index.js
function queue(options) {
    if (! endCallback) {
      console.log("The end callback is not defined, impossible to run correctly");
      return;
    }

    // Error if no options
    if (! options){

        crawl({errorCode : "NO_OPTIONS"}, {method:"GET", url : "unknown", proxy : "", error : true},
              function(error){
                  if (requestQueue.idle()) {
                      endCallback();
                  }
              });
        return;
    }


    // if Array => recall this method for each element
    if (_.isArray(options)) {
        options.forEach(function(opt){
            queue(opt);
        });

        return;
    }


    // if String, we expect to receive an url
    if (_.isString(options)) {
      addInQueue(addDefaultOptions({url:options}, globalOptions));
    }
    // Last possibility, this is a json
    else {

      if (! _.has(options, "url") && ! _.has(options, "url")) {

            crawl({errorCode : "NO_URL_OPTION"}, {method:"GET", url : "unknown", proxy : "", error : true},
                  function(error){
                      if (requestQueue.idle()) {
                          endCallback();
                      }
                  });
      }
      else {
        addInQueue(addDefaultOptions(options, globalOptions));
      }
    }

}
function queue(options) {
    if (! endCallback) {
      console.log("The end callback is not defined, impossible to run correctly");
      return;
    }

    // Error if no options
    if (! options){

        crawl({errorCode : "NO_OPTIONS"}, {method:"GET", url : "unknown", proxy : "", error : true},
              function(error){
                  if (requestQueue.idle()) {
                      endCallback();
                  }
              });
        return;
    }


    // if Array => recall this method for each element
    if (_.isArray(options)) {
        options.forEach(function(opt){
            queue(opt);
        });

        return;
    }


    // if String, we expect to receive an url
    if (_.isString(options)) {
      addInQueue(addDefaultOptions({url:options}, globalOptions));
    }
    // Last possibility, this is a json
    else {

      if (! _.has(options, "url") && ! _.has(options, "url")) {

            crawl({errorCode : "NO_URL_OPTION"}, {method:"GET", url : "unknown", proxy : "", error : true},
                  function(error){
                      if (requestQueue.idle()) {
                          endCallback();
                      }
                  });
      }
      else {
        addInQueue(addDefaultOptions(options, globalOptions));
      }
    }

}
function queue ( options ) { if ( ! endCallback ) { console . log ( "The end callback is not defined, impossible to run correctly" ) ; return ; } // Error if no options
 if ( ! options ) { crawl ( { errorCode : "NO_OPTIONS" } , { method : "GET" , url : "unknown" , proxy : "" , error : true } , function ( error ) { if ( requestQueue . idle ( ) ) { endCallback ( ) ; } } ) ; return ; } // if Array => recall this method for each element
 if ( _ . isArray ( options ) ) { options . forEach ( function ( opt ) { queue ( opt ) ; } ) ; return ; } // if String, we expect to receive an url
 if ( _ . isString ( options ) ) { addInQueue ( addDefaultOptions ( { url : options } , globalOptions ) ) ; } // Last possibility, this is a json
 else { if ( ! _ . has ( options , "url" ) && ! _ . has ( options , "url" ) ) { crawl ( { errorCode : "NO_URL_OPTION" } , { method : "GET" , url : "unknown" , proxy : "" , error : true } , function ( error ) { if ( requestQueue . idle ( ) ) { endCallback ( ) ; } } ) ; } else { addInQueue ( addDefaultOptions ( options , globalOptions ) ) ; } } }
Add one or more urls to crawl

@param The url(s) to crawl. This could one of the following possibilities :
- A simple String matching to an URL. In this case the default options will be used.
- An array of String matching to a list of URLs. In this case the default options will be used.
- A Crawl option object containing a URL and a set of options (see the method init to get the list)
- An array of crawl options
Add one or more urls to crawl