lib/queue/request-queue.js
function queue(options, callback) {


        // if skipDuplicates, don't crawl twice the same url
        if (options.skipDuplicates) {
            store.getStore().checkInCrawlHistory(options.url, function(error, isInCrawlHistory) {

              if (isInCrawlHistory) {
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" :  "Don't crawl this url - Option skipDuplicates=true & the url has already been crawled" });
                  callback();
              }
              else {

                  //numOfUrl++
                  //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
                  requestQueue.push(options);
                  callback();

              }

            });

        }
        else {
            //numOfUrl++
            //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );

            log.info({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
            requestQueue.push(options);
            callback();
        }

  }
function queue(options, callback) {


        // if skipDuplicates, don't crawl twice the same url
        if (options.skipDuplicates) {
            store.getStore().checkInCrawlHistory(options.url, function(error, isInCrawlHistory) {

              if (isInCrawlHistory) {
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" :  "Don't crawl this url - Option skipDuplicates=true & the url has already been crawled" });
                  callback();
              }
              else {

                  //numOfUrl++
                  //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );
                  log.debug({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
                  requestQueue.push(options);
                  callback();

              }

            });

        }
        else {
            //numOfUrl++
            //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url );

            log.info({"url" : options.url, "step" : "queue-resquester.queue", "message" : "Add in the request queue"});
            requestQueue.push(options);
            callback();
        }

  }
function queue ( options , callback ) { // if skipDuplicates, don't crawl twice the same url if ( options . skipDuplicates ) { store . getStore ( ) . checkInCrawlHistory ( options . url , function ( error , isInCrawlHistory ) { if ( isInCrawlHistory ) { log . debug ( { "url" : options . url , "step" : "queue-resquester.queue" , "message" : "Don't crawl this url - Option skipDuplicates=true & the url has already been crawled" } ) ; callback ( ) ; } else { //numOfUrl++ //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url ); log . debug ( { "url" : options . url , "step" : "queue-resquester.queue" , "message" : "Add in the request queue" } ) ; requestQueue . push ( options ) ; callback ( ) ; } } ) ; } else { //numOfUrl++ //console.log("Queue length : " + requestQueue.length() + " - Running : " + requestQueue.running() + " - Total Add :" + numOfUrl + " - Total End : " + endUrl + " url : " + options.url ); log . info ( { "url" : options . url , "step" : "queue-resquester.queue" , "message" : "Add in the request queue" } ) ; requestQueue . push ( options ) ; callback ( ) ; } }
Add a new url to crawl in the queue.
Check the desired options and add it to the request queue

@param the options used to crawl the url
Add a new url to crawl in the queue . Check the desired options and add it to the request queue