index.js
function init(options, callback, proxies, logLevel) {

      if (! callback) {
        log.error("The end callback is not defined");
      }

      // Default options
      globalOptions = createDefaultOptions();

      // Merge default options values & overridden values provided by the arg options
      if (options) {
        _.extend(globalOptions, options);
      }

      // if using rateLimits we want to use only one connection with delay between requests
      if (globalOptions.rateLimits !== 0) {
          globalOptions.maxConnections = 1;
      }

      // create the crawl store
      store.createStore(globalOptions.storeModuleName, globalOptions.storeParams ? globalOptions.storeParams : null);

      // If the options object contains an new implementation of the updateDepth method
      if (globalOptions.updateDepth) {
        updateDepthFn = globalOptions.updateDepth;
      }

      // Init the crawl queue
      endCallback = callback;
      requestQueue.init(globalOptions, crawl, recrawl, endCallback, proxies);

      if (logLevel) {
        console.log("Change Log level into :" + logLevel);
        log.level(logLevel);
      }
      else {
        log.level("info"); // By default debug level is not activated
      }
  }
function init(options, callback, proxies, logLevel) {

      if (! callback) {
        log.error("The end callback is not defined");
      }

      // Default options
      globalOptions = createDefaultOptions();

      // Merge default options values & overridden values provided by the arg options
      if (options) {
        _.extend(globalOptions, options);
      }

      // if using rateLimits we want to use only one connection with delay between requests
      if (globalOptions.rateLimits !== 0) {
          globalOptions.maxConnections = 1;
      }

      // create the crawl store
      store.createStore(globalOptions.storeModuleName, globalOptions.storeParams ? globalOptions.storeParams : null);

      // If the options object contains an new implementation of the updateDepth method
      if (globalOptions.updateDepth) {
        updateDepthFn = globalOptions.updateDepth;
      }

      // Init the crawl queue
      endCallback = callback;
      requestQueue.init(globalOptions, crawl, recrawl, endCallback, proxies);

      if (logLevel) {
        console.log("Change Log level into :" + logLevel);
        log.level(logLevel);
      }
      else {
        log.level("info"); // By default debug level is not activated
      }
  }
function init ( options , callback , proxies , logLevel ) { if ( ! callback ) { log . error ( "The end callback is not defined" ) ; } // Default options
 globalOptions = createDefaultOptions ( ) ; // Merge default options values & overridden values provided by the arg options
 if ( options ) { _ . extend ( globalOptions , options ) ; } // if using rateLimits we want to use only one connection with delay between requests
 if ( globalOptions . rateLimits !== 0 ) { globalOptions . maxConnections = 1 ; } // create the crawl store
 store . createStore ( globalOptions . storeModuleName , globalOptions . storeParams ? globalOptions . storeParams : null ) ; // If the options object contains an new implementation of the updateDepth method
 if ( globalOptions . updateDepth ) { updateDepthFn = globalOptions . updateDepth ; } // Init the crawl queue
 endCallback = callback ; requestQueue . init ( globalOptions , crawl , recrawl , endCallback , proxies ) ; if ( logLevel ) { console . log ( "Change Log level into :" + logLevel ) ; log . level ( logLevel ) ; } else { log . level ( "info" ) ; // By default debug level is not activated
 } }
Init the crawler

@param options used to customize the crawler.

The current options attributes are :
- skipDuplicates        : if true skips URLs that were already crawled - default is true
- maxConnections        : the number of connections used to crawl - default is 5
- rateLimits            : number of milliseconds to delay between each requests (Default 0).
Note that this option will force crawler to use only one connection
- externalDomains       : if true crawl the  external domains. This option can crawl a lot of different linked domains, default = false.
- externalHosts         : if true crawl the others hosts on the same domain, default = false.
- firstExternalLinkOnly : crawl only the first link found for external domains/hosts. externalHosts or externalDomains should be = true
- scripts               : if true crawl script tags
- links                 : if true crawl link tags
- linkTypes             : the type of the links tags to crawl (match to the rel attribute), default : ["canonical", "stylesheet"]
- images                : if true crawl images
- protocols             : list of the protocols to crawl, default = ["http", "https"]
- timeout               : timeout per requests in milliseconds (Default 20000)
- retries               : number of retries if the request fails (default 3)
- retryTimeout          : number of milliseconds to wait before retrying (Default 10000)
- depthLimit            : the depth limit for the crawl
- followRedirect        : if true, the crawl will not return the 301, it will follow directly the redirection
- proxyList             : the list of proxies (see the project simple-proxies on npm)
- storeModuleName       : the npm nodule name used for the store implementation, by default  : memory-store
- storeParams           : the params to pass to the store module when create it.
- queueModuleName       : the npm module name used for the job queue. By default : async-queue
- queueParams           : the params to pass to the job queue when create it.

+ all options provided by nodejs request : https://github.com/request/request

@param callback() called when all URLs have been crawled
@param proxies to used when making http requests (optional)
@param logLevel : a new log level (eg. "debug") (optional, default value is info)
Init the crawler