lib/queue/request-queue.js
function execHttp(options, callback) {

      if (proxyList) {
        options.proxy  = proxyList.getProxy().getUrl();
      }
      log.debug({"url" : options.url, "step" : "queue-requester.execHttp", "message" : "Execute the request"});
      request.get(options, function(error, result) {
        log.debug({"url" : options.url, "step" : "queue-requester.execHttp", "message" : "Execute the request done"});

        // TODO : review this solution/try to understand the origin of this problem
        // some sites provide inconsistent response for some urls (status 40* instead of 200).
        // In such case, it should be nice to retry (if the option retry400 == true)
        if (result.statusCode && result.statusCode >= 400 && result.statusCode <= 499 && result.retry400) {
           error = new Error("40* Error");
           error.code = result.statusCode;
        }

        if (error) {
          onRequestError(error, result, function(error){
            process.nextTick(function() {callback(error);});
          });
        }
        else {
          onCrawl(null, result, function(error){
            process.nextTick(function() {callback(error);});
          });
        }

      });


  }
function execHttp(options, callback) {

      if (proxyList) {
        options.proxy  = proxyList.getProxy().getUrl();
      }
      log.debug({"url" : options.url, "step" : "queue-requester.execHttp", "message" : "Execute the request"});
      request.get(options, function(error, result) {
        log.debug({"url" : options.url, "step" : "queue-requester.execHttp", "message" : "Execute the request done"});

        // TODO : review this solution/try to understand the origin of this problem
        // some sites provide inconsistent response for some urls (status 40* instead of 200).
        // In such case, it should be nice to retry (if the option retry400 == true)
        if (result.statusCode && result.statusCode >= 400 && result.statusCode <= 499 && result.retry400) {
           error = new Error("40* Error");
           error.code = result.statusCode;
        }

        if (error) {
          onRequestError(error, result, function(error){
            process.nextTick(function() {callback(error);});
          });
        }
        else {
          onCrawl(null, result, function(error){
            process.nextTick(function() {callback(error);});
          });
        }

      });


  }
function execHttp ( options , callback ) { if ( proxyList ) { options . proxy = proxyList . getProxy ( ) . getUrl ( ) ; } log . debug ( { "url" : options . url , "step" : "queue-requester.execHttp" , "message" : "Execute the request" } ) ; request . get ( options , function ( error , result ) { log . debug ( { "url" : options . url , "step" : "queue-requester.execHttp" , "message" : "Execute the request done" } ) ; // TODO : review this solution/try to understand the origin of this problem // some sites provide inconsistent response for some urls (status 40* instead of 200). // In such case, it should be nice to retry (if the option retry400 == true) if ( result . statusCode && result . statusCode >= 400 && result . statusCode <= 499 && result . retry400 ) { error = new Error ( "40* Error" ) ; error . code = result . statusCode ; } if ( error ) { onRequestError ( error , result , function ( error ) { process . nextTick ( function ( ) { callback ( error ) ; } ) ; } ) ; } else { onCrawl ( null , result , function ( error ) { process . nextTick ( function ( ) { callback ( error ) ; } ) ; } ) ; } } ) ; }
Execute an http request

@param The options to used for the request
@param callback executed when the request is finished
Execute an http request