lib/crawler.js
function processReceivedData() {
            if (dataReceived || queueItem.fetched) {
                return;
            }

            responseBuffer = responseBuffer.slice(0, responseLengthReceived);
            dataReceived = true;
            timeDataReceived = Date.now();

            crawler.queue.update(queueItem.id, {
                stateData: {
                    downloadTime: timeDataReceived - timeHeadersReceived,
                    requestTime: timeDataReceived - timeCommenced,
                    actualDataSize: responseBuffer.length,
                    sentIncorrectSize: responseBuffer.length !== responseLength
                }
            }, function (error, queueItem) {
                if (error) {
                    // Remove this request from the open request map
                    crawler._openRequests.splice(
                        crawler._openRequests.indexOf(response.req), 1);

                    return crawler.emit("queueerror", error, queueItem);
                }

                // First, save item to cache (if we're using a cache!)
                if (crawler.cache && crawler.cache.setCacheData instanceof Function) {
                    crawler.cache.setCacheData(queueItem, responseBuffer);
                }

                // No matter the value of `crawler.decompressResponses`, we still
                // decompress the response if it's gzipped or deflated. This is
                // because we always provide the discoverResources method with a
                // decompressed buffer
                if (/(gzip|deflate)/.test(queueItem.stateData.headers["content-encoding"])) {
                    zlib.unzip(responseBuffer, function(error, decompressedBuffer) {
                        if (error) {
                            /**
                             * Fired when an error was encountered while unzipping the response data
                             * @event Crawler#gziperror
                             * @param {QueueItem} queueItem           The queue item for which the unzipping failed
                             * @param {String|Buffer} responseBody    If {@link Crawler#decodeResponses} is true, this will be the decoded HTTP response. Otherwise it will be the raw response buffer.
                             * @param {http.IncomingMessage} response The [http.IncomingMessage]{@link https://nodejs.org/api/http.html#http_class_http_incomingmessage} for the request's response
                             */
                            crawler.emit("gziperror", queueItem, error, responseBuffer);
                            emitFetchComplete(responseBuffer);
                        } else {
                            var responseBody = crawler.decompressResponses ? decompressedBuffer : responseBuffer;
                            emitFetchComplete(responseBody, decompressedBuffer);
                        }
                    });
                } else {
                    emitFetchComplete(responseBuffer);
                }
            });
        }
function processReceivedData() {
            if (dataReceived || queueItem.fetched) {
                return;
            }

            responseBuffer = responseBuffer.slice(0, responseLengthReceived);
            dataReceived = true;
            timeDataReceived = Date.now();

            crawler.queue.update(queueItem.id, {
                stateData: {
                    downloadTime: timeDataReceived - timeHeadersReceived,
                    requestTime: timeDataReceived - timeCommenced,
                    actualDataSize: responseBuffer.length,
                    sentIncorrectSize: responseBuffer.length !== responseLength
                }
            }, function (error, queueItem) {
                if (error) {
                    // Remove this request from the open request map
                    crawler._openRequests.splice(
                        crawler._openRequests.indexOf(response.req), 1);

                    return crawler.emit("queueerror", error, queueItem);
                }

                // First, save item to cache (if we're using a cache!)
                if (crawler.cache && crawler.cache.setCacheData instanceof Function) {
                    crawler.cache.setCacheData(queueItem, responseBuffer);
                }

                // No matter the value of `crawler.decompressResponses`, we still
                // decompress the response if it's gzipped or deflated. This is
                // because we always provide the discoverResources method with a
                // decompressed buffer
                if (/(gzip|deflate)/.test(queueItem.stateData.headers["content-encoding"])) {
                    zlib.unzip(responseBuffer, function(error, decompressedBuffer) {
                        if (error) {
                            /**
                             * Fired when an error was encountered while unzipping the response data
                             * @event Crawler#gziperror
                             * @param {QueueItem} queueItem           The queue item for which the unzipping failed
                             * @param {String|Buffer} responseBody    If {@link Crawler#decodeResponses} is true, this will be the decoded HTTP response. Otherwise it will be the raw response buffer.
                             * @param {http.IncomingMessage} response The [http.IncomingMessage]{@link https://nodejs.org/api/http.html#http_class_http_incomingmessage} for the request's response
                             */
                            crawler.emit("gziperror", queueItem, error, responseBuffer);
                            emitFetchComplete(responseBuffer);
                        } else {
                            var responseBody = crawler.decompressResponses ? decompressedBuffer : responseBuffer;
                            emitFetchComplete(responseBody, decompressedBuffer);
                        }
                    });
                } else {
                    emitFetchComplete(responseBuffer);
                }
            });
        }
function processReceivedData ( ) { if ( dataReceived || queueItem . fetched ) { return ; } responseBuffer = responseBuffer . slice ( 0 , responseLengthReceived ) ; dataReceived = true ; timeDataReceived = Date . now ( ) ; crawler . queue . update ( queueItem . id , { stateData : { downloadTime : timeDataReceived - timeHeadersReceived , requestTime : timeDataReceived - timeCommenced , actualDataSize : responseBuffer . length , sentIncorrectSize : responseBuffer . length !== responseLength } } , function ( error , queueItem ) { if ( error ) { // Remove this request from the open request map crawler . _openRequests . splice ( crawler . _openRequests . indexOf ( response . req ) , 1 ) ; return crawler . emit ( "queueerror" , error , queueItem ) ; } // First, save item to cache (if we're using a cache!) if ( crawler . cache && crawler . cache . setCacheData instanceof Function ) { crawler . cache . setCacheData ( queueItem , responseBuffer ) ; } // No matter the value of `crawler.decompressResponses`, we still // decompress the response if it's gzipped or deflated. This is // because we always provide the discoverResources method with a // decompressed buffer if ( / (gzip|deflate) / . test ( queueItem . stateData . headers [ "content-encoding" ] ) ) { zlib . unzip ( responseBuffer , function ( error , decompressedBuffer ) { if ( error ) { /**
                             * Fired when an error was encountered while unzipping the response data
                             * @event Crawler#gziperror
                             * @param {QueueItem} queueItem           The queue item for which the unzipping failed
                             * @param {String|Buffer} responseBody    If {@link Crawler#decodeResponses} is true, this will be the decoded HTTP response. Otherwise it will be the raw response buffer.
                             * @param {http.IncomingMessage} response The [http.IncomingMessage]{@link https://nodejs.org/api/http.html#http_class_http_incomingmessage} for the request's response
                             */ crawler . emit ( "gziperror" , queueItem , error , responseBuffer ) ; emitFetchComplete ( responseBuffer ) ; } else { var responseBody = crawler . decompressResponses ? decompressedBuffer : responseBuffer ; emitFetchComplete ( responseBody , decompressedBuffer ) ; } } ) ; } else { emitFetchComplete ( responseBuffer ) ; } } ) ; }
Function for dealing with 200 responses
Function for dealing with 200 responses