lib/queue.js
function() {
    Array.call(this);

    /**
     * Speeds up {@link FetchQueue.oldestUnfetchedItem} by storing the index at
     * which the latest oldest unfetched queue item was found.
     * @name FetchQueue._oldestUnfetchedIndex
     * @private
     * @type {Number}
     */
    Object.defineProperty(this, "_oldestUnfetchedIndex", {
        enumerable: false,
        writable: true,
        value: 0
    });

    /**
     * Serves as a cache for what URL's have been fetched. Keys are URL's,
     * values are booleans.
     * @name FetchQueue._scanIndex
     * @private
     * @type {Object}
     */
    Object.defineProperty(this, "_scanIndex", {
        enumerable: false,
        writable: true,
        value: {}
    });

    /**
     * Controls what properties can be operated on with the
     * {@link FetchQueue#min}, {@link FetchQueue#avg} and {@link FetchQueue#max}
     * methods.
     * @name FetchQueue._allowedStatistics
     * @type {Array}
     */
    Object.defineProperty(this, "_allowedStatistics", {
        enumerable: false,
        writable: true,
        value: [
            "actualDataSize",
            "contentLength",
            "downloadTime",
            "requestLatency",
            "requestTime"
        ]
    });
}
function() {
    Array.call(this);

    /**
     * Speeds up {@link FetchQueue.oldestUnfetchedItem} by storing the index at
     * which the latest oldest unfetched queue item was found.
     * @name FetchQueue._oldestUnfetchedIndex
     * @private
     * @type {Number}
     */
    Object.defineProperty(this, "_oldestUnfetchedIndex", {
        enumerable: false,
        writable: true,
        value: 0
    });

    /**
     * Serves as a cache for what URL's have been fetched. Keys are URL's,
     * values are booleans.
     * @name FetchQueue._scanIndex
     * @private
     * @type {Object}
     */
    Object.defineProperty(this, "_scanIndex", {
        enumerable: false,
        writable: true,
        value: {}
    });

    /**
     * Controls what properties can be operated on with the
     * {@link FetchQueue#min}, {@link FetchQueue#avg} and {@link FetchQueue#max}
     * methods.
     * @name FetchQueue._allowedStatistics
     * @type {Array}
     */
    Object.defineProperty(this, "_allowedStatistics", {
        enumerable: false,
        writable: true,
        value: [
            "actualDataSize",
            "contentLength",
            "downloadTime",
            "requestLatency",
            "requestTime"
        ]
    });
}
function ( ) { Array . call ( this ) ; /**
     * Speeds up {@link FetchQueue.oldestUnfetchedItem} by storing the index at
     * which the latest oldest unfetched queue item was found.
     * @name FetchQueue._oldestUnfetchedIndex
     * @private
     * @type {Number}
     */ Object . defineProperty ( this , "_oldestUnfetchedIndex" , { enumerable : false , writable : true , value : 0 } ) ; /**
     * Serves as a cache for what URL's have been fetched. Keys are URL's,
     * values are booleans.
     * @name FetchQueue._scanIndex
     * @private
     * @type {Object}
     */ Object . defineProperty ( this , "_scanIndex" , { enumerable : false , writable : true , value : { } } ) ; /**
     * Controls what properties can be operated on with the
     * {@link FetchQueue#min}, {@link FetchQueue#avg} and {@link FetchQueue#max}
     * methods.
     * @name FetchQueue._allowedStatistics
     * @type {Array}
     */ Object . defineProperty ( this , "_allowedStatistics" , { enumerable : false , writable : true , value : [ "actualDataSize" , "contentLength" , "downloadTime" , "requestLatency" , "requestTime" ] } ) ; }
QueueItems represent resources in the queue that have been fetched, or will be eventually.
@typedef {Object} QueueItem
@property {Number} id A unique ID assigned by the queue when the queue item is added
@property {String} url The complete, canonical URL of the resource
@property {String} protocol The protocol of the resource (http, https)
@property {String} host The full domain/hostname of the resource
@property {Number} port The port of the resource
@property {String} path The URL path, including the query string
@property {String} uriPath The URL path, excluding the query string
@property {Number} depth How many steps simplecrawler has taken from the initial page (which is depth 1) to this resource.
@property {String} referrer The URL of the resource where the URL of this queue item was discovered
@property {Boolean} fetched Has the request for this item been completed? You can monitor this as requests are processed.
@property {'queued'|'spooled'|'headers'|'downloaded'|'redirected'|'notfound'|'failed'} status The internal status of the item.
@property {Object} stateData An object containing state data and other information about the request.
@property {Number} stateData.requestLatency The time (in ms) taken for headers to be received after the request was made.
@property {Number} stateData.requestTime The total time (in ms) taken for the request (including download time.)
@property {Number} stateData.downloadTime The total time (in ms) taken for the resource to be downloaded.
@property {Number} stateData.contentLength The length (in bytes) of the returned content. Calculated based on the `content-length` header.
@property {String} stateData.contentType The MIME type of the content.
@property {Number} stateData.code The HTTP status code returned for the request. Note that this code is `600` if an error occurred in the client and a fetch operation could not take place successfully.
@property {Object} stateData.headers An object containing the header information returned by the server. This is the object node returns as part of the `response` object.
@property {Number} stateData.actualDataSize The length (in bytes) of the returned content. Calculated based on what is actually received, not the `content-length` header.
@property {Boolean} stateData.sentIncorrectSize True if the data length returned by the server did not match what we were told to expect by the `content-length` header.
 
FetchQueue handles {@link QueueItem}s and provides a few utility methods for querying them
@class
QueueItems represent resources in the queue that have been fetched or will be eventually .