src/index.js
function(url, options, callback) {
    var crawler       = Crawler.crawl(url.toLowerCase()),
        opts          = options || {},
        maxPages      = opts.maxPages || 10,
        parsedPages   = [],         // Store parsed pages in this array
        seoParser     = this.meta,  // Reference to `meta` method to call during crawl
        crawlResults  = [];         // Store results in this array and then return it to caller

    // Crawler settings
    crawler.interval            = opts.interval || 250;         // Time between spooling up new requests
    crawler.maxDepth            = opts.depth || 2;              // Maximum deptch of crawl
    crawler.maxConcurrency      = opts.concurrency || 2;        // Number of processes to spawn at a time
    crawler.timeout             = opts.timeout || 1000;         // Milliseconds to wait for server to send headers
    crawler.downloadUnsupported = opts.unsupported || false;    // Save resources by only downloading files Simple Crawler can parse
                                                                // The user agent string to provide - Be cool and don't trick people
    crawler.userAgent           = opts.useragent || 'SEO Checker v1 (https://github.com/Clever-Labs/seo-checker)';

    // Only fetch HTML! You should always set this option unless you have a good reason not to
    if (opts.htmlOnly === true) { // Being explicit about truthy values here
      var htmlCondition = crawler.addFetchCondition(function(parsedURL) {
        return !parsedURL.path.match(/\.jpg|jpeg|png|gif|js|txt|css|pdf$/i);
      });
    }

    crawler.on('fetchcomplete', function(queueItem, responseBuffer, response) {
      if (queueItem.stateData.code === 200) {
        crawlResults.push({ url: queueItem.url, body: responseBuffer.toString() });
      }
      if (crawlResults.length >= maxPages) {
        this.stop(); // Stop the crawler
        crawlResults.forEach(function(page, index, results) {
          parsedPages.push({url: page.url, results: seoParser(page.body)});
        });
        if (!callback) {
          return parsedPages;
        } else {
          callback(parsedPages);
        }
      }
    });
  }
function(url, options, callback) {
    var crawler       = Crawler.crawl(url.toLowerCase()),
        opts          = options || {},
        maxPages      = opts.maxPages || 10,
        parsedPages   = [],         // Store parsed pages in this array
        seoParser     = this.meta,  // Reference to `meta` method to call during crawl
        crawlResults  = [];         // Store results in this array and then return it to caller

    // Crawler settings
    crawler.interval            = opts.interval || 250;         // Time between spooling up new requests
    crawler.maxDepth            = opts.depth || 2;              // Maximum deptch of crawl
    crawler.maxConcurrency      = opts.concurrency || 2;        // Number of processes to spawn at a time
    crawler.timeout             = opts.timeout || 1000;         // Milliseconds to wait for server to send headers
    crawler.downloadUnsupported = opts.unsupported || false;    // Save resources by only downloading files Simple Crawler can parse
                                                                // The user agent string to provide - Be cool and don't trick people
    crawler.userAgent           = opts.useragent || 'SEO Checker v1 (https://github.com/Clever-Labs/seo-checker)';

    // Only fetch HTML! You should always set this option unless you have a good reason not to
    if (opts.htmlOnly === true) { // Being explicit about truthy values here
      var htmlCondition = crawler.addFetchCondition(function(parsedURL) {
        return !parsedURL.path.match(/\.jpg|jpeg|png|gif|js|txt|css|pdf$/i);
      });
    }

    crawler.on('fetchcomplete', function(queueItem, responseBuffer, response) {
      if (queueItem.stateData.code === 200) {
        crawlResults.push({ url: queueItem.url, body: responseBuffer.toString() });
      }
      if (crawlResults.length >= maxPages) {
        this.stop(); // Stop the crawler
        crawlResults.forEach(function(page, index, results) {
          parsedPages.push({url: page.url, results: seoParser(page.body)});
        });
        if (!callback) {
          return parsedPages;
        } else {
          callback(parsedPages);
        }
      }
    });
  }
function ( url , options , callback ) { var crawler = Crawler . crawl ( url . toLowerCase ( ) ) , opts = options || { } , maxPages = opts . maxPages || 10 , parsedPages = [ ] , // Store parsed pages in this array seoParser = this . meta , // Reference to `meta` method to call during crawl crawlResults = [ ] ; // Store results in this array and then return it to caller // Crawler settings crawler . interval = opts . interval || 250 ; // Time between spooling up new requests crawler . maxDepth = opts . depth || 2 ; // Maximum deptch of crawl crawler . maxConcurrency = opts . concurrency || 2 ; // Number of processes to spawn at a time crawler . timeout = opts . timeout || 1000 ; // Milliseconds to wait for server to send headers crawler . downloadUnsupported = opts . unsupported || false ; // Save resources by only downloading files Simple Crawler can parse // The user agent string to provide - Be cool and don't trick people crawler . userAgent = opts . useragent || 'SEO Checker v1 (https://github.com/Clever-Labs/seo-checker)' ; // Only fetch HTML! You should always set this option unless you have a good reason not to if ( opts . htmlOnly === true ) { // Being explicit about truthy values here var htmlCondition = crawler . addFetchCondition ( function ( parsedURL ) { return ! parsedURL . path . match ( / \.jpg|jpeg|png|gif|js|txt|css|pdf$ / i ) ; } ) ; } crawler . on ( 'fetchcomplete' , function ( queueItem , responseBuffer , response ) { if ( queueItem . stateData . code === 200 ) { crawlResults . push ( { url : queueItem . url , body : responseBuffer . toString ( ) } ) ; } if ( crawlResults . length >= maxPages ) { this . stop ( ) ; // Stop the crawler crawlResults . forEach ( function ( page , index , results ) { parsedPages . push ( { url : page . url , results : seoParser ( page . body ) } ) ; } ) ; if ( ! callback ) { return parsedPages ; } else { callback ( parsedPages ) ; } } } ) ; }
Generate SEO data for multiple pages of a site at once

`url` [String] - The URL to begin the crawl
`options` [Object] - Options to pass to the crawler. Uses a subset of the `simplecrawler` lib's options:
- `maxPages` [Number] - The max number of pages to crawl (defaults to 10)
- `interval` [Number] - Delay between each request for a new page
- `maxDepth` [Number] - Depth of crawl. See simplecrawler docs for an explanation
- `maxConcurrency` [Number] - Number of processes to spawn at a time
- `timeout` [Number] - Time to wait for a server response before moving on
- `downloadUnsupported` [Boolean] - Determines whether crawler downloads files it cannot parse
- `userAgent` [String] - The UA string to send with requests
- `htmlOnly` [Boolean] - Tells crawler not to crawl any non-HTML text/html pages. This is a required option and has no default

Returns an array of objects containing SEO data and URL. Example return value:

[{
url: 'http://example.com/page1.html',
results: { <results object identical to signature of this.meta()'s return value> }
}]
Generate SEO data for multiple pages of a site at once