tasks/generator/Scraper.js
function()
    {
        var url       = this.tableUrl;
        var _this     = this;
        var d         = new Q.defer();
        var tableInfo;

        this.getTableInfo()
        .then(function(info) {
            tableInfo = info;
            return Q.nfcall(request, url + '/index.html');
        })
        .then(function(data) {
            var $        = cheerio.load(data);
            var list     = $('#tables li');
            var requests = [];

            list.each(function() {
                var $this = this;
                var title = $this.find('a').text();
                var link  = $this.find('a').attr('href');

                // Once we've got the table, add the description and fire an
                // update back on the promise
                var p = _this.getTableFields(url + '/' + link);
                p.then(function(t) { t.description = tableInfo[t.tableName]; });
                p.then(function(s) { d.notify(s.tableName); });
                requests.push(p);
            });

            Q.spread(requests, function() {
                d.resolve(_(arguments).toArray());
            });

        });

        return d.promise;
    }
function()
    {
        var url       = this.tableUrl;
        var _this     = this;
        var d         = new Q.defer();
        var tableInfo;

        this.getTableInfo()
        .then(function(info) {
            tableInfo = info;
            return Q.nfcall(request, url + '/index.html');
        })
        .then(function(data) {
            var $        = cheerio.load(data);
            var list     = $('#tables li');
            var requests = [];

            list.each(function() {
                var $this = this;
                var title = $this.find('a').text();
                var link  = $this.find('a').attr('href');

                // Once we've got the table, add the description and fire an
                // update back on the promise
                var p = _this.getTableFields(url + '/' + link);
                p.then(function(t) { t.description = tableInfo[t.tableName]; });
                p.then(function(s) { d.notify(s.tableName); });
                requests.push(p);
            });

            Q.spread(requests, function() {
                d.resolve(_(arguments).toArray());
            });

        });

        return d.promise;
    }
function ( ) { var url = this . tableUrl ; var _this = this ; var d = new Q . defer ( ) ; var tableInfo ; this . getTableInfo ( ) . then ( function ( info ) { tableInfo = info ; return Q . nfcall ( request , url + '/index.html' ) ; } ) . then ( function ( data ) { var $ = cheerio . load ( data ) ; var list = $ ( '#tables li' ) ; var requests = [ ] ; list . each ( function ( ) { var $this = this ; var title = $this . find ( 'a' ) . text ( ) ; var link = $this . find ( 'a' ) . attr ( 'href' ) ; // Once we've got the table, add the description and fire an // update back on the promise var p = _this . getTableFields ( url + '/' + link ) ; p . then ( function ( t ) { t . description = tableInfo [ t . tableName ] ; } ) ; p . then ( function ( s ) { d . notify ( s . tableName ) ; } ) ; requests . push ( p ) ; } ) ; Q . spread ( requests , function ( ) { d . resolve ( _ ( arguments ) . toArray ( ) ) ; } ) ; } ) ; return d . promise ; }
Load all the tables up
Load all the tables up