lib/index.js
function initialize(options) {
    // log
	config.logfile = options.logfile || 'logs/spider.log';
	log4js.configure({
		appenders: [
			{ type: 'console', category: 'console'},
			{ type: 'dateFile', 
				filename: config.logfile, 
				pattern: 'yyyy-MM-dd',
				category: 'default'
			} 
		],
		replaceConsole: true
	});
    logger = log4js.getLogger('default');
	if (!options.startURLs) {
		logger.error('At least one URL must be specified!');
		return -1;
	}
	if (!options.filter) {
		logger.error('Filter must be specified!');
		return -1;
	}

	config.filter = options.filter;
	config.startURLs = options.startURLs;
	config.domains = options.startURLs.map(function(url) {
		return URL.parse(url).host;
	});
	config.interval = options.interval || 2 * 1000;
	config.domains.forEach(function(host) {
		taskQueues[host] = [];
	});
	config.timeout = options.timeout || 10 * 1000;

	// push all URLs into taskQueues
	config.startURLs.forEach(function(url) {
		var urlObj = URL.parse(url);
		var queue = new TaskQueue(urlObj.host);
		queue.tasks.push(url);
		taskQueues.push(queue);
	});
}
function initialize(options) {
    // log
	config.logfile = options.logfile || 'logs/spider.log';
	log4js.configure({
		appenders: [
			{ type: 'console', category: 'console'},
			{ type: 'dateFile', 
				filename: config.logfile, 
				pattern: 'yyyy-MM-dd',
				category: 'default'
			} 
		],
		replaceConsole: true
	});
    logger = log4js.getLogger('default');
	if (!options.startURLs) {
		logger.error('At least one URL must be specified!');
		return -1;
	}
	if (!options.filter) {
		logger.error('Filter must be specified!');
		return -1;
	}

	config.filter = options.filter;
	config.startURLs = options.startURLs;
	config.domains = options.startURLs.map(function(url) {
		return URL.parse(url).host;
	});
	config.interval = options.interval || 2 * 1000;
	config.domains.forEach(function(host) {
		taskQueues[host] = [];
	});
	config.timeout = options.timeout || 10 * 1000;

	// push all URLs into taskQueues
	config.startURLs.forEach(function(url) {
		var urlObj = URL.parse(url);
		var queue = new TaskQueue(urlObj.host);
		queue.tasks.push(url);
		taskQueues.push(queue);
	});
}
function initialize ( options ) { // log config . logfile = options . logfile || 'logs/spider.log' ; log4js . configure ( { appenders : [ { type : 'console' , category : 'console' } , { type : 'dateFile' , filename : config . logfile , pattern : 'yyyy-MM-dd' , category : 'default' } ] , replaceConsole : true } ) ; logger = log4js . getLogger ( 'default' ) ; if ( ! options . startURLs ) { logger . error ( 'At least one URL must be specified!' ) ; return - 1 ; } if ( ! options . filter ) { logger . error ( 'Filter must be specified!' ) ; return - 1 ; } config . filter = options . filter ; config . startURLs = options . startURLs ; config . domains = options . startURLs . map ( function ( url ) { return URL . parse ( url ) . host ; } ) ; config . interval = options . interval || 2 * 1000 ; config . domains . forEach ( function ( host ) { taskQueues [ host ] = [ ] ; } ) ; config . timeout = options . timeout || 10 * 1000 ; // push all URLs into taskQueues config . startURLs . forEach ( function ( url ) { var urlObj = URL . parse ( url ) ; var queue = new TaskQueue ( urlObj . host ) ; queue . tasks . push ( url ) ; taskQueues . push ( queue ) ; } ) ; }
Initialize the spider, with some configurations.
options.filter
options.startURLS
options.interval
options.logfile
options.timeout
Initialize the spider with some configurations . options . filter options . startURLS options . interval options . logfile options . timeout