Easy Crawler

A simple webcrawler for Node.js supporting Concurrent Connections and Queues.

Installation

npm install easycrawler

Usage

var Crawler = require('easycrawler');

var crawler = new Crawler({
	thread: 5,
	logs: true,
	depth: 2,
	headers : {'user-agent' : 'foobar'},
	onlyCrawl : ['reddit', 'reddit.com'], //will only crawl urls containing these strings
	reject : ['rutube'], //will reject links containing rutube
	onSuccess : function(data){
		//console.log(data.url);
		//console.log(data.body);
	},
	onError : function(data){
		//console.log(data.url);
		//console.log(data.status);
	},
	onFinished : function(urls){
		//console.log(urls.crawled);//urls.crawled for visited urls;
		//console.log(urls.discovered);//urls.discovered for discovered urls
	}
});


crawler.crawl('http://www.reddit.com/');