Crawler

Simple site crawler for node.js

Install

npm install site-crawler

Example Codes

var Crawler = require('site-crawler')

var site = 'https://techcrunch.com'

var crawler = new Crawler({
	// default is 10
	concurrency:10
})
crawler
.on('found',function(url,next) {
	var ok = url.startsWith(site)
	if(ok) console.error('found:',url)
	// set null argument for next if reject crawling this url.(or you can modify url)
	next(ok ? url : null)
})
.on('crawl',function(url,res,$,next) {
	// res is response object of resuest module
	// $ is cheerio object
	console.error('\tcrawl:',$('title').text())
	next()
})
.on('error',function(url,err) {
	console.error('\terror:',url,':',err.statusCode)
})
.on('complete',function() {
	console.log('done.')
})
crawler.start(site)

Tests

cd crawler
npm test

Licence

MIT