easy-down
v1.0.4
Published
``` npm install --save easy-down ```
Downloads
14
Readme
Node 爬虫 功能库
安装
npm install --save easy-down
并发控制示例
const { TaskPool } = require('easy-down')
// 示例数据
const list = new Array(100).fill(1)
// 执行函数
const fn = (index) => {
return new Promise((r, j) => {
setTimeout(() => {
index % 2 === 0 ? r() : j()
}, index * 100 * Math.random());
})
}
// 并发数10
const taskPool = new TaskPool(10)
list.forEach((v, i) => taskPool.push(fn, i))
taskPool.on('success', (task) => {
console.log(task, '---------------成功回调')
})
taskPool.on('error', (task) => {
console.log(task, '---------------失败回调')
})
taskPool.on('full', (length) => {
console.warn('底层系统处理不能及时完成,排队中,目前队列长度为:' + length);
})
图片爬取示例
const { getElemet, FilePic, utils } = require('easy-down')
const path = require('path')
const baseUrl = 'https://www.3733.com/game/'
// 实例化-下载并发数控制为10
const filePic = new FilePic(30)
const getList = async(url) => {
const { $ } = await getElemet(url)
const lastEl = $('ul.pagination li').last()
let nextPage = lastEl.find('a')?.attr('href') || ''
nextPage = nextPage? (baseUrl + nextPage) : ''
const list = []
$('.pic_box .taptap-top-card').each(function(e) {
// 标题
const title = $(this).find('.card-middle-title h4').text()
// 封面
const cover = $(this).find('.card-left-image img').attr('src')
// 游戏截图
const detailsImgs = []
$(this).find('.top-card-right .card-right-image').each(function() {
const url = $(this).find('img').attr('src')
detailsImgs.push(url)
})
list.push({title, cover, detailsImgs})
});
return { list, nextPage }
}
const app = async (url = baseUrl) => {
// 获取想要的数据
const { list, nextPage } = await getList(url)
for (let i = 0; i < list.length; i++) {
const item = list[i];
const dir = path.join(__dirname, `/images/${item.title}`)
await utils.dirExists(dir)
// 单张指定名称下载
filePic.taskDownPic(item.cover, dir, item.title)
// 批量下载
filePic.taskGroupPic(item.detailsImgs, path.join(dir, '游戏截图'))
}
if(nextPage) {
// 等待一秒
await utils.sleep(1000)
app(nextPage)
}
}
app()