easy-down v1.0.4

Node 爬虫 功能库

安装

npm install --save easy-down

并发控制示例

const { TaskPool } = require('easy-down')

// 示例数据
const list = new Array(100).fill(1)
// 执行函数
const fn = (index) => {
   return new Promise((r, j) => {
        setTimeout(() => {
            index % 2 === 0 ? r() : j()
        }, index * 100 * Math.random());
    })
}

// 并发数10
const taskPool = new TaskPool(10)
list.forEach((v, i) => taskPool.push(fn, i))

taskPool.on('success', (task) => {
    console.log(task, '---------------成功回调')
})

taskPool.on('error', (task) => {
    console.log(task, '---------------失败回调')
})

taskPool.on('full', (length) => {
    console.warn('底层系统处理不能及时完成，排队中，目前队列长度为:' + length);
})

图片爬取示例

const { getElemet, FilePic, utils } = require('easy-down')
const path = require('path')
const baseUrl = 'https://www.3733.com/game/'

// 实例化-下载并发数控制为10
const filePic = new FilePic(30)
const getList = async(url) => {
    const { $ } = await getElemet(url)
    const lastEl = $('ul.pagination li').last()
    let nextPage = lastEl.find('a')?.attr('href') || ''
    nextPage = nextPage? (baseUrl + nextPage) : ''
    const list = []
    $('.pic_box .taptap-top-card').each(function(e) {
        // 标题
        const title = $(this).find('.card-middle-title h4').text()
        // 封面
        const cover = $(this).find('.card-left-image img').attr('src')
        // 游戏截图
        const detailsImgs = []
        $(this).find('.top-card-right .card-right-image').each(function() {
            const url = $(this).find('img').attr('src')
            detailsImgs.push(url)
        })
        list.push({title, cover, detailsImgs})
    });
    return { list, nextPage }
}

const app = async (url = baseUrl) => {
   // 获取想要的数据
   const { list, nextPage } = await getList(url)
   for (let i = 0; i < list.length; i++) {
    const item = list[i];
    const dir = path.join(__dirname, `/images/${item.title}`)
    await utils.dirExists(dir)
    // 单张指定名称下载
    filePic.taskDownPic(item.cover, dir, item.title)
    // 批量下载
    filePic.taskGroupPic(item.detailsImgs, path.join(dir, '游戏截图'))
   }
   if(nextPage) {
    // 等待一秒
    await utils.sleep(1000)
    app(nextPage)
   }
}
app()