Egg-crawler NPM

egg-crawler

Crawler tool for egg.

Install

$ npm i egg-crawler --save

or

$ yarn add egg-crawler

Usage

// {app_root}/config/plugin.js
exports.crawler = {
  enable: true,
  package: 'egg-crawler',
};

Example

Create a crawler

// {app_root}/crawler/profile.js
import { Crawler } from 'egg-crawler';

class ProfileCrawler extends Crawler {
  // class instantiation name
  name: string = 'profile';

  async run() {
    // this.scraper -> is scrape-it instantiation
    // const { app, ctx } = this -> that can use every ctx module

    const data = await this.scraper('https://ionicabizau.net', {
      title: '.header h1',
      desc: '.header h2',
      avatar: {
        selector: '.header img',
        attr: 'src',
      },
    });

    return data;
  }
}

Run it

// {app_root}/controller/profile.js
import { Controller } from 'egg';

class ProfileController extends Controller {
  async info() {
    const { ctx, app } = this;
    const data = await app.crawler.profile.run();

    ctx.body = data;
  }
}