@modalnetworks/rpacore v0.1.0

CoreApplication Modalnetworks

How to use this package

Creating your scrapper

#SourceScrapper.ts

import { ScrapperContract, Common } from "@modalnetworks/rpacore";
import { saveToCSV } from "@modalnetworks/rpacore/dist/common/Files";
import { stripTags } from "@modalnetworks/rpacore/dist/common/string";
import { delay } from "@modalnetworks/rpacore/dist/common/timer";
import { PageResult, SearchResult , SearchParams } from "@modalnetworks/rpacore/dist/types";
import { saveToCSV } from "@modalnetworks/rpacore/dist/common/Files";

export class SourceScrapper implements ScrapperContract {
  readonly name = "sourceOne";
  readonly url = "{your_url_source}";

  static browser?: any;

  async open() {
    SourceScrapper.browser = await Browser.initialize();
  }
  async close() {
    await SourceScrapper.browser.close();
  }

  async search(params: SearchParams): Promise<SearchResult | any> {
    return new Promise(async (resolve) => {
      const browser = SourceScrapper.browser;
      console.log(`scrapper ${this.url}`);
      const page = await browser.newPage();
      await page.goto(this.url);
      await page.type("#_3_keywords", params.query);
      await page.click("#_3_search");
   

      resolve({ items, page });
    });
  }
  toPage({ items, page }: SearchResult): Promise<PageResult> {
    return new Promise(async (resolve) => {
     
      
      resolve(...);
    });
  }
  toSaveFile({ items }: PageResult): Promise<any> {
    return new Promise(async (resolve) => {
      await saveToCSV(
        items.map((row) => {
          row.titulo = stripTags(row.titulo);
          row.ementa = String(row.ementa).replace(/\s+/gi, " ");
          row.integra = String(row.integra).replace(/\s+/gi, " ");
          return row;
        }),
        `./csv/${this.name}`
      );
      resolve(true);
    });
  }
}