0.0.37 • Published 2 months ago
@soralinks/news-scrapers v0.0.37
news-scrapers
Scraping news organizations' websites for headlines
Install
npm install @soralinks/news-scrapers
Use in your app
import {
NewsScraperType,
NewsScraperResponse,
NewsScraperFactory
} from '@soralinks/news-scrapers';
(async () => {
const factory = new NewsScraperFactory();
const scrapers = await factory.createScrapers();
const results = await Promise.allSettled(
scrapers.map(async (scraper) => {
return scraper.scrape(NewsScraperType.POLITICS);
}),
);
// @ts-ignore
const scraperResponses: NewsScraperResponse[] = results.map(result => {
if (result.status === 'fulfilled') {
return result.value;
}
return undefined;
}).filter(Boolean);
console.log(`scraperResponses: ${JSON.stringify(scraperResponses, null, 2)}`);
})();
Logging
// To turn on logging, set the following environment variables:
// LOGGING_ABC_SCRAPER = 'on'
// LOGGING_AP_SCRAPER = 'on'
// LOGGING_BBC_SCRAPER = 'on'
// LOGGING_CNN_SCRAPER = 'on'
// LOGGING_DISPATCH_SCRAPER = 'on'
// LOGGING_CS_MONITOR_SCRAPER = 'on'
// LOGGING_EPOCH_TIMES_SCRAPER='on'
// LOGGING_FOX_SCRAPER = 'on'
// LOGGING_NEWSNATION_SCRAPER = 'on'
// LOGGING_NEWSWEEK_SCRAPER = 'on'
// LOGGING_NY_POST_SCRAPER = 'on'
// LOGGING_WASH_EXAM_SCRAPER = 'on'
// LOGGING_WSJ_SCRAPER = 'on'
// Note that error logging is always on
0.0.37
2 months ago
0.0.36
2 months ago
0.0.32
2 months ago
0.0.33
2 months ago
0.0.34
2 months ago
0.0.35
2 months ago
0.0.31
2 months ago
0.0.30
4 months ago
0.0.29
4 months ago
0.0.28
4 months ago
0.0.24
4 months ago
0.0.25
4 months ago
0.0.26
4 months ago
0.0.27
4 months ago
0.0.22
4 months ago
0.0.23
4 months ago
0.0.20
5 months ago
0.0.21
5 months ago
0.0.19
5 months ago
0.0.18
5 months ago
0.0.17
5 months ago
0.0.16
5 months ago
0.0.15
5 months ago
0.0.14
5 months ago
0.0.13
6 months ago
0.0.12
6 months ago
0.0.11
6 months ago
0.0.10
6 months ago
0.0.9
6 months ago
0.0.8
6 months ago
0.0.7
6 months ago
0.0.6
6 months ago
0.0.5
6 months ago
0.0.4
6 months ago
0.0.3
6 months ago
0.0.2
6 months ago
0.0.1
6 months ago