1.0.0 • Published 7 years ago
request-manager v1.0.0
request-manager
A NodeJS module to manage batch requests with the 'request' module. The timing and batch amount can be set and updated at any point in time.
Usage
Install from NPM:
npm install request-manager --save
Example usage:
const RequestManager = require('request-manager');
links = []; // Some full array of links
// Initialize a RequestManager instance
var reqm = new RequestManager({
linkArray: links, // required.
numberConcurrent: 3, // optional, default 5. Must be >= 1
waitTime: 500, // optional, default 0 milliseconds. Must be >= 0.
requestOptions: { // optional, default none. These are the options that can be specified for the 'request' module
encoding: null, // Save raw responses
headers: { // Set a custom user agent
'User-Agent': 'Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405'
}
}
});
// Start requesting the links. This can be cancelled with reqm.stop()
reqm.start();
// Cancels the currently running RequestManager. If reqm.start() is
// called again, it starts back from the beginning.
reqm.stop();
// Every response gets sent back on a 'data' event
reqm.on('data', (err, returned) => {
// 'err' is any error from the 'request' module
// 'returned' is an object that consists of:
// 'resp': the response from the 'request' module
// 'data': the body of the response
// 'link': the link that was requested
// 'index': the index of the link in the provided link array
// From here, the results of the request can be managed.
// NOTE: All requests are transparently handled. Even if a request returns an error
// message (eg., 404), it is always sent back here (no re-requests are attempted)
});
// When all the links have been requested and returned, the 'end' event
// is emitted
reqm.on('end', () => {
console.log('Finished requesting all the links');
});
// Any general error message generated by the module emits an 'error' event
reqm.on('error', (err) => {
console.log(err);
});
// Special functions
// These functions can be called while 'request-manager' is running. The
// values they update will be applied once the current batch of requests
// is complete.
// The new waitTime in milliseconds. Must be greater than or equal to 0
reqm.updateWaitTime(newWaitTime);
// The new batch amount. Must be greater than or equal to 1
reqm.updateBatchAmount(newBatchAmount);
1.0.0
7 years ago