aspider v3.0.2

aspider

Node.js configurable spider.

Installing

npm install aspider

Examples

1. Create a file aspider.json in your project root directory:

{
  "qq": {
    "url": "http://www.qq.com",
    "selectors": {
      "guess": "#guess"
    }
  }
}

2. Require module aspider, and call function run:

var aspider = require('aspider');

aspider.run('qq', (error, response, body, selectors) => {
  console.log('error', error);
  console.log('response', response);
  console.log('body', body);
  console.log('selectors', selectors.guess.text());
});

API

configure file (default: aspider.json)

{
  "aspider-key": {
    "url": "request url",
    "method": "GET/POST (default: GET)",
    "decode": "response decoded mode (default: UTF8)",
    "proxy": "network proxy",
    "timeout": request timeout (default: 10000),
    "headers": {  
      "header-key": "header-value"
    },
    "params": {  
      "param-key": "param-value"
    },
    "selectors": {
      "selector-key": "selector-value"
    }    
  }
}

{
  "qq": {
    "url": "http://www.qq.com",
    "timeout": 10000
  }
}

.use(path)

Set the configure file path, if your configure file's name isn't aspider.json.

aspider.use('./test.json');

.getValue(key)

Get the aspider-value from the configure file.

var qq = aspider.getValue('qq');

.run(key, callback)

Run to request by aspider-key.

aspider.run('qq', (error, response, body, selectors) => {

});

.run(value, callback)

Run to request by aspider-value.

var qq = aspider.getValue('qq');
qq.timeout = 5000;
aspider.run(qq, (error, response, body, selectors) => {

});