3.0.0 • Published 2 months ago

pelias-dbclient v3.0.0

Weekly downloads
158
License
MIT
Repository
github
Last release
2 months ago

This repository is part of the Pelias project. Pelias is an open-source, open-data geocoder originally sponsored by Mapzen. Our official user documentation is here.

Pelias Elasticsearch database client

This module provides a Node.js stream for bulk-inserting documents into Elasticsearch.

Greenkeeper badge

Install Dependencies

$ npm install

Usage

This module returns “streamFactory” —a function that produces a transforming stream. The stream puts documents into elasticsearch during import pipeline. Note: this stream triggers finish event only after all documents are stored into elasticsearch.

'use strict';

// some_importer.js

const streamify = require('stream-array');
const through = require('through2');
const Document = require('pelias-model').Document;
const dbMapper = require('pelias-model').createDocumentMapperStream;
const dbclient = require('pelias-dbclient');

const elasticsearch = require('elasticsearch');
const config = require('pelias-config').generate();
const elasticDeleteQuery = require('elastic-deletebyquery');

const timestamp = Date.now();

const stream = streamify([1, 2, 3])
  .pipe(through.obj((item, enc, next) => {
    const uniqueId = [ 'docType', item ].join(':'); // documents with the same id will be updated
    const doc = new Document( 'sourceType', 'venue', uniqueId );
    doc.timestamp = timestamp;
    next(null, doc);
  }))
  .pipe(dbMapper())
  .pipe(dbclient()); // put documents into elasticsearch

stream.on('finish', () => {
  const client = new elasticsearch.Client(config.esclient);
  elasticDeleteQuery(client);

  const options = {
    index: config.schema.indexName,
    body: {
      query: {
        "bool": {
          "must": [
            {"term": { "source":  "sourceType" }}
          ],
          "must_not": [
            {"term": { "timestamp":  timestamp }}
          ]
        }
      }
    }
  };

  client.deleteByQuery(options, (err, response) => {
    console.log('The elements deleted are: %s', response.elements);
  });
});

Contributing

Please fork and pull request against upstream master on a feature branch.

Pretty please; provide unit tests and script fixtures in the test directory.

Running Unit Tests

$ npm test

Continuous Integration

CI tests every release against all currently supported Node.js versions.

3.0.0

2 months ago

2.15.0

1 year ago

2.14.0

4 years ago

2.13.1

4 years ago

2.13.0

4 years ago

2.12.1

5 years ago

2.12.0

5 years ago

2.11.0

5 years ago

2.10.1

5 years ago

2.10.0

5 years ago

2.9.0

5 years ago

2.8.1

5 years ago

2.8.0

5 years ago

2.7.0

5 years ago

2.6.2

5 years ago

2.6.1

6 years ago

2.6.0

6 years ago

2.5.6

6 years ago

2.5.5

6 years ago

2.5.4

6 years ago

2.5.3

6 years ago

2.5.2

6 years ago

2.5.1

6 years ago

2.5.0

6 years ago

2.4.1

6 years ago

2.4.0

6 years ago

2.3.1

6 years ago

2.3.0

7 years ago

2.2.7

7 years ago

2.2.6

7 years ago

2.2.5

7 years ago

2.2.4

7 years ago

2.2.3

7 years ago

2.2.2

7 years ago

2.2.1

7 years ago

2.2.0

7 years ago

2.1.2

7 years ago

2.1.1

7 years ago

2.1.0

7 years ago

2.0.0

8 years ago

1.0.2

8 years ago

1.0.1

8 years ago

1.0.0

8 years ago

0.0.9

9 years ago

0.0.8

9 years ago

0.0.7

9 years ago

0.0.6

9 years ago

0.0.5

9 years ago

0.0.4

9 years ago

0.0.3

9 years ago

0.0.2

9 years ago

0.0.1

9 years ago

0.0.0

9 years ago