Changes-index NPM

changes-index

create indexes from a leveldb changes feed

This package provides a way to create a materialized view on top of an append-only log.

To update an index, just change the index code and delete the indexed data.

example

Create a change feed and set up an index. Here we'll create an index for keys that start with the prefix 'user!' on the name and hackerspace properties.

var level = require('level');
var sublevel = require('subleveldown');
var changes = require('changes-feed');
var changesdown = require('changesdown');
var chi = require('../');

var argv = require('minimist')(process.argv.slice(2));

var up = level('/tmp/test.db', { valueEncoding: 'json' });
var feed = changes(sublevel(up, 'feed'));
var db = changesdown(sublevel(up, 'db'), feed, { valueEncoding: 'json' });

var indexes = chi({
    ixdb: level('/tmp/index.db', { valueEncoding: 'json' }),
    chdb: db,
    feed: feed
});
indexes.add(function (row, cb) {
    if (/^user!/.test(row.key)) {
        cb(null, {
            'user.name': row.value.name,
            'user.space': row.value.hackerspace
        });
    }
    else cb()
});

now we can create users:

if (argv._[0] === 'create') {
    var id = require('crypto').randomBytes(16).toString('hex');
    var name = argv._[1], space = argv._[2];
    var value = { name: name, hackerspace: space };
    
    userExists(name, function (err, ex) {
        if (err) return console.error(err);
        if (ex) return console.error('name in use');
        
        db.put('user!' + id, value, function (err) {
            if (err) console.error(err);
        });
    });
    
    function userExists (name, cb) {
        indexes.createReadStream('user.name', name, { gte: name, lte: name })
            .pipe(through.obj(write, end))
        ;
        function write (row, enc, next) { cb(null, true) }
        function end () { cb(null, false) }
    }
}

or clear (and implicitly regenerate) an existing index:

else if (argv._[0] === 'clear') {
    indexes.clear(argv._[1], function (err) {
        if (err) console.error(err);
    });
}

With these indexes we can list users by name and space:

else if (argv._[0] === 'by-name') {
    indexes.createReadStream('user.name', argv)
        .on('data', console.log)
    ;
}
else if (argv._[0] === 'by-space') {
    indexes.createReadStream('user.space', argv)
        .on('data', console.log)
    ;
}

methods

var chi = require('changes-index')

var indexes = chi(opts)

You must provide:

opts.ixdb - levelup database to use for indexing
opts.chdb - wrapped changesdown levelup database to lookup primary records
opts.feed - changes-feed handle wired up to the chdb

indexes.add(fn)

Create an index from a function fn(row, cb) that will be called for each put and delete. Your function fn must call cb(err, ix) with ix, an object mapping index names to values. The values from ix will be sorted according to the algorithm from bytewise.

indexes.createReadStream(name, opts)

Create a readable object-mode stream of the primary documents inserted into chdb based on the index given by name.

The stream will produce row objects with:

row.key - the key name put into changedown
row.value - the value put into changedown
row.index - the index value generated by the index function
row.exists - whether the key existed prior to this operation
row.prev - the previous set of indexes or null if the key was created
row.change - the monotonically increasing change sequence from changes-feed

This read stream can be bounded by all the usual levelup options:

opts.lt
opts.lte
opts.gt
opts.gte
opts.limit
opts.reverse

plus:

opts.eq

which is the same as setting opts.gte and opts.lte to the same value. This isn't common in ordinary levelup but is very common when dealing with indexes that map to other keys.