changes-index

create indexes from a leveldb changes feed

This package provides a way to create a materialized view on top of an append-only log.

To update an index, just change the index code and delete the indexed data.

example

Create a change feed and set up an index. Here we'll create an index for keys that start with the prefix 'user!' on the name and hackerspace properties.

var level = require('level');
var sublevel = require('subleveldown');
var changes = require('changes-feed');
var changesdown = require('changesdown');
var chi = require('../');

var argv = require('minimist')(process.argv.slice(2));

var up = level('/tmp/test.db', { valueEncoding: 'json' });
var feed = changes(sublevel(up, 'feed'));
var db = changesdown(sublevel(up, 'db'), feed, { valueEncoding: 'json' });

var indexes = chi({
    ixdb: level('/tmp/index.db', { valueEncoding: 'json' }),
    chdb: db,
    feed: feed
});
indexes.add(function (row, cb) {
    if (/^user!/.test(row.key)) {
        cb(null, {
            'user.name': row.value.name,
            'user.space': row.value.hackerspace
        });
    }
    else cb()
});

now we can create users:

if (argv._[0] === 'create') {
    var id = require('crypto').randomBytes(16).toString('hex');
    var name = argv._[1], space = argv._[2];
    var value = { name: name, hackerspace: space };
    
    userExists(name, function (err, ex) {
        if (err) return console.error(err);
        if (ex) return console.error('name in use');
        
        db.put('user!' + id, value, function (err) {
            if (err) console.error(err);
        });
    });
    
    function userExists (name, cb) {
        indexes.createReadStream('user.name', name, { gte: name, lte: name })
            .pipe(through.obj(write, end))
        ;
        function write (row, enc, next) { cb(null, true) }
        function end () { cb(null, false) }
    }
}

or clear (and implicitly regenerate) an existing index:

else if (argv._[0] === 'clear') {
    indexes.clear(argv._[1], function (err) {
        if (err) console.error(err);
    });
}

With these indexes we can list users by name and space:

else if (argv._[0] === 'by-name') {
    indexes.createReadStream('user.name', argv)
        .on('data', console.log)
    ;
}
else if (argv._[0] === 'by-space') {
    indexes.createReadStream('user.space', argv)
        .on('data', console.log)
    ;
}

methods

var chi = require('changes-index')

var indexes = chi(opts)

You must provide:

• opts.ixdb - levelup database to use for indexing
• opts.chdb - wrapped changesdown levelup database to lookup primary records
• opts.feed - changes-feed handle wired up to the chdb

indexes.add(fn)

Create an index from a function fn(row, cb) that will be called for each put and delete. Your function fn must call cb(err, ix) with ix, an object mapping index names to values. The values from ix will be sorted according to the algorithm from bytewise.

indexes.createReadStream(name, opts)

Create a readable object-mode stream of the primary documents inserted into chdb based on the index given by name.

The stream will produce row objects with:

• row.key - the key name put into changedown
• row.value - the value put into changedown
• row.index - the index value generated by the index function
• row.exists - whether the key existed prior to this operation
• row.prev - the previous set of indexes or null if the key was created
• row.change - the monotonically increasing change sequence from changes-feed

This read stream can be bounded by all the usual levelup options:

• opts.lt
• opts.lte
• opts.gt
• opts.gte
• opts.limit
• opts.reverse

plus:

• opts.eq

which is the same as setting opts.gte and opts.lte to the same value. This isn't common in ordinary levelup but is very common when dealing with indexes that map to other keys.

indexes.clear(name, cb)

Delete the index for name, calling cb(err) when finished.

versioning

The internals of this module may change between patch releases, which may affect how data is stored on disk.

When you upgrade this package over existing data, you should delete the indexes first.

license

MIT