dirdb
v2.0.0
Published
DirDB key-value directory database
Downloads
34
Maintainers
Readme
DirDB - lightning fast database
DirDB key-value directory database
$ npm install dirdb
Run tests
Browse module (e.g. node_modules/dirdb
) install directory, and run tests:
$ npm test
# or
$ node test.js
Compare test results with travis run tests.
Include in your script
const dirdb = require('dirdb');
Define the database root directory
First time, make sure the directory exists, is empty, and have the right user permissions mode.
const db = new dirdb('/dir/path/name');
Chain calls
All async method functions returns the object they belong (db
core, or db.client()
stream).
obj. // < can be: db or db.client() stream
methodAsync(..., (...) => { ... }). // and so on...
methodAsync(..., (...) => { ... });
// ASYNC call example
db. // < can be: db or db.client() stream, see below
put(dirname, key, value1, e => {}). // and so on...
add(dirname, key, value2, e => {});
isdir(dirname)
If dirname
exist, return/callback object dirconfig
, or undefined
if not.
dirname
- String directory table name (folder), without slashes, e.g.name
// SYNC
const dirconfig = db.isdir(dirname);
// ASYNC
db.isdir(dirname, dirconfig => { });
mkdir(dirname[, options])
Make a directory by name, not path, e.g. name
. If dirname
exist, throw/callback error
. Return/callback name
on success. For more options
, see below.
// SYNC
const name = db.mkdir(dirname);
// ASYNC
db.mkdir(dirname, (error, name) => {
if (error) { throw error; }
});
rmdir(dirname)
Remove dir name and its contents, throw/callback error
if dirname
not exists.
// SYNC
db.rmdir(dirname);
// ASYNC
db.rmdir(dirname, error => {
if (error) { throw error; }
});
list()
Return/callback dbconfig
object { dirname: dirconfig, ... }
.
// SYNC
const dbconfig = db.list();
// ASYNC
db.list(dbconfig => { });
put(dirname, key, value[, callback])
Throw/callback error
if key exists. Return/callback { uid, hash, path }
if success.
dirname
- String directory table name, without slasheskey
- String|Buffervalue
- String|Buffer
Return { uid, hash, path }
uid
- String unique id ( birthDateInt36.Index )hash
- String key hash ( using:dirconfig
algorithm + digest )path
- String path key ( filepath.k
) and value ( filepath.v
)
// SYNC
const { uid, hash, path } = db.put(dirname, key, value);
// ASYNC
db.put(dirname, key, value, (error, uid, hash, path) => {
if (error) { throw error; }
});
set(dirname, key, value[, callback])
Overwrite value if key exists, or create, if not. Return/callback { uid, hash, path }
if success.
// SYNC
const { uid, hash, path } = db.set(dirname, key, value);
// ASYNC
db.set(dirname, key, value, (error, uid, hash, path) => {
if (error) { throw error; }
});
add(dirname, key, value[, callback])
Append value
if key exists, or create, if not. Return/callback { uid, hash, path }
if success.
// SYNC
const { uid, hash, path } = db.add(dirname, key, value);
// ASYNC
db.add(dirname, key, value, (error, uid, hash, path) => {
if (error) { throw error; }
});
get(dirname, key[, callback])
Read key value
. Throw/callback error
if key not exists. Return/callback { value, uid, hash, path }
if success.
// SYNC
const { value, uid, hash, path } = db.get(dirname, key); // value is Buffer
// ASYNC
db.get(dirname, key, (error, value, uid, hash, path) => { // value is Buffer or undefined if error
if (error) { throw error; }
});
del(dirname, key[, callback])
Delete key
. Throw/callback error
if key not exists. Return/callback uid
if success.
// SYNC
const uid = db.del(dirname, key);
// ASYNC
db.del(dirname, key, (error, uid) => { // uid is String or undefined if error
if (error) { throw error; }
});
keys(dirname[, range[, callback]])
Return/callback object keylist
if success.
range
- Object{ start: Number, end: Number }
keylist
- Object{ uid: keyhash, ... }
// SYNC
const keylistA = db.keys(dirname); // without range select, return all
const keylistB = db.keys(dirname, { start: 1 }); // without end point, return all except first key ( index: 1, 2, ... )
const keylistC = db.keys(dirname, { start: 0, end: 2 }); // return first two keys ( index: 0 and 1 )
// ASYNC
db.keys(dirname, (error, keylist) => { // without range select, return all
if (error) { throw error; }
});
// without start point, return first two keys ( index: 0 and 1 )
db.keys(dirname, { end: 2 }, (error, keylist) => { // keylist is Object or undefined if error
if (error) { throw error; }
});
val(dirname, uid, keyhash[, callback])
Throw/callback error
if key not exists. Return/callback { key, value, path }
if success. See the above keylist
object for uid
and keyhash
.
// SYNC
const { key, value, path } = db.val(dirname, uid, keyhash); // key and value is Buffer
// ASYNC
db.val(dirname, uid, keyhash, (error, key, value, path) => { // key and value is Buffer or undefined if error
if (error) { throw error; }
});
stats()
Throw/callback error
if key not exists. Return/callback { uid, hash, path, stats }
if success.
stats
- Object, key value filelstat
fs.Stats
// SYNC
const { uid, hash, path, stats } = db.stats(dirname, key);
// ASYNC
db.stats(dirname, key, (error, uid, hash, path, stats) => {
if (error) { throw error; }
// fs.Stats file: path + '.v'
// require('fs').lstat(path + '.v', (error, stats) => {})
console.log('Birth time', stats.birthtime);
console.log('Modified time', stats.mtime);
console.log('Key value size', stats.size);
});
setgc(dirname, option)
Set dirname
GC boolean option. When delete a key using del()
function, if GC is enabled (true), the directory where the key-value was saved, is deleted if is empty. Throw/callback error
if dirname not exists. Return/callback dirconfig
if success.
// SYNC
const dirconfig = db.setgc(dirname, true);
// ASYNC
db.setgc(dirname, false, (error, dirconfig) => { // dirconfig is Object or undefined if error
if (error) { throw error; }
console.log('gc', dirconfig.gc);
});
server()
Server stream object. See the stream / socket examples below, of how to pipe server stream into client stream.
client([sync])
Client stream object. Server call method functions is sync false
(by default), for sync set true
. The sync/async server method can be set individually on any client function, with the last argument.
// call SYNC method functions on server
const clientS = db.client(true);
// call ASYNC method functions on server
const clientA = db.client(); // or false
const client = db.client();
client.set(dirname, key, value, (error, uid) => {
if (error) { throw error; }
}, true); // true: call SYNC method function on server
client.del(dirname, key, (error, uid) => {
if (error) { throw error; }
}, false); // false: call ASYNC method function on server
Stream example
const client = db.client();
client.pipe(db.server()).pipe(client);
client.put(dirname, key, value, (error, uid) => {
if (error) { throw error; }
});
Socket stream example
const net = require('net');
net.createServer(socket => socket.pipe(db.server()).pipe(socket)).
listen(function() { // socket server listen to a random port and address
const a = this.address(); // get the socket server port and address
net.connect(a.port, a.address, function() {
const client = db.client(); // default ASYNC methods on DB core
this.pipe(client).pipe(this);
client.set(dirname, key, value, (error, uid) => {
if (error) { throw error; }
});
});
});
mkdir(dirname[, options])
dirname
- String directory table name, without slashesoptions
- Object, see below
Directory table options
level
- Number, key hash directory divisor, default3
, minim0
and max limited byalgorithm
anddigest
value, see belowdmode
- Number, directory mode, default0o700
fmode
- Number, file mode, default0o600
algorithm
- String, key hash algorithm, defaultmd5
, possible:md5
|sha1
|sha256
|sha512
digest
- String, key hash digest, defaultbase64
, possible:base64
|hex
compress
- String, zlib compress type, defaultnone
, possible:none
|deflate
|gzip
gc
- Boolean, run garbage collector after key delete, defaulttrue
You can overwrite the default directory options:
const db = new dirdb('/dir/pathname', {
// overwrite default options
level: 3,
digest: 'hex',
gc: false
});
mkdir()
options example
High level, means high directory divisor. To increase I/O speed on high number of keys entries, make sure a high level value is defined on db.mkdir
options. If there is only few key entries on directory, high level value will decrease the I/O speed.
// dir name 'logs'
db.mkdir('logs', {
level: 4,
algorithm: 'sha1',
digest: 'hex',
// WARNING: when use db.add() "append" on compress other than 'none'
// make sure, the key value will not be corrupted if append function will be used
compress: 'gzip'
});
// sha1 = 35 unique characters , level = 4
console.log('divisor', Math.pow(35, 4)); // 1500625
// key entries are stored on 1500625 max sub-directories
Few examples of how to calculate the directory divisor, at maximum level
md5-base64
Math.pow(64, 22) - 64 unique characters, 22 long max levelmd5-hex
Math.pow(35, 32) - 35 unique characters, 32 long max levelsha1-base64
Math.pow(64, 27) - 64 unique characters, 27 long max levelsha1-hex
Math.pow(35, 40) - 35 unique characters, 40 long max level
High Availability, Backup and Restore Operations
DirDB is very light and simple database, because of that, HA / Back-Up / Restore can be done with tools like rsync or unison.
# Backup example
$ rsync -abqz dirA dirB
# High Availability example
$ unison -auto dirA/ dirB/
dirA
local directory ( primary )dirB
remote / network directory ( secondary )
// High Availability, using unison
const dbA = new dirdb(dirA); // primary DB
// Host A ( local )
net.createServer(socket => socket.pipe(dbA.server()).pipe(socket));
// dirB > ssh://[email protected]/home/alice/dirB
// OR
// dirB > socket://remote_host:port_num/path/to/dirB
const dbB = new dirdb(dirB); // secondary DB
// Host B ( remote ) - another node.js server
net.createServer(socket => socket.pipe(dbB.server()).pipe(socket));
DB core only (not client), async stream write/read big values
const fs = require('fs');
// db core, write big value, source file '/path/big/file.ext'
db.put(dirname, key, '', (error, uid, hash, path) => { // or db.set() to overwrite, if key exists
if (error) { throw error; }
fs.createReadStream('/path/big/file.ext').pipe(fs.createWriteStream(path + '.v'));
});
// db core, append big value, source file '/path/big/file.ext'
db.add(dirname, key, '', (error, uid, hash, path) => {
if (error) { throw error; }
fs.createReadStream('/path/big/file.ext').pipe(fs.createWriteStream(path + '.v', { flags: 'a' }));
});
// db core, read big value
db.stats(dirname, key, (error, uid, hash, path, stats) => {
if (error) { throw error; }
fs.createReadStream(path + '.v').on('data', data => {});
});
For more info, consult or run the test.js file.
DirDB is licensed under the MIT license. See the included LICENSE
file for more details.