├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── example.js ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | db 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "0.12" 5 | - 4 6 | - 5 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | @node_modules/.bin/tape test.js 4 | 5 | .PHONY: test 6 | 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # level-pathwise 3 | 4 | Turn a [leveldb](http://leveldb.org) into one huge object of arbitrary size! Efficiently and atomically update and read parts of it! 5 | 6 | [![build status](https://secure.travis-ci.org/juliangruber/level-pathwise.svg)](http://travis-ci.org/juliangruber/level-pathwise) 7 | 8 | ## Example 9 | 10 | ```js 11 | var Pathwise = require('level-pathwise'); 12 | var level = require('level'); 13 | 14 | var store = new Pathwise(level('db')); 15 | 16 | // insert an object on the root level 17 | 18 | store.put([], { 19 | foo: { 20 | bar: ['beep', 'boop'], 21 | baz: 'bleep' 22 | } 23 | }, function(err){}); 24 | 25 | // read it out 26 | 27 | store.get([], function(err, obj){ 28 | // => { 29 | // foo: { 30 | // bar: ['beep', 'boop'], 31 | // baz: 'bleep' 32 | // } 33 | // } 34 | }); 35 | 36 | // read only a subsection of the object, 37 | // like data.foo.bar 38 | 39 | store.get(['foo', 'bar'], function(err, obj){ 40 | // => ['beep', 'boop'] 41 | }); 42 | 43 | // extend an object, 44 | // like data.foo.key = 'value' 45 | 46 | store.put([], { 47 | foo: { 48 | key: 'value' 49 | } 50 | }) 51 | 52 | // read the direct children of a path 53 | 54 | store.children(['foo'], function(err, children){ 55 | // => ['bar', 'baz'] 56 | }); 57 | 58 | // remove some data, 59 | // like delete data.foo.baz 60 | 61 | store.del(['foo', 'baz'], function(err){}); 62 | 63 | // perform several updates in one atomic step, 64 | // like data.i.said.what = 'what'; delete data.foo; 65 | 66 | store.batch([ 67 | { type: 'put', path: [], data: { i: said: { what: 'what' } } }, 68 | { type: 'del', path: ['foo'] } 69 | ], function(){}); 70 | ``` 71 | 72 | ## Installation 73 | 74 | ```bash 75 | $ npm install level-pathwise 76 | ``` 77 | 78 | ## API 79 | 80 | ### Pathwise(db) 81 | 82 | Instantiate a new pathwise store, using `db`. 83 | 84 | ### #put(path, object[, opts], fn) 85 | 86 | Store `object` at `path`. 87 | 88 | Options: 89 | 90 | - `batch`: LevelUP batch object to use 91 | 92 | ### #get(path, fn) 93 | 94 | Get the object at `path` with all its children. 95 | 96 | ### #del(path[, opts], fn) 97 | 98 | Delete the object at `path` with all its children. 99 | 100 | Options: 101 | 102 | - `batch`: LevelUP batch object to use 103 | 104 | ### #children(path, fn) 105 | 106 | Get the direct children of `path`. 107 | 108 | ### #batch(ops, fn) 109 | 110 | Execute multiple `get` and `del` operations in one atomic batch. `ops` is an array with objects of type 111 | 112 | - `{ type: 'put', path: path, data: data }` 113 | - `{ type: 'del', path: path }` 114 | 115 | ## License 116 | 117 | MIT 118 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var Pathwise = require('./'); 2 | var level = require('level'); 3 | 4 | var db = level('db'); 5 | var store = new Pathwise(db); 6 | 7 | store.put([], { 8 | foo: { 9 | bar: ['beep', 'boop'], 10 | baz: 'bleep' 11 | } 12 | }, function(err){ 13 | if (err) throw err; 14 | 15 | store.get([], function(err, obj){ 16 | if (err) throw err; 17 | 18 | console.log('=> %j', obj); 19 | 20 | store.del(['foo', 'bar'], function(err){ 21 | if (err) throw err; 22 | 23 | store.get([], function(err, obj){ 24 | if (err) throw err; 25 | 26 | console.log('=> %j', obj); 27 | 28 | store.get(['foo', 'baz'], function(err, obj){ 29 | if (err) throw err; 30 | 31 | console.log('=> %j', obj); 32 | 33 | store.children(['foo', 'baz'], function(err, children){ 34 | if (err) throw err; 35 | 36 | console.log('=> %j', children); 37 | 38 | store.children([], function(err, children){ 39 | if (err) throw err; 40 | 41 | console.log('=> %j', children); 42 | 43 | store.batch([ 44 | { type: 'put', path: [], data: { i: { said: { what: 'yo' } } } }, 45 | { type: 'del', path: ['foo'] } 46 | ], function(err){ 47 | if (err) throw err; 48 | 49 | store.get([], function(err, obj){ 50 | if (err) throw err; 51 | 52 | console.log('=> %j', obj); 53 | }); 54 | }); 55 | }); 56 | }); 57 | }); 58 | }); 59 | }); 60 | }); 61 | }); 62 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var defaults = require('levelup-defaults'); 3 | var bytewise = require('bytewise'); 4 | var type = require('component-type'); 5 | var after = require('after'); 6 | var streamToArray = require('stream-to-array'); 7 | 8 | module.exports = Pathwise; 9 | 10 | function Pathwise(db){ 11 | assert(db, 'db required'); 12 | this._db = defaults(db, { 13 | keyEncoding: bytewise, 14 | valueEncoding: 'json' 15 | }); 16 | } 17 | 18 | Pathwise.prototype.put = function(path, obj, opts, fn){ 19 | if (typeof opts == 'function') { 20 | fn = opts; 21 | opts = {}; 22 | } 23 | if (!opts) { 24 | opts = {}; 25 | } 26 | if (!fn) { 27 | fn = () => {}; 28 | } 29 | var batch = opts.batch || this._db.batch(); 30 | this._write(batch, path, obj, fn); 31 | if (opts.batch) setImmediate(fn); 32 | else batch.write(fn); 33 | }; 34 | 35 | Pathwise.prototype._write = function(batch, key, obj, fn){ 36 | var self = this; 37 | switch(type(obj)) { 38 | case 'object': 39 | var keys = Object.keys(obj); 40 | var next = after(keys.length, fn); 41 | keys.forEach(function(k){ 42 | self._write(batch, key.concat(k), obj[k], next); 43 | }); 44 | break; 45 | case 'array': 46 | this._write(batch, key, arrToObj(obj), fn); 47 | break; 48 | default: 49 | batch.put(bytewise.encode(key), JSON.stringify(obj)); 50 | break; 51 | } 52 | }; 53 | 54 | Pathwise.prototype.batch = function(ops, fn) { 55 | var self = this; 56 | var batch = this._db.batch(); 57 | var next = after(ops.length, function(err){ 58 | if (err) return fn(err); 59 | batch.write(fn); 60 | }); 61 | ops.forEach(function(op){ 62 | if (op.type == 'put') self.put(op.path, op.data, { batch: batch }, next); 63 | else if (op.type == 'del') self.del(op.path, { batch: batch }, next); 64 | }); 65 | }; 66 | 67 | Pathwise.prototype.get = function(path, fn){ 68 | var ret = {}; 69 | var el = ret; 70 | 71 | streamToArray(this._db.createReadStream({ 72 | start: path, 73 | end: path.concat(undefined) 74 | }), function(err, data){ 75 | if (err) return fn(err); 76 | 77 | data.forEach(function(kv){ 78 | var segs = kv.key.slice(path.length); 79 | if (segs.length) { 80 | segs.forEach(function(seg, idx){ 81 | if (!el[seg]) { 82 | if (idx == segs.length - 1) { 83 | el[seg] = kv.value; 84 | } else { 85 | el[seg] = {}; 86 | } 87 | } 88 | el = el[seg]; 89 | }); 90 | el = ret; 91 | } else { 92 | ret = kv.value; 93 | } 94 | }); 95 | fn(null, ret); 96 | }); 97 | }; 98 | 99 | Pathwise.prototype.del = function(path, opts, fn){ 100 | if (typeof opts == 'function') { 101 | fn = opts; 102 | opts = {}; 103 | } 104 | var batch = opts.batch || this._db.batch(); 105 | 106 | streamToArray(this._db.createKeyStream({ 107 | start: path, 108 | end: path.concat(undefined) 109 | }), function(err, keys){ 110 | if (err) return fn(err); 111 | keys.forEach(function(key){ batch.del(bytewise.encode(key)) }); 112 | if (opts.batch) fn(); 113 | else batch.write(fn); 114 | }); 115 | }; 116 | 117 | Pathwise.prototype.children = function(path, fn) { 118 | streamToArray(this._db.createReadStream({ 119 | start: path, 120 | end: path.concat(undefined) 121 | }), function(err, kv){ 122 | if (err) return fn(err); 123 | fn(null, kv.map(function(_kv){ 124 | return _kv.key[path.length] || _kv.value; 125 | })); 126 | }); 127 | } 128 | 129 | function arrToObj(arr){ 130 | var obj = {}; 131 | arr.forEach(function(el, idx){ 132 | obj[idx] = el; 133 | }); 134 | return obj; 135 | } 136 | 137 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "level-pathwise", 3 | "version": "4.0.1", 4 | "description": "Turn a leveldb into one object of arbitrary size. Efficiently+atomically update+read parts", 5 | "scripts": { 6 | "test": "make test" 7 | }, 8 | "repository": "juliangruber/level-pathwise", 9 | "devDependencies": { 10 | "level": "^3.0.0", 11 | "level-mem": "^2.0.0", 12 | "tape": "^4.2.0" 13 | }, 14 | "license": "MIT", 15 | "dependencies": { 16 | "after": "^0.8.2", 17 | "bytewise": "^1.1.0", 18 | "component-type": "^1.2.1", 19 | "levelup-defaults": "^2.0.0", 20 | "stream-to-array": "^2.3.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var test = require('tape'); 2 | var Pathwise = require('./'); 3 | var level = require('level-mem'); 4 | var bytewise = require('bytewise'); 5 | 6 | test('Pathwise', function(t){ 7 | t.throws(function(){ 8 | new Pathwise(); 9 | }); 10 | t.ok(new Pathwise(level())); 11 | 12 | t.test('#put(path, obj, fn)', function(t){ 13 | t.test('object', function(t){ 14 | var p = new Pathwise(level()); 15 | var o = { foo: 'bar', bar: 'baz' }; 16 | p.put([], o, function(err){ 17 | t.error(err); 18 | p.get([], function(err, obj){ 19 | t.error(err); 20 | t.deepEqual(obj, o); 21 | t.end(); 22 | }); 23 | }); 24 | }); 25 | t.test('array', function(t){ 26 | var p = new Pathwise(level()); 27 | var a = ['foo', 'bar']; 28 | p.put([], a, function(err){ 29 | t.error(err); 30 | p.get([], function(err, array){ 31 | t.error(err); 32 | t.deepEqual(array, a); 33 | t.end(); 34 | }); 35 | }); 36 | }); 37 | t.test('other', function(t){ 38 | var p = new Pathwise(level()); 39 | p.put([], 3, function(err){ 40 | t.error(err); 41 | p.get([], function(err, other){ 42 | t.error(err); 43 | t.equal(other, 3); 44 | t.end(); 45 | }); 46 | }); 47 | }); 48 | t.test('integration', function(t){ 49 | var p = new Pathwise(level()); 50 | var o = { foo: 'bar', bar: ['baz', { yo: 9 }] }; 51 | p.put([], o, function(err){ 52 | t.error(err); 53 | p.get([], function(err, obj){ 54 | t.error(err); 55 | t.deepEqual(obj, o); 56 | t.end(); 57 | }); 58 | }); 59 | }); 60 | t.test('extend', function(t){ 61 | var p = new Pathwise(level()); 62 | p.put([], { foo: { hi: 'you' } }, function(err){ 63 | t.error(err); 64 | p.put([], { foo: { beep: 'boop' } }, function(err){ 65 | t.error(err); 66 | p.get([], function(err, obj){ 67 | t.error(err); 68 | t.deepEqual(obj, { 69 | foo: { 70 | hi: 'you', 71 | beep: 'boop' 72 | } 73 | }); 74 | t.end(); 75 | }); 76 | }); 77 | }); 78 | }); 79 | t.end(); 80 | }); 81 | 82 | t.test('#put(path, obj, { batch }, fn)', function(t){ 83 | var db = level(); 84 | var p = new Pathwise(db); 85 | var b = db.batch(); 86 | var nextTick = true; 87 | p.put([], { foo: 'bar', beep: 'boop' }, { batch: b }, function(err){ 88 | nextTick = false; 89 | t.error(err); 90 | t.deepEqual(b.ops, [ 91 | { type: 'put', key: bytewise.encode(['foo']), value: JSON.stringify('bar') }, 92 | { type: 'put', key: bytewise.encode(['beep']), value: JSON.stringify('boop') } 93 | ]); 94 | t.end(); 95 | }); 96 | t.ok(nextTick); 97 | }); 98 | 99 | t.test('#batch(ops, fn)', function(t){ 100 | var p = new Pathwise(level()); 101 | p.batch([{ type: 'put', path: [], data: 'hey' }], function(err){ 102 | t.error(err); 103 | p.get([], function(err, data){ 104 | t.error(err); 105 | t.equal(data, 'hey'); 106 | 107 | p.batch([{ type: 'del', path: [] }], function(err){ 108 | t.error(err); 109 | p.get([], function(err, data){ 110 | t.error(err); 111 | t.deepEqual(data, {}); 112 | t.end(); 113 | }); 114 | }); 115 | }); 116 | }); 117 | }); 118 | 119 | t.test('#get(path, fn)', function(t){ 120 | var p = new Pathwise(level()); 121 | var o = { foo: { bar: 'baz' } }; 122 | p.put([], o, function(err){ 123 | t.error(err); 124 | p.get([], function(err, data){ 125 | t.error(err); 126 | t.deepEqual(data, o); 127 | p.get(['foo'], function(err, data){ 128 | t.error(err); 129 | t.deepEqual(data, o.foo); 130 | p.get(['foo', 'bar'], function(err, data){ 131 | t.error(err); 132 | t.equal(data, o.foo.bar); 133 | t.end(); 134 | }); 135 | }); 136 | }); 137 | }); 138 | }); 139 | 140 | t.test('#del(path, fn)', function(t){ 141 | var p = new Pathwise(level()); 142 | p.put([], { foo: { bar: 'baz', beep: 'boop' } }, function(err){ 143 | t.error(err); 144 | p.del(['foo', 'bar'], function(err){ 145 | t.error(err); 146 | p.get([], function(err, data){ 147 | t.error(err); 148 | t.deepEqual(data, { foo: { beep: 'boop' } }); 149 | p.del([], function(err){ 150 | t.error(err); 151 | p.get([], function(err, data){ 152 | t.error(err); 153 | t.deepEqual(data, {}); 154 | t.end(); 155 | }); 156 | }); 157 | }); 158 | }); 159 | }); 160 | }); 161 | 162 | t.test('#del(path, { batch }, fn)', function(t){ 163 | var db = level(); 164 | var p = new Pathwise(db); 165 | p.put([], { foo: 'bar', beep: 'boop' }, function(err){ 166 | t.error(err); 167 | 168 | var b = db.batch(); 169 | var nextTick = true; 170 | p.del([], { batch: b }, function(err){ 171 | nextTick = false; 172 | t.deepEqual(b.ops, [ 173 | { type: 'del', key: bytewise.encode(['beep']) }, 174 | { type: 'del', key: bytewise.encode(['foo']) } 175 | ]); 176 | t.end(); 177 | }); 178 | t.ok(nextTick); 179 | }); 180 | }); 181 | 182 | t.test('#children(path, fn)', function(t){ 183 | var p = new Pathwise(level()); 184 | p.put([], { foo: { bar: 'baz' } }, function(err){ 185 | t.error(err); 186 | p.children([], function(err, children){ 187 | t.error(err); 188 | t.deepEqual(children, ['foo']); 189 | p.children(['foo'], function(err, children){ 190 | t.error(err); 191 | t.deepEqual(children, ['bar']); 192 | p.children(['foo', 'bar'], function(err, children){ 193 | t.error(err); 194 | t.deepEqual(children, ['baz']); 195 | t.end(); 196 | }); 197 | }); 198 | }); 199 | }); 200 | }); 201 | 202 | t.end(); 203 | }); 204 | --------------------------------------------------------------------------------