├── .gitignore ├── .travis.yml ├── test ├── helpers │ └── create.js ├── flags.js ├── history.js ├── update.js ├── closest.js ├── extension.js ├── reconnect.js ├── watch.js ├── hidden.js ├── deletes.js ├── collisions.js ├── iterator.js ├── conditions.js ├── diff.js └── basic.js ├── example.js ├── schema.proto ├── LICENSE ├── lib ├── watch.js ├── history.js ├── trie.js ├── batch.js ├── get.js ├── node.js ├── extension.js ├── del.js ├── put.js ├── iterator.js ├── diff.js └── messages.js ├── package.json ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | db 3 | sandbox.js 4 | sandbox 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "node" 5 | - "lts/*" 6 | - "10" 7 | cache: 8 | npm: false 9 | -------------------------------------------------------------------------------- /test/helpers/create.js: -------------------------------------------------------------------------------- 1 | const ram = require('random-access-memory') 2 | const hypertrie = require('../../') 3 | 4 | module.exports = function (key, opts) { 5 | return module.exports.create(key, opts) 6 | } 7 | 8 | module.exports.create = function (key, opts) { 9 | opts = Object.assign({ valueEncoding: 'json' }, opts) 10 | return hypertrie(ram, key, opts) 11 | } 12 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const hypertrie = require('./') 2 | const ram = require('random-access-memory') 3 | 4 | const db = hypertrie(ram) 5 | 6 | const batch = new Array(200) 7 | for (var i = 0; i < batch.length; i++) { 8 | batch[i] = {key: 'a/#' + i, value: '#' + i} 9 | } 10 | 11 | db.batch(batch, function () { 12 | db.createReadStream() 13 | .on('data', data => console.log(data.key)) 14 | .on('end', _ => console.log('(end)')) 15 | }) 16 | -------------------------------------------------------------------------------- /test/flags.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('deletes preverses nearby flags', function (t) { 5 | const db = create() 6 | 7 | db.put('ho', 'hi', function () { 8 | db.put('hi', 'ho', { flags: 100 }, function () { 9 | db.del('ho', function () { 10 | db.get('hi', function (err, node) { 11 | t.error(err, 'no error') 12 | t.same(node.flags, 100) 13 | t.end() 14 | }) 15 | }) 16 | }) 17 | }) 18 | }) 19 | -------------------------------------------------------------------------------- /schema.proto: -------------------------------------------------------------------------------- 1 | // The Header is DEP-0007 compatible. 2 | message Header { 3 | required string type = 1; 4 | optional bytes metadata = 2; 5 | optional string subtype = 3; 6 | } 7 | 8 | message Node { 9 | required string key = 1; 10 | optional bytes valueBuffer = 2; 11 | optional bytes trieBuffer = 3; 12 | optional uint64 seq = 4; 13 | optional uint64 flags = 5; 14 | } 15 | 16 | message Extension { 17 | message Get { 18 | optional uint64 head = 1; 19 | optional string key = 2; 20 | } 21 | 22 | message Iterator { 23 | optional uint64 head = 1; 24 | optional string key = 2; 25 | optional uint64 flags = 3; 26 | optional bytes checkpoint = 4; 27 | } 28 | 29 | message Cache { 30 | required uint64 start = 1; 31 | required uint64 end = 2; 32 | repeated uint64 blocks = 3 [packed=true]; 33 | } 34 | 35 | optional Cache cache = 1; 36 | optional Iterator iterator = 2; 37 | optional Get get = 3; 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/watch.js: -------------------------------------------------------------------------------- 1 | const inherits = require('inherits') 2 | const events = require('events') 3 | 4 | module.exports = Watch 5 | 6 | function Watch (db, prefix, onchange) { 7 | events.EventEmitter.call(this) 8 | 9 | this._db = db 10 | this._prefix = prefix 11 | this._destroyed = false 12 | this._closest = 0 13 | this._updated = false 14 | this._kicking = false 15 | this._index = 0 16 | 17 | if (onchange) this.on('change', onchange) 18 | this._db._addWatch(this) 19 | this.update() 20 | } 21 | 22 | inherits(Watch, events.EventEmitter) 23 | 24 | Watch.prototype.destroy = function () { 25 | this._db._removeWatch(this) 26 | this._destroyed = true 27 | } 28 | 29 | Watch.prototype.update = function () { 30 | if (this._destroyed) return 31 | if (!this._kicking) this._kick() 32 | else this._updated = true 33 | } 34 | 35 | Watch.prototype._done = function (closest) { 36 | this._kicking = false 37 | 38 | if (closest > this._closest) { 39 | this._closest = closest 40 | this._updated = false 41 | this.emit('change') 42 | return 43 | } 44 | 45 | if (this._updated) { 46 | this._updated = false 47 | this._kick() 48 | } 49 | } 50 | 51 | Watch.prototype._kick = function () { 52 | const self = this 53 | this._kicking = true 54 | this._db.get(this._prefix, {prefix: true}, done) 55 | 56 | function done (_, node) { 57 | self._done(node ? node.seq : 0) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hypertrie", 3 | "version": "5.1.3", 4 | "description": "Distributed single writer key/value store", 5 | "main": "index.js", 6 | "dependencies": { 7 | "array-lru": "^1.1.1", 8 | "bulk-write-stream": "^1.1.4", 9 | "codecs": "^2.0.0", 10 | "hypercore": "^9.4.1", 11 | "hypercore-protocol": "^8.0.0", 12 | "inherits": "^2.0.3", 13 | "inspect-custom-symbol": "^1.1.0", 14 | "is-options": "^1.0.1", 15 | "mutexify": "^1.2.0", 16 | "nanoiterator": "^1.2.0", 17 | "protocol-buffers-encodings": "^1.1.0", 18 | "siphash24-universal": "^1.0.0", 19 | "thunky": "^1.0.2", 20 | "unordered-set": "^2.0.1", 21 | "varint": "^5.0.0" 22 | }, 23 | "devDependencies": { 24 | "compare": "^2.0.0", 25 | "protocol-buffers": "^4.0.4", 26 | "random-access-memory": "^3.0.0", 27 | "standard": "^11.0.1", 28 | "stream-collector": "^1.0.1", 29 | "tape": "^4.9.0" 30 | }, 31 | "scripts": { 32 | "test": "standard && tape test/*.js", 33 | "protobuf": "protocol-buffers schema.proto -o lib/messages.js" 34 | }, 35 | "repository": { 36 | "type": "git", 37 | "url": "https://github.com/mafintosh/hypertrie.git" 38 | }, 39 | "author": "Mathias Buus (@mafintosh)", 40 | "license": "MIT", 41 | "bugs": { 42 | "url": "https://github.com/mafintosh/hypertrie/issues" 43 | }, 44 | "homepage": "https://github.com/mafintosh/hypertrie#readme" 45 | } 46 | -------------------------------------------------------------------------------- /lib/history.js: -------------------------------------------------------------------------------- 1 | const Nanoiterator = require('nanoiterator') 2 | const inherits = require('inherits') 3 | 4 | module.exports = History 5 | 6 | function History (db, opts) { 7 | if (!opts) opts = {} 8 | if (typeof opts.gt === 'number') opts.gte = opts.gt + 1 9 | if (typeof opts.lt === 'number') opts.lte = opts.lt - 1 10 | 11 | Nanoiterator.call(this) 12 | 13 | this._gte = ifNumber(opts.gte, 1) 14 | this._lte = ifNumber(opts.lte, -1) 15 | this._reverse = !!(opts && opts.reverse) 16 | this._db = db 17 | this._live = !!(opts && opts.live) 18 | } 19 | 20 | inherits(History, Nanoiterator) 21 | 22 | History.prototype._open = function (cb) { 23 | const self = this 24 | 25 | if (this._live && !this._reverse) { 26 | this._lte = Infinity 27 | return cb(null) 28 | } 29 | 30 | this._db.head(onhead) 31 | 32 | function onhead (err, head) { 33 | if (err) return cb(err) 34 | const headSeq = head ? head.seq : -1 35 | self._lte = self._lte === -1 ? headSeq : Math.min(self._lte, headSeq) 36 | cb(null) 37 | } 38 | } 39 | 40 | History.prototype._next = function (cb) { 41 | if (this._gte > this._lte) return cb(null, null) 42 | this._db.getBySeq(this._reverse ? this._lte-- : this._gte++, done) 43 | 44 | function done (err, node) { 45 | if (err) return cb(err) 46 | if (!node) return cb(null, null) 47 | cb(null, node.final()) 48 | } 49 | } 50 | 51 | function ifNumber (n, def) { 52 | return typeof n === 'number' ? n : def 53 | } 54 | -------------------------------------------------------------------------------- /test/history.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('basic history', function (t) { 5 | const db = create() 6 | 7 | db.put('hello', 'world', function (err) { 8 | t.error(err, 'no error') 9 | 10 | const ite = db.history() 11 | 12 | ite.next(function (err, data) { 13 | t.error(err, 'no error') 14 | t.same(data.key, 'hello') 15 | t.same(data.value, 'world') 16 | }) 17 | ite.next(function (err, data) { 18 | t.error(err, 'no error') 19 | t.same(data, null) 20 | t.end() 21 | }) 22 | }) 23 | }) 24 | 25 | tape('multiple entries history', function (t) { 26 | const db = create() 27 | 28 | db.put('hello', 'hi', function (err) { 29 | t.error(err, 'no error') 30 | db.put('hi', 'ho', function (err) { 31 | t.error(err, 'no error') 32 | db.put('hello', 'world', function (err) { 33 | t.error(err, 'no error') 34 | 35 | const ite = db.history() 36 | 37 | ite.next(function (err, data) { 38 | t.error(err, 'no error') 39 | t.same(data.key, 'hello') 40 | t.same(data.value, 'hi') 41 | }) 42 | ite.next(function (err, data) { 43 | t.error(err, 'no error') 44 | t.same(data.key, 'hi') 45 | t.same(data.value, 'ho') 46 | }) 47 | ite.next(function (err, data) { 48 | t.error(err, 'no error') 49 | t.same(data.key, 'hello') 50 | t.same(data.value, 'world') 51 | }) 52 | ite.next(function (err, data) { 53 | t.error(err, 'no error') 54 | t.same(data, null) 55 | t.end() 56 | }) 57 | }) 58 | }) 59 | }) 60 | }) 61 | -------------------------------------------------------------------------------- /lib/trie.js: -------------------------------------------------------------------------------- 1 | const varint = require('varint') 2 | 3 | exports.encode = function (trie) { 4 | const buf = Buffer.alloc(65536) 5 | var i, j 6 | var offset = 0 7 | 8 | varint.encode(trie.length, buf, offset) 9 | offset += varint.encode.bytes 10 | 11 | for (i = 0; i < trie.length; i++) { 12 | const bucket = trie[i] 13 | if (!bucket) continue 14 | 15 | var bit = 1 16 | var bitfield = 0 17 | 18 | varint.encode(i, buf, offset) 19 | offset += varint.encode.bytes 20 | 21 | for (j = 0; j < bucket.length; j++) { 22 | const seq = bucket[j] 23 | if (seq) bitfield |= bit 24 | bit *= 2 25 | } 26 | 27 | varint.encode(bitfield, buf, offset) 28 | offset += varint.encode.bytes 29 | 30 | for (j = 0; j < bucket.length; j++) { 31 | const seq = bucket[j] 32 | if (seq) { 33 | varint.encode(seq, buf, offset) 34 | offset += varint.encode.bytes 35 | } 36 | } 37 | } 38 | 39 | return buf.slice(0, offset) 40 | } 41 | 42 | exports.decode = function (buf) { 43 | var offset = 0 44 | 45 | const len = varint.decode(buf, offset) 46 | offset += varint.decode.bytes 47 | 48 | const trie = new Array(len) 49 | 50 | while (offset < buf.length) { 51 | const i = varint.decode(buf, offset) 52 | offset += varint.decode.bytes 53 | 54 | var bitfield = varint.decode(buf, offset) 55 | var pos = 0 56 | 57 | const bucket = trie[i] = new Array(32 - Math.clz32(bitfield)) 58 | offset += varint.decode.bytes 59 | 60 | while (bitfield) { 61 | const bit = bitfield & 1 62 | 63 | if (bit) { 64 | bucket[pos] = varint.decode(buf, offset) 65 | offset += varint.decode.bytes 66 | } 67 | 68 | bitfield = (bitfield - bit) / 2 69 | pos++ 70 | } 71 | } 72 | 73 | return trie 74 | } 75 | -------------------------------------------------------------------------------- /lib/batch.js: -------------------------------------------------------------------------------- 1 | const Put = require('./put') 2 | const Delete = require('./del') 3 | 4 | module.exports = Batch 5 | 6 | function Batch (db, ops, cb) { 7 | this._db = db 8 | this._ops = ops 9 | this._callback = cb 10 | this._head = null 11 | this._nodes = [] 12 | this._offset = 0 13 | this._op = null 14 | this._start() 15 | } 16 | 17 | Batch.prototype.get = function (seq) { 18 | if (seq < this._offset) return null 19 | return this._nodes[seq - this._offset] 20 | } 21 | 22 | Batch.prototype.head = function () { 23 | return this._head 24 | } 25 | 26 | Batch.prototype.append = function (node) { 27 | node.seq = this._offset + this._nodes.length 28 | node.preencode() 29 | this._nodes.push(node) 30 | } 31 | 32 | Batch.prototype._finalize = function (err) { 33 | const self = this 34 | if (err) return done(err) 35 | 36 | const buffers = new Array(this._nodes.length) 37 | for (var i = 0; i < buffers.length; i++) { 38 | buffers[i] = this._nodes[i].encode() 39 | } 40 | 41 | this._db.feed.append(buffers, done) 42 | 43 | function done (err) { 44 | self._release(self._callback, err, self._nodes) 45 | } 46 | } 47 | 48 | Batch.prototype._start = function () { 49 | const self = this 50 | this._db._lock(function (release) { 51 | self._release = release 52 | self._db.ready(function () { 53 | self._offset = self._db.feed.length 54 | self._db.head(function (err, head) { 55 | if (err) return self._finalize(err) 56 | self._head = head 57 | self._update() 58 | }) 59 | }) 60 | }) 61 | } 62 | 63 | Batch.prototype._update = function () { 64 | var i = 0 65 | const self = this 66 | 67 | loop(null, null) 68 | 69 | function loop (err, head) { 70 | if (err) return self._finalize(err) 71 | if (i === self._ops.length) return self._finalize(null) 72 | if (head) self._head = head 73 | 74 | const {type, key, value, hidden, flags} = self._ops[i++] 75 | if (type === 'del') self._op = new Delete(self._db, key, { batch: self, hidden }, loop) 76 | else self._op = new Put(self._db, key, value === undefined ? null : value, { batch: self, del: 0, hidden, flags }, loop) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /test/update.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | const HypercoreProtocol = require('hypercore-protocol') 4 | 5 | tape('get without alwaysUpdate returns null', t => { 6 | const trie1 = create() 7 | var trie2 = null 8 | 9 | trie1.ready(() => { 10 | trie2 = create(trie1.key) 11 | trie1.put('a', 'b', () => { 12 | trie2.get('a', (err, node) => { 13 | t.error(err, 'no error') 14 | t.same(node, null) 15 | t.end() 16 | }) 17 | }) 18 | replicate(trie1, trie2, { live: true }) 19 | }) 20 | }) 21 | 22 | tape('get with alwaysUpdate will wait for an update', t => { 23 | const trie1 = create({ alwaysUpdate: true }) 24 | var trie2 = null 25 | 26 | trie1.ready(() => { 27 | trie2 = create(trie1.key, { alwaysUpdate: true, valueEncoding: 'utf8' }) 28 | trie1.put('a', 'b', () => { 29 | trie2.get('a', (err, node) => { 30 | t.error(err, 'no error') 31 | t.same(node.key, 'a') 32 | t.same(node.value, 'b') 33 | t.end() 34 | }) 35 | }) 36 | replicate(trie1, trie2, { live: true }) 37 | }) 38 | }) 39 | 40 | tape('replication with an empty peer is not problematic', t => { 41 | const trie1 = create({ alwaysUpdate: true }) 42 | const emptyPeer = { 43 | replicate: (isInitiator, opts) => { 44 | const stream = new HypercoreProtocol(isInitiator, { ...opts, live: true }) 45 | stream.on('discovery-key', dkey => { 46 | stream.close(dkey) 47 | }) 48 | return stream 49 | } 50 | } 51 | var trie2 = null 52 | 53 | trie1.ready(() => { 54 | trie2 = create(trie1.key, { alwaysUpdate: true, valueEncoding: 'utf8' }) 55 | trie1.put('a', 'b', () => { 56 | trie2.get('a', (err, node) => { 57 | t.error(err, 'no error') 58 | t.same(node.key, 'a') 59 | t.same(node.value, 'b') 60 | t.end() 61 | }) 62 | }) 63 | replicate(trie1, trie2, { live: true }) 64 | replicate(trie1, emptyPeer, { live: true }) 65 | }) 66 | }) 67 | 68 | function replicate (trie1, trie2, opts) { 69 | const stream = trie1.replicate(true, opts) 70 | return stream.pipe(trie2.replicate(false, opts)).pipe(stream) 71 | } 72 | -------------------------------------------------------------------------------- /test/closest.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('closest put node is a prefix', function (t) { 5 | t.plan(4) 6 | 7 | const db = create() 8 | 9 | db.put('a', 'hello', err => { 10 | t.error(err, 'no error') 11 | db.put('b', 'goodbye', err => { 12 | t.error(err, 'no error') 13 | db.put('a/b', 'something', { 14 | closest: true, 15 | condition: (closest, newNode, cb) => { 16 | t.same(closest.key, 'a') 17 | return cb(null, true) 18 | } 19 | }, err => { 20 | t.error(err, 'no error') 21 | }) 22 | }) 23 | }) 24 | }) 25 | 26 | tape('closest put node is a prefix, batch insertion with flags', function (t) { 27 | t.plan(3) 28 | 29 | const db = create() 30 | 31 | db.batch([ 32 | { type: 'put', key: 'a', value: 'hello', flags: 1 }, 33 | { type: 'put', key: 'b', value: 'goodbye' } 34 | ], err => { 35 | t.error(err, 'no error') 36 | db.put('a/b', 'something', { 37 | closest: true, 38 | condition: (closest, newNode, cb) => { 39 | t.same(closest.key, 'a') 40 | return cb(null, true) 41 | } 42 | }, err => { 43 | t.error(err, 'no error') 44 | }) 45 | }) 46 | }) 47 | 48 | tape('closest node to non-existing delete path is a prefix', function (t) { 49 | t.plan(4) 50 | 51 | const db = create() 52 | 53 | db.batch([ 54 | { type: 'put', key: 'a', value: 'hello' }, 55 | { type: 'put', key: 'b', value: 'goodbye' }, 56 | { type: 'put', key: 'a/a', value: 'world' }, 57 | { type: 'put', key: 'b/a', value: 'dog' }, 58 | { type: 'put', key: 'b/b', value: 'otter' } 59 | ], err => { 60 | t.error(err, 'no error') 61 | db.del('a/a/b', { 62 | closest: true, 63 | condition: (closest, cb) => { 64 | t.same(closest.key, 'a/a') 65 | return cb(null, true) 66 | } 67 | }, (err, node) => { 68 | t.error(err, 'no error') 69 | t.false(node) 70 | }) 71 | }) 72 | }) 73 | 74 | // TODO: Fix this issue. 75 | tape.skip('closest node with one hidden node', function (t) { 76 | t.plan(3) 77 | 78 | const db = create() 79 | 80 | db.put('a', 'hello', { hidden: true }, err => { 81 | t.error(err, 'no error') 82 | db.get('b', { closest: true }, (err, node) => { 83 | t.error(err, 'no error') 84 | t.false(node) 85 | }) 86 | }) 87 | }) 88 | -------------------------------------------------------------------------------- /test/extension.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('extension get', function (t) { 5 | t.plan(5) 6 | 7 | const db = create() 8 | 9 | db.on('extension-get', function () { 10 | t.pass('got extension message') 11 | }) 12 | 13 | db.batch([ 14 | { key: 'hi', value: 'hi' }, 15 | { key: 'hi1', value: 'hi' }, 16 | { key: 'hi2', value: 'hi' }, 17 | { key: 'hi3', value: 'hi' }, 18 | { key: 'hi4', value: 'hi' } 19 | ], function () { 20 | const clone = create(db.key, { alwaysUpdate: true, sparse: true }) 21 | 22 | replicate(db, clone) 23 | 24 | clone.get('hi', function (err, node) { 25 | t.error(err, 'no error') 26 | t.same(node.value, 'hi') 27 | clone.get('hi', function (err, node) { 28 | t.error(err, 'no error') 29 | t.same(node.value, 'hi') 30 | }) 31 | }) 32 | }) 33 | }) 34 | 35 | tape('extension iterator', function (t) { 36 | t.plan(4) 37 | 38 | const db = create() 39 | 40 | db.on('extension-iterator', function () { 41 | t.pass('got extension message') 42 | }) 43 | 44 | db.batch([ 45 | { key: 'hi', value: 'hi' }, 46 | { key: 'hi1', value: 'hi' }, 47 | { key: 'hi2', value: 'hi' }, 48 | { key: 'hi3', value: 'hi' }, 49 | { key: 'hi4', value: 'hi' } 50 | ], function () { 51 | const clone = create(db.key, { alwaysUpdate: true, sparse: true }) 52 | 53 | replicate(db, clone) 54 | 55 | const ite = clone.iterator() 56 | ite.next(function (err, data) { 57 | t.error(err, 'no error') 58 | t.ok(!!data, 'got data') 59 | 60 | clone.iterator().next(function () { 61 | t.pass('same iteration') 62 | }) 63 | }) 64 | }) 65 | }) 66 | 67 | tape('extension sparse mitm', function (t) { 68 | t.plan(1) 69 | 70 | const db = create() 71 | 72 | db.on('extension-get', function () { 73 | t.fail('got extension message') 74 | }) 75 | 76 | db.batch([ 77 | { key: 'hi', value: 'hi' }, 78 | { key: 'hi1', value: 'hi' }, 79 | { key: 'hi2', value: 'hi' }, 80 | { key: 'hi3', value: 'hi' }, 81 | { key: 'hi4', value: 'hi' } 82 | ], function () { 83 | const clone = create(db.key, { alwaysUpdate: true, sparse: true }) 84 | const clone2 = create(db.key, { alwaysUpdate: true, sparse: true }) 85 | 86 | clone.on('extension-get', function () { 87 | t.pass('got extension message') 88 | }) 89 | 90 | replicate(db, clone) 91 | replicate(clone, clone2) 92 | 93 | clone.get('hi4', { extension: false }, function () { 94 | clone.feed.on('download', function () { 95 | t.fail('mitm downloading') 96 | }) 97 | clone2.get('hi', () => {}) 98 | clone2.iterator().next(() => {}) 99 | }) 100 | }) 101 | }) 102 | 103 | function replicate (a, b) { 104 | const s = a.replicate(true, { live: true }) 105 | s.pipe(b.replicate(false, { live: true })).pipe(s) 106 | } 107 | -------------------------------------------------------------------------------- /test/reconnect.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('reconnecting', function (t) { 5 | const db = create() 6 | 7 | db.put('hello', 'world', function () { 8 | const clone = create(db.key, { sparse: true, alwaysUpdate: true, alwaysReconnect: true }) 9 | 10 | const s = db.replicate(true, { live: true }) 11 | 12 | s.pipe(clone.replicate(false, { live: true })).pipe(s) 13 | 14 | clone.on('reconnected', function () { 15 | clone.feed.on('download', function () { 16 | t.fail('should not need to download any data') 17 | }) 18 | clone.get('hello', function (_, node) { 19 | t.same(node.value, 'verden') 20 | t.end() 21 | }) 22 | }) 23 | 24 | clone.get('hello', function (err, node) { 25 | t.error(err, 'no error') 26 | t.same(node.value, 'world') 27 | 28 | db.batch([ 29 | { key: 'hello', value: 'verden' }, 30 | { key: 'foo', value: 'bar' }, 31 | { key: 'bar', value: 'baz' }, 32 | { key: 'baz', value: 'foo' } 33 | ], function (err) { 34 | t.error(err, 'no error') 35 | 36 | clone.get('baz', function (err, node) { 37 | t.error(err, 'no error') 38 | t.same(node.value, 'foo') 39 | }) 40 | }) 41 | }) 42 | }) 43 | }) 44 | 45 | tape('reconnecting twice', function (t) { 46 | const db = create() 47 | 48 | db.put('hello', 'world', function () { 49 | const clone = create(db.key, { sparse: true, alwaysUpdate: true, alwaysReconnect: true }) 50 | 51 | const s = db.replicate(true, { live: true }) 52 | 53 | s.pipe(clone.replicate(false, { live: true })).pipe(s) 54 | 55 | clone.once('reconnected', function () { 56 | clone.get('hello', function (_, node) { 57 | t.same(node.value, 'verden') 58 | 59 | const puts = new Array(128) 60 | 61 | for (let i = 0; i < puts.length; i++) { 62 | puts[i] = { key: '#' + i, value: i } 63 | } 64 | 65 | db.batch(puts, function () { 66 | let dl = 0 67 | 68 | clone.once('reconnected', function () { 69 | t.ok(dl < puts.length / 2, 'small diff') 70 | t.ok(dl > 1, 'diff larger than one message') 71 | 72 | clone.feed.on('download', function () { 73 | t.fail('should not need to download any data') 74 | }) 75 | 76 | clone.get('hello', function (_, node) { 77 | t.same(node.value, 'verden') 78 | t.end() 79 | }) 80 | }) 81 | 82 | clone.feed.on('download', function () { 83 | dl++ 84 | }) 85 | 86 | clone.get('baz', () => {}) 87 | }) 88 | }) 89 | }) 90 | 91 | clone.get('hello', function (err, node) { 92 | t.error(err, 'no error') 93 | t.same(node.value, 'world') 94 | 95 | db.batch([ 96 | { key: 'hello', value: 'verden' }, 97 | { key: 'foo', value: 'bar' }, 98 | { key: 'bar', value: 'baz' }, 99 | { key: 'baz', value: 'foo' } 100 | ], function (err) { 101 | t.error(err, 'no error') 102 | clone.get('baz', () => {}) 103 | }) 104 | }) 105 | }) 106 | }) 107 | -------------------------------------------------------------------------------- /test/watch.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('basic watch', function (t) { 5 | const db = create() 6 | 7 | db.watch(function () { 8 | t.pass('watch triggered') 9 | t.end() 10 | }) 11 | 12 | db.put('hello', 'world') 13 | }) 14 | 15 | tape('watch prefix', function (t) { 16 | const db = create() 17 | var changed = false 18 | 19 | db.watch('foo', function () { 20 | t.ok(changed) 21 | t.end() 22 | }) 23 | 24 | db.put('hello', 'world', function (err) { 25 | t.error(err) 26 | setImmediate(function () { 27 | changed = true 28 | db.put('foo/bar', 'baz') 29 | }) 30 | }) 31 | }) 32 | 33 | tape('recursive watch', function (t) { 34 | t.plan(20) 35 | 36 | const db = create() 37 | var i = 0 38 | 39 | db.watch('foo', function () { 40 | if (i === 20) return 41 | t.pass('watch triggered') 42 | db.put('foo', 'bar-' + (++i)) 43 | }) 44 | 45 | db.put('foo', 'bar') 46 | }) 47 | 48 | tape('watch and stop watching', function (t) { 49 | const db = create() 50 | var once = true 51 | 52 | const w = db.watch('foo', function () { 53 | t.ok(once) 54 | once = false 55 | w.destroy() 56 | db.put('foo/bar/baz', 'qux', function () { 57 | t.end() 58 | }) 59 | }) 60 | 61 | db.put('foo/bar', 'baz') 62 | }) 63 | 64 | tape('watch, watch and stop watch first one', function (t) { 65 | t.plan(5 + 1) 66 | 67 | const db = create() 68 | var bs = 5 69 | 70 | db.ready(function () { 71 | const clone = create(db.key, { sparse: true }) 72 | 73 | const a = clone.watch('foo', function () { 74 | t.pass('got a update') 75 | a.destroy() 76 | }) 77 | 78 | const b = clone.watch('foo', function () { 79 | t.pass('got b update') 80 | 81 | if (!--bs) { 82 | b.destroy() 83 | t.end() 84 | return 85 | } 86 | 87 | db.put('foo/bar', 'baz') 88 | }) 89 | 90 | db.put('foo/bar', 'baz') 91 | 92 | const stream = db.replicate(true, { live: true }) 93 | stream.pipe(clone.replicate(false, { live: true })).pipe(stream) 94 | }) 95 | }) 96 | 97 | tape('remote watch', function (t) { 98 | const db = create() 99 | 100 | db.ready(function () { 101 | const clone = create(db.key) 102 | 103 | for (var i = 0; i < 100; i++) db.put('hello-' + i, 'world-' + i) 104 | db.put('flush', 'flush', function () { 105 | clone.watch(function () { 106 | t.pass('remote watch triggered') 107 | t.end() 108 | }) 109 | 110 | const stream = db.replicate(true) 111 | stream.pipe(clone.replicate(false)).pipe(stream) 112 | }) 113 | }) 114 | }) 115 | 116 | tape('remote watch with sparse mode and live replication', function (t) { 117 | const db = create(null, { sparse: true }) 118 | 119 | db.ready(function () { 120 | const clone = create(db.key, { alwaysUpdate: true, sparse: true }) 121 | // The ready triggers an update, which we do not want to trigger a watch event, so we must call this first. 122 | clone.ready(function () { 123 | const stream = db.replicate(true, { live: true }) 124 | stream.pipe(clone.replicate(false, { live: true })).pipe(stream) 125 | 126 | var watcher = clone.watch(function () { 127 | t.pass('remote watch triggered') 128 | watcher.destroy() 129 | t.end() 130 | }) 131 | 132 | setTimeout(function () { 133 | db.put('hello', 'world') 134 | }, 50) 135 | }) 136 | }) 137 | }) 138 | -------------------------------------------------------------------------------- /lib/get.js: -------------------------------------------------------------------------------- 1 | const Node = require('./node') 2 | 3 | module.exports = Get 4 | 5 | function Get (db, key, opts, cb) { 6 | this._db = db 7 | this._node = new Node({key, flags: (opts && opts.hidden) ? Node.Flags.HIDDEN : 0}, 0, null, db.hash) 8 | this._callback = cb 9 | this._prefix = !!(opts && opts.prefix) 10 | this._closest = !!(opts && opts.closest) 11 | this._length = this._node.length - (this._prefix ? 1 : 0) 12 | this._onseq = (opts && opts.onseq) || null 13 | this._options = opts ? { wait: opts.wait, timeout: opts.timeout, onwait: null } : { onwait: null } 14 | this._extension = (!opts || opts.extension !== false) ? this._db._extension : null 15 | this._extensionSent = !this._extension 16 | this._head = 0 17 | this._onheadseq = (opts && opts.onheadseq) || null 18 | this._options.onwait = this._extension ? this._sendExt.bind(this) : ((opts && opts.onwait) || null) 19 | 20 | this._start() 21 | } 22 | 23 | Get.prototype._start = function () { 24 | const self = this 25 | this._db.headSeq(this._options, onheadseq) 26 | 27 | function onheadseq (err, seq) { 28 | if (err) return self._callback(err, null) 29 | if (!seq) return onhead(null, null) 30 | 31 | if (self._onheadseq) self._onheadseq(seq) 32 | self._head = seq 33 | 34 | if (self._onseq) self._onseq(seq) 35 | self._db.getBySeq(seq, self._options, onhead) 36 | } 37 | 38 | function onhead (err, head) { 39 | if (err) return self._callback(err, null) 40 | self._update(0, head) 41 | } 42 | } 43 | 44 | Get.prototype._sendExt = function () { 45 | if (this._head && this._extension && !this._extensionSent) { 46 | this._options.onwait = null 47 | this._extensionSent = true 48 | this._extension.get(this._head, this._node.key) 49 | } 50 | } 51 | 52 | Get.prototype._update = function (i, head) { 53 | if (!head) return this._callback(null, null) 54 | 55 | const node = this._node 56 | 57 | for (; i < this._length; i++) { 58 | const val = node.path(i) 59 | const checkCollision = Node.terminator(i) 60 | 61 | if (head.path(i) === val) { 62 | if (!checkCollision || !node.collides(head, i)) continue 63 | } 64 | 65 | const bucket = head.trie[i] || [] 66 | 67 | if (checkCollision) return this._updateHeadCollides(i, bucket, val) 68 | 69 | const seq = bucket[val] 70 | if (!seq) return this._callback(null, this._closest ? head.final() : null) 71 | 72 | return this._updateHead(i, seq) 73 | } 74 | 75 | this._callback(null, head.final()) 76 | } 77 | 78 | Get.prototype._updateHeadCollides = function (i, bucket, val) { 79 | const self = this 80 | var missing = 1 81 | var error = null 82 | var node = null 83 | 84 | for (var j = val; j < bucket.length; j += 5) { 85 | const seq = bucket[j] 86 | if (!seq) break 87 | missing++ 88 | 89 | if (this._onseq) this._onseq(seq) 90 | this._db.getBySeq(seq, this._options, onnode) 91 | } 92 | 93 | onnode(null, null) 94 | 95 | function onnode (err, n) { 96 | if (err) error = err 97 | else if (n && !n.collides(self._node, i)) node = n 98 | if (--missing) return 99 | 100 | if (!node || error) return self._callback(error, this._closest ? this._node : null) 101 | self._update(i + 1, node) 102 | } 103 | } 104 | 105 | Get.prototype._updateHead = function (i, seq) { 106 | const self = this 107 | 108 | if (this._onseq) this._onseq(seq) 109 | this._db.getBySeq(seq, this._options, onnode) 110 | 111 | function onnode (err, node) { 112 | if (err) return self._callback(err, null) 113 | self._update(i + 1, node) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /lib/node.js: -------------------------------------------------------------------------------- 1 | const siphash24 = require('siphash24-universal') 2 | const inspect = require('inspect-custom-symbol') 3 | const messages = require('./messages') 4 | const trie = require('./trie') 5 | 6 | const KEY = Buffer.alloc(16) 7 | 8 | module.exports = Node 9 | 10 | const Flags = { 11 | HIDDEN: 1 12 | } 13 | 14 | function Node (data, seq, enc, userHash) { 15 | this.seq = seq || 0 16 | this.key = normalizeKey(data.key) 17 | this.value = data.value !== undefined ? data.value : null 18 | this.keySplit = split(this.key) 19 | this.hash = userHash ? userHash(this.key) : hash(this.keySplit) 20 | this.trie = data.trieBuffer ? trie.decode(data.trieBuffer) : (data.trie || []) 21 | this.trieBuffer = null 22 | this.valueBuffer = data.valueBuffer || null 23 | this.length = this.hash.length * 4 + 1 + 1 24 | this.valueEncoding = enc 25 | 26 | this._finalized = false 27 | this._flags = data.flags || 0 28 | this.flags = this._flags 29 | } 30 | Node.Flags = Flags 31 | 32 | Node.prototype[inspect] = function (depth, opts) { 33 | return ((opts && opts.stylize) || defaultStylize)({seq: this.seq, key: this.key, value: this.value}, 'object') 34 | } 35 | 36 | Object.defineProperty(Node.prototype, 'hidden', { 37 | enumerable: true, 38 | get: function () { 39 | return !!(this._flags & Flags.HIDDEN) 40 | } 41 | }) 42 | 43 | Node.prototype.path = function (i) { 44 | if (!i) return this.hidden ? 1 : 0 45 | i-- 46 | const hash = this.hash 47 | const j = i >> 2 48 | if (j >= hash.length) return 4 49 | return (hash[j] >> (2 * (i & 3))) & 3 50 | } 51 | 52 | Node.prototype.compare = function (other) { 53 | const min = Math.min(this.length, other.length) 54 | for (var i = 0; i < min; i++) { 55 | const diff = this.path(i) - other.path(i) 56 | if (diff !== 0) return diff 57 | } 58 | return 0 59 | } 60 | 61 | Node.prototype.final = function () { 62 | if (this._finalized) return this 63 | 64 | if (this.valueBuffer === null) this.value = null 65 | else this.value = this.valueEncoding ? this.valueEncoding.decode(this.valueBuffer) : this.valueBuffer 66 | 67 | // The flags are shifted in order to both hide the internal flags and support user-defined flags. 68 | this.flags = this._flags >> 8 69 | 70 | this._finalized = true 71 | return this 72 | } 73 | 74 | Node.prototype.preencode = function () { 75 | if (!this.trieBuffer) this.trieBuffer = trie.encode(this.trie) 76 | if (!this.valueBuffer) this.valueBuffer = ((this.value !== null) && this.valueEncoding) ? this.valueEncoding.encode(this.value) : this.value 77 | } 78 | 79 | Node.prototype.encode = function () { 80 | this.preencode() 81 | return messages.Node.encode(this) 82 | } 83 | 84 | Node.prototype.collides = function (node, i) { 85 | if (!i) return false 86 | if (i === this.length - 1) return this.key !== node.key 87 | const j = Math.floor((i - 1) / 32) 88 | return this.keySplit[j] !== node.keySplit[j] 89 | } 90 | 91 | Node.decode = function (buf, seq, enc, hash) { 92 | return new Node(messages.Node.decode(buf), seq, enc, hash) 93 | } 94 | 95 | Node.terminator = function (i) { 96 | return i > 0 && (i & 31) === 0 97 | } 98 | 99 | Node.normalizeKey = normalizeKey 100 | 101 | function hash (keys) { 102 | const buf = Buffer.allocUnsafe(8 * keys.length) 103 | 104 | for (var i = 0; i < keys.length; i++) { 105 | const key = Buffer.from(keys[i]) 106 | const j = i * 8 107 | siphash24(buf.slice(j, j + 8), key, KEY) 108 | } 109 | 110 | return buf 111 | } 112 | 113 | function split (key) { 114 | const list = key.split('/') 115 | if (list[0] === '') list.shift() 116 | if (list[list.length - 1] === '') list.pop() 117 | return list 118 | } 119 | 120 | function normalizeKey (key) { 121 | if (!key.length) return '' 122 | return key[0] === '/' ? key.slice(1) : key 123 | } 124 | 125 | function defaultStylize (val) { 126 | return val 127 | } 128 | -------------------------------------------------------------------------------- /lib/extension.js: -------------------------------------------------------------------------------- 1 | const MAX_ACTIVE = 32 2 | const FLUSH_BATCH = 128 3 | const MAX_PASSIVE_BATCH = 2048 4 | const MAX_ACTIVE_BATCH = MAX_PASSIVE_BATCH + FLUSH_BATCH 5 | 6 | const { Extension } = require('./messages') 7 | 8 | class Batch { 9 | constructor (outgoing, from) { 10 | this.blocks = [] 11 | this.start = 0 12 | this.end = 0 13 | this.outgoing = outgoing 14 | this.from = from 15 | } 16 | 17 | push (seq) { 18 | const len = this.blocks.push(seq) 19 | if (len === 1 || seq < this.start) this.start = seq 20 | if (len === 1 || seq >= this.end) this.end = seq + 1 21 | if (len >= FLUSH_BATCH) { 22 | this.send() 23 | this.clear() 24 | } 25 | } 26 | 27 | send () { 28 | if (!this.blocks.length) return 29 | this.outgoing.send(Extension.encode({ cache: { blocks: this.blocks, start: this.start, end: this.end } }), this.from) 30 | } 31 | 32 | clear () { 33 | this.start = this.end = 0 34 | this.blocks = [] 35 | } 36 | } 37 | 38 | class HypertrieExtension { 39 | constructor (trie) { 40 | this.encoding = null 41 | this.outgoing = null 42 | this.trie = trie 43 | this.active = 0 44 | } 45 | 46 | onmessage (buf, from) { 47 | const message = decode(buf) 48 | 49 | if (!message) return 50 | if (message.cache) this.oncache(message.cache, from) 51 | if (message.iterator) this.oniterator(message.iterator, from) 52 | if (message.get) this.onget(message.get, from) 53 | } 54 | 55 | get (head, key) { 56 | this.outgoing.broadcast(Extension.encode({ get: { head, key } })) 57 | } 58 | 59 | iterator (head, key, flags, checkpoint) { 60 | this.outgoing.broadcast(Extension.encode({ iterator: { head, key, flags, checkpoint } })) 61 | return MAX_PASSIVE_BATCH 62 | } 63 | 64 | oncache (message, from) { 65 | if (!message.blocks.length) return 66 | if (message.blocks.length > MAX_ACTIVE_BATCH) message.blocks = message.blocks.slice(0, MAX_ACTIVE_BATCH) 67 | 68 | this.trie.feed.download(message) 69 | } 70 | 71 | oniterator (message, from) { 72 | if (message.key === null && !message.checkpoint) return 73 | 74 | if (this.active >= MAX_ACTIVE) return 75 | this.active++ 76 | this.trie.emit('extension-iterator', message.key) 77 | 78 | const self = this 79 | let total = 0 80 | 81 | const checkpointed = !!message.checkpoint 82 | const b = new Batch(this.outgoing, from) 83 | const ite = message.key 84 | ? this.trie.checkout(message.head + 1).iterator(message.key, { extension: false, wait: false, onseq }) 85 | : this.trie.iterator({ extension: false, wait: false, checkpoint: message.checkpoint, onseq }) 86 | 87 | ite.next(onnext) 88 | 89 | function onseq (seq) { 90 | if (checkpointed && !ite.opened) return 91 | total++ 92 | b.push(seq) 93 | } 94 | 95 | function onnext (err, node) { 96 | if (err || node === null || total >= MAX_ACTIVE_BATCH) { 97 | self.active-- 98 | b.send() 99 | } else { 100 | ite.next(onnext) 101 | } 102 | } 103 | } 104 | 105 | onget (message, from) { 106 | if (!message.key) return 107 | 108 | if (this.active >= MAX_ACTIVE) return 109 | this.active++ 110 | this.trie.emit('extension-get', message.key) 111 | 112 | const self = this 113 | const b = new Batch(this.outgoing, from) 114 | this.trie.checkout(message.head + 1).get(message.key, { extension: false, wait: false, onseq }, ondone) 115 | 116 | function onseq (seq) { 117 | b.push(seq) 118 | } 119 | 120 | function ondone () { 121 | self.active-- 122 | b.send() 123 | } 124 | } 125 | } 126 | 127 | HypertrieExtension.BATCH_SIZE = MAX_PASSIVE_BATCH 128 | 129 | module.exports = HypertrieExtension 130 | 131 | function decode (buf) { 132 | try { 133 | return Extension.decode(buf) 134 | } catch (err) { 135 | return null 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /test/hidden.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('hidden put is hidden', function (t) { 5 | const db = create() 6 | 7 | db.put('hello', 'world', { hidden: true }, function (err) { 8 | t.error(err, 'no error') 9 | db.get('hello', function (err, node) { 10 | t.error(err, 'no error') 11 | t.same(node, null) 12 | db.get('hello', { hidden: true }, function (err, node) { 13 | t.error(err, 'no error') 14 | t.same(node.value, 'world') 15 | t.end() 16 | }) 17 | }) 18 | }) 19 | }) 20 | 21 | tape('hidden and non hidden do not collide', function (t) { 22 | const db = create() 23 | 24 | db.put('hello', 'hidden', { hidden: true }, function (err) { 25 | t.error(err, 'no error') 26 | db.put('hello', 'not hidden', function (err) { 27 | t.error(err, 'no error') 28 | db.get('hello', function (err, node) { 29 | t.error(err, 'no error') 30 | t.same(node.value, 'not hidden') 31 | db.get('hello', { hidden: true }, function (err, node) { 32 | t.error(err, 'no error') 33 | t.same(node.value, 'hidden') 34 | t.end() 35 | }) 36 | }) 37 | }) 38 | }) 39 | }) 40 | 41 | tape('batch hidden and non hidden do not collide', function (t) { 42 | const db = create() 43 | 44 | db.batch([ 45 | { type: 'put', key: 'hello', value: 'hidden', hidden: true }, 46 | { type: 'put', key: 'hello', value: 'not hidden' } 47 | ], function (err) { 48 | t.error(err, 'no error') 49 | db.get('hello', function (err, node) { 50 | t.error(err, 'no error') 51 | t.same(node.value, 'not hidden') 52 | db.get('hello', { hidden: true }, function (err, node) { 53 | t.error(err, 'no error') 54 | t.same(node.value, 'hidden') 55 | t.end() 56 | }) 57 | }) 58 | }) 59 | }) 60 | 61 | tape('put 4 non hidden and one hidden', function (t) { 62 | const db = create() 63 | 64 | db.put('a', 'a', function () { 65 | db.put('hello', 'hello', { hidden: true }, function () { 66 | db.put('b', 'b', function () { 67 | db.put('c', 'c', function () { 68 | db.put('d', 'd', function () { 69 | db.get('hello', { hidden: true }, function (err, node) { 70 | t.error(err, 'no error') 71 | t.same(node.value, 'hello') 72 | t.end() 73 | }) 74 | }) 75 | }) 76 | }) 77 | }) 78 | }) 79 | }) 80 | 81 | tape('hidden iterators', function (t) { 82 | const db = create() 83 | 84 | db.batch([ 85 | { type: 'put', key: 'a', value: 'a' }, 86 | { type: 'put', key: 'hello', value: 'hello', hidden: true }, 87 | { type: 'put', key: 'b', value: 'b' }, 88 | { type: 'put', key: 'c', value: 'c' }, 89 | { type: 'put', key: 'world', value: 'world', hidden: true }, 90 | { type: 'put', key: 'd', value: 'd' } 91 | ], function (err) { 92 | t.error(err, 'no error') 93 | db.list(function (err, nodes) { 94 | t.error(err, 'no error') 95 | const values = nodes.map(n => n.value).sort() 96 | t.same(values, [ 'a', 'b', 'c', 'd' ]) 97 | db.list('', { hidden: true }, function (err, nodes) { 98 | t.error(err, 'no error') 99 | const values = nodes.map(n => n.value).sort() 100 | t.same(values, [ 'hello', 'world' ]) 101 | t.end() 102 | }) 103 | }) 104 | }) 105 | }) 106 | 107 | tape('hidden deletes', function (t) { 108 | t.plan(4) 109 | 110 | const db = create() 111 | 112 | db.put('a', 'a', { hidden: true }, function () { 113 | db.put('a', 'b', function () { 114 | db.del('a', { hidden: true }, function () { 115 | db.get('a', function (err, node) { 116 | t.error(err, 'no error') 117 | t.same(node.value, 'b') 118 | }) 119 | db.get('a', { hidden: true }, function (err, node) { 120 | t.error(err, 'no error') 121 | t.same(node, null) 122 | }) 123 | }) 124 | }) 125 | }) 126 | }) 127 | -------------------------------------------------------------------------------- /test/deletes.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | const messages = require('../lib/messages') 5 | 6 | tape('basic delete', function (t) { 7 | const db = create() 8 | 9 | db.put('hello', 'world', function () { 10 | db.get('hello', function (err, node) { 11 | t.error(err, 'no error') 12 | t.same(node.value, 'world') 13 | db.del('hello', function (err) { 14 | t.error(err, 'no error') 15 | db.get('hello', function (err, node) { 16 | t.error(err, 'no error') 17 | t.ok(!node, 'was deleted') 18 | 19 | db.iterator().next(function (err, node) { 20 | t.error(err, 'no error') 21 | t.ok(!node, 'db is empty') 22 | t.end() 23 | }) 24 | }) 25 | }) 26 | }) 27 | }) 28 | }) 29 | 30 | tape('delete one in many', function (t) { 31 | t.plan(1 + 2 + 2 + 2) 32 | 33 | const db = create() 34 | const batch = [] 35 | 36 | for (var i = 0; i < 50; i++) { 37 | batch.push({key: '' + i, value: '' + i}) 38 | } 39 | 40 | db.batch(batch, function () { 41 | db.del('42', done) 42 | }) 43 | 44 | function done (err) { 45 | t.error(err, 'no error') 46 | db.get('42', function (err, node) { 47 | t.error(err, 'no error') 48 | t.ok(!node, 'was deleted') 49 | }) 50 | db.get('43', function (err, node) { 51 | t.error(err, 'no error') 52 | t.same(node.value, '43') 53 | }) 54 | db.get('15', function (err, node) { 55 | t.error(err, 'no error') 56 | t.same(node.value, '15') 57 | }) 58 | } 59 | }) 60 | 61 | tape('delete one in many (iteration)', function (t) { 62 | const db = create() 63 | const batch = [] 64 | 65 | for (var i = 0; i < 50; i++) { 66 | batch.push({key: '' + i, value: '' + i}) 67 | } 68 | 69 | const keys = batch.map(b => b.key) 70 | 71 | db.batch(batch, function () { 72 | db.del('42', done) 73 | }) 74 | 75 | function done (err) { 76 | t.error(err, 'no error') 77 | 78 | const ite = db.iterator() 79 | const actual = [] 80 | 81 | ite.next(function loop (err, node) { 82 | if (err) return t.error(err, 'no error') 83 | 84 | if (!node) { 85 | const expected = keys.slice(0, 42).concat(keys.slice(43)) 86 | t.same(actual.sort(), expected.sort(), 'all except deleted one') 87 | t.end() 88 | return 89 | } 90 | 91 | actual.push(node.value) 92 | ite.next(loop) 93 | }) 94 | } 95 | }) 96 | 97 | tape('delete many in many (iteration)', function (t) { 98 | const db = create() 99 | const batch = [] 100 | 101 | for (var i = 0; i < 50; i++) { 102 | batch.push({key: '' + i, value: '' + i}) 103 | } 104 | 105 | const keys = batch.map(b => b.key) 106 | 107 | db.batch(batch, function () { 108 | const dels = batch.slice(0, 25).map(toDel) 109 | db.batch(dels, done) 110 | }) 111 | 112 | function done (err) { 113 | t.error(err, 'no error') 114 | 115 | const ite = db.iterator() 116 | const actual = [] 117 | 118 | ite.next(function loop (err, node) { 119 | if (err) return t.error(err, 'no error') 120 | 121 | if (!node) { 122 | const expected = keys.slice(25) 123 | t.same(actual.sort(), expected.sort(), 'all except deleted ones') 124 | t.end() 125 | return 126 | } 127 | 128 | actual.push(node.value) 129 | ite.next(loop) 130 | }) 131 | } 132 | }) 133 | 134 | tape('deletion with a single record, and a valueEncoding', function (t) { 135 | const db = create(null, { valueEncoding: messages.Header }) 136 | 137 | db.put('hello', { type: 'some-type' }, function (err) { 138 | t.error(err, 'no error') 139 | db.del('hello', function (err) { 140 | t.error(err, 'no error') 141 | db.get('hello', function (err, val) { 142 | t.error(err, 'no error') 143 | t.same(val, null) 144 | t.end() 145 | }) 146 | }) 147 | }) 148 | }) 149 | 150 | function toDel (e) { 151 | return { 152 | type: 'del', 153 | key: e.key 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /lib/del.js: -------------------------------------------------------------------------------- 1 | const Put = require('./put') 2 | const Node = require('./node') 3 | 4 | module.exports = Delete 5 | 6 | function Delete (db, key, { batch, condition = null, hidden = false, closest = false }, cb) { 7 | this._db = db 8 | this._key = key 9 | this._callback = cb 10 | this._release = null 11 | this._put = null 12 | this._batch = batch 13 | this._condition = condition 14 | this._node = new Node({key, flags: hidden ? Node.Flags.HIDDEN : 0}, null, null, db.hash) 15 | this._length = this._node.length 16 | this._returnClosest = closest 17 | this._closest = 0 18 | 19 | if (this._batch) this._update(0, this._batch.head()) 20 | else this._lock() 21 | } 22 | 23 | Delete.prototype._lock = function () { 24 | const self = this 25 | this._db._lock(function (release) { 26 | self._release = release 27 | self._start() 28 | }) 29 | } 30 | 31 | Delete.prototype._start = function () { 32 | const self = this 33 | this._db.head(onhead) 34 | 35 | function onhead (err, head) { 36 | if (err) return self._finalize(err, null) 37 | if (!head) return self._finalize(null, null) 38 | self._update(0, head) 39 | } 40 | } 41 | 42 | Delete.prototype._finalize = function (err, node) { 43 | if (!this._release) this._callback(err, node) 44 | else this._release(this._callback, err, node) 45 | } 46 | 47 | Delete.prototype._splice = function (closest, node) { 48 | const key = closest ? closest.key : '' 49 | const valueBuffer = closest ? closest.valueBuffer : null 50 | const hidden = closest ? closest.hidden : node.hidden 51 | const flags = closest ? closest.flags >> 8 : 0 52 | const self = this 53 | if (this._condition) this._condition(node.final(), oncondition) 54 | else del() 55 | 56 | function oncondition (err, proceed) { 57 | if (err) return done(err) 58 | if (!proceed) return done(null) 59 | return del() 60 | } 61 | 62 | function del () { 63 | self._put = new Put(self._db, key, null, { batch: self._batch, del: node.seq, hidden, valueBuffer, flags }, done) 64 | } 65 | 66 | function done (err, node) { 67 | self._finalize(err, node) 68 | } 69 | } 70 | 71 | Delete.prototype._update = function (i, head) { 72 | const self = this 73 | if (!head) return terminate() 74 | 75 | const node = this._node 76 | 77 | for (; i < this._length; i++) { 78 | const val = node.path(i) 79 | const bucket = head.trie[i] || [] 80 | 81 | if (head.path(i) === val) { 82 | const closest = firstSeq(bucket, val) 83 | if (closest) this._closest = closest 84 | continue 85 | } 86 | 87 | const seq = bucket[val] 88 | if (!seq) return terminate() 89 | 90 | this._closest = head.seq 91 | this._updateHead(i, seq) 92 | return 93 | } 94 | 95 | // TODO: collisions 96 | if (node.key !== head.key) return terminate() 97 | this._spliceClosest(head) 98 | 99 | function terminate () { 100 | if (self._condition && self._returnClosest) { 101 | return self._condition(head && head.final(), (err, proceed) => { 102 | if (err) return self._finalize(err) 103 | return self._finalize(null, null) 104 | }) 105 | } 106 | return self._finalize(null, null) 107 | } 108 | } 109 | 110 | Delete.prototype._spliceClosest = function (head) { 111 | if (!this._closest) return this._splice(null, head) 112 | 113 | const self = this 114 | 115 | this._get(this._closest, function (err, closest) { 116 | if (err) return self._finalize(err, null) 117 | self._splice(closest, head) 118 | }) 119 | } 120 | 121 | Delete.prototype._get = function (seq, onnode) { 122 | const node = this._batch && this._batch.get(seq) 123 | if (node) return process.nextTick(onnode, null, node) 124 | this._db.getBySeq(seq, onnode) 125 | } 126 | 127 | Delete.prototype._updateHead = function (i, seq) { 128 | const self = this 129 | this._get(seq, onnode) 130 | 131 | function onnode (err, node) { 132 | if (err) return self._finalize(err, null) 133 | self._update(i + 1, node) 134 | } 135 | } 136 | 137 | function firstSeq (bucket, val) { 138 | for (var i = 0; i < bucket.length; i++) { 139 | if (i === val) continue 140 | const seq = bucket[i] 141 | if (seq) return seq 142 | } 143 | return 0 144 | } 145 | -------------------------------------------------------------------------------- /test/collisions.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('two keys with same siphash', function (t) { 5 | t.plan(2 + 2) 6 | 7 | const db = create() 8 | 9 | db.put('idgcmnmna', 'a', function () { 10 | db.put('mpomeiehc', 'b', function () { 11 | db.get('idgcmnmna', function (err, node) { 12 | t.error(err, 'no error') 13 | t.same(node.value, 'a') 14 | }) 15 | db.get('mpomeiehc', function (err, node) { 16 | t.error(err, 'no error') 17 | t.same(node.value, 'b') 18 | }) 19 | }) 20 | }) 21 | }) 22 | 23 | tape('two keys with same siphash batch', function (t) { 24 | t.plan(1 + 3 * 2) 25 | 26 | const db = create() 27 | 28 | db.batch([ 29 | {key: 'idgcmnmna', value: 'a'}, 30 | {key: 'mpomeiehc', value: 'b'}, 31 | {key: 'foo', value: 'bar'} 32 | ], function (err) { 33 | t.error(err, 'no error') 34 | db.get('idgcmnmna', same('a')) 35 | db.get('mpomeiehc', same('b')) 36 | db.get('foo', same('bar')) 37 | }) 38 | 39 | function same (v) { 40 | return function (err, node) { 41 | t.error(err, 'no error') 42 | t.same(node.value, v) 43 | } 44 | } 45 | }) 46 | 47 | tape('two keys with same siphash (iterator)', function (t) { 48 | const db = create() 49 | 50 | db.put('idgcmnmna', 'a', function () { 51 | db.put('mpomeiehc', 'b', function () { 52 | const ite = db.iterator() 53 | 54 | ite.next(function (err, node) { 55 | t.error(err, 'no error') 56 | t.same(node.value, 'a') 57 | }) 58 | ite.next(function (err, node) { 59 | t.error(err, 'no error') 60 | t.same(node.value, 'b') 61 | }) 62 | ite.next(function (err, node) { 63 | t.error(err, 'no error') 64 | t.same(node, null) 65 | t.end() 66 | }) 67 | }) 68 | }) 69 | }) 70 | 71 | tape('two prefixes with same siphash (iterator)', function (t) { 72 | const db = create() 73 | 74 | db.put('idgcmnmna/a', 'a', function () { 75 | db.put('mpomeiehc/b', 'b', function () { 76 | const ite = db.iterator('idgcmnmna') 77 | 78 | ite.next(function (err, node) { 79 | t.error(err, 'no error') 80 | t.same(node.value, 'a') 81 | }) 82 | ite.next(function (err, node) { 83 | t.error(err, 'no error') 84 | t.same(node, null) 85 | t.end() 86 | }) 87 | }) 88 | }) 89 | }) 90 | 91 | tape('two prefixes with same key', function (t) { 92 | const db = create() 93 | 94 | db.put('idgcmnmna/a', 'a', function () { 95 | db.put('mpomeiehc/a', 'a', function () { 96 | const ite = db.iterator({recursive: false}) 97 | 98 | ite.next(function (err, node) { 99 | t.error(err, 'no error') 100 | t.same(node.value, 'a') 101 | }) 102 | ite.next(function (err, node) { 103 | t.error(err, 'no error') 104 | t.same(node.value, 'a') 105 | t.end() 106 | }) 107 | }) 108 | }) 109 | }) 110 | 111 | tape('sorts based on key when colliding', function (t) { 112 | const db1 = create() 113 | const db2 = create() 114 | 115 | db1.batch([ 116 | {key: 'idgcmnmna'}, 117 | {key: 'mpomeiehc'}, 118 | {key: 'a'}, 119 | {key: 'b'}, 120 | {key: 'c'} 121 | ], function () { 122 | db2.batch([ 123 | {key: 'b'}, 124 | {key: 'mpomeiehc'}, 125 | {key: 'a'}, 126 | {key: 'idgcmnmna'}, 127 | {key: 'c'} 128 | ], function () { 129 | const i1 = db1.iterator() 130 | const i2 = db2.iterator() 131 | 132 | i1.next(function loop (err, n1) { 133 | t.error(err, 'no error') 134 | i2.next(function (err, n2) { 135 | t.error(err, 'no error') 136 | if (!n1 && !n2) return t.end() 137 | t.same(n1.key, n2.key) 138 | i1.next(loop) 139 | }) 140 | }) 141 | }) 142 | }) 143 | }) 144 | 145 | tape('two keys with same siphash (diff)', function (t) { 146 | const db = create() 147 | 148 | db.batch([ 149 | {key: 'idgcmnmna'}, 150 | {key: 'mpomeiehc'}, 151 | {key: 'a'}, 152 | {key: 'b'}, 153 | {key: 'c'} 154 | ], function () { 155 | const ite = db.diff(0) 156 | const found = {} 157 | 158 | ite.next(function loop (err, node) { 159 | t.error(err, 'no error') 160 | if (!node) { 161 | t.same(found, { 162 | idgcmnmna: true, 163 | mpomeiehc: true, 164 | a: true, 165 | b: true, 166 | c: true 167 | }) 168 | return t.end() 169 | } 170 | found[node.key] = true 171 | ite.next(loop) 172 | }) 173 | }) 174 | }) 175 | -------------------------------------------------------------------------------- /test/iterator.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('basic iteration', function (t) { 5 | const db = create() 6 | const vals = ['a', 'b', 'c'] 7 | const expected = toMap(vals) 8 | 9 | put(db, vals, function (err) { 10 | t.error(err, 'no error') 11 | all(db.iterator(), function (err, map) { 12 | t.error(err, 'no error') 13 | t.same(map, expected, 'iterated all values') 14 | t.end() 15 | }) 16 | }) 17 | }) 18 | 19 | tape('iterate a big db', function (t) { 20 | const db = create() 21 | const vals = range(1000, '#') 22 | const expected = toMap(vals) 23 | 24 | put(db, vals, function (err) { 25 | t.error(err, 'no error') 26 | all(db.iterator(), function (err, map) { 27 | t.error(err, 'no error') 28 | t.same(map, expected, 'iterated all values') 29 | t.end() 30 | }) 31 | }) 32 | }) 33 | 34 | tape('prefix basic iteration', function (t) { 35 | var vals = ['foo/a', 'foo/b', 'foo/c'] 36 | const db = create() 37 | const expected = toMap(vals) 38 | 39 | vals = vals.concat(['a', 'b', 'c']) 40 | 41 | put(db, vals, function (err) { 42 | t.error(err, 'no error') 43 | all(db.iterator('foo'), function (err, map) { 44 | t.error(err, 'no error') 45 | t.same(map, expected, 'iterated all values') 46 | t.end() 47 | }) 48 | }) 49 | }) 50 | 51 | tape('empty prefix iteration', function (t) { 52 | const db = create() 53 | const vals = ['foo/a', 'foo/b', 'foo/c'] 54 | const expected = {} 55 | 56 | put(db, vals, function (err) { 57 | t.error(err, 'no error') 58 | all(db.iterator('bar'), function (err, map) { 59 | t.error(err, 'no error') 60 | t.same(map, expected, 'iterated all values') 61 | t.end() 62 | }) 63 | }) 64 | }) 65 | 66 | tape('prefix iterate a big db', function (t) { 67 | var vals = range(1000, 'foo/#') 68 | const db = create() 69 | const expected = toMap(vals) 70 | 71 | vals = vals.concat(range(1000, '#')) 72 | 73 | put(db, vals, function (err) { 74 | t.error(err, 'no error') 75 | all(db.iterator('foo'), function (err, map) { 76 | t.error(err, 'no error') 77 | t.same(map, expected, 'iterated all values') 78 | t.end() 79 | }) 80 | }) 81 | }) 82 | 83 | tape('non recursive iteration', function (t) { 84 | const db = create() 85 | const vals = [ 86 | 'a', 87 | 'a/b/c/d', 88 | 'a/c', 89 | 'b', 90 | 'b/b/c', 91 | 'c/a', 92 | 'c' 93 | ] 94 | 95 | put(db, vals, function (err) { 96 | t.error(err, 'no error') 97 | all(db.iterator({recursive: false}), function (err, map) { 98 | t.error(err, 'no error') 99 | const keys = Object.keys(map).map(k => k.split('/')[0]) 100 | t.same(keys.sort(), ['a', 'b', 'c'], 'iterated all values') 101 | t.end() 102 | }) 103 | }) 104 | }) 105 | 106 | tape('mixed nested and non nexted iteration', function (t) { 107 | const db = create() 108 | const vals = ['a', 'a/a', 'a/b', 'a/c', 'a/a/a', 'a/a/b', 'a/a/c'] 109 | const expected = toMap(vals) 110 | 111 | put(db, vals, function (err) { 112 | t.error(err, 'no error') 113 | all(db.iterator(), function (err, map) { 114 | t.error(err, 'no error') 115 | t.same(map, expected, 'iterated all values') 116 | t.end() 117 | }) 118 | }) 119 | }) 120 | 121 | tape('list buffers an iterator', function (t) { 122 | const db = create() 123 | 124 | put(db, ['a', 'b', 'b/c'], function (err) { 125 | t.error(err, 'no error') 126 | db.list(function (err, all) { 127 | t.error(err, 'no error') 128 | t.same(all.map(v => v.key).sort(), ['a', 'b', 'b/c']) 129 | db.list('b', {gt: true}, function (err, all) { 130 | t.error(err, 'no error') 131 | t.same(all.length, 1) 132 | t.same(all[0].key, 'b/c') 133 | t.end() 134 | }) 135 | }) 136 | }) 137 | }) 138 | 139 | function range (n, v) { 140 | // #0, #1, #2, ... 141 | return new Array(n).join('.').split('.').map((a, i) => v + i) 142 | } 143 | 144 | function toMap (list) { 145 | const map = {} 146 | for (var i = 0; i < list.length; i++) { 147 | map[list[i]] = list[i] 148 | } 149 | return map 150 | } 151 | 152 | function all (ite, cb) { 153 | const vals = {} 154 | 155 | ite.next(function loop (err, node) { 156 | if (err) return cb(err) 157 | if (!node) return cb(null, vals) 158 | const key = Array.isArray(node) ? node[0].key : node.key 159 | if (vals[key]) return cb(new Error('duplicate node for ' + key)) 160 | vals[key] = Array.isArray(node) ? node.map(n => n.value).sort() : node.value 161 | ite.next(loop) 162 | }) 163 | } 164 | 165 | function put (db, vals, cb) { 166 | db.batch(vals.map(v => ({key: v, value: v})), cb) 167 | } 168 | -------------------------------------------------------------------------------- /lib/put.js: -------------------------------------------------------------------------------- 1 | const Node = require('./node') 2 | 3 | module.exports = Put 4 | 5 | function putDefaultOptions (opts) { 6 | return Object.assign({}, { 7 | condition: null, 8 | closest: false, 9 | hidden: false, 10 | valueBuffer: null, 11 | flags: 0 12 | }, opts) 13 | } 14 | 15 | function Put (db, key, value, opts, cb) { 16 | let { hidden, condition, valueBuffer, flags, batch, del, closest } = putDefaultOptions(opts) 17 | 18 | this._db = db 19 | 20 | // The flags are shifted in order to both hide the internal flags and support user-defined flags. 21 | flags = (flags << 8) | (hidden ? Node.Flags.HIDDEN : 0) 22 | 23 | this._node = new Node({key, value, valueBuffer, flags}, 0, db.valueEncoding, db.hash) 24 | this._callback = cb 25 | this._release = null 26 | this._batch = batch 27 | this._closest = closest 28 | this._condition = condition 29 | this._error = null 30 | this._pending = 0 31 | this._del = del 32 | this._finalized = false 33 | this._head = null 34 | 35 | if (this._batch) this._update(0, this._batch.head()) 36 | else if (this._del) this._start() 37 | else this._lock() 38 | } 39 | 40 | Put.prototype._lock = function () { 41 | const self = this 42 | this._db._lock(function (release) { 43 | self._release = release 44 | self._start() 45 | }) 46 | } 47 | 48 | Put.prototype._start = function () { 49 | const self = this 50 | this._db.head(onhead) 51 | 52 | function onhead (err, head) { 53 | if (err) return self._finalize(err) 54 | self._update(0, head) 55 | } 56 | } 57 | 58 | Put.prototype._finalize = function (err) { 59 | const self = this 60 | 61 | this._finalized = true 62 | if (this._pending) { 63 | if (err) this._error = err 64 | return 65 | } 66 | 67 | if (this._error) err = this._error 68 | if (err) return done(err) 69 | 70 | const closest = this._head 71 | if (this._head && this._head.key !== this._node.key) this._head = null 72 | if (this._condition) { 73 | const conditionNode = this._closest ? closest && closest.final() : this._head && this._head.final() 74 | this._condition(conditionNode, this._node, oncondition) 75 | } else insert() 76 | 77 | function oncondition (err, proceed) { 78 | if (err) return done(err) 79 | if (!proceed) return done(null) 80 | return insert() 81 | } 82 | 83 | function insert () { 84 | if (self._batch) { 85 | self._batch.append(self._node) 86 | return done(null, self._node) 87 | } 88 | 89 | self._node.seq = self._db.feed.length 90 | self._db.feed.append(self._node.encode(), done) 91 | } 92 | 93 | function done (err) { 94 | const node = err ? null : self._node 95 | if (self._release) self._release(self._callback, err, node) 96 | else self._callback(err, node) 97 | } 98 | } 99 | 100 | Put.prototype._push = function (i, val, seq) { 101 | if (seq !== this._del) push(this._node.trie, i, val, seq) 102 | } 103 | 104 | Put.prototype._pushCollidable = function (i, val, seq) { 105 | if (seq === this._del) return 106 | 107 | const self = this 108 | this._pending++ 109 | this._get(seq, function (err, node) { 110 | if (err) this._error = err 111 | else if (node.collides(self._node, i)) push(self._node.trie, i, val, seq) 112 | if (!--self._pending && self._finalized) self._finalize(null) 113 | }) 114 | } 115 | 116 | Put.prototype._update = function (i, head) { 117 | if (!head) return this._finalize(null) 118 | 119 | const node = this._node 120 | 121 | for (; i < node.length; i++) { 122 | // check for collision at the end (4) or if it's a prefix terminator 123 | const checkCollision = Node.terminator(i) 124 | const val = node.path(i) 125 | const bucket = head.trie[i] || [] 126 | const headVal = head.path(i) 127 | for (var j = 0; j < bucket.length; j++) { 128 | // if same hash prefix, if no collision check is needed just continue 129 | if (j === val && !checkCollision) continue 130 | 131 | const seq = bucket[j] 132 | if (!seq) continue // skip no-ops 133 | 134 | if (!checkCollision) { // TODO: can prob optimise this with a || j !== val 135 | this._push(i, j, seq) 136 | } else { 137 | this._pushCollidable(i, j, seq) 138 | } 139 | } 140 | 141 | // we copied the head bucket, if this is still the closest node, continue 142 | // if no collision is possible 143 | if (headVal === val && (!checkCollision || !node.collides(head, i))) continue 144 | 145 | this._push(i, headVal, head.seq) 146 | 147 | if (checkCollision) return this._updateHeadCollidable(i, bucket, val) 148 | 149 | const seq = bucket[val] 150 | if (!seq) break 151 | return this._updateHead(i, seq) 152 | } 153 | 154 | this._head = head 155 | 156 | this._finalize(null) 157 | } 158 | 159 | Put.prototype._get = function (seq, cb) { 160 | const node = this._batch && this._batch.get(seq) 161 | if (node) return process.nextTick(cb, null, node) 162 | this._db.getBySeq(seq, cb) 163 | } 164 | 165 | Put.prototype._updateHeadCollidable = function (i, bucket, val) { 166 | const self = this 167 | var missing = 1 168 | var error = null 169 | var node = null 170 | 171 | for (var j = val; j < bucket.length; j += 5) { 172 | const seq = bucket[j] 173 | if (!seq) break 174 | missing++ 175 | this._get(seq, onnode) 176 | } 177 | 178 | onnode(null, null) 179 | 180 | function onnode (err, n) { 181 | if (err) error = err 182 | else if (n && !n.collides(self._node, i)) node = n 183 | if (--missing) return 184 | 185 | if (!node) return self._finalize(error) 186 | self._update(i + 1, node) 187 | } 188 | } 189 | 190 | Put.prototype._updateHead = function (i, seq) { 191 | const self = this 192 | this._get(seq, onnode) 193 | 194 | function onnode (err, node) { 195 | if (err) return self._finalize(err) 196 | self._update(i + 1, node) 197 | } 198 | } 199 | 200 | function push (trie, i, val, seq) { 201 | while (val >= 5) val -= 5 202 | 203 | const bucket = trie[i] || (trie[i] = []) 204 | while (bucket.length > val && bucket[val]) val += 5 205 | 206 | if (bucket.indexOf(seq) === -1) bucket[val] = seq 207 | } 208 | -------------------------------------------------------------------------------- /test/conditions.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | 4 | tape('condition: put only if changed', function (t) { 5 | const db = create() 6 | db.put('hello', 'world', { condition: onlyIfChanged }, err => { 7 | t.error(err, 'no error') 8 | db.put('hello', 'world', { condition: onlyIfChanged }, err => { 9 | t.error(err, 'no error') 10 | t.same(db.version, 2) 11 | t.end() 12 | }) 13 | }) 14 | 15 | function onlyIfChanged (oldNode, newNode, cb) { 16 | if (!oldNode) return cb(null, true) 17 | if (oldNode && !newNode) return cb(new Error('Cannot insert a null value (use delete)')) 18 | if (oldNode.value === newNode.value) return cb(null, false) 19 | return cb(null, true) 20 | } 21 | }) 22 | 23 | tape('condition: put only if the value is null', function (t) { 24 | const db = create() 25 | db.put('hello', 'world', { condition: onlyIfNull }, err => { 26 | t.error(err, 'no error') 27 | db.put('hello', 'friend', { condition: onlyIfNull }, err => { 28 | t.error(err, 'no error') 29 | t.same(db.version, 2) 30 | t.end() 31 | }) 32 | }) 33 | 34 | function onlyIfNull (oldNode, newNode, cb) { 35 | if (!newNode) return cb(new Error('Cannot insert a null value (use delete)')) 36 | if (oldNode) return cb(null, false) 37 | return cb(null, true) 38 | } 39 | }) 40 | 41 | tape('condition: put only if value is null, nested paths', function (t) { 42 | const db = create() 43 | db.put('/a/b', 'world', { condition: onlyIfNull }, err => { 44 | t.error(err, 'no error') 45 | db.put('/a/b/c', 'friend', { condition: onlyIfNull }, err => { 46 | t.error(err, 'no error') 47 | t.same(db.version, 3) 48 | t.end() 49 | }) 50 | }) 51 | 52 | function onlyIfNull (oldNode, newNode, cb) { 53 | if (!newNode) return cb(new Error('Cannot insert a null value (use delete)')) 54 | if (oldNode) return cb(null, false) 55 | return cb(null, true) 56 | } 57 | }) 58 | 59 | tape('condition: two keys with same siphash', function (t) { 60 | const db = create() 61 | var pending = 2 62 | 63 | db.put('idgcmnmna', 'a', function () { 64 | db.put('mpomeiehc', 'b', { condition: onlyIfNull }, function (err) { 65 | t.error(err, 'no error') 66 | t.same(db.version, 3) 67 | testKey('idgcmnmna', ifValueMatches('a')) 68 | testKey('mpomeiehc', ifValueMatches('b')) 69 | }) 70 | }) 71 | 72 | function testKey (key, condition) { 73 | db.put(key, 'c', { condition }, function (err) { 74 | t.error(err, 'no error') 75 | db.get(key, function (err, node) { 76 | t.error(err, 'no error') 77 | t.same(node.value, 'c') 78 | if (!--pending) return t.end() 79 | }) 80 | }) 81 | } 82 | 83 | function ifValueMatches (val) { 84 | return function (oldNode, newNode, cb) { 85 | if (oldNode && oldNode.value === val) return cb(null, true) 86 | return cb(null, false) 87 | } 88 | } 89 | 90 | function onlyIfNull (oldNode, newNode, cb) { 91 | if (!newNode) return cb(new Error('Cannot insert a null value (use delete)')) 92 | if (oldNode) return cb(null, false) 93 | return cb(null, true) 94 | } 95 | }) 96 | 97 | tape('condition: delete only a certain value', function (t) { 98 | const db = create() 99 | db.put('hello', 'world', err => { 100 | t.error(err, 'no error') 101 | db.del('hello', { condition: deleteGuard('friend') }, err => { 102 | t.error(err, 'no error') 103 | db.get('hello', (err, node) => { 104 | t.error(err, 'no error') 105 | t.same(node.value, 'world') 106 | doDelete() 107 | }) 108 | }) 109 | }) 110 | 111 | function doDelete () { 112 | db.del('hello', { condition: deleteGuard('world') }, err => { 113 | t.error(err, 'no error') 114 | db.get('hello', (err, node) => { 115 | t.error(err, 'no error') 116 | t.true(node === null) 117 | t.end() 118 | }) 119 | }) 120 | } 121 | 122 | function deleteGuard (value) { 123 | return function (node, cb) { 124 | if (node && node.value === value) return cb(null, true) 125 | return cb(null, false) 126 | } 127 | } 128 | }) 129 | 130 | tape('condition: async condition', function (t) { 131 | const db = create() 132 | db.put('hello', 'world', { condition: afterWork }, err => { 133 | t.error(err, 'no error') 134 | db.put('hello', 'world', { condition: afterWork }, err => { 135 | t.error(err, 'no error') 136 | t.same(db.version, 3) 137 | db.get('hello', (err, node) => { 138 | t.error(err, 'no error') 139 | t.same(node.value, 'world') 140 | t.end() 141 | }) 142 | }) 143 | }) 144 | 145 | function afterWork (oldNode, newNode, cb) { 146 | setTimeout(() => { 147 | return cb(null, true) 148 | }, 200) 149 | } 150 | }) 151 | 152 | tape('condition: deletion closest with similar paths', function (t) { 153 | const db = create() 154 | db.put('a', 'hello world', err => { 155 | t.error(err, 'no error') 156 | db.del('a/b', { condition: closestGuard('a'), closest: true }, err => { 157 | t.true(err && err.closest, 'closest was a') 158 | db.get('a', (err, node) => { 159 | t.error(err, 'no error') 160 | t.same(node.value, 'hello world') 161 | t.end() 162 | }) 163 | }) 164 | }) 165 | 166 | function closestGuard (key) { 167 | return function (closest, cb) { 168 | if (closest && closest.key === key) { 169 | const err = new Error('Closest key was incorrect') 170 | err.closest = true 171 | return cb(err) 172 | } 173 | return cb(null, true) 174 | } 175 | } 176 | }) 177 | 178 | tape('condition: deletion closest with multiple hops', function (t) { 179 | const db = create() 180 | db.put('a', 'hello world', err => { 181 | t.error(err, 'no error') 182 | db.put('b', 'blah', err => { 183 | t.error(err, 'no error') 184 | db.del('a/b', { condition: closestGuard('a'), closest: true }, err => { 185 | t.true(err && err.closest, 'closest was a') 186 | db.get('a', (err, node) => { 187 | t.error(err, 'no error') 188 | t.same(node.value, 'hello world') 189 | t.end() 190 | }) 191 | }) 192 | }) 193 | }) 194 | 195 | function closestGuard (key) { 196 | return function (closest, cb) { 197 | if (closest && closest.key === key) { 198 | const err = new Error('Closest key was incorrect') 199 | err.closest = true 200 | return cb(err) 201 | } 202 | return cb(null, true) 203 | } 204 | } 205 | }) 206 | -------------------------------------------------------------------------------- /lib/iterator.js: -------------------------------------------------------------------------------- 1 | const Nanoiterator = require('nanoiterator') 2 | const inherits = require('inherits') 3 | const varint = require('varint') 4 | const Node = require('./node') 5 | const { BATCH_SIZE } = require('./extension') 6 | 7 | const SORT_ORDER = [4, 0, 1, 2, 3].reverse() 8 | const REVERSE_SORT_ORDER = SORT_ORDER.slice(0).reverse() 9 | 10 | module.exports = Iterator 11 | 12 | function Iterator (db, prefix, opts) { 13 | Nanoiterator.call(this) 14 | 15 | if (opts && opts.flags) { 16 | opts.recursive = opts.flags & 1 17 | opts.reverse = opts.flags & 2 18 | opts.gt = opts.flags & 4 19 | opts.hidden = opts.flags & 8 20 | } 21 | 22 | this._checkpoint = (opts && opts.checkpoint) || null 23 | this._prefix = Node.normalizeKey(prefix || '') 24 | this._recursive = !opts || opts.recursive !== false 25 | this._order = (opts && opts.reverse) ? REVERSE_SORT_ORDER : SORT_ORDER 26 | this._random = !!(opts && opts.random) 27 | this._extension = (opts && opts.extension === false) ? null : db._extension 28 | this._extensionState = this._extension ? { missing: 0, head: 0, checkpoint: false } : null 29 | this._onseq = (opts && opts.onseq) || null 30 | this._start = 0 31 | this._end = 0 32 | this._db = db 33 | this._stack = [] 34 | this._callback = null 35 | this._pending = 0 36 | this._error = null 37 | this._gt = !!(opts && opts.gt) 38 | this._needsSort = [] 39 | this._options = opts ? { extension: opts.extension, wait: opts.wait, timeout: opts.timeout, hidden: !!opts.hidden, onseq: opts.onseq, onwait: null } : { onwait: null } 40 | this._flags = (this._recursive ? 1 : 0) | (this._order === REVERSE_SORT_ORDER ? 2 : 0) | (this._gt ? 4 : 0) | ((this._options && this._options.hidden) ? 8 : 0) 41 | if (this._extensionState) this._options.onwait = this._sendExt.bind(this) 42 | } 43 | 44 | inherits(Iterator, Nanoiterator) 45 | 46 | Iterator.prototype._open = function (cb) { 47 | const self = this 48 | const opts = Object.assign(this._options || {}, { prefix: true, extension: false, onheadseq }) 49 | const prefix = this._db.get(this._prefix, opts, onnode) 50 | 51 | function onnode (err, node) { 52 | if (err) return cb(err) 53 | if (node && !self._checkpoint) self._stack.push({i: prefix._length, seq: node.seq, node}) 54 | self._start = prefix._length 55 | if (self._recursive) self._end = Infinity 56 | else self._end = prefix._length + 32 57 | if (self._extensionState) self._extensionState.checkpoint = true 58 | if (self._checkpoint) self._openCheckpoint(cb) 59 | else cb(null) 60 | } 61 | 62 | function onheadseq (seq) { 63 | const ext = self._extensionState 64 | if (ext && !ext.head) ext.head = seq 65 | } 66 | } 67 | 68 | Iterator.prototype._sendExt = function () { 69 | if (this._extensionState.missing > 0 || !this._extensionState.head) return 70 | this._extensionState.missing = BATCH_SIZE 71 | this._extension.iterator(this._extensionState.head, this._prefix, this._flags, this._extensionState.checkpoint ? this.checkpoint() : null) 72 | } 73 | 74 | Iterator.prototype._openCheckpoint = function (cb) { 75 | var ptr = 0 76 | 77 | this._callback = cb 78 | 79 | while (ptr < this._checkpoint.length) { 80 | const i = varint.decode(this._checkpoint, ptr) 81 | ptr += varint.decode.bytes 82 | const seq = varint.decode(this._checkpoint, ptr) 83 | ptr += varint.decode.bytes 84 | this._push(i, seq) 85 | } 86 | 87 | if (!this._pending) { 88 | this._callback = null 89 | cb(null) 90 | } 91 | } 92 | 93 | Iterator.prototype.checkpoint = function () { 94 | const buf = Buffer.alloc(this._stack.length * 8 * 2) 95 | var ptr = 0 96 | 97 | for (var i = 0; i < this._stack.length; i++) { 98 | const s = this._stack[i] 99 | varint.encode(s.i, buf, ptr) 100 | ptr += varint.encode.bytes 101 | varint.encode(s.seq, buf, ptr) 102 | ptr += varint.encode.bytes 103 | } 104 | 105 | return buf.slice(0, ptr) 106 | } 107 | 108 | Iterator.prototype._next = function (cb) { 109 | var j 110 | 111 | while (this._stack.length) { 112 | const top = this._stack.pop() 113 | const len = Math.min(top.node.length, this._end) 114 | const i = top.i++ 115 | 116 | if (i >= len) return cb(null, top.node.final()) 117 | 118 | const bucket = top.node.trie[i] || [] 119 | const order = this._random ? randomOrder() : this._order 120 | 121 | for (j = 0; j < order.length; j++) { 122 | var val = order[j] 123 | if (val !== 4 || !this._gt || i !== this._start) { 124 | const len = this._stack.length 125 | if (top.node.path(i) === val) this._stack.push(top) 126 | for (; val < bucket.length; val += 5) { 127 | const seq = bucket[val] 128 | if (seq) this._push(i + 1, seq) 129 | } 130 | if (this._stack.length - len > 1) { 131 | this._needsSort.push(len, this._stack.length) 132 | } 133 | } 134 | } 135 | 136 | if (!this._pending) continue 137 | this._callback = cb 138 | return 139 | } 140 | 141 | cb(null, null) 142 | } 143 | 144 | Iterator.prototype._push = function (i, seq) { 145 | const self = this 146 | const top = {i, seq, node: null} 147 | 148 | this._pending++ 149 | this._stack.push(top) 150 | 151 | if (this._onseq) this._onseq(seq) 152 | if (this._extensionState && this._extensionState.missing > 0) this._extensionState.missing-- 153 | 154 | this._db.getBySeq(seq, this._options, onnode) 155 | 156 | function onnode (err, node) { 157 | if (node) top.node = node 158 | else if (err) self._error = err 159 | if (!--self._pending) self._continue() 160 | } 161 | } 162 | 163 | Iterator.prototype._sort = function () { 164 | // only ran when there are potential collisions to make sure 165 | // the iterator sorts consistently 166 | while (this._needsSort.length) { 167 | const end = this._needsSort.pop() 168 | const start = this._needsSort.pop() 169 | sort(this._stack, start, end) 170 | } 171 | } 172 | 173 | Iterator.prototype._continue = function () { 174 | const callback = this._callback 175 | const err = this._error 176 | this._callback = this._error = null 177 | if (err) return callback(err) 178 | if (this._needsSort.length) this._sort() 179 | if (!this.opened) return callback(null) 180 | this._next(callback) 181 | } 182 | 183 | function sort (list, from, to) { 184 | // only ran on short lists so the simple o(n^2) algo is fine 185 | for (var i = from + 1; i < to; i++) { 186 | for (var j = i; j > from; j--) { 187 | const a = list[j] 188 | const b = list[j - 1] 189 | if (a.node.key <= b.node.key) break 190 | list[j] = b 191 | list[j - 1] = a 192 | } 193 | } 194 | } 195 | 196 | function randomOrder () { 197 | const order = [0, 1, 2, 3, 4] 198 | for (let i = 0; i < order.length - 1; i++) { 199 | const n = i + Math.floor(Math.random() * (order.length - i)) 200 | const tmp = order[i] 201 | order[i] = order[n] 202 | order[n] = tmp 203 | } 204 | return order 205 | } 206 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hypertrie 2 | 3 | Distributed single writer key/value store 4 | 5 | ``` 6 | npm install hypertrie 7 | ``` 8 | 9 | [![Build Status](https://travis-ci.org/mafintosh/hypertrie.svg?branch=master)](https://travis-ci.org/mafintosh/hypertrie) 10 | 11 | Uses a rolling hash array mapped trie to index key/value data on top of a [hypercore](https://github.com/mafintosh/hypercore). 12 | 13 | Useful if you just want a straight forward single writer kv store or if you are looking for a building block for building more complex multiwriter databases on top. 14 | 15 | ## Usage 16 | 17 | ```js 18 | const hypertrie = require('hypertrie') 19 | const db = hypertrie('./trie.db', {valueEncoding: 'json'}) 20 | 21 | db.put('hello', 'world', function () { 22 | db.get('hello', console.log) 23 | }) 24 | ``` 25 | 26 | ## API 27 | 28 | #### `db = hypertrie(storage, [key], [options])` 29 | 30 | Create a new database. Options include: 31 | 32 | ``` 33 | { 34 | feed: aHypercore, // use this feed instead of loading storage 35 | valueEncoding: 'json', // set value encoding 36 | subtype: undefined, // set subtype in the header message at feed.get(0) 37 | alwaysUpdate: true // perform an ifAvailable update prior to every head operation 38 | } 39 | ``` 40 | 41 | If you set `options.feed` then you can set `storage` to null. 42 | 43 | #### `db.get(key, [options], callback)` 44 | 45 | Lookup a key. Returns a result node if found or `null` otherwise. 46 | Options are passed through to hypercore's get method. 47 | 48 | #### `db.put(key, value, [options], [callback])` 49 | 50 | Insert a value. 51 | 52 | Options can include: 53 | ``` 54 | { 55 | condition: function (oldNode, newNode, cb(err, bool)) { ... } 56 | } 57 | ``` 58 | The optional `condition` function provides atomic compare-and-swap semantics, allowing you to optionally abort a put based on the current and intended node values. 59 | The condition callback should be used as follows: 60 | 1. `cb(new Error(...))`: Abort with an error that will be forwarded through the `put`. 61 | 2. `cb(null, false)`: Abort the put, but do not produce an error. 62 | 3. `cb(null, true)`: Proceed with the put. 63 | 64 | #### `db.del(key, [options], [callback])` 65 | 66 | Delete a key from the database. 67 | 68 | Options can include: 69 | ``` 70 | { 71 | condition: function (oldNode, cb(err, bool)) { ... } 72 | } 73 | ``` 74 | The optional `condition` function behaves the same as the one in `put`, minus the `newNode` parameter. 75 | 76 | #### `db.batch(batch, [callback])` 77 | 78 | Insert/delete multiple values atomically. 79 | The batch objects should look like this: 80 | 81 | ```js 82 | { 83 | type: 'put' | 'del', 84 | key: 'key/we/are/updating', 85 | value: optionalValue 86 | } 87 | ``` 88 | 89 | #### `const watcher = db.watch(prefix, [onchange])` 90 | 91 | Watch a prefix of the db and get notified when it changes. 92 | 93 | When there is a change `watcher.on('change')` is emitted. 94 | Use `watcher.destroy()` to stop watching. 95 | 96 | #### `db.on('ready')` 97 | 98 | Emitted when the db has loaded it's internal state. 99 | 100 | You do not need to wait for this unless noted in the docs. 101 | 102 | #### `db.version` 103 | 104 | Returns the current version of the db (an incrementing integer). 105 | 106 | Only available after `ready` has been emitted. 107 | 108 | #### `db.key` 109 | 110 | Returns the db public key. You need to pass this to other instances 111 | you want to replicate with. 112 | 113 | Only available after `ready` has been emitted. 114 | 115 | #### `db.discoveryKey` 116 | 117 | Returns the db discovery key. Can be used to find other db peers. 118 | 119 | Only available after `ready` has been emitted. 120 | 121 | #### `checkoutDb = db.checkout(version)` 122 | 123 | Returns a new db instance checked out at the version specified. 124 | 125 | #### `checkoutDb = db.snapshot()` 126 | 127 | Same as checkout but just returns the latest version as a checkout. 128 | 129 | #### `stream = db.replicate(isInitiator, [options])` 130 | 131 | Returns a hypercore replication stream for the db. Pipe this together with another hypertrie instance. 132 | 133 | Replicate takes an `isInitiator` boolean which is used to indicate if this replication stream is the passive/active replicator. 134 | 135 | All options are forwarded to hypercores replicate method. 136 | 137 | #### `ite = db.iterator(prefix, [options])` 138 | 139 | Returns a [nanoiterator](https://github.com/mafintosh/nanoiterator) that iterates 140 | the latest values in the prefix specified. 141 | 142 | Options include: 143 | 144 | ```js 145 | { 146 | recursive: true, 147 | random: false // does a random order iteration 148 | } 149 | ``` 150 | 151 | If you set `recursive: false` it will only iterate the immediate children (similar to readdir) 152 | 153 | Additional options are passed through to hypercore's get method. 154 | 155 | #### `stream = db.createReadStream(prefix, [options])` 156 | 157 | Same as above but as a stream 158 | 159 | #### `db.list(prefix, [options], callback)` 160 | 161 | Creates an iterator for the prefix with the specified options and buffers it into an array that is passed to the callback. 162 | 163 | #### `stream = db.createWriteStream()` 164 | 165 | A writable stream you can write batch objects to, to update the db. 166 | 167 | #### `ite = db.history([options])` 168 | 169 | Returns a [nanoiterator](https://github.com/mafintosh/nanoiterator) that iterates over the feed in causal order. 170 | 171 | Options include: 172 | 173 | ```js 174 | { 175 | gt: seq, 176 | lt: seq, 177 | gte: seq, 178 | lte: seq, 179 | reverse: false, 180 | live: false // set to true to keep iterating forever 181 | } 182 | ``` 183 | 184 | #### `stream = db.createHistoryStream([options])` 185 | 186 | Same as above but as a stream 187 | 188 | #### `ite = db.diff(version, [prefix], [options])` 189 | 190 | Returns a [nanoiterator](https://github.com/mafintosh/nanoiterator) that iterates the diff between the current db and the version you specifiy. The objects returned look like this 191 | 192 | ```js 193 | { 194 | key: 'node-key-that-is-updated', 195 | left: , 196 | right: 197 | } 198 | ``` 199 | 200 | If a node is in the current db but not in the version you are diffing against 201 | `left` will be set to the current node and `right` will be null and vice versa. 202 | 203 | Options include: 204 | 205 | ```js 206 | { 207 | skipLeftNull: false, 208 | skipRightNull: false, 209 | hidden: false, // set to true to diff the hidden keyspace 210 | checkpoint: 211 | } 212 | ``` 213 | 214 | The order of messages emitted for a specific diff is predictable (ordered by key hash). It is possible to resume a diff at any position. To do so, call the `.checkpoint` method on the diff iterator. It returns a serialized buffer of the current position within the diff. To resume, create a new diff between the same versions and pass the checkpoint buffer as an option. 215 | 216 | #### `stream = db.createDiffStream(version, [prefix])` 217 | 218 | Same as above but as a stream 219 | 220 | ## License 221 | 222 | MIT 223 | -------------------------------------------------------------------------------- /test/diff.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const collect = require('stream-collector') 3 | const create = require('./helpers/create') 4 | const cmp = require('compare') 5 | 6 | tape('empty diff', function (t) { 7 | const db = create() 8 | 9 | const rs = db.createDiffStream(0, 'a') 10 | collect(rs, function (err, actual) { 11 | t.error(err, 'no error') 12 | t.deepEqual(actual, [], 'diff as expected') 13 | t.end() 14 | }) 15 | }) 16 | 17 | tape('implicit checkout', function (t) { 18 | const db = create() 19 | 20 | db.put('a', '2', function (err) { 21 | t.error(err, 'no error') 22 | const rs = db.createDiffStream(0, 'a') 23 | collect(rs, function (err, actual) { 24 | t.error(err, 'no error') 25 | t.equals(actual.length, 1) 26 | t.equals(actual[0].key, 'a') 27 | t.equals(actual[0].left.key, 'a') 28 | t.equals(actual[0].left.value, '2') 29 | t.equals(actual[0].right, null) 30 | t.end() 31 | }) 32 | }) 33 | }) 34 | 35 | tape('new value', function (t) { 36 | const db = create() 37 | 38 | db.put('a', '1', function (err) { 39 | t.error(err, 'no error') 40 | db.put('a', '2', function (err) { 41 | t.error(err, 'no error') 42 | const rs = db.createDiffStream(0, 'a') 43 | collect(rs, function (err, actual) { 44 | t.error(err, 'no error') 45 | t.equals(actual.length, 1) 46 | t.equals(actual[0].key, 'a') 47 | t.equals(actual[0].left.value, '2') 48 | t.equals(actual[0].right, null) 49 | t.end() 50 | }) 51 | }) 52 | }) 53 | }) 54 | 55 | tape('two new nodes', function (t) { 56 | const db = create() 57 | 58 | db.put('a/foo', 'quux', function (err) { 59 | t.error(err, 'no error') 60 | db.put('a/bar', 'baz', function (err) { 61 | t.error(err, 'no error') 62 | const rs = db.createDiffStream(0, 'a') 63 | collect(rs, function (err, actual) { 64 | t.error(err, 'no error') 65 | actual.sort(sort) 66 | t.equals(actual.length, 2) 67 | t.equals(actual[0].key, 'a/bar') 68 | t.equals(actual[0].left.value, 'baz') 69 | t.equals(actual[0].right, null) 70 | t.equals(actual[1].key, 'a/foo') 71 | t.equals(actual[1].left.value, 'quux') 72 | t.equals(actual[1].right, null) 73 | t.end() 74 | }) 75 | }) 76 | }) 77 | }) 78 | 79 | tape('checkout === head', function (t) { 80 | const db = create() 81 | 82 | db.put('a', '2', function (err) { 83 | t.error(err, 'no error') 84 | const rs = db.createDiffStream(db, 'a') 85 | collect(rs, function (err, actual) { 86 | t.error(err, 'no error') 87 | t.equals(actual.length, 0) 88 | t.end() 89 | }) 90 | }) 91 | }) 92 | 93 | tape('new value, twice', function (t) { 94 | const db = create() 95 | const snap = db.snapshot() 96 | 97 | db.put('/a', '1', function (err) { 98 | t.error(err, 'no error') 99 | db.put('/a', '2', function (err) { 100 | t.error(err, 'no error') 101 | const rs = db.createDiffStream(snap, 'a') 102 | collect(rs, function (err, actual) { 103 | t.error(err, 'no error') 104 | t.equals(actual.length, 1) 105 | t.equals(actual[0].left.key, 'a') 106 | t.equals(actual[0].left.value, '2') 107 | t.equals(actual[0].right, null) 108 | t.end() 109 | }) 110 | }) 111 | }) 112 | }) 113 | 114 | tape('untracked value', function (t) { 115 | const db = create() 116 | 117 | db.put('a', '1', function (err) { 118 | t.error(err, 'no error') 119 | const snap = db.snapshot() 120 | db.put('a', '2', function (err) { 121 | t.error(err, 'no error') 122 | db.put('b', '17', function (err) { 123 | t.error(err, 'no error') 124 | const rs = db.createDiffStream(snap, 'a') 125 | collect(rs, function (err, actual) { 126 | t.error(err, 'no error') 127 | t.equals(actual.length, 1) 128 | t.equals(actual[0].key, 'a') 129 | t.equals(actual[0].left.value, '2') 130 | t.equals(actual[0].right.value, '1') 131 | t.end() 132 | }) 133 | }) 134 | }) 135 | }) 136 | }) 137 | 138 | tape('diff root', function (t) { 139 | const db = create() 140 | 141 | db.put('a', '1', function (err) { 142 | t.error(err, 'no error') 143 | const snap = db.snapshot() 144 | db.put('a', '2', function (err) { 145 | t.error(err, 'no error') 146 | db.put('b', '17', function (err) { 147 | t.error(err, 'no error') 148 | const rs = db.createDiffStream(snap) 149 | collect(rs, function (err, actual) { 150 | t.error(err, 'no error') 151 | actual.sort(sort) 152 | t.equals(actual.length, 2) 153 | t.equals(actual[0].key, 'a') 154 | t.equals(actual[0].left.value, '2') 155 | t.equals(actual[0].key, 'a') 156 | t.equals(actual[0].right.value, '1') 157 | t.equals(actual[1].key, 'b') 158 | t.equals(actual[1].left.value, '17') 159 | t.equals(actual[1].right, null) 160 | t.end() 161 | }) 162 | }) 163 | }) 164 | }) 165 | }) 166 | 167 | tape('updated value', function (t) { 168 | const db = create() 169 | 170 | db.put('a/d/r', '1', function (err) { 171 | t.error(err, 'no error') 172 | const snap = db.snapshot() 173 | db.put('a/d/r', '3', function (err) { 174 | t.error(err, 'no error') 175 | const rs = db.createDiffStream(snap, 'a') 176 | collect(rs, function (err, actual) { 177 | t.error(err, 'no error') 178 | t.equals(actual.length, 1) 179 | t.equals(actual[0].key, 'a/d/r') 180 | t.equals(actual[0].left.value, '3') 181 | t.equals(actual[0].key, 'a/d/r') 182 | t.equals(actual[0].right.value, '1') 183 | t.end() 184 | }) 185 | }) 186 | }) 187 | }) 188 | 189 | tape('small diff on big db', function (t) { 190 | const db = create() 191 | var nodes = 0 192 | 193 | db.batch(range(10000), function (err) { 194 | t.error(err, 'no error') 195 | const snap = db.snapshot() 196 | db.put('42', '42*', function (err) { 197 | t.error(err, 'no error') 198 | const rs = db.createDiffStream(snap, {onnode}) 199 | collect(rs, function (err, actual) { 200 | t.error(err, 'no error') 201 | t.equals(actual.length, 1) 202 | t.equals(actual[0].key, '42') 203 | t.equals(actual[0].left.value, '42*') 204 | t.equals(actual[0].key, '42') 205 | t.equals(actual[0].right.value, '42') 206 | t.ok(nodes < 50) 207 | t.end() 208 | }) 209 | }) 210 | }) 211 | 212 | function onnode () { 213 | nodes++ 214 | } 215 | }) 216 | 217 | tape('diff on hidden', function (t) { 218 | const db = create() 219 | db.put('secret', 'hyper', { hidden: true }, err => { 220 | t.error(err, 'no error') 221 | db.put('public', 'knowledge', err => { 222 | t.error(err, 'no error') 223 | let secdiff = db.createDiffStream(0, { hidden: true }) 224 | let pubdiff = db.createDiffStream(0) 225 | collect(secdiff, (err, actual) => { 226 | t.error(err, 'no error') 227 | t.equal(actual.length, 1) 228 | t.equal(actual[0].key, 'secret') 229 | collect(pubdiff, (err, actual) => { 230 | t.error(err, 'no error') 231 | t.equal(actual.length, 1) 232 | t.equal(actual[0].key, 'public') 233 | t.end() 234 | }) 235 | }) 236 | }) 237 | }) 238 | }) 239 | 240 | tape('diff checkpoints', function (t) { 241 | const db = create() 242 | const batchSize = 20 243 | db.batch(range(100), function (err) { 244 | t.error(err, 'no error') 245 | start() 246 | }) 247 | 248 | function start () { 249 | const snap = db.snapshot() 250 | let batches = [] 251 | run(snap, 0, null, finish) 252 | 253 | function finish (err, data, checkpoint) { 254 | t.error(err, 'no error') 255 | if (data && data.length) batches.push(data) 256 | if (checkpoint) run(snap, 0, checkpoint, finish) 257 | else finalize() 258 | } 259 | 260 | function finalize () { 261 | t.equal(batches.length, 5, 'number of batches') 262 | const set = new Set() 263 | for (let batch of batches) { 264 | t.equal(batch.length, 20, 'batch size') 265 | batch.forEach(val => set.add(val)) 266 | } 267 | t.equal(set.size, 100, 'no duplicate values') 268 | t.end() 269 | } 270 | } 271 | 272 | function run (db, from, checkpoint, cb) { 273 | const diff = db.diff(from, { checkpoint }) 274 | const data = [] 275 | diff.next(next) 276 | function next (err, msg) { 277 | if (err) finish(err) 278 | if (!msg) return finish(null, false) 279 | data.push(msg.key) 280 | if (data.length >= batchSize) finish(null, true) 281 | else diff.next(next) 282 | } 283 | 284 | function finish (err, hasMore) { 285 | t.error(err, 'no error') 286 | let newCheckpoint 287 | if (hasMore) newCheckpoint = diff.checkpoint() 288 | cb(null, data, newCheckpoint) 289 | } 290 | } 291 | }) 292 | 293 | function range (n) { 294 | return Array(n).join('.').split('.').map((_, i) => '' + i).map(kv) 295 | } 296 | 297 | function kv (v) { 298 | return {type: 'put', key: v, value: v} 299 | } 300 | 301 | function sort (a, b) { 302 | var ak = (a.left || a.right).key 303 | var bk = (b.left || b.right).key 304 | return cmp(ak, bk) 305 | } 306 | -------------------------------------------------------------------------------- /lib/diff.js: -------------------------------------------------------------------------------- 1 | const Nanoiterator = require('nanoiterator') 2 | const inherits = require('inherits') 3 | const Node = require('./node') 4 | const varint = require('varint') 5 | 6 | module.exports = Diff 7 | 8 | function Diff (db, checkout, prefix, opts) { 9 | Nanoiterator.call(this) 10 | 11 | this._db = db 12 | this._prefix = prefix || '' 13 | this._checkout = checkout 14 | this._stack = [] 15 | this._pending = 0 16 | this._error = null 17 | this._callback = null 18 | this._left = [] 19 | this._right = [] 20 | this._onnode = (opts && opts.onnode) || null 21 | this._hidden = !!(opts && opts.hidden) 22 | this._needsCheck = [] 23 | this._skipLeftNull = !!(opts && opts.skipLeftNull) 24 | this._skipRightNull = !!(opts && opts.skipRightNull) 25 | this._checkpoint = (opts && opts.checkpoint) || null 26 | this._reconnect = !!(opts && opts.reconnect) 27 | this._pendingQueue = [] 28 | this.maxInflight = (opts && opts.maxInflight) || (this._reconnect ? 4 : Infinity) 29 | if (this._reconnect) this._skipRightNull = true 30 | } 31 | 32 | inherits(Diff, Nanoiterator) 33 | 34 | Diff.prototype._open = function (cb) { 35 | if (this._checkpoint) return this._openCheckpoint(cb) 36 | 37 | const self = this 38 | const opts = {onnode: this._onnode, prefix: true, hidden: this._hidden} 39 | const get = this._db.get(this._prefix, opts, function (err, a) { 40 | if (err) return cb(err) 41 | self._checkout.get(self._prefix, opts, function (err, b) { 42 | if (err) return cb(err) 43 | self._stack.push({i: get._length, left: a, right: b, skip: false}) 44 | cb(null) 45 | }) 46 | }) 47 | } 48 | 49 | Diff.prototype._openCheckpoint = function (cb) { 50 | const self = this 51 | const buf = this._checkpoint 52 | var ptr = 0 53 | 54 | loop() 55 | 56 | function loop () { 57 | if (ptr >= buf.length) return cb(null) 58 | 59 | const i = varint.decode(buf, ptr) 60 | ptr += varint.decode.bytes 61 | const l = varint.decode(buf, ptr) 62 | ptr += varint.decode.bytes 63 | const r = varint.decode(buf, ptr) 64 | ptr += varint.decode.bytes 65 | 66 | self._db.getBySeq(l, function (err, left) { 67 | if (err) return cb(err) 68 | self._db.getBySeq(r, function (err, right) { 69 | if (err) return cb(err) 70 | self._stack.push({i, left, right, skip: false}) 71 | loop() 72 | }) 73 | }) 74 | } 75 | } 76 | 77 | Diff.prototype.checkpoint = function () { 78 | const buf = Buffer.alloc(this._stack.length * 8 * 3) 79 | var ptr = 0 80 | 81 | for (var i = 0; i < this._stack.length; i++) { 82 | const s = this._stack[i] 83 | if (s.skip) continue 84 | varint.encode(s.i, buf, ptr) 85 | ptr += varint.encode.bytes 86 | varint.encode(s.left ? s.left.seq : 0, buf, ptr) 87 | ptr += varint.encode.bytes 88 | varint.encode(s.right ? s.right.seq : 0, buf, ptr) 89 | ptr += varint.encode.bytes 90 | } 91 | return buf.slice(0, ptr) 92 | } 93 | 94 | Diff.prototype._finalize = function () { 95 | const callback = this._callback 96 | if (!callback) return 97 | 98 | if (this.closed) return callback(new Error('Iterator closed')) 99 | 100 | const err = this._error 101 | this._callback = this._error = null 102 | if (err) return callback(err) 103 | 104 | while (this._needsCheck.length) { 105 | const end = this._needsCheck.pop() 106 | const start = this._needsCheck.pop() 107 | this._maybeCollides(start, end) 108 | } 109 | 110 | this._next(callback) 111 | } 112 | 113 | Diff.prototype._next = function (cb) { 114 | this._nextAsync(cb) 115 | } 116 | 117 | Diff.prototype._has = async function (seq) { 118 | this._pending++ 119 | try { 120 | return await this._db.feed.has(seq) 121 | } catch (err) { 122 | this._error = err 123 | } finally { 124 | this._pending-- 125 | } 126 | } 127 | 128 | Diff.prototype._nextAsync = async function (cb) { 129 | if (this._pending) { 130 | this._callback = cb 131 | return 132 | } 133 | 134 | if (this._error) return cb(this._error) 135 | 136 | while (this._stack.length) { 137 | const {i, left, right, skip} = this._stack.pop() 138 | 139 | if (skip || seq(left) === seq(right)) continue 140 | 141 | const doneLeft = done(left, i) 142 | const doneRight = done(right, i) 143 | 144 | if (doneLeft && doneRight) return call(cb, left, right) 145 | 146 | if (!right && left && this._skipRightNull) continue 147 | if (right && !left && this._skipLeftNull) continue 148 | 149 | const leftVal = left ? left.path(i) : 5 150 | const rightVal = right ? right.path(i) : 6 151 | const leftBucket = trie(left, i) 152 | const rightBucket = trie(right, i) 153 | 154 | for (var j = 0; j < 5; j++) { 155 | const leftSeq = leftVal === j ? left.seq : 0 156 | const rightSeq = rightVal === j ? right.seq : 0 157 | const len = this._stack.length 158 | var leftLen = this._stack.length 159 | var rightLen = this._stack.length 160 | var val 161 | 162 | if (leftSeq !== rightSeq) { 163 | if (!doneLeft && leftSeq && notInBucket(j, leftSeq, rightBucket)) { 164 | set(this._pushStack(leftLen++, i + 1), true, left) 165 | } 166 | if (!doneRight && rightSeq && notInBucket(j, rightSeq, leftBucket)) { 167 | set(this._pushStack(rightLen++, i + 1), false, right) 168 | } 169 | } 170 | 171 | if (!doneLeft) { 172 | const pushLeft = !this._skipRightNull || (rightBucket[j] && (!this._reconnect || await this._hasSeqInBucket(rightBucket, j))) 173 | for (val = j; val < leftBucket.length; val += 5) { 174 | const seq = leftBucket[val] 175 | if (!seq) break 176 | if (seq !== rightSeq && notInBucket(j, seq, rightBucket)) { 177 | const top = this._pushStack(leftLen++, i + 1) 178 | if (pushLeft || top.right) this._getNode(seq, top, true) 179 | else top.skip = true 180 | } 181 | } 182 | } 183 | 184 | if (!doneRight) { 185 | const pushRight = !this._skipLeftNull || leftBucket[j] 186 | for (val = j; val < rightBucket.length; val += 5) { 187 | const seq = rightBucket[val] 188 | if (!seq) break 189 | if (seq !== leftSeq && notInBucket(j, seq, leftBucket) && (!this._reconnect || await this._has(seq))) { 190 | const top = this._pushStack(rightLen++, i + 1) 191 | if (pushRight || top.left) this._getNode(seq, top, false) 192 | else top.skip = true 193 | } 194 | } 195 | } 196 | 197 | if (Node.terminator(i) && this._stack.length > len) { 198 | if (!this._pending) this._maybeCollides(len, this._stack.length) 199 | else this._needsCheck.push(len, this._stack.length) 200 | } 201 | } 202 | 203 | if (doneLeft) return call(cb, left, null) 204 | if (doneRight) return call(cb, null, right) 205 | 206 | if (!this._pending) continue 207 | this._callback = cb 208 | return 209 | } 210 | 211 | cb(null, null) 212 | } 213 | 214 | Diff.prototype._hasSeqInBucket = async function (bucket, val) { 215 | for (; val < bucket.length; val += 5) { 216 | if (bucket[val] && await this._has(bucket[val])) return true 217 | } 218 | return false 219 | } 220 | 221 | Diff.prototype._maybeCollides = function (start, end) { 222 | // all nodes, start -> end, share the same hash 223 | // we need to check that there are no collisions 224 | 225 | // much simpler and *much* more likely - only one node 226 | if (end - start === 1) { 227 | const top = this._stack[start] 228 | if (collides(top)) { 229 | this._stack.push({i: top.i, left: null, right: top.right, skip: top.skip}) 230 | top.right = null 231 | } 232 | return 233 | } 234 | 235 | // very unlikely, but multiple collisions or a trie reordering 236 | // due to a collision being deleted 237 | 238 | for (var i = start; i < end; i++) { 239 | const top = this._stack[i] 240 | if (collides(top) || !top.left) { 241 | const right = top.right 242 | for (var j = start; j < end; j++) { 243 | const other = this._stack[j] 244 | if (other.left && !other.left.collides(right)) { 245 | top.right = other.right 246 | other.right = right 247 | i-- // revisit top again, as it might still collide 248 | break 249 | } 250 | } 251 | if (top.right === right && top.left) { 252 | this._stack.push({i: top.i, left: null, right, skip: top.skip}) 253 | top.right = null 254 | } 255 | } 256 | } 257 | } 258 | 259 | Diff.prototype._pushStack = function (len, i) { 260 | if (this._stack.length === len) this._stack.push({i, left: null, right: null, skip: false}) 261 | return this._stack[len] 262 | } 263 | 264 | Diff.prototype._getNode = function (seq, top, left) { 265 | const self = this 266 | this._pending++ 267 | 268 | const inflight = this._pending - this._pendingQueue.length 269 | if (inflight >= this.maxInflight) { 270 | this._pendingQueue.push([seq, top, left]) 271 | return 272 | } 273 | 274 | this._db.getBySeq(seq, onnode) 275 | 276 | function onnode (err, node) { 277 | if (self._onnode && node) self._onnode(node) 278 | if (node) set(top, left, node) 279 | else if (err) self._error = err 280 | if (!--self._pending) self._finalize() 281 | 282 | if (self._pendingQueue.length && self._pending - self._pendingQueue.length < self.maxInflight) { 283 | const [seq, top, left] = self._pendingQueue.pop() 284 | self._pending-- 285 | self._getNode(seq, top, left) 286 | } 287 | } 288 | } 289 | 290 | function notInBucket (val, seq, bucket) { 291 | for (; val < bucket.length; val += 5) { 292 | if (bucket[val] === seq) return false 293 | } 294 | return true 295 | } 296 | 297 | function set (top, left, node) { 298 | if (left) top.left = node 299 | else top.right = node 300 | } 301 | 302 | function call (cb, left, right) { 303 | cb(null, { 304 | key: left ? left.key : right.key, 305 | left: left && left.final(), 306 | right: right && right.final() 307 | }) 308 | } 309 | 310 | function trie (node, i) { 311 | return (node && node.trie[i]) || [] 312 | } 313 | 314 | function seq (node) { 315 | return node ? node.seq : 0 316 | } 317 | 318 | function done (node, i) { 319 | return !!node && i >= node.length 320 | } 321 | 322 | function collides (top) { 323 | if (!top.left || !top.right || !Node.terminator(top.i)) return false 324 | return top.left.collides(top.right, top.i) 325 | } 326 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const events = require('events') 2 | 3 | const mutexify = require('mutexify') 4 | const thunky = require('thunky') 5 | const codecs = require('codecs') 6 | const bulk = require('bulk-write-stream') 7 | const toStream = require('nanoiterator/to-stream') 8 | const isOptions = require('is-options') 9 | const hypercore = require('hypercore') 10 | const inherits = require('inherits') 11 | const alru = require('array-lru') 12 | const set = require('unordered-set') 13 | 14 | const Extension = require('./lib/extension') 15 | const Node = require('./lib/node') 16 | const Get = require('./lib/get') 17 | const Put = require('./lib/put') 18 | const Batch = require('./lib/batch') 19 | const Delete = require('./lib/del') 20 | const History = require('./lib/history') 21 | const Iterator = require('./lib/iterator') 22 | const Watch = require('./lib/watch') 23 | const Diff = require('./lib/diff') 24 | const { Header } = require('./lib/messages') 25 | 26 | module.exports = HyperTrie 27 | 28 | function HyperTrie (storage, key, opts) { 29 | if (!(this instanceof HyperTrie)) return new HyperTrie(storage, key, opts) 30 | 31 | if (isOptions(key)) { 32 | opts = key 33 | key = null 34 | } 35 | 36 | if (!opts) opts = {} 37 | 38 | events.EventEmitter.call(this) 39 | 40 | this.id = null 41 | this.key = null 42 | this.discoveryKey = null 43 | this.secretKey = null 44 | this.metadata = opts.metadata || null 45 | this.hash = opts.hash || null 46 | this.valueEncoding = opts.valueEncoding ? codecs(opts.valueEncoding) : null 47 | this.alwaysUpdate = !!opts.alwaysUpdate 48 | this.alwaysReconnect = !!opts.alwaysReconnect 49 | this.subtype = opts.subtype 50 | 51 | const feedOpts = Object.assign({}, opts, { valueEncoding: 'binary' }) 52 | this.feed = opts.feed || hypercore(storage, key, feedOpts) 53 | this.feed.maxRequests = opts.maxRequests || 256 // set max requests higher since the payload is small 54 | this.opened = false 55 | this.ready = thunky(this._ready.bind(this)) 56 | 57 | this._extension = opts.extension === false ? null : ((opts.extension === true ? null : opts.extension) || new Extension(this)) 58 | if (this._extension && !this._extension.outgoing) this._extension.outgoing = this.feed.registerExtension('hypertrie', this._extension) 59 | 60 | this._watchers = [] 61 | this._checkout = (opts && opts.checkout) || 0 62 | this._cache = (opts && opts.cache) || alru((opts && opts.cacheSize) || 32768) 63 | this._lock = mutexify() 64 | 65 | if (this.feed !== opts.feed) this.feed.on('error', this._onerror.bind(this)) 66 | if (!this._checkout) this.feed.on('append', this._onappend.bind(this)) 67 | } 68 | 69 | inherits(HyperTrie, events.EventEmitter) 70 | 71 | Object.defineProperty(HyperTrie.prototype, 'version', { 72 | enumerable: true, 73 | get: function () { 74 | return this._checkout || this.feed.length 75 | } 76 | }) 77 | 78 | HyperTrie.prototype._removeWatch = function (w) { 79 | set.remove(this._watchers, w) 80 | } 81 | 82 | HyperTrie.prototype._addWatch = function (w) { 83 | const self = this 84 | 85 | set.add(this._watchers, w) 86 | if (this._watchers.length > 1 || !this.feed.sparse) return 87 | 88 | this.feed.update({ ifAvailable: false }, function loop () { 89 | if (self._watchers.length === 0) return 90 | self.feed.update({ ifAvailable: false }, loop) 91 | }) 92 | } 93 | 94 | HyperTrie.prototype.reconnect = function (from, opts) { 95 | opts = opts ? Object.assign({}, opts, { reconnect: true }) : { reconnect: true } 96 | return this.diff(from, opts) 97 | } 98 | 99 | HyperTrie.prototype._onerror = function (err) { 100 | this.emit('error', err) 101 | } 102 | 103 | HyperTrie.prototype._onappend = function () { 104 | for (var i = 0; i < this._watchers.length; i++) { 105 | this._watchers[i].update() 106 | } 107 | 108 | this.emit('append') 109 | } 110 | 111 | HyperTrie.prototype._ready = function (cb) { 112 | const self = this 113 | 114 | this.feed.ready(function (err) { 115 | if (err) return done(err) 116 | 117 | if (self.feed.length || !self.feed.writable) return done(null) 118 | self.feed.append(Header.encode({ 119 | type: 'hypertrie', 120 | metadata: self.metadata, 121 | subtype: self.subtype 122 | }), done) 123 | 124 | function done (err) { 125 | if (err) return cb(err) 126 | if (self._checkout === -1) self._checkout = self.feed.length 127 | self.id = self.feed.id 128 | self.key = self.feed.key 129 | self.discoveryKey = self.feed.discoveryKey 130 | self.secretKey = self.feed.secretKey 131 | self.opened = true 132 | self.emit('ready') 133 | 134 | if (self.alwaysReconnect) { 135 | var from = self.feed.length 136 | var active = null 137 | 138 | self.feed.on('append', function () { 139 | if (!from) { 140 | from = self.feed.length 141 | return 142 | } 143 | 144 | if (active) active.destroy() 145 | 146 | self.emit('reconnecting') 147 | const r = active = self.reconnect(from) 148 | active.next(function loop (err, data) { 149 | if (r !== active) return 150 | 151 | if (err || !data) { 152 | active = null 153 | from = self.feed.length 154 | if (!err) self.emit('reconnected') 155 | return 156 | } 157 | 158 | active.next(loop) 159 | }) 160 | }) 161 | } 162 | 163 | cb(null) 164 | } 165 | }) 166 | } 167 | 168 | HyperTrie.getMetadata = function (feed, cb) { 169 | feed.get(0, (err, msg) => { 170 | if (err) return cb(err) 171 | 172 | try { 173 | var header = Header.decode(msg) 174 | } catch (err) { 175 | return cb(err) 176 | } 177 | 178 | cb(null, header.metadata) 179 | }) 180 | } 181 | 182 | HyperTrie.prototype.getMetadata = function (cb) { 183 | HyperTrie.getMetadata(this.feed, cb) 184 | } 185 | 186 | HyperTrie.prototype.setMetadata = function (metadata) { 187 | // setMetadata can only be called before this.ready is first called. 188 | if (this.feed.length || !this.feed.writable) throw new Error('The metadata must be set before any puts have occurred.') 189 | this.metadata = metadata 190 | } 191 | 192 | HyperTrie.prototype.replicate = function (isInitiator, opts) { 193 | return this.feed.replicate(isInitiator, opts) 194 | } 195 | 196 | HyperTrie.prototype.checkout = function (version) { 197 | if (version === 0) version = 1 198 | return new HyperTrie(null, null, { 199 | checkout: version || 1, 200 | valueEncoding: this.valueEncoding, 201 | feed: this.feed, 202 | extension: this._extension === null ? false : this._extension, 203 | cache: this._cache 204 | }) 205 | } 206 | 207 | HyperTrie.prototype.snapshot = function () { 208 | return this.checkout(this.version) 209 | } 210 | 211 | HyperTrie.prototype.headSeq = function (opts, cb) { 212 | const self = this 213 | 214 | if (!this.opened) return readyAndHeadSeq(this, opts, cb) 215 | if (this._checkout !== 0) return process.nextTick(cb, null, this._checkout - 1) 216 | if (this.alwaysUpdate && (!opts || opts.wait !== false)) this.feed.update({ hash: false, ifAvailable: true }, onupdated) 217 | else process.nextTick(onupdated) 218 | 219 | function onupdated () { 220 | if (self.feed.length < 2) return cb(null, 0) 221 | cb(null, self.feed.length - 1) 222 | } 223 | } 224 | 225 | HyperTrie.prototype.head = function (opts, cb) { 226 | if (typeof opts === 'function') return this.head(null, opts) 227 | 228 | const self = this 229 | this.headSeq(opts, function (err, seq) { 230 | if (err) return cb(err) 231 | if (!seq) return cb(null, null) 232 | self.getBySeq(seq, opts, cb) 233 | }) 234 | } 235 | 236 | HyperTrie.prototype.list = function (prefix, opts, cb) { 237 | if (typeof prefix === 'function') return this.list('', null, prefix) 238 | if (typeof opts === 'function') return this.list(prefix, null, opts) 239 | 240 | const ite = this.iterator(prefix, opts) 241 | const res = [] 242 | 243 | ite.next(function loop (err, node) { 244 | if (err) return cb(err) 245 | if (!node) return cb(null, res) 246 | res.push(node) 247 | ite.next(loop) 248 | }) 249 | } 250 | 251 | HyperTrie.prototype.iterator = function (prefix, opts) { 252 | if (isOptions(prefix)) return this.iterator('', prefix) 253 | return new Iterator(this, prefix, opts) 254 | } 255 | 256 | HyperTrie.prototype.createReadStream = function (prefix, opts) { 257 | return toStream(this.iterator(prefix, opts)) 258 | } 259 | 260 | HyperTrie.prototype.history = function (opts) { 261 | return new History(this, opts) 262 | } 263 | 264 | HyperTrie.prototype.createHistoryStream = function (opts) { 265 | return toStream(this.history(opts)) 266 | } 267 | 268 | HyperTrie.prototype.diff = function (other, prefix, opts) { 269 | if (Buffer.isBuffer(other)) return this.diff(0, prefix, Object.assign(opts || {}, { checkpoint: other })) 270 | if (isOptions(prefix)) return this.diff(other, null, prefix) 271 | const checkout = (typeof other === 'number' || !other) ? this.checkout(other) : other 272 | return new Diff(this, checkout, prefix, opts) 273 | } 274 | 275 | HyperTrie.prototype.createDiffStream = function (other, prefix, opts) { 276 | return toStream(this.diff(other, prefix, opts)) 277 | } 278 | 279 | HyperTrie.prototype.get = function (key, opts, cb) { 280 | if (typeof opts === 'function') return this.get(key, null, opts) 281 | return new Get(this, key, opts, cb) 282 | } 283 | 284 | HyperTrie.prototype.watch = function (key, onchange) { 285 | if (typeof key === 'function') return this.watch('', key) 286 | return new Watch(this, key, onchange) 287 | } 288 | 289 | HyperTrie.prototype.batch = function (ops, cb) { 290 | return new Batch(this, ops, cb || noop) 291 | } 292 | 293 | HyperTrie.prototype.put = function (key, value, opts, cb) { 294 | if (typeof opts === 'function') return this.put(key, value, null, opts) 295 | opts = Object.assign({}, opts, { 296 | batch: null, 297 | del: 0 298 | }) 299 | return new Put(this, key, value, opts, cb || noop) 300 | } 301 | 302 | HyperTrie.prototype.del = function (key, opts, cb) { 303 | if (typeof opts === 'function') return this.del(key, null, opts) 304 | opts = Object.assign({}, opts, { 305 | batch: null 306 | }) 307 | return new Delete(this, key, opts, cb) 308 | } 309 | 310 | HyperTrie.prototype.createWriteStream = function (opts) { 311 | const self = this 312 | return bulk.obj(write) 313 | 314 | function write (batch, cb) { 315 | if (batch.length && Array.isArray(batch[0])) batch = flatten(batch) 316 | self.batch(batch, cb) 317 | } 318 | } 319 | 320 | HyperTrie.prototype.getBySeq = function (seq, opts, cb) { 321 | if (typeof opts === 'function') return this.getBySeq(seq, null, opts) 322 | if (seq < 1) return process.nextTick(cb, null, null) 323 | const self = this 324 | 325 | const cached = this._cache.get(seq) 326 | if (cached) return process.nextTick(onnode, null, cached) 327 | this.feed.get(seq, opts, onnode) 328 | 329 | function onnode (err, val) { 330 | if (err) return cb(err) 331 | const node = Node.decode(val, seq, self.valueEncoding, self.hash) 332 | self._cache.set(seq, val) 333 | // early exit for the key: '' nodes we write to reset the db 334 | if (node.value === null && node.key === '') return cb(null, null) 335 | cb(null, node) 336 | } 337 | } 338 | 339 | function noop () {} 340 | 341 | function readyAndHeadSeq (self, opts, cb) { 342 | self.ready(function (err) { 343 | if (err) return cb(err) 344 | self.headSeq(opts, cb) 345 | }) 346 | } 347 | 348 | function flatten (list) { 349 | const result = [] 350 | for (var i = 0; i < list.length; i++) { 351 | const next = list[i] 352 | for (var j = 0; j < next.length; j++) result.push(next[j]) 353 | } 354 | return result 355 | } 356 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const create = require('./helpers/create') 3 | const Readable = require('stream').Readable 4 | 5 | tape('basic put/get', function (t) { 6 | const db = create() 7 | db.put('hello', 'world', function (err, node) { 8 | t.same(node.key, 'hello') 9 | t.same(node.value, 'world') 10 | t.error(err, 'no error') 11 | db.get('hello', function (err, node) { 12 | t.error(err, 'no error') 13 | t.same(node.key, 'hello', 'same key') 14 | t.same(node.value, 'world', 'same value') 15 | t.end() 16 | }) 17 | }) 18 | }) 19 | 20 | tape('get on empty db', function (t) { 21 | const db = create() 22 | 23 | db.get('hello', function (err, node) { 24 | t.error(err, 'no error') 25 | t.same(node, null, 'node is not found') 26 | t.end() 27 | }) 28 | }) 29 | 30 | tape('not found', function (t) { 31 | const db = create() 32 | db.put('hello', 'world', function (err) { 33 | t.error(err, 'no error') 34 | db.get('hej', function (err, node) { 35 | t.error(err, 'no error') 36 | t.same(node, null, 'node is not found') 37 | t.end() 38 | }) 39 | }) 40 | }) 41 | 42 | tape('leading / is ignored', function (t) { 43 | t.plan(7) 44 | const db = create() 45 | db.put('/hello', 'world', function (err) { 46 | t.error(err, 'no error') 47 | db.get('/hello', function (err, node) { 48 | t.error(err, 'no error') 49 | t.same(node.key, 'hello', 'same key') 50 | t.same(node.value, 'world', 'same value') 51 | }) 52 | db.get('hello', function (err, node) { 53 | t.error(err, 'no error') 54 | t.same(node.key, 'hello', 'same key') 55 | t.same(node.value, 'world', 'same value') 56 | }) 57 | }) 58 | }) 59 | 60 | tape('multiple put/get', function (t) { 61 | t.plan(8) 62 | 63 | const db = create() 64 | 65 | db.put('hello', 'world', function (err) { 66 | t.error(err, 'no error') 67 | db.put('world', 'hello', function (err) { 68 | t.error(err, 'no error') 69 | db.get('hello', function (err, node) { 70 | t.error(err, 'no error') 71 | t.same(node.key, 'hello', 'same key') 72 | t.same(node.value, 'world', 'same value') 73 | }) 74 | db.get('world', function (err, node) { 75 | t.error(err, 'no error') 76 | t.same(node.key, 'world', 'same key') 77 | t.same(node.value, 'hello', 'same value') 78 | }) 79 | }) 80 | }) 81 | }) 82 | 83 | tape('overwrites', function (t) { 84 | const db = create() 85 | 86 | db.put('hello', 'world', function (err) { 87 | t.error(err, 'no error') 88 | db.get('hello', function (err, node) { 89 | t.error(err, 'no error') 90 | t.same(node.key, 'hello', 'same key') 91 | t.same(node.value, 'world', 'same value') 92 | db.put('hello', 'verden', function (err) { 93 | t.error(err, 'no error') 94 | db.get('hello', function (err, node) { 95 | t.error(err, 'no error') 96 | t.same(node.key, 'hello', 'same key') 97 | t.same(node.value, 'verden', 'same value') 98 | t.end() 99 | }) 100 | }) 101 | }) 102 | }) 103 | }) 104 | 105 | tape('put/gets namespaces', function (t) { 106 | t.plan(8) 107 | 108 | const db = create() 109 | 110 | db.put('hello/world', 'world', function (err) { 111 | t.error(err, 'no error') 112 | db.put('world', 'hello', function (err) { 113 | t.error(err, 'no error') 114 | db.get('hello/world', function (err, node) { 115 | t.error(err, 'no error') 116 | t.same(node.key, 'hello/world', 'same key') 117 | t.same(node.value, 'world', 'same value') 118 | }) 119 | db.get('world', function (err, node) { 120 | t.error(err, 'no error') 121 | t.same(node.key, 'world', 'same key') 122 | t.same(node.value, 'hello', 'same value') 123 | }) 124 | }) 125 | }) 126 | }) 127 | 128 | tape('put in tree', function (t) { 129 | t.plan(8) 130 | 131 | const db = create() 132 | 133 | db.put('hello', 'a', function (err) { 134 | t.error(err, 'no error') 135 | db.put('hello/world', 'b', function (err) { 136 | t.error(err, 'no error') 137 | db.get('hello', function (err, node) { 138 | t.error(err, 'no error') 139 | t.same(node.key, 'hello', 'same key') 140 | t.same(node.value, 'a', 'same value') 141 | }) 142 | db.get('hello/world', function (err, node) { 143 | t.error(err, 'no error') 144 | t.same(node.key, 'hello/world', 'same key') 145 | t.same(node.value, 'b', 'same value') 146 | }) 147 | }) 148 | }) 149 | }) 150 | 151 | tape('put in tree reverse order', function (t) { 152 | t.plan(8) 153 | 154 | const db = create() 155 | 156 | db.put('hello/world', 'b', function (err) { 157 | t.error(err, 'no error') 158 | db.put('hello', 'a', function (err) { 159 | t.error(err, 'no error') 160 | db.get('hello', function (err, node) { 161 | t.error(err, 'no error') 162 | t.same(node.key, 'hello', 'same key') 163 | t.same(node.value, 'a', 'same value') 164 | }) 165 | db.get('hello/world', function (err, node) { 166 | t.error(err, 'no error') 167 | t.same(node.key, 'hello/world', 'same key') 168 | t.same(node.value, 'b', 'same value') 169 | }) 170 | }) 171 | }) 172 | }) 173 | 174 | tape('multiple put in tree', function (t) { 175 | t.plan(13) 176 | 177 | const db = create() 178 | 179 | db.put('hello/world', 'b', function (err) { 180 | t.error(err, 'no error') 181 | db.put('hello', 'a', function (err) { 182 | t.error(err, 'no error') 183 | db.put('hello/verden', 'c', function (err) { 184 | t.error(err, 'no error') 185 | db.put('hello', 'd', function (err) { 186 | t.error(err, 'no error') 187 | db.get('hello', function (err, node) { 188 | t.error(err, 'no error') 189 | t.same(node.key, 'hello', 'same key') 190 | t.same(node.value, 'd', 'same value') 191 | }) 192 | db.get('hello/world', function (err, node) { 193 | t.error(err, 'no error') 194 | t.same(node.key, 'hello/world', 'same key') 195 | t.same(node.value, 'b', 'same value') 196 | }) 197 | db.get('hello/verden', function (err, node) { 198 | t.error(err, 'no error') 199 | t.same(node.key, 'hello/verden', 'same key') 200 | t.same(node.value, 'c', 'same value') 201 | }) 202 | }) 203 | }) 204 | }) 205 | }) 206 | }) 207 | 208 | tape('insert 100 values and get them all', function (t) { 209 | const db = create() 210 | const max = 100 211 | var i = 0 212 | 213 | t.plan(3 * max) 214 | 215 | loop() 216 | 217 | function loop () { 218 | if (i === max) return validate() 219 | db.put('#' + i, '#' + (i++), loop) 220 | } 221 | 222 | function validate () { 223 | for (var i = 0; i < max; i++) { 224 | db.get('#' + i, same('#' + i)) 225 | } 226 | } 227 | 228 | function same (key) { 229 | return function (err, node) { 230 | t.error(err, 'no error') 231 | t.same(node.key, key, 'same key') 232 | t.same(node.value, key, 'same value') 233 | } 234 | } 235 | }) 236 | 237 | tape('race works', function (t) { 238 | t.plan(40) 239 | 240 | var missing = 10 241 | const db = create() 242 | 243 | for (var i = 0; i < 10; i++) db.put('#' + i, '#' + i, done) 244 | 245 | function done (err) { 246 | t.error(err, 'no error') 247 | if (--missing) return 248 | for (var i = 0; i < 10; i++) same('#' + i) 249 | } 250 | 251 | function same (val) { 252 | db.get(val, function (err, node) { 253 | t.error(err, 'no error') 254 | t.same(node.key, val, 'same key') 255 | t.same(node.value, val, 'same value') 256 | }) 257 | } 258 | }) 259 | 260 | tape('version', function (t) { 261 | const db = create() 262 | 263 | db.ready(function () { 264 | t.same(db.version, 1) 265 | db.put('hello', 'world', function () { 266 | t.same(db.version, 2) 267 | db.put('hello', 'verden', function () { 268 | t.same(db.version, 3) 269 | db.checkout(2).get('hello', function (err, node) { 270 | t.error(err, 'no error') 271 | t.same(node.value, 'world') 272 | t.end() 273 | }) 274 | }) 275 | }) 276 | }) 277 | }) 278 | 279 | tape('basic batch', function (t) { 280 | t.plan(1 + 3 + 3) 281 | 282 | const db = create() 283 | 284 | db.batch([ 285 | {key: 'hello', value: 'world'}, 286 | {key: 'hej', value: 'verden'}, 287 | {key: 'hello', value: 'welt'} 288 | ], function (err) { 289 | t.error(err, 'no error') 290 | db.get('hello', function (err, node) { 291 | t.error(err, 'no error') 292 | t.same(node.key, 'hello') 293 | t.same(node.value, 'welt') 294 | }) 295 | db.get('hej', function (err, node) { 296 | t.error(err, 'no error') 297 | t.same(node.key, 'hej') 298 | t.same(node.value, 'verden') 299 | }) 300 | }) 301 | }) 302 | 303 | tape('batch with del', function (t) { 304 | t.plan(1 + 1 + 3 + 2) 305 | 306 | const db = create() 307 | 308 | db.batch([ 309 | {key: 'hello', value: 'world'}, 310 | {key: 'hej', value: 'verden'}, 311 | {key: 'hello', value: 'welt'} 312 | ], function (err) { 313 | t.error(err, 'no error') 314 | db.batch([ 315 | {key: 'hello', value: 'verden'}, 316 | {type: 'del', key: 'hej'} 317 | ], function (err) { 318 | t.error(err, 'no error') 319 | db.get('hello', function (err, node) { 320 | t.error(err, 'no error') 321 | t.same(node.key, 'hello') 322 | t.same(node.value, 'verden') 323 | }) 324 | db.get('hej', function (err, node) { 325 | t.error(err, 'no error') 326 | t.same(node, null) 327 | }) 328 | }) 329 | }) 330 | }) 331 | 332 | tape('multiple batches', function (t) { 333 | t.plan(19) 334 | 335 | const db = create() 336 | 337 | db.batch([{ 338 | type: 'put', 339 | key: 'foo', 340 | value: 'foo' 341 | }, { 342 | type: 'put', 343 | key: 'bar', 344 | value: 'bar' 345 | }], function (err, nodes) { 346 | t.error(err) 347 | t.same(2, nodes.length) 348 | same('foo', 'foo') 349 | same('bar', 'bar') 350 | db.batch([{ 351 | type: 'put', 352 | key: 'foo', 353 | value: 'foo2' 354 | }, { 355 | type: 'put', 356 | key: 'bar', 357 | value: 'bar2' 358 | }, { 359 | type: 'put', 360 | key: 'baz', 361 | value: 'baz' 362 | }], function (err, nodes) { 363 | t.error(err) 364 | t.same(3, nodes.length) 365 | same('foo', 'foo2') 366 | same('bar', 'bar2') 367 | same('baz', 'baz') 368 | }) 369 | }) 370 | 371 | function same (key, val) { 372 | db.get(key, function (err, node) { 373 | t.error(err, 'no error') 374 | t.same(node.key, key) 375 | t.same(node.value, val) 376 | }) 377 | } 378 | }) 379 | 380 | tape('createWriteStream', function (t) { 381 | t.plan(10) 382 | var db = create() 383 | var writer = db.createWriteStream() 384 | 385 | writer.write([{ 386 | type: 'put', 387 | key: 'foo', 388 | value: 'foo' 389 | }, { 390 | type: 'put', 391 | key: 'bar', 392 | value: 'bar' 393 | }]) 394 | 395 | writer.write({ 396 | type: 'put', 397 | key: 'baz', 398 | value: 'baz' 399 | }) 400 | 401 | writer.end(function (err) { 402 | t.error(err, 'no error') 403 | same('foo', 'foo') 404 | same('bar', 'bar') 405 | same('baz', 'baz') 406 | }) 407 | 408 | function same (key, val) { 409 | db.get(key, function (err, node) { 410 | t.error(err, 'no error') 411 | t.same(node.key, key) 412 | t.same(node.value, val) 413 | }) 414 | } 415 | }) 416 | 417 | tape('createWriteStream pipe', function (t) { 418 | t.plan(10) 419 | var index = 0 420 | const db = create() 421 | const writer = db.createWriteStream() 422 | const reader = new Readable({ 423 | objectMode: true, 424 | read: function (size) { 425 | var value = (index < 1000) ? { 426 | type: 'put', 427 | key: 'foo' + index, 428 | value: index++ 429 | } : null 430 | this.push(value) 431 | } 432 | }) 433 | reader.pipe(writer) 434 | writer.on('finish', function (err) { 435 | t.error(err, 'no error') 436 | same('foo1', 1) 437 | same('foo50', 50) 438 | same('foo999', 999) 439 | }) 440 | 441 | function same (key, val) { 442 | db.get(key, function (err, node) { 443 | t.error(err, 'no error') 444 | t.same(node.key, key) 445 | t.same(node.value, val) 446 | }) 447 | } 448 | }) 449 | 450 | tape('can insert falsy values', function (t) { 451 | t.plan(2 * 2 + 3 + 1) 452 | 453 | const db = create(null, {valueEncoding: 'json'}) 454 | 455 | db.put('hello', 0, function () { 456 | db.put('world', false, function () { 457 | db.get('hello', function (err, node) { 458 | t.error(err, 'no error') 459 | t.same(node && node.value, 0) 460 | }) 461 | db.get('world', function (err, node) { 462 | t.error(err, 'no error') 463 | t.same(node && node.value, false) 464 | }) 465 | 466 | const ite = db.iterator() 467 | const result = {} 468 | 469 | ite.next(function loop (err, node) { 470 | t.error(err, 'no error') 471 | 472 | if (!node) { 473 | t.same(result, {hello: 0, world: false}) 474 | return 475 | } 476 | 477 | result[node.key] = node.value 478 | ite.next(loop) 479 | }) 480 | }) 481 | }) 482 | }) 483 | 484 | tape('can put/get a null value', function (t) { 485 | t.plan(3) 486 | 487 | const db = create(null, {valueEncoding: 'json'}) 488 | db.put('some key', null, function (err) { 489 | t.error(err, 'no error') 490 | db.get('some key', function (err, node) { 491 | t.error(err, 'no error') 492 | t.same(node.value, null) 493 | }) 494 | }) 495 | }) 496 | 497 | tape('can create with metadata', function (t) { 498 | const db = create(null, { 499 | valueEncoding: 'json', 500 | metadata: 'A piece of metadata' 501 | }) 502 | db.ready(function (err) { 503 | t.error(err, 'no error') 504 | db.getMetadata(function (err, metadata) { 505 | t.error(err, 'no error') 506 | t.same(metadata, Buffer.from('A piece of metadata')) 507 | t.end() 508 | }) 509 | }) 510 | }) 511 | 512 | tape('can support a custom hash function', function (t) { 513 | const db = create(null, { 514 | valueEncoding: 'utf-8', 515 | hash: function (key) { 516 | return Buffer.from(key) 517 | } 518 | }) 519 | db.ready(function (err) { 520 | t.error(err, 'no error') 521 | db.put('hello', 'world', function (err) { 522 | t.error(err, 'no error') 523 | db.get('hello', function (err, node) { 524 | t.error(err, 'no error') 525 | t.same(node.value, 'world') 526 | t.end() 527 | }) 528 | }) 529 | }) 530 | }) 531 | 532 | tape('can delete with a custom hash function', function (t) { 533 | const db = create(null, { 534 | valueEncoding: 'utf-8', 535 | hash: function (key) { 536 | return Buffer.from(key) 537 | } 538 | }) 539 | db.ready(function (err) { 540 | t.error(err, 'no error') 541 | db.put('hello', 'world', function (err) { 542 | t.error(err, 'no error') 543 | db.get('hello', function (err, node) { 544 | t.error(err, 'no error') 545 | t.same(node.value, 'world') 546 | db.del('hello', function (err) { 547 | t.error(err, 'no error') 548 | db.get('hello', function (err, node) { 549 | t.error(err, 'no error') 550 | t.false(node) 551 | t.end() 552 | }) 553 | }) 554 | }) 555 | }) 556 | }) 557 | }) 558 | -------------------------------------------------------------------------------- /lib/messages.js: -------------------------------------------------------------------------------- 1 | // This file is auto generated by the protocol-buffers compiler 2 | 3 | /* eslint-disable quotes */ 4 | /* eslint-disable indent */ 5 | /* eslint-disable no-redeclare */ 6 | /* eslint-disable camelcase */ 7 | 8 | // Remember to `npm install --save protocol-buffers-encodings` 9 | var encodings = require('protocol-buffers-encodings') 10 | var varint = encodings.varint 11 | var skip = encodings.skip 12 | 13 | var Header = exports.Header = { 14 | buffer: true, 15 | encodingLength: null, 16 | encode: null, 17 | decode: null 18 | } 19 | 20 | var Node = exports.Node = { 21 | buffer: true, 22 | encodingLength: null, 23 | encode: null, 24 | decode: null 25 | } 26 | 27 | var Extension = exports.Extension = { 28 | buffer: true, 29 | encodingLength: null, 30 | encode: null, 31 | decode: null 32 | } 33 | 34 | defineHeader() 35 | defineNode() 36 | defineExtension() 37 | 38 | function defineHeader () { 39 | Header.encodingLength = encodingLength 40 | Header.encode = encode 41 | Header.decode = decode 42 | 43 | function encodingLength (obj) { 44 | var length = 0 45 | if (!defined(obj.type)) throw new Error("type is required") 46 | var len = encodings.string.encodingLength(obj.type) 47 | length += 1 + len 48 | if (defined(obj.metadata)) { 49 | var len = encodings.bytes.encodingLength(obj.metadata) 50 | length += 1 + len 51 | } 52 | if (defined(obj.subtype)) { 53 | var len = encodings.string.encodingLength(obj.subtype) 54 | length += 1 + len 55 | } 56 | return length 57 | } 58 | 59 | function encode (obj, buf, offset) { 60 | if (!offset) offset = 0 61 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 62 | var oldOffset = offset 63 | if (!defined(obj.type)) throw new Error("type is required") 64 | buf[offset++] = 10 65 | encodings.string.encode(obj.type, buf, offset) 66 | offset += encodings.string.encode.bytes 67 | if (defined(obj.metadata)) { 68 | buf[offset++] = 18 69 | encodings.bytes.encode(obj.metadata, buf, offset) 70 | offset += encodings.bytes.encode.bytes 71 | } 72 | if (defined(obj.subtype)) { 73 | buf[offset++] = 26 74 | encodings.string.encode(obj.subtype, buf, offset) 75 | offset += encodings.string.encode.bytes 76 | } 77 | encode.bytes = offset - oldOffset 78 | return buf 79 | } 80 | 81 | function decode (buf, offset, end) { 82 | if (!offset) offset = 0 83 | if (!end) end = buf.length 84 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 85 | var oldOffset = offset 86 | var obj = { 87 | type: "", 88 | metadata: null, 89 | subtype: "" 90 | } 91 | var found0 = false 92 | while (true) { 93 | if (end <= offset) { 94 | if (!found0) throw new Error("Decoded message is not valid") 95 | decode.bytes = offset - oldOffset 96 | return obj 97 | } 98 | var prefix = varint.decode(buf, offset) 99 | offset += varint.decode.bytes 100 | var tag = prefix >> 3 101 | switch (tag) { 102 | case 1: 103 | obj.type = encodings.string.decode(buf, offset) 104 | offset += encodings.string.decode.bytes 105 | found0 = true 106 | break 107 | case 2: 108 | obj.metadata = encodings.bytes.decode(buf, offset) 109 | offset += encodings.bytes.decode.bytes 110 | break 111 | case 3: 112 | obj.subtype = encodings.string.decode(buf, offset) 113 | offset += encodings.string.decode.bytes 114 | break 115 | default: 116 | offset = skip(prefix & 7, buf, offset) 117 | } 118 | } 119 | } 120 | } 121 | 122 | function defineNode () { 123 | Node.encodingLength = encodingLength 124 | Node.encode = encode 125 | Node.decode = decode 126 | 127 | function encodingLength (obj) { 128 | var length = 0 129 | if (!defined(obj.key)) throw new Error("key is required") 130 | var len = encodings.string.encodingLength(obj.key) 131 | length += 1 + len 132 | if (defined(obj.valueBuffer)) { 133 | var len = encodings.bytes.encodingLength(obj.valueBuffer) 134 | length += 1 + len 135 | } 136 | if (defined(obj.trieBuffer)) { 137 | var len = encodings.bytes.encodingLength(obj.trieBuffer) 138 | length += 1 + len 139 | } 140 | if (defined(obj.seq)) { 141 | var len = encodings.varint.encodingLength(obj.seq) 142 | length += 1 + len 143 | } 144 | if (defined(obj.flags)) { 145 | var len = encodings.varint.encodingLength(obj.flags) 146 | length += 1 + len 147 | } 148 | return length 149 | } 150 | 151 | function encode (obj, buf, offset) { 152 | if (!offset) offset = 0 153 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 154 | var oldOffset = offset 155 | if (!defined(obj.key)) throw new Error("key is required") 156 | buf[offset++] = 10 157 | encodings.string.encode(obj.key, buf, offset) 158 | offset += encodings.string.encode.bytes 159 | if (defined(obj.valueBuffer)) { 160 | buf[offset++] = 18 161 | encodings.bytes.encode(obj.valueBuffer, buf, offset) 162 | offset += encodings.bytes.encode.bytes 163 | } 164 | if (defined(obj.trieBuffer)) { 165 | buf[offset++] = 26 166 | encodings.bytes.encode(obj.trieBuffer, buf, offset) 167 | offset += encodings.bytes.encode.bytes 168 | } 169 | if (defined(obj.seq)) { 170 | buf[offset++] = 32 171 | encodings.varint.encode(obj.seq, buf, offset) 172 | offset += encodings.varint.encode.bytes 173 | } 174 | if (defined(obj.flags)) { 175 | buf[offset++] = 40 176 | encodings.varint.encode(obj.flags, buf, offset) 177 | offset += encodings.varint.encode.bytes 178 | } 179 | encode.bytes = offset - oldOffset 180 | return buf 181 | } 182 | 183 | function decode (buf, offset, end) { 184 | if (!offset) offset = 0 185 | if (!end) end = buf.length 186 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 187 | var oldOffset = offset 188 | var obj = { 189 | key: "", 190 | valueBuffer: null, 191 | trieBuffer: null, 192 | seq: 0, 193 | flags: 0 194 | } 195 | var found0 = false 196 | while (true) { 197 | if (end <= offset) { 198 | if (!found0) throw new Error("Decoded message is not valid") 199 | decode.bytes = offset - oldOffset 200 | return obj 201 | } 202 | var prefix = varint.decode(buf, offset) 203 | offset += varint.decode.bytes 204 | var tag = prefix >> 3 205 | switch (tag) { 206 | case 1: 207 | obj.key = encodings.string.decode(buf, offset) 208 | offset += encodings.string.decode.bytes 209 | found0 = true 210 | break 211 | case 2: 212 | obj.valueBuffer = encodings.bytes.decode(buf, offset) 213 | offset += encodings.bytes.decode.bytes 214 | break 215 | case 3: 216 | obj.trieBuffer = encodings.bytes.decode(buf, offset) 217 | offset += encodings.bytes.decode.bytes 218 | break 219 | case 4: 220 | obj.seq = encodings.varint.decode(buf, offset) 221 | offset += encodings.varint.decode.bytes 222 | break 223 | case 5: 224 | obj.flags = encodings.varint.decode(buf, offset) 225 | offset += encodings.varint.decode.bytes 226 | break 227 | default: 228 | offset = skip(prefix & 7, buf, offset) 229 | } 230 | } 231 | } 232 | } 233 | 234 | function defineExtension () { 235 | var Get = Extension.Get = { 236 | buffer: true, 237 | encodingLength: null, 238 | encode: null, 239 | decode: null 240 | } 241 | 242 | var Iterator = Extension.Iterator = { 243 | buffer: true, 244 | encodingLength: null, 245 | encode: null, 246 | decode: null 247 | } 248 | 249 | var Cache = Extension.Cache = { 250 | buffer: true, 251 | encodingLength: null, 252 | encode: null, 253 | decode: null 254 | } 255 | 256 | defineGet() 257 | defineIterator() 258 | defineCache() 259 | 260 | function defineGet () { 261 | Get.encodingLength = encodingLength 262 | Get.encode = encode 263 | Get.decode = decode 264 | 265 | function encodingLength (obj) { 266 | var length = 0 267 | if (defined(obj.head)) { 268 | var len = encodings.varint.encodingLength(obj.head) 269 | length += 1 + len 270 | } 271 | if (defined(obj.key)) { 272 | var len = encodings.string.encodingLength(obj.key) 273 | length += 1 + len 274 | } 275 | return length 276 | } 277 | 278 | function encode (obj, buf, offset) { 279 | if (!offset) offset = 0 280 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 281 | var oldOffset = offset 282 | if (defined(obj.head)) { 283 | buf[offset++] = 8 284 | encodings.varint.encode(obj.head, buf, offset) 285 | offset += encodings.varint.encode.bytes 286 | } 287 | if (defined(obj.key)) { 288 | buf[offset++] = 18 289 | encodings.string.encode(obj.key, buf, offset) 290 | offset += encodings.string.encode.bytes 291 | } 292 | encode.bytes = offset - oldOffset 293 | return buf 294 | } 295 | 296 | function decode (buf, offset, end) { 297 | if (!offset) offset = 0 298 | if (!end) end = buf.length 299 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 300 | var oldOffset = offset 301 | var obj = { 302 | head: 0, 303 | key: "" 304 | } 305 | while (true) { 306 | if (end <= offset) { 307 | decode.bytes = offset - oldOffset 308 | return obj 309 | } 310 | var prefix = varint.decode(buf, offset) 311 | offset += varint.decode.bytes 312 | var tag = prefix >> 3 313 | switch (tag) { 314 | case 1: 315 | obj.head = encodings.varint.decode(buf, offset) 316 | offset += encodings.varint.decode.bytes 317 | break 318 | case 2: 319 | obj.key = encodings.string.decode(buf, offset) 320 | offset += encodings.string.decode.bytes 321 | break 322 | default: 323 | offset = skip(prefix & 7, buf, offset) 324 | } 325 | } 326 | } 327 | } 328 | 329 | function defineIterator () { 330 | Iterator.encodingLength = encodingLength 331 | Iterator.encode = encode 332 | Iterator.decode = decode 333 | 334 | function encodingLength (obj) { 335 | var length = 0 336 | if (defined(obj.head)) { 337 | var len = encodings.varint.encodingLength(obj.head) 338 | length += 1 + len 339 | } 340 | if (defined(obj.key)) { 341 | var len = encodings.string.encodingLength(obj.key) 342 | length += 1 + len 343 | } 344 | if (defined(obj.flags)) { 345 | var len = encodings.varint.encodingLength(obj.flags) 346 | length += 1 + len 347 | } 348 | if (defined(obj.checkpoint)) { 349 | var len = encodings.bytes.encodingLength(obj.checkpoint) 350 | length += 1 + len 351 | } 352 | return length 353 | } 354 | 355 | function encode (obj, buf, offset) { 356 | if (!offset) offset = 0 357 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 358 | var oldOffset = offset 359 | if (defined(obj.head)) { 360 | buf[offset++] = 8 361 | encodings.varint.encode(obj.head, buf, offset) 362 | offset += encodings.varint.encode.bytes 363 | } 364 | if (defined(obj.key)) { 365 | buf[offset++] = 18 366 | encodings.string.encode(obj.key, buf, offset) 367 | offset += encodings.string.encode.bytes 368 | } 369 | if (defined(obj.flags)) { 370 | buf[offset++] = 24 371 | encodings.varint.encode(obj.flags, buf, offset) 372 | offset += encodings.varint.encode.bytes 373 | } 374 | if (defined(obj.checkpoint)) { 375 | buf[offset++] = 34 376 | encodings.bytes.encode(obj.checkpoint, buf, offset) 377 | offset += encodings.bytes.encode.bytes 378 | } 379 | encode.bytes = offset - oldOffset 380 | return buf 381 | } 382 | 383 | function decode (buf, offset, end) { 384 | if (!offset) offset = 0 385 | if (!end) end = buf.length 386 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 387 | var oldOffset = offset 388 | var obj = { 389 | head: 0, 390 | key: "", 391 | flags: 0, 392 | checkpoint: null 393 | } 394 | while (true) { 395 | if (end <= offset) { 396 | decode.bytes = offset - oldOffset 397 | return obj 398 | } 399 | var prefix = varint.decode(buf, offset) 400 | offset += varint.decode.bytes 401 | var tag = prefix >> 3 402 | switch (tag) { 403 | case 1: 404 | obj.head = encodings.varint.decode(buf, offset) 405 | offset += encodings.varint.decode.bytes 406 | break 407 | case 2: 408 | obj.key = encodings.string.decode(buf, offset) 409 | offset += encodings.string.decode.bytes 410 | break 411 | case 3: 412 | obj.flags = encodings.varint.decode(buf, offset) 413 | offset += encodings.varint.decode.bytes 414 | break 415 | case 4: 416 | obj.checkpoint = encodings.bytes.decode(buf, offset) 417 | offset += encodings.bytes.decode.bytes 418 | break 419 | default: 420 | offset = skip(prefix & 7, buf, offset) 421 | } 422 | } 423 | } 424 | } 425 | 426 | function defineCache () { 427 | Cache.encodingLength = encodingLength 428 | Cache.encode = encode 429 | Cache.decode = decode 430 | 431 | function encodingLength (obj) { 432 | var length = 0 433 | if (!defined(obj.start)) throw new Error("start is required") 434 | var len = encodings.varint.encodingLength(obj.start) 435 | length += 1 + len 436 | if (!defined(obj.end)) throw new Error("end is required") 437 | var len = encodings.varint.encodingLength(obj.end) 438 | length += 1 + len 439 | if (defined(obj.blocks)) { 440 | var packedLen = 0 441 | for (var i = 0; i < obj.blocks.length; i++) { 442 | if (!defined(obj.blocks[i])) continue 443 | var len = encodings.varint.encodingLength(obj.blocks[i]) 444 | packedLen += len 445 | } 446 | if (packedLen) { 447 | length += 1 + packedLen + varint.encodingLength(packedLen) 448 | } 449 | } 450 | return length 451 | } 452 | 453 | function encode (obj, buf, offset) { 454 | if (!offset) offset = 0 455 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 456 | var oldOffset = offset 457 | if (!defined(obj.start)) throw new Error("start is required") 458 | buf[offset++] = 8 459 | encodings.varint.encode(obj.start, buf, offset) 460 | offset += encodings.varint.encode.bytes 461 | if (!defined(obj.end)) throw new Error("end is required") 462 | buf[offset++] = 16 463 | encodings.varint.encode(obj.end, buf, offset) 464 | offset += encodings.varint.encode.bytes 465 | if (defined(obj.blocks)) { 466 | var packedLen = 0 467 | for (var i = 0; i < obj.blocks.length; i++) { 468 | if (!defined(obj.blocks[i])) continue 469 | packedLen += encodings.varint.encodingLength(obj.blocks[i]) 470 | } 471 | if (packedLen) { 472 | buf[offset++] = 26 473 | varint.encode(packedLen, buf, offset) 474 | offset += varint.encode.bytes 475 | } 476 | for (var i = 0; i < obj.blocks.length; i++) { 477 | if (!defined(obj.blocks[i])) continue 478 | encodings.varint.encode(obj.blocks[i], buf, offset) 479 | offset += encodings.varint.encode.bytes 480 | } 481 | } 482 | encode.bytes = offset - oldOffset 483 | return buf 484 | } 485 | 486 | function decode (buf, offset, end) { 487 | if (!offset) offset = 0 488 | if (!end) end = buf.length 489 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 490 | var oldOffset = offset 491 | var obj = { 492 | start: 0, 493 | end: 0, 494 | blocks: [] 495 | } 496 | var found0 = false 497 | var found1 = false 498 | while (true) { 499 | if (end <= offset) { 500 | if (!found0 || !found1) throw new Error("Decoded message is not valid") 501 | decode.bytes = offset - oldOffset 502 | return obj 503 | } 504 | var prefix = varint.decode(buf, offset) 505 | offset += varint.decode.bytes 506 | var tag = prefix >> 3 507 | switch (tag) { 508 | case 1: 509 | obj.start = encodings.varint.decode(buf, offset) 510 | offset += encodings.varint.decode.bytes 511 | found0 = true 512 | break 513 | case 2: 514 | obj.end = encodings.varint.decode(buf, offset) 515 | offset += encodings.varint.decode.bytes 516 | found1 = true 517 | break 518 | case 3: 519 | var packedEnd = varint.decode(buf, offset) 520 | offset += varint.decode.bytes 521 | packedEnd += offset 522 | while (offset < packedEnd) { 523 | obj.blocks.push(encodings.varint.decode(buf, offset)) 524 | offset += encodings.varint.decode.bytes 525 | } 526 | break 527 | default: 528 | offset = skip(prefix & 7, buf, offset) 529 | } 530 | } 531 | } 532 | } 533 | 534 | Extension.encodingLength = encodingLength 535 | Extension.encode = encode 536 | Extension.decode = decode 537 | 538 | function encodingLength (obj) { 539 | var length = 0 540 | if (defined(obj.cache)) { 541 | var len = Cache.encodingLength(obj.cache) 542 | length += varint.encodingLength(len) 543 | length += 1 + len 544 | } 545 | if (defined(obj.iterator)) { 546 | var len = Iterator.encodingLength(obj.iterator) 547 | length += varint.encodingLength(len) 548 | length += 1 + len 549 | } 550 | if (defined(obj.get)) { 551 | var len = Get.encodingLength(obj.get) 552 | length += varint.encodingLength(len) 553 | length += 1 + len 554 | } 555 | return length 556 | } 557 | 558 | function encode (obj, buf, offset) { 559 | if (!offset) offset = 0 560 | if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) 561 | var oldOffset = offset 562 | if (defined(obj.cache)) { 563 | buf[offset++] = 10 564 | varint.encode(Cache.encodingLength(obj.cache), buf, offset) 565 | offset += varint.encode.bytes 566 | Cache.encode(obj.cache, buf, offset) 567 | offset += Cache.encode.bytes 568 | } 569 | if (defined(obj.iterator)) { 570 | buf[offset++] = 18 571 | varint.encode(Iterator.encodingLength(obj.iterator), buf, offset) 572 | offset += varint.encode.bytes 573 | Iterator.encode(obj.iterator, buf, offset) 574 | offset += Iterator.encode.bytes 575 | } 576 | if (defined(obj.get)) { 577 | buf[offset++] = 26 578 | varint.encode(Get.encodingLength(obj.get), buf, offset) 579 | offset += varint.encode.bytes 580 | Get.encode(obj.get, buf, offset) 581 | offset += Get.encode.bytes 582 | } 583 | encode.bytes = offset - oldOffset 584 | return buf 585 | } 586 | 587 | function decode (buf, offset, end) { 588 | if (!offset) offset = 0 589 | if (!end) end = buf.length 590 | if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") 591 | var oldOffset = offset 592 | var obj = { 593 | cache: null, 594 | iterator: null, 595 | get: null 596 | } 597 | while (true) { 598 | if (end <= offset) { 599 | decode.bytes = offset - oldOffset 600 | return obj 601 | } 602 | var prefix = varint.decode(buf, offset) 603 | offset += varint.decode.bytes 604 | var tag = prefix >> 3 605 | switch (tag) { 606 | case 1: 607 | var len = varint.decode(buf, offset) 608 | offset += varint.decode.bytes 609 | obj.cache = Cache.decode(buf, offset, offset + len) 610 | offset += Cache.decode.bytes 611 | break 612 | case 2: 613 | var len = varint.decode(buf, offset) 614 | offset += varint.decode.bytes 615 | obj.iterator = Iterator.decode(buf, offset, offset + len) 616 | offset += Iterator.decode.bytes 617 | break 618 | case 3: 619 | var len = varint.decode(buf, offset) 620 | offset += varint.decode.bytes 621 | obj.get = Get.decode(buf, offset, offset + len) 622 | offset += Get.decode.bytes 623 | break 624 | default: 625 | offset = skip(prefix & 7, buf, offset) 626 | } 627 | } 628 | } 629 | } 630 | 631 | function defined (val) { 632 | return val !== null && val !== undefined && (typeof val !== 'number' || !isNaN(val)) 633 | } 634 | --------------------------------------------------------------------------------