├── .gitignore ├── LICENSE ├── README.md ├── example.js ├── index.js ├── lib ├── get.js ├── hash.js ├── node.js ├── put.js ├── storage.js └── trie-builder.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | sandbox.js 3 | *.db 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Hyperdivision 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tinybox 2 | 3 | Tiny, single file, scalable key value store based on HAMTs 4 | 5 | ``` 6 | npm install tinybox 7 | ``` 8 | 9 | Uses the [Hypertrie](https://github.com/mafintosh/hypertrie) trie without the replication parts, 10 | and auto compacts. 11 | 12 | Still under development but the storage format should be stable. Upcoming features include batching, 13 | deletions and getting all values out of the store. 14 | 15 | ## Usage 16 | 17 | ``` js 18 | const TinyBox = require('tinybox') 19 | 20 | const db = new TinyBox('./db') 21 | 22 | db.put('hello', 'world', function () { 23 | db.get('hello', console.log) 24 | }) 25 | ``` 26 | 27 | ## API 28 | 29 | #### `db = new TinyBox(storage)` 30 | 31 | Create a new tiny store. Storage can be any [random-access-storage](https://github.com/random-access-storage) instance. 32 | For conveinience you can pass a filename as storage as well. 33 | 34 | #### `db.get(key, callback)` 35 | 36 | Looks up a value. Key can be a buffer or string. If the key does not exist null is passed, otherwise 37 | the a Node object looking like this: 38 | 39 | ```js 40 | { 41 | key: , 42 | value: 43 | } 44 | ``` 45 | 46 | #### `db.put(key, [value], [callback])` 47 | 48 | Insert a key and optional value. 49 | 50 | ## License 51 | 52 | MIT 53 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const Tinystore = require('./') 2 | const t = new Tinystore(require('random-access-file')('test.db')) 3 | 4 | t.put('hi', null) 5 | t.put('ho', Buffer.alloc(9)) 6 | t.put('ha', null) 7 | t.put('hi', Buffer.alloc(120)) 8 | t.put('hx', null) 9 | t.put('soo', null) 10 | 11 | for (let i = 0; i < 1e4; i++) { 12 | t.put('hi-' + i, 'hi-' + i) 13 | } 14 | 15 | console.time() 16 | t.flush(function () { 17 | console.timeEnd() 18 | t.get('hi-4240', print) 19 | t.get('hi', print) 20 | t.get('ho', print) 21 | t.get('ha', print) 22 | t.get('hx', print) 23 | t.get('soo', print) 24 | t.get('hi-424', print) 25 | }) 26 | 27 | function print (err, node) { 28 | if (err) throw err 29 | console.log(node.key.toString() + ' --> ' + (node.value ? node.value.length : 0) + ' bytes', node.seq) 30 | } 31 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const Storage = require('./lib/storage') 2 | const get = require('./lib/get') 3 | const put = require('./lib/put') 4 | const mutexify = require('mutexify') 5 | const thunky = require('thunky') 6 | 7 | module.exports = class Tinystore { 8 | constructor (file) { 9 | this.data = new Storage(file) 10 | this.lock = mutexify() 11 | this.opened = false 12 | this.ready = thunky(open.bind(this)) 13 | } 14 | 15 | get (key, cb) { 16 | if (!Buffer.isBuffer(key)) key = Buffer.from(key) 17 | if (!this.opened) return openAndGet(this, key, cb) 18 | get(this, key, cb) 19 | } 20 | 21 | put (key, value, cb) { 22 | if (!cb) cb = noop 23 | if (!Buffer.isBuffer(key)) key = Buffer.from(key) 24 | if (!this.opened) return openAndPut(this, key, value || null, cb) 25 | if (value && !Buffer.isBuffer(value)) value = Buffer.from(value) 26 | put(this, key, value || null, cb) 27 | } 28 | 29 | flush (cb) { 30 | this.ready((err) => { 31 | if (err) return cb(err) 32 | this.lock(unlock => unlock(cb, null)) 33 | }) 34 | } 35 | } 36 | 37 | function noop () {} 38 | 39 | function openAndGet (self, key, cb) { 40 | self.ready(function (err) { 41 | if (err) return cb(err) 42 | self.get(key, cb) 43 | }) 44 | } 45 | 46 | function openAndPut (self, key, value, cb) { 47 | self.ready(function (err) { 48 | if (err) return cb(err) 49 | self.put(key, value, cb) 50 | }) 51 | } 52 | 53 | function open (cb) { 54 | this.data.open((err) => { 55 | if (err) return cb(err) 56 | this.opened = true 57 | cb(null) 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /lib/get.js: -------------------------------------------------------------------------------- 1 | const Node = require('./node') 2 | 3 | module.exports = get 4 | 5 | function get (store, key, cb) { 6 | const target = new Node(key, null, null) 7 | 8 | update(store.data, store.data.latest, target, 0, cb) 9 | } 10 | 11 | function getNodeAndUpdate (storage, target, i, seq, cb) { 12 | storage.get(seq, function (err, node) { 13 | if (err) return cb(err) 14 | update(storage, node, target, i + 1, cb) 15 | }) 16 | } 17 | 18 | function update (storage, head, target, i, cb) { 19 | if (!head) return cb(null, null) 20 | 21 | for (; i < target.hash.length; i++) { 22 | const val = target.hash.get(i) 23 | if (val === head.hash.get(i)) continue 24 | 25 | if (i >= head.links.length) return cb(null, null) 26 | 27 | const link = head.links[i] 28 | if (!link) return cb(null, null) 29 | 30 | const seq = link[val] 31 | if (!seq) return cb(null, null) 32 | 33 | getNodeAndUpdate(storage, target, i, seq, cb) 34 | return 35 | } 36 | 37 | if (head.key.equals(target.key)) return cb(null, head) 38 | cb(null, null) 39 | } 40 | -------------------------------------------------------------------------------- /lib/hash.js: -------------------------------------------------------------------------------- 1 | const sodium = require('sodium-universal') 2 | 3 | const KEY = Buffer.alloc(32).fill('tiny-store') 4 | 5 | module.exports = class HashPath { 6 | constructor (key) { 7 | this.hash = hash(key) 8 | this.length = this.hash.length * 4 + 1 9 | } 10 | 11 | get (i) { 12 | const j = i >> 2 13 | if (j >= this.hash.length) return 4 14 | return (this.hash[j] >> (2 * (i & 3))) & 3 15 | } 16 | } 17 | 18 | function hash (data) { 19 | const out = Buffer.allocUnsafe(8) 20 | sodium.crypto_shorthash(out, data, KEY) 21 | return out 22 | } 23 | -------------------------------------------------------------------------------- /lib/node.js: -------------------------------------------------------------------------------- 1 | const varint = require('varint') 2 | const Hash = require('./hash') 3 | const TrieBuilder = require('./trie-builder') 4 | 5 | const EMPTY = Buffer.alloc(0) 6 | 7 | class Node { 8 | constructor (key, value = null, links = null) { 9 | this.seq = 0 // set by storage 10 | this.blockSize = 0 // set by storage 11 | this.key = key 12 | this.value = value 13 | this.trieBuilder = links ? null : new TrieBuilder() 14 | this.links = links 15 | this.hash = new Hash(this.key) 16 | } 17 | 18 | finalise () { 19 | const { deflated, links } = this.trieBuilder.finalise() 20 | const buf = Buffer.allocUnsafe(encodingLength(this, deflated)) 21 | const val = this.value || EMPTY 22 | 23 | this.links = links 24 | this.trieBuilder = null 25 | 26 | let offset = 9 // we need 9 bytes for the storage header 27 | 28 | varint.encode(this.key.length, buf, offset) 29 | offset += varint.encode.bytes 30 | 31 | this.key.copy(buf, offset) 32 | offset += this.key.length 33 | 34 | varint.encode(deflated.length, buf, offset) 35 | offset += varint.encode.bytes 36 | 37 | deflated.copy(buf, offset) 38 | offset += deflated.length 39 | 40 | varint.encode(val.length, buf, offset) 41 | offset += varint.encode.bytes 42 | 43 | val.copy(buf, offset) 44 | offset += val.length 45 | 46 | return buf 47 | } 48 | 49 | static decode (buf) { 50 | let offset = 9 51 | 52 | const kl = varint.decode(buf, offset) 53 | offset += varint.decode.bytes 54 | const key = buf.slice(offset, offset += kl) 55 | 56 | const tl = varint.decode(buf, offset) 57 | offset += varint.decode.bytes 58 | const trieBuffer = buf.slice(offset, offset += tl) 59 | 60 | const vl = varint.decode(buf, offset) 61 | offset += varint.decode.bytes 62 | const value = vl ? buf.slice(offset, offset += vl) : null 63 | 64 | const links = TrieBuilder.inflate(trieBuffer) 65 | 66 | return new Node(key, value, links) 67 | } 68 | } 69 | 70 | function encodingLength (node, trieBuffer) { // excluding the free list link 71 | const val = node.value || EMPTY 72 | 73 | const nl = varint.encodingLength(node.key.length) 74 | const vl = varint.encodingLength(val.length) 75 | const tl = varint.encodingLength(trieBuffer.length) 76 | 77 | return 9 + nl + node.key.length + vl + val.length + tl + trieBuffer.length 78 | } 79 | 80 | module.exports = Node 81 | -------------------------------------------------------------------------------- /lib/put.js: -------------------------------------------------------------------------------- 1 | const Node = require('./node') 2 | 3 | module.exports = put 4 | 5 | function put (store, key, value, cb) { 6 | const target = new Node(key, value) 7 | 8 | store.lock(function (unlock) { 9 | update(store.data, store.data.latest, target, 0, unlock, cb) 10 | }) 11 | } 12 | 13 | function update (storage, head, target, i, unlock, cb) { 14 | if (!head) { 15 | finalize(storage, null, target, unlock, cb) 16 | return 17 | } 18 | 19 | for (; i < target.hash.length; i++) { 20 | const headVal = head.hash.get(i) 21 | const headLink = i < head.links.length ? head.links[i] : null 22 | const val = target.hash.get(i) 23 | 24 | if (headLink) { 25 | for (let j = 0; j < headLink.length; j++) { 26 | if (j === val || !headLink[j]) continue // we are closest 27 | target.trieBuilder.addLink(i, j, headLink[j]) 28 | } 29 | } 30 | 31 | if (val === headVal) continue 32 | target.trieBuilder.addLink(i, headVal, head.seq) 33 | 34 | if (!headLink) break 35 | const seq = headLink[val] 36 | if (!seq) break 37 | 38 | getNodeAndUpdate(storage, target, i, seq, unlock, cb) 39 | return 40 | } 41 | 42 | finalize(storage, head, target, unlock, cb) 43 | } 44 | 45 | function getNodeAndUpdate (storage, target, i, seq, unlock, cb) { 46 | storage.get(seq, function (err, node) { 47 | if (err) return cb(err) 48 | update(storage, node, target, i + 1, unlock, cb) 49 | }) 50 | } 51 | 52 | function finalize (storage, head, target, unlock, cb) { 53 | storage.append(target, function (err) { 54 | if (err) return done(err) 55 | 56 | if (head && head.key.equals(target.key)) { // if same, we can free old one 57 | storage.free(head, done) 58 | return 59 | } 60 | 61 | done(null) 62 | }) 63 | 64 | function done (err) { 65 | if (err) return unlock(cb, err) 66 | storage.writeHeader(unlock.bind(null, cb)) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lib/storage.js: -------------------------------------------------------------------------------- 1 | const uint64be = require('uint64be') 2 | const raf = require('random-access-file') 3 | const Node = require('./node') 4 | 5 | const MIN_SIZE = 128 6 | const MIN_BITS = 31 - Math.clz32(MIN_SIZE) - 1 7 | 8 | // | 0 | 1 | 2 | 8 | 16 | 24 | 536 | 537 9 | // header: | | | | | | | | 10 | 11 | module.exports = class Storage { 12 | constructor (file) { 13 | if (typeof file === 'string') file = raf(file) 14 | 15 | this.file = file 16 | this.header = null 17 | this.latest = null 18 | this.tick = 0 19 | this.size = 0 20 | this.top = 0 21 | } 22 | 23 | get (index, cb) { 24 | this.getBuffer(index, function (err, data) { 25 | if (err) return cb(err, null) 26 | const node = Node.decode(data) 27 | node.seq = index 28 | node.blockSize = data[8] 29 | cb(null, node) 30 | }) 31 | } 32 | 33 | getBuffer (index, cb) { 34 | const self = this 35 | const offset = this.offset(index) 36 | 37 | this.file.read(offset, offset + MIN_SIZE > this.size ? this.size - offset : MIN_SIZE, function (err, data) { 38 | if (err) return cb(null, data) 39 | const id = data[8] 40 | if (id === 0) return cb(null, data) 41 | const blkSize = MIN_SIZE << id 42 | self.file.read(offset, offset + blkSize > self.size ? self.size - offset : blkSize, cb) 43 | }) 44 | } 45 | 46 | offset (index) { 47 | return (index - 1) * MIN_SIZE + 4096 48 | } 49 | 50 | append (node, cb) { 51 | const self = this 52 | 53 | const blk = node.finalise() 54 | const id = node.blockSize = sizeId(blk.length) 55 | 56 | const ptr = 24 + id * 16 57 | const free = uint64be.decode(this.header, ptr) 58 | const prev = uint64be.decode(this.header, ptr + 8) 59 | const isTop = !free 60 | 61 | const index = free || this.top || 1 62 | const offset = this.offset(index) 63 | 64 | // encode header and preserve link 65 | uint64be.encode(isTop ? 0 : prev, blk, 0) 66 | blk[8] = id 67 | 68 | this.file.write(offset, blk, function (err) { 69 | if (err) return cb(err) 70 | 71 | const size = offset + blk.length 72 | if (size > self.size) self.size = size 73 | 74 | if (isTop) return onfreeused(null, null) 75 | if (!prev) return onfreeused(null, null) 76 | 77 | self.file.read(self.offset(prev), 10, onfreeused) 78 | 79 | function onfreeused (err, data) { 80 | if (err) return cb(err) 81 | 82 | uint64be.encode(index, self.header, 8) 83 | 84 | if (isTop) { 85 | self.top = index + (1 << id) 86 | uint64be.encode(self.top, self.header, 16) 87 | } else { 88 | const index = data ? uint64be.decode(data, 0) : 0 89 | uint64be.encode(prev, self.header, ptr) 90 | uint64be.encode(index, self.header, ptr + 8) 91 | } 92 | 93 | self.latest = node 94 | node.seq = index 95 | 96 | cb(null) 97 | } 98 | }) 99 | } 100 | 101 | free (node, cb) { 102 | const index = node.seq 103 | const header = this.header 104 | const ptr = 24 + node.blockSize * 16 105 | const freed = Buffer.allocUnsafe(8) 106 | const prev = uint64be.decode(header, ptr) 107 | 108 | uint64be.encode(prev, freed, 0) 109 | 110 | this.file.write(this.offset(index), freed, function (err) { 111 | if (err) return cb(err) 112 | 113 | uint64be.encode(index, header, ptr) 114 | uint64be.encode(prev, header, ptr + 8) 115 | cb(null) 116 | }) 117 | } 118 | 119 | writeHeader (cb) { 120 | const self = this 121 | 122 | const tick = (this.tick + 1) & 255 123 | const offset = (tick & 1) * 2048 124 | 125 | this.header[536] = tick 126 | this.file.write(offset, this.header, function (err) { 127 | if (!err) return cb(null) 128 | self.tick = tick 129 | cb(err) 130 | }) 131 | } 132 | 133 | open (cb) { 134 | const self = this 135 | 136 | this.file.read(2048, 537, function (_, a) { 137 | self.file.read(0, 537, function (err, b) { 138 | if (err) return onheader(Buffer.alloc(537)) 139 | if (!a) return onheader(b) 140 | 141 | const aDist = (a[536] - b[536]) & 255 142 | const bDist = (b[536] - a[536]) & 255 143 | 144 | if (aDist < bDist) onheader(a) 145 | else onheader(b) 146 | }) 147 | }) 148 | 149 | function onheader (header) { 150 | self.tick = header[536] 151 | self.header = header 152 | self.top = uint64be.decode(header, 16) 153 | 154 | const latest = uint64be.decode(header, 8) 155 | self.file.stat(function (_, st) { 156 | if (st) self.size = st.size 157 | if (!latest) return cb(null) 158 | 159 | self.get(latest, function (err, node) { 160 | if (err) return cb(err) 161 | self.latest = node 162 | cb(null) 163 | }) 164 | }) 165 | } 166 | } 167 | } 168 | 169 | function sizeId (blockSize) { 170 | const zeros = Math.clz32((blockSize - 1) >> MIN_BITS) 171 | return zeros === 32 ? 0 : 31 - zeros 172 | } 173 | -------------------------------------------------------------------------------- /lib/trie-builder.js: -------------------------------------------------------------------------------- 1 | const varint = require('varint') 2 | 3 | module.exports = class TrieBuilder { 4 | constructor () { 5 | this.offset = -1 6 | this.result = [] 7 | this.end = 0 8 | this.buckets = 0 9 | } 10 | 11 | bucket (offset) { 12 | if (this.end > 0) { 13 | this.result[this.end - 1] = this.result.length - this.end 14 | } 15 | this.result.push(offset, 0) 16 | this.end = this.result.length 17 | this.buckets++ 18 | this.offset = offset 19 | } 20 | 21 | addLink (offset, val, seq) { 22 | if (this.offset !== offset) this.bucket(offset) 23 | this.result.push(val, seq) 24 | return this.result.length - 1 25 | } 26 | 27 | updateLink (ptr, seq) { 28 | this.result[ptr] = seq 29 | } 30 | 31 | finalise () { 32 | if (this.end > 0) { 33 | this.result[this.end - 1] = this.result.length - this.end 34 | } 35 | 36 | const links = new Array(this.buckets && this.offset + 1) 37 | const buffer = Buffer.allocUnsafe(this.result.length * 8) 38 | 39 | let ptr = 0 40 | let pos = 0 41 | 42 | while (pos < this.result.length) { 43 | const bucket = this.result[pos] 44 | const count = this.result[pos + 1] 45 | 46 | const start = pos + 2 47 | const end = start + count 48 | 49 | const tmp = [0, 0, 0, 0, 0] 50 | let bitfield = 0 51 | 52 | for (let i = start; i < end; i += 2) { 53 | let val = this.result[i] 54 | const seq = this.result[i + 1] 55 | 56 | let updated = bitfield | (1 << val) 57 | while (updated === bitfield) { 58 | val += 5 59 | updated = bitfield | (1 << val) 60 | tmp.push(0, 0, 0, 0, 0) 61 | } 62 | 63 | bitfield = updated 64 | tmp[val] = seq 65 | } 66 | 67 | varint.encode(bucket, buffer, ptr) 68 | ptr += varint.encode.bytes 69 | 70 | varint.encode(bitfield, buffer, ptr) 71 | ptr += varint.encode.bytes 72 | 73 | links[bucket] = tmp 74 | 75 | for (let i = 0; i < tmp.length; i++) { 76 | if (tmp[i] > 0) { 77 | varint.encode(tmp[i], buffer, ptr) 78 | ptr += varint.encode.bytes 79 | } 80 | } 81 | 82 | pos = end 83 | } 84 | 85 | return { deflated: buffer.slice(0, ptr), links } 86 | } 87 | 88 | static inflate (buf) { 89 | if (!buf.length) return [] 90 | 91 | const offsets = [] 92 | const buckets = [] 93 | 94 | let pos = 0 95 | while (pos < buf.length) { 96 | const offset = varint.decode(buf, pos) 97 | pos += varint.decode.bytes 98 | 99 | let bitfield = varint.decode(buf, pos) 100 | pos += varint.decode.bytes 101 | 102 | const vals = [] 103 | 104 | while (bitfield > 0) { 105 | const zeros = Math.clz32(bitfield) 106 | bitfield &= (0x7fffffff >>> zeros) 107 | vals.push(31 - zeros) 108 | } 109 | 110 | const seqs = new Array(vals[vals.length - 1] <= 5 ? 5 : 30) 111 | seqs.fill(0) 112 | 113 | for (let i = vals.length - 1; i >= 0; i--) { 114 | seqs[vals[i]] = varint.decode(buf, pos) 115 | pos += varint.decode.bytes 116 | } 117 | 118 | buckets.push(seqs) 119 | offsets.push(offset) 120 | } 121 | 122 | const links = new Array(offsets[offsets.length - 1]) 123 | for (let i = 0; i < offsets.length; i++) links[offsets[i]] = buckets[i] 124 | return links 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tinybox", 3 | "version": "0.0.0", 4 | "description": "Tiny, single file, scalable key value store based on HAMTs", 5 | "main": "index.js", 6 | "dependencies": { 7 | "mutexify": "^1.2.0", 8 | "random-access-file": "^2.1.3", 9 | "sodium-universal": "^2.0.0", 10 | "thunky": "^1.0.3", 11 | "uint64be": "^2.0.2", 12 | "varint": "^5.0.0" 13 | }, 14 | "devDependencies": { 15 | "standard": "^13.0.1" 16 | }, 17 | "scripts": { 18 | "test": "standard" 19 | }, 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/hyperdivision/tinybox.git" 23 | }, 24 | "author": "Mathias Buus (@mafintosh)", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/hyperdivision/tinybox/issues" 28 | }, 29 | "homepage": "https://github.com/hyperdivision/tinybox" 30 | } 31 | --------------------------------------------------------------------------------