├── .gitignore ├── LICENSE ├── README.md ├── example.js ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | db 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # smalltable 2 | 3 | An on-disk key-value store with very few features 4 | 5 | ``` 6 | npm install smalltable 7 | ``` 8 | 9 | ## Usage 10 | 11 | ``` js 12 | var smalltable = require('smalltable') 13 | var db = smalltable('my.db', { 14 | hash: 'sha1', // use sha1 to hash the keys 15 | valueLength: 5 // values take up 5 bytes 16 | }) 17 | 18 | db.put('hello', 'world', function (err) { 19 | if (err) throw err 20 | db.get('hello', function (err, value) { 21 | if (err) throw err 22 | console.log('hello -> ' + value) 23 | }) 24 | }) 25 | ``` 26 | 27 | ## API 28 | 29 | #### `db = smalltable(filename, options)` 30 | 31 | Create a new db instance. The data will stored in a file called `filename`. 32 | 33 | Options include: 34 | 35 | ``` js 36 | { 37 | hash: 'sha1', // optional key hashing algorithm 38 | keyLength: 5, // optional key byte length 39 | valueLength: 5, // value byte length 40 | values: 128, // optional inital value count 41 | truncate: false // optional truncate the database 42 | } 43 | ``` 44 | 45 | If you don't specify `hash` you have to specify `keyLength`. If you store more than `values` values the database will be automatically expanded to contain more values. Currently this requires re-indexing all values stored in the database. 46 | 47 | #### `db.put(key, value, [cb])` 48 | 49 | Store a value. If the value if less than `valueLength` it will be zero-padded. 50 | 51 | #### `db.get(key, cb)` 52 | 53 | Retrive a value. Callback is called with `cb(err, buffer)`. If the key isn't found an error is returned. 54 | 55 | #### `db.del(key, [cb])` 56 | 57 | Delete a value 58 | 59 | #### `db.flush(cb)` 60 | 61 | Flushes the database by performing an fsync on the underlying file descriptor 62 | 63 | #### `db.close(cb)` 64 | 65 | Closes the underlying file descriptor. 66 | 67 | #### `stream = db.list()` 68 | 69 | Get a stream of all values `{key: key, value: value}` in the database. If `hash` was set in the constructor the key will be the hash of the key. 70 | 71 | ## License 72 | 73 | MIT 74 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var smalltable = require('./') 2 | 3 | var db = smalltable('./db', { 4 | keyLength: 5, 5 | valueLength: 5 6 | }) 7 | 8 | db.put('hello', 'world', function (err) { 9 | if (err) throw err 10 | db.get('hello', function (err, value) { 11 | if (err) throw err 12 | console.log('hello -> ' + value) 13 | }) 14 | }) 15 | 16 | // db.list().on('data', function (data) { 17 | // console.log(data.value.toString()) 18 | // }) 19 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var open = require('random-access-open') 2 | var crypto = require('crypto') 3 | var equals = require('buffer-equals') 4 | var stream = require('readable-stream') 5 | var thunky = require('thunky') 6 | var fs = require('fs') 7 | var util = require('util') 8 | 9 | var MAX_PROBES = 64 10 | 11 | module.exports = SmallTable 12 | 13 | function SmallTable (filename, opts) { 14 | if (!(this instanceof SmallTable)) return new SmallTable(filename, opts) 15 | var self = this 16 | 17 | if (!opts) opts = {} 18 | if (!opts.keyLength && !opts.hash) throw new Error('opts.keyLength or opts.hash is required') 19 | if (!opts.valueLength) throw new Error('opts.valueLength is required') 20 | 21 | this.filename = filename 22 | this.hash = opts.hash 23 | this.keyLength = opts.keyLength || crypto.createHash(this.hash).update('').digest().length 24 | this.valueLength = opts.valueLength 25 | this.chunkLength = this.valueLength + this.keyLength 26 | this.blankKey = blank(this.keyLength) 27 | this.blankValue = blank(this.valueLength) 28 | 29 | this.fd = 0 30 | this.size = 0 31 | this.values = 0 32 | this._wait = true 33 | 34 | this.open = thunky(ready) 35 | this.open() 36 | 37 | function ready (cb) { 38 | if (opts.truncate) fs.open(filename, 'w+', next) 39 | else open(filename, next) 40 | 41 | function next (err, fd) { 42 | if (err) return cb(err) 43 | fs.fstat(fd, function (err, st) { 44 | if (err) return onerror(fd, err) 45 | var size = st.size || ((opts.values || 64 * MAX_PROBES) * self.chunkLength) 46 | fs.ftruncate(fd, size, function (err) { 47 | if (err) return onerror(fd, err) 48 | self.size = size 49 | self.fd = fd 50 | self.values = Math.floor(self.size / self.chunkLength) 51 | self._wait = false 52 | cb() 53 | }) 54 | }) 55 | } 56 | 57 | function onerror (fd, err) { 58 | fs.close(fd, function () { 59 | cb(err) 60 | }) 61 | } 62 | } 63 | } 64 | 65 | SmallTable.prototype.del = function (key, cb) { 66 | if (!cb) cb = noop 67 | if (typeof key === 'string') key = Buffer(key) 68 | if (this.hash) key = crypto.createHash(this.hash).update(key).digest() 69 | 70 | if (this._wait) this._defer(key, this.blankValue, cb) 71 | else this._visit(key, this.blankValue, cb) 72 | } 73 | 74 | SmallTable.prototype.put = function (key, value, cb) { 75 | if (!cb) cb = noop 76 | if (typeof key === 'string') key = Buffer(key) 77 | if (typeof value === 'string') value = Buffer(value) 78 | if (this.hash) key = crypto.createHash(this.hash).update(key).digest() 79 | 80 | if (this._wait) this._defer(key, value, cb) 81 | else this._visit(key, value, cb) 82 | } 83 | 84 | SmallTable.prototype.get = function (key, cb) { 85 | if (typeof key === 'string') key = Buffer(key) 86 | if (this.hash) key = crypto.createHash(this.hash).update(key).digest() 87 | 88 | if (this._wait) this._defer(key, null, cb) 89 | else this._visit(key, null, cb) 90 | } 91 | 92 | SmallTable.prototype.flush = function (cb) { 93 | if (!this.fd) return this.open(this.flush.bind(this, cb)) 94 | fs.fsync(this.fd, cb || noop) 95 | } 96 | 97 | SmallTable.prototype.list = function () { 98 | return new ReadStream(this) 99 | } 100 | 101 | SmallTable.prototype._expand = function (cb) { 102 | if (!cb) cb = noop 103 | 104 | var self = this 105 | var buf = Buffer(65536 - (65536 % this.chunkLength)) 106 | var pos = 0 107 | var missing = 0 108 | var expanded = new SmallTable(this.filename + '.expand', { 109 | valueLength: this.valueLength, 110 | keyLength: this.keyLength, 111 | values: this.values * 2, 112 | truncate: true 113 | }) 114 | 115 | this._wait = true 116 | this.open = thunky(grow) 117 | this.open(cb) 118 | 119 | function grow (cb) { 120 | var error = null 121 | 122 | expanded.open(function (err) { 123 | if (err) return cb(err) 124 | fs.read(self.fd, buf, 0, buf.length, 0, onread) 125 | }) 126 | 127 | function done () { 128 | fs.rename(expanded.filename, self.filename, function (err) { 129 | if (err) return cb(err) 130 | var oldFd = self.fd 131 | self.fd = expanded.fd 132 | fs.close(oldFd, function (err) { 133 | if (err) return cb(err) 134 | self.size = expanded.size 135 | self.values = expanded.values 136 | cb() 137 | }) 138 | }) 139 | } 140 | 141 | function check (err) { 142 | if (err) error = err 143 | if (--missing) return 144 | if (error) return cb(error) 145 | fs.read(self.fd, buf, 0, buf.length, pos, onread) 146 | } 147 | 148 | function onread (err, bytes) { 149 | if (err) return cb(err) 150 | if (!bytes) return done() 151 | 152 | for (var i = 0; i < bytes; i += self.chunkLength) { 153 | if (i + self.chunkLength > bytes) break 154 | pos += self.chunkLength 155 | var key = buf.slice(i, i + self.keyLength) 156 | if (!equals(key, self.blankKey)) { 157 | missing++ 158 | var val = buf.slice(i + self.keyLength, i + self.keyLength + self.valueLength) 159 | expanded._visit(key, val, check) 160 | } 161 | } 162 | 163 | if (!missing) { 164 | missing = 1 165 | check() 166 | } 167 | } 168 | } 169 | } 170 | 171 | SmallTable.prototype._defer = function (key, value, cb) { 172 | var self = this 173 | this.open(function (err) { 174 | if (err) return cb(err) 175 | self._visit(key, value, cb) 176 | }) 177 | } 178 | 179 | SmallTable.prototype._visit = function (hash, value, cb) { 180 | var pos = toNumber(hash) % this.values 181 | 182 | var writing = !!value 183 | var self = this 184 | var buf = Buffer(this.chunkLength) 185 | var probes = MAX_PROBES 186 | var flushing = false 187 | 188 | fs.read(this.fd, buf, 0, buf.length, this.chunkLength * pos, check) 189 | 190 | function check (err, bytes) { 191 | if (err) return cb(err) 192 | if (flushing) return cb(null) 193 | if (!bytes) buf.fill(0) 194 | 195 | var oldKey = buf.slice(0, hash.length) 196 | 197 | if (writing) { 198 | if (equals(self.blankKey, oldKey) || equals(hash, oldKey)) { 199 | if (value === self.blankValue) self.blankKey.copy(buf) 200 | else hash.copy(buf) 201 | value.copy(buf, hash.length) 202 | flushing = true 203 | fs.write(self.fd, buf, 0, buf.length, self.chunkLength * pos, check) 204 | return 205 | } 206 | } else { 207 | if (equals(hash, oldKey)) return cb(null, buf.slice(hash.length)) 208 | } 209 | 210 | if (!probes) { 211 | if (!writing) return cb(new Error('Could not find key')) 212 | self._grow(hash, value, cb) 213 | return 214 | } 215 | 216 | if (pos === self.values - 1) pos = 0 217 | else pos++ 218 | probes-- 219 | 220 | fs.read(self.fd, buf, 0, buf.length, self.chunkLength * pos, check) 221 | } 222 | } 223 | 224 | SmallTable.prototype._grow = function (hash, value, cb) { 225 | var self = this 226 | this._expand(function (err) { 227 | if (err) return cb(err) 228 | self._visit(hash, value, cb) 229 | }) 230 | } 231 | 232 | function ReadStream (table) { 233 | stream.Readable.call(this, {objectMode: true}) 234 | 235 | var pos = 0 236 | var buf = Buffer(65536 - (65536 % table.chunkLength)) 237 | 238 | this.destroyed = false 239 | this._table = table 240 | this._reading = false 241 | this._kick = kick 242 | 243 | var self = this 244 | 245 | function kick (err) { 246 | if (err) return self.destroy(err) 247 | fs.read(table.fd, buf, 0, buf.length, pos, onread) 248 | } 249 | 250 | function onread (err, bytes) { 251 | if (err) return self.destroy(err) 252 | if (!bytes) return self.push(null) 253 | 254 | var flushed = true 255 | for (var i = 0; i < bytes; i += table.chunkLength) { 256 | if (i + table.chunkLength > bytes) break 257 | pos += table.chunkLength 258 | 259 | var key = buf.slice(i, i + table.keyLength) 260 | if (!equals(key, table.blankKey)) { 261 | var copyKey = Buffer(table.keyLength) 262 | key.copy(copyKey) 263 | var copyValue = Buffer(table.valueLength) 264 | buf.copy(copyValue, 0, i + table.keyLength, i + table.keyLength + table.valueLength) 265 | flushed = self.push({key: copyKey, value: copyValue}) 266 | if (self.destroyed) return 267 | } 268 | } 269 | self._reading = false 270 | if (flushed) self._read() 271 | } 272 | } 273 | 274 | util.inherits(ReadStream, stream.Readable) 275 | 276 | ReadStream.prototype.destroy = function (err) { 277 | if (this.destroyed) return 278 | this.destroyed = true 279 | if (err) this.emit('error', err) 280 | this.emit('close') 281 | } 282 | 283 | ReadStream.prototype._read = function () { 284 | if (this._reading || this.destroyed) return 285 | this._reading = true 286 | this._table.open(this._kick) 287 | } 288 | 289 | function toNumber (buf) { 290 | if (buf.length < 6) { 291 | switch (buf.length) { 292 | case 0: return 0 293 | case 1: return buf[0] 294 | case 2: return 256 * buf[0] + buf[1] 295 | case 3: return 65536 * buf.readUInt16BE(0) + buf[2] 296 | case 4: return buf.readUInt32BE(0) 297 | case 5: return 256 * buf.readUInt32BE(0) + buf[4] 298 | } 299 | } 300 | return 65536 * buf.readUInt32BE(0) + buf.readUInt16BE(4) 301 | } 302 | 303 | function noop () {} 304 | 305 | function blank (n) { 306 | var buf = Buffer(n) 307 | buf.fill(0) 308 | return buf 309 | } 310 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "smalltable", 3 | "version": "1.0.1", 4 | "description": "An on-disk key-value store with very few features", 5 | "main": "index.js", 6 | "dependencies": { 7 | "buffer-equals": "^1.0.3", 8 | "random-access-open": "^1.1.0", 9 | "readable-stream": "^2.0.5", 10 | "thunky": "^0.1.0" 11 | }, 12 | "devDependencies": { 13 | "standard": "^5.4.1" 14 | }, 15 | "scripts": { 16 | "test": "standard" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/mafintosh/smalltable.git" 21 | }, 22 | "author": "Mathias Buus (@mafintosh)", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/mafintosh/smalltable/issues" 26 | }, 27 | "homepage": "https://github.com/mafintosh/smalltable" 28 | } 29 | --------------------------------------------------------------------------------