├── .gitignore ├── README.md ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | sandbox.js 3 | sandbox/ 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bitbase 2 | 3 | WIP, database for bits 4 | 5 | ## Usage 6 | 7 | ``` js 8 | const Bitbase = require('bitbase') 9 | const raf = require('random-access-file') 10 | 11 | const bb = new Bitbase(raf('bits.db')) 12 | 13 | bb.set(42, true, function (err) { 14 | if (err) throw err 15 | bb.get(42, function (err, bit) { 16 | if (err) throw err 17 | console.log('bb -> ' + bit) 18 | }) 19 | }) 20 | ``` 21 | 22 | ## LICENSE 23 | 24 | MIT 25 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const Nanoresource = require('nanoresource') 2 | 3 | const FACTOR = 32768 4 | // const MAX_BITS = 64 5 | const STEPS = new Array(4) // Math.ceil(MAX_BITS / Math.log2(FACTOR))) 6 | 7 | STEPS[0] = 0 8 | for (let i = 1; i < STEPS.length; i++) { 9 | const n = FACTOR * STEPS[i - 1] + 1 10 | STEPS[i] = n 11 | } 12 | 13 | const LEAF_SIZE = 4096 + 128 + 4 + 128 + 4 14 | const NODE_SIZE = 4096 + 128 + 4 + 4096 + 128 + 4 15 | 16 | class Node { 17 | constructor (x, y, index, storage) { 18 | this.x = x 19 | this.y = y 20 | this.index = index 21 | this.storage = storage 22 | this.allOne = null 23 | this.oneOne = null 24 | this.buffer = null 25 | this.dirty = false 26 | } 27 | 28 | load (cb) { 29 | if (this.buffer) return cb(null, this) 30 | // const size = 4096 + 128 + 4 + 4096 + 128 + 4 31 | const size = this.y === 0 ? LEAF_SIZE : NODE_SIZE 32 | this.storage.read(this.byteOffset(), size, (err, buf) => { 33 | if (err) buf = Buffer.alloc(size) 34 | // if (err) return cb(err) 35 | if (this.allOne === null) this.onload(buf) 36 | cb(null, this) 37 | }) 38 | } 39 | 40 | byteOffset () { 41 | const leaves = this.x === 0 && this.y !== 0 42 | ? Math.pow(FACTOR, this.y - 1) 43 | : Math.pow(FACTOR, this.y) * this.x 44 | 45 | return LEAF_SIZE * leaves + NODE_SIZE * (this.index - leaves) 46 | } 47 | 48 | save (cb) { 49 | this.dirty = false 50 | this.storage.write(this.byteOffset(), this.buffer, cb) 51 | } 52 | 53 | onload (buf) { 54 | this.buffer = buf 55 | 56 | if (this.y === 0) { 57 | this.oneOne = [ 58 | new Uint32Array(buf.buffer, buf.byteOffset, 1024), 59 | new Uint32Array(buf.buffer, buf.byteOffset + 4096, 32), 60 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128, 1) 61 | ] 62 | 63 | const cpy = Buffer.concat([ buf.slice(0, 4096) ]) 64 | 65 | this.allOne = [ 66 | new Uint32Array(cpy.buffer, cpy.byteOffset, 1024), 67 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128 + 4, 32), 68 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128 + 4 + 128, 1) 69 | ] 70 | } else { 71 | this.oneOne = [ 72 | new Uint32Array(buf.buffer, buf.byteOffset, 1024), 73 | new Uint32Array(buf.buffer, buf.byteOffset + 4096, 32), 74 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128, 1) 75 | ] 76 | 77 | this.allOne = [ 78 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128 + 4, 1024), 79 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128 + 4 + 4096, 32), 80 | new Uint32Array(buf.buffer, buf.byteOffset + 4096 + 128 + 4 + 4096 + 128, 1) 81 | ] 82 | } 83 | } 84 | 85 | updateOneOne (b, upd) { 86 | for (let i = 0; i < 3; i++) { 87 | let r = b & 31 88 | let prev = this.oneOne[i][b >>>= 5] 89 | 90 | this.oneOne[i][b] = upd !== 0 91 | ? (prev | (0x80000000 >>> r)) 92 | : (prev & ~(0x80000000 >>> r)) 93 | 94 | upd = this.oneOne[i][b] 95 | if (upd === prev) return false 96 | this.dirty = true 97 | } 98 | 99 | return upd !== 0 100 | } 101 | 102 | updateAllOne (b, upd) { 103 | for (let i = 0; i < 3; i++) { 104 | let r = b & 31 105 | let prev = this.allOne[i][b >>>= 5] 106 | 107 | this.allOne[i][b] = upd === 0xffffffff 108 | ? (prev | (0x80000000 >>> r)) 109 | : (prev & ~(0x80000000 >>> r)) 110 | 111 | upd = this.allOne[i][b] 112 | if (upd === prev) return false 113 | this.dirty = true 114 | } 115 | 116 | return upd === 0xffffffff 117 | } 118 | 119 | last () { 120 | let n = 31 - ctz32(this.oneOne[2][0]) 121 | if (n === -1) return -1 122 | n = 32 * n + (31 - ctz32(this.oneOne[1][n])) 123 | return 32 * n + (31 - ctz32(this.oneOne[0][n])) 124 | } 125 | 126 | first () { 127 | let n = Math.clz32(this.oneOne[2][0]) 128 | if (n === 32) return -1 129 | n = 32 * n + Math.clz32(this.oneOne[1][n]) 130 | return 32 * n + Math.clz32(this.oneOne[0][n]) 131 | } 132 | 133 | child (n) { 134 | return new Node(this.x * FACTOR + n, this.y - 1, this.childIndex(n), this.storage) 135 | } 136 | 137 | childIndex (n) { 138 | if (this.x !== 0) return this.index + 1 + n * STEPS[this.y] 139 | if (n === 0) return STEPS[this.y - 1] 140 | return this.index + 1 + (n - 1) * STEPS[this.y] 141 | } 142 | } 143 | 144 | module.exports = class Bits extends Nanoresource { 145 | constructor (storage) { 146 | super() 147 | this.storage = storage 148 | this.root = new Node(0, 0, STEPS[0], storage) 149 | this.length = FACTOR 150 | 151 | this._path = new Uint16Array(5) 152 | this._offsets = new Uint16Array(this._path.buffer, this._path.byteOffset + 2, 4) 153 | this._cache = new Map() 154 | } 155 | 156 | _grow () { 157 | this.length *= 32768 158 | this.root = new Node(0, this.root.y + 1, STEPS[this.root.y + 1], this.storage) 159 | } 160 | 161 | _growAndSet (i, bit, cb) { 162 | const old = this.root 163 | 164 | this._grow() 165 | this.root.onload(Buffer.alloc(NODE_SIZE)) // y > 0 always here 166 | this.root.updateOneOne(0, old.oneOne[2][0]) 167 | this.root.updateAllOne(0, old.allOne[2][0]) 168 | 169 | this.root.save((err) => { 170 | if (err) return cb(err) 171 | this.set(i, bit, cb) 172 | }) 173 | } 174 | 175 | _open (cb) { 176 | this.storage.stat((err, st) => { 177 | if (err) return this.root.load(cb) 178 | 179 | const size = 4096 + 128 + 4 + 4096 + 128 + 4 180 | const maxIndex = st.size / size 181 | while (STEPS[this.root.y + 1] < maxIndex) { 182 | this._grow() 183 | } 184 | 185 | this.root.load(cb) 186 | }) 187 | } 188 | 189 | first (cb) { 190 | this.open(err => { 191 | if (err) return cb(err) 192 | 193 | let factor = Math.pow(32768, this.root.y) 194 | let offset = 0 195 | 196 | visit(null, this.root) 197 | 198 | function visit (err, node) { 199 | if (err) return cb(err) 200 | const n = node.first() 201 | if (n === -1) return cb(null, -1) 202 | if (node.y === 0) return cb(null, offset + n) 203 | offset += factor * n 204 | factor /= 32768 205 | node.child(n).load(visit) 206 | } 207 | }) 208 | } 209 | 210 | last (cb) { 211 | this.open(err => { 212 | if (err) return cb(err) 213 | 214 | let factor = Math.pow(32768, this.root.y) 215 | let offset = 0 216 | 217 | visit(null, this.root) 218 | 219 | function visit (err, node) { 220 | if (err) return cb(err) 221 | const n = node.last() 222 | if (n === -1) return cb(null, -1) 223 | if (node.y === 0) return cb(null, offset + n) 224 | offset += factor * n 225 | factor /= 32768 226 | node.child(n).load(visit) 227 | } 228 | }) 229 | } 230 | 231 | set (index, bit, cb) { 232 | this.open(err => { 233 | if (err) return cb(err) 234 | 235 | const path = new Uint16Array(4) 236 | factor(index, path) 237 | if (index >= this.length && bit) { 238 | this._growAndSet(index, bit, cb) 239 | return 240 | } 241 | 242 | const parents = new Array(path.length) 243 | 244 | parents[this.root.y] = this.root 245 | for (let y = this.root.y - 1; y >= 0; y--) { 246 | parents[y] = parents[y + 1].child(path[y + 1]) 247 | } 248 | 249 | let missing = this.root.y + 1 250 | const len = missing 251 | for (let i = 0; i < len; i++) parents[i].load(ondone) 252 | 253 | function ondone (err) { 254 | if (err) throw err 255 | if (--missing) return 256 | 257 | for (let i = 0; i < len; i++) { 258 | if (!parents[i].updateOneOne(path[i], bit ? 1 : 0)) break 259 | } 260 | for (let i = 0; i < len; i++) { 261 | if (!parents[i].updateAllOne(path[i], bit ? 0xffffffff : 0)) break 262 | } 263 | 264 | missing = 1 265 | for (let i = 0; i < len; i++) { 266 | if (!parents[i].dirty) break 267 | missing++ 268 | parents[i].save(done) 269 | } 270 | done(null) 271 | 272 | function done (err) { 273 | if (err) throw err 274 | if (!--missing) cb(null) 275 | } 276 | } 277 | }) 278 | } 279 | 280 | get (index, cb) { 281 | this.open(err => { 282 | if (err) return cb(err) 283 | 284 | if (index >= this.length) return cb(null, false) 285 | 286 | const path = new Uint16Array(4) 287 | factor(index, path) 288 | 289 | this.root.load(loop) 290 | 291 | function loop (err, node) { 292 | if (err) return cb(err) 293 | 294 | const i = path[node.y] 295 | const r = 0x80000000 >>> (i & 31) 296 | const b = i >>> 5 297 | 298 | if ((node.allOne[0][b] & r) !== 0) { 299 | return cb(null, true) 300 | } 301 | 302 | if ((node.oneOne[0][b] & r) === 0) { 303 | // console.log(node.oneOne[0][b], node.oneOne[0], b) 304 | return cb(null, false) 305 | } 306 | 307 | node.child(path[node.y]).load(loop) 308 | } 309 | }) 310 | } 311 | } 312 | 313 | function factor (n, out) { 314 | n = (n - (out[0] = (n & 32767))) / 32768 315 | n = (n - (out[1] = (n & 32767))) / 32768 316 | out[3] = ((n - (out[2] = (n & 32767))) / 32768) & 32767 317 | } 318 | 319 | function ctz32 (v) { 320 | let c = 32 321 | v &= -v 322 | if (v) c-- 323 | if (v & 0x0000FFFF) c -= 16 324 | if (v & 0x00FF00FF) c -= 8 325 | if (v & 0x0F0F0F0F) c -= 4 326 | if (v & 0x33333333) c -= 2 327 | if (v & 0x55555555) c -= 1 328 | return c 329 | } 330 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitbase", 3 | "version": "0.0.0", 4 | "description": "Bit database", 5 | "author": "Mathias Buus (@mafintosh)", 6 | "license": "MIT", 7 | "repository": "mafintosh/bitbase", 8 | "scripts": { 9 | "test": "standard && tape test.js" 10 | }, 11 | "dependencies": { 12 | "nanoresource": "^1.0.0" 13 | }, 14 | "devDependencies": { 15 | "standard": "^12.0.1", 16 | "tape": "^4.10.2" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const Bitfield = require('./') 3 | 4 | tape('set and get', function (t) { 5 | const bits = create() 6 | 7 | bits.set(4242, true, function (err) { 8 | t.error(err, 'no error') 9 | bits.get(4242, function (err, bit) { 10 | t.error(err, 'no error') 11 | t.ok(bit, 'should be set') 12 | bits.get(4243, function (err, bit) { 13 | t.error(err, 'no error') 14 | t.notOk(bit, 'should not be set') 15 | t.end() 16 | }) 17 | }) 18 | }) 19 | }) 20 | 21 | tape('set and unset', function (t) { 22 | const bits = create() 23 | 24 | bits.set(4242, true, function (err) { 25 | t.error(err, 'no error') 26 | bits.get(4242, function (err, bit) { 27 | t.error(err, 'no error') 28 | t.ok(bit, 'should be set') 29 | bits.set(4242, false, function (err, bit) { 30 | t.error(err, 'no error') 31 | bits.get(4242, function (err, bit) { 32 | t.error(err, 'no error') 33 | t.notOk(bit, 'should not be set') 34 | t.end() 35 | }) 36 | }) 37 | }) 38 | }) 39 | }) 40 | 41 | function create () { 42 | return new Bitfield(require('random-access-memory')()) 43 | } 44 | --------------------------------------------------------------------------------