├── LICENSE ├── README.md ├── hashtable.js ├── index.js ├── multi.js ├── package-lock.json ├── package.json └── test ├── bench.js ├── flumeview-index.js ├── index.js ├── multi.js ├── reconstruct.js └── simple.js /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 'Dominic Tarr' 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flumeview-hashtable 2 | 3 | A in-memory hashtable based flumeview. 4 | 5 | Creates an unordered key:value mapping over a flumedb. 6 | the key for each indexed record must be unique, and 7 | the value will be the record stored in the log. 8 | Since flumedb uses integer keys for the log implementation, 9 | the hashtable is just a sparse array of integers, implemented 10 | on a buffer. This means that a relatively small buffer (say 1mb) 11 | can hold hundreds of thousands of references, and since buffers 12 | are not traversed by the garbage collector, this is not a burden 13 | on your app. Also, there is no disk-io while building the index 14 | so if the only disk access is very slow (i.e. browser) this is great. 15 | 16 | ## example 17 | 18 | ``` js 19 | //you need to provide a hash function. 20 | //it doesn't actually need to be cryptographic 21 | 22 | function hash (v) { 23 | return crypto.createHash('sha256') 24 | .update(v.toString()) 25 | .digest().readUInt32BE(0) 26 | } 27 | 28 | //default getKey function 29 | function getKey (data) { 30 | return data.key 31 | } 32 | 33 | var db = Flume( 34 | Log( 35 | '/tmp/test-flumeview-hashtable'+Date.now(), 36 | {blockSize: 1024, codec: require('flumecodec/json')} 37 | )) 38 | .use('index', require('../')(1, hash, getKey)) 39 | 40 | db.append({key: 'foo', value: ...}, function (err) { 41 | if(err) throw err 42 | db.index.get('foo', function (err, value) { 43 | 44 | }) 45 | }) 46 | ``` 47 | 48 | ## append-only resize 49 | 50 | this implements a simple resize strategy, when the hashtable starts to become saturated, 51 | a second hashtable twice the size is allocated. when querying, look in the largest 52 | hashtable first. 53 | 54 | ## TODO 55 | 56 | * migrate records accessed into the large table, and throw out 57 | the smaller ones when possible. 58 | * maybe implement multiple records? for not quite unique values? 59 | * support different key sizes. 60 | 61 | ## api 62 | 63 | ### FlumeViewHashTable = require('flumeview-hashtable') 64 | 65 | #### FlumeViewHashTable(version, hash, getKey, minSlots) => fvht 66 | 67 | create a flumeview based on a hash table. This view is suitable only 68 | for unique keys. 69 | `version` is an integer. If you change any of the supplied functions 70 | or the version of this library, `version` must be changed. 71 | 72 | `hash` and `getKey` are passed to [HashTable](./#HashTable) module. 73 | see documentation below. 74 | 75 | `minSlots` is the size of the smallest hashtable to start with. 76 | 77 | `fvht` supports the required flumeview apis (`since, createSink, destroy, close`) 78 | and also exposes methods `get` and `load` 79 | 80 | #### fvht.get(key, cb(err, data, seq)) 81 | 82 | retrive a record by `key`. 83 | 84 | #### fvht.load() => float 85 | 86 | returns the current load, `count / slots`. when this is near 0.5 it will make the hashtable 87 | bigger, which will decrease the load. 88 | 89 | ### HashTable = require('flumeview-hashtable/hashtable') 90 | 91 | create hashtables that look up data asynchronously. 92 | 93 | #### HashTable(hash, matches, get) => createHashTable(buffer) => ht 94 | 95 | takes `hash` `matches` and `get` and returns a factory function that accepts 96 | a buffer, and returns a hashtable instance. 97 | 98 | ``` js 99 | //initialize a hash table with 100 | var createHashTable = HashTable(hash, matches, get) 101 | 102 | //pass a buffer for the hashtable to use. 103 | var ht = createHashTable(Buffer.alloc(4*1024*1024)) 104 | //or give the number of slots desired. 105 | var ht = createHashTable(1024*1024) 106 | ``` 107 | 108 | setting up a new hash table requires passing in 3 methods that will be used 109 | to hash input values and return results. 110 | 111 | ##### `hash(key) => index` 112 | 113 | Takes a key (of any type, but that you indend to pass to `ht.add(key, seq)` later 114 | and returns a 32 bit int hash value, that will be used as the index within the hashtable. 115 | the value `seq` will be placed at the first empty slot after modulus of `index` and the number 116 | of slots in the hashtable. 117 | 118 | `hash` would usually contain a hash function, but if the key is already a hash, 119 | it's okay to just read a integer from it. 120 | 121 | `index` must not ever be zero because zero represents an empty space in the filter. 122 | 123 | ##### `matches(data, key) => boolean` 124 | 125 | when you call `get(key, cb)` this method is used to check each value, 126 | after it is returned by `get`. `data` is the record retrived, and `key` is the query. 127 | 128 | ##### `get(seq, cb(err, data, seq))` 129 | 130 | retrive `data` at key. 131 | if you added `ht.add(key, index)` then did `ht.get(key)` `get(index, cb)` will be called. 132 | 133 | #### `ht.slots => int` 134 | 135 | a read only property, the maximum capacity of the hashtable. 136 | 137 | #### `ht.count => int` 138 | 139 | a read only property, the current occupied capacity of the hashtable. 140 | hashtables become inefficient when they become full, so usually you want 141 | to move to a new hashtable when `ht.count >= ht.slots/2` 142 | 143 | #### `ht.add(key, seq)` 144 | 145 | add a `key`, pointing at `seq`. `seq` must be a 32 bit int. 146 | The return value for get is expected to be already stored at `seq`. 147 | 148 | #### `ht.get(key, cb(err, data, seq))` 149 | 150 | retrive a previously added key. This will look up the `seq`, and call `get(seq, cb)` 151 | (which was passed to `createHashTable(hash, matches, get)`). 152 | 153 | #### `ht.buffer => Buffer` 154 | 155 | a read only property. The buffer making up the hash table. 156 | The first two UInt32LE's (unsigned 32 bit big endian int) store the `slots` and the `count`. 157 | the rest are slots that may have indexes stored. 158 | 159 | #### `ht.load()` 160 | 161 | return `count / slots` when greater than 0.5 it's time to allocate a new hashtable. 162 | when a new hashtable is created, it will be twice the size of the previous one. 163 | 164 | ### createMulti = require('flumeview-hashtable/multi') 165 | 166 | create progressively larger hashtables, so they remain efficient. 167 | 168 | #### createMulti(createHashTable(buffer)) => MultiHashTable 169 | 170 | given the initialized `createHashTable` function, returns an `MultiHashTable` instance. 171 | this has the same api as HashTable, except it handles resizing automatically. 172 | 173 | #### mht.count => int 174 | 175 | the number of items currently in the collection of hash tables. 176 | 177 | #### mht.slots => int 178 | 179 | the number of available slots in the collection of hash tables. 180 | 181 | #### mht.get(key, cb(err, data, seq)) 182 | 183 | call with key and returns the first data found that `matches` the `key` 184 | 185 | ### mht.add(key, seq) 186 | 187 | adds `key` to the hashtable. The key is only added to the last created, 188 | largest hashtable. 189 | 190 | ### mht.buffer() => [buffer,...] 191 | 192 | returns an array of all buffers in the collection of hashtables. 193 | 194 | ## License 195 | 196 | MIT 197 | -------------------------------------------------------------------------------- /hashtable.js: -------------------------------------------------------------------------------- 1 | function NotFound (key) { 2 | var err = new Error('Not Found:'+key) 3 | err.notFound = true 4 | err.name = 'NotFoundError' 5 | return err 6 | } 7 | 8 | module.exports = function (hash, matches, get) { 9 | return function (buffer) { 10 | var size = 4, slots 11 | var self 12 | var count = 0 13 | var HEADER = 8 //[slots, count] 14 | 15 | if('number' === typeof buffer) { 16 | slots = buffer 17 | buffer = new Buffer(HEADER+buffer*size) 18 | buffer.fill(0) 19 | buffer.writeUInt32BE(slots, 0) 20 | } 21 | else { 22 | slots = (buffer.length-HEADER)/size 23 | var _slots = buffer.readUInt32BE(0) 24 | if(_slots != slots) 25 | throw new Error('mismatch number of slots, expected:'+_slots + ', got:'+slots) 26 | //XXX reads count, but doesn't use update it later!!! 27 | count = buffer.readUInt32BE(4) 28 | } 29 | 30 | function _get (i) { 31 | return buffer.readUInt32BE(HEADER + (i%slots)*size) 32 | } 33 | function _set (i, v) { 34 | buffer.writeUInt32BE(v, HEADER + (i%slots)*size) 35 | } 36 | 37 | return self = { 38 | count: count, 39 | slots: slots, 40 | get: function (key, cb) { 41 | ;(function next (i) { 42 | var k = _get(i) 43 | if(k === 0) { 44 | cb(NotFound(key)) 45 | } 46 | else 47 | get(k, function (err, data) { 48 | if(err) cb(err) 49 | else if(matches(data, key)) { 50 | cb(null, data, k) 51 | } 52 | else next(i + 1) 53 | }) 54 | })(hash(key)) 55 | }, 56 | _get: _get, 57 | add: function (key, index) { 58 | var i = hash(key) 59 | while(true) { 60 | var j = _get(i) 61 | if(j === 0) { 62 | _set(i, index) 63 | buffer.writeUInt32BE(self.count = ++count, 4) 64 | 65 | return true 66 | } 67 | else if(j === index) 68 | return false 69 | else 70 | i++ 71 | } 72 | }, 73 | buffer: buffer, 74 | load: function () { 75 | return count / slots 76 | } 77 | } 78 | } 79 | } 80 | 81 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var Obv = require('obv') 3 | var HashTable = require('./hashtable') 4 | var Drain = require('pull-stream/sinks/drain') 5 | var AtomicFile = require('atomic-file/buffer') 6 | var path = require('path') 7 | var AsyncSingle = require('async-single') 8 | var Multi = require('./multi') 9 | //round up to the next power of 2 10 | function nextPowerOf2 (n) { 11 | return Math.pow(2, Math.ceil(Math.log(n)/Math.LN2)) 12 | } 13 | 14 | module.exports = function (version, hash, getKey, minSlots) { 15 | getKey = getKey || function (data) { return data.key } 16 | 17 | return function (log, name) { 18 | var since = Obv() 19 | var ht, buffer, mt 20 | var filename = path.join(path.dirname(log.filename), name+'.ht') 21 | var state = AtomicFile(filename) 22 | 23 | var HT = HashTable(hash, function (data, key) { 24 | return key === getKey(data) 25 | }, function (offset, cb) { 26 | log.get(offset-1, cb) 27 | }) 28 | 29 | var _seq 30 | since(function (value) { 31 | if(!_seq) _seq = value 32 | else if(value < _seq) 33 | console.error('seq decreased:'+value+', was:'+_seq) 34 | _seq = value 35 | }) 36 | 37 | state.get(function (err, buffer) { 38 | //version, items, seq, count, hashtable... 39 | if('string' == typeof buffer) throw new Error('expected buffer, found string') 40 | //TODO: implement restoring for multitable. 41 | if(!buffer) 42 | initialize(minSlots || 65536) 43 | else { 44 | //check that version is correct 45 | if(version !== buffer.readUInt32BE(0)) 46 | return initialize(minSlots || 65536) 47 | else { 48 | mt = Multi(HT, buffer.slice(8)) 49 | since.set(buffer.readUInt32BE(4)-1) 50 | } 51 | } 52 | }) 53 | 54 | function rebuild (target) { 55 | state.destroy(function () { 56 | initialize(target) 57 | }) 58 | } 59 | 60 | function initialize (target) { 61 | mt = Multi(HT, [HT(target)]) 62 | //sequence and items are already zero 63 | since.set(-1) 64 | } 65 | 66 | function getBuffer () { 67 | var header = new Buffer(8) 68 | header.writeUInt32BE(version, 0) 69 | header.writeUInt32BE(since.value+1, 4) //sequence 70 | return Buffer.concat([header].concat(mt.buffer())) 71 | } 72 | 73 | var async = AsyncSingle(function (value, cb) { 74 | if(state) { 75 | if(value) state.set(getBuffer(), cb) 76 | else state.destroy(cb) 77 | } else cb() 78 | }, {min: 100, max: 500}) 79 | 80 | return { 81 | methods: {get: 'async', load: 'sync'}, 82 | since: since, 83 | createSink: function (cb) { 84 | var rebuilding = false 85 | return Drain(function (data) { 86 | mt.add(getKey(data.value), data.seq+1) 87 | since.set(data.seq) 88 | async.write(mt) 89 | }, function (err) { 90 | if(!rebuilding) 91 | cb(err !== true ? err : null) 92 | }) 93 | }, 94 | get: function (key, cb) { 95 | var called = false 96 | mt.get(key, function (err, value, seq) { 97 | if(called) throw new Error('called already!') 98 | called = true 99 | cb(err, value, seq) 100 | }) 101 | }, 102 | destroy: function (cb) { 103 | async.write(null, cb) 104 | }, 105 | load: function () { return ht.load() }, 106 | _buffer: function () {return buffer}, 107 | close: function (cb) { 108 | async.close(cb) 109 | } 110 | } 111 | } 112 | } 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /multi.js: -------------------------------------------------------------------------------- 1 | 2 | //instead of one single hashtable, 3 | //use a series of increasing size 4 | //so that we do not need to rebuild the table. 5 | 6 | module.exports = function (createHashtable, tables) { 7 | var count = 0, slots = 0 8 | if(Buffer.isBuffer(tables)) { 9 | var buffer = tables 10 | tables = [] 11 | var start = 0 12 | while(start < buffer.length) { 13 | var slots = buffer.readUInt32BE(start) 14 | var end = start + 8 + slots * 4 15 | tables.push(createHashtable(buffer.slice(start, end))) 16 | start = end 17 | } 18 | } 19 | 20 | for(var i = 0; i < tables.length; i++) { 21 | count += tables[i].count 22 | slots += tables[i].slots 23 | } 24 | var self 25 | return self = { 26 | count: count, 27 | 28 | slots: slots, 29 | 30 | get: function (key, cb) { 31 | ;(function next (i) { 32 | tables[i].get(key, function (err, value, seq) { 33 | if(value) cb(null, value, seq) 34 | else if(i) next(i-1) 35 | else cb(err) 36 | }) 37 | })(tables.length-1) 38 | }, 39 | 40 | add: function (key, index) { 41 | var last = tables[tables.length-1] 42 | if(last.load() >= 0.5) { 43 | tables.push(last = createHashtable(last.slots*2)) 44 | self.slots = slots += last.slots 45 | } 46 | if(last.add(key, index)) { 47 | self.count = ++ count 48 | return true 49 | } 50 | return false 51 | }, 52 | 53 | load: function () { 54 | return count/slots 55 | }, 56 | 57 | buffer: function () { 58 | return tables.map(function (e) { return e.buffer }) 59 | } 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flumeview-hashtable", 3 | "version": "1.1.1", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "aligned-block-file": { 8 | "version": "1.1.1", 9 | "resolved": "https://registry.npmjs.org/aligned-block-file/-/aligned-block-file-1.1.1.tgz", 10 | "integrity": "sha1-WzORWxgNMhHAUXfeLrOoVj3OzHQ=", 11 | "dev": true, 12 | "requires": { 13 | "hashlru": "2.2.0", 14 | "int53": "0.2.4", 15 | "mkdirp": "0.5.1", 16 | "obv": "0.0.0", 17 | "uint48be": "1.0.2" 18 | }, 19 | "dependencies": { 20 | "obv": { 21 | "version": "0.0.0", 22 | "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.0.tgz", 23 | "integrity": "sha1-7eq4Ro+R1BkzYu1/kdC5bdOaecE=", 24 | "dev": true 25 | } 26 | } 27 | }, 28 | "append-batch": { 29 | "version": "0.0.1", 30 | "resolved": "https://registry.npmjs.org/append-batch/-/append-batch-0.0.1.tgz", 31 | "integrity": "sha1-kiSFjlVpl8zAfxHx7poShTKqDSU=", 32 | "dev": true 33 | }, 34 | "async-single": { 35 | "version": "1.0.4", 36 | "resolved": "https://registry.npmjs.org/async-single/-/async-single-1.0.4.tgz", 37 | "integrity": "sha512-VjGRJuVcDpS024FycoY55o/LwjiG7FoITbAoha1YEMVp+R5aPkTiFvcH1a8TuglQJQFzk4rQZzg6NcvdPwlUNQ==" 38 | }, 39 | "atomic-file": { 40 | "version": "1.1.3", 41 | "resolved": "https://registry.npmjs.org/atomic-file/-/atomic-file-1.1.3.tgz", 42 | "integrity": "sha512-T0T/st8MAsbBm1PKBMP+PgKPYlPB+EFak0MbFrJZwzQ3QcgU9kq55BbPNgFnJhHi/pmrJvSQaio6KJUUupDuJg==" 43 | }, 44 | "balanced-match": { 45 | "version": "1.0.0", 46 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 47 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 48 | "dev": true 49 | }, 50 | "brace-expansion": { 51 | "version": "1.1.8", 52 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", 53 | "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", 54 | "dev": true, 55 | "requires": { 56 | "balanced-match": "1.0.0", 57 | "concat-map": "0.0.1" 58 | } 59 | }, 60 | "concat-map": { 61 | "version": "0.0.1", 62 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 63 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 64 | "dev": true 65 | }, 66 | "cont": { 67 | "version": "1.0.3", 68 | "resolved": "https://registry.npmjs.org/cont/-/cont-1.0.3.tgz", 69 | "integrity": "sha1-aHTx6TX8qZ0EjK6qrZoK6wILzOA=", 70 | "dev": true, 71 | "requires": { 72 | "continuable": "1.2.0", 73 | "continuable-para": "1.2.0", 74 | "continuable-series": "1.2.0" 75 | } 76 | }, 77 | "continuable": { 78 | "version": "1.2.0", 79 | "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.2.0.tgz", 80 | "integrity": "sha1-CCd0aNQRNiAAdMz4cpQwjRafJbY=", 81 | "dev": true 82 | }, 83 | "continuable-hash": { 84 | "version": "0.1.4", 85 | "resolved": "https://registry.npmjs.org/continuable-hash/-/continuable-hash-0.1.4.tgz", 86 | "integrity": "sha1-gcdNQXcdjJJ4Ph4A5fEbNNbfx4w=", 87 | "dev": true, 88 | "requires": { 89 | "continuable": "1.1.8" 90 | }, 91 | "dependencies": { 92 | "continuable": { 93 | "version": "1.1.8", 94 | "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz", 95 | "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U=", 96 | "dev": true 97 | } 98 | } 99 | }, 100 | "continuable-list": { 101 | "version": "0.1.6", 102 | "resolved": "https://registry.npmjs.org/continuable-list/-/continuable-list-0.1.6.tgz", 103 | "integrity": "sha1-h88G7FgHFuEN/5X7C4TF8OisrF8=", 104 | "dev": true, 105 | "requires": { 106 | "continuable": "1.1.8" 107 | }, 108 | "dependencies": { 109 | "continuable": { 110 | "version": "1.1.8", 111 | "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz", 112 | "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U=", 113 | "dev": true 114 | } 115 | } 116 | }, 117 | "continuable-para": { 118 | "version": "1.2.0", 119 | "resolved": "https://registry.npmjs.org/continuable-para/-/continuable-para-1.2.0.tgz", 120 | "integrity": "sha1-RFUQ9klFndD8NchyAVFGEicxxYM=", 121 | "dev": true, 122 | "requires": { 123 | "continuable-hash": "0.1.4", 124 | "continuable-list": "0.1.6" 125 | } 126 | }, 127 | "continuable-series": { 128 | "version": "1.2.0", 129 | "resolved": "https://registry.npmjs.org/continuable-series/-/continuable-series-1.2.0.tgz", 130 | "integrity": "sha1-MkM5euk6cdZVswJoNKUVkLlYueg=", 131 | "dev": true 132 | }, 133 | "deep-equal": { 134 | "version": "1.0.1", 135 | "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", 136 | "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", 137 | "dev": true 138 | }, 139 | "define-properties": { 140 | "version": "1.1.2", 141 | "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", 142 | "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", 143 | "dev": true, 144 | "requires": { 145 | "foreach": "2.0.5", 146 | "object-keys": "1.0.11" 147 | } 148 | }, 149 | "defined": { 150 | "version": "1.0.0", 151 | "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", 152 | "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=", 153 | "dev": true 154 | }, 155 | "es-abstract": { 156 | "version": "1.7.0", 157 | "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.7.0.tgz", 158 | "integrity": "sha1-363ndOAb/Nl/lhgCmMRJyGI/uUw=", 159 | "dev": true, 160 | "requires": { 161 | "es-to-primitive": "1.1.1", 162 | "function-bind": "1.1.0", 163 | "is-callable": "1.1.3", 164 | "is-regex": "1.0.4" 165 | } 166 | }, 167 | "es-to-primitive": { 168 | "version": "1.1.1", 169 | "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz", 170 | "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=", 171 | "dev": true, 172 | "requires": { 173 | "is-callable": "1.1.3", 174 | "is-date-object": "1.0.1", 175 | "is-symbol": "1.0.1" 176 | } 177 | }, 178 | "explain-error": { 179 | "version": "1.0.4", 180 | "resolved": "https://registry.npmjs.org/explain-error/-/explain-error-1.0.4.tgz", 181 | "integrity": "sha1-p5PTrAytTGq1cemWj7urbLJTKSk=", 182 | "dev": true 183 | }, 184 | "flumecodec": { 185 | "version": "0.0.0", 186 | "resolved": "https://registry.npmjs.org/flumecodec/-/flumecodec-0.0.0.tgz", 187 | "integrity": "sha1-Ns4Gq+Lg4BxE3WnyoWUwWiMgZJs=", 188 | "dev": true, 189 | "requires": { 190 | "level-codec": "6.2.0" 191 | } 192 | }, 193 | "flumedb": { 194 | "version": "0.4.2", 195 | "resolved": "https://registry.npmjs.org/flumedb/-/flumedb-0.4.2.tgz", 196 | "integrity": "sha1-bqeQFvN5kkISa0sOsCnx86HeWoU=", 197 | "dev": true, 198 | "requires": { 199 | "cont": "1.0.3", 200 | "explain-error": "1.0.4", 201 | "obv": "0.0.1", 202 | "pull-cont": "0.0.0", 203 | "pull-stream": "3.6.0" 204 | } 205 | }, 206 | "flumelog-offset": { 207 | "version": "3.2.2", 208 | "resolved": "https://registry.npmjs.org/flumelog-offset/-/flumelog-offset-3.2.2.tgz", 209 | "integrity": "sha512-ZU51QP3KCNGiv4VoX4MhbMAB1izAdyUNKkBHp7WKiNk5F8ZOiXAJ2u+8qIYW4GUOAWU52KAm9Ugx6C6TPCUKMQ==", 210 | "dev": true, 211 | "requires": { 212 | "aligned-block-file": "1.1.1", 213 | "append-batch": "0.0.1", 214 | "explain-error": "1.0.4", 215 | "hashlru": "2.2.0", 216 | "int53": "0.2.4", 217 | "looper": "4.0.0", 218 | "ltgt": "2.2.0", 219 | "obv": "0.0.1", 220 | "pull-cursor": "2.1.3", 221 | "uint48be": "1.0.2" 222 | } 223 | }, 224 | "for-each": { 225 | "version": "0.3.2", 226 | "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.2.tgz", 227 | "integrity": "sha1-LEBFC5NI6X8oEyJZO6lnBLmr1NQ=", 228 | "dev": true, 229 | "requires": { 230 | "is-function": "1.0.1" 231 | } 232 | }, 233 | "foreach": { 234 | "version": "2.0.5", 235 | "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", 236 | "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=", 237 | "dev": true 238 | }, 239 | "fs.realpath": { 240 | "version": "1.0.0", 241 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 242 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 243 | "dev": true 244 | }, 245 | "function-bind": { 246 | "version": "1.1.0", 247 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.0.tgz", 248 | "integrity": "sha1-FhdnFMgBeY5Ojyz391KUZ7tKV3E=", 249 | "dev": true 250 | }, 251 | "glob": { 252 | "version": "7.1.2", 253 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 254 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 255 | "dev": true, 256 | "requires": { 257 | "fs.realpath": "1.0.0", 258 | "inflight": "1.0.6", 259 | "inherits": "2.0.3", 260 | "minimatch": "3.0.4", 261 | "once": "1.4.0", 262 | "path-is-absolute": "1.0.1" 263 | } 264 | }, 265 | "has": { 266 | "version": "1.0.1", 267 | "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz", 268 | "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=", 269 | "dev": true, 270 | "requires": { 271 | "function-bind": "1.1.0" 272 | } 273 | }, 274 | "hashlru": { 275 | "version": "2.2.0", 276 | "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.2.0.tgz", 277 | "integrity": "sha1-eTpYlD+QKupXgXfXsDNfE/JpS3E=", 278 | "dev": true 279 | }, 280 | "inflight": { 281 | "version": "1.0.6", 282 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 283 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 284 | "dev": true, 285 | "requires": { 286 | "once": "1.4.0", 287 | "wrappy": "1.0.2" 288 | } 289 | }, 290 | "inherits": { 291 | "version": "2.0.3", 292 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 293 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 294 | "dev": true 295 | }, 296 | "int53": { 297 | "version": "0.2.4", 298 | "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", 299 | "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y=", 300 | "dev": true 301 | }, 302 | "is-callable": { 303 | "version": "1.1.3", 304 | "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz", 305 | "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI=", 306 | "dev": true 307 | }, 308 | "is-date-object": { 309 | "version": "1.0.1", 310 | "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", 311 | "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", 312 | "dev": true 313 | }, 314 | "is-function": { 315 | "version": "1.0.1", 316 | "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.1.tgz", 317 | "integrity": "sha1-Es+5i2W1fdPRk6MSH19uL0N2ArU=", 318 | "dev": true 319 | }, 320 | "is-regex": { 321 | "version": "1.0.4", 322 | "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", 323 | "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", 324 | "dev": true, 325 | "requires": { 326 | "has": "1.0.1" 327 | } 328 | }, 329 | "is-symbol": { 330 | "version": "1.0.1", 331 | "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz", 332 | "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=", 333 | "dev": true 334 | }, 335 | "level-codec": { 336 | "version": "6.2.0", 337 | "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-6.2.0.tgz", 338 | "integrity": "sha1-pLUkS7akwvcj1oodZOmAxTYn2dQ=", 339 | "dev": true 340 | }, 341 | "looper": { 342 | "version": "4.0.0", 343 | "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz", 344 | "integrity": "sha1-dwat7VmpntygbmtUu4bI7BnJUVU=", 345 | "dev": true 346 | }, 347 | "ltgt": { 348 | "version": "2.2.0", 349 | "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.0.tgz", 350 | "integrity": "sha1-tlul/LNJopkkyOMz98alVi8uSEI=", 351 | "dev": true 352 | }, 353 | "minimatch": { 354 | "version": "3.0.4", 355 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 356 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 357 | "dev": true, 358 | "requires": { 359 | "brace-expansion": "1.1.8" 360 | } 361 | }, 362 | "minimist": { 363 | "version": "0.0.8", 364 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 365 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", 366 | "dev": true 367 | }, 368 | "mkdirp": { 369 | "version": "0.5.1", 370 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 371 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 372 | "dev": true, 373 | "requires": { 374 | "minimist": "0.0.8" 375 | } 376 | }, 377 | "object-inspect": { 378 | "version": "1.2.2", 379 | "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.2.2.tgz", 380 | "integrity": "sha1-yCEV5PzIiK6hTWTCLk8X9qcNXlo=", 381 | "dev": true 382 | }, 383 | "object-keys": { 384 | "version": "1.0.11", 385 | "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", 386 | "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=", 387 | "dev": true 388 | }, 389 | "obv": { 390 | "version": "0.0.1", 391 | "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.1.tgz", 392 | "integrity": "sha1-yyNhBjQVNvDaxIFeBnCCIcrX+14=" 393 | }, 394 | "once": { 395 | "version": "1.4.0", 396 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 397 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 398 | "dev": true, 399 | "requires": { 400 | "wrappy": "1.0.2" 401 | } 402 | }, 403 | "path-is-absolute": { 404 | "version": "1.0.1", 405 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 406 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 407 | "dev": true 408 | }, 409 | "path-parse": { 410 | "version": "1.0.5", 411 | "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz", 412 | "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME=", 413 | "dev": true 414 | }, 415 | "pull-cont": { 416 | "version": "0.0.0", 417 | "resolved": "https://registry.npmjs.org/pull-cont/-/pull-cont-0.0.0.tgz", 418 | "integrity": "sha1-P6xIuBrJe3W6ATMgiLDOevjBvg4=", 419 | "dev": true 420 | }, 421 | "pull-cursor": { 422 | "version": "2.1.3", 423 | "resolved": "https://registry.npmjs.org/pull-cursor/-/pull-cursor-2.1.3.tgz", 424 | "integrity": "sha1-Ah6X19I9eK9mrda4wcF/1v2xK2w=", 425 | "dev": true, 426 | "requires": { 427 | "looper": "4.0.0", 428 | "ltgt": "2.2.0", 429 | "pull-stream": "3.6.0" 430 | } 431 | }, 432 | "pull-stream": { 433 | "version": "3.6.0", 434 | "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.6.0.tgz", 435 | "integrity": "sha1-WdAzpoFdTjCX1Hw9KxiTqeWKI1E=" 436 | }, 437 | "resolve": { 438 | "version": "1.3.3", 439 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.3.3.tgz", 440 | "integrity": "sha1-ZVkHw0aahoDcLeOidaj91paR8OU=", 441 | "dev": true, 442 | "requires": { 443 | "path-parse": "1.0.5" 444 | } 445 | }, 446 | "resumer": { 447 | "version": "0.0.0", 448 | "resolved": "https://registry.npmjs.org/resumer/-/resumer-0.0.0.tgz", 449 | "integrity": "sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k=", 450 | "dev": true, 451 | "requires": { 452 | "through": "2.3.8" 453 | } 454 | }, 455 | "string.prototype.trim": { 456 | "version": "1.1.2", 457 | "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz", 458 | "integrity": "sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=", 459 | "dev": true, 460 | "requires": { 461 | "define-properties": "1.1.2", 462 | "es-abstract": "1.7.0", 463 | "function-bind": "1.1.0" 464 | } 465 | }, 466 | "tape": { 467 | "version": "4.7.0", 468 | "resolved": "https://registry.npmjs.org/tape/-/tape-4.7.0.tgz", 469 | "integrity": "sha512-ePzu2KfZYVtq0v+KKGxBJ9HJWYZ4MaQWeGabD+KpVdMKRen3NJPf6EiwA5BxfMkhQPGtCwnOFWelcB39bhOUng==", 470 | "dev": true, 471 | "requires": { 472 | "deep-equal": "1.0.1", 473 | "defined": "1.0.0", 474 | "for-each": "0.3.2", 475 | "function-bind": "1.1.0", 476 | "glob": "7.1.2", 477 | "has": "1.0.1", 478 | "inherits": "2.0.3", 479 | "minimist": "1.2.0", 480 | "object-inspect": "1.2.2", 481 | "resolve": "1.3.3", 482 | "resumer": "0.0.0", 483 | "string.prototype.trim": "1.1.2", 484 | "through": "2.3.8" 485 | }, 486 | "dependencies": { 487 | "minimist": { 488 | "version": "1.2.0", 489 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", 490 | "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", 491 | "dev": true 492 | } 493 | } 494 | }, 495 | "test-flumeview-index": { 496 | "version": "2.0.1", 497 | "resolved": "https://registry.npmjs.org/test-flumeview-index/-/test-flumeview-index-2.0.1.tgz", 498 | "integrity": "sha512-x4npsHa93zMsQ8BhPoMjDXOv/uMx3u9ZLlKtgscOh0nEdP/NuIxB7kffaRc9590KbA7ZvxWszDnioErwJ5sI5g==", 499 | "dev": true, 500 | "requires": { 501 | "tape": "4.7.0" 502 | } 503 | }, 504 | "through": { 505 | "version": "2.3.8", 506 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 507 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", 508 | "dev": true 509 | }, 510 | "uint48be": { 511 | "version": "1.0.2", 512 | "resolved": "https://registry.npmjs.org/uint48be/-/uint48be-1.0.2.tgz", 513 | "integrity": "sha512-jNn1eEi81BLiZfJkjbiAKPDMj7iFrturKazqpBu0aJYLr6evgkn+9rgkX/gUwPBj5j2Ri5oUelsqC/S1zmpWBA==", 514 | "dev": true 515 | }, 516 | "wrappy": { 517 | "version": "1.0.2", 518 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 519 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 520 | "dev": true 521 | } 522 | } 523 | } 524 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flumeview-hashtable", 3 | "description": "flumeview key index as a memory hashtable (very fast)", 4 | "version": "1.1.1", 5 | "homepage": "https://github.com/dominictarr/flumeview-hashtable", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/dominictarr/flumeview-hashtable.git" 9 | }, 10 | "dependencies": { 11 | "async-single": "^1.0.5", 12 | "atomic-file": "^1.1.3", 13 | "obv": "0.0.1", 14 | "pull-stream": "^3.6.0" 15 | }, 16 | "devDependencies": { 17 | "deep-equal": "^1.0.1", 18 | "flumecodec": "0.0.0", 19 | "flumedb": "^0.4.2", 20 | "flumelog-offset": "^3.2.2", 21 | "tape": "^4.7.0", 22 | "test-flumeview-index": "^2.0.1" 23 | }, 24 | "scripts": { 25 | "test": "set -e; for t in test/*.js; do node $t; done" 26 | }, 27 | "author": "'Dominic Tarr' (dominictarr.com)", 28 | "license": "MIT" 29 | } 30 | -------------------------------------------------------------------------------- /test/bench.js: -------------------------------------------------------------------------------- 1 | var Log = require('flumelog-offset') 2 | var Flume = require('flumedb') 3 | var crypto = require('crypto') 4 | function hash (v) { 5 | return crypto.createHash('sha256') 6 | .update(v.toString()) 7 | .digest().readUInt32BE(0) 8 | } 9 | 10 | require('test-flumeview-index/bench')(function (file) { 11 | return Flume( 12 | Log( 13 | file+'/log.offset', 14 | {blockSize: 1024, codec: require('flumecodec/json')} 15 | )) 16 | .use('index', require('../')(1, hash, null, Math.pow(2, 16))) 17 | }) 18 | -------------------------------------------------------------------------------- /test/flumeview-index.js: -------------------------------------------------------------------------------- 1 | 2 | var Log = require('flumelog-offset') 3 | var Flume = require('flumedb') 4 | var crypto = require('crypto') 5 | function hash (v) { 6 | return crypto.createHash('sha256') 7 | .update(v.toString()) 8 | .digest().readUInt32BE(0) 9 | } 10 | 11 | require('test-flumeview-index')(function (seed) { 12 | return Flume( 13 | Log( 14 | '/tmp/test-flumeview-hashtable'+seed+'/log.offset', 15 | {blockSize: 1024, codec: require('flumecodec/json')} 16 | )) 17 | .use('index', require('../')(1, hash)) 18 | }) 19 | 20 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | 2 | var tape = require('tape') 3 | var crypto = require('crypto') 4 | 5 | function hash (v) { 6 | return crypto.createHash('sha256').update(v.toString()).digest().readUInt32BE(0) 7 | } 8 | 9 | function matches (k, data) { 10 | return k === hash(data+'value') 11 | } 12 | 13 | function get(k, cb) { 14 | cb(null, k) 15 | } 16 | 17 | tape('random integers', function (t) { 18 | 19 | var ht = require('../hashtable')(hash, matches, get)(2000) 20 | 21 | var r = hash('hello random'+Date.now()) 22 | 23 | var k = ht.add('random2'+Date.now(), r) 24 | // t.equal(ht._get(k), r) 25 | console.log(k, r) 26 | 27 | var c = 0 28 | for(var i = 0; i < 1000; i++) { 29 | c++ 30 | var k = ht.add(i, hash(i+'value')) 31 | 32 | ht.get(i, function (err, j, seq) { 33 | if(err) throw err 34 | t.equal(j, hash(i+'value')) 35 | 36 | // Now we just want to double-check that `seq` actually contains the 37 | // value `j` when we look it up manually. 38 | ht._get(seq, function (err, val) { 39 | t.error(err) 40 | t.equal(val, j) 41 | }) 42 | }) 43 | t.notEqual(k, i) 44 | } 45 | console.log(c) 46 | //1001 because we add one before the loop 47 | t.equal(ht.count, 1001) 48 | t.equal(ht.load(), 0.5005) 49 | t.end() 50 | }) 51 | 52 | tape('overwrite', function (t) { 53 | 54 | var ht = require('../hashtable')(hash, matches, get)(10) 55 | 56 | t.equal(ht.add(1, 1), true) 57 | t.equal(ht.count, 1) 58 | t.equal(ht.add(1, 1), false) 59 | t.equal(ht.count, 1) 60 | console.log(ht.buffer) 61 | t.end() 62 | }) 63 | 64 | 65 | -------------------------------------------------------------------------------- /test/multi.js: -------------------------------------------------------------------------------- 1 | var Multi = require('../multi') 2 | var tape = require('tape') 3 | var crypto = require('crypto') 4 | 5 | function hash (v) { 6 | return crypto.createHash('sha256').update(v.toString()).digest().readUInt32BE(0) 7 | } 8 | 9 | function matches (k, data) { 10 | return k === hash(data+'value') 11 | } 12 | 13 | function get(k, cb) { 14 | cb(null, k) 15 | } 16 | 17 | tape('growing, with random integers', function (t) { 18 | 19 | var createHT = require('../hashtable')(hash, matches, get) 20 | var table = createHT(100) 21 | 22 | var mt = Multi(createHT, [table]) 23 | 24 | var r = hash('hello random'+Date.now()) 25 | 26 | var c = 0 27 | for(var i = 0; i < 1000; i++) { 28 | c++ 29 | var k = mt.add(i, hash(i+'value')) 30 | mt.get(i, function (err, j) { 31 | if(err) throw err 32 | t.equal(j, hash(i+'value')) 33 | }) 34 | t.notEqual(k, i) 35 | } 36 | console.log(mt.count, mt.slots, mt.load()) 37 | t.equal(mt.count, 1000) 38 | t.equal(mt.count, c) 39 | t.ok(mt.load() < 0.5001) 40 | t.end() 41 | }) 42 | 43 | 44 | tape('overwrite', function (t) { 45 | var createHT = require('../hashtable')(hash, matches, get) 46 | var mt = Multi(createHT, [createHT(10)]) 47 | 48 | t.equal(mt.add(1, 1), true) 49 | t.equal(mt.count, 1) 50 | t.equal(mt.add(1, 1), false) 51 | t.equal(mt.count, 1) 52 | t.end() 53 | }) 54 | 55 | tape('restore', function (t) { 56 | var createHT = require('../hashtable')(hash, matches, get) 57 | 58 | var createHT = require('../hashtable')(hash, matches, get) 59 | var mt = Multi(createHT, [createHT(10)]) 60 | 61 | var N = 100 62 | for(var i = 0; i < N; i++) { 63 | var k = mt.add(i, hash(i+'value')) 64 | } 65 | 66 | var buf = Buffer.concat(mt.buffer()) 67 | var mt2 = Multi(createHT, buf) 68 | 69 | for(var i = 0; i < N; i++) { 70 | mt2.get(i, function (err, value) { 71 | if(err) throw err 72 | t.equal(value, hash(i+'value')) 73 | }) 74 | } 75 | 76 | t.end() 77 | }) 78 | 79 | 80 | -------------------------------------------------------------------------------- /test/reconstruct.js: -------------------------------------------------------------------------------- 1 | 2 | var tape = require('tape') 3 | var crypto = require('crypto') 4 | 5 | function hash (v) { 6 | return crypto.createHash('sha256').update(v.toString()).digest().readUInt32BE(0) 7 | } 8 | 9 | function matches (k, data) { 10 | return k === data+1 11 | } 12 | 13 | function get(k, cb) { 14 | cb(null, k) 15 | } 16 | 17 | 18 | tape('reconstruct', function (t) { 19 | 20 | var HT = require('../hashtable')(hash, matches, get) 21 | var ht = HT(10) 22 | var N = 10 23 | for(var i = 0; i < N; i++) 24 | ht.add(i, i+1) //(i<<8)|0x0f) 25 | 26 | console.log("DUMP", ht.buffer) 27 | 28 | for(var j = 0; j < N; j++) 29 | ht.get(j, function (err, value) { 30 | if(err) throw err 31 | t.equal(value, j+1) 32 | }) 33 | 34 | var ht2 = HT(ht.buffer) 35 | t.equal(ht2.slots, ht.slots) 36 | t.equal(ht2.count, ht.count) 37 | 38 | for(var k = 0; k < N; k++) 39 | ht2.get(k, function (err, value) { 40 | if(err) throw err 41 | t.equal(value, k+1) 42 | }) 43 | 44 | t.end() 45 | }) 46 | 47 | 48 | -------------------------------------------------------------------------------- /test/simple.js: -------------------------------------------------------------------------------- 1 | const Log = require('flumelog-offset') 2 | const Flume = require('flumedb') 3 | const crypto = require('crypto') 4 | const test = require('tape') 5 | 6 | function hash (v) { 7 | return crypto.createHash('sha256') 8 | .update(v.toString()) 9 | .digest().readUInt32BE(0) 10 | } 11 | 12 | const db = Flume( 13 | Log( 14 | '/tmp/test-flumeview-hashtable-' + String(Math.random()) + '/log.offset', 15 | {blockSize: 1024, codec: require('flumecodec/json')} 16 | )) 17 | .use('index', require('../')(1, hash)) 18 | 19 | test('no errors, correct value and sequence number', (t) => { 20 | const item = { key: 'foo', value: 'bar' } 21 | 22 | db.append(item, function (err) { 23 | t.error(err) 24 | db.index.get(item.key, function (err, val, seq) { 25 | t.error(err) 26 | t.deepEqual(val, item) 27 | t.equal(seq, 1) 28 | t.end() 29 | }) 30 | }) 31 | }) 32 | --------------------------------------------------------------------------------