├── .npmignore ├── .travis.yml ├── README.md ├── ex.js ├── index.js ├── lib └── state.js ├── package.json └── test ├── adder.js ├── basic.js ├── batch.js ├── fork.js ├── pause-resume.js ├── states.js ├── sync.js └── version.js /.npmignore: -------------------------------------------------------------------------------- 1 | ex.js 2 | test/ 3 | .travis.yml 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '8' 4 | - '10' 5 | os: 6 | - windows 7 | - osx 8 | - linux 9 | notifications: 10 | email: false 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # multifeed-index 2 | 3 | > Build an index over a set of hypercores. 4 | 5 | Traverses a set of hypercores as a [multifeed][multifeed] and calls a user 6 | indexing function to build an index. 7 | 8 | ## Purpose 9 | 10 | A [multifeed][multifeed] is a set of append-only logs (feeds), with the 11 | property that only the feed's author can write new entries to it. 12 | 13 | One type of document that you might write to such a feed are key -> value 14 | pairs. Maybe documents looks like this: 15 | 16 | ```js 17 | { 18 | id: '23482934', 19 | name: 'quinn', 20 | gender: null 21 | } 22 | ``` 23 | 24 | So some key, `'23482934'`, can map to this document. How could you make 25 | the lookup from `'23482934'` to the aforementioned document fast? What if 26 | there are thousands of such entries? 27 | 28 | You'd probably want to build some kind of index. One that iterates over every 29 | entry in every feed, and also listens for new entries that get added. Then you 30 | could create an efficient data structure (say, maybe with 31 | [level](https://github.com/Level/level)) that can map each key to a value 32 | quickly. 33 | 34 | Good news: this is essentially what `multifeed-index` does! 35 | 36 | It does this by taking a multifeed you give it, along with three functions that 37 | the user provides: 38 | 39 | 1. `storeState(state, cb)`: multifeed-index will give you a `Buffer` object, 40 | `state`, to store somewhere of your choosing. This could be in memory, to a 41 | database, to a JSON file, to browser storage, or whatever makes sense for 42 | your program. The module doesn't want to tie you down to a specific storage 43 | method. 44 | 2. `fetchState(cb)`: Like `storeState`, except this function will be called by 45 | multifeed-index when it needs to retrieve the state. Your job is to call the 46 | callback `cb` with the same `state` Buffer that was given to `storeState` 47 | earlier. 48 | 3. `clearIndex(cb)`: This only gets called when you change the `version` of the 49 | multifeed-index (see below). This function should delete the entire index 50 | *and* whatever `state` it stored earlier, so that the new version of the 51 | index can be regenerated from scratch. 52 | 53 | ## Example 54 | 55 | Let's build a key-value index using this and 56 | [unordered-materialized-kv](https://github.com/substack/unordered-materialized-kv): 57 | 58 | ```js 59 | var multifeed = require('multifeed') 60 | var indexer = require('multifeed-index') 61 | var umkv = require('unordered-materialized-kv') 62 | var ram = require('random-access-memory') 63 | var memdb = require('memdb') 64 | 65 | var multi = multifeed(ram, { valueEncoding: 'json' }) 66 | 67 | var kv = umkv(memdb()) 68 | 69 | var kvView = indexer({ 70 | version: 1, // setting a different number will cause the index to be purged and rebuilt 71 | log: multi, 72 | maxBatch: 5, 73 | batch: function (nodes, next) { 74 | var ops = nodes.map(function (node) { 75 | return { 76 | id: node.key.toString('hex') + '@' + node.seq, 77 | key: node.value.key, 78 | links: node.value.links 79 | } 80 | }) 81 | kv.batch(ops, next) 82 | } 83 | }) 84 | 85 | function append (w, data, cb) { 86 | w.append(data, function (err) { 87 | if (err) return cb(err) 88 | var id = w.key.toString('hex') + '@' + (w.length - 1) 89 | cb(null, id) 90 | }) 91 | } 92 | 93 | multi.writer(function (_, w) { 94 | append(w, { 95 | key: 'foo', 96 | value: 'bax', 97 | links: [] 98 | }, function (_, id1) { 99 | console.log('id-1', id1) 100 | append(w, { 101 | key: 'foo', 102 | value: 'bax', 103 | links: [id1] 104 | }, function (_, id2) { 105 | console.log('id-2', id2) 106 | kvView.ready(function () { 107 | kv.get('foo', function (_, res) { 108 | console.log(res) 109 | }) 110 | }) 111 | }) 112 | }) 113 | }) 114 | ``` 115 | 116 | outputs 117 | 118 | ``` 119 | id-1 9736a3ff7ae522ca80b7612fed5aefe8cfb40e0a43199174e47d78703abaa22f@0 120 | id-2 9736a3ff7ae522ca80b7612fed5aefe8cfb40e0a43199174e47d78703abaa22f@1 121 | [ 122 | '9736a3ff7ae522ca80b7612fed5aefe8cfb40e0a43199174e47d78703abaa22f@1' 123 | ] 124 | ``` 125 | 126 | ## API 127 | 128 | > var Index = require('multifeed-index') 129 | 130 | ### var index = Index(opts) 131 | 132 | Required `opts` include: 133 | 134 | - `log`: a [multifeed](https://github.com/noffle/multifeed) instance 135 | - `batch`: a mapping function, to be called on 1+ nodes at a time in the 136 | hypercores of `log`. 137 | - `storeState`: Function of the form `function (state, cb)`. Called by the 138 | indexer when there is a new indexing state to save. The user can store this 139 | Buffer object whereever/however they'd like. 140 | - `fetchState`: Function of the form `function (cb)`. Called by the indexer to 141 | seed the indexing process when the indexer is created. Expects `cb` to be 142 | called with `(err, state)`, where `state` is a Buffer that was previously 143 | given to `opts.storeState`. 144 | 145 | The `batch` function expects params `(nodes, next)`. `next` should be called 146 | when you are done processing. Each `node` of `nodes` is of the form 147 | 148 | ```js 149 | { 150 | key: 'hex-string-of-hypercore-public-key', 151 | seq: 14, 152 | value: 153 | } 154 | ``` 155 | 156 | Optional `opts` include: 157 | 158 | - `version`: the version of the index. If increased, any indexes built with an 159 | earlier version will be purged and rebuilt. This is useful for when you 160 | change the data format of the index and want all peers to rebuild to use this 161 | format. Defaults to `1`. 162 | - `maxBatch`: maximum batch size of nodes to process in one `batch` call. 163 | Defaults to `50`. 164 | - `clearIndex`: Function of the form `function (cb)`. Called by the indexer 165 | when a new version for the index has been passed in (via `opts.version`) and 166 | the index needs to be cleared & regenerated. 167 | 168 | ### index.ready(cb) 169 | 170 | Registers the callback `cb()` to fire when the indexes have "caught up" to the 171 | latest known change. The `cb()` function fires exactly once. You may call 172 | `index.ready()` multiple times with different functions. 173 | 174 | ### index.pause([cb]) 175 | 176 | Pauses the indexing process. Whatever batches of entries are currently being 177 | processed will finish first. If a callback `cb` is given, it will be called 178 | once pending entries are processed and the indexer is fully paused. 179 | 180 | ### index.resume() 181 | 182 | Synchronously restarts a paused indexer. 183 | 184 | ### var state = index.getState() 185 | 186 | Returns the current state of the indexer. 187 | 188 | `state` is an object that looks like 189 | 190 | ```js 191 | { 192 | state: 'indexing', // one of ['idle', 'indexing', 'paused', 'error'] 193 | context: { 194 | totalBlocks: 50, // total # of blocks known of across all feeds 195 | indexedBlocks: 18, // total # of blocks indexed so far 196 | prevIndexedBlocks: 0, // total # of blocks indexed as of the previous indexing run 197 | indexStartTime: 1583184761581, // ms since epoch when the last indexing run started 198 | error: null // error state (indexing is unusable if this is set) 199 | } 200 | } 201 | ``` 202 | 203 | ### index.on('indexed', function (nodes) {}) 204 | 205 | Event emitted when entries have finished being indexed. 206 | 207 | ### index.on('error', function (err) {}) 208 | 209 | Event emitted when an error within multifeed-index has occurred. This is very 210 | important to listen on, lest things suddenly seem to break and it's not 211 | immediately clear why. 212 | 213 | ### index.on('state-update', function (state) {}) 214 | 215 | Event emitted when the internal state of the indexer changes. It has the same 216 | form as the `state` object returned by `index.getState()` above. 217 | 218 | ## Install 219 | 220 | ``` 221 | $ npm install multifeed-index 222 | ``` 223 | 224 | ## See Also 225 | - [hyperlog-index](https://github.com/substack/hyperlog-index) 226 | - [hyperdb-index](https://github.com/noffle/hyperdb-index) 227 | 228 | ## License 229 | 230 | ISC 231 | 232 | [multifeed]: https://github.com/noffle/multifeed 233 | 234 | -------------------------------------------------------------------------------- /ex.js: -------------------------------------------------------------------------------- 1 | var hypercore = require('hypercore') 2 | var multifeed = require('multifeed') 3 | var indexer = require('.') 4 | var umkv = require('unordered-materialized-kv') 5 | var ram = require('random-access-memory') 6 | var memdb = require('memdb') 7 | 8 | var multi = multifeed(hypercore, ram, { valueEncoding: 'json' }) 9 | 10 | var kv = umkv(memdb()) 11 | 12 | var kvView = indexer({ 13 | version: 1, // setting a different number will cause the index to be purged and rebuilt 14 | log: multi, 15 | maxBatch: 5, 16 | batch: function (nodes, next) { 17 | var ops = nodes.map(function (node) { 18 | return { 19 | id: node.key.toString('hex') + '@' + node.seq, 20 | key: node.value.key, 21 | links: node.value.links 22 | } 23 | }) 24 | kv.batch(ops, next) 25 | } 26 | }) 27 | 28 | function append (w, data, cb) { 29 | w.append(data, function (err) { 30 | if (err) return cb(err) 31 | var id = w.key.toString('hex') + '@' + (w.length - 1) 32 | cb(null, id) 33 | }) 34 | } 35 | 36 | multi.writer(function (_, w) { 37 | append(w, { 38 | key: 'foo', 39 | value: 'bax', 40 | links: [] 41 | }, function (_, id1) { 42 | console.log('id-1', id1) 43 | append(w, { 44 | key: 'foo', 45 | value: 'bax', 46 | links: [id1] 47 | }, function (_, id2) { 48 | console.log('id-2', id2) 49 | kvView.ready(function () { 50 | kv.get('foo', function (_, res) { 51 | console.log(res) 52 | }) 53 | }) 54 | }) 55 | }) 56 | }) 57 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var inherits = require('inherits') 2 | var EventEmitter = require('events').EventEmitter 3 | var IndexState = require('./lib/state') 4 | var clone = require('clone') 5 | 6 | module.exports = Indexer 7 | 8 | var State = { 9 | PreIndexing: 'preindexing', 10 | Indexing: 'indexing', 11 | Idle: 'idle', 12 | Paused: 'paused', 13 | Error: 'error' 14 | } 15 | 16 | function Indexer (opts) { 17 | if (!(this instanceof Indexer)) return new Indexer(opts) 18 | 19 | if (!opts) throw new Error('missing opts param') 20 | if (!opts.log) throw new Error('missing opts param "log"') 21 | if (!opts.batch) throw new Error('missing opts param "batch"') 22 | if (!allOrNone(!!opts.storeState, !!opts.fetchState)) { 23 | throw new Error('either none or all of (opts.storeState, opts.fetchState) must be provided') 24 | } 25 | if (!unset(opts.version) && typeof opts.version !== 'number') throw new Error('opts.version must be a number') 26 | // TODO: support forward & backward indexing from newest 27 | 28 | this._version = unset(opts.version) ? 1 : opts.version 29 | this._log = opts.log 30 | this._batch = opts.batch 31 | this._maxBatch = unset(opts.maxBatch) ? 50 : opts.maxBatch 32 | 33 | // Is there another pending indexing run? 34 | this._pending = false 35 | 36 | this._state = { 37 | state: State.Indexing, 38 | context: { 39 | totalBlocks: 0, 40 | indexedBlocks: 0, 41 | prevIndexedBlocks: 0, 42 | indexStartTime: Date.now(), 43 | error: null 44 | } 45 | } 46 | 47 | this._at = null 48 | // bind methods to this so we can pass them directly to event listeners 49 | this._freshRun = this._run.bind(this, false) 50 | this._onNewFeed = this._onNewFeed.bind(this) 51 | 52 | if (!opts.storeState && !opts.fetchState && !opts.clearIndex) { 53 | // In-memory storage implementation 54 | var state 55 | this._storeIndexState = function (buf, cb) { 56 | state = buf 57 | process.nextTick(cb) 58 | } 59 | this._fetchIndexState = function (cb) { 60 | process.nextTick(cb, null, state) 61 | } 62 | this._clearIndex = function (cb) { 63 | state = null 64 | process.nextTick(cb) 65 | } 66 | } else { 67 | this._storeIndexState = opts.storeState 68 | this._fetchIndexState = opts.fetchState 69 | this._clearIndex = opts.clearIndex || null 70 | } 71 | 72 | var self = this 73 | 74 | this._onError = function (err) { 75 | self._setState(State.Error, { error: err }) 76 | self.emit('error', err) 77 | } 78 | 79 | this._log.ready(function () { 80 | self._fetchIndexState(function (err, state) { 81 | if (err && !err.notFound) { 82 | self._onError(err) 83 | return 84 | } 85 | if (!state) { 86 | start() 87 | return 88 | } 89 | 90 | try { 91 | state = IndexState.deserialize(state) 92 | } catch (e) { 93 | self._onError(e) 94 | return 95 | } 96 | 97 | // Wipe existing index if versions don't match (and there's a 'clearIndex' implementation) 98 | var storedVersion = state.version 99 | if (storedVersion !== self._version && self._clearIndex) { 100 | self._clearIndex(function (err) { 101 | if (err) { 102 | self._onError(err) 103 | } else { 104 | start() 105 | } 106 | }) 107 | } else { 108 | start() 109 | } 110 | }) 111 | }) 112 | 113 | function start () { 114 | self._setState(State.Idle) 115 | self._freshRun() 116 | } 117 | 118 | this._log.on('feed', this._onNewFeed) 119 | 120 | this.setMaxListeners(1024) 121 | } 122 | 123 | inherits(Indexer, EventEmitter) 124 | 125 | Indexer.prototype._onNewFeed = function (feed, idx) { 126 | var self = this 127 | feed.setMaxListeners(128) 128 | feed.ready(function () { 129 | // It's possible these listeners are already attached. Ensure they are 130 | // removed before attaching them to avoid attaching them twice 131 | feed.removeListener('append', self._freshRun) 132 | feed.removeListener('download', self._freshRun) 133 | feed.on('append', self._freshRun) 134 | feed.on('download', self._freshRun) 135 | if (self._state.state === State.Idle) self._freshRun() 136 | }) 137 | } 138 | 139 | Indexer.prototype.pause = function (cb) { 140 | cb = cb || function () {} 141 | var self = this 142 | 143 | if (this._state.state === State.Paused || this._wantPause) { 144 | process.nextTick(cb) 145 | } else if (this._state.state === State.Idle) { 146 | self._setState(State.Paused) 147 | process.nextTick(cb) 148 | } else { 149 | this._wantPause = true 150 | this.once('pause', function () { 151 | self._wantPause = false 152 | self._setState(State.Paused) 153 | cb() 154 | }) 155 | } 156 | } 157 | 158 | Indexer.prototype.resume = function () { 159 | if (this._state.state !== State.Paused) return 160 | 161 | this._setState(State.Idle) 162 | this._freshRun() 163 | } 164 | 165 | Indexer.prototype.ready = function (fn) { 166 | if (this._state.state === State.Idle || this._state.state === State.Paused) process.nextTick(fn) 167 | else this.once('ready', fn) 168 | } 169 | 170 | Indexer.prototype._run = function (continuedRun) { 171 | if (this._wantPause) { 172 | this._wantPause = false 173 | this._pending = true 174 | this.emit('pause') 175 | return 176 | } 177 | if (!continuedRun && this._state.state !== State.Idle) { 178 | this._pending = true 179 | return 180 | } 181 | var self = this 182 | 183 | this._state.state = State.PreIndexing 184 | 185 | var didWork = false 186 | 187 | // load state from storage 188 | if (!this._at) { 189 | this._fetchIndexState(function (err, state) { 190 | if (err && !err.notFound) return self._onError(err) 191 | if (!state) { 192 | if (!self._clearIndex) return resetAt() 193 | self._clearIndex(function (err) { 194 | if (err) return self._onError(err) 195 | resetAt() 196 | }) 197 | } else { 198 | self._at = IndexState.deserialize(state).keys 199 | withState() 200 | } 201 | 202 | function resetAt () { 203 | self._at = {} 204 | self._log.feeds().forEach(function (feed) { 205 | self._at[feed.key.toString('hex')] = { 206 | key: feed.key, 207 | min: 0, 208 | max: 0 209 | } 210 | }) 211 | withState() 212 | } 213 | 214 | function withState () { 215 | self._log.feeds().forEach(function (feed) { 216 | feed.setMaxListeners(128) 217 | // The ready() method also adds these events listeners. Try to remove 218 | // them first so that they aren't added twice. 219 | feed.removeListener('append', self._freshRun) 220 | feed.removeListener('download', self._freshRun) 221 | feed.on('append', self._freshRun) 222 | feed.on('download', self._freshRun) 223 | }) 224 | 225 | work() 226 | } 227 | }) 228 | } else { 229 | work() 230 | } 231 | 232 | function work () { 233 | var feeds = self._log.feeds() 234 | var nodes = [] 235 | 236 | // Check if there is anything new. 237 | var indexedBlocks = Object.values(self._at).reduce((accum, entry) => accum + entry.max, 0) 238 | var totalBlocks = self._log.feeds().reduce((accum, feed) => accum + feed.length, 0) 239 | 240 | // Bail if no work needs to happen. 241 | if (indexedBlocks === totalBlocks) { 242 | return done() 243 | } 244 | 245 | if (!continuedRun) { 246 | const context = { 247 | indexStartTime: Date.now(), 248 | prevIndexedBlocks: self._state.context.indexedBlocks, 249 | indexedBlocks: indexedBlocks, 250 | totalBlocks: totalBlocks 251 | } 252 | self._setState(State.Indexing, context) 253 | } 254 | 255 | ;(function collect (i) { 256 | if (i >= feeds.length) return done() 257 | 258 | feeds[i].ready(function () { 259 | var key = feeds[i].key.toString('hex') 260 | 261 | if (self._at[key] === undefined) { 262 | self._at[key] = { key: feeds[i].key, min: 0, max: 0 } 263 | } 264 | 265 | // prefer to process forward 266 | var at = self._at[key].max 267 | var to = Math.min(feeds[i].length, at + self._maxBatch) 268 | 269 | if (!feeds[i].has(at, to)) { 270 | return collect(i + 1) 271 | } else if (at < to) { 272 | // TODO: This waits for all of the blocks to be available, and 273 | // actually blocks ALL indexing until it's ready. The intention is to 274 | // get min(maxBatch, feed.length-at) CONTIGUOUS entries 275 | feeds[i].getBatch(at, to, {wait: false}, function (err, res) { 276 | if (err || !res.length) { 277 | return collect(i + 1) 278 | } 279 | for (var j = 0; j < res.length; j++) { 280 | var node = res[j] 281 | nodes.push({ 282 | key: feeds[i].key.toString('hex'), 283 | seq: j + at, 284 | value: node 285 | }) 286 | } 287 | 288 | didWork = true 289 | self._batch(nodes, function (err) { 290 | if (err) return done(err) 291 | self._at[key].max += nodes.length 292 | self._storeIndexState(IndexState.serialize(self._at, self._version), function (err) { 293 | if (err) return done(err) 294 | self.emit('indexed', nodes) 295 | done() 296 | }) 297 | }) 298 | }) 299 | } else { 300 | collect(i + 1) 301 | } 302 | }) 303 | })(0) 304 | 305 | function done (err) { 306 | if (err) { 307 | self._onError(err) 308 | return 309 | } 310 | 311 | if (didWork || self._pending) { 312 | self._state.context.totalBlocks = self._log.feeds().reduce( 313 | (accum, feed) => accum + feed.length, 0) 314 | self._state.context.indexedBlocks = Object.values(self._at).reduce( 315 | (accum, entry) => accum + entry.max, 0) 316 | 317 | self._pending = false 318 | self._run(true) 319 | } else { 320 | if (self._wantPause) { 321 | self._wantPause = false 322 | self._pending = true 323 | self.emit('pause') 324 | } else { 325 | // Don't do a proper state change if this is the first run and 326 | // nothing had to be indexed, since it would look like Idle -> Idle 327 | // to API consumers. 328 | if (continuedRun) self._setState(State.Idle) 329 | else self._state.state = State.Idle 330 | 331 | self.emit('ready') 332 | } 333 | } 334 | } 335 | } 336 | } 337 | 338 | // Set state to `state` and apply updates `context` to the state context. Also 339 | // emits a `state-update` event. 340 | Indexer.prototype._setState = function (state, context) { 341 | if (state === this._state.state) return 342 | 343 | if (!context) context = {} 344 | 345 | this._state.state = state 346 | this._state.context = Object.assign({}, this._state.context, context) 347 | this.emit('state-update', clone(this._state, false)) 348 | } 349 | 350 | Indexer.prototype.getState = function () { 351 | const state = clone(this._state, false) 352 | 353 | // hidden states 354 | if (state.state === State.PreIndexing) state.state = State.Idle 355 | 356 | return state 357 | } 358 | 359 | function allOrNone (a, b) { 360 | return (!!a && !!b) || (!a && !b) 361 | } 362 | 363 | function unset (x) { 364 | return x === null || x === undefined 365 | } 366 | -------------------------------------------------------------------------------- /lib/state.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | serialize: serializeState, 3 | deserialize: deserializeState 4 | } 5 | 6 | /* 7 | * KeyState: { key: Buffer<32>, min: Number, max: Number } 8 | * ViewState: { keys: {HexString<32>}, version: Number } 9 | */ 10 | 11 | // Number -> Buffer<4> 12 | function int32buf (n) { 13 | var buf = Buffer.alloc(4) 14 | buf.writeUInt32LE(n, 0) 15 | return buf 16 | } 17 | 18 | // KeyState, Number -> Buffer 19 | function serializeState (at, version) { 20 | at = values(at) 21 | var len = int32buf(at.length) 22 | var bufs = [len] 23 | for (var i = 0; i < at.length; i++) { 24 | bufs.push(at[i].key) 25 | bufs.push(int32buf(at[i].min)) 26 | bufs.push(int32buf(at[i].max)) 27 | } 28 | 29 | if (version && typeof version === 'number') bufs.push(int32buf(version)) 30 | 31 | return Buffer.concat(bufs) 32 | } 33 | 34 | // Buffer -> ViewState 35 | function deserializeState (buf) { 36 | var state = { keys: {} } 37 | var len = buf.readUInt32LE(0) 38 | for (var i = 0; i < len; i++) { 39 | var pos = 4 + i * 40 40 | var key = buf.slice(pos, pos + 32) 41 | var min = buf.readUInt32LE(pos + 32) 42 | var max = buf.readUInt32LE(pos + 36) 43 | state.keys[key.toString('hex')] = { 44 | key: key, 45 | min: min, 46 | max: max 47 | } 48 | } 49 | 50 | // Read 'version', if there are any unread bytes left. 51 | if (4 + len * 40 + 4 <= buf.length) { 52 | var version = buf.readUInt32LE(4 + len * 40) 53 | state.version = version 54 | } else { 55 | state.version = 1 56 | } 57 | 58 | return state 59 | } 60 | 61 | // {String:A} -> [A] 62 | function values (dict) { 63 | return Object.keys(dict).map(k => dict[k]) 64 | } 65 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multifeed-index", 3 | "description": "build an index over a set of hypercores", 4 | "author": "noffle", 5 | "version": "3.4.2", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "standard && tape test/*.js", 9 | "lint": "standard" 10 | }, 11 | "license": "ISC", 12 | "dependencies": { 13 | "clone": "^2.1.2", 14 | "inherits": "^2.0.3" 15 | }, 16 | "devDependencies": { 17 | "hypercore": "^8.3.0", 18 | "level-mem": "^5.0.1", 19 | "multifeed": "^5.0.0", 20 | "random-access-memory": "^3.1.1", 21 | "standard": "^11.0.1", 22 | "tape": "^4.9.2", 23 | "unordered-materialized-kv": "^1.2.1" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /test/adder.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var ram = require('random-access-memory') 4 | var index = require('..') 5 | var tmp = require('os').tmpdir 6 | var rimraf = require('rimraf') 7 | var path = require('path') 8 | var versions = require('../lib/state') 9 | 10 | test('empty + ready called', function (t) { 11 | t.plan(1) 12 | 13 | var db = multifeed(ram, { valueEncoding: 'json' }) 14 | var version = null 15 | var idx = index({ 16 | log: db, 17 | batch: function (nodes, next) { 18 | next() 19 | }, 20 | fetchState: function (cb) { cb(null, version) }, 21 | storeState: function (s, cb) { version = s; cb(null) }, 22 | clearIndex: function (cb) { version = null; cb(null) } 23 | }) 24 | 25 | idx.ready(function () { 26 | t.ok(true) 27 | }) 28 | }) 29 | 30 | test('adder', function (t) { 31 | t.plan(7) 32 | 33 | var db = multifeed(ram, { valueEncoding: 'json' }) 34 | 35 | var sum = 0 36 | var version = null 37 | 38 | var idx = index({ 39 | log: db, 40 | maxBatch: 1, 41 | batch: function (nodes, next) { 42 | nodes.forEach(function (node) { sum += node.value.value }) 43 | next() 44 | 45 | if (!--pending) done() 46 | }, 47 | fetchState: function (cb) { cb(null, version) }, 48 | storeState: function (s, cb) { version = s; cb(null) }, 49 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 50 | }) 51 | 52 | var pending = 3 53 | db.writer(function (err, w) { 54 | t.error(err, 'got writer') 55 | w.append({value: 17}, function (err) { t.error(err, 'wrote 17') }) 56 | w.append({value: 12}, function (err) { t.error(err, 'wrote 12') }) 57 | w.append({value: 1}, function (err) { t.error(err, 'wrote 1') }) 58 | }) 59 | 60 | function done () { 61 | idx.ready(function () { 62 | var finalVersion = values(versions.deserialize(version).keys) 63 | t.equal(finalVersion.length, 1, 'correct # of keys') 64 | t.equal(finalVersion[0].max, 3, 'correct # of entries') 65 | t.equal(sum, 30, 'correct sum') 66 | }) 67 | } 68 | }) 69 | 70 | test('adder: picks up where it left off', function (t) { 71 | t.plan(6) 72 | 73 | var db = multifeed(ram, { valueEncoding: 'json' }) 74 | 75 | var version = null 76 | var pending = 3 77 | 78 | var idx = index({ 79 | log: db, 80 | maxBatch: 1, 81 | batch: function (nodes, next) { 82 | next() 83 | if (!--pending) done() 84 | }, 85 | fetchState: function (cb) { cb(null, version) }, 86 | storeState: function (s, cb) { version = s; cb(null) }, 87 | clearIndex: function (cb) { version = null; cb(null) } 88 | }) 89 | 90 | var writer 91 | db.writer(function (err, w) { 92 | t.error(err, 'created a writer') 93 | writer = w 94 | w.append({value: 17}, function (err) { t.error(err, 'appended 17') }) 95 | w.append({value: 12}, function (err) { t.error(err, 'appended 12') }) 96 | w.append({value: 1}, function (err) { t.error(err, 'appended 1') }) 97 | }) 98 | 99 | function done () { 100 | idx.ready(function () { 101 | var pending = 1 102 | var sum = 0 103 | var version2 = version.slice() 104 | index({ 105 | log: db, 106 | maxBatch: 1, 107 | batch: function (nodes, next) { 108 | nodes.forEach(function (node) { sum += node.value.value }) 109 | 110 | if (!--pending) { 111 | t.equals(sum, 7, 'processed only last item') 112 | } 113 | next() 114 | }, 115 | fetchState: function (cb) { cb(null, version2) }, 116 | storeState: function (s, cb) { version2 = s; cb(null) }, 117 | clearIndex: function (cb) { version = null; cb(null) } 118 | }) 119 | writer.append({value: 7}, function (err) { t.error(err, 'appended 7') }) 120 | }) 121 | } 122 | }) 123 | 124 | test('adder /w slow versions', function (t) { 125 | t.plan(7) 126 | 127 | var db = multifeed(ram, { valueEncoding: 'json' }) 128 | 129 | var sum = 0 130 | var version = null 131 | 132 | var idx = index({ 133 | log: db, 134 | maxBatch: 1, 135 | batch: function (nodes, next) { 136 | nodes.forEach(function (node) { sum += node.value.value }) 137 | next() 138 | }, 139 | fetchState: function (cb) { 140 | setTimeout(function () { cb(null, version) }, 100) 141 | }, 142 | storeState: function (s, cb) { version = s; setTimeout(cb, 100) }, 143 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 144 | }) 145 | 146 | var pending = 3 147 | db.writer(function (err, w) { 148 | t.error(err) 149 | w.append({value: 17}, done) 150 | w.append({value: 12}, done) 151 | w.append({value: 1}, done) 152 | }) 153 | 154 | function done (err) { 155 | t.error(err) 156 | if (!--pending) { 157 | idx.ready(function () { 158 | var finalVersion = values(versions.deserialize(version).keys) 159 | t.equal(finalVersion.length, 1) 160 | t.equal(finalVersion[0].max, 3) 161 | t.equal(sum, 30) 162 | }) 163 | } 164 | } 165 | }) 166 | 167 | test('adder /w many concurrent PUTs', function (t) { 168 | t.plan(204) 169 | 170 | var db = multifeed(ram, { valueEncoding: 'json' }) 171 | 172 | var sum = 0 173 | var version = null 174 | 175 | var idx = index({ 176 | log: db, 177 | maxBatch: 1, 178 | batch: function (nodes, next) { 179 | nodes.forEach(function (node) { sum += node.value.value }) 180 | next() 181 | 182 | if (!--pending) done() 183 | }, 184 | fetchState: function (cb) { cb(null, version) }, 185 | storeState: function (s, cb) { version = s; cb(null) }, 186 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 187 | }) 188 | 189 | var pending = 200 190 | var expectedSum = 0 191 | 192 | db.writer(function (err, w) { 193 | t.error(err) 194 | for (var i = 0; i < pending; i++) { 195 | var n = Math.floor(Math.random() * 10) 196 | expectedSum += n 197 | w.append({value: n}, function (err) { t.error(err) }) 198 | } 199 | }) 200 | 201 | function done () { 202 | idx.ready(function () { 203 | var finalVersion = values(versions.deserialize(version).keys) 204 | t.equal(finalVersion.length, 1) 205 | t.equal(finalVersion[0].max, 200) 206 | t.equal(sum, expectedSum) 207 | }) 208 | } 209 | }) 210 | 211 | test('adder /w index made AFTER db population', function (t) { 212 | t.plan(204) 213 | 214 | var db = multifeed(ram, { valueEncoding: 'json' }) 215 | 216 | var sum = 0 217 | var version = null 218 | 219 | var pending = 200 220 | var expectedSum = 0 221 | 222 | db.writer(function (err, w) { 223 | t.error(err) 224 | for (var i = 0; i < pending; i++) { 225 | var n = Math.floor(Math.random() * 10) 226 | expectedSum += n 227 | w.append({value: n}, function (err) { 228 | t.error(err) 229 | if (!--pending) done() 230 | }) 231 | } 232 | }) 233 | 234 | function done () { 235 | var idx = index({ 236 | log: db, 237 | maxBatch: 1, 238 | batch: function (nodes, next) { 239 | nodes.forEach(function (node) { sum += node.value.value }) 240 | next() 241 | }, 242 | fetchState: function (cb) { cb(null, version) }, 243 | storeState: function (s, cb) { version = s; cb(null) }, 244 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 245 | }) 246 | idx.ready(function () { 247 | var finalVersion = values(versions.deserialize(version).keys) 248 | t.equal(finalVersion.length, 1) 249 | t.equal(finalVersion[0].max, 200) 250 | t.equal(sum, expectedSum) 251 | }) 252 | } 253 | }) 254 | 255 | test('adder /w async storage', function (t) { 256 | t.plan(7) 257 | 258 | var db = multifeed(ram, { valueEncoding: 'json' }) 259 | 260 | var sum = 0 261 | var version = null 262 | 263 | function getSum (cb) { 264 | setTimeout(function () { cb(sum) }, Math.floor(Math.random() * 200)) 265 | } 266 | function setSum (newSum, cb) { 267 | setTimeout(function () { 268 | sum = newSum 269 | cb() 270 | }, Math.floor(Math.random() * 200)) 271 | } 272 | 273 | var idx = index({ 274 | log: db, 275 | maxBatch: 1, 276 | batch: function (nodes, next) { 277 | nodes.forEach(function (node) { 278 | if (typeof node.value.value === 'number') { 279 | getSum(function (theSum) { 280 | theSum += node.value.value 281 | setSum(theSum, function () { 282 | next() 283 | if (!--pending) done() 284 | }) 285 | }) 286 | } 287 | }) 288 | }, 289 | fetchState: function (cb) { cb(null, version) }, 290 | storeState: function (s, cb) { version = s; cb(null) }, 291 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 292 | }) 293 | 294 | var pending = 3 295 | db.writer(function (err, w) { 296 | t.error(err) 297 | w.append({value: 17}, function (err) { t.error(err) }) 298 | w.append({value: 12}, function (err) { t.error(err) }) 299 | w.append({value: 1}, function (err) { t.error(err) }) 300 | }) 301 | 302 | function done () { 303 | idx.ready(function () { 304 | var finalVersion = values(versions.deserialize(version).keys) 305 | t.equal(finalVersion.length, 1) 306 | t.equal(finalVersion[0].max, 3) 307 | t.equal(sum, 30) 308 | }) 309 | } 310 | }) 311 | 312 | test('adder /w async storage: ready', function (t) { 313 | t.plan(7) 314 | 315 | var db = multifeed(ram, { valueEncoding: 'json' }) 316 | 317 | var sum = 0 318 | var version = null 319 | 320 | function getSum (cb) { 321 | setTimeout(function () { cb(sum) }, Math.floor(Math.random() * 100)) 322 | } 323 | function setSum (newSum, cb) { 324 | setTimeout(function () { 325 | sum = newSum 326 | cb() 327 | }, Math.floor(Math.random() * 100)) 328 | } 329 | 330 | var idx = index({ 331 | log: db, 332 | maxBatch: 1, 333 | batch: function (nodes, next) { 334 | nodes.forEach(function (node) { 335 | if (typeof node.value.value === 'number') { 336 | getSum(function (theSum) { 337 | theSum += node.value.value 338 | setSum(theSum, function () { 339 | next() 340 | }) 341 | }) 342 | } 343 | }) 344 | }, 345 | fetchState: function (cb) { cb(null, version) }, 346 | storeState: function (s, cb) { version = s; cb(null) }, 347 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 348 | }) 349 | 350 | db.writer(function (err, w) { 351 | t.error(err) 352 | w.append({value: 17}, function (err) { 353 | t.error(err) 354 | w.append({value: 12}, function (err) { 355 | t.error(err) 356 | w.append({value: 1}, function (err) { 357 | t.error(err) 358 | idx.ready(function () { 359 | getSum(function (theSum) { 360 | var finalVersion = values(versions.deserialize(version).keys) 361 | t.equal(finalVersion.length, 1) 362 | t.equal(finalVersion[0].max, 3) 363 | t.equals(theSum, 30) 364 | }) 365 | }) 366 | }) 367 | }) 368 | }) 369 | }) 370 | }) 371 | 372 | test('fs: adder', function (t) { 373 | t.plan(57) 374 | 375 | var id = String(Math.random()).substring(2) 376 | var dir = path.join(tmp(), 'hyperdb-index-test-' + id) 377 | var db = multifeed(dir, { valueEncoding: 'json' }) 378 | 379 | var sum = 0 380 | var version = null 381 | 382 | var idx = index({ 383 | log: db, 384 | maxBatch: 1, 385 | batch: function (nodes, next) { 386 | nodes.forEach(function (node) { 387 | if (typeof node.value.value === 'number') sum += node.value.value 388 | }) 389 | next() 390 | }, 391 | fetchState: function (cb) { 392 | setTimeout(function () { cb(null, version) }, 50) 393 | }, 394 | storeState: function (s, cb) { 395 | setTimeout(function () { version = s; cb(null) }, 50) 396 | }, 397 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 398 | }) 399 | 400 | var pending = 50 401 | var expectedSum = 0 402 | db.writer(function (err, w) { 403 | t.error(err) 404 | for (var i = 0; i < pending; i++) { 405 | var value = i * 2 + 1 406 | expectedSum += value 407 | w.append({value: value}, function (err) { 408 | t.error(err) 409 | if (!--pending) done() 410 | }) 411 | } 412 | }) 413 | 414 | function done (err) { 415 | t.error(err) 416 | idx.ready(function () { 417 | var finalVersion = values(versions.deserialize(version).keys) 418 | t.equal(finalVersion.length, 1) 419 | t.equal(sum, expectedSum, 'sum of all nodes is as expected') 420 | t.equal(finalVersion[0].max, 50) 421 | db.close(function (err) { 422 | t.error(err) 423 | rimraf(dir, function (err) { 424 | t.error(err) 425 | }) 426 | }) 427 | }) 428 | } 429 | }) 430 | 431 | test('adder + sync', function (t) { 432 | t.plan(14) 433 | 434 | createTwo(function (db1, db2) { 435 | var sum1 = 0 436 | var sum2 = 0 437 | var version1 = null 438 | var version2 = null 439 | 440 | var pending = 4 441 | var idx1 = index({ 442 | log: db1, 443 | maxBatch: 1, 444 | batch: function (nodes, next) { 445 | nodes.forEach(function (node) { 446 | if (typeof node.value.value === 'number') sum1 += node.value.value 447 | }) 448 | next() 449 | 450 | if (!--pending) done() 451 | }, 452 | fetchState: function (cb) { cb(null, version1) }, 453 | storeState: function (s, cb) { version1 = s; cb(null) }, 454 | clearIndex: function (cb) { version1 = null; sum1 = 0; cb(null) } 455 | }) 456 | 457 | var idx2 = index({ 458 | log: db2, 459 | maxBatch: 1, 460 | batch: function (nodes, next) { 461 | nodes.forEach(function (node) { 462 | if (typeof node.value.value === 'number') sum2 += node.value.value 463 | }) 464 | next() 465 | 466 | if (!--pending) done() 467 | }, 468 | fetchState: function (cb) { cb(null, version2) }, 469 | storeState: function (s, cb) { version2 = s; cb(null) }, 470 | clearIndex: function (cb) { version1 = null; sum2 = 0; cb(null) } 471 | }) 472 | 473 | db1.writer(function (err, w) { 474 | t.error(err) 475 | w.append({value: 17}, function (err) { t.error(err) }) 476 | w.append({value: 12}, function (err) { t.error(err) }) 477 | w.append({value: 1}, function (err) { t.error(err) }) 478 | }) 479 | db2.writer(function (err, w) { 480 | t.error(err) 481 | w.append({value: 9}, function (err) { t.error(err) }) 482 | }) 483 | 484 | function done () { 485 | replicate(db1, db2, function () { 486 | idx1.ready(function () { 487 | idx2.ready(function () { 488 | var finalVersion = values(versions.deserialize(version1).keys) 489 | t.equal(finalVersion.length, 2) 490 | t.equal(finalVersion[0].max, 3) 491 | t.equal(finalVersion[1].max, 1) 492 | t.equal(sum1, 39) 493 | 494 | finalVersion = values(versions.deserialize(version2).keys) 495 | t.equal(finalVersion.length, 2) 496 | t.equal(finalVersion[0].max, 1) 497 | t.equal(finalVersion[1].max, 3) 498 | t.equal(sum2, 39) 499 | 500 | t.end() 501 | }) 502 | }) 503 | }) 504 | } 505 | }) 506 | }) 507 | 508 | function createTwo (cb) { 509 | var a = multifeed(ram, {valueEncoding: 'json'}) 510 | a.ready(function () { 511 | var b = multifeed(ram, {valueEncoding: 'json'}) 512 | b.ready(function () { 513 | cb(a, b) 514 | }) 515 | }) 516 | } 517 | 518 | function replicate (a, b, cb) { 519 | var stream = a.replicate(true) 520 | stream.pipe(b.replicate(false)).pipe(stream).on('end', cb) 521 | } 522 | 523 | function values (dict) { 524 | return Object.keys(dict).map(k => dict[k]) 525 | } 526 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var indexer = require('..') 4 | var umkv = require('unordered-materialized-kv') 5 | var ram = require('random-access-memory') 6 | var memdb = require('level-mem') 7 | 8 | test('kv: create index then data', function (t) { 9 | t.plan(10) 10 | 11 | var multi = multifeed(ram, { valueEncoding: 'json' }) 12 | 13 | var kv = umkv(memdb()) 14 | 15 | var hyperkv = indexer({ 16 | log: multi, 17 | batch: function (nodes, next) { 18 | var batch = nodes.map(function (node) { 19 | return { 20 | id: node.key.toString('hex') + '@' + node.seq, 21 | key: node.value.key, 22 | links: node.value.links 23 | } 24 | }) 25 | kv.batch(batch, next) 26 | } 27 | }) 28 | 29 | function append (w, data, cb) { 30 | w.append(data, function (err) { 31 | t.error(err) 32 | var id = w.key.toString('hex') + '@' + (w.length - 1) 33 | cb(null, id) 34 | }) 35 | } 36 | 37 | hyperkv.ready(function () { 38 | kv.get('foo', function (err, res) { 39 | t.ok(err, 'foo not inserted yet') 40 | t.equals(err.notFound, true, 'not found error from level') 41 | }) 42 | }) 43 | 44 | multi.writer(function (err, w) { 45 | t.error(err) 46 | append(w, { 47 | key: 'foo', 48 | value: 'bax', 49 | links: [] 50 | }, function (err, id1) { 51 | t.error(err, 'no append error 1') 52 | append(w, { 53 | key: 'foo', 54 | value: 'bax', 55 | links: [id1] 56 | }, function (err, id2) { 57 | t.error(err, 'no append error 1') 58 | hyperkv.ready(function () { 59 | kv.get('foo', function (err, res) { 60 | t.error(err) 61 | t.equals(res.length, 1) 62 | t.equals(res[0], w.key.toString('hex') + '@1') 63 | }) 64 | }) 65 | }) 66 | }) 67 | }) 68 | }) 69 | 70 | test('indexed event', function (t) { 71 | t.plan(4) 72 | 73 | var multi = multifeed(ram, { valueEncoding: 'json' }) 74 | 75 | var entries = [1, 2, 3, 4, 5, 6] 76 | multi.writer(function (err, w) { 77 | t.error(err) 78 | w.append(entries, function (err) { 79 | t.error(err) 80 | w.append(10, function (err) { 81 | t.error(err) 82 | counter.ready(function () { 83 | t.equals(count, 7, 'count matches') 84 | }) 85 | }) 86 | }) 87 | }) 88 | 89 | var count = 0 90 | 91 | var counter = indexer({ 92 | log: multi, 93 | batch: function (nodes, next) { 94 | next() 95 | } 96 | }) 97 | 98 | counter.on('indexed', function (msgs) { 99 | count += msgs.length 100 | }) 101 | }) 102 | -------------------------------------------------------------------------------- /test/batch.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var ram = require('random-access-memory') 4 | var index = require('..') 5 | 6 | test('batch size', function (t) { 7 | t.plan(6) 8 | 9 | var db = multifeed(ram, { valueEncoding: 'json' }) 10 | 11 | var pending = 3 12 | db.writer(function (err, w) { 13 | t.error(err) 14 | w.append({value: 17}, function (err) { t.error(err); write() }) 15 | w.append({value: 12}, function (err) { t.error(err); write() }) 16 | w.append({value: 1}, function (err) { t.error(err); write() }) 17 | }) 18 | 19 | function write () { 20 | if (--pending) return 21 | var version = null 22 | var sum = 0 23 | var idx = index({ 24 | log: db, 25 | maxBatch: 10, 26 | batch: function (nodes, next) { 27 | t.equals(nodes.length, 3, 'correct batch size') 28 | nodes.forEach(function (node) { sum += node.value.value }) 29 | next() 30 | }, 31 | fetchState: function (cb) { cb(null, version) }, 32 | storeState: function (s, cb) { version = s; cb(null) }, 33 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 34 | }) 35 | 36 | idx.ready(function () { 37 | t.equal(sum, 30, 'correct sum') 38 | }) 39 | } 40 | }) 41 | -------------------------------------------------------------------------------- /test/fork.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var ram = require('random-access-memory') 4 | var memdb = require('level-mem') 5 | var umkv = require('unordered-materialized-kv') 6 | var indexer = require('../') 7 | 8 | test('kv merge fork', function (t) { 9 | t.plan(19) 10 | var a = multifeed(ram, { valueEncoding: 'json' }) 11 | var b = multifeed(ram, { valueEncoding: 'json' }) 12 | var akv = umkv(memdb()) 13 | var bkv = umkv(memdb()) 14 | var ai = indexer({ 15 | log: a, 16 | batch: batchFn(akv) 17 | }) 18 | var bi = indexer({ 19 | log: b, 20 | batch: batchFn(bkv) 21 | }) 22 | function batchFn (kv) { 23 | return function (nodes, next) { 24 | kv.batch(nodes.map(function (node) { 25 | return { 26 | id: node.value.id, 27 | key: 'X', 28 | links: node.value.links 29 | } 30 | }), next) 31 | } 32 | } 33 | 34 | var pending = 2 35 | a.ready(onready) 36 | b.ready(onready) 37 | function onready () { 38 | if (--pending === 0) populateFirst() 39 | } 40 | function populateFirst () { 41 | a.writer(function (err, w) { 42 | t.error(err) 43 | w.append({ id: 1, value: 100, links: [] }, function (err) { 44 | t.error(err) 45 | sync(a, b, populateSecond) 46 | }) 47 | }) 48 | } 49 | function populateSecond (err) { 50 | var pending = 2 51 | t.error(err) 52 | a.writer(function (err, w) { 53 | t.error(err) 54 | w.append({ id: 2, value: 200, links: [1] }, function (err) { 55 | t.error(err) 56 | if (--pending === 0) sync(a, b, readyCheckForked) 57 | }) 58 | }) 59 | b.writer(function (err, w) { 60 | t.error(err) 61 | w.append({ id: 3, value: 300, links: [1] }, function (err) { 62 | t.error(err) 63 | if (--pending === 0) sync(a, b, readyCheckForked) 64 | }) 65 | }) 66 | } 67 | function readyCheckForked (err) { 68 | t.error(err) 69 | var pending = 2 70 | ai.ready(onready) 71 | bi.ready(onready) 72 | function onready () { if (--pending === 0) checkForked() } 73 | } 74 | function checkForked () { 75 | var pending = 2 76 | akv.get('X', function (err, heads) { 77 | t.error(err) 78 | t.deepEqual(heads.sort(), ['2', '3']) 79 | if (--pending === 0) populateThird() 80 | }) 81 | bkv.get('X', function (err, heads) { 82 | t.error(err) 83 | t.deepEqual(heads.sort(), ['2', '3']) 84 | if (--pending === 0) populateThird() 85 | }) 86 | } 87 | function populateThird () { 88 | a.writer(function (err, w) { 89 | t.error(err) 90 | w.append({ id: 4, value: 400, links: [2, 3] }, function (err) { 91 | t.error(err) 92 | sync(a, b, readyCheckFinal) 93 | }) 94 | }) 95 | } 96 | function readyCheckFinal (err) { 97 | t.error(err) 98 | var pending = 2 99 | ai.ready(onready) 100 | bi.ready(onready) 101 | function onready () { if (--pending === 0) checkFinal() } 102 | } 103 | function checkFinal () { 104 | akv.get('X', function (err, heads) { 105 | t.error(err) 106 | t.deepEqual(heads, ['4']) 107 | }) 108 | bkv.get('X', function (err, heads) { 109 | t.error(err) 110 | t.deepEqual(heads, ['4']) 111 | }) 112 | } 113 | }) 114 | 115 | function sync (a, b, cb) { 116 | var pending = 2 117 | var sa = a.replicate(true) 118 | var sb = b.replicate(false) 119 | sa.pipe(sb).pipe(sa) 120 | sa.on('error', cb) 121 | sb.on('error', cb) 122 | sa.on('end', onend) 123 | sb.on('end', onend) 124 | function onend () { if (--pending === 0) cb() } 125 | } 126 | -------------------------------------------------------------------------------- /test/pause-resume.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var indexer = require('..') 4 | var ram = require('random-access-memory') 5 | 6 | test('pause while ready', function (t) { 7 | t.plan(4) 8 | 9 | var count = 0 10 | var multi = multifeed(ram, { valueEncoding: 'json' }) 11 | var counter = indexer({ 12 | log: multi, 13 | batch: function (nodes, next) { 14 | next() 15 | } 16 | }) 17 | 18 | var entries = [1, 2, 3, 4, 5, 6] 19 | multi.writer(function (err, w) { 20 | t.error(err) 21 | w.append(entries, function (err) { 22 | t.error(err) 23 | counter.ready(function () { 24 | counter.pause(function () { 25 | w.append(10, function (err) { 26 | t.error(err) 27 | counter.resume() 28 | counter.ready(function () { 29 | t.equals(count, 31, 'count matches') 30 | }) 31 | }) 32 | }) 33 | }) 34 | }) 35 | }) 36 | 37 | counter.on('indexed', function (msgs) { 38 | count = msgs.reduce(function (accum, msg) { 39 | return accum + msg.value 40 | }, count) 41 | }) 42 | }) 43 | 44 | test('pause while indexing', function (t) { 45 | t.plan(5) 46 | 47 | var count = 0 48 | var multi = multifeed(ram, { valueEncoding: 'json' }) 49 | var counter = indexer({ 50 | log: multi, 51 | batch: function (nodes, next) { 52 | next() 53 | } 54 | }) 55 | 56 | var entries = [1, 2, 3, 4, 5, 6] 57 | multi.writer(function (err, w) { 58 | t.error(err) 59 | w.append(entries, function (err) { 60 | t.error(err) 61 | counter.pause(function () { 62 | w.append(10, function (err) { 63 | t.error(err) 64 | counter.ready(function () { 65 | t.equals(count, 21, 'count matches') 66 | counter.resume() 67 | counter.ready(function () { 68 | t.equals(count, 31, 'count matches') 69 | }) 70 | }) 71 | }) 72 | }) 73 | }) 74 | }) 75 | 76 | counter.on('indexed', function (msgs) { 77 | count = msgs.reduce(function (accum, msg) { 78 | return accum + msg.value 79 | }, count) 80 | }) 81 | }) 82 | -------------------------------------------------------------------------------- /test/states.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var indexer = require('..') 4 | var ram = require('random-access-memory') 5 | 6 | test('state is idle on creation', function (t) { 7 | t.plan(1) 8 | 9 | var multi = multifeed(ram, { valueEncoding: 'json' }) 10 | 11 | var idx = indexer({ 12 | log: multi, 13 | batch: function (nodes, next) { 14 | next() 15 | } 16 | }) 17 | 18 | idx.ready(function () { 19 | t.equals(idx.getState().state, 'idle') 20 | }) 21 | }) 22 | 23 | test('create a writer', function (t) { 24 | t.plan(2) 25 | 26 | var multi = multifeed(ram, { valueEncoding: 'json' }) 27 | 28 | var idx = indexer({ 29 | log: multi, 30 | batch: function (nodes, next) { 31 | next() 32 | } 33 | }) 34 | 35 | idx.once('state-update', function (state) { 36 | t.equals(state.state, 'idle') 37 | }) 38 | 39 | multi.writer(function (err, w) { 40 | t.error(err) 41 | }) 42 | }) 43 | 44 | test('create a writer and write blocks', function (t) { 45 | t.plan(8) 46 | 47 | var multi = multifeed(ram, { valueEncoding: 'json' }) 48 | 49 | var idx = indexer({ 50 | log: multi, 51 | batch: function (nodes, next) { 52 | next() 53 | } 54 | }) 55 | 56 | idx.ready(function () { 57 | idx.once('state-update', function (state) { 58 | t.equals(state.state, 'indexing') 59 | t.equals(idx.getState().context.totalBlocks, 1) 60 | t.equals(idx.getState().context.indexedBlocks, 0) 61 | 62 | idx.once('state-update', function (state) { 63 | t.equals(state.state, 'idle') 64 | t.equals(idx.getState().context.totalBlocks, 1) 65 | t.equals(idx.getState().context.indexedBlocks, 1) 66 | }) 67 | }) 68 | 69 | multi.writer(function (err, w) { 70 | t.error(err) 71 | w.append({ 72 | key: 'foo', 73 | value: 'bax' 74 | }, function (err, id1) { 75 | t.error(err, 'append is ok') 76 | }) 77 | }) 78 | }) 79 | }) 80 | 81 | test('pausing when nothing is indexing', function (t) { 82 | t.plan(3) 83 | 84 | var multi = multifeed(ram, { valueEncoding: 'json' }) 85 | 86 | var idx = indexer({ 87 | log: multi, 88 | batch: function (nodes, next) { 89 | next() 90 | } 91 | }) 92 | 93 | idx.ready(function () { 94 | t.equals(idx.getState().state, 'idle') 95 | idx.pause(function () { 96 | t.equals(idx.getState().state, 'paused') 97 | idx.resume() 98 | t.equals(idx.getState().state, 'idle') 99 | }) 100 | }) 101 | }) 102 | 103 | test('pausing while indexing', function (t) { 104 | t.plan(16) 105 | 106 | var multi = multifeed(ram, { valueEncoding: 'json' }) 107 | 108 | var idx = indexer({ 109 | log: multi, 110 | batch: function (nodes, next) { 111 | next() 112 | } 113 | }) 114 | 115 | idx.ready(function () { 116 | t.equals(idx.getState().state, 'idle') 117 | 118 | multi.writer(function (err, w) { 119 | t.error(err) 120 | 121 | w.append({ 122 | key: 'foo', 123 | value: 'bax' 124 | }, function (err, id1) { 125 | t.error(err, 'append is ok') 126 | 127 | t.equals(idx.getState().state, 'indexing') 128 | t.equals(idx.getState().context.totalBlocks, 1) 129 | t.equals(idx.getState().context.indexedBlocks, 0) 130 | 131 | idx.pause(function () { 132 | t.equals(idx.getState().state, 'paused') 133 | t.equals(idx.getState().context.totalBlocks, 1) 134 | t.equals(idx.getState().context.indexedBlocks, 1) 135 | 136 | w.append({ 137 | key: 'quux', 138 | value: 'simmel' 139 | }) 140 | 141 | idx.resume() 142 | t.equals(idx.getState().state, 'idle') 143 | 144 | idx.once('state-update', function (state) { 145 | t.equals(idx.getState().state, 'indexing') 146 | t.equals(idx.getState().context.totalBlocks, 2) 147 | t.equals(idx.getState().context.indexedBlocks, 1) 148 | 149 | idx.once('state-update', function (state) { 150 | t.equals(idx.getState().state, 'idle') 151 | t.equals(idx.getState().context.totalBlocks, 2) 152 | t.equals(idx.getState().context.indexedBlocks, 2) 153 | }) 154 | }) 155 | }) 156 | }) 157 | }) 158 | }) 159 | }) 160 | 161 | test('error state', function (t) { 162 | t.plan(2) 163 | 164 | var multi = multifeed(ram, { valueEncoding: 'json' }) 165 | const error = new Error('boom!') 166 | 167 | var idx = indexer({ 168 | log: multi, 169 | fetchState: function (cb) { process.nextTick(cb, error) }, 170 | storeState: function (state, cb) { process.nextTick(cb, error) }, 171 | batch: function (nodes, next) { 172 | next() 173 | } 174 | }) 175 | idx.on('error', function () {}) 176 | 177 | idx.once('state-update', function (state) { 178 | t.equals(state.state, 'error') 179 | t.deepEquals(state.context.error, error) 180 | }) 181 | }) 182 | -------------------------------------------------------------------------------- /test/sync.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var ram = require('random-access-memory') 4 | var index = require('..') 5 | 6 | test('multiple feeds', function (t) { 7 | createTwo(function (a, b) { 8 | var sums = [0, 0] 9 | var version1 = null 10 | var version2 = null 11 | 12 | var pending = 5 13 | a.writer(function (err, w) { 14 | t.error(err) 15 | w.append({value: 17}, function (err) { t.error(err); sync() }) 16 | w.append({value: 12}, function (err) { t.error(err); sync() }) 17 | w.append({value: 1}, function (err) { t.error(err); sync() }) 18 | }) 19 | b.writer(function (err, w) { 20 | t.error(err) 21 | w.append({value: 11}, function (err) { t.error(err); sync() }) 22 | w.append({value: 3}, function (err) { t.error(err); sync() }) 23 | }) 24 | 25 | function batchFn (sumId, nodes, next) { 26 | nodes.forEach(function (node) { 27 | if (typeof node.value.value === 'number') sums[sumId] += node.value.value 28 | }) 29 | next() 30 | } 31 | 32 | function sync () { 33 | if (--pending) return 34 | replicate(a, b, function () { 35 | doIndex() 36 | }) 37 | } 38 | 39 | function doIndex () { 40 | var idx1 = index({ 41 | log: a, 42 | maxBatch: 50, 43 | batch: batchFn.bind(null, 0), 44 | fetchState: function (cb) { cb(null, version1) }, 45 | storeState: function (s, cb) { version1 = s; cb(null) }, 46 | clearIndex: function (cb) { process.nextTick(cb) } 47 | }) 48 | var idx2 = index({ 49 | log: b, 50 | maxBatch: 50, 51 | batch: batchFn.bind(null, 1), 52 | fetchState: function (cb) { cb(null, version2) }, 53 | storeState: function (s, cb) { version2 = s; cb(null) }, 54 | clearIndex: function (cb) { process.nextTick(cb) } 55 | }) 56 | 57 | idx1.ready(function () { 58 | idx2.ready(function () { 59 | t.equals(sums[0], 44, 'db A sum matches') 60 | t.equals(sums[1], 44, 'db B sum matches') 61 | t.end() 62 | }) 63 | }) 64 | } 65 | }) 66 | }) 67 | 68 | function createTwo (cb) { 69 | var a = multifeed(ram, {valueEncoding: 'json'}) 70 | a.ready(function () { 71 | var b = multifeed(ram, {valueEncoding: 'json'}) 72 | b.ready(function () { 73 | cb(a, b) 74 | }) 75 | }) 76 | } 77 | 78 | function replicate (a, b, cb) { 79 | var stream = a.replicate(true) 80 | stream.pipe(b.replicate(false)).pipe(stream).on('end', cb) 81 | } 82 | -------------------------------------------------------------------------------- /test/version.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var multifeed = require('multifeed') 3 | var raf = require('random-access-file') 4 | var rimraf = require('rimraf') 5 | var tmp = require('tmp') 6 | var index = require('..') 7 | 8 | // 1. build index 9 | // 2. bump version 10 | // 3. ensure index is wiped 11 | // 4. rebuild index 12 | // 5. check correctness 13 | test('version: reopening index @ same version -> no re-index', function (t) { 14 | var tmpdir = tmp.dirSync().name 15 | 16 | var files = [] 17 | var storage = function (name) { 18 | var file = raf(tmpdir + '/' + name) 19 | files.push(file) 20 | return file 21 | } 22 | 23 | var version = null 24 | var sum = 0 25 | 26 | indexV1(function () { 27 | indexV2(function () { 28 | t.end() 29 | }) 30 | }) 31 | 32 | function wipeFiles (cb) { 33 | rimraf(tmpdir, cb) 34 | } 35 | 36 | function closeFiles (cb) { 37 | var pending = 1 38 | files.forEach(function (file) { 39 | file.close(function () { 40 | if (!--pending) cb() 41 | }) 42 | }) 43 | if (!--pending) cb() 44 | } 45 | 46 | function cleanup (cb) { 47 | closeFiles(function () { 48 | wipeFiles(cb) 49 | }) 50 | } 51 | 52 | function writeData (db, cb) { 53 | var pending = 3 54 | db.writer(function (err, w) { 55 | t.error(err) 56 | w.append({value: 17}, function (err) { t.error(err); done() }) 57 | w.append({value: 12}, function (err) { t.error(err); done() }) 58 | w.append({value: 1}, function (err) { t.error(err); done() }) 59 | }) 60 | 61 | function done () { 62 | if (--pending) return 63 | cb() 64 | } 65 | } 66 | 67 | function indexV1 (cb) { 68 | var db = multifeed(storage, { valueEncoding: 'json' }) 69 | 70 | writeData(db, function () { 71 | var idx = index({ 72 | log: db, 73 | maxBatch: 10, 74 | batch: function (nodes, next) { 75 | t.equals(nodes.length, 3, 'correct batch size') 76 | nodes.forEach(function (node) { sum += node.value.value }) 77 | next() 78 | }, 79 | fetchState: function (cb) { cb(null, version) }, 80 | storeState: function (s, cb) { version = s; cb(null) }, 81 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 82 | }) 83 | 84 | idx.ready(function () { 85 | t.equal(sum, 30) 86 | closeFiles(cb) 87 | }) 88 | }) 89 | } 90 | 91 | function indexV2 (cb) { 92 | var db = multifeed(storage, { valueEncoding: 'json' }) 93 | 94 | var idx = index({ 95 | log: db, 96 | maxBatch: 10, 97 | batch: function (nodes, next) { 98 | t.fail('batch should not be called') 99 | next() 100 | }, 101 | fetchState: function (cb) { cb(null, version) }, 102 | storeState: function (s, cb) { version = s; cb(null) }, 103 | clearIndex: function (cb) { t.fail('clearIndex should not be called') } 104 | }) 105 | 106 | idx.ready(function () { 107 | t.equal(sum, 30) 108 | cleanup(cb) 109 | }) 110 | } 111 | }) 112 | 113 | test('version: reopening index @ new version -> re-index -> reopen -> no re-index', function (t) { 114 | t.plan(11) 115 | 116 | var tmpdir = tmp.dirSync().name 117 | 118 | var files = [] 119 | var storage = function (name) { 120 | var file = raf(tmpdir + '/' + name) 121 | files.push(file) 122 | return file 123 | } 124 | 125 | var version = null 126 | var sum = 0 127 | 128 | indexV1(function () { 129 | indexV2(function () { 130 | indexV3(function () { 131 | t.end() 132 | }) 133 | }) 134 | }) 135 | 136 | function wipeFiles (cb) { 137 | rimraf(tmpdir, cb) 138 | } 139 | 140 | function closeFiles (cb) { 141 | var pending = 1 142 | files.forEach(function (file) { 143 | file.close(function () { 144 | if (!--pending) cb() 145 | }) 146 | }) 147 | if (!--pending) cb() 148 | } 149 | 150 | function cleanup (cb) { 151 | closeFiles(function () { 152 | wipeFiles(cb) 153 | }) 154 | } 155 | 156 | function writeData (db, cb) { 157 | var pending = 3 158 | db.writer(function (err, w) { 159 | t.error(err) 160 | w.append({value: 17}, function (err) { t.error(err); done() }) 161 | w.append({value: 12}, function (err) { t.error(err); done() }) 162 | w.append({value: 1}, function (err) { t.error(err); done() }) 163 | }) 164 | 165 | function done () { 166 | if (--pending) return 167 | cb() 168 | } 169 | } 170 | 171 | function indexV1 (cb) { 172 | var db = multifeed(storage, { valueEncoding: 'json' }) 173 | 174 | writeData(db, function () { 175 | var idx = index({ 176 | log: db, 177 | maxBatch: 10, 178 | batch: function (nodes, next) { 179 | t.equals(nodes.length, 3, 'correct batch size') 180 | nodes.forEach(function (node) { sum += node.value.value }) 181 | next() 182 | }, 183 | fetchState: function (cb) { cb(null, version) }, 184 | storeState: function (s, cb) { version = s; cb(null) }, 185 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 186 | }) 187 | 188 | idx.ready(function () { 189 | t.equal(sum, 30) 190 | closeFiles(cb) 191 | }) 192 | }) 193 | } 194 | 195 | function indexV2 (cb) { 196 | var db = multifeed(storage, { valueEncoding: 'json' }) 197 | 198 | var batchCalls = 0 199 | var idx = index({ 200 | log: db, 201 | version: 2, 202 | maxBatch: 10, 203 | batch: function (nodes, next) { 204 | batchCalls++ 205 | t.equals(nodes.length, 3, 'correct batch size') 206 | nodes.forEach(function (node) { sum += node.value.value }) 207 | next() 208 | }, 209 | fetchState: function (cb) { cb(null, version) }, 210 | storeState: function (s, cb) { version = s; cb(null) }, 211 | clearIndex: function (cb) { version = null; sum = 0; cb(null) } 212 | }) 213 | 214 | idx.ready(function () { 215 | t.equal(batchCalls, 1) 216 | t.equal(sum, 30) 217 | cb() 218 | }) 219 | } 220 | 221 | function indexV3 (cb) { 222 | var db = multifeed(storage, { valueEncoding: 'json' }) 223 | 224 | var idx = index({ 225 | log: db, 226 | version: 2, 227 | maxBatch: 10, 228 | batch: function (nodes, next) { 229 | t.fail('batch should not be called') 230 | next() 231 | }, 232 | fetchState: function (cb) { cb(null, version) }, 233 | storeState: function (s, cb) { version = s; cb(null) }, 234 | clearIndex: function (cb) { t.fail('clearIndex should not be called') } 235 | }) 236 | 237 | idx.ready(function () { 238 | t.equal(sum, 30) 239 | cleanup(function () { 240 | t.ok('cleanup') 241 | }) 242 | }) 243 | } 244 | }) 245 | --------------------------------------------------------------------------------