├── .gitignore ├── .jshintrc ├── .travis.yml ├── LICENSE ├── README.md ├── level-mapped-index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "predef": [ ] 3 | , "bitwise": false 4 | , "camelcase": false 5 | , "curly": false 6 | , "eqeqeq": false 7 | , "forin": false 8 | , "immed": false 9 | , "latedef": false 10 | , "newcap": true 11 | , "noarg": true 12 | , "noempty": true 13 | , "nonew": true 14 | , "plusplus": false 15 | , "quotmark": true 16 | , "regexp": false 17 | , "undef": true 18 | , "unused": true 19 | , "strict": false 20 | , "trailing": true 21 | , "maxlen": 120 22 | , "asi": true 23 | , "boss": true 24 | , "debug": true 25 | , "eqnull": true 26 | , "esnext": true 27 | , "evil": true 28 | , "expr": true 29 | , "funcscope": false 30 | , "globalstrict": false 31 | , "iterator": false 32 | , "lastsemic": true 33 | , "laxbreak": true 34 | , "laxcomma": true 35 | , "loopfunc": true 36 | , "multistr": false 37 | , "onecase": false 38 | , "proto": false 39 | , "regexdash": false 40 | , "scripturl": true 41 | , "smarttabs": false 42 | , "shadow": false 43 | , "sub": true 44 | , "supernew": false 45 | , "validthis": true 46 | , "browser": true 47 | , "couch": false 48 | , "devel": false 49 | , "dojo": false 50 | , "mootools": false 51 | , "node": true 52 | , "nonstandard": true 53 | , "prototypejs": false 54 | , "rhino": false 55 | , "worker": true 56 | , "wsh": false 57 | , "nomen": false 58 | , "onevar": true 59 | , "passfail": false 60 | } -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.8 4 | - "0.10" 5 | branches: 6 | only: 7 | - master 8 | notifications: 9 | email: 10 | - rod@vagg.org 11 | 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013, Rod Vagg (the "Original Author") 2 | All rights reserved. 3 | 4 | MIT +no-false-attribs License 5 | 6 | Permission is hereby granted, free of charge, to any person 7 | obtaining a copy of this software and associated documentation 8 | files (the "Software"), to deal in the Software without 9 | restriction, including without limitation the rights to use, 10 | copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the 12 | Software is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice shall be 16 | included in all copies or substantial portions of the Software. 17 | 18 | Distributions of all or part of the Software intended to be used 19 | by the recipients as they would use the unmodified Software, 20 | containing modifications that substantially alter, remove, or 21 | disable functionality of the Software, outside of the documented 22 | configuration mechanisms provided by the Software, shall be 23 | modified such that the Original Author's bug reporting email 24 | addresses and urls are either replaced with the contact information 25 | of the parties responsible for the changes, or removed entirely. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 29 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 31 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 32 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 33 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 34 | OTHER DEALINGS IN THE SOFTWARE. 35 | 36 | 37 | Except where noted, this license applies to any and all software 38 | programs and associated documentation files created by the 39 | Original Author, when distributed with the Software. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mapped Index for LevelDB [![Build Status](https://secure.travis-ci.org/rvagg/node-level-mapped-index.png)](http://travis-ci.org/rvagg/node-level-mapped-index) 2 | 3 | ![LevelDB Logo](https://twimg0-a.akamaihd.net/profile_images/3360574989/92fc472928b444980408147e5e5db2fa_bigger.png) 4 | 5 | A simple and flexible indexer for LevelDB, built on [LevelUP](https://github.com/rvagg/node-levelup) and [Map Reduce](https://github.com/dominictarr/map-reduce/); allowing asynchronous index calculation. 6 | 7 | After initialising Mapped Index, your LevelUP instance will have some new methods that let you register new indexes and fetch values from them. 8 | 9 | ```js 10 | // requires levelup and level-sublevel packages 11 | const levelup = require('levelup') 12 | , mappedIndex = require('level-mapped-index') 13 | , sublevel = require('level-sublevel') 14 | 15 | levelup('/tmp/foo.db', function (err, db) { 16 | 17 | // set up our LevelUP instance 18 | db = sublevel(db) 19 | db = mappedIndex(db) 20 | 21 | // register 2 indexes: 22 | 23 | // first index is named 'id' and indexes the 'id' property 24 | // of each entry 25 | db.registerIndex('id', function (key, value, emit) { 26 | value = JSON.parse(value) 27 | // if the value has a property 'id', register this entry 28 | // by calling emit() with just the indexable value 29 | if (value.id) emit(value.id) 30 | }) 31 | 32 | // second index is named 'bleh' and indexes the 'bleh' property 33 | db.registerIndex('bleh', function (key, value, emit) { 34 | value = JSON.parse(value) 35 | // in this case we're just going to index any entries that have a 36 | // 'boom' property equal to 'bam!' 37 | if (value.boom == 'bam!') emit(String(value.boom)) 38 | }) 39 | 40 | // ... use the database 41 | }) 42 | ``` 43 | 44 | In this example we're using the `registerIndex()` method to register two indexes. You must supply an index name (String) and a function that will parse and register individual entries for this index. Your function receives the key and the value of the entry and an `emit()` function. You call `emit()` with a single argument, the property for this entry that you are indexing on. The `emit()` function *does not need to be called* for each entry, only entries relevant to your index. 45 | 46 | ~~Note that the register method has the signature: registerIndex([ mapDb, ] indexName, indexFn). So you can provide your own custom *sublevel* or even a totally separate LevelUP instance to store the indexing data if that suits your needs (perhaps you're a little OCD about polluting your main store with map-reduce & index cruft?)~~ 47 | 48 | Now we put some values into our database: 49 | 50 | ```js 51 | db.put('foo1', JSON.stringify({ one : 'ONE' , id : '1' })) 52 | db.put('foo2', JSON.stringify({ two : 'TWO' , id : '2' , boom: 'bam!' })) 53 | db.put('foo3', JSON.stringify({ three : 'THREE' , id : '3' , boom: 'bam!' })) 54 | db.put('foo4', JSON.stringify({ four : 'FOUR' , id : '4' , boom : 'fizzle...' })) 55 | ``` 56 | 57 | *Map Reduce* processes these entries and passes them each to our index functions that we registered earlier. Our index references are stored in the same database, namespaced, so that they can be efficiently retrieved when required: 58 | 59 | ```js 60 | db.getBy('id', '1', function (err, data) { 61 | // `data` will equal: 62 | // [{ key: 'foo1', value: '{"one":"ONE","key":"1"}' }] 63 | }) 64 | 65 | db.getBy('bleh', 'bam!', function (err, data) { 66 | // `data` will equal: 67 | // [ 68 | // { key: 'foo2', value: '{"two":"TWO","key":"2","boom":"bam!"}' } 69 | // , { key: 'foo3', value: '{"three":"THREE","key":"3","boom":"bam!"}' } 70 | // ] 71 | }) 72 | ``` 73 | 74 | Our LevelUP instance has been augmented with a `getBy()` method that takes 3 arguments: the index name, the value on that index we are looking for and a callback function. Our callback will receive two arguments, an error and an array of objects containing `'key'` and `'value'` properties for each indexed entry. You will receive empty arrays where your indexed value finds no corresponding entries. 75 | 76 | It is **important to note** that your entries are not stored in duplicate, only the primary keys are stored for each index entry so an additional look-up is required to fetch each complete entry. 77 | 78 | You can also ask for a stream of your indexed entries in a similar manner: 79 | 80 | ```js 81 | db.createIndexedStream('id', '1') 82 | .on('data', function (data) { 83 | // this will be called once, and data will equal: 84 | // { key: 'foo1', value: '{"one":"ONE","key":"1"}' } 85 | }) 86 | .on('error', function () { 87 | // ... 88 | }) 89 | .on('end', function () { 90 | // ... 91 | }) 92 | 93 | db.createIndexedStream('bleh', 'bam!') 94 | .on('data', function (data) { 95 | // this will be called twice, and data will equal: 96 | // { key: 'foo2', value: '{"two":"TWO","key":"2","boom":"bam!"}' } 97 | // { key: 'foo3', value: '{"three":"THREE","key":"3","boom":"bam!"}' } 98 | }) 99 | .on('error', function () { 100 | // ... 101 | }) 102 | .on('end', function () { 103 | // ... 104 | }) 105 | ``` 106 | 107 | Of course this method is preferable if you are likely to have a large number of entries for each index value, otherwise `getBy()` will buffer each entry before returning them to you on the callback. 108 | 109 | ## Licence 110 | 111 | level-mapped-index is Copyright (c) 2013 Rod Vagg [@rvagg](https://twitter.com/rvagg) and licensed under the MIT licence. All rights not explicitly granted in the MIT license are reserved. See the included LICENSE file for more details. 112 | -------------------------------------------------------------------------------- /level-mapped-index.js: -------------------------------------------------------------------------------- 1 | const mapReduce = require('map-reduce') 2 | , xtend = require('xtend') 3 | , through2 = require('through2') 4 | , bytewise = require('bytewise') 5 | 6 | var mapReducePrefix = 'mi/' 7 | 8 | function register (db, mapDb, indexName, indexer) { 9 | if (typeof indexName == 'function') { 10 | indexer = indexName 11 | 12 | if (typeof mapDb == 'string') { 13 | indexName = mapDb 14 | mapDb = mapReducePrefix + mapDb 15 | } else 16 | indexName = mapDb._prefix 17 | } 18 | 19 | function emit (id, value, _emit) { 20 | indexer(id, value, function (value) { 21 | _emit(value, id) 22 | }) 23 | } 24 | 25 | var mapper = mapReduce(db, mapDb, emit) 26 | db._mappedIndexes[indexName] = typeof mapDb == 'string' ? mapper : mapDb 27 | 28 | return db 29 | } 30 | 31 | function indexedStream (db, indexName, key, options) { 32 | if (!db._mappedIndexes[indexName]) 33 | throw new Error('No such index: ' + indexName) 34 | 35 | if (!options) 36 | options = {} 37 | 38 | var start = encode(key) 39 | // strip 00 (end of array) 40 | var end = start.substring(0, start.length - 2) + '~' 41 | 42 | if (options.substringMatch) { 43 | // strip 0000 (end of string + end of array) 44 | end = start.substring(0, start.length - 4) + '~' 45 | } 46 | 47 | options = xtend(options || {}, { 48 | start: start, 49 | end: end 50 | }) 51 | 52 | var stream = db._mappedIndexes[indexName] 53 | .createReadStream(options) 54 | .pipe(through2({ objectMode: true }, function (data, enc, callback) { 55 | db.get(data.value, function (err, value) { 56 | if (err) 57 | return callback(err) 58 | callback(null, { key: data.value, value: value }) 59 | }) 60 | })) 61 | 62 | stream.on('end', function () { 63 | process.nextTick(stream.emit.bind(stream, 'close')) 64 | }) 65 | 66 | return stream 67 | } 68 | 69 | function getBy (db, indexName, key, options, callback) { 70 | var data = [] 71 | if (typeof options === 'function') { 72 | callback = options 73 | options = {} 74 | } 75 | db.createIndexedStream(indexName, key, options) 76 | .on('data', function (_data) { 77 | data.push(_data) 78 | }) 79 | .on('error', function (err) { 80 | callback(err) 81 | callback = null 82 | }) 83 | .on('close', function () { 84 | callback && callback(null, data) 85 | }) 86 | } 87 | 88 | function setup (db, opts) { 89 | if (db._mappedIndexes) return 90 | 91 | db._mappedIndexes = {} 92 | db.registerIndex = register.bind(null, db) 93 | db.createIndexedStream = indexedStream.bind(null, db) 94 | db.getBy = getBy.bind(null, db) 95 | 96 | opts = opts || {} 97 | if (opts.mapReducePrefix) { 98 | mapReducePrefix = opts.mapReducePrefix 99 | } 100 | 101 | return db 102 | } 103 | 104 | // this is the key encoding scheme used by map-reduce 6.0 105 | function encode(key) { 106 | if(!Array.isArray(key)) { 107 | key = [String(key)] 108 | } 109 | return bytewise.encode([2].concat(key)).toString('hex') 110 | } 111 | 112 | module.exports = setup -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "level-mapped-index", 3 | "description": "Simple indexing for LevelUP", 4 | "version": "0.5.3", 5 | "homepage": "https://github.com/rvagg/node-level-mapped-index", 6 | "authors": [ 7 | "Rod Vagg (https://github.com/rvagg)" 8 | ], 9 | "keywords": [ 10 | "leveldb", 11 | "levelup", 12 | "index", 13 | "indexing" 14 | ], 15 | "main": "./level-mapped-index.js", 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/rvagg/node-level-mapped-index.git" 19 | }, 20 | "dependencies": { 21 | "bytewise": "~0.6.1", 22 | "map-reduce": "~6.0.0", 23 | "through2": "~0.2.1", 24 | "xtend": "~2.1.1" 25 | }, 26 | "peerDependencies": {}, 27 | "devDependencies": { 28 | "tap": "*", 29 | "level": "*", 30 | "level-sublevel": "*", 31 | "rimraf": "*", 32 | "after": "*", 33 | "delayed": "*" 34 | }, 35 | "scripts": { 36 | "test": "./node_modules/.bin/tap ./test.js" 37 | }, 38 | "license": "MIT" 39 | } 40 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | , levelup = require('level') 3 | , sublevel = require('level-sublevel') 4 | , rimraf = require('rimraf') 5 | , after = require('after') 6 | , delayed = require('delayed').delayed 7 | , mappedIndex = require('./') 8 | 9 | function writeTestData (db, cb) { 10 | db.put('foo1', JSON.stringify({'one':'ONE','key':'1'}), cb) 11 | db.put('foo2', JSON.stringify({'two':'TWO','key':'2','bleh':true}), cb) 12 | db.put('foo3', JSON.stringify({'three':'THREE','key':'3','bleh':true}), cb) 13 | db.put('foo14', JSON.stringify({'fourteen':'FOURTEEN','key':'14'}), cb) 14 | } 15 | 16 | function verifyGetBy (t, db, cb) { 17 | db.getBy('key', '1', function (err, data) { 18 | t.notOk(err, 'no error') 19 | 20 | t.deepEqual(data, [{ 21 | key : 'foo1' 22 | , value : '{"one":"ONE","key":"1"}' 23 | }], 'correct values') 24 | 25 | cb() 26 | }) 27 | 28 | db.getBy('bleh', 'true', function (err, data) { 29 | t.notOk(err, 'no error') 30 | 31 | t.deepEqual(data, [ 32 | { key: 'foo2', value: '{"two":"TWO","key":"2","bleh":true}' } 33 | , { key: 'foo3', value: '{"three":"THREE","key":"3","bleh":true}' } 34 | ], 'correct values') 35 | 36 | cb() 37 | }) 38 | } 39 | 40 | function verifyStream (t, db, cb) { 41 | var i = 0, j = 0 42 | 43 | db.createIndexedStream('key', '1') 44 | .on('data', function (data) { 45 | t.ok(i++ === 0, 'first and only entry') 46 | t.equal(data.key, 'foo1', 'correct key') 47 | t.equal(data.value, '{"one":"ONE","key":"1"}', 'correct value') 48 | }) 49 | .on('error', function (err) { 50 | t.notOk(err, 'got error from stream') 51 | }) 52 | .on('end', function () { 53 | i = -1 54 | }) 55 | .on('close', function () { 56 | t.equal(i, -1, '"end" was emitted') 57 | cb() 58 | }) 59 | 60 | db.createIndexedStream('bleh', 'true') 61 | .on('data', function (data) { 62 | t.ok(j++ < 2, 'only two entries') 63 | t.equal(data.key, j == 1 ? 'foo2' : 'foo3', 'correct key') 64 | t.equal( 65 | data.value 66 | , j == 1 67 | ? '{"two":"TWO","key":"2","bleh":true}' 68 | : '{"three":"THREE","key":"3","bleh":true}' 69 | , 'correct value' 70 | ) 71 | }) 72 | .on('error', function (err) { 73 | t.notOk(err, 'got error from stream') 74 | }) 75 | .on('end', function () { 76 | j = -1 77 | }) 78 | .on('close', function () { 79 | t.equal(j, -1, '"end" was emitted') 80 | cb() 81 | }) 82 | } 83 | 84 | test('test simple index', function (t) { 85 | var location = '__mapped-index-' + Math.random() 86 | levelup(location, function (err, db) { 87 | t.notOk(err, 'no error') 88 | 89 | db = sublevel(db) 90 | db = mappedIndex(db, { 91 | mapReducePrefix: 'by!' 92 | }) 93 | 94 | var end = after(4, function () { 95 | rimraf(location, t.end.bind(t)) 96 | }) 97 | 98 | , cb = after(4, delayed(function (err) { 99 | t.notOk(err, 'no error') 100 | verifyGetBy(t, db, end) 101 | verifyStream(t, db, end) 102 | }, 0.1)) 103 | 104 | db.registerIndex('key', function (id, value, emit) { 105 | value = JSON.parse(value) 106 | if (value.key) 107 | emit(value.key) 108 | }) 109 | 110 | db.registerIndex('bleh', function (id, value, emit) { 111 | value = JSON.parse(value) 112 | if (value.bleh) 113 | emit(String(value.bleh)) 114 | }) 115 | 116 | writeTestData(db, cb) 117 | }) 118 | }) 119 | 120 | test('test index with sublevel mapDb', function (t) { 121 | var location = '__mapped-index-' + Math.random() 122 | levelup(location, function (err, db) { 123 | t.notOk(err, 'no error') 124 | 125 | db = sublevel(db) 126 | db = mappedIndex(db) 127 | var idxdb = db.sublevel('indexsublevel') 128 | , end = after(4, function () { 129 | rimraf(location, t.end.bind(t)) 130 | }) 131 | 132 | , cb = after(4, delayed(function (err) { 133 | t.notOk(err, 'no error') 134 | verifyGetBy(t, db, end) 135 | verifyStream(t, db, end) 136 | }, 0.1)) 137 | 138 | // this index is to be stored in a sublevel 139 | db.registerIndex(idxdb, 'key', function (id, value, emit) { 140 | value = JSON.parse(value) 141 | if (value.key) 142 | emit(value.key) 143 | }) 144 | 145 | db.registerIndex('bleh', function (id, value, emit) { 146 | value = JSON.parse(value) 147 | if (value.bleh) 148 | emit(String(value.bleh)) 149 | }) 150 | 151 | writeTestData(db, cb) 152 | }) 153 | }) 154 | 155 | test('test index with sublevel mapDb, no name', function (t) { 156 | var location = '__mapped-index-' + Math.random() 157 | levelup(location, function (err, db) { 158 | t.notOk(err, 'no error') 159 | 160 | db = sublevel(db) 161 | db = mappedIndex(db) 162 | var idxdb = db.sublevel('key') // index name comes from sublevel 163 | , end = after(4, function () { 164 | rimraf(location, t.end.bind(t)) 165 | }) 166 | 167 | , cb = after(4, delayed(function (err) { 168 | t.notOk(err, 'no error') 169 | verifyGetBy(t, db, end) 170 | verifyStream(t, db, end) 171 | }, 0.1)) 172 | 173 | // this index is to be stored in a sublevel, no index name provided 174 | db.registerIndex(idxdb, function (id, value, emit) { 175 | value = JSON.parse(value) 176 | if (value.key) 177 | emit(value.key) 178 | }) 179 | 180 | db.registerIndex('bleh', function (id, value, emit) { 181 | value = JSON.parse(value) 182 | if (value.bleh) 183 | emit(String(value.bleh)) 184 | }) 185 | 186 | writeTestData(db, cb) 187 | }) 188 | }) 189 | 190 | // TODO: level-hooks doesn't allow this anymore 191 | // 192 | // test('test index with separate mapDb', function (t) { 193 | // var location1 = '__mapped-index-' + Math.random() 194 | // , location2 = '__mapped-index-' + Math.random() 195 | // levelup(location1, function (err, db) { 196 | // t.notOk(err, 'no error') 197 | // levelup(location2, function (err, idxdb) { 198 | // t.notOk(err, 'no error') 199 | 200 | // db = sublevel(db) 201 | // idxdb = sublevel(idxdb) 202 | // db = mappedIndex(db) 203 | // var end = after(4, function () { 204 | // rimraf(location1, function () { 205 | // rimraf(location2, t.end.bind(t)) 206 | // }) 207 | // }) 208 | 209 | // , cb = after(4, delayed(function (err) { 210 | // t.notOk(err, 'no error') 211 | // verifyGetBy(t, db, end) 212 | // verifyStream(t, db, end) 213 | // }, 0.1)) 214 | 215 | // // this index is to be stored in a separate db 216 | // db.registerIndex(idxdb, 'key', function (id, value, emit) { 217 | // value = JSON.parse(value) 218 | // if (value.key) 219 | // emit(value.key) 220 | // }) 221 | 222 | // db.registerIndex('bleh', function (id, value, emit) { 223 | // value = JSON.parse(value) 224 | // if (value.bleh) 225 | // emit(String(value.bleh)) 226 | // }) 227 | 228 | // writeTestData(db, cb) 229 | // }) 230 | // }) 231 | // }) --------------------------------------------------------------------------------