├── .gitignore ├── .travis.yml ├── test ├── error.js ├── fixtures │ ├── sum.js │ └── insert.js ├── intercept.js ├── order.js ├── unsafe.js ├── prehook.js └── prehook2.js ├── package.json ├── LICENSE ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/* 3 | npm_debug.log 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.6 4 | - 0.8 5 | -------------------------------------------------------------------------------- /test/error.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | 3 | var Hooks = require('../') 4 | 5 | var assert = require('assert') 6 | var mac = require('macgyver')().autoValidate() 7 | 8 | var db = level('map-reduce-prehook-test') 9 | 10 | Hooks(db) 11 | 12 | db.hooks.pre({min: 'a', max:'z'}, function (ch, add) { 13 | console.log(ch) 14 | add(ch) //this should cause an error 15 | }) 16 | 17 | db.put('c', 'whatever', mac(function (err) { 18 | 19 | console.log('expect error:', err) 20 | assert.ok(err) 21 | }).once()) 22 | 23 | 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "level-hooks", 3 | "description": "pre/post hooks for leveldb", 4 | "version": "4.5.0", 5 | "homepage": "https://github.com/dominictarr/level-hooks", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/dominictarr/level-hooks.git" 9 | }, 10 | "author": "Dominic Tarr (http://bit.ly/dominictarr)", 11 | "dependencies": { 12 | "string-range": "~1.2" 13 | }, 14 | "devDependencies": { 15 | "rimraf": "~2.0.2", 16 | "macgyver": "~1.9", 17 | "range-bucket": "0.0.0", 18 | "level-test": "~1.4.0" 19 | }, 20 | "scripts": { 21 | "test": "set -e; for t in test/*.js; do node $t; done" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /test/fixtures/sum.js: -------------------------------------------------------------------------------- 1 | 2 | var levelup = require('levelup') 3 | var rimraf = require('rimraf') 4 | var pad = require('pad') 5 | 6 | function genSum (path, cb) { 7 | rimraf(path, function () { 8 | levelup(path, {createIfMissing: true}, function (err, db) { 9 | 10 | //install plugin system 11 | require('../../use')(db) 12 | 13 | var l = 1e3, i = 0 14 | var stream = db.writeStream() 15 | while(l--) 16 | stream.write({key: pad(6, ''+ ++i, '0'), value: JSON.stringify(i)}) 17 | stream.end() 18 | if(cb) stream.on('close', function () { 19 | cb(null, db) 20 | }) 21 | }) 22 | }) 23 | } 24 | 25 | if(!module.parent) { 26 | genSum('/tmp/map-reduce-sum-test') 27 | } 28 | 29 | module.exports = genSum 30 | 31 | -------------------------------------------------------------------------------- /test/intercept.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | var hooks = require('..') 3 | 4 | var assert = require('assert') 5 | var mac = require('macgyver')().autoValidate() 6 | 7 | var db = level('map-reduce-intercept-test') 8 | 9 | hooks(db) 10 | var _batch = [] 11 | //hook keys that start with a word character 12 | db.hooks.pre(/^\w/, mac(function (ch, add) { 13 | 14 | _batch.push(ch) 15 | var a 16 | add(a = {key: '~h', value: 'hello', type: 'put'}) 17 | _batch.push(a) 18 | }).atLeast(1)) 19 | 20 | //assert that it really became a batch 21 | db.on('batch', mac(function (batch) { 22 | console.log('batch', _batch) 23 | assert.deepEqual(_batch, batch.map(function (e) { 24 | return {key: ''+e.key, value: ''+ e.value, type: e.type} 25 | })) 26 | }).once()) 27 | 28 | 29 | db.put('hello' , 'whatever' , mac(function (){ 30 | 31 | }).once()) 32 | 33 | 34 | -------------------------------------------------------------------------------- /test/order.js: -------------------------------------------------------------------------------- 1 | 2 | var assert = require('assert') 3 | var alphabet = '$&[{}(=*)+]!#%7531902468`"\'?^/@\|-_abcdefghijklmnopqrstuvwxyz,.:;<>\\~' 4 | console.log(alphabet.split('').sort().join('')) 5 | function randomLetter(n) { 6 | var a = alphabet[~~(Math.random()*26)] 7 | return (n ? a + randomLetter(n - 1) : a).toUpperCase() 8 | } 9 | 10 | var sep = ',' 11 | 12 | function toKey(g) { 13 | return g.map(function (e) { 14 | return encodeURIComponent(e) 15 | }).join(sep) 16 | } 17 | 18 | function fromKey (a) { 19 | return a.split(sep).map(decodeURIComponent) 20 | } 21 | 22 | var groups = [] 23 | 24 | function gen (a) { 25 | var l = 3 26 | a = a || [] 27 | if(a.length > 3) return 28 | 29 | while(l --) { 30 | var _a = a.slice() 31 | _a.push(randomLetter(3)) 32 | gen(_a) 33 | _a.unshift(_a.length) 34 | groups.push(toKey(_a)) 35 | } 36 | } 37 | 38 | gen() 39 | 40 | -------------------------------------------------------------------------------- /test/unsafe.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | 3 | var Hooks = require('../') 4 | 5 | var assert = require('assert') 6 | var mac = require('macgyver')().autoValidate() 7 | 8 | var db = level('map-reduce-prehook-test') 9 | 10 | Hooks(db) 11 | 12 | //safe: false means do not prevent me from inserting into the same range. 13 | //when this option is set, the user's hook is responsible for not 14 | //causing a stack overflow. 15 | db.hooks.pre({min: 'a', max:'z', safe: false}, function (ch, add) { 16 | console.log(ch) 17 | //this is an absurd example 18 | if(ch.key !== 'p') 19 | add({key: 'p', value: ch.key, type: 'put'}) //this should cause an error 20 | }) 21 | 22 | db.put('c', 'whatever', mac(function (err) { 23 | assert.ifError(err) 24 | db.get('p', mac(function (err, c) { 25 | assert.ifError(err) 26 | assert.equal(c, 'c') 27 | }).once()) 28 | }).once()) 29 | 30 | 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 'Dominic Tarr' 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /test/fixtures/insert.js: -------------------------------------------------------------------------------- 1 | 2 | var levelup = require('levelup') 3 | var rimraf = require('rimraf') 4 | 5 | // if(!module.parent) { 6 | 7 | // var dir = '/tmp/map-reduce-sum-test' 8 | 9 | // rimraf(dir, function () { 10 | 11 | // levelup(dir, {createIfMissing: true}, function (err, db) { 12 | 13 | // var l = 10e3, i = 1 14 | // var stream = db.writeStream() 15 | // while(l--) 16 | // stream.write({key: JSON.stringify(i++), value: JSON.stringify(i)}) 17 | // }) 18 | 19 | // }) 20 | 21 | // } 22 | 23 | module.exports = sum 24 | 25 | function sum(db, list, callback) { 26 | rimraf(db, function (err) { 27 | if (err) { 28 | return callback(err) 29 | } 30 | 31 | levelup(db, { createIfMissing: true }, function (err, db) { 32 | if (err) { 33 | return callback(err) 34 | } 35 | 36 | var stream = db.writeStream() 37 | 38 | list.forEach(function (item) { 39 | stream.write(item) 40 | }) 41 | 42 | stream.end() 43 | 44 | stream.on("close", function () { 45 | callback(null) 46 | }) 47 | }) 48 | }) 49 | } 50 | 51 | -------------------------------------------------------------------------------- /test/prehook.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | 3 | var Hooks = require('../') 4 | 5 | var assert = require('assert') 6 | var mac = require('macgyver')().autoValidate() 7 | 8 | var db = level('map-reduce-prehook-test') 9 | 10 | var SEQ = 0 11 | 12 | Hooks(db) 13 | 14 | db.hooks.pre(/^\w/, mac(function (ch, add, batch) { 15 | //iterate backwards so you can push without breaking stuff. 16 | 17 | assert.ok(Array.isArray(batch)) 18 | assert.notEqual(batch.indexOf(ch), -1) 19 | var key = ch.key 20 | add({ 21 | type: 'put', 22 | key: new Buffer('~log~'+ ++SEQ), 23 | value: new Buffer(JSON.stringify({ 24 | type: ch.type, 25 | key: key.toString(), 26 | time: Date.now() 27 | })) 28 | }) 29 | add({type: 'put', key: new Buffer('~seq'), value: new Buffer(SEQ.toString())}) 30 | 31 | }).atLeast(1)) 32 | 33 | var n = 3 34 | 35 | var next = mac(function () { 36 | console.log('test', n) 37 | if(--n) return 38 | 39 | db.get('~seq', mac(function (err, val) { 40 | console.log('seq=', ''+val) 41 | assert.equal(Number(''+val), 3) 42 | db.createReadStream({start: '~log~', end: '~log~~'}) 43 | .on('data', function (data) { 44 | console.log(data.key.toString(), data.value.toString()) 45 | }) 46 | }).once()) 47 | }).times(3) 48 | 49 | db.put('hello' , 'whatever' , next) 50 | db.put('hi' , 'message' , next) 51 | db.put('yoohoo', 'test 1, 2', next) 52 | 53 | -------------------------------------------------------------------------------- /test/prehook2.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | 3 | var Hooks = require('../') 4 | 5 | var assert = require('assert') 6 | var mac = require('macgyver')().autoValidate() 7 | 8 | var db = level('map-reduce-prehook-test') 9 | 10 | var SEQ = 0, LOGSEQ = 0 11 | 12 | Hooks(db) 13 | 14 | db.hooks.pre(/^\w/, mac(function (ch, add) { 15 | //iterate backwards so you can push without breaking stuff. 16 | var key = ch.key 17 | add({ 18 | type: 'put', 19 | key: ++SEQ, 20 | value: key.toString() 21 | }, '~log~') 22 | 23 | add({ 24 | type: 'put', key: new Buffer('~seq'), 25 | value: new Buffer(SEQ.toString()) 26 | }) 27 | 28 | }).atLeast(1)) 29 | 30 | var removeLogHook = db.hooks.pre('~log', mac(function (ch, add) { 31 | //iterate backwards so you can push without breaking stuff. 32 | console.log('LOG2', ch) 33 | var key = ch.key 34 | add({ 35 | type: 'put', 36 | key: ++LOGSEQ, 37 | value: Date.now(), 38 | prefix: '~LOGSEQ~' 39 | }) 40 | 41 | }).atLeast(1)) 42 | 43 | 44 | var n = 4 45 | 46 | var next = mac(function () { 47 | console.log('test', n) 48 | if(--n) return 49 | 50 | db.get('~seq', mac(function (err, val) { 51 | console.log('seq=', ''+val) 52 | assert.equal(Number(''+val), 4) 53 | db.readStream({start: '~log~', end: '~log~~'}) 54 | .on('data', function (data) { 55 | console.log(data.key.toString(), data.value.toString()) 56 | }) 57 | }).once()) 58 | 59 | var all = {} 60 | 61 | db.readStream() 62 | .on('data', function (data) { 63 | all[data.key.toString()] = data.value.toString() 64 | }) 65 | .on('end', function () { 66 | console.log(all) 67 | 68 | //these will be times, and will have changed. 69 | delete all['~LOGSEQ~1'] 70 | delete all['~LOGSEQ~2'] 71 | delete all['~LOGSEQ~3'] 72 | 73 | assert.deepEqual(all, { 74 | hello: 'whatever', 75 | hi: 'message', 76 | thing: 'WHATEVER', 77 | yoohoo: 'test 1, 2', 78 | '~log~1': 'hello', 79 | '~log~2': 'hi', 80 | '~log~3': 'yoohoo', 81 | '~log~4': 'thing', 82 | '~seq': '4' 83 | }) 84 | 85 | }) 86 | 87 | }).times(4) 88 | 89 | db.put('hello' , 'whatever' , next) 90 | db.put('hi' , 'message' , next) 91 | db.put('yoohoo', 'test 1, 2', next) 92 | 93 | removeLogHook() 94 | 95 | db.put('thing' , 'WHATEVER' , next) 96 | 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pre/Post hooks for leveldb 2 | 3 | Intercept put/delete/batch operations on levelup. 4 | 5 | ## Warning - Breaking Changes 6 | 7 | The API for implementing pre hooks has changed. 8 | Instead of mutating an array at once, the prehook 9 | is called on each change `hook(change, add)` 10 | and may call `add(_change)` to add a new item into the batch. 11 | 12 | Also, attaching hooks to leveldb is now simpler 13 | ``` js 14 | var Hooks = require('level-hooks') 15 | Hooks(db) //previously: Hooks()(db) 16 | ``` 17 | 18 | ## Example 19 | 20 | ``` js 21 | var levelup = require('levelup') 22 | var timestamp = require('monotonic-timestamp') 23 | var hooks = require('level-hooks') 24 | 25 | levelup(file, {createIfMissing: true}, function (err, db) { 26 | 27 | //install hooks onto db. 28 | hooks(db) 29 | 30 | db.hooks.pre({start: '', end: '~'}, function (change, add) { 31 | //change is same pattern as the an element in the batch array. 32 | //add a log to record every put operation. 33 | add({type: 'put', key: '~log-'+timestamp()+'-'+change.type, value: change.key}) 34 | }) 35 | 36 | //add a hook that responds after an operation has completed. 37 | db.hooks.post(function (ch) { 38 | //{type: 'put'|'del', key: ..., value: ...} 39 | }) 40 | 41 | }) 42 | ``` 43 | 44 | Used by [map-reduce](https://github.com/dominictarr/map-reduce) 45 | to make map-reduce durable across crashes! 46 | 47 | ## API 48 | 49 | ### rm = db.hooks.pre (range?, hook(change, add(op), batch)) 50 | 51 | If `prefix` is a `string` or `object` that defines the range the pre-hook triggers on. 52 | If `prefix` is a string, then the hook only triggers on keys that _start_ with that 53 | string. If the hook is an object it must be of form `{start: START, end: END}` 54 | 55 | `hook` is a function, and will be called on each item in the batch 56 | (if it was a `put` or `del`, it will be called on the change) 57 | `op` is always of the form `{key: key, value: value, type:'put' | 'del'}` 58 | 59 | Pass additional changes to `add` to add them to the batch. 60 | If add is passed a string as the second argument it will prepend that prefix 61 | to any keys you add. 62 | 63 | You can check what opperations are currently in the batch with the third argument. 64 | Do not modify the `batch` directly, instead use `add` 65 | 66 | To veto (remove) the current change call `add(false)`. 67 | 68 | `db.hooks.pre` returns a function that will remove the hook when called. 69 | 70 | #### unsafe mode 71 | 72 | normally, pre hooks prevent you from inserting into the hooked range 73 | when the hook is triggered. However, sometimes you do need to do this. 74 | In those cases, pass in a range with `{start: START, end: END, safe: false}` 75 | and level-hooks will not error. If you use this option, your hook must 76 | avoid triggering in a loop itself. 77 | 78 | ### rm = db.hooks.post (range?, hook) 79 | 80 | Post hooks do not offer any chance to change the value. 81 | but do take a range option, just like `pre` 82 | 83 | `db.hooks.post` returns a function that will remove the hook when called. 84 | 85 | 86 | ## License 87 | 88 | MIT 89 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var ranges = require('string-range') 2 | 3 | module.exports = function (db) { 4 | 5 | if(db.hooks) { 6 | return 7 | } 8 | 9 | var posthooks = [] 10 | var prehooks = [] 11 | 12 | function getPrefix (p) { 13 | return p && ( 14 | 'string' === typeof p ? p 15 | : 'string' === typeof p.prefix ? p.prefix 16 | : 'function' === typeof p.prefix ? p.prefix() 17 | : '' 18 | ) 19 | } 20 | 21 | function getKeyEncoding (db) { 22 | if(db && db._getKeyEncoding) 23 | return db._getKeyEncoding(db) 24 | } 25 | 26 | function getValueEncoding (db) { 27 | if(db && db._getValueEncoding) 28 | return db._getValueEncoding(db) 29 | } 30 | 31 | function remover (array, item) { 32 | return function () { 33 | var i = array.indexOf(item) 34 | if(!~i) return false 35 | array.splice(i, 1) 36 | return true 37 | } 38 | } 39 | 40 | db.hooks = { 41 | post: function (prefix, hook) { 42 | if(!hook) hook = prefix, prefix = '' 43 | var h = {test: ranges.checker(prefix), hook: hook} 44 | posthooks.push(h) 45 | return remover(posthooks, h) 46 | }, 47 | pre: function (prefix, hook) { 48 | if(!hook) hook = prefix, prefix = '' 49 | var h = { 50 | test: ranges.checker(prefix), 51 | hook: hook, 52 | safe: false !== prefix.safe 53 | } 54 | prehooks.push(h) 55 | return remover(prehooks, h) 56 | }, 57 | posthooks: posthooks, 58 | prehooks: prehooks 59 | } 60 | 61 | //POST HOOKS 62 | 63 | function each (e) { 64 | if(e && e.type) { 65 | posthooks.forEach(function (h) { 66 | if(h.test(e.key)) h.hook(e) 67 | }) 68 | } 69 | } 70 | 71 | db.on('put', function (key, val) { 72 | each({type: 'put', key: key, value: val}) 73 | }) 74 | db.on('del', function (key, val) { 75 | each({type: 'del', key: key, value: val}) 76 | }) 77 | db.on('batch', function onBatch (ary) { 78 | ary.forEach(each) 79 | }) 80 | 81 | //PRE HOOKS 82 | 83 | var put = db.put 84 | var del = db.del 85 | var batch = db.batch 86 | 87 | function callHooks (isBatch, b, opts, cb) { 88 | try { 89 | b.forEach(function hook(e, i) { 90 | prehooks.forEach(function (h) { 91 | if(h.test(String(e.key))) { 92 | //optimize this? 93 | //maybe faster to not create a new object each time? 94 | //have one object and expose scope to it? 95 | var context = { 96 | add: function (ch, db) { 97 | if(typeof ch === 'undefined') { 98 | return this 99 | } 100 | if(ch === false) 101 | return delete b[i] 102 | var prefix = ( 103 | getPrefix(ch.prefix) || 104 | getPrefix(db) || 105 | h.prefix || '' 106 | ) 107 | //don't leave a circular json object there incase using multilevel. 108 | if(prefix) ch.prefix = prefix 109 | ch.key = prefix + ch.key 110 | if(h.safe && h.test(String(ch.key))) { 111 | //this usually means a stack overflow. 112 | throw new Error('prehook cannot insert into own range') 113 | } 114 | var ke = ch.keyEncoding || getKeyEncoding(ch.prefix) 115 | var ve = ch.valueEncoding || getValueEncoding(ch.prefix) 116 | if(ke) ch.keyEncoding = ke 117 | if(ve) ch.valueEncoding = ve 118 | 119 | b.push(ch) 120 | hook(ch, b.length - 1) 121 | return this 122 | }, 123 | put: function (ch, db) { 124 | if('object' === typeof ch) ch.type = 'put' 125 | return this.add(ch, db) 126 | }, 127 | del: function (ch, db) { 128 | if('object' === typeof ch) ch.type = 'del' 129 | return this.add(ch, db) 130 | }, 131 | veto: function () { 132 | return this.add(false) 133 | } 134 | } 135 | h.hook.call(context, e, context.add, b) 136 | } 137 | }) 138 | }) 139 | } catch (err) { 140 | return (cb || opts)(err) 141 | } 142 | b = b.filter(function (e) { 143 | return e && e.type //filter out empty items 144 | }) 145 | 146 | if(b.length == 1 && !isBatch) { 147 | var change = b[0] 148 | return change.type == 'put' 149 | ? put.call(db, change.key, change.value, opts, cb) 150 | : del.call(db, change.key, opts, cb) 151 | } 152 | return batch.call(db, b, opts, cb) 153 | } 154 | 155 | db.put = function (key, value, opts, cb ) { 156 | var batch = [{key: key, value: value, type: 'put'}] 157 | return callHooks(false, batch, opts, cb) 158 | } 159 | 160 | db.del = function (key, opts, cb) { 161 | var batch = [{key: key, type: 'del'}] 162 | return callHooks(false, batch, opts, cb) 163 | } 164 | 165 | db.batch = function (batch, opts, cb) { 166 | return callHooks(true, batch, opts, cb) 167 | } 168 | } 169 | --------------------------------------------------------------------------------