├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── changesdown.js ├── encoding.js ├── example.js ├── index.js ├── package.json ├── schema.proto └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | db 3 | changes 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # changesdown 2 | 3 | levelup that uses a leveldown that writes to a changes feed to store its state 4 | 5 | ``` 6 | npm install changesdown 7 | ``` 8 | 9 | [![build status](http://img.shields.io/travis/mafintosh/changesdown.svg?style=flat)](http://travis-ci.org/mafintosh/changesdown) 10 | 11 | ## Usage 12 | 13 | ``` js 14 | var changesdown = require('changesdown') 15 | var changes = require('changes-feed') 16 | var level = require('level') 17 | 18 | var feed = changes(level('changes')) 19 | var db = changesdown(level('db'), feed) 20 | 21 | db.put('hello', 'world', function() { 22 | db.get('hello', function(err, value) { 23 | console.log(value) // should print world 24 | }) 25 | }) 26 | 27 | db.createChangesStream({live:true}) 28 | .on('data', function(data) { 29 | console.log('change:', data.value) // should print some changes 30 | }) 31 | ``` 32 | 33 | ## API 34 | 35 | #### `db = changesdown(levelup, changesFeed, [options])` 36 | 37 | Returns a new levelup (`db`) that reads and writes from the changes feed. 38 | The levelup you pass in is used to store a view of the feed. 39 | 40 | Any options passed will be forwarded to the levelup constructor. 41 | 42 | #### `stream = db.createChangesStream(opts)` 43 | 44 | Read from the changes stream and decode the changes value 45 | with the same encoding that was used in the levelup. 46 | 47 | For example if you pass `{valueEncoding: 'json'}` the values 48 | will be decoded as JSON instead of buffers 49 | 50 | ## License 51 | 52 | MIT 53 | -------------------------------------------------------------------------------- /changesdown.js: -------------------------------------------------------------------------------- 1 | var util = require('util') 2 | var through = require('through2') 3 | var subdown = require('subleveldown/leveldown') 4 | var abstract = require('abstract-leveldown') 5 | var pump = require('pump') 6 | var encoding = require('./encoding') 7 | 8 | var ChangesDOWN = function(location, changes, db) { 9 | if (!(this instanceof ChangesDOWN)) return new ChangesDOWN(location, changes, db) 10 | abstract.AbstractLevelDOWN.call(this, location) 11 | 12 | this.leveldown = subdown(db, 'd') 13 | this.meta = subdown(db, 'm') 14 | this.changes = changes 15 | this.change = 0 16 | this.cbs = {} 17 | } 18 | 19 | util.inherits(ChangesDOWN, abstract.AbstractLevelDOWN) 20 | 21 | ChangesDOWN.prototype.setDb = function() { 22 | return this.leveldown.setDb.apply(this.leveldown, arguments) 23 | } 24 | 25 | ChangesDOWN.prototype.getProperty = function() { 26 | return this.leveldown.getProperty.apply(this.leveldown, arguments) 27 | } 28 | 29 | ChangesDOWN.prototype.approximateSize = function() { 30 | this.leveldown.approximateSize.apply(this.leveldown, arguments) 31 | } 32 | 33 | ChangesDOWN.prototype.close = function() { 34 | this.leveldown.close.apply(this.leveldown, arguments) 35 | } 36 | 37 | var toBuffer = function(val) { 38 | if (Buffer.isBuffer(val)) return val 39 | if (typeof val === 'string') return new Buffer(val) 40 | return val 41 | } 42 | 43 | ChangesDOWN.prototype._open = function(options, cb) { 44 | var self = this 45 | var changes = this.changes 46 | 47 | var index = function(data, enc, cb) { 48 | self.change = data.change 49 | 50 | var value = encoding.decode(data.value) 51 | 52 | var predone = function(err) { 53 | if (err) return done(err) 54 | self.meta.put('indexed', ''+data.change, done) 55 | } 56 | 57 | var done = function(err) { 58 | var saved = self.cbs[data.change] 59 | delete self.cbs[data.change] 60 | if (saved) saved(err) 61 | cb(err) 62 | } 63 | 64 | if (value.type === 'put') return self.leveldown.put(value.key, value.value, predone) 65 | if (value.type === 'del') return self.leveldown.del(value.key, predone) 66 | if (value.type === 'batch') return self.leveldown.batch(value.batch, predone) 67 | 68 | cb() 69 | } 70 | 71 | this.meta.open(options, function() { 72 | self.leveldown.open(options, function(err) { 73 | if (err) return cb(err) 74 | self.meta.get('indexed', function(err, change) { 75 | if (!change) change = new Buffer('0') 76 | 77 | pump(changes.createReadStream({since:Number(change.toString())}), through.obj(index), function(err) { 78 | if (err) return cb(err) 79 | pump(changes.createReadStream({live:true, since:self.change}), through.obj(index)) 80 | cb() 81 | }) 82 | }) 83 | }) 84 | }) 85 | } 86 | 87 | ChangesDOWN.prototype._put = function(key, value, options, cb) { 88 | this._append(encoding.encode({ 89 | type: 'put', 90 | key: toBuffer(key), 91 | value: toBuffer(value) 92 | }), cb) 93 | } 94 | 95 | ChangesDOWN.prototype._batch = function(batch, options, cb) { 96 | batch = batch.map(function(b) { 97 | return { 98 | type: b.type, 99 | key: toBuffer(b.key), 100 | value: toBuffer(b.value) 101 | } 102 | }) 103 | 104 | this._append(encoding.encode({ 105 | type: 'batch', 106 | batch: batch 107 | }), cb) 108 | } 109 | 110 | ChangesDOWN.prototype._del = function(key, options, cb) { 111 | this._append(encoding.encode({ 112 | type: 'del', 113 | key: toBuffer(key) 114 | }), cb) 115 | } 116 | 117 | ChangesDOWN.prototype._append = function(value, cb) { 118 | var self = this 119 | this.changes.append(value, function(err, node) { 120 | if (err) return cb(err) 121 | if (self.change >= node.change) return cb() 122 | self.cbs[node.change] = cb 123 | }) 124 | } 125 | 126 | ChangesDOWN.prototype.get = function(key, options, cb) { 127 | this.leveldown.get.apply(this.leveldown, arguments) 128 | } 129 | 130 | ChangesDOWN.prototype.iterator = function() { 131 | return this.leveldown.iterator.apply(this.leveldown, arguments) 132 | } 133 | 134 | module.exports = ChangesDOWN 135 | -------------------------------------------------------------------------------- /encoding.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var protobuf = require('protocol-buffers') 3 | 4 | module.exports = protobuf(fs.readFileSync(__dirname+'/schema.proto')).Entry 5 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var changesdown = require('changesdown') 2 | var changes = require('changes-feed') 3 | var level = require('level') 4 | 5 | var feed = changes(level('changes')) 6 | var db = changesdown(level('db'), feed) 7 | 8 | db.put('hello', 'world', function() { 9 | db.get('hello', function(err, value) { 10 | console.log(value) // should print world 11 | }) 12 | }) 13 | 14 | db.createChangesStream({live:true}) 15 | .on('data', function(data) { 16 | console.log('change:', data.value) // should print some changes 17 | }) 18 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var changesdown = require('./changesdown') 2 | var subleveldown = require('subleveldown') 3 | var through = require('through2') 4 | var pump = require('pump') 5 | var levelup = require('levelup') 6 | var encoding = require('./encoding') 7 | 8 | var decoder = function (name) { 9 | switch (name) { 10 | case 'binary': 11 | return function (val) { 12 | return val 13 | } 14 | 15 | case 'utf-8': 16 | case 'utf8': 17 | return function (val) { 18 | return val.toString() 19 | } 20 | 21 | case 'json': 22 | return function (val) { 23 | return JSON.parse(val.toString()) 24 | } 25 | } 26 | } 27 | 28 | module.exports = function(db, changes, opts) { 29 | if (!opts) opts = {} 30 | 31 | opts.db = function(location) { 32 | return changesdown(location, changes, db) 33 | } 34 | 35 | var result = levelup(db.location || 'no-location', opts) 36 | 37 | var decodeKey = decoder(opts.keyEncoding || 'utf-8') 38 | var decodeValue = decoder(opts.valueEncoding || 'binary') 39 | var decode = function (entry) { 40 | if (entry.type === 'put' || entry.type === 'del') return {type: entry.type, key: decodeKey(entry.key), value: decodeValue(entry.value)} 41 | return {type: 'batch', batch: entry.batch.map(decode)} 42 | } 43 | 44 | result.createChangesStream = function (opts) { 45 | var format = function (data, enc, cb) { 46 | data.value = decode(encoding.decode(data.value)) 47 | cb(null, data) 48 | } 49 | 50 | return pump(changes.createReadStream(opts), through.obj(format)) 51 | } 52 | 53 | return result 54 | } 55 | 56 | module.exports.encoding = encoding 57 | module.exports.encode = encoding.encode 58 | module.exports.decode = encoding.decode -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "changesdown", 3 | "version": "2.3.1", 4 | "description": "levelup that uses a leveldown that writes to a changes feed to store its state", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "tape test.js" 8 | }, 9 | "dependencies": { 10 | "abstract-leveldown": "^2.1.0", 11 | "brfs": "^1.4.0", 12 | "levelup": "^0.19.0", 13 | "protocol-buffers": "^2.4.6", 14 | "pump": "^1.0.0", 15 | "subleveldown": "^1.1.0", 16 | "through2": "^0.6.3" 17 | }, 18 | "devDependencies": { 19 | "changes-feed": "^1.0.0", 20 | "memdb": "^0.2.0", 21 | "tape": "^3.2.0" 22 | }, 23 | "browserify": { 24 | "transform": [ 25 | "brfs" 26 | ] 27 | }, 28 | "repository": { 29 | "type": "git", 30 | "url": "https://github.com/mafintosh/changesdown.git" 31 | }, 32 | "author": "Mathias Buus (@mafintosh)", 33 | "license": "MIT", 34 | "bugs": { 35 | "url": "https://github.com/mafintosh/changesdown/issues" 36 | }, 37 | "homepage": "https://github.com/mafintosh/changesdown" 38 | } 39 | -------------------------------------------------------------------------------- /schema.proto: -------------------------------------------------------------------------------- 1 | message Entry { 2 | required string type = 1; 3 | optional bytes key = 2; 4 | optional bytes value = 3; 5 | repeated Entry batch = 4; 6 | } 7 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | var memdb = require('memdb') 3 | var changesdown = require('./') 4 | var changes = require('changes-feed') 5 | 6 | tape('works', function(t) { 7 | var feed = changes(memdb()) 8 | var db = changesdown(memdb(), feed) 9 | 10 | db.put(new Buffer('hello'), new Buffer('world'), function() { 11 | db.get(new Buffer('hello'), function(err, val) { 12 | t.notOk(err, 'no err') 13 | t.same(val, new Buffer('world')) 14 | t.end() 15 | }) 16 | }) 17 | }) 18 | 19 | tape('batches', function(t) { 20 | var feed = changes(memdb()) 21 | var db = changesdown(memdb(), feed) 22 | 23 | db.batch([{ 24 | type: 'put', 25 | key: new Buffer('hello'), 26 | value: new Buffer('world') 27 | }, { 28 | type: 'put', 29 | key: new Buffer('hej'), 30 | value: new Buffer('verden') 31 | }], function() { 32 | db.get(new Buffer('hello'), function(err, val) { 33 | t.notOk(err, 'no err') 34 | t.same(val, new Buffer('world')) 35 | db.get(new Buffer('hej'), function(err, val) { 36 | t.notOk(err, 'no err') 37 | t.same(val, new Buffer('verden')) 38 | t.end() 39 | }) 40 | }) 41 | }) 42 | }) 43 | 44 | tape('can reset db view', function(t) { 45 | var feed = changes(memdb()) 46 | var db = changesdown(memdb(), feed) 47 | 48 | db.put(new Buffer('hello'), new Buffer('world'), function() { 49 | db.get(new Buffer('hello'), function(err, val) { 50 | t.notOk(err, 'no err') 51 | t.same(val, new Buffer('world')) 52 | 53 | var db = changesdown(memdb(), feed) 54 | db.get(new Buffer('hello'), function(err, val) { 55 | t.notOk(err, 'no err') 56 | t.same(val, new Buffer('world')) 57 | t.end() 58 | }) 59 | }) 60 | }) 61 | }) --------------------------------------------------------------------------------