├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── client.js ├── index.js ├── lib ├── buffered-opener.js ├── cache.js ├── client-opener.js ├── map.js ├── remote-open.js ├── schema.js └── stream.js ├── package.json └── test ├── crdt.js ├── expected.js ├── map-reduce.js ├── open.js ├── open2.js ├── open3.js ├── reopen.js ├── reopen2.js ├── script.js ├── simple-id.js └── view.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/* 3 | npm_debug.log 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.6 4 | - 0.8 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 'Dominic Tarr' 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # level-scuttlebutt 2 | 3 | Plugin to add persistence and querying [scuttlebutt](https://github.com/scuttlebutt) documents 4 | stored in leveldb. 5 | 6 | Instead of representing an object as a single document, scuttlebutt represents a document as 7 | a series of immutable transactions. The 'document' is modified by appending a new transaction. 8 | Old transactions that are no longer relevant can be cleaned up, but you can never modify a 9 | transaction in place. As it turns out, leveldb (a log-structured merge tree) is optimized for 10 | exactly this sort of data. 11 | 12 | Must be used with [level-sublevel](https://github.com/dominictarr/level-sublevel) 13 | 14 | # Example 15 | 16 | ``` js 17 | var levelup = require("levelup") 18 | var level_scuttlebutt = require("level-scuttlebutt") 19 | var SubLevel = require('level-sublevel') 20 | 21 | 22 | //create a leveldb instance... 23 | //levelup must be extended with SubLevel! 24 | var db = SubLevel(levelup(DB_FILE)) 25 | 26 | 27 | 28 | //a scuttlebutt model. 29 | var Model = require('scuttlebutt/model') 30 | 31 | //level-scuttlebutt needs to have an unique identifier of the current instance 32 | var udid = require('udid')('app-name') 33 | 34 | 35 | //patch it with level-scuttlebutt. 36 | var sbDb = db.sublevel('scuttlebutt') //add a scuttlebutt 'table' 37 | 38 | //k 39 | level_scuttlebutt(sbDb, udid, function (name) { 40 | //create a scuttlebutt instance given a name. 41 | //the key will match the start of the name. 42 | return new Model() 43 | //now is a good time to customize the scuttlebutt instance. 44 | }) 45 | 46 | //open a scuttlebutt instance by name. 47 | sbDb.open(name, function (err, model) { 48 | model.on('change:key', console.log) //... 49 | model.set('key', value) 50 | 51 | // when you're done get rid of it 52 | model.dispose() 53 | }) 54 | 55 | //the toJSON values are stored in the db, 56 | //so you can just use any other map reduce library on it! 57 | sbDb.views['all'] = 58 | mapReduce(sbDb, 'all', 59 | function (key, json, emit) { 60 | return emit(key.split('!'), 1) 61 | }, 62 | function (acc, item) { 63 | return '' + (Number(acc) + Number(item)) 64 | }, 65 | '0' 66 | ) 67 | 68 | ``` 69 | 70 | ## Initialization 71 | 72 | Add `level-scuttlebutt` plugin to the `db` object 73 | `var level_scuttlebutt = require('level-scuttlebutt'); level_scuttlebutt(db, ID, schema)` 74 | 75 | `ID` is a unique string that identifies the node (the machine) and should be 76 | tied to the leveldb instance. 77 | I suggest using [udid](https://github.com/dominictarr/udid). 78 | 79 | `schema` should be a function that takes a string (the name of the scuttlebutt instance) 80 | and returns and empty scuttlebutt instance. 81 | You can use [scuttlebutt-schema](https://github.com/dominictarr/scuttlebutt-schema). 82 | 83 | ## Queries 84 | 85 | Use some other `level-*` plugin for queries! 86 | 87 | [map-reduce](https://github.com/dominictarr/map-reduce), 88 | [level-map-merge](https://github.com/dominictarr/level-map-merge) 89 | 90 | ### Example 91 | 92 | get the 10 last edited documents! 93 | 94 | ``` js 95 | sbDb.views['latest10'] 96 | = 97 | MapReduce(sdb, 'latest10', 98 | function (key, json) { 99 | var name = key 100 | var obj = JSON.parse(json) 101 | //emit 0-many group-value pairs. 102 | //value must be a string or a buffer. 103 | this.emit([], JSON.stringify({name: name, time: Date.now(), length: obj.text.length})) 104 | }, 105 | //merge the latest value into the accumulator. 106 | function (acc, value) { 107 | var all = JSON.parse(acc).concat(JSON.parse(value)) 108 | //sort by time, decending. 109 | all.sort(function (a, b) { 110 | return b.time - a.time 111 | }) 112 | //top ten most recent 113 | var all = all.slice(0, 10) 114 | return JSON.stringify(all) 115 | }, 116 | //the first value for the accumulator. 117 | //since we are parsing it, it needs to be valid JSON. 118 | '[]' 119 | }) 120 | ``` 121 | 122 | 123 | 124 | ## License 125 | 126 | MIT 127 | -------------------------------------------------------------------------------- /client.js: -------------------------------------------------------------------------------- 1 | 2 | var Schema = require('./lib/schema') 3 | var Client = require('./lib/client-opener') 4 | var Buffered = require('./lib/buffered-opener') 5 | 6 | module.exports = function (schema, id) { 7 | schema = Schema(schema, id) 8 | var c = Client() 9 | var b = Buffered(schema, id) 10 | c.on('open', function () { 11 | b.swap(c) 12 | }) 13 | b.createStream = 14 | b.createRemoteStream = c.createStream 15 | return b 16 | } 17 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var duplex = require('duplex') 3 | 4 | var LiveStream = require('level-live-stream') 5 | var REDIS = require('redis-protocol-stream') 6 | 7 | var makeSchema = require('./lib/schema') 8 | var BufferedOpener 9 | = require('./lib/buffered-opener') 10 | var MakeCreateStream 11 | = require('./lib/stream') 12 | var ranges = require('map-reduce/range') 13 | 14 | //need a seperator that sorts early. 15 | //use NULL instead? 16 | 17 | module.exports = function (db, id, schema) { 18 | 19 | //none of these should be used. 20 | var sep = '\x00' 21 | var localDb = db.sublevel('sb') 22 | var replicateDb = db.sublevel('replicate') 23 | var vectorDb = db.sublevel('vector') 24 | var sources = {} 25 | 26 | if('string' !== typeof id) 27 | schema = id, id = null 28 | 29 | id = id || uuid() 30 | 31 | var match = makeSchema(schema, id) 32 | 33 | //create a new scuttlebutt attachment. 34 | //a document that is modeled as a range of keys, 35 | //rather than as a single {key: value} pair 36 | 37 | //WHY DID I DO THIS? - remove this and it works. 38 | //but it seems to be problem with r-array... 39 | function checkOld (id, ts) { 40 | return false 41 | if(sources[id] && sources[id] >= ts) return true 42 | sources[id] = ts 43 | } 44 | 45 | var _batch = [], queued = false 46 | 47 | function save() { 48 | if(!queued) { 49 | process.nextTick(function () { 50 | var batch = _batch 51 | _batch = [] 52 | db.batch(batch, function (err) { 53 | if(err) return db.emit('error', err) 54 | 55 | queued = false 56 | if(!_batch.length) db.emit('drain') 57 | else save() 58 | }) 59 | }) 60 | } 61 | queued = true 62 | } 63 | 64 | db.scuttlebutt = function () { 65 | var args = [].slice.call(arguments) 66 | return db.scuttlebutt.open.apply(null, args) 67 | } 68 | 69 | function key() { 70 | return [].slice.call(arguments).join(sep) 71 | } 72 | 73 | function insertBatch (_id, doc_id, ts, value, emitter) { 74 | ts = ts.toString() 75 | 76 | _batch.push({ 77 | key: key(doc_id, ts, _id), 78 | value: value, type: 'put', 79 | prefix: localDb 80 | }) 81 | 82 | _batch.push({ 83 | //the second time, so that documents can be rapidly replicated. 84 | key: key(_id, ts, doc_id), 85 | value: value, type: 'put', 86 | prefix: replicateDb 87 | }) 88 | 89 | _batch.push({ 90 | //also, update the vector clock for this replication range, 91 | //so that it's easy to recall what are the latest documents are. 92 | //this vector clock is for all the documents, not just this one... 93 | key: _id, value: ''+ts, type: 'put', 94 | prefix: vectorDb 95 | }) 96 | 97 | //or do this async? 98 | //YES, do this async, providing a way to wait until the given 99 | //maps have been flushed. 100 | 101 | if(emitter && emitter.toJSON) 102 | _batch.push({ 103 | key: doc_id, 104 | value: JSON.stringify(emitter.toJSON()), 105 | type: 'put', 106 | prefix: db 107 | }) 108 | 109 | save() 110 | } 111 | 112 | function deleteBatch (_id, doc_id, ts) { 113 | 114 | _batch.push({ 115 | key: key(doc_id, ts, _id), 116 | type: 'del', prefix: localDb 117 | }) 118 | 119 | _batch.push({ 120 | key: key(_id, ts, doc_id), 121 | type: 'del', prefix: replicateDb 122 | }) 123 | 124 | save() 125 | } 126 | 127 | var dbO = new EventEmitter() 128 | dbO.open = function (key, tail, callback) { 129 | if('function' === typeof tail) callback = tail, tail = true 130 | 131 | if(!key) throw new Error('must provide a doc_id') 132 | var scuttlebutt 133 | if('string' === typeof key) { 134 | scuttlebutt = match(key) 135 | scuttlebutt.name = key 136 | 137 | 'function' === scuttlebutt.setId 138 | ? scuttlebutt.setId(id) 139 | : scuttlebutt.id = id 140 | 141 | } else { 142 | scuttlebutt = key 143 | key = scuttlebutt.name 144 | } 145 | 146 | var stream = LiveStream(localDb, { 147 | start: [key, 0].join(sep), 148 | end : [key, '~'].join(sep) 149 | }) 150 | .on('data', function (data) { 151 | //ignore deletes, 152 | //deletes must be an update. 153 | if(data.type == 'del') return 154 | 155 | var ary = data.key.split(sep) 156 | var ts = Number(ary[1]) 157 | var source = ary[2] 158 | var change = JSON.parse(data.value) 159 | 160 | scuttlebutt._update([change, ts, source]) 161 | }) 162 | 163 | //this scuttlebutt instance is up to date with the db. 164 | 165 | var ready = false 166 | function onReady () { 167 | if(ready) return 168 | ready = true 169 | scuttlebutt.emit('sync') 170 | if(callback) callback(null, scuttlebutt) 171 | } 172 | 173 | stream.once('sync', onReady) 174 | stream.once('end' , onReady) 175 | 176 | scuttlebutt.once('dispose', function () { 177 | //levelup/read-stream throws if the stream has already ended 178 | //but it's just a user error, not a serious problem. 179 | try { stream.destroy() } catch (_) { } 180 | }) 181 | 182 | //write the update twice, 183 | //the first time, to store the document. 184 | //maybe change scuttlebutt so that value is always a string? 185 | //If i write a bunch of batches, will they come out in order? 186 | //because I think updates are expected in order, or it will break. 187 | 188 | function onUpdate (update) { 189 | var value = update[0], ts = update[1], id = update[2] 190 | insertBatch (id, key, ts, JSON.stringify(value), scuttlebutt) 191 | } 192 | 193 | //If the user has passed in a scuttlebutt instead of a key, 194 | //then it may have history, so save that. 195 | scuttlebutt.history().forEach(onUpdate) 196 | 197 | //then, track real-time updates 198 | scuttlebutt.on('_update', onUpdate) 199 | 200 | //an update is now no longer significant 201 | //can clean it from the database, 202 | //this isn't to save space so much as it is to 203 | //save time on future reads. 204 | scuttlebutt.on('_remove', function (update) { 205 | var ts = update[1], id = update[2] 206 | deleteBatch (id, key, ts) 207 | }) 208 | 209 | return scuttlebutt 210 | } 211 | 212 | dbO.createStream = function () { 213 | var mx = MuxDemux(function (stream) { 214 | if(!db) return stream.error('cannot access database this end') 215 | 216 | if('string' === typeof stream.meta) { 217 | var ts = through().pause() 218 | //TODO. make mux-demux pause. 219 | 220 | stream.pipe(ts) 221 | //load the scuttlebutt with the callback, 222 | //and then connect the stream to the client 223 | //so that the 'sync' event fires the right time, 224 | //and the open method works on the client too. 225 | opener.open(stream.meta, function (err, doc) { 226 | ts.pipe(doc.createStream()).pipe(stream) 227 | ts.resume() 228 | }) 229 | } else if(Array.isArray(stream.meta)) { 230 | //reduce the 10 most recently modified documents. 231 | opener.view.apply(null, stream.meta) 232 | .pipe(through(function (data) { 233 | this.queue({ 234 | key: data.key.toString(), 235 | value: data.value.toString() 236 | }) 237 | })) 238 | .pipe(stream) 239 | } 240 | }) 241 | //clean up 242 | function onClose () { mx.end() } 243 | 244 | db.once('close', onClose) 245 | mx.once('close', function () { db.removeListener('close', onClose) }) 246 | 247 | return mx 248 | } 249 | 250 | //THIS IS JUST LEGACY STUFF NOW, 251 | //TODO: rewrite the streaming/client api, 252 | //so this disappears. 253 | 254 | db.views = {} 255 | dbO.view = function (name, opts) { 256 | if(!opts) 257 | opts = name, name = opts.name 258 | if(opts.range) { 259 | var r = ranges.range(opts.range) 260 | opts.start = r.start 261 | opts.end = r.end 262 | } 263 | if(db.views[name]) 264 | return LiveStream(db.views[name], opts) 265 | throw new Error('no view named:', name) 266 | } 267 | 268 | db.on('close', function () { 269 | opener.emit('close') 270 | }) 271 | 272 | var opener = BufferedOpener(schema, id).swap(dbO) 273 | 274 | db.open = 275 | db.scuttlebutt.open = opener.open 276 | db.scuttlebutt._opener = dbO 277 | db.scuttlebutt.view = opener.view 278 | db.createRemoteStream = 279 | db.scuttlebutt.createRemoteStream = MakeCreateStream(opener) 280 | 281 | db.createReplicateStream = 282 | db.scuttlebutt.createReplicateStream = function (opts) { 283 | opts = opts || {} 284 | var yourClock, myClock 285 | var d = duplex () 286 | var outer = REDIS.serialize(d) 287 | d.on('_data', function (data) { 288 | if(data.length === 1) { 289 | //like a telephone, say 290 | if(''+data[0] === 'BYE') { 291 | d._end() 292 | } else { 293 | //data should be {id: ts} 294 | yourClock = JSON.parse(data.shift()) 295 | start() 296 | } 297 | } else { 298 | //maybe increment the clock for this node, 299 | //so that when we detect that a record has been written, 300 | //can avoid updating the model twice when recieving 301 | var id = ''+data[0] 302 | var ts = Number(''+data[1]) 303 | 304 | if(!myClock || !myClock[id] || myClock[id] < ts) { 305 | var doc_id = data[2] 306 | var value = data[3] 307 | 308 | insertBatch(id, doc_id, ts, value) 309 | myClock[id] = ts 310 | } 311 | } 312 | }) 313 | 314 | function start() { 315 | if(!(myClock && yourClock)) return 316 | 317 | var clock = {} 318 | for(var id in myClock) 319 | clock[id] = '' 320 | 321 | for(var id in yourClock) 322 | clock[id] = yourClock[id] 323 | 324 | var started = 0 325 | for(var id in clock) { 326 | (function (id) { 327 | started ++ 328 | var _opts = { 329 | start: [id, clock[id]].join(sep), 330 | end : [id, '\xff' ].join(sep), 331 | tail : opts.tail 332 | } 333 | //TODO, merge stream that efficiently handles back pressure 334 | //when reading from many streams. 335 | var stream = LiveStream(replicateDb, _opts) 336 | .on('data', function (data) { 337 | var ary = data.key.split(sep) 338 | ary.push(data.value) 339 | d._data(ary) 340 | }) 341 | .once('end', function () { 342 | if(--started) return 343 | if(opts.tail === false) d._data(['BYE']) 344 | }) 345 | 346 | d.on('close', stream.destroy.bind(stream)) 347 | 348 | })(id); 349 | } 350 | } 351 | db.scuttlebutt.vectorClock(function (err, clock) { 352 | myClock = clock 353 | d._data([JSON.stringify(clock)]) 354 | start() 355 | }) 356 | 357 | return outer 358 | } 359 | 360 | //read the vector clock. {id: ts, ...} pairs. 361 | 362 | db.vectorClock = 363 | db.scuttlebutt.vectorClock = function (cb) { 364 | var clock = {} 365 | vectorDb.createReadStream() 366 | .on('data', function (data) { 367 | clock[data.key] = Number(''+data.value) 368 | }) 369 | .on('close', function () { 370 | cb(null, clock) 371 | }) 372 | } 373 | } 374 | 375 | -------------------------------------------------------------------------------- /lib/buffered-opener.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var Cache = require('./cache') 3 | var through = require('through') 4 | 5 | module.exports = function (schema, id) { 6 | var opener = new EventEmitter() 7 | var _opener 8 | 9 | if('string' !== typeof id) 10 | throw new Error('id must be string') 11 | 12 | opener.swap = function (newOpener) { 13 | if(_opener) throw new Error('already has opener, expects opener.emit("close")') 14 | if(!newOpener) throw new Error('cannot swap null') 15 | _opener = newOpener 16 | _opener.once('close', function () { 17 | _opener = null 18 | }) 19 | ready() 20 | return opener 21 | } 22 | 23 | var toOpen = [], toView = [], _open, _view 24 | 25 | //cache at this level, not on scuttlebutt.open 26 | 27 | opener.open = Cache(schema, id, function (scuttlebutt, tail, cb) { 28 | if('object' !== typeof scuttlebutt) 29 | throw new Error('expected Scuttlebutt') 30 | 31 | if(!_opener) 32 | return toOpen.push([scuttlebutt, tail, cb]) 33 | 34 | _opener.open(scuttlebutt, tail, cb) 35 | return scuttlebutt 36 | }) 37 | 38 | var cache = opener.open.local 39 | 40 | opener.view = function () { 41 | var args = [].slice.call(arguments) 42 | if(_opener) 43 | return _opener.view.apply(null, args) 44 | var stream = through() 45 | toView.push({args: args, stream: stream}) 46 | return stream 47 | } 48 | 49 | function ready () { 50 | var opening = {} 51 | while(_opener && toOpen.length) { 52 | var args = toOpen.shift() 53 | opening[args[0].name] = true 54 | _opener.open.apply(null, args) 55 | } 56 | while(_opener && toView.length) { 57 | var v = toView.shift() 58 | _opener.view.apply(null, v.args) 59 | .on('error', function (err) { 60 | //because stream errors are not propagated... 61 | v.stream.emit('error', err) 62 | }).pipe(v.stream) 63 | } 64 | 65 | //reopen anything that was closed.. 66 | for(var key in cache) 67 | if(!opening[key]) { 68 | _opener.open(cache[key], true, function () {}) 69 | } 70 | } 71 | 72 | return opener 73 | } 74 | -------------------------------------------------------------------------------- /lib/cache.js: -------------------------------------------------------------------------------- 1 | var Schema = require('./schema') 2 | var TIMEOUT = 100 3 | 4 | module.exports = function (schema, id, open, onCache) { 5 | var local = cached.local = {} 6 | schema = Schema(schema, id) 7 | if('string' != typeof id) 8 | throw new Error('id must be string') 9 | if('function' !== typeof open) 10 | throw new Error('open must be function') 11 | function cached (name, tail, cb) { 12 | if('function' == typeof tail) 13 | cb = tail, tail = true 14 | 15 | //user *must not* pass a Scuttlebutt instance to cached open. 16 | if('string' !== typeof name) 17 | throw new Error('name must be string') 18 | 19 | //so if this is passed a ready instance... 20 | //and there is a cached instance... then pipe them together. 21 | //hmm, but the caching should be ontop of the other layer... 22 | //will this work? 23 | //ah, if a scuttlebutt is passed, and there isn't one in the cache 24 | //clone it and put the clone in the cache! 25 | //else, pipe it to the cached instace. 26 | 27 | //actually... should combine this into level-scuttlebutt 28 | //then only one cache will be necessary. 29 | 30 | //which will work better for reconnections. 31 | 32 | //...so, need lots of tests here. 33 | 34 | //yes, only one cache, the outer most open does not support 35 | //opening with a scuttlebutt - but remote, and 36 | //level-scuttlebutt do! 37 | 38 | //then there is only one cache... 39 | //so, you always control the point of caching. 40 | //no "russian doll" caching. 41 | 42 | var cached = local[name] 43 | if(cached && 'function' === typeof cached.clone) { 44 | var n = cached.clone() 45 | n.name = name 46 | cb(null, n) 47 | return n 48 | } 49 | 50 | var clone 51 | var scuttlebutt = local[name] = schema(name) 52 | if(!scuttlebutt) 53 | return cb(new Error('no constructor for:'+name)) 54 | 55 | scuttlebutt.name = name 56 | scuttlebutt.id = id 57 | open(scuttlebutt, tail, function (err, scuttlebutt) { 58 | if(err) return cb(err) 59 | cb(null, clone) 60 | }) 61 | 62 | //will callback an error 63 | if(!scuttlebutt) return 64 | 65 | scuttlebutt.name = name 66 | 67 | //only scuttlebutts with clone can be cleanly cached. 68 | if('function' === typeof scuttlebutt.clone) { 69 | local[name] = scuttlebutt 70 | clone = scuttlebutt.clone() 71 | clone._parent = scuttlebutt 72 | clone.name = name 73 | if(onCache) onCache('clone', scuttlebutt.name) 74 | //okay... have something to dispose the scuttlebutt when there are 0 streams. 75 | //hmm, count streams... and emit an event 'unstream' or something? 76 | //okay, if all the steams have closed but this one, then it means no one is using this, 77 | //so close... 78 | //TODO add this to level-scuttlebutt. 79 | 80 | //OH, hang on... maybe DOMAINS is the right thing to use here... 81 | 82 | var timer = null 83 | scuttlebutt.on('unclone', function (n) { 84 | if(n === 0) { 85 | clearTimeout(timer) 86 | timer = setTimeout(function () { 87 | if(scuttlebutt._clones) return 88 | scuttlebutt.dispose() 89 | if(onCache) onCache('uncache', scuttlebutt.name) 90 | }, TIMEOUT*1.5) 91 | //if an emitter was passed, imet 92 | } else if(n > 1) 93 | clearTimeout(timer) 94 | }) 95 | scuttlebutt.on('dispose', function () { 96 | delete local[name] 97 | }) 98 | } 99 | 100 | return scuttlebutt 101 | } 102 | 103 | return cached 104 | } 105 | 106 | 107 | -------------------------------------------------------------------------------- /lib/client-opener.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var MuxDemux = require('mux-demux') 3 | var remoteOpen = function (connect) { 4 | //pass in a string name, or a scuttlebutt instance 5 | //you want to reconnect to the server. 6 | return function (scuttlebutt, tail, cb) { 7 | if('function' == typeof tail) 8 | cb = tail, tail = true 9 | 10 | if('object' !== typeof scuttlebutt) 11 | throw new Error('expect Scuttlebutt, not ' + typeof scuttlebutt) 12 | 13 | var es = scuttlebutt.createStream(), stream 14 | try { 15 | stream = connect(scuttlebutt.name) 16 | } catch (err) { return cb(err) } 17 | 18 | if(!stream) 19 | return cb(new Error('unable to connect')) 20 | 21 | stream.pipe(es).pipe(stream) 22 | 23 | var ready = false 24 | es.once('sync', function () { 25 | if(ready) return 26 | ready = true 27 | 28 | //cb the stream we are loading the scuttlebutt from, 29 | //incase it errors after we cb? 30 | //I'm not sure about this usecase. 31 | //Actually, just leave that feature out! 32 | //that way I don't have to break API when I realize it was a bad idea. 33 | if(cb) cb(null, scuttlebutt) 34 | if(!tail) es.end() 35 | }) 36 | //hmm, this has no way to detect that the stream has errored 37 | stream.once('error', function (err) { 38 | if(!ready) return cb(err) 39 | }) 40 | 41 | return scuttlebutt 42 | } 43 | } 44 | 45 | module.exports = function () { 46 | 47 | var opener = new EventEmitter() 48 | var mx = null 49 | 50 | opener.open = remoteOpen(function (name) { 51 | if(!mx) throw new Error('scuttlebutt remoteOpener must be connected') 52 | return mx.createStream(name) 53 | }) 54 | 55 | opener.view = function () { 56 | var args = [].slice.call(arguments) 57 | return mx.createStream(args) 58 | } 59 | 60 | //create stream... 61 | opener.createStream = function () { 62 | if(mx) throw new Error('remoteOpener may only connect to one server') 63 | 64 | mx = MuxDemux(function (stream) { 65 | stream.error(new Error('remoteOpener is client only - cannot recieve stream')) 66 | }) 67 | 68 | var ended = false 69 | 70 | function onEnd () { 71 | if(ended) return ended = true 72 | mx.removeListener('end', onEnd) 73 | mx.removeListener('close', onEnd) 74 | mx.removeListener('error', onEnd) 75 | mx.removeAllListeners() 76 | mx = null 77 | opener.emit('close') 78 | } 79 | 80 | mx.on('close', onEnd) 81 | mx.on('end', onEnd) 82 | mx.on('error', onEnd) 83 | 84 | process.nextTick(function () { 85 | opener.emit('open', mx) 86 | mx.resume() 87 | }) 88 | return mx 89 | } 90 | 91 | return opener 92 | } 93 | 94 | -------------------------------------------------------------------------------- /lib/map.js: -------------------------------------------------------------------------------- 1 | var mapReduce = require('map-reduce') 2 | 3 | function merge (a, b) { 4 | for (var k in b) { 5 | a[k] = b[k] 6 | } 7 | return a 8 | } 9 | 10 | module.exports = function (db) { 11 | 12 | if(db.scuttlebutt.addMap) return 13 | 14 | mapReduce(db) 15 | 16 | db.scuttlebutt.addView = 17 | db.scuttlebutt.addMapReduce = function (opts) { 18 | 19 | opts = merge({ 20 | name: 'default', 21 | start: db.scuttlebutt.range.start, 22 | end: db.scuttlebutt.range.end, 23 | keyMap: function (data) { 24 | var d = data.key.toString().split('\0') 25 | d.pop();d.pop() 26 | return d.pop() 27 | }, 28 | load: function (name, cb) { 29 | db.scuttlebutt(name, false, function (err, s) { 30 | cb(null, s) 31 | s.dispose() 32 | }) 33 | } 34 | //the user should pass in map, and/or reduce 35 | }, opts) 36 | 37 | db.mapReduce.add(opts) 38 | 39 | return db.scuttlebutt 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lib/remote-open.js: -------------------------------------------------------------------------------- 1 | module.exports = function (connect) { 2 | //pass in a string name, or a scuttlebutt instance 3 | //you want to reconnect to the server. 4 | return function (scuttlebutt, tail, cb) { 5 | if('function' == typeof tail) 6 | cb = tail, tail = true 7 | 8 | if('object' !== typeof scuttlebutt) 9 | throw new Error('expect Scuttlebutt, not ' + typeof scuttlebutt) 10 | 11 | var es = scuttlebutt.createStream() 12 | var stream = connect(scuttlebutt.name) 13 | 14 | if(!stream) 15 | return cb(new Error('unable to connect')) 16 | 17 | stream.pipe(es).pipe(stream) 18 | 19 | var ready = false 20 | es.once('sync', function () { 21 | if(ready) return 22 | ready = true 23 | 24 | //cb the stream we are loading the scuttlebutt from, 25 | //incase it errors after we cb? 26 | //I'm not sure about this usecase. 27 | //Actually, just leave that feature out! 28 | //that way I don't have to break API when I realize it was a bad idea. 29 | if(cb) cb(null, scuttlebutt) 30 | if(!tail) es.end() 31 | }) 32 | //hmm, this has no way to detect that the stream has errored 33 | stream.once('error', function (err) { 34 | if(!ready) return cb(err) 35 | }) 36 | 37 | return scuttlebutt 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /lib/schema.js: -------------------------------------------------------------------------------- 1 | var parserx = require('parse-regexp') 2 | 3 | //don't make the timeout too large, because it will prevent the process from exiting... 4 | var exports = module.exports = function (schema, id) { 5 | if('function' == typeof schema) 6 | return schema 7 | if('__proto__' === id) 8 | throw new Error('__proto__ is invalid id') 9 | 10 | if(!id) throw new Error('schema requires node id') 11 | 12 | var rules = [] 13 | for (var p in schema) { 14 | rules.push({rx: parserx(p) || p, fn: schema[p]}) 15 | } 16 | 17 | function match (key) { 18 | if('object' === typeof key) return key 19 | for (var i in rules) { 20 | var r = rules[i] 21 | var m = key.match(r.rx) 22 | if(m && m.index === 0) { 23 | var scuttlebutt = r.fn(key) 24 | scuttlebutt.name = key 25 | 'function' === scuttlebutt.setId ? scuttlebutt.setId(id) : scuttlebutt.id = id 26 | return scuttlebutt 27 | } 28 | } 29 | } 30 | 31 | match.schema = schema 32 | match.rules = rules 33 | 34 | return match 35 | } 36 | 37 | exports.schema = exports 38 | 39 | exports.cache = 40 | function (schema, open, onCache) { 41 | var local = cached.local = {} 42 | function cached (name, tail, cb) { 43 | if('function' == typeof tail) 44 | cb = tail, tail = true 45 | 46 | //user *must not* pass a Scuttlebutt instance to cached open. 47 | if('string' !== typeof name) 48 | throw new Error('name must be string') 49 | 50 | //so if this is passed a ready instance... 51 | //and there is a cached instance... then pipe them together. 52 | //hmm, but the caching should be ontop of the other layer... 53 | //will this work? 54 | //ah, if a scuttlebutt is passed, and there isn't one in the cache 55 | //clone it and put the clone in the cache! 56 | //else, pipe it to the cached instace. 57 | 58 | //actually... should combine this into level-scuttlebutt 59 | //then only one cache will be necessary. 60 | 61 | //which will work better for reconnections. 62 | 63 | //...so, need lots of tests here. 64 | 65 | //yes, only one cache, the outer most open does not support 66 | //opening with a scuttlebutt - but remote, and 67 | //level-scuttlebutt do! 68 | 69 | //then there is only one cache... 70 | //so, you always control the point of caching. 71 | //no "russian doll" caching. 72 | 73 | var cached = local[name] 74 | if(cached && 'function' === typeof cached.clone) { 75 | var n = cached.clone() 76 | n.name = name 77 | cb(null, n) 78 | return n 79 | } 80 | var clone 81 | var scuttlebutt = local[name] = schema(name) 82 | scuttlebutt.name = name 83 | open(scuttlebutt, tail, function (err, scuttlebutt) { 84 | if(err) return cb(err) 85 | cb(null, clone) 86 | }) 87 | 88 | //will callback an error 89 | if(!scuttlebutt) return 90 | 91 | scuttlebutt.name = name 92 | 93 | //only scuttlebutts with clone can be cleanly cached. 94 | if('function' === typeof scuttlebutt.clone) { 95 | local[name] = scuttlebutt 96 | clone = scuttlebutt.clone() 97 | clone.name = name 98 | if(onCache) onCache('clone', scuttlebutt.name) 99 | //okay... have something to dispose the scuttlebutt when there are 0 streams. 100 | //hmm, count streams... and emit an event 'unstream' or something? 101 | //okay, if all the steams have closed but this one, then it means no one is using this, 102 | //so close... 103 | //TODO add this to level-scuttlebutt. 104 | 105 | //OH, hang on... maybe DOMAINS is the right thing to use here... 106 | 107 | var timer = null 108 | scuttlebutt.on('unstream', function (n) { 109 | if(n === 1) { 110 | clearTimeout(timer) 111 | timer = setTimeout(function () { 112 | scuttlebutt.dispose() 113 | if(onCache) onCache('uncache', scuttlebutt.name) 114 | }, TIMEOUT*1.5) 115 | //if an emitter was passed, imet 116 | } else if(n > 1) 117 | clearTimeout(timer) 118 | }) 119 | scuttlebutt.on('dispose', function () { 120 | delete local[name] 121 | }) 122 | } 123 | 124 | return scuttlebutt 125 | } 126 | 127 | return cached 128 | } 129 | 130 | exports.sync = 131 | exports.open = function (schema, connect) { 132 | //pass in a string name, or a scuttlebutt instance 133 | //you want to reconnect to the server. 134 | return function (name, tail, cb) { 135 | if('function' == typeof tail) 136 | cb = tail, tail = true 137 | 138 | if('string' === typeof name) 139 | throw new Error('expect cache to create Scuttlebutt for open') 140 | 141 | var scuttlebutt = name 142 | /* 143 | if('string' === typeof name) 144 | scuttlebutt = schema(name) 145 | else { 146 | scuttlebutt = name 147 | name = scuttlebutt.name 148 | } 149 | */ 150 | var es = scuttlebutt.createStream() 151 | var stream = connect(scuttlebutt.name) 152 | 153 | if(!stream) 154 | return cb(new Error('unable to connect')) 155 | 156 | stream.pipe(es).pipe(stream) 157 | 158 | var ready = false 159 | es.once('sync', function () { 160 | if(ready) return 161 | ready = true 162 | 163 | //cb the stream we are loading the scuttlebutt from, 164 | //incase it errors after we cb? 165 | //I'm not sure about this usecase. 166 | //Actually, just leave that feature out! 167 | //that way I don't have to break API when I realize it was a bad idea. 168 | if(cb) cb(null, scuttlebutt) 169 | if(!tail) es.end() 170 | }) 171 | //hmm, this has no way to detect that the stream has errored 172 | stream.once('error', function (err) { 173 | if(!ready) return cb(err) 174 | }) 175 | 176 | return scuttlebutt 177 | } 178 | } 179 | 180 | -------------------------------------------------------------------------------- /lib/stream.js: -------------------------------------------------------------------------------- 1 | var MuxDemux = require('mux-demux') 2 | var through = require('through') 3 | 4 | module.exports = function (opener) { 5 | return function () { 6 | return MuxDemux(function (stream) { 7 | 8 | if('string' === typeof stream.meta) { 9 | var ts = through().pause() 10 | //TODO. make mux-demux pause. 11 | 12 | stream.pipe(ts) 13 | //load the scuttlebutt with the callback, 14 | //and then connect the stream to the client 15 | //so that the 'sync' event fires the right time, 16 | //and the open method works on the client too. 17 | opener.open(stream.meta, function (err, doc) { 18 | ts.pipe(doc.createStream()).pipe(stream) 19 | ts.resume() 20 | }) 21 | } else if(Array.isArray(stream.meta)) { 22 | //reduce the 10 most recently modified documents. 23 | try { 24 | opener.view.apply(null, stream.meta) 25 | .pipe(through(function (data) { 26 | this.queue(data) 27 | })) 28 | .pipe(stream) 29 | } catch (err) { 30 | stream.error(err) //emit error on other end! 31 | } 32 | } 33 | }) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "level-scuttlebutt", 3 | "version": "5.0.8", 4 | "homepage": "https://github.com/dominictarr/level-scuttlebutt", 5 | "repository": { 6 | "type": "git", 7 | "url": "git://github.com/dominictarr/level-scuttlebutt.git" 8 | }, 9 | "dependencies": { 10 | "monotonic-timestamp": "~0.0.7", 11 | "node-uuid": "~1.4.0", 12 | "duplex": "~1.0.0", 13 | "redis-protocol-stream": "~0.1.2", 14 | "parse-regexp": "~0.0.1", 15 | "mux-demux": "~3.5.0", 16 | "through": "~2.1.0", 17 | "level-live-stream": "~1.4", 18 | "map-reduce": "~6.0.2" 19 | }, 20 | "devDependencies": { 21 | "level-test": "^1.6.2", 22 | "rimraf": "^2.0.2", 23 | "tape": "^2", 24 | "delay-stream": "^0.0.1", 25 | "scuttlebutt": "^5.5", 26 | "macgyver": "^1.10.1", 27 | "level-sublevel": "^5.2.0", 28 | "crdt": "^3.6.2" 29 | }, 30 | "scripts": { 31 | "test": "set -e; for t in test/*.js; do node $t; done" 32 | }, 33 | "author": "'Dominic Tarr' (http://dominictarr.com)", 34 | "license": "MIT" 35 | } -------------------------------------------------------------------------------- /test/crdt.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | , assert = require('assert') 3 | , sublevel = require('level-sublevel') 4 | // , udid = require('udid')('example-app') 5 | , scuttlebutt = require('../') 6 | , Doc = require('crdt').Doc 7 | , test = require('tape') 8 | 9 | // setup db 10 | newDB = function (opts) { 11 | var db = sublevel(level('test-level-scuttlebutt-crdt', opts)) 12 | scuttlebutt(db, 'udid', function(name) {return new Doc;}); 13 | return db 14 | } 15 | 16 | test('modifying a sequence persists correctly', function(t) { 17 | t.plan(1) 18 | var DB = newDB() 19 | 20 | DB.open('one-doc', function(err, doc1) { 21 | var seq = doc1.createSeq('session', 'one'); 22 | doc1.on('_remove', function (update) { 23 | console.error('_REMOVE', update) 24 | }) 25 | seq.on('_update', console.error) 26 | 27 | seq.push({id: 'a'}); 28 | seq.push({id: 'b'}); 29 | seq.push({id: 'c'}); 30 | console.log(seq.toJSON()) 31 | seq.after('a', 'b'); 32 | console.log(JSON.stringify(doc1.history())) 33 | 34 | var firstOutput = seq.toJSON() 35 | 36 | // is 'drain' the right event to listen for here? 37 | DB.on('drain', function(){ 38 | DB.close(function(err){ 39 | if (err) console.log('err', err); 40 | 41 | // reopen DB 42 | var anotherDB = newDB({clean: false}) 43 | 44 | anotherDB.open('one-doc', function(err, doc2) { 45 | var seq2 = doc2.createSeq('session', 'one'); 46 | 47 | var secondOutput = seq2.toJSON() 48 | console.log(doc2.history()) 49 | t.same(secondOutput, firstOutput) 50 | }) 51 | }) 52 | }) 53 | }) 54 | }) 55 | -------------------------------------------------------------------------------- /test/expected.js: -------------------------------------------------------------------------------- 1 | var levelup = require('level-test')(); 2 | var SubLevel = require('level-sublevel') 3 | var levelScuttlebutt = require('../'); 4 | var Model = require('scuttlebutt/model'); 5 | 6 | require('tape')('test', function (t) { 7 | var db = SubLevel(levelup('test-level-scuttlebutt-expected')) 8 | levelScuttlebutt(db, "TEST", function (name) { 9 | 10 | var m = new Model(); 11 | m.set('foo', 'BAR') 12 | return m 13 | }); 14 | 15 | db.open('some-name', function (err, model) { 16 | if(err) throw err 17 | t.deepEqual(model.toJSON(), {foo: 'BAR'}) 18 | t.end() 19 | }); 20 | }) 21 | -------------------------------------------------------------------------------- /test/map-reduce.js: -------------------------------------------------------------------------------- 1 | 2 | var level = require('level-test')() 3 | var Model = require('scuttlebutt/model') 4 | var assert = require('assert') 5 | var rimraf = require('rimraf') 6 | var MapReduce = require('map-reduce') 7 | var Scuttlebutt = require('..') 8 | var SubLevel = require('level-sublevel') 9 | 10 | require('tape')('scuttlebutt: map-reduce', function (t) { 11 | 12 | var db = SubLevel(level('level-scuttlebutt-example')) 13 | 14 | Scuttlebutt(db, 'THIS', { 15 | test: function () { 16 | return new Model() 17 | } 18 | }) 19 | 20 | var mapDb = 21 | MapReduce(db, 'test', 22 | function (key, model, emit) { 23 | model = JSON.parse(model) 24 | if(!model) return 25 | emit('square', Math.pow(Number(model.number), 2)) 26 | emit('cube', Math.pow(Number(model.number), 3)) 27 | }, 28 | function (sum, n) { 29 | return Number(sum) + Number(n) 30 | }, 31 | 0) 32 | 33 | 'abcde'.split('').forEach(function (e, i) { 34 | db.scuttlebutt.open('test-'+e, false, function (err, t) { 35 | t.set('number', i) 36 | setTimeout(function () { 37 | var l = 10 38 | var int = setInterval(function () { 39 | t.set('number', i * 2 * l) 40 | if(!--l) { 41 | clearInterval(int) 42 | t.dispose() 43 | } 44 | }, 200) 45 | 46 | }, 1000) 47 | }) 48 | }) 49 | 50 | var sq, cu 51 | 52 | mapDb.on('reduce', function (group, sum) { 53 | console.log('reduce->', group, sum) 54 | try { 55 | assert.deepEqual([['square'], 120], [group, sum]) 56 | console.log('sq') 57 | sq = true 58 | t.ok(true, "eventually ['square']: 120") 59 | } catch (_) { } 60 | 61 | try { 62 | assert.deepEqual([['cube'], 800], [group, sum]) 63 | console.log('cu') 64 | cu = true 65 | t.ok(true, "eventually ['cube']: 800") 66 | } catch (_) { } 67 | 68 | if(sq && cu) 69 | t.end() 70 | }) 71 | }) 72 | -------------------------------------------------------------------------------- /test/open.js: -------------------------------------------------------------------------------- 1 | //test to open scuttlebutt from leveldb. 2 | 3 | var level = require('level-test')() 4 | var SubLevel = require('level-sublevel') 5 | 6 | var Model = require('scuttlebutt/model') 7 | var LevelScuttlebutt = require('..') 8 | var Client = require('../client') 9 | var mac = require('macgyver')().autoValidate() 10 | 11 | var tape = require('tape') 12 | 13 | tape('local open, remote open', function (t) { 14 | var path = 'test-scuttlebutt-remote' 15 | t.plan(2) 16 | var db = SubLevel(level(path)) 17 | 18 | var schema = {test: function () { return new Model} } 19 | 20 | LevelScuttlebutt(db, 'TEST', schema) 21 | var local = db.scuttlebutt 22 | var client = Client(schema, 'TEST-CLIENT') 23 | 24 | local.open('test1', mac(function (err, a) { 25 | if(err) t.fail(err) 26 | 27 | console.log('OPEN') 28 | a.set('x', Math.random()) 29 | a.set('y', Math.random()) 30 | a.set('z', Math.random()) 31 | console.log('A', a.history(), a._parent.history()) 32 | 33 | setTimeout(function () { 34 | client.open('test1', mac(function (err, b) { 35 | if(err) t.fail(err) 36 | 37 | console.log('B', b.history()) 38 | t.notStrictEqual(a, b) 39 | t.deepEqual(b.history(), a.history()) 40 | t.end() 41 | }).once()) 42 | 43 | }, 500) 44 | }).once()) 45 | var ls = local.createRemoteStream() 46 | var rs = client.createStream() 47 | 48 | rs.on('data', function (d) { console.log('rs>>', d) }) 49 | ls.on('data', function (d) { console.log('ls>>', d) }) 50 | 51 | ls.pipe(rs).pipe(ls) 52 | 53 | }) 54 | 55 | tape('parallel open', function (t) { 56 | var path = 'test-scuttlebutt-remote2' 57 | t.plan(2) 58 | 59 | var db = SubLevel(level(path)) 60 | var schema = {test: Model} 61 | LevelScuttlebutt(db, 'test', schema) 62 | 63 | var a,b 64 | 65 | var local = db.scuttlebutt 66 | var remote = Client(schema, 'TEST-CLIENT') 67 | local.open('test1', mac(function (err, _a) { 68 | if(err) t.fail(err) 69 | a = _a 70 | a.set('x', Math.random()) 71 | a.set('y', Math.random()) 72 | a.set('z', Math.random()) 73 | 74 | if(a && b) next() 75 | }).once()) 76 | 77 | remote.open('test1', mac(function (err, _b) { 78 | b = _b 79 | if(a && b) next() 80 | }).once()) 81 | 82 | function next () { 83 | console.log('END') 84 | t.notStrictEqual(a, b) 85 | t.deepEqual(b.history(), a.history()) 86 | t.end() 87 | } 88 | 89 | var ls = local.createRemoteStream() 90 | var rs = remote.createStream() 91 | ls.pipe(rs).pipe(ls) 92 | }) 93 | 94 | -------------------------------------------------------------------------------- /test/open2.js: -------------------------------------------------------------------------------- 1 | //test to open scuttlebutt from leveldb. 2 | 3 | var level = require('level-test')() 4 | var SubLevel = require('level-sublevel') 5 | 6 | var Model = require('scuttlebutt/model') 7 | var LevelScuttlebutt 8 | = require('..') 9 | var Client = require('../client') 10 | var mac = require('macgyver')().autoValidate() 11 | var tape = require('tape') 12 | 13 | function create (c) { 14 | return function () { 15 | return new c 16 | } 17 | } 18 | 19 | tape('local open, remote open', function (t) { 20 | var path = 'test-scuttlebutt-remote' 21 | t.plan(1) 22 | var db = SubLevel(level(path)) 23 | var schema = {foo: create(Model), bar: create(Model)} 24 | LevelScuttlebutt(db, 'test', schema) 25 | var local = db.scuttlebutt 26 | var remote = Client(schema, 'test-client') 27 | var a, b 28 | 29 | remote.open('foo1', mac(function (err, _a) { 30 | if(err) t.fail(err) 31 | a = _a 32 | console.log('OPEN foo1') 33 | a.on('change', mac(function (k, r) { 34 | console.log(this.name, k, r) 35 | }).times(3)) 36 | 37 | a.set('x', Math.random()) 38 | a.set('y', Math.random()) 39 | a.set('z', Math.random()) 40 | console.log('A', a.history()) 41 | 42 | if(a && b) t.notDeepEqual(a.history(), b.history()), t.end() 43 | }).once()) 44 | 45 | 46 | remote.open('bar1', mac(function (err, _b) { 47 | b = _b 48 | console.log('OPEN bar1') 49 | b.on('change', mac(function (k, r) { 50 | console.log(this.name, k, r) 51 | }).times(3)) 52 | b.set('a', Math.random()) 53 | b.set('b', Math.random()) 54 | b.set('c', Math.random()) 55 | 56 | 57 | console.log('B', b.history()) 58 | 59 | if(a && b) t.notDeepEqual(a.history(), b.history()), t.end() 60 | }).once()) 61 | 62 | console.log('>>>>>>>>>>>>>>>>') 63 | var ls = local.createRemoteStream() 64 | console.log('<<<<<<<<<<<<<<<<') 65 | 66 | var rs = remote.createStream() 67 | console.log(rs) 68 | rs.on('data', console.log) 69 | // ls.on('data', console.log) 70 | 71 | ls.pipe(rs).pipe(ls) 72 | 73 | }) 74 | 75 | 76 | -------------------------------------------------------------------------------- /test/open3.js: -------------------------------------------------------------------------------- 1 | //test to open scuttlebutt from leveldb. 2 | 3 | var level = require('level-test')() 4 | var SubLevel = require('level-sublevel') 5 | 6 | var Model = require('scuttlebutt/model') 7 | var LevelScuttlebutt = require('..') 8 | var Client = require('../client') 9 | var mac = require('macgyver')().autoValidate() 10 | 11 | var tape = require('tape') 12 | 13 | 14 | tape('remote open, local open', function (t) { 15 | var path = 'test-scuttlebutt-remote3' 16 | t.plan(2) 17 | 18 | var db = SubLevel(level(path)) 19 | var schema = {test: Model} 20 | LevelScuttlebutt(db, 'test', schema) 21 | var local = db.scuttlebutt 22 | var remote = Client(schema, 'test-client') 23 | 24 | remote.open('test1', mac(function remoteOpen (err, a) { 25 | if(err) t.fail(err) 26 | a.set('x', Math.random()) 27 | a.set('y', Math.random()) 28 | a.set('z', Math.random()) 29 | 30 | local.open('test1', mac(function localOpen (err, b) { 31 | t.notStrictEqual(a, b) 32 | console.log('A', a.history()) 33 | console.log('B', b.history()) 34 | t.deepEqual(b.history(), a.history()) 35 | t.end() 36 | }).once()) 37 | 38 | }).once()) 39 | var ls = local.createRemoteStream() 40 | var rs = remote.createStream() 41 | 42 | ls.pipe(rs).pipe(ls) 43 | }) 44 | 45 | -------------------------------------------------------------------------------- /test/reopen.js: -------------------------------------------------------------------------------- 1 | 2 | var level = require('level-test')() 3 | var SubLevel = require('level-sublevel') 4 | var rimraf = require('rimraf') 5 | var delay = require('delay-stream') 6 | var Model = require('scuttlebutt/model') 7 | var tape = require('tape') 8 | 9 | 10 | tape('test', function (t) { 11 | 12 | var db = SubLevel(level('level-scuttlebutt-test-A')) 13 | 14 | require('../')(db, 'test1', { 15 | test: function () { 16 | return Model() 17 | } 18 | }) 19 | 20 | var m = new Model() 21 | 22 | m.name = 'test-model' 23 | 24 | m.set('x', Math.random()) 25 | m.set('y', Math.random()) 26 | m.set('z', Math.random()) 27 | 28 | var opener = db.scuttlebutt._opener //Opener(db) 29 | 30 | opener.open(m, function () { 31 | console.log('reopened') 32 | 33 | db.scuttlebutt(m.name, function (err, _m) { 34 | 35 | console.log(_m.history(), m.history()) 36 | t.notStrictEqual(_m, m) 37 | t.deepEqual(_m.history(), m.history()) 38 | t.end() 39 | }) 40 | }) 41 | 42 | }) 43 | -------------------------------------------------------------------------------- /test/reopen2.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | var SubLevel = require('level-sublevel') 3 | var delay = require('delay-stream') 4 | var Model = require('scuttlebutt/model') 5 | var LevelScuttlebutt = require('../') 6 | var Client = require('../client') 7 | var mac = require('macgyver')().autoValidate() 8 | var tape = require('tape') 9 | 10 | tape('test', function (t) { 11 | 12 | 13 | 14 | db = SubLevel(level('level-scuttlebutt-test-A')) 15 | 16 | var schema = { 17 | test: function () { 18 | return Model() 19 | } 20 | } 21 | 22 | LevelScuttlebutt(db, 'test1', schema) 23 | 24 | //open a scuttlebutt, then close the connection to the database, 25 | //then reopen the connection, then the scuttlebutt should be reconnected. 26 | 27 | var local = db.scuttlebutt 28 | var remote = Client(schema, 'test1-client') 29 | 30 | remote.open('test1', mac(function remoteOpen (err, a) { 31 | if(err) t.fail(err) 32 | a.set('x', Math.random()) 33 | a.set('y', Math.random()) 34 | a.set('z', Math.random()) 35 | 36 | local.open('test1', mac(function localOpen (err, b) { 37 | t.notStrictEqual(a, b) 38 | console.log('A', a.history()) 39 | console.log('B', b.history()) 40 | t.deepEqual(b.history(), a.history()) 41 | // t.end() 42 | b.set('a', Math.random()) 43 | b.set('b', Math.random()) 44 | b.set('c', Math.random()) 45 | 46 | t.deepEqual(b.history(), a.history()) 47 | 48 | rs.end() 49 | 50 | a.set('i', Math.random()) 51 | b.set('j', Math.random()) 52 | a.set('k', Math.random()) 53 | 54 | t.notDeepEqual(b.history(), a.history()) 55 | 56 | console.log('pre disconnect') 57 | console.log('A', a.history()) 58 | console.log('B', b.history()) 59 | 60 | var ls2 = local.createRemoteStream() 61 | var rs2 = remote.createStream() 62 | 63 | ls2.pipe(rs2).pipe(ls2) 64 | 65 | ls2.resume() 66 | rs2.resume() 67 | 68 | process.nextTick(function () { 69 | process.nextTick(function () { 70 | console.log('post reconnect') 71 | console.log('A', a.history()) 72 | console.log('B', b.history()) 73 | t.deepEqual(b.history(), a.history()) 74 | t.end() 75 | }) 76 | }) 77 | }).once()) 78 | 79 | }).once()) 80 | 81 | var ls = local.createRemoteStream() 82 | var rs = remote.createStream() 83 | 84 | ls.pipe(rs).pipe(ls) 85 | 86 | }) 87 | -------------------------------------------------------------------------------- /test/script.js: -------------------------------------------------------------------------------- 1 | var SubLevel = require('level-sublevel') 2 | var level = require('level-test')() 3 | var delay = require('delay-stream') 4 | var Model = require('scuttlebutt/model') 5 | var tape = require('tape') 6 | 7 | tape('test replication', function (t) { 8 | 9 | var A, B 10 | 11 | function randomData(db, id, cb) { 12 | require('..')(db, id, { 13 | test: function () { 14 | return Model() 15 | } 16 | }) 17 | 18 | db.sublevel('sb') 19 | .post(function (op) { 20 | console.error(op) 21 | if(undefined === op.value && op.type !== 'del') 22 | throw new Error('value is undefined') 23 | }) 24 | 25 | db.scuttlebutt('test1', function (err, emitter) { 26 | var letters = "ABCDEFGHIJK" 27 | var l = 5 28 | 29 | while(l --> 0) 30 | emitter.set(letters[~~(Math.random()*letters.length)], 'Date: ' + new Date()) 31 | 32 | setTimeout(cb, 1000) 33 | 34 | }) 35 | } 36 | 37 | 38 | randomData(A = SubLevel(level('level-scuttlebutt-test-A', {encoding: 'utf8'})), 'A', next) 39 | randomData(B = SubLevel(level('level-scuttlebutt-test-B'), {encoding: 'utf8'}), 'B', next) 40 | 41 | var z = 2 42 | 43 | function next() { 44 | if(--z) return 45 | var streamA = A.scuttlebutt.createReplicateStream({tail: false}) 46 | var streamB = B.scuttlebutt.createReplicateStream({tail: false}) 47 | 48 | streamA.pipe(delay(100)).pipe(streamB).pipe(delay(100)).pipe(streamA) 49 | 50 | streamA.pipe(process.stderr, {end: false}) 51 | streamB.pipe(process.stderr, {end: false}) 52 | 53 | var n = 2, vecA, vecB 54 | 55 | streamA.on('end', function () { 56 | A.scuttlebutt.vectorClock(function (err, vec) { 57 | console.log('streamA end') 58 | vecA = vec; next() 59 | }) 60 | }) 61 | 62 | streamB.on('end', function () { 63 | B.scuttlebutt.vectorClock(function (err, vec) { 64 | console.log('streamB end') 65 | vecB = vec; next() 66 | }) 67 | }) 68 | 69 | function next() { 70 | if(--n) return 71 | console.log(vecA, vecB) 72 | t.deepEqual(vecA, vecB) 73 | t.end() 74 | } 75 | } 76 | 77 | }) 78 | -------------------------------------------------------------------------------- /test/simple-id.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | //test to open scuttlebutt from leveldb. 4 | 5 | var rimraf = require('rimraf') 6 | var levelup = require('levelup') 7 | var SubLevel = require('level-sublevel') 8 | 9 | var Model = require('scuttlebutt/model') 10 | var LevelScuttlebutt = require('..') 11 | var Client = require('../client') 12 | var mac = require('macgyver')().autoValidate() 13 | 14 | var tape = require('tape') 15 | 16 | function initDb(suffix) { 17 | var path = '/tmp/test-scuttlebutt-'+suffix 18 | rimraf.sync(path) 19 | return SubLevel(levelup(path)) 20 | } 21 | 22 | 23 | tape('sets correct id with object', function (t) { 24 | t.plan(2) 25 | var db = initDb('object') 26 | var ID = '#' + Math.random().toString(16).substring(2) 27 | LevelScuttlebutt(db, ID, { 28 | test: function () { return new Model} 29 | }) 30 | 31 | var a = db.open('test1', mac(function (err, a) { 32 | t.equal(a.id, ID) 33 | t.end() 34 | }).once()) 35 | t.equal(a.id, ID) 36 | }) 37 | 38 | tape('sets correct id with function', function (t) { 39 | t.plan(2) 40 | var db = initDb('function') 41 | var ID = '#' + Math.random().toString(16).substring(2) 42 | LevelScuttlebutt(db, ID, function () { return new Model}) 43 | 44 | var a = db.open('test1', mac(function (err, a) { 45 | t.equal(a.id, ID) 46 | t.end() 47 | }).once()) 48 | t.equal(a.id, ID) 49 | }) 50 | 51 | -------------------------------------------------------------------------------- /test/view.js: -------------------------------------------------------------------------------- 1 | var level = require('level-test')() 2 | var SubLevel = require('level-sublevel') 3 | var delay = require('delay-stream') 4 | var Model = require('scuttlebutt/model') 5 | var LevelScuttlebutt 6 | = require('../') 7 | var MapReduce = require('map-reduce') 8 | var Client = require('../client') 9 | var mac = require('macgyver')().autoValidate() 10 | var tape = require('tape') 11 | 12 | tape('test', function (t) { 13 | 14 | var A, B 15 | 16 | var db = SubLevel(level('level-scuttlebutt-test-A')) 17 | 18 | var schema = { 19 | test: function () { 20 | return Model() 21 | } 22 | } 23 | 24 | LevelScuttlebutt(db, 'test1', schema) 25 | 26 | db.views['all'] = 27 | MapReduce(db, 'all', 28 | function (key, scuttle, emit) { 29 | console.log(key.split('!'), scuttle) 30 | return emit(key.split('!'), 1) 31 | }, 32 | function (acc, item) { 33 | return '' + (Number(acc) + Number(item)) 34 | }, 35 | '0' 36 | ) 37 | 38 | //open a scuttlebutt, then close the connection to the database, 39 | //then reopen the connection, then the scuttlebutt should be reconnected. 40 | 41 | var local = db.scuttlebutt 42 | var remote = Client(schema, 'test1-client') 43 | 44 | local.open('test!thing1', mac(function remoteOpen (err, a) { 45 | if(err) t.fail(err) 46 | a.set('x', Math.random()) 47 | a.set('y', Math.random()) 48 | a.set('z', Math.random()) 49 | }).once()) 50 | 51 | local.open('test!thing2', mac(function remoteOpen (err, a) { 52 | if(err) t.fail(err) 53 | a.set('x', Math.random()) 54 | a.set('y', Math.random()) 55 | a.set('z', Math.random()) 56 | }).once()) 57 | 58 | var rv = [] 59 | var lv = [] 60 | var ended = 0 61 | function onEnd () { 62 | if(!ended++) return 63 | t.deepEqual(rv, lv) 64 | // console.log(rv, lv) 65 | // console.log('passed?') 66 | t.end() 67 | } 68 | 69 | remote.view({ 70 | name: 'all', range: ['test', true] 71 | }).on('data', function (data) { 72 | console.log('remote view', data) 73 | rv.push(data) 74 | if(rv.length > 1) onEnd() 75 | }) 76 | 77 | local.view({ 78 | name: 'all', range: ['test', true] 79 | }).on('data', function (data) { 80 | console.log('local view', data) 81 | lv.push(data) 82 | if(lv.length > 1) onEnd() 83 | }) 84 | 85 | var ls = local.createRemoteStream() 86 | var rs = remote.createStream() 87 | 88 | ls.pipe(rs).pipe(ls) 89 | 90 | }) 91 | --------------------------------------------------------------------------------