├── .gitignore ├── .travis.yml ├── LICENSE.APACHE2 ├── LICENSE.MIT ├── README.markdown ├── events.js ├── index.js ├── keygen ├── model.js ├── package.json ├── security.js ├── test ├── cleanup.js ├── clone.js ├── dispose.js ├── events.js ├── header.js ├── index.js ├── integrate.js ├── integrate2.js ├── iterate.js ├── keys │ ├── test1 │ ├── test1.pem │ ├── test1.pub │ ├── test2 │ ├── test2.pem │ └── test2.pub ├── meta.js ├── model.js ├── persist.js ├── secure.js ├── sendclock.js ├── sync.js └── unstream.js └── util.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/* 3 | npm-debug.log 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.6 4 | - 0.8 5 | - '0.10' 6 | -------------------------------------------------------------------------------- /LICENSE.APACHE2: -------------------------------------------------------------------------------- 1 | Apache License, Version 2.0 2 | 3 | Copyright (c) 2012 Dominic Tarr 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /LICENSE.MIT: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2012 Dominic Tarr 4 | 5 | Permission is hereby granted, free of charge, 6 | to any person obtaining a copy of this software and 7 | associated documentation files (the "Software"), to 8 | deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, 10 | merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom 12 | the Software is furnished to do so, 13 | subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 21 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 22 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # scuttlebutt 2 | 3 | A base-class for real-time replication. 4 | 5 | [![travis](https://secure.travis-ci.org/dominictarr/scuttlebutt.png?branch=master)](https://travis-ci.org/dominictarr/scuttlebutt) 6 | 7 | [![browser support](http://ci.testling.com/dominictarr/scuttlebutt.png)](http://ci.testling.com/dominictarr/scuttlebutt) 8 | 9 | This seems like a silly name, but I assure you, this is real science. 10 | Read this: http://www.cs.cornell.edu/home/rvr/papers/flowgossip.pdf 11 | 12 | Or, if you're lazy: http://en.wikipedia.org/wiki/Scuttlebutt (laziness will get you nowhere, btw) 13 | 14 | ## secure-scuttlebutt 15 | 16 | Creating this module eventually lead me to [secure-scuttlebutt](https://github.com/dominictarr/secure-scuttlebutt) 17 | it is based on a similar replication protocol as scuttlebutt, but puts security at the forefront. 18 | 19 | ## Subclasses 20 | 21 | Scuttlebutt is intended to be subclassed into a variety of data-models. 22 | 23 | Two implementations are provided as examples [scuttlebutt/model](#scuttlebuttmodel) and 24 | [scuttlebutt/events](#scuttlebuttevents) 25 | 26 | subclasses: 27 | 28 | * [crdt](https://github.com/dominictarr/crdt) higher-level, with sets and sequences. 29 | * [r-array](https://github.com/dominictarr/r-array) Replicatable Array. 30 | * [r-edit](https://github.com/dominictarr/r-edit) Collaborative Text Editing. 31 | * [append-only](https://github.com/Raynos/append-only) news feed. 32 | * [scuttlebucket](https://github.com/dominictarr/scuttlebucket) combine multiple scuttlebutts into one. 33 | * [expiry-model](https://github.com/Raynos/expiry-model) memory capped model with expiring keys. 34 | * [r-value](https://github.com/dominictarr/r-value) replicate a single value. 35 | * [gossip-object](https://github.com/vigour-io/gossip-object) like scuttlebutt/model, but supports nested objects. 36 | * [scuttleboat](https://github.com/kumavis/scuttleboat) a dynamic scuttlebucket! 37 | * [r-string](https://github.com/marcelklehr/r-string) character-wise collaborative text editing. 38 | 39 | ### Replication 40 | 41 | Any Scuttlebutt subclass is replicated with createStream. 42 | Create a server and then connect a client to it: 43 | 44 | ``` js 45 | var Model = require('scuttlebutt/model') 46 | var net = require('net') 47 | 48 | var s = new Model() 49 | var z = new Model() 50 | 51 | net.createServer(function (stream) { 52 | 53 | stream.pipe(s.createStream()).pipe(stream) 54 | 55 | }).listen(8000, function () { 56 | 57 | //then connect the client! 58 | var stream = net.connect(8000) 59 | stream.pipe(z.createStream()).pipe(stream) 60 | 61 | }) 62 | ``` 63 | 64 | ### Gotchas 65 | 66 | Scuttlebutt is always duplex. 67 | Scuttlebutt does a handshake on connecting to another scuttlebutt, 68 | and this won't work unless both sides are connected. 69 | 70 | #### Right 71 | 72 | ``` js 73 | stream.pipe(model.createStream()).pipe(stream) 74 | ``` 75 | 76 | #### WRONG! 77 | 78 | ``` js 79 | wrongStream.pipe(model2.createStream()) 80 | ``` 81 | 82 | Also, when creating a server, scuttlebutt needs a stream for EACH connection. 83 | 84 | #### Right 85 | 86 | ``` js 87 | net.createServer(function (stream) { 88 | stream.pipe(model.createStream()).pipe(stream) 89 | }).listen(port) 90 | ``` 91 | 92 | #### WRONG! 93 | this will use one stream for many connections! 94 | ``` js 95 | var wrongStream = model.createStream() 96 | net.createServer(function (stream) { 97 | stream.pipe(wrongStream).pipe(stream) 98 | }).listen(port) 99 | ``` 100 | 101 | ### Errors and use in PRODUCTION 102 | 103 | If have are using scuttlebutt in production, you must register 104 | on `'error'` listener in case someone sends invalid data to it. 105 | 106 | ** Any stream that gets parsed should have an error listener! ** 107 | 108 | ``` js 109 | net.createServer(function (stream) { 110 | var ms = m.createStream() 111 | stream.pipe(ms).pipe(stream) 112 | ms.on('error', function () { 113 | stream.destroy() 114 | }) 115 | stream.on('error', function () { 116 | ms.destroy() 117 | }) 118 | }).listen(9999) 119 | ``` 120 | 121 | Otherwise, if someone tries to connect to port `9999` with a different 122 | protocol (say, HTTP) this will emit an error. You must handle this and 123 | close the connection / log the error. 124 | 125 | Also, you should handle errors on `stream`, stream may error if the client 126 | responsible for it crashes. 127 | 128 | ### Persistence 129 | 130 | Persist by saving to at least one writable stream. 131 | 132 | ``` js 133 | var Model = require('scuttlebutt/model') //or some other subclass... 134 | var fs = require('fs') 135 | var m = new Model() 136 | 137 | //stream FROM disk. 138 | fs.createReadStream(file).pipe(m.createWriteStream()) 139 | 140 | //stream TO disk. 141 | m.on('sync', function () { 142 | m.createReadStream().pipe(fs.createWriteStream(file)) 143 | }) 144 | ``` 145 | Use `on('sync',...` to wait until the persisted state is in the file 146 | before writing to disk. 147 | (Make sure you rotate files, else there is an edge case where if the process 148 | crashes before the history has been written some data will be lost 149 | /*this is where link to module for that will go*/) 150 | 151 | You may use [kv](https://github.com/dominictarr/kv) to get streams 152 | to local storage. 153 | 154 | ## read only mode. 155 | 156 | Sometimes you want to use scuttlebutt to send data one way, 157 | from a `master` instance to a `slave` instance. 158 | 159 | ``` js 160 | var s1 = master.createStream({writable: false, sendClock: true}) 161 | var s2 = slave.createStream({readable: false, sendClock: true}) 162 | ``` 163 | 164 | `master` will emit updates, but not accept them, over this stream. 165 | This checking is per stream - so it's possible to attach `master` to 166 | another master node and have master nodes replicate each way. 167 | 168 | ## Implementing Custom Scuttlebutts 169 | 170 | A custom Scuttlebutt is a data model that inherits from `Scuttlebutt`. It must provide an implementation of `history()` and `applyUpdate()`. 171 | 172 | See [r-value](https://github.com/dominictarr/r-value) for a demonstration of the simplest possible Scuttlebutt, one that replicates a a single value. 173 | 174 | ### Scuttlebutt#history(sources) 175 | 176 | `sources` is a hash of source_ids: timestamps. 177 | History must return an array of all known events from all sources 178 | That occur after the given timestamps for each source. 179 | 180 | The array MUST be in order by timestamp. 181 | 182 | ### Scuttlebutt#applyUpdate (update) 183 | 184 | Each update is of the form `[change, timestamp, source]` (see [Protocol](#Protocol) below). 185 | 186 | Possibly apply a given update to the subclasses model. 187 | Return 'true' if the update was applied. (See scuttlebutt/model.js 188 | for an example of a subclass that does not apply every update.) 189 | 190 | ### Scuttlebutt#createStream (opts) 191 | 192 | Create a duplex stream to replicate with a remote endpoint. 193 | 194 | The stream returned here emits a special `'header'` event with the id of the 195 | local and remote nodes and the vector clock. You can set metadata on the header 196 | object using `opts.meta`. 197 | 198 | #### Examples 199 | 200 | Connect two `Model` scuttlebutts locally. 201 | 202 | ``` js 203 | var Model = require('scuttlebutt/model') 204 | 205 | var a = new Model() 206 | var b = new Model() 207 | 208 | a.set(key, value) 209 | 210 | b.on('update', console.log) 211 | 212 | var s = a.createStream() 213 | s.pipe(b.createStream()).pipe(s) 214 | ``` 215 | 216 | ### scuttlebutt/events 217 | 218 | A reliable event emmitter. Multiple instances of an emitter 219 | may be connected to each other and will remember events, 220 | so that they may be present after a disconnection or crash. 221 | 222 | With this approach it is also possible to persist events to disk, 223 | making them durable over crashes. 224 | 225 | ``` js 226 | var Emitter = require('scuttlebutt/events') 227 | var emitter = new Emitter() 228 | ``` 229 | 230 | #### emit (event, data) 231 | 232 | Emit an event. Only one argument is permitted. 233 | 234 | #### on (event, listener) 235 | 236 | Add an event listener. 237 | 238 | ### scuttlebutt/model 239 | 240 | A replicateable `Model` object, a simple key-value store. 241 | 242 | ``` js 243 | var Model = require('scuttlebutt/model') 244 | var model = new Model() 245 | ``` 246 | 247 | 248 | #### get (key) 249 | 250 | Get a property. 251 | 252 | #### set (key, value) 253 | 254 | Set a property. 255 | 256 | #### on('update', function ([key, value], source, updateId)) 257 | 258 | Emmitted when a property changes. 259 | If `source !== this.id` 260 | then it was a remote update. 261 | 262 | ## Protocol 263 | 264 | Messages are sent in this format: 265 | 266 | ``` js 267 | [change, timestamp, source] 268 | ``` 269 | 270 | `source` is the id of the node which originated this message. 271 | Timestamp is the time when the message was created. 272 | This message is created using `Scuttlebutt#localUpdate(key, value)`. 273 | 274 | When two `Scuttlebutts` are piped together, they both exchange their current list 275 | of sources. This is an object of `{source_id: latest_timestamp_for_source_id}` 276 | After receiving this message, `Scuttlebutt` sends any messages not yet 277 | known by the other end. This is the heart of Scuttlebutt Reconciliation. 278 | 279 | ## Security 280 | 281 | Scuttlebutt has an (optional) heavy duty security model using public keys. 282 | This enables a high level of security even in peer-to-peer applications. 283 | You can be sure that a given message is from the node that sent it, 284 | even if you did not receive the messasge from them directly. 285 | 286 | ## Enabling Security 287 | 288 | ``` js 289 | var model = require('scuttlebutt/model') 290 | var security = require('scuttlebutt/security') 291 | var keys = {} 292 | var m = new Model(security(keys, PRIVATE, PUBLIC)) 293 | ``` 294 | 295 | ## Security API 296 | 297 | When security is enabled, each scuttlebutt message is signed with a private key. 298 | It is then possible for any scuttlebutt instance to be confident about the 299 | authenticity of the message by verifying it against the source's public key. 300 | 301 | This is possible even if the verifying node received the message from an intermediate node. 302 | 303 | Security is activated by passing in a security object to the contructor of a scuttlebutt 304 | subclass. 305 | 306 | Use the included implementation: 307 | 308 | ``` js 309 | var security = require('scuttlebutt/security')(keys, PRIVATE, PUBLIC) 310 | var Model = require('scuttlebutt/model') 311 | 312 | var m = new Model(security) 313 | ``` 314 | 315 | See 316 | [scuttlebutt/security.js](https://github.com/dominictarr/scuttlebutt/blob/master/security.js) 317 | for a simple example implementation. 318 | 319 | `sign(update)` should sign the `update` with the instance's private key. 320 | `verify(update, cb)` should verify the update, using public key associated with the 321 | `source` field in the update. Verification may be asynchronous. `verify` must callback 322 | `cb(err, boolean)` where boolean indicates whether or not the signature is valid. 323 | Only callback in error in the most extreme circumstances. 324 | If there was no known key for the required source then that should be treated as a 325 | verification failure. If it is not possible to reach the key database (or whatever) 326 | then the request should be retried until it is available. 327 | 328 | > Note: although the API supports asynchronous verification, 329 | > it's probably a good idea to load keys into memory so that messages can be verified 330 | > and signed synchronously. 331 | 332 | `createId()` returns a new id for the current node. This is used in the example security 333 | implementation to return a id that is a hash of the public key. This makes it impossible 334 | for rogue nodes to attempt to associate a old node id with a new public key. 335 | 336 | ## Generating Keys. 337 | 338 | Generate an ssh private key, and a PEM encoded public key. 339 | ``` 340 | ssh-keygen -f $KEYNAME -b $LENGTH -N $PASSWORD -q 341 | ssh-keygen -e -f $KEYNAME.pub -m PEM > $KEYNAME.pem 342 | 343 | ``` 344 | `$LENGTH` must be `>= 786`, shorter is faster but less secure. 345 | password may be empty `''`. 346 | 347 | `$KEYNAME` is the private key, and `$KEYNAME.pem` is the public key 348 | to use with Scuttlebutt. 349 | 350 | -------------------------------------------------------------------------------- /events.js: -------------------------------------------------------------------------------- 1 | var Scuttlebutt = require('./') 2 | var inherits = require('util').inherits 3 | var each = require('iterate').each 4 | var u = require('./util') 5 | var EventEmitter = require('events').EventEmitter 6 | 7 | module.exports = ReliableEventEmitter 8 | 9 | inherits(ReliableEventEmitter, Scuttlebutt) 10 | 11 | function ReliableEventEmitter (opts) { 12 | if(!(this instanceof ReliableEventEmitter)) return new ReliableEventEmitter(opts) 13 | Scuttlebutt.call(this, opts) 14 | } 15 | 16 | var emit = EventEmitter.prototype.emit 17 | var emitter = ReliableEventEmitter.prototype 18 | 19 | emitter.emit = function (event) { 20 | if(event === '__proto__') 21 | throw new Error('__proto__ is illegal event name') 22 | var args = [].slice.call(arguments) 23 | if(event == 'newListener') 24 | return emit.apply(this, args) 25 | return this.localUpdate(args) 26 | } 27 | 28 | var on = EventEmitter.prototype.on 29 | 30 | emitter.on = function (event, listener) { 31 | if(event === '__proto__') 32 | throw new Error('__proto__ is invalid event') 33 | return on.call(this, event, listener) 34 | } 35 | 36 | emitter.applyUpdate = function (update) { 37 | var key = update[0][0] 38 | this.events = this.events || {} 39 | this.events[key] = this.events[key] || [] 40 | this.events[key].push(update) 41 | //emit the event. 42 | emit.apply(this, update[0]) 43 | return true 44 | } 45 | 46 | 47 | emitter.history = function (filter) { 48 | var self = this 49 | var h = [] 50 | this.events = this.events || {} 51 | each(this.events, function (es) { 52 | each(es, function (e) { 53 | if(u.filter(e, filter)) 54 | h.push(e) 55 | }) 56 | }) 57 | return u.sort(h) 58 | } 59 | 60 | 61 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var i = require('iterate') 3 | var duplex = require('duplex') 4 | var inherits = require('util').inherits 5 | var serializer = require('stream-serializer') 6 | var u = require('./util') 7 | var timestamp = require('monotonic-timestamp') 8 | 9 | exports = 10 | module.exports = Scuttlebutt 11 | 12 | exports.createID = u.createID 13 | exports.updateIsRecent = u.filter 14 | exports.filter = u.filter 15 | exports.timestamp = timestamp 16 | 17 | function dutyOfSubclass() { 18 | throw new Error('method must be implemented by subclass') 19 | } 20 | 21 | function validate (data) { 22 | if(!(Array.isArray(data) 23 | && 'string' === typeof data[2] 24 | && '__proto__' !== data[2] //THIS WOULD BREAK STUFF 25 | && 'number' === typeof data[1] 26 | )) return false 27 | 28 | return true 29 | } 30 | 31 | inherits (Scuttlebutt, EventEmitter) 32 | 33 | function Scuttlebutt (opts) { 34 | 35 | if(!(this instanceof Scuttlebutt)) return new Scuttlebutt(opts) 36 | var id = 'string' === typeof opts ? opts : opts && opts.id 37 | this.sources = {} 38 | this.setMaxListeners(Number.MAX_VALUE) 39 | //count how many other instances we are replicating to. 40 | this._streams = 0 41 | if(opts && opts.sign && opts.verify) { 42 | this.setId(opts.id || opts.createId()) 43 | this._sign = opts.sign 44 | this._verify = opts.verify 45 | } else { 46 | this.setId(id || u.createId()) 47 | } 48 | } 49 | 50 | var sb = Scuttlebutt.prototype 51 | 52 | var emit = EventEmitter.prototype.emit 53 | 54 | sb.applyUpdate = dutyOfSubclass 55 | sb.history = dutyOfSubclass 56 | 57 | sb.localUpdate = function (trx) { 58 | this._update([trx, timestamp(), this.id]) 59 | return this 60 | } 61 | 62 | sb._update = function (update) { 63 | //validated when it comes into the stream 64 | var ts = update[1] 65 | var source = update[2] 66 | 67 | //if this message is old for it's source, ignore it. it's out of 68 | //order. each node must emit it's changes in order! 69 | //emit an 'old_data' event because i'll want to track how many 70 | //unnecessary messages are sent. 71 | 72 | var latest = this.sources[source] 73 | if(latest && latest >= ts) 74 | return emit.call(this, 'old_data', update), false 75 | 76 | this.sources[source] = ts 77 | 78 | var self = this 79 | function didVerification (err, verified) { 80 | 81 | // I'm not sure how what should happen if a async verification 82 | // errors. if it's an key not found - that is a verification fail, 83 | // not a error. if it's genunie error, really you should queue and 84 | // try again? or replay the message later 85 | // -- this should be done my the security plugin though, not scuttlebutt. 86 | 87 | if(err) 88 | return emit.call(self, 'error', err) 89 | 90 | if(!verified) 91 | return emit.call(self, 'unverified_data', update) 92 | 93 | if(self.applyUpdate(update)) 94 | emit.call(self, '_update', update) //write to stream. 95 | } 96 | 97 | if(source !== this.id) { 98 | if(this._verify) 99 | this._verify(update, didVerification) 100 | else 101 | didVerification(null, true) 102 | } else { 103 | if(this._sign) { 104 | //could make this async easily enough. 105 | update[3] = this._sign(update) 106 | } 107 | didVerification(null, true) 108 | } 109 | 110 | return true 111 | } 112 | 113 | sb.createStream = function (opts) { 114 | var self = this 115 | //the sources for the remote end. 116 | var sources = {}, other 117 | var syncSent = false, syncRecv = false 118 | 119 | this._streams ++ 120 | 121 | opts = opts || {} 122 | var d = duplex() 123 | d.name = opts.name 124 | var outer = serializer(opts && opts.wrapper)(d) 125 | outer.inner = d 126 | 127 | d.writable = opts.writable !== false 128 | d.readable = opts.readable !== false 129 | 130 | syncRecv = !d.writable 131 | syncSent = !d.readable 132 | 133 | var tail = opts.tail !== false //default to tail=true 134 | 135 | function start (data) { 136 | //when the digest is recieved from the other end, 137 | //send the history. 138 | //merge with the current list of sources. 139 | if (!data || !data.clock) { 140 | d.emit('error'); 141 | return d._end() 142 | } 143 | 144 | sources = data.clock 145 | 146 | i.each(self.history(sources), function (data) {d._data(data)}) 147 | 148 | //the _update listener must be set after the history is queued. 149 | //otherwise there is a race between the first client message 150 | //and the next update (which may come in on another stream) 151 | //this problem will probably not be encountered until you have 152 | //thousands of scuttlebutts. 153 | 154 | self.on('_update', onUpdate) 155 | 156 | d._data('SYNC') 157 | syncSent = true 158 | //when we have sent all history 159 | outer.emit('header', data) 160 | outer.emit('syncSent') 161 | //when we have recieved all histoyr 162 | //emit 'synced' when this stream has synced. 163 | if(syncRecv) outer.emit('sync'), outer.emit('synced') 164 | if(!tail) d._end() 165 | } 166 | 167 | d 168 | .on('_data', function (data) { 169 | //if it's an array, it's an update. 170 | if(Array.isArray(data)) { 171 | //check whether we are accepting writes. 172 | if(!d.writable) 173 | return 174 | if(validate(data)) 175 | return self._update(data) 176 | } 177 | //if it's an object, it's a scuttlebut digest. 178 | else if('object' === typeof data && data) 179 | start(data) 180 | else if('string' === typeof data && data == 'SYNC') { 181 | syncRecv = true 182 | outer.emit('syncRecieved') 183 | if(syncSent) outer.emit('sync'), outer.emit('synced') 184 | } 185 | }).on('_end', function () { 186 | d._end() 187 | }) 188 | .on('close', function () { 189 | self.removeListener('_update', onUpdate) 190 | self.removeListener('dispose', dispose) 191 | //emit the number of streams that are remaining... 192 | //this will be used for memory management... 193 | self._streams -- 194 | emit.call(self, 'unstream', self._streams) 195 | }) 196 | 197 | if(opts && opts.tail === false) { 198 | outer.on('sync', function () { 199 | process.nextTick(function () { 200 | d._end() 201 | }) 202 | }) 203 | } 204 | function onUpdate (update) { //value, source, ts 205 | if(!validate(update) || !u.filter(update, sources)) 206 | return 207 | 208 | d._data(update) 209 | 210 | //really, this should happen before emitting. 211 | var ts = update[1] 212 | var source = update[2] 213 | sources[source] = ts 214 | } 215 | 216 | function dispose () { 217 | d.end() 218 | } 219 | 220 | var outgoing = { id : self.id, clock : self.sources } 221 | 222 | if (opts && opts.meta) outgoing.meta = opts.meta 223 | 224 | if(d.readable) { 225 | d._data(outgoing) 226 | if(!d.writable && !opts.clock) 227 | start({clock:{}}) 228 | 229 | } else if (opts.sendClock) { 230 | //send my current clock. 231 | //so the other side knows what to send 232 | d._data(outgoing) 233 | } 234 | 235 | self.once('dispose', dispose) 236 | 237 | return outer 238 | } 239 | 240 | sb.createWriteStream = function (opts) { 241 | opts = opts || {} 242 | opts.writable = true; opts.readable = false 243 | return this.createStream(opts) 244 | } 245 | 246 | sb.createReadStream = function (opts) { 247 | opts = opts || {} 248 | opts.writable = false; opts.readable = true 249 | return this.createStream(opts) 250 | } 251 | 252 | sb.dispose = function () { 253 | emit.call(this, 'dispose') 254 | } 255 | 256 | sb.setId = function (id) { 257 | if('__proto__' === id) throw new Error('__proto__ is invalid id') 258 | if(id == null) throw new Error('null is not invalid id') 259 | this.id = id 260 | return this 261 | } 262 | 263 | function streamDone(stream, listener) { 264 | 265 | function remove () { 266 | stream.removeListener('end', onDone) 267 | stream.removeListener('error', onDone) 268 | stream.removeListener('close', onDone) 269 | } 270 | function onDone (arg) { 271 | remove() 272 | listener.call(this, arg) 273 | } 274 | 275 | //this makes emitter.removeListener(event, listener) still work 276 | onDone.listener = listener 277 | 278 | stream.on('end', onDone) 279 | stream.on('error', onDone) 280 | stream.on('close', onDone) 281 | } 282 | 283 | //create another instance of this scuttlebutt, 284 | //that is in sync and attached to this instance. 285 | sb.clone = function () { 286 | var A = this 287 | var B = new (A.constructor) 288 | B.setId(A.id) //same id. think this will work... 289 | 290 | A._clones = (A._clones || 0) + 1 291 | 292 | var a = A.createStream({wrapper: 'raw'}) 293 | var b = B.createStream({wrapper: 'raw'}) 294 | 295 | //all updates must be sync, so make sure pause never happens. 296 | a.pause = b.pause = function noop(){} 297 | 298 | streamDone(b, function () { 299 | A._clones-- 300 | emit.call(A, 'unclone', A._clones) 301 | }) 302 | 303 | a.pipe(b).pipe(a) 304 | //resume both streams, so that the new instance is brought up to date immediately. 305 | a.resume() 306 | b.resume() 307 | 308 | return B 309 | } 310 | 311 | -------------------------------------------------------------------------------- /keygen: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | [ "x$1" = x ] && { 4 | echo "USAGE ./keygen NAME"; 5 | exit 1; 6 | } 7 | 8 | ssh-keygen -q -f "$1" -b 768 -N '' 9 | ssh-keygen -e -f "$1".pub -m PEM > "$1".pem 10 | 11 | -------------------------------------------------------------------------------- /model.js: -------------------------------------------------------------------------------- 1 | var Scuttlebutt = require('./index') 2 | var inherits = require('util').inherits 3 | var each = require('iterate').each 4 | var u = require('./util') 5 | 6 | module.exports = Model 7 | 8 | inherits(Model, Scuttlebutt) 9 | 10 | function Model (opts) { 11 | if(!(this instanceof Model)) return new Model(opts) 12 | Scuttlebutt.call(this, opts) 13 | this.store = {} 14 | } 15 | 16 | var m = Model.prototype 17 | 18 | m.set = function (k, v) { 19 | if(k==='__proto__') return u.protoIsIllegal(this) 20 | this.localUpdate([k, v]) 21 | return this 22 | } 23 | 24 | 25 | m.get = function (k) { 26 | if(k==='__proto__') return u.protoIsIllegal(this) 27 | if(this.store[k]) 28 | return this.store[k][0][1] 29 | } 30 | 31 | m.keys = function () { 32 | var a = [] 33 | for (var k in this.store) 34 | a.push(k) 35 | return a 36 | } 37 | 38 | m.forEach = 39 | m.each = function (iter) { 40 | for (var k in this.store) 41 | iter(this.store[k][0][1], k, this.store) 42 | return this 43 | } 44 | 45 | //return this history since sources. 46 | //sources is a hash of { ID: TIMESTAMP } 47 | 48 | m.applyUpdate = function (update) { 49 | var key = update[0][0] 50 | if('__proto__' === key) return u.protoIsIllegal(this) 51 | //ignore if we already have a more recent value 52 | 53 | if('undefined' !== typeof this.store[key] 54 | && this.store[key][1] > update[1]) 55 | return this.emit('_remove', update) 56 | 57 | if(this.store[key]) this.emit('_remove', this.store[key]) 58 | 59 | this.store[key] = update 60 | 61 | this.emit.apply(this, ['update'].concat(update)) 62 | this.emit('change', key, update[0][1]) 63 | this.emit('change:'+key, update[0][1]) 64 | 65 | return true 66 | } 67 | 68 | m.history = function (sources) { 69 | var self = this 70 | var h = [] 71 | each(this.store, function (e) { 72 | if(u.filter(e, sources)) 73 | h.push(e) 74 | }) 75 | return u.sort(h) 76 | } 77 | 78 | m.toJSON = function () { 79 | var o = {}, notNull = false 80 | for (var k in this.store) { 81 | var v = this.get(k) 82 | if(v != null) 83 | o[k] = this.get(k) 84 | } 85 | return o 86 | } 87 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "Dominic Tarr (http://dominictarr.com)", 3 | "name": "scuttlebutt", 4 | "description": "replicate data via scuttlebutt protocol", 5 | "version": "5.6.15", 6 | "homepage": "https://github.com/dominictarr/scuttlebutt", 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/dominictarr/scuttlebutt.git" 10 | }, 11 | "dependencies": { 12 | "iterate": "0.1.0", 13 | "duplex": "~1.0.0", 14 | "stream-serializer": "1.1.2", 15 | "monotonic-timestamp": "~0.0.8" 16 | }, 17 | "devDependencies": { 18 | "macgyver": "~1.10", 19 | "event-stream": "~3.0", 20 | "tape": "~0.1.5", 21 | "request": "~2.16.6" 22 | }, 23 | "testling": { 24 | "files": "test/*.js", 25 | "browsers": { 26 | "ie": [ 27 | 8, 28 | 9, 29 | 10 30 | ], 31 | "firefox": [ 32 | 17, 33 | 18 34 | ], 35 | "chrome": [ 36 | 23, 37 | 24 38 | ], 39 | "safari": [ 40 | 5, 41 | 6 42 | ], 43 | "opera": [ 44 | 12 45 | ] 46 | } 47 | }, 48 | "scripts": { 49 | "test": "set -e; for test in test/*.js; do node $test; done" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /security.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto') 2 | 3 | // 4 | //security is pluggable - or can be left out. 5 | // 6 | //there is a sign method, and a verify method (which may be async) 7 | //hmm, after thinking through some possible attacks, 8 | //https://github.com/dominictarr/scuttlebutt/issues/6 9 | //I've realized that it is necessary to let the security plugin set 10 | //the ID. hmm, that means there will need to be an async initialization step... 11 | // 12 | //The security will either need to shell out to `ssh-keygen` or to read a key pair 13 | //from the file system. hmm, There should only be a single instance of security per 14 | //process (except when testing), so maybe init security, then pass it to a new scuttlebutt) 15 | //there may be many scuttlebutt instances but they should use the same key pair. 16 | // 17 | //that is up to security - maybe init the security when the app starts - indeed, 18 | //probably generate the key during installation, and then `readFileSync()` 19 | // 20 | //Yup, because we don't want to be regenerating that stuff live. 21 | // 22 | // we do need the security to be able to set the key though! 23 | 24 | //exactly what init api the security plugin wants to use is it's own business. 25 | //it doesn't have to pass keys in like this... 26 | 27 | var algorithm = 'RSA-SHA1' 28 | var format = 'base64' 29 | var hashAlg = 'SHA1' 30 | 31 | module.exports = function (keys, 32 | //THIS IS SERIOUS BUSINESS! 33 | PRIVATE, PUBLIC 34 | //THEREFORE THE CAPS MUST BE LOCKED 35 | ) { 36 | return { 37 | sign: function (update) { 38 | var data = JSON.stringify(update) 39 | return crypto.createSign(algorithm).update(data).sign(PRIVATE, format) 40 | }, 41 | verify: function (update, cb) { 42 | var _update = update.slice() 43 | var sig = _update.pop() 44 | var id = update[2] 45 | var data = JSON.stringify(_update) 46 | var key = keys[id] 47 | if(!key) return cb(null, false) 48 | cb(null, crypto.createVerify(algorithm).update(data).verify(key, sig, format)) 49 | }, 50 | createId: function () { 51 | //hash of public key. 52 | return crypto.createHash(hashAlg).update(PUBLIC).digest(format) 53 | }, 54 | publicKey: PUBLIC 55 | } 56 | } 57 | 58 | -------------------------------------------------------------------------------- /test/cleanup.js: -------------------------------------------------------------------------------- 1 | require('tape')('cleanup', function (t) { 2 | var Events = require('../events') 3 | var e = new Events() 4 | 5 | //test that _update listeners are cleaned up! 6 | 7 | var l = 20 8 | function cs () { 9 | console.log('cs', l, e.listeners('_update').length) 10 | var s = e.createStream() 11 | 12 | if(l--) 13 | process.nextTick(function () { 14 | s.end() 15 | cs() 16 | t.ok(e.listeners('_update').length <= 2) 17 | }) 18 | else 19 | t.end() 20 | 21 | } 22 | 23 | cs() 24 | 25 | 26 | }) 27 | -------------------------------------------------------------------------------- /test/clone.js: -------------------------------------------------------------------------------- 1 | 2 | var Model = require('../model') 3 | var tape = require('tape') 4 | 5 | tape('clone() -> another instance', function (t) { 6 | var a = new Model() 7 | var b = a.clone() 8 | t.equal(b.constructor, Model) 9 | t.end() 10 | }) 11 | 12 | tape('clone() -> deepEqual history', function (t) { 13 | var a = new Model() 14 | a.set('foo', 'bar') 15 | var b = a.clone() 16 | 17 | t.deepEqual(b.history(), a.history()) 18 | t.end() 19 | 20 | }) 21 | 22 | tape('clone() -> updates apply to both instances', function (t) { 23 | var a = new Model() 24 | a.set('foo', 'bar') 25 | 26 | var b = a.clone() 27 | t.deepEqual(b.history(), a.history()) 28 | 29 | b.set('quux', 'zaff') 30 | t.deepEqual(b.history(), a.history()) 31 | 32 | t.end() 33 | }) 34 | 35 | 36 | tape('clone() -> dispose triggers unclone event', function (t) { 37 | var a = new Model(), uncloned = false 38 | a.set('foo', 'bar') 39 | 40 | var b = a.clone() 41 | t.deepEqual(b.history(), a.history()) 42 | 43 | t.equal(a._clones, 1) 44 | 45 | b.set('quux', 'zaff') 46 | t.deepEqual(b.history(), a.history()) 47 | 48 | a.on('unclone', function (clones) { 49 | uncloned = true 50 | t.equal(clones, 0, 'should have zero clones') 51 | }) 52 | 53 | b.dispose() 54 | 55 | t.equal(uncloned, true) 56 | t.equal(a._clones, 0) 57 | t.end() 58 | }) 59 | -------------------------------------------------------------------------------- /test/dispose.js: -------------------------------------------------------------------------------- 1 | require('tape')('dispose', function (t) { 2 | /* 3 | we need to be able to stream the state of the model to disk, 4 | and then call doc.dispose() 5 | and end the stream. 6 | */ 7 | 8 | var EE = require('../events') 9 | 10 | var es = require('event-stream') 11 | var mac = require('macgyver')().autoValidate() 12 | 13 | var emitter = new EE() 14 | var ended = false 15 | 16 | var fs = require('fs') 17 | 18 | var s = emitter.createReadStream({tail: true}) 19 | s.on('readable', function () { 20 | console.log('READABLE') 21 | }) 22 | // s.pipe(fs.createWriteStream('/tmp/dispose-test')) 23 | s.pipe(es.writeArray(function (err, ary) { 24 | console.log(ary) 25 | ended = true 26 | })) 27 | 28 | emitter.emit('message', 'hello') 29 | emitter.emit('message', 'hello') 30 | 31 | emitter.on('dispose', function () { 32 | console.log('DISPOSE') 33 | }) 34 | 35 | process.nextTick(function () { 36 | console.log('dispose') 37 | emitter.dispose() 38 | t.ok(ended, 'dispose must trigger end on all streams') 39 | t.end() 40 | }) 41 | 42 | }) 43 | -------------------------------------------------------------------------------- /test/events.js: -------------------------------------------------------------------------------- 1 | require('tape')('events', function (t) { 2 | var ReliableEventEmitter = require('../events') 3 | var mac = require('macgyver')().autoValidate() 4 | 5 | function allow (update, cb) { 6 | return cb(null, true) 7 | } 8 | 9 | 10 | var A = new ReliableEventEmitter('a') 11 | var B = new ReliableEventEmitter('b') 12 | 13 | function log (data) { 14 | console.log('LOG', this.id, data) 15 | } 16 | 17 | function old (data) { 18 | console.log('OLD', data, 19 | this.sources[data[2]]) 20 | } 21 | 22 | var _a = [], _b = [] 23 | 24 | A.on('a', log) 25 | A.on('a', mac(function (data) { _a.push(data) }).times(6)) 26 | 27 | B.on('a', log) 28 | B.on('a', mac(function (data) { _b.push(data) }).times(6)) 29 | 30 | A.emit('a', 'aardvark') 31 | A.emit('a', 'antelope') 32 | A.emit('a', 'anteater') 33 | 34 | B.emit('a', 'armadillo') 35 | B.emit('a', 'alligator') 36 | B.emit('a', 'amobea') 37 | 38 | var s 39 | (s = A.createStream()).pipe(B.createStream()).pipe(s) 40 | 41 | process.nextTick(function () { 42 | t.deepEqual(_a.sort(), _b.sort()) 43 | console.log(_a.sort()) 44 | t.end() 45 | }) 46 | 47 | }) 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/header.js: -------------------------------------------------------------------------------- 1 | require('tape')('header', function (t) { 2 | 3 | var mac = require('macgyver')().autoValidate() 4 | 5 | var Model = require('../model') 6 | 7 | var a = new Model() 8 | var b = new Model() 9 | var c = new Model() 10 | 11 | var as = a.createStream() 12 | var bs = b.createStream() 13 | var b2s = b.createStream() 14 | var cs = c.createStream() 15 | 16 | var n = 3 17 | 18 | as.on('header', mac(function (h) { 19 | t.equal(h.id, b.id) 20 | end() 21 | }).once()) 22 | 23 | var ix = 0 24 | bs.on('header', mac(function (h) { 25 | t.equal(h.id, ix === 0 ? a.id : c.id) 26 | ix ++ 27 | end() 28 | }).once()) 29 | 30 | b2s.on('header', mac(function (h) { 31 | t.equal(h.id, ix === 0 ? a.id : c.id) 32 | ix ++ 33 | end() 34 | }).once()) 35 | 36 | cs.on('header', mac(function (h) { 37 | t.equal(h.id, b.id) 38 | end() 39 | }).once()) 40 | 41 | as.pipe(bs).pipe(as) 42 | b2s.pipe(cs).pipe(b2s) 43 | 44 | function end () { 45 | if(--n) return 46 | t.end() 47 | } 48 | 49 | }) 50 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | 3 | var gossip = require('../model') 4 | var i = require('iterate') 5 | 6 | var timestamp = require('monotonic-timestamp') 7 | 8 | var createId = require('../util').createId 9 | 10 | function test(name, test) { 11 | console.log('#', name) 12 | tape(name, function (t) { 13 | test(gossip(), t) 14 | }) 15 | } 16 | 17 | test('updates appear in histroy', function (g, t) { 18 | var key = 'key' 19 | var value = Math.random() 20 | var source = 'source' //gossip.createID() 21 | var ts = timestamp() 22 | 23 | 24 | t.equal(g._update([[key, value], ts, source]) 25 | , true 26 | , 'update returns true to indicate was not old') 27 | 28 | console.log(g.store) 29 | t.equal(g.get(key), value) 30 | 31 | t.deepEqual(g.history(), [[['key', value], ts, source]]) 32 | 33 | var value2 = Math.random() 34 | //older timestamps are not appled. 35 | t.equal(g._update([[key, value2], ts - 1, source]) 36 | , false 37 | , 'write returns false to indicate update did not apply') 38 | 39 | //the second update was older, so must not be in the history 40 | t.deepEqual(g.history(), [[['key', value], ts, source]]) 41 | 42 | t.equal(g.get(key), value) 43 | 44 | t.end() 45 | }) 46 | 47 | test('can filter histroy with {sources: timestamps}', function (g, t) { 48 | var A = '#A' 49 | var B = '#B' 50 | var C = '#C' 51 | var ts = timestamp() 52 | 53 | g._update([['A', 'aaa'], ts, A]) 54 | g._update([['B', 'bbb'], ts, B]) 55 | g._update([['C', 'ccc'], ts, C]) 56 | 57 | //filter should only return timestamps that are after 58 | //the given timestamps. 59 | var filter = {} 60 | filter[A] = ts 61 | filter[B] = ts 62 | filter[C] = ts 63 | 64 | t.deepEqual( 65 | g.history(filter) 66 | , []) 67 | 68 | filter[B] = ts - 1 69 | 70 | t.deepEqual( 71 | g.history(filter) 72 | , [[['B', 'bbb'], ts, B]]) 73 | 74 | //if an item is not available, it 75 | 76 | filter[C] = null 77 | t.deepEqual( 78 | g.history(filter) 79 | , [ [['B', 'bbb'], ts, B] 80 | , [['C', 'ccc'], ts, C]]) 81 | 82 | t.end() 83 | }) 84 | 85 | -------------------------------------------------------------------------------- /test/integrate.js: -------------------------------------------------------------------------------- 1 | require('tape')('integrate 1', function (t) { 2 | 3 | var gossip = require('../model') 4 | var assert = require('assert') 5 | 6 | var g1 = gossip() 7 | var g2 = gossip() 8 | var s1, s2 9 | (s1 = g1.createStream()) 10 | .pipe(s2 = g2.createStream()).pipe(s1) 11 | 12 | s1.on('data', function (d) { console.log("s1", d)}) 13 | s2.on('data', function (d) { console.log("s2", d)}) 14 | 15 | //I like to have streams that work sync. 16 | //if you can do that, you know it's tight. 17 | s1.resume() 18 | s2.resume() 19 | 20 | //process.nextTick(function () { 21 | 22 | var value = Math.random() 23 | 24 | g1.set('key', value) 25 | 26 | t.equal(g2.get('key'), g1.get('key')) 27 | t.end() 28 | }) 29 | -------------------------------------------------------------------------------- /test/integrate2.js: -------------------------------------------------------------------------------- 1 | require('tape')('integrate 2', function (t) { 2 | var gossip = require('../model') 3 | 4 | var g1 = gossip() 5 | var g2 = gossip() 6 | var g3 = gossip() 7 | 8 | function sync(g, h) { 9 | var s = g.createStream({wrapper: 'raw'}) 10 | var r = h.createStream({wrapper: 'raw'}) 11 | g.on('old_data', function (d) { 12 | console.log('old_data', d, g.id, h.id) 13 | }) 14 | g.on('update', function () { 15 | console.log(g.id, 'key', g.get('key')) 16 | }) 17 | s.pipe(r).pipe(s) 18 | s.resume() 19 | r.resume() 20 | } 21 | 22 | sync(g1, g2) 23 | sync(g2, g3) 24 | sync(g3, g1) 25 | 26 | var value = Math.random() 27 | 28 | g1.set('key', value) 29 | 30 | t.equal(g3.get('key'), g1.get('key')) 31 | t.equal(g2.get('key'), g1.get('key')) 32 | 33 | t.end() 34 | }) 35 | -------------------------------------------------------------------------------- /test/iterate.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var Model = require('../model') 3 | 4 | test('keys', function (t) { 5 | 6 | var m = Model() 7 | 8 | m.set('a', 1) 9 | m.set('o', 2) 10 | m.set('e', 3) 11 | m.set('u', 4) 12 | m.set('i', 5) 13 | 14 | t.deepEqual(m.keys(), 'aoeui'.split('')) 15 | var s = 0 16 | 17 | m.each(function (v) { 18 | console.log(v) 19 | s = s + v 20 | }) 21 | 22 | t.equal(s, 1+2+3+4+5) 23 | 24 | t.end() 25 | 26 | }) 27 | -------------------------------------------------------------------------------- /test/keys/test1: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIBywIBAAJhAM6LVHtQlBQat6J2v03EefWS9DbVj9/CKTeRrFNguWIi/vfiSlFA 3 | I+CakhbIiIXdmzgM1avAKCwlThUWFmdcJ5x1sb7FoSCAb5OtirZnMxPpcYK/ewAI 4 | pcihorgZ2u92BwIDAQABAmEAxdvJMVxOjjfN5G928Yue/XYqRoEtE0APdfExsKm+ 5 | zVkTAOmhIdacx+OqwPKXfg17kwx7VLZVmZePNmZhztM2NdiMtboQx2GUV/la+f9O 6 | pRpqorL3du9heQQYo4q0hxCxAjEA+ffOhjnaZfv64XRgWccloUmqubpOKsDXjlhU 7 | O9gWfmHKKVd+vrpEkgUZCVACXisJAjEA04dFNkK/cYMylJzL9RqPMMUSfTKbMrVE 8 | JzgxN5UKw1WGIMXp7ByzKibh2f+DCQyPAjAPt9ZjuFWUXiDrdl7spkomdzRmE2IA 9 | 7DlhuQoq7S6U6d9FdDwDEEFpkSp+3GoZs8kCMQCm0poNLxsZBOWROw/HoEipp+Lr 10 | BkxL85VqcPCv60VvxDViB3RzGDdc2QlqCg9nxZUCMDPQp5X9CyEygiRRCqNIbDnR 11 | fx5/eYd5KSmGXR6jgJIw+lrsjkvKkLSQG65zHRZ6ZQ== 12 | -----END RSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /test/keys/test1.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PUBLIC KEY----- 2 | MGgCYQDOi1R7UJQUGreidr9NxHn1kvQ21Y/fwik3kaxTYLliIv734kpRQCPgmpIW 3 | yIiF3Zs4DNWrwCgsJU4VFhZnXCecdbG+xaEggG+TrYq2ZzMT6XGCv3sACKXIoaK4 4 | GdrvdgcCAwEAAQ== 5 | -----END RSA PUBLIC KEY----- 6 | -------------------------------------------------------------------------------- /test/keys/test1.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQDOi1R7UJQUGreidr9NxHn1kvQ21Y/fwik3kaxTYLliIv734kpRQCPgmpIWyIiF3Zs4DNWrwCgsJU4VFhZnXCecdbG+xaEggG+TrYq2ZzMT6XGCv3sACKXIoaK4Gdrvdgc= dominic@asus 2 | -------------------------------------------------------------------------------- /test/keys/test2: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIBywIBAAJhAOucT/+VWAr8a0bf6BQuz4y+lvB1RZy434CPr3u+LNVRpiZJ/CDa 3 | U2XAX6S2sxx9irlIB1hIe4CB6J1UOBBfaOG3mz4b2AJ1ypr1ExjtvnM2h97C9GXD 4 | 0WKGaogV9j1TMwIDAQABAmBhttZGvX1iMQWcw0fQ9wkE6hZXa4uPT+0BuJWi0GBv 5 | qAksYAq85U8pJ/uCqHLoIfgWzRXj8rKxSvz8zUTqCkropWahIlDsvwixkIkHTzNw 6 | /FjSgFyT58x8+VaUtWkk1AECMQD8hebvdXiVtYo8zji1Xojc/B/mVFPusg3TLp41 7 | xJ0KIYR/sAevQjc/oeonUjd9SLMCMQDu2swLwxZ4cIimbTD6yh/sxK5n7tQ4YoXh 8 | 8pZExhhnihbcLj9ea0uAdaD8LFAUC4ECMQD1zZ1yf6VGDPUnpRD8Mq4EdYLToEgm 9 | 87iTVTB5ZA38241vATkptsmyrfgQGG6dDBMCMHBcwUxvM+zok4AnMbloyGfrhlgi 10 | Q/dacbz/D62+utBKZ8KghvL16oi9zUOT3P/xAQIxAPbCCFofzX2rlEdoEW+TE3Rl 11 | dKfHJ9pYl/P0zPkTPNoP7crGGS11Q/HcONOUHO/wtQ== 12 | -----END RSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /test/keys/test2.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PUBLIC KEY----- 2 | MGgCYQDrnE//lVgK/GtG3+gULs+MvpbwdUWcuN+Aj697vizVUaYmSfwg2lNlwF+k 3 | trMcfYq5SAdYSHuAgeidVDgQX2jht5s+G9gCdcqa9RMY7b5zNofewvRlw9FihmqI 4 | FfY9UzMCAwEAAQ== 5 | -----END RSA PUBLIC KEY----- 6 | -------------------------------------------------------------------------------- /test/keys/test2.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQDrnE//lVgK/GtG3+gULs+MvpbwdUWcuN+Aj697vizVUaYmSfwg2lNlwF+ktrMcfYq5SAdYSHuAgeidVDgQX2jht5s+G9gCdcqa9RMY7b5zNofewvRlw9FihmqIFfY9UzM= dominic@asus 2 | -------------------------------------------------------------------------------- /test/meta.js: -------------------------------------------------------------------------------- 1 | require('tape')('meta', function (t) { 2 | var mac = require('macgyver')().autoValidate() 3 | 4 | var Model = require('../model') 5 | 6 | var a = new Model() 7 | var b = new Model() 8 | var c = new Model() 9 | 10 | var as = a.createStream({ meta: 'A' }) 11 | var bs = b.createStream({ meta: 'B' }) 12 | var cs = c.createStream({ meta: 'C' }) 13 | 14 | as.on('header', mac(function (h) { 15 | t.equal(h.id, b.id) 16 | t.equal(h.meta, 'B') 17 | }).once()) 18 | 19 | var ix = 0 20 | bs.on('header', mac(function (h) { 21 | t.equal(h.id, ix === 0 ? a.id : c.id) 22 | t.equal(h.meta, ix === 0 ? 'A' : 'C') 23 | ix ++ 24 | }).times(2)) 25 | 26 | cs.on('header', mac(function (h) { 27 | t.equal(h.id, b.id) 28 | t.equal(h.meta, 'B') 29 | t.end() 30 | }).once()) 31 | 32 | as.pipe(bs).pipe(as) 33 | bs.pipe(cs).pipe(bs) 34 | 35 | }) 36 | -------------------------------------------------------------------------------- /test/model.js: -------------------------------------------------------------------------------- 1 | require('tape')('model', function (t) { 2 | var Model = require('../model') 3 | var mac = require('macgyver')().autoValidate() 4 | 5 | var a = new Model() 6 | 7 | var expected = { 8 | key: Math.random() 9 | } 10 | 11 | t.deepEqual({}, a.toJSON()) 12 | 13 | a.once('change', mac(function (key, value) { 14 | t.ok(expected[key] !== undefined) 15 | t.equal(value, expected[key]) 16 | t.deepEqual(a.toJSON(), expected) 17 | next() 18 | }).atLeast(1)) 19 | 20 | a.once('change:key', mac(function (value) { 21 | t.equal(value, expected.key) 22 | next() 23 | }).once()) 24 | 25 | 26 | var n = 2 27 | function next () { 28 | if(--n) return 29 | 30 | a.set('key', null) 31 | t.equal(a.get('key'), null) 32 | t.deepEqual(a.toJSON(), {}) 33 | 34 | t.end() 35 | } 36 | 37 | a.set('key', expected.key) 38 | 39 | 40 | }) 41 | 42 | -------------------------------------------------------------------------------- /test/persist.js: -------------------------------------------------------------------------------- 1 | require('tape')('persist', function (t) { 2 | //need a stream that ends after it has syncronized two scuttlebutts. 3 | 4 | var EE = require('../events') 5 | var es = require('event-stream') 6 | var mac = require('macgyver')() 7 | 8 | mac.autoValidate() 9 | 10 | var a = new EE() 11 | var b = new EE() 12 | 13 | var ary = [] 14 | 15 | a.createReadStream(/*{wrapper: 'raw'}*/) 16 | .pipe(es.log('>>')) 17 | .on('end', mac('end').once()) 18 | .pipe(es.writeArray(function (_, ary) { 19 | es.from(ary).pipe(b.createWriteStream(/*{wrapper: 'raw'}*/)) 20 | .on('close', mac(function () { 21 | t.deepEqual(a.history(), b.history()) 22 | t.end() 23 | 24 | console.log('ARY', b.history()) 25 | }).once()) 26 | })) 27 | 28 | b.on('_update', mac('_update').times(3)) 29 | 30 | a.on('message', mac(function (m) { console.log(m) }).times(3)) 31 | 32 | var l = 3 33 | while(l--) { 34 | a.emit('message', 'hello_' + new Date()) 35 | } 36 | 37 | a.dispose() //end all streams 38 | 39 | }) 40 | -------------------------------------------------------------------------------- /test/secure.js: -------------------------------------------------------------------------------- 1 | require('tape')('secure', function (t) { 2 | //don't run test in the browser, because don't have readFileSync 3 | if(process.title === 'browser') return t.end() 4 | 5 | var crypto = require('crypto') 6 | var fs = require('fs') 7 | var mac = require('macgyver')() 8 | 9 | var PRIVATE = fs.readFileSync(__dirname + '/keys/test1') 10 | var PUBLIC = fs.readFileSync(__dirname + '/keys/test1.pem') 11 | 12 | var PRIVATE2 = fs.readFileSync(__dirname + '/keys/test2') 13 | var PUBLIC2 = fs.readFileSync(__dirname + '/keys/test2.pem') 14 | 15 | var keys = {} 16 | 17 | var ids = {} 18 | 19 | function getKey(id) { 20 | return keys[id] 21 | } 22 | 23 | var security = require('../security') 24 | var secure = security(keys, PRIVATE, PUBLIC) 25 | var secure2 = security(keys, PRIVATE2, PUBLIC2) 26 | var me_id 27 | keys[me_id = secure.createId()] = PUBLIC 28 | 29 | var sign = secure.sign = mac(secure.sign).atLeast(1) 30 | 31 | var verify = secure.verify = mac(secure.verify).atLeast(1) 32 | 33 | //check the verify and sing methods are correct. 34 | 35 | var update = [['id', 'value'], Date.now(), me_id] 36 | update.push(sign(update)) 37 | var isVerified = false 38 | verify(update, function (err, verified) { 39 | t.strictEqual(verified, true) 40 | isVerified = true 41 | }) 42 | t.ok(isVerified) 43 | 44 | var Emitter = require('../events') 45 | 46 | var e = new Emitter(secure) 47 | ids.e = e.id 48 | var d = new Emitter(security(keys, '', '')) 49 | 50 | //emitting from f should be ignored. because the signature is no good. 51 | var f = new Emitter(secure2) 52 | ids.f = f.id 53 | 54 | e.emit('hello', {world: true}) 55 | 56 | var es = e.createStream() 57 | 58 | es.pipe(d.createStream()).pipe(es) 59 | 60 | //this should be the signed update. 61 | d.on('hello', mac(function (message) { 62 | console.log(message) 63 | t.deepEqual(message, {world: true}) 64 | // assert.equal(id, ids.e) 65 | t.end() 66 | }).once()) 67 | 68 | var fs = f.createStream() 69 | 70 | fs.pipe(d.createStream()).pipe(fs) 71 | 72 | //should be the next update 73 | var n = 0 74 | d.on('unverified_data', mac(function (update) { 75 | t.equal(update[2], ids.f) 76 | t.equal(update[0][1], 'ignore me') 77 | t.equal(update[0][0], 'hello') 78 | }).once()) 79 | 80 | f.emit('hello', 'ignore me') 81 | }) 82 | -------------------------------------------------------------------------------- /test/sendclock.js: -------------------------------------------------------------------------------- 1 | var Model = require('../model') 2 | 3 | var a = Model('A') 4 | var b = Model('B') 5 | var c = Model('C') 6 | 7 | var tape = require('tape') 8 | 9 | tape('send clock', function (t) { 10 | 11 | var s1 = a.createStream({writable: false, sendClock: true}) 12 | var s2 = b.createStream({readable: false, sendClock: true}) 13 | 14 | s1.pipe(s2).pipe(s1) 15 | s1.resume(); s2.resume() 16 | 17 | a.set('foo', 'bar') 18 | console.log(b.get('foo')) 19 | 20 | t.equal(b.get('foo'), 'bar') 21 | 22 | b.set('foo', 'baz') 23 | 24 | //b has changed locally 25 | t.equal(b.get('foo'), 'baz') 26 | //a has NOT changed 27 | t.equal(a.get('foo'), 'bar') 28 | 29 | //set a again 30 | a.set('foo', 'bar') 31 | 32 | //b has changed locally 33 | t.equal(b.get('foo'), 'bar') 34 | 35 | var s3 = b.createStream({writable: false, sendClock: true}) 36 | var s4 = c.createStream({readable: false, sendClock: true}) 37 | 38 | s3.pipe(s4).pipe(s3) 39 | 40 | s3.resume(); s4.resume() 41 | 42 | t.equal(b.get('foo'), 'bar') 43 | 44 | t.end() 45 | }) 46 | -------------------------------------------------------------------------------- /test/sync.js: -------------------------------------------------------------------------------- 1 | require('tape')('sync', function (t) { 2 | //need a stream that ends after it has syncronized two scuttlebutts. 3 | 4 | var EE = require('../events') 5 | var assert = require('assert') 6 | var es = require('event-stream') 7 | var mac = require('macgyver')().autoValidate() 8 | 9 | var a = new EE() 10 | var b = new EE() 11 | var synced = false 12 | var as = a.createStream({end: true, wrapper: 'json', name: 'a'}) 13 | var bs = b.createStream({end: true, wrapper: 'json', name: 'b'}) 14 | 15 | as.on('sync', mac(function () { 16 | console.log('A SYNC!') 17 | synced = true 18 | t.deepEqual(a.history(), b.history()) 19 | }).once()) 20 | 21 | bs.on('sync', mac(function () { 22 | console.log('B SYNC!') 23 | next(function () { 24 | t.deepEqual(a.history(), b.history()) 25 | }) 26 | }).once()) 27 | 28 | as.on('end', function () { 29 | console.log('A.END()') 30 | }) 31 | 32 | bs.on('end', function () { 33 | console.log('B.END()') 34 | }) 35 | 36 | 37 | a.emit('event', 1) 38 | a.emit('event', 2) 39 | a.emit('event', 3) 40 | 41 | b.emit('event', 4) 42 | b.emit('event', 5) 43 | b.emit('event', 6) 44 | 45 | t.equal(synced, false) 46 | 47 | as.pipe(es.log('AB>')).pipe(bs).pipe(es.log('BA>')).pipe(as) 48 | 49 | var next = process.nextTick 50 | 51 | next(function () { 52 | 53 | console.log(a.history()) 54 | console.log(b.history()) 55 | 56 | t.deepEqual(a.history(), b.history()) 57 | t.end() 58 | }) 59 | 60 | }) 61 | -------------------------------------------------------------------------------- /test/unstream.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | 3 | var Model = require('../model') 4 | 5 | tape('unstream', function (t) { 6 | 7 | var m = new Model() 8 | 9 | var s = m.createStream() 10 | 11 | m.on('unstream', function (n) { 12 | t.equal(n, 0) 13 | t.end() 14 | }) 15 | 16 | s.end() 17 | 18 | }) 19 | 20 | 21 | tape('unstream x2', function (t) { 22 | 23 | var m = new Model() 24 | 25 | var N = 2 26 | var s = m.createStream() 27 | var z = m.createStream() 28 | 29 | m.on('unstream', function (n) { 30 | t.equal(n, --N) 31 | if(!N) t.end() 32 | }) 33 | 34 | s.end() 35 | z.end() 36 | }) 37 | 38 | tape('unstream dispose', function (t) { 39 | 40 | var m = new Model() 41 | 42 | var N = 2 43 | var s = m.createStream() 44 | var z = m.createStream() 45 | 46 | m.on('unstream', function (n) { 47 | t.equal(n, --N) 48 | if(!N) t.end() 49 | }) 50 | 51 | m.dispose() 52 | }) 53 | 54 | -------------------------------------------------------------------------------- /util.js: -------------------------------------------------------------------------------- 1 | exports.createId = 2 | function () { 3 | return [1,1,1].map(function () { 4 | return Math.random().toString(16).substring(2).toUpperCase() 5 | }).join('') 6 | } 7 | 8 | exports.filter = function (update, sources) { 9 | var ts = update[1] 10 | var source = update[2] 11 | return (!sources || !sources[source] || sources[source] < ts) 12 | } 13 | 14 | exports.protoIsIllegal = function (s) { 15 | s.emit('invalid', new Error('"__proto__" is illegal property name')) 16 | return null 17 | } 18 | 19 | function invalidUpdate(t) { 20 | t.emit('invalid', new Error('invalid update')) 21 | } 22 | 23 | exports.validUpdate = function (t, update) { 24 | if(!Array.isArray(update)) return invalidUpdate(t) 25 | if('string' !== typeof update[1] || 'number' !== typeof update[2]) 26 | return invalidUpdate(t) 27 | } 28 | 29 | exports.sort = function (hist) { 30 | return hist.sort(function (a, b) { 31 | //sort by timestamps, then ids. 32 | //there should never be a pair with equal timestamps 33 | //and ids. 34 | return a[1] - b[1] || (a[2] > b[2] ? 1 : -1) 35 | }) 36 | } 37 | --------------------------------------------------------------------------------