├── .gitignore ├── LICENSE ├── README.md ├── index.js ├── lib ├── messages.js └── protocol.js ├── messages.js ├── package.json └── schema.proto /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # peervision 2 | 3 | WIP (a live p2p streaming protocol) 4 | 5 | ``` 6 | npm install peervision 7 | ``` 8 | 9 | ## Usage 10 | 11 | ``` js 12 | var peervision = require('peervision') 13 | 14 | var producer = peervision() 15 | 16 | producer.append(new Buffer('some data')) 17 | producer.append(new Buffer('some more data')) 18 | 19 | console.log('stream id is', producer.id) 20 | 21 | var client = peervision(producer.id) 22 | 23 | var stream = client.createStream() 24 | 25 | stream.pipe(producer.createStream()).pipe(stream) 26 | 27 | client.get(0, function (err, buf) { 28 | console.log(buf) // some data 29 | }) 30 | 31 | client.get(1, function (err, buf) { 32 | console.log(buf) // some more data 33 | }) 34 | ``` 35 | 36 | THIS CURRENTLY STILL A WORK IN PROGRESS. 37 | CURRENTLY ALL DATA IS STORED IN MEMORY. 38 | 39 | ## How does it work? 40 | 41 | peervision uses a flat merkle tree where every bottom 42 | indirectly verifies the entire previous tree using [flat-tree](https://github.com/mafintosh/flat-tree) and 43 | signs the latest node using elliptic curve cryptography. 44 | 45 | (more details to be added here obviously) 46 | 47 | ## License 48 | 49 | MIT 50 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var events = require('events') 2 | var util = require('util') 3 | var protocol = require('./lib/protocol') 4 | var bufferEquals = require('buffer-equals') 5 | var signatures = require('sodium-signatures') 6 | var tree = require('flat-tree') 7 | var crypto = require('crypto') 8 | var bitfield = require('bitfield') 9 | var eos = require('end-of-stream') 10 | var map = require('numeric-id-map') 11 | 12 | var HASH_NONE = new Buffer([0]) // enum to indicate no buffer 13 | var HASH_DUPLICATE = new Buffer([1]) // enum to indicate buffer is a dup 14 | 15 | module.exports = Peervision 16 | 17 | function Peervision (id, opts) { 18 | if (!(this instanceof Peervision)) return new Peervision(id, opts) 19 | 20 | this.keypair = id ? {publicKey: id} : signatures.keyPair() 21 | this.id = this.keypair.publicKey 22 | 23 | this.tree = [] 24 | this.signatures = [] 25 | this.digests = [] 26 | this.blocks = [] 27 | this.have = bitfield(1, {grow: Infinity}) 28 | this.peers = [] 29 | 30 | this._pendingRequests = map() 31 | this._requests = map() 32 | this._head = -1 33 | 34 | events.EventEmitter.call(this) 35 | } 36 | 37 | util.inherits(Peervision, events.EventEmitter) 38 | 39 | Peervision.prototype.createStream = function () { 40 | var self = this 41 | 42 | var stream = protocol({ 43 | protocol: 1, 44 | blocks: this.have.buffer.slice(0, Math.ceil(this.blocks.length / 8)) 45 | }) 46 | 47 | stream.head = -1 48 | stream.blocks = null 49 | 50 | stream.on('handshake', function (handshake) { 51 | stream.blocks = bitfield(handshake.blocks ? handshake.blocks : 1, {grow: Infinity}) 52 | stream.head = getHead(stream.blocks) 53 | self._update() 54 | }) 55 | 56 | stream.on('have', function (have) { 57 | if (have.index > stream.head) stream.head = have.index 58 | stream.blocks.set(have.index) 59 | self._update() 60 | }) 61 | 62 | stream.on('request', function (request) { 63 | var hashes = [] 64 | for (var i = 0; i < request.tree.length; i++) { 65 | hashes[i] = self._getTree(request.tree[i]) 66 | } 67 | 68 | if (!request.digest) { 69 | self.emit('upload', request.index, self.blocks[request.index]) 70 | } 71 | 72 | stream.response({ 73 | id: request.id, 74 | tree: hashes, 75 | signature: request.signature ? self.signatures[request.index] : null, 76 | data: request.digest ? self.digests[request.index] : self.blocks[request.index] 77 | }) 78 | }) 79 | 80 | this.peers.push(stream) 81 | eos(stream, function () { 82 | self.peers.splice(self.peers.indexOf(stream), 1) 83 | }) 84 | 85 | return stream 86 | } 87 | 88 | Peervision.prototype.get = function (index, cb) { 89 | if (!cb) cb = noop 90 | var self = this 91 | var i = this._getPeer(index) 92 | 93 | if (i === -1) { 94 | this._pendingRequests.add([index, done]) 95 | return 96 | } 97 | 98 | this._get(this.peers[i], index, done) 99 | 100 | function done (err, blk) { 101 | if (err && err.retry) return self.get(index, cb) 102 | cb(err, blk) 103 | } 104 | } 105 | 106 | Peervision.prototype._get = function (peer, index, cb) { 107 | if (this.blocks[index]) return cb(null, this.blocks[index]) 108 | 109 | var self = this 110 | var head = this.digests.length - 1 111 | 112 | if (peer.head <= head || index <= head) send() 113 | else this._getHead(peer, send) // TODO: if the piece we need is peer.head we can save a roundtrip 114 | 115 | function send (err) { 116 | if (err) return cb(err) 117 | if (self.blocks[index]) return cb(null, self.blocks[index]) 118 | 119 | var treeIndex = tree.index(0, index) 120 | 121 | var roots = tree.fullRoots(treeIndex) 122 | var treeCache = [] 123 | var treeIndexes = [] 124 | var treeDups = [] 125 | 126 | var needed = treeIndex 127 | var checksum 128 | 129 | for (var i = 0; i < roots.length; i++) { 130 | pushIndex(self, roots[i], treeIndexes, treeCache) 131 | } 132 | 133 | while (!(checksum = self.tree[needed])) { 134 | var sibling = tree.sibling(needed) 135 | pushIndex(self, sibling, treeIndexes, treeCache, treeDups) 136 | needed = tree.parent(needed) 137 | } 138 | 139 | peer.request({index: index, tree: treeIndexes, digest: false}, function (err, res) { 140 | if (err) return cb(err) 141 | 142 | treeCache = mergeTree(treeCache, res, treeDups) 143 | needed = treeIndex 144 | 145 | var digest = createHash().update(res.data).digest() 146 | var treeDigest = treeHash(treeCache, roots.length, digest) 147 | var sum = treeDigest 148 | 149 | for (var i = roots.length; i < treeCache.length; i++) { 150 | var sibling = tree.sibling(needed) 151 | var siblingSum = treeCache[i] 152 | 153 | if (needed > sibling) { // swap so "sum" is always left sibling 154 | var tmp = sum 155 | sum = siblingSum 156 | siblingSum = tmp 157 | } 158 | 159 | sum = createHash().update(sum).update(siblingSum).digest() 160 | needed = tree.parent(needed) 161 | 162 | // quick hash to push the possible parents to the response so they'll get stored on validation 163 | res.tree.push(sum) 164 | treeIndexes.push(needed) 165 | } 166 | 167 | if (!bufferEquals(sum, checksum)) return cb(new Error('Tree checksum mismatch')) 168 | 169 | // everything is fine - store the response 170 | 171 | for (var i = 0; i < treeIndexes.length; i++) self.tree[treeIndexes[i]] = res.tree[i] 172 | self.tree[treeIndex] = treeDigest 173 | self.blocks[index] = res.data 174 | self.digests[index] = digest 175 | self.signatures[index] = res.signature 176 | self._have(index) 177 | self.emit('download', index, res.data) 178 | 179 | cb(null, res.data) 180 | }) 181 | } 182 | } 183 | 184 | Peervision.prototype._getHead = function (peer, cb) { 185 | var self = this 186 | var peerHead = peer.head 187 | var treeHead = tree.index(0, peerHead) 188 | var roots = tree.fullRoots(treeHead) 189 | var treeCache = [] 190 | var treeIndexes = [] 191 | var treeDups = [] 192 | 193 | var prevRoots = tree.fullRoots(tree.index(0, Math.max(0, this.digests.length - 1))) 194 | var prevRootParents = [] 195 | 196 | for (var i = 0; i < roots.length; i++) { 197 | pushIndex(this, roots[i], treeIndexes, treeCache) 198 | } 199 | 200 | treeIndexes.push(treeHead) 201 | treeCache.push(HASH_NONE) 202 | 203 | // filter out dups 204 | var filter = [] 205 | for (var j = 0; j < prevRoots.length; j++) { 206 | if (roots.indexOf(prevRoots[j]) === -1) filter.push(prevRoots[j]) 207 | } 208 | prevRoots = filter 209 | 210 | // migrate from old roots to new roots 211 | for (var j = 0; j < prevRoots.length; j++) { 212 | var needed = prevRoots[j] 213 | var root = -1 214 | while ((root = roots.indexOf(needed)) === -1) { 215 | var sibling = tree.sibling(needed) 216 | pushIndex(this, sibling, treeIndexes, treeCache, treeDups) 217 | needed = tree.parent(needed) 218 | } 219 | prevRootParents.push(root) 220 | } 221 | 222 | peer.request({index: peerHead, tree: treeIndexes, digest: false, signature: true}, function (err, res) { 223 | if (self.blocks.length > peerHead) return cb() // this request is no longer relevant 224 | if (err) return cb(err) 225 | 226 | mergeTree(treeCache, res, treeDups) 227 | 228 | var peerTreeDigest = treeCache[roots.length] 229 | var signed = signatures.verify(peerTreeDigest, res.signature, self.id) 230 | if (!signed) return cb(new Error('Tree signature is invalid')) 231 | 232 | var digest = createHash().update(res.data).digest() 233 | var treeDigest = treeHash(treeCache, roots.length, digest) 234 | 235 | if (!bufferEquals(treeDigest, peerTreeDigest)) return cb(new Error('Tree checksum mismatch')) 236 | 237 | // haxx - this is mostly duplicate code :/ 238 | needed = prevRoots.length ? prevRoots.shift() : -1 239 | 240 | var sum = self.tree[needed] 241 | for (var i = roots.length + 1; i < treeCache.length; i++) { 242 | if (needed !== roots[prevRootParents[0]]) { 243 | var sibling = tree.sibling(needed) 244 | var siblingSum = treeCache[i] 245 | 246 | if (needed > sibling) { // swap so "sum" is always left sibling 247 | var tmp = sum 248 | sum = siblingSum 249 | siblingSum = tmp 250 | } 251 | sum = createHash().update(sum).update(siblingSum).digest() 252 | needed = tree.parent(needed) 253 | } 254 | 255 | // quick hash to push the possible parents to the response so they'll get stored on validation 256 | res.tree.push(sum) 257 | treeIndexes.push(needed) 258 | 259 | if (needed === roots[prevRootParents[0]]) { 260 | var cached = treeCache[prevRootParents.shift()] 261 | if (!bufferEquals(cached, sum)) { 262 | return cb(new Error('Tree checksum mismatch')) 263 | } 264 | needed = prevRoots.length ? prevRoots.shift() : -1 265 | sum = self.tree[needed] 266 | } 267 | } 268 | 269 | if (needed !== -1) return cb(new Error('Tree checksum mismatch')) 270 | 271 | // everything is fine - store the response 272 | 273 | for (var i = 0; i < treeIndexes.length; i++) self.tree[treeIndexes[i]] = res.tree[i] 274 | self.blocks[peerHead] = res.data 275 | self.digests[peerHead] = digest 276 | self.signatures[peerHead] = res.signature 277 | self.emit('download', peerHead, res.data) 278 | 279 | self._have(peerHead) 280 | 281 | cb(null) 282 | }) 283 | } 284 | 285 | Peervision.prototype._getPeer = function (index) { 286 | var selected = -1 287 | var found = 1 288 | 289 | for (var i = 0; i < this.peers.length; i++) { 290 | var p = this.peers[i] 291 | if (p && p.blocks && p.blocks.get(index)) { 292 | if (Math.random() < (1 / found++)) selected = i 293 | } 294 | } 295 | 296 | return selected 297 | } 298 | 299 | Peervision.prototype.append = function (block, cb) { 300 | if (!this.keypair.secretKey) throw new Error('Only the producer can append chunks') 301 | 302 | var index = this.blocks.length 303 | var treeIndex = tree.index(0, index) 304 | var digest = createHash().update(block).digest() 305 | var roots = tree.fullRoots(treeIndex) 306 | 307 | var hash = createHash() 308 | hash.update(digest) 309 | for (var i = 0; i < roots.length; i++) { 310 | hash.update(this._getTree(roots[i])) 311 | } 312 | 313 | var treeDigest = hash.digest() 314 | var signature = signatures.sign(treeDigest, this.keypair.secretKey) 315 | 316 | this.blocks[index] = block 317 | this.digests[index] = digest 318 | this.signatures[index] = signature 319 | this.tree[treeIndex] = treeDigest 320 | 321 | this._have(index) 322 | 323 | if (cb) cb() 324 | } 325 | 326 | Peervision.prototype._have = function (index) { 327 | this.have.set(index, true) 328 | for (var i = 0; i < this.peers.length; i++) { 329 | this.peers[i].have({index: index}) 330 | } 331 | 332 | if (index > this._head) { 333 | this._head = index 334 | this.emit('head', index) 335 | } 336 | } 337 | 338 | Peervision.prototype._update = function () { 339 | if (!this._pendingRequests.length) return 340 | 341 | for (var i = 0; i < this._pendingRequests.length; i++) { 342 | var pair = this._pendingRequests.get(i) 343 | if (!pair) continue 344 | 345 | var index = pair[0] 346 | var cb = pair[1] 347 | 348 | var peer = this._getPeer(index) 349 | if (peer === -1) continue 350 | 351 | this._pendingRequests.remove(i) 352 | this._get(this.peers[peer], index, cb) 353 | } 354 | } 355 | 356 | Peervision.prototype._getTree = function (index) { 357 | if (index === -1) throw new Error('Invalid index') 358 | if (this.tree[index]) return this.tree[index] 359 | 360 | var hash = createHash() 361 | .update(this._getTree(tree.leftChild(index))) 362 | .update(this._getTree(tree.rightChild(index))) 363 | .digest() 364 | 365 | this.tree[index] = hash 366 | return hash 367 | } 368 | 369 | function noop () {} 370 | 371 | function pushIndex (self, index, treeIndexes, treeCache, treeDups) { 372 | if (treeDups) { 373 | var i = treeIndexes.indexOf(index) 374 | if (i > -1) { 375 | treeCache.push(HASH_DUPLICATE) 376 | treeDups.push(i) 377 | return 378 | } 379 | } 380 | 381 | var hash = self.tree[index] 382 | 383 | if (!hash) { 384 | treeIndexes.push(index) 385 | treeCache.push(HASH_NONE) 386 | } else { 387 | treeCache.push(hash) 388 | } 389 | } 390 | 391 | function mergeTree (tree, res, dups) { 392 | var offset = 0 393 | var dupsOffset = 0 394 | 395 | for (var i = 0; i < tree.length; i++) { 396 | if (tree[i] === HASH_NONE) tree[i] = res.tree[offset++] 397 | if (tree[i] === HASH_DUPLICATE) tree[i] = res.tree[dups[dupsOffset++]] 398 | } 399 | 400 | return tree 401 | } 402 | 403 | function treeHash (roots, end, digest) { 404 | var hash = createHash() 405 | hash.update(digest) 406 | for (var i = 0; i < end; i++) hash.update(roots[i]) 407 | return hash.digest() 408 | } 409 | 410 | function createHash () { 411 | return crypto.createHash('sha256') 412 | } 413 | 414 | function getHead (bitfield) { 415 | var end = bitfield.buffer.length * 8 416 | var max = -1 417 | for (var i = end - 8; i < end; i++) { 418 | if (bitfield.get(i)) max = i 419 | } 420 | return max 421 | } 422 | -------------------------------------------------------------------------------- /lib/messages.js: -------------------------------------------------------------------------------- 1 | var protobuf = require('protocol-buffers') 2 | var fs = require('fs') 3 | 4 | module.exports = protobuf(fs.readFileSync(__dirname + '/../schema.proto', 'utf-8')) 5 | -------------------------------------------------------------------------------- /lib/protocol.js: -------------------------------------------------------------------------------- 1 | var messages = require('./messages') 2 | var lpstream = require('length-prefixed-stream') 3 | var duplexify = require('duplexify') 4 | var util = require('util') 5 | 6 | var DESTROYED = new Error('Stream was destroyed') 7 | DESTROYED.retry = true 8 | 9 | var ENCODERS = [ 10 | messages.Handshake, 11 | messages.Have, 12 | messages.Request, 13 | messages.Response 14 | ] 15 | 16 | module.exports = ProtocolStream 17 | 18 | function ProtocolStream (opts) { 19 | if (!(this instanceof ProtocolStream)) return new ProtocolStream(opts) 20 | if (!opts) opts = {} 21 | 22 | this._encode = lpstream.encode() 23 | this._decode = lpstream.decode({limit: 5 * 1024 * 1024}) 24 | this._requests = [] 25 | this._handshook = false 26 | this.destroyed = false 27 | 28 | duplexify.call(this, this._decode, this._encode) 29 | var self = this 30 | 31 | this._decode.on('data', function (data) { 32 | self._parse(data) 33 | }) 34 | 35 | this._send(0, opts) 36 | } 37 | 38 | util.inherits(ProtocolStream, duplexify) 39 | 40 | ProtocolStream.prototype.destroy = function (err) { 41 | if (this.destroyed) return 42 | this.destroyed = true 43 | 44 | while (this._requests.length) { 45 | var cb = this._requests.shift() 46 | if (cb) cb(err || DESTROYED) 47 | } 48 | 49 | if (err) this.emit('error', err) 50 | this.emit('close') 51 | } 52 | 53 | ProtocolStream.prototype._parse = function (buf) { 54 | if (this.destroyed) return 55 | 56 | var dec = ENCODERS[buf[0]] 57 | 58 | try { 59 | var data = dec.decode(buf, 1) 60 | } catch (err) { 61 | return this.destroy(err) 62 | } 63 | 64 | if (!this._handshook && buf[0] !== 0) { 65 | this.emit('error', new Error('First message should be a handshake')) 66 | return 67 | } 68 | 69 | switch (buf[0]) { 70 | case 0: 71 | this._handshook = true 72 | this.emit('handshake', data) 73 | return 74 | 75 | case 1: 76 | this.emit('have', data) 77 | return 78 | 79 | case 2: 80 | this.emit('request', data) 81 | return 82 | 83 | case 3: 84 | if (this._requests.length > data.id) { 85 | var cb = this._requests[data.id] 86 | this._requests[data.id] = null 87 | while (this._requests.length && !this._requests[this._requests.length - 1]) this._requests.pop() 88 | if (cb) cb(null, data) 89 | } 90 | this.emit('response', data) 91 | return 92 | } 93 | 94 | this.emit('unknown', buf) 95 | } 96 | 97 | ProtocolStream.prototype.have = function (have) { 98 | this._send(1, have) 99 | } 100 | 101 | ProtocolStream.prototype.request = function (req, cb) { 102 | req.id = this._requests.indexOf(null) 103 | if (req.id === -1) req.id = this._requests.push(null) - 1 104 | this._requests[req.id] = cb 105 | this._send(2, req) 106 | } 107 | 108 | ProtocolStream.prototype.response = function (res) { 109 | this._send(3, res) 110 | } 111 | 112 | ProtocolStream.prototype._send = function (type, data, cb) { 113 | var enc = ENCODERS[type] 114 | var buf = new Buffer(1 + enc.encodingLength(data)) 115 | buf[0] = type 116 | enc.encode(data, buf, 1) 117 | this._encode.write(buf, cb) 118 | } 119 | -------------------------------------------------------------------------------- /messages.js: -------------------------------------------------------------------------------- 1 | var protobuf = require('protocol-buffers') 2 | var fs = require('fs') 3 | 4 | module.exports = protobuf(fs.readFileSync(__dirname + '/schema.proto', 'utf-8')) 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "peervision", 3 | "version": "0.0.7", 4 | "description": "(WIP) a p2p live streaming protocol", 5 | "main": "index.js", 6 | "dependencies": { 7 | "bitfield": "^1.1.2", 8 | "brfs": "^1.4.1", 9 | "buffer-equals": "^1.0.3", 10 | "duplexify": "^3.4.2", 11 | "end-of-stream": "^1.1.0", 12 | "flat-tree": "^1.1.1", 13 | "length-prefixed-stream": "^1.4.0", 14 | "numeric-id-map": "^1.1.0", 15 | "protocol-buffers": "^3.1.3", 16 | "sodium-signatures": "^1.0.1" 17 | }, 18 | "browserify": { 19 | "transform": [ 20 | "brfs" 21 | ] 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/mafintosh/peervision.git" 26 | }, 27 | "author": "Mathias Buus (@mafintosh)", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/mafintosh/peervision/issues" 31 | }, 32 | "homepage": "https://github.com/mafintosh/peervision" 33 | } 34 | -------------------------------------------------------------------------------- /schema.proto: -------------------------------------------------------------------------------- 1 | message Handshake { 2 | optional uint32 version = 1 [default = 1]; 3 | optional bytes blocks = 2; 4 | } 5 | 6 | message Have { 7 | required uint64 index = 1; 8 | } 9 | 10 | message Request { 11 | required uint64 id = 1; 12 | optional uint64 index = 2; 13 | optional bool digest = 3; 14 | optional bool signature = 4; 15 | repeated uint64 tree = 5; 16 | } 17 | 18 | message Response { 19 | required uint64 id = 1; 20 | repeated bytes tree = 2; 21 | optional bytes signature = 3; 22 | optional bytes data = 4; 23 | } 24 | --------------------------------------------------------------------------------