├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── collaborators.md ├── index.js ├── lib ├── hash.js └── messages.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '5' 4 | - '4' 5 | - '0.12' 6 | - '0.10' 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # subgraph 2 | 3 | Content addressable graph where every node has at most a single link to another node 4 | 5 | ``` 6 | npm install subgraph 7 | ``` 8 | 9 | [![build status](http://img.shields.io/travis/mafintosh/subgraph.svg?style=flat)](http://travis-ci.org/mafintosh/subgraph) 10 | 11 | ## Usage 12 | 13 | ``` js 14 | var subgraph = require('subgraph') 15 | var sg = subgraph(levelupInstance) 16 | 17 | var ws = sg.createAppendStream() 18 | 19 | ws.write('hello') 20 | ws.write('world') 21 | 22 | ws.end(function () { 23 | var rs = sg.createReadStream(ws.key) 24 | 25 | rs.on('data', function (node) { 26 | console.log(node) // first {value: 'world'} then {value: 'hello'} 27 | }) 28 | }) 29 | ``` 30 | 31 | ## API 32 | 33 | #### `var sg = subgraph(levelupInstance, [options])` 34 | 35 | Create a new subgraph instance. Options include: 36 | 37 | ``` js 38 | { 39 | prefix: 'optional-sublevel-prefix' 40 | } 41 | ``` 42 | 43 | #### `var ws = sg.createAppendStream([link])` 44 | 45 | Create an append stream. The values you write to it will be linked together. 46 | When the stream emits `finish` it will have a `.key` property that contains the latest link 47 | and a `.length` property that contains the number of nodes written 48 | 49 | Optionally you can provide a `link` in the constructor for the first node to append to. 50 | 51 | #### `var rs = sg.createReadStream(key)` 52 | 53 | Create a read stream from a key. 54 | Will read out values in reverse order of writes to the append stream. 55 | 56 | #### `var ws = sg.createWriteStream(key)` 57 | 58 | Create a write stream from a key. Will verify that the values written matches the key when hashed. 59 | 60 | #### `sg.add(link, value, [cb])` 61 | 62 | Shorthand for only adding a single value 63 | 64 | #### `sg.get(key, cb)` 65 | 66 | Shorthand for getting a single value 67 | 68 | #### `sg.root(key, cb)` 69 | 70 | Returns the root of a stream. If a write stream was ended prematurely / destroyed the root returned will 71 | have a link property. 72 | 73 | ``` js 74 | sg.root(someKey, function (err, node) { 75 | console.log('root is', node) 76 | console.log('the stream is partially written?', !!root.link) 77 | }) 78 | ``` 79 | 80 | ## License 81 | 82 | MIT 83 | -------------------------------------------------------------------------------- /collaborators.md: -------------------------------------------------------------------------------- 1 | ## Collaborators 2 | 3 | subgraph is only possible due to the excellent work of the following collaborators: 4 | 5 | 6 | 7 | 8 |
maxogdenGitHub/maxogden
mafintoshGitHub/mafintosh
karissaGitHub/karissa
9 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var bulk = require('bulk-write-stream') 2 | var from = require('from2') 3 | var equals = require('buffer-equals') 4 | var messages = require('./lib/messages') 5 | var hash = require('./lib/hash') 6 | 7 | module.exports = Subgraph 8 | 9 | function Subgraph (db, opts) { 10 | if (!(this instanceof Subgraph)) return new Subgraph(db, opts) 11 | if (!opts) opts = {} 12 | this.prefix = opts.prefix ? '!' + opts.prefix + '!' : '' 13 | this.db = db 14 | } 15 | 16 | Subgraph.prototype.get = function (key, cb) { 17 | this.db.get(this.prefix + key.toString('hex'), {valueEncoding: messages.Node}, cb) 18 | } 19 | 20 | Subgraph.prototype.add = function (link, value, cb) { 21 | var self = this 22 | 23 | if (!link) return add(0, value, null) 24 | this.get(link, function (err, entry) { 25 | if (err) return cb(err) 26 | add(entry.index + 1, value, link) 27 | }) 28 | 29 | function add (index, value, link) { 30 | var key = hash(index, value, link) 31 | var entry = {index: index, value: value, link: link} 32 | self.db.put(self.prefix + key.toString('hex'), messages.Node.encode(entry), function (err) { 33 | if (err) return cb(err) 34 | cb(null, key) 35 | }) 36 | } 37 | } 38 | 39 | Subgraph.prototype.root = function (key, cb) { 40 | var rs = this.createReadStream(key) 41 | var root = null 42 | 43 | rs.on('data', function (data) { 44 | root = data 45 | }) 46 | 47 | rs.on('error', done) 48 | rs.on('end', done) 49 | 50 | function done (err) { 51 | if (err) return cb(err) 52 | cb(null, root) 53 | } 54 | } 55 | 56 | Subgraph.prototype.createAppendStream = function (link) { 57 | var self = this 58 | var count = 0 59 | var stream = bulk.obj(write, flush) 60 | var first = true 61 | 62 | return stream 63 | 64 | function write (datas, cb) { 65 | if (first && link) { 66 | first = false 67 | init(datas, cb) 68 | return 69 | } 70 | 71 | var batch = new Array(datas.length) 72 | 73 | for (var i = 0; i < datas.length; i++) { 74 | var data = toBuffer(datas[i], 'utf-8') 75 | var index = count++ 76 | var node = messages.Node.encode({ 77 | value: data, 78 | link: link, 79 | index: index 80 | }) 81 | 82 | link = hash(index, data, link) 83 | batch[i] = {type: 'put', key: self.prefix + link.toString('hex'), value: node} 84 | } 85 | 86 | self.db.batch(batch, cb) 87 | } 88 | 89 | function init (datas, cb) { 90 | self.get(link, function (err, node) { 91 | if (err) return cb(err) 92 | count = node.index + 1 93 | write(datas, cb) 94 | }) 95 | } 96 | 97 | function flush (cb) { 98 | stream.length = count 99 | stream.key = link 100 | cb() 101 | } 102 | } 103 | 104 | Subgraph.prototype.createWriteStream = function (link) { 105 | if (!link) throw new Error('key is required') 106 | link = toBuffer(link) 107 | 108 | var self = this 109 | var stream = bulk.obj(write) 110 | 111 | return stream 112 | 113 | function write (datas, cb) { 114 | var batch = new Array(datas.length) 115 | 116 | for (var i = 0; i < datas.length; i++) { 117 | var data = datas[i] 118 | if (!equals(hash(data.index, data.value, data.link), link)) return cb(new Error('Checksum mismatch')) 119 | batch[i] = { 120 | type: 'put', 121 | key: self.prefix + link.toString('hex'), 122 | value: messages.Node.encode(data) 123 | } 124 | link = data.link 125 | } 126 | 127 | self.db.batch(batch, cb) 128 | } 129 | } 130 | 131 | Subgraph.prototype.createReadStream = function (link) { 132 | if (!link) throw new Error('key is required') 133 | link = toBuffer(link) 134 | 135 | var self = this 136 | var stream = from.obj(read) 137 | stream.length = -1 138 | 139 | return stream 140 | 141 | function read (size, cb) { 142 | if (!link) return cb(null, null) 143 | self.db.get(self.prefix + link.toString('hex'), {valueEncoding: messages.Node}, function (err, node) { 144 | if (err && err.notFound) { 145 | if (first) ready(0) 146 | return cb(null, null) 147 | } 148 | if (err) return cb(err) 149 | var first = stream.length === -1 150 | if (first) ready(node.index + 1) 151 | link = node.link 152 | cb(null, node) 153 | }) 154 | } 155 | 156 | function ready (length) { 157 | stream.length = length 158 | stream.emit('ready') 159 | } 160 | } 161 | 162 | function toBuffer (buf, enc) { 163 | if (typeof buf === 'string') return new Buffer(buf, enc) 164 | return buf 165 | } 166 | -------------------------------------------------------------------------------- /lib/hash.js: -------------------------------------------------------------------------------- 1 | var framedHash = require('framed-hash') 2 | 3 | module.exports = hash 4 | 5 | function hash (index, data, link) { 6 | var sha = framedHash('sha256') 7 | sha.update(index.toString()) 8 | sha.update(data) 9 | if (link) sha.update(link) 10 | return sha.digest() 11 | } 12 | -------------------------------------------------------------------------------- /lib/messages.js: -------------------------------------------------------------------------------- 1 | var protobuf = require('protocol-buffers') 2 | 3 | module.exports = protobuf([ 4 | 'message Node {', 5 | ' required bytes value = 1;', 6 | ' optional bytes link = 2;', 7 | ' required uint64 index = 3;', 8 | '}' 9 | ].join('\n')) 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "subgraph", 3 | "version": "2.1.0", 4 | "description": "Content addressable graph where every node has at most a single link to another node", 5 | "main": "index.js", 6 | "dependencies": { 7 | "buffer-equals": "^1.0.3", 8 | "bulk-write-stream": "^1.1.1", 9 | "framed-hash": "^1.1.0", 10 | "from2": "^2.1.0", 11 | "protocol-buffers": "^3.1.3" 12 | }, 13 | "devDependencies": { 14 | "memdb": "^1.0.1", 15 | "standard": "^5.3.1", 16 | "tape": "^4.2.2" 17 | }, 18 | "scripts": { 19 | "test": "standard && tape test.js" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/mafintosh/subgraph.git" 24 | }, 25 | "author": "Mathias Buus (@mafintosh)", 26 | "license": "MIT", 27 | "bugs": { 28 | "url": "https://github.com/mafintosh/subgraph/issues" 29 | }, 30 | "homepage": "https://github.com/mafintosh/subgraph" 31 | } 32 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | var memdb = require('memdb') 3 | var subgraph = require('./') 4 | 5 | tape('add one value', function (t) { 6 | var sg = subgraph(memdb()) 7 | 8 | sg.add(null, 'hello', function (err, key) { 9 | t.error(err, 'no error') 10 | t.ok(key, 'had key') 11 | t.end() 12 | }) 13 | }) 14 | 15 | tape('add two values', function (t) { 16 | var sg = subgraph(memdb()) 17 | 18 | sg.add(null, 'hello', function (err, link) { 19 | t.error(err, 'no error') 20 | sg.add(link, 'world', function (err, key) { 21 | t.error(err, 'no error') 22 | t.ok(key, 'had key') 23 | sg.get(key, function (err, entry) { 24 | t.error(err, 'no error') 25 | t.same(entry.value.toString(), 'world') 26 | t.same(entry.link, link) 27 | t.same(entry.index, 1) 28 | t.end() 29 | }) 30 | }) 31 | }) 32 | }) 33 | 34 | tape('append stream', function (t) { 35 | var sg = subgraph(memdb()) 36 | var stream = sg.createAppendStream() 37 | 38 | stream.write('hello') 39 | stream.write('world') 40 | stream.write('verden') 41 | 42 | stream.end(function () { 43 | t.ok(stream.key, 'has key') 44 | sg.get(stream.key, function (err, entry) { 45 | t.error(err, 'no error') 46 | t.same(entry.value.toString(), 'verden') 47 | t.same(entry.index, 2) 48 | t.end() 49 | }) 50 | }) 51 | }) 52 | 53 | tape('append stream + prefix', function (t) { 54 | var sg = subgraph(memdb(), {prefix: 'test'}) 55 | var stream = sg.createAppendStream() 56 | 57 | stream.write('hello') 58 | stream.write('world') 59 | stream.write('verden') 60 | 61 | stream.end(function () { 62 | t.ok(stream.key, 'has key') 63 | sg.get(stream.key, function (err, entry) { 64 | t.error(err, 'no error') 65 | t.same(entry.value.toString(), 'verden') 66 | t.same(entry.index, 2) 67 | t.end() 68 | }) 69 | }) 70 | }) 71 | 72 | tape('append stream with link', function (t) { 73 | var sg = subgraph(memdb()) 74 | var stream = sg.createAppendStream() 75 | 76 | stream.write('hello') 77 | stream.write('world') 78 | 79 | stream.end(function () { 80 | var stream2 = sg.createAppendStream(stream.key) 81 | stream2.write('verden') 82 | stream2.end(function () { 83 | t.ok(stream2.key, 'has key') 84 | sg.get(stream2.key, function (err, entry) { 85 | t.error(err, 'no error') 86 | t.same(entry.value.toString(), 'verden') 87 | t.same(entry.index, 2) 88 | t.end() 89 | }) 90 | }) 91 | }) 92 | }) 93 | 94 | tape('read stream', function (t) { 95 | var sg = subgraph(memdb()) 96 | var stream = sg.createAppendStream() 97 | 98 | stream.write('hello') 99 | stream.write('world') 100 | stream.write('verden') 101 | 102 | stream.end(function () { 103 | var rs = sg.createReadStream(stream.key) 104 | var expected = [{value: 'verden', index: 2}, {value: 'world', index: 1}, {value: 'hello', index: 0}] 105 | 106 | rs.on('ready', function () { 107 | t.same(rs.length, 3, 'length is 3') 108 | }) 109 | rs.on('data', function (data) { 110 | t.same(expected[0].value, data.value.toString(), 'expected value') 111 | t.same(expected[0].index, data.index, 'expected index') 112 | expected.shift() 113 | }) 114 | rs.on('end', function () { 115 | t.same(expected.length, 0, 'no more data') 116 | t.end() 117 | }) 118 | }) 119 | }) 120 | 121 | tape('partial read stream', function (t) { 122 | var sg = subgraph(memdb()) 123 | var stream = sg.createAppendStream() 124 | 125 | stream.write('hello') 126 | stream.write('world') 127 | stream.write('verden') 128 | 129 | stream.end(function () { 130 | sg.get(stream.key, function (err, entry) { 131 | t.error(err, 'no error') 132 | 133 | var rs = sg.createReadStream(entry.link) 134 | var expected = [{value: 'world', index: 1}, {value: 'hello', index: 0}] 135 | 136 | rs.on('ready', function () { 137 | t.same(rs.length, 2, 'length is 2') 138 | }) 139 | rs.on('data', function (data) { 140 | t.same(expected[0].value, data.value.toString(), 'expected value') 141 | t.same(expected[0].index, data.index, 'expected index') 142 | expected.shift() 143 | }) 144 | rs.on('end', function () { 145 | t.same(expected.length, 0, 'no more data') 146 | t.end() 147 | }) 148 | }) 149 | }) 150 | }) 151 | 152 | tape('read stream to write stream', function (t) { 153 | var sg = subgraph(memdb()) 154 | var copy = subgraph(memdb()) 155 | var stream = sg.createAppendStream() 156 | 157 | stream.write('hello') 158 | stream.write('world') 159 | stream.write('verden') 160 | 161 | stream.end(function () { 162 | sg.createReadStream(stream.key).pipe(copy.createWriteStream(stream.key)).on('finish', verify) 163 | }) 164 | 165 | function verify () { 166 | var rs = copy.createReadStream(stream.key) 167 | var expected = [{value: 'verden', index: 2}, {value: 'world', index: 1}, {value: 'hello', index: 0}] 168 | 169 | rs.on('ready', function () { 170 | t.same(rs.length, 3, 'length is 3') 171 | }) 172 | rs.on('data', function (data) { 173 | t.same(expected[0].value, data.value.toString(), 'expected value') 174 | t.same(expected[0].index, data.index, 'expected index') 175 | expected.shift() 176 | }) 177 | rs.on('end', function () { 178 | t.same(expected.length, 0, 'no more data') 179 | t.end() 180 | }) 181 | } 182 | }) 183 | 184 | tape('read stream to bad write stream', function (t) { 185 | var sg = subgraph(memdb()) 186 | var copy = subgraph(memdb()) 187 | var stream = sg.createAppendStream() 188 | 189 | stream.write('hello') 190 | stream.write('world') 191 | stream.write('verden') 192 | 193 | stream.end(function () { 194 | sg.createReadStream(stream.key).pipe(copy.createWriteStream('deadbeaf')).on('error', onerror) 195 | }) 196 | 197 | function onerror (err) { 198 | t.ok(err, 'had error') 199 | t.end() 200 | } 201 | }) 202 | 203 | tape('resumable write stream', function (t) { 204 | var sg = subgraph(memdb()) 205 | var copy = subgraph(memdb()) 206 | var stream = sg.createAppendStream() 207 | 208 | stream.write('hello') 209 | stream.write('world') 210 | stream.write('verden') 211 | 212 | stream.end(function () { 213 | var rs = sg.createReadStream(stream.key) 214 | var ws = copy.createWriteStream(stream.key) 215 | rs.pipe(ws) 216 | rs.on('data', function () { 217 | rs.destroy() 218 | ws.end(function () { 219 | copy.root(stream.key, function (err, root) { 220 | t.error(err, 'no error') 221 | t.ok(root.link, 'is resumable') 222 | var rs = sg.createReadStream(root.link) 223 | var ws = copy.createWriteStream(root.link) 224 | rs.pipe(ws).on('finish', function () { 225 | t.same(rs.length, 2, 'wrote to entries') 226 | copy.root(stream.key, function (err, root) { 227 | t.error(err, 'no error') 228 | t.ok(!root.link, 'no longer resumable') 229 | t.end() 230 | }) 231 | }) 232 | }) 233 | }) 234 | }) 235 | }) 236 | }) 237 | --------------------------------------------------------------------------------