├── LICENSE ├── README.md ├── example └── replicate.js ├── index.js ├── package.json └── test └── basic.js /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, James Halliday 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this list 8 | of conditions and the following disclaimer. 9 | 10 | Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hyperdb-sneakernet 2 | 3 | > Peer to peer field replication for a [hyperdb][] you can send around on a USB 4 | > stick. 5 | 6 | `hyperdb-sneakernet` stores a live hyperdb copy on a USB stick suitable for 7 | replicating with existing hyperdbs and also for authorizing new writers. 8 | 9 | If the target directory doesn't exist, a new hyperdb will be created there, 10 | authorized by the source hyperdb, and replicated to. 11 | 12 | If the target directory does exist, it will be authorized by the source hyperdb 13 | (if needed) and replicated with the source hyperdb. 14 | 15 | [hyperdb]: https://github.com/mafintosh/hyperdb 16 | 17 | ## Example 18 | 19 | ``` js 20 | var replicate = require('hyperdb-sneakernet') 21 | var hyperdb = require('hyperdb') 22 | 23 | var db = hyperdb('log.db', { valueEncoding: 'json' }) 24 | 25 | replicate(db, '/media/usb/log.db') 26 | ``` 27 | 28 | ## API 29 | 30 | ```js 31 | var replicate = require('hyperdb-sneakernet') 32 | ``` 33 | 34 | ### replicate(db, dir[, opts={}], cb) 35 | 36 | Performs replication between the hyperdb `db` and the directory located at 37 | `dir`. If no diectory exists at `dir` then a brand new hyperdb database will be 38 | created there and replicated to. 39 | 40 | `cb` is a callback function of the form `function (err) {}`, and is called upon 41 | an error, or successful completion. 42 | 43 | ## Install 44 | 45 | With [npm](https://npmjs.org/) installed, run 46 | 47 | ``` 48 | $ npm install hyperdb-sneakernet 49 | ``` 50 | 51 | ## License 52 | 53 | ISC 54 | -------------------------------------------------------------------------------- /example/replicate.js: -------------------------------------------------------------------------------- 1 | var replicate = require('../') 2 | var hyperdb = require('hyperdb') 3 | 4 | var db = hyperdb(process.argv[2], { valueEncoding: 'json' }) 5 | replicate(db, 'outfile.tgz', function (err) { 6 | if (err) console.error(err) 7 | else console.db('ok') 8 | }) 9 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var hyperdb = require('hyperdb') 2 | var fs = require('fs') 3 | var onend = require('end-of-stream') 4 | 5 | module.exports = function (local, dir, opts, cb) { 6 | if (typeof opts === 'function') { 7 | cb = opts 8 | opts = {} 9 | } 10 | cb = cb || (function () {}) 11 | 12 | var freshRemote = !fs.existsSync(dir) 13 | var remote 14 | 15 | local.ready(function () { 16 | remote = freshRemote ? hyperdb(dir, local.key, local.valueEncoding) : hyperdb(dir, local.valueEncoding) 17 | 18 | remote.ready(function () { 19 | // Bail if both local and remote shared keys don't match 20 | if (!local.key.equals(remote.key)) { 21 | return cb(new Error('shared hyperdb keys do not match')) 22 | } 23 | 24 | var localAuthorized, remoteAuthorized 25 | local.authorized(local.local.key, function (err, authLocal) { 26 | if (err) return cb(err) 27 | remote.authorized(remote.local.key, function (err, authRemote) { 28 | if (err) return cb(err) 29 | check(authLocal, authRemote) 30 | }) 31 | }) 32 | }) 33 | }) 34 | 35 | function check (localAuthorized, remoteAuthorized) { 36 | if (localAuthorized && remoteAuthorized) { 37 | // Existing local; existing remote 38 | replicate(local, remote, cb) 39 | } else if (!localAuthorized && remoteAuthorized) { 40 | // Fresh local; existing remote 41 | remote.authorize(local.local.key, function (err) { 42 | if (err) return cb(err) 43 | replicate(local, remote, cb) 44 | }) 45 | } else if (localAuthorized && !remoteAuthorized) { 46 | // Existing local; fresh remote 47 | local.authorize(remote.local.key, function (err) { 48 | if (err) return cb(err) 49 | replicate(local, remote, cb) 50 | }) 51 | } else { 52 | // Neither feed is authorized 53 | return cb(new Error('neither feed is authorized to write')) 54 | } 55 | } 56 | } 57 | 58 | // HyperDB, HyperDB => Error 59 | function replicate (local, remote, cb) { 60 | var rr = remote.replicate() 61 | var lr = local.replicate() 62 | onend(rr, doneReplication) 63 | onend(lr, doneReplication) 64 | rr.pipe(lr).pipe(rr) 65 | 66 | var pending = 2 67 | var error 68 | function doneReplication (err) { 69 | if (err) error = err 70 | if (--pending) return 71 | cb(error) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hyperdb-sneakernet", 3 | "version": "3.0.0", 4 | "description": "Peer to peer replication for hyperdb using files you can send around on a USB stick.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "tape test/*.js" 8 | }, 9 | "keywords": [ 10 | "hyperdb", 11 | "usb", 12 | "sneakernet", 13 | "replication", 14 | "sync" 15 | ], 16 | "authors": [ 17 | "substack", 18 | "noffle" 19 | ], 20 | "license": "ISC", 21 | "dependencies": { 22 | "debug": "^2.2.0", 23 | "end-of-stream": "^1.1.0", 24 | "graceful-fs": "^4.1.9", 25 | "hyperdb": "3.0.0-1", 26 | "once": "^1.3.3", 27 | "pump": "^1.0.1", 28 | "tar-fs": "^1.13.2" 29 | }, 30 | "devDependencies": { 31 | "tape": "^4.5.1", 32 | "tmp": "0.0.29" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | var sneaker = require('..') 2 | var test = require('tape') 3 | var hyperdb = require('hyperdb') 4 | var tmp = require('tmp').dir 5 | var path = require('path') 6 | var fs = require('fs') 7 | var onend = require('end-of-stream') 8 | 9 | function emptyFixture (key, done) { 10 | if (typeof key === 'function' && !done) { 11 | done = key 12 | key = null 13 | } 14 | var db = null 15 | tmp({unsafeCleanup: true}, function (err, dir, cleanup) { 16 | if (err) return done(err) 17 | db = key ? hyperdb(dir, key, {valueEncoding: 'json'}) : hyperdb(dir, {valueEncoding: 'json'}) 18 | db.ready(function () { 19 | done(null, db, dir, cleanup) 20 | }) 21 | }) 22 | } 23 | 24 | function fixture (done) { 25 | emptyFixture(function (err, db, dir, cleanup) { 26 | db.put('hello', 'world', function(err, node) { 27 | if (err) return done(err) 28 | db.put('hello', 'there', function(err, node) { 29 | if (err) return done(err) 30 | done(err, db, dir, cleanup) 31 | }) 32 | }) 33 | }) 34 | } 35 | 36 | test('different shared keys', function (t) { 37 | emptyFixture(function (err, db0, dir0, cleanup0) { 38 | t.notOk(err) 39 | emptyFixture(function (err, db1, dir1, cleanup1) { 40 | t.notOk(err) 41 | sneaker(db0, dir1, function (err) { 42 | t.ok(err) 43 | t.equals(err.message, 'shared hyperdb keys do not match') 44 | cleanup0() 45 | cleanup1() 46 | t.end() 47 | }) 48 | }) 49 | }) 50 | }) 51 | 52 | test('neither db authorized to write', function (t) { 53 | emptyFixture(function (err, db0, dir0, cleanup0) { 54 | t.notOk(err) 55 | emptyFixture(db0.key, function (err, db1, dir1, cleanup0) { 56 | t.notOk(err) 57 | emptyFixture(db0.key, function (err, db2, dir2, cleanup1) { 58 | t.notOk(err) 59 | sneaker(db1, dir2, function (err) { 60 | t.ok(err) 61 | t.equals(err.message, 'neither feed is authorized to write') 62 | cleanup0() 63 | cleanup1() 64 | t.end() 65 | }) 66 | }) 67 | }) 68 | }) 69 | }) 70 | 71 | test('existing local; existing remote', function (t) { 72 | t.plan(9) 73 | 74 | emptyFixture(function (err, db0, dir0, cleanup0) { 75 | t.notOk(err) 76 | emptyFixture(db0.key, function (err, db1, dir1, cleanup1) { 77 | t.notOk(err) 78 | 79 | db0.authorize(db1.local.key, function (err) { 80 | t.notOk(err) 81 | replicate(db0, db1, function (err) { 82 | t.notOk(err) 83 | populate() 84 | }) 85 | }) 86 | 87 | function populate () { 88 | db0.put('foo', 'bar', function (err) { 89 | t.notOk(err) 90 | db1.put('bax', 'quux', function (err) { 91 | t.notOk(err) 92 | sneaker(db0, dir1, function (err) { 93 | t.notOk(err) 94 | check() 95 | }) 96 | }) 97 | }) 98 | } 99 | 100 | function check () { 101 | getContent(db0, function (err, res) { 102 | t.notOk(err) 103 | t.deepEquals(res, [ 104 | { key: '', value: null }, 105 | { key: 'foo', value: 'bar' }, 106 | { key: 'bax', value: 'quux' } 107 | ]) 108 | 109 | cleanup0() 110 | cleanup1() 111 | }) 112 | } 113 | }) 114 | }) 115 | }) 116 | 117 | test('existing local; fresh remote', function (t) { 118 | var db1 119 | 120 | emptyFixture(function (err, db0, dir0, cleanup0) { 121 | t.notOk(err) 122 | tmp({unsafeCleanup: true}, function (err, dir1, cleanup1) { 123 | t.notOk(err) 124 | dir1 = path.join(dir1, 'db') 125 | populate() 126 | 127 | function populate () { 128 | db0.put('foo', 'bar', function (err) { 129 | t.notOk(err) 130 | sneaker(db0, dir1, function (err) { 131 | t.notOk(err) 132 | db1 = hyperdb(dir1, db0.key, {valueEncoding: 'json'}) 133 | db1.ready(check) 134 | }) 135 | }) 136 | } 137 | 138 | function check () { 139 | getContent(db1, function (err, res) { 140 | t.notOk(err) 141 | t.deepEquals(res, [ 142 | { key: 'foo', value: 'bar' }, 143 | { key: '', value: null } 144 | ]) 145 | 146 | cleanup0() 147 | cleanup1() 148 | t.end() 149 | }) 150 | } 151 | }) 152 | }) 153 | }) 154 | 155 | // this case wouldn't really happen in practice 156 | test('fresh local; existing remote', function (t) { 157 | t.plan(7) 158 | 159 | emptyFixture(function (err, db0, dir0, cleanup0) { 160 | t.notOk(err) 161 | emptyFixture(db0.key, function (err, db1, dir1, cleanup1) { 162 | t.notOk(err) 163 | 164 | populate() 165 | 166 | function populate () { 167 | db0.put('foo', 'bar', function (err) { 168 | t.notOk(err) 169 | db1.put('bax', 'quux', function (err) { 170 | t.notOk(err) 171 | sneaker(db0, dir1, function (err) { 172 | t.notOk(err) 173 | check() 174 | }) 175 | }) 176 | }) 177 | } 178 | 179 | function check () { 180 | getContent(db0, function (err, res) { 181 | t.notOk(err) 182 | t.deepEquals(res, [ 183 | { key: 'foo', value: 'bar' }, 184 | { key: '', value: null }, 185 | { key: 'bax', value: 'quux' } 186 | ]) 187 | 188 | cleanup0() 189 | cleanup1() 190 | }) 191 | } 192 | }) 193 | }) 194 | }) 195 | 196 | // HyperDB, HyperDB => Error 197 | function replicate (local, remote, cb) { 198 | var rr = remote.replicate() 199 | var lr = local.replicate() 200 | onend(rr, doneReplication) 201 | onend(lr, doneReplication) 202 | rr.pipe(lr).pipe(rr) 203 | 204 | var pending = 2 205 | var error 206 | function doneReplication (err) { 207 | if (err) error = err 208 | if (--pending) return 209 | cb(error) 210 | } 211 | } 212 | 213 | function getContent (db, cb) { 214 | var res = [] 215 | db.createHistoryStream() 216 | .on('data', function (node) { 217 | res.push({key: node.key, value: node.value}) 218 | }) 219 | .once('end', cb.bind(null, null, res)) 220 | .once('error', cb) 221 | } 222 | --------------------------------------------------------------------------------