├── .gitignore ├── .github └── workflows │ └── test-node.yml ├── package.json ├── test ├── helpers │ └── index.js └── basic.js ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.github/workflows/test-node.yml: -------------------------------------------------------------------------------- 1 | name: Build Status 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | jobs: 11 | build: 12 | strategy: 13 | matrix: 14 | node-version: [14.x] 15 | os: [ubuntu-16.04, macos-latest, windows-latest] 16 | runs-on: ${{ matrix.os }} 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Use Node.js ${{ matrix.node-version }} 20 | uses: actions/setup-node@v1 21 | with: 22 | node-version: ${{ matrix.node-version }} 23 | - run: npm install 24 | - run: npm test 25 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@hyperswarm/replicator", 3 | "version": "2.0.2", 4 | "description": "Replicates data structures easily using hyperswarm", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "standard && tape test/*.js" 8 | }, 9 | "author": "Mathias Buus (@mafintosh)", 10 | "license": "MIT", 11 | "dependencies": { 12 | "hypercore-protocol": "^8.0.7", 13 | "hyperswarm": "^2.13.0", 14 | "pump": "^3.0.0" 15 | }, 16 | "devDependencies": { 17 | "dht-rpc": "^4.9.6", 18 | "hypercore": "^9.7.5", 19 | "random-access-memory": "^3.1.2", 20 | "standard": "^14.3.1", 21 | "tape": "^5.1.1" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/hyperswarm/replicator.git" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/helpers/index.js: -------------------------------------------------------------------------------- 1 | const dht = require('dht-rpc') 2 | const tape = require('tape') 3 | const Replicator = require('../../') 4 | 5 | module.exports = { get, append, ready, test } 6 | 7 | function get (core, seq) { 8 | return new Promise((resolve, reject) => { 9 | core.get(seq, (err, data) => { 10 | if (err) return reject(err) 11 | resolve(data) 12 | }) 13 | }) 14 | } 15 | 16 | function append (core, data) { 17 | return new Promise((resolve, reject) => { 18 | core.append(data, (err) => { 19 | if (err) return reject(err) 20 | resolve() 21 | }) 22 | }) 23 | } 24 | 25 | function ready (core) { 26 | return new Promise((resolve, reject) => { 27 | core.ready((err) => { 28 | if (err) return reject(err) 29 | resolve() 30 | }) 31 | }) 32 | } 33 | 34 | function test (msg, fn) { 35 | tape(msg, function (t) { 36 | return new Promise((resolve, reject) => { 37 | const bootstraper = dht({ ephemeral: true }) 38 | 39 | bootstraper.listen(0, async function () { 40 | const replicators = [makeReplicator(), makeReplicator()] 41 | 42 | let missing = replicators.length 43 | for (const r of replicators) { 44 | r.on('close', () => { 45 | if (--missing > 0) return 46 | bootstraper.destroy() 47 | }) 48 | } 49 | 50 | try { 51 | await fn(t, ...replicators) 52 | resolve() 53 | } catch (err) { 54 | reject(err) 55 | } finally { 56 | for (const r of replicators) r.destroy() 57 | } 58 | 59 | function makeReplicator () { 60 | const bootstrap = ['localhost:' + bootstraper.address().port] 61 | return new Replicator({ bootstrap }) 62 | } 63 | }) 64 | }) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | const hypercore = require('hypercore') 2 | const ram = require('random-access-memory') 3 | const { get, append, test } = require('./helpers') 4 | 5 | test('basic', async function (t, replicator, clone) { 6 | const core = hypercore(ram) 7 | 8 | replicator.add(core, { announce: true, lookup: false }) 9 | 10 | await append(core, 'test') 11 | 12 | const coreClone = hypercore(ram, core.key) 13 | 14 | clone.add(coreClone, { lookup: true, announce: false }) 15 | 16 | t.same(await get(coreClone, 0), Buffer.from('test')) 17 | }) 18 | 19 | test('multi core swarm', async function (t, replicator, clone) { 20 | const a = hypercore(ram) 21 | const b = hypercore(ram) 22 | 23 | replicator.add(a, { announce: true, lookup: false }) 24 | replicator.add(b, { announce: true, lookup: false }) 25 | 26 | await append(a, 'a test') 27 | await append(b, 'b test') 28 | 29 | const aClone = hypercore(ram, a.key) 30 | const bClone = hypercore(ram, b.key) 31 | 32 | clone.add(bClone, { lookup: true, announce: false }) 33 | clone.add(aClone, { lookup: true, announce: false }) 34 | 35 | t.same(await get(aClone, 0), Buffer.from('a test')) 36 | t.same(await get(bClone, 0), Buffer.from('b test')) 37 | }) 38 | 39 | test('multi core swarm higher latency', async function (t, replicator, clone) { 40 | const a = hypercore(ram) 41 | const b = hypercore(ram) 42 | 43 | replicator.add(a, { announce: true, lookup: false }) 44 | 45 | await append(a, 'a test') 46 | await append(b, 'b test') 47 | 48 | const aClone = hypercore(ram, a.key) 49 | const bClone = hypercore(ram, b.key) 50 | 51 | clone.add(bClone, { lookup: true, announce: false }) 52 | clone.add(aClone, { lookup: true, announce: false }) 53 | 54 | replicator.on('discovery-key', function () { 55 | replicator.add(b, { announce: true, lookup: false }) 56 | }) 57 | 58 | t.same(await get(aClone, 0), Buffer.from('a test')) 59 | t.same(await get(bClone, 0), Buffer.from('b test')) 60 | }) 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @hyperswarm/replicator 2 | 3 | Replicate data structures easily using hyperswarm 4 | 5 | ## Install 6 | 7 | ```js 8 | npm install @hyperswarm/replicator 9 | ``` 10 | 11 | ## Usage 12 | 13 | You data structure has to support a .replicate() stream, then you can replicate 14 | them using the hyperswarm replicator. 15 | 16 | ```js 17 | const Replicator = require('@hyperswarm/replicator') 18 | 19 | const r = new Replicator() 20 | 21 | r.add(aHypercore, { 22 | live: true // passed to .replicate 23 | }) 24 | ``` 25 | 26 | ## API 27 | 28 | #### `r = new Replicator([options])` 29 | 30 | Make a new replicator. Options include: 31 | 32 | ```js 33 | { 34 | bootstrap: [...], // optional set the DHT bootstrap servers 35 | } 36 | ``` 37 | 38 | #### `promise = r.add(hyperDataStructure, [options])` 39 | 40 | Add a hyper* data structure to replicate. 41 | 42 | ```js 43 | { 44 | live: bool, // passed to .replicate 45 | upload: bool, // passed to .replicate 46 | download: bool, // passed to .replicate 47 | encrypt: bool, // passed to .replicate 48 | discoveryKey: , // optionally set your own discovery key 49 | announce: true, // should the swarm announce you? 50 | lookup: true, // should the swarm do lookups for you? 51 | keyPair: { publicKey, secretKey }, // noise keypair used for the connection 52 | onauthenticate (remotePublicKey, done) // the onauthenticate hook to verify remote key pairs 53 | } 54 | ``` 55 | 56 | Promise resolves when the data structure has been fully added. 57 | 58 | 59 | #### `promise = r.remove(hyperDataStructure)` 60 | 61 | Remove a data structure from replication. 62 | Promise resolves when the data structure has been fully removed. 63 | 64 | #### `r.swarm` 65 | 66 | The associated hyperswarm instance. 67 | 68 | #### `r.on('discovery-key', (discoveryKey, remoteStream) => ...)` 69 | 70 | Emitted when a remote asks for a discovery key of a data structure you are 71 | not currently replicating. 72 | 73 | #### `r = Replicator.replicate(hyperDataStructure[s])` 74 | 75 | Easy "one off" replication of one or more data structures. 76 | 77 | ## License 78 | 79 | MIT 80 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const pump = require('pump') 2 | const hyperswarm = require('hyperswarm') 3 | const Protocol = require('hypercore-protocol') 4 | const { EventEmitter } = require('events') 5 | 6 | const promises = Symbol.for('hypercore.promises') 7 | 8 | class Event { 9 | constructor () { 10 | this.triggered = false 11 | this.fns = new Set() 12 | } 13 | 14 | on (fn) { 15 | if (this.triggered) return fn() 16 | this.fns.add(fn) 17 | } 18 | 19 | emit () { 20 | this.triggered = true 21 | for (const fn of this.fns) fn() 22 | } 23 | } 24 | 25 | module.exports = class Replicator extends EventEmitter { 26 | constructor (options = {}) { 27 | super() 28 | 29 | this.swarm = hyperswarm({ 30 | announceLocalAddress: !!options.announceLocalAddress, 31 | preferredPort: 49737, 32 | bootstrap: options.bootstrap, 33 | queue: { 34 | multiplex: true 35 | } 36 | }) 37 | 38 | this.createStream = options.createStream || ((init) => new Protocol(init, options)) 39 | this.swarm.on('connection', this._onconnection.bind(this)) 40 | this.swarming = new Map() 41 | this.streams = new Set() 42 | } 43 | 44 | static replicate (cores, options) { 45 | const r = new Replicator({ createStream () { return null } }) 46 | if (!Array.isArray(cores)) cores = [cores] 47 | 48 | for (const core of cores) { 49 | r.add(core, options).catch(() => {}) 50 | } 51 | 52 | return r 53 | } 54 | 55 | get dht () { 56 | return this.swarm.network && this.swarm.network.discovery && this.swarm.network.discovery.dht 57 | } 58 | 59 | _onconnection (socket, info) { 60 | this.emit('connection', socket, info) 61 | 62 | let stream = this.createStream(info.client) 63 | 64 | for (const { core, options, one } of this.swarming.values()) { 65 | stream = core.replicate(info.client, { ...options, stream }) 66 | one.emit() 67 | } 68 | 69 | this.emit('replication-stream', stream, info) 70 | 71 | pump(stream, socket, stream) 72 | stream.on('discovery-key', this._ondiscoverykey.bind(this, stream)) 73 | 74 | this.streams.add(stream) 75 | stream.on('close', () => this.streams.delete(stream)) 76 | } 77 | 78 | _ondiscoverykey (stream, discoveryKey) { 79 | const key = discoveryKey.toString('hex') 80 | 81 | if (!this.swarming.has(key)) { 82 | this.emit('discovery-key', discoveryKey, stream) 83 | return 84 | } 85 | 86 | const { core, options, one } = this.swarming.get(key) 87 | core.replicate(false, { ...options, stream }) 88 | one.emit() 89 | } 90 | 91 | listen () { 92 | this.swarm.listen() 93 | } 94 | 95 | destroy () { 96 | return new Promise((resolve, reject) => { 97 | this.swarm.destroy((err) => { 98 | if (err) return reject(err) 99 | this.emit('close') 100 | resolve() 101 | }) 102 | }) 103 | } 104 | 105 | async add (core, options = {}) { 106 | await ready(core) 107 | 108 | const key = core.discoveryKey.toString('hex') 109 | const { announce, lookup } = options 110 | const defaultLookup = lookup === undefined && announce === undefined 111 | const added = this.swarming.has(key) 112 | 113 | const one = new Event() 114 | const all = new Event() 115 | 116 | this.swarming.set(key, { core, options, one, all }) 117 | 118 | if (announce || lookup || defaultLookup) { 119 | this.swarm.join(core.discoveryKey, { announce: !!announce, lookup: !!lookup || defaultLookup }) 120 | this.swarm.flush(onflush) 121 | } else { 122 | onflush() 123 | } 124 | 125 | if (core.timeouts) { // current timeout api support ... 126 | const { update, get } = core.timeouts 127 | if (update) core.timeouts.update = (cb) => one.on(() => update(cb)) 128 | if (get) core.timeouts.get = (cb) => all.on(() => get(cb)) 129 | } 130 | 131 | if (!added) { 132 | for (const stream of this.streams) { 133 | core.replicate(false, { ...options, stream }) 134 | one.emit() 135 | } 136 | } 137 | 138 | this.emit('add', core, options) 139 | 140 | function onflush () { 141 | one.emit() 142 | all.emit() 143 | } 144 | } 145 | 146 | async remove (core) { 147 | await ready(core) 148 | 149 | const key = core.discoveryKey.toString('hex') 150 | 151 | if (!this.swarming.has(key)) return 152 | 153 | const { options } = this.swarming.get(key) 154 | this.swarming.delete(key) 155 | this.swarm.leave(core.discoveryKey) 156 | 157 | this.emit('remove', core, options) 158 | } 159 | } 160 | 161 | function ready (core) { 162 | if (!core.ready) return 163 | if (core[promises]) return core.ready() 164 | 165 | return new Promise((resolve, reject) => { 166 | const p = core.ready((err) => { 167 | if (err) return reject(err) 168 | resolve() 169 | }) 170 | 171 | if (p && typeof p.then === 'function') { 172 | p.then(resolve, reject) 173 | } 174 | }) 175 | } 176 | --------------------------------------------------------------------------------