├── .gitignore ├── LICENSE ├── README.md ├── bin.js ├── example.js ├── index.js ├── package.json ├── schema.proto └── store.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hyperdht 2 | 3 | A DHT that supports peer discovery and distributed hole punching 4 | 5 | ``` 6 | npm install hyperdht 7 | ``` 8 | 9 | ## Usage 10 | 11 | First run a bootstrap node 12 | 13 | ``` sh 14 | npm install -g dht-rpc-bootstrap 15 | dht-rpc-bootstrap --port=10000 16 | ``` 17 | 18 | ``` js 19 | var hyperdht = require('hyperdht') 20 | 21 | var a = hyperdht({ 22 | bootstrap: ['localhost:10000'] 23 | }) 24 | 25 | var b = hyperdht({ 26 | bootstrap: ['localhost:10000'] 27 | }) 28 | 29 | a.ready(function () { 30 | // announce on a 32 byte key 31 | var key = new Buffer('01234567012345670123456701234567') 32 | 33 | b.announce(key, {port: 10000}, function (err) { 34 | if (err) throw err 35 | 36 | var stream = a.lookup(key) 37 | 38 | stream.on('data', function (data) { 39 | console.log('found peers:', data) 40 | }) 41 | }) 42 | }) 43 | ``` 44 | 45 | ## Usage 46 | 47 | #### `var dht = hyperdht([options])` 48 | 49 | Create a new dht. Options are passed to the [dht-rpc](https://github.com/mafintosh/dht-rpc) constructor 50 | 51 | #### `var stream = dht.announce(key, [options], [callback])` 52 | 53 | Announce that you are listening on a key. Options include 54 | 55 | ``` js 56 | { 57 | port: 10000, // port you are listening. If omitted the udp sockets port is used 58 | localAddress: { 59 | host: '192.168.1.2', // announce that you are listening on a local address also 60 | port: 8888 61 | } 62 | } 63 | ``` 64 | 65 | The returned stream will emit peers as they are discovered during the announce face. 66 | The data events look like this 67 | 68 | ``` js 69 | { 70 | node: { 71 | host: '10.4.2.4', // dht node's host 72 | port: 42424 // dht node's port 73 | }, 74 | peers: [{ 75 | host: '4.41.3.4', // a peer host 76 | port: 4244, // a peer port 77 | }], 78 | localPeers: [{ 79 | host: '192.168.3.4', // a local peer host 80 | port: 4244, // a local peer port 81 | }] 82 | } 83 | ``` 84 | 85 | Local peers will only contain addresses that share the first two parts of your local address (`192.168` in the example). 86 | Both `peers` and `localPeers` will *not* contain your own address. 87 | 88 | If you provide the callback the stream will be buffers and an array of results is passed. 89 | Note that you should keep announcing yourself at regular intervals (fx every 4-5min) 90 | 91 | #### `var stream = dht.lookup(key, [options], [callback])` 92 | 93 | Find peers but do not announce. Accepts the same options as `announce` and returns a similar stream. 94 | 95 | #### `dht.unannounce(key, [options], [callback])` 96 | 97 | Remove yourself from the DHT. Pass the same options as you used to announce yourself. 98 | 99 | #### `dht.ping(peer, callback)` 100 | 101 | Ping a another DHT peer. Useful if you want to see if you can connect to a node without holepunching. 102 | 103 | #### `dht.holepunch(peer, node, callback)` 104 | 105 | UDP holepunch to another peer. Pass the same node as was returned in the announce/lookup stream 106 | for the peer. 107 | 108 | #### `dht.ready(callback)` 109 | 110 | Wait for the dht to be fully bootstrapped. You do not need to call this before annnouncing / querying. 111 | 112 | #### `dht.destroy(callback)` 113 | 114 | Destroy the dht and stop listening. 115 | 116 | #### `dht.bootstrap([callback])` 117 | 118 | Re-bootstrap the DHT. Call this at regular intervals if you do not announce/lookup any keys. 119 | 120 | #### `dht.listen(port, callback)` 121 | 122 | Explicitly listen on a port. If you do not call this a random port will be chosen for you. 123 | 124 | ## Command line tool 125 | 126 | There is a command line tool available as well which is useful if you want to run a long lived dht node on a server or similar. 127 | 128 | ``` sh 129 | npm install -g hyperdht 130 | hyperdht --help 131 | ``` 132 | 133 | To run a node simply pass it the addresses of your bootstrap servers 134 | 135 | ``` sh 136 | hyperdht --bootstrap=localhost:10000 --bootstrap=localhost:10001 137 | ``` 138 | 139 | ## License 140 | 141 | MIT 142 | -------------------------------------------------------------------------------- /bin.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var minimist = require('minimist') 4 | var hyperdht = require('./') 5 | 6 | var argv = minimist(process.argv, { 7 | alias: { 8 | bootstrap: 'b', 9 | port: 'p' 10 | } 11 | }) 12 | 13 | if (argv.help) { 14 | console.error( 15 | 'Usage: hyperdht [options]\n' + 16 | ' --bootstrap, -b host:port\n' + 17 | ' --port, -p listen-port\n' + 18 | ' --quiet' 19 | ) 20 | process.exit(0) 21 | } 22 | 23 | var dht = hyperdht({ 24 | bootstrap: argv.bootstrap 25 | }) 26 | 27 | dht.on('announce', function (key, peer) { 28 | if (!argv.quiet) console.log('announce:', key.toString('hex'), peer) 29 | }) 30 | 31 | dht.on('unannounce', function (key, peer) { 32 | if (!argv.quiet) console.log('unannounce:', key.toString('hex'), peer) 33 | }) 34 | 35 | dht.on('lookup', function (key) { 36 | if (!argv.quiet) console.log('lookup:', key.toString('hex')) 37 | }) 38 | 39 | dht.listen(argv.port, function () { 40 | console.log('hyperdht listening on ' + dht.address().port) 41 | }) 42 | 43 | dht.ready(function loop () { 44 | if (!argv.quiet) console.log('bootstrapped...') 45 | setTimeout(bootstrap, Math.floor((5 + Math.random() * 60) * 1000)) 46 | 47 | function bootstrap () { 48 | dht.bootstrap(loop) 49 | } 50 | }) 51 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var hyperdht = require('./') 2 | 3 | var a = hyperdht({ 4 | bootstrap: ['localhost:10000'] 5 | }) 6 | 7 | var b = hyperdht({ 8 | bootstrap: ['localhost:10000'] 9 | }) 10 | 11 | a.ready(function () { 12 | // announce on a 32 byte key 13 | var key = new Buffer('01234567012345670123456701234567') 14 | 15 | console.log('announcing port...') 16 | b.announce(key, {port: 10000}, function (err) { 17 | if (err) throw err 18 | 19 | var stream = a.lookup(key) 20 | 21 | stream.on('data', function (data) { 22 | console.log('found peers:', data) 23 | }) 24 | 25 | stream.on('end', function () { 26 | console.log('unannouncing...') 27 | b.unannounce(key, {port: 10000}, function (err) { 28 | if (err) throw err 29 | process.exit() 30 | }) 31 | }) 32 | }) 33 | }) 34 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var dht = require('dht-rpc') 2 | var protobuf = require('protocol-buffers') 3 | var ipv4 = require('ipv4-peers') 4 | var equals = require('buffer-equals') 5 | var store = require('./store') 6 | var fs = require('fs') 7 | var inherits = require('inherits') 8 | var events = require('events') 9 | var path = require('path') 10 | 11 | var PEERS = new Buffer(1024) 12 | var LOCAL_PEERS = new Buffer(1024) 13 | 14 | var messages = protobuf(fs.readFileSync(path.join(__dirname, 'schema.proto'))) 15 | 16 | module.exports = HyperDHT 17 | 18 | function HyperDHT (opts) { 19 | if (!(this instanceof HyperDHT)) return new HyperDHT(opts) 20 | if (!opts) opts = {} 21 | events.EventEmitter.call(this) 22 | 23 | if (opts.bootstrap) { 24 | opts.bootstrap = [].concat(opts.bootstrap).map(parseBootstrap) 25 | } 26 | 27 | this.dht = dht(opts) 28 | this.id = this.dht.id 29 | this.socket = this.dht.socket.socket 30 | this._store = store() 31 | 32 | var self = this 33 | 34 | this.dht.on('error', function (err) { 35 | self.emit('error', err) 36 | }) 37 | 38 | this.dht.on('query:peers', function (data, cb) { 39 | self._onquery(data, false, cb) 40 | }) 41 | 42 | this.dht.on('update:peers', function (data, cb) { 43 | self._onquery(data, true, cb) 44 | }) 45 | } 46 | 47 | inherits(HyperDHT, events.EventEmitter) 48 | 49 | HyperDHT.prototype._onquery = function (data, update, cb) { 50 | var req = data.value && decode(messages.Request, data.value) 51 | if (!req) return cb() 52 | var res = this._processPeers(req, data, update) 53 | if (!res) return cb() 54 | cb(null, messages.Response.encode(res)) 55 | } 56 | 57 | HyperDHT.prototype._processPeers = function (req, data, update) { 58 | var from = {host: data.node.host, port: req.port || data.node.port} 59 | if (!from.port) return null // TODO: nessesary? check dht-rpc / udp-socket 60 | 61 | var key = data.target.toString('hex') 62 | var peer = encodePeer(from) 63 | 64 | if (update && req.type === 1) { 65 | var id = from.host + ':' + from.port 66 | var stored = { 67 | localFilter: null, 68 | localPeer: null, 69 | peer: peer 70 | } 71 | 72 | if (req.localAddress && decodePeer(req.localAddress)) { 73 | stored.localFilter = req.localAddress.slice(0, 2) 74 | stored.localPeer = req.localAddress.slice(2) 75 | } 76 | 77 | this.emit('announce', data.target, from) 78 | this._store.put(key, id, stored) 79 | } else if (update && req.type === 2) { 80 | this.emit('unannounce', data.target, from) 81 | this._store.del(key, from.host + ':' + from.port) 82 | return null 83 | } else if (req.type === 0) { 84 | this.emit('lookup', data.target) 85 | } 86 | 87 | var off1 = 0 88 | var off2 = 0 89 | var next = this._store.iterator(key) 90 | var filter = req.localAddress && req.localAddress.length === 6 && req.localAddress.slice(0, 2) 91 | 92 | while (off1 + off2 < 900) { 93 | var n = next() 94 | if (!n) break 95 | if (equals(n.peer, peer)) continue 96 | 97 | n.peer.copy(PEERS, off1) 98 | off1 += 6 99 | 100 | if (n.localPeer && filter && filter[0] === n.localFilter[0] && filter[1] === n.localFilter[1]) { 101 | if (!equals(n.localPeer, req.localAddress)) { 102 | n.localPeer.copy(LOCAL_PEERS, off2) 103 | off2 += 4 104 | } 105 | } 106 | } 107 | 108 | if (!off1 && !off2) return null 109 | 110 | return { 111 | peers: PEERS.slice(0, off1), 112 | localPeers: off2 ? LOCAL_PEERS.slice(0, off2) : null 113 | } 114 | } 115 | 116 | HyperDHT.prototype._processPeersLocal = function (key, req, stream) { 117 | if (!this._store.has(key.toString('hex'))) return 118 | 119 | var data = {node: {id: this.dht.id, host: '127.0.0.1', port: this.dht.address().port}, target: key} 120 | var res = this._processPeers(req, data, false) 121 | if (!res) return 122 | 123 | stream.push({ 124 | node: data.node, 125 | peers: ipv4.decode(res.peers), 126 | localPeers: decodeLocalPeers(res.localPeers, req.localAddress) 127 | }) 128 | } 129 | 130 | HyperDHT.prototype.address = function () { 131 | return this.dht.address() 132 | } 133 | 134 | HyperDHT.prototype.bootstrap = function (cb) { 135 | this.dht.bootstrap(cb) 136 | } 137 | 138 | HyperDHT.prototype.listen = function (port, cb) { 139 | this.dht.listen(port, cb) 140 | } 141 | 142 | HyperDHT.prototype.destroy = function (cb) { 143 | this.dht.destroy(cb) 144 | } 145 | 146 | HyperDHT.prototype.ready = function (cb) { 147 | this.dht.ready(cb) 148 | } 149 | 150 | HyperDHT.prototype.ping = function (peer, cb) { 151 | this.dht.ping(peer, cb) 152 | } 153 | 154 | HyperDHT.prototype.holepunch = function (peer, ref, cb) { 155 | if (ref.id && equals(ref.id, this.id)) return cb() 156 | this.dht.holepunch(peer, ref, cb) 157 | } 158 | 159 | HyperDHT.prototype.announce = function (key, opts, cb) { 160 | if (typeof opts === 'function') return this.announce(key, null, opts) 161 | if (typeof opts === 'number') opts = {port: opts} 162 | if (!opts) opts = {} 163 | 164 | var localAddress = encodePeer(opts.localAddress) 165 | var req = { 166 | type: 1, 167 | port: opts.port || 0, 168 | localAddress: localAddress 169 | } 170 | 171 | var map = mapper(localAddress) 172 | var stream = this.dht.update({ 173 | target: key, 174 | command: 'peers', 175 | value: messages.Request.encode(req) 176 | }, { 177 | query: true, 178 | map: map 179 | }, cb) 180 | 181 | this._processPeersLocal(key, req, stream) 182 | 183 | return stream 184 | } 185 | 186 | HyperDHT.prototype.unannounce = function (key, opts, cb) { 187 | if (typeof opts === 'function') return this.unannounce(key, null, opts) 188 | if (typeof opts === 'number') opts = {port: opts} 189 | if (!opts) opts = {} 190 | 191 | var req = { 192 | type: 2, 193 | port: opts.port || 0, 194 | localAddress: encodePeer(opts.localAddress) 195 | } 196 | 197 | this.dht.update({ 198 | target: key, 199 | command: 'peers', 200 | value: messages.Request.encode(req) 201 | }, cb) 202 | } 203 | 204 | HyperDHT.prototype.lookup = function (key, opts, cb) { 205 | if (typeof opts === 'function') return this.lookup(key, null, opts) 206 | if (!opts) opts = {} 207 | if (opts.localAddress && !opts.localAddress.port) opts.localAddress.port = 0 208 | 209 | var localAddress = encodePeer(opts.localAddress) 210 | var req = { 211 | type: 0, 212 | localAddress: localAddress, 213 | port: opts.port || 0 214 | } 215 | 216 | var map = mapper(localAddress) 217 | var stream = this.dht.query({ 218 | target: key, 219 | command: 'peers', 220 | value: messages.Request.encode(req) 221 | }, { 222 | map: map 223 | }, cb) 224 | 225 | this._processPeersLocal(key, req, stream) 226 | 227 | return stream 228 | } 229 | 230 | function mapper (localAddress) { 231 | return map 232 | 233 | function map (data) { 234 | var res = decode(messages.Response, data.value) 235 | if (!res) return null 236 | 237 | var peers = res.peers && decode(ipv4, res.peers) 238 | if (!peers) return null 239 | 240 | var v = { 241 | node: data.node, 242 | peers: peers, 243 | localPeers: decodeLocalPeers(res.localPeers, localAddress) 244 | } 245 | 246 | return v 247 | } 248 | } 249 | 250 | function decodeLocalPeers (buf, localAddress) { 251 | var localPeers = [] 252 | if (!localAddress || !buf) return localPeers 253 | 254 | for (var i = 0; i < buf.length; i += 4) { 255 | if (buf.length - i < 4) return localPeers 256 | 257 | var port = buf.readUInt16BE(i + 2) 258 | if (!port || port === 65536) continue 259 | 260 | localPeers.push({ 261 | host: localAddress[0] + '.' + localAddress[1] + '.' + buf[i] + '.' + buf[i + 1], 262 | port: port 263 | }) 264 | } 265 | 266 | return localPeers 267 | } 268 | 269 | function parseBootstrap (node) { 270 | return node.indexOf(':') === -1 ? node + ':49737' : node 271 | } 272 | 273 | function encodePeer (p) { 274 | return p && ipv4.encode([p]) 275 | } 276 | 277 | function decodePeer (b) { 278 | var p = b && decode(ipv4, b) 279 | return p && p[0] 280 | } 281 | 282 | function decode (enc, buf) { 283 | try { 284 | return enc.decode(buf) 285 | } catch (err) { 286 | return null 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hyperdht", 3 | "version": "1.3.2", 4 | "description": "A DHT that supports peer discovery and distributed hole punching", 5 | "main": "index.js", 6 | "dependencies": { 7 | "buffer-equals": "^1.0.4", 8 | "dht-rpc": "^3.0.1", 9 | "inherits": "^2.0.3", 10 | "ipv4-peers": "^1.1.1", 11 | "minimist": "^1.2.0", 12 | "protocol-buffers": "^3.2.1", 13 | "unordered-set": "^2.0.0" 14 | }, 15 | "devDependencies": { 16 | "standard": "^8.0.0" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/mafintosh/hyperdht.git" 21 | }, 22 | "scripts": { 23 | "test": "standard" 24 | }, 25 | "bin": { 26 | "hyperdht": "./bin.js" 27 | }, 28 | "author": "Mathias Buus (@mafintosh)", 29 | "license": "MIT", 30 | "bugs": { 31 | "url": "https://github.com/mafintosh/hyperdht/issues" 32 | }, 33 | "homepage": "https://github.com/mafintosh/hyperdht" 34 | } 35 | -------------------------------------------------------------------------------- /schema.proto: -------------------------------------------------------------------------------- 1 | message Request { 2 | required uint32 type = 1; 3 | optional uint32 port = 2; 4 | optional bytes localAddress = 3; 5 | } 6 | 7 | message Response { 8 | optional bytes peers = 1; 9 | optional bytes localPeers = 2; 10 | } 11 | -------------------------------------------------------------------------------- /store.js: -------------------------------------------------------------------------------- 1 | var set = require('unordered-set') 2 | var empty = [] 3 | 4 | module.exports = Store 5 | 6 | function Store (opts) { 7 | if (!(this instanceof Store)) return new Store(opts) 8 | if (!opts) opts = {} 9 | 10 | this._values = {} 11 | this._lists = {} 12 | this._pValues = {} 13 | this._pLists = {} 14 | this._size = 0 15 | this.max = opts.max || 65536 16 | this.maxAge = opts.maxAge || 12 * 60 * 1000 17 | } 18 | 19 | Store.prototype.has = function (key) { 20 | return !!(this._lists[key] || this._pLists[key]) 21 | } 22 | 23 | Store.prototype.put = function (key, id, val) { 24 | var k = key + '@' + id 25 | var list = this._lists[key] 26 | var prev = this._values[k] 27 | 28 | if (prev) { 29 | prev.age = Date.now() 30 | prev.value = val 31 | return 32 | } 33 | 34 | prev = this._pValues[k] 35 | if (prev) { 36 | set.remove(prev.list, prev) 37 | this._pValues[k] = null 38 | } 39 | 40 | if (!list) { 41 | this._size++ 42 | list = this._lists[key] = [] 43 | } 44 | 45 | var v = { 46 | _index: list.length, 47 | list: list, 48 | value: val, 49 | age: Date.now() 50 | } 51 | 52 | this._values[k] = v 53 | this._size++ 54 | list.push(v) 55 | 56 | if (this._size > this.max) this._gc() 57 | } 58 | 59 | Store.prototype.del = function (key, id) { 60 | var k = key + '@' + id 61 | 62 | var v = this._values[k] 63 | if (v) { 64 | this._values[k] = null 65 | set.remove(v.list, v) 66 | } 67 | 68 | v = this._pValues[k] 69 | if (v) { 70 | this._pValues[k] = null 71 | set.remove(v.list, v) 72 | } 73 | } 74 | 75 | Store.prototype._gc = function () { 76 | this._pValues = this._values 77 | this._pLists = this._lists 78 | this._values = {} 79 | this._lists = {} 80 | this._size = 0 81 | } 82 | 83 | Store.prototype.iterator = function (key) { 84 | var list = this._lists[key] || empty 85 | var prevList = this._pLists[key] || empty 86 | var missing = list.length + prevList.length 87 | var maxAge = this.maxAge 88 | var now = Date.now() 89 | 90 | var i = 0 91 | var pi = 0 92 | 93 | return function () { 94 | while (true) { 95 | if (!missing) return null 96 | 97 | var next = (Math.random() * missing) | 0 98 | var n = null 99 | missing-- 100 | 101 | if (next < list.length - i) { 102 | set.swap(list, list[i], list[next + i]) 103 | n = list[i++] 104 | } else { 105 | set.swap(prevList, prevList[pi], prevList[next - (list.length - i)]) 106 | n = prevList[pi++] 107 | } 108 | 109 | if (now - n.age > maxAge) continue 110 | return n.value 111 | } 112 | } 113 | } 114 | --------------------------------------------------------------------------------