├── .datignore ├── .eslintrc ├── .gitignore ├── .travis.yml ├── README.md ├── bin.js ├── index.html ├── index.js ├── package.json └── test.js /.datignore: -------------------------------------------------------------------------------- 1 | **/node_modules/** 2 | .nyc_output/ 3 | package-lock.json 4 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "standard" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | .nyc_output/ 4 | .dat/ 5 | .idea/ 6 | .DS_Store 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 8 4 | - 6 5 | after_success: 6 | - npm i -g nyc coveralls 7 | - nyc npm test && nyc report --reporter=text-lcov | coveralls 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # **Deprecated** 3 | 4 | **Please use the canonical version at [@garbados/dat-gateway](https://github.com/garbados/dat-gateway/)** 5 | -------------------------------------------------------------------------------- /bin.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | const DatGateway = require('.') 6 | const os = require('os') 7 | const mkdirp = require('mkdirp') 8 | const pkg = require('./package.json') 9 | 10 | require('yargs') 11 | .version(pkg.version) 12 | .command({ 13 | command: '$0', 14 | aliases: ['start'], 15 | builder: function (yargs) { 16 | yargs.options({ 17 | port: { 18 | alias: 'p', 19 | description: 'Port for the gateway to listen on.', 20 | default: 3000 21 | }, 22 | dir: { 23 | alias: 'd', 24 | description: 'Directory to use as a cache.', 25 | coerce: function (value) { 26 | return value.replace('~', os.homedir()) 27 | }, 28 | default: '~/.dat-gateway', 29 | normalize: true 30 | }, 31 | max: { 32 | alias: 'm', 33 | description: 'Maximum number of archives allowed in the cache.', 34 | default: 20 35 | }, 36 | period: { 37 | description: 'Number of milliseconds between cleaning the cache of expired archives.', 38 | default: 60 * 1000 // every minute 39 | }, 40 | ttl: { 41 | alias: 't', 42 | description: 'Number of milliseconds before archives expire.', 43 | default: 10 * 60 * 1000 // ten minutes 44 | }, 45 | redirect: { 46 | alias: 'r', 47 | description: 'Whether to use subdomain redirects', 48 | default: false 49 | } 50 | }) 51 | }, 52 | handler: function (argv) { 53 | const { dir, max, period, port, ttl, redirect } = argv 54 | mkdirp.sync(dir) // make sure it exists 55 | const gateway = new DatGateway({ dir, max, period, ttl, redirect }) 56 | gateway 57 | .load() 58 | .then(() => { 59 | return gateway.listen(port) 60 | }) 61 | .then(function () { 62 | console.log('[dat-gateway] Now listening on port ' + port) 63 | }) 64 | .catch(console.error) 65 | } 66 | }) 67 | .alias('h', 'help') 68 | .config() 69 | .parse() 70 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dat-gateway 5 | 6 | 7 | 42 | 43 | 44 |
45 |
46 |

dat-gateway

47 |

portal to a decentralized web

48 |
49 |

You can use this gateway to visit content hosted over the p2p Dat protocol.

50 |

So why not visit something?

51 |

Enter Dat keys in the URL like this: /:key/:path or use the form below.

52 |

53 |

54 | 55 | 56 |
57 |

58 |
59 |
60 |

61 | 62 | View the source. Forked from @garbados. 63 | 64 |

65 |
66 |
67 |
68 | 69 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const DatLibrarian = require('dat-librarian') 4 | const fs = require('fs') 5 | const http = require('http') 6 | const hyperdriveHttp = require('hyperdrive-http') 7 | const path = require('path') 8 | const Websocket = require('websocket-stream') 9 | const url = require('url') 10 | const hexTo32 = require('hex-to-32') 11 | 12 | const BASE_32_KEY_LENGTH = 52 13 | const ERR_404 = 'Not found' 14 | const ERR_500 = 'Server error' 15 | 16 | function log () { 17 | let msg = arguments[0] 18 | arguments[0] = '[dat-gateway] ' + msg 19 | if (process.env.DEBUG || process.env.LOG) { 20 | console.log.apply(console, arguments) 21 | } 22 | } 23 | 24 | module.exports = 25 | class DatGateway extends DatLibrarian { 26 | constructor ({ dir, dat, max, net, period, ttl, redirect }) { 27 | dat = dat || {} 28 | if (typeof dat.temp === 'undefined') { 29 | dat.temp = dat.temp || true // store dats in memory only 30 | } 31 | super({ dir, dat, net }) 32 | this.redirect = redirect 33 | this.max = max 34 | this.ttl = ttl 35 | this.period = period 36 | this.lru = {} 37 | if (this.ttl && this.period) { 38 | this.cleaner = setInterval(() => { 39 | log('Checking for expired archives...') 40 | const tasks = Object.keys(this.dats).filter((key) => { 41 | const now = Date.now() 42 | let lastRead = this.lru[key] 43 | return (lastRead && ((now - lastRead) > this.ttl)) 44 | }).map((key) => { 45 | log('Deleting expired archive %s', key) 46 | delete this.lru[key] 47 | return this.remove(key) 48 | }) 49 | return Promise.all(tasks) 50 | }, this.period) 51 | } 52 | } 53 | 54 | load () { 55 | log('Setting up...') 56 | return this.getHandler().then((handler) => { 57 | log('Setting up server...') 58 | this.server = http.createServer(handler) 59 | const websocketHandler = this.getWebsocketHandler() 60 | this.websocketServer = Websocket.createServer({ 61 | perMessageDeflate: false, 62 | server: this.server 63 | }, websocketHandler) 64 | }).then(() => { 65 | log('Loading pre-existing archives...') 66 | // load pre-existing archives 67 | return super.load() 68 | }) 69 | } 70 | 71 | /** 72 | * Promisification of server.listen() 73 | * @param {Number} port Port to listen on. 74 | * @return {Promise} Promise that resolves once the server has started listening. 75 | */ 76 | listen (port) { 77 | return new Promise((resolve, reject) => { 78 | this.server.listen(port, (err) => { 79 | if (err) return reject(err) 80 | else return resolve() 81 | }) 82 | }) 83 | } 84 | 85 | close () { 86 | if (this.cleaner) clearInterval(this.cleaner) 87 | return new Promise((resolve) => { 88 | if (this.server) this.server.close(resolve) 89 | else resolve() 90 | }).then(() => { 91 | return super.close() 92 | }) 93 | } 94 | 95 | getIndexHtml () { 96 | return new Promise((resolve, reject) => { 97 | let filePath = path.join(__dirname, 'index.html') 98 | fs.readFile(filePath, 'utf-8', (err, html) => { 99 | if (err) return reject(err) 100 | else return resolve(html) 101 | }) 102 | }) 103 | } 104 | 105 | getWebsocketHandler () { 106 | return (stream, req) => { 107 | stream.on('error', function (e) { 108 | log('getWebsocketHandler has error: ' + e) 109 | }) 110 | const urlParts = req.url.split('/') 111 | const address = urlParts[1] 112 | if (!address) { 113 | stream.end('Must provide archive key') 114 | return Promise.resolve() 115 | } 116 | return this.addIfNew(address).then((dat) => { 117 | const archive = dat.archive 118 | const replication = archive.replicate({ 119 | live: true 120 | }) 121 | 122 | // Relay error events 123 | replication.on('error', function (e) { 124 | stream.emit('error', e) 125 | }) 126 | stream.pipe(replication).pipe(stream) 127 | }).catch((e) => { 128 | stream.end(e.message) 129 | }) 130 | } 131 | } 132 | 133 | getHandler () { 134 | return this.getIndexHtml().then((welcome) => { 135 | return (req, res) => { 136 | res.setHeader('Access-Control-Allow-Origin', '*') 137 | const start = Date.now() 138 | // TODO redirect /:key to /:key/ 139 | let requestURL = `http://${req.headers.host}${req.url}` 140 | let urlParts = url.parse(requestURL) 141 | let pathParts = urlParts.pathname.split('/').slice(1) 142 | let hostnameParts = urlParts.hostname.split('.') 143 | 144 | let subdomain = hostnameParts[0] 145 | let isRedirecting = this.redirect && (subdomain.length === BASE_32_KEY_LENGTH) 146 | 147 | let address = isRedirecting ? hexTo32.decode(subdomain) : pathParts[0] 148 | let path = (isRedirecting ? pathParts : pathParts.slice(1)).join('/') 149 | 150 | const logError = (err, end) => log('[%s] %s %s | ERROR %s [%i ms]', address, req.method, path, err.message, end - start) 151 | log('[%s] %s %s', address, req.method, path) 152 | 153 | // return index 154 | if (!isRedirecting && !address) { 155 | res.writeHead(200) 156 | res.end(welcome) 157 | return Promise.resolve() 158 | } 159 | 160 | // redirect to subdomain 161 | if (!isRedirecting && this.redirect) { 162 | return DatLibrarian.resolve(address).then((resolvedAddress) => { 163 | // TODO: Detect DatDNS addresses 164 | let encodedAddress = hexTo32.encode(resolvedAddress) 165 | let redirectURL = `http://${encodedAddress}.${urlParts.host}/${path}${urlParts.search || ''}` 166 | 167 | log('Redirecting %s to %s', address, redirectURL) 168 | res.setHeader('Location', redirectURL) 169 | res.writeHead(302) 170 | res.end() 171 | }).catch((e) => { 172 | const end = Date.now() 173 | logError(e, end) 174 | res.writeHead(500) 175 | res.end(ERR_500) 176 | }) 177 | } 178 | 179 | // Return a Dat DNS entry without fetching it from the archive 180 | if (path === '.well-known/dat') { 181 | return DatLibrarian.resolve(address).then((resolvedAddress) => { 182 | log('Resolving address %s to %s', address, resolvedAddress) 183 | 184 | res.writeHead(200) 185 | res.end(`dat://${resolvedAddress}\nttl=3600`) 186 | }).catch((e) => { 187 | const end = Date.now() 188 | logError(e, end) 189 | res.writeHead(500) 190 | res.end(ERR_500) 191 | }) 192 | } 193 | 194 | // return the archive 195 | return this.addIfNew(address).then((dat) => { 196 | // handle it!! 197 | const end = Date.now() 198 | log('[%s] %s %s | OK [%i ms]', address, req.method, path, end - start) 199 | req.url = `/${path}` 200 | dat.onrequest(req, res) 201 | }).catch((e) => { 202 | const end = Date.now() 203 | logError(e, end) 204 | if (e.message.indexOf('not found') > -1) { 205 | res.writeHead(404) 206 | res.end(ERR_404) 207 | } else { 208 | res.writeHead(500) 209 | res.end(ERR_500) 210 | } 211 | }) 212 | } 213 | }) 214 | } 215 | 216 | addIfNew (address) { 217 | return DatLibrarian.resolve(address).then((key) => { 218 | if (this.keys.indexOf(key) === -1) { 219 | return this.add(address) 220 | } else { 221 | this.lru[key] = Date.now() 222 | return this.get(key) 223 | } 224 | }) 225 | } 226 | 227 | clearOldest () { 228 | const sortOldestFirst = Object.keys(this.lru).sort((a, b) => { 229 | return this.lru[a] - this.lru[b] 230 | }) 231 | const oldest = sortOldestFirst[0] 232 | return this.remove(oldest) 233 | } 234 | 235 | add () { 236 | if (this.keys.length >= this.max) { 237 | // Delete the oldest item when we reach capacity and try again 238 | return this.clearOldest().then(() => this.add.apply(this, arguments)) 239 | } 240 | return super.add.apply(this, arguments).then((dat) => { 241 | log('Adding HTTP handler to archive...') 242 | if (!dat.onrequest) dat.onrequest = hyperdriveHttp(dat.archive, { live: true, exposeHeaders: true }) 243 | return new Promise((resolve) => { 244 | /* 245 | Wait for the archive to populate OR for 3s to pass, 246 | so that addresses for archives which don't exist 247 | don't hold us up all night. 248 | */ 249 | let isDone = false 250 | const done = () => { 251 | if (isDone) return null 252 | isDone = true 253 | const key = dat.archive.key.toString('hex') 254 | this.lru[key] = Date.now() 255 | return resolve(dat) 256 | } 257 | dat.archive.metadata.update(1, done) 258 | setTimeout(done, 3000) 259 | }) 260 | }) 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dat-gateway", 3 | "description": "A Dat-to-HTTP gateway and helper library.", 4 | "version": "2.0.1-beta", 5 | "main": "index.js", 6 | "bin": "bin.js", 7 | "scripts": { 8 | "start": "bin.js", 9 | "test": "standard && dependency-check . --unused --no-dev && mocha" 10 | }, 11 | "dependencies": { 12 | "dat-librarian": "^1.1.2-alpha", 13 | "hex-to-32": "^2.0.0", 14 | "hyperdrive-http": "^4.2.2", 15 | "websocket-stream": "^5.1.2", 16 | "yargs": "^11.0.0" 17 | }, 18 | "devDependencies": { 19 | "axios": "^0.18.0", 20 | "delay": "^2.0.0", 21 | "dependency-check": "^2.9.1", 22 | "eslint": "^4.19.1", 23 | "eslint-config-standard": "^11.0.0", 24 | "eslint-plugin-import": "^2.12.0", 25 | "eslint-plugin-node": "^6.0.1", 26 | "eslint-plugin-promise": "^3.7.0", 27 | "eslint-plugin-standard": "^3.1.0", 28 | "mocha": "^5.0.0", 29 | "random-access-memory": "^2.4.0", 30 | "rimraf": "^2.6.2", 31 | "standard": "^10.0.3" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "git+https://github.com/garbados/dat-gateway.git" 36 | }, 37 | "keywords": [ 38 | "dat", 39 | "http", 40 | "https" 41 | ], 42 | "authors": [ 43 | "Paul Frazee ", 44 | "Diana Thayer " 45 | ], 46 | "license": "Apache-2.0" 47 | } 48 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | /* global describe it before after */ 2 | 3 | const assert = require('assert') 4 | const http = require('http') 5 | const DatGateway = require('.') 6 | const rimraf = require('rimraf') 7 | const hyperdrive = require('hyperdrive') 8 | const ram = require('random-access-memory') 9 | const websocket = require('websocket-stream') 10 | 11 | const dir = 'fixtures' 12 | const ttl = 4000 13 | const period = 1000 14 | 15 | describe('dat-gateway', function () { 16 | this.timeout(0) 17 | 18 | before(function () { 19 | this.gateway = new DatGateway({dir, ttl, period}) 20 | return this.gateway.load().then(() => { 21 | return this.gateway.listen(5917) 22 | }) 23 | }) 24 | 25 | after(function () { 26 | return this.gateway.close().then(() => { 27 | rimraf.sync(dir) 28 | }) 29 | }) 30 | 31 | it('should exist', function () { 32 | assert.equal(this.gateway.dir, dir) 33 | }) 34 | 35 | it('should handle requests', function () { 36 | return new Promise((resolve) => { 37 | const req = http.get('http://localhost:5917/garbados.hashbase.io/icons/favicon.ico', resolve) 38 | req.on('error', console.log) 39 | }).then((res) => { 40 | // should display empty index, s.t. an attacker cannot determine 41 | assert.equal(res.statusCode, 200) 42 | }).catch((e) => { 43 | console.error(e) 44 | throw e 45 | }) 46 | }) 47 | 48 | it('should handle requests for dead addresses', function () { 49 | return new Promise((resolve) => { 50 | http.get('http://localhost:5917/af75142d92dd1e456cf2a7e58a37f891fe42a1e49ce2a5a7859de938e38f4642', resolve) 51 | }).then((res) => { 52 | // show blank index 53 | assert.equal(res.statusCode, 200) 54 | }).catch((e) => { 55 | console.error(e) 56 | throw e 57 | }) 58 | }) 59 | 60 | it('should proactively deleted expired archives', function () { 61 | return new Promise((resolve) => { 62 | const checker = setInterval(() => { 63 | // assert that they have been deleted 64 | if (this.gateway.keys.length === 0) { 65 | clearInterval(checker) 66 | return resolve() 67 | } 68 | }, ttl) 69 | }) 70 | }) 71 | 72 | it('should handle websockets for replication', function () { 73 | // Key for gardos.hashbase.io 74 | const key = 'c33bc8d7c32a6e905905efdbf21efea9ff23b00d1c3ee9aea80092eaba6c4957' 75 | 76 | const url = `ws://localhost:5917/${key}` 77 | 78 | let socket = null 79 | 80 | return new Promise((resolve, reject) => { 81 | const archive = hyperdrive(ram, Buffer.from(key, 'hex')) 82 | archive.once('error', reject) 83 | archive.once('ready', () => { 84 | socket = websocket(url) 85 | 86 | socket.pipe(archive.replicate({ 87 | live: true 88 | })).pipe(socket) 89 | 90 | setTimeout(() => { 91 | archive.readFile('/icons/favicon.ico', (e, content) => { 92 | if (e) reject(e) 93 | else resolve(content) 94 | }) 95 | }, 3000) 96 | }) 97 | }).then((content) => { 98 | socket.end() 99 | }, (e) => { 100 | socket.end() 101 | console.error(e.message) 102 | throw e 103 | }) 104 | }) 105 | }) 106 | --------------------------------------------------------------------------------