├── .gitignore ├── cli.js ├── index.js ├── package.json └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var minimist = require('minimist') 4 | var http = require('http') 5 | var Archiver = require('hypercore-archiver') 6 | var archiverServer = require('.') 7 | 8 | var argv = minimist(process.argv.slice(2), { 9 | alias: { 10 | httpPort: 'p' 11 | }, 12 | default: { 13 | httpPort: process.env.PORT || 8080, 14 | datPort: 3282, 15 | archiveDir: 'dats', 16 | swarm: true, 17 | http: true 18 | }, 19 | boolean: ['swarm', 'http'] 20 | }) 21 | 22 | var archiver = Archiver(argv.archiveDir) 23 | var datServer = archiverServer(archiver, argv) 24 | 25 | if (argv.http) { 26 | var server = http.createServer() 27 | server.on('request', datServer.httpRequest) 28 | server.listen(argv.httpPort, function () { 29 | console.log('Server is listening on port ' + argv.httpPort) 30 | }) 31 | } 32 | 33 | if (argv.swarm) { 34 | datServer.swarm.on('listening', function () { 35 | console.log('Listening for connections on the Dat Network') 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Swarm = require('discovery-swarm') 2 | var swarmDefaults = require('datland-swarm-defaults') 3 | var hyperdriveHttp = require('hyperdrive-http') 4 | var lru = require('lru') 5 | var hyperdrive = require('hyperdrive') 6 | var debug = require('debug')('archiver-server') 7 | 8 | module.exports = function (archiver, opts) { 9 | opts = opts || {} 10 | opts.swarm = opts.swarm || true 11 | opts.http = opts.http || true 12 | 13 | // Dat Swarm options 14 | opts.datPort = opts.datPort || false 15 | opts.tcp = opts.tcp || true 16 | opts.utp = opts.utp || true 17 | 18 | return { 19 | swarm: opts.swarm ? createSwarm(archiver, opts) : null, 20 | httpRequest: opts.http ? hyperdriveHttp(getArchive(archiver, opts)) : null 21 | } 22 | } 23 | 24 | function getArchive (archiver, opts) { 25 | var drive = hyperdrive(archiver.db) 26 | var cache = lru(opts.cacheSize || 100) 27 | cache.on('evict', function (item) { 28 | // TODO ? 29 | }) 30 | 31 | return function (dat, cb) { 32 | if (!dat.key) return cb('please provide key') // TODO: fix bug? 33 | debug('Archive HTTP request', JSON.stringify(dat)) 34 | 35 | var archive = cache.get(archiver.discoveryKey(new Buffer(dat.key, 'hex')).toString('hex')) 36 | if (archive) return cb(null, archive) 37 | debug('Getting archive:', dat.key) 38 | 39 | archiver.get(dat.key, function (err, feed, contentFeed) { 40 | debug('got archive', err) 41 | if (err || !feed) return cb('not found') 42 | if (!contentFeed) return cb('TODO: hypercore feed, not archive') 43 | debug('got archive') 44 | 45 | archive = drive.createArchive(dat.key, { 46 | metadata: feed, 47 | content: contentFeed 48 | }) 49 | 50 | cache.set(archive.discoveryKey.toString('hex'), archive) 51 | cb(null, archive) 52 | }) 53 | } 54 | } 55 | 56 | function createSwarm (archiver, opts) { 57 | if (!archiver) throw new Error('hypercore archiver required') 58 | if (!opts) opts = {} 59 | 60 | var timeouts = [] 61 | var swarmOpts = swarmDefaults({ 62 | utp: opts.utp, 63 | tcp: opts.tcp, 64 | dht: opts.dht, 65 | dns: opts.dns, 66 | hash: false, 67 | stream: function () { 68 | return archiver.replicate() // TODO: can you do {upload, download} here? 69 | } 70 | }) 71 | var swarm = Swarm(swarmOpts) 72 | swarm.once('error', function () { 73 | if (!opts.dontShare) swarm.listen(0) 74 | }) 75 | swarm.once('close', function () { 76 | timeouts.forEach(function (timeout) { 77 | clearTimeout(timeout) 78 | }) 79 | }) 80 | if (!opts.dontShare) { 81 | if (opts.datPort) swarm.listen(opts.datPort) 82 | else swarm.listen() 83 | } 84 | 85 | archiver.changes(function (err, feed) { 86 | if (err) throw err 87 | swarm.join(feed.discoveryKey) 88 | debug('Changes feed available at: ' + feed.key.toString('hex')) 89 | }) 90 | 91 | archiver.list().on('data', function (key) { 92 | if (!opts.dht) serveArchive(key) 93 | else { 94 | // random timeout so it doesn't flood DHT 95 | timeouts.push(setTimeout(function () { 96 | serveArchive(key) 97 | }, Math.floor(Math.random() * 30 * 1000))) 98 | } 99 | }) 100 | archiver.on('add', serveArchive) 101 | archiver.on('remove', function (key) { 102 | swarm.leave(archiver.discoveryKey(key)) 103 | }) 104 | 105 | return swarm 106 | 107 | function serveArchive (key) { 108 | debug(`Serving Archive ${key.toString('hex')} on Dat`) 109 | swarm.join(archiver.discoveryKey(key), {dontShare: opts.dontShare}) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "archiver-server", 3 | "version": "2.2.0", 4 | "description": "Serve keys found in a `hypercore-archiver` with `discovery-swarm`.", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "DEBUG=archiver-server node cli.js", 8 | "test": "standard" 9 | }, 10 | "author": "Joe Hand (https://joeahand.com/)", 11 | "license": "MIT", 12 | "dependencies": { 13 | "datland-swarm-defaults": "^1.0.2", 14 | "debug": "^2.3.3", 15 | "discovery-swarm": "^4.0.2", 16 | "hypercore-archiver": "^3.0.0", 17 | "hyperdrive": "^7.11.0", 18 | "hyperdrive-http": "^3.4.2", 19 | "lru": "^3.1.0", 20 | "random-access-file": "^1.3.1" 21 | }, 22 | "devDependencies": { 23 | "debug": "^2.3.3", 24 | "standard": "^8.6.0" 25 | }, 26 | "repository": { 27 | "type": "git", 28 | "url": "git+https://github.com/joehand/archiver-server.git" 29 | }, 30 | "bugs": { 31 | "url": "https://github.com/joehand/archiver-server/issues" 32 | }, 33 | "homepage": "https://github.com/joehand/archiver-server#readme" 34 | } 35 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Archiver-Server 2 | 3 | Serve Dat Archives stored in a [hypercore-archiver](https://github.com/mafintosh/hypercore-archiver). 4 | 5 | Archives can be served over HTTP and the Dat Network (via `discovery-swarm`). 6 | 7 | ## Usage 8 | 9 | ### Serve on Dat Network 10 | 11 | Serve archives in a hypercore-archiver with discovery-swarm. 12 | 13 | ```js 14 | var Archiver = require('hypercore-archiver') 15 | var archiverServer = require('archiver-server') 16 | 17 | var archives = Archiver('archives', {swarm: true}) 18 | var datServer = archiverServer(archives) 19 | 20 | datServer.swarm.on('listening', function () { 21 | console.log('Listening for connections on the Dat Network') 22 | }) 23 | 24 | // (Later) Any archives added will be available over discovery-swarm network 25 | archives.add(key) 26 | ``` 27 | 28 | ### Serve Over HTTP 29 | 30 | ```js 31 | var http = require('http') 32 | var Archiver = require('hypercore-archiver') 33 | var archiverServer = require('archiver-server') 34 | 35 | var archiver = Archiver('archives') 36 | var datServer = archiverServer(archiver, {http: true}) 37 | 38 | // Bring your own HTTP server and handle requests 39 | var server = http.createServer() 40 | server.on('request', datServer.httpRequest) 41 | server.listen(argv.httpPort, function () { 42 | console.log('Server is listening on port ' + port) 43 | }) 44 | 45 | // (Later) Any archives added will be available over HTTP 46 | archives.add(key) 47 | ``` 48 | 49 | ### CLI 50 | 51 | Archiver-server provides a basic CLI utility. There is currently no interface to add/remove archives to the hypercore-archiver, so it may be difficult to add use the CLI except for testing on preexisting archiver directories. 52 | 53 | Run `npm start` to run the CLI in debug mode. 54 | 55 | Options: 56 | 57 | * `--httpPort 8080`: Port for HTTP server 58 | * `--datPort 3282`: Port for Dat Network 59 | * `--archiveDir dats`: Directory for `hypercore-archiver` storage 60 | * `--swarm` (boolean): Serve archives on the Dat Network 61 | * `--http` (boolean): Serve archives over HTTP 62 | 63 | ## API 64 | 65 | ### var server = archiverServer(archiver, [opts]) 66 | 67 | Create a server for a `hypercore-archiver`. Use `http` and `swarm` to specify which server types to use. 68 | 69 | Options include: 70 | 71 | ```js 72 | opts = { 73 | http: true, // Return onrequest function to serve over HTTP 74 | swarm: true, // Serve over Dat Network 75 | utp: true, // Passed to Discovery-Swarm 76 | tcp: true, // Passed to Discovery-Swarm 77 | datPort: 3282 // Passed to Discovery-Swarm 78 | } 79 | ``` 80 | 81 | #### `server.swarm` 82 | 83 | `discovery-swarm` for your archives. Automatically connects. 84 | 85 | #### `server.httpRequest` 86 | 87 | Bring your own HTTP server. Use `server.httpRequest` for your http server's request function. 88 | 89 | HTTP requires hypercore-archiver `^2.3.0`. 90 | 91 | ## License 92 | 93 | MIT 94 | --------------------------------------------------------------------------------