├── .dockerignore ├── .eslintrc.json ├── .gitignore ├── .prettierrc ├── Dockerfile ├── README.md ├── docker-compose.yml ├── index.js ├── lib ├── config.js ├── datWrapper.js ├── errorPlugin.js ├── initServer.js ├── routes.js └── swarmManager.js ├── package.json ├── run.js ├── test ├── fixtures │ ├── dummy.txt │ └── swarmMan │ │ ├── clone │ │ └── .gitkeep │ │ ├── d1 │ │ └── a.txt │ │ └── d2 │ │ └── b.txt ├── helpers │ ├── index.js │ └── testContext.js ├── test-datShare.js └── test-swarmManager.js └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Example user template template 3 | ### Example user template 4 | 5 | # IntelliJ project files 6 | .idea 7 | *.iml 8 | out 9 | gen### JetBrains template 10 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 11 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 12 | 13 | # User-specific stuff 14 | .idea/**/workspace.xml 15 | .idea/**/tasks.xml 16 | .idea/**/dictionaries 17 | .idea/**/shelf 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # CMake 33 | cmake-build-debug/ 34 | cmake-build-release/ 35 | 36 | # Mongo Explorer plugin 37 | .idea/**/mongoSettings.xml 38 | 39 | # File-based project format 40 | *.iws 41 | 42 | # IntelliJ 43 | out/ 44 | 45 | # mpeltonen/sbt-idea plugin 46 | .idea_modules/ 47 | 48 | # JIRA plugin 49 | atlassian-ide-plugin.xml 50 | 51 | # Cursive Clojure plugin 52 | .idea/replstate.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | fabric.properties 59 | 60 | # Editor-based Rest Client 61 | .idea/httpRequests 62 | ### Node template 63 | # Logs 64 | logs 65 | *.log 66 | npm-debug.log* 67 | yarn-debug.log* 68 | yarn-error.log* 69 | 70 | # Runtime data 71 | pids 72 | *.pid 73 | *.seed 74 | *.pid.lock 75 | 76 | # Directory for instrumented libs generated by jscoverage/JSCover 77 | lib-cov 78 | 79 | # Coverage directory used by tools like istanbul 80 | coverage 81 | 82 | # nyc test coverage 83 | .nyc_output 84 | 85 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 86 | .grunt 87 | 88 | # Bower dependency directory (https://bower.io/) 89 | bower_components 90 | 91 | # node-waf configuration 92 | .lock-wscript 93 | 94 | # Compiled binary addons (https://nodejs.org/api/addons.html) 95 | build/Release 96 | 97 | # Dependency directories 98 | node_modules/ 99 | jspm_packages/ 100 | 101 | # TypeScript v1 declaration files 102 | typings/ 103 | 104 | # Optional npm cache directory 105 | .npm 106 | 107 | # Optional eslint cache 108 | .eslintcache 109 | 110 | # Optional REPL history 111 | .node_repl_history 112 | 113 | # Output of 'npm pack' 114 | *.tgz 115 | 116 | # Yarn Integrity file 117 | .yarn-integrity 118 | 119 | # dotenv environment variables file 120 | .env 121 | 122 | # next.js build output 123 | .next 124 | 125 | data/ 126 | .eslintrc.json 127 | .gitignore 128 | .prettierrc 129 | datTracker.db 130 | docker-compose.yml 131 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "extends": ["plugin:prettier/recommended"], 4 | "parserOptions": { 5 | "ecmaVersion": 9 6 | }, 7 | "env": { 8 | "es6": true, 9 | "node": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | .idea/**/dbnavigator.xml 20 | 21 | # Gradle 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # CMake 26 | cmake-build-debug/ 27 | cmake-build-release/ 28 | 29 | # Mongo Explorer plugin 30 | .idea/**/mongoSettings.xml 31 | 32 | # File-based project format 33 | *.iws 34 | 35 | # IntelliJ 36 | out/ 37 | 38 | # mpeltonen/sbt-idea plugin 39 | .idea_modules/ 40 | 41 | # JIRA plugin 42 | atlassian-ide-plugin.xml 43 | 44 | # Cursive Clojure plugin 45 | .idea/replstate.xml 46 | 47 | # Crashlytics plugin (for Android Studio and IntelliJ) 48 | com_crashlytics_export_strings.xml 49 | crashlytics.properties 50 | crashlytics-build.properties 51 | fabric.properties 52 | 53 | # Editor-based Rest Client 54 | .idea/httpRequests 55 | ### Example user template template 56 | ### Example user template 57 | 58 | # IntelliJ project files 59 | .idea 60 | *.iml 61 | out 62 | gen### Node template 63 | # Logs 64 | logs 65 | *.log 66 | npm-debug.log* 67 | yarn-debug.log* 68 | yarn-error.log* 69 | 70 | # Runtime data 71 | pids 72 | *.pid 73 | *.seed 74 | *.pid.lock 75 | 76 | # Directory for instrumented libs generated by jscoverage/JSCover 77 | lib-cov 78 | 79 | # Coverage directory used by tools like istanbul 80 | coverage 81 | 82 | # nyc test coverage 83 | .nyc_output 84 | 85 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 86 | .grunt 87 | 88 | # Bower dependency directory (https://bower.io/) 89 | bower_components 90 | 91 | # node-waf configuration 92 | .lock-wscript 93 | 94 | # Compiled binary addons (https://nodejs.org/api/addons.html) 95 | build/Release 96 | 97 | # Dependency directories 98 | node_modules/ 99 | jspm_packages/ 100 | 101 | # TypeScript v1 declaration files 102 | typings/ 103 | 104 | # Optional npm cache directory 105 | .npm 106 | 107 | # Optional eslint cache 108 | .eslintcache 109 | 110 | # Optional REPL history 111 | .node_repl_history 112 | 113 | # Output of 'npm pack' 114 | *.tgz 115 | 116 | # Yarn Integrity file 117 | .yarn-integrity 118 | 119 | # dotenv environment variables file 120 | .env 121 | 122 | # next.js build output 123 | .next 124 | 125 | redis/dump.rdb 126 | data 127 | dbDir 128 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "semi": true, 4 | "tabWidth": 2, 5 | "singleQuote": true 6 | } 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:11.13.0 2 | 3 | VOLUME /data 4 | 5 | ENV ROOT_DIR /data 6 | ENV SWARM_PORT 3282 7 | ENV NODE_ENV production 8 | ENV DEBUG "SwarmManager" 9 | 10 | EXPOSE 3282 11 | EXPOSE 3000 12 | 13 | RUN mkdir -p /usr/src/app 14 | WORKDIR /usr/src/app 15 | 16 | COPY package.json /usr/src/app 17 | RUN yarn install 18 | 19 | COPY . /usr/src/app 20 | 21 | #RUN useradd -ms /bin/bash -u 1000 datuser 22 | 23 | USER node 24 | 25 | CMD node run.js -r $ROOT_DIR -s $SWARM_PORT 26 | 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | dat-share 2 | ======================= 3 | 4 | Webrecorders dat integration backend. 5 | 6 | ### Installation 7 | To use this project you must first install its dependencies 8 | 9 | ```bash 10 | $ yarn install 11 | # or "npm install" 12 | ``` 13 | 14 | ### Usage 15 | dat-share provides a cli to help you use this project. 16 | 17 | The commands available to you are displayed below 18 | 19 | ```bash 20 | $ ./run.js --help 21 | Usage: run [options] 22 | 23 | Options: 24 | -V, --version output the version number 25 | -p, --port [port] The port the api server is to bind to (default: 3000) 26 | -h, --host [host] The host address the server is listen on (default: "127.0.0.1") 27 | -s, --swarm-port [port] The port the swarm is to bind to (default: 3282) 28 | -r, --rootDir The root directory that contains the contents to be shared via dat 29 | -l --log should logging be enabled for both the api server and swarm manager 30 | --help output usage information 31 | ``` 32 | 33 | Some configuration of the server can be done via the environment variables listed below 34 | - `SWARM_API_HOST`: the host the api server will use (e.g. 127.0.0.1) 35 | - `SWARM_API_PORT`: the port the api server will listen on (e.g. 3000) 36 | - `SWARM_PORT`: the port the swarm will listen on (e.g. 3282) 37 | - `SWARM_ROOT`: the root directory that contains the contents to be shared via dat 38 | - `LOG`: should logging be enabled (exists **yes**, does not exists **no**) 39 | - `Debug=SwarmManager`: enables logging of the actions performed by the swarm manager only 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | dat-share: 5 | build: . 6 | image: webrecorder/dat-share 7 | ports: 8 | - "3282:3282" 9 | - "3000:3000" 10 | volumes: 11 | - ./data/storage:/data 12 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | exports.DatWrapper = require('./lib/datWrapper'); 2 | exports.SwarmManager = require('./lib/swarmManager'); 3 | exports.initServer = require('./lib/initServer'); 4 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const program = require('commander'); 3 | const chalk = require('chalk').default; 4 | const fs = require('fs-extra'); 5 | const pkg = require('../package'); 6 | 7 | /** 8 | * Prints a warning for invalid environment config values 9 | * @param {string} key 10 | * @param {string} value 11 | * @param {*} defaultValue 12 | */ 13 | function invalidValue(key, value, defaultValue) { 14 | console.log(chalk.bold.red(`Invalid value for ${key}: ${value}`)); 15 | console.log(chalk.bold.red(`Using default value: ${defaultValue}`)); 16 | } 17 | 18 | function convertEnvInt(key, defaultValue) { 19 | const envValue = process.env[key]; 20 | let value = defaultValue; 21 | if (envValue != null) { 22 | try { 23 | value = parseInt(envValue, 10); 24 | } catch (e) { 25 | invalidValue(key, envValue, defaultValue); 26 | } 27 | if (isNaN(value)) { 28 | invalidValue(key, envValue, defaultValue); 29 | value = defaultValue; 30 | } 31 | } 32 | return value; 33 | } 34 | 35 | function validateArgs(prog) { 36 | const rootDir = prog.rootDir; 37 | let isError = false; 38 | if (!rootDir) { 39 | console.log( 40 | chalk.bold.red('The rootDir argument was not supplied and is required') 41 | ); 42 | isError = true; 43 | } else if (!fs.pathExistsSync(rootDir)) { 44 | console.log( 45 | chalk.bold.red( 46 | `The directory specified by the rootDir argument (${rootDir}) does not exist` 47 | ) 48 | ); 49 | isError = true; 50 | } else if (!fs.statSync(rootDir).isDirectory()) { 51 | console.log( 52 | chalk.bold.red( 53 | `The value for the rootDir argument (${rootDir}) is not a directory` 54 | ) 55 | ); 56 | isError = true; 57 | } 58 | 59 | if (!isError && isNaN(prog.swarmPort)) { 60 | isError = true; 61 | console.log( 62 | chalk.bold.red('The value for the swarmPort argument is not a number') 63 | ); 64 | } 65 | 66 | if (!isError && isNaN(prog.port)) { 67 | isError = true; 68 | console.log( 69 | chalk.bold.red('The value for the port argument is not a number') 70 | ); 71 | } 72 | 73 | if (isError) { 74 | program.help(chalk.bold.red); 75 | } 76 | } 77 | 78 | /** 79 | * Returns the default port the api server will listen on. 80 | * If the env variable SWARM_MAN_API_HOST is set returns it's value 81 | * otherwise returns 127.0.0.1 82 | * @return {string} 83 | */ 84 | function getDefaultHost() { 85 | if (process.env.SWARM_API_HOST != null) { 86 | return process.env.SWARM_API_HOST; 87 | } 88 | return process.env.NODE_ENV === 'production' ? '0.0.0.0' : '127.0.0.1'; 89 | } 90 | 91 | program 92 | .version(pkg.version) 93 | .option( 94 | '-p, --port [port]', 95 | 'The port the api server is to bind to', 96 | convertEnvInt('SWARM_API_PORT', 3000) 97 | ) 98 | .option( 99 | '-h, --host [host]', 100 | 'The host address the server is listen on', 101 | getDefaultHost() 102 | ) 103 | .option( 104 | '-s, --swarm-port [port]', 105 | 'The port the swarm is to bind to', 106 | convertEnvInt('SWARM_PORT', 3282) 107 | ) 108 | .option( 109 | '-r, --rootDir ', 110 | 'The root directory that contains the contents to be shared via dat', 111 | process.env.SWARM_ROOT 112 | ) 113 | .option( 114 | '-l --log', 115 | 'should logging be enabled for both the api server and swarm manager' 116 | ) 117 | .parse(process.argv); 118 | 119 | if (process.env.NODE_ENV !== 'test') validateArgs(program); 120 | 121 | if (program.log) { 122 | process.env.DEBUG = 'SwarmManager'; 123 | } 124 | 125 | module.exports = { 126 | host: program.host, 127 | port: program.port, 128 | log: program.log, 129 | swarmManager: { 130 | port: program.swarmPort, 131 | rootDir: program.rootDir, 132 | }, 133 | fastifyOpts: { 134 | trustProxy: true, 135 | logger: program.log, 136 | }, 137 | }; 138 | -------------------------------------------------------------------------------- /lib/datWrapper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const path = require('path'); 3 | const fs = require('fs-extra'); 4 | const DatStorage = require('dat-storage'); 5 | const hyperdrive = require('hyperdrive'); 6 | const importFiles = require('dat-node/lib/import-files'); 7 | const S3HybridStorage = require('dat-s3-hybrid-storage'); 8 | 9 | /** 10 | * @desc Wrapper around both dat-node and dat-json that provides ease of use 11 | * for both 12 | */ 13 | class DatWrapper { 14 | /** 15 | * 16 | * @param {Hyperdrive} drive 17 | * @param {function(): Promise} importer 18 | */ 19 | constructor(drive, importer) { 20 | /** 21 | * @desc The wrapped hyperdrive 22 | * @type {Hyperdrive} 23 | */ 24 | this.drive = drive; 25 | 26 | /** 27 | * @desc Source url or File dir for importing files, or null for read-only 28 | * @type {function(): Promise} 29 | */ 30 | this.importFiles = importer; 31 | 32 | /** 33 | * @desc are we sharing this dat (joined the swarm) 34 | * @type {boolean} 35 | * @private 36 | */ 37 | this._sharing = false; 38 | } 39 | 40 | /** 41 | * @desc Create a Dat instance, archive storage, and ready the archive. 42 | * @param {string} dir - Path to the directory of the dat 43 | * @param {string} fullDir - Full path to directory of the dat 44 | * @param {Object} [options = {}] - Dat-node options and any hyperdrive init options. 45 | * @param {string|Buffer} [options.key] - Hyperdrive key 46 | * @param {Boolean} [options.latest = true] - Create storage if it does not exit. 47 | * @param {string} [options.dir] 48 | * @return {Promise} The new Dat instance 49 | */ 50 | static async create(dir, fullDir, options = {}) { 51 | let storage = null; 52 | let drive = null; 53 | let importer = null; 54 | 55 | const opts = { latest: true, dir: fullDir, ...options }; 56 | 57 | // S3 SUPPORT! 58 | if (process.env.S3_ROOT) { 59 | await fs.ensureDir(fullDir); 60 | 61 | const srcUrl = process.env.S3_ROOT + dir; 62 | const s3_storage = new S3HybridStorage(srcUrl, fullDir); 63 | 64 | storage = s3_storage.storage(); 65 | 66 | drive = hyperdrive(storage, opts); 67 | 68 | importer = function() { 69 | return s3_storage.importer().importFiles(drive); 70 | }; 71 | } else { 72 | storage = DatStorage(fullDir); 73 | 74 | drive = hyperdrive(storage, opts); 75 | 76 | importer = function() { 77 | return new Promise(resolve => { 78 | importFiles(drive, fullDir, { indexing: true }, () => { 79 | resolve(); 80 | }); 81 | }); 82 | }; 83 | } 84 | 85 | const hasDat = fs.existsSync(path.join(fullDir, '.dat')); 86 | 87 | await new Promise((resolve, reject) => { 88 | drive.on('error', reject); 89 | drive.ready(() => { 90 | drive.removeListener('error', reject); 91 | drive.resumed = !!(hasDat || (drive.metadata.has(0) && drive.version)); 92 | resolve(); 93 | }); 94 | }); 95 | 96 | return new DatWrapper(drive, importer); 97 | } 98 | 99 | /** 100 | * @desc Are we sharing this dat (joined the swarm) 101 | * @return {boolean} - True if this dat has joined the swarm otherwise false 102 | */ 103 | sharing() { 104 | return this._sharing; 105 | } 106 | 107 | /** 108 | * @desc Stat a path in the archive 109 | * @param {string} archivePath - The path in the archive to stat 110 | * @return {Promise} 111 | */ 112 | stat(archivePath) { 113 | return new Promise((resolve, reject) => { 114 | this.drive.stat(archivePath, (error, stats) => { 115 | if (error) return reject(error); 116 | resolve(stats); 117 | }); 118 | }); 119 | } 120 | 121 | /** 122 | * @desc Retrieve this dats hyperdrive 123 | * @return {Object} - The dats hyperdrive 124 | */ 125 | archive() { 126 | return this.drive; 127 | } 128 | 129 | /** 130 | * @desc Close the underlying hyperdrive 131 | * @returns {Promise} 132 | */ 133 | close() { 134 | return new Promise((resolve, reject) => { 135 | this.closeCB(resolve); 136 | }); 137 | } 138 | 139 | /** 140 | * @desc Close the underlying hyperdrive 141 | * @param {function()} [cb] 142 | */ 143 | closeCB(cb) { 144 | this.drive.close(cb); 145 | } 146 | 147 | /** 148 | * @desc Retrieve the dat's archive discoveryKey 149 | * @param {string?} [as = 'hex'] - The format of the discoveryKey's string representation 150 | * @return {string} - The dats discoveryKey 151 | */ 152 | discoveryKey(as = 'hex') { 153 | return this.drive.discoveryKey.toString(as); 154 | } 155 | 156 | /** 157 | * @desc Retrieve the dat's archive key 158 | * @param {string?} [as = 'hex'] - The format of the key's string representation 159 | * @return {string} - The dat's archive key 160 | */ 161 | key(as = 'hex') { 162 | return this.drive.key.toString(as); 163 | } 164 | 165 | /** 166 | * @desc Join the swarm (share this dat) 167 | * @param {Swarm} swarm - The swarm instance to join 168 | * @param {function?} [cb] - A callback that is called once first discovery happens 169 | */ 170 | joinSwarmCB(swarm, cb) { 171 | this._sharing = true; 172 | swarm.join(this.drive.discoveryKey, { announce: true }, cb); 173 | } 174 | 175 | /** 176 | * @desc Join the swarm (share this dat) 177 | * @param {Swarm} swarm - The swarm instance to leave 178 | * @return {Promise} 179 | */ 180 | joinSwarm(swarm) { 181 | return new Promise(resolve => { 182 | this.joinSwarmCB(swarm, () => { 183 | resolve(); 184 | }); 185 | }); 186 | } 187 | 188 | /** 189 | * @desc Leave the swarm 190 | * @param {Swarm} swarm - The swarm instance to leave 191 | */ 192 | leaveSwarm(swarm) { 193 | if (!this._sharing) return; 194 | swarm.leave(this.drive.discoveryKey); 195 | this._sharing = false; 196 | } 197 | 198 | /** 199 | * @desc Replicate the dat via the supplied hypercoreProtocol stream 200 | * @param {Protocol} stream - The hypercoreProtocol stream to replicate to 201 | * @return {Protocol} - The hypercoreProtocol stream replicated to 202 | */ 203 | replicate(stream) { 204 | return this.drive.replicate({ 205 | stream: stream, 206 | live: true, 207 | upload: true, 208 | download: true, 209 | }); 210 | } 211 | } 212 | 213 | module.exports = DatWrapper; 214 | -------------------------------------------------------------------------------- /lib/errorPlugin.js: -------------------------------------------------------------------------------- 1 | const fp = require('fastify-plugin'); 2 | 3 | module.exports = 4 | process.env.NODE_ENV === 'production' 5 | ? require('fastify-boom') 6 | : fp( 7 | function fastifyErrorPage(fastify, options, next) { 8 | fastify.setErrorHandler(function errorHandler(error, request, reply) { 9 | console.error(error); 10 | if (error && error.isBoom) { 11 | reply 12 | .code(error.output.statusCode) 13 | .type('application/json') 14 | .headers(error.output.headers) 15 | .send(error.output.payload); 16 | 17 | return; 18 | } 19 | 20 | reply.send(error || new Error('Got non-error: ' + error)); 21 | }); 22 | 23 | next(); 24 | }, 25 | { 26 | fastify: '>=0.43', 27 | name: 'fastify-boom', 28 | } 29 | ); 30 | -------------------------------------------------------------------------------- /lib/initServer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const fastify = require('fastify'); 3 | const SwarmManager = require('./swarmManager'); 4 | 5 | /** 6 | * 7 | * @param {Object} config 8 | * @return {Promise} 9 | */ 10 | module.exports = async function initServer(config) { 11 | const swarmMan = new SwarmManager(config.swarmManager); 12 | const server = fastify(config.fastifyOpts); 13 | server 14 | .decorate('conf', config) 15 | .decorate('swarmManager', swarmMan) 16 | .use(require('cors')()) 17 | .register(require('fastify-graceful-shutdown'), { timeout: 3000 }) 18 | .register(require('fastify-swagger'), { 19 | routePrefix: '/swagger', 20 | exposeRoute: true, 21 | swagger: { 22 | info: { 23 | title: 'dat-share', 24 | description: "Webrecorder's dat integration", 25 | version: require('../package').version, 26 | }, 27 | host: 'localhost', 28 | schemes: ['http'], 29 | consumes: ['application/json'], 30 | produces: ['application/json'], 31 | tags: [{ name: 'dat', description: 'Dat Endpoints' }], 32 | }, 33 | }) 34 | .register(require('./errorPlugin')) 35 | .register(require('./routes')) 36 | .addHook('onClose', async (server, done) => { 37 | await swarmMan.close(done); 38 | }) 39 | .ready(() => { 40 | swarmMan.initSwarm(); 41 | }); 42 | const listeningOn = await server.listen(config.port, config.host); 43 | console.log( 44 | `Dat Share api server listening on\n${ 45 | listeningOn.startsWith('http://127.0.0.1') 46 | ? listeningOn.replace('http://127.0.0.1', 'http://localhost') 47 | : listeningOn 48 | }` 49 | ); 50 | console.log(server.printRoutes()); 51 | return server; 52 | }; 53 | -------------------------------------------------------------------------------- /lib/routes.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const Boom = require('boom'); 3 | 4 | /** 5 | * @desc Initialize the dat-share routes 6 | * @param {fastify.FastifyInstance} server 7 | * @param {Object} opts 8 | * @param {function} next 9 | */ 10 | module.exports = function datRoutesInit(server, opts, next) { 11 | /** @type {SwarmManager} */ 12 | const swarmManager = server.swarmManager; 13 | 14 | const validateCollDirRequest = async body => { 15 | if (!body) { 16 | throw Boom.badRequest('No request body'); 17 | } 18 | if (!body.collDir) { 19 | throw Boom.badRequest('Did not supply collDir'); 20 | } 21 | server.log.info(swarmManager.actualDirPath(body.collDir)); 22 | const exists = await swarmManager.canShareDir(body.collDir); 23 | if (!exists) { 24 | throw Boom.notFound( 25 | 'The collection requested to be shared does not exist', 26 | body.collDir 27 | ); 28 | } 29 | }; 30 | 31 | server.route({ 32 | method: 'POST', 33 | url: '/init', 34 | schema: { 35 | description: 36 | `Initialize a collection's dat and receive the collections discovery and dat key. 37 | If the collection was previously initialized and or currently being shared,` + 38 | `no action is performed other than to return it's discovery and dat key.`, 39 | summary: `Initialize a collection's dat`, 40 | tags: ['dat'], 41 | body: { 42 | type: 'object', 43 | properties: { 44 | collDir: { 45 | type: 'string', 46 | description: 'Path to the collection to be initialized', 47 | }, 48 | }, 49 | }, 50 | response: { 51 | 200: { 52 | type: 'object', 53 | properties: { 54 | discoveryKey: { 55 | type: 'string', 56 | description: 'The discovery key associated with the collection', 57 | }, 58 | datKey: { 59 | type: 'string', 60 | description: 'The dat key associated with the collection', 61 | }, 62 | }, 63 | }, 64 | 400: { 65 | type: 'object', 66 | properties: { 67 | statusCode: { type: 'number' }, 68 | error: { 69 | type: 'string', 70 | description: 'The type of Error thrown', 71 | }, 72 | message: { 73 | type: 'string', 74 | description: 'The thrown Errors message', 75 | }, 76 | }, 77 | }, 78 | 404: { 79 | type: 'object', 80 | properties: { 81 | statusCode: { type: 'number' }, 82 | error: { 83 | type: 'string', 84 | description: 'The type of Error thrown', 85 | }, 86 | message: { 87 | type: 'string', 88 | description: 'The thrown Errors message', 89 | }, 90 | }, 91 | }, 92 | }, 93 | }, 94 | async handler(request, reply) { 95 | const { collDir } = request.body; 96 | return swarmManager.initDat(collDir); 97 | }, 98 | }); 99 | 100 | server.route({ 101 | method: 'POST', 102 | url: '/share', 103 | schema: { 104 | description: `Import the collections files and start sharing a collection via the dat protocol. 105 | If the collection was not previously initialized it is initialized.`, 106 | summary: 'Start sharing a collection via the dat protocol.', 107 | tags: ['dat'], 108 | body: { 109 | type: 'object', 110 | properties: { 111 | collDir: { 112 | type: 'string', 113 | description: 'Path to the collection to be shared', 114 | }, 115 | }, 116 | }, 117 | response: { 118 | 200: { 119 | type: 'object', 120 | properties: { 121 | discoveryKey: { 122 | type: 'string', 123 | description: 'The discovery key associated with the collection', 124 | }, 125 | datKey: { 126 | type: 'string', 127 | description: 'The dat key associated with the collection', 128 | }, 129 | }, 130 | }, 131 | 400: { 132 | type: 'object', 133 | properties: { 134 | statusCode: { type: 'number' }, 135 | error: { 136 | type: 'string', 137 | description: 'The type of Error thrown', 138 | }, 139 | message: { 140 | type: 'string', 141 | description: 'The thrown Errors message', 142 | }, 143 | }, 144 | }, 145 | 404: { 146 | type: 'object', 147 | properties: { 148 | statusCode: { type: 'number' }, 149 | error: { 150 | type: 'string', 151 | description: 'The type of Error thrown', 152 | }, 153 | message: { 154 | type: 'string', 155 | description: 'The thrown Errors message', 156 | }, 157 | }, 158 | }, 159 | }, 160 | }, 161 | async handler(request, reply) { 162 | await validateCollDirRequest(request.body); 163 | const { collDir } = request.body; 164 | return swarmManager.shareDir(collDir); 165 | }, 166 | }); 167 | 168 | server.route({ 169 | method: 'POST', 170 | url: '/unshare', 171 | schema: { 172 | description: 'Stop sharing a collection via the dat protocol.', 173 | summary: 'Stop sharing a collection via the dat protocol.', 174 | tags: ['dat'], 175 | body: { 176 | type: 'object', 177 | collDir: { 178 | type: 'string', 179 | description: 'Path to the collection to be un-shared', 180 | }, 181 | }, 182 | response: { 183 | 200: { 184 | type: 'object', 185 | description: 'Default response', 186 | properties: { 187 | success: { 188 | type: 'boolean', 189 | description: 190 | 'Indicates if the un-share operation was successful or not', 191 | }, 192 | }, 193 | }, 194 | 400: { 195 | type: 'object', 196 | properties: { 197 | statusCode: { type: 'number' }, 198 | error: { 199 | type: 'string', 200 | description: 'The type of Error thrown', 201 | }, 202 | message: { 203 | type: 'string', 204 | description: 'The thrown Errors message', 205 | }, 206 | }, 207 | }, 208 | 404: { 209 | type: 'object', 210 | properties: { 211 | statusCode: { type: 'number' }, 212 | error: { 213 | type: 'string', 214 | description: 'The type of Error thrown', 215 | }, 216 | message: { 217 | type: 'string', 218 | description: 'The thrown Errors message', 219 | }, 220 | }, 221 | }, 222 | }, 223 | }, 224 | async handler(request, reply) { 225 | await validateCollDirRequest(request.body); 226 | const { collDir } = request.body; 227 | return { success: swarmManager.unshareDir(collDir) }; 228 | }, 229 | }); 230 | 231 | server.route({ 232 | method: 'POST', 233 | url: '/sync', 234 | schema: { 235 | description: 236 | 'Sync collections currently being shared by un-sharing all ' + 237 | 'collection not provided in the post-body and share any that were not being shared.', 238 | summary: 'Sync collections currently being shared', 239 | tags: ['dat'], 240 | body: { 241 | type: 'object', 242 | properties: { 243 | dirs: { 244 | type: 'array', 245 | description: `List of collection paths to be sync'd`, 246 | items: { 247 | type: 'string', 248 | }, 249 | }, 250 | }, 251 | }, 252 | response: { 253 | 200: { 254 | type: 'object', 255 | properties: { 256 | results: { 257 | type: 'array', 258 | description: 'The successful results of the sync', 259 | items: { 260 | type: 'object', 261 | properties: { 262 | discoveryKey: { 263 | type: 'string', 264 | description: 265 | 'The discovery key associated with the collection', 266 | }, 267 | datKey: { 268 | type: 'string', 269 | description: 'The dat key associated with the collection', 270 | }, 271 | dir: { 272 | type: 'string', 273 | description: `The directory sync'd`, 274 | }, 275 | }, 276 | }, 277 | }, 278 | errors: { 279 | type: 'array', 280 | description: 281 | 'The un-successful results of the sync (An error was thrown)', 282 | items: { 283 | type: 'object', 284 | properties: { 285 | error: { 286 | type: 'string', 287 | description: 'The message of the thrown Error', 288 | }, 289 | dir: { 290 | type: 'string', 291 | description: `The directory not sync'd`, 292 | }, 293 | }, 294 | }, 295 | }, 296 | }, 297 | }, 298 | 400: { 299 | type: 'object', 300 | properties: { 301 | statusCode: { type: 'number' }, 302 | error: { 303 | type: 'string', 304 | description: 'The type of Error thrown', 305 | }, 306 | message: { 307 | type: 'string', 308 | description: 'The thrown Errors message', 309 | }, 310 | }, 311 | }, 312 | }, 313 | }, 314 | async handler(request, reply) { 315 | if (!request.body) { 316 | throw Boom.badRequest('No request body'); 317 | } 318 | if (!request.body.dirs) { 319 | throw Boom.badRequest('Did not supply collDir'); 320 | } 321 | return swarmManager.sync(request.body); 322 | }, 323 | }); 324 | 325 | server.route({ 326 | method: 'GET', 327 | url: '/numSharing', 328 | schema: { 329 | description: 'Retrieve the number of collections currently being shared', 330 | summary: 'Retrieve the number of collections currently being shared', 331 | tags: ['dat'], 332 | response: { 333 | 200: { 334 | type: 'object', 335 | properties: { 336 | num: { 337 | type: 'number', 338 | description: 'The number of collections shared', 339 | }, 340 | }, 341 | }, 342 | }, 343 | }, 344 | async handler(request, reply) { 345 | return { num: swarmManager.numSharing() }; 346 | }, 347 | }); 348 | 349 | server.route({ 350 | method: 'GET', 351 | url: '/numDats', 352 | schema: { 353 | description: 'Retrieve the number of collections with dats', 354 | summary: 'Retrieve the number of collections with dats', 355 | tags: ['dat'], 356 | response: { 357 | 200: { 358 | type: 'object', 359 | properties: { 360 | num: { 361 | type: 'number', 362 | description: 'The number of collections with dats', 363 | }, 364 | }, 365 | }, 366 | }, 367 | }, 368 | async handler(request, reply) { 369 | return { num: swarmManager.numDats() }; 370 | }, 371 | }); 372 | 373 | next(); 374 | }; 375 | -------------------------------------------------------------------------------- /lib/swarmManager.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const fs = require('fs-extra'); 3 | const path = require('path'); 4 | const crypto = require('crypto'); 5 | const EventEmitter = require('eventemitter3'); 6 | const swarmDefaults = require('dat-swarm-defaults'); 7 | const discoverySwarm = require('discovery-swarm'); 8 | const hypercoreProtocol = require('hypercore-protocol'); 9 | const debug = require('debug')('SwarmManager'); 10 | const DatWrapper = require('./datWrapper'); 11 | 12 | function noop() {} 13 | 14 | function validateArgs(rootDir, port) { 15 | const undefRd = rootDir == null; 16 | const undefPort = port == null; 17 | if (undefRd || undefPort) { 18 | throw new Error( 19 | `new SwarmManager({rootDir, port}): ${ 20 | undefRd && undefPort 21 | ? 'both rootDir and port are' 22 | : undefRd 23 | ? 'rootDir is' 24 | : 'port is' 25 | } undefined` 26 | ); 27 | } 28 | if (typeof rootDir !== 'string') { 29 | throw new Error( 30 | `new SwarmManager(rootDir, port): rootDir should be a 'string' received '${typeof rootDir}'` 31 | ); 32 | } 33 | } 34 | 35 | /** 36 | * @desc Class for managing the discovery swarm and dats shared 37 | */ 38 | class SwarmManager extends EventEmitter { 39 | /** 40 | * @desc Construct a new instance of SwarmManager 41 | * @param {{rootDir: string, port: number}} opts - Config options 42 | */ 43 | constructor({ rootDir, port }) { 44 | super(); 45 | validateArgs(rootDir, port); 46 | 47 | /** 48 | * @desc The port the discovery swarm will listen on 49 | * @type {number} 50 | */ 51 | this.port = port; 52 | 53 | /** 54 | * @desc The root path of the directories to be shared 55 | * @type {string} 56 | */ 57 | this.rootDir = rootDir; 58 | 59 | /** 60 | * @desc The discovery swarm instance 61 | * @type {Swarm} 62 | * @private 63 | */ 64 | this._swarm = null; 65 | 66 | /** 67 | * @desc A map of active discovery keys to their dats 68 | * @type {Map} 69 | * @private 70 | */ 71 | this._dkToDat = new Map(); 72 | 73 | /** 74 | * @desc A map of active dat directories to their discovery key 75 | * @type {Map} 76 | * @private 77 | */ 78 | this._dirToDk = new Map(); 79 | 80 | /** 81 | * @desc indicates if the the swarm close event was caused by us closing it 82 | * @type {boolean} 83 | * @private 84 | */ 85 | this._closeServerShutDown = false; 86 | 87 | /** 88 | * @desc An id for ourselves 89 | * @type {Buffer} 90 | */ 91 | this.networkId = crypto.randomBytes(32); 92 | this._connIdCounter = 0; // for debugging 93 | 94 | this._replicate = this._replicate.bind(this); 95 | } 96 | 97 | /** 98 | * @desc Returns the number of managed dats that we are actively sharing 99 | * @return {number} 100 | */ 101 | numSharing() { 102 | let sharing = 0; 103 | for (const dw of this._dkToDat.values()) { 104 | if (dw.sharing()) { 105 | sharing += 1; 106 | } 107 | } 108 | return sharing; 109 | } 110 | 111 | /** 112 | * @desc Retrieve the number of managed dats 113 | * @return {number} 114 | */ 115 | numDats() { 116 | return this._dkToDat.size; 117 | } 118 | 119 | /** 120 | * @desc Retrieve the tracked dat associated wih the supplied discovery key 121 | * @param {string} discoveryKey - The discoveryKey associated with an active dat 122 | * @return {DatWrapper | undefined} - The dat associated with the discovery key if it exists 123 | */ 124 | getDat(discoveryKey) { 125 | return this._dkToDat.get(discoveryKey); 126 | } 127 | 128 | /** 129 | * @desc Retrieve the discoveryKey associated with the supplied directory 130 | * @param {string} dir - The directory to retrieve the discoveryKey for 131 | * @return {string | undefined} - The discovery key associated with the directory if it exists 132 | */ 133 | getDiscoveryKeyForDir(dir) { 134 | return this._dirToDk.get(dir); 135 | } 136 | 137 | /** 138 | * @desc Retrieve the tracked dat associated wih the supplied directory 139 | * @param {string} dir - The full path to a actively shared directory 140 | * @return {DatWrapper | undefined} - The actively shared dat if it exists 141 | */ 142 | getDatForDir(dir) { 143 | const dk = this.getDiscoveryKeyForDir(dir); 144 | return this.getDat(dk); 145 | } 146 | 147 | /** 148 | * @desc Determine if we are actively managing the directory 149 | * @param {string} dir - The directory to check 150 | * @return {boolean} - True if the directory is actively shared otherwise false 151 | */ 152 | isActiveDir(dir) { 153 | return this._dirToDk.has(dir); 154 | } 155 | 156 | /** 157 | * @desc Determine if the supplied discovery key is associated with an active dat 158 | * @param {string} discoveryKey - The discovery key to check 159 | * @return {boolean} - True if the discovery key is associated with an active dat otherwise false 160 | */ 161 | isActiveDiscoveryKey(discoveryKey) { 162 | return this._dkToDat.has(discoveryKey); 163 | } 164 | 165 | /** 166 | * @desc Checks for the existence (sharability) of the supplied dir path 167 | * @param {string} directory - The directory under {@link rootDir} to check if it exists (can be shared) 168 | * @return {Promise} - True if the directory can be shared otherwise false 169 | */ 170 | canShareDir(directory) { 171 | return fs.pathExists(this.actualDirPath(directory)); 172 | } 173 | 174 | /** 175 | * @desc Initialize the swarm and re-init dats from state 176 | */ 177 | initSwarm() { 178 | this._swarm = discoverySwarm( 179 | swarmDefaults({ 180 | hash: false, 181 | stream: this._replicate, 182 | }) 183 | ); 184 | 185 | this._swarm.on('listening', () => { 186 | debug('Swarm Listening...'); 187 | this.emit('listening'); 188 | }); 189 | 190 | this._swarm.on('close', () => { 191 | if (!this._closeServerShutDown) { 192 | debug('swarm closed but not server we have an issue'); 193 | } else { 194 | debug('swarm closed'); 195 | } 196 | this.emit('close', this._closeServerShutDown); 197 | }); 198 | 199 | this._swarm.on('error', err => { 200 | debug('Swarm error: %O', err); 201 | }); 202 | 203 | this._swarm.listen(this.port); 204 | } 205 | 206 | /** 207 | * @desc Unshare and remove all tracked dats not in the set of keys to be kept (sync'd) 208 | * @param {Array} toSync 209 | * @private 210 | */ 211 | _doSyncRemoval(toSync) { 212 | const keep = new Set(); 213 | for (let i = 0; i < toSync.length; ++i) { 214 | keep.add(toSync[i]); 215 | } 216 | for (const [haveDir, dk] of this._dirToDk.entries()) { 217 | if (!keep.has(haveDir)) { 218 | this.unshareDir(haveDir); 219 | this._dirToDk.delete(haveDir); 220 | this._dkToDat.delete(dk); 221 | } 222 | } 223 | } 224 | 225 | /** 226 | * @desc Bulk share directories 227 | * @param {Object} toSync 228 | * @property {Array} toSync.dirs 229 | * @return {Promise<{results: Array<{dir: string, discoveryKey: string, datKey: string}>, errors: Array<{dir: string, error: string}>}>} 230 | */ 231 | async sync({ dirs }) { 232 | debug(`Sync: ${dirs}`); 233 | this._doSyncRemoval(dirs); 234 | const shared = { results: [], errors: [] }; 235 | for (let i = 0; i < dirs.length; ++i) { 236 | const dir = dirs[i]; 237 | const canShare = await this.canShareDir(dir); 238 | if (!canShare) { 239 | shared.errors.push({ 240 | dir, 241 | error: `Directory ${dir} can not be shared`, 242 | }); 243 | continue; 244 | } 245 | 246 | try { 247 | let results = await this.shareDir(dir); 248 | results[dir] = dir; 249 | shared.results.push(results); 250 | } catch (e) { 251 | shared.errors.push({ dir, error: e.toString() }); 252 | } 253 | } 254 | return shared; 255 | } 256 | 257 | /** 258 | * @desc Initialize a dat in the supplied directory 259 | * @param {string} directory 260 | * @return {Promise<{discoveryKey: string, datKey: string}>} 261 | */ 262 | async initDat(directory) { 263 | debug(`Init DAT: ${directory}`); 264 | 265 | const { discoveryKey, datKey } = await this._loadDat(directory); 266 | return { discoveryKey, datKey }; 267 | } 268 | 269 | /** 270 | * @desc Share a directory using dat. 271 | * @param {string} directory - The directory to share via dat 272 | * @param {boolean} [skipImport = false] - Should importing files be skipped 273 | * @return {Promise<{discoveryKey: string, datKey: string}>} - Information about the dat 274 | */ 275 | async shareDir(directory, skipImport = false) { 276 | const { discoveryKey, datKey, dat } = await this._loadDat(directory); 277 | 278 | if (!skipImport) { 279 | await dat.importFiles(); 280 | } 281 | 282 | if (!dat.sharing()) { 283 | this._shareDat(dat, discoveryKey, datKey); 284 | } 285 | 286 | return { discoveryKey, datKey }; 287 | } 288 | 289 | /** 290 | * @desc Stop sharing a directory 291 | * @param {string} dir - The directory to unshare 292 | * @return {boolean} - True if the directory was unshared otherwise false 293 | */ 294 | unshareDir(dir) { 295 | debug(`UnShare Dir: ${dir}`); 296 | if (!this.isActiveDir(dir)) { 297 | return false; 298 | } 299 | const discoveryKey = this.getDiscoveryKeyForDir(dir); 300 | const existingDat = this.getDat(discoveryKey); 301 | if (!existingDat.sharing()) { 302 | return true; 303 | } 304 | existingDat.leaveSwarm(this._swarm); 305 | existingDat.closeCB(noop); 306 | this._unmarkDatAsActive(dir, discoveryKey); 307 | return true; 308 | } 309 | 310 | /** 311 | * @desc Close the swarm connection 312 | * @param {function} [cb] - Optional callback that is called once the swarm connection was closed 313 | */ 314 | async close(cb) { 315 | this._closeServerShutDown = true; 316 | await this._closeAndClear(); 317 | this._swarm.close(cb); 318 | } 319 | 320 | async _closeAndClear() { 321 | const toClose = []; 322 | for (const dat of this._dkToDat.values()) { 323 | toClose.push(dat.close()); 324 | } 325 | try { 326 | await Promise.all(toClose); 327 | } catch (e) {} 328 | this._dirToDk.clear(); 329 | this._dkToDat.clear(); 330 | } 331 | 332 | /** 333 | * @desc Clear the active dat state 334 | * @private 335 | */ 336 | _clearActive() { 337 | for (const dat of this._dkToDat.values()) { 338 | dat.closeCB(noop); 339 | } 340 | this._dirToDk.clear(); 341 | this._dkToDat.clear(); 342 | } 343 | 344 | /** 345 | * @desc Convert a path to a directory under {@link rootDir} to its full fs path 346 | * @param {string} directory - A directory under {@link rootDir} 347 | * @return {string} - The full path to the directory 348 | */ 349 | actualDirPath(directory) { 350 | if (directory.startsWith(this.rootDir)) { 351 | return directory; 352 | } 353 | return path.join(this.rootDir, directory); 354 | } 355 | 356 | /** 357 | * @desc Initialize or load cached dat 358 | * @param {string} dir - The full path to the directory to initialize the dat in. 359 | * Returns the previously created dat if no initialization is needed. 360 | * @return {Promise<{discoveryKey: string, datKey: string, dat: DatWrapper}>} 361 | * @private 362 | */ 363 | async _loadDat(dir) { 364 | if (this.isActiveDir(dir)) { 365 | const discoveryKey = this.getDiscoveryKeyForDir(dir); 366 | const dat = this.getDat(discoveryKey); 367 | const datKey = dat.key(); 368 | 369 | return { discoveryKey, datKey, dat }; 370 | } else { 371 | const fullDir = this.actualDirPath(dir); 372 | 373 | const dat = await DatWrapper.create(dir, fullDir); 374 | const discoveryKey = dat.discoveryKey(); 375 | const datKey = dat.key(); 376 | 377 | this._markDatAsActive(dir, discoveryKey, dat); 378 | 379 | return { discoveryKey, datKey, dat }; 380 | } 381 | } 382 | 383 | /** 384 | * @desc Add dat information to the tracking state 385 | * @param {string} dir - The directory path to associate with the discovery key 386 | * @param {string} discoveryKey - The discoveryKey to associate with the Dat 387 | * @param {DatWrapper} dat - The dat associated with the dir and discovery key 388 | * @private 389 | */ 390 | _markDatAsActive(dir, discoveryKey, dat) { 391 | this._dirToDk.set(dir, discoveryKey); 392 | this._dkToDat.set(discoveryKey, dat); 393 | } 394 | 395 | /** 396 | * @desc Remove dat information from the tracking state 397 | * @param {string} dir - The directory path to disassociate with the discovery key 398 | * @param {string} discoveryKey - The discoveryKey to disassociate with a Dat 399 | * @private 400 | */ 401 | _unmarkDatAsActive(dir, discoveryKey) { 402 | this._dirToDk.delete(dir); 403 | this._dkToDat.delete(discoveryKey); 404 | } 405 | 406 | /** 407 | * @desc Replicate a dat if the streams discovery key (dk) is known to us 408 | * @param {Object} info - The info about the connection 409 | * @return {Protocol} - The hypecoreProtocol stream 410 | * @private 411 | */ 412 | _replicate(info) { 413 | let connId = ++this._connIdCounter; 414 | debug( 415 | `replicating to connection ${connId}:${info.type} ${info.host}:${ 416 | info.port 417 | }` 418 | ); 419 | const stream = hypercoreProtocol({ 420 | id: this.networkId, 421 | live: true, 422 | encrypt: true, 423 | }); 424 | 425 | stream.on('error', function(err) { 426 | debug(`Connection ${connId}: replication stream error: ${err}`); 427 | }); 428 | stream.on('close', function() { 429 | debug(`Connection ${connId}: stream closed`); 430 | }); 431 | stream.on('end', function() { 432 | debug(`Connection ${connId}: stream ended`); 433 | }); 434 | 435 | stream.on('feed', dk => { 436 | debug('OnFeed'); 437 | dk = dk.toString('hex'); 438 | const dat = this.getDat(dk); 439 | if (dat) { 440 | debug( 441 | `Connection ${connId}: DAT found, (discoveryKey: ${dk}, datKey: ${dat.key()}), uploading...` 442 | ); 443 | this.emit('replicating', dk); 444 | dat.replicate(stream); 445 | } else { 446 | debug(`Connection ${connId}: DAT not found (discoveryKey: ${dk})...`); 447 | } 448 | }); 449 | 450 | return stream; 451 | } 452 | 453 | /** 454 | * @desc Start sharing the supplied dat by having it join the swarm 455 | * @param {DatWrapper} dat - The dat to have join the swarm 456 | * @param {string?} [dk] - The dat's discovery key 457 | * @param {string?} [datKey] - The dat key 458 | * @private 459 | */ 460 | _shareDat(dat, dk, datKey) { 461 | if (!dk) { 462 | dk = dat.discoveryKey(); 463 | } 464 | if (!datKey) { 465 | datKey = dat.key(); 466 | } 467 | debug(`Sharing DAT: ${datKey}`); 468 | 469 | dat.joinSwarm(this._swarm).then(() => { 470 | this.emit('shared-dat', dk); 471 | debug(`Added discoveryKey to swarm: ${dk}`); 472 | }); 473 | } 474 | } 475 | 476 | module.exports = SwarmManager; 477 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dat-share", 3 | "version": "1.0.0", 4 | "description": "Webrecorders dat intergration", 5 | "main": "./index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git@github.com:webrecorder/dat-share.git" 9 | }, 10 | "author": "Webrecorder", 11 | "contributors": [ 12 | "John Berlin", 13 | "Ilya Kreymer" 14 | ], 15 | "license": "MIT", 16 | "private": false, 17 | "keywords": [ 18 | "webrecorder", 19 | "dat", 20 | "s3", 21 | "aws", 22 | "hyperdrive", 23 | "storage" 24 | ], 25 | "dependencies": { 26 | "aws-sdk": "^2.437.0", 27 | "boom": "^7.3.0", 28 | "chalk": "^2.4.2", 29 | "commander": "^2.20.0", 30 | "cors": "^2.8.5", 31 | "dat": "^13.12.2", 32 | "dat-encoding": "^5.0.1", 33 | "dat-node": "^3.5.15", 34 | "dat-s3-hybrid-storage": "webrecorder/dat-s3-hybrid-storage#develop", 35 | "dat-storage": "^1.1.1", 36 | "dat-swarm-defaults": "^1.0.2", 37 | "debug": "^4.1.1", 38 | "discovery-swarm": "^5.1.4", 39 | "eventemitter3": "^3.1.0", 40 | "fastify": "^2.2.0", 41 | "fastify-boom": "^0.1.0", 42 | "fastify-graceful-shutdown": "^2.0.1", 43 | "fastify-plugin": "^1.5.0", 44 | "fastify-swagger": "^2.3.2", 45 | "fs-extra": "^7.0.1", 46 | "hypercore-protocol": "^6.9.0", 47 | "hyperdrive": "^9.14.5", 48 | "supports-color": "^6.1.0" 49 | }, 50 | "devDependencies": { 51 | "@types/fs-extra": "^5.0.5", 52 | "@types/node": "^11.13.4", 53 | "ava": "^1.4.1", 54 | "babel-eslint": "^10.0.1", 55 | "eslint": "^5.16.0", 56 | "eslint-config-prettier": "^4.1.0", 57 | "eslint-plugin-prettier": "^3.0.1", 58 | "prettier": "^1.16.4", 59 | "prettier-eslint-cli": "^4.7.1", 60 | "request": "^2.88.0", 61 | "request-promise": "^4.2.4" 62 | }, 63 | "scripts": { 64 | "format": "prettier --write 'lib/**/*.js' 'test/**/*.js'", 65 | "lint": "eslint lib test", 66 | "start": "node run.js", 67 | "test": "ava --verbose -c 1 --serial" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /run.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict'; 3 | const util = require('util'); 4 | const config = require('./lib/config'); 5 | const initServer = require('./lib/initServer'); 6 | 7 | console.log( 8 | `Dat Share API server starting with config\n${util.inspect(config, { 9 | depth: null, 10 | compact: false, 11 | })}\n` 12 | ); 13 | 14 | initServer(config).catch(error => { 15 | console.error(error); 16 | }); 17 | -------------------------------------------------------------------------------- /test/fixtures/dummy.txt: -------------------------------------------------------------------------------- 1 | hi this is something blah 2 | -------------------------------------------------------------------------------- /test/fixtures/swarmMan/clone/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webrecorder/dat-share/5dce8ff674fa9f69c3fe15849fc55598397b6c85/test/fixtures/swarmMan/clone/.gitkeep -------------------------------------------------------------------------------- /test/fixtures/swarmMan/d1/a.txt: -------------------------------------------------------------------------------- 1 | Hi this is a.txt 2 | -------------------------------------------------------------------------------- /test/fixtures/swarmMan/d2/b.txt: -------------------------------------------------------------------------------- 1 | Hi this is b.txt 2 | -------------------------------------------------------------------------------- /test/helpers/index.js: -------------------------------------------------------------------------------- 1 | export TestContext from './testContext'; 2 | -------------------------------------------------------------------------------- /test/helpers/testContext.js: -------------------------------------------------------------------------------- 1 | const cp = require('child_process'); 2 | const path = require('path'); 3 | const EventEmitter = require('eventemitter3'); 4 | const fs = require('fs-extra'); 5 | const config = require('../../lib/config'); 6 | const initServer = require('../../lib/initServer'); 7 | 8 | module.exports = class TestContext extends EventEmitter { 9 | constructor() { 10 | super(); 11 | config.swarmManager.rootDir = path.join( 12 | __dirname, 13 | '..', 14 | 'fixtures', 15 | 'swarmMan' 16 | ); 17 | config.swarmManager.port = 3282; 18 | this._config = config; 19 | this.dir1 = 'd1'; 20 | this.dir2 = 'd2'; 21 | this.notUnderRoot = 'notUnderRoot'; 22 | this.shouldCreateDir = 'idonotexist'; 23 | this._cleanupables = []; 24 | } 25 | 26 | get rootDir() { 27 | return this._config.swarmManager.rootDir; 28 | } 29 | 30 | get smConfig() { 31 | return this._config.swarmManager; 32 | } 33 | 34 | get swarmPort() { 35 | return this._config.swarmManager.port; 36 | } 37 | 38 | get dir1DatPath() { 39 | return path.join(this.rootDir, this.dir1, '.dat'); 40 | } 41 | 42 | get dir2DatPath() { 43 | return path.join(this.rootDir, this.dir2, '.dat'); 44 | } 45 | 46 | hasCleanUp() { 47 | return this._cleanupables.length > 0; 48 | } 49 | 50 | async cleanUp() { 51 | const cleanUp = []; 52 | for (let i = 0; i < this._cleanupables.length; ++i) { 53 | cleanUp.push(this._cleanupables[i]()); 54 | } 55 | try { 56 | await Promise.all(cleanUp); 57 | } catch (e) { 58 | console.error(e); 59 | } 60 | this._cleanupables = []; 61 | } 62 | 63 | cloneDat(datKey, which) { 64 | const cloneTo = path.join(this.rootDir, 'clone'); 65 | const datP = path.join(cloneTo, '.dat'); 66 | const contentP = path.join( 67 | cloneTo, 68 | which === this.dir1 ? 'a.txt' : 'b.txt' 69 | ); 70 | this._cleanupables.push(async () => { 71 | try { 72 | await fs.remove(datP); 73 | } catch (e) {} 74 | try { 75 | await fs.remove(contentP); 76 | } catch (e) {} 77 | }); 78 | return new Promise((resolve, reject) => { 79 | cp.exec( 80 | `yarn run dat clone -d ${cloneTo} ${datKey}`, 81 | { 82 | env: process.env, 83 | }, 84 | (error, stdout, stderr) => { 85 | if (error) return reject(error); 86 | resolve({ 87 | cloneDir: { 88 | datP, 89 | contentP, 90 | }, 91 | stdout, 92 | stderr, 93 | }); 94 | } 95 | ); 96 | }); 97 | } 98 | 99 | async startServer() { 100 | this.server = await initServer(config); 101 | this.server.swarmManager.on('replicating', key => 102 | this.emit('replicating', key) 103 | ); 104 | this.server.swarmManager.on('shared-dat', key => 105 | this.emit('shared-dat', key) 106 | ); 107 | this._cleanupables.push(async () => { 108 | try { 109 | await this.closeServer(); 110 | } catch (e) { 111 | console.error(e); 112 | } 113 | }); 114 | } 115 | 116 | closeServer() { 117 | return new Promise((resolve, reject) => { 118 | const to = setTimeout( 119 | () => reject(new Error('failed to close server withing 10 seconds')), 120 | 10000 121 | ); 122 | this.server.close(() => { 123 | clearTimeout(to); 124 | resolve(); 125 | }); 126 | }); 127 | } 128 | 129 | startSwarmMan(swarmMam) { 130 | return new Promise((resolve, reject) => { 131 | swarmMam.initSwarm(); 132 | const to = setTimeout( 133 | () => reject(new Error('failed to start swarm man after 5sec')), 134 | 5000 135 | ); 136 | swarmMam.once('listening', () => { 137 | this._cleanupables.push(async () => { 138 | try { 139 | await this.shutDownSwarmMan(swarmMam); 140 | } catch (e) {} 141 | }); 142 | clearTimeout(to); 143 | resolve(); 144 | }); 145 | }); 146 | } 147 | 148 | shutDownSwarmMan(swarmMan) { 149 | return Promise.all([ 150 | swarmMan.close(), 151 | new Promise((resolve, reject) => { 152 | const to = setTimeout( 153 | () => reject(new Error('failed to close swarm man after 4sec')), 154 | 4000 155 | ); 156 | swarmMan.once('close', () => { 157 | clearTimeout(to); 158 | resolve(); 159 | }); 160 | }), 161 | ]); 162 | } 163 | 164 | /** 165 | * @param {string} dir 166 | */ 167 | deferredDatCleanup(dir) { 168 | this._cleanupables.push(async () => { 169 | try { 170 | await fs.remove(path.join(this.rootDir, dir, '.dat')); 171 | } catch (e) {} 172 | }); 173 | } 174 | 175 | /** 176 | * @param {Promise | function } promOrFn 177 | */ 178 | addCleanUpable(promOrFn) { 179 | const tst = promOrFn[Symbol.toStringTag]; 180 | if (tst === 'AsyncFunction') { 181 | this._cleanupables.push(promOrFn); 182 | } else if (tst === 'Promise') { 183 | this._cleanupables.push(() => promOrFn); 184 | } else { 185 | this._cleanupables.push(async () => { 186 | try { 187 | typeof promOrFn.then === 'function' 188 | ? await promOrFn 189 | : await promOrFn(); 190 | } catch (e) {} 191 | }); 192 | } 193 | } 194 | }; 195 | -------------------------------------------------------------------------------- /test/test-datShare.js: -------------------------------------------------------------------------------- 1 | import test from 'ava'; 2 | import * as fs from 'fs-extra'; 3 | import rp from 'request-promise'; 4 | import { TestContext } from './helpers'; 5 | 6 | test.before(async t => { 7 | t.context = new TestContext(); 8 | await t.context.startServer(); 9 | }); 10 | 11 | test.after.always(async t => { 12 | await t.context.cleanUp(); 13 | }); 14 | 15 | test.serial('post /init should initialize the dat', async t => { 16 | const res = await rp({ 17 | method: 'POST', 18 | uri: 'http://localhost:3000/init', 19 | body: { 20 | collDir: t.context.dir1, 21 | }, 22 | json: true, 23 | }); 24 | t.truthy(res, 'The post /init request should return a response'); 25 | t.true(res.datKey != null, 'The response should send a non null dataKey'); 26 | t.true( 27 | res.discoveryKey != null, 28 | 'The response should send a non null discoveryKey' 29 | ); 30 | t.true( 31 | await fs.pathExists(t.context.dir1DatPath), 32 | 'The .dat should be created' 33 | ); 34 | t.context.deferredDatCleanup(t.context.dir1); 35 | }); 36 | 37 | test.serial( 38 | 'get /numSharing initialize the dat should return 0 but get /numDats should return 1', 39 | async t => { 40 | let res = await rp({ 41 | method: 'GET', 42 | uri: 'http://localhost:3000/numSharing', 43 | json: true, 44 | }); 45 | t.truthy(res, 'The post /numSharing request should return a response'); 46 | t.is(res.num, 0, 'The response should indicate we have 0 shared dats'); 47 | res = await rp({ 48 | method: 'GET', 49 | uri: 'http://localhost:3000/numDats', 50 | json: true, 51 | }); 52 | t.truthy(res, 'The post /numDats request should return a response'); 53 | t.is(res.num, 1, 'The response should indicate we have 1 dat'); 54 | } 55 | ); 56 | 57 | test.serial('post /share after post /init should share the dat', async t => { 58 | const res = await rp({ 59 | method: 'POST', 60 | uri: 'http://localhost:3000/share', 61 | body: { 62 | collDir: t.context.dir1, 63 | }, 64 | json: true, 65 | }); 66 | const sharedProm = new Promise((resolve, reject) => { 67 | const to = setTimeout( 68 | () => reject(new Error('failed to share dat after 10sec')), 69 | 10000 70 | ); 71 | t.context.once('shared-dat', dk => { 72 | clearTimeout(to); 73 | resolve(dk); 74 | }); 75 | }); 76 | t.truthy(res, 'The post /share request should return a response'); 77 | t.true(res.datKey != null, 'The response should send a non null dataKey'); 78 | t.true( 79 | res.discoveryKey != null, 80 | 'The response should send a non null discoveryKey' 81 | ); 82 | const serverKey = await sharedProm; 83 | t.is( 84 | res.discoveryKey, 85 | serverKey, 86 | 'The responses discoveryKey should be equal to the one the sever uses to join the swarm' 87 | ); 88 | let replicatingKey = ''; 89 | t.context.once('replicating', key => { 90 | replicatingKey = key; 91 | }); 92 | const { cloneDir } = await t.context.cloneDat(res.datKey, t.context.dir1); 93 | t.true( 94 | (await fs.stat(cloneDir.datP)).isDirectory(), 95 | 'After cloning the dat cloned dat directory should exist' 96 | ); 97 | t.true( 98 | (await fs.stat(cloneDir.contentP)).isFile(), 99 | 'After cloning the dats content directory should exist' 100 | ); 101 | t.is( 102 | (await fs.readdir(cloneDir.datP)).length, 103 | 10, 104 | 'After cloning the dats content directory should contain content' 105 | ); 106 | t.is( 107 | replicatingKey, 108 | serverKey, 109 | 'The replicatingKey should be equal to the one the sever uses to join the swarm' 110 | ); 111 | t.is( 112 | replicatingKey, 113 | res.discoveryKey, 114 | 'The replicatingKey should be equal to the responses discoveryKey' 115 | ); 116 | }); 117 | 118 | test.serial( 119 | 'get /numSharing sharing the dat should return 1 and get /numDats should return 1', 120 | async t => { 121 | let res = await rp({ 122 | method: 'GET', 123 | uri: 'http://localhost:3000/numSharing', 124 | json: true, 125 | }); 126 | t.truthy(res, 'The post /numSharing request should return a response'); 127 | t.is(res.num, 1, 'The response should indicate we are sharing 1 dat'); 128 | res = await rp({ 129 | method: 'GET', 130 | uri: 'http://localhost:3000/numDats', 131 | json: true, 132 | }); 133 | t.truthy(res, 'The post /numDats request should return a response'); 134 | t.is(res.num, 1, 'The response should indicate we have 1 dat'); 135 | } 136 | ); 137 | -------------------------------------------------------------------------------- /test/test-swarmManager.js: -------------------------------------------------------------------------------- 1 | import test from 'ava'; 2 | import * as fs from 'fs-extra'; 3 | import path from 'path'; 4 | import { TestContext } from './helpers'; 5 | import SwarmManager from '../lib/swarmManager'; 6 | 7 | test.beforeEach(t => { 8 | t.context = new TestContext(); 9 | }); 10 | 11 | test.afterEach.always(async t => { 12 | if (t.context.hasCleanUp()) { 13 | await t.context.cleanUp(); 14 | } 15 | }); 16 | 17 | test('creating a new SwarmManager should not initialize dats or swarm', async t => { 18 | const { context } = t; 19 | const swarmMan = new SwarmManager(context.smConfig); 20 | t.is( 21 | swarmMan.rootDir, 22 | context.rootDir, 23 | 'the SwarmManager rootDir should be equal to the one used to create it' 24 | ); 25 | t.is( 26 | swarmMan.port, 27 | context.smConfig.port, 28 | 'the SwarmManager port should be equal to the value of swarmPort used to create it' 29 | ); 30 | t.is( 31 | swarmMan.numDats(), 32 | 0, 33 | 'the SwarmManager should not indicate it has dats right after creation' 34 | ); 35 | t.is( 36 | swarmMan.numSharing(), 37 | 0, 38 | 'the SwarmManager should not indicate it is sharing dats right after creation' 39 | ); 40 | t.is( 41 | swarmMan._dkToDat.size, 42 | 0, 43 | 'the SwarmManager should not indicate it has dats right after creation' 44 | ); 45 | t.is( 46 | swarmMan._dirToDk.size, 47 | 0, 48 | 'the SwarmManager should not indicate it has dats right after creation' 49 | ); 50 | t.is( 51 | swarmMan._connIdCounter, 52 | 0, 53 | 'the SwarmManager should not have recieved connections after creation' 54 | ); 55 | t.is( 56 | swarmMan._swarm, 57 | null, 58 | 'the SwarmManager should not have joined the swarm after creation' 59 | ); 60 | t.false( 61 | swarmMan._closeServerShutDown, 62 | 'the SwarmManager should not have shut down after creation' 63 | ); 64 | t.truthy( 65 | swarmMan.networkId, 66 | 'the SwarmManager should have assigned itself a network id' 67 | ); 68 | }); 69 | 70 | test('SwarmManager should indicate if it can share a directory if it exists under rootdir otherwise should indicate cant', async t => { 71 | const { context } = t; 72 | const swarmMan = new SwarmManager(context.smConfig); 73 | t.true( 74 | await swarmMan.canShareDir(context.dir1), 75 | 'should indicate ability to share a directory under rootdir' 76 | ); 77 | t.true( 78 | await swarmMan.canShareDir(context.dir2), 79 | 'should indicate ability to share a directory under rootdir' 80 | ); 81 | t.false( 82 | await swarmMan.canShareDir(context.notUnderRoot), 83 | 'should indicate inability to share a directory not under rootdir' 84 | ); 85 | }); 86 | 87 | test('SwarmManager.actualDirPath should join the supplied path to the rootdir correctly', async t => { 88 | const { context } = t; 89 | const swarmMan = new SwarmManager(context.smConfig); 90 | 91 | const fp1 = swarmMan.actualDirPath(context.dir1); 92 | t.true( 93 | fp1.startsWith(context.rootDir), 94 | 'actualDirPath(dir1) should start with rootDir' 95 | ); 96 | t.true(await fs.pathExists(fp1), 'fp1 should exist'); 97 | t.true((await fs.stat(fp1)).isDirectory(), 'fp1 should be a dir'); 98 | t.false( 99 | swarmMan 100 | .actualDirPath(fp1) 101 | .startsWith(path.join(context.rootDir, context.rootDir)), 102 | 'actualDirPath should not prefix rootDir if the supplied path starts with it' 103 | ); 104 | 105 | const fp2 = swarmMan.actualDirPath(context.dir2); 106 | t.true( 107 | fp2.startsWith(context.rootDir), 108 | 'actualDirPath(dir2) should start with rootDir' 109 | ); 110 | t.true(await fs.pathExists(fp2), 'fp2 should exist'); 111 | t.true((await fs.stat(fp2)).isDirectory(), 'fp2 should be a dir'); 112 | t.false( 113 | swarmMan 114 | .actualDirPath(fp2) 115 | .startsWith(path.join(context.rootDir, context.rootDir)), 116 | 'actualDirPath should not prefix rootDir if the supplied path starts with it' 117 | ); 118 | 119 | const fp3 = swarmMan.actualDirPath(context.notUnderRoot); 120 | t.true( 121 | fp3.startsWith(context.rootDir), 122 | 'actualDirPath(notUnderRoot) should start with rootDir' 123 | ); 124 | t.false(await fs.pathExists(fp3), 'fp3 should not exist'); 125 | t.false( 126 | swarmMan 127 | .actualDirPath(fp3) 128 | .startsWith(path.join(context.rootDir, context.rootDir)), 129 | 'actualDirPath should not prefix rootDir if the supplied path starts with it' 130 | ); 131 | }); 132 | 133 | test.serial( 134 | 'SwarmManager should not emit "listening" when the swarm is listening and "close" when the swarm closes', 135 | async t => { 136 | const { context } = t; 137 | const swarmMan = new SwarmManager(context.smConfig); 138 | 139 | const listening = await new Promise((resolve, reject) => { 140 | swarmMan.initSwarm(); 141 | swarmMan.once('listening', () => resolve(true)); 142 | setTimeout(() => resolve(false), 4000); 143 | }); 144 | t.true(listening, 'SwarmManager failed to emit listening'); 145 | 146 | swarmMan.close().then(); 147 | const closed = await new Promise((resolve, reject) => { 148 | swarmMan.once('close', () => resolve(true)); 149 | setTimeout(() => resolve(false), 4000); 150 | }); 151 | t.true( 152 | closed, 153 | 'SwarmManager failed to emit close or the swarm did not close' 154 | ); 155 | } 156 | ); 157 | 158 | test.serial( 159 | 'SwarmManager.initDat should initialize a dat in the supplied directory', 160 | async t => { 161 | const { context } = t; 162 | context.deferredDatCleanup(context.dir1); 163 | const swarmMan = new SwarmManager(context.smConfig); 164 | 165 | const datInfo = await swarmMan.initDat(context.dir1); 166 | t.truthy(datInfo, 'The return value of initDat should be non-null'); 167 | t.true( 168 | typeof datInfo === 'object', 169 | 'The return value of initDat should be an object' 170 | ); 171 | t.true( 172 | typeof datInfo.discoveryKey === 'string', 173 | 'datInfo.discoveryKey should be a string' 174 | ); 175 | t.true( 176 | typeof datInfo.datKey === 'string', 177 | 'datInfo.datKey should be a string' 178 | ); 179 | 180 | const dirP = swarmMan.actualDirPath(context.dir1); 181 | t.true( 182 | swarmMan.isActiveDir(context.dir1), 183 | "Once a directory is init'd, isActiveDir should return true" 184 | ); 185 | t.is( 186 | swarmMan.numDats(), 187 | 1, 188 | "Once a directory is init'd, numDats should return 1" 189 | ); 190 | t.is( 191 | swarmMan.numSharing(), 192 | 0, 193 | "Once a directory is init'd, numSharing should be 0" 194 | ); 195 | 196 | const dat = swarmMan.getDat(datInfo.discoveryKey); 197 | t.truthy( 198 | dat, 199 | "Once a directory is init'd, getDat should return the dat associated with the directory" 200 | ); 201 | t.true( 202 | swarmMan.getDatForDir(context.dir1) === dat, 203 | "Once a directory is init'd, the dat retrieved using getDatForDir, should match the dat returned from getDat" 204 | ); 205 | t.true( 206 | swarmMan.getDiscoveryKeyForDir(context.dir1) === datInfo.discoveryKey, 207 | "Once a directory is init'd, the discovery key returned by getDiscoveryKeyForDir should match the datInfo's" 208 | ); 209 | t.true( 210 | dat.discoveryKey('hex') === datInfo.discoveryKey, 211 | "Once a directory is init'd, dat.discoveryKey('hex') should match the datInfo's" 212 | ); 213 | t.true( 214 | dat.key('hex') === datInfo.datKey, 215 | "Once a directory is init'd, dat.key('hex') should match the datInfo's" 216 | ); 217 | t.false( 218 | dat.sharing(), 219 | "Once a directory is init'd, the dat retrieved using getDat, should not indicate it is being shared after init" 220 | ); 221 | 222 | const reinit = await swarmMan.initDat(context.dir1); 223 | t.true( 224 | typeof reinit === 'object', 225 | 'The return value of initDat should be an object' 226 | ); 227 | t.true( 228 | typeof reinit.discoveryKey === 'string', 229 | 'The return value of initDat 2x should have a discoveryKey property' 230 | ); 231 | t.true( 232 | reinit.discoveryKey === datInfo.discoveryKey, 233 | 'The return value of initDat 2x should have a discoveryKey property equal to first init value' 234 | ); 235 | t.true( 236 | typeof reinit.datKey === 'string', 237 | 'The return value of initDat 2x should have a datKey property' 238 | ); 239 | t.true( 240 | reinit.datKey === datInfo.datKey, 241 | 'The return value of initDat 2x should have a datKey property equal to first init value' 242 | ); 243 | 244 | t.is( 245 | swarmMan.numDats(), 246 | 1, 247 | 'The return value of numDats after init 2x should be 1' 248 | ); 249 | t.is( 250 | swarmMan.numSharing(), 251 | 0, 252 | 'The return value of numSharing after init 2x should be 0' 253 | ); 254 | 255 | context.addCleanUpable(dat.close()); 256 | } 257 | ); 258 | 259 | test.serial( 260 | 'SwarmManager should share and unshare a directory after it was inited', 261 | async t => { 262 | const { context } = t; 263 | const swarmMan = new SwarmManager(context.smConfig); 264 | await context.startSwarmMan(swarmMan); 265 | const tTO = setTimeout( 266 | () => 267 | t.fail('SwarmMan share unshare failed to complete after 60 seconds'), 268 | 60000 269 | ); 270 | context.deferredDatCleanup(context.dir1); 271 | const initInfo = await swarmMan.initDat(context.dir1); 272 | const shareInfo = await swarmMan.shareDir(context.dir1); 273 | t.is( 274 | initInfo.datKey, 275 | shareInfo.datKey, 276 | 'The initInfo.datKey should equal shareInfo.datKey' 277 | ); 278 | t.is( 279 | initInfo.discoveryKey, 280 | shareInfo.discoveryKey, 281 | 'The initInfo.discoveryKey should equal shareInfo.discoveryKey' 282 | ); 283 | const sharedDK = await new Promise((resolve, reject) => { 284 | const to = setTimeout( 285 | () => reject(new Error('failed to share dat after 4sec')), 286 | 10000 287 | ); 288 | swarmMan.once('shared-dat', dk => { 289 | clearTimeout(to); 290 | resolve(dk); 291 | }); 292 | }); 293 | t.is( 294 | sharedDK, 295 | shareInfo.discoveryKey, 296 | 'The sharedDK should be equal to shareInfo.discoveryKey' 297 | ); 298 | t.is( 299 | sharedDK, 300 | initInfo.discoveryKey, 301 | 'The sharedDK should be equal to initInfo.discoveryKey' 302 | ); 303 | t.is( 304 | swarmMan.numDats(), 305 | 1, 306 | 'After init and share numDats should still be 1' 307 | ); 308 | t.is( 309 | swarmMan.numSharing(), 310 | 1, 311 | 'After init and share numSharing should be 1' 312 | ); 313 | 314 | const dat = swarmMan.getDat(shareInfo.discoveryKey); 315 | t.true( 316 | dat.sharing(), 317 | 'After init and share the dat returned from getDat should indicate it is being shared' 318 | ); 319 | let replicatingKey = ''; 320 | swarmMan.once('replicating', key => { 321 | replicatingKey = key; 322 | }); 323 | 324 | const { cloneDir } = await context.cloneDat(shareInfo.datKey, context.dir1); 325 | t.true( 326 | (await fs.stat(cloneDir.datP)).isDirectory(), 327 | 'After cloning the dat cloned dat directory should exist' 328 | ); 329 | t.true( 330 | (await fs.stat(cloneDir.contentP)).isFile(), 331 | 'After cloning the dats content directory should exist' 332 | ); 333 | t.is( 334 | (await fs.readdir(cloneDir.datP)).length, 335 | 10, 336 | 'After cloning the dats content directory should contain content' 337 | ); 338 | 339 | t.is( 340 | replicatingKey, 341 | dat.discoveryKey('hex'), 342 | 'the replicating key from emitted from SwarmMan should equal the dats' 343 | ); 344 | t.is( 345 | replicatingKey, 346 | initInfo.discoveryKey, 347 | 'the replicating key from emitted from SwarmMan should equal initInfo.discoveryKey' 348 | ); 349 | t.is( 350 | replicatingKey, 351 | shareInfo.discoveryKey, 352 | 'the replicating key from emitted from SwarmMan should equal shareInfo.discoveryKey' 353 | ); 354 | 355 | t.true( 356 | swarmMan.unshareDir(context.dir1), 357 | 'SwarmMan should return true from unshareDir when unsharing a dir that is shared' 358 | ); 359 | t.is( 360 | swarmMan.numDats(), 361 | 0, 362 | 'After unsharing a directory numDats should be 0' 363 | ); 364 | t.is( 365 | swarmMan.numSharing(), 366 | 0, 367 | 'After unsharing a directory numSharing should be 0' 368 | ); 369 | t.false( 370 | swarmMan.isActiveDir(swarmMan.actualDirPath(context.dir1)), 371 | 'After unsharing a directory isActiveDir should return false for the unshared dir' 372 | ); 373 | 374 | clearTimeout(tTO); 375 | } 376 | ); 377 | 378 | test.serial('SwarmManager should sync directories correctly', async t => { 379 | const { context } = t; 380 | const swarmMan = new SwarmManager(context.smConfig); 381 | await context.startSwarmMan(swarmMan); 382 | const tTO = setTimeout( 383 | () => t.fail('SwarmMan sync failed to complete after 60 seconds'), 384 | 60000 385 | ); 386 | context.deferredDatCleanup(context.dir1); 387 | context.deferredDatCleanup(context.dir2); 388 | await swarmMan.initDat(context.dir1); 389 | const d1ShareInfo = await swarmMan.shareDir(context.dir1); 390 | const sharedDK1 = await new Promise((resolve, reject) => { 391 | const to = setTimeout( 392 | () => reject(new Error('failed to share dat after 4sec')), 393 | 10000 394 | ); 395 | swarmMan.once('shared-dat', dk => { 396 | clearTimeout(to); 397 | resolve(dk); 398 | }); 399 | }); 400 | t.is( 401 | sharedDK1, 402 | d1ShareInfo.discoveryKey, 403 | 'sharedDK1 should equal d1ShareInfo.discoveryKey' 404 | ); 405 | t.is(swarmMan.numDats(), 1, 'numDats should be 1'); 406 | t.is(swarmMan.numSharing(), 1, 'numSharing should be 1'); 407 | 408 | const dat1 = swarmMan.getDatForDir(context.dir1); 409 | const { errors, results } = await swarmMan.sync({ dirs: [context.dir2] }); 410 | t.is( 411 | errors.length, 412 | 0, 413 | 'syncing should not return errors for an existing dir' 414 | ); 415 | t.is(results.length, 1, 'syncing should return 1 result for syncing one dir'); 416 | t.false( 417 | dat1.sharing(), 418 | 'the previously shared dat after syncing should indicate it is not being shared' 419 | ); 420 | 421 | const { dir, discoveryKey, datKey } = results[0]; 422 | t.is(swarmMan.numDats(), 1, 'after sync numDats should be 1'); 423 | t.is(swarmMan.numSharing(), 1, 'after sync numSharing should be 1'); 424 | 425 | const sharedDat = swarmMan.getDatForDir(context.dir2); 426 | t.is( 427 | discoveryKey, 428 | sharedDat.discoveryKey('hex'), 429 | 'the newly shared dat (from sync) discoveryKey should equal the sync results discoveryKey' 430 | ); 431 | t.is( 432 | datKey, 433 | sharedDat.key('hex'), 434 | 'the newly shared dat (from sync) datKey should equal the sync results datKey' 435 | ); 436 | t.true( 437 | sharedDat.sharing(), 438 | 'the newly shared dat (from sync) datKey should indicate it is being shared' 439 | ); 440 | 441 | let replicatingKey = ''; 442 | swarmMan.once('replicating', key => { 443 | replicatingKey = key; 444 | }); 445 | 446 | const { cloneDir } = await context.cloneDat(datKey, context.dir2); 447 | t.true( 448 | (await fs.stat(cloneDir.datP)).isDirectory(), 449 | 'After cloning the dat cloned dat directory should exist' 450 | ); 451 | t.true( 452 | (await fs.stat(cloneDir.contentP)).isFile(), 453 | 'After cloning the dats content directory should exist' 454 | ); 455 | t.is( 456 | (await fs.readdir(cloneDir.datP)).length, 457 | 10, 458 | 'After cloning the dats content directory should contain content' 459 | ); 460 | 461 | t.is( 462 | replicatingKey, 463 | sharedDat.discoveryKey('hex'), 464 | 'replicating key should equal shared dat discoveryKey ' 465 | ); 466 | t.is( 467 | replicatingKey, 468 | discoveryKey, 469 | 'replicating key should equal discovery key' 470 | ); 471 | 472 | t.true( 473 | swarmMan.unshareDir(context.dir2), 474 | 'SwarmMan should return true from unshareDir when unsharing a dir that is shared' 475 | ); 476 | t.is( 477 | swarmMan.numDats(), 478 | 0, 479 | 'After unsharing a directory numDats should be 0' 480 | ); 481 | t.is( 482 | swarmMan.numSharing(), 483 | 0, 484 | 'After unsharing a directory numSharing should be 0' 485 | ); 486 | t.false( 487 | swarmMan.isActiveDir(context.dir2), 488 | 'After unsharing a directory isActiveDir should return false for the unshared dir' 489 | ); 490 | clearTimeout(tTO); 491 | }); 492 | --------------------------------------------------------------------------------