├── test ├── scaffold │ └── testdat1 │ │ ├── hello.txt │ │ └── dat.json ├── lib │ ├── util.js │ ├── dat.js │ └── server.js ├── quotas.js ├── activity.js ├── admin.js ├── users.js └── archives.js ├── .gitignore ├── contributors.yml ├── docs ├── schemas │ ├── access-scopes.md │ ├── events.md │ └── leveldb.md ├── components │ ├── triggers.md │ ├── locks.md │ └── jobs.md ├── flows │ ├── forgot-password.md │ ├── registration.md │ └── dat-ownership-proof.md ├── README.md └── webapis.md ├── lib ├── sanitizers.js ├── lock.js ├── templates │ ├── mail │ │ ├── support.js │ │ ├── verification.js │ │ ├── verify-update-email.js │ │ └── forgot-password.js │ └── directory-listing-page.js ├── validators.js ├── config.js ├── proofs.js ├── crypto.js ├── mailer.js ├── sessions.js ├── const.js ├── apis │ ├── service.js │ ├── admin.js │ ├── archive-files.js │ ├── archives.js │ └── users.js ├── helpers.js ├── dbs │ ├── activity.js │ ├── archives.js │ └── users.js ├── index.js └── archiver.js ├── nodecompat.js ├── bin.js ├── config.defaults.yml ├── CONTRIBUTING.md ├── package.json ├── readme.md └── index.js /test/scaffold/testdat1/hello.txt: -------------------------------------------------------------------------------- 1 | world -------------------------------------------------------------------------------- /test/scaffold/testdat1/dat.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Test Dat 1", 3 | "description": "The first test dat" 4 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .hypercloud 4 | config.*.yml 5 | !config.defaults.yml 6 | test/scaffold/testdat1/.dat 7 | -------------------------------------------------------------------------------- /contributors.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Paul Frazee 3 | catchphrase: "If it ain't P2P, it ain't free" 4 | website: https://twitter.com/pfrazee 5 | --- 6 | name: Joe Hand 7 | catchphrase: "Building things by hand." 8 | website: https://joeahand.com 9 | -------------------------------------------------------------------------------- /docs/schemas/access-scopes.md: -------------------------------------------------------------------------------- 1 | # Access Scopes 2 | 3 | Scopes are hypercloud's term for "permissions" 4 | 5 | - `user` - base permission, must be set for any non-public API to be used 6 | - `admin:dats` - control dats via API 7 | - `admin:users` - control users via API -------------------------------------------------------------------------------- /lib/sanitizers.js: -------------------------------------------------------------------------------- 1 | const bytes = require('bytes') 2 | const { DAT_URL_REGEX } = require('./const') 3 | 4 | exports.toDatDomain = value => { 5 | return DAT_URL_REGEX.exec(value)[1] + '/' 6 | } 7 | 8 | exports.toBytes = value => { 9 | return bytes(value) 10 | } 11 | -------------------------------------------------------------------------------- /test/lib/util.js: -------------------------------------------------------------------------------- 1 | const os = require('os') 2 | const path = require('path') 3 | const fs = require('fs') 4 | 5 | exports.mktmpdir = function () { 6 | if (fs.mkdtempSync) { 7 | return fs.mkdtempSync(os.tmpdir() + path.sep + 'hypercloud-test-') 8 | } 9 | var p = (os.tmpdir() + path.sep + 'beaker-test-' + Date.now()) 10 | fs.mkdirSync(p) 11 | return p 12 | } 13 | -------------------------------------------------------------------------------- /lib/lock.js: -------------------------------------------------------------------------------- 1 | var AwaitLock = require('await-lock') 2 | 3 | // wraps await-lock in a simpler interface, with many possible locks 4 | // usage: 5 | /* 6 | var lock = require('./lock') 7 | async function foo () { 8 | var release = await lock('bar') 9 | // ... 10 | release() 11 | } 12 | */ 13 | 14 | var locks = {} 15 | module.exports = async function (key) { 16 | if (!(key in locks)) locks[key] = new AwaitLock() 17 | 18 | var lock = locks[key] 19 | await lock.acquireAsync() 20 | return lock.release.bind(lock) 21 | } 22 | -------------------------------------------------------------------------------- /lib/templates/mail/support.js: -------------------------------------------------------------------------------- 1 | exports.subject = function (params) { 2 | return params.subject 3 | } 4 | 5 | exports.text = function (params) { 6 | console.log(params) 7 | return ` 8 | ${params.username},\n 9 | \n 10 | ${params.message}\n 11 | \n 12 | Thanks,\n 13 | The ${params.brandname} team 14 | ` 15 | } 16 | 17 | exports.html = function (params) { 18 | return ` 19 |

${params.username},

20 |

${params.message}

21 |

Thanks,
The ${params.brandname} team

22 | ` 23 | } 24 | -------------------------------------------------------------------------------- /nodecompat.js: -------------------------------------------------------------------------------- 1 | var vMajor = +(/^v([\d]+)/.exec(process.version)[1]) 2 | if (vMajor < 6) { 3 | console.log('Detected node version <6, transpiling es2015 features') 4 | try { 5 | require('babel-register')({ presets: ['es2015', 'transform-async-to-generator'] }) 6 | } catch (e) { 7 | console.log('Call `npm run install-transpiler` first. You\'re on node <6, so we need extra deps.') 8 | process.exit(1) 9 | } 10 | } else if (vMajor === 6) { 11 | console.log('Detected node version 6, transpiling async to generators') 12 | require('babel-register')({ plugins: ['transform-async-to-generator'] }) 13 | } 14 | -------------------------------------------------------------------------------- /docs/schemas/events.md: -------------------------------------------------------------------------------- 1 | ## UsersDB 2 | 3 | Emits: 4 | 5 | ```js 6 | usersDB.on('create', (record) => {}) 7 | usersDB.on('put', (record) => {}) 8 | usersDB.on('del', (record) => {}) 9 | usersDB.on('add-archive', ({userId, archiveKey, name}, record) => {}) 10 | usersDB.on('remove-archive', ({userId, archiveKey}, record) => {}) 11 | ``` 12 | 13 | ## ArchivesDB 14 | 15 | ```js 16 | archivesDB.emit('create', (record) => {}) 17 | archivesDB.emit('put', (record) => {}) 18 | archivesDB.emit('del', (record) => {}) 19 | archivesDB.emit('add-hosting-user', ({key, userId}, record) => {}) 20 | archivesDB.emit('remove-hosting-user', ({key, userId}, record) => {}) 21 | ``` -------------------------------------------------------------------------------- /bin.js: -------------------------------------------------------------------------------- 1 | require('./nodecompat') 2 | var config = require('./lib/config') 3 | var createApp = require('./index') 4 | var log = require('debug')('LE') 5 | 6 | var app = createApp(config) 7 | if (config.letsencrypt) { 8 | var greenlockExpress = require('greenlock-express') 9 | var debug = config.letsencrypt.debug !== false 10 | greenlockExpress.create({ 11 | server: debug ? 'staging' : 'https://acme-v01.api.letsencrypt.org/directory', 12 | debug, 13 | approveDomains: app.approveDomains, 14 | app, 15 | log 16 | }).listen(80, 443) 17 | } else { 18 | app.listen(config.port, () => { 19 | console.log(`server started on http://127.0.0.1:${config.port}`) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /lib/templates/mail/verification.js: -------------------------------------------------------------------------------- 1 | exports.subject = function () { 2 | return 'Verify your email address' 3 | } 4 | 5 | exports.text = function (params) { 6 | return ` 7 | \n 8 | Welcome, ${params.username}, to ${params.brandname}.\n 9 | \n 10 | To verify your account, follow this link:\n 11 | \n 12 | ${params.emailVerificationLink}\n 13 | \n 14 | \n 15 | ` 16 | } 17 | 18 | exports.html = function (params) { 19 | return ` 20 |

Welcome, ${params.username}, to ${params.brandname}.

21 |

To verify your account, follow this link:

22 |

${params.emailVerificationLink}

23 | ` 24 | } 25 | -------------------------------------------------------------------------------- /lib/validators.js: -------------------------------------------------------------------------------- 1 | const bytes = require('bytes') 2 | const { DAT_URL_REGEX, DAT_KEY_REGEX, DAT_NAME_REGEX } = require('./const') 3 | 4 | exports.isDatURL = value => { 5 | return DAT_URL_REGEX.test(value) 6 | } 7 | 8 | exports.isDatHash = value => { 9 | return value.length === 64 && DAT_KEY_REGEX.test(value) 10 | } 11 | 12 | exports.isDatName = value => { 13 | return DAT_NAME_REGEX.test(value) 14 | } 15 | 16 | exports.isScopesArray = value => { 17 | return Array.isArray(value) && value.filter(v => typeof v !== 'string').length === 0 18 | } 19 | 20 | exports.isSimpleEmail = value => { 21 | return typeof value === 'string' && value.indexOf('+') === -1 22 | } 23 | 24 | exports.isBytes = value => { 25 | return !!bytes(value) 26 | } 27 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var extend = require('deep-extend') 4 | var yaml = require('js-yaml') 5 | 6 | function load (name) { 7 | var str, doc 8 | var filepath = path.join(__dirname, `../config.${name}.yml`) 9 | 10 | try { 11 | str = fs.readFileSync(filepath, 'utf8') 12 | } catch (e) { 13 | return {} 14 | } 15 | 16 | try { 17 | doc = yaml.safeLoad(str) 18 | } catch (e) { 19 | console.log('Failed to parse', filepath, e) 20 | return {} 21 | } 22 | 23 | return doc 24 | } 25 | 26 | // load the config 27 | var env = process.env.NODE_ENV || 'development' 28 | var defaultCfg = load('defaults') 29 | var envCfg = load(env) 30 | module.exports = extend(defaultCfg, envCfg, { env }) 31 | -------------------------------------------------------------------------------- /lib/templates/mail/verify-update-email.js: -------------------------------------------------------------------------------- 1 | exports.subject = function () { 2 | return 'Verify your email address' 3 | } 4 | 5 | exports.text = function (params) { 6 | return ` 7 | \n 8 | Hi ${params.username},\n 9 | \n 10 | You requested to change the email address assocatied with your account at ${params.brandname}. 11 | \n 12 | To verify this change, follow this link:\n 13 | \n 14 | ${params.emailVerificationLink}\n 15 | \n 16 | \n 17 | ` 18 | } 19 | 20 | exports.html = function (params) { 21 | return ` 22 |

Hi, ${params.username}.

23 |

You requested to change the email address associated with your account at ${params.brandname}.

24 |

To verify this change, follow this link:

25 |

${params.emailVerificationLink}

26 | ` 27 | } 28 | -------------------------------------------------------------------------------- /config.defaults.yml: -------------------------------------------------------------------------------- 1 | dir: ./.hypercloud 2 | brandname: Hypercloud 3 | hostname: hypercloud.local 4 | port: 8080 5 | letsencrypt: false 6 | ui: hypercloud-ui-vanilla 7 | sites: false 8 | rateLimiting: true 9 | defaultDiskUsageLimit: 100mb 10 | pm2: false 11 | 12 | # processing jobs 13 | jobs: 14 | popularArchivesIndex: 30s 15 | userDiskUsage: 5m 16 | deleteDeadArchives: 5m 17 | 18 | # user settings 19 | registration: 20 | open: false 21 | allowed: 22 | - alice@mail.com 23 | - bob@mail.com 24 | reservedNames: 25 | - admin 26 | - root 27 | - support 28 | - noreply 29 | - users 30 | - archives 31 | admin: 32 | email: '' 33 | password: '' 34 | 35 | # email settings 36 | email: 37 | transport: stub 38 | sender: '"Hypercloud" ' 39 | 40 | # login sessions 41 | sessions: 42 | algorithm: HS256 43 | secret: THIS MUST BE REPLACED! 44 | expiresIn: 1h -------------------------------------------------------------------------------- /lib/proofs.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var jwt = require('jsonwebtoken') 3 | 4 | // exported api 5 | // = 6 | 7 | module.exports = class Proofs { 8 | constructor (config) { 9 | this.secret = config.proofs.secret 10 | this.options = config.proofs 11 | delete this.options.secret 12 | assert(this.secret, 'config.proofs.secret is required') 13 | assert(this.options.algorithm, 'config.sessions.algorithm is required') 14 | } 15 | 16 | verify (token) { 17 | try { 18 | // return decoded session or null on failure 19 | return jwt.verify(token, this.secret, { algorithms: [this.options.algorithm] }) 20 | } catch (e) { 21 | return null 22 | } 23 | } 24 | 25 | generate (userRecord) { 26 | return jwt.sign( 27 | { 28 | id: userRecord.id, 29 | profileURL: userRecord.profileURL 30 | }, 31 | this.secret, 32 | { algorithm: this.options.algorithm } 33 | ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /docs/components/triggers.md: -------------------------------------------------------------------------------- 1 | # Triggers Component 2 | 3 | Any file-indexing is handled by Triggers, which watch for changes to specific paths and archives, then queue jobs automatically when a change is detected. 4 | 5 | In some ways, Triggers are an alternative control mechanism to HTTP requests. Rather than a GET/POST, clients write and upload files which cause updates in the hypercloud instance. 6 | 7 | ## Triggers API 8 | 9 | ```js 10 | triggers.add(pathSpec, handler) // add a trigger & handler function 11 | triggers.list() // list the active triggers 12 | triggers.remove(handerId) // remove a trigger & handler 13 | ``` 14 | 15 | The `pathSpec` may be a string or regex. 16 | 17 | Example usage: 18 | 19 | ```js 20 | triggers.add('/proofs/hypercloud.com', (archive, entry) => { 21 | jobs.queue('verify-profile-dat', { datUrl: archive.url }) 22 | }) 23 | 24 | triggers.add(new RegExp('/dats/[a-z0-9]'), (archive, entry) => { 25 | // ... 26 | }) 27 | ``` -------------------------------------------------------------------------------- /docs/components/locks.md: -------------------------------------------------------------------------------- 1 | # Locks Component 2 | 3 | Locks are used internally to create regions of async code that will only be entered one at a time. Locks are necessary to coordinate multi-step changes to the level databases. 4 | 5 | Take care to coordinate the locks across the codebase. Some lock identifiers need to be reused in multiple code regions. Be careful not to use an identifier twice in a row, without first releasing, since that will stall the request. 6 | 7 | ## Usage 8 | 9 | ```js 10 | var lock = require('./lock') 11 | 12 | async function foo () { 13 | var release = await lock('bar') 14 | try { 15 | // do work 16 | } finally { 17 | release() 18 | } 19 | } 20 | ``` 21 | 22 | Be sure to always use a try/finally block. 23 | 24 | ## Locks in use 25 | 26 | - `users`. Must be used any time updates are made to the users DB. 27 | - `archives`. Must be used any time an update is made to the archives DB. 28 | - `archiver-job`. Used to make sure only one job runs at once (to avoid overloading the thread). -------------------------------------------------------------------------------- /lib/templates/mail/forgot-password.js: -------------------------------------------------------------------------------- 1 | exports.subject = function () { 2 | return 'Forgotten password reset' 3 | } 4 | 5 | exports.text = function (params) { 6 | return ` 7 | \n 8 | **Forgotten password reset for ${params.username}.**\n 9 | \n 10 | We received a request at ${params.hostname} to reset your password.\n 11 | \n 12 | If this was you, follow this link:\n 13 | \n 14 | ${params.forgotPasswordLink}\n 15 | \n 16 | If you did not request to reset your password, please ignore this email.\n 17 | \n 18 | \n 19 | ` 20 | } 21 | 22 | exports.html = function (params) { 23 | return ` 24 |

Forgotten password reset for ${params.username}.

25 |

We received a request at ${params.hostname} to reset your password. If this was you, follow this link:

26 |

${params.forgotPasswordLink}

27 |

If you did not request to reset your password, please ignore this email.

28 | ` 29 | } 30 | -------------------------------------------------------------------------------- /docs/flows/forgot-password.md: -------------------------------------------------------------------------------- 1 | # Forgot Password Flow 2 | 3 | ## Step 1. Trigger flow (POST /v1/forgot-password) 4 | 5 | User POSTS to `/v1/forgot-password` with body: 6 | 7 | ``` 8 | { 9 | email: String 10 | } 11 | ``` 12 | 13 | A random 32-byte email-verification nonce is created and saved the user record. The user record indicates: 14 | 15 | forgotPasswordNonce|passwordHash|passwordSalt 16 | --------------------------------------------- 17 | XXX|old|old 18 | 19 | Server sends an email to the user with the `forgotPasswordNonce`. 20 | 21 | Server responds 200 with JSON indicating to check email. 22 | 23 | ## Step 2. Update password (POST /v1/account/password) 24 | 25 | User POSTS `/v1/account/password` with body: 26 | 27 | ``` 28 | { 29 | username: String, username of the account 30 | nonce: String, verification nonce 31 | newPassword: String, new password 32 | } 33 | ``` 34 | 35 | Server updates user record to indicate: 36 | 37 | forgotPasswordNonce|passwordHash|passwordSalt 38 | --------------------------------------------- 39 | null|new|new -------------------------------------------------------------------------------- /docs/flows/registration.md: -------------------------------------------------------------------------------- 1 | # User Registration Flow 2 | 3 | ## Step 1. Register (POST /v1/register) 4 | 5 | User POSTS to `/v1/register` with body: 6 | 7 | ``` 8 | { 9 | email: String 10 | username: String 11 | password: String 12 | } 13 | ``` 14 | 15 | Server creates a new account for the user. A random 32-byte email-verification nonce is created. The user record indicates: 16 | 17 | scopes|isEmailVerified|emailVerifyNonce 18 | ------|---------------|---------------- 19 | none|false|XXX 20 | 21 | Server sends an email to the user with the `emailVerifyNonce`. 22 | 23 | Server responds 200 with HTML/JSON indicating to check email. 24 | 25 | ## Step 2. Verify (GET or POST /v1/verify) 26 | 27 | User GETS or POSTS `/v1/verify` with query-params or body: 28 | 29 | ``` 30 | { 31 | username: String, username of the account 32 | nonce: String, verification nonce 33 | } 34 | ``` 35 | 36 | Server updates user record to indicate: 37 | 38 | scopes|isEmailVerified|emailVerifyNonce 39 | ------|---------------|---------------- 40 | user|true|null 41 | 42 | Sever generates session JWT and responds 200 with auth=token. -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Docs 2 | 3 | - [Contributing Guidelines](../CONTRIBUTING.md) 4 | 5 | ### APIs 6 | 7 | - [Web API](./webapis.md). Complete description of all endpoints. 8 | 9 | ### Flows 10 | 11 | - [Registration Flow](./flows/registration.md). User-registration and verification. 12 | - [Forgot Password Flow](./flows/forgot-password.md). User password reset. 13 | - [Dat Ownership Proof Flow](./flows/dat-ownership-proof.md). How ownership of a dat by a specific user is verified. 14 | 15 | ### Components 16 | 17 | - [Jobs](./components/jobs.md). Behaviors that either get triggered by a message, or auto-triggered by the scheduler. 18 | - [Triggers](./components/triggers.md). Any file-indexing is handled by Triggers, which watch for changes to specific paths and archives, then queue jobs automatically when a change is detected. 19 | - [Locks](./components/locks.md). Locks are used internally to create regions of async code that will only be entered one at a time. 20 | 21 | ### Schemas 22 | 23 | - [Access Scopes](./schemas/access-scopes.md). The different permissions available to users. 24 | - [LevelDB](./schemas/leveldb.md). LevelDB layout and objects. 25 | - [Events](./schemas/events.md). Events emitted by various components. -------------------------------------------------------------------------------- /lib/crypto.js: -------------------------------------------------------------------------------- 1 | var promisify = require('es6-promisify') 2 | var { createHash, randomBytes } = require('crypto') 3 | 4 | // promisify some methods 5 | randomBytes = promisify(randomBytes) 6 | 7 | // exported api 8 | // = 9 | 10 | exports.randomBytes = randomBytes 11 | 12 | exports.shasum = shasum 13 | function shasum (buf) { 14 | if (typeof buf !== 'string' && !Buffer.isBuffer(buf)) { 15 | buf = JSON.stringify(buf) 16 | } 17 | return createHash('sha256') 18 | .update(buf, Buffer.isBuffer(buf) ? null : 'utf8') 19 | .digest('hex') 20 | } 21 | 22 | exports.hashPassword = async function (password) { 23 | // generate a new salt and hash the password with it 24 | var passwordSalt = await randomBytes(16) 25 | password = Buffer.from(password, 'utf8') 26 | return { 27 | passwordHash: shasum(Buffer.concat([passwordSalt, password])), 28 | passwordSalt: passwordSalt.toString('hex') 29 | } 30 | } 31 | 32 | exports.verifyPassword = function (password, userRecord) { 33 | // verify that the given password, when hashed, is the same as the password on record 34 | var passwordHash = shasum(Buffer.concat([ 35 | Buffer.from(userRecord.passwordSalt, 'hex'), 36 | Buffer.from(password, 'utf8') 37 | ])) 38 | return (passwordHash === userRecord.passwordHash) 39 | } 40 | -------------------------------------------------------------------------------- /docs/flows/dat-ownership-proof.md: -------------------------------------------------------------------------------- 1 | # Dat Ownership Proof Flow 2 | 3 | This describes a process for asserting ownership of a Dat by writing a pre-defined payload, then syncing the dat to hypercloud. 4 | 5 | > This spec was originally part of the registration flow. It's now being preserved, as a general-purpose flow, until we have a deployment plan for it. 6 | 7 | ## Step 1. Claim ownership (POST /v1/archives/claim) 8 | 9 | User POSTS `/v1/archives/claim` while authenticated with body (JSON): 10 | 11 | ``` 12 | { 13 | key: String, they key of the dat, or 14 | url: String, the url of the dat 15 | } 16 | ``` 17 | 18 | Server generates the `proof` (a non-expiring JWT) with the following content: 19 | 20 | ``` 21 | { 22 | id: String, id of the user 23 | url: String, the URL of the dat 24 | } 25 | ``` 26 | 27 | Server responds 200 with the body: 28 | 29 | ``` 30 | { 31 | proof: String, the encoded JWT 32 | hostname: String, the hostname of this service 33 | } 34 | ``` 35 | 36 | ## Step 2. Write proof 37 | 38 | User writes the `proof` to the `/proofs/:hostname` file of their profile dat. User then syncs the updated dat to the service. 39 | 40 | User GETS `/v1/archives/:key?view=proofs` periodically to watch for successful sync. 41 | 42 | ## Step 3. Validate claim 43 | 44 | Server receives proof-file in the dat. After checking the JWT signature, the server updates archive record to indicate the verified ownership. -------------------------------------------------------------------------------- /lib/mailer.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var nodemailer = require('nodemailer') 3 | var templates = { 4 | verification: require('./templates/mail/verification'), 5 | 'forgot-password': require('./templates/mail/forgot-password'), 6 | 'verify-update-email': require('./templates/mail/verify-update-email'), 7 | 'support': require('./templates/mail/support') 8 | } 9 | 10 | // exported api 11 | // = 12 | 13 | module.exports = class Mailer { 14 | constructor (config) { 15 | this.hostname = config.hostname 16 | this.brandname = config.brandname 17 | this.sender = config.email.sender 18 | this._mailer = nodemailer.createTransport(config.email) 19 | } 20 | 21 | async send (tmpl, params) { 22 | assert(params.email) 23 | params = Object.assign({}, params, this) 24 | tmpl = templates[tmpl] 25 | try { 26 | return await this._mailer.sendMail({ 27 | from: this.sender, 28 | to: params.email, 29 | subject: tmpl.subject(params), 30 | text: tmpl.text(params), 31 | html: tmpl.html(params) 32 | }) 33 | } catch (err) { 34 | this.logError(err, tmpl, params) 35 | throw err 36 | } 37 | } 38 | 39 | get transport () { 40 | return this._mailer.transporter 41 | } 42 | 43 | logError (err, tmpl, params) { 44 | console.error('[ERROR] Failed to send email', tmpl, 'To:', params.email, 'Error:', err) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lib/sessions.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var jwt = require('jsonwebtoken') 3 | 4 | // exported api 5 | // = 6 | 7 | module.exports = class Sessions { 8 | constructor (config) { 9 | this.options = config.sessions 10 | this.secret = config.sessions.secret 11 | delete this.options.secret 12 | assert(this.secret, 'config.sessions.secret is required') 13 | assert(this.options.algorithm, 'config.sessions.algorithm is required') 14 | } 15 | 16 | middleware () { 17 | return (req, res, next) => { 18 | // pull token out of auth or cookie header 19 | var authHeader = req.header('authorization') 20 | if (authHeader && authHeader.indexOf('Bearer') > -1) { 21 | res.locals.session = this.verify(authHeader.slice('Bearer '.length)) 22 | } else if (req.cookies && req.cookies.sess) { 23 | res.locals.session = this.verify(req.cookies.sess) 24 | } 25 | next() 26 | } 27 | } 28 | 29 | verify (token) { 30 | try { 31 | // return decoded session or null on failure 32 | return jwt.verify(token, this.secret, { algorithms: [this.options.algorithm] }) 33 | } catch (e) { 34 | return null 35 | } 36 | } 37 | 38 | generate (userRecord) { 39 | return jwt.sign( 40 | { 41 | id: userRecord.id, 42 | scopes: userRecord.scopes 43 | }, 44 | this.secret, 45 | this.options 46 | ) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /lib/const.js: -------------------------------------------------------------------------------- 1 | exports.DAT_KEY_REGEX = /([0-9a-f]{64})/i 2 | exports.DAT_URL_REGEX = /^(dat:\/\/[0-9a-f]{64})/i 3 | exports.DAT_NAME_REGEX = /^([0-9a-zA-Z-]*)$/i 4 | 5 | exports.NotFoundError = class NotFoundError extends Error { 6 | constructor (message) { 7 | super(message) 8 | this.name = 'NotFoundError' 9 | this.status = 404 10 | this.body = { 11 | message: message || 'Resource not found', 12 | notFound: true 13 | } 14 | } 15 | 16 | } 17 | 18 | exports.UnauthorizedError = class UnauthorizedError extends Error { 19 | constructor (message) { 20 | super(message) 21 | this.name = 'UnauthorizedError' 22 | this.status = 401 23 | this.body = { 24 | message: message || 'You must sign in to access this resource', 25 | notAuthorized: true 26 | } 27 | } 28 | } 29 | 30 | exports.ForbiddenError = class ForbiddenError extends Error { 31 | constructor (message) { 32 | super(message) 33 | this.name = 'ForbiddenError' 34 | this.status = 403 35 | this.body = { 36 | message: message || 'You dont have the rights to access this resource', 37 | forbidden: true 38 | } 39 | } 40 | } 41 | 42 | exports.NotImplementedError = class NotImplementedError extends Error { 43 | constructor (message) { 44 | super(message) 45 | this.name = 'NotImplementedError' 46 | this.status = 501 47 | this.body = { 48 | message: message || 'Resources not yet implemented', 49 | notImplemented: true 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /test/lib/dat.js: -------------------------------------------------------------------------------- 1 | const Dat = require('dat-node') 2 | const util = require('./util') 3 | 4 | exports.makeDatFromFolder = function (dir, cb) { 5 | Dat(dir, (err, dat) => { 6 | if (err) return cb(err) 7 | 8 | dat.importFiles(() => { 9 | dat.joinNetwork() 10 | 11 | var key = dat.key.toString('hex') 12 | console.log('created dat', key, 'from', dir) 13 | cb(null, dat, key) 14 | }) 15 | }) 16 | } 17 | 18 | exports.downloadDatFromSwarm = function (key, { timeout = 5e3 }, cb) { 19 | var dir = util.mktmpdir() 20 | Dat(dir, {key}, (err, dat) => { 21 | if (err) return cb(err) 22 | 23 | dat.joinNetwork() 24 | dat.network.once('connection', (...args) => { 25 | console.log('got connection') 26 | }) 27 | 28 | dat.archive.metadata.on('download', (index, block) => { 29 | console.log('meta download event', index) 30 | }) 31 | 32 | var to = setTimeout(() => cb(new Error('timed out waiting for download')), timeout) 33 | dat.archive.metadata.on('sync', () => { 34 | console.log('meta download finished') 35 | }) 36 | dat.archive.once('content', () => { 37 | console.log('opened') 38 | dat.archive.content.on('download', (index, block) => { 39 | console.log('content download event', index) 40 | }) 41 | dat.archive.content.on('sync', () => { 42 | console.log('content download finished') 43 | clearTimeout(to) 44 | dat.close() 45 | cb(null, dat, key) 46 | }) 47 | }) 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We <3 PRs. 4 | 5 | Follow this guide for making changes, and then adding yourself to the in-app contributors page. 6 | 7 | ## Making Changes 8 | 9 | * Create a topic branch from where you want to base your work. 10 | * This is usually the master branch. 11 | * Make commits of logical units. 12 | * Make sure your commit messages are in the proper format. If appropriate, [use an extended commit to describe the changes involved.](https://git-scm.com/book/ch5-2.html) 13 | 14 | ```` 15 | The short-line description is capitalized at front, and <50 chars. 16 | 17 | If you feel you need to write more about your commit, do so here. This can 18 | help future developers understand the logic of the changes you made, and 19 | sometimes that future developer is you! 20 | ```` 21 | 22 | * Make sure you have added the necessary tests for your changes. 23 | * Run _all_ the tests to assure nothing else was accidentally broken. 24 | * Update the documentation. Add new documentation files as needed. 25 | 26 | ## Common reasons a pull-request will not be accepted 27 | 28 | * The changes need to have tests added. 29 | * The changes need to be documented. 30 | * The changes don't pass the `standard` formatting test. 31 | 32 | Make sure you update tests and docs! 33 | 34 | ## Adding yourself to the Contributors page 35 | 36 | After your first successful PR, you should create a second PR to add yourself to the contributors.yml doc. 37 | 38 | Open `./contributors.yml` and add to the bottom a line that follows this format: 39 | 40 | ```yaml 41 | --- 42 | name: Bob Robertson 43 | catchphrase: That's-a spicey meatball! 44 | website: https://bobs-homepage.com 45 | ``` 46 | 47 | Of course, you'll want to put your own name, catchphrase, and website. -------------------------------------------------------------------------------- /lib/apis/service.js: -------------------------------------------------------------------------------- 1 | var {NotImplementedError} = require('../const') 2 | 3 | // exported api 4 | // = 5 | 6 | module.exports = class ServicesAPI { 7 | constructor (cloud) { 8 | this.config = cloud.config 9 | this.usersDB = cloud.usersDB 10 | this.activityDB = cloud.activityDB 11 | this.archivesDB = cloud.archivesDB 12 | } 13 | 14 | async frontpage (req, res, next) { 15 | var contentType = req.accepts(['html', 'json']) 16 | if (contentType === 'json') throw new NotImplementedError() 17 | next() 18 | } 19 | 20 | async explore (req, res, next) { 21 | if (req.query.view === 'activity') { 22 | return res.json({ 23 | activity: await this.activityDB.listGlobalEvents({ 24 | limit: 25, 25 | lt: req.query.start, 26 | reverse: true 27 | }) 28 | }) 29 | } 30 | if (req.query.view === 'popular') { 31 | return res.json({ 32 | popular: (await this.archivesDB.list({ 33 | sort: 'popular', 34 | limit: 25, 35 | cursor: req.query.start 36 | })).map(mapArchiveObject) 37 | }) 38 | } 39 | if (req.query.view === 'recent') { 40 | return res.json({ 41 | recent: (await this.archivesDB.list({ 42 | sort: 'createdAt', 43 | limit: 25, 44 | cursor: req.query.start 45 | })).map(mapArchiveObject) 46 | }) 47 | } 48 | next() 49 | } 50 | } 51 | 52 | function mapArchiveObject (archive) { 53 | return { 54 | key: archive.key, 55 | numPeers: archive.numPeers, 56 | name: archive.name, 57 | title: archive.manifest ? archive.manifest.title : null, 58 | description: archive.manifest ? archive.manifest.description : null, 59 | owner: archive.owner ? archive.owner.username : null, 60 | createdAt: archive.createdAt 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /lib/helpers.js: -------------------------------------------------------------------------------- 1 | var promisify = require('es6-promisify') 2 | var through2 = require('through2') 3 | var identifyFiletype = require('identify-filetype') 4 | var mime = require('mime') 5 | 6 | // config default mimetype 7 | mime.default_type = 'text/plain' 8 | 9 | exports.promisify = promisify 10 | exports.promisifyModule = function (module, methods) { 11 | for (var m of methods) { 12 | module[m] = promisify(module[m], module) 13 | } 14 | } 15 | 16 | exports.pluralize = function (num, base, suffix = 's') { 17 | if (num === 1) { 18 | return base 19 | } 20 | return base + suffix 21 | } 22 | 23 | exports.makeSafe = function (str) { 24 | return str.replace(//g, '>').replace(/&/g, '&').replace(/"/g, '') 25 | } 26 | 27 | var identify = 28 | exports.identify = function (name, chunk) { 29 | // try to identify the type by the chunk contents 30 | var mimeType 31 | var identifiedExt = (chunk) ? identifyFiletype(chunk) : false 32 | if (identifiedExt) { 33 | mimeType = mime.lookup(identifiedExt) 34 | } 35 | if (!mimeType) { 36 | // fallback to using the entry name 37 | mimeType = mime.lookup(name) 38 | } 39 | 40 | // hackish fix 41 | // the svg test can be a bit aggressive: html pages with 42 | // inline svgs can be falsely interpretted as svgs 43 | // double check that 44 | if (identifiedExt === 'svg' && mime.lookup(name) === 'text/html') { 45 | return 'text/html' 46 | } 47 | 48 | return mimeType 49 | } 50 | 51 | exports.identifyStream = function (name, cb) { 52 | var first = true 53 | return through2(function (chunk, enc, cb2) { 54 | if (first) { 55 | first = false 56 | cb(identify(name, chunk)) 57 | } 58 | this.push(chunk) 59 | cb2() 60 | }) 61 | } 62 | 63 | exports.wait = function (ms, value) { 64 | return new Promise(resolve => { 65 | setTimeout(() => resolve(value), ms) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /test/lib/server.js: -------------------------------------------------------------------------------- 1 | const request = require('request-promise-native') 2 | const createApp = require('../../index.js') 3 | const util = require('./util') 4 | 5 | var portCounter = 10000 6 | 7 | module.exports = function (cb) { 8 | if (process.env.REMOTE_URL) { 9 | return createRemoteApp(cb) 10 | } else { 11 | return createLocalApp(cb) 12 | } 13 | } 14 | 15 | function createRemoteApp (cb) { 16 | var url = process.env.REMOTE_URL 17 | console.log(`connecting to ${url}`) 18 | var app = { 19 | url, 20 | isRemote: true, 21 | req: request.defaults({ baseUrl: url, timeout: 10e3, resolveWithFullResponse: true, simple: false }), 22 | close: cb => cb() 23 | } 24 | cb() 25 | return app 26 | } 27 | 28 | function createLocalApp (cb) { 29 | // setup config 30 | // = 31 | 32 | var tmpdir = util.mktmpdir() 33 | var config = { 34 | hostname: 'test.local', 35 | dir: tmpdir, 36 | port: portCounter++, 37 | defaultDiskUsageLimit: '100mb', 38 | admin: { 39 | password: 'foobar' 40 | }, 41 | jobs: { 42 | popularArchivesIndex: '15m', 43 | userDiskUsage: '30m', 44 | deleteDeadArchives: '30m' 45 | }, 46 | registration: { 47 | open: true, 48 | reservedNames: ['reserved', 'blacklisted'] 49 | }, 50 | email: { 51 | transport: 'mock', 52 | sender: '"Test Server" ' 53 | }, 54 | sessions: { 55 | algorithm: 'HS256', 56 | secret: 'super secret', 57 | expiresIn: '1h' 58 | }, 59 | proofs: { 60 | algorithm: 'HS256', 61 | secret: 'super secret 2' 62 | } 63 | } 64 | 65 | // create server 66 | // = 67 | 68 | var app = createApp(config) 69 | var server = app.listen(config.port, (err) => { 70 | console.log(`server started on http://127.0.0.1:${config.port}`) 71 | app.cloud.whenAdminCreated(() => cb(err)) 72 | }) 73 | 74 | app.isRemote = false 75 | app.url = `http://127.0.0.1:${config.port}` 76 | app.req = request.defaults({ 77 | baseUrl: app.url, 78 | resolveWithFullResponse: true, 79 | simple: false 80 | }) 81 | 82 | // wrap app.close to stop the server 83 | var orgClose = app.close 84 | app.close = cb => orgClose.call(app, () => server.close(cb)) 85 | 86 | return app 87 | } 88 | -------------------------------------------------------------------------------- /docs/schemas/leveldb.md: -------------------------------------------------------------------------------- 1 | # Level Database Schema 2 | 3 | Layout and schemas of the data in the LevelDB. 4 | 5 | ## Layout 6 | 7 | - `main` 8 | - `archives`: Map of `key => Archive object`. 9 | - `archives-index`: Index of `createdAt => key` 10 | - `accounts`: Map of `id => Account object`. 11 | - `accounts-index`: Index of `username => id`, `email => id`, `profileUrl => id`. 12 | - `global-activity`: Map of `timestamp => Event object`. 13 | - `global-activity-users-index`: Set of `username:timestamp => null` for doing user filtering. 14 | - `dead-archives`: Map of `key => undefined`. A listing of archives with no hosting users, and which need to be deleted. 15 | 16 | ## Archive object 17 | 18 | Schema: 19 | 20 | ``` 21 | { 22 | key: String, the archive key 23 | 24 | hostingUsers: Array(String), list of user-ids hosting the archive 25 | 26 | updatedAt: Number, the timestamp of the last update 27 | createdAt: Number, the timestamp of creation time 28 | } 29 | ``` 30 | 31 | ## Account object 32 | 33 | Schema: 34 | 35 | ``` 36 | { 37 | id: String, the assigned id 38 | username: String, the chosen username 39 | passwordHash: String, hashed password 40 | passwordSalt: String, salt used on hashed password 41 | 42 | email: String 43 | pendingEmail: String, the user's new email address pending verification 44 | profileURL: String, the url of the profile dat 45 | archives: [{ 46 | key: String, uploaded archive's key 47 | name: String, optional shortname for the archive 48 | }, ..] 49 | scopes: Array(String), the user's access scopes 50 | suspension: String, if suspended, will be set to an explanation 51 | updatedAt: Number, the timestamp of the last update 52 | createdAt: Number, the timestamp of creation time 53 | 54 | isEmailVerified: Boolean 55 | emailVerifyNonce: String, the random verification nonce (register flow) 56 | 57 | forgotPasswordNonce: String, the random verification nonce (forgot password flow) 58 | 59 | isProfileDatVerified: Boolean 60 | profileVerifyToken: String, the profile verification token (stored so the user can refetch it) 61 | } 62 | ``` 63 | 64 | ## Event object 65 | 66 | Schema: 67 | 68 | ``` 69 | { 70 | ts: Number, the timestamp of the event 71 | userid: String, the user who made the change 72 | username: String, the name of the user who made the change 73 | action: String, the label for the action 74 | params: Object, a set of arbitrary KVs relevant to the action 75 | } 76 | ``` -------------------------------------------------------------------------------- /docs/components/jobs.md: -------------------------------------------------------------------------------- 1 | # Jobs Component 2 | 3 | Jobs are behaviors that either get triggered by a message, or auto-triggered by the scheduler. 4 | 5 | ## Jobs API 6 | 7 | The jobs manager is an event broker. When it comes time to scale horizontally, it will be internally rewritten to use RabbitMQ. 8 | 9 | ```js 10 | jobs.queue(name[, data]) // add a one-time job 11 | jobs.requeue(job) // remove, then re-add the job to the queue 12 | jobs.markDone(job) // remove the job from the queue 13 | jobs.addHandler(name, job => ...) // add a handler for the job 14 | jobs.removeHandler(handlerId) // remove a handler 15 | ``` 16 | 17 | Example of setting up jobs: 18 | 19 | ```js 20 | jobs.queue('verify-profile-dat', { userId: '...', url: '...' }) 21 | jobs.queue('clean-unverified-users') 22 | ``` 23 | 24 | Example of handling jobs: 25 | 26 | ```js 27 | var { hostname } = config 28 | jobs.addHandler('verify-profile-dat', job => { 29 | var { userId, url } = job.data 30 | readDatFile(`${url}/proofs/${hostname}`, (err, data) => { 31 | // ... 32 | jobs.markDone(job) 33 | }) 34 | }) 35 | ``` 36 | 37 | ## Scheduler API 38 | 39 | The scheduler adds cron-style timers/intervals to queue jobs at certain times. 40 | 41 | ```js 42 | scheduler.add(name, when[, data]) // schedule a job (cron syntax) 43 | scheduler.list([name]) // list active jobs 44 | scheduler.remove(scheduleId) // remove a scheduled job 45 | ``` 46 | 47 | Example of scheduling jobs: 48 | 49 | ```js 50 | // should be run during app startup 51 | scheduler.add('clean-unverified-users', '0 0 0 * * *') // run at midnight every day 52 | ``` 53 | 54 | ## Jobs 55 | 56 | ### Verify Profile Dat 57 | 58 | - Name: `verify-profile-dat' 59 | - Task: Read the proof file in the profile, verify the proof, and update the user record. 60 | - Data: 61 | - `userId`: ID of the account that is attached to the profile 62 | - `url`: URL of the profile-dat 63 | - Preconditions: 64 | - User account should have its email verified 65 | - Profile-dat and the proof-file should be locally available 66 | 67 | ### Dead Archive Cleanup 68 | 69 | - Name: `clean-dead-archives` 70 | - Task: Deletes any archives referenced in [`dead-archives`](https://github.com/joehand/hypercloud/wiki/Archives-Schema#layout) (no hosting users). 71 | 72 | ### Unverified User Cleanup 73 | 74 | - Name: `clean-unverified-users` 75 | - Task: Deletes any user records older than a day with `isEmailVerified==false` -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hypercloud", 3 | "version": "0.0.3", 4 | "description": "a p2p ☁", 5 | "main": "index.js", 6 | "scripts": { 7 | "install-transpiler": "npm install babel-preset-es2015", 8 | "start": "node bin.js", 9 | "test": "ava test/*.js && standard" 10 | }, 11 | "keywords": [ 12 | "hyperdrive", 13 | "dat", 14 | "data", 15 | "cloud", 16 | "p2p" 17 | ], 18 | "ava": { 19 | "serial": true, 20 | "require": [ 21 | "./nodecompat.js" 22 | ] 23 | }, 24 | "author": "Joe Hand (https://joeahand.com/)", 25 | "license": "MIT", 26 | "dependencies": { 27 | "await-lock": "^1.1.2", 28 | "babel-plugin-transform-async-to-generator": "^6.16.0", 29 | "babel-register": "^6.18.0", 30 | "body-parser": "^1.17.2", 31 | "bytes": "^2.5.0", 32 | "co-express": "^1.2.2", 33 | "cookie-parser": "^1.4.3", 34 | "dat-encoding": "^4.0.2", 35 | "datland-swarm-defaults": "^1.0.2", 36 | "debug": "^2.6.8", 37 | "deep-extend": "^0.4.2", 38 | "es6-promisify": "^5.0.0", 39 | "express": "^4.15.3", 40 | "express-rate-limit": "^2.8.0", 41 | "express-server-sent-events": "^1.1.0", 42 | "express-validator": "^3.1.2", 43 | "get-folder-size": "^1.0.0", 44 | "greenlock-express": "^2.0.11", 45 | "hypercloud-ui-vanilla": "^1.0.0", 46 | "hyperdrive": "^9.3.0", 47 | "identify-filetype": "^1.0.0", 48 | "js-yaml": "^3.8.4", 49 | "jsonwebtoken": "^7.4.1", 50 | "level": "^1.7.0", 51 | "level-promise": "^2.1.1", 52 | "level-simple-indexes": "^2.2.0", 53 | "lru": "^3.1.0", 54 | "mime": "^1.3.6", 55 | "mkdirp": "^0.5.1", 56 | "monotonic-timestamp": "0.0.9", 57 | "monotonic-timestamp-base36": "^1.0.0", 58 | "ms": "^1.0.0", 59 | "nicedate": "^1.0.0", 60 | "nodemailer": "^2.7.0", 61 | "nodemailer-ses-transport": "^1.5.0", 62 | "nodemailer-stub-transport": "^1.1.0", 63 | "pauls-dat-api": "^4.2.1", 64 | "pmx": "^1.2.0", 65 | "pretty-bytes": "^4.0.2", 66 | "range-parser": "^1.2.0", 67 | "request": "^2.79.0", 68 | "request-promise-native": "^1.0.4", 69 | "rimraf": "^2.6.1", 70 | "stream-collector": "^1.0.1", 71 | "subleveldown": "^2.1.0", 72 | "through2": "^2.0.3", 73 | "uuid": "^3.0.1", 74 | "vhost": "^3.0.2" 75 | }, 76 | "devDependencies": { 77 | "ava": "^0.17.0", 78 | "dat-node": "^3.3.2", 79 | "memdb": "^1.3.1", 80 | "nodemailer-mock-transport": "^1.3.0", 81 | "standard": "^8.6.0" 82 | }, 83 | "repository": { 84 | "type": "git", 85 | "url": "git+https://github.com/datprotocol/hypercloud.git" 86 | }, 87 | "bugs": { 88 | "url": "https://github.com/datprotocol/hypercloud/issues" 89 | }, 90 | "homepage": "https://github.com/datprotocol/hypercloud#readme" 91 | } 92 | -------------------------------------------------------------------------------- /lib/dbs/activity.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var levelPromise = require('level-promise') 3 | var sublevel = require('subleveldown') 4 | var collect = require('stream-collector') 5 | var mtb36 = require('monotonic-timestamp-base36') 6 | var through2 = require('through2') 7 | 8 | // constants 9 | // = 10 | 11 | // used in the users-index-db 12 | const SEPARATOR = '!' 13 | const USERKEY = (key, username) => `${username}${SEPARATOR}${key}` 14 | 15 | // valid actions 16 | const ACTIONS = [ 17 | 'add-archive', 18 | 'del-archive' 19 | ] 20 | 21 | // exported api 22 | // = 23 | 24 | class ActivityDB { 25 | constructor (cloud) { 26 | // create levels 27 | this.globalActivityDB = sublevel(cloud.db, 'global-activity', { valueEncoding: 'json' }) 28 | this.usersIndexDB = sublevel(cloud.db, 'global-activity-users-index') 29 | 30 | // promisify 31 | levelPromise.install(this.globalActivityDB) 32 | } 33 | 34 | // basic ops 35 | // = 36 | 37 | async writeGlobalEvent (record) { 38 | assert(record && typeof record === 'object') 39 | assert(typeof record.userid === 'string', 'Valid userid type') 40 | assert(typeof record.username === 'string', 'Valid username type') 41 | assert(ACTIONS.includes(record.action), 'Valid action type') 42 | record = Object.assign({}, ActivityDB.defaults(), record) 43 | record.ts = Date.now() 44 | var key = mtb36() 45 | await Promise.all([ 46 | this.globalActivityDB.put(key, record), 47 | this.usersIndexDB.put(USERKEY(key, record.username), null) 48 | ]) 49 | return record 50 | } 51 | 52 | async delGlobalEvent (key) { 53 | assert(typeof key === 'string') 54 | var record = await this.globalActivityDB.get(key) 55 | await Promise.all([ 56 | this.globalActivityDB.del(key), 57 | this.usersIndexDB.del(USERKEY(key, record.username)) 58 | ]) 59 | } 60 | 61 | // getters 62 | // = 63 | 64 | listGlobalEvents (opts) { 65 | return new Promise((resolve, reject) => { 66 | collect(this.globalActivityDB.createReadStream(opts), (err, res) => { 67 | if (err) reject(err) 68 | else resolve(res.map(toNiceObj)) 69 | }) 70 | }) 71 | } 72 | 73 | listUserEvents (username, opts = {}) { 74 | return new Promise((resolve, reject) => { 75 | // update the start/end 76 | if (opts.lt) opts.lt = USERKEY(opts.lt, username) 77 | if (opts.gt) opts.gt = USERKEY(opts.gt, username) 78 | if (opts.lte) opts.lte = USERKEY(opts.lte, username) 79 | if (opts.gte) opts.gte = USERKEY(opts.gte, username) 80 | 81 | // set range edges 82 | if (!opts.lt && !opts.lte) { 83 | opts.lte = USERKEY('\xff', username) 84 | } 85 | if (!opts.gt && !opts.gte) { 86 | opts.gt = USERKEY('', username) 87 | } 88 | 89 | // fetch the index range 90 | var self = this 91 | var stream = this.usersIndexDB.createReadStream(opts) 92 | .pipe(through2.obj(function (entry, enc, cb) { 93 | // load the record 94 | var [username, key] = entry.key.split(SEPARATOR) 95 | self.globalActivityDB.get(key, (err, value) => { 96 | if (value) { 97 | value.key = key 98 | this.push(value) 99 | } 100 | cb() 101 | }) 102 | })) 103 | collect(stream, (err, res) => { 104 | if (err) reject(err) 105 | else resolve(res) // no need to use toNiceObj 106 | }) 107 | }) 108 | } 109 | } 110 | module.exports = ActivityDB 111 | 112 | // default user-record values 113 | ActivityDB.defaults = () => ({ 114 | ts: null, 115 | userid: null, 116 | username: null, 117 | action: null, 118 | params: {} 119 | }) 120 | 121 | // helper to convert {key:, value:} to just {values...} 122 | function toNiceObj (obj) { 123 | obj.value.key = obj.key 124 | return obj.value 125 | } 126 | -------------------------------------------------------------------------------- /lib/templates/directory-listing-page.js: -------------------------------------------------------------------------------- 1 | const prettyBytes = require('pretty-bytes') 2 | const path = require('path') 3 | const {pluralize, makeSafe} = require('../helpers') 4 | const {stat, readdir} = require('pauls-dat-api') 5 | 6 | const styles = `` 22 | 23 | module.exports = async function renderDirectoryListingPage (archive, dirPath) { 24 | // list files 25 | var names = [] 26 | try { names = await readdir(archive, dirPath) } catch (e) {} 27 | 28 | // stat each file 29 | var entries = await Promise.all(names.map(async (name) => { 30 | var entry 31 | var entryPath = path.join(dirPath, name) 32 | try { entry = await stat(archive, entryPath) } catch (e) { return false } 33 | entry.path = entryPath 34 | entry.name = name 35 | return entry 36 | })) 37 | entries = entries.filter(Boolean) 38 | 39 | // sort the listing 40 | entries.sort((a, b) => { 41 | // directories on top 42 | if (a.isDirectory() && !b.isDirectory()) return -1 43 | if (!a.isDirectory() && b.isDirectory()) return 1 44 | // alphabetical after that 45 | return a.name.localeCompare(b.name) 46 | }) 47 | 48 | // show the updog if path is not top 49 | var updog = '' 50 | if (dirPath !== '/' && dirPath !== '') { 51 | updog = `
..
` 52 | } 53 | 54 | // render entries 55 | var totalFiles = 0 56 | entries = entries.map(entry => { 57 | totalFiles++ 58 | var url = makeSafe(entry.path) 59 | if (!url.startsWith('/')) url = '/' + url // all urls should have a leading slash 60 | if (entry.isDirectory() && !url.endsWith('/')) url += '/' // all dirs should have a trailing slash 61 | var type = entry.isDirectory() ? 'directory' : 'file' 62 | return `
${makeSafe(entry.name)}
` 63 | }).join('') 64 | 65 | // render summary 66 | var summary = `
${totalFiles} ${pluralize(totalFiles, 'file')}
` 67 | 68 | // render final 69 | return '' + styles + updog + entries + summary 70 | } 71 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | var level = require('level') 2 | var assert = require('assert') 3 | var path = require('path') 4 | var fs = require('fs') 5 | var wrap = require('co-express') 6 | 7 | var {hashPassword} = require('./crypto') 8 | var Sessions = require('./sessions') 9 | var Mailer = require('./mailer') 10 | var lock = require('./lock') 11 | var Archiver = require('./archiver') 12 | var UsersAPI = require('./apis/users') 13 | var ArchivesAPI = require('./apis/archives') 14 | var ArchiveFilesAPI = require('./apis/archive-files') 15 | var ServiceAPI = require('./apis/service') 16 | var AdminAPI = require('./apis/admin') 17 | var UsersDB = require('./dbs/users') 18 | var ArchivesDB = require('./dbs/archives') 19 | var ActivityDB = require('./dbs/activity') 20 | 21 | class Hypercloud { 22 | constructor (config) { 23 | assert(config, 'hypercloud requires options') 24 | assert(config.hostname, 'config.hostname is required') 25 | assert(config.dir || config.db, 'hypercloud requires a dir or db option') 26 | 27 | // fallback config 28 | config.env = config.env || 'development' 29 | 30 | // setup config 31 | var {dir, db} = config 32 | if (dir) { 33 | // ensure the target dir exists 34 | console.log('Data directory:', dir) 35 | try { 36 | fs.accessSync(dir, fs.F_OK) 37 | } catch (e) { 38 | fs.mkdirSync(dir) 39 | } 40 | } 41 | if (!db && dir) { 42 | // allocate a leveldb 43 | db = level(path.join(dir, 'db'), { valueEncoding: 'json' }) 44 | } 45 | assert(db, 'database was not created') 46 | this.config = config 47 | this.db = db 48 | 49 | // state guards 50 | var adminCreatedPromise = new Promise(resolve => { 51 | this._adminCreated = resolve 52 | }) 53 | this.whenAdminCreated = adminCreatedPromise.then.bind(adminCreatedPromise) 54 | 55 | // init components 56 | this.lock = lock 57 | this.sessions = new Sessions(config) 58 | // this.proofs = new Proofs(config) TODO 59 | this.mailer = new Mailer(config) 60 | this.archiver = new Archiver(this) 61 | this.usersDB = new UsersDB(this) 62 | this.archivesDB = new ArchivesDB(this) 63 | this.activityDB = new ActivityDB(this) 64 | 65 | // init apis 66 | this.api = { 67 | users: new UsersAPI(this), 68 | archives: new ArchivesAPI(this), 69 | archiveFiles: new ArchiveFilesAPI(this), 70 | service: new ServiceAPI(this), 71 | admin: new AdminAPI(this) 72 | } 73 | 74 | // wrap all APIs in co-express handling 75 | wrapAll(this.api.users) 76 | wrapAll(this.api.archives) 77 | wrapAll(this.api.archiveFiles) 78 | wrapAll(this.api.service) 79 | wrapAll(this.api.admin) 80 | 81 | // load all archives 82 | var ps = [] 83 | this.archivesDB.archivesDB.createKeyStream().on('data', key => { 84 | ps.push(this.archiver.loadArchive(key).then(null, err => null)) 85 | }).on('end', async () => { 86 | await Promise.all(ps) 87 | // compute user disk usage and swarm archives accordingly 88 | this.archiver.computeUserDiskUsageAndSwarm() 89 | // create the popular-archives index 90 | this.archiver.computePopularIndex() 91 | }) 92 | } 93 | 94 | async setupAdminUser () { 95 | try { 96 | // is the admin-user config wellformed? 97 | var adminConfig = this.config.admin 98 | if (!adminConfig || !adminConfig.password) { 99 | console.log('Admin user not created: must set password in config') 100 | return this._adminCreated(false) // abort if not 101 | } 102 | 103 | // upsert the admin user with these creds 104 | var method = 'put' 105 | let {passwordHash, passwordSalt} = await hashPassword(adminConfig.password) 106 | var adminRecord = await this.usersDB.getByUsername('admin') 107 | if (!adminRecord) { 108 | method = 'create' 109 | adminRecord = { 110 | username: 'admin', 111 | scopes: ['user', 'admin:dats', 'admin:users'], 112 | isEmailVerified: true 113 | } 114 | } 115 | adminRecord.passwordHash = passwordHash 116 | adminRecord.passwordSalt = passwordSalt 117 | if (adminConfig.email) adminRecord.email = adminConfig.email 118 | await this.usersDB[method](adminRecord) 119 | console.log((method === 'create' ? 'Created' : 'Updated'), 'admin record') 120 | this._adminCreated(true) 121 | } catch (e) { 122 | console.error('[ERROR] While trying to create admin user:', e) 123 | this._adminCreated(false) 124 | } 125 | } 126 | 127 | async close (cb) { 128 | await this.archiver.closeAllArchives() 129 | cb() 130 | } 131 | } 132 | 133 | module.exports = Hypercloud 134 | 135 | function wrapAll (api) { 136 | for (let methodName of Object.getOwnPropertyNames(Object.getPrototypeOf(api))) { 137 | let method = api[methodName] 138 | if (typeof method === 'function' && methodName.charAt(0) !== '_') { 139 | api[methodName] = wrap(method.bind(api)) 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /test/quotas.js: -------------------------------------------------------------------------------- 1 | var test = require('ava') 2 | var path = require('path') 3 | var createTestServer = require('./lib/server.js') 4 | var { makeDatFromFolder } = require('./lib/dat.js') 5 | 6 | var app 7 | var sessionToken, auth, authUser 8 | var testDat, testDatKey 9 | 10 | test.cb('start test server', t => { 11 | app = createTestServer(async err => { 12 | t.ifError(err) 13 | 14 | // login 15 | var res = await app.req.post({ 16 | uri: '/v1/login', 17 | json: { 18 | 'username': 'admin', 19 | 'password': 'foobar' 20 | } 21 | }) 22 | if (res.statusCode !== 200) throw new Error('Failed to login as admin') 23 | sessionToken = res.body.sessionToken 24 | auth = { bearer: sessionToken } 25 | 26 | t.end() 27 | }) 28 | }) 29 | 30 | test('register and login bob', async t => { 31 | // register bob 32 | var res = await app.req.post({ 33 | uri: '/v1/register', 34 | json: { 35 | email: 'bob@example.com', 36 | username: 'bob', 37 | password: 'foobar' 38 | } 39 | }) 40 | if (res.statusCode !== 201) throw new Error('Failed to register bob user') 41 | 42 | // check sent mail and extract the verification nonce 43 | var lastMail = app.cloud.mailer.transport.sentMail.pop() 44 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 45 | 46 | // verify via GET 47 | res = await app.req.get({ 48 | uri: '/v1/verify', 49 | qs: { 50 | username: 'bob', 51 | nonce: emailVerificationNonce 52 | }, 53 | json: true 54 | }) 55 | if (res.statusCode !== 200) throw new Error('Failed to verify bob user') 56 | 57 | // login bob 58 | res = await app.req.post({ 59 | uri: '/v1/login', 60 | json: { 61 | 'username': 'bob', 62 | 'password': 'foobar' 63 | } 64 | }) 65 | if (res.statusCode !== 200) throw new Error('Failed to login as bob') 66 | sessionToken = res.body.sessionToken 67 | authUser = { bearer: sessionToken } 68 | }) 69 | 70 | test('set bobs quota to something really small', async t => { 71 | var res = await app.req.post({ 72 | uri: '/v1/admin/users/bob', 73 | json: {diskQuota: '5b'}, 74 | auth 75 | }) 76 | t.is(res.statusCode, 200, '200 updated') 77 | }) 78 | 79 | test.cb('share test-dat 1', t => { 80 | makeDatFromFolder(path.join(__dirname, '/scaffold/testdat1'), (err, d, dkey) => { 81 | t.ifError(err) 82 | testDat = d 83 | testDatKey = dkey 84 | t.end() 85 | }) 86 | }) 87 | 88 | test('user disk usage is zero', async t => { 89 | var res = await app.req.get({url: '/v1/admin/users/bob', json: true, auth}) 90 | t.is(res.statusCode, 200, '200 got user data') 91 | t.deepEqual(res.body.diskUsage, 0, 'disk usage is zero') 92 | }) 93 | 94 | test('add archive', async t => { 95 | var json = {key: testDatKey} 96 | var res = await app.req.post({uri: '/v1/archives/add', json, auth: authUser}) 97 | t.is(res.statusCode, 200, '200 added dat') 98 | }) 99 | 100 | test.cb('check archive status and wait till synced', t => { 101 | var to = setTimeout(() => { 102 | throw new Error('Archive did not sync') 103 | }, 15e3) 104 | 105 | checkStatus() 106 | async function checkStatus () { 107 | var res = await app.req({uri: `/v1/archives/${testDatKey}`, qs: {view: 'status'}, json: true, auth}) 108 | var progress = res.body && res.body.progress ? res.body.progress : 0 109 | if (progress === 1) { 110 | clearTimeout(to) 111 | console.log('synced!') 112 | t.end() 113 | } else { 114 | console.log('progress', progress * 100, '%') 115 | setTimeout(checkStatus, 300) 116 | } 117 | } 118 | }) 119 | 120 | test('archive is still downloading', async t => { 121 | // check archive record 122 | var res = await app.req.get({url: `/v1/admin/archives/${testDatKey}`, json: true, auth}) 123 | t.is(res.statusCode, 200, '200 got archive data') 124 | t.deepEqual(res.body.swarmOpts, {upload: true, download: true}, 'is still downloading') 125 | }) 126 | 127 | test('compute disk usage', async t => { 128 | // run usage-compute job 129 | await app.cloud.archiver.computeUserDiskUsageAndSwarm() 130 | }) 131 | 132 | test('user disk usage now exceeds the disk quota', async t => { 133 | // check user record 134 | var res = await app.req.get({url: '/v1/admin/users/bob', json: true, auth}) 135 | t.is(res.statusCode, 200, '200 got user data') 136 | t.truthy(res.body.diskUsage > res.body.diskQuota, 'disk quota is exceeded') 137 | 138 | // check archive record 139 | var res = await app.req.get({url: `/v1/admin/archives/${testDatKey}`, json: true, auth}) 140 | t.is(res.statusCode, 200, '200 got archive data') 141 | t.deepEqual(res.body.swarmOpts, {upload: true, download: false}, 'no longer downloading') 142 | t.truthy(res.body.diskUsage > 0, 'response has disk usage') 143 | }) 144 | 145 | test('add archive now fails', async t => { 146 | var json = {key: 'f'.repeat(64)} 147 | var res = await app.req.post({uri: '/v1/archives/add', json, auth: authUser}) 148 | t.is(res.statusCode, 422, '422 denied') 149 | t.truthy(res.body.outOfSpace, 'disk quota is exceeded') 150 | }) 151 | 152 | test.cb('stop test server', t => { 153 | app.close(() => { 154 | testDat.close(() => { 155 | t.pass('closed') 156 | t.end() 157 | }) 158 | }) 159 | }) 160 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | [![deprecated](http://badges.github.io/stability-badges/dist/deprecated.svg)](github.com/beakerbrowser/hashbase) See [hashbase](github.com/beakerbrowser/hashbase) for similar functionality. 2 | 3 | More info on active projects and modules at [dat-ecosystem.org](https://dat-ecosystem.org/) 4 | 5 | --- 6 | 7 | # Hypercloud ☁ 8 | 9 | Hypercloud is a public peer service for [Dat](https://datproject.org) archives. It provides a HTTP-accessible interface for creating an account and uploading Dats. 10 | 11 | Features: 12 | 13 | - Simple Dat uploading and hosting 14 | - Easy to replicate Dats, Users, or entire datasets between Hypercloud deployments 15 | - Configurable user management 16 | - Easy to self-deploy 17 | 18 | Links: 19 | 20 | - **[Get Involved](https://github.com/joehand/hypercloud/wiki)** 21 | - **[Documentation](./docs)** 22 | 23 | ## Setup 24 | 25 | Clone this repository, then run 26 | 27 | ``` 28 | npm install 29 | cp config.defaults.yml config.development.yml 30 | ``` 31 | 32 | Modify `config.development.yml` to fit your needs, then start the server with `npm start`. 33 | 34 | ## Configuration 35 | 36 | Before deploying the service, you absolutely *must* modify the following config. 37 | 38 | #### Basics 39 | 40 | ```yaml 41 | dir: ./.hypercloud # where to store the data 42 | brandname: Hypercloud # the title of your service 43 | hostname: hypercloud.local # the hostname of your service 44 | port: 8080 # the port to run the service on 45 | rateLimiting: true # rate limit the HTTP requests? 46 | defaultDiskUsageLimit: 100mb # default maximum disk usage for each user 47 | pm2: false # set to true if you're using https://keymetrics.io/ 48 | ``` 49 | 50 | #### Lets Encrypt 51 | 52 | You can enable lets-encrypt to automatically provision TLS certs using this config: 53 | 54 | ```yaml 55 | letsencrypt: 56 | debug: false # debug mode? must be set to 'false' to use live config 57 | email: 'foo@bar.com' # email to register domains under 58 | ``` 59 | 60 | If enabled, `port` will be ignored and the server will register at ports 80 and 443. 61 | 62 | #### Admin Account 63 | 64 | The admin user has its credentials set by the config yaml at load. If you change the password while the server is running, then restart the server, the password will be reset to whatever is in the config. 65 | 66 | ```yaml 67 | admin: 68 | email: 'foo@bar.com' 69 | password: myverysecretpassword 70 | ``` 71 | 72 | #### UI Module 73 | 74 | The frontend can be replaced with a custom npm module. The default is [hypercloud-ui-vanilla](https://npm.im/hypercloud-ui-vanilla). 75 | 76 | ```yaml 77 | ui: hypercloud-ui-vanilla 78 | ``` 79 | 80 | #### HTTP Sites 81 | 82 | Hypercloud can host the archives as HTTP sites. This has the added benefit of enabling [dat-dns shortnames](npm.im/dat-dns) for the archives. There are two possible schemes: 83 | 84 | ```yaml 85 | sites: per-user 86 | ``` 87 | 88 | Per-user will host archives at `username.hostname/archivename`, in a scheme similar to GitHub Pages. If the archive-name is == to the username, it will be hosted at `username.hostname`. 89 | 90 | Note that, in this scheme, a DNS shortname is only provided for the user archive (`username.hostname`). 91 | 92 | ```yaml 93 | sites: per-archive 94 | ``` 95 | 96 | Per-archive will host archives at `archivename-username.hostname`. If the archive-name is == to the username, it will be hosted at `username.hostname`. 97 | 98 | By default, HTTP Sites are disabled. 99 | 100 | #### Closed Registration 101 | 102 | For a private instance, use closed registration with a whitelist of allowed emails: 103 | 104 | ```yaml 105 | registration: 106 | open: false 107 | allowed: 108 | - alice@mail.com 109 | - bob@mail.com 110 | ``` 111 | 112 | #### Reserved Usernames 113 | 114 | Use reserved usernames to blacklist usernames which collide with frontend routes, or which might be used maliciously. 115 | 116 | ```yaml 117 | registration: 118 | reservedNames: 119 | - admin 120 | - root 121 | - support 122 | - noreply 123 | - users 124 | - archives 125 | ``` 126 | 127 | #### Session Tokens 128 | 129 | Hypercloud uses Json Web Tokens to manage sessions. You absolutely *must* replace the `secret` with a random string before deployment. 130 | 131 | ```yaml 132 | sessions: 133 | algorithm: HS256 # probably dont update this 134 | secret: THIS MUST BE REPLACED! # put something random here 135 | expiresIn: 1h # how long do sessions live? 136 | ``` 137 | 138 | #### Jobs 139 | 140 | Hypercloud runs some jobs periodically. You can configure how frequently they run. 141 | 142 | ```yaml 143 | # processing jobs 144 | jobs: 145 | popularArchivesIndex: 30s # compute the index of archives sorted by num peers 146 | userDiskUsage: 5m # compute how much disk space each user is using 147 | deleteDeadArchives: 5m # delete removed archives from disk 148 | ``` 149 | 150 | #### Emailer 151 | 152 | *Todo, sorry* 153 | 154 | ## Tests 155 | 156 | Run the tests with 157 | 158 | ``` 159 | npm test 160 | ``` 161 | 162 | To run the tests against a running server, specify the env var: 163 | 164 | ``` 165 | REMOTE_URL=http://{hostname}/ npm test 166 | ``` 167 | 168 | ## License 169 | 170 | MIT 171 | -------------------------------------------------------------------------------- /lib/apis/admin.js: -------------------------------------------------------------------------------- 1 | const {NotFoundError, UnauthorizedError, ForbiddenError} = require('../const') 2 | const lock = require('../lock') 3 | 4 | // exported api 5 | // = 6 | 7 | module.exports = class AdminAPI { 8 | constructor (cloud) { 9 | this.usersDB = cloud.usersDB 10 | this.archiver = cloud.archiver 11 | this.mailer = cloud.mailer 12 | this.config = cloud.config 13 | } 14 | 15 | async listUsers (req, res) { 16 | // check perms 17 | if (!res.locals.session) throw new UnauthorizedError() 18 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 19 | 20 | // fetch 21 | var users = await this.usersDB.list({ 22 | cursor: req.query.cursor, 23 | limit: req.query.limit ? +req.query.limit : 25, 24 | sort: req.query.sort, 25 | reverse: +req.query.reverse === 1 26 | }) 27 | 28 | // respond 29 | res.status(200) 30 | res.json({users}) 31 | } 32 | 33 | async getUser (req, res) { 34 | // check perms 35 | if (!res.locals.session) throw new UnauthorizedError() 36 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 37 | 38 | // fetch 39 | var user = await this._getUser(req.params.id) 40 | 41 | // respond 42 | res.status(200) 43 | res.json(user) 44 | } 45 | 46 | async updateUser (req, res) { 47 | // check perms 48 | if (!res.locals.session) throw new UnauthorizedError() 49 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 50 | 51 | // validate & sanitize input 52 | req.checkBody('username').optional() 53 | .isAlphanumeric().withMessage('Can only be letters and numbers.') 54 | .isLength({ min: 3, max: 16 }).withMessage('Must be 3 to 16 characters.') 55 | req.checkBody('email', 'Must be a valid email').optional() 56 | .isEmail() 57 | .isLength({ min: 3, max: 100 }) 58 | req.checkBody('scopes', 'Must be an array of strings.').optional() 59 | .isScopesArray() 60 | req.checkBody('diskQuota', 'Must be a byte size.').optional() 61 | .isBytes() 62 | ;(await req.getValidationResult()).throw() 63 | if (req.body.diskQuota) req.sanitizeBody('diskQuota').toBytes() 64 | var {username, email, scopes, diskQuota} = req.body 65 | 66 | var release = await lock('users') 67 | try { 68 | // fetch 69 | var user = await this._getUser(req.params.id) 70 | 71 | // update 72 | if (typeof username !== 'undefined') user.username = username 73 | if (typeof email !== 'undefined') user.email = email 74 | if (typeof scopes !== 'undefined') user.scopes = scopes 75 | if (typeof diskQuota !== 'undefined') user.diskQuota = diskQuota 76 | await this.usersDB.put(user) 77 | } finally { 78 | release() 79 | } 80 | 81 | // respond 82 | res.status(200) 83 | res.json(user) 84 | } 85 | 86 | async suspendUser (req, res) { 87 | // check perms 88 | if (!res.locals.session) throw new UnauthorizedError() 89 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 90 | 91 | var release = await lock('users') 92 | try { 93 | // fetch user record 94 | var user = await this._getUser(req.params.id) 95 | 96 | // update record 97 | var scopeIndex = user.scopes.indexOf('user') 98 | if (scopeIndex !== -1) user.scopes.splice(scopeIndex, 1) 99 | user.suspension = req.body && req.body.reason ? req.body.reason : true 100 | await this.usersDB.put(user) 101 | } finally { 102 | release() 103 | } 104 | 105 | // respond 106 | res.status(200).end() 107 | } 108 | 109 | async unsuspendUser (req, res) { 110 | // check perms 111 | if (!res.locals.session) throw new UnauthorizedError() 112 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 113 | 114 | var release = await lock('users') 115 | try { 116 | // fetch user record 117 | var user = await this._getUser(req.params.id) 118 | 119 | // update record 120 | var scopeIndex = user.scopes.indexOf('user') 121 | if (scopeIndex === -1) user.scopes.push('user') 122 | user.suspension = null 123 | await this.usersDB.put(user) 124 | } finally { 125 | release() 126 | } 127 | 128 | // respond 129 | res.status(200).end() 130 | } 131 | 132 | async getArchive (req, res) { 133 | // check perms 134 | if (!res.locals.session) throw new UnauthorizedError() 135 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 136 | 137 | // fetch from memory 138 | var archive = await this.archiver.getArchive(req.params.key) 139 | if (!archive) { 140 | throw new NotFoundError() 141 | } 142 | res.status(200) 143 | res.json({ 144 | key: req.params.key, 145 | numPeers: archive.numPeers, 146 | manifest: await this.archiver.getManifest(req.params.key), 147 | swarmOpts: archive.swarmOpts, 148 | diskUsage: archive.diskUsage 149 | }) 150 | } 151 | 152 | async sendEmail (req, res) { 153 | // check perms 154 | if (!res.locals.session) throw new UnauthorizedError() 155 | if (!res.locals.session.scopes.includes('admin:users')) throw new ForbiddenError() 156 | 157 | var {message, subject, username} = req.body 158 | if (!(message && subject && username)) { 159 | return res.status(422).json({ 160 | message: 'Must include a message and subject line' 161 | }) 162 | } 163 | 164 | // fetch user record 165 | var userRecord = await this.usersDB.getByUsername(username) 166 | 167 | if (!userRecord) throw new NotFoundError() 168 | 169 | this.mailer.send('support', { 170 | email: userRecord.email, 171 | subject, 172 | message, 173 | username, 174 | brandname: this.config.brandname 175 | }) 176 | res.status(200).end() 177 | } 178 | 179 | async _getUser (id) { 180 | // try to fetch by id, username, and email 181 | var user = await this.usersDB.getByID(id) 182 | if (user) return user 183 | 184 | user = await this.usersDB.getByUsername(id) 185 | if (user) return user 186 | 187 | user = await this.usersDB.getByEmail(id) 188 | if (user) return user 189 | 190 | throw new NotFoundError() 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /test/activity.js: -------------------------------------------------------------------------------- 1 | var test = require('ava') 2 | var createTestServer = require('./lib/server.js') 3 | 4 | var app 5 | var sessionToken, auth, authUser 6 | var fakeDatKey1 = 'a'.repeat(64) 7 | var fakeDatKey2 = 'b'.repeat(64) 8 | 9 | test.cb('start test server', t => { 10 | app = createTestServer(async err => { 11 | t.ifError(err) 12 | 13 | // login 14 | var res = await app.req.post({ 15 | uri: '/v1/login', 16 | json: { 17 | 'username': 'admin', 18 | 'password': 'foobar' 19 | } 20 | }) 21 | if (res.statusCode !== 200) throw new Error('Failed to login as admin') 22 | sessionToken = res.body.sessionToken 23 | auth = { bearer: sessionToken } 24 | 25 | t.end() 26 | }) 27 | }) 28 | 29 | test('register and login bob', async t => { 30 | // register bob 31 | var res = await app.req.post({ 32 | uri: '/v1/register', 33 | json: { 34 | email: 'bob@example.com', 35 | username: 'bob', 36 | password: 'foobar' 37 | } 38 | }) 39 | if (res.statusCode !== 201) throw new Error('Failed to register bob user') 40 | 41 | // check sent mail and extract the verification nonce 42 | var lastMail = app.cloud.mailer.transport.sentMail.pop() 43 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 44 | 45 | // verify via GET 46 | res = await app.req.get({ 47 | uri: '/v1/verify', 48 | qs: { 49 | username: 'bob', 50 | nonce: emailVerificationNonce 51 | }, 52 | json: true 53 | }) 54 | if (res.statusCode !== 200) throw new Error('Failed to verify bob user') 55 | 56 | // login bob 57 | res = await app.req.post({ 58 | uri: '/v1/login', 59 | json: { 60 | 'username': 'bob', 61 | 'password': 'foobar' 62 | } 63 | }) 64 | if (res.statusCode !== 200) throw new Error('Failed to login as bob') 65 | sessionToken = res.body.sessionToken 66 | authUser = { bearer: sessionToken } 67 | }) 68 | 69 | test('do some activity', async t => { 70 | var res 71 | var json 72 | 73 | // add an archive as admin 74 | json = {key: fakeDatKey1, name: 'fakedat1'} 75 | res = await app.req.post({uri: '/v1/archives/add', json, auth}) 76 | t.is(res.statusCode, 200, '200 added dat') 77 | 78 | // add an archive as bob 79 | json = {key: fakeDatKey2, name: 'fakedat2'} 80 | res = await app.req.post({uri: '/v1/archives/add', json, auth: authUser}) 81 | t.is(res.statusCode, 200, '200 added dat') 82 | 83 | // remove an archive as admin 84 | json = {key: fakeDatKey1} 85 | res = await app.req.post({uri: '/v1/archives/remove', json, auth}) 86 | t.is(res.statusCode, 200, '200 removed dat') 87 | }) 88 | 89 | test('get global activity', async t => { 90 | // no offset 91 | var res = await app.req.get({url: '/v1/explore?view=activity', json: true}) 92 | t.is(res.statusCode, 200, '200 got activity') 93 | t.is(res.body.activity.length, 3) 94 | t.is(res.body.activity[0].username, 'admin') 95 | t.is(res.body.activity[0].action, 'del-archive') 96 | t.is(res.body.activity[0].params.name, 'fakedat1') 97 | t.is(res.body.activity[0].params.key, fakeDatKey1) 98 | t.is(res.body.activity[1].username, 'bob') 99 | t.is(res.body.activity[1].action, 'add-archive') 100 | t.is(res.body.activity[1].params.name, 'fakedat2') 101 | t.is(res.body.activity[1].params.key, fakeDatKey2) 102 | t.is(res.body.activity[2].username, 'admin') 103 | t.is(res.body.activity[2].action, 'add-archive') 104 | t.is(res.body.activity[2].params.name, 'fakedat1') 105 | t.is(res.body.activity[2].params.key, fakeDatKey1) 106 | 107 | // with offset 108 | var start = res.body.activity[0].key 109 | res = await app.req.get({url: '/v1/explore', qs: {view: 'activity', start}, json: true}) 110 | t.is(res.statusCode, 200, '200 got activity') 111 | t.is(res.body.activity.length, 2) 112 | t.is(res.body.activity[0].username, 'bob') 113 | t.is(res.body.activity[0].action, 'add-archive') 114 | t.is(res.body.activity[0].params.name, 'fakedat2') 115 | t.is(res.body.activity[0].params.key, fakeDatKey2) 116 | t.is(res.body.activity[1].username, 'admin') 117 | t.is(res.body.activity[1].action, 'add-archive') 118 | t.is(res.body.activity[1].params.name, 'fakedat1') 119 | t.is(res.body.activity[1].params.key, fakeDatKey1) 120 | }) 121 | 122 | test('get user activity', async t => { 123 | // no offset 124 | var res = await app.req.get({url: '/v1/users/admin?view=activity', json: true}) 125 | t.is(res.statusCode, 200, '200 got activity') 126 | t.is(res.body.activity.length, 2) 127 | t.is(res.body.activity[0].username, 'admin') 128 | t.is(res.body.activity[0].action, 'del-archive') 129 | t.is(res.body.activity[0].params.key, fakeDatKey1) 130 | t.is(res.body.activity[0].params.name, 'fakedat1') 131 | t.is(res.body.activity[1].username, 'admin') 132 | t.is(res.body.activity[1].action, 'add-archive') 133 | t.is(res.body.activity[1].params.key, fakeDatKey1) 134 | t.is(res.body.activity[1].params.name, 'fakedat1') 135 | var start = res.body.activity[0].key 136 | 137 | res = await app.req.get({url: '/v1/users/bob?view=activity', json: true}) 138 | t.is(res.statusCode, 200, '200 got activity') 139 | t.is(res.body.activity.length, 1) 140 | t.is(res.body.activity[0].username, 'bob') 141 | t.is(res.body.activity[0].action, 'add-archive') 142 | t.is(res.body.activity[0].params.key, fakeDatKey2) 143 | t.is(res.body.activity[0].params.name, 'fakedat2') 144 | 145 | // with offset 146 | res = await app.req.get({url: '/v1/users/admin', qs: {view: 'activity', start}, json: true}) 147 | t.is(res.statusCode, 200, '200 got activity') 148 | t.is(res.body.activity.length, 1) 149 | t.is(res.body.activity[0].username, 'admin') 150 | t.is(res.body.activity[0].action, 'add-archive') 151 | t.is(res.body.activity[0].params.key, fakeDatKey1) 152 | t.is(res.body.activity[0].params.name, 'fakedat1') 153 | 154 | res = await app.req.get({url: '/v1/users/bob', qs: {view: 'activity', start}, json: true}) 155 | t.is(res.statusCode, 200, '200 got activity') 156 | t.is(res.body.activity.length, 1) 157 | t.is(res.body.activity[0].username, 'bob') 158 | t.is(res.body.activity[0].action, 'add-archive') 159 | t.is(res.body.activity[0].params.key, fakeDatKey2) 160 | t.is(res.body.activity[0].params.name, 'fakedat2') 161 | }) 162 | 163 | test.cb('stop test server', t => { 164 | app.close(() => { 165 | t.pass('closed') 166 | t.end() 167 | }) 168 | }) 169 | -------------------------------------------------------------------------------- /lib/apis/archive-files.js: -------------------------------------------------------------------------------- 1 | const {NotFoundError} = require('../const') 2 | const pda = require('pauls-dat-api') 3 | const parseRange = require('range-parser') 4 | const {identifyStream} = require('../helpers') 5 | const directoryListingPage = require('../templates/directory-listing-page') 6 | 7 | const CSP = ` 8 | default-src 'self'; 9 | script-src 'self' 'unsafe-eval' 'unsafe-inline'; 10 | style-src 'self' 'unsafe-inline'; 11 | img-src 'self' data: blob:; 12 | object-src 'none'; 13 | `.replace(/\n/g, ' ') 14 | 15 | // exported api 16 | // = 17 | 18 | module.exports = class ArchiveFilesAPI { 19 | constructor (cloud) { 20 | this.config = cloud.config 21 | this.usersDB = cloud.usersDB 22 | this.archivesDB = cloud.archivesDB 23 | this.archiver = cloud.archiver 24 | } 25 | 26 | async _getArchiveRecord (req, {topLevel} = {}) { 27 | var username, archname, userRecord, archiveRecord 28 | const findFn = test => a => a.name.toLowerCase() === test 29 | 30 | if (this.config.sites === 'per-archive') { 31 | let vhostParts = req.vhost[0].split('-') 32 | if (vhostParts.length === 1) { 33 | // user.domain 34 | username = archname = vhostParts[0] 35 | } else { 36 | // archive-user.domain 37 | archname = vhostParts.slice(0, -1).join('-') 38 | username = vhostParts[vhostParts.length - 1] 39 | } 40 | 41 | // lookup user record 42 | userRecord = await this.usersDB.getByUsername(username) 43 | if (!userRecord) throw new NotFoundError() 44 | 45 | // lookup archive record 46 | archiveRecord = userRecord.archives.find(findFn(archname)) 47 | if (!archiveRecord) throw new NotFoundError() 48 | return archiveRecord 49 | } else { 50 | // user.domain/archive 51 | username = req.vhost[0] 52 | archname = req.path.split('/')[1] 53 | 54 | // lookup user record 55 | userRecord = await this.usersDB.getByUsername(username) 56 | if (!userRecord) throw new NotFoundError() 57 | 58 | if (!topLevel && archname) { 59 | // lookup archive record 60 | archiveRecord = userRecord.archives.find(findFn(archname)) 61 | if (archiveRecord) { 62 | archiveRecord.isNotToplevel = true 63 | return archiveRecord 64 | } 65 | } 66 | 67 | // look up archive record at username 68 | archiveRecord = userRecord.archives.find(findFn(username)) 69 | if (!archiveRecord) throw new NotFoundError() 70 | return archiveRecord 71 | } 72 | } 73 | 74 | async getDNSFile (req, res) { 75 | // get the archive record 76 | var archiveRecord = await this._getArchiveRecord(req, {topLevel: true}) 77 | 78 | // respond 79 | res.status(200).end('dat://' + archiveRecord.key + '/\nTTL=3600') 80 | } 81 | 82 | async getFile (req, res) { 83 | var fileReadStream 84 | var headersSent = false 85 | var archiveRecord = await this._getArchiveRecord(req) 86 | 87 | // skip the archivename if the archive was not found by subdomain 88 | var reqPath = archiveRecord.isNotToplevel ? req.path.split('/').slice(2).join('/') : req.path 89 | 90 | // track whether the request has been aborted by client 91 | // if, after some async, we find `aborted == true`, then we just stop 92 | var aborted = false 93 | req.once('aborted', () => { 94 | aborted = true 95 | }) 96 | 97 | // get the archive 98 | var archive = await this.archiver.loadArchive(archiveRecord.key) 99 | if (!archive) { 100 | throw NotFoundError() 101 | } 102 | if (aborted) return 103 | 104 | // find an entry 105 | var filepath = decodeURIComponent(reqPath) 106 | if (!filepath) filepath = '/' 107 | var isFolder = filepath.endsWith('/') 108 | var entry 109 | const tryStat = async (path) => { 110 | if (entry) return 111 | try { 112 | entry = await pda.stat(archive, path) 113 | entry.path = path 114 | } catch (e) {} 115 | } 116 | if (isFolder) { 117 | await tryStat(filepath + 'index.html') 118 | await tryStat(filepath) 119 | } else { 120 | await tryStat(filepath) 121 | await tryStat(filepath + '.html') // fallback to .html 122 | } 123 | if (aborted) return 124 | 125 | // handle folder 126 | if ((!entry && isFolder) || (entry && entry.isDirectory())) { 127 | res.writeHead(200, 'OK', { 128 | 'Content-Type': 'text/html', 129 | 'Content-Security-Policy': CSP 130 | }) 131 | return res.end(await directoryListingPage(archive, filepath)) 132 | } 133 | 134 | // handle not found 135 | if (!entry) { 136 | throw new NotFoundError() 137 | } 138 | 139 | // handle range 140 | var statusCode = 200 141 | res.setHeader('Accept-Ranges', 'bytes') 142 | var range = req.headers.range && parseRange(entry.size, req.headers.range) 143 | if (range && range.type === 'bytes') { 144 | range = range[0] // only handle first range given 145 | statusCode = 206 146 | res.setHeader('Content-Range', 'bytes ' + range.start + '-' + range.end + '/' + entry.size) 147 | res.setHeader('Content-Length', range.end - range.start + 1) 148 | } else { 149 | if (entry.size) { 150 | res.setHeader('Content-Length', entry.size) 151 | } 152 | } 153 | 154 | // caching if-match (not if range is used) 155 | const ETag = 'block-' + entry.offset 156 | if (statusCode === 200 && req.headers['if-none-match'] === ETag) { 157 | return res.status(304, { 158 | 'Content-Security-Policy': CSP 159 | }).end() 160 | } 161 | 162 | // fetch the entry and stream the response 163 | fileReadStream = archive.createReadStream(entry.path, range) 164 | fileReadStream 165 | .pipe(identifyStream(entry.path, mimeType => { 166 | // send headers, now that we can identify the data 167 | headersSent = true 168 | var headers = { 169 | 'Content-Type': mimeType, 170 | 'Content-Security-Policy': CSP, 171 | 'Cache-Control': 'public, max-age: 60', 172 | ETag 173 | } 174 | res.writeHead(statusCode, 'OK', headers) 175 | })) 176 | .pipe(res) 177 | 178 | // handle empty files 179 | fileReadStream.once('end', () => { 180 | if (!headersSent) { 181 | // no content 182 | headersSent = true 183 | res.writeHead(200, 'OK', { 184 | 'Content-Security-Policy': CSP 185 | }) 186 | res.end('\n') 187 | } 188 | }) 189 | 190 | // handle read-stream errors 191 | fileReadStream.once('error', _ => { 192 | if (!headersSent) { 193 | headersSent = true 194 | res.status(500).send('Failed to read file') 195 | } 196 | }) 197 | 198 | // abort if the client aborts 199 | req.once('aborted', () => { 200 | if (fileReadStream) { 201 | fileReadStream.destroy() 202 | } 203 | }) 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /lib/dbs/archives.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events') 2 | var assert = require('assert') 3 | var levelPromise = require('level-promise') 4 | var createIndexer = require('level-simple-indexes') 5 | var sublevel = require('subleveldown') 6 | var collect = require('stream-collector') 7 | var lock = require('../lock') 8 | var { promisifyModule } = require('../helpers') 9 | 10 | // exported api 11 | // = 12 | 13 | class ArchivesDB extends EventEmitter { 14 | constructor (cloud) { 15 | super() 16 | this.config = cloud.config 17 | this.archiver = cloud.archiver 18 | this.usersDB = cloud.usersDB 19 | 20 | // create levels and indexer 21 | this.archivesDB = sublevel(cloud.db, 'archives', { valueEncoding: 'json' }) 22 | this.deadArchivesDB = sublevel(cloud.db, 'dead-archives') 23 | this.indexDB = sublevel(cloud.db, 'archives-index') 24 | this.indexer = createIndexer(this.indexDB, { 25 | keyName: 'key', 26 | properties: ['createdAt'], 27 | map: (key, next) => { 28 | this.getExtraByKey(key) 29 | .catch(next) 30 | .then(res => next(null, res)) 31 | } 32 | }) 33 | 34 | // promisify 35 | levelPromise.install(this.archivesDB) 36 | promisifyModule(this.indexer, ['findOne', 'addIndexes', 'removeIndexes', 'updateIndexes']) 37 | } 38 | 39 | // basic ops 40 | // = 41 | 42 | async create (record) { 43 | assert(record && typeof record === 'object') 44 | assert(typeof record.key === 'string') 45 | record = Object.assign({}, ArchivesDB.defaults(), record) 46 | record.createdAt = Date.now() 47 | await this.put(record) 48 | await this.indexer.updateIndexes(record) 49 | this.emit('create', record) 50 | return record 51 | } 52 | 53 | async put (record) { 54 | assert(typeof record.key === 'string') 55 | record.updatedAt = Date.now() 56 | await this.archivesDB.put(record.key, record) 57 | this.emit('put', record) 58 | } 59 | 60 | async del (record) { 61 | assert(record && typeof record === 'object') 62 | assert(typeof record.key === 'string') 63 | await this.archivesDB.del(record.key) 64 | await this.indexer.removeIndexes(record) 65 | /* dont await */ this.deadArchivesDB.del(record.key) 66 | this.emit('del', record) 67 | } 68 | 69 | // getters 70 | // = 71 | 72 | async getByKey (key) { 73 | assert(typeof key === 'string') 74 | try { 75 | return await this.archivesDB.get(key) 76 | } catch (e) { 77 | if (e.notFound) return null 78 | throw e 79 | } 80 | } 81 | 82 | async getOrCreateByKey (key) { 83 | var release = await lock('archives:goc:' + key) 84 | try { 85 | var archiveRecord = await this.getByKey(key) 86 | if (!archiveRecord) { 87 | archiveRecord = await this.create({ key }) 88 | } 89 | } finally { 90 | release() 91 | } 92 | return archiveRecord 93 | } 94 | 95 | async getExtraByKey (key) { 96 | var archive = this.archiver.archives[key] 97 | var record = await this.getByKey(key) 98 | if (record.hostingUsers[0]) { 99 | record.owner = await this.usersDB.getByID(record.hostingUsers[0]) 100 | record.name = record.owner.archives.find(a => a.key === key).name 101 | record.niceUrl = record.name === record.owner.username 102 | ? `${record.owner.username}.${this.config.hostname}` 103 | : `${record.name}-${record.owner.username}.${this.config.hostname}` 104 | } else { 105 | record.owner = null 106 | record.name = '' 107 | record.niceUrl = null 108 | } 109 | record.numPeers = archive ? archive.numPeers : 0 110 | record.manifest = archive ? await this.archiver.getManifest(archive.key) : null 111 | return record 112 | } 113 | 114 | list ({cursor, limit, reverse, sort} = {}) { 115 | // 'popular' uses a special index 116 | if (sort === 'popular') { 117 | return this._listPopular({cursor, limit, reverse}) 118 | } 119 | 120 | return new Promise((resolve, reject) => { 121 | var opts = {limit, reverse} 122 | if (typeof cursor !== 'undefined') { 123 | // set cursor according to reverse 124 | if (reverse) opts.lt = cursor 125 | else opts.gt = cursor 126 | } 127 | // fetch according to sort 128 | var stream 129 | if (sort === 'createdAt') stream = this.indexer.find('createdAt', opts) 130 | else stream = this.archivesDB.createValueStream(opts) 131 | // collect into an array 132 | collect(stream, (err, res) => { 133 | if (err) reject(err) 134 | else resolve(res) 135 | }) 136 | }) 137 | } 138 | 139 | async _listPopular ({cursor, limit, reverse}) { 140 | cursor = cursor || 0 141 | limit = limit || 25 142 | 143 | // slice and dice the index 144 | var index = this.archiver.indexes.popular 145 | if (reverse) index = index.slice().reverse() 146 | index = index.slice(cursor, cursor + limit) 147 | 148 | // fetch the record for each item 149 | return Promise.all(index.map(indexEntry => ( 150 | this.getExtraByKey(indexEntry.key) 151 | ))) 152 | } 153 | 154 | // highlevel updates 155 | // = 156 | 157 | async addHostingUser (key, userId) { 158 | // fetch/create record 159 | var archiveRecord = await this.getOrCreateByKey(key) 160 | 161 | // add user 162 | if (archiveRecord.hostingUsers.includes(userId)) { 163 | return // already hosting 164 | } 165 | 166 | // TEMPORARY 167 | // only allow one hosting user per archive 168 | if (archiveRecord.hostingUsers.length > 0) { 169 | throw new Error('Cant add another user') 170 | } 171 | 172 | // update records 173 | archiveRecord.hostingUsers.push(userId) 174 | await this.put(archiveRecord) 175 | this.emit('add-hosting-user', {key, userId}, archiveRecord) 176 | 177 | // track dead archives 178 | /* dont await */ this.updateDeadArchives(key, archiveRecord.hostingUsers.length) 179 | } 180 | 181 | async removeHostingUser (key, userId) { 182 | // fetch/create record 183 | var archiveRecord = await this.getOrCreateByKey(key) 184 | 185 | // remove user 186 | var index = archiveRecord.hostingUsers.indexOf(userId) 187 | if (index === -1) { 188 | return // not already hosting 189 | } 190 | archiveRecord.hostingUsers.splice(index, 1) 191 | 192 | // update records 193 | await this.put(archiveRecord) 194 | this.emit('remove-hosting-user', {key, userId}, archiveRecord) 195 | 196 | // track dead archives 197 | /* dont await */ this.updateDeadArchives(key, archiveRecord.hostingUsers.length) 198 | } 199 | 200 | // internal tracking 201 | // = 202 | 203 | listDeadArchiveKeys () { 204 | return new Promise((resolve, reject) => { 205 | collect(this.deadArchivesDB.createKeyStream(), (err, res) => { 206 | if (err) reject(err) 207 | else resolve(res) 208 | }) 209 | }) 210 | } 211 | 212 | async updateDeadArchives (key, numHostingUsers) { 213 | try { 214 | if (numHostingUsers === 0) { 215 | await this.deadArchivesDB.put(key, '') 216 | } else { 217 | await this.deadArchivesDB.del(key) 218 | } 219 | } catch (e) {} 220 | } 221 | 222 | } 223 | module.exports = ArchivesDB 224 | 225 | // default user-record values 226 | ArchivesDB.defaults = () => ({ 227 | key: null, 228 | 229 | hostingUsers: [], 230 | 231 | updatedAt: 0, 232 | createdAt: 0 233 | }) 234 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var express = require('express') 2 | var bodyParser = require('body-parser') 3 | var cookieParser = require('cookie-parser') 4 | var expressValidator = require('express-validator') 5 | var RateLimit = require('express-rate-limit') 6 | var vhost = require('vhost') 7 | var bytes = require('bytes') 8 | 9 | var Hypercloud = require('./lib') 10 | var customValidators = require('./lib/validators') 11 | var customSanitizers = require('./lib/sanitizers') 12 | var packageJson = require('./package.json') 13 | 14 | module.exports = function (config) { 15 | if (config.pm2) { 16 | var pmx = require('pmx').init({ 17 | http: true, // HTTP routes logging (default: true) 18 | ignore_routes: [], // Ignore http routes with this pattern (Default: []) 19 | errors: true, // Exceptions logging (default: true) 20 | custom_probes: true, // Auto expose JS Loop Latency and HTTP req/s as custom metrics 21 | network: true, // Network monitoring at the application level 22 | ports: true // Shows which ports your app is listening on (default: false) 23 | }) 24 | } 25 | 26 | addConfigHelpers(config) 27 | var cloud = new Hypercloud(config) 28 | cloud.version = packageJson.version 29 | cloud.setupAdminUser() 30 | 31 | var app = express() 32 | app.cloud = cloud 33 | app.config = config 34 | app.approveDomains = approveDomains(config, cloud) 35 | 36 | app.locals = { 37 | session: false, // default session value 38 | errors: false, // common default value 39 | appInfo: { 40 | version: packageJson.version, 41 | brandname: config.brandname, 42 | hostname: config.hostname, 43 | port: config.port 44 | } 45 | } 46 | 47 | app.use(cookieParser()) 48 | app.use(bodyParser.json()) 49 | app.use(bodyParser.urlencoded()) 50 | app.use(expressValidator({ customValidators, customSanitizers })) 51 | app.use(cloud.sessions.middleware()) 52 | if (config.rateLimiting) { 53 | app.use(new RateLimit({windowMs: 1e3, max: 100, delayMs: 0})) // general rate limit 54 | app.use('/v1/verify', actionLimiter(24, 'Too many accounts created from this IP, please try again after an hour')) 55 | app.use('/v1/login', actionLimiter(1, 'Too many login attempts from this IP, please try again after an hour')) 56 | } 57 | 58 | // http gateway 59 | // = 60 | 61 | if (config.sites) { 62 | var httpGatewayApp = express() 63 | httpGatewayApp.get('/.well-known/dat', cloud.api.archiveFiles.getDNSFile) 64 | httpGatewayApp.get('*', cloud.api.archiveFiles.getFile) 65 | app.use(vhost('*.' + config.hostname, httpGatewayApp)) 66 | } 67 | 68 | // service apis 69 | // = 70 | 71 | app.get('/', cloud.api.service.frontpage) 72 | app.get('/v1/explore', cloud.api.service.explore) 73 | 74 | // user & auth apis 75 | // = 76 | 77 | app.post('/v1/register', cloud.api.users.doRegister) 78 | app.all('/v1/verify', cloud.api.users.verify) 79 | app.get('/v1/account', cloud.api.users.getAccount) 80 | app.post('/v1/account', cloud.api.users.updateAccount) 81 | app.post('/v1/account/password', cloud.api.users.updateAccountPassword) 82 | app.post('/v1/account/email', cloud.api.users.updateAccountEmail) 83 | app.post('/v1/login', cloud.api.users.doLogin) 84 | app.get('/v1/logout', cloud.api.users.doLogout) 85 | app.post('/v1/forgot-password', cloud.api.users.doForgotPassword) 86 | app.get('/v1/users/:username([^/]{3,})', cloud.api.users.get) 87 | 88 | // archives apis 89 | // = 90 | 91 | app.post('/v1/archives/add', cloud.api.archives.add) 92 | app.post('/v1/archives/remove', cloud.api.archives.remove) 93 | app.get('/v1/archives/:key([0-9a-f]{64})', cloud.api.archives.get) 94 | app.get('/v1/users/:username([^/]{3,})/:archivename', cloud.api.archives.getByName) 95 | 96 | // admin apis 97 | // = 98 | 99 | app.get('/v1/admin/users', cloud.api.admin.listUsers) 100 | app.get('/v1/admin/users/:id', cloud.api.admin.getUser) 101 | app.post('/v1/admin/users/:id', cloud.api.admin.updateUser) 102 | app.post('/v1/admin/users/:id/suspend', cloud.api.admin.suspendUser) 103 | app.post('/v1/admin/users/:id/unsuspend', cloud.api.admin.unsuspendUser) 104 | app.post('/v1/admin/users/:username/send-email', cloud.api.admin.sendEmail) 105 | app.get('/v1/admin/archives/:key', cloud.api.admin.getArchive) 106 | 107 | // (json) error-handling fallback 108 | // = 109 | 110 | app.use((err, req, res, next) => { 111 | var contentType = req.accepts('json') 112 | if (!contentType) { 113 | return next() 114 | } 115 | 116 | // validation errors 117 | if ('isEmpty' in err) { 118 | return res.status(422).json({ 119 | message: 'There were errors in your submission', 120 | invalidInputs: true, 121 | details: err.mapped() 122 | }) 123 | } 124 | 125 | // common errors 126 | if ('status' in err) { 127 | res.status(err.status) 128 | res.json(err.body) 129 | return 130 | } 131 | 132 | // general uncaught error 133 | console.error('[ERROR]', err) 134 | res.status(500) 135 | var error = { 136 | message: 'Internal server error', 137 | internalError: true 138 | } 139 | res.json(error) 140 | }) 141 | 142 | // ui module handlers 143 | // = 144 | 145 | if (config.ui) { 146 | app.use(require(config.ui)({cloud, config})) 147 | } 148 | 149 | // error handling 150 | // = 151 | 152 | process.on('unhandledRejection', (reason, p) => { 153 | console.log('Unhandled Rejection at: Promise', p, 'reason:', reason) 154 | }) 155 | 156 | // shutdown 157 | // = 158 | 159 | app.close = cloud.close.bind(cloud) 160 | 161 | return app 162 | } 163 | 164 | function actionLimiter (perHour, message) { 165 | return new RateLimit({ 166 | windowMs: perHour * 60 * 60 * 1000, 167 | delayMs: 0, 168 | max: 5, // start blocking after 5 requests 169 | message 170 | }) 171 | } 172 | 173 | function addConfigHelpers (config) { 174 | config.getUserDiskQuota = (userRecord) => { 175 | return userRecord.diskQuota || bytes(config.defaultDiskUsageLimit) 176 | } 177 | config.getUserDiskQuotaPct = (userRecord) => { 178 | return userRecord.diskUsage / config.getUserDiskQuota(userRecord) 179 | } 180 | } 181 | 182 | function approveDomains (config, cloud) { 183 | return async (options, certs, cb) => { 184 | var {domain} = options 185 | options.agreeTos = true 186 | options.email = config.letsencrypt.email 187 | 188 | // toplevel domain? 189 | if (domain === config.hostname) { 190 | return cb(null, {options, certs}) 191 | } 192 | 193 | // try looking up the site 194 | try { 195 | var archiveName 196 | var userName 197 | var domainParts = domain.split('.') 198 | if (config.sites === 'per-user') { 199 | // make sure the user record exists 200 | userName = domainParts[0] 201 | await cloud.usersDB.getByUsername(userName) 202 | return cb(null, {options, certs}) 203 | } else if (config.sites === 'per-archive') { 204 | // make sure the user and archive records exists 205 | if (domainParts.length === 3) { 206 | userName = archiveName = domainParts[0] 207 | } else { 208 | archiveName = domainParts[0] 209 | userName = domainParts[1] 210 | } 211 | let userRecord = await cloud.usersDB.getByUsername(userName) 212 | let archiveRecord = userRecord.archives.find(a => a.name === archiveName) 213 | if (archiveRecord) { 214 | return cb(null, {options, certs}) 215 | } 216 | } 217 | } catch (e) {} 218 | cb(new Error('Invalid domain')) 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /lib/dbs/users.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var monotonicTimestamp = require('monotonic-timestamp') 3 | var levelPromise = require('level-promise') 4 | var createIndexer = require('level-simple-indexes') 5 | var sublevel = require('subleveldown') 6 | var collect = require('stream-collector') 7 | var EventEmitter = require('events') 8 | var { promisifyModule } = require('../helpers') 9 | var lock = require('../lock') 10 | 11 | // exported api 12 | // = 13 | 14 | class UsersDB extends EventEmitter { 15 | constructor (cloud) { 16 | super() 17 | // create levels and indexer 18 | this.accountsDB = sublevel(cloud.db, 'accounts', { valueEncoding: 'json' }) 19 | this.indexDB = sublevel(cloud.db, 'accounts-index') 20 | this.indexer = createIndexer(this.indexDB, { 21 | keyName: 'id', 22 | properties: ['email', 'username', 'profileURL'], 23 | map: (id, next) => { 24 | this.getByID(id) 25 | .catch(next) 26 | .then(res => next(null, res)) 27 | } 28 | }) 29 | 30 | // promisify 31 | levelPromise.install(this.accountsDB) 32 | levelPromise.install(this.indexDB) 33 | promisifyModule(this.indexer, ['findOne', 'addIndexes', 'removeIndexes', 'updateIndexes']) 34 | } 35 | 36 | // basic ops 37 | // = 38 | 39 | async create (record) { 40 | assert(record && typeof record === 'object') 41 | record = Object.assign({}, UsersDB.defaults(), record) 42 | record.id = monotonicTimestamp().toString(36) 43 | record.createdAt = Date.now() 44 | await this.put(record) 45 | this.emit('create', record) 46 | return record 47 | } 48 | 49 | async put (record) { 50 | assert(typeof record.id === 'string') 51 | var release = await lock('users:write:' + record.id) 52 | try { 53 | record.updatedAt = Date.now() 54 | await this.accountsDB.put(record.id, record) 55 | await this.indexer.updateIndexes(record) 56 | } finally { 57 | release() 58 | } 59 | this.emit('put', record) 60 | } 61 | 62 | // update() is a put() that uses locks 63 | // - use this when outside of a locking transaction 64 | async update (id, updates) { 65 | assert(typeof id === 'string') 66 | assert(updates && typeof updates === 'object') 67 | var release = await lock('users') 68 | try { 69 | var record = await this.getByID(id) 70 | for (var k in updates) { 71 | if (k === 'id' || typeof updates[k] === 'undefined') { 72 | continue // dont allow that! 73 | } 74 | record[k] = updates[k] 75 | } 76 | record.updatedAt = Date.now() 77 | await this.accountsDB.put(record.id, record) 78 | await this.indexer.updateIndexes(record) 79 | } finally { 80 | release() 81 | } 82 | this.emit('put', record) 83 | return record 84 | } 85 | 86 | async del (record) { 87 | assert(typeof record.id === 'string') 88 | var release = await lock('users:write:' + record.id) 89 | try { 90 | await this.accountsDB.del(record.id) 91 | await this.indexer.removeIndexes(record) 92 | } finally { 93 | release() 94 | } 95 | this.emit('del', record) 96 | } 97 | 98 | // getters 99 | // = 100 | // TODO 101 | // do these getters need to be put behind locks along w/the rights? 102 | // the index-based reads (email, username) involve 2 separate reads, not atomic 103 | // -prf 104 | 105 | async isEmailTaken (email) { 106 | var record = await this.getByEmail(email) 107 | return !!record 108 | } 109 | 110 | async isUsernameTaken (username) { 111 | var record = await this.getByUsername(username) 112 | return !!record 113 | } 114 | 115 | async getByID (id) { 116 | assert(typeof id === 'string') 117 | try { 118 | return await this.accountsDB.get(id) 119 | } catch (e) { 120 | if (e.notFound) return null 121 | throw e 122 | } 123 | } 124 | 125 | async getByEmail (email) { 126 | assert(typeof email === 'string') 127 | return this.indexer.findOne('email', email) 128 | } 129 | 130 | async getByUsername (username) { 131 | assert(typeof username === 'string') 132 | return this.indexer.findOne('username', username) 133 | } 134 | 135 | async getByProfileURL (profileURL) { 136 | assert(typeof profileURL === 'string') 137 | return this.indexer.findOne('profileURL', profileURL) 138 | } 139 | 140 | list ({cursor, limit, reverse, sort} = {}) { 141 | return new Promise((resolve, reject) => { 142 | var opts = {limit, reverse} 143 | // find indexes require a start- and end-point 144 | if (sort && sort !== 'id') { 145 | if (reverse) { 146 | opts.lt = cursor || '\xff' 147 | opts.gte = '\x00' 148 | } else { 149 | opts.gt = cursor || '\x00' 150 | opts.lte = '\xff' 151 | } 152 | } else if (typeof cursor !== 'undefined') { 153 | // set cursor according to reverse 154 | if (reverse) opts.lt = cursor 155 | else opts.gt = cursor 156 | } 157 | // fetch according to sort 158 | var stream 159 | if (sort === 'username') stream = this.indexer.find('username', opts) 160 | else if (sort === 'email') stream = this.indexer.find('email', opts) 161 | else stream = this.accountsDB.createValueStream(opts) 162 | // collect into an array 163 | collect(stream, (err, res) => { 164 | if (err) reject(err) 165 | else resolve(res) 166 | }) 167 | }) 168 | } 169 | 170 | createValueStream (opts) { 171 | return this.accountsDB.createValueStream(opts) 172 | } 173 | 174 | // highlevel updates 175 | // = 176 | 177 | async addArchive (userId, archiveKey, name) { 178 | var release = await lock('users:update:' + userId) 179 | try { 180 | // fetch/create record 181 | var userRecord = await this.getByID(userId) 182 | 183 | // add/update archive 184 | var archive = userRecord.archives.find(a => a.key === archiveKey) 185 | if (!archive) { 186 | archive = Object.assign({}, UsersDB.archiveDefaults()) 187 | archive.key = archiveKey 188 | if (typeof name !== 'undefined') archive.name = name 189 | userRecord.archives.push(archive) 190 | } else { 191 | if (typeof name !== 'undefined') archive.name = name 192 | } 193 | 194 | // update records 195 | await this.put(userRecord) 196 | } finally { 197 | release() 198 | } 199 | this.emit('add-archive', {userId, archiveKey, name}, userRecord) 200 | } 201 | 202 | async removeArchive (userId, archiveKey) { 203 | var release = await lock('users:update:' + userId) 204 | try { 205 | // fetch/create record 206 | var userRecord = await this.getByID(userId) 207 | 208 | // remove archive 209 | var index = userRecord.archives.findIndex(a => a.key === archiveKey) 210 | if (index === -1) { 211 | return // not already hosting 212 | } 213 | userRecord.archives.splice(index, 1) 214 | 215 | // update records 216 | await this.put(userRecord) 217 | } finally { 218 | release() 219 | } 220 | this.emit('remove-archive', {userId, archiveKey}, userRecord) 221 | } 222 | } 223 | module.exports = UsersDB 224 | 225 | // default user-record values 226 | UsersDB.defaults = () => ({ 227 | username: null, 228 | passwordHash: null, 229 | passwordSalt: null, 230 | 231 | email: null, 232 | profileURL: null, 233 | scopes: [], 234 | suspension: null, 235 | archives: [], 236 | updatedAt: 0, 237 | createdAt: 0, 238 | 239 | diskUsage: 0, 240 | diskQuota: null, 241 | 242 | isEmailVerified: false, 243 | emailVerifyNonce: null, 244 | 245 | forgotPasswordNonce: null, 246 | 247 | isProfileDatVerified: false, 248 | profileVerifyToken: null 249 | }) 250 | 251 | // default user-record archive values 252 | UsersDB.archiveDefaults = () => ({ 253 | key: null, 254 | name: null 255 | }) 256 | -------------------------------------------------------------------------------- /lib/apis/archives.js: -------------------------------------------------------------------------------- 1 | const sse = require('express-server-sent-events') 2 | const {DAT_KEY_REGEX, NotFoundError, UnauthorizedError, ForbiddenError, NotImplementedError} = require('../const') 3 | const {wait} = require('../helpers') 4 | const lock = require('../lock') 5 | 6 | // exported api 7 | // = 8 | 9 | module.exports = class ArchivesAPI { 10 | constructor (cloud) { 11 | this.config = cloud.config 12 | this.usersDB = cloud.usersDB 13 | this.archivesDB = cloud.archivesDB 14 | this.activityDB = cloud.activityDB 15 | this.archiver = cloud.archiver 16 | } 17 | 18 | async add (req, res) { 19 | // validate session 20 | if (!res.locals.session) throw new UnauthorizedError() 21 | if (!res.locals.session.scopes.includes('user')) throw new ForbiddenError() 22 | 23 | // validate & sanitize input 24 | req.checkBody('key').optional().isDatHash() 25 | req.checkBody('url').optional().isDatURL() 26 | req.checkBody('name').optional() 27 | .isDatName().withMessage('Names must only contain characters, numbers, and dashes.') 28 | .isLength({ min: 3, max: 63 }).withMessage('Names must be 3-63 characters long.') 29 | ;(await req.getValidationResult()).throw() 30 | if (req.body.url) req.sanitizeBody('url').toDatDomain() 31 | var { key, url, name } = req.body 32 | 33 | // only allow one or the other 34 | if ((!key && !url) || (key && url)) { 35 | return res.status(422).json({ 36 | message: 'Must provide a key or url', 37 | invalidInputs: true 38 | }) 39 | } 40 | if (url) { 41 | key = DAT_KEY_REGEX.exec(url)[1] 42 | } 43 | 44 | var release = await Promise.all([lock('users'), lock('archives')]) 45 | try { 46 | // fetch user's record 47 | var userRecord = await this.usersDB.getByID(res.locals.session.id) 48 | 49 | // check that the user has the available quota 50 | if (this.config.getUserDiskQuotaPct(userRecord) >= 1) { 51 | return res.status(422).json({ 52 | message: 'You have exceeded your disk usage', 53 | outOfSpace: true 54 | }) 55 | } 56 | 57 | if (name && userRecord.archives.find(a => a.name === name)) { 58 | return res.status(422).json({ 59 | message: 'There were errors in your submission', 60 | details: { 61 | name: { 62 | msg: `You already have an archive named ${name}. Please select a new name.` 63 | } 64 | } 65 | }) 66 | } 67 | 68 | // update the records 69 | // TEMPORARY we have to do addHostingUser first, and cancel if that fails 70 | // await Promise.all([ 71 | // this.usersDB.addArchive(userRecord.id, key, name), 72 | // this.archivesDB.addHostingUser(key, userRecord.id) 73 | // ]) 74 | try { 75 | await this.archivesDB.addHostingUser(key, userRecord.id) 76 | } catch (e) { 77 | return res.status(422).json({ 78 | message: 'This archive is already being hosted by someone else' 79 | }) 80 | } 81 | await this.usersDB.addArchive(userRecord.id, key, name) 82 | } finally { 83 | release[0]() 84 | release[1]() 85 | } 86 | 87 | // record the event 88 | /* dont await */ this.activityDB.writeGlobalEvent({ 89 | userid: userRecord.id, 90 | username: userRecord.username, 91 | action: 'add-archive', 92 | params: {key, name} 93 | }) 94 | 95 | // add to the swarm 96 | this.archiver.loadArchive(key).then(() => { 97 | this.archiver._swarm(key, {upload: true, download: true}) 98 | }) 99 | 100 | // respond 101 | res.status(200).end() 102 | } 103 | 104 | async remove (req, res) { 105 | // validate session 106 | if (!res.locals.session) throw new UnauthorizedError() 107 | if (!res.locals.session.scopes.includes('user')) throw new ForbiddenError() 108 | 109 | // validate & sanitize input 110 | req.checkBody('key').optional().isDatHash() 111 | req.checkBody('url').optional().isDatURL() 112 | ;(await req.getValidationResult()).throw() 113 | if (req.body.url) req.sanitizeBody('url').toDatDomain() 114 | var { key, url } = req.body 115 | 116 | // only allow one or the other 117 | if ((!key && !url) || (key && url)) { 118 | return res.status(422).json({ 119 | message: 'Must provide a key or url', 120 | invalidInputs: true 121 | }) 122 | } 123 | if (url) { 124 | key = DAT_KEY_REGEX.exec(url)[1] 125 | } 126 | 127 | var release = await Promise.all([lock('users'), lock('archives')]) 128 | try { 129 | // fetch the user 130 | var userRecord = await this.usersDB.getByID(res.locals.session.id) 131 | 132 | // find the archive name 133 | var archiveRecord = await this.archivesDB.getExtraByKey(key) 134 | var name = archiveRecord.name 135 | 136 | // update the records 137 | await Promise.all([ 138 | this.usersDB.removeArchive(res.locals.session.id, key), 139 | this.archivesDB.removeHostingUser(key, res.locals.session.id) 140 | ]) 141 | } finally { 142 | release[0]() 143 | release[1]() 144 | } 145 | 146 | // record the event 147 | /* dont await */ this.activityDB.writeGlobalEvent({ 148 | userid: userRecord.id, 149 | username: userRecord.username, 150 | action: 'del-archive', 151 | params: {key, name} 152 | }) 153 | 154 | // remove from the swarm 155 | var archive = await this.archivesDB.getByKey(key) 156 | if (!archive.hostingUsers.length) { 157 | /* dont await */ this.archiver.closeArchive(key) 158 | } 159 | 160 | // respond 161 | res.status(200).end() 162 | } 163 | 164 | async get (req, res) { 165 | if (req.query.view === 'status') { 166 | return this.archiveStatus(req, res) 167 | } 168 | 169 | // give info about the archive 170 | // TODO 171 | throw NotImplementedError() 172 | } 173 | 174 | async getByName (req, res) { 175 | // validate & sanitize input 176 | req.checkParams('username').isAlphanumeric().isLength({ min: 3, max: 16 }) 177 | req.checkParams('archivename').isDatName().isLength({ min: 3, max: 64 }) 178 | ;(await req.getValidationResult()).throw() 179 | var { username, archivename } = req.params 180 | 181 | // lookup user 182 | var userRecord = await this.usersDB.getByUsername(username) 183 | if (!userRecord) throw new NotFoundError() 184 | 185 | // lookup archive 186 | const findFn = (DAT_KEY_REGEX.test(archivename)) 187 | ? a => a.key === archivename 188 | : a => a.name === archivename 189 | var archive = userRecord.archives.find(findFn) 190 | if (!archive) throw new NotFoundError() 191 | 192 | // lookup manifest 193 | var manifest = await this.archiver.getManifest(archive.key) 194 | 195 | // respond 196 | res.status(200).json({ 197 | user: username, 198 | key: archive.key, 199 | name: archive.name, 200 | title: manifest ? manifest.title : '', 201 | description: manifest ? manifest.description : '' 202 | }) 203 | } 204 | 205 | async archiveStatus (req, res) { 206 | var type = req.accepts(['json', 'text/event-stream']) 207 | if (type === 'text/event-stream') { 208 | sse(req, res, () => this._getArchiveProgressEventStream(req, res)) 209 | } else { 210 | let progress = await this._getArchiveProgress(req.params.key) 211 | res.status(200).json({ progress }) 212 | } 213 | } 214 | 215 | async _getArchive (key) { 216 | var archive = this.archiver.getArchive(key) 217 | if (!archive) { 218 | if (!this.archiver.isLoadingArchive(key)) { 219 | throw new NotFoundError() 220 | } 221 | archive = await this.archiver.loadArchive(key) 222 | } 223 | return archive 224 | } 225 | 226 | _getArchiveProgressEventStream (req, res) { 227 | let to 228 | let done = false 229 | let self = this 230 | async function send () { 231 | let progress = await self._getArchiveProgress(req.params.key) 232 | if (done) return 233 | res.sse('data: ' + progress + '\n\n') 234 | to = setTimeout(send, 1e3) 235 | } 236 | res.once('close', () => { 237 | done = true 238 | clearTimeout(to) 239 | }) 240 | send() 241 | } 242 | 243 | async _getArchiveProgress (key) { 244 | // fetch the archive 245 | var archive = await Promise.race([ 246 | this._getArchive(key), 247 | wait(5e3, false) 248 | ]) 249 | if (!archive) return 0 250 | var {metadata, content} = archive 251 | 252 | // some data missing, report progress at zero 253 | if (!metadata || !metadata.length || !content || !content.length) { 254 | return 0 255 | } 256 | 257 | // calculate & respond 258 | var need = metadata.length + content.length 259 | var remaining = blocksRemaining(metadata) + blocksRemaining(content) 260 | return (need - remaining) / need 261 | } 262 | } 263 | 264 | function blocksRemaining (feed) { 265 | var remaining = 0 266 | for (var i = 0; i < feed.length; i++) { 267 | if (!feed.has(i)) remaining++ 268 | } 269 | return remaining 270 | } 271 | -------------------------------------------------------------------------------- /docs/webapis.md: -------------------------------------------------------------------------------- 1 | # Web APIs Overview 2 | 3 | Service APIs 4 | 5 | ``` 6 | GET / - entry endpoint 7 | GET /v1/explore - get info about activity on the server 8 | ``` 9 | 10 | Archive APIs 11 | 12 | ``` 13 | GET /v1/archives/:archiveKey 14 | GET /v1/users/:username/:archiveName 15 | POST /v1/archives/add 16 | POST /v1/archives/remove 17 | ``` 18 | 19 | User APIs 20 | 21 | ``` 22 | GET /v1/users/:username 23 | POST /v1/register 24 | GET /v1/verify 25 | POST /v1/verify 26 | POST /v1/login 27 | POST /v1/logout 28 | GET /v1/account - get my info & settings 29 | POST /v1/account - update my settings 30 | ``` 31 | 32 | Admin APIs 33 | 34 | ``` 35 | GET /v1/admin/users - query users 36 | GET /v1/admin/users/:id - get user info & settings 37 | POST /v1/admin/users/:id - update user settings 38 | POST /v1/admin/users/:id/suspend - suspend a user account 39 | POST /v1/admin/users/:id/unsuspend - unsuspend a user account 40 | GET /v1/admin/archives/:key - get archive information 41 | POST /v1/admin/users/:username/send-email - send an email to the user 42 | ``` 43 | 44 | ## Service APIs 45 | 46 | ### GET / 47 | 48 | Home page. 49 | 50 | Response: TODO 51 | 52 | ### GET /v1/users/:username 53 | 54 | Lookup user profile. 55 | 56 | Response: 57 | 58 | ``` 59 | { 60 | username: String, from user's account object 61 | createdAt: Number, the timestamp of creation time 62 | } 63 | ``` 64 | 65 | Response when `?view=archives`: 66 | 67 | ``` 68 | { 69 | archives: [{ 70 | key: String, dat key 71 | name: String, optional shortname assigned by the user 72 | title: String, optional title extracted from the dat's manifest file 73 | description: String, optional description extracted from the dat's manifest file 74 | }] 75 | } 76 | ``` 77 | 78 | Response when `?view=activity`: 79 | 80 | ``` 81 | { 82 | activity: [{ 83 | key: String, event's id 84 | userid: String, the user who made the change 85 | username: String, the name of the user who made the change 86 | action: String, the label for the action 87 | params: Object, a set of arbitrary KVs relevant to the action 88 | }, ...] 89 | } 90 | ``` 91 | 92 | Additional query params when `?view=activity`: 93 | 94 | - start: For pagination. The key of the event to start after. 95 | 96 | ### GET /v1/users/:username/:archivename 97 | 98 | Lookup archive info. `archivename` can be the user-specified shortname, or the archive key. 99 | 100 | Response: 101 | 102 | ``` 103 | { 104 | user: String, the owning user's name 105 | key: String, the key of the dat 106 | name: String, optional shortname assigned by the user 107 | title: String, optional title extracted from the dat's manifest file 108 | description: String, optional description extracted from the dat's manifest file 109 | } 110 | ``` 111 | 112 | ### GET /v1/explore 113 | 114 | Response body when `?view=activity`: 115 | 116 | ``` 117 | { 118 | activity: [{ 119 | key: String, event's id 120 | userid: String, the user who made the change 121 | username: String, the name of the user who made the change 122 | action: String, the label for the action 123 | params: Object, a set of arbitrary KVs relevant to the action 124 | }, ...] 125 | } 126 | ``` 127 | 128 | Additional query params when `?view=activity`: 129 | 130 | - start: For pagination. The key of the event to start after. 131 | 132 | Response body when `?view=popular`: 133 | 134 | ``` 135 | { 136 | popular: [{ 137 | key: String, the archive's key 138 | numPeers: Number, the number of peers replicating the archive 139 | name: String, the name given to the archive by its owner 140 | title: String, optional title extracted from the dat's manifest file 141 | description: String, optional description extracted from the dat's manifest file 142 | owner: String, the username of the owning author 143 | createdAt: Number, the timestamp of the archive's upload 144 | }, ...] 145 | } 146 | ``` 147 | 148 | Additional query params when `?view=popular`: 149 | 150 | - start: For pagination. Should be an offset. 151 | 152 | Response body when `?view=recent`: 153 | 154 | ``` 155 | { 156 | recent: [{ 157 | key: String, the archive's key 158 | numPeers: Number, the number of peers replicating the archive 159 | name: String, the name given to the archive by its owner 160 | title: String, optional title extracted from the dat's manifest file 161 | description: String, optional description extracted from the dat's manifest file 162 | owner: String, the username of the owning author 163 | createdAt: Number, the timestamp of the archive's upload 164 | }, ...] 165 | } 166 | ``` 167 | 168 | Additional query params when `?view=recent`: 169 | 170 | - start: For pagination. Should be a timestamp. 171 | 172 | ## Archive APIs 173 | 174 | ### GET /v1/archives/:archiveKey 175 | 176 | Response when `?view=status` and `Accept: text/event-stream`: 177 | 178 | - Data event is emitted every 1000ms 179 | - Event contains a number, percentage (from 0 to 1) of upload progress 180 | 181 | Response when `?view=status` and `Accept: application/json`: 182 | 183 | ``` 184 | { 185 | progress: Number, a percentage (from 0 to 1) of upload progress 186 | } 187 | ``` 188 | 189 | ### POST /v1/archives/add 190 | 191 | Request body. Can supply `key` or `url`: 192 | 193 | ``` 194 | { 195 | key: String 196 | url: String 197 | name: String, optional shortname for the archive 198 | } 199 | ``` 200 | 201 | Adds the archive to the user's account. If the archive already exists, the request will update the settings (eg the name). 202 | 203 | ### POST /v1/archives/remove 204 | 205 | Request body. Can supply `key` or `url`: 206 | 207 | ``` 208 | { 209 | key: String 210 | url: String 211 | } 212 | ``` 213 | 214 | Removes the archive from the user's account. If no users are hosting the archive anymore, the archive will be deleted. 215 | 216 | ## User APIs 217 | 218 | ### POST /v1/login 219 | 220 | Request body. All fields required: 221 | 222 | ``` 223 | { 224 | email: String 225 | password: String 226 | } 227 | ``` 228 | 229 | Generates a session JWT and provides it in response headers. 230 | 231 | ### POST /v1/register 232 | 233 | [Step 1 of the register flow](./flows/registration.md#step-1-register-post-v1register) 234 | 235 | Request body. All fields required: 236 | 237 | ``` 238 | { 239 | email: String 240 | username: String 241 | password: String 242 | } 243 | ``` 244 | 245 | ### GET|POST /v1/verify 246 | 247 | [Step 2 of the register flow](./flows/registration.md#step-2-verify-get-or-post-v1verify) 248 | 249 | Request body. All fields required: 250 | 251 | ``` 252 | { 253 | username: String, username of the account 254 | nonce: String, verification nonce 255 | } 256 | ``` 257 | 258 | Like `/v1/login`, generates a session JWT and provides it in response headers. 259 | 260 | ### GET /v1/account 261 | 262 | Responds with the authenticated user's [account object](https://github.com/datprotocol/hypercloud/wiki/Users-Schema#account-object). 263 | 264 | Response body: 265 | 266 | ``` 267 | { 268 | email: String, the user's email address 269 | username: String, the chosen username 270 | diskUsage: Number, the number of bytes currently used by this account's archives 271 | diskQuota: Number, the number of bytes allowed to be used by this account's archives 272 | updatedAt: Number, the timestamp of the last update to the account 273 | createdAt: Number, the timestamp of when the account was created 274 | } 275 | ``` 276 | 277 | ### POST /v1/account 278 | 279 | Updates the authenticated user's [account object](https://github.com/datprotocol/hypercloud/wiki/Users-Schema#account-object) 280 | 281 | Request body: 282 | 283 | All fields are optional. If a field is omitted, no change is made. 284 | 285 | ``` 286 | { 287 | username: String, the chosen username 288 | } 289 | ``` 290 | 291 | ## Admin APIs 292 | 293 | ### GET /v1/admin/users 294 | 295 | Run queries against the users DB. 296 | 297 | Query params: 298 | 299 | - `cursor`. Key value to start listing from. 300 | - `limit`. How many records to fetch. 301 | - `sort`. Values: `id` `username` `email`. Default `id` (which is also ordered by creation time) 302 | - `reverse`. Reverse the sort. (1 means true.) 303 | 304 | Response body: 305 | 306 | ``` 307 | { 308 | users: [{ 309 | email: String, the user's email address 310 | username: String, the chosen username 311 | isEmailVerified: Boolean 312 | scopes: Array of strings, what is this user's perms? 313 | diskUsage: Number, how many bytes the user is using 314 | diskQuota: Number, how many bytes the user is allowed 315 | updatedAt: Number, the timestamp of the last update 316 | createdAt: Number, the timestamp of creation time 317 | }, ...] 318 | } 319 | ``` 320 | 321 | Scope: `admin:users` 322 | 323 | ### GET /v1/admin/users/:id 324 | 325 | Response body: 326 | 327 | ``` 328 | { 329 | email: String, the user's email address 330 | username: String, the chosen username 331 | isEmailVerified: Boolean 332 | emailVerifyNonce: String, the random verification nonce 333 | scopes: Array of strings, what is this user's perms? 334 | diskUsage: Number, how many bytes the user is using 335 | diskQuota: Number, how many bytes the user is allowed 336 | updatedAt: Number, the timestamp of the last update 337 | createdAt: Number, the timestamp of creation time 338 | } 339 | ``` 340 | 341 | Scope: `admin:users` 342 | 343 | ### POST /v1/admin/users/:id 344 | 345 | Request body: 346 | 347 | All fields are optional. If a field is omitted, no change is made. 348 | 349 | ``` 350 | { 351 | email: String, the user's email address 352 | username: String, the chosen username 353 | scopes: Array of strings, what is this user's perms? 354 | diskQuota: String, a description of how many bytes the user is allowed (eg '5mb') 355 | } 356 | ``` 357 | 358 | Scope: `admin:users` 359 | 360 | ### POST /v1/admin/users/:id/suspend 361 | 362 | Scope: `admin:users` 363 | 364 | ### POST /v1/admin/users/:id/unsuspend 365 | 366 | Scope: `admin:users` 367 | 368 | ### GET /v1/admin/archives/:key 369 | 370 | Response body: 371 | 372 | ``` 373 | { 374 | key: String, archive key 375 | numPeers: Number, number of active peers 376 | manifest: Object, the archive's manifest object (dat.json) 377 | swarmOpts: { 378 | download: Boolean, is it downloading? 379 | upload: Boolean, is it uploading? 380 | } 381 | } 382 | ``` -------------------------------------------------------------------------------- /lib/archiver.js: -------------------------------------------------------------------------------- 1 | var path = require('path') 2 | var promisify = require('es6-promisify') 3 | var hyperdrive = require('hyperdrive') 4 | var datEncoding = require('dat-encoding') 5 | var discoverySwarm = require('discovery-swarm') 6 | var swarmDefaults = require('datland-swarm-defaults') 7 | var mkdirp = require('mkdirp') 8 | var rimraf = require('rimraf') 9 | var getFolderSize = require('get-folder-size') 10 | var ms = require('ms') 11 | var pda = require('pauls-dat-api') 12 | var lock = require('./lock') 13 | var debug = require('debug')('archiver') 14 | var debugJobs = require('debug')('jobs') 15 | 16 | mkdirp = promisify(mkdirp) 17 | rimraf = promisify(rimraf) 18 | getFolderSize = promisify(getFolderSize) 19 | 20 | // exported api 21 | // = 22 | 23 | module.exports = class Archiver { 24 | constructor (cloud) { 25 | this.cloud = cloud 26 | this.config = cloud.config 27 | this.archives = {} 28 | this.loadPromises = {} 29 | 30 | // periodically construct the indexes 31 | this.indexes = {popular: []} 32 | this._startJob(this.computePopularIndex, 'popularArchivesIndex') 33 | this._startJob(this.computeUserDiskUsageAndSwarm, 'userDiskUsage') 34 | this._startJob(this.deleteDeadArchives, 'deleteDeadArchives') 35 | } 36 | 37 | // methods 38 | // = 39 | 40 | getArchive (key) { 41 | return this.archives[key] 42 | } 43 | 44 | isLoadingArchive (key) { 45 | return key in this.loadPromises 46 | } 47 | 48 | // load archive (wrapper) manages load promises 49 | async loadArchive (key) { 50 | key = datEncoding.toStr(key) 51 | 52 | // fallback to archive if it exists 53 | if (key in this.archives) { 54 | return this.archives[key] 55 | } 56 | 57 | // fallback to the promise, if it exists 58 | if (key in this.loadPromises) { 59 | return this.loadPromises[key] 60 | } 61 | 62 | // ensure the folder exists 63 | var archivePath = this._getArchiveFilesPath(key) 64 | await mkdirp(archivePath) 65 | 66 | // run and cache the promise 67 | var p = this._loadArchiveInner(archivePath, key) 68 | this.loadPromises[key] = p 69 | 70 | // when done, clear the promise 71 | const clear = () => delete this.loadPromises[key] 72 | p.then(clear, clear) 73 | 74 | // when done, save the archive instance 75 | p.then(archive => { this.archives[key] = archive }) 76 | 77 | return p 78 | } 79 | 80 | async closeArchive (key) { 81 | key = datEncoding.toStr(key) 82 | var archive = this.archives[key] 83 | if (archive) { 84 | this._swarm(archive, {download: false, upload: false}) 85 | await new Promise(resolve => archive.close(resolve)) 86 | delete this.archives[key] 87 | } 88 | } 89 | 90 | async closeAllArchives () { 91 | return Promise.all(Object.keys(this.archives).map(key => 92 | this.closeArchive(key) 93 | )) 94 | } 95 | 96 | // helper only reads manifest from disk if DNE or changed 97 | async getManifest (key) { 98 | var archive = this.archives[datEncoding.toStr(key)] 99 | if (!archive) { 100 | return null 101 | } 102 | try { 103 | var st = await pda.stat(archive, '/dat.json') 104 | if (archive.manifest) { 105 | if (st.offset === archive.manifest._offset) { 106 | // use cached 107 | return archive.manifest 108 | } 109 | } 110 | archive.manifest = await pda.readManifest(archive) 111 | archive.manifest._offset = st.offset 112 | return archive.manifest 113 | } catch (e) { 114 | if (!e.notFound) { 115 | console.error('Failed to load manifest for', archive.key, e) 116 | } 117 | return null 118 | } 119 | } 120 | 121 | async computePopularIndex () { 122 | var release = await lock('archiver-job') 123 | try { 124 | debugJobs('START Compute popular archives index') 125 | var start = Date.now() 126 | var popular = Object.keys(this.archives) 127 | popular.sort((aKey, bKey) => ( 128 | this.archives[bKey].numPeers - this.archives[aKey].numPeers 129 | )) 130 | this.indexes.popular = popular.slice(0, 100).map(key => ( 131 | {key, numPeers: this.archives[key].numPeers} 132 | )) 133 | } catch (e) { 134 | console.error(e) 135 | debugJobs('FAILED Compute popular archives index (%dms)', (Date.now() - start)) 136 | } finally { 137 | debugJobs('FINISH Compute popular archives index (%dms)', (Date.now() - start)) 138 | release() 139 | } 140 | } 141 | 142 | async computeUserDiskUsageAndSwarm () { 143 | var release = await lock('archiver-job') 144 | try { 145 | debugJobs('START Compute user quota usage') 146 | var start = Date.now() 147 | var users = await this.cloud.usersDB.list() 148 | await Promise.all(users.map(async (userRecord) => { 149 | // sum the disk usage of each archive 150 | var diskUsage = 0 151 | await Promise.all(userRecord.archives.map(async (archiveRecord) => { 152 | var path = this._getArchiveFilesPath(archiveRecord.key) 153 | var archive = this.getArchive(archiveRecord.key) 154 | var archiveUsage = await getFolderSize(path) 155 | if (archive) archive.diskUsage = archiveUsage 156 | diskUsage += archiveUsage 157 | })) 158 | 159 | // store on the user record 160 | userRecord.diskUsage = diskUsage 161 | await this.cloud.usersDB.update(userRecord.id, {diskUsage}) 162 | 163 | // reconfigure swarms based on quota overages 164 | var quotaPct = this.config.getUserDiskQuotaPct(userRecord) 165 | userRecord.archives.forEach(archiveRecord => { 166 | this._swarm(archiveRecord.key, { 167 | upload: true, // always upload 168 | download: quotaPct < 1 // only download if the user has capacity 169 | }) 170 | }) 171 | })) 172 | } catch (e) { 173 | console.error(e) 174 | debugJobs('FAILED Compute user quota usage (%dms)', (Date.now() - start)) 175 | } finally { 176 | debugJobs('FINISH Compute user quota usage (%dms)', (Date.now() - start)) 177 | release() 178 | } 179 | } 180 | 181 | async deleteDeadArchives () { 182 | var release = await lock('archiver-job') 183 | try { 184 | debugJobs('START Delete dead archives') 185 | var start = Date.now() 186 | var deadArchiveKeys = await this.cloud.archivesDB.listDeadArchiveKeys() 187 | await Promise.all(deadArchiveKeys.map(async (archiveKey) => { 188 | // delete files 189 | var archivePath = this._getArchiveFilesPath(archiveKey) 190 | await rimraf(archivePath, {disableGlob: true}) 191 | })) 192 | } catch (e) { 193 | console.error(e) 194 | debugJobs('FAILED Delete dead archives (%dms)', (Date.now() - start)) 195 | } finally { 196 | debugJobs('FINISH Delete dead archives (%dms)', (Date.now() - start)) 197 | release() 198 | } 199 | } 200 | 201 | // internal 202 | // = 203 | 204 | _getArchiveFilesPath (key) { 205 | return path.join(this.config.dir, 'archives', key.slice(0, 2), key.slice(2)) 206 | } 207 | 208 | _startJob (method, configKey) { 209 | var i = setInterval(method.bind(this), ms(this.config.jobs[configKey])) 210 | i.unref() 211 | } 212 | 213 | // load archive (inner) main load logic 214 | async _loadArchiveInner (archivePath, key) { 215 | // create the archive instance 216 | var archive = hyperdrive(archivePath, key, {sparse: false}) 217 | archive.replicationStreams = [] // list of all active replication streams 218 | archive.numPeers = 1 // num of active peers (1 for self) 219 | archive.manifest = null // cached manifest 220 | archive.diskUsage = 0 // cached disk usage 221 | 222 | // wait for ready 223 | await new Promise((resolve, reject) => { 224 | archive.ready(err => { 225 | if (err) reject(err) 226 | else resolve() 227 | }) 228 | }) 229 | return archive 230 | } 231 | 232 | // swarm archive 233 | _swarm (archive, opts) { 234 | if (typeof archive === 'string') { 235 | archive = this.getArchive(archive) 236 | if (!archive) return 237 | } 238 | 239 | // are any opts changed? 240 | var so = archive.swarmOpts 241 | if (so && so.download === opts.download && so.upload === opts.upload) { 242 | return 243 | } 244 | 245 | // close existing swarm 246 | if (archive.swarm) { 247 | archive.replicationStreams.forEach(stream => stream.destroy()) // stop all active replications 248 | archive.replicationStreams.length = 0 249 | archive.swarm.destroy() 250 | archive.swarm = null 251 | } 252 | 253 | // done? 254 | if (opts.download === false && opts.upload === false) { 255 | return 256 | } 257 | 258 | // join the swarm 259 | var swarm = discoverySwarm(swarmDefaults({ 260 | hash: false, 261 | utp: true, 262 | tcp: true, 263 | stream: (info) => { 264 | var key = datEncoding.toStr(archive.key) 265 | var dkey = datEncoding.toStr(archive.discoveryKey) 266 | var chan = dkey.slice(0, 6) + '..' + dkey.slice(-2) 267 | var keyStrShort = key.slice(0, 6) + '..' + key.slice(-2) 268 | debug('new connection chan=%s type=%s host=%s key=%s', chan, info.type, info.host, keyStrShort) 269 | 270 | // create the replication stream 271 | var stream = archive.replicate({ 272 | download: opts.download, 273 | upload: opts.upload, 274 | live: true 275 | }) 276 | stream.isActivePeer = false 277 | archive.replicationStreams.push(stream) 278 | stream.once('close', () => { 279 | var rs = archive.replicationStreams 280 | var i = rs.indexOf(stream) 281 | if (i !== -1) rs.splice(rs.indexOf(stream), 1) 282 | archive.numPeers = countActivePeers(archive) 283 | }) 284 | 285 | // timeout the connection after 5s if handshake does not occur 286 | var TO = setTimeout(() => { 287 | debug('handshake timeout chan=%s type=%s host=%s key=%s', chan, info.type, info.host, keyStrShort) 288 | stream.destroy(new Error('Timed out waiting for handshake')) 289 | }, 5000) 290 | stream.once('handshake', () => { 291 | stream.isActivePeer = true 292 | archive.numPeers = countActivePeers(archive) 293 | clearTimeout(TO) 294 | }) 295 | 296 | // debugging 297 | stream.on('error', err => debug('error chan=%s type=%s host=%s key=%s', chan, info.type, info.host, keyStrShort, err)) 298 | stream.on('close', () => debug('closing connection chan=%s type=%s host=%s key=%s', chan, info.type, info.host, keyStrShort)) 299 | return stream 300 | } 301 | })) 302 | swarm.listen(this.config.datPort) 303 | swarm.on('error', err => debug('Swarm error for', datEncoding.toStr(archive.key), err)) 304 | swarm.join(archive.discoveryKey, { announce: !(opts.upload === false) }) 305 | 306 | debug('Swarming archive', datEncoding.toStr(archive.key), 'discovery key', datEncoding.toStr(archive.discoveryKey)) 307 | archive.swarm = swarm 308 | archive.swarmOpts = opts 309 | } 310 | } 311 | 312 | function countActivePeers (archive) { 313 | return archive.replicationStreams.reduce((acc, stream) => ( 314 | acc + (stream.isActivePeer ? 1 : 0) 315 | ), 1) // start from one to include self 316 | } 317 | -------------------------------------------------------------------------------- /test/admin.js: -------------------------------------------------------------------------------- 1 | var test = require('ava') 2 | var bytes = require('bytes') 3 | var createTestServer = require('./lib/server.js') 4 | 5 | var app 6 | var sessionToken, auth 7 | 8 | test.cb('start test server', t => { 9 | app = createTestServer(async err => { 10 | t.ifError(err) 11 | 12 | // login 13 | var res = await app.req.post({ 14 | uri: '/v1/login', 15 | json: { 16 | 'username': 'admin', 17 | 'password': 'foobar' 18 | } 19 | }) 20 | if (res.statusCode !== 200) throw new Error('Failed to login as admin') 21 | sessionToken = res.body.sessionToken 22 | auth = { bearer: sessionToken } 23 | 24 | t.end() 25 | }) 26 | }) 27 | 28 | async function registerUser (username) { 29 | // register 30 | var res = await app.req.post({ 31 | uri: '/v1/register', 32 | json: { 33 | email: `${username}@example.com`, 34 | username: username, 35 | password: 'foobar' 36 | } 37 | }) 38 | if (res.statusCode !== 201) throw new Error(`Failed to register ${username} user`) 39 | 40 | // check sent mail and extract the verification nonce 41 | var lastMail = app.cloud.mailer.transport.sentMail.pop() 42 | var nonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 43 | 44 | // verify via GET 45 | res = await app.req.get({ 46 | uri: '/v1/verify', 47 | qs: {username, nonce}, 48 | json: true 49 | }) 50 | if (res.statusCode !== 200) throw new Error(`Failed to verify ${username} user`) 51 | } 52 | 53 | test('register alice, bob, and carla', async t => { 54 | await registerUser('alice') 55 | await registerUser('carla') 56 | await registerUser('bob') 57 | }) 58 | 59 | test('list users', async t => { 60 | var res 61 | 62 | // no params 63 | res = await app.req.get({ 64 | uri: '/v1/admin/users', 65 | json: true, 66 | auth 67 | }) 68 | t.is(res.statusCode, 200, '200 got users') 69 | t.is(res.body.users.length, 4, 'no params') 70 | t.is(res.body.users[0].username, 'admin') 71 | t.is(res.body.users[1].username, 'alice') 72 | t.is(res.body.users[2].username, 'carla') 73 | t.is(res.body.users[3].username, 'bob') 74 | 75 | // reverse sort 76 | res = await app.req.get({ 77 | uri: '/v1/admin/users', 78 | qs: {reverse: 1}, 79 | json: true, 80 | auth 81 | }) 82 | t.is(res.statusCode, 200, '200 got users') 83 | t.is(res.body.users.length, 4, 'reverse sort') 84 | t.is(res.body.users[0].username, 'bob') 85 | t.is(res.body.users[1].username, 'carla') 86 | t.is(res.body.users[2].username, 'alice') 87 | t.is(res.body.users[3].username, 'admin') 88 | 89 | // with limit 90 | res = await app.req.get({ 91 | uri: '/v1/admin/users', 92 | qs: {limit: 1}, 93 | json: true, 94 | auth 95 | }) 96 | t.is(res.statusCode, 200, '200 got users') 97 | t.is(res.body.users.length, 1, 'with limit') 98 | t.is(res.body.users[0].username, 'admin') 99 | 100 | // with limit and offset 101 | res = await app.req.get({ 102 | uri: '/v1/admin/users', 103 | qs: {limit: 1, cursor: res.body.users[0].id}, 104 | json: true, 105 | auth 106 | }) 107 | t.is(res.statusCode, 200, '200 got users') 108 | t.is(res.body.users.length, 1, 'with limit and offset') 109 | t.is(res.body.users[0].username, 'alice') 110 | 111 | // reverse with limit 112 | res = await app.req.get({ 113 | uri: '/v1/admin/users', 114 | qs: {limit: 1, reverse: 1}, 115 | json: true, 116 | auth 117 | }) 118 | t.is(res.statusCode, 200, '200 got users') 119 | t.is(res.body.users.length, 1, 'reverse with limit') 120 | t.is(res.body.users[0].username, 'bob') 121 | 122 | // reverse with limit and offset 123 | res = await app.req.get({ 124 | uri: '/v1/admin/users', 125 | qs: {limit: 1, reverse: 1, cursor: res.body.users[0].id}, 126 | json: true, 127 | auth 128 | }) 129 | t.is(res.statusCode, 200, '200 got users') 130 | t.is(res.body.users.length, 1, 'reverse with limit and offset') 131 | t.is(res.body.users[0].username, 'carla') 132 | 133 | // by username 134 | res = await app.req.get({ 135 | uri: '/v1/admin/users', 136 | qs: {sort: 'username'}, 137 | json: true, 138 | auth 139 | }) 140 | t.is(res.statusCode, 200, '200 got users') 141 | // console.log(res.body.users) 142 | t.is(res.body.users.length, 4, 'by username') 143 | t.is(res.body.users[0].username, 'admin') 144 | t.is(res.body.users[1].username, 'alice') 145 | t.is(res.body.users[2].username, 'bob') 146 | t.is(res.body.users[3].username, 'carla') 147 | 148 | // by username reverse sort 149 | res = await app.req.get({ 150 | uri: '/v1/admin/users', 151 | qs: {sort: 'username', reverse: 1}, 152 | json: true, 153 | auth 154 | }) 155 | t.is(res.statusCode, 200, '200 got users') 156 | t.is(res.body.users.length, 4, 'by username reverse sort') 157 | t.is(res.body.users[0].username, 'carla') 158 | t.is(res.body.users[1].username, 'bob') 159 | t.is(res.body.users[2].username, 'alice') 160 | t.is(res.body.users[3].username, 'admin') 161 | 162 | // by username with limit 163 | res = await app.req.get({ 164 | uri: '/v1/admin/users', 165 | qs: {sort: 'username', limit: 1}, 166 | json: true, 167 | auth 168 | }) 169 | t.is(res.statusCode, 200, '200 got users') 170 | t.is(res.body.users.length, 1, 'by username with limit') 171 | t.is(res.body.users[0].username, 'admin', 'by username with limit') 172 | 173 | // by username with offset 174 | res = await app.req.get({ 175 | uri: '/v1/admin/users', 176 | qs: {sort: 'username', cursor: 'admin'}, 177 | json: true, 178 | auth 179 | }) 180 | t.is(res.statusCode, 200, '200 got users') 181 | t.is(res.body.users.length, 3, 'by username with offset') 182 | t.is(res.body.users[0].username, 'alice', 'by username with offset') 183 | t.is(res.body.users[1].username, 'bob', 'by username with offset') 184 | t.is(res.body.users[2].username, 'carla', 'by username with offset') 185 | 186 | // by username with limit and offset 187 | res = await app.req.get({ 188 | uri: '/v1/admin/users', 189 | qs: {sort: 'username', limit: 1, cursor: 'admin'}, 190 | json: true, 191 | auth 192 | }) 193 | t.is(res.statusCode, 200, '200 got users') 194 | t.is(res.body.users.length, 1, 'by username with limit and offset') 195 | t.is(res.body.users[0].username, 'alice', 'by username with limit and offset') 196 | 197 | // by username reverse with limit 198 | res = await app.req.get({ 199 | uri: '/v1/admin/users', 200 | qs: {sort: 'username', limit: 1, reverse: 1}, 201 | json: true, 202 | auth 203 | }) 204 | t.is(res.statusCode, 200, '200 got users') 205 | t.is(res.body.users.length, 1, 'by username reverse with limit') 206 | t.is(res.body.users[0].username, 'carla') 207 | 208 | // by username reverse with limit and offset 209 | res = await app.req.get({ 210 | uri: '/v1/admin/users', 211 | qs: {sort: 'username', limit: 1, reverse: 1, cursor: res.body.users[0].username}, 212 | json: true, 213 | auth 214 | }) 215 | t.is(res.statusCode, 200, '200 got users') 216 | t.is(res.body.users.length, 1, 'by username reverse with limit and offset') 217 | t.is(res.body.users[0].username, 'bob') 218 | }) 219 | 220 | test('get user', async t => { 221 | var res 222 | 223 | // fetch listing 224 | res = await app.req.get({ 225 | uri: '/v1/admin/users', 226 | json: true, 227 | auth 228 | }) 229 | t.is(res.statusCode, 200, '200 got users') 230 | t.is(res.body.users.length, 4) 231 | var testUser = res.body.users[1] 232 | 233 | // by id 234 | res = await app.req.get({ 235 | uri: `/v1/admin/users/${testUser.id}`, 236 | json: true, 237 | auth 238 | }) 239 | t.is(res.statusCode, 200, '200 got users') 240 | t.is(res.body.username, testUser.username) 241 | 242 | // by username 243 | res = await app.req.get({ 244 | uri: `/v1/admin/users/${testUser.username}`, 245 | json: true, 246 | auth 247 | }) 248 | t.is(res.statusCode, 200, '200 got users') 249 | t.is(res.body.username, testUser.username) 250 | 251 | // by email 252 | res = await app.req.get({ 253 | uri: `/v1/admin/users/${testUser.email}`, 254 | json: true, 255 | auth 256 | }) 257 | t.is(res.statusCode, 200, '200 got users') 258 | t.is(res.body.username, testUser.username) 259 | }) 260 | 261 | test('fully update carla', async t => { 262 | var res = await app.req.post({ 263 | uri: '/v1/admin/users/carla', 264 | json: { 265 | username: 'carlita', 266 | email: 'carlita@example.com', 267 | scopes: ['user', 'admin:users'], 268 | diskQuota: '5mb' 269 | }, 270 | auth 271 | }) 272 | t.is(res.statusCode, 200, '200 updated') 273 | 274 | res = await app.req.get({ 275 | uri: '/v1/admin/users/carlita', 276 | json: true, 277 | auth 278 | }) 279 | t.is(res.statusCode, 200, '200 got') 280 | t.is(res.body.username, 'carlita', 'is updated') 281 | t.is(res.body.email, 'carlita@example.com', 'is updated') 282 | t.deepEqual(res.body.scopes, ['user', 'admin:users'], 'is updated') 283 | t.deepEqual(res.body.diskQuota, bytes('5mb'), 'is updated') 284 | }) 285 | 286 | test('partially update carlita', async t => { 287 | var res = await app.req.post({ 288 | uri: '/v1/admin/users/carlita', 289 | json: { 290 | scopes: ['user'] 291 | }, 292 | auth 293 | }) 294 | t.is(res.statusCode, 200, '200 updated') 295 | 296 | res = await app.req.get({ 297 | uri: '/v1/admin/users/carlita', 298 | json: true, 299 | auth 300 | }) 301 | t.is(res.statusCode, 200, '200 got') 302 | t.is(res.body.username, 'carlita', 'is updated') 303 | t.is(res.body.email, 'carlita@example.com', 'is updated') 304 | t.deepEqual(res.body.scopes, ['user'], 'is updated') 305 | t.deepEqual(res.body.diskQuota, bytes('5mb'), 'is updated') 306 | }) 307 | 308 | test('suspend bob', async t => { 309 | var res = await app.req.post({ 310 | uri: '/v1/admin/users/bob/suspend', 311 | json: {reason: 'A total jerk'}, 312 | auth 313 | }) 314 | t.is(res.statusCode, 200, '200 suspended') 315 | }) 316 | 317 | test('bob cant login when suspended', async t => { 318 | var res = await app.req.post({ 319 | uri: '/v1/login', 320 | json: { 321 | username: 'bob', 322 | password: 'foobar' 323 | } 324 | }) 325 | t.is(res.statusCode, 403, '403 cant login when suspended') 326 | }) 327 | 328 | test('unsuspend bob', async t => { 329 | var res = await app.req.post({ 330 | uri: '/v1/admin/users/bob/unsuspend', 331 | json: true, 332 | auth 333 | }) 334 | t.is(res.statusCode, 200, '200 suspended') 335 | }) 336 | 337 | test('bob can login when unsuspended', async t => { 338 | var res = await app.req.post({ 339 | uri: '/v1/login', 340 | json: { 341 | username: 'bob', 342 | password: 'foobar' 343 | } 344 | }) 345 | t.is(res.statusCode, 200, '200 can login when unsuspended') 346 | }) 347 | 348 | test('send support email', async t => { 349 | var res, lastMail 350 | 351 | res = await app.req.post({ 352 | uri: '/v1/admin/users/alice/send-email', 353 | json: { 354 | username: 'alice', 355 | subject: 'The subject line', 356 | message: 'The message' 357 | }, 358 | auth 359 | }) 360 | t.is(res.statusCode, 200) 361 | 362 | // check sent mail and extract the verification nonce 363 | lastMail = app.cloud.mailer.transport.sentMail.pop() 364 | t.truthy(lastMail) 365 | t.is(lastMail.data.subject, 'The subject line') 366 | t.truthy(lastMail.data.text.includes('The message')) 367 | }) 368 | 369 | test.cb('stop test server', t => { 370 | app.close(() => { 371 | t.pass('closed') 372 | t.end() 373 | }) 374 | }) 375 | -------------------------------------------------------------------------------- /test/users.js: -------------------------------------------------------------------------------- 1 | var test = require('ava') 2 | var createTestServer = require('./lib/server.js') 3 | 4 | var app 5 | 6 | // NOTE the test accounts created, when all tests are run sequentially: 7 | // - alice 8 | // - bob 9 | // - carla (invalid, never verifies email) 10 | 11 | test.cb('start test server', t => { 12 | app = createTestServer(err => { 13 | t.ifError(err) 14 | t.end() 15 | }) 16 | }) 17 | 18 | test('register and POST verify', async t => { 19 | var res, lastMail 20 | 21 | // register alice 22 | res = await app.req.post({ 23 | uri: '/v1/register', 24 | json: { 25 | email: 'alice@example.com', 26 | username: 'alice', 27 | password: 'foobar' 28 | } 29 | }) 30 | t.is(res.statusCode, 201, '201 created user') 31 | t.truthy(res.body.id) 32 | t.is(res.body.email, 'alice@example.com') 33 | 34 | // check sent mail and extract the verification nonce 35 | lastMail = app.cloud.mailer.transport.sentMail.pop() 36 | t.truthy(lastMail) 37 | t.is(lastMail.data.subject, 'Verify your email address') 38 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 39 | 40 | // verify via POST 41 | res = await app.req.post({ 42 | uri: '/v1/verify', 43 | json: { 44 | username: 'alice', 45 | nonce: emailVerificationNonce 46 | } 47 | }) 48 | t.is(res.statusCode, 200, '200 verified user') 49 | }) 50 | 51 | test('register and GET verify', async t => { 52 | var res, lastMail 53 | 54 | // register bob 55 | res = await app.req.post({ 56 | uri: '/v1/register', 57 | json: { 58 | email: 'bob@example.com', 59 | username: 'bob', 60 | password: 'foobar' 61 | } 62 | }) 63 | t.is(res.statusCode, 201, '201 created user') 64 | 65 | // check sent mail and extract the verification nonce 66 | lastMail = app.cloud.mailer.transport.sentMail.pop() 67 | t.truthy(lastMail) 68 | t.is(lastMail.data.subject, 'Verify your email address') 69 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 70 | 71 | // verify via GET 72 | res = await app.req.get({ 73 | uri: '/v1/verify', 74 | qs: { 75 | username: 'bob', 76 | nonce: emailVerificationNonce 77 | }, 78 | json: true 79 | }) 80 | t.is(res.statusCode, 200, '200 verified user') 81 | }) 82 | 83 | test('register validation', async t => { 84 | async function run (inputs, badParam) { 85 | var res = await app.req.post({uri: '/v1/register', json: inputs}) 86 | t.is(res.statusCode, 422, '422 bad input') 87 | t.is(res.body.invalidInputs, true, 'invalidInputs') 88 | } 89 | 90 | await run({ email: 'bob@example.com', username: 'bob' }, 'password') // missing password 91 | await run({ email: 'bob@example.com', password: 'foobar' }, 'username') // missing username 92 | await run({ username: 'bob', password: 'foobar' }, 'email') // missing email 93 | await run({ email: 'bob@example.com', username: 'bob', password: 'a' }, 'password') // password too short 94 | await run({ email: 'bob@example.com', username: 'a', password: 'foobar' }, 'username') // username too short 95 | await run({ email: 'bob@example.com', username: 'bob.boy', password: 'foobar' }, 'username') // username has invalid chars 96 | await run({ email: 'asdf', username: 'bob', password: 'foobar' }, 'email') // invalid email 97 | await run({ email: 'bob+foo@example.com', username: 'bob', password: 'foobar' }, 'email') // invalid email 98 | }) 99 | 100 | test('register blocks reserved usernames', async t => { 101 | async function run (inputs) { 102 | var res = await app.req.post({uri: '/v1/register', json: inputs}) 103 | t.is(res.statusCode, 422, '422 bad input') 104 | t.is(res.body.reservedName, true, 'reservedName') 105 | } 106 | 107 | await run({ email: 'bob@example.com', username: 'blacklisted', password: 'foobar' }) 108 | await run({ email: 'bob@example.com', username: 'reserved', password: 'foobar' }) 109 | await run({ email: 'bob@example.com', username: 'RESERVED', password: 'foobar' }) 110 | }) 111 | 112 | test('verify validation', async t => { 113 | async function run (type, inputs, badParam) { 114 | var res = await (type === 'post' 115 | ? app.req.post({uri: '/v1/verify', json: inputs}) 116 | : app.req.get({url: '/v1/verify', qs: inputs, json: true})) 117 | t.is(res.statusCode, 422, '422 bad input') 118 | t.is(res.body.invalidInputs, true, 'invalidInputs') 119 | } 120 | 121 | // register carla 122 | var res = await app.req.post({ 123 | uri: '/v1/register', 124 | json: { 125 | email: 'carla@example.com', 126 | username: 'carla', 127 | password: 'foobar' 128 | } 129 | }) 130 | t.is(res.statusCode, 201, '201 created user') 131 | 132 | await run('get', { username: 'carla' }, 'nonce') // missing nonce 133 | await run('post', { username: 'carla' }, 'nonce') // missing nonce 134 | await run('get', { nonce: 'asdf' }, 'username') // missing username 135 | await run('post', { nonce: 'asdf' }, 'username') // missing username 136 | }) 137 | 138 | test('cant register an already-registered user', async t => { 139 | var res 140 | 141 | // email collision on fully-registered account 142 | res = await app.req.post({ 143 | uri: '/v1/register', 144 | json: { 145 | email: 'alice@example.com', 146 | username: 'rando', 147 | password: 'foobar' 148 | } 149 | }) 150 | t.is(res.statusCode, 422, '422 bad input') 151 | t.is(res.body.emailNotAvailable, true, 'emailNotAvailable') 152 | 153 | // username collision on fully-registered account 154 | res = await app.req.post({ 155 | uri: '/v1/register', 156 | json: { 157 | email: 'rando@example.com', 158 | username: 'alice', 159 | password: 'foobar' 160 | } 161 | }) 162 | t.is(res.statusCode, 422, '422 bad input') 163 | t.is(res.body.usernameNotAvailable, true, 'usernameNotAvailable') 164 | 165 | // email collision on half-registered account 166 | res = await app.req.post({ 167 | uri: '/v1/register', 168 | json: { 169 | email: 'carla@example.com', 170 | username: 'rando', 171 | password: 'foobar' 172 | } 173 | }) 174 | t.is(res.statusCode, 422, '422 bad input') 175 | t.is(res.body.emailNotAvailable, true, 'emailNotAvailable') 176 | 177 | // username collision on half-registered account 178 | res = await app.req.post({ 179 | uri: '/v1/register', 180 | json: { 181 | email: 'rando@example.com', 182 | username: 'carla', 183 | password: 'foobar' 184 | } 185 | }) 186 | t.is(res.statusCode, 422, '422 bad input') 187 | t.is(res.body.usernameNotAvailable, true, 'usernameNotAvailable') 188 | }) 189 | 190 | test('cant verify a username that hasnt been registered', async t => { 191 | var res = await app.req.get({ 192 | uri: '/v1/verify', 193 | json: true, 194 | qs: { 195 | username: 'rando', 196 | nonce: 'asdf' 197 | } 198 | }) 199 | t.is(res.statusCode, 422, '422 bad input') 200 | t.is(res.body.invalidUsername, true, 'invalidUsername') 201 | }) 202 | 203 | test('verify fails with incorrect nonce', async t => { 204 | var res = await app.req.get({ 205 | uri: '/v1/verify', 206 | json: true, 207 | qs: { 208 | username: 'carla', 209 | nonce: 'asdf' 210 | } 211 | }) 212 | t.is(res.statusCode, 422, '422 bad input') 213 | t.is(res.body.invalidNonce, true, 'invalidNonce') 214 | }) 215 | 216 | test('login', async t => { 217 | var res = await app.req.post({ 218 | uri: '/v1/login', 219 | json: { 220 | username: 'bob', 221 | password: 'foobar' 222 | } 223 | }) 224 | t.is(res.statusCode, 200, '200 got token') 225 | t.truthy(res.body.sessionToken, 'got token in response') 226 | }) 227 | 228 | test('login configured admin user', async t => { 229 | var res = await app.req.post({ 230 | uri: '/v1/login', 231 | json: { 232 | username: 'admin', 233 | password: 'foobar' 234 | } 235 | }) 236 | t.is(res.statusCode, 200, '200 got token') 237 | t.truthy(res.body.sessionToken, 'got token in response') 238 | }) 239 | 240 | test('login validation', async t => { 241 | async function run (inputs, badParam) { 242 | var res = await app.req.post({uri: '/v1/login', json: inputs}) 243 | t.is(res.statusCode, 422, '422 bad input') 244 | t.is(res.body.invalidInputs, true, 'invalidInputs') 245 | } 246 | 247 | await run({ username: 'bob' }, 'password') // missing password 248 | await run({ password: 'foobar' }, 'username') // missing username 249 | }) 250 | 251 | test('cant login with invalid credentials', async t => { 252 | var res 253 | 254 | res = await app.req.post({ 255 | uri: '/v1/login', 256 | json: { 257 | username: 'rando', 258 | password: 'foobar' 259 | } 260 | }) 261 | t.is(res.statusCode, 422, '422 bad input') 262 | t.truthy(res.body.invalidCredentials, 'invalidCredentials') 263 | 264 | res = await app.req.post({ 265 | uri: '/v1/login', 266 | json: { 267 | username: 'bob', 268 | password: 'asdfasdf' 269 | } 270 | }) 271 | t.is(res.statusCode, 422, '422 bad input') 272 | t.truthy(res.body.invalidCredentials, 'invalidCredentials') 273 | }) 274 | 275 | test('login and get profile', async t => { 276 | // login 277 | var res = await app.req.post({ 278 | uri: '/v1/login', 279 | json: { 280 | username: 'bob', 281 | password: 'foobar' 282 | } 283 | }) 284 | t.is(res.statusCode, 200, '200 got token') 285 | 286 | // get profile 287 | var auth = {bearer: res.body.sessionToken} 288 | res = await app.req.get({url: '/v1/account', auth, json: true}) 289 | t.is(res.statusCode, 200, '200 got profile') 290 | t.is(res.body.email, 'bob@example.com', 'email is included') 291 | t.is(res.body.username, 'bob', 'username is included') 292 | }) 293 | 294 | test('login and change email', async t => { 295 | var res, lastMail 296 | 297 | // login 298 | res = await app.req.post({ 299 | uri: '/v1/login', 300 | json: { 301 | username: 'bob', 302 | password: 'foobar' 303 | } 304 | }) 305 | 306 | t.is(res.statusCode, 200, '200 got token') 307 | t.truthy(res.body.sessionToken, 'got token in response') 308 | 309 | var auth = {bearer: res.body.sessionToken} 310 | 311 | // try to change email to a duplicate email address 312 | res = await app.req.post({ 313 | url: '/v1/account/email', 314 | auth, 315 | json: { 316 | newEmail: 'bob@example.com', 317 | password: 'foobar' 318 | } 319 | }) 320 | t.is(res.statusCode, 422) 321 | 322 | // try to change email with invalid password 323 | res = await app.req.post({ 324 | url: '/v1/account/email', 325 | auth, 326 | json: { 327 | newEmail: 'bob@example.com', 328 | password: 'barfoo' 329 | } 330 | }) 331 | t.is(res.statusCode, 422) 332 | 333 | // change the email address 334 | res = await app.req.post({ 335 | url: '/v1/account/email', 336 | auth, 337 | json: { 338 | newEmail: 'bob2@example.com', 339 | password: 'foobar' 340 | } 341 | }) 342 | t.is(res.statusCode, 200) 343 | 344 | // verify that the user's email address does not change until it's verified 345 | res = await app.req.get({url: '/v1/account', auth, json: true}) 346 | t.is(res.body.email, 'bob@example.com') 347 | 348 | // check sent mail and extract the verification nonce 349 | lastMail = app.cloud.mailer.transport.sentMail.pop() 350 | t.truthy(lastMail) 351 | t.is(lastMail.data.subject, 'Verify your email address') 352 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 353 | 354 | // verify via POST 355 | res = await app.req.post({ 356 | uri: '/v1/verify', 357 | json: { 358 | username: 'bob', 359 | nonce: emailVerificationNonce 360 | } 361 | }) 362 | t.is(res.statusCode, 200, '200 verified user') 363 | 364 | // verify that the user's email was updated 365 | res = await app.req.get({url: '/v1/account', auth, json: true}) 366 | t.is(res.body.email, 'bob2@example.com') 367 | }) 368 | 369 | test('login and change password', async t => { 370 | // login 371 | var res = await app.req.post({ 372 | uri: '/v1/login', 373 | json: { 374 | username: 'bob', 375 | password: 'foobar' 376 | } 377 | }) 378 | t.is(res.statusCode, 200, '200 got token') 379 | t.truthy(res.body.sessionToken, 'got token in response') 380 | 381 | // change password 382 | var auth = {bearer: res.body.sessionToken} 383 | res = await app.req.post({ 384 | url: '/v1/account/password', 385 | auth, 386 | json: { 387 | oldPassword: 'foobar', 388 | newPassword: 'foobaz' 389 | } 390 | }) 391 | t.is(res.statusCode, 200, '200 password changed') 392 | 393 | // login with new password 394 | res = await app.req.post({ 395 | uri: '/v1/login', 396 | json: { 397 | username: 'bob', 398 | password: 'foobaz' 399 | } 400 | }) 401 | t.is(res.statusCode, 200, '200 got token') 402 | t.truthy(res.body.sessionToken, 'got token in response') 403 | }) 404 | 405 | test('forgot password flow', async t => { 406 | var res, lastMail 407 | 408 | // start the flow 409 | res = await app.req.post({ 410 | uri: '/v1/forgot-password', 411 | json: { 412 | email: 'bob@example.com' 413 | } 414 | }) 415 | t.is(res.statusCode, 200, '200 started forgot password flow') 416 | 417 | // check sent mail and extract the verification nonce 418 | var sentMail = app.cloud.mailer.transport.sentMail 419 | await waitUntil(() => sentMail[sentMail.length - 1].data.subject === 'Forgotten password reset') 420 | lastMail = sentMail.pop() 421 | t.truthy(lastMail) 422 | t.is(lastMail.data.subject, 'Forgotten password reset') 423 | var forgotPasswordNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 424 | 425 | // update password 426 | res = await app.req.post({ 427 | uri: '/v1/account/password', 428 | json: { 429 | username: 'bob', 430 | nonce: forgotPasswordNonce, 431 | newPassword: 'fooblah' 432 | } 433 | }) 434 | t.is(res.statusCode, 200, '200 updated password') 435 | 436 | // login with new password 437 | res = await app.req.post({ 438 | uri: '/v1/login', 439 | json: { 440 | username: 'bob', 441 | password: 'fooblah' 442 | } 443 | }) 444 | t.is(res.statusCode, 200, '200 got token') 445 | t.truthy(res.body.sessionToken, 'got token in response') 446 | }) 447 | 448 | test('forgot password flow rejects bad nonces', async t => { 449 | var res 450 | 451 | // update password 452 | res = await app.req.post({ 453 | uri: '/v1/account/password', 454 | json: { 455 | username: 'bob', 456 | nonce: 'bs', 457 | newPassword: 'fooblah' 458 | } 459 | }) 460 | t.is(res.statusCode, 422, '422 bad nonce') 461 | }) 462 | 463 | test.cb('stop test server', t => { 464 | app.close(() => { 465 | t.pass('closed') 466 | t.end() 467 | }) 468 | }) 469 | 470 | function waitUntil (pred) { 471 | return new Promise(resolve => { 472 | var i = setInterval(() => { 473 | if (pred()) { 474 | clearInterval(i) 475 | resolve() 476 | } 477 | }, 50) 478 | }) 479 | } 480 | -------------------------------------------------------------------------------- /test/archives.js: -------------------------------------------------------------------------------- 1 | var test = require('ava') 2 | var path = require('path') 3 | var fs = require('fs') 4 | var promisify = require('es6-promisify') 5 | var createTestServer = require('./lib/server.js') 6 | var { makeDatFromFolder, downloadDatFromSwarm } = require('./lib/dat.js') 7 | 8 | var app 9 | var sessionToken, auth, authUser 10 | var testDat, testDatKey 11 | var fsstat = promisify(fs.stat, fs) 12 | 13 | test.cb('start test server', t => { 14 | app = createTestServer(async err => { 15 | t.ifError(err) 16 | 17 | // login 18 | var res = await app.req.post({ 19 | uri: '/v1/login', 20 | json: { 21 | 'username': 'admin', 22 | 'password': 'foobar' 23 | } 24 | }) 25 | if (res.statusCode !== 200) throw new Error('Failed to login as admin') 26 | sessionToken = res.body.sessionToken 27 | auth = { bearer: sessionToken } 28 | 29 | t.end() 30 | }) 31 | }) 32 | 33 | test('register and login bob', async t => { 34 | // register bob 35 | var res = await app.req.post({ 36 | uri: '/v1/register', 37 | json: { 38 | email: 'bob@example.com', 39 | username: 'bob', 40 | password: 'foobar' 41 | } 42 | }) 43 | if (res.statusCode !== 201) throw new Error('Failed to register bob user') 44 | 45 | // check sent mail and extract the verification nonce 46 | var lastMail = app.cloud.mailer.transport.sentMail.pop() 47 | var emailVerificationNonce = /([0-9a-f]{64})/.exec(lastMail.data.text)[0] 48 | 49 | // verify via GET 50 | res = await app.req.get({ 51 | uri: '/v1/verify', 52 | qs: { 53 | username: 'bob', 54 | nonce: emailVerificationNonce 55 | }, 56 | json: true 57 | }) 58 | if (res.statusCode !== 200) throw new Error('Failed to verify bob user') 59 | 60 | // login bob 61 | res = await app.req.post({ 62 | uri: '/v1/login', 63 | json: { 64 | 'username': 'bob', 65 | 'password': 'foobar' 66 | } 67 | }) 68 | if (res.statusCode !== 200) throw new Error('Failed to login as bob') 69 | sessionToken = res.body.sessionToken 70 | authUser = { bearer: sessionToken } 71 | }) 72 | 73 | test.cb('share test-dat', t => { 74 | makeDatFromFolder(path.join(__dirname, '/scaffold/testdat1'), (err, d, dkey) => { 75 | t.ifError(err) 76 | testDat = d 77 | testDatKey = dkey 78 | t.end() 79 | }) 80 | }) 81 | 82 | test('user disk usage is zero', async t => { 83 | var res = await app.req.get({url: '/v1/account', json: true, auth}) 84 | t.is(res.statusCode, 200, '200 got account data') 85 | t.deepEqual(res.body.diskUsage, 0, 'disk usage is zero') 86 | }) 87 | 88 | test('add archive', async t => { 89 | var json = {key: testDatKey} 90 | var res = await app.req.post({uri: '/v1/archives/add', json, auth}) 91 | t.is(res.statusCode, 200, '200 added dat') 92 | }) 93 | 94 | test.cb('check archive status and wait till synced', t => { 95 | var to = setTimeout(() => { 96 | throw new Error('Archive did not sync') 97 | }, 15e3) 98 | 99 | checkStatus() 100 | async function checkStatus () { 101 | var res = await app.req({uri: `/v1/archives/${testDatKey}`, qs: {view: 'status'}, json: true, auth}) 102 | var progress = res.body && res.body.progress ? res.body.progress : 0 103 | if (progress === 1) { 104 | clearTimeout(to) 105 | console.log('synced!') 106 | t.end() 107 | } else { 108 | console.log('progress', progress * 100, '%') 109 | setTimeout(checkStatus, 300) 110 | } 111 | } 112 | }) 113 | 114 | test('read back archive', async t => { 115 | var res = await app.req.get({url: '/v1/users/admin?view=archives', json: true, auth}) 116 | t.is(res.statusCode, 200, '200 got user data') 117 | t.deepEqual(res.body.archives[0], { 118 | key: testDatKey, 119 | name: null, 120 | title: 'Test Dat 1', 121 | description: 'The first test dat' 122 | }) 123 | 124 | res = await app.req.get({url: '/v1/users/admin/' + testDatKey, json: true, auth}) 125 | t.is(res.statusCode, 200, '200 got dat data') 126 | t.deepEqual(res.body, { 127 | user: 'admin', 128 | key: testDatKey, 129 | name: null, 130 | title: 'Test Dat 1', 131 | description: 'The first test dat' 132 | }) 133 | }) 134 | 135 | test('user disk usage is now non-zero', async t => { 136 | // run usage-compute job 137 | await app.cloud.archiver.computeUserDiskUsageAndSwarm() 138 | 139 | // check data 140 | var res = await app.req.get({url: '/v1/account', json: true, auth}) 141 | t.is(res.statusCode, 200, '200 got account data') 142 | t.truthy(res.body.diskUsage > 0, 'disk usage is greater than zero') 143 | }) 144 | 145 | // TEMPORARY - hypercloud only allows one hosting user per archive 146 | // test('add duplicate archive as another user', async t => { 147 | // var json = {key: testDatKey} 148 | // var res = await app.req.post({uri: '/v1/archives/add', json, auth: authUser}) 149 | // t.is(res.statusCode, 200, '200 added dat') 150 | 151 | // res = await app.req.get({url: '/v1/users/bob?view=archives', json: true, auth: authUser}) 152 | // t.is(res.statusCode, 200, '200 got user data') 153 | // t.deepEqual(res.body.archives[0], { 154 | // key: testDatKey, 155 | // name: null, 156 | // title: 'Test Dat 1', 157 | // description: 'The first test dat' 158 | // }) 159 | 160 | // res = await app.req.get({url: '/v1/users/bob/' + testDatKey, json: true, auth: authUser}) 161 | // t.is(res.statusCode, 200, '200 got dat data') 162 | // t.deepEqual(res.body, { 163 | // user: 'bob', 164 | // key: testDatKey, 165 | // name: null, 166 | // title: 'Test Dat 1', 167 | // description: 'The first test dat' 168 | // }) 169 | // }) 170 | test('dont allow duplicate archives as another user', async t => { 171 | var json = {key: testDatKey} 172 | var res = await app.req.post({uri: '/v1/archives/add', json, auth: authUser}) 173 | t.is(res.statusCode, 422, '422 rejected') 174 | }) 175 | 176 | test('add archive that was already added', async t => { 177 | var json = {key: testDatKey} 178 | var res = await app.req.post({uri: '/v1/archives/add', json, auth}) 179 | t.is(res.statusCode, 200, '200 added dat') 180 | 181 | res = await app.req.get({url: '/v1/users/admin?view=archives', json: true, auth}) 182 | t.is(res.statusCode, 200, '200 got user data') 183 | t.deepEqual(res.body.archives[0], { 184 | key: testDatKey, 185 | name: null, 186 | title: 'Test Dat 1', 187 | description: 'The first test dat' 188 | }) 189 | 190 | res = await app.req.get({url: '/v1/users/admin/' + testDatKey, json: true, auth}) 191 | t.is(res.statusCode, 200, '200 got dat data') 192 | t.deepEqual(res.body, { 193 | user: 'admin', 194 | key: testDatKey, 195 | name: null, 196 | title: 'Test Dat 1', 197 | description: 'The first test dat' 198 | }) 199 | }) 200 | 201 | test('change archive name', async t => { 202 | // change name the first time 203 | var json = {key: testDatKey, name: 'test-archive'} 204 | var res = await app.req.post({uri: '/v1/archives/add', json, auth}) 205 | t.is(res.statusCode, 200, '200 added dat') 206 | 207 | res = await app.req.get({url: '/v1/users/admin?view=archives', json: true, auth}) 208 | t.is(res.statusCode, 200, '200 got user data') 209 | t.deepEqual(res.body.archives[0], { 210 | key: testDatKey, 211 | name: 'test-archive', 212 | title: 'Test Dat 1', 213 | description: 'The first test dat' 214 | }) 215 | 216 | res = await app.req.get({url: '/v1/users/admin/test-archive', json: true, auth}) 217 | t.is(res.statusCode, 200, '200 got dat data by name') 218 | t.deepEqual(res.body, { 219 | user: 'admin', 220 | key: testDatKey, 221 | name: 'test-archive', 222 | title: 'Test Dat 1', 223 | description: 'The first test dat' 224 | }) 225 | 226 | res = await app.req.get({url: '/v1/users/admin/' + testDatKey, json: true, auth}) 227 | t.is(res.statusCode, 200, '200 got dat data by key') 228 | t.deepEqual(res.body, { 229 | user: 'admin', 230 | key: testDatKey, 231 | name: 'test-archive', 232 | title: 'Test Dat 1', 233 | description: 'The first test dat' 234 | }) 235 | 236 | // change to invalid names 237 | json = {key: testDatKey, name: 'invalid$name'} 238 | res = await app.req.post({uri: '/v1/archives/add', json, auth}) 239 | t.is(res.statusCode, 422, '422 invalid name') 240 | 241 | // change name the second time 242 | json = {key: testDatKey, name: 'test--dat'} 243 | res = await app.req.post({uri: '/v1/archives/add', json, auth}) 244 | t.is(res.statusCode, 200, '200 added dat') 245 | 246 | res = await app.req.get({url: '/v1/users/admin?view=archives', json: true, auth}) 247 | t.is(res.statusCode, 200, '200 got user data') 248 | t.deepEqual(res.body.archives[0], { 249 | key: testDatKey, 250 | name: 'test--dat', 251 | title: 'Test Dat 1', 252 | description: 'The first test dat' 253 | }) 254 | 255 | res = await app.req.get({url: '/v1/users/admin/test--dat', json: true, auth}) 256 | t.is(res.statusCode, 200, '200 got dat data by name') 257 | t.deepEqual(res.body, { 258 | user: 'admin', 259 | key: testDatKey, 260 | name: 'test--dat', 261 | title: 'Test Dat 1', 262 | description: 'The first test dat' 263 | }) 264 | 265 | res = await app.req.get({url: '/v1/users/admin/' + testDatKey, json: true, auth}) 266 | t.is(res.statusCode, 200, '200 got dat data by key') 267 | t.deepEqual(res.body, { 268 | user: 'admin', 269 | key: testDatKey, 270 | name: 'test--dat', 271 | title: 'Test Dat 1', 272 | description: 'The first test dat' 273 | }) 274 | 275 | res = await app.req.get({url: '/v1/users/admin/test-archive', json: true, auth}) 276 | t.is(res.statusCode, 404, '404 old name not found') 277 | }) 278 | 279 | test('dont allow two archives with same name for given user', async t => { 280 | // add archive 281 | var json = {key: testDatKey, name: 'test-duplicate-archive'} 282 | var res = await app.req.post({uri: '/v1/archives/add', json, auth}) 283 | t.is(res.statusCode, 200, '200 added dat') 284 | 285 | // add the archive again 286 | res = await app.req.post({uri: '/v1/archives/add', json, auth}) 287 | t.is(res.statusCode, 422, '422 name already in use') 288 | }) 289 | 290 | test.cb('archive is accessable via dat swarm', t => { 291 | console.log('closing origin testdat swarm') 292 | testDat.close(() => { 293 | console.log('downloading from server swarm') 294 | downloadDatFromSwarm(testDatKey, { timeout: 15e3 }, (err, receivedDat) => { 295 | t.ifError(err) 296 | t.is(testDat.archive.content.blocks, receivedDat.archive.content.blocks, 'got all content blocks') 297 | t.end() 298 | }) 299 | }) 300 | }) 301 | 302 | test('list archives by popularity', async t => { 303 | // manually compute popular index 304 | app.cloud.archiver.computePopularIndex() 305 | 306 | var res = await app.req.get({uri: '/v1/explore?view=popular', json: true}) 307 | t.is(res.statusCode, 200, '200 got popular') 308 | t.is(res.body.popular.length, 1, 'got 1 archive') 309 | for (var i = 0; i < 1; i++) { 310 | let archive = res.body.popular[i] 311 | t.truthy(typeof archive.key === 'string', 'has key') 312 | t.truthy(typeof archive.numPeers === 'number', 'has numPeers') 313 | t.truthy(typeof archive.name === 'string', 'has name') 314 | t.truthy(typeof archive.title === 'string', 'has title') 315 | t.truthy(typeof archive.description === 'string', 'has description') 316 | t.truthy(typeof archive.owner === 'string', 'has owner') 317 | t.truthy(typeof archive.createdAt === 'number', 'has createdAt') 318 | } 319 | }) 320 | 321 | test('list archives by recency', async t => { 322 | var res = await app.req.get({uri: '/v1/explore?view=recent', json: true}) 323 | t.is(res.statusCode, 200, '200 got recent') 324 | t.is(res.body.recent.length, 1, 'got 1 archive') 325 | for (var i = 0; i < 1; i++) { 326 | let archive = res.body.recent[i] 327 | t.truthy(typeof archive.key === 'string', 'has key') 328 | t.truthy(typeof archive.numPeers === 'number', 'has numPeers') 329 | t.truthy(typeof archive.name === 'string', 'has name') 330 | t.truthy(typeof archive.title === 'string', 'has title') 331 | t.truthy(typeof archive.description === 'string', 'has description') 332 | t.truthy(typeof archive.owner === 'string', 'has owner') 333 | t.truthy(typeof archive.createdAt === 'number', 'has createdAt') 334 | } 335 | }) 336 | 337 | test('remove archive', async t => { 338 | var json = {key: testDatKey} 339 | var res = await app.req.post({uri: '/v1/archives/remove', json, auth}) 340 | t.is(res.statusCode, 200, '200 removed dat') 341 | }) 342 | 343 | // TEMPORARY only 1 owner per archive allowed 344 | // test('check archive status after removed by one user, not all', async t => { 345 | // var res = await app.req({uri: `/v1/archives/${testDatKey}`, qs: {view: 'status'}, auth}) 346 | // t.is(res.statusCode, 200, '200 got dat') 347 | // }) 348 | 349 | // test('remove archive as other user', async t => { 350 | // var json = {key: testDatKey} 351 | // var res = await app.req.post({uri: '/v1/archives/remove', json, auth: authUser}) 352 | // t.is(res.statusCode, 200, '200 removed dat') 353 | // }) 354 | 355 | test('remove archive that was already removed', async t => { 356 | var json = {key: testDatKey} 357 | var res = await app.req.post({uri: '/v1/archives/remove', json, auth}) 358 | t.is(res.statusCode, 200, '200 removed dat') 359 | }) 360 | 361 | test('check archive status after removed', async t => { 362 | var res = await app.req({uri: `/v1/archives/${testDatKey}`, qs: {view: 'status'}, auth}) 363 | t.is(res.statusCode, 404, '404 not found') 364 | 365 | res = await app.req.get({url: '/v1/users/admin?view=archives', json: true, auth}) 366 | t.is(res.statusCode, 200, '200 got user data') 367 | t.is(res.body.archives.length, 0) 368 | 369 | res = await app.req.get({url: '/v1/users/admin/' + testDatKey, json: true, auth}) 370 | t.is(res.statusCode, 404, '404 not found') 371 | 372 | res = await app.req.get({url: '/v1/users/admin/testdat', json: true, auth}) 373 | t.is(res.statusCode, 404, '404 not found') 374 | }) 375 | 376 | test('delete dead archives job', async t => { 377 | // folder exists 378 | var stat = await fsstat(app.cloud.archiver._getArchiveFilesPath(testDatKey)) 379 | t.truthy(stat) 380 | 381 | // run job 382 | await app.cloud.archiver.deleteDeadArchives() 383 | 384 | // folder does not exist 385 | await t.throws(fsstat(app.cloud.archiver._getArchiveFilesPath(testDatKey))) 386 | }) 387 | 388 | test('archive status wont stall on archive that fails to sync', async t => { 389 | // add a fake archive 390 | var fakeKey = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' 391 | var json = {key: fakeKey} 392 | var res = await app.req({uri: '/v1/archives/add', method: 'POST', json, auth}) 393 | t.same(res.statusCode, 200, '200 status') 394 | 395 | // now ask for the status. since the archive is never found, this should timeout 396 | res = await app.req({uri: `/v1/archives/${fakeKey}`, qs: {view: 'status'}}) 397 | t.same(res.statusCode, 200, '200 status') 398 | }) 399 | 400 | test.cb('stop test server', t => { 401 | app.close(() => { 402 | testDat.close(() => { 403 | t.pass('closed') 404 | t.end() 405 | }) 406 | }) 407 | }) 408 | -------------------------------------------------------------------------------- /lib/apis/users.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var querystring = require('querystring') 3 | var {randomBytes, hashPassword, verifyPassword} = require('../crypto') 4 | var {UnauthorizedError, ForbiddenError, NotFoundError} = require('../const') 5 | var lock = require('../lock') 6 | 7 | // exported api 8 | // = 9 | 10 | module.exports = class UsersAPI { 11 | constructor (cloud) { 12 | this.config = cloud.config 13 | this.archiver = cloud.archiver 14 | this.usersDB = cloud.usersDB 15 | this.activityDB = cloud.activityDB 16 | this.sessions = cloud.sessions 17 | this.proofs = cloud.proofs 18 | this.mailer = cloud.mailer 19 | } 20 | 21 | async doRegister (req, res) { 22 | // validate & sanitize input 23 | req.checkBody('username') 24 | .isAlphanumeric().withMessage('Can only be letters and numbers.') 25 | .isLength({ min: 3, max: 16 }).withMessage('Must be 3 to 16 characters.') 26 | req.checkBody('email', 'Must be a valid email') 27 | .isEmail({ allow_utf8_local_part: false }) 28 | .isSimpleEmail() 29 | .isLength({ min: 3, max: 100 }) 30 | req.checkBody('password', 'Must be 6 to 100 characters.').isLength({ min: 6, max: 100 }) 31 | ;(await req.getValidationResult()).throw() 32 | var { username, email, password } = req.body 33 | 34 | // check email if registration is closed 35 | if (!this.config.registration.open) { 36 | if (!this.config.registration.allowed.includes(email)) { 37 | let error = { 38 | message: 'Your email has not been whitelisted for registration by the admin.', 39 | emailNotWhitelisted: true 40 | } 41 | return res.status(422).json(error) 42 | } 43 | } 44 | 45 | // check if the username is reserved 46 | var {reservedNames} = this.config.registration 47 | if (reservedNames && Array.isArray(reservedNames) && reservedNames.length > 0) { 48 | if (reservedNames.indexOf(username.toLowerCase()) !== -1) { 49 | let error = { 50 | message: 'That username is reserved, please choose another.', 51 | reservedName: true 52 | } 53 | return res.status(422).json(error) 54 | } 55 | } 56 | 57 | // generate email verification nonce 58 | let emailVerificationNonce = (await randomBytes(32)).toString('hex') 59 | 60 | // salt and hash password 61 | let {passwordHash, passwordSalt} = await hashPassword(password) 62 | 63 | var release = await lock('users') 64 | try { 65 | // check email & username availability 66 | let error = false 67 | if (await this.usersDB.isEmailTaken(email)) { 68 | error = { 69 | message: 'Email is not available', 70 | emailNotAvailable: true 71 | } 72 | } else if (await this.usersDB.isUsernameTaken(username)) { 73 | error = { 74 | message: 'Username is not available', 75 | usernameNotAvailable: true 76 | } 77 | } 78 | 79 | // render error 80 | if (error) { 81 | return res.status(422).json(error) 82 | } 83 | 84 | // create user record 85 | var record = await this.usersDB.create({ 86 | username, 87 | email, 88 | passwordHash, 89 | passwordSalt, 90 | emailVerificationNonce 91 | }) 92 | } finally { 93 | release() 94 | } 95 | 96 | // send email 97 | var qs = querystring.stringify({ 98 | username, nonce: emailVerificationNonce 99 | }) 100 | this.mailer.send('verification', { 101 | email, 102 | username, 103 | emailVerificationNonce, 104 | emailVerificationLink: `https://${this.config.hostname}/v1/verify?${qs}` 105 | }) 106 | // log the verification link 107 | if (this.config.env === 'development') { 108 | console.log('Verify link for', username) 109 | console.log(`https://${this.config.hostname}/v1/verify?${qs}`) 110 | } 111 | 112 | // respond 113 | res.status(201).json({id: record.id, email: record.email}) 114 | } 115 | 116 | async verify (req, res) { 117 | var contentType = req.accepts(['html', 'json']) 118 | 119 | // validate & sanitize input 120 | req.check('username').isAlphanumeric().isLength({ min: 3, max: 16 }) 121 | req.check('nonce').isLength({ min: 3, max: 100 }) 122 | ;(await req.getValidationResult()).throw() 123 | var username = req.query.username || req.body.username 124 | var nonce = req.query.nonce || req.body.nonce 125 | 126 | var release = await lock('users') 127 | try { 128 | // fetch user record 129 | var userRecord = await this.usersDB.getByUsername(username) 130 | if (!userRecord) { 131 | return res.status(422).json({ 132 | message: 'Invalid username', 133 | invalidUsername: true 134 | }) 135 | } 136 | 137 | // compare email nonce 138 | if (nonce !== userRecord.emailVerificationNonce) { 139 | return res.status(422).json({ 140 | message: 'Invalid verification code', 141 | invalidNonce: true 142 | }) 143 | } 144 | 145 | // update user record 146 | userRecord.emailVerificationNonce = null 147 | userRecord.isEmailVerified = true 148 | 149 | // handle account email changes 150 | if (userRecord.pendingEmail) { 151 | userRecord.email = userRecord.pendingEmail 152 | userRecord.pendingEmail = null 153 | } 154 | if (!userRecord.scopes.includes('user')) { 155 | userRecord.scopes.push('user') 156 | } 157 | await this.usersDB.put(userRecord) 158 | } finally { 159 | release() 160 | } 161 | 162 | // generate session token 163 | var sessionToken = this.sessions.generate(userRecord) 164 | res.cookie('sess', sessionToken, { 165 | domain: this.config.hostname, 166 | httpOnly: true, 167 | secure: (this.config.env !== 'development'), 168 | sameSite: 'Lax' 169 | }) 170 | 171 | // respond 172 | if (contentType === 'html') { 173 | res.redirect('/?verified=true') 174 | } else { 175 | res.status(200).end() 176 | } 177 | } 178 | 179 | async getAccount (req, res) { 180 | // validate session 181 | if (!res.locals.session) throw new UnauthorizedError() 182 | 183 | // fetch user record 184 | var userRecord = await this.usersDB.getByID(res.locals.session.id) 185 | if (!userRecord) { 186 | return res.status(500).json({ 187 | message: 'Session user record not found', 188 | userRecordNotFound: true 189 | }) 190 | } 191 | 192 | // respond 193 | res.status(200).json({ 194 | email: userRecord.email, 195 | username: userRecord.username, 196 | // profileURL: userRecord.profileURL, TODO 197 | // profileVerifyToken: userRecord.profileVerifyToken, TODO 198 | diskUsage: userRecord.diskUsage, 199 | diskQuota: this.config.getUserDiskQuota(userRecord), 200 | updatedAt: userRecord.updatedAt, 201 | createdAt: userRecord.createdAt 202 | }) 203 | } 204 | 205 | async updateAccount (req, res) { 206 | // TODO: support username changes -prf 207 | // TODO: support profileURL changes -prf 208 | 209 | // validate session 210 | if (!res.locals.session) throw new UnauthorizedError() 211 | 212 | // validate & sanitize input 213 | req.checkBody('profileURL').isDatURL() 214 | ;(await req.getValidationResult()).throw() 215 | req.sanitizeBody('profileURL').toDatDomain() 216 | var { profileURL } = req.body 217 | 218 | var release = await lock('users') 219 | try { 220 | // fetch user record 221 | var userRecord = await this.usersDB.getByID(res.locals.session.id) 222 | if (!userRecord) { 223 | return res.status(500).json({ 224 | message: 'Session user record not found', 225 | userRecordNotFound: true 226 | }) 227 | } 228 | 229 | // new profile dat? 230 | if (profileURL && profileURL !== userRecord.profileURL) { 231 | // remove old profile-dat from swarm 232 | // TODO 233 | 234 | // add new profile-dat to swarm 235 | // TODO 236 | 237 | // generate a new proof & update record 238 | userRecord.profileVerifyToken = this.proofs.generate(userRecord) 239 | userRecord.isProfileDatVerified = false 240 | userRecord.profileURL = profileURL 241 | } 242 | 243 | // update user record 244 | await this.usersDB.put(userRecord) 245 | } finally { 246 | release() 247 | } 248 | 249 | // respond 250 | res.status(200).end() 251 | } 252 | 253 | async updateAccountPassword (req, res) { 254 | var userRecord 255 | var session = res.locals.session 256 | 257 | var release = await lock('users') 258 | try { 259 | // handle inputs based on whether this is an in-session update, or a forgot-password flow 260 | if (session) { 261 | // validate inputs 262 | req.checkBody('oldPassword', 'Must be 6 to 100 characters.').isLength({ min: 6, max: 100 }) 263 | req.checkBody('newPassword', 'Must be 6 to 100 characters.').isLength({ min: 6, max: 100 }) 264 | ;(await req.getValidationResult()).throw() 265 | let { oldPassword } = req.body 266 | 267 | // verify old password 268 | try { 269 | userRecord = await this.usersDB.getByID(session.id) 270 | assert(userRecord.isEmailVerified) 271 | assert(verifyPassword(oldPassword, userRecord)) 272 | } catch (e) { 273 | return res.status(422).json({ 274 | message: 'Invalid password', 275 | invalidCredentials: true 276 | }) 277 | } 278 | } else { 279 | // validate inputs 280 | req.checkBody('username').isAlphanumeric().isLength({ min: 3, max: 16 }) 281 | req.checkBody('nonce').isLength({ min: 3, max: 100 }) 282 | req.checkBody('newPassword', 'Must be 6 to 100 characters.').isLength({ min: 6, max: 100 }) 283 | ;(await req.getValidationResult()).throw() 284 | let { username, nonce } = req.body 285 | 286 | // fetch user record 287 | userRecord = await this.usersDB.getByUsername(username) 288 | if (!userRecord) { 289 | return res.status(422).json({ 290 | message: 'Invalid username', 291 | invalidUsername: true 292 | }) 293 | } 294 | 295 | // compare email nonce 296 | if (nonce !== userRecord.forgotPasswordNonce) { 297 | return res.status(422).json({ 298 | message: 'Invalid verification code', 299 | invalidNonce: true 300 | }) 301 | } 302 | } 303 | 304 | // salt and hash the new password 305 | let {passwordHash, passwordSalt} = await hashPassword(req.body.newPassword) 306 | 307 | // update user record 308 | Object.assign(userRecord, { 309 | passwordHash, 310 | passwordSalt, 311 | forgotPasswordNonce: null 312 | }) 313 | await this.usersDB.put(userRecord) 314 | } finally { 315 | release() 316 | } 317 | 318 | // respond 319 | res.status(200).end() 320 | } 321 | 322 | async updateAccountEmail (req, res) { 323 | var userRecord 324 | var session = res.locals.session 325 | 326 | // verify inputs 327 | req.checkBody('newEmail', 'Must be a valid email') 328 | .isEmail({ allow_utf8_local_part: false }) 329 | .isSimpleEmail() 330 | .isLength({ min: 3, max: 100 }) 331 | req.checkBody('password', 'Invalid password.').isLength({ min: 6, max: 100 }) 332 | ;(await req.getValidationResult()).throw() 333 | let { newEmail, password } = req.body 334 | 335 | // generate email verification nonce 336 | let emailVerificationNonce = (await randomBytes(32)).toString('hex') 337 | 338 | var release = await lock('users') 339 | try { 340 | // fetch user record 341 | userRecord = await this.usersDB.getByID(res.locals.session.id) 342 | if (!userRecord) { 343 | return res.status(500).json({ 344 | message: 'Session user record not found', 345 | userRecordNotFound: true 346 | }) 347 | } 348 | 349 | // verify password 350 | try { 351 | assert(verifyPassword(password, userRecord)) 352 | } catch (e) { 353 | return res.status(422).json({ 354 | message: 'Invalid password', 355 | invalidCredentials: true 356 | }) 357 | } 358 | 359 | // check email availability 360 | let error = false 361 | if (await this.usersDB.isEmailTaken(newEmail)) { 362 | error = { 363 | message: 'Email is not available', 364 | emailNotAvailable: true 365 | } 366 | } 367 | 368 | // render error 369 | if (error) { 370 | return res.status(422).json(error) 371 | } 372 | 373 | // update user record 374 | userRecord.pendingEmail = newEmail 375 | userRecord.emailVerificationNonce = emailVerificationNonce 376 | await this.usersDB.put(userRecord) 377 | } finally { 378 | release() 379 | } 380 | 381 | // send email 382 | var qs = querystring.stringify({ 383 | username: userRecord.username, 384 | nonce: emailVerificationNonce 385 | }) 386 | 387 | this.mailer.send('verify-update-email', { 388 | email: newEmail, 389 | username: userRecord.username, 390 | emailVerificationNonce, 391 | emailVerificationLink: `https://${this.config.hostname}/v1/verify?${qs}` 392 | }) 393 | // log the verification link 394 | if (this.config.env === 'development') { 395 | console.log('Verify link for', userRecord.username) 396 | console.log(`https://${this.config.hostname}/v1/verify?${qs}`) 397 | } 398 | 399 | // respond 400 | res.status(200).end() 401 | } 402 | 403 | async doLogin (req, res) { 404 | // validate & sanitize input 405 | req.checkBody('username', 'Invalid username.').isAlphanumeric().isLength({ min: 3, max: 16 }) 406 | req.checkBody('password', 'Invalid password.').isLength({ min: 6, max: 100 }) 407 | ;(await req.getValidationResult()).throw() 408 | var { username, password } = req.body 409 | 410 | var userRecord 411 | try { 412 | // fetch user record & check credentials 413 | userRecord = await this.usersDB.getByUsername(username) 414 | assert(userRecord.isEmailVerified) 415 | assert(verifyPassword(password, userRecord)) 416 | } catch (e) { 417 | return res.status(422).json({ 418 | message: 'Invalid username/password', 419 | invalidCredentials: true 420 | }) 421 | } 422 | 423 | // check for a suspension 424 | if (userRecord.suspension) { 425 | throw new ForbiddenError('Your account has been suspended.') 426 | } 427 | 428 | // generate session token 429 | var sessionToken = this.sessions.generate(userRecord) 430 | res.cookie('sess', sessionToken, { 431 | domain: this.config.hostname, 432 | httpOnly: true, 433 | secure: (this.config.env !== 'development'), 434 | sameSite: 'Lax' 435 | }) 436 | 437 | // respond 438 | res.status(200).json({ sessionToken }) 439 | } 440 | 441 | async doLogout (req, res) { 442 | res.clearCookie('sess', { 443 | domain: this.config.hostname, 444 | httpOnly: true, 445 | secure: (this.config.env !== 'development'), 446 | sameSite: 'Lax' 447 | }) 448 | res.redirect('/') 449 | } 450 | 451 | async doForgotPassword (req, res) { 452 | // validate & sanitize input 453 | req.checkBody('email', 'Must be a valid email').isEmail().isLength({ min: 3, max: 100 }) 454 | ;(await req.getValidationResult()).throw() 455 | var {email} = req.body 456 | 457 | // generate the nonce 458 | let forgotPasswordNonce = (await randomBytes(32)).toString('hex') 459 | 460 | var release = await lock('users') 461 | try { 462 | // fetch user record 463 | var userRecord = await this.usersDB.getByEmail(email) 464 | 465 | // send a response immediately so user list can't be enumerated 466 | res.status(200).end() 467 | 468 | if (!userRecord) { 469 | return 470 | } 471 | 472 | // save the email verification nonce 473 | Object.assign(userRecord, {forgotPasswordNonce}) 474 | await this.usersDB.put(userRecord) 475 | } finally { 476 | release() 477 | } 478 | 479 | // send email 480 | var qs = querystring.stringify({ 481 | username: userRecord.username, 482 | nonce: forgotPasswordNonce 483 | }) 484 | this.mailer.send('forgot-password', { 485 | email, 486 | username: userRecord.username, 487 | forgotPasswordNonce, 488 | forgotPasswordLink: `https://${this.config.hostname}/reset-password?${qs}` 489 | }) 490 | 491 | // log the verification link 492 | if (this.config.env === 'development') { 493 | console.log('Forgot-password link for', userRecord.username) 494 | console.log(`https://${this.config.hostname}/reset-password?${qs}`) 495 | } 496 | } 497 | 498 | async get (req, res) { 499 | // validate & sanitize input 500 | req.checkParams('username').isAlphanumeric().isLength({ min: 3, max: 16 }) 501 | ;(await req.getValidationResult()).throw() 502 | var { username } = req.params 503 | 504 | // lookup user 505 | var userRecord = await this.usersDB.getByUsername(username) 506 | if (!userRecord) throw new NotFoundError() 507 | 508 | // respond 509 | switch (req.query.view) { 510 | case 'archives': 511 | await Promise.all(userRecord.archives.map(async (archive) => { 512 | var manifest = await this.archiver.getManifest(archive.key) 513 | if (manifest) { 514 | archive.title = manifest.title 515 | archive.description = manifest.description 516 | } else { 517 | archive.title = '' 518 | archive.description = '' 519 | } 520 | })) 521 | res.status(200).json({ 522 | archives: userRecord.archives 523 | }) 524 | break 525 | 526 | case 'activity': 527 | res.status(200).json({ 528 | activity: await this.activityDB.listUserEvents(username, { 529 | limit: 25, 530 | lt: req.query.start, 531 | reverse: true 532 | }) 533 | }) 534 | break 535 | 536 | default: 537 | res.status(200).json({ 538 | username, 539 | createdAt: userRecord.createdAt 540 | }) 541 | break 542 | } 543 | } 544 | } 545 | --------------------------------------------------------------------------------