├── .babelrc ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .postcssrc.js ├── Dockerfile ├── LICENSE ├── README.md ├── build ├── build.js ├── check-versions.js ├── dev-client.js ├── dev-server.js ├── utils.js ├── vue-loader.conf.js ├── webpack.base.conf.js ├── webpack.dev.conf.js └── webpack.prod.conf.js ├── config ├── database.json └── default.yaml ├── db └── .gitkeep ├── docs ├── api.md └── migrating-sqlite-to-postgres.md ├── images ├── room_overlay.png └── user_overlay.png ├── index.html ├── index.js ├── migratedb.js ├── migrations ├── 20170319174849-create-membership-events-table.js ├── 20170319182954-create-room-links-table.js ├── 20170323032523-create-enrolled-users-table.js ├── 20170323040115-add-unlisted-flag-to-membership-events.js ├── 20170324010445-add-unlisted-flag-to-room-links.js ├── 20170326170627-add-nodes-table.js ├── 20170326170635-add-node-versions-table.js ├── 20170326170639-add-links-table.js ├── 20170326170646-add-timeline-events-table.js ├── 20170326170650-add-state-events-table.js ├── 20170329015645-migrate-states-to-events.js ├── 20170402235601-redact-known-bad-events.js ├── 20170409075941-add-primary-alias-column-to-node-versions.js ├── 20170415204327-add-vacuum-rules-to-psql.js ├── 20170415235415-add-node-meta-table.js ├── 20170415235447-add-node-meta-reference-to-nodes.js ├── 20170415235907-add-node-id-to-node-meta.js ├── 20170416000043-add-primary-alias-to-node-meta.js ├── 20170416001033-create-node-meta.js ├── 20170424035611-add-node-aliases-table.js ├── 20170905023708-add-stats-to-node-meta.js ├── 20170909012946-add-dnt-table.js └── sqls │ ├── 20170416001033-create-node-meta-down.sql │ ├── 20170416001033-create-node-meta-up.sql │ ├── pg │ ├── 20170329015645-migrate-states-to-events-up.sql │ ├── 20170402235601-redact-known-bad-events-down.sql │ ├── 20170402235601-redact-known-bad-events-up.sql │ └── 20170415204327-add-vacuum-rules-to-psql-up.sql │ └── sqlite3 │ ├── 20170329015645-migrate-states-to-events-up.sql │ ├── 20170402235601-redact-known-bad-events-down.sql │ ├── 20170402235601-redact-known-bad-events-up.sql │ └── 20170415204327-add-vacuum-rules-to-psql-up.sql ├── package-lock.json ├── package.json ├── src ├── LogService.js ├── VoyagerBot.js ├── api │ └── ApiHandler.js ├── matrix │ ├── CommandProcessor.js │ ├── MatrixClientLite.js │ └── filter_template.json └── storage │ ├── VoyagerStore.js │ └── models │ ├── dnt.js │ ├── links.js │ ├── node_aliases.js │ ├── node_meta.js │ ├── node_versions.js │ ├── nodes.js │ ├── state_events.js │ └── timeline_events.js ├── static ├── .gitkeep └── img │ ├── favicon.ico │ └── favicon.png ├── web-config ├── dev.env.js ├── index.js └── prod.env.js └── web-src ├── App.vue ├── components ├── graph │ ├── graph.vue │ ├── script.js │ ├── style.css │ └── template.html ├── landing │ ├── landing.vue │ ├── script.js │ ├── style.css │ └── template.html └── stats │ ├── script.js │ ├── sort-icon │ ├── script.js │ ├── sort-icon.vue │ ├── style.css │ └── template.html │ ├── stat-box │ ├── script.js │ ├── stat-box.vue │ ├── style.css │ └── template.html │ ├── stats.vue │ ├── style.css │ └── template.html ├── main.js └── vendorconf ├── http.js └── router.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", 5 | { 6 | "modules": false, 7 | "targets": { 8 | "browsers": [ 9 | "> 1%", 10 | "last 2 versions", 11 | "not ie <= 8" 12 | ] 13 | } 14 | } 15 | ], 16 | "stage-2" 17 | ], 18 | "plugins": [ 19 | "transform-runtime" 20 | ], 21 | "env": { 22 | "test": { 23 | "presets": [ 24 | "env", 25 | "stage-2" 26 | ], 27 | "plugins": [ 28 | "istanbul" 29 | ] 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | network_sample.json 3 | 4 | web-dist/ 5 | 6 | logs/ 7 | 8 | db/*.db 9 | db/*.db-journal 10 | start.sh 11 | 12 | app/test2.json 13 | 14 | .tmp 15 | 16 | config/production.yaml 17 | config/development.yaml 18 | 19 | config/database-orig.json 20 | config/database-target.json 21 | 22 | db/sync-token 23 | db/localstorage 24 | db/voyager_local_storage* 25 | db/mtx_client_lite_localstorage* 26 | 27 | .DS_Store 28 | node_modules/ 29 | dist/ 30 | npm-debug.log* 31 | yarn-debug.log* 32 | yarn-error.log* 33 | 34 | # Editor directories and files 35 | .idea 36 | *.suo 37 | *.ntvs* 38 | *.njsproj 39 | *.sln 40 | 41 | 42 | # Logs 43 | logs 44 | *.log 45 | npm-debug.log* 46 | 47 | # Runtime data 48 | pids 49 | *.pid 50 | *.seed 51 | 52 | # Directory for instrumented libs generated by jscoverage/JSCover 53 | lib-cov 54 | 55 | # Coverage directory used by tools like istanbul 56 | coverage 57 | 58 | # nyc test coverage 59 | .nyc_output 60 | 61 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 62 | .grunt 63 | 64 | # node-waf configuration 65 | .lock-wscript 66 | 67 | # Compiled binary addons (http://nodejs.org/api/addons.html) 68 | build/Release 69 | 70 | # Dependency directories 71 | node_modules 72 | jspm_packages 73 | 74 | # Optional npm cache directory 75 | .npm 76 | 77 | # Optional REPL history 78 | .node_repl_history 79 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | build/*.js 2 | config/*.js 3 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | // http://eslint.org/docs/user-guide/configuring 2 | 3 | module.exports = { 4 | root: true, 5 | parser: 'babel-eslint', 6 | parserOptions: { 7 | sourceType: 'module' 8 | }, 9 | env: { 10 | browser: true, 11 | }, 12 | // https://github.com/standard/standard/blob/master/docs/RULES-en.md 13 | extends: 'standard', 14 | // required to lint *.vue files 15 | plugins: [ 16 | 'html' 17 | ], 18 | // add your custom rules here 19 | 'rules': { 20 | // allow paren-less arrow functions 21 | 'arrow-parens': 0, 22 | // allow async-await 23 | 'generator-star-spacing': 0, 24 | // allow debugger during development 25 | 'no-debugger': process.env.NODE_ENV === 'production' ? 2 : 0, 26 | 'semi': 0, 27 | 'indent': 0, 28 | 'quotes': 0, 29 | 'no-extend-native': 0, 30 | } 31 | }; 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | network_sample.json 3 | 4 | web-dist/ 5 | 6 | logs/ 7 | 8 | db/*.db 9 | db/*.db-journal 10 | start.sh 11 | 12 | app/test2.json 13 | 14 | .tmp 15 | 16 | config/production.yaml 17 | config/development.yaml 18 | 19 | config/database-orig.json 20 | config/database-target.json 21 | 22 | db/sync-token 23 | db/localstorage 24 | db/voyager_local_storage* 25 | db/mtx_client_lite_localstorage* 26 | 27 | .DS_Store 28 | node_modules/ 29 | dist/ 30 | npm-debug.log* 31 | yarn-debug.log* 32 | yarn-error.log* 33 | 34 | # Editor directories and files 35 | .idea 36 | *.suo 37 | *.ntvs* 38 | *.njsproj 39 | *.sln 40 | 41 | 42 | # Logs 43 | logs 44 | *.log 45 | npm-debug.log* 46 | 47 | # Runtime data 48 | pids 49 | *.pid 50 | *.seed 51 | 52 | # Directory for instrumented libs generated by jscoverage/JSCover 53 | lib-cov 54 | 55 | # Coverage directory used by tools like istanbul 56 | coverage 57 | 58 | # nyc test coverage 59 | .nyc_output 60 | 61 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 62 | .grunt 63 | 64 | # node-waf configuration 65 | .lock-wscript 66 | 67 | # Compiled binary addons (http://nodejs.org/api/addons.html) 68 | build/Release 69 | 70 | # Dependency directories 71 | node_modules 72 | jspm_packages 73 | 74 | # Optional npm cache directory 75 | .npm 76 | 77 | # Optional REPL history 78 | .node_repl_history 79 | -------------------------------------------------------------------------------- /.postcssrc.js: -------------------------------------------------------------------------------- 1 | // https://github.com/michael-ciniawsky/postcss-load-config 2 | 3 | module.exports = { 4 | "plugins": { 5 | // to edit target browsers: use "browserslist" field in package.json 6 | "autoprefixer": {} 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16-bullseye 2 | WORKDIR /app 3 | COPY . /app 4 | RUN chown -R node /app 5 | USER node 6 | RUN npm install 7 | RUN npm run build 8 | ENV NODE_ENV=production 9 | VOLUME ["/app/db", "/app/config"] 10 | EXPOSE 8184 11 | CMD node index.js 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # matrix-voyager-bot 2 | 3 | This is a [[matrix]](https://matrix.org) bot that travels the federation simply based upon user input to rooms it participates in. 4 | 5 | Whenever the bot is invited to a room, or someone leaves an alias to a room in their message, the bot will try to join that room and add it to the network graph. If the bot is then kicked or banned, it removes the node from the graph. 6 | 7 | The bot currently goes by the name of `@voyager:t2bot.io` and has its network graph published [here](https://voyager.t2bot.io/). 8 | 9 | Questions? Ask away in [#voyager-bot:matrix.org](https://matrix.to/#/#voyager-bot:matrix.org) 10 | 11 | # Usage 12 | 13 | 1. Invite `@voyager:t2bot.io` to a room 14 | 2. Send a message with a room alias (eg: `Hello! Please join #voyager-bot:matrix.org!`) 15 | 3. Wait a moment while the bot collects some information and joins the room 16 | 17 | # Building your own 18 | 19 | *Note*: You'll need to have access to an account that the bot can use to get the access token. 20 | 21 | 1. Clone this repository 22 | 2. `npm install` 23 | 3. Copy `config/default.yaml` to `config/production.yaml` 24 | 4. Edit the values of `config/production.yaml` and `config/database.json` to match your needs 25 | 5. Run the bot with `NODE_ENV=production node index.js` 26 | 27 | # But... why? 28 | 29 | There's no real benefit to having this bot in the room, seeing as it just listens and joins other rooms. This is not intended to be a functional bot - just a fun project that builds a pretty graph. 30 | 31 | # How to remove the bot from a room 32 | 33 | There are 2 options to remove the bot from the room: 34 | 1. Kick it (someone can still invite it back or relink an alias of the room) 35 | 2. Ban it (if you'd like it to stay gone) 36 | 37 | The bot does record who kicked/banned it and what the reason given was. The bot will remove any applicable nodes from the graph. 38 | 39 | -------------------------------------------------------------------------------- /build/build.js: -------------------------------------------------------------------------------- 1 | require('./check-versions')(); 2 | 3 | process.env.NODE_ENV = 'production'; 4 | 5 | var ora = require('ora'); 6 | var rm = require('rimraf'); 7 | var path = require('path'); 8 | var chalk = require('chalk'); 9 | var webpack = require('webpack'); 10 | var config = require('../web-config'); 11 | var webpackConfig = require('./webpack.prod.conf'); 12 | 13 | var spinner = ora('building for production...'); 14 | spinner.start(); 15 | 16 | rm(path.join(config.build.assetsRoot, config.build.assetsSubDirectory), err => { 17 | if (err) throw err; 18 | webpack(webpackConfig, function (err, stats) { 19 | spinner.stop(); 20 | if (err) throw err; 21 | process.stdout.write(stats.toString({ 22 | colors: true, 23 | modules: false, 24 | children: false, 25 | chunks: false, 26 | chunkModules: false 27 | }) + '\n\n'); 28 | 29 | if (stats.hasErrors()) { 30 | console.log(chalk.red(' Build failed with errors.\n')); 31 | process.exit(1); 32 | } 33 | 34 | console.log(chalk.cyan(' Build complete.\n')); 35 | console.log(chalk.yellow( 36 | ' Tip: built files are meant to be served over an HTTP server.\n' + 37 | ' Opening index.html over file:// won\'t work.\n' 38 | )); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /build/check-versions.js: -------------------------------------------------------------------------------- 1 | var chalk = require('chalk'); 2 | var semver = require('semver'); 3 | var packageConfig = require('../package.json'); 4 | var shell = require('shelljs'); 5 | function exec(cmd) { 6 | return require('child_process').execSync(cmd).toString().trim(); 7 | } 8 | 9 | var versionRequirements = [ 10 | { 11 | name: 'node', 12 | currentVersion: semver.clean(process.version), 13 | versionRequirement: packageConfig.engines.node 14 | } 15 | ]; 16 | 17 | if (shell.which('npm')) { 18 | versionRequirements.push({ 19 | name: 'npm', 20 | currentVersion: exec('npm --version'), 21 | versionRequirement: packageConfig.engines.npm 22 | }); 23 | } 24 | 25 | module.exports = function () { 26 | var warnings = []; 27 | for (var i = 0; i < versionRequirements.length; i++) { 28 | var mod = versionRequirements[i]; 29 | if (!semver.satisfies(mod.currentVersion, mod.versionRequirement)) { 30 | warnings.push(mod.name + ': ' + 31 | chalk.red(mod.currentVersion) + ' should be ' + 32 | chalk.green(mod.versionRequirement) 33 | ); 34 | } 35 | } 36 | 37 | if (warnings.length) { 38 | console.log(''); 39 | console.log(chalk.yellow('To use this template, you must update following to modules:')); 40 | console.log(); 41 | for (var i = 0; i < warnings.length; i++) { 42 | var warning = warnings[i]; 43 | console.log(' ' + warning); 44 | } 45 | console.log(); 46 | process.exit(1); 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /build/dev-client.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | require('eventsource-polyfill'); 3 | var hotClient = require('webpack-hot-middleware/client?noInfo=true&reload=true'); 4 | 5 | hotClient.subscribe(function (event) { 6 | if (event.action === 'reload') { 7 | window.location.reload(); 8 | } 9 | }); 10 | -------------------------------------------------------------------------------- /build/dev-server.js: -------------------------------------------------------------------------------- 1 | require('./check-versions')(); 2 | 3 | var config = require('../web-config'); 4 | if (!process.env.NODE_ENV) { 5 | process.env.NODE_ENV = JSON.parse(config.dev.env.NODE_ENV); 6 | } 7 | 8 | var opn = require('opn'); 9 | var path = require('path'); 10 | var express = require('express'); 11 | var webpack = require('webpack'); 12 | var proxyMiddleware = require('http-proxy-middleware'); 13 | var webpackConfig = require('./webpack.dev.conf'); 14 | 15 | // default port where dev server listens for incoming traffic 16 | var port = process.env.PORT || config.dev.port; 17 | // automatically open browser, if not set will be false 18 | var autoOpenBrowser = !!config.dev.autoOpenBrowser; 19 | // Define HTTP proxies to your custom API backend 20 | // https://github.com/chimurai/http-proxy-middleware 21 | var proxyTable = config.dev.proxyTable; 22 | 23 | var app = express(); 24 | var compiler = webpack(webpackConfig); 25 | 26 | var devMiddleware = require('webpack-dev-middleware')(compiler, { 27 | publicPath: webpackConfig.output.publicPath, 28 | quiet: true 29 | }); 30 | 31 | var hotMiddleware = require('webpack-hot-middleware')(compiler, { 32 | log: false, 33 | heartbeat: 2000 34 | }); 35 | // force page reload when html-webpack-plugin template changes 36 | compiler.plugin('compilation', function (compilation) { 37 | compilation.plugin('html-webpack-plugin-after-emit', function (data, cb) { 38 | hotMiddleware.publish({action: 'reload'}); 39 | cb(); 40 | }) 41 | }); 42 | 43 | // proxy api requests 44 | Object.keys(proxyTable).forEach(function (context) { 45 | var options = proxyTable[context]; 46 | if (typeof options === 'string') { 47 | options = {target: options}; 48 | } 49 | app.use(proxyMiddleware(options.filter || context, options)); 50 | }); 51 | 52 | // handle fallback for HTML5 history API 53 | app.use(require('connect-history-api-fallback')()); 54 | 55 | // serve webpack bundle output 56 | app.use(devMiddleware); 57 | 58 | // enable hot-reload and state-preserving 59 | // compilation error display 60 | app.use(hotMiddleware); 61 | 62 | // serve pure static assets 63 | var staticPath = path.posix.join(config.dev.assetsPublicPath, config.dev.assetsSubDirectory); 64 | app.use(staticPath, express.static('./static')); 65 | 66 | var uri = 'http://localhost:' + port; 67 | 68 | var _resolve; 69 | var readyPromise = new Promise(resolve => { 70 | _resolve = resolve; 71 | }); 72 | 73 | console.log('> Starting dev server...'); 74 | devMiddleware.waitUntilValid(() => { 75 | console.log('> Listening at ' + uri + '\n'); 76 | // when env is testing, don't need open it 77 | if (autoOpenBrowser && process.env.NODE_ENV !== 'testing') { 78 | opn(uri); 79 | } 80 | _resolve(); 81 | }); 82 | 83 | var server = app.listen(port); 84 | 85 | module.exports = { 86 | ready: readyPromise, 87 | close: () => { 88 | server.close(); 89 | } 90 | }; 91 | -------------------------------------------------------------------------------- /build/utils.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var config = require('../web-config'); 3 | var ExtractTextPlugin = require('extract-text-webpack-plugin'); 4 | 5 | exports.assetsPath = function (_path) { 6 | var assetsSubDirectory = process.env.NODE_ENV === 'production' 7 | ? config.build.assetsSubDirectory 8 | : config.dev.assetsSubDirectory; 9 | return path.posix.join(assetsSubDirectory, _path); 10 | }; 11 | 12 | exports.cssLoaders = function (options) { 13 | options = options || {}; 14 | 15 | var cssLoader = { 16 | loader: 'css-loader', 17 | options: { 18 | minimize: process.env.NODE_ENV === 'production', 19 | sourceMap: options.sourceMap 20 | } 21 | }; 22 | 23 | // generate loader string to be used with extract text plugin 24 | function generateLoaders(loader, loaderOptions) { 25 | var loaders = [cssLoader]; 26 | if (loader) { 27 | loaders.push({ 28 | loader: loader + '-loader', 29 | options: Object.assign({}, loaderOptions, { 30 | sourceMap: options.sourceMap 31 | }) 32 | }); 33 | } 34 | 35 | // Extract CSS when that option is specified 36 | // (which is the case during production build) 37 | if (options.extract) { 38 | return ExtractTextPlugin.extract({ 39 | use: loaders, 40 | fallback: 'vue-style-loader' 41 | }); 42 | } else { 43 | return ['vue-style-loader'].concat(loaders); 44 | } 45 | } 46 | 47 | // https://vue-loader.vuejs.org/en/configurations/extract-css.html 48 | return { 49 | css: generateLoaders(), 50 | postcss: generateLoaders(), 51 | less: generateLoaders('less'), 52 | sass: generateLoaders('sass', {indentedSyntax: true}), 53 | scss: generateLoaders('sass'), 54 | stylus: generateLoaders('stylus'), 55 | styl: generateLoaders('stylus') 56 | }; 57 | }; 58 | 59 | // Generate loaders for standalone style files (outside of .vue) 60 | exports.styleLoaders = function (options) { 61 | var output = []; 62 | var loaders = exports.cssLoaders(options); 63 | for (var extension in loaders) { 64 | var loader = loaders[extension]; 65 | output.push({ 66 | test: new RegExp('\\.' + extension + '$'), 67 | use: loader 68 | }); 69 | } 70 | return output; 71 | }; 72 | -------------------------------------------------------------------------------- /build/vue-loader.conf.js: -------------------------------------------------------------------------------- 1 | var utils = require('./utils'); 2 | var config = require('../web-config'); 3 | var isProduction = process.env.NODE_ENV === 'production'; 4 | 5 | module.exports = { 6 | loaders: utils.cssLoaders({ 7 | sourceMap: isProduction 8 | ? config.build.productionSourceMap 9 | : config.dev.cssSourceMap, 10 | extract: isProduction 11 | }), 12 | transformToRequire: { 13 | video: 'src', 14 | source: 'src', 15 | img: 'src', 16 | image: 'xlink:href' 17 | } 18 | }; 19 | -------------------------------------------------------------------------------- /build/webpack.base.conf.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var fs = require('fs'); 3 | var utils = require('./utils'); 4 | var config = require('../web-config'); 5 | var vueLoaderConfig = require('./vue-loader.conf'); 6 | 7 | function resolve(dir) { 8 | return path.join(__dirname, '..', dir); 9 | } 10 | 11 | module.exports = { 12 | entry: { 13 | app: './web-src/main.js' 14 | }, 15 | output: { 16 | path: config.build.assetsRoot, 17 | filename: '[name].js', 18 | publicPath: process.env.NODE_ENV === 'production' 19 | ? config.build.assetsPublicPath 20 | : config.dev.assetsPublicPath 21 | }, 22 | resolve: { 23 | extensions: ['.js', '.vue', '.json'], 24 | alias: { 25 | 'vue$': 'vue/dist/vue.esm.js', 26 | '@': resolve('web-src'), 27 | }, 28 | symlinks: false 29 | }, 30 | module: { 31 | rules: [ 32 | { 33 | test: /\.(js|vue)$/, 34 | loader: 'eslint-loader', 35 | enforce: 'pre', 36 | include: [resolve('web-src'), resolve('test')], 37 | options: { 38 | formatter: require('eslint-friendly-formatter') 39 | } 40 | }, 41 | { 42 | test: /\.vue$/, 43 | loader: 'vue-loader', 44 | options: vueLoaderConfig 45 | }, 46 | { 47 | test: /\.js$/, 48 | loader: 'babel-loader', 49 | include: [resolve('web-src'), resolve('test')] 50 | }, 51 | { 52 | test: /\.(png|jpe?g|gif|svg|ico)(\?.*)?$/, 53 | loader: 'url-loader', 54 | options: { 55 | limit: 10000, 56 | name: utils.assetsPath('img/[name].[hash:7].[ext]') 57 | } 58 | }, 59 | { 60 | test: /\.(mp4|webm|ogg|mp3|wav|flac|aac)(\?.*)?$/, 61 | loader: 'url-loader', 62 | options: { 63 | limit: 10000, 64 | name: utils.assetsPath('media/[name].[hash:7].[ext]') 65 | } 66 | }, 67 | { 68 | test: /\.(woff2?|eot|ttf|otf)(\?.*)?$/, 69 | loader: 'url-loader', 70 | options: { 71 | limit: 10000, 72 | name: utils.assetsPath('fonts/[name].[hash:7].[ext]') 73 | } 74 | } 75 | ] 76 | } 77 | }; 78 | -------------------------------------------------------------------------------- /build/webpack.dev.conf.js: -------------------------------------------------------------------------------- 1 | var utils = require('./utils'); 2 | var webpack = require('webpack'); 3 | var config = require('../web-config'); 4 | var merge = require('webpack-merge'); 5 | var baseWebpackConfig = require('./webpack.base.conf'); 6 | var HtmlWebpackPlugin = require('html-webpack-plugin'); 7 | var FriendlyErrorsPlugin = require('friendly-errors-webpack-plugin'); 8 | var DashboardPlugin = require("webpack-dashboard/plugin"); 9 | 10 | // add hot-reload related code to entry chunks 11 | Object.keys(baseWebpackConfig.entry).forEach(function (name) { 12 | baseWebpackConfig.entry[name] = ['./build/dev-client'].concat(baseWebpackConfig.entry[name]) 13 | }); 14 | 15 | module.exports = merge(baseWebpackConfig, { 16 | module: { 17 | rules: utils.styleLoaders({sourceMap: config.dev.cssSourceMap}) 18 | }, 19 | // cheap-module-eval-source-map is faster for development 20 | devtool: '#cheap-module-eval-source-map', 21 | plugins: [ 22 | new DashboardPlugin(), 23 | new webpack.DefinePlugin({ 24 | 'process.env': config.dev.env 25 | }), 26 | // https://github.com/glenjamin/webpack-hot-middleware#installation--usage 27 | new webpack.HotModuleReplacementPlugin(), 28 | new webpack.NoEmitOnErrorsPlugin(), 29 | // https://github.com/ampedandwired/html-webpack-plugin 30 | new HtmlWebpackPlugin({ 31 | filename: 'index.html', 32 | template: 'index.html', 33 | inject: true 34 | }), 35 | new FriendlyErrorsPlugin() 36 | ] 37 | }); 38 | -------------------------------------------------------------------------------- /build/webpack.prod.conf.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var utils = require('./utils'); 3 | var webpack = require('webpack'); 4 | var config = require('../web-config'); 5 | var merge = require('webpack-merge'); 6 | var baseWebpackConfig = require('./webpack.base.conf'); 7 | var CopyWebpackPlugin = require('copy-webpack-plugin'); 8 | var HtmlWebpackPlugin = require('html-webpack-plugin'); 9 | var ExtractTextPlugin = require('extract-text-webpack-plugin'); 10 | var OptimizeCSSPlugin = require('optimize-css-assets-webpack-plugin'); 11 | 12 | var env = config.build.env; 13 | 14 | var webpackConfig = merge(baseWebpackConfig, { 15 | module: { 16 | rules: utils.styleLoaders({ 17 | sourceMap: config.build.productionSourceMap, 18 | extract: true 19 | }) 20 | }, 21 | devtool: config.build.productionSourceMap ? '#source-map' : false, 22 | output: { 23 | path: config.build.assetsRoot, 24 | filename: utils.assetsPath('js/[name].[chunkhash].js'), 25 | chunkFilename: utils.assetsPath('js/[id].[chunkhash].js') 26 | }, 27 | plugins: [ 28 | // http://vuejs.github.io/vue-loader/en/workflow/production.html 29 | new webpack.DefinePlugin({ 30 | 'process.env': env 31 | }), 32 | new webpack.optimize.UglifyJsPlugin({ 33 | compress: { 34 | warnings: false 35 | }, 36 | sourceMap: true 37 | }), 38 | // extract css into its own file 39 | new ExtractTextPlugin({ 40 | filename: utils.assetsPath('css/[name].[contenthash].css') 41 | }), 42 | // Compress extracted CSS. We are using this plugin so that possible 43 | // duplicated CSS from different components can be deduped. 44 | new OptimizeCSSPlugin({ 45 | cssProcessorOptions: { 46 | safe: true 47 | } 48 | }), 49 | // generate dist index.html with correct asset hash for caching. 50 | // you can customize output by editing /index.html 51 | // see https://github.com/ampedandwired/html-webpack-plugin 52 | new HtmlWebpackPlugin({ 53 | filename: config.build.index, 54 | template: 'index.html', 55 | inject: true, 56 | minify: { 57 | removeComments: true, 58 | collapseWhitespace: true, 59 | removeAttributeQuotes: true 60 | // more options: 61 | // https://github.com/kangax/html-minifier#options-quick-reference 62 | }, 63 | // necessary to consistently work with multiple chunks via CommonsChunkPlugin 64 | chunksSortMode: 'dependency' 65 | }), 66 | // keep module.id stable when vender modules does not change 67 | new webpack.HashedModuleIdsPlugin(), 68 | // split vendor js into its own file 69 | new webpack.optimize.CommonsChunkPlugin({ 70 | name: 'vendor', 71 | minChunks: function (module, count) { 72 | // any required modules inside node_modules are extracted to vendor 73 | return ( 74 | module.resource && 75 | /\.js$/.test(module.resource) && 76 | module.resource.indexOf( 77 | path.join(__dirname, '../node_modules') 78 | ) === 0 79 | ) 80 | } 81 | }), 82 | // extract webpack runtime and module manifest to its own file in order to 83 | // prevent vendor hash from being updated whenever app bundle is updated 84 | new webpack.optimize.CommonsChunkPlugin({ 85 | name: 'manifest', 86 | chunks: ['vendor'] 87 | }), 88 | // copy custom static assets 89 | new CopyWebpackPlugin([ 90 | { 91 | from: path.resolve(__dirname, '../static'), 92 | to: config.build.assetsSubDirectory, 93 | ignore: ['.*'] 94 | } 95 | ]) 96 | ] 97 | }); 98 | 99 | if (config.build.productionGzip) { 100 | var CompressionWebpackPlugin = require('compression-webpack-plugin'); 101 | 102 | webpackConfig.plugins.push( 103 | new CompressionWebpackPlugin({ 104 | asset: '[path].gz[query]', 105 | algorithm: 'gzip', 106 | test: new RegExp( 107 | '\\.(' + 108 | config.build.productionGzipExtensions.join('|') + 109 | ')$' 110 | ), 111 | threshold: 10240, 112 | minRatio: 0.8 113 | }) 114 | ); 115 | } 116 | 117 | if (config.build.bundleAnalyzerReport) { 118 | var BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; 119 | webpackConfig.plugins.push(new BundleAnalyzerPlugin()); 120 | } 121 | 122 | module.exports = webpackConfig; 123 | -------------------------------------------------------------------------------- /config/database.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultEnv": { 3 | "ENV": "NODE_ENV" 4 | }, 5 | "development": { 6 | "driver": "sqlite3", 7 | "filename": "db/development.db" 8 | }, 9 | "production": { 10 | "driver": "pg", 11 | "host": "localhost", 12 | "database": "voyager", 13 | "username": "voyager", 14 | "password": "voyager" 15 | } 16 | } -------------------------------------------------------------------------------- /config/default.yaml: -------------------------------------------------------------------------------- 1 | # The configuration for the matrix portion of the bot. The bot's matrix account must already 2 | # be created proiot to running the bot. 3 | matrix: 4 | # The homeserver URL for the bot. For example, https://matrix.org 5 | homeserverUrl: "https://t2bot.io" 6 | 7 | # The access token for the bot that authenticates it on the above homeserver. 8 | accessToken: "YOUR_TOKEN_HERE" 9 | 10 | # The Matrix ID for the bot's account. For example, @voyager:t2bot.io 11 | userId: "@voyager:t2bot.io" 12 | 13 | # The web settings for the bot (serves the graph) 14 | web: 15 | # The port to run the webserver on 16 | port: 8184 17 | 18 | # The address to bind to (0.0.0.0 for all interfaces) 19 | address: '0.0.0.0' 20 | 21 | # Advanced settings to control behaviour of the bot 22 | bot: 23 | # if enabled, the bot will process all node update requests on startup. This is disabled 24 | # by default to prevent the bot from attempting to use significant resources on startup. 25 | processNodeUpdatesOnStartup: false 26 | 27 | # If node updates are enabled, which nodes should be updated on startup? 28 | nodeUpdatesOnStartup: 29 | rooms: true 30 | users: true 31 | 32 | # Settings for controlling how logging works 33 | logging: 34 | file: logs/voyager.log 35 | console: true 36 | consoleLevel: info 37 | fileLevel: verbose 38 | rotate: 39 | size: 52428800 # bytes, default is 50mb 40 | count: 5 -------------------------------------------------------------------------------- /db/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/db/.gitkeep -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | # Voyager API Docs 2 | 3 | *TODO: Convert this to swagger or some other proper documentation* 4 | 5 | All resources can be reached with [https://voyager.t2bot.io](https://voyager.t2bot.io) 6 | 7 | ### Link Types 8 | * `message` - A message in a room that contained an alias or room ID 9 | * `topic` - A topic in the source room contained an alias or room ID 10 | * `invite` - The bot was invited to the target room by the source user 11 | * `self_link` - The source user decided to link themselves to the target room 12 | 13 | **Exist, but selectively exposed by endpoints** 14 | * `kick` - A user has kicked the bot from the room 15 | * `ban` - A user has banned the bot from the room 16 | * `soft_kick` - A user has asked the bot to leave peacefully (does not redact room node) 17 | 18 | *Note:* Most endpoints do not support these extra link types. Endpoints that do will indicate as such. 19 | 20 | ### Event Types 21 | * `node_added` - A new node has been created 22 | * `node_updated` - An existing node has been updated, see metadata 23 | * `node_removed` - An existing node has been removed 24 | * `link_added` - A link has been created (may occur multiple times for each source and target pair) 25 | * `link_removed` - An existing link has been removed 26 | 27 | ## `GET /api/v1/network` 28 | 29 | Gets information about the graph network. 30 | 31 | **Request parameters:** 32 | * `limit` - `int`, the number of results to retrieve (up to 10,000). Default is 1000, minimum 1. 33 | * `since` - `long`, the timestamp to filter against. Default is the beginning of time. 34 | 35 | **Response:** 36 | ```javascript 37 | { 38 | total: 1000, // total links in this result set 39 | remaining: 10, // the number of results not included in this response that are after `since` 40 | redacted: 0, // the number of results marked as redacted (these will not be in the results) 41 | hidden: 0, // the number of results marked as invisible (these will not be in the results) 42 | results: { 43 | // Note: This only includes nodes that are relevant to the `links` 44 | nodes: [{ 45 | id: 1234, 46 | firstIntroduced: 1234567890, // milliseconds since epoch 47 | meta: { 48 | type: 'user', // or 'room' 49 | displayName: "Some User", 50 | avatarUrl: "https://...", // not included if the node doesn't have an avatar 51 | objectId: "@user:domain.com", // Rooms will be a Room ID. Anonymous nodes don't have this. 52 | isAnonymous: false 53 | } 54 | }], 55 | 56 | // Links now all have a weight of 1 and may be duplicated (source to target). 57 | links: [{ 58 | id: 1234, 59 | timestamp: 1234567890, // milliseconds since epoch 60 | meta: { 61 | sourceNodeId: 1234, 62 | targetNodeId: 1235, 63 | type: 'message' // any of the normal link types 64 | } 65 | }] 66 | } 67 | } 68 | ``` 69 | 70 | If there are no events/links for the given range, the following is returned as `200 OK`: 71 | ```javascript 72 | { 73 | total: 0, 74 | remaining: 0, 75 | results: { 76 | nodes: [], 77 | links: [] 78 | } 79 | } 80 | ``` 81 | 82 | ## `GET /api/v1/nodes` 83 | 84 | Gets all known nodes (users or rooms). 85 | 86 | **Response:** 87 | ```javascript 88 | [{ 89 | id: 1234, 90 | firstIntroduced: 1234567890 // milliseconds since epoch 91 | meta: { 92 | type: 'user', // or 'room' 93 | displayName: "Some User", 94 | avatarUrl: "https://...", // not included if the node doesn't have an avatar 95 | objectId: "@user:domain.com", // Rooms will be a Room ID. Anonymous nodes don't have this. 96 | isAnonymous: false 97 | } 98 | }] 99 | ``` 100 | 101 | ## `GET /api/v1/nodes/{id}` 102 | 103 | Gets information about a particular node 104 | 105 | **Response:** 106 | ```javascript 107 | { 108 | id: 1234, 109 | firstIntroduced: 1234567890 // milliseconds since epoch 110 | meta: { 111 | type: 'user', // or 'room' 112 | displayName: "Some User", 113 | avatarUrl: "https://...", // not included if the node doesn't have an avatar 114 | objectId: "@user:domain.com", // Rooms will be a Room ID. Anonymous nodes don't have this. 115 | isAnonymous: false 116 | } 117 | } 118 | ``` 119 | 120 | If the node is not found, `404 Not Found` is returned. 121 | 122 | ## `GET /api/v1/nodes/publicRooms` 123 | 124 | Gets a list of public room nodes with extra information about their composition. 125 | 126 | **Response:** 127 | ```javascript 128 | [ 129 | { 130 | id: 1234, 131 | firstIntroduced: 1234567890, // milliseconds since epoch 132 | meta: { 133 | type: 'room', 134 | displayName: 'Some Display Name', 135 | avatarUrl: 'https://...', // not included if the avatar is missing 136 | objectId: '!someroom:domain.com', 137 | isAnonymous: false, 138 | primaryAlias: '#somewhere:domain.com', // always present for rooms on this endpoint 139 | stats: { 140 | users: 123, // number of users in the room (cached) 141 | servers: 123, // number of servers in the room (cached) 142 | aliases: 123 // number of aliases for the room (cached) 143 | } 144 | } 145 | } 146 | ] 147 | ``` 148 | 149 | ## `GET /api/v1/events` 150 | 151 | Gets all known events. This will include state events for the 'extra' link types. 152 | 153 | **Request Query Params** 154 | * `limit` - `int`, the number of results to retrieve (up to 10,000). Default is 1000, minimum 1. 155 | * `since` - `long`, the timestamp to filter against. Default is the beginning of time. 156 | 157 | **Response:** 158 | ```javascript 159 | { 160 | total: 1000, 161 | remaining: 10, // the number of results not included in this response that are after `since` 162 | results: { 163 | events: [{ 164 | id: 1234, // an ID for this event (sequential to dedupe timestamp) 165 | type: "node_updated", // any of the event types available 166 | timestamp: 1234567890, // milliseconds since epoch 167 | nodeId: 1234, 168 | meta: { // may not be included if it doesn't apply to the event, or if the relevant node no longer exists 169 | displayName: 'Some Name', 170 | avatarUrl: 'https://...', 171 | isAnonymous: false 172 | } 173 | }] 174 | } 175 | } 176 | ``` 177 | 178 | If no events were found for the given range, the following is returned as `200 OK`: 179 | ```javascript 180 | { 181 | total: 0, 182 | remaining: 0, 183 | results: { 184 | events: [] 185 | } 186 | } 187 | ``` 188 | 189 | ### Some examples of other event types 190 | 191 | **node_added** 192 | ```javascript 193 | { 194 | id: 1234, 195 | type: "node_added", 196 | timestamp: 1234567890, // milliseconds since epoch 197 | nodeId: 1234, 198 | meta: { // Not present if the node no longer exists on the graph 199 | displayName: 'Some Name', 200 | avatarUrl: 'https://...', 201 | isAnonymous: false, 202 | type: 'room', // or 'user' 203 | objectId: '!room:domain.com' // only present if not anonymous 204 | } 205 | } 206 | ``` 207 | 208 | **node_updated** 209 | ```javascript 210 | { 211 | id: 1234, 212 | type: "node_updated", 213 | timestamp: 1234567890, // milliseconds since epoch 214 | nodeId: 1234, 215 | meta: { // Not present if the node no longer exists on the graph 216 | // One or more of these fields will be present, depending on the change 217 | displayName: 'Some Name', 218 | avatarUrl: 'https://...', 219 | isAnonymous: false 220 | } 221 | } 222 | ``` 223 | 224 | **node_removed** 225 | ```javascript 226 | { 227 | id: 1234, 228 | type: "node_removed", 229 | timestamp: 1234567890, // milliseconds since epoch 230 | nodeId: 1234 231 | } 232 | ``` 233 | 234 | **link_added** 235 | ```javascript 236 | { 237 | id: 1234, 238 | type: "link_added", 239 | timestamp: 1234567890, // milliseconds since epoch 240 | linkId: 1234, 241 | meta: { 242 | sourceNodeId: 1234, 243 | targetNodeId: 1235, 244 | type: 'message' // any of the link types 245 | } 246 | } 247 | ``` 248 | 249 | **link_removed** 250 | ```javascript 251 | { 252 | id: 1234, 253 | type: "link_removed", 254 | timestamp: 1234567890, // milliseconds since epoch 255 | linkId: 1234, 256 | meta: { 257 | sourceNodeId: 1234, 258 | targetNodeId: 1235, 259 | type: 'message' // any of the link types 260 | } 261 | } 262 | ``` 263 | 264 | ## `GET /api/v1/stats` 265 | 266 | Gets various stats about voyager's participation in the network. 267 | 268 | **Response:** 269 | ```javascript 270 | { 271 | users: 123, // The total number of users discovered 272 | rooms: 123, // The total number of rooms discovered 273 | aliases: 123, // The total number of room aliases discovered (current) 274 | servers: 123, // The total number of discovered servers 275 | mentions: 123 // The total number of times a room alias has been mentioned (successfully) 276 | } 277 | ``` -------------------------------------------------------------------------------- /docs/migrating-sqlite-to-postgres.md: -------------------------------------------------------------------------------- 1 | # Migrating from sqlite3 to postgres 2 | 3 | 1. Copy `config/database.json` to `config/database-orig.json` 4 | 2. Edit `config/database.json` to be the 'new' database settings 5 | 3. Edit `config/database-orig.json` to be the 'old' database settings 6 | 4. Run `NODE_ENV=production node migratedb.js config/database-orig.json config/database.json` (assuming you're migrating your `production` database) 7 | * This might take a while depending on the size of your database. 8 | 5. Run the bot normally -------------------------------------------------------------------------------- /images/room_overlay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/images/room_overlay.png -------------------------------------------------------------------------------- /images/user_overlay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/images/user_overlay.png -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Voyager 8 | 9 | 10 | 11 | 12 |
13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var VoyagerBot = require("./src/VoyagerBot"); 2 | var VoyagerStore = require("./src/storage/VoyagerStore"); 3 | var ApiHandler = require("./src/api/ApiHandler"); 4 | var log = require("./src/LogService"); 5 | 6 | log.info("index", "Bootstrapping bot"); 7 | var db = new VoyagerStore(); 8 | db.prepare().then(() => { 9 | var bot = new VoyagerBot(db); 10 | bot.start(); 11 | 12 | var api = new ApiHandler(db, bot); 13 | api.start(); 14 | }); 15 | -------------------------------------------------------------------------------- /migratedb.js: -------------------------------------------------------------------------------- 1 | var DBMigrate = require("db-migrate"); 2 | var log = require("./src/LogService"); 3 | var Sequelize = require('sequelize'); 4 | var DbModels = require("./src/storage/VoyagerStore").models; 5 | 6 | var args = process.argv.slice(2); 7 | if (args.length !== 2) { 8 | log.error("migratedb", "Missing source and/or target configuration"); 9 | process.exit(1); 10 | } 11 | 12 | var sourceDbConfig = require("./" + args[0]); 13 | var targetDbConfig = require("./" + args[1]); 14 | 15 | var env = process.env.NODE_ENV || 'development'; 16 | 17 | var sourceDbConfigEnv = sourceDbConfig[env]; 18 | var targetDbConfigEnv = targetDbConfig[env]; 19 | 20 | function setupOrm(configPath, dbConfigEnv) { 21 | var driverMap = { 22 | // 'sqlite3': 'sqlite', 23 | 'pg': 'postgres' 24 | }; 25 | process.env.VOYAGER_DB_CONF_MIGRATE = "../" + configPath; 26 | var dbMigrate = DBMigrate.getInstance(true, { 27 | config: configPath, 28 | env: env 29 | }); 30 | return dbMigrate.up().then(() => { 31 | var opts = { 32 | host: dbConfigEnv.host || 'localhost', 33 | dialect: driverMap[dbConfigEnv.driver], 34 | pool: { 35 | max: 5, 36 | min: 0, 37 | idle: 10000 38 | }, 39 | logging: i => log.info("migratedb [SQL: " + configPath + "]", i) 40 | }; 41 | 42 | if (opts.dialect == 'sqlite') 43 | opts.storage = dbConfigEnv.filename; 44 | 45 | return new Sequelize(dbConfigEnv.database || 'voyager', dbConfigEnv.username, dbConfigEnv.password, opts); 46 | }); 47 | } 48 | 49 | function bindModels(orm) { 50 | var models = {}; 51 | models.Links = orm.import(__dirname + "/src/storage/models/links"); 52 | models.NodeVersions = orm.import(__dirname + "/src/storage/models/node_versions"); 53 | models.Nodes = orm.import(__dirname + "/src/storage/models/nodes"); 54 | models.NodeMeta = orm.import(__dirname + "/src/storage/models/node_meta"); 55 | models.StateEvents = orm.import(__dirname + "/src/storage/models/state_events"); 56 | models.TimelineEvents = orm.import(__dirname + "/src/storage/models/timeline_events"); 57 | return models; 58 | } 59 | 60 | function promiseIter(set, fn) { 61 | return new Promise((resolve, reject) => { 62 | var i = 0; 63 | var handler = () => { 64 | i++; 65 | if (i >= set.length) resolve(); 66 | else return fn(set[i]).then(handler, reject); 67 | }; 68 | fn(set[i]).then(handler, reject); 69 | }); 70 | } 71 | 72 | var source = null; 73 | var target = null; 74 | var sourceModels = null; 75 | var targetModels = null; 76 | 77 | // This process is incredibly slow, however it is only intended to be run once. 78 | setupOrm(args[0], sourceDbConfigEnv).then(orm => source = orm).then(() => sourceModels = bindModels(source)).then(() => { 79 | return setupOrm(args[1], targetDbConfigEnv).then(orm => target = orm).then(() => targetModels = bindModels(target)); 80 | }).then(() => { 81 | log.info("migratedb", "Fetching all Nodes..."); 82 | return sourceModels.Nodes.findAll(); 83 | }).then(nodes => { 84 | return promiseIter(nodes.map(r => new DbModels.Node(r)), n => { 85 | var nodeMeta = null; 86 | return sourceModels.NodeMeta.findOne({where: {nodeId: n.id}}) 87 | .then(meta => targetModels.NodeMeta.create(new DbModels.NodeMeta(meta))) 88 | .then(meta => { 89 | nodeMeta = meta; 90 | n.firstTimestamp = new Date(n.firstTimestamp); 91 | n.nodeMetaId = meta.id; 92 | return targetModels.Nodes.create(n); 93 | }).then(node => { 94 | nodeMeta.nodeId = node.id; 95 | return nodeMeta.save(); 96 | }); 97 | }); 98 | }).then(() => { 99 | log.info("migratedb", "Fetching all Links..."); 100 | return sourceModels.Links.findAll(); 101 | }).then(links => { 102 | return promiseIter(links.map(r => new DbModels.Link(r)), k => { 103 | k.timestamp = new Date(k.timestamp); 104 | return targetModels.Links.create(k); 105 | }); 106 | }).then(() => { 107 | log.info("migratedb", "Fetching all Node Versions..."); 108 | return sourceModels.NodeVersions.findAll(); 109 | }).then(versions => { 110 | return promiseIter(versions.map(r => new DbModels.NodeVersion(r)), v => { 111 | return targetModels.NodeVersions.create(v); 112 | }); 113 | }).then(() => { 114 | log.info("migratedb", "Fetching all Timeline Events..."); 115 | return sourceModels.TimelineEvents.findAll(); 116 | }).then(events => { 117 | return promiseIter(events.map(r => new DbModels.TimelineEvent(r)), e => { 118 | e.timestamp = new Date(e.timestamp); 119 | return targetModels.TimelineEvents.create(e); 120 | }); 121 | }).then(() => { 122 | log.info("migratedb", "Fetching all State Events..."); 123 | return sourceModels.StateEvents.findAll(); 124 | }).then(events => { 125 | return promiseIter(events.map(r => new DbModels.StateEvent(r)), e => { 126 | e.timestamp = new Date(e.timestamp); 127 | return targetModels.StateEvents.create(e); 128 | }); 129 | }).then(() => { 130 | if (targetDbConfigEnv.driver == 'pg') { 131 | log.info("migratedb", "Updating sequences..."); 132 | return target.query("SELECT setval('links_id_seq', COALESCE((SELECT MAX(id)+1 FROM links), 1), false)", {type: Sequelize.QueryTypes.SELECT}) 133 | .then(() => target.query("SELECT setval('node_versions_id_seq', COALESCE((SELECT MAX(id)+1 FROM node_versions), 1), false)", {type: Sequelize.QueryTypes.SELECT})) 134 | .then(() => target.query("SELECT setval('nodes_id_seq', COALESCE((SELECT MAX(id)+1 FROM nodes), 1), false)", {type: Sequelize.QueryTypes.SELECT})) 135 | .then(() => target.query("SELECT setval('node_meta_id_seq', COALESCE((SELECT MAX(id)+1 FROM node_meta), 1), false)", {type: Sequelize.QueryTypes.SELECT})) 136 | .then(() => target.query("SELECT setval('state_events_id_seq', COALESCE((SELECT MAX(id)+1 FROM state_events), 1), false)", {type: Sequelize.QueryTypes.SELECT})) 137 | .then(() => target.query("SELECT setval('timeline_events_id_seq', COALESCE((SELECT MAX(id)+1 FROM timeline_events), 1), false)", {type: Sequelize.QueryTypes.SELECT})) 138 | } 139 | }).then(() => { 140 | log.info("migratedb", "Done migration. Cleaning up..."); 141 | }).catch(err => { 142 | log.error("migratedb", err); 143 | throw err; 144 | }); 145 | -------------------------------------------------------------------------------- /migrations/20170319174849-create-membership-events-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('membership_events', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true}, 20 | event_id: 'string', 21 | type: 'string', 22 | sender: 'string', 23 | room_id: 'string', 24 | timestamp: 'timestamp', 25 | message: 'string', 26 | error: 'string' 27 | }); 28 | }; 29 | 30 | exports.down = function (db) { 31 | return db.dropTable('membership_events'); 32 | }; 33 | 34 | exports._meta = { 35 | "version": 1 36 | }; 37 | -------------------------------------------------------------------------------- /migrations/20170319182954-create-room-links-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('room_links', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true}, 20 | event_id: 'string', 21 | parsed_value: 'string', 22 | type: 'string', 23 | sender: 'string', 24 | to_room_id: 'string', 25 | from_room_id: 'string', 26 | timestamp: 'timestamp', 27 | message: 'string', 28 | error: 'string' 29 | }); 30 | }; 31 | 32 | exports.down = function (db) { 33 | return db.dropTable('room_links'); 34 | }; 35 | 36 | exports._meta = { 37 | "version": 1 38 | }; 39 | -------------------------------------------------------------------------------- /migrations/20170323032523-create-enrolled-users-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('enrolled_users', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true}, 20 | user_id: 'string' 21 | }); 22 | }; 23 | 24 | exports.down = function (db) { 25 | return db.dropTable('enrolled_users'); 26 | }; 27 | 28 | exports._meta = { 29 | "version": 1 30 | }; 31 | -------------------------------------------------------------------------------- /migrations/20170323040115-add-unlisted-flag-to-membership-events.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('membership_events', 'unlisted', {type: 'boolean', defaultValue: false}); 19 | }; 20 | 21 | exports.down = function (db) { 22 | return db.removeColumn('membership_events', 'unlisted'); 23 | }; 24 | 25 | exports._meta = { 26 | "version": 1 27 | }; 28 | -------------------------------------------------------------------------------- /migrations/20170324010445-add-unlisted-flag-to-room-links.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('room_links', 'unlisted', {type: 'boolean', defaultValue: false}); 19 | }; 20 | 21 | exports.down = function (db) { 22 | return db.removeColumn('room_links', 'unlisted'); 23 | }; 24 | 25 | exports._meta = { 26 | "version": 1 27 | }; 28 | -------------------------------------------------------------------------------- /migrations/20170326170627-add-nodes-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('nodes', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | type: {type: 'string', notNull: true}, 21 | objectId: {type: 'string', notNull: true}, 22 | isReal: {type: 'boolean', notNull: true}, 23 | isRedacted: {type: 'boolean', notNull: true}, 24 | firstTimestamp: {type: 'timestamp', notNull: true} 25 | }); 26 | }; 27 | 28 | exports.down = function (db) { 29 | return db.dropTable('nodes'); 30 | }; 31 | 32 | exports._meta = { 33 | "version": 1 34 | }; 35 | -------------------------------------------------------------------------------- /migrations/20170326170635-add-node-versions-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('node_versions', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | nodeId: {type: 'int', foreignKey: {name: 'fk_node_version_node', table: 'nodes', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, notNull: true}, 21 | displayName: {type: 'string', notNull: false}, 22 | avatarUrl: {type: 'string', notNull: false}, 23 | isAnonymous: {type: 'boolean', notNull: false} 24 | }); 25 | }; 26 | 27 | exports.down = function (db) { 28 | return db.dropTable('node_versions'); 29 | }; 30 | 31 | exports._meta = { 32 | "version": 1 33 | }; 34 | -------------------------------------------------------------------------------- /migrations/20170326170639-add-links-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('links', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | type: {type: 'string', notNull: true}, 21 | sourceNodeId: { 22 | type: 'int', 23 | foreignKey: {name: 'fk_links_source_node_id_nodes_node_id', table: 'nodes', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 24 | notNull: true 25 | }, 26 | targetNodeId: { 27 | type: 'int', 28 | foreignKey: {name: 'fk_links_target_node_id_nodes_node_id', table: 'nodes', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 29 | notNull: true 30 | }, 31 | timestamp: {type: 'timestamp', notNull: true}, 32 | isVisible: {type: 'boolean', notNull: true}, 33 | isRedacted: {type: 'boolean', notNull: true} 34 | }); 35 | }; 36 | 37 | exports.down = function (db) { 38 | return db.dropTable('links'); 39 | }; 40 | 41 | exports._meta = { 42 | "version": 1 43 | }; 44 | -------------------------------------------------------------------------------- /migrations/20170326170646-add-timeline-events-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('timeline_events', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | linkId: { 21 | type: 'int', 22 | foreignKey: {name: 'fk_timeline_events_links', table: 'links', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 23 | notNull: true 24 | }, 25 | message: {type: 'string', notNull: false}, 26 | matrixEventId: {type: 'string', notNull: true}, 27 | timestamp: {type: 'timestamp', notNull: true} 28 | }); 29 | }; 30 | 31 | exports.down = function (db) { 32 | return db.dropTable('timeline_events'); 33 | }; 34 | 35 | exports._meta = { 36 | "version": 1 37 | }; 38 | -------------------------------------------------------------------------------- /migrations/20170326170650-add-state-events-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('state_events', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | type: {type: 'string', notNull: true}, 21 | linkId: { 22 | type: 'int', 23 | foreignKey: {name: 'fk_state_events_links', table: 'links', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 24 | notNull: false 25 | }, 26 | nodeId: { 27 | type: 'int', 28 | foreignKey: {name: 'fk_state_events_nodes', table: 'nodes', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 29 | notNull: false 30 | }, 31 | nodeVersionId: { 32 | type: 'int', 33 | foreignKey: {name: 'fk_state_events_node_versions', table: 'node_versions', mapping: 'id', rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'}}, 34 | notNull: false 35 | }, 36 | timestamp: {type: 'timestamp', notNull: true} 37 | }); 38 | }; 39 | 40 | exports.down = function (db) { 41 | return db.dropTable('state_events'); 42 | }; 43 | 44 | exports._meta = { 45 | "version": 1 46 | }; 47 | -------------------------------------------------------------------------------- /migrations/20170329015645-migrate-states-to-events.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var Promise; 9 | 10 | var dbConfig = require(process.env.VOYAGER_DB_CONF_MIGRATE || "../config/database.json"); 11 | var dbConfigEnv = dbConfig[process.env.NODE_ENV || 'development']; 12 | 13 | /** 14 | * We receive the dbmigrate dependency from dbmigrate initially. 15 | * This enables us to not have to rely on NODE_PATH. 16 | */ 17 | exports.setup = function (options, seedLink) { 18 | dbm = options.dbmigrate; 19 | type = dbm.dataType; 20 | seed = seedLink; 21 | Promise = options.Promise; 22 | }; 23 | 24 | exports.up = function (db) { 25 | var filePath = path.join(__dirname, 'sqls', dbConfigEnv.driver, '20170329015645-migrate-states-to-events-up.sql'); 26 | return new Promise(function (resolve, reject) { 27 | fs.readFile(filePath, {encoding: 'utf-8'}, function (err, data) { 28 | if (err) return reject(err); 29 | console.log('received data: ' + data); 30 | 31 | resolve(data); 32 | }); 33 | }).then(function (data) { 34 | return db.runSql(data); 35 | }); 36 | }; 37 | 38 | exports.down = function (db) { 39 | throw new Error("Unsupported"); 40 | }; 41 | 42 | exports._meta = { 43 | "version": 1 44 | }; 45 | -------------------------------------------------------------------------------- /migrations/20170402235601-redact-known-bad-events.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var Promise; 9 | 10 | var dbConfig = require(process.env.VOYAGER_DB_CONF_MIGRATE || "../config/database.json"); 11 | var dbConfigEnv = dbConfig[process.env.NODE_ENV || 'development']; 12 | 13 | /** 14 | * We receive the dbmigrate dependency from dbmigrate initially. 15 | * This enables us to not have to rely on NODE_PATH. 16 | */ 17 | exports.setup = function (options, seedLink) { 18 | dbm = options.dbmigrate; 19 | type = dbm.dataType; 20 | seed = seedLink; 21 | Promise = options.Promise; 22 | }; 23 | 24 | exports.up = function (db) { 25 | var filePath = path.join(__dirname, 'sqls', dbConfigEnv.driver, '20170402235601-redact-known-bad-events-up.sql'); 26 | return new Promise(function (resolve, reject) { 27 | fs.readFile(filePath, {encoding: 'utf-8'}, function (err, data) { 28 | if (err) return reject(err); 29 | console.log('received data: ' + data); 30 | 31 | resolve(data); 32 | }); 33 | }) 34 | .then(function (data) { 35 | return db.runSql(data); 36 | }); 37 | }; 38 | 39 | exports.down = function (db) { 40 | var filePath = path.join(__dirname, 'sqls', dbConfigEnv.driver, '20170402235601-redact-known-bad-events-down.sql'); 41 | return new Promise(function (resolve, reject) { 42 | fs.readFile(filePath, {encoding: 'utf-8'}, function (err, data) { 43 | if (err) return reject(err); 44 | console.log('received data: ' + data); 45 | 46 | resolve(data); 47 | }); 48 | }) 49 | .then(function (data) { 50 | return db.runSql(data); 51 | }); 52 | }; 53 | 54 | exports._meta = { 55 | "version": 1 56 | }; 57 | -------------------------------------------------------------------------------- /migrations/20170409075941-add-primary-alias-column-to-node-versions.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('node_versions', 'primaryAlias', {type: 'string', notNull: false}); 19 | }; 20 | 21 | exports.down = function (db) { 22 | return db.removeColumn('node_versions', 'primaryAlias'); 23 | }; 24 | 25 | exports._meta = { 26 | "version": 1 27 | }; 28 | -------------------------------------------------------------------------------- /migrations/20170415204327-add-vacuum-rules-to-psql.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var Promise; 9 | 10 | var dbConfig = require(process.env.VOYAGER_DB_CONF_MIGRATE || "../config/database.json"); 11 | var dbConfigEnv = dbConfig[process.env.NODE_ENV || 'development']; 12 | 13 | /** 14 | * We receive the dbmigrate dependency from dbmigrate initially. 15 | * This enables us to not have to rely on NODE_PATH. 16 | */ 17 | exports.setup = function (options, seedLink) { 18 | dbm = options.dbmigrate; 19 | type = dbm.dataType; 20 | seed = seedLink; 21 | Promise = options.Promise; 22 | }; 23 | 24 | exports.up = function (db) { 25 | var filePath = path.join(__dirname, 'sqls', dbConfigEnv.driver, '20170415204327-add-vacuum-rules-to-psql-up.sql'); 26 | return new Promise(function (resolve, reject) { 27 | fs.readFile(filePath, {encoding: 'utf-8'}, function (err, data) { 28 | if (err) return reject(err); 29 | console.log('received data: ' + data); 30 | 31 | console.log("!!!! It is recommended to run `VACUUM ANALYZE ` ON ALL TABLES. See migration script for example."); 32 | resolve(data); 33 | }); 34 | }).then(function (data) { 35 | return db.runSql(data); 36 | }); 37 | }; 38 | 39 | exports.down = function (db) { 40 | throw new Error("Unsupported"); 41 | }; 42 | 43 | exports._meta = { 44 | "version": 1 45 | }; 46 | -------------------------------------------------------------------------------- /migrations/20170415235415-add-node-meta-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('node_meta', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true, notNull: true}, 20 | displayName: {type: 'string', notNull: false}, 21 | avatarUrl: {type: 'string', notNull: false}, 22 | isAnonymous: {type: 'boolean', notNull: false} 23 | }); 24 | }; 25 | 26 | exports.down = function (db) { 27 | return db.dropTable('node_meta'); 28 | }; 29 | 30 | exports._meta = { 31 | "version": 1 32 | }; 33 | -------------------------------------------------------------------------------- /migrations/20170415235447-add-node-meta-reference-to-nodes.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('nodes', 'nodeMetaId', { 19 | type: 'int', 20 | foreignKey: { 21 | name: 'fk_node_node_meta', 22 | table: 'node_meta', 23 | mapping: 'id', 24 | rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'} 25 | }, 26 | notNull: false 27 | }); 28 | }; 29 | 30 | exports.down = function (db) { 31 | return db.removeColumn('nodes', 'nodeMetaId'); 32 | }; 33 | 34 | exports._meta = { 35 | "version": 1 36 | }; 37 | -------------------------------------------------------------------------------- /migrations/20170415235907-add-node-id-to-node-meta.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('node_meta', 'nodeId', { 19 | type: 'int', 20 | foreignKey: { 21 | name: 'fk_node_meta_nodes', 22 | table: 'nodes', 23 | mapping: 'id', 24 | rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'} 25 | }, 26 | notNull: false 27 | }); 28 | }; 29 | 30 | exports.down = function (db) { 31 | return db.removeColumn('node_meta', 'nodeId'); 32 | }; 33 | 34 | exports._meta = { 35 | "version": 1 36 | }; 37 | -------------------------------------------------------------------------------- /migrations/20170416000043-add-primary-alias-to-node-meta.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.addColumn('node_meta', 'primaryAlias', {type: 'string', notNull: false}); 19 | }; 20 | 21 | exports.down = function (db) { 22 | return db.removeColumn('node_meta', 'primaryAlias'); 23 | }; 24 | 25 | exports._meta = { 26 | "version": 1 27 | }; 28 | -------------------------------------------------------------------------------- /migrations/20170416001033-create-node-meta.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var Promise; 9 | 10 | /** 11 | * We receive the dbmigrate dependency from dbmigrate initially. 12 | * This enables us to not have to rely on NODE_PATH. 13 | */ 14 | exports.setup = function(options, seedLink) { 15 | dbm = options.dbmigrate; 16 | type = dbm.dataType; 17 | seed = seedLink; 18 | Promise = options.Promise; 19 | }; 20 | 21 | exports.up = function(db) { 22 | var filePath = path.join(__dirname, 'sqls', '20170416001033-create-node-meta-up.sql'); 23 | return new Promise( function( resolve, reject ) { 24 | fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ 25 | if (err) return reject(err); 26 | console.log('received data: ' + data); 27 | 28 | resolve(data); 29 | }); 30 | }) 31 | .then(function(data) { 32 | return db.runSql(data); 33 | }); 34 | }; 35 | 36 | exports.down = function(db) { 37 | var filePath = path.join(__dirname, 'sqls', '20170416001033-create-node-meta-down.sql'); 38 | return new Promise( function( resolve, reject ) { 39 | fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ 40 | if (err) return reject(err); 41 | console.log('received data: ' + data); 42 | 43 | resolve(data); 44 | }); 45 | }) 46 | .then(function(data) { 47 | return db.runSql(data); 48 | }); 49 | }; 50 | 51 | exports._meta = { 52 | "version": 1 53 | }; 54 | -------------------------------------------------------------------------------- /migrations/20170424035611-add-node-aliases-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('node_aliases', { 19 | id: {type: 'int', primaryKey: true, autoIncrement: true}, 20 | nodeId: { 21 | type: 'int', 22 | foreignKey: { 23 | name: 'fk_node_aliases_nodes', 24 | table: 'nodes', 25 | mapping: 'id', 26 | rules: {onDelete: 'CASCADE', onUpdate: 'CASCADE'} 27 | }, 28 | notNull: false 29 | }, 30 | alias: 'string' 31 | }); 32 | }; 33 | 34 | exports.down = function (db) { 35 | return db.dropTable('node_aliases'); 36 | }; 37 | 38 | exports._meta = { 39 | "version": 1 40 | }; 41 | -------------------------------------------------------------------------------- /migrations/20170905023708-add-stats-to-node-meta.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return Promise.all([ 19 | db.addColumn('node_meta', 'userCount', {type: 'int', notNull: false}), 20 | db.addColumn('node_meta', 'serverCount', {type: 'int', notNull: false}), 21 | db.addColumn('node_meta', 'aliasCount', {type: 'int', notNull: false}) 22 | ]); 23 | }; 24 | 25 | exports.down = function (db) { 26 | return Promise.all([ 27 | db.removeColumn('node_meta', 'userCount'), 28 | db.removeColumn('node_meta', 'serverCount'), 29 | db.removeColumn('node_meta', 'aliasCount') 30 | ]); 31 | }; 32 | 33 | exports._meta = { 34 | "version": 1 35 | }; 36 | -------------------------------------------------------------------------------- /migrations/20170909012946-add-dnt-table.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var dbm; 4 | var type; 5 | var seed; 6 | 7 | /** 8 | * We receive the dbmigrate dependency from dbmigrate initially. 9 | * This enables us to not have to rely on NODE_PATH. 10 | */ 11 | exports.setup = function (options, seedLink) { 12 | dbm = options.dbmigrate; 13 | type = dbm.dataType; 14 | seed = seedLink; 15 | }; 16 | 17 | exports.up = function (db) { 18 | return db.createTable('dnt', { 19 | userId: {type: 'string', primaryKey: true, notNull: true}, 20 | isDnt: {type: 'boolean', notNull: false} 21 | }); 22 | }; 23 | 24 | exports.down = function (db) { 25 | return db.dropTable('dnt'); 26 | }; 27 | 28 | exports._meta = { 29 | "version": 1 30 | }; 31 | -------------------------------------------------------------------------------- /migrations/sqls/20170416001033-create-node-meta-down.sql: -------------------------------------------------------------------------------- 1 | -- Nothing to do -------------------------------------------------------------------------------- /migrations/sqls/20170416001033-create-node-meta-up.sql: -------------------------------------------------------------------------------- 1 | -- Cross-compatible with sqlite and pg 2 | 3 | INSERT INTO node_meta ("nodeId", "displayName", "avatarUrl", "isAnonymous", "primaryAlias") 4 | SELECT nodes.id, 5 | (SELECT "displayName" from node_versions where "nodeId" = nodes.id AND "displayName" IS NOT NULL ORDER BY id DESC LIMIT 1), 6 | (SELECT "avatarUrl" from node_versions where "nodeId" = nodes.id AND "avatarUrl" IS NOT NULL ORDER BY id DESC LIMIT 1), 7 | (SELECT "isAnonymous" from node_versions where "nodeId" = nodes.id AND "isAnonymous" IS NOT NULL ORDER BY id DESC LIMIT 1), 8 | (SELECT "primaryAlias" from node_versions where "nodeId" = nodes.id AND "primaryAlias" IS NOT NULL ORDER BY id DESC LIMIT 1) 9 | FROM nodes; 10 | 11 | UPDATE nodes SET "nodeMetaId" = (SELECT node_meta.id FROM node_meta WHERE "nodeId" = nodes.id); -------------------------------------------------------------------------------- /migrations/sqls/pg/20170329015645-migrate-states-to-events-up.sql: -------------------------------------------------------------------------------- 1 | -- Repair database state first 2 | delete from room_links where "timestamp" is null; 3 | 4 | -- Find all the room nodes first 5 | create table _room_nodes ("room_id" text, "timestamp" timestamp, "is_redacted" boolean); 6 | insert into _room_nodes select distinct e."from_room_id", to_timestamp(0), false from room_links as e; 7 | insert into _room_nodes select distinct e."to_room_id", to_timestamp(0), false from room_links as e where e."to_room_id" is not null and e."to_room_id" not in (select "room_id" from _room_nodes); 8 | insert into _room_nodes select distinct e."room_id", to_timestamp(0), false from membership_events as e where e."room_id" is not null and e."room_id" not in (select _room_nodes."room_id" from _room_nodes); 9 | update _room_nodes set "timestamp" = (select min(e."timestamp") from room_links as e where e."to_room_id" = "room_id" or e."from_room_id" = "room_id"); 10 | update _room_nodes set "timestamp" = (select min(e2."timestamp") from membership_events as e2 where e2."room_id" = _room_nodes."room_id") where _room_nodes."timestamp" > (select min(e2."timestamp") from membership_events as e2 where e2."room_id" = _room_nodes."room_id") or _room_nodes."timestamp" is null; 11 | update _room_nodes set "is_redacted" = (select case when count(*) > 0 then true else false end from membership_events as e where e."room_id" = _room_nodes."room_id" and (e."type" = 'kick' or e."type" = 'ban')); 12 | 13 | -- Find all the user nodes 14 | create table _user_nodes ("user_id" text, "timestamp" timestamp, "is_redacted" boolean); 15 | insert into _user_nodes select distinct e."sender", to_timestamp(0), false from membership_events as e; 16 | update _user_nodes set "timestamp" = (select min(e."timestamp") from membership_events as e where e."sender" = "user_id"); 17 | 18 | -- Insert the nodes into the Nodes table 19 | insert into nodes ("type", "objectId", "isReal", "firstTimestamp", "isRedacted") select 'room', "room_id", true, "timestamp", "is_redacted" from _room_nodes; 20 | insert into nodes ("type", "objectId", "isReal", "firstTimestamp", "isRedacted") select 'user', "user_id", true, "timestamp", "is_redacted" from _user_nodes; 21 | 22 | -- Clean up temporary tables 23 | drop table _room_nodes; 24 | drop table _user_nodes; 25 | 26 | -- Create timeline events and links from old messages 27 | alter table links add column "_legacy_link_id" integer; 28 | insert into links ("type", "sourceNodeId", "targetNodeId", "timestamp", "isVisible", "isRedacted", _legacy_link_id) select 'message', (select "id" from nodes where "objectId" = e."from_room_id"), (select "id" from nodes where "objectId" = e."to_room_id"), e."timestamp", true, false, "id" from room_links as e where e."to_room_id" is not null; 29 | insert into timeline_events ("linkId", "timestamp", "message", "matrixEventId") select links."id", (select e."timestamp" from room_links as e where e."id" = links."_legacy_link_id"), (select e."message" from room_links as e where e."id" = links."_legacy_link_id"), (select e."event_id" from room_links as e where e."id" = links."_legacy_link_id") from links; 30 | 31 | select * from membership_events; 32 | -- Create timeline events and links from old membership events 33 | insert into links ("type", "sourceNodeId", "targetNodeId", "timestamp", "isVisible", "isRedacted", _legacy_link_id) select e."type", (select "id" from nodes where "objectId" = e."sender"), (select "id" from nodes where "objectId" = e."room_id"), e."timestamp", (select case when e."unlisted" = true then false else true end), (select case when e."type" = 'kick' or e."type" = 'ban' then true else false end), e."id" from membership_events as e where e."room_id" is not null; 34 | insert into timeline_events ("linkId", "timestamp", "message", "matrixEventId") select links."id", (select e."timestamp" from membership_events as e where e."id" = links."_legacy_link_id"), (select e."message" from membership_events as e where e."id" = links."_legacy_link_id"), (select e."event_id" from membership_events as e where e.id = links."_legacy_link_id") from links where links."type" <> 'message'; 35 | update links set "isRedacted" = true where "isVisible" = false and "type" = 'self_link'; 36 | 37 | -- Create all the state events for links 38 | insert into state_events ("type", "linkId", "timestamp") select 'link_added', links."id", (select e."timestamp" from timeline_events as e where e."linkId" = links."id") from links; 39 | insert into state_events ("type", "linkId", "timestamp") select 'link_removed', links."id", (select e."timestamp" + interval '1 second' from timeline_events as e where e."linkId" = links."id") from links where links."isVisible" = true; 40 | 41 | -- Create placeholder versions for the nodes we know about 42 | -- These will later be populated by the bot on first start 43 | insert into node_versions ("nodeId", "isAnonymous") select nodes."id", case when (select count(*) from enrolled_users where "user_id" = nodes."objectId") > 0 then false else true end from nodes; 44 | 45 | -- Create all the state events for nodes 46 | insert into state_events ("type", "nodeId", "nodeVersionId", "timestamp") select 'node_added', nodes."id", (select node_versions."id" from node_versions where node_versions."nodeId" = nodes."id"), nodes."firstTimestamp" from nodes; 47 | insert into state_events ("type", "nodeId", "nodeVersionId", "timestamp") select 'node_removed', nodes."id", (select node_versions."id" from node_versions where node_versions."nodeId" = nodes."id"), (select max(e."timestamp") from timeline_events as e join links as e2 on e2."id" = e."linkId" where e2."targetNodeId" = nodes."id" and (e2."type" = 'kick' or e2."type" = 'ban')) from nodes where nodes."isRedacted" = true; -------------------------------------------------------------------------------- /migrations/sqls/pg/20170402235601-redact-known-bad-events-down.sql: -------------------------------------------------------------------------------- 1 | update links set "isRedacted" = false where "id" in (select "linkId" from timeline_events where "matrixEventId" = '$1490298427314CIwPU:matrix.magnap.dk'); -------------------------------------------------------------------------------- /migrations/sqls/pg/20170402235601-redact-known-bad-events-up.sql: -------------------------------------------------------------------------------- 1 | update links set "isRedacted" = true where "id" in (select "linkId" from timeline_events where "matrixEventId" = '$1490298427314CIwPU:matrix.magnap.dk'); -------------------------------------------------------------------------------- /migrations/sqls/pg/20170415204327-add-vacuum-rules-to-psql-up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE node_versions SET (autovacuum_vacuum_scale_factor = 0.0); 2 | ALTER TABLE node_versions SET (autovacuum_vacuum_threshold = 5000); 3 | ALTER TABLE node_versions SET (autovacuum_analyze_scale_factor = 0.0); 4 | ALTER TABLE node_versions SET (autovacuum_analyze_threshold = 5000); 5 | 6 | ALTER TABLE nodes SET (autovacuum_vacuum_scale_factor = 0.0); 7 | ALTER TABLE nodes SET (autovacuum_vacuum_threshold = 5000); 8 | ALTER TABLE nodes SET (autovacuum_analyze_scale_factor = 0.0); 9 | ALTER TABLE nodes SET (autovacuum_analyze_threshold = 5000); 10 | 11 | ALTER TABLE links SET (autovacuum_vacuum_scale_factor = 0.0); 12 | ALTER TABLE links SET (autovacuum_vacuum_threshold = 5000); 13 | ALTER TABLE links SET (autovacuum_analyze_scale_factor = 0.0); 14 | ALTER TABLE links SET (autovacuum_analyze_threshold = 5000); 15 | 16 | ALTER TABLE state_events SET (autovacuum_vacuum_scale_factor = 0.0); 17 | ALTER TABLE state_events SET (autovacuum_vacuum_threshold = 5000); 18 | ALTER TABLE state_events SET (autovacuum_analyze_scale_factor = 0.0); 19 | ALTER TABLE state_events SET (autovacuum_analyze_threshold = 5000); 20 | 21 | ALTER TABLE timeline_events SET (autovacuum_vacuum_scale_factor = 0.0); 22 | ALTER TABLE timeline_events SET (autovacuum_vacuum_threshold = 5000); 23 | ALTER TABLE timeline_events SET (autovacuum_analyze_scale_factor = 0.0); 24 | ALTER TABLE timeline_events SET (autovacuum_analyze_threshold = 5000); 25 | 26 | -- This can't be run in a transaction, but it is recommended to still run this. 27 | --VACUUM ANALYZE node_versions; 28 | --VACUUM ANALYZE nodes; 29 | --VACUUM ANALYZE links; 30 | --VACUUM ANALYZE state_events; 31 | --VACUUM ANALYZE timeline_events; -------------------------------------------------------------------------------- /migrations/sqls/sqlite3/20170329015645-migrate-states-to-events-up.sql: -------------------------------------------------------------------------------- 1 | -- Repair database state first 2 | delete from room_links where timestamp is null; 3 | 4 | -- Find all the room nodes first 5 | create table _room_nodes (room_id text, timestamp text, is_redacted integer); 6 | insert into _room_nodes select distinct e.from_room_id, 0, 0 from room_links as e; 7 | insert into _room_nodes select distinct e.to_room_id, 0, 0 from room_links as e where e.to_room_id is not null and e.to_room_id not in (select room_id from _room_nodes); 8 | insert into _room_nodes select distinct e.room_id, 0, 0 from membership_events as e where e.room_id is not null and e.room_id not in (select _room_nodes.room_id from _room_nodes); 9 | update _room_nodes set timestamp = (select min(e.timestamp) from room_links as e where e.to_room_id = room_id or e.from_room_id = room_id); 10 | update _room_nodes set timestamp = (select min(e2.timestamp) from membership_events as e2 where e2.room_id = _room_nodes.room_id) where _room_nodes.timestamp > (select min(e2.timestamp) from membership_events as e2 where e2.room_id = _room_nodes.room_id) or _room_nodes.timestamp is null; 11 | update _room_nodes set is_redacted = (select case when count(*) > 0 then 1 else 0 end from membership_events as e where e.room_id = _room_nodes.room_id and (e.type = 'kick' or e.type = 'ban')); 12 | 13 | -- Find all the user nodes 14 | create table _user_nodes (user_id text, timestamp text, is_redacted integer); 15 | insert into _user_nodes select distinct e.sender, 0, 0 from membership_events as e; 16 | update _user_nodes set timestamp = (select min(e.timestamp) from membership_events as e where e.sender = user_id); 17 | 18 | -- Insert the nodes into the Nodes table 19 | insert into nodes (type, objectId, isReal, firstTimestamp, isRedacted) select 'room', room_id, 1, timestamp, is_redacted from _room_nodes; 20 | insert into nodes (type, objectId, isReal, firstTimestamp, isRedacted) select 'user', user_id, 1, timestamp, is_redacted from _user_nodes; 21 | 22 | -- Clean up temporary tables 23 | drop table _room_nodes; 24 | drop table _user_nodes; 25 | 26 | -- Create timeline events and links from old messages 27 | alter table links add column _legacy_link_id; 28 | insert into links (type, sourceNodeId, targetNodeId, timestamp, isVisible, isRedacted, _legacy_link_id) select 'message', (select id from nodes where objectId = e.from_room_id), (select id from nodes where objectId = e.to_room_id), e.timestamp, 1, 0, id from room_links as e where e.to_room_id is not null; 29 | insert into timeline_events (linkId, timestamp, message, matrixEventId) select links.id, (select e.timestamp from room_links as e where e.id = links._legacy_link_id), (select e.message from room_links as e where e.id = links._legacy_link_id), (select e.event_id from room_links as e where e.id = links._legacy_link_id) from links; 30 | 31 | select * from membership_events; 32 | -- Create timeline events and links from old membership events 33 | insert into links (type, sourceNodeId, targetNodeId, timestamp, isVisible, isRedacted, _legacy_link_id) select e.type, (select id from nodes where objectId = e.sender), (select id from nodes where objectId = e.room_id), e.timestamp, (select case when e.unlisted = 1 then 0 else 1 end), (select case when e.type = 'kick' or e.type = 'ban' then 1 else 0 end), e.id from membership_events as e where e.room_id is not null; 34 | insert into timeline_events (linkId, timestamp, message, matrixEventId) select links.id, (select e.timestamp from membership_events as e where e.id = links._legacy_link_id), (select e.message from membership_events as e where e.id = links._legacy_link_id), (select e.event_id from membership_events as e where e.id = links._legacy_link_id) from links where links.type <> 'message'; 35 | update links set isRedacted = 1 where isVisible = 0 and type = 'self_link'; 36 | 37 | -- Create all the state events for links 38 | insert into state_events (type, linkId, timestamp) select 'link_added', links.id, (select e.timestamp from timeline_events as e where e.linkId = links.id) from links; 39 | insert into state_events (type, linkId, timestamp) select 'link_removed', links.id, (select e.timestamp + 1 from timeline_events as e where e.linkId = links.id) from links where links.isVisible = 0; 40 | 41 | -- Create placeholder versions for the nodes we know about 42 | -- These will later be populated by the bot on first start 43 | insert into node_versions (nodeId, isAnonymous) select nodes.id, case when (select count(*) from enrolled_users where user_id = nodes.objectId) > 0 then 0 else 1 end from nodes; 44 | 45 | -- Create all the state events for nodes 46 | insert into state_events (type, nodeId, nodeVersionId, timestamp) select 'node_added', nodes.id, (select node_versions.id from node_versions where node_versions.nodeId = nodes.id), nodes.firstTimestamp from nodes; 47 | insert into state_events (type, nodeId, nodeVersionId, timestamp) select 'node_removed', nodes.id, (select node_versions.id from node_versions where node_versions.nodeId = nodes.id), (select max(e.timestamp) from timeline_events as e join links as e2 on e2.id = e.linkId where e2.targetNodeId = nodes.id and (e2.type = 'kick' or e2.type = 'ban')) from nodes where nodes.isRedacted = 1; -------------------------------------------------------------------------------- /migrations/sqls/sqlite3/20170402235601-redact-known-bad-events-down.sql: -------------------------------------------------------------------------------- 1 | update links set isRedacted = 0 where id in (select linkId from timeline_events where matrixEventId = '$1490298427314CIwPU:matrix.magnap.dk'); -------------------------------------------------------------------------------- /migrations/sqls/sqlite3/20170402235601-redact-known-bad-events-up.sql: -------------------------------------------------------------------------------- 1 | update links set isRedacted = 1 where id in (select linkId from timeline_events where matrixEventId = '$1490298427314CIwPU:matrix.magnap.dk'); -------------------------------------------------------------------------------- /migrations/sqls/sqlite3/20170415204327-add-vacuum-rules-to-psql-up.sql: -------------------------------------------------------------------------------- 1 | -- Nothing to do - does not apply -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "matrix-voyager-bot", 3 | "version": "1.0.0", 4 | "license": "GPL-3.0", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/turt2live/matrix-voyager-bot.git" 8 | }, 9 | "dependencies": { 10 | "bluebird": "^3.5.0", 11 | "chalk": "^2.1.0", 12 | "config": "^1.26.2", 13 | "db-migrate": "^1.0.0-beta.16", 14 | "db-migrate-pg": "^1.2.2", 15 | "events": "^1.1.1", 16 | "express": "^4.15.4", 17 | "js-yaml": "^3.13.1", 18 | "lodash": "^4.17.21", 19 | "lodash.sortedindex": "^4.1.0", 20 | "moment": "^2.29.2", 21 | "node-cache": "^4.1.1", 22 | "node-localstorage": "^1.3.0", 23 | "node-natural-sort": "^0.8.6", 24 | "pg": "^8.7.1", 25 | "pg-hstore": "^2.3.4", 26 | "promise-map": "^1.2.0", 27 | "q": "^1.5.0", 28 | "reflect-metadata": "^0.1.10", 29 | "request": "^2.81.0", 30 | "sequelize": "^5.22.4", 31 | "string_score": "^0.1.22", 32 | "string-hash": "^1.1.3", 33 | "vue": "^2.4.2", 34 | "vue-resource": "^1.3.4", 35 | "vue-router": "^2.7.0", 36 | "winston": "^2.3.1" 37 | }, 38 | "devDependencies": { 39 | "autoprefixer": "^7.1.2", 40 | "babel-core": "^6.22.1", 41 | "babel-eslint": "^7.1.1", 42 | "babel-loader": "^7.1.1", 43 | "babel-plugin-transform-runtime": "^6.22.0", 44 | "babel-preset-env": "^1.3.2", 45 | "babel-preset-stage-2": "^6.22.0", 46 | "babel-register": "^6.22.0", 47 | "chalk": "^2.0.1", 48 | "connect-history-api-fallback": "^1.3.0", 49 | "copy-webpack-plugin": "^4.0.1", 50 | "css-loader": "^0.28.0", 51 | "cssnano": "^3.10.0", 52 | "d3": "^4.10.2", 53 | "eslint": "^4.18.2", 54 | "eslint-config-standard": "^6.2.1", 55 | "eslint-friendly-formatter": "^3.0.0", 56 | "eslint-loader": "^1.7.1", 57 | "eslint-plugin-html": "^3.0.0", 58 | "eslint-plugin-promise": "^3.4.0", 59 | "eslint-plugin-standard": "^2.0.1", 60 | "eventsource-polyfill": "^0.9.6", 61 | "express": "^4.14.1", 62 | "extract-text-webpack-plugin": "^2.0.0", 63 | "file-loader": "^0.11.1", 64 | "friendly-errors-webpack-plugin": "^1.1.3", 65 | "html-webpack-plugin": "^2.28.0", 66 | "http-proxy-middleware": "^0.17.3", 67 | "numeral": "^2.0.6", 68 | "opn": "^5.1.0", 69 | "optimize-css-assets-webpack-plugin": "^2.0.0", 70 | "ora": "^1.2.0", 71 | "rimraf": "^2.6.0", 72 | "semver": "^5.3.0", 73 | "shelljs": "^0.7.6", 74 | "url-loader": "^0.5.8", 75 | "vue-awesome": "^2.3.3", 76 | "vue-loader": "^13.0.4", 77 | "vue-style-loader": "^3.0.1", 78 | "vue-template-compiler": "^2.4.2", 79 | "webpack": "^2.6.1", 80 | "webpack-bundle-analyzer": "^3.3.2", 81 | "webpack-dashboard": "^1.0.0-5", 82 | "webpack-dev-middleware": "^1.10.0", 83 | "webpack-hot-middleware": "^2.18.0", 84 | "webpack-merge": "^4.1.0" 85 | }, 86 | "scripts": { 87 | "dev": "node build/dev-server.js", 88 | "start": "node build/dev-server.js", 89 | "build": "node build/build.js", 90 | "lint": "eslint --ext .js,.vue src" 91 | }, 92 | "engines": { 93 | "node": ">= 4.0.0", 94 | "npm": ">= 3.0.0" 95 | }, 96 | "browserslist": [ 97 | "> 1%", 98 | "last 2 versions", 99 | "not ie <= 8" 100 | ] 101 | } 102 | -------------------------------------------------------------------------------- /src/LogService.js: -------------------------------------------------------------------------------- 1 | var winston = require("winston"); 2 | var chalk = require("chalk"); 3 | var config = require("config"); 4 | var fs = require('fs'); 5 | var moment = require('moment'); 6 | 7 | try { 8 | fs.mkdirSync('logs') 9 | } catch (err) { 10 | if (err.code !== 'EEXIST') throw err 11 | } 12 | 13 | const TERM_COLORS = { 14 | error: "red", 15 | warn: "yellow", 16 | info: "blue", 17 | verbose: "white", 18 | silly: "grey", 19 | }; 20 | 21 | function winstonColorFormatter(options) { 22 | options.level = chalk[TERM_COLORS[options.level]](options.level); 23 | return winstonFormatter(options); 24 | } 25 | 26 | function winstonFormatter(options) { 27 | return options.timestamp() + ' ' + options.level + ' ' + (options.message ? options.message : '') + 28 | (options.meta && Object.keys(options.meta).length ? '\n\t' + JSON.stringify(options.meta) : '' ); 29 | } 30 | 31 | function getTimestamp() { 32 | return moment().format('MMM-D-YYYY HH:mm:ss.SSS Z'); 33 | } 34 | 35 | var loggingConfig = config.get('logging'); 36 | 37 | var transports = []; 38 | transports.push(new (winston.transports.File)({ 39 | json: false, 40 | name: "file", 41 | filename: loggingConfig.file, 42 | timestamp: getTimestamp, 43 | formatter: winstonFormatter, 44 | level: loggingConfig.fileLevel, 45 | maxsize: loggingConfig.rotate.size, 46 | maxFiles: loggingConfig.rotate.count, 47 | zippedArchive: false 48 | })); 49 | 50 | if (loggingConfig.console) { 51 | transports.push(new (winston.transports.Console)({ 52 | json: false, 53 | name: "console", 54 | timestamp: getTimestamp, 55 | formatter: winstonColorFormatter, 56 | level: loggingConfig.consoleLevel 57 | })); 58 | } 59 | 60 | var log = new winston.Logger({ 61 | transports: transports, 62 | levels: { 63 | error: 0, 64 | warn: 1, 65 | info: 2, 66 | verbose: 3, 67 | silly: 4 68 | } 69 | }); 70 | 71 | function doLog(level, module, messageOrObject) { 72 | if (typeof(messageOrObject) === 'object' && !(messageOrObject instanceof Error)) 73 | messageOrObject = JSON.stringify(messageOrObject); 74 | 75 | if (messageOrObject instanceof Error) { 76 | var err = messageOrObject; 77 | messageOrObject = err.message + "\n" + err.stack; 78 | } 79 | 80 | var message = "[" + module + "] " + messageOrObject; 81 | log.log(level, message); 82 | } 83 | 84 | class LogService { 85 | static info(module, message) { 86 | doLog('info', module, message); 87 | } 88 | 89 | static warn(module, message) { 90 | doLog('warn', module, message); 91 | } 92 | 93 | static error(module, message) { 94 | doLog('error', module, message); 95 | } 96 | 97 | static verbose(module, message) { 98 | doLog('verbose', module, message); 99 | } 100 | 101 | static silly(module, message) { 102 | doLog('silly', module, message); 103 | } 104 | } 105 | 106 | module.exports = LogService; -------------------------------------------------------------------------------- /src/VoyagerBot.js: -------------------------------------------------------------------------------- 1 | var CommandProcessor = require("./matrix/CommandProcessor"); 2 | var LocalStorage = require("node-localstorage").LocalStorage; 3 | var config = require("config"); 4 | var log = require("./LogService"); 5 | var naturalSort = require("node-natural-sort"); 6 | var MatrixClientLite = require("./matrix/MatrixClientLite"); 7 | var _ = require("lodash"); 8 | var Promise = require('bluebird'); 9 | var moment = require('moment'); 10 | 11 | const STATS_CACHE_MS = 1 * 60 * 60 * 1000; // 1 hour 12 | 13 | /** 14 | * The main entry point for the bot. Handles most of the business logic and bot actions 15 | */ 16 | class VoyagerBot { 17 | 18 | /** 19 | * Creates a new VoyagerBot 20 | * @param {VoyagerStore} store the store to use 21 | */ 22 | constructor(store) { 23 | this._localStorage = new LocalStorage("db/voyager_local_storage", 100 * 1024 * 1024); // quota is 100mb 24 | 25 | this._nodeUpdateQueue = []; 26 | this._processingNodes = false; 27 | this._queuedObjectIds = []; 28 | this._queueNodesForUpdate = config.get('bot.processNodeUpdatesOnStartup'); 29 | this._queueUsersOnStartup = config.get('bot.nodeUpdatesOnStartup.users'); 30 | this._queueRoomsOnStartup = config.get('bot.nodeUpdatesOnStartup.rooms'); 31 | 32 | this._store = store; 33 | this._commandProcessor = new CommandProcessor(this, store); 34 | this._statsCache = {}; 35 | 36 | this._client = new MatrixClientLite(config['matrix']['homeserverUrl'], config['matrix']['accessToken'], config['matrix']['userId']); 37 | 38 | this._loadPendingNodeUpdates(); 39 | 40 | this._client.on('room_invite', this._onInvite.bind(this)); 41 | this._client.on('room_message', this._onRoomMessage.bind(this)); 42 | this._client.on('room_leave', this._onRoomLeave.bind(this)); 43 | this._client.on('room_avatar', this._onRoomUpdated.bind(this)); 44 | this._client.on('room_name', this._onRoomUpdated.bind(this)); 45 | this._client.on('room_join_rules', this._onRoomUpdated.bind(this)); 46 | this._client.on('room_aliases', this._onRoomUpdated.bind(this)); 47 | this._client.on('room_canonical_alias', this._onRoomUpdated.bind(this)); 48 | this._client.on('user_avatar', this._onUserUpdated.bind(this)); 49 | this._client.on('user_name', this._onUserUpdated.bind(this)); 50 | } 51 | 52 | /** 53 | * Starts the voyager bot 54 | */ 55 | start() { 56 | this._client.start().then(() => { 57 | log.info("VoyagerBot", "Enabling node updates now that the bot is syncing"); 58 | this._queueNodesForUpdate = true; 59 | 60 | this._tryUpdateNodeVersions(); 61 | 62 | this._processNodeVersions(); 63 | setInterval(() => this._processNodeVersions(), 15000); 64 | }); 65 | } 66 | 67 | _onRoomUpdated(roomId, event) { 68 | this._queueNodeUpdate({objectId: roomId, type: 'room'}); 69 | } 70 | 71 | _onUserUpdated(roomId, event) { 72 | this._queueNodeUpdate({objectId: event['sender'], inRoom: roomId, type: 'user'}); 73 | } 74 | 75 | _onRoomMessage(roomId, event) { 76 | if (event['sender'] === this._client.selfId) return; // self - ignore 77 | 78 | var body = event['content']['body']; 79 | if (!body) return; // likely redacted 80 | 81 | if (body.startsWith("!voyager")) { 82 | this._commandProcessor.processCommand(roomId, event, body.substring("!voyager".length).trim().split(" ")).catch(err => { 83 | log.error("VoyagerBot", "Error processing command " + body); 84 | log.error("VoyagerBot", err); 85 | this._commandProcessor._reply(roomId, event, "There was an error processing your command"); // HACK: Should not be calling private methods 86 | }); 87 | return; 88 | } 89 | 90 | this._store.isDnt(event['sender']).then(dnt => { 91 | if (dnt) { 92 | log.warn("VoyagerBot", "Received message from " + event['sender'] + " but the user has set DNT. Ignoring message."); 93 | return; 94 | } //else log.silly("VoyagerBot", "User " + event['sender'] + " does not have DNT"); 95 | 96 | var matches = body.match(/[#!][a-zA-Z0-9.\-_#=]+:[a-zA-Z0-9.\-_]+[a-zA-Z0-9]/g); 97 | if (!matches) return; 98 | 99 | var promise = Promise.resolve(); 100 | _.forEach(matches, match => promise = promise.then(() => this._processMatchedLink(roomId, event, match))); 101 | 102 | promise.then(() => this._client.sendReadReceipt(roomId, event['event_id'])); 103 | }); 104 | } 105 | 106 | _onRoomLeave(roomId, event) { 107 | if (event['state_key'] === this._client.selfId) { 108 | if (event['sender'] === this._client.selfId) { 109 | // Probably admin action or we soft kicked. 110 | // TODO: If not already a soft kick, record as soft kick (#130) 111 | } else if (event['content']['membership'] === 'ban') { 112 | this._onBan(roomId, event); 113 | } else if (event['unsigned']['prev_content'] && event['unsigned']['prev_content']['membership'] === 'ban') { 114 | // TODO: Handled unbanned state? 115 | log.info("VoyagerBot", event['sender'] + " has unbanned the bot in " + roomId); 116 | } else this._onKick(roomId, event); 117 | } 118 | } 119 | 120 | _processMatchedLink(inRoomId, event, matchedValue, retryCount = 0) { 121 | var roomId; 122 | var sourceNode; 123 | var targetNode; 124 | 125 | return this._client.joinRoom(matchedValue).then(rid => { 126 | roomId = rid; 127 | return this.getNode(roomId, 'room'); 128 | }, err => { 129 | if (err.httpStatus == 500 && retryCount < 5) { 130 | return this._processMatchedLink(event, matchedValue, ++retryCount); 131 | } 132 | 133 | log.error("VoyagerBot", err); 134 | return Promise.resolve(); // TODO: Record failed event as unlinkable node 135 | }).then(node => { 136 | if (!roomId) return Promise.resolve(); 137 | targetNode = node; 138 | 139 | return this.getNode(inRoomId, 'room'); 140 | }).then(node => { 141 | if (!roomId) return Promise.resolve(); 142 | sourceNode = node; 143 | return this._store.createLink(sourceNode, targetNode, 'message', event['origin_server_ts']); 144 | }).then(link => { 145 | if (!link) return Promise.resolve(); 146 | return this._store.createTimelineEvent(link, event['origin_server_ts'], event['event_id'], 'Matched: ' + matchedValue); 147 | }); 148 | } 149 | 150 | _onInvite(roomId, event) { 151 | var sourceNode; 152 | var targetNode; 153 | 154 | if (event.__voyagerRepeat) { 155 | log.info("VoyagerBot", "Attempt #" + event.__voyagerRepeat + " to retry event " + event['event_id']); 156 | } 157 | 158 | return this._client.joinRoom(roomId) 159 | .then(() => Promise.all([this.getNode(event['sender'], 'user'), this.getNode(roomId, 'room')])) 160 | .then(nodes => { 161 | sourceNode = nodes[0]; 162 | targetNode = nodes[1]; 163 | return this._store.findLinkByTimeline(sourceNode, targetNode, 'invite', event['event_id']) 164 | }) 165 | .then(existingLink => { 166 | if (existingLink) return Promise.resolve(); 167 | else return this._store.createLink(sourceNode, targetNode, 'invite', event['origin_server_ts']) 168 | .then(link => this._store.createTimelineEvent(link, event['origin_server_ts'], event['event_id'])); 169 | }) 170 | .then(() => this._tryUpdateRoomNodeVersion(roomId)) 171 | .catch(err => { 172 | log.error("VoyagerBot", err); 173 | 174 | // Sometimes the error is nested under another object 175 | if (err['body']) err = err['body']; 176 | 177 | // Convert the error to an object if we can 178 | if (typeof(err) === 'string') { 179 | try { 180 | err = JSON.parse(err); 181 | } catch (e) { 182 | } 183 | } 184 | 185 | if ((err['errcode'] == "M_FORBIDDEN" || err['errcode'] == "M_GUEST_ACCESS_FORBIDDEN") && (!event.__voyagerRepeat || event.__voyagerRepeat < 25)) { // 25 is arbitrary 186 | event.__voyagerRepeat = (event.__voyagerRepeat ? event.__voyagerRepeat : 0) + 1; 187 | log.info("VoyagerBot", "Forbidden as part of event " + event['event_id'] + " - will retry for attempt #" + event.__voyagerRepeat + " shortly."); 188 | setTimeout(() => this._onInvite(roomId, event), 1000); // try again later 189 | } else if (event.__voyagerRepeat) { 190 | log.error("VoyagerBot", "Failed to retry event " + event['event_id']); 191 | } 192 | }); 193 | } 194 | 195 | _onKick(roomId, event) { 196 | return this._addKickBan(roomId, event, 'kick'); 197 | } 198 | 199 | _onBan(roomId, event) { 200 | return this._addKickBan(roomId, event, 'ban'); 201 | } 202 | 203 | _addKickBan(roomId, event, type) { 204 | var roomNode; 205 | var userNode; 206 | var kickbanLink; 207 | 208 | log.info("VoyagerBot", "Recording " + type + " for " + roomId + " made by " + event['sender']); 209 | 210 | return this.getNode(event['sender'], 'user').then(node => { 211 | userNode = node; 212 | return this.getNode(roomId, 'room'); 213 | }).then(node => { 214 | roomNode = node; 215 | return this._store.redactNode(roomNode); 216 | }).then(() => { 217 | return this._store.createLink(userNode, roomNode, type, event['origin_server_ts'], false, true); 218 | }).then(link => { 219 | kickbanLink = link; 220 | var reason = (event['content'] || {})['reason'] || null; 221 | return this._store.createTimelineEvent(kickbanLink, event['origin_server_ts'], event['event_id'], reason); 222 | }); 223 | } 224 | 225 | getNode(objectId, type) { 226 | return this._store.getNode(type, objectId).then(node => { 227 | if (node) return Promise.resolve(node); 228 | 229 | if (type == 'user') 230 | return this._createUserNode(objectId); 231 | else if (type == 'room') 232 | return this._createRoomNode(objectId); 233 | else throw new Error("Unexpected node type: " + type); 234 | }); 235 | } 236 | 237 | _createUserNode(userId) { 238 | return this._getUserVersion(userId).then(version => this._store.createNode('user', userId, version)); 239 | } 240 | 241 | _createRoomNode(roomId) { 242 | return this._getRoomVersion(roomId).then(version => this._store.createNode('room', roomId, version, version.aliases)); 243 | } 244 | 245 | _getUserVersion(userId) { 246 | var version = { 247 | displayName: "", 248 | avatarUrl: "", 249 | isAnonymous: !this._store.isEnrolled(userId), 250 | primaryAlias: null, // users can't have aliases 251 | }; 252 | 253 | // Don't get profile information if the user isn't public 254 | if (version.isAnonymous) { 255 | return Promise.resolve(version); 256 | } 257 | 258 | return this._client.getUserInfo(userId).then(userInfo => { 259 | version.displayName = userInfo['displayname']; 260 | version.avatarUrl = userInfo['avatar_url']; 261 | 262 | if (!version.avatarUrl || version.avatarUrl.trim().length == 0) 263 | version.avatarUrl = null; 264 | else version.avatarUrl = this._client.convertMediaToThumbnail(version.avatarUrl, 256, 256); 265 | 266 | if (!version.displayName || version.displayName.trim().length == 0) 267 | version.displayName = null; 268 | 269 | return version; 270 | }); 271 | } 272 | 273 | _getRoomVersion(roomId) { 274 | var version = { 275 | displayName: null, 276 | avatarUrl: null, 277 | isAnonymous: true, 278 | primaryAlias: null, 279 | aliases: [], 280 | stats: {users: 0, servers: 0} // aliases handled by above array 281 | }; 282 | 283 | return this._client.getRoomState(roomId).then(state => { 284 | var roomMembers = []; // displayNames (strings) 285 | var joinedMembers = []; // same as room members 286 | var matrixDotOrgAliases = []; // special case handling 287 | var servers = []; 288 | 289 | var tryAddServer = (component) => { 290 | var serverParts = component.split(':'); 291 | var server = serverParts[serverParts.length - 1]; 292 | if (servers.indexOf(server) == -1) 293 | servers.push(server); 294 | }; 295 | 296 | var chain = Promise.resolve(); 297 | state.map(event => chain = chain.then(() => { 298 | if (event['type'] === 'm.room.join_rules') { 299 | log.silly("VoyagerBot", "m.room.join_rules for " + roomId + " is " + event['content']['join_rule']); 300 | version.isAnonymous = event['content']['join_rule'] !== 'public'; 301 | } else if (event['type'] === 'm.room.member') { 302 | if (event['user_id'] === this._client.selfId) return; // skip ourselves, always 303 | log.silly("VoyagerBot", "m.room.member of " + event['user_id'] + " in " + roomId + " is " + event['membership']); 304 | 305 | var displayName = event['content']['displayname']; 306 | if (!displayName || displayName.trim().length === 0) 307 | displayName = event['user_id']; 308 | 309 | roomMembers.push(displayName); 310 | if (event['membership'] === 'join' || event['membership'] === 'invite') joinedMembers.push(displayName); 311 | tryAddServer(event['user_id']); 312 | 313 | // Create the node, but don't bother updating the information for it 314 | return this.getNode(event['user_id'], 'user').then(n => log.silly("VoyagerBot", "Got node for " + n.objectId + ": " + n.id)); 315 | } else if (event['type'] === 'm.room.aliases') { 316 | if (event['content']['aliases']) { 317 | log.silly("VoyagerBot", "m.room.aliases for " + roomId + " on domain " + event['state_key'] + " is: " + event['content']['aliases'].join(', ')); 318 | for (var alias of event['content']['aliases']) { 319 | version.aliases.push(alias); 320 | if (alias.endsWith(":matrix.org")) matrixDotOrgAliases.push(alias); 321 | tryAddServer(alias); 322 | } 323 | } else log.silly("VoyagerBot", "m.room.aliases for " + roomId + " on domain " + event['state_key'] + " is empty/null"); 324 | } else if (event['type'] === 'm.room.canonical_alias') { 325 | log.silly("VoyagerBot", "m.room.canonical_alias for " + roomId + " is " + event['content']['alias']); 326 | version.primaryAlias = event['content']['alias']; 327 | if (event['content']['alias']) tryAddServer(event['content']['alias']); 328 | } else if (event['type'] === 'm.room.name') { 329 | log.silly("VoyagerBot", "m.room.name for " + roomId + " is " + event['content']['name']); 330 | version.displayName = event['content']['name']; 331 | } else if (event['type'] === 'm.room.avatar') { 332 | log.silly("VoyagerBot", "m.room.avatar for " + roomId + " is " + event['content']['url']); 333 | if (event['content']['url'] && event['content']['url'].trim().length > 0) 334 | version.avatarUrl = this._client.convertMediaToThumbnail(event['content']['url'], 256, 256); 335 | } else log.silly("VoyagerBot", "Not handling state event " + event['type'] + " in room " + roomId); 336 | })); 337 | 338 | return chain.then(() => { 339 | // Populate stats 340 | version.stats.users = joinedMembers.length; 341 | version.stats.servers = servers.length; 342 | 343 | // HACK: This is technically against spec, but we'll pick a reasonable default for a room's alias if there is none. 344 | if (!version.primaryAlias && version.aliases.length > 0) 345 | version.primaryAlias = (matrixDotOrgAliases.length > 0 ? matrixDotOrgAliases[0] : version.aliases[0]); 346 | 347 | // Now that we've processed room state: determine the room name 348 | if (version.displayName && version.displayName.trim().length > 0) return version; // we're done :) 349 | 350 | matrixDotOrgAliases.sort(); 351 | version.aliases.sort(); 352 | joinedMembers.sort(naturalSort({caseSensitive: false})); 353 | roomMembers.sort(naturalSort({caseSensitive: false})); 354 | 355 | // Display name logic (according to matrix spec) | http://matrix.org/docs/spec/client_server/r0.2.0.html#id222 356 | // 1. Use m.room.name (handled above) 357 | // 2. Use m.room.canonical_alias 358 | // a. *Against Spec* Use m.room.aliases, picking matrix.org aliases over other aliases, if no canonical alias 359 | // 3. Use joined/invited room members (not including self) 360 | // a. 1 member - use their display name 361 | // b. 2 members - use their display names, lexically sorted 362 | // c. 3+ members - use first display name, lexically, and show 'and N others' 363 | // 4. Consider left users and repeat #3 ("Empty room (was Alice and Bob)") 364 | // 5. Show 'Empty Room' - this shouldn't happen as it is an error condition in the spec 365 | 366 | // using canonical alias 367 | if (version.primaryAlias && version.primaryAlias.trim().length > 0) { 368 | version.displayName = version.primaryAlias; 369 | return version; 370 | } 371 | 372 | // using other aliases, against spec, preferring matrix.org 373 | if (version.aliases.length > 0) { 374 | if (matrixDotOrgAliases.length > 0) { 375 | version.displayName = matrixDotOrgAliases[0]; 376 | } else version.displayName = version.aliases[0]; 377 | return version; 378 | } 379 | 380 | // pick the appropriate collection of members 381 | var memberArray = joinedMembers; 382 | if (memberArray.length === 0) memberArray = roomMembers; 383 | 384 | // build a room name using those members 385 | if (memberArray.length === 1) { 386 | version.displayName = memberArray[0]; 387 | return version; 388 | } else if (memberArray.length === 2) { 389 | version.displayName = memberArray[0] + " and " + memberArray[1]; 390 | return version; 391 | } else if (memberArray.length > 2) { 392 | version.displayName = memberArray[0] + " and " + (memberArray.length - 1) + " others"; 393 | return version; 394 | } 395 | 396 | // weird fallback scenario (alone in room) 397 | version.displayName = "Empty Room"; 398 | 399 | return version; 400 | }); 401 | }); 402 | } 403 | 404 | getRoomStateEvents(roomId, type, stateKey) { 405 | return this._client.getRoomStateEvents(roomId, type, stateKey); 406 | } 407 | 408 | sendNotice(roomId, message) { 409 | return this._client.sendNotice(roomId, message); 410 | } 411 | 412 | leaveRoom(roomId) { 413 | return this._client.leaveRoom(roomId); 414 | } 415 | 416 | matchRoomSharedWith(roomIdOrAlias, userId) { 417 | return this._client.getJoinedRooms().then(joinedRooms => { 418 | var promiseChain = Promise.resolve(); 419 | _.forEach(joinedRooms, roomId => { 420 | promiseChain = promiseChain 421 | .then(() => this._client.getRoomState(roomId)) 422 | .then(state => { 423 | var isMatch = roomIdOrAlias === roomId; 424 | var isMember = false; 425 | 426 | for (var event of state) { 427 | if (event['type'] === 'm.room.canonical_alias' && event['content']['alias'] === roomIdOrAlias) { 428 | isMatch = true; 429 | } else if (event['type'] === 'm.room.aliases' && event['content']['aliases'].indexOf(roomIdOrAlias) !== -1) { 430 | isMatch = true; 431 | } else if (event['type'] === 'm.room.member' && event['user_id'] === userId && event['membership'] === 'join') { 432 | isMember = true; 433 | } 434 | 435 | if (isMatch && isMember) break; // to save a couple clock cycles 436 | } 437 | 438 | if (isMatch && isMember) return Promise.reject(roomId); // reject === break loop 439 | else return Promise.resolve(); // resolve === try next room 440 | }); 441 | }); 442 | 443 | // Invert the success and fail because of how the promise chain is dealt with 444 | return promiseChain.then(() => Promise.resolve(null), roomId => Promise.resolve(roomId)); 445 | }); 446 | } 447 | 448 | _queueNodeUpdate(nodeMeta, doSave = true) { 449 | if (!nodeMeta.objectId) { 450 | log.warn("VoyagerBot", "Unexpected node: " + JSON.stringify(nodeMeta)); 451 | return; 452 | } 453 | 454 | //if (nodeMeta.type === 'user') { 455 | // log.warn("VoyagerBot", "Skipping user node update for " + nodeMeta.objectId); 456 | // return; 457 | //} 458 | 459 | if (this._queuedObjectIds.indexOf(nodeMeta.objectId) !== -1) { 460 | log.info("VoyagerBot", "Node update queue attempt for " + nodeMeta.objectId + " - skipped because the node is already queued"); 461 | return; 462 | } 463 | 464 | this._nodeUpdateQueue.push(nodeMeta); 465 | this._queuedObjectIds.push(nodeMeta.objectId); 466 | if (doSave) this._savePendingNodeUpdates(); 467 | 468 | log.info("VoyagerBot", "Queued update for " + nodeMeta.objectId); 469 | } 470 | 471 | _savePendingNodeUpdates() { 472 | log.info("VoyagerBot", "Saving queued node updates"); 473 | this._localStorage.setItem("voyager_node_update_queue", JSON.stringify(this._nodeUpdateQueue)); 474 | } 475 | 476 | _loadPendingNodeUpdates() { 477 | var pendingNodeUpdates = this._localStorage.getItem("voyager_node_update_queue"); 478 | if (pendingNodeUpdates) { 479 | var nodeUpdatesAsArray = JSON.parse(pendingNodeUpdates); 480 | for (var update of nodeUpdatesAsArray) { 481 | update.retryCount = 0; 482 | if (update.node && !update.objectId) { 483 | update.objectId = update.node; 484 | update.node = null; 485 | } 486 | this._queueNodeUpdate(update, /*doSave:*/false); 487 | } 488 | } 489 | log.info("VoyagerBot", "Loaded " + this._nodeUpdateQueue.length + " previously pending node updates"); 490 | } 491 | 492 | _processNodeVersions() { 493 | if (this._processingNodes) { 494 | log.warn("VoyagerBot", "Already processing nodes from queue - skipping interval check"); 495 | return; 496 | } 497 | 498 | this._processingNodes = true; 499 | var nodesToProcess = this._nodeUpdateQueue.splice(0, 2500); 500 | this._savePendingNodeUpdates(); 501 | 502 | log.info("VoyagerBot", "Processing " + nodesToProcess.length + " pending node updates. " + this._nodeUpdateQueue.length + " remaining"); 503 | 504 | var promiseChain = Promise.resolve(); 505 | _.forEach(nodesToProcess, node => { 506 | promiseChain = promiseChain.then(() => { 507 | var idx = this._queuedObjectIds.indexOf(node.objectId); 508 | if (idx !== -1) this._queuedObjectIds.splice(idx, 1); 509 | 510 | var promise = Promise.resolve(); 511 | 512 | try { 513 | switch (node.type) { 514 | case "room": 515 | promise = this._tryUpdateRoomNodeVersion(node.objectId); 516 | break; 517 | case "user": 518 | promise = this._tryUpdateUserNodeVersion(node.objectId); 519 | break; 520 | default: 521 | log.warn("VoyagerBot", "Could not handle node in update queue: " + JSON.stringify(node)); 522 | return Promise.resolve(); 523 | } 524 | } catch (error) { 525 | promise = Promise.reject(error); 526 | } 527 | 528 | return promise.then(() => log.info("VoyagerBot", "Completed update for " + node.objectId)).catch(err => { 529 | log.error("VoyagerBot", "Error updating node " + node.objectId); 530 | log.error("VoyagerBot", err); 531 | 532 | if (node.retryCount >= 5) { 533 | log.error("VoyagerBot", "Not retrying node update for node " + node.objectId + " due to the maximum number of retries reached (5)"); 534 | return; 535 | } 536 | 537 | if (!node.retryCount) node.retryCount = 0; 538 | node.retryCount++; 539 | 540 | log.warn("VoyagerBot", "Re-queueing node " + node.objectId + " for updates due to failure. This will be retry #" + node.retryCount); 541 | 542 | this._queueNodeUpdate(node); 543 | }); 544 | }); 545 | }); 546 | 547 | promiseChain.then(() => { 548 | log.info("VoyagerBot", "Processed " + nodesToProcess.length + " node updates. " + this._nodeUpdateQueue.length + " remaining"); 549 | this._processingNodes = false; 550 | }).catch(err => { 551 | log.info("VoyagerBot", "Processed " + nodesToProcess.length + " node updates (with errors). " + this._nodeUpdateQueue.length + " remaining"); 552 | log.error("VoyagerBot", err); 553 | this._processingNodes = false; 554 | }); 555 | } 556 | 557 | _tryUpdateNodeVersions() { 558 | if (!this._queueNodesForUpdate) { 559 | log.verbose("VoyagerBot", "Skipping state updates for all nodes - node updates are disabled"); 560 | return; 561 | } 562 | 563 | var promises = []; 564 | 565 | if (this._queueRoomsOnStartup) { 566 | promises.push(this._client.getJoinedRooms().then(joinedRooms => { 567 | _.forEach(joinedRooms, roomId => this._queueNodeUpdate({ 568 | objectId: roomId, 569 | type: 'room' 570 | }, /*saveQueue:*/false)); 571 | }).catch(err => { 572 | log.warn("VoyagerBot", "Failed to update rooms from startup: Matrix error."); 573 | })); 574 | } 575 | 576 | if (this._queueUsersOnStartup) { 577 | promises.push(this._store.getNodesByType('user').then(users => { 578 | _.forEach(users, user => this._queueNodeUpdate({ 579 | objectId: user.objectId, 580 | type: 'user' 581 | }, /*saveQueue:*/false)); 582 | })); 583 | } 584 | 585 | Promise.all(promises).then(() => this._savePendingNodeUpdates()); 586 | } 587 | 588 | _tryUpdateUserNodeVersion(userId) { 589 | if (!userId) { 590 | log.warn("VoyagerBot", "Try update user node failed: User ID was null"); 591 | return Promise.resolve(); 592 | } 593 | log.info("VoyagerBot", "Attempting an update for user node: " + userId); 594 | 595 | var userNode; 596 | var userMeta; 597 | 598 | // We won't bother updating the user information, just create the user 599 | return this.getNode(userId, 'user'); 600 | 601 | //return this.getNode(userId, 'user').then(node => { 602 | // userNode = node; 603 | 604 | // return this._store.getCurrentNodeState(userNode); 605 | //}).then(meta=> { 606 | // userMeta = meta; 607 | // return this._getUserVersion(userId); 608 | //}).then(realVersion => { 609 | // return this._tryUpdateNodeVersion(userNode, userMeta, realVersion); 610 | //}); 611 | } 612 | 613 | _tryUpdateRoomNodeVersion(roomId) { 614 | if (!roomId) { 615 | log.warn("VoyagerBot", "Try update room node failed: Room ID was null"); 616 | return Promise.resolve(); 617 | } 618 | log.info("VoyagerBot", "Attempting an update for room node: " + roomId); 619 | 620 | var roomNode; 621 | var roomMeta; 622 | var roomAliases; 623 | 624 | return this.getNode(roomId, 'room').then(node => { 625 | roomNode = node; 626 | 627 | return this._store.getCurrentNodeState(roomNode); 628 | }).then(meta => { 629 | roomMeta = meta; 630 | 631 | return this._store.getNodeAliases(roomNode); 632 | }).then(aliases => { 633 | roomAliases = aliases || []; 634 | return this._getRoomVersion(roomId); 635 | }).then(realVersion => { 636 | return this._tryUpdateNodeVersion(roomNode, roomMeta, realVersion, roomAliases); 637 | }); 638 | } 639 | 640 | _replaceNulls(obj, defs) { 641 | for (var key in obj) { 642 | if (obj[key] === null || obj[key] === undefined) { 643 | if (defs[key] !== null && defs[key] !== undefined) { 644 | obj[key] = defs[key]; 645 | } 646 | } 647 | } 648 | } 649 | 650 | _tryUpdateNodeVersion(node, meta, currentVersion, storedAliases) { 651 | var newVersion = {}; 652 | var updated = false; 653 | var aliasesUpdated = false; 654 | 655 | var defaults = { 656 | displayName: '', 657 | avatarUrl: '', 658 | isAnonymous: true, 659 | primaryAlias: '' 660 | }; 661 | 662 | // Ensure that `null != ''` doesn't end up triggering an update 663 | this._replaceNulls(meta, defaults); 664 | this._replaceNulls(currentVersion, defaults); 665 | 666 | if (currentVersion.displayName != meta.displayName) { 667 | newVersion.displayName = currentVersion.displayName || ''; 668 | updated = true; 669 | } 670 | if (currentVersion.avatarUrl != meta.avatarUrl) { 671 | newVersion.avatarUrl = currentVersion.avatarUrl || ''; 672 | updated = true; 673 | } 674 | if (currentVersion.isAnonymous != meta.isAnonymous) { 675 | newVersion.isAnonymous = currentVersion.isAnonymous; 676 | updated = true; 677 | } 678 | if (currentVersion.primaryAlias != meta.primaryAlias && node.type == 'room') { 679 | newVersion.primaryAlias = currentVersion.primaryAlias || ''; 680 | updated = true; 681 | } 682 | 683 | if (currentVersion.aliases) { 684 | newVersion.aliasCount = storedAliases.length; 685 | if (currentVersion.aliases.length != storedAliases.length) { 686 | aliasesUpdated = true; 687 | } else { 688 | for (var newAlias of storedAliases) { 689 | if (currentVersion.aliases.indexOf(newAlias.alias) === -1) { 690 | aliasesUpdated = true; 691 | break; 692 | } 693 | } 694 | } 695 | } 696 | 697 | if (node.id == 54) 698 | console.log(currentVersion); 699 | newVersion.userCount = (currentVersion.stats ? currentVersion.stats.users : 0); 700 | newVersion.serverCount = (currentVersion.stats ? currentVersion.stats.servers : 0); 701 | 702 | var statsUpdated = 703 | meta.userCount != newVersion.userCount 704 | || meta.aliasCount != newVersion.aliasCount 705 | || meta.serverCount != newVersion.serverCount; 706 | 707 | var versionPromise = Promise.resolve(); 708 | var aliasPromise = Promise.resolve(); 709 | 710 | if (updated || statsUpdated) { 711 | log.info("VoyagerBot", "Updating meta for node " + node.objectId + " to: " + JSON.stringify(newVersion)); 712 | 713 | var oldValues = {}; 714 | for (var key in newVersion) { 715 | oldValues[key] = meta[key]; 716 | } 717 | log.info("VoyagerBot", "Old meta for node " + node.objectId + " was (changed properties only): " + JSON.stringify(oldValues)); 718 | 719 | versionPromise = this._store.createNodeVersion(node, newVersion); 720 | } 721 | 722 | if (aliasesUpdated) { 723 | log.info("VoyagerBot", "Updating aliases for node " + node.objectId + " to " + JSON.stringify(currentVersion.aliases) + " from " + JSON.stringify(storedAliases)); 724 | aliasPromise = this._store.setNodeAliases(node, currentVersion.aliases); 725 | } 726 | 727 | return Promise.all([versionPromise, aliasPromise]); 728 | } 729 | } 730 | 731 | module.exports = VoyagerBot; 732 | -------------------------------------------------------------------------------- /src/api/ApiHandler.js: -------------------------------------------------------------------------------- 1 | var express = require("express"); 2 | var log = require("./../LogService"); 3 | var config = require("config"); 4 | var moment = require('moment'); 5 | 6 | const USE_SAMPLE = false; 7 | const STATS_CACHE_TIME_MS = 1 * 60 * 60 * 1000; // 1 hour 8 | 9 | /** 10 | * Processes and controls API requests 11 | */ 12 | class ApiHandler { 13 | 14 | /** 15 | * Creates a new API handler 16 | * @param {VoyagerStore} store the store to use 17 | * @param {VoyagerBot} bot the bot to use 18 | */ 19 | constructor(store, bot) { 20 | this._store = store; 21 | this._bot = bot; 22 | this._lastStats = null; 23 | 24 | this._app = express(); 25 | this._app.use(express.static('web-dist')); 26 | 27 | this._app.get('/api/v1/network', this._getNetwork.bind(this)); 28 | this._app.get('/api/v1/nodes', this._getNodes.bind(this)); 29 | this._app.get('/api/v1/nodes/publicRooms', this._getPublicRooms.bind(this)); 30 | this._app.get('/api/v1/nodes/:id', this._getNode.bind(this)); 31 | this._app.get('/api/v1/events', this._getEvents.bind(this)); 32 | this._app.get('/api/v1/stats', this._getStats.bind(this)); 33 | } 34 | 35 | start() { 36 | this._app.listen(config.get('web.port'), config.get('web.address')); 37 | log.info("ApiHandler", "API Listening on " + config.get("web.address") + ":" + config.get("web.port")); 38 | } 39 | 40 | _getStats(request, response) { 41 | if (this._lastStats) { 42 | var msDiff = moment().valueOf() - this._lastStats.updated; 43 | if (msDiff < STATS_CACHE_TIME_MS) { 44 | response.setHeader("Content-Type", "application/json"); 45 | response.send(JSON.stringify(this._lastStats)); 46 | return; 47 | } 48 | } 49 | 50 | log.info("ApiHandler", "Stats expired. Getting updated stats to cache..."); 51 | this._store.getBasicStats().then(stats => { 52 | log.info("ApiHandler", "Got updated stats. " + JSON.stringify(stats)); 53 | stats.updated = moment().valueOf(); 54 | this._lastStats = stats; 55 | response.setHeader("Content-Type", "application/json"); 56 | response.send(JSON.stringify(this._lastStats)); 57 | }).catch(err => { 58 | log.error("ApiHandler", err); 59 | response.sendStatus(500); 60 | }); 61 | } 62 | 63 | _getNetwork(request, response) { 64 | if (USE_SAMPLE) { 65 | var json = require("./network_sample.json"); 66 | response.setHeader("Content-Type", "application/json"); 67 | response.send(JSON.stringify(json)); 68 | return; 69 | } 70 | 71 | var limit = Math.max(0, Math.min(10000, request.query.limit || 1000)); 72 | var since = Math.max(0, request.query.since || 0); 73 | 74 | var handledNodeIds = []; 75 | var nodes = []; 76 | var links = []; 77 | var remaining = 0; 78 | var redactedLinks = 0; 79 | var hiddenLinks = 0; 80 | 81 | log.info("ApiHandler", "Getting events for query since=" + since + " limit=" + limit); 82 | this._store.getTimelineEventsPaginated(since, limit).then(dto => { 83 | log.info("ApiHandler", "Got " + dto.events.length + " events (" + dto.remaining + " remaining) for query since=" + since + " limit=" + limit); 84 | 85 | remaining = dto.remaining; 86 | 87 | var bannedRooms = []; 88 | 89 | for (var event of dto.events) { 90 | if (event.link.type != 'kick' && event.link.type != 'ban') continue; 91 | 92 | bannedRooms.push(event.targetNode.id); 93 | } 94 | 95 | for (var event of dto.events) { 96 | if (event.link.isRedacted || bannedRooms.indexOf(event.sourceNode.id) !== -1 || bannedRooms.indexOf(event.targetNode.id) !== -1) { 97 | redactedLinks++; 98 | continue; 99 | } 100 | 101 | if (!event.link.isVisible) { 102 | hiddenLinks++; 103 | continue; 104 | } 105 | 106 | if (handledNodeIds.indexOf(event.sourceNode.id) === -1) { 107 | nodes.push(this._nodeToJsonObject(event.sourceNode, event.sourceNodeMeta)); 108 | handledNodeIds.push(event.sourceNode.id); 109 | } 110 | if (handledNodeIds.indexOf(event.targetNode.id) === -1) { 111 | nodes.push(this._nodeToJsonObject(event.targetNode, event.targetNodeMeta)); 112 | handledNodeIds.push(event.targetNode.id); 113 | } 114 | 115 | if (!event.targetNode.isRedacted && !event.sourceNode.isRedacted) 116 | links.push(this._linkToJsonObject(event.link)); 117 | } 118 | 119 | var payload = { 120 | total: links.length, 121 | remaining: remaining, 122 | redacted: redactedLinks, 123 | hidden: hiddenLinks, 124 | results: { 125 | nodes: nodes, 126 | links: links 127 | } 128 | }; 129 | response.setHeader("Content-Type", "application/json"); 130 | response.send(JSON.stringify(payload)); 131 | }, err => { 132 | log.error("ApiHandler", err); 133 | response.sendStatus(500); 134 | }).catch(err => { 135 | log.error("ApiHandler", err); 136 | response.sendStatus(500); 137 | }); 138 | } 139 | 140 | _getNodes(request, response) { 141 | this._store.getAllNodes().then(nodes => { 142 | var payload = nodes.map(r => this._nodeToJsonObject(r, r.currentMeta)); 143 | response.setHeader("Content-Type", "application/json"); 144 | response.send(JSON.stringify(payload)); 145 | }, err => { 146 | log.error("ApiHandler", err); 147 | response.sendStatus(500); 148 | }).catch(err => { 149 | log.error("ApiHandler", err); 150 | response.sendStatus(500); 151 | }); 152 | } 153 | 154 | _getPublicRooms(request, response) { 155 | this._store.getPublicRooms().then(nodes => { 156 | var promise = Promise.resolve(); 157 | var mapped = nodes.map(r => this._nodeToJsonObject(r, r.currentMeta)); 158 | 159 | promise.then(() => { 160 | response.setHeader("Content-Type", "application/json"); 161 | response.send(JSON.stringify(mapped)); 162 | }); 163 | }).catch(err => { 164 | log.error("ApiHandler", err); 165 | response.sendStatus(500); 166 | }); 167 | } 168 | 169 | _getNode(request, response) { 170 | this._store.getNodeById(request.params.id).then(node => { 171 | if (!node) { 172 | response.setHeader("Content-Type", "application/json"); 173 | response.status(404); 174 | response.send("{}"); 175 | } else { 176 | this._store.getCurrentNodeState(node).then(meta => { 177 | var payload = this._nodeToJsonObject(node, meta); 178 | response.setHeader("Content-Type", "application/json"); 179 | response.send(JSON.stringify(payload)); 180 | }, err => { 181 | log.error("ApiHandler", err); 182 | response.sendStatus(500); 183 | }); 184 | } 185 | }, err => { 186 | log.error("ApiHandler", err); 187 | response.sendStatus(500); 188 | }).catch(err => { 189 | log.error("ApiHandler", err); 190 | response.sendStatus(500); 191 | }); 192 | } 193 | 194 | _getEvents(request, response) { 195 | var limit = Math.max(0, Math.min(10000, request.query.limit || 1000)); 196 | var since = Math.max(0, request.query.since || 0); 197 | 198 | var events = []; 199 | var remaining = 0; 200 | 201 | this._store.getStateEventsPaginated(since, limit).then(dto => { 202 | remaining = dto.remaining; 203 | 204 | for (var event of dto.events) { 205 | var obj = { 206 | id: event.stateEvent.id, 207 | type: event.stateEvent.type, 208 | timestamp: event.stateEvent.timestamp, 209 | meta: null 210 | }; 211 | 212 | if (event.node && event.nodeVersion) { 213 | obj.nodeId = event.node.id; 214 | obj.nodeVersionId = event.nodeVersion.id; 215 | 216 | var tempMeta = this._nodeToJsonObject(event.node, event.nodeVersion, true); 217 | obj.meta = tempMeta.meta; 218 | } 219 | 220 | if (event.link) { 221 | obj.linkId = event.link.id; 222 | 223 | var tempMeta = this._linkToJsonObject(event.link); 224 | obj.meta = tempMeta.meta; 225 | } 226 | 227 | events.push(obj); 228 | } 229 | 230 | var payload = { 231 | total: events.length, 232 | remaining: remaining, 233 | results: { 234 | events: events 235 | } 236 | }; 237 | response.setHeader("Content-Type", "application/json"); 238 | response.send(JSON.stringify(payload)); 239 | }, err => { 240 | log.error("ApiHandler", err); 241 | response.sendStatus(500); 242 | }).catch(err => { 243 | log.error("ApiHandler", err); 244 | response.sendStatus(500); 245 | }); 246 | } 247 | 248 | _nodeToJsonObject(node, meta, allowEmptyStrings = false) { 249 | var obj = { 250 | id: node.id, 251 | firstIntroduced: node.firstTimestamp, 252 | meta: { 253 | type: node.type, 254 | isAnonymous: meta.isAnonymous === null ? true : meta.isAnonymous, 255 | stats: { 256 | users: meta.userCount ? meta.userCount : 0, 257 | servers: meta.serverCount ? meta.serverCount : 0, 258 | aliases: meta.aliasCount ? meta.aliasCount : 0 259 | } 260 | } 261 | }; 262 | 263 | if (!obj.meta.isAnonymous) { 264 | obj.meta.objectId = node.objectId; 265 | if (meta.displayName !== null && (meta.displayName !== '' && !allowEmptyStrings)) obj.meta.displayName = meta.displayName; 266 | if (meta.avatarUrl !== null && (meta.avatarUrl !== '' && !allowEmptyStrings)) obj.meta.avatarUrl = meta.avatarUrl; 267 | if (meta.primaryAlias !== null && (meta.primaryAlias !== '' && !allowEmptyStrings)) obj.meta.primaryAlias = meta.primaryAlias; 268 | } 269 | 270 | return obj; 271 | } 272 | 273 | _linkToJsonObject(link) { 274 | var obj = { 275 | id: link.id, 276 | timestamp: link.timestamp, 277 | meta: { 278 | sourceNodeId: link.sourceNodeId, 279 | targetNodeId: link.targetNodeId, 280 | type: link.type 281 | } 282 | }; 283 | 284 | return obj; 285 | } 286 | } 287 | 288 | module.exports = ApiHandler; -------------------------------------------------------------------------------- /src/matrix/CommandProcessor.js: -------------------------------------------------------------------------------- 1 | var log = require("./../LogService"); 2 | var Promise = require('bluebird'); 3 | 4 | require("string_score"); // automagically adds itself as "words".score(...) 5 | 6 | /** 7 | * Processes bot commands from Matrix 8 | */ 9 | class CommandProcessor { 10 | 11 | /** 12 | * Creates a new command processor 13 | * @param {VoyagerBot} bot the bot instance this processor is for 14 | * @param {VoyagerStore} store the store to use 15 | */ 16 | constructor(bot, store) { 17 | this._bot = bot; 18 | this._store = store; 19 | } 20 | 21 | /** 22 | * Processes a command from Matrix 23 | * @param {string} roomId the room the event happened in 24 | * @param {*} event the event 25 | * @param {string[]} cmdArguments the arguments to the command 26 | * @returns {Promise<*>} resolves when processing complete 27 | */ 28 | processCommand(roomId, event, cmdArguments) { 29 | if (cmdArguments.length == 0) { 30 | return this._reply(roomId, event, 'Unknown command. Try !voyager help'); 31 | } 32 | 33 | if (cmdArguments[0] == 'help') { 34 | return this._sendHelp(roomId, event); 35 | } else if (cmdArguments[0] == 'enroll' || cmdArguments[0] == 'showme') { 36 | return this._store.setEnrolled(event['sender'], true).then(() => this._reply(roomId, event, "Your name and avatar will appear on the graph.")); 37 | } else if (cmdArguments[0] == 'withdraw' || cmdArguments[0] == 'hideme') { 38 | return this._store.setEnrolled(event['sender'], false).then(() => this._reply(roomId, event, "Your name and avatar will no longer appear on the graph.")); 39 | } else if (cmdArguments[0] == 'linkme') { 40 | return this._handleSelfLink(roomId, event, /*isLinking=*/true, cmdArguments[1]); 41 | } else if (cmdArguments[0] == 'unlinkme') { 42 | return this._handleSelfLink(roomId, event, /*isLinking=*/false, cmdArguments[1]); 43 | } else if (cmdArguments[0] == 'search') { 44 | return this._handleSearch(roomId, event, cmdArguments.splice(1)); 45 | } else if (cmdArguments[0] == 'leave') { 46 | return this._handleSoftKick(roomId, event); 47 | } else if (cmdArguments[0] == 'addme') { 48 | return this._handleSelfRedact(roomId, event, /*isAdding=*/true); 49 | } else if (cmdArguments[0] == 'removeme') { 50 | return this._handleSelfRedact(roomId, event, /*isAdding=*/false); 51 | } else if (cmdArguments[0] == 'dnt' || cmdArguments[0] == 'donottrack' || cmdArguments[0] == 'untrackme') { 52 | return this._handleDnt(roomId, event, /*isTracking=*/false); 53 | } else if (cmdArguments[0] == 'trackme') { 54 | return this._handleDnt(roomId, event, /*isTracking=*/true); 55 | } else return this._reply(roomId, event, "Unknown command. Try !voyager help"); 56 | } 57 | 58 | _reply(roomId, event, message) { 59 | return this._bot.sendNotice(roomId, event['sender'] + ": " + message); 60 | } 61 | 62 | _sendHelp(roomId, event) { 63 | return this._bot.sendNotice(roomId, 64 | "!voyager showme - Sets your name and avatar to be visible on the graph\n" + 65 | "!voyager hideme - Hides your name and avatar from the graph\n" + 66 | "!voyager linkme [room] - Links your user account to the specified room (defaults to current room)\n" + 67 | "!voyager unlinkme [room] - Removes your self-links from the specified room (defaults to current room)\n" + 68 | "!voyager search - Searches for rooms that have the specified keywords\n" + 69 | "!voyager leave - Forces the bot to leave the room, but keep the room on the graph\n" + 70 | "!voyager removeme - Takes your user node, and associated links, off of the graph\n" + 71 | "!voyager addme - Adds your user node, and associated links, to the graph\n" + 72 | "!voyager dnt - The bot will read your messages, but not follow any links to rooms in them\n" + 73 | "!voyager trackme - The bot will read and follow rooms links in your messages. This is the default.\n" + 74 | "!voyager help - This menu\n" + 75 | "\n" + 76 | "View the current graph online at https://voyager.t2bot.io" 77 | ); 78 | } 79 | 80 | _handleDnt(roomId, event, isTracking) { 81 | return this._store.setDnt(event['sender'], !isTracking).then(() => { 82 | return this._reply(roomId, event, isTracking ? "I'll follow room links you post" : "I'll stop following links to rooms you post in rooms"); 83 | }); 84 | } 85 | 86 | _handleSoftKick(roomId, event) { 87 | return this._bot.getRoomStateEvents(roomId, 'm.room.power_levels', /*stateKey:*/'') 88 | .then(powerLevels => { 89 | if (!powerLevels) 90 | return this._reply(roomId, event, "Error processing command: Could not find m.room.power_levels state event").then(() => Promise.reject("Missing m.room.power_levels in room " + roomId)); 91 | 92 | var powerLevel = powerLevels['users'][event['sender']]; 93 | if (!powerLevel && powerLevel !== 0) powerLevel = powerLevels['users_default']; 94 | if (powerLevel < powerLevels['kick']) 95 | return this._reply(roomId, event, "You must be at least power level " + powerLevels['kick'] + " to kick me from the room").then(() => Promise.reject(event['sender'] + " does not have permission to kick in room " + roomId)); 96 | }) 97 | .then(() => Promise.all([this._bot.getNode(event['sender'], 'user'), this._bot.getNode(roomId, 'room')])) 98 | .then(userRoomNodes => this._store.createLink(userRoomNodes[0], userRoomNodes[1], 'soft_kick', event['origin_server_ts'], false, false)) 99 | .then(link => this._store.createTimelineEvent(link, event['origin_server_ts'], event['event_id'], 'Soft kicked')) 100 | .then(() => this._bot.leaveRoom(roomId)) 101 | .catch(err => { 102 | log.error("CommandProcessor", err); 103 | }); 104 | } 105 | 106 | _handleSearch(roomId, event, keywords) { 107 | if (keywords.length == 0) 108 | return this._reply(roomId, event, "No keywords specified. Try !voyager search "); 109 | 110 | return this._store.findNodesMatching(keywords).then(results => { 111 | // We have to score these ourselves now (the database just does a rough contains check to get a smaller dataset) 112 | for (var result of results) { 113 | result.rank = 0; 114 | result.rank += result.mentionCount * 0.1; // 10% of mention count is added to score to bump numbers 115 | for (var keyword of keywords) { 116 | if (result.primaryAlias) result.rank += result.meta.primaryAlias.toLowerCase().split(':', 2)[0].score(keyword.toLowerCase()); 117 | if (result.displayName) result.rank += result.meta.displayName.toLowerCase().score(keyword.toLowerCase()); 118 | 119 | if (result.aliases) { 120 | // We only take the highest alias rank for other aliases to avoid the case where 121 | // a room may have several available aliases, all of which are there to just bump 122 | // the score up a bit. 123 | var highestAliasRank = 0; 124 | for (var alias of result.aliases) { 125 | var rank = alias.alias.toLowerCase().split(':', 2)[0].score(keyword.toLowerCase()); 126 | if (rank > highestAliasRank) 127 | highestAliasRank = rank; 128 | } 129 | result.rank += highestAliasRank; 130 | } 131 | } 132 | } 133 | 134 | results.sort((a, b) => { 135 | return b.rank - a.rank; 136 | }); 137 | 138 | return results; 139 | }).then(sortedResults => { 140 | var sample = sortedResults.splice(0, 5); 141 | if (sample.length == 0) 142 | return this._reply(roomId, event, "No results for keywords: " + keywords); 143 | 144 | var response = "Found the following rooms:\n"; 145 | for (var result of sample) 146 | response += (sample.indexOf(result) + 1) + ". " + (result.meta.primaryAlias || result.aliases[0].alias) + (result.meta.displayName ? " | " + result.meta.displayName : "") + "\n" 147 | return this._reply(roomId, event, response); 148 | }); 149 | } 150 | 151 | _handleSelfRedact(roomId, event, isAdding) { 152 | return this._bot.getNode(event['sender'], 'user') 153 | .then(node => { 154 | if (isAdding && !node.isRedacted) 155 | return this._reply(roomId, event, "You are already available on the graph"); 156 | 157 | if (!isAdding && node.isRedacted) 158 | return this._reply(roomId, event, "You are already removed from the graph"); 159 | 160 | if (isAdding) 161 | return this._store.unredactNode(node).then(() => this._reply(roomId, event, "You have been restored to the graph")); 162 | else return this._store.redactNode(node).then(() => this._reply(roomId, event, "You have been removed from the graph")); 163 | }); 164 | } 165 | 166 | _handleSelfLink(inRoomId, event, isLinking, roomArg) { 167 | var alias = inRoomId; 168 | var roomId; 169 | var userNode; 170 | var roomNode; 171 | var link; 172 | 173 | if (!roomArg) roomArg = inRoomId; 174 | 175 | return this._bot.matchRoomSharedWith(roomArg, event['sender']).then(roomId => { 176 | if (!roomId) 177 | return this._reply(inRoomId, event, "You do not appear to be in the room " + roomArg +" or the room does not exist.").then(() => Promise.reject("Sender not in room or room missing: " + roomArg)); 178 | return roomId; 179 | }).then(id => { 180 | roomId = id; 181 | return this._bot.getNode(event['sender'], 'user'); 182 | }).then(n => { 183 | userNode = n; 184 | return this._bot.getNode(roomId, 'room'); 185 | }).then(n => { 186 | roomNode = n; 187 | return this._store.findLink(userNode, roomNode, 'self_link'); 188 | }).then(sl => { 189 | link = sl; 190 | 191 | if (link && isLinking) return this._reply(inRoomId, event, "You are already linked to " + alias); 192 | if (!link && !isLinking) return this._reply(inRoomId, event, "You are not linked to " + alias); 193 | 194 | if (!link && isLinking) { 195 | return this._store.createLink(userNode, roomNode, 'self_link', event['origin_server_ts']) 196 | .then(link => this._store.createTimelineEvent(link, event['origin_server_ts'], event['event_id'])) 197 | .then(() => this._store.setEnrolled(event['sender'], true)) 198 | .then(() => this._reply(inRoomId, event, "You have been linked to " + alias + " and are no longer anonymous")); 199 | } 200 | 201 | if (link && !isLinking) { 202 | return this._store.redactLink(link) 203 | .then(() => this._store.createTimelineEvent(link, event['origin_server_ts'], event['id'])) 204 | .then(() => this._reply(inRoomId, event, "You are no longer linked to " + alias)); 205 | } 206 | 207 | throw new Error("Invalid state. isLinking = " + isLinking + ", link = " + link); 208 | }).catch(err => { 209 | log.error("CommandProcessor", err); 210 | }); 211 | } 212 | } 213 | 214 | module.exports = CommandProcessor; -------------------------------------------------------------------------------- /src/matrix/MatrixClientLite.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var log = require("../LogService"); 3 | var filterJson = require("./filter_template.json"); 4 | var LocalStorage = require("node-localstorage").LocalStorage; 5 | var Promise = require('bluebird'); 6 | var EventEmitter = require('events'); 7 | var _ = require('lodash'); 8 | 9 | /** 10 | * Represents a lightweight matrix client with minimal functionality. Fires the following events: 11 | * * "room_leave" (roomId, leaveEvent) 12 | * * "room_join" (roomId) 13 | * * "room_invite" (roomId, inviteEvent) 14 | * * "room_message" (roomId, messageEvent) - only fired for joined rooms 15 | * * "room_name" (roomId, nameEvent) 16 | * * "room_avatar" (roomId, avatarEvent) 17 | * * "room_join_rules" (roomId, joinRulesEvent) 18 | * * "user_name" (roomId, nameEvent) 19 | * * "user_avatar" (roomId, avatarEvent) 20 | */ 21 | class MatrixLiteClient extends EventEmitter { 22 | 23 | /** 24 | * Creates a new matrix client 25 | * @param {string} homeserverUrl the homeserver base url 26 | * @param {string} accessToken the matrix access token 27 | * @param {string} selfId the ID of the user owning the token 28 | */ 29 | constructor(homeserverUrl, accessToken, selfId) { 30 | super(); 31 | this.selfId = selfId; 32 | this._accessToken = accessToken; 33 | this._homeserverUrl = homeserverUrl; 34 | this._requestId = 0; 35 | this._stopSyncPromise = null; 36 | 37 | if (this._homeserverUrl.endsWith('/')) 38 | this._homeserverUrl = this._homeserverUrl.substr(0, this._homeserverUrl.length - 2); 39 | 40 | // Note: We use localstorage because we don't need the complexity of a database, and this makes resetting state a lot easier. 41 | this._kvStore = new LocalStorage("db/mtx_client_lite_localstorage", 100 * 1024 * 1024); // quota is 100mb 42 | 43 | this._joinedRooms = []; 44 | 45 | log.verbose("MatrixClientLite", "New client created for " + this.selfId + " at homeserver " + this._homeserverUrl); 46 | } 47 | 48 | /** 49 | * Starts the matrix client with the designated filter. If no filter is specified, a new one will be created from 50 | * a local file. 51 | * @returns {Promise<*>} resolves when the client has started 52 | */ 53 | start(filter = null) { 54 | if (!filter || typeof(filter) !== 'object') { 55 | log.verbose("MatrixClientLite", "No filter given to start method. Assuming defaults"); 56 | filter = filterJson; 57 | } 58 | 59 | var createFilter = false; 60 | 61 | var existingFilter = this._kvStore.getItem("m.filter"); 62 | if (existingFilter) { 63 | existingFilter = JSON.parse(existingFilter); 64 | log.verbose("MatrixClientLite", "Found existing filter. Checking consistency with given filter"); 65 | if (JSON.stringify(existingFilter.filter) == JSON.stringify(filter)) { 66 | log.verbose("MatrixClientLite", "Filters are the same - not creating a new one"); 67 | this._filterId = existingFilter.id; 68 | } else { 69 | createFilter = true; 70 | } 71 | } else createFilter = true; 72 | 73 | var filterPromise = Promise.resolve(); 74 | if (createFilter) { 75 | log.verbose("MatrixClientLite", "Creating new filter"); 76 | filterPromise = this._do("POST", "/_matrix/client/r0/user/" + this.selfId + "/filter", null, filter).then(response => { 77 | this._filterId = response["filter_id"]; 78 | this._kvStore.removeItem("m.synctoken"); // clear token because we've got a new filter 79 | this._kvStore.setItem("m.filter", JSON.stringify({ 80 | id: this._filterId, 81 | filter: filter 82 | })); 83 | }); 84 | } 85 | 86 | return this.getJoinedRooms().then(roomIds => { 87 | this._joinedRooms = roomIds; 88 | return filterPromise; 89 | }).then(() => { 90 | log.info("MatrixClientLite", "Starting sync with filter ID " + this._filterId); 91 | this._startSync(); 92 | }); 93 | } 94 | 95 | /** 96 | * Stops the client 97 | * @returns {Promise<*>} resolves when the client has stopped 98 | */ 99 | stop() { 100 | log.info("MatrixClientLite", "Stop requested"); 101 | this._stopSyncPromise = new Promise(); 102 | return this._stopSyncPromise; 103 | } 104 | 105 | _startSync() { 106 | var syncToken = this._kvStore.getItem("m.synctoken"); 107 | 108 | var promiseWhile = Promise.method(() => { 109 | if (this._stopSyncPromise) { 110 | log.info("MatrixClientLite", "Client stop requested - stopping"); 111 | this._stopSyncPromise.resolve(); 112 | return; 113 | } 114 | 115 | return this._doSync(syncToken).then(response => { 116 | syncToken = response["next_batch"]; 117 | this._kvStore.setItem("m.synctoken", syncToken); 118 | log.info("MatrixClientLite", "Received sync. Next token: " + syncToken); 119 | 120 | this._processSync(response); 121 | }, error => null).then(promiseWhile.bind(this)); // errors are already reported, so suppress 122 | }); 123 | promiseWhile(); // start the loop 124 | } 125 | 126 | _doSync(syncToken) { 127 | log.info("MatrixClientLite", "Doing sync with token: " + syncToken); 128 | var conf = { 129 | filter: this._filterId, 130 | //since: syncToken, // can't have this here or synapse complains when it's null 131 | full_state: false, 132 | timeout: 10000 133 | }; 134 | if (syncToken) conf['since'] = syncToken; 135 | 136 | // timeout is 30s if we have a token, otherwise 10min 137 | return this._do("GET", "/_matrix/client/r0/sync", conf, null, (syncToken ? 30000 : 600000)); 138 | } 139 | 140 | _processSync(data) { 141 | if (!data['rooms']) return; 142 | 143 | // process leaves 144 | var leftRooms = data['rooms']['leave']; 145 | if (!leftRooms) leftRooms = {}; 146 | _.forEach(_.keys(leftRooms), roomId => { 147 | var roomInfo = leftRooms[roomId]; 148 | if (!roomInfo['timeline'] || !roomInfo['timeline']['events']) return; 149 | 150 | var leaveEvent = null; 151 | for (var event of roomInfo['timeline']['events']) { 152 | if (event['type'] !== 'm.room.member') continue; 153 | if (event['state_key'] !== this.selfId) continue; 154 | if (leaveEvent && leaveEvent['unsigned']['age'] < event['unsigned']['age']) continue; 155 | 156 | leaveEvent = event; 157 | } 158 | 159 | if (!leaveEvent) { 160 | log.warn("MatrixClientLite", "Left room " + roomId + " without a leave event in /sync"); 161 | return; 162 | } 163 | 164 | this.emit("room_leave", roomId, leaveEvent); 165 | }); 166 | 167 | // process invites 168 | var inviteRooms = data['rooms']['invite']; 169 | if (!inviteRooms) inviteRooms = {}; 170 | _.forEach(_.keys(inviteRooms), roomId => { 171 | var roomInfo = inviteRooms[roomId]; 172 | if (!roomInfo['invite_state'] || !roomInfo['invite_state']['events']) return; 173 | 174 | var inviteEvent = null; 175 | for (var event of roomInfo['invite_state']['events']) { 176 | if (event['type'] !== 'm.room.member') continue; 177 | if (event['state_key'] !== this.selfId) continue; 178 | if (event['membership'] !== 'invite') continue; 179 | if (inviteEvent && inviteEvent['unsigned']['age'] < event['unsigned']['age']) continue; 180 | 181 | inviteEvent = event; 182 | } 183 | 184 | if (!inviteEvent) { 185 | log.warn("MatrixClientLite", "Invited to room " + roomId + " without an invite event in /sync"); 186 | return; 187 | } 188 | 189 | this.emit("room_invite", roomId, inviteEvent); 190 | }); 191 | 192 | // process joined rooms and their messages 193 | var joinedRooms = data['rooms']['join']; 194 | if (!joinedRooms) joinedRooms = {}; 195 | var roomIds = _.keys(joinedRooms); 196 | for (var roomId of roomIds) { 197 | this.emit("room_join", roomId); 198 | 199 | var roomInfo = joinedRooms[roomId]; 200 | if (!roomInfo['timeline'] || !roomInfo['timeline']['events']) continue; 201 | 202 | for (var event of roomInfo['timeline']['events']) { 203 | if (event['type'] === 'm.room.message') this.emit("room_message", roomId, event); 204 | else if (event['type'] === 'm.room.member') this._checkMembershipUpdate(roomId, event); 205 | else if (event['type'] === 'm.room.name') this.emit("room_name", roomId, event); 206 | else if (event['type'] === 'm.room.avatar') this.emit("room_avatar", roomId, event); 207 | else if (event['type'] === 'm.room.join_rules') this.emit("room_join_rules", roomId, event); 208 | else if (event['type'] === 'm.room.canonical_alias') this.emit("room_join_rules", roomId, event); 209 | else if (event['type'] === 'm.room.aliases') this.emit("room_join_rules", roomId, event); 210 | else log.silly("MatrixClientLite", "Not handling sync event " + event['type']); 211 | } 212 | } 213 | } 214 | 215 | _checkMembershipUpdate(roomId, event) { 216 | var current = event['content']; 217 | var prev = event['unsigned']['prev_content']; 218 | if (!prev) return; 219 | 220 | if (prev['avatar_url'] !== current['avatar_url']) this.emit("user_avatar", roomId, event); 221 | if (prev['displayname'] !== current['displayname']) this.emit("user_name", roomId, event); 222 | 223 | // else no change 224 | } 225 | 226 | /** 227 | * Gets the room state for the given room. Returned as raw events. 228 | * @param {string} roomId the room ID to get state for 229 | * @returns {Promise<*[]>} resolves to the room's state 230 | */ 231 | getRoomState(roomId) { 232 | return this._do("GET", "/_matrix/client/r0/rooms/" + roomId + "/state"); 233 | } 234 | 235 | /** 236 | * Gets the state events for a given room of a given type under the given state key. 237 | * @param {string} roomId the room ID 238 | * @param {string} type the event type 239 | * @param {String} stateKey the state key, falsey if not needed 240 | * @returns {Promise<*|*[]>} resolves to the state event(s) 241 | */ 242 | getRoomStateEvents(roomId, type, stateKey) { 243 | return this._do("GET", "/_matrix/client/r0/rooms/" + roomId + "/state/" + type + "/" + (stateKey ? stateKey : '')); 244 | } 245 | 246 | /** 247 | * Gets information on a given user 248 | * @param {string} userId the user ID to lookup 249 | * @returns {Promise<*>} information on the user 250 | */ 251 | getUserInfo(userId) { 252 | return this._do("GET", "/_matrix/client/r0/profile/" + userId); 253 | } 254 | 255 | /** 256 | * Joins the given room 257 | * @param {string} roomIdOrAlias the room ID or alias to join 258 | * @returns {Promise} resolves to the joined room ID 259 | */ 260 | joinRoom(roomIdOrAlias) { 261 | if (this._joinedRooms.indexOf(roomIdOrAlias) !== -1) { 262 | log.info("MatrixClientLite", "No-oping join: Already joined room"); 263 | return Promise.resolve(roomIdOrAlias); 264 | } 265 | 266 | var isRoom = roomIdOrAlias[0] === '!'; 267 | var originalId = roomIdOrAlias; 268 | roomIdOrAlias = encodeURIComponent(roomIdOrAlias); 269 | 270 | // TODO: Make this better 271 | var joinPromise = isRoom ? Promise.resolve({room_id: originalId}) : this._do("GET", "/_matrix/client/r0/directory/room/" + roomIdOrAlias); 272 | 273 | // Do a directory lookup to get the room ID 274 | return joinPromise.then(response => { 275 | if (this._joinedRooms.indexOf(response['room_id']) !== -1) { 276 | log.info("MatrixClientLite", "No-oping join: Already joined room"); 277 | return response; 278 | } 279 | 280 | // Actually do the join because we aren't joined 281 | return this._do("POST", "/_matrix/client/r0/join/" + roomIdOrAlias); 282 | }).then(response => { 283 | return response['room_id']; 284 | }); 285 | } 286 | 287 | /** 288 | * Gets a list of joined room IDs 289 | * @returns {Promise} resolves to a list of room IDs the client participates in 290 | */ 291 | getJoinedRooms() { 292 | return this._do("GET", "/_matrix/client/r0/joined_rooms").then(response => response['joined_rooms']); 293 | } 294 | 295 | /** 296 | * Leaves the given room 297 | * @param {string} roomId the room ID to leave 298 | * @returns {Promise<*>} resolves when left 299 | */ 300 | leaveRoom(roomId) { 301 | return this._do("POST", "/_matrix/client/r0/rooms/" + roomId + "/leave"); 302 | } 303 | 304 | /** 305 | * Sends a read receipt for an event in a room 306 | * @param {string} roomId the room ID to send the receipt to 307 | * @param {string} eventId the event ID to set the receipt at 308 | * @returns {Promise<*>} resolves when the receipt has been sent 309 | */ 310 | sendReadReceipt(roomId, eventId) { 311 | return this._do("POST", "/_matrix/client/r0/rooms/" + roomId + "/receipt/m.read/" + eventId); 312 | } 313 | 314 | /** 315 | * Sends a notice to the given room 316 | * @param {string} roomId the room ID to send the notice to 317 | * @param {string} text the text to send 318 | * @returns {Promise} resolves to the event ID that represents the message 319 | */ 320 | sendNotice(roomId, text) { 321 | var txnId = (new Date().getTime()) + "LR" + this._requestId; 322 | return this._do("PUT", "/_matrix/client/r0/rooms/" + roomId + "/send/m.room.message/" + txnId, null, { 323 | body: text, 324 | msgtype: "m.notice" 325 | }).then(response => { 326 | return response['event_id']; 327 | }); 328 | } 329 | 330 | /** 331 | * Converts a media URI to a thumbnail URL 332 | * @param {string} mxcUri the mxc uri 333 | * @param {number} width the width in pixels for the thumbnail 334 | * @param {number} height the height in pixels for the thumbnail 335 | * @returns {string} the URL to get the thumbnail at 336 | */ 337 | convertMediaToThumbnail(mxcUri, width, height) { 338 | var shorthand = mxcUri.substring("mxc://".length).split("?")[0].split("#")[0]; // split off path components 339 | 340 | // shorthand is serverName/mediaId (which matches the URL format) 341 | return this._homeserverUrl + "/_matrix/media/r0/thumbnail/" + shorthand + "?width=" + width + "&height=" + height; 342 | } 343 | 344 | _do(method, endpoint, qs = null, body = null, timeout = 60000, raw = false) { 345 | if (!endpoint.startsWith('/')) 346 | endpoint = '/' + endpoint; 347 | 348 | var requestId = ++this._requestId; 349 | 350 | var url = this._homeserverUrl + endpoint; 351 | 352 | log.verbose("MatrixLiteClient (REQ-" + requestId + ")", method + " " + url); 353 | 354 | if (!qs) qs = {}; 355 | qs['access_token'] = this._accessToken; 356 | 357 | var cleanAndStringify = (obj) => { 358 | var clone = JSON.parse(JSON.stringify(obj)); 359 | if (clone['access_token']) clone['access_token'] = ''; 360 | return JSON.stringify(clone); 361 | }; 362 | 363 | if (qs) log.verbose("MatrixLiteClient (REQ-" + requestId + ")", "qs = " + cleanAndStringify(qs)); 364 | if (body) log.verbose("MatrixLiteClient (REQ-" + requestId + ")", "body = " + cleanAndStringify(body)); 365 | 366 | var params = { 367 | url: url, 368 | method: method, 369 | json: body, 370 | qs: qs, 371 | timeout: timeout, 372 | }; 373 | 374 | return new Promise((resolve, reject) => { 375 | request(params, (err, response, body) => { 376 | if (err) { 377 | log.error("MatrixLiteClient (REQ-" + requestId + ")", err); 378 | reject(err); 379 | } else { 380 | if (typeof(body) === 'string') { 381 | try { 382 | body = JSON.parse(body); 383 | } catch (e) { 384 | } 385 | } 386 | 387 | log.verbose("MatrixLiteClient (REQ-" + requestId + " RESP-H" + response.statusCode + ")", response.body); 388 | if (response.statusCode < 200 || response.statusCode >= 300) { 389 | log.error("MatrixLiteClient (REQ-" + requestId + ")", response.body); 390 | reject(response); 391 | } else resolve(raw ? response : body); 392 | } 393 | }); 394 | }); 395 | } 396 | } 397 | 398 | module.exports = MatrixLiteClient; 399 | -------------------------------------------------------------------------------- /src/matrix/filter_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_format": "client", 3 | "account_data": { 4 | "limit": 0, 5 | "types": [] 6 | }, 7 | "presence": { 8 | "limit": 0, 9 | "types": [] 10 | }, 11 | "room": { 12 | "include_leave": true, 13 | "account_data": { 14 | "types": [] 15 | }, 16 | "timeline": { 17 | }, 18 | "ephemeral": { 19 | "types": [] 20 | }, 21 | "state": { 22 | "types": [] 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /src/storage/models/dnt.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('dnt', { 3 | userId: { 4 | type: DataTypes.STRING, 5 | allowNull: false, 6 | primaryKey: true, 7 | field: 'userId' 8 | }, 9 | isDnt: { 10 | type: DataTypes.BOOLEAN, 11 | allowNull: false, 12 | field: 'isDnt' 13 | } 14 | }, { 15 | tableName: 'dnt', 16 | underscored: false, 17 | timestamps: false 18 | }); 19 | }; 20 | -------------------------------------------------------------------------------- /src/storage/models/links.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('links', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | primaryKey: true, 7 | autoIncrement: true, 8 | field: 'id' 9 | }, 10 | type: { 11 | type: DataTypes.STRING, 12 | allowNull: false, 13 | field: 'type' 14 | }, 15 | sourceNodeId: { 16 | type: DataTypes.INTEGER, 17 | allowNull: false, 18 | field: 'sourceNodeId', 19 | references: { 20 | model: "nodes", 21 | key: "id" 22 | } 23 | }, 24 | targetNodeId: { 25 | type: DataTypes.INTEGER, 26 | allowNull: false, 27 | field: 'targetNodeId', 28 | references: { 29 | model: "nodes", 30 | key: "id" 31 | } 32 | }, 33 | timestamp: { 34 | type: DataTypes.TIME, 35 | allowNull: false, 36 | field: 'timestamp' 37 | }, 38 | isVisible: { 39 | type: DataTypes.BOOLEAN, 40 | allowNull: false, 41 | field: 'isVisible' 42 | }, 43 | isRedacted: { 44 | type: DataTypes.BOOLEAN, 45 | allowNull: false, 46 | field: 'isRedacted' 47 | } 48 | }, { 49 | tableName: 'links', 50 | underscored: false, 51 | timestamps: false 52 | }); 53 | }; 54 | -------------------------------------------------------------------------------- /src/storage/models/node_aliases.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('nodeAliases', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | nodeId: { 11 | type: DataTypes.INTEGER, 12 | allowNull: false, 13 | field: 'nodeId', 14 | references: { 15 | model: "nodes", 16 | key: "id" 17 | } 18 | }, 19 | alias: { 20 | type: DataTypes.STRING, 21 | allowNull: true, 22 | field: 'alias' 23 | } 24 | }, { 25 | tableName: 'node_aliases', 26 | underscored: false, 27 | timestamps: false 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /src/storage/models/node_meta.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('nodeMeta', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | displayName: { 11 | type: DataTypes.STRING, 12 | allowNull: true, 13 | field: 'displayName' 14 | }, 15 | avatarUrl: { 16 | type: DataTypes.STRING, 17 | allowNull: true, 18 | field: 'avatarUrl' 19 | }, 20 | isAnonymous: { 21 | type: DataTypes.BOOLEAN, 22 | allowNull: true, 23 | field: 'isAnonymous' 24 | }, 25 | primaryAlias: { 26 | type: DataTypes.STRING, 27 | allowNull: true, 28 | field: 'primaryAlias' 29 | }, 30 | userCount: { 31 | type: DataTypes.INTEGER, 32 | allowNull: true, 33 | field: 'userCount' 34 | }, 35 | serverCount: { 36 | type: DataTypes.INTEGER, 37 | allowNull: true, 38 | field: 'serverCount' 39 | }, 40 | aliasCount: { 41 | type: DataTypes.INTEGER, 42 | allowNull: true, 43 | field: 'aliasCount' 44 | } 45 | }, { 46 | tableName: 'node_meta', 47 | underscored: false, 48 | timestamps: false 49 | }); 50 | }; 51 | -------------------------------------------------------------------------------- /src/storage/models/node_versions.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('nodeVersions', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | nodeId: { 11 | type: DataTypes.INTEGER, 12 | allowNull: false, 13 | field: 'nodeId', 14 | references: { 15 | model: "nodes", 16 | key: "id" 17 | } 18 | }, 19 | displayName: { 20 | type: DataTypes.STRING, 21 | allowNull: true, 22 | field: 'displayName' 23 | }, 24 | avatarUrl: { 25 | type: DataTypes.STRING, 26 | allowNull: true, 27 | field: 'avatarUrl' 28 | }, 29 | isAnonymous: { 30 | type: DataTypes.BOOLEAN, 31 | allowNull: true, 32 | field: 'isAnonymous' 33 | }, 34 | primaryAlias: { 35 | type: DataTypes.STRING, 36 | allowNull: true, 37 | field: 'primaryAlias' 38 | } 39 | }, { 40 | tableName: 'node_versions', 41 | underscored: false, 42 | timestamps: false 43 | }); 44 | }; 45 | -------------------------------------------------------------------------------- /src/storage/models/nodes.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('nodes', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | nodeMetaId: { 11 | type: DataTypes.INTEGER, 12 | allowNull: false, 13 | field: 'nodeMetaId', 14 | references: { 15 | model: "nodeMeta", 16 | key: "id" 17 | } 18 | }, 19 | type: { 20 | type: DataTypes.STRING, 21 | allowNull: false, 22 | field: 'type' 23 | }, 24 | objectId: { 25 | type: DataTypes.STRING, 26 | allowNull: false, 27 | field: 'objectId' 28 | }, 29 | isReal: { 30 | type: DataTypes.BOOLEAN, 31 | allowNull: false, 32 | field: 'isReal' 33 | }, 34 | isRedacted: { 35 | type: DataTypes.BOOLEAN, 36 | allowNull: false, 37 | field: 'isRedacted' 38 | }, 39 | firstTimestamp: { 40 | type: DataTypes.TIME, 41 | allowNull: false, 42 | field: 'firstTimestamp' 43 | } 44 | }, { 45 | tableName: 'nodes', 46 | underscored: false, 47 | timestamps: false 48 | }); 49 | }; 50 | -------------------------------------------------------------------------------- /src/storage/models/state_events.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('stateEvents', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | type: { 11 | type: DataTypes.STRING, 12 | allowNull: false, 13 | field: 'type' 14 | }, 15 | linkId: { 16 | type: DataTypes.INTEGER, 17 | allowNull: true, 18 | field: 'linkId' 19 | }, 20 | nodeId: { 21 | type: DataTypes.INTEGER, 22 | allowNull: true, 23 | field: 'nodeId', 24 | references: { 25 | model: "nodes", 26 | key: "id" 27 | } 28 | }, 29 | nodeVersionId: { 30 | type: DataTypes.INTEGER, 31 | allowNull: true, 32 | field: 'nodeVersionId', 33 | references: { 34 | model: "nodeVersions", 35 | key: "id" 36 | } 37 | }, 38 | timestamp: { 39 | type: DataTypes.TIME, 40 | allowNull: false, 41 | field: 'timestamp' 42 | } 43 | }, { 44 | tableName: 'state_events', 45 | underscored: false, 46 | timestamps: false 47 | }); 48 | }; 49 | -------------------------------------------------------------------------------- /src/storage/models/timeline_events.js: -------------------------------------------------------------------------------- 1 | module.exports = function (sequelize, DataTypes) { 2 | return sequelize.define('timelineEvents', { 3 | id: { 4 | type: DataTypes.INTEGER, 5 | allowNull: false, 6 | autoIncrement: true, 7 | primaryKey: true, 8 | field: 'id' 9 | }, 10 | linkId: { 11 | type: DataTypes.INTEGER, 12 | allowNull: false, 13 | field: 'linkId', 14 | references: { 15 | model: "links", 16 | key: "id" 17 | } 18 | }, 19 | message: { 20 | type: DataTypes.STRING, 21 | allowNull: true, 22 | field: 'message' 23 | }, 24 | matrixEventId: { 25 | type: DataTypes.STRING, 26 | allowNull: false, 27 | field: 'matrixEventId' 28 | }, 29 | timestamp: { 30 | type: DataTypes.TIME, 31 | allowNull: false, 32 | field: 'timestamp' 33 | } 34 | }, { 35 | tableName: 'timeline_events', 36 | underscored: false, 37 | timestamps: false 38 | }); 39 | }; 40 | -------------------------------------------------------------------------------- /static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/static/.gitkeep -------------------------------------------------------------------------------- /static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/static/img/favicon.ico -------------------------------------------------------------------------------- /static/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/static/img/favicon.png -------------------------------------------------------------------------------- /web-config/dev.env.js: -------------------------------------------------------------------------------- 1 | var merge = require('webpack-merge'); 2 | var prodEnv = require('./prod.env'); 3 | 4 | module.exports = merge(prodEnv, { 5 | NODE_ENV: '"development"' 6 | }); 7 | -------------------------------------------------------------------------------- /web-config/index.js: -------------------------------------------------------------------------------- 1 | // see http://vuejs-templates.github.io/webpack for documentation. 2 | var path = require('path'); 3 | 4 | module.exports = { 5 | build: { 6 | env: require('./prod.env'), 7 | index: path.resolve(__dirname, '../web-dist/index.html'), 8 | assetsRoot: path.resolve(__dirname, '../web-dist'), 9 | assetsSubDirectory: 'static', 10 | assetsPublicPath: '/', 11 | productionSourceMap: true, 12 | // Gzip off by default as many popular static hosts such as 13 | // Surge or Netlify already gzip all static assets for you. 14 | // Before setting to `true`, make sure to: 15 | // npm install --save-dev compression-webpack-plugin 16 | productionGzip: false, 17 | productionGzipExtensions: ['js', 'css'], 18 | // Run the build command with an extra argument to 19 | // View the bundle analyzer report after build finishes: 20 | // `npm run build --report` 21 | // Set to `true` or `false` to always turn it on or off 22 | bundleAnalyzerReport: process.env.npm_config_report 23 | }, 24 | dev: { 25 | env: require('./dev.env'), 26 | port: 8080, 27 | autoOpenBrowser: true, 28 | assetsSubDirectory: 'static', 29 | assetsPublicPath: '/', 30 | proxyTable: { 31 | '/api': { 32 | target: 'http://localhost:8184', 33 | secure: false 34 | } 35 | }, 36 | // CSS Sourcemaps off by default because relative paths are "buggy" 37 | // with this option, according to the CSS-Loader README 38 | // (https://github.com/webpack/css-loader#sourcemaps) 39 | // In our experience, they generally work as expected, 40 | // just be aware of this issue when enabling this option. 41 | cssSourceMap: false 42 | } 43 | }; 44 | -------------------------------------------------------------------------------- /web-config/prod.env.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | NODE_ENV: '"production"' 3 | }; 4 | -------------------------------------------------------------------------------- /web-src/App.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 13 | 14 | 40 | -------------------------------------------------------------------------------- /web-src/components/graph/graph.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /web-src/components/graph/script.js: -------------------------------------------------------------------------------- 1 | var d3 = require("d3"); 2 | export default { 3 | name: 'graph', 4 | data () { 5 | return { 6 | isLoading: true, 7 | error: null, 8 | graph: null, 9 | nodeHover: {x: 0, y: 0, item: null, is: false}, 10 | linkHover: {x: 0, y: 0, item: null, is: false}, 11 | transformStr: "", 12 | width: Math.max(document.documentElement.clientWidth, window.innerWidth || 0), 13 | height: Math.max(document.documentElement.clientHeight, window.innerHeight || 0) - 4, 14 | 15 | hasBoundZoom: false 16 | }; 17 | }, 18 | mounted () { 19 | return this.$http.get("/api/v1/network", {params: {since: 0, limit: 10000}}).then(response => { 20 | this.graph = this.processNetwork(response.body.results); 21 | this.genGraph(); 22 | this.stylize(); 23 | this.isLoading = false; 24 | }).catch(error => { 25 | this.error = "There was a problem loading the graph data. Please try again later."; 26 | this.isLoading = false; 27 | console.error(error); 28 | }); 29 | }, 30 | updated () { 31 | if (this.hasBoundZoom) { 32 | return; 33 | } 34 | 35 | this.hasBoundZoom = true; 36 | d3.select("#graphsvg").call(d3.zoom() 37 | .scaleExtent([-1, 10]) 38 | .on('zoom', () => { 39 | this.transformStr = "translate(" + d3.event.transform.x + "," + d3.event.transform.y + ")" 40 | + "scale(" + d3.event.transform.k + "," + d3.event.transform.k + ")"; 41 | })); 42 | }, 43 | methods: { 44 | isNodeHovered (node) { 45 | if (!this.nodeHover.item || !this.nodeHover.is) { 46 | return true; 47 | } 48 | 49 | for (var link of node.directLinks) { 50 | if (link.target.id === this.nodeHover.item.id || link.source.id === this.nodeHover.item.id) { 51 | return true; 52 | } 53 | } 54 | 55 | return false; 56 | }, 57 | isLinkHovered (link) { 58 | if (!this.nodeHover.item || !this.nodeHover.is) { 59 | return true; 60 | } 61 | return link.target.id === this.nodeHover.item.id || link.source.id === this.nodeHover.item.id; 62 | }, 63 | enterItem (item, state, event) { 64 | state.x = event.clientX; 65 | state.y = event.clientY; 66 | state.item = item; 67 | state.is = true; 68 | }, 69 | exitItem (state) { 70 | state.is = false; 71 | }, 72 | getLinkText (link) { 73 | if (!link) { 74 | return ""; 75 | } 76 | 77 | return link.value + ' ' + link.type.replace(/_ /g, ' ') + (link.value !== 1 ? 's' : '') + ' from ' + link.source.name + ' to ' + link.target.name; 78 | }, 79 | getFillForText (text) { 80 | let hash = text.hashCode(); 81 | if (hash < 0) hash = hash * -1; 82 | 83 | const options = [ 84 | "#ae71c6", 85 | "#71c6a8", 86 | "#a9c671", 87 | "#7189c6", 88 | "#c46fa8" 89 | ]; 90 | 91 | return options[hash % options.length]; 92 | }, 93 | getLinkParams (link) { 94 | var dx = (link.target.x - link.source.x) / 0.1; 95 | var dy = (link.target.y - link.source.y) / 0.1; 96 | var dr = Math.sqrt((dx * dx) + (dy * dy)); 97 | 98 | var hasRelatedLinks = link.relatedLinkTypes && link.relatedLinkTypes.length > 1; 99 | if (!hasRelatedLinks && (link.inverseCount === 0 || link.value === 0)) { 100 | return "M" + link.source.x + "," + link.source.y + " L" + link.target.x + "," + link.target.y; 101 | } 102 | 103 | var shouldInvert = hasRelatedLinks ? (link.relatedLinkTypes.indexOf(link.type) !== 0) : false; 104 | var sx = shouldInvert ? link.target.x : link.source.x; 105 | var sy = shouldInvert ? link.target.y : link.source.y; 106 | var tx = shouldInvert ? link.source.x : link.target.x; 107 | var ty = shouldInvert ? link.source.y : link.target.y; 108 | 109 | return "M" + sx + "," + sy + "A" + dr + "," + dr + " 0 0,1 " + tx + "," + ty; 110 | }, 111 | downloadSvg () { 112 | var svgData = document.getElementById("graphsvg").outerHTML; 113 | var svgBlob = new Blob([svgData], {type: "image/svg+xml;charset=utf-8"}); 114 | var svgUrl = URL.createObjectURL(svgBlob); 115 | var downloadLink = document.createElement("a"); 116 | downloadLink.href = svgUrl; 117 | downloadLink.download = "voyager.svg"; 118 | document.body.appendChild(downloadLink); 119 | downloadLink.click(); 120 | document.body.removeChild(downloadLink); 121 | }, 122 | genGraph () { 123 | this.simulation = d3.forceSimulation(this.graph.nodes) 124 | .force("link", d3.forceLink(this.graph.links).id(n => n.id).distance(k => Math.sqrt(k.value) * 75)) 125 | .force("charge", d3.forceManyBody().strength(n => Math.max(-400, n.linkCount * -40))) 126 | .force("center", d3.forceCenter(this.width / 2, this.height / 2)) 127 | .force("collide", d3.forceCollide(i => i.type === 'room' ? 20 : 15).strength(0.5)); 128 | this.simulation.force("link").links(this.graph.links); 129 | this.simulation.stop(); 130 | 131 | for (let i = 0, n = Math.ceil(Math.log(this.simulation.alphaMin()) / Math.log(1 - this.simulation.alphaDecay())); i < n; ++i) { 132 | this.simulation.tick(); 133 | } 134 | }, 135 | stylize () { 136 | for (var node of this.graph.nodes) { 137 | node.r = node.type === "room" ? 15 : 8; 138 | } 139 | 140 | for (var link of this.graph.links) { 141 | switch (link.type) { 142 | case "invite": 143 | link.stroke = "#10b748"; 144 | continue; 145 | case "self_link": 146 | link.stroke = "#694bcc"; 147 | continue; 148 | case "kick": 149 | link.stroke = "#dd5e1a"; 150 | continue; 151 | case "ban": 152 | link.stroke = "#ff2626"; 153 | continue; 154 | case "message": 155 | default: 156 | link.stroke = "#999"; 157 | } 158 | } 159 | }, 160 | processNetwork (network) { 161 | const nodes = []; 162 | const links = []; 163 | const nodeIndexMap = {}; 164 | const nodeLinksMap = []; 165 | 166 | for (let networkNode of network.nodes) { 167 | let node = { 168 | id: networkNode.id, 169 | name: networkNode.meta.displayName || networkNode.meta.objectId || 'Matrix ' + networkNode.meta.type, 170 | group: networkNode.meta.type, 171 | type: networkNode.meta.type, 172 | avatarUrl: networkNode.meta.avatarUrl, 173 | isAnonymous: networkNode.meta.isAnonymous, 174 | primaryAlias: networkNode.meta.primaryAlias, 175 | linkCount: 0, 176 | directLinks: [] 177 | }; 178 | nodeIndexMap[networkNode.id] = nodes.length; 179 | nodes.push(node); 180 | } 181 | 182 | let linkMap = {}; 183 | let linkTypesMap = {}; 184 | for (let networkLink of network.links) { 185 | const key = networkLink.meta.sourceNodeId + " to " + networkLink.meta.targetNodeId + " for " + networkLink.meta.type; 186 | const typeKey = networkLink.meta.sourceNodeId + " and " + networkLink.meta.targetNodeId; 187 | const inverseTypeKey = networkLink.meta.sourceNodeId + " and " + networkLink.meta.targetNodeId; 188 | 189 | let typeArray = linkTypesMap[typeKey] ? linkTypesMap[typeKey] : linkTypesMap[inverseTypeKey]; 190 | if (!typeArray) { 191 | typeArray = linkTypesMap[typeKey] = []; 192 | } 193 | 194 | if (typeArray.indexOf(networkLink.meta.type) === -1) { 195 | typeArray.push(networkLink.meta.type); 196 | } 197 | 198 | if (!linkMap[key]) { 199 | linkMap[key] = { 200 | count: 0, 201 | sourceNodeId: networkLink.meta.sourceNodeId, 202 | targetNodeId: networkLink.meta.targetNodeId, 203 | type: networkLink.meta.type, 204 | relatedLinkTypes: typeArray 205 | }; 206 | } 207 | 208 | linkMap[key].count++; 209 | } 210 | 211 | for (let linkKey in linkMap) { 212 | const aggregateLink = linkMap[linkKey]; 213 | const inverseLinkKey = aggregateLink.targetNodeId + " to " + aggregateLink.sourceNodeId + " for " + aggregateLink.type; 214 | const oppositeAggregateLink = linkMap[inverseLinkKey]; 215 | 216 | let link = { 217 | sourceNode: nodeIndexMap[aggregateLink.sourceNodeId], 218 | targetNode: nodeIndexMap[aggregateLink.targetNodeId], 219 | source: aggregateLink.sourceNodeId, 220 | target: aggregateLink.targetNodeId, 221 | value: aggregateLink.count, 222 | type: aggregateLink.type, 223 | inverseCount: oppositeAggregateLink ? oppositeAggregateLink.count : 0, 224 | relatedLinkTypes: aggregateLink.relatedLinkTypes 225 | }; 226 | links.push(link); 227 | 228 | let sourceNode = nodes[link.sourceNode]; 229 | let targetNode = nodes[link.targetNode]; 230 | 231 | sourceNode.linkCount++; 232 | targetNode.linkCount++; 233 | sourceNode.directLinks.push(link); 234 | targetNode.directLinks.push(link); 235 | 236 | if (nodeLinksMap.indexOf(sourceNode.id + "," + targetNode.id) === -1) { 237 | nodeLinksMap.push(sourceNode.id + "," + targetNode.id); 238 | } 239 | if (nodeLinksMap.indexOf(targetNode.id + "," + sourceNode.id) === -1) { 240 | nodeLinksMap.push(targetNode.id + "," + sourceNode.id); 241 | } 242 | } 243 | 244 | return {links, nodes, nodeLinks: nodeLinksMap}; 245 | } 246 | } 247 | }; 248 | -------------------------------------------------------------------------------- /web-src/components/graph/style.css: -------------------------------------------------------------------------------- 1 | div.tooltip { 2 | position: absolute; 3 | top: 0; 4 | left: 0; 5 | text-align: left; 6 | padding: 5px; 7 | border-radius: 3px; 8 | pointer-events: none; 9 | opacity: 0; 10 | color: #000; 11 | font-size: 14px; 12 | transition: opacity 0.5s; 13 | } 14 | 15 | div.tooltip.node-tooltip, div.tooltip.link-tooltip { 16 | background: #bfdbff; 17 | border: 1px solid #98c3f9; 18 | } 19 | 20 | .graph, .graph .graph-container { 21 | padding: 0; 22 | margin: 0; 23 | height: 100%; 24 | } 25 | 26 | #graphsvg { 27 | padding: 0; 28 | margin: 0; 29 | } 30 | 31 | .nav { 32 | position: absolute; 33 | top: 0; 34 | right: 150px; 35 | background-color: #ccc; 36 | border: 1px solid #aaa; 37 | border-top: none; 38 | border-radius: 0 0 7px 7px; 39 | padding: 5px 10px; 40 | } 41 | 42 | .nav a { 43 | text-decoration: none; 44 | } -------------------------------------------------------------------------------- /web-src/components/graph/template.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | View the stats 4 |
5 |
6 |

Loading...

7 |
8 |
9 |

{{ error }}

10 |
11 |
12 | 13 |
15 | {{ getLinkText(linkHover.item) }} 16 |
17 |
19 | {{ nodeHover.item ? nodeHover.item.name : '' }}
20 | {{ nodeHover.item ? nodeHover.item.primaryAlias : '' }} 21 |
22 | 23 | 24 | 25 | 26 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 53 | {{ (node.isAnonymous ? (node.type === 'room' ? '#' : '@') : (node.name[0] === '!' || node.name[0] === '@' || node.name[0] === '#' ? node.name[1] : node.name[0])).toUpperCase() }} 54 | 55 | 56 | 57 |
58 |
-------------------------------------------------------------------------------- /web-src/components/landing/landing.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /web-src/components/landing/script.js: -------------------------------------------------------------------------------- 1 | export default { 2 | name: 'landing', 3 | data () { 4 | return { 5 | 6 | }; 7 | }, 8 | mounted () { 9 | }, 10 | methods: { 11 | } 12 | }; 13 | -------------------------------------------------------------------------------- /web-src/components/landing/style.css: -------------------------------------------------------------------------------- 1 | .landing { 2 | width: 960px; 3 | margin: auto; 4 | } 5 | 6 | .button { 7 | text-decoration: none; 8 | color: #2c3e50; 9 | padding: 15px 25px; 10 | border: 1px solid #2c3e50; 11 | border-radius: 5px; 12 | -webkit-transition: background-color 250ms; 13 | -moz-transition: background-color 250ms; 14 | -ms-transition: background-color 250ms; 15 | -o-transition: background-color 250ms; 16 | transition: background-color 250ms; 17 | } 18 | 19 | .button:hover { 20 | background-color: #98c3f9; 21 | } -------------------------------------------------------------------------------- /web-src/components/landing/template.html: -------------------------------------------------------------------------------- 1 |
2 |

Matrix Traveler (Voyager)

3 |

Voyager is a bot that travels through Matrix trying to find new rooms. It does this by sitting in rooms and waiting for someone to mention another room, at which point it tries to join that room. Each new room it discovers is mapped to a public graph.

4 |

Learn more about the bot on Github or in #voyager:t2bot.io.

5 |

 

6 | View the graph 7 | View the stats 8 |
-------------------------------------------------------------------------------- /web-src/components/stats/script.js: -------------------------------------------------------------------------------- 1 | import StatBox from "./stat-box/stat-box.vue"; 2 | import SortIcon from "./sort-icon/sort-icon.vue"; 3 | 4 | export default { 5 | name: 'stats', 6 | components: {StatBox, SortIcon}, 7 | data () { 8 | return { 9 | isLoading: true, 10 | error: null, 11 | stats: { 12 | rooms: 2575, 13 | users: 250123, 14 | aliases: 4123, 15 | mentions: 10000, 16 | servers: 24576 17 | }, 18 | sortBy: "id", 19 | sortDir: "natural", 20 | rooms: [], 21 | seenRooms: JSON.parse(localStorage.getItem("t2l-voyager.seenNodes") || "[]") 22 | }; 23 | }, 24 | computed: { 25 | sortedRooms: function() { 26 | var getProp = (item, prop) => { 27 | if (prop === "name") return item.meta.displayName; 28 | if (prop === "alias") return item.meta.primaryAlias; 29 | if (prop === "users") return item.meta.stats.users; 30 | if (prop === "servers") return item.meta.stats.servers; 31 | if (prop === "aliases") return item.meta.stats.aliases; 32 | if (prop === "id") return item.id; 33 | return item[prop]; 34 | }; 35 | 36 | return this.rooms.sort((a, b) => { 37 | var order = 0; 38 | if (getProp(a, this.sortBy) < getProp(b, this.sortBy)) order = -1; 39 | if (getProp(a, this.sortBy) > getProp(b, this.sortBy)) order = 1; 40 | 41 | return this.sortDir === "asc" ? order : -order; 42 | }); 43 | } 44 | }, 45 | mounted () { 46 | this.$http.get('/api/v1/stats').then(response => { 47 | this.stats = response.body; 48 | return this.$http.get('/api/v1/nodes/publicRooms'); 49 | }).then(response => { 50 | this.rooms = response.body; 51 | this.isLoading = false; 52 | 53 | var ids = []; 54 | for (var room of this.rooms) { 55 | ids.push(room.id); 56 | } 57 | localStorage.setItem("t2l-voyager.seenNodes", JSON.stringify(ids)); 58 | }).catch(error => { 59 | this.error = "There was a problem loading the data. Please try again later."; 60 | this.isLoading = false; 61 | console.error(error); 62 | }); 63 | }, 64 | methods: { 65 | isNew (item) { 66 | if (this.seenRooms.length === 0) return false; 67 | return this.seenRooms.indexOf(item.id) === -1; 68 | }, 69 | setSort (column) { 70 | if (this.sortBy !== column) { 71 | this.sortBy = column; 72 | this.sortDir = "asc"; 73 | } else { 74 | if (this.sortDir === "asc") this.sortDir = "desc"; 75 | else if (this.sortDir === "desc") this.sortDir = "natural"; 76 | else if (this.sortDir === "natural") this.sortDir = "asc"; 77 | } 78 | 79 | if (this.sortDir === "natural") 80 | this.sortBy = 'id'; 81 | } 82 | } 83 | }; 84 | -------------------------------------------------------------------------------- /web-src/components/stats/sort-icon/script.js: -------------------------------------------------------------------------------- 1 | export default { 2 | name: 'sort-icon', 3 | props: ['current', 'direction', 'watch'] 4 | }; 5 | -------------------------------------------------------------------------------- /web-src/components/stats/sort-icon/sort-icon.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /web-src/components/stats/sort-icon/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turt2live/matrix-voyager-bot/8d5326f93fed3e413a639e151e57bc7dc17cb1e5/web-src/components/stats/sort-icon/style.css -------------------------------------------------------------------------------- /web-src/components/stats/sort-icon/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /web-src/components/stats/stat-box/script.js: -------------------------------------------------------------------------------- 1 | export default { 2 | name: 'stat-box', 3 | props: ['name', 'value', 'icon', 'color'] 4 | }; 5 | -------------------------------------------------------------------------------- /web-src/components/stats/stat-box/stat-box.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /web-src/components/stats/stat-box/style.css: -------------------------------------------------------------------------------- 1 | .stat-box { 2 | background-color: #fff; 3 | border-radius: 2px; 4 | border: 1px solid #ddd; 5 | } 6 | 7 | .stat-box .icon { 8 | float: left; 9 | padding: 0 15px; 10 | background-color: pink; 11 | width: 48px; 12 | height: 100%; 13 | display: flex; 14 | align-items: center; 15 | justify-content: center; 16 | } 17 | 18 | .stat-box .content { 19 | text-align: left; 20 | padding: 5px; 21 | margin-left: 85px; 22 | } 23 | 24 | .stat-box .content .title { 25 | text-transform: uppercase; 26 | font-size: 0.8em; 27 | color: #888; 28 | } 29 | 30 | .stat-box .content .stat { 31 | margin-top: 4px; 32 | display: block; 33 | font-size: 2.5em; 34 | } -------------------------------------------------------------------------------- /web-src/components/stats/stat-box/template.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
5 |
6 | {{ name }} 7 | {{ value | formatNumber }} 8 |
9 |
-------------------------------------------------------------------------------- /web-src/components/stats/stats.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /web-src/components/stats/style.css: -------------------------------------------------------------------------------- 1 | .stats { 2 | } 3 | 4 | .stats-container { 5 | min-width: 960px; 6 | max-width: 1500px; 7 | margin: auto; 8 | padding-top: 25px; 9 | } 10 | 11 | .stats-header { 12 | display: flex; 13 | align-items: stretch; 14 | } 15 | 16 | .stat-box { 17 | flex-grow: 1; 18 | flex-basis: 0; 19 | margin: 20px 10px; 20 | } 21 | 22 | .room-list { 23 | width: calc(100% - 20px); 24 | margin: 10px; 25 | background-color: #fff; 26 | border-collapse: collapse; 27 | border: 1px solid #ddd; 28 | } 29 | 30 | .room-list th, .room-list td { 31 | text-align: left; 32 | border-top: 1px solid #ddd; 33 | padding: 5px; 34 | } 35 | 36 | .room-list td.metric { 37 | text-align: center; 38 | } 39 | 40 | .room-list .left-data:first-child { 41 | width: 250px; 42 | max-width: 350px; 43 | } 44 | 45 | .room-list .right-metric { 46 | max-width: 110px; 47 | width: 85px; 48 | text-align: center; 49 | } 50 | 51 | .room-list .new-room { 52 | background-color: #ddaf00; 53 | padding: 3px; 54 | border-radius: 3px; 55 | } 56 | 57 | .room-icon { 58 | width: 20px; 59 | height: 20px; 60 | border-radius: 20px; 61 | vertical-align: middle; 62 | } 63 | 64 | .room-icon.missing { 65 | display: inline-block; 66 | background-color: #aac1ff; 67 | text-align: center; 68 | font-size: 1.1em; 69 | } 70 | 71 | .nav { 72 | position: absolute; 73 | top: 0; 74 | right: 150px; 75 | background-color: #ccc; 76 | border: 1px solid #aaa; 77 | border-top: none; 78 | border-radius: 0 0 7px 7px; 79 | padding: 5px 10px; 80 | } 81 | 82 | .nav a { 83 | text-decoration: none; 84 | } 85 | 86 | .sortable { 87 | cursor: pointer; 88 | } -------------------------------------------------------------------------------- /web-src/components/stats/template.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | View the graph 4 |
5 |
6 |

Loading...

7 |
8 |
9 |

{{ error }}

10 |
11 |
12 |
13 | 14 | 15 | 16 | 17 | 18 |
19 | 20 |
21 | 22 | 23 | 27 | 31 | 35 | 39 | 43 | 44 | 45 | 46 | 47 | 56 | 57 | 58 | 59 | 60 | 61 | 62 |
24 | 25 | Room Name 26 | 28 | 29 | Room Alias 30 | 32 | 33 | Users 34 | 36 | 37 | Servers 38 | 40 | 41 | Aliases 42 |
48 | 50 |
{{ room.meta.displayName[0].toUpperCase() }}
52 | 53 | New! 54 | {{ room.meta.displayName }} 55 | 		{{ room.meta.primaryAlias }}{{ room.meta.stats.users | formatNumber }}{{ room.meta.stats.servers | formatNumber }}{{ room.meta.stats.aliases | formatNumber }}
63 | 64 | -------------------------------------------------------------------------------- /web-src/main.js: -------------------------------------------------------------------------------- 1 | // The Vue build version to load with the `import` command 2 | // (runtime-only or standalone) has been set in webpack.base.conf with an alias. 3 | import Vue from "vue"; 4 | import App from "./App"; 5 | import router from "./vendorconf/router"; 6 | import http from "./vendorconf/http"; 7 | import 'vue-awesome/icons'; 8 | import Icon from 'vue-awesome/components/Icon' 9 | import numeral from 'numeral'; 10 | 11 | Vue.config.productionTip = false; 12 | 13 | Vue.component("icon", Icon); 14 | Vue.filter('formatNumber', (value) => { 15 | return numeral(value).format("0,0"); 16 | }); 17 | 18 | /* eslint-disable no-new */ 19 | new Vue({ 20 | el: '#app', 21 | router, 22 | http, 23 | template: '', 24 | components: {App} 25 | }); 26 | 27 | String.prototype.hashCode = function () { 28 | var hash = 0; 29 | if (this.length === 0) return hash; 30 | for (var i = 0; i < this.length; i++) { 31 | var chr = this.charCodeAt(i); 32 | hash = ((hash << 5) - hash) + chr; 33 | hash |= 0; // Convert to 32bit integer 34 | } 35 | return hash; 36 | }; 37 | -------------------------------------------------------------------------------- /web-src/vendorconf/http.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import Resource from "vue-resource"; 3 | 4 | Vue.use(Resource); 5 | 6 | export default {}; 7 | -------------------------------------------------------------------------------- /web-src/vendorconf/router.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import Router from "vue-router"; 3 | import Graph from "@/components/graph/graph"; 4 | import Stats from "@/components/stats/stats"; 5 | import Landing from "@/components/landing/landing"; 6 | 7 | Vue.use(Router); 8 | 9 | export default new Router({ 10 | routes: [ 11 | { 12 | path: '/', 13 | name: 'Landing', 14 | component: Landing 15 | }, 16 | { 17 | path: '/graph', 18 | name: 'Graph', 19 | component: Graph 20 | }, 21 | { 22 | path: '/stats', 23 | name: 'Stats', 24 | component: Stats 25 | } 26 | ] 27 | }); 28 | --------------------------------------------------------------------------------