├── .babelrc ├── .dockerignore ├── .eslintrc ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── config.js ├── index.js ├── package.json └── src ├── admin ├── checkAdmin.js ├── index.js ├── updateUser.js └── users.js ├── client ├── exynize.js ├── index.js └── webpack.config.js ├── components ├── create.js ├── delete.js ├── execute.js ├── get.js ├── getAll.js └── index.js ├── db ├── bus.js ├── component.js ├── connection.js ├── index.js ├── pipeline.js ├── pipelineLog.js └── user.js ├── index.js ├── logger └── index.js ├── pipes ├── create.js ├── delete.js ├── execute.js ├── get.js ├── getAll.js ├── index.js ├── input.js ├── inputSocket.js ├── log.js ├── result.js ├── resultSocket.js ├── runner │ ├── index.js │ ├── runPipeline.js │ └── runner.js ├── start.js ├── statusSocket.js └── stop.js ├── runner ├── compileWithRabbit.js ├── index.js ├── runWithRabbit.js ├── service.js ├── serviceWithRabbit.js └── stopWithRabbit.js ├── server.js ├── sockutil ├── authedSocket.js └── index.js ├── users ├── checkToken.js ├── hash.js ├── index.js ├── login.js ├── register.js ├── resetPassword.js ├── resetPasswordAccept.js ├── resetPasswordChange.js ├── sendEmail.js └── verify.js ├── util ├── asyncRequest.js └── index.js └── webpack └── webpack.config.base.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015", "stage-1"], 3 | "plugins": ["typecheck", "syntax-flow", "transform-flow-strip-types"] 4 | } 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | src/static/ 3 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | // I want to use babel-eslint for parsing! 3 | "parser": "babel-eslint", 4 | "env": { 5 | // I write for browser 6 | "browser": true, 7 | // in CommonJS 8 | "node": true, 9 | // use ES6 10 | "es6": true 11 | }, 12 | "ecmaProperties": { 13 | // enable JSX support 14 | "jsx": true 15 | }, 16 | "plugins": [ 17 | // enable react plugin 18 | "react" 19 | ], 20 | // To give you an idea how to override rule options 21 | "rules": { 22 | // Possible Errors 23 | "comma-dangle": 0, 24 | "no-console": 0, 25 | "no-debugger": 2, 26 | "no-dupe-keys": 2, 27 | "no-dupe-args": 2, 28 | "no-empty": 2, 29 | "no-extra-boolean-cast": 2, 30 | "no-extra-semi": 2, 31 | "no-invalid-regexp": 2, 32 | "no-irregular-whitespace": 2, 33 | "quote-props": [2, "consistent-as-needed", {"keywords": true}], 34 | "no-sparse-arrays": 2, 35 | "no-unreachable": 2, 36 | "use-isnan": 2, 37 | "valid-jsdoc": 2, 38 | "valid-typeof": 2, 39 | // Best Practices 40 | "consistent-return": 1, 41 | "curly": 2, 42 | "default-case": 2, 43 | "dot-notation": 2, 44 | "eqeqeq": 2, 45 | "no-alert": 2, 46 | "no-caller": 2, 47 | "no-else-return": 2, 48 | "no-eq-null": 2, 49 | "no-eval": 2, 50 | "no-extend-native": 2, 51 | "no-floating-decimal": 2, 52 | "no-implied-eval": 2, 53 | "no-iterator": 2, 54 | "no-labels": 2, 55 | "no-loop-func": 1, 56 | "no-lone-blocks": 2, 57 | "no-multi-spaces": 2, 58 | "no-native-reassign": 2, 59 | "no-new": 2, 60 | "no-new-func": 2, 61 | "no-new-wrappers": 2, 62 | "no-proto": 2, 63 | "no-redeclare": 2, 64 | "no-return-assign": 2, 65 | "no-script-url": 2, 66 | "no-self-compare": 2, 67 | "no-sequences": 2, 68 | "no-throw-literal": 2, 69 | "no-unused-expressions": 2, 70 | "no-void": 2, 71 | "radix": 2, 72 | "yoda": 0, 73 | // Strict Mode 74 | "strict": 0, 75 | // Variables 76 | "no-catch-shadow": 2, 77 | "no-delete-var": 2, 78 | "no-shadow": 2, 79 | "no-shadow-restricted-names": 2, 80 | "no-undef": 2, 81 | "no-unused-vars": [2, {"vars": "all", "args": "after-used"}], 82 | "no-use-before-define": 2, 83 | // Node 84 | "handle-callback-err": 2, 85 | "no-new-require": 2, 86 | "no-path-concat": 2, 87 | // Stylistic Issues 88 | "indent": 2, // 4 spaces 89 | "camelcase": 2, 90 | "comma-spacing": [2, {"before": false, "after": true}], 91 | "comma-style": [2, "last"], 92 | "eol-last": 2, 93 | "func-style": [2, "expression"], 94 | "max-nested-callbacks": [2, 3], 95 | "no-array-constructor": 2, 96 | "no-mixed-spaces-and-tabs": 2, 97 | "no-multiple-empty-lines": [1, {"max": 2}], 98 | "no-nested-ternary": 2, 99 | "no-new-object": 2, 100 | "semi-spacing": [2, {"before": false, "after": true}], 101 | "no-spaced-func": 2, 102 | "no-trailing-spaces": 2, 103 | "no-underscore-dangle": 2, 104 | "no-extra-parens": [2, "functions"], 105 | "quotes": [1, "single"], 106 | "semi": [2, "always"], 107 | "space-before-function-paren": [1, {"anonymous": "never", "named": "never"}], 108 | "keyword-spacing": 1, 109 | "space-before-blocks": [1, "always"], 110 | "object-curly-spacing": [1, "never"], 111 | "array-bracket-spacing": [1, "never"], 112 | "computed-property-spacing": [1, "never"], 113 | "space-in-parens": [1, "never"], 114 | "key-spacing": [1, {"beforeColon": false, "afterColon": true}], 115 | "space-infix-ops": 2, 116 | // complexity rules 117 | "max-depth": [2, 3], 118 | "max-statements": [1, 20], 119 | "complexity": [1, 3], 120 | "max-len": [2, 120], 121 | "max-params": [2, 3], 122 | // jsx rules 123 | "jsx-quotes": 1, 124 | "react/jsx-no-undef": 1, 125 | "react/jsx-uses-react": 1, 126 | "react/jsx-uses-vars": 1, 127 | "react/no-did-mount-set-state": 1, 128 | "react/no-did-update-set-state": 1, 129 | "react/no-multi-comp": 1, 130 | "react/react-in-jsx-scope": 1, 131 | "react/self-closing-comp": 1, 132 | "react/wrap-multilines": 1, 133 | // ES6 134 | "prefer-const": 2, 135 | "object-shorthand": [2, "always"], 136 | "no-var": 2 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # node modules 2 | node_modules/ 3 | 4 | # compiled client-side components 5 | src/static/ 6 | 7 | # logs 8 | *.log 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 0.9.0 / 2016-02-22 4 | ================== 5 | 6 | * public pipeline results support 7 | * only allow pipeline actions if invoked by owner 8 | 9 | 0.8.0 / 2016-02-22 10 | ================== 11 | 12 | * add configurable expiration time for all rabbitmq messages 13 | * better error handling and reporting 14 | 15 | 0.7.0 / 2016-02-16 16 | ================== 17 | 18 | * allow using http requests as input 19 | 20 | 0.6.1 / 2016-02-12 21 | ================== 22 | 23 | * update to microwork 0.7 for reconnect support 24 | 25 | 0.6.0 / 2016-02-12 26 | ================== 27 | 28 | * use docker volume to persist compiled components upon upgrades 29 | * add pipeline delete functionality 30 | * add component delete method 31 | * change repubsub to standalone library 32 | * replace amqplib with microwork 33 | 34 | 0.5.0 / 2016-01-28 35 | ================== 36 | 37 | New features: 38 | * add username support, validate email and username uniqueness during registration 39 | * add way to get component with refname and username 40 | * allow getting pipeline info by user and refName 41 | * replace bootstrap-material with simpler bootswatch paper 42 | * add pipeline status socket 43 | * use better URIs for pipeline results 44 | 45 | Fixes and minor tweaks: 46 | * fetch only public or user's pipelines' 47 | * correctly cleanup after compilation using rabbit 48 | * fix pipeline log output 49 | * fix error when querying zero size log 50 | * fix queue handling in services 51 | * fix issue with processors responses and add better rabbitmq cleanup on pipeline teardown 52 | * remove private source from full component info 53 | * fix execution and testing of pipelines with private components 54 | * fix webpack require modules path 55 | * simplify babel config 56 | * update deps, remove unused ones 57 | * correctly fill out pipeline details and users 58 | * only select needed user fields while getting pipelines 59 | * respect private source flag 60 | * only show public and user-owned components in list 61 | 62 | 0.4.4 / 2016-01-07 63 | ================== 64 | 65 | * correctly kill component runners on pipeline stop 66 | 67 | 0.4.3 / 2016-01-07 68 | ================== 69 | 70 | * use runner to compile render components 71 | * support for component versions 72 | * generate kebab-case refnames for components and pipelines 73 | 74 | 0.4.2 / 2016-01-05 75 | ================== 76 | 77 | * change removed await* to await Promise.all 78 | 79 | 0.4.1 / 2015-12-21 80 | ================== 81 | 82 | * allow disabling email validation 83 | 84 | 0.4.0 / 2015-12-21 85 | ================== 86 | 87 | * better tags for logging 88 | * make pipelines in test and prod modes run over rabbit 89 | * fix concurrent component testing 90 | * migrate test component execution to new runner over rabbitmq 91 | * exclude react and react-dom from compiled render components 92 | * adjust readme a bit 93 | 94 | 0.3.0 / 2015-12-16 95 | ================== 96 | 97 | * change from access request to plain registration 98 | * allow specifying hostname for emails using env var 99 | 100 | v0.2.0 / 2015-12-15 101 | =================== 102 | 103 | * fix license field format in package.json & format license as markdown 104 | * update dockerfile and npm scripts to use env vars 105 | * allow using env vars for configuration 106 | * move email config to config.js, remove exynize email creds >_> 107 | * blacklist user fields that are normally not required 108 | * fix sending status on requests with no response 109 | * adjust license text 110 | 111 | v0.1.0 / 2015-12-10 112 | =================== 113 | 114 | * add npm script to start server without db 115 | * first commit for open source version 116 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node 2 | 3 | # Define working directory 4 | WORKDIR /opt/app 5 | 6 | # Install top dependencies w/ caching 7 | COPY package.json /opt/app/package.json 8 | RUN npm install --silent 9 | 10 | # Bundle source 11 | COPY . /opt/app 12 | 13 | # Create volume for compiled UI components 14 | VOLUME /opt/app/src/static 15 | # Expose port 8080 16 | EXPOSE 8080 17 | # Define default command. 18 | CMD ["node", "index.js"] 19 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | Exynize REST, including all of its components and all of its derivates, is available under different licensing options designed to accommodate the needs of various users. 4 | 5 | ## Community license (GPL-3.0) 6 | 7 | Exynize REST licensed under the GNU General Public License v3 (GPL-3.0) is appropriate for the development 8 | of applications based on Exynize platform provided you can comply with the terms and conditions 9 | of the GNU General Public License v3 (GPL-3.0). 10 | 11 | ## Commercial license 12 | 13 | Exynize REST licensed under Commercial license is appropriate for development of proprietary/commercial 14 | software where you do not want to share any source code with third parties or otherwise cannot comply with the terms 15 | of the GPL-3.0. 16 | To obtain the commercial license please contact team@exynize.com 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Exynize REST server for Exynize platform 2 | 3 | > Exynize REST server that provides a way to run user-created components and pipelines in node. Part of Exynize platform. 4 | 5 | ## About Exynize platform 6 | 7 | Exynize platform aims to simplifying the workflow and allow rapid creation of data processing pipelines and visualisations. 8 | Current version of the platform allows: 9 | - constructing pipelines right in your browsers with very little effort, 10 | - writing processing component as if you was dealing with a single data item, 11 | - re-using existing processing modules in new pipelines, 12 | - creating real-time processing and visualisation without thinking about doing real-time at all, 13 | - spending time on doing actual work, not fiddling with scaffolding. 14 | 15 | More info on the platform as well as some demoes of its capabilities can be found in the following article on Medium 16 | > [Building data processing and visualisation pipelines in the browser with Exynize](https://medium.com/the-data-experience/building-data-processing-and-visualisation-pipelines-in-the-browser-with-exynize-372ab15e848c#.cq73g7k7q) 17 | 18 | ## Getting started 19 | 20 | ### Requirements 21 | 22 | For Exynize REST to function properly, you'll need to have following things installed: 23 | 24 | - node.js v4.x or later 25 | - npm v3.x or later 26 | - docker (used to start RethinkDB) or RethinkDB 27 | 28 | Alternatively you can use docker environment provided with a supplied Dockerfile. 29 | 30 | ### Installation 31 | 32 | 1. Clone the repository and cd into new folder: `git clone git@github.com:Exynize/exynize-rest.git && cd exynize-rest` 33 | 2. Execute `npm install` 34 | 3. Execute `npm start` (this will also start a docker container with RethinkDB) 35 | 4. Exynize REST will start working on `http://localhost:8080` 36 | 37 | If you have local RethinkDB instance, you can just use `npm run server` in step 3. 38 | 39 | Alternatively, you can use Dockerfile to assemble docker container and then start it with a link to your RethinkDB instance. 40 | 41 | ## License 42 | 43 | Dual licensed under GPL-3.0 and commercial license. 44 | See LICENSE.md file for more details. 45 | -------------------------------------------------------------------------------- /config.js: -------------------------------------------------------------------------------- 1 | export const auth = { 2 | salt: process.env.EXYNIZE_AUTH_SALT || 'default-auth-salt', 3 | }; 4 | export const db = { 5 | host: process.env.EXYNIZE_DB_HOST || 'docker.dev', 6 | database: process.env.EXYNIZE_DB_NAME || 'exynizedb', 7 | user: '', 8 | password: '', 9 | }; 10 | 11 | export const requireEmailValidation = process.env.EXYNIZE_MAIL_VALIDATION ? 12 | process.env.EXYNIZE_MAIL_VALIDATION === '1' : true; 13 | 14 | export const email = { 15 | host: process.env.EXYNIZE_MAIL_HOST || 'mail.server.com', 16 | port: process.env.EXYNIZE_MAIL_PORT || 465, 17 | secure: process.env.EXYNIZE_MAIL_SECURE ? true : false, 18 | auth: { 19 | user: process.env.EXYNIZE_MAIL_USER || 'user@server.com', 20 | pass: process.env.EXYNIZE_MAIL_PASS || 'password' 21 | }, 22 | }; 23 | 24 | export const jwtconf = { 25 | secret: process.env.EXYNIZE_JWT_SECRET || 'default-jwt-secret', 26 | }; 27 | 28 | export const rabbit = { 29 | host: process.env.RABBITMQ_NODENAME || 'docker.dev', 30 | exchange: 'exynize.components.exchange', 31 | messageExpiration: process.env.RABBITMQ_EXPIRATION || 500, 32 | }; 33 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // override promises with bluebird for extended functionality 2 | global.Promise = require('bluebird'); 3 | // register babel 4 | require('babel-core/register'); 5 | require('babel-polyfill'); 6 | // load app 7 | require('./src'); 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "exynize-rest", 3 | "version": "0.9.0", 4 | "description": "Exynize REST server, part of Exynize platform", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "npm run start-db && npm run start-rabbit && npm run server", 8 | "server": "EXYNIZE_MAIL_VALIDATION=0 nodemon --ignore ./src/static index.js", 9 | "start-db": "docker run -d -p 28015:28015 -p 8080:8080 --name rethinkpipes rethinkdb", 10 | "stop-db": "docker stop rethinkpipes", 11 | "start-rabbit": "docker run -d -p 5672:5672 -p 8081:15672 --name exynize-rabbit rabbitmq:management", 12 | "stop-rabbit": "docker stop exynize-rabbit", 13 | "build-docker": "docker build -t exynize-rest .", 14 | "start-docker": "docker run -it --rm --link rethinkpipes:rdb -e EXYNIZE_DB_HOST=rdb --name exynize-rest exynize-rest" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git@github.com:Exynize/exynize-rest.git" 19 | }, 20 | "author": "Tim Ermilov (http://codezen.net)", 21 | "license": "SEE LICENSE IN LICENSE.md", 22 | "homepage": "https://github.com/Exynize/exynize-rest", 23 | "dependencies": { 24 | "babel-core": "^6.3.15", 25 | "babel-loader": "^6.2.0", 26 | "babel-plugin-typecheck": "^3.4.6", 27 | "babel-polyfill": "^6.3.14", 28 | "babel-preset-es2015": "^6.3.13", 29 | "babel-preset-react": "^6.3.13", 30 | "babel-preset-stage-1": "^6.3.13", 31 | "bluebird": "^3.0.6", 32 | "body-parser": "^1.14.1", 33 | "bootstrap": "^3.3.6", 34 | "bootswatch": "^3.3.6", 35 | "cors": "^2.7.1", 36 | "css-loader": "^0.23.0", 37 | "express": "^4.13.3", 38 | "express-ws": "^1.0.0-rc.2", 39 | "file-loader": "^0.8.5", 40 | "json-loader": "^0.5.4", 41 | "jsonwebtoken": "^5.4.1", 42 | "lodash": "^4.0.0", 43 | "method-override": "^2.3.5", 44 | "microwork": "^0.8.0", 45 | "node-uuid": "^1.4.7", 46 | "nodemailer": "^2.0.0", 47 | "react": "^0.14.3", 48 | "react-dom": "^0.14.3", 49 | "rethinkdb": "^2.2.0", 50 | "rethinkdb-pubsub": "^1.0.1", 51 | "rx": "^4.0.6", 52 | "style-loader": "^0.13.0", 53 | "url-loader": "^0.5.7", 54 | "webpack": "^1.12.6", 55 | "winston": "^2.1.0" 56 | }, 57 | "devDependencies": { 58 | "nodemon": "^1.8.1" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/admin/checkAdmin.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | 3 | // action 4 | export default (req, res, next) => { 5 | logger.debug('validating admin access for:', req.userInfo); 6 | if (req.userInfo && req.userInfo.isAdmin) { 7 | logger.debug('admin access validated for:', req.userInfo); 8 | return next(); 9 | } 10 | 11 | logger.debug('admin access rejected for:', req.userInfo); 12 | return next(new Error('Access not allowed!')); 13 | }; 14 | -------------------------------------------------------------------------------- /src/admin/index.js: -------------------------------------------------------------------------------- 1 | import checkAdmin from './checkAdmin'; 2 | import checkToken from '../users/checkToken'; 3 | import users from './users'; 4 | import updateUser from './updateUser'; 5 | 6 | export default (app) => { 7 | // get users 8 | app 9 | .route('/api/admin/users') 10 | .all(checkToken) 11 | .all(checkAdmin) 12 | .get(users); 13 | 14 | // update user 15 | app 16 | .route('/api/admin/users') 17 | .all(checkToken) 18 | .all(checkAdmin) 19 | .post(updateUser); 20 | }; 21 | -------------------------------------------------------------------------------- /src/admin/updateUser.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {User} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | const {id, isAdmin, isEmailValid} = req.body; 7 | logger.debug('updating user with:', req.body); 8 | // update user 9 | await User.update(id, { 10 | isAdmin, 11 | isEmailValid, 12 | }); 13 | logger.debug('user updated!'); 14 | res.sendStatus(204); 15 | }; 16 | 17 | export default asyncRequest.bind(null, handler); 18 | -------------------------------------------------------------------------------- /src/admin/users.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {User} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | logger.debug('getting all users'); 7 | // find user 8 | const users = await User.findAll({}); 9 | // check if user was found 10 | if (!users) { 11 | res.status(401).json({error: 'Could not get users!'}); 12 | return; 13 | } 14 | logger.debug('got users: ', users); 15 | res.status(200).json({users}); 16 | }; 17 | 18 | export default asyncRequest.bind(null, handler); 19 | -------------------------------------------------------------------------------- /src/client/exynize.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | 4 | // import styles 5 | // styles 6 | import 'bootstrap/dist/css/bootstrap.min.css'; 7 | import 'bootswatch/paper/bootstrap.min.css'; 8 | 9 | const App = React.createClass({ 10 | getInitialState() { 11 | return { 12 | data: [], 13 | }; 14 | }, 15 | 16 | componentWillMount() { 17 | const ws = new WebSocket('ws://' + window.location.host + window.location.pathname); 18 | ws.onopen = () => ws.send(JSON.stringify({token: window.location.search.split('=')[1]})); 19 | ws.onclose = () => console.log('done'); 20 | ws.onmessage = (event) => { 21 | const resp = JSON.parse(event.data); 22 | console.log('message', resp); 23 | if (Array.isArray(resp)) { 24 | this.setState({data: [...resp, ...this.state.data]}); 25 | } else { 26 | this.setState({data: [resp, ...this.state.data]}); 27 | } 28 | }; 29 | }, 30 | 31 | render() { 32 | if (this.props.children) { 33 | return React.cloneElement(this.props.children, this.state); 34 | } 35 | 36 | return
No render set!
; 37 | }, 38 | }); 39 | 40 | window.App = App; 41 | window.React = React; 42 | window.ReactDOM = ReactDOM; 43 | -------------------------------------------------------------------------------- /src/client/index.js: -------------------------------------------------------------------------------- 1 | // start webpack 2 | import webpack from 'webpack'; 3 | import config from './webpack.config.js'; 4 | import logger from '../logger'; 5 | 6 | // create a compiler instance 7 | const compiler = webpack(config); 8 | 9 | compiler.run(function(err) { 10 | if (err) { 11 | logger.error('error compiling webpack:', err); 12 | return; 13 | } 14 | 15 | logger.debug('compiled webpack'); 16 | }); 17 | -------------------------------------------------------------------------------- /src/client/webpack.config.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import path from 'path'; 3 | import baseConfig from '../webpack/webpack.config.base'; 4 | 5 | const cfg = _.merge({}, baseConfig, { 6 | context: path.resolve(__dirname), 7 | entry: path.join(__dirname, 'exynize.js'), 8 | output: { 9 | filename: 'exynize.min.js', 10 | }, 11 | resolve: { 12 | modulesDirectories: [path.join(__dirname, '..', '..', 'node_modules')], 13 | }, 14 | }, (a, b) => { 15 | if (_.isArray(a)) { 16 | return a.concat(b); 17 | } 18 | }); 19 | 20 | export default cfg; 21 | -------------------------------------------------------------------------------- /src/components/create.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import fs from 'fs'; 3 | import {join} from 'path'; 4 | import {asyncRequest} from '../util'; 5 | import logger from '../logger'; 6 | import {Component} from '../db'; 7 | import {compileWithRabbit} from '../runner'; 8 | 9 | const handler = async (req, res, next) => { 10 | const {id, name, description, version, source, params, type, isPublic, isSourcePublic} = req.body; 11 | const refName = _.kebabCase(name); 12 | logger.debug('creating component', 13 | name, refName, description, version, source, params, 14 | type, isPublic, isSourcePublic); 15 | // create data 16 | const componentData = { 17 | name, 18 | refName, 19 | version, 20 | description, 21 | source, 22 | type, 23 | params, 24 | isPublic: Boolean(isPublic), 25 | isSourcePublic: Boolean(isSourcePublic), 26 | }; 27 | let result; 28 | if (id) { 29 | result = await Component.update(id, componentData); 30 | logger.debug('updated component: ', result); 31 | } else { 32 | // create new component 33 | result = await Component.create({ 34 | ...componentData, 35 | user: req.userInfo.id, 36 | }); 37 | logger.debug('created component: ', result); 38 | } 39 | 40 | if (result.inserted === 1 || result.replaced === 1) { 41 | // if render, compile source 42 | if (type === 'render') { 43 | const newId = result.inserted === 1 ? result.generated_keys[0] : id; 44 | const filepath = join(__dirname, '..', 'static', newId + '.min.js'); 45 | const compiled = await compileWithRabbit(newId, source); 46 | logger.debug('compiled webpack'); 47 | fs.writeFileSync(filepath, compiled, 'utf8'); 48 | res.sendStatus(201); 49 | return; 50 | } 51 | 52 | res.sendStatus(201); 53 | return; 54 | } 55 | 56 | res.status(500).json({message: 'error inserting component!'}); 57 | }; 58 | 59 | export default asyncRequest.bind(null, handler); 60 | -------------------------------------------------------------------------------- /src/components/delete.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Component} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | logger.debug('deleting component', req.params.user, req.params.component); 7 | // find component 8 | const component = await Component.getByUserAndRef(req.params.user, req.params.component); 9 | const result = await Component.del(component.id); 10 | if (result.deleted === 1) { 11 | // return 12 | res.sendStatus(204); 13 | } else { 14 | res.status(500).json(`Oops, couldn't delete component! Something went wrong!`); 15 | } 16 | }; 17 | 18 | export default asyncRequest.bind(null, handler); 19 | -------------------------------------------------------------------------------- /src/components/execute.js: -------------------------------------------------------------------------------- 1 | import uuid from 'node-uuid'; 2 | import logger from '../logger'; 3 | import {authedSocket} from '../sockutil'; 4 | import {runWithRabbit} from '../runner'; 5 | 6 | export default (ws) => { 7 | let sub; 8 | 9 | const start = (data) => { 10 | const {source, componentType, args} = data; 11 | const id = uuid.v4(); 12 | logger.debug('executing component source:', source); 13 | logger.debug('executing component type:', componentType); 14 | logger.debug('with args:', args); 15 | logger.debug('with id:', id); 16 | try { 17 | sub = runWithRabbit({source, componentType, args, id, mode: 'test'}) 18 | .subscribe( 19 | execRes => { 20 | logger.debug('exec result:', execRes); 21 | // if socket is not open, log and return 22 | if (ws.readyState !== 1) { 23 | logger.debug('[execute] socket is already closed!'); 24 | return; 25 | } 26 | 27 | ws.send(JSON.stringify(execRes)); 28 | }, 29 | error => { 30 | logger.debug('exec error:', error); 31 | // if socket is not open, log and return 32 | if (ws.readyState !== 1) { 33 | logger.debug('[execute] socket is already closed!'); 34 | return; 35 | } 36 | 37 | ws.send(JSON.stringify({error})); 38 | ws.close(); 39 | }, 40 | () => { 41 | logger.debug('exec done!'); 42 | // if socket is not open, log and return 43 | if (ws.readyState !== 1) { 44 | logger.debug('[execute] socket is already closed!'); 45 | return; 46 | } 47 | 48 | ws.close(); 49 | } 50 | ); 51 | } catch (e) { 52 | logger.debug('subscribe error:', e); 53 | // if socket is not open, log and return 54 | if (ws.readyState !== 1) { 55 | logger.debug('[execute] socket is already closed!'); 56 | return; 57 | } 58 | 59 | if (e.message.indexOf('subscribe is not a function') !== -1) { 60 | ws.send(JSON.stringify({error: 'Your function MUST return an Observable!'})); 61 | } else { 62 | ws.send(JSON.stringify({error: e.message})); 63 | } 64 | ws.close(); 65 | } 66 | }; 67 | 68 | const end = () => { 69 | if (sub) { 70 | sub.dispose(); 71 | } 72 | }; 73 | 74 | authedSocket(ws, {start, end}); 75 | }; 76 | -------------------------------------------------------------------------------- /src/components/get.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Component} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | logger.debug('searching for component', req.params.user, req.params.component); 7 | // find component 8 | const component = await Component.getByUserAndRef(req.params.user, req.params.component); 9 | // delete source if flag is set and user is not owner 10 | if (!component.isSourcePublic && component.user.id !== req.userInfo.id) { 11 | // otherwise - delete source and return 12 | delete component.source; 13 | } 14 | // return 15 | res.status(200).json(component); 16 | }; 17 | 18 | export default asyncRequest.bind(null, handler); 19 | -------------------------------------------------------------------------------- /src/components/getAll.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import logger from '../logger'; 3 | import {Component} from '../db'; 4 | import {asyncRequest} from '../util'; 5 | 6 | const handler = async (req, res) => { 7 | logger.debug('searching for components'); 8 | // find components 9 | const result = await Component.find( 10 | r.row('isPublic').eq(true).or(r.row('user').eq(req.userInfo.id)) 11 | ); 12 | // remove source where needed 13 | const components = result.map(component => { 14 | // always show full info to creator 15 | if (component.user.id === req.userInfo.id) { 16 | return component; 17 | } 18 | // show source if flag is set 19 | if (component.isSourcePublic) { 20 | return component; 21 | } 22 | // otherwise - delete source and return 23 | delete component.source; 24 | return component; 25 | }); 26 | // return 27 | res.status(200).json(components); 28 | }; 29 | 30 | export default asyncRequest.bind(null, handler); 31 | -------------------------------------------------------------------------------- /src/components/index.js: -------------------------------------------------------------------------------- 1 | import checkToken from '../users/checkToken'; 2 | import getComponents from './getAll'; 3 | import getComponent from './get'; 4 | import createComponent from './create'; 5 | import executeComponent from './execute'; 6 | import deleteComponent from './delete'; 7 | 8 | export default (app) => { 9 | // get all public 10 | app 11 | .route('/api/components') 12 | .all(checkToken) 13 | .get(getComponents); 14 | 15 | // get one 16 | app 17 | .route('/api/component/:user/:component') 18 | .all(checkToken) 19 | .get(getComponent) 20 | .delete(deleteComponent); 21 | 22 | // create new 23 | app 24 | .route('/api/components') 25 | .all(checkToken) 26 | .post(createComponent); 27 | 28 | app.ws('/api/components/exec', executeComponent); 29 | }; 30 | -------------------------------------------------------------------------------- /src/db/bus.js: -------------------------------------------------------------------------------- 1 | import {Exchange} from 'rethinkdb-pubsub'; 2 | import {db as dbConfig} from '../../config'; 3 | 4 | export const testExchange = new Exchange('exynize_rest_exchange', { 5 | db: dbConfig.database, 6 | host: dbConfig.host, 7 | port: dbConfig.port, 8 | }); 9 | -------------------------------------------------------------------------------- /src/db/component.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {rdb} from './connection'; 3 | 4 | const userFields = ['id', 'email', 'username']; 5 | 6 | const table = async function() { 7 | const {db, connection} = await rdb(); 8 | const t = db.table('components'); 9 | return {db, t, connection}; 10 | }; 11 | 12 | const find = async function(pattern) { 13 | const {db, t, connection} = await table(); 14 | const cursor = await t.filter(pattern) 15 | .merge(c => ({user: db.table('users').get(c('user')).pluck(userFields)})) 16 | .run(connection); 17 | let result = []; 18 | try { 19 | result = await cursor.toArray(); 20 | } catch (err) { 21 | // check if it's just nothing found error 22 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 23 | logger.debug('error, no users found'); 24 | } else { 25 | throw err; 26 | } 27 | } 28 | connection.close(); 29 | return result; 30 | }; 31 | 32 | const getByUserAndRef = async function(username, refName) { 33 | const {db, t, connection} = await table(); 34 | const cursor = await t.eqJoin('user', db.table('users')) 35 | .filter({left: {refName}, right: {username}}) 36 | .map(row => row('left').merge({user: row('right').pluck(userFields)})) 37 | .limit(1) 38 | .run(connection); 39 | let result = {}; 40 | try { 41 | result = await cursor.next(); 42 | } catch (err) { 43 | // check if it's just nothing found error 44 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 45 | logger.debug('error, no components found'); 46 | } else { 47 | throw err; 48 | } 49 | } 50 | connection.close(); 51 | return result; 52 | }; 53 | 54 | const get = async function(id: string|Object) { 55 | const {db, t, connection} = await table(); 56 | let result = null; 57 | try { 58 | result = await t.get(id) 59 | .merge(c => ({user: db.table('users').get(c('user')).pluck(userFields)})) 60 | .run(connection); 61 | } catch (err) { 62 | // check if it's just nothing found error 63 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 64 | logger.debug('error, no components found'); 65 | } else { 66 | throw err; 67 | } 68 | } 69 | connection.close(); 70 | return result; 71 | }; 72 | 73 | const create = async function(data: Object) { 74 | const {t, connection} = await table(); 75 | const res = t.insert(data).run(connection); 76 | return res; 77 | }; 78 | 79 | const update = async function(pattern: string|Object, data: Object) { 80 | const {t, connection} = await table(); 81 | logger.debug('updating component:', pattern, 'with:', data); 82 | return t.get(pattern).update(data).run(connection); 83 | }; 84 | 85 | const del = async function(id) { 86 | const {t, connection} = await table(); 87 | const result = await t.get(id).delete().run(connection); 88 | connection.close(); 89 | return result; 90 | }; 91 | 92 | export const Component = { 93 | find, 94 | get, 95 | getByUserAndRef, 96 | create, 97 | update, 98 | del, 99 | }; 100 | -------------------------------------------------------------------------------- /src/db/connection.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import logger from '../logger'; 3 | import {db as dbConfig} from '../../config'; 4 | 5 | const setup = async function() { 6 | // connect 7 | const connection = await r.connect({host: dbConfig.host, port: dbConfig.port}); 8 | // create db 9 | try { 10 | await r.dbCreate(dbConfig.database).run(connection); 11 | } catch (e) { 12 | logger.debug('db already exists! done.'); 13 | return; 14 | } 15 | 16 | // create tables and indices 17 | await Promise.all([ 18 | // users 19 | r.db(dbConfig.database).tableCreate('users').run(connection), 20 | // components 21 | r.db(dbConfig.database).tableCreate('components').run(connection), 22 | // pipelines 23 | r.db(dbConfig.database).tableCreate('pipelines').run(connection), 24 | // pipelines execution log 25 | r.db(dbConfig.database).tableCreate('pipelineLog').run(connection), 26 | // pipeline communication bus 27 | r.db(dbConfig.database).tableCreate('exynize_rest_exchange').run(connection), 28 | ]); 29 | }; 30 | 31 | const rdb = async function() { 32 | const connection = await r.connect({host: dbConfig.host, port: dbConfig.port}); 33 | const db = r.db(dbConfig.database); 34 | return {db, connection}; 35 | }; 36 | 37 | export {setup, rdb}; 38 | -------------------------------------------------------------------------------- /src/db/index.js: -------------------------------------------------------------------------------- 1 | import {setup} from './connection'; 2 | 3 | // setup db 4 | export default setup; 5 | 6 | // export 7 | export {User} from './user'; 8 | export {Component} from './component'; 9 | export {Pipeline} from './pipeline'; 10 | export {PipelineLog} from './pipelineLog'; 11 | export {testExchange} from './bus'; 12 | -------------------------------------------------------------------------------- /src/db/pipeline.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {rdb} from './connection'; 3 | 4 | const userFields = ['id', 'email', 'username']; 5 | const deletedUser = {username: 'deleted'}; 6 | const deletedComponent = {user: -1, name: 'component', description: 'This component was deleted'}; 7 | 8 | const table = async function() { 9 | const {db, connection} = await rdb(); 10 | const t = db.table('pipelines'); 11 | return {db, t, connection}; 12 | }; 13 | 14 | const mergeWithComponents = (db, pipe) => ({ 15 | source: pipe('source').merge( 16 | comp => db.table('components') 17 | .get(comp('id')) 18 | .default(deletedComponent) 19 | .merge(c => ({user: db.table('users').get(c('user')).default(deletedUser).pluck(userFields)})) 20 | ), 21 | components: pipe('components').merge( 22 | comp => db.table('components') 23 | .get(comp('id')) 24 | .default(deletedComponent) 25 | .merge(c => ({user: db.table('users').get(c('user')).default(deletedUser).pluck(userFields)})) 26 | ), 27 | render: pipe('render').merge( 28 | comp => db.table('components') 29 | .get(comp('id')) 30 | .default(deletedComponent) 31 | .merge(c => ({user: db.table('users').get(c('user')).default(deletedUser).pluck(userFields)})) 32 | ), 33 | }); 34 | 35 | const find = async function(pattern) { 36 | const {db, t, connection} = await table(); 37 | const cursor = await t.filter(pattern) 38 | .merge(pipe => ({user: db.table('users').get(pipe('user')).pluck(userFields)})) 39 | .merge(pipe => mergeWithComponents(db, pipe)) 40 | .run(connection); 41 | let result = []; 42 | try { 43 | result = await cursor.toArray(); 44 | } catch (err) { 45 | // check if it's just nothing found error 46 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 47 | logger.debug('error, no pipelines found'); 48 | } else { 49 | throw err; 50 | } 51 | } 52 | connection.close(); 53 | return result; 54 | }; 55 | 56 | const getByUserAndRef = async function(username, refName) { 57 | const {db, t, connection} = await table(); 58 | const cursor = await t.eqJoin('user', db.table('users')) 59 | .filter({left: {refName}, right: {username}}) 60 | .map(row => row('left').merge({user: row('right').pluck(userFields)})) 61 | .merge(pipe => mergeWithComponents(db, pipe)) 62 | .limit(1) 63 | .run(connection); 64 | let result = {}; 65 | try { 66 | result = await cursor.next(); 67 | } catch (err) { 68 | // check if it's just nothing found error 69 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 70 | logger.debug('error, no components found'); 71 | } else { 72 | throw err; 73 | } 74 | } 75 | connection.close(); 76 | return result; 77 | }; 78 | 79 | const get = async function(id: string|Object) { 80 | const {db, t, connection} = await table(); 81 | let result = null; 82 | try { 83 | result = await t.get(id) 84 | .merge(pipe => ({user: db.table('users').get(pipe('user')).pluck(userFields)})) 85 | .merge(pipe => mergeWithComponents(db, pipe)) 86 | .run(connection); 87 | } catch (err) { 88 | // check if it's just nothing found error 89 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 90 | logger.debug('error, no pipelines found'); 91 | } else { 92 | throw err; 93 | } 94 | } 95 | connection.close(); 96 | return result; 97 | }; 98 | 99 | const create = async function(data: Object) { 100 | const {t, connection} = await table(); 101 | const result = await t.insert(data).run(connection); 102 | connection.close(); 103 | return result; 104 | }; 105 | 106 | const update = async function(pattern: string|Object, data: Object) { 107 | const {t, connection} = await table(); 108 | logger.debug('updating pipeline:', pattern, 'with:', data); 109 | const result = await t.get(pattern).update(data).run(connection); 110 | connection.close(); 111 | return result; 112 | }; 113 | 114 | const changes = async function(id: string) { 115 | const {t, connection} = await table(); 116 | logger.debug('subscribing to pipeline status:', id); 117 | return t.get(id).changes().run(connection); 118 | }; 119 | 120 | const del = async function(id) { 121 | const {t, connection} = await table(); 122 | const result = await t.get(id).delete().run(connection); 123 | connection.close(); 124 | return result; 125 | }; 126 | 127 | export const Pipeline = { 128 | find, 129 | get, 130 | getByUserAndRef, 131 | create, 132 | update, 133 | changes, 134 | del, 135 | }; 136 | -------------------------------------------------------------------------------- /src/db/pipelineLog.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import logger from '../logger'; 3 | import {rdb} from './connection'; 4 | 5 | const table = async function() { 6 | const {db, connection} = await rdb(); 7 | const t = db.table('pipelineLog'); 8 | return {db, t, connection}; 9 | }; 10 | 11 | const find = async function(pattern) { 12 | const {t, connection} = await table(); 13 | const cursor = await t.filter(pattern).group('sessionId').run(connection); 14 | let result = []; 15 | try { 16 | result = await cursor.toArray(); 17 | } catch (err) { 18 | // check if it's just nothing found error 19 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 20 | logger.debug('error, no pipeline log found'); 21 | } else { 22 | throw err; 23 | } 24 | } 25 | connection.close(); 26 | return result; 27 | }; 28 | 29 | const latest = async function(pattern) { 30 | const {t, connection} = await table(); 31 | const cursor = await t.filter(pattern) 32 | .group('sessionId').ungroup().map(it => it('reduction')) 33 | .orderBy(r.desc(it => it('added_on'))) 34 | .nth(0).orderBy(r.desc('added_on')) 35 | .default([]) 36 | .run(connection); 37 | let result = null; 38 | try { 39 | result = await cursor.toArray(); 40 | } catch (err) { 41 | // check if it's just nothing found error 42 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 43 | logger.debug('error, no pipeline log found'); 44 | } else { 45 | throw err; 46 | } 47 | } 48 | connection.close(); 49 | return result; 50 | }; 51 | 52 | const create = async function(data: Object) { 53 | const {t, connection} = await table(); 54 | logger.debug('inserting pipeline log:', data); 55 | const res = await t.insert(data).run(connection); 56 | connection.close(); 57 | return res; 58 | }; 59 | 60 | 61 | export const PipelineLog = { 62 | find, 63 | latest, 64 | create, 65 | }; 66 | -------------------------------------------------------------------------------- /src/db/user.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {rdb} from './connection'; 3 | 4 | const withoutFields = ['password', 'passwordReset', 'verifyId']; 5 | 6 | const table = async function() { 7 | const {db, connection} = await rdb(); 8 | const t = db.table('users'); 9 | return {t, connection}; 10 | }; 11 | 12 | const find = async function(pattern) { 13 | const {t, connection} = await table(); 14 | const cursor = await t.filter(pattern) 15 | .without(withoutFields) 16 | .limit(1) 17 | .run(connection); 18 | let result; 19 | try { 20 | result = await cursor.next(); 21 | } catch (err) { 22 | // check if it's just nothing found error 23 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 24 | logger.debug('error, no users found'); 25 | } else { 26 | throw err; 27 | } 28 | } 29 | connection.close(); 30 | return result; 31 | }; 32 | 33 | const findAll = async function(pattern) { 34 | const {t, connection} = await table(); 35 | const cursor = await t.filter(pattern) 36 | .without(withoutFields) 37 | .run(connection); 38 | let result; 39 | try { 40 | result = await cursor.toArray(); 41 | } catch (err) { 42 | // check if it's just nothing found error 43 | if (err.name === 'ReqlDriverError' && err.message === 'No more rows in the cursor.') { 44 | logger.debug('error, no users found'); 45 | } else { 46 | throw err; 47 | } 48 | } 49 | connection.close(); 50 | return result; 51 | }; 52 | 53 | const create = async function(data: Object) { 54 | const {t, connection} = await table(); 55 | // check for email 56 | let existingUserCount = await t.filter({email: data.email}).count().run(connection); 57 | if (existingUserCount > 0) { 58 | throw new Error('User with given email already exists!'); 59 | } 60 | // check for username 61 | existingUserCount = await t.filter({username: data.username}).count().run(connection); 62 | if (existingUserCount > 0) { 63 | throw new Error('User with given username already exists!'); 64 | } 65 | const res = await t.insert(data).run(connection); 66 | const id = res.generated_keys[0]; 67 | const result = await find({id}); 68 | connection.close(); 69 | return result; 70 | }; 71 | 72 | const update = async function(pattern: Object|string, data: Object|Function) { 73 | const {t, connection} = await table(); 74 | const res = await t.get(pattern).update(data).run(connection); 75 | connection.close(); 76 | return res; 77 | }; 78 | 79 | export const User = { 80 | find, 81 | findAll, 82 | create, 83 | update, 84 | }; 85 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import './server'; 2 | -------------------------------------------------------------------------------- /src/logger/index.js: -------------------------------------------------------------------------------- 1 | import winston from 'winston'; 2 | 3 | export const consoleTransport = new winston.transports.Console({ 4 | level: process.env.NODE_ENV === 'production' ? 'info' : 'debug', 5 | prettyPrint: process.env.NODE_ENV !== 'production', 6 | colorize: process.env.NODE_ENV !== 'production', 7 | timestamp: process.env.NODE_ENV !== 'production', 8 | label: 'rest-api', 9 | }); 10 | 11 | const logger = new winston.Logger({ 12 | transports: [consoleTransport], 13 | }); 14 | 15 | export default logger; 16 | -------------------------------------------------------------------------------- /src/pipes/create.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import logger from '../logger'; 3 | import {Pipeline} from '../db'; 4 | import {asyncRequest} from '../util'; 5 | 6 | const handler = async (req, res) => { 7 | const {id, name, source, components, render, isPublic} = req.body; 8 | const refName = _.kebabCase(name); 9 | logger.debug('creating pipleine', name, refName, source, components, render, isPublic, 'with id:', id); 10 | // create new component 11 | let result; 12 | if (id) { 13 | result = await Pipeline.update(id, { 14 | name, 15 | refName, 16 | source, 17 | components, 18 | render, 19 | isPublic: Boolean(isPublic), 20 | }); 21 | logger.debug('updated pipeline: ', result); 22 | } else { 23 | result = await Pipeline.create({ 24 | name, 25 | refName, 26 | source, 27 | components, 28 | render, 29 | status: 'off', 30 | message: '', 31 | isPublic: Boolean(isPublic), 32 | user: req.userInfo.id, 33 | }); 34 | logger.debug('created pipeline: ', result); 35 | } 36 | 37 | if (result.inserted === 1 || result.replaced === 1) { 38 | res.sendStatus(201); 39 | return; 40 | } 41 | 42 | res.status(500).json({error: 'error inserting pipeline!'}); 43 | }; 44 | 45 | export default asyncRequest.bind(null, handler); 46 | -------------------------------------------------------------------------------- /src/pipes/delete.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Pipeline} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | logger.debug('deleting pipeline', req.params.user, req.params.pipeline); 7 | // only allow deleting if owner 8 | if (req.params.user !== req.userInfo.username) { 9 | return res.status(403).json(`You don't have permission to do this!`); 10 | } 11 | // find pipeline 12 | const pipeline = await Pipeline.getByUserAndRef(req.params.user, req.params.pipeline); 13 | const result = await Pipeline.del(pipeline.id); 14 | if (result.deleted === 1) { 15 | res.sendStatus(204); 16 | } else { 17 | res.status(500).json(`Oops, couldn't delete pipeline! Something went wrong!`); 18 | } 19 | }; 20 | 21 | export default asyncRequest.bind(null, handler); 22 | -------------------------------------------------------------------------------- /src/pipes/execute.js: -------------------------------------------------------------------------------- 1 | import {runPipeline} from './runner/runPipeline'; 2 | import {authedSocket} from '../sockutil'; 3 | import logger from '../logger'; 4 | 5 | export default (ws) => { 6 | let cleanRunners; 7 | 8 | const start = async (data: Object) => { 9 | const {pipeline: pipelineJSON} = data; 10 | const pipeline = JSON.parse(pipelineJSON); 11 | logger.debug('[execute] executing pipe:', JSON.stringify(pipeline, null, 4)); 12 | 13 | // get source 14 | const {stream, clean} = await runPipeline(pipeline); 15 | cleanRunners = clean; 16 | // subscribe 17 | stream.subscribe( 18 | resp => { 19 | // logger.debug('[execute] response:', resp); 20 | // if socket is not open, log and return 21 | if (ws.readyState !== 1) { 22 | logger.debug('[execute] socket is already closed!'); 23 | return; 24 | } 25 | 26 | ws.send(JSON.stringify(resp)); 27 | }, 28 | error => { 29 | logger.error('[execute] error:', error); 30 | // if socket is not open, log and return 31 | if (ws.readyState !== 1) { 32 | logger.debug('[execute] socket is already closed!'); 33 | return; 34 | } 35 | 36 | ws.send(JSON.stringify({error})); 37 | ws.close(); 38 | }, 39 | () => { 40 | logger.debug('[execute] done!'); 41 | // if socket is not open, log and return 42 | if (ws.readyState !== 1) { 43 | logger.debug('[execute] socket is already closed!'); 44 | return; 45 | } 46 | 47 | ws.close(); 48 | } 49 | ); 50 | }; 51 | 52 | const end = () => { 53 | cleanRunners(); 54 | }; 55 | 56 | authedSocket(ws, {start, end}); 57 | }; 58 | -------------------------------------------------------------------------------- /src/pipes/get.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Pipeline} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | logger.debug('searching for pipeline', req.params.user, req.params.pipeline); 7 | // only allow getting owned pipelines 8 | if (req.params.user !== req.userInfo.username) { 9 | return res.status(403).json(`You don't have permission to do that!`); 10 | } 11 | // find pipeline 12 | const pipeline = await Pipeline.getByUserAndRef(req.params.user, req.params.pipeline); 13 | // return 14 | res.status(200).json(pipeline); 15 | }; 16 | 17 | export default asyncRequest.bind(null, handler); 18 | -------------------------------------------------------------------------------- /src/pipes/getAll.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import _ from 'lodash'; 3 | import logger from '../logger'; 4 | import {Pipeline} from '../db'; 5 | import {asyncRequest} from '../util'; 6 | 7 | const toPrivate = (c) => _.omit( 8 | c.isPublic ? c : { 9 | name: 'Private component', 10 | description: 'This component is private.', 11 | user: {username: 'private'}, 12 | }, 13 | ['args', 'params'] 14 | ); 15 | 16 | const handler = async (req, res) => { 17 | logger.debug('searching for pipelines'); 18 | // find pipelines 19 | const pipelines = await Pipeline.find( 20 | r.row('isPublic').eq(true).or(r.row('user').eq(req.userInfo.id)) 21 | ); 22 | // filter out private components 23 | const filteredPipelines = pipelines.map(p => { 24 | const {components, render, source} = p; 25 | return { 26 | ...p, 27 | components: components.map(toPrivate), 28 | render: toPrivate(render), 29 | source: toPrivate(source), 30 | }; 31 | }); 32 | // return 33 | res.status(200).json(filteredPipelines); 34 | }; 35 | 36 | export default asyncRequest.bind(null, handler); 37 | -------------------------------------------------------------------------------- /src/pipes/index.js: -------------------------------------------------------------------------------- 1 | import checkToken from '../users/checkToken'; 2 | // pipelines listing 3 | import getPipes from './getAll'; 4 | // pipeline management 5 | import getPipe from './get'; 6 | import deletePipe from './delete'; 7 | import createPipe from './create'; 8 | import executePipe from './execute'; 9 | import startPipe from './start'; 10 | import stopPipe from './stop'; 11 | // pipeline results / status 12 | import getPipeLog from './log'; 13 | import pipeStatusSocket from './statusSocket'; 14 | import getPipeSocket from './resultSocket'; 15 | import getPipeResult from './result'; 16 | // inputs 17 | import pipelineInput from './input'; 18 | import pipelineInputSocket from './inputSocket'; 19 | 20 | export default (app) => { 21 | // get all public 22 | app 23 | .route('/api/pipes') 24 | .all(checkToken) 25 | .get(getPipes); 26 | 27 | // create new 28 | app 29 | .route('/api/pipes') 30 | .all(checkToken) 31 | .post(createPipe); 32 | 33 | // start 34 | app 35 | .route('/api/pipes/:id/start') 36 | .all(checkToken) 37 | .post(startPipe); 38 | 39 | // stop 40 | app 41 | .route('/api/pipes/:id/stop') 42 | .all(checkToken) 43 | .post(stopPipe); 44 | 45 | // execution log 46 | app 47 | .route('/api/pipes/:id/log') 48 | .all(checkToken) 49 | .get(getPipeLog); 50 | 51 | // get/delete pipeline 52 | app 53 | .route('/api/pipes/:user/:pipeline') 54 | .all(checkToken) 55 | .get(getPipe) 56 | .delete(deletePipe); 57 | 58 | // latest execution result 59 | app 60 | .route('/api/pipes/:user/:pipeline/result') 61 | .get(getPipeResult); 62 | 63 | // pipeline inputs: 64 | // pipeline REST input 65 | app 66 | .route('/api/pipes/:user/:pipeline/input') 67 | .get(pipelineInput) 68 | .post(pipelineInput) 69 | .put(pipelineInput); 70 | // pipeline socket input 71 | app.ws('/api/pipes/:user/:pipeline/input', pipelineInputSocket); 72 | 73 | // execute pipe 74 | app.ws('/api/pipes/exec', executePipe); 75 | // execution socket 76 | app.ws('/api/pipes/:user/:pipeline/result', getPipeSocket); 77 | // status socket 78 | app.ws('/api/pipes/:id/status', pipeStatusSocket); 79 | }; 80 | -------------------------------------------------------------------------------- /src/pipes/input.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import logger from '../logger'; 3 | import {Pipeline} from '../db'; 4 | import {asyncRequest} from '../util'; 5 | import service from '../runner/service'; 6 | import {rabbit} from '../../config'; 7 | 8 | const handler = async (req, res) => { 9 | logger.debug('searching for pipeline', req.params.user, req.params.pipeline); 10 | // find pipeline 11 | const pipeline = await Pipeline.getByUserAndRef(req.params.user, req.params.pipeline); 12 | if (!pipeline) { 13 | return res.sendStatus(404); 14 | } 15 | // send 16 | const data = { 17 | type: 'incoming-http-request', 18 | ..._.pick(req, ['body', 'cookies', 'method', 'query']) 19 | }; 20 | const request = {id: pipeline.instance.sourceId, data}; 21 | await service.send('runner.command', request, {expiration: rabbit.messageExpiration}); 22 | // return 23 | res.sendStatus(202); 24 | }; 25 | 26 | export default asyncRequest.bind(null, handler); 27 | -------------------------------------------------------------------------------- /src/pipes/inputSocket.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Pipeline} from '../db'; 3 | import service from '../runner/service'; 4 | import {rabbit} from '../../config'; 5 | 6 | export default async (ws, req) => { 7 | const {user, pipeline: pipelineName} = req.params; 8 | // find pipeline 9 | logger.debug('starting socket with id:', user, pipelineName); 10 | // get pipeline 11 | const pipeline = await Pipeline.getByUserAndRef(user, pipelineName); 12 | if (!pipeline) { 13 | // if socket is not open, log and return 14 | if (ws.readyState !== 1) { 15 | logger.debug('socket is already closed!'); 16 | return; 17 | } 18 | 19 | ws.send({error: 'pipeline not found'}); 20 | ws.close(); 21 | return; 22 | } 23 | logger.debug('found pipeline', pipeline.id); 24 | // wait for messages 25 | ws.on('message', async (msg) => { 26 | const incData = JSON.parse(msg); 27 | // send 28 | const data = { 29 | type: 'incoming-http-request', 30 | method: 'ws', 31 | ...incData, 32 | }; 33 | const request = {id: pipeline.instance.sourceId, data}; 34 | await service.send('runner.command', request, {expiration: rabbit.messageExpiration}); 35 | }); 36 | }; 37 | -------------------------------------------------------------------------------- /src/pipes/log.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {PipelineLog} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | const {id} = req.params; 7 | logger.debug('getting log for', id); 8 | const pipelineLog = await PipelineLog.find({pipeline: id}); 9 | // say we're good 10 | res.status(200).json(pipelineLog); 11 | }; 12 | 13 | export default asyncRequest.bind(null, handler); 14 | -------------------------------------------------------------------------------- /src/pipes/result.js: -------------------------------------------------------------------------------- 1 | // start webpack 2 | import logger from '../logger'; 3 | import {Pipeline, PipelineLog} from '../db'; 4 | import {asyncRequest} from '../util'; 5 | import {requestToToken, checkStringToken} from '../users/checkToken'; 6 | 7 | const publicOrAuth = async (req, res, user, pipeline) => { 8 | // check public status / auth 9 | if (!pipeline.isPublic) { 10 | const token = requestToToken(req); 11 | try { 12 | const authUser = await checkStringToken(token); 13 | logger.info('user found!', authUser); 14 | if (authUser.username !== user) { 15 | throw new Error(`User mismatch!`); 16 | } 17 | } catch (e) { 18 | throw new Error(`You don't have permission to do that!`); 19 | } 20 | } 21 | }; 22 | 23 | const json = async (req, res) => { 24 | const {user, pipeline: pipelineName} = req.params; 25 | logger.debug('getting json log for', user, pipelineName); 26 | // get log 27 | const pipeline = await Pipeline.getByUserAndRef(user, pipelineName); 28 | // check public status / auth 29 | await publicOrAuth(req, res, user, pipeline); 30 | // get data 31 | const pipelineLog = await PipelineLog.latest({pipeline: pipeline.id}); 32 | // say we're good 33 | res.status(200).json(pipelineLog); 34 | }; 35 | 36 | const html = async (req, res) => { 37 | const {user, pipeline: pipelineName} = req.params; 38 | logger.debug('getting html result for', user, pipelineName); 39 | const pipeline = await Pipeline.getByUserAndRef(user, pipelineName); 40 | // check public status / auth 41 | await publicOrAuth(req, res, user, pipeline); 42 | // get render id 43 | const renderId = pipeline.render.id; 44 | const staticPrefix = process.env.NODE_ENV === 'production' ? '/api' : ''; 45 | logger.debug('got compiled render:', renderId); 46 | res.send(` 47 | 48 | 49 | 50 | 51 | Exynize Pipeline: ${user}/${pipelineName} 52 | 53 | 54 |
55 | 56 | 57 | 65 | 66 | 67 | `); 68 | }; 69 | 70 | export default (req, res) => { 71 | res.format({ 72 | json() { 73 | asyncRequest(json, req, res); 74 | }, 75 | 76 | html() { 77 | asyncRequest(html, req, res); 78 | }, 79 | 80 | default() { 81 | // log the request and respond with 406 82 | res.status(406).send('Not Acceptable'); 83 | } 84 | }); 85 | }; 86 | -------------------------------------------------------------------------------- /src/pipes/resultSocket.js: -------------------------------------------------------------------------------- 1 | import {testExchange, Pipeline, PipelineLog} from '../db'; 2 | import {checkStringToken} from '../users/checkToken'; 3 | import logger from '../logger'; 4 | 5 | export default (ws, req) => { 6 | const {user, pipeline: pipelineName} = req.params; 7 | logger.debug('getting socket for', user, pipelineName); 8 | 9 | // listen for message 10 | ws.on('message', async (msg) => { 11 | const data = JSON.parse(msg); 12 | 13 | logger.debug('starting socket with id:', user, pipelineName); 14 | // get pipeline 15 | const pipeline = await Pipeline.getByUserAndRef(user, pipelineName); 16 | logger.debug('found pipeline', pipeline.id); 17 | // check for auth if needed 18 | if (!pipeline.isPublic) { 19 | // check auth 20 | try { 21 | const authUser = await checkStringToken(data.token); 22 | if (!authUser || authUser.username !== user) { 23 | throw new Error('Not authorized'); 24 | } 25 | } catch (e) { 26 | // if socket is not open, log and return 27 | if (ws.readyState !== 1) { 28 | logger.debug('socket is already closed!'); 29 | return; 30 | } 31 | 32 | ws.send(JSON.stringify({error: e.message})); 33 | ws.close(); 34 | return; 35 | } 36 | } 37 | // if pipeline is not running, just send latest results from DB 38 | const pipelineLog = await PipelineLog.latest({pipeline: pipeline.id}); 39 | const res = pipelineLog.map(it => it.data); 40 | // if socket is not open, log and return 41 | if (ws.readyState !== 1) { 42 | logger.debug('socket is already closed!'); 43 | return; 44 | } 45 | 46 | ws.send(JSON.stringify(res)); 47 | // if pipeline is not running, just die 48 | if (pipeline.status !== 'running') { 49 | ws.close(); 50 | return; 51 | } 52 | 53 | // otherwise subscribe to current socket result 54 | testExchange 55 | .queue(topic => topic.eq(pipeline.id + '.out')) 56 | .subscribe((topic, payload) => { 57 | logger.debug('[SOCKET-RESPONSE] for topic:', topic, 'got payload:', payload); 58 | // if socket is not open 59 | if (ws.readyState !== 1) { 60 | logger.debug('socket is already closed!'); 61 | return; 62 | } 63 | 64 | // if done - close socket 65 | if (payload.done) { 66 | ws.close(); 67 | return; 68 | } 69 | 70 | ws.send(JSON.stringify(payload.data)); 71 | }); 72 | }); 73 | }; 74 | -------------------------------------------------------------------------------- /src/pipes/runner/index.js: -------------------------------------------------------------------------------- 1 | // override promises with bluebird for extended functionality 2 | global.Promise = require('bluebird'); 3 | // register babel 4 | require('babel-core/register'); 5 | require('babel-polyfill'); 6 | // load app 7 | require('./runner'); 8 | -------------------------------------------------------------------------------- /src/pipes/runner/runPipeline.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import uuid from 'node-uuid'; 3 | import {Component, Pipeline} from '../../db'; 4 | import logger from '../../logger'; 5 | import {runWithRabbit, serviceWithRabbit, stopWithRabbit} from '../../runner'; 6 | 7 | export const runPipeline = async (pipeline, saveInstanceIDs = false) => { 8 | // logger.debug('running pipeline:', pipeline); 9 | const toKill = []; 10 | // get source 11 | const {source} = pipeline; 12 | const sourceId = uuid.v4(); 13 | logger.debug('starting source with id:', sourceId); 14 | // get sources for componnets without source 15 | logger.debug('getting sources for components...'); 16 | const componentsWithSource = pipeline.components.filter(c => c.source); 17 | const componentsWithoutSource = pipeline.components.filter(c => !c.source); 18 | for (const comp of componentsWithoutSource) { 19 | logger.debug('component does not have source:', comp); 20 | const c = await Component.get(comp.id); 21 | const res = { 22 | ...comp, 23 | ...c, 24 | }; 25 | componentsWithSource.push(res); 26 | } 27 | logger.debug('all comps:'); 28 | logger.debug(JSON.stringify(componentsWithSource)); 29 | // init source 30 | const srcRx = runWithRabbit({ 31 | source: source.source, 32 | componentType: 'source', 33 | args: source.args, 34 | id: sourceId, 35 | }) 36 | // remove id from kill list when it's done 37 | .doOnCompleted(() => { 38 | logger.debug('source done, removing from kill list', toKill); 39 | _.remove(toKill, it => it === sourceId); 40 | logger.debug('removed source from kill list', toKill); 41 | }); 42 | // say we need to kill it 43 | toKill.push(sourceId); 44 | // init components with ids 45 | const components = componentsWithSource //pipeline.components 46 | // map with new metadata 47 | .map(comp => { 48 | // assign id and type 49 | comp.id = uuid.v4(); 50 | comp.componentType = 'processor'; 51 | // say we need to kill it 52 | toKill.push(comp.id); 53 | return comp; 54 | }) 55 | // map to services 56 | .map(comp => serviceWithRabbit(comp)); 57 | 58 | // if needed, save instance IDs to db 59 | if (saveInstanceIDs && pipeline.id) { 60 | logger.debug('saving instance ids:', pipeline.id); 61 | await Pipeline.update(pipeline.id, { 62 | instance: { 63 | sourceId, 64 | componentsIds: toKill.filter(id => id !== sourceId), 65 | } 66 | }); 67 | } 68 | 69 | // reduce to stream and return 70 | const stream = components.reduce((s, fn) => s.flatMap(fn), srcRx); 71 | 72 | // return 73 | return { 74 | stream, 75 | clean() { 76 | logger.debug('got clean command:', toKill); 77 | return toKill.map(id => stopWithRabbit(id)); 78 | }, 79 | }; 80 | }; 81 | -------------------------------------------------------------------------------- /src/pipes/runner/runner.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import uuid from 'node-uuid'; 3 | import logger from '../../logger'; 4 | import setupDb, {testExchange, PipelineLog, Pipeline} from '../../db'; 5 | import {runPipeline} from './runPipeline'; 6 | import service from '../../runner/service'; 7 | 8 | // get args 9 | const [,, id, pipelineJSON] = process.argv; 10 | const pipeline = JSON.parse(pipelineJSON); 11 | logger.debug('running child with id', id); 12 | 13 | // generate sessionId 14 | const sessionId = uuid.v4(); 15 | 16 | // promises to do before exit 17 | const promises = []; 18 | 19 | // setup db, then do work 20 | setupDb().then(async () => { 21 | // get reply topic 22 | const replyTopic = testExchange.topic(id + '.out'); 23 | 24 | // delayed exit command 25 | const delayedExit = ({ 26 | status = 'done', 27 | message = 'success', 28 | } = {}) => { 29 | // do delayed notification that we're done 30 | setTimeout(() => { 31 | promises.push(replyTopic.publish({ 32 | data: [], 33 | done: true 34 | })); 35 | // if executed in production - update status to done 36 | promises.push(Pipeline.update(id, {status, message})); 37 | // shut down microwork service 38 | promises.push(service.stop()); 39 | // do delayed close to say we're done 40 | Promise.all(promises).then(() => process.exit()); 41 | }, 500); 42 | }; 43 | 44 | // if executed in production - set pipeline status to 'running' 45 | promises.push(Pipeline.update(id, { 46 | status: 'running', 47 | message: '' 48 | })); 49 | 50 | // start pipeline 51 | logger.debug('executing pipeline..'); 52 | const {stream, clean} = await runPipeline(pipeline, true); 53 | 54 | // listen for commands 55 | testExchange 56 | .queue(topic => topic.eq(id + '.in')) 57 | .subscribe((topic, payload) => { 58 | logger.debug('[IN] for topic:', topic, 'got payload:', payload); 59 | if (payload.command === 'kill') { 60 | clean().forEach(p => promises.push(p)); 61 | delayedExit(); 62 | } 63 | }); 64 | 65 | // subscribe to results 66 | stream.subscribe( 67 | data => { 68 | logger.debug('[OUT] seding pipeline response:', data, 'to topic:', id); 69 | promises.push(replyTopic.publish({ 70 | data, done: false 71 | })); 72 | // if executed in production - push to persistent db log 73 | promises.push(PipelineLog.create({ 74 | pipeline: id, 75 | sessionId, 76 | data, 77 | added_on: r.now(), // eslint-disable-line 78 | })); 79 | }, 80 | e => { 81 | logger.error('[OUT] error in pipline:', e); 82 | // if executed in production - push error to persistent db log 83 | promises.push(PipelineLog.create({ 84 | pipeline: id, 85 | sessionId, 86 | data: { 87 | type: 'error', 88 | error: e, 89 | }, 90 | added_on: r.now(), // eslint-disable-line 91 | })); 92 | // schedule exit 93 | logger.debug('[OUT] scheduling exit...'); 94 | clean().forEach(p => promises.push(p)); 95 | delayedExit({ 96 | status: 'error', 97 | message: e.message || JSON.stringify(e) 98 | }); 99 | }, () => { 100 | logger.debug('[OUT] pipeline done, scheduling exit'); 101 | clean().forEach(p => promises.push(p)); 102 | delayedExit(); 103 | } 104 | ); 105 | }); 106 | -------------------------------------------------------------------------------- /src/pipes/start.js: -------------------------------------------------------------------------------- 1 | import {join} from 'path'; 2 | import {fork} from 'child_process'; 3 | import logger from '../logger'; 4 | import {Pipeline} from '../db'; 5 | import {asyncRequest} from '../util'; 6 | 7 | const handler = async (req, res) => { 8 | const {id} = req.params; 9 | const pipelines = await Pipeline.find({id}); 10 | logger.debug('found pipelines for start: ', JSON.stringify(pipelines)); 11 | if (pipelines.length !== 1) { 12 | logger.debug('Found more than one pipeline, wtf?!'); 13 | return res.status(500).json({message: 'Found more than one pipeline, wtf?!'}); 14 | } 15 | 16 | const pipeline = pipelines[0]; 17 | logger.debug('starting pipleine', JSON.stringify(pipeline)); 18 | // only allow starting if owner 19 | if (pipeline.user.username !== req.userInfo.username) { 20 | return res.status(403).json(`You don't have permission to do this!`); 21 | } 22 | 23 | // die if it's already running 24 | if (pipeline.status === 'running') { 25 | logger.debug('pipleine already running, dying', pipeline.id); 26 | // say we're good 27 | return res.status(500).json({error: 'Pipeline already running!'}); 28 | } 29 | // fork child 30 | fork(join(__dirname, 'runner', 'index.js'), [id, JSON.stringify(pipeline)]); 31 | // say we're good 32 | res.sendStatus(204); 33 | }; 34 | 35 | export default asyncRequest.bind(null, handler); 36 | -------------------------------------------------------------------------------- /src/pipes/statusSocket.js: -------------------------------------------------------------------------------- 1 | import {Pipeline} from '../db'; 2 | import {authedSocket} from '../sockutil'; 3 | import logger from '../logger'; 4 | 5 | export default (ws, req) => { 6 | const {id} = req.params; 7 | let cursor; 8 | logger.debug('getting status socket for', id); 9 | 10 | const start = async () => { 11 | logger.debug('starting status socket with id:', id); 12 | // subscribe to pipeline updates 13 | try { 14 | cursor = await Pipeline.changes(id); 15 | } catch (e) { 16 | // if socket is not open, log and return 17 | if (ws.readyState !== 1) { 18 | logger.debug('socket is already closed!'); 19 | return; 20 | } 21 | 22 | ws.close(); 23 | return; 24 | } 25 | logger.debug('found pipeline status feed!'); 26 | 27 | cursor.each((err, message) => { 28 | if (err) { 29 | ws.close(); 30 | return; 31 | } 32 | 33 | // if socket is not open, log and return 34 | if (ws.readyState !== 1) { 35 | logger.debug('socket is already closed!'); 36 | return; 37 | } 38 | 39 | const res = message.new_val; 40 | logger.debug('pipeline update: ', res); 41 | ws.send(JSON.stringify(res)); 42 | }); 43 | }; 44 | 45 | // close cursor once user disconnects 46 | const end = () => { 47 | if (cursor) { 48 | cursor.close(); 49 | } 50 | }; 51 | 52 | authedSocket(ws, {start, end}); 53 | }; 54 | -------------------------------------------------------------------------------- /src/pipes/stop.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {Pipeline, testExchange} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | const {id} = req.params; 7 | const pipelines = await Pipeline.find({id}); 8 | logger.debug('found pipelines for stop: ', JSON.stringify(pipelines)); 9 | if (pipelines.length !== 1) { 10 | logger.debug('Found more than one pipeline, wtf?!'); 11 | res.status(500).json({message: 'Found more than one pipeline, wtf?!'}); 12 | return; 13 | } 14 | 15 | const pipeline = pipelines[0]; 16 | logger.debug('stopping pipleine', JSON.stringify(pipeline)); 17 | // only allow starting if owner 18 | if (pipeline.user.username !== req.userInfo.username) { 19 | return res.status(403).json(`You don't have permission to do this!`); 20 | } 21 | // get topic 22 | const childTopic = testExchange.topic(pipeline.id + '.in'); 23 | await childTopic.publish({command: 'kill'}); 24 | // say we're good 25 | res.sendStatus(204); 26 | }; 27 | 28 | export default asyncRequest.bind(null, handler); 29 | -------------------------------------------------------------------------------- /src/runner/compileWithRabbit.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import service from './service'; 3 | import {rabbit} from '../../config'; 4 | 5 | export const compileWithRabbit = (id, source) => new Promise((resolve, reject) => { 6 | logger.debug('compiling'); 7 | const run = async () => { 8 | let cleanup = () => {}; 9 | logger.debug('[cwr]: run'); 10 | const topic = 'runner.compileResult.' + id; 11 | const tag = await service.subscribe(topic, (msg) => { 12 | logger.debug('[cwr]: got message.'); 13 | // reject if error 14 | if (msg.error) { 15 | logger.error('[cwr]: error, ', msg.error); 16 | cleanup(); 17 | return reject(msg.error); 18 | } 19 | // return data 20 | logger.debug('[cwr]: done...'); 21 | cleanup(); 22 | return resolve(msg.data); 23 | }, {exclusive: true}); 24 | // send 25 | logger.debug('[cwr]: sending:', id, source); 26 | service.send('runner.compile', {id, source}, {expiration: rabbit.messageExpiration}); 27 | // define cleanup 28 | cleanup = async () => { 29 | logger.debug('[cwr]: cleanup...'); 30 | await service.unsubscribe(topic, tag); 31 | }; 32 | }; 33 | // trigger 34 | run(); 35 | }); 36 | -------------------------------------------------------------------------------- /src/runner/index.js: -------------------------------------------------------------------------------- 1 | export {compileWithRabbit} from './compileWithRabbit'; 2 | export {runWithRabbit} from './runWithRabbit'; 3 | export {serviceWithRabbit} from './serviceWithRabbit'; 4 | export {stopWithRabbit} from './stopWithRabbit'; 5 | -------------------------------------------------------------------------------- /src/runner/runWithRabbit.js: -------------------------------------------------------------------------------- 1 | import Rx from 'rx'; 2 | import logger from '../logger'; 3 | import service from './service'; 4 | import {rabbit} from '../../config'; 5 | 6 | export const runWithRabbit = (data) => Rx.Observable.create(obs => { 7 | let cachedConsumerTag; 8 | 9 | const topic = 'runner.result.' + data.id; 10 | 11 | const returnByType = { 12 | result: obs.onNext.bind(obs), 13 | error: obs.onError.bind(obs), 14 | done: obs.onCompleted.bind(obs), 15 | }; 16 | 17 | const run = async () => { 18 | logger.debug('[rwr]: run'); 19 | cachedConsumerTag = await service.subscribe(topic, (msg) => { 20 | // return depending on type 21 | returnByType[msg.type](msg.data); 22 | }, {exclusive: true}); 23 | service.send('runner.execute', data, {expiration: rabbit.messageExpiration}); 24 | }; 25 | 26 | // run and catch error 27 | run().catch(e => { 28 | logger.error('[rwr]: ERROR ', e); 29 | obs.onError(e); 30 | }); 31 | // cleanup 32 | return async () => { 33 | logger.debug('[rwr]: cleanup'); 34 | await service.send('runner.kill', {id: data.id}, {expiration: rabbit.messageExpiration}); 35 | await service.unsubscribe(topic, cachedConsumerTag); 36 | }; 37 | }); 38 | -------------------------------------------------------------------------------- /src/runner/service.js: -------------------------------------------------------------------------------- 1 | import Microwork from 'microwork'; 2 | import {rabbit} from '../../config'; 3 | import {consoleTransport} from '../logger'; 4 | 5 | const service = new Microwork({host: rabbit.host, exchange: rabbit.exchange, loggingTransports: [consoleTransport]}); 6 | 7 | export default service; 8 | -------------------------------------------------------------------------------- /src/runner/serviceWithRabbit.js: -------------------------------------------------------------------------------- 1 | import uuid from 'node-uuid'; 2 | import {Observable} from 'rx'; 3 | import logger from '../logger'; 4 | import service from './service'; 5 | import {rabbit} from '../../config'; 6 | 7 | export const serviceWithRabbit = (cfg) => { 8 | // run processor 9 | const run = async () => { 10 | // send 11 | await service.send('runner.execute', cfg, {expiration: rabbit.messageExpiration}); 12 | logger.debug('[svc]: sent execute to runner'); 13 | }; 14 | 15 | // run and catch error 16 | run().catch(e => logger.error('[svc]: ', e)); 17 | 18 | // return new handler 19 | return (data) => Observable.create(obs => { 20 | let cachedConsumerTag; 21 | // generate unique ID for current transaction 22 | const id = uuid.v4(); 23 | const topic = 'runner.result.' + id; 24 | // return by type mapping 25 | const returnByType = { 26 | result: obs.onNext.bind(obs), 27 | error: obs.onError.bind(obs), 28 | done: obs.onCompleted.bind(obs), 29 | }; 30 | const runCommand = async () => { 31 | cachedConsumerTag = await service.subscribe(topic, (msg) => { 32 | // log 33 | logger.debug('[svc]: got message:', msg.type, 'for:', id); 34 | // return depending on type 35 | returnByType[msg.type](msg.data); 36 | }); 37 | // send 38 | const request = {id: cfg.id, responseId: id, data}; 39 | await service.send('runner.command', request, {expiration: rabbit.messageExpiration}); 40 | // logger.debug('[svc]: sent', data, 'to', id); 41 | }; 42 | // run command 43 | runCommand().catch(e => { 44 | logger.error('[svc]: ERROR ', e); 45 | obs.onError(e); 46 | }); 47 | // return cleanup 48 | return async () => { 49 | logger.debug('[svc]: cleanup'); 50 | await service.unsubscribe(topic, cachedConsumerTag); 51 | }; 52 | }); 53 | }; 54 | -------------------------------------------------------------------------------- /src/runner/stopWithRabbit.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import service from './service'; 3 | import {rabbit} from '../../config'; 4 | 5 | export const stopWithRabbit = (id) => { 6 | logger.debug('stopping', id); 7 | return service.send('runner.kill', {id}, {expiration: rabbit.messageExpiration}); 8 | }; 9 | -------------------------------------------------------------------------------- /src/server.js: -------------------------------------------------------------------------------- 1 | import {join} from 'path'; 2 | import express from 'express'; 3 | import expressWs from 'express-ws'; 4 | import bodyParser from 'body-parser'; 5 | import methodOverride from 'method-override'; 6 | import cors from 'cors'; 7 | import logger from './logger'; 8 | import setupUsers from './users'; 9 | import setupAdmin from './admin'; 10 | import setupComponents from './components'; 11 | import setupPipes from './pipes'; 12 | import setupDb from './db'; 13 | // compile client code 14 | import './client'; 15 | 16 | // init app 17 | const app = express(); 18 | // parse request bodies (req.body) 19 | app.use(bodyParser.json()); 20 | app.use(bodyParser.urlencoded({extended: true})); 21 | // support _method (PUT in forms etc) 22 | app.use(methodOverride()); 23 | // enable CORS headers 24 | app.use(cors()); 25 | // enable CORS pre-flights 26 | app.options('*', cors()); 27 | // enable websockets on routes 28 | expressWs(app); 29 | // error handling inside of express 30 | app.use((err, req, res, next) => { // eslint-disable-line 31 | logger.error(err.stack); 32 | res.status(500).send('Something broke!'); 33 | }); 34 | // static files 35 | const staticPrefix = process.env.NODE_ENV === 'production' ? '/api' : ''; 36 | app.use(staticPrefix + '/static', express.static(join(__dirname, 'static'))); 37 | // output all uncaught exceptions 38 | process.on('uncaughtException', err => logger.error('uncaught exception:', err)); 39 | process.on('unhandledRejection', error => logger.error('unhandled rejection:', error)); 40 | 41 | 42 | // setup api 43 | // users & auth 44 | setupUsers(app); 45 | // admin 46 | setupAdmin(app); 47 | // components 48 | setupComponents(app); 49 | // pipes 50 | setupPipes(app); 51 | 52 | // wait for DB setup 53 | setupDb().then(() => { 54 | // start server 55 | const server = app.listen(8080, () => { 56 | const host = server.address().address; 57 | const port = server.address().port; 58 | logger.info(`Exynize platform REST API listening at http://${host}:${port}`); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /src/sockutil/authedSocket.js: -------------------------------------------------------------------------------- 1 | import {checkStringToken} from '../users/checkToken'; 2 | 3 | export const authedSocket = (ws: Object, {start, end} = {}) => { 4 | ws.on('message', async (msg) => { 5 | const data = JSON.parse(msg); 6 | 7 | // check if it's end request 8 | if (data.end) { 9 | end(); 10 | ws.close(); 11 | return; 12 | } 13 | 14 | // check auth 15 | try { 16 | const user = await checkStringToken(data.token); 17 | if (!user) { 18 | throw new Error('Not authorized'); 19 | } 20 | } catch (e) { 21 | // if socket is not open, return 22 | if (ws.readyState !== 1) { 23 | return; 24 | } 25 | 26 | ws.send(JSON.stringify({error: e.message})); 27 | ws.close(); 28 | return; 29 | } 30 | 31 | start(data); 32 | }); 33 | }; 34 | -------------------------------------------------------------------------------- /src/sockutil/index.js: -------------------------------------------------------------------------------- 1 | export {authedSocket} from './authedSocket'; 2 | -------------------------------------------------------------------------------- /src/users/checkToken.js: -------------------------------------------------------------------------------- 1 | import jwt from 'jsonwebtoken'; 2 | import logger from '../logger'; 3 | import {User} from '../db'; 4 | import {jwtconf} from '../../config'; 5 | 6 | // token from request 7 | export const requestToToken = (req) => req.body.token || req.query.token || req.headers['x-access-token']; 8 | 9 | export const checkStringToken = async (token: string) : Object => { 10 | logger.debug('checking token: ', token); 11 | if (!token) { 12 | logger.debug('no broken'); 13 | throw new Error('No auth token provided!'); 14 | } 15 | 16 | let decoded; 17 | try { 18 | // FIXME ignoreExpiration 19 | decoded = jwt.verify(token, jwtconf.secret, {ignoreExpiration: process.env.NODE_ENV !== 'production'}); 20 | } catch (e) { 21 | logger.error('Error decoding token', e); 22 | throw e; 23 | } 24 | logger.debug('decoded: ', decoded); 25 | const {email, id} = decoded; 26 | logger.debug('searching for: ', email, id); 27 | // find user 28 | const user = await User.find({email, id}); 29 | if (user) { 30 | logger.info('user found!', user); 31 | return user; 32 | } 33 | 34 | throw new Error('Not logged in!'); 35 | }; 36 | 37 | // action 38 | export default async (req, res, next) => { 39 | const token = requestToToken(req); 40 | try { 41 | const user = await checkStringToken(token); 42 | logger.info('user found!', user); 43 | req.userInfo = user; 44 | return next(); 45 | } catch (e) { 46 | return next(e); 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /src/users/hash.js: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | import {auth} from '../../config'; 3 | 4 | export default (string: string) : string => { 5 | const shasum = crypto.createHash('sha1'); 6 | shasum.update(string); 7 | shasum.update(auth.salt); 8 | const d = shasum.digest('hex'); 9 | return d; 10 | }; 11 | -------------------------------------------------------------------------------- /src/users/index.js: -------------------------------------------------------------------------------- 1 | import login from './login'; 2 | import register from './register'; 3 | import verify from './verify'; 4 | import resetPassword from './resetPassword'; 5 | import resetPasswordChange from './resetPasswordChange'; 6 | import resetPasswordAccept from './resetPasswordAccept'; 7 | 8 | export default (app) => { 9 | app.route('/api/login').post(login); 10 | app.route('/api/verify/:id').get(verify); 11 | app.route('/api/password/reset').post(resetPassword); 12 | app.route('/api/password/reset/:id').get(resetPasswordChange); 13 | app.route('/api/password/reset/:id').post(resetPasswordAccept); 14 | app.route('/api/register').post(register); 15 | }; 16 | -------------------------------------------------------------------------------- /src/users/login.js: -------------------------------------------------------------------------------- 1 | import jwt from 'jsonwebtoken'; 2 | import hash from './hash'; 3 | import logger from '../logger'; 4 | import {User} from '../db'; 5 | import {jwtconf} from '../../config'; 6 | import {asyncRequest} from '../util'; 7 | 8 | const handler = async (req, res) => { 9 | const {email, password} = req.body; 10 | const hashedPassword = hash(password); 11 | logger.debug('searching for: ', email, hashedPassword); 12 | // find user 13 | const user = await User.find({ 14 | email, 15 | password: hashedPassword, 16 | isEmailValid: true, 17 | }); 18 | // check if user was found 19 | if (!user) { 20 | res.status(401).json({error: 'Incorrect email or password!'}); 21 | return; 22 | } 23 | logger.debug('got user: ', user); 24 | // generate token 25 | const token = jwt.sign(user, jwtconf.secret, {expiresIn: '1d'}); 26 | res.status(200).json({token}); 27 | }; 28 | 29 | export default asyncRequest.bind(null, handler); 30 | -------------------------------------------------------------------------------- /src/users/register.js: -------------------------------------------------------------------------------- 1 | import uuid from 'node-uuid'; 2 | import hash from './hash'; 3 | import sendEmail from './sendEmail'; 4 | import logger from '../logger'; 5 | import {User} from '../db'; 6 | import {asyncRequest} from '../util'; 7 | import {requireEmailValidation} from '../../config'; 8 | 9 | const handler = async (req, res) => { 10 | const host = process.env.EXYNIZE_HOST || req.get('host'); 11 | const {email, username, password} = req.body; 12 | const hashedPassword = hash(password); 13 | const verifyId = uuid.v4(); 14 | logger.debug('adding: ', email, hashedPassword); 15 | // find user 16 | const user = await User.create({ 17 | email, 18 | username, 19 | password: hashedPassword, 20 | verifyId, 21 | isEmailValid: !requireEmailValidation, 22 | }); 23 | 24 | if (!user) { 25 | logger.debug('unknown error while creating user during registration!'); 26 | res.status(500).json({error: 'Error while creating user!'}); 27 | return; 28 | } 29 | 30 | if (requireEmailValidation) { 31 | // send email 32 | const verifyLink = `http://${host}/api/verify/${verifyId}`; 33 | const text = `Hi ${username}, 34 | Please Click on the link to verify your email: ${verifyLink}`; 35 | const html = `Hi ${username},
36 | Please Click on the link to verify your email.
37 | Click here to verify
38 | Or open this in a browser: ${verifyLink}.`; 39 | 40 | // send email 41 | await sendEmail({ 42 | to: email, 43 | subject: 'Exynize: Confirm Your Email', 44 | text, 45 | html, 46 | }); 47 | } 48 | 49 | logger.debug('created user: ', user); 50 | res.sendStatus(201); 51 | }; 52 | 53 | export default asyncRequest.bind(null, handler); 54 | -------------------------------------------------------------------------------- /src/users/resetPassword.js: -------------------------------------------------------------------------------- 1 | import r from 'rethinkdb'; 2 | import uuid from 'node-uuid'; 3 | import logger from '../logger'; 4 | import {User} from '../db'; 5 | import sendEmail from './sendEmail'; 6 | import {asyncRequest} from '../util'; 7 | 8 | const handler = async (req, res) => { 9 | const host = process.env.EXYNIZE_HOST || req.get('host'); 10 | const {email} = req.body; 11 | const resetId = uuid.v4(); 12 | logger.debug('reset pass for: ', email, 'with resetId:', resetId); 13 | // find user 14 | const user = await User.find({email}); 15 | // send email 16 | if (user) { 17 | logger.debug('user found:', user, ', sending reset password email'); 18 | // save token to db 19 | await User.update(user.id, { 20 | passwordReset: { 21 | token: resetId, 22 | date: r.now(), 23 | }, 24 | }); 25 | // generate email 26 | const resetLink = `http://${host}/api/password/reset/${resetId}`; 27 | const text = `Hi there, 28 | Please Click on the link to reset your password: ${resetId}`; 29 | const html = `Hi there,
30 | Please Click on the link to reset your password.
31 | Click here to reset password
32 | Or open this in a browser: ${resetLink}.`; 33 | 34 | // send email 35 | await sendEmail({ 36 | to: email, 37 | subject: 'Exynize: Password Reset', 38 | text, 39 | html, 40 | }); 41 | } 42 | 43 | logger.debug('password reset for user: ', user); 44 | res.sendStatus(204); 45 | }; 46 | 47 | export default asyncRequest.bind(null, handler); 48 | -------------------------------------------------------------------------------- /src/users/resetPasswordAccept.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {User} from '../db'; 3 | import hash from './hash'; 4 | import {asyncRequest} from '../util'; 5 | 6 | const handler = async (req, res) => { 7 | const {id: resetId} = req.params; 8 | const {password, passwordRepeat} = req.body; 9 | logger.debug('reset pass accept for: ', resetId, password, passwordRepeat); 10 | // redirect back if passwords not equal 11 | if (password !== passwordRepeat) { 12 | const error = encodeURIComponent('Passwords must match!'); 13 | res.redirect('/api/password/reset/' + resetId + '?error=' + error); 14 | return; 15 | } 16 | 17 | // find user 18 | const user = await User.find({ 19 | passwordReset: { 20 | token: resetId 21 | } 22 | }); 23 | 24 | // send email 25 | if (!user) { 26 | logger.debug('user found:', user, ', sending reset password email'); 27 | res.status(500).send(`Error! Password reset request not found!`); 28 | return; 29 | } 30 | 31 | // save to db 32 | const hashedPassword = hash(password); 33 | logger.debug('saving new password for: ', user.email, user.id, hashedPassword); 34 | await User.update(user.id, { 35 | password: hashedPassword, 36 | passwordReset: {token: '-1', date: 0}, 37 | }); 38 | 39 | logger.debug('password reset for user: ', user); 40 | res.status(200).send(` 41 | 42 | 43 | Password was reset! You can login now. 44 | 45 | 46 | `); 47 | }; 48 | 49 | export default asyncRequest.bind(null, handler); 50 | -------------------------------------------------------------------------------- /src/users/resetPasswordChange.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {User} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | const {id: resetId} = req.params; 7 | if (resetId === '-1') { 8 | res.status(500).send(`Error! Password reset request not found!`); 9 | return; 10 | } 11 | const errorMessage = req.query.error || ''; 12 | logger.debug('reset pass for: ', resetId); 13 | // find user 14 | const user = await User.find({passwordReset: {token: resetId}}); 15 | // check for user and time validity 16 | const now = new Date().getTime() - 60 * 60 * 1000; // 60 mins expiration 17 | if (!user || user.passwordReset.date.getTime() < now) { 18 | logger.debug('error during password reset with user or date:', user); 19 | if (user) { 20 | await User.update(user.id, {passwordReset: {token: '-1', date: 0}}); 21 | } 22 | res.status(500).send(`Error! Password reset request not found!`); 23 | return; 24 | } 25 | 26 | logger.debug('password reset for user: ', user); 27 | res.status(200).send(` 28 | 29 | 30 | ${errorMessage} 31 |
32 | 33 | 34 | 35 |
36 | 37 | 38 | `); 39 | }; 40 | 41 | export default asyncRequest.bind(null, handler); 42 | -------------------------------------------------------------------------------- /src/users/sendEmail.js: -------------------------------------------------------------------------------- 1 | import nodemailer from 'nodemailer'; 2 | import logger from '../logger'; 3 | import {email} from '../../config'; 4 | 5 | // create transporter 6 | const transporter = nodemailer.createTransport(email); 7 | 8 | // export send function 9 | export default ({to, subject, text, html}) => new Promise((resolve, reject) => { 10 | transporter.sendMail({ 11 | from: 'Exynize Bot ', // sender address 12 | to, // list of receivers 13 | subject, // Subject line 14 | text, // plaintext body 15 | html // html body 16 | }, (error, info) => { 17 | if (error) { 18 | logger.error('error sending email:', error); 19 | return reject(error); 20 | } 21 | 22 | logger.debug('Message sent:', info); 23 | resolve(info); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/users/verify.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | import {User} from '../db'; 3 | import {asyncRequest} from '../util'; 4 | 5 | const handler = async (req, res) => { 6 | const {id: verifyId} = req.params; 7 | logger.debug('verifying email for', verifyId); 8 | if (verifyId === '0') { 9 | res.status(401).send('Incorrect verification token!'); 10 | return; 11 | } 12 | // find user 13 | const user = await User.find({ 14 | verifyId, 15 | isEmailValid: false, 16 | }); 17 | // check if user was found 18 | if (!user) { 19 | res.status(401).send('Incorrect verification token!'); 20 | return; 21 | } 22 | logger.debug('got user: ', user); 23 | await User.update(user.id, {isEmailValid: true, verifyId: '0'}); 24 | res.status(200).send(`Your email was successfully activated! You can login now.`); 25 | }; 26 | 27 | export default asyncRequest.bind(null, handler); 28 | -------------------------------------------------------------------------------- /src/util/asyncRequest.js: -------------------------------------------------------------------------------- 1 | import logger from '../logger'; 2 | 3 | /** 4 | * Wraps express.js async handler function with catch to correctly handle errors 5 | * @param {Function} asyncFn async handler function for express 6 | * @param {Object} req express request object 7 | * @param {Object} res express response object 8 | * @return {void} 9 | */ 10 | export const asyncRequest = (asyncFn, req, res) => 11 | asyncFn(req, res) 12 | .catch(e => { 13 | logger.error(e); 14 | res.status(500).json({message: e.message}); 15 | }); 16 | -------------------------------------------------------------------------------- /src/util/index.js: -------------------------------------------------------------------------------- 1 | export {asyncRequest} from './asyncRequest'; 2 | -------------------------------------------------------------------------------- /src/webpack/webpack.config.base.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | 3 | export default { 4 | devtool: process.env.NODE_ENV === 'production' ? undefined : '#inline-source-map', 5 | debug: process.env.NODE_ENV === 'production' ? false : true, 6 | output: { 7 | path: path.resolve(__dirname, '..', 'static'), 8 | publicPath: '/static/', 9 | }, 10 | resolve: { 11 | root: path.resolve(__dirname), 12 | extensions: ['', '.js', '.jsx'], 13 | modulesDirectories: [path.join(__dirname, '..', '..', 'node_modules')], 14 | }, 15 | module: { 16 | loaders: [ 17 | { 18 | test: /\.css$/, 19 | loaders: ['style', 'css'], 20 | }, 21 | { 22 | test: /\.jsx?$/, 23 | exclude: /node_modules/, 24 | loader: 'babel', 25 | query: { 26 | presets: ['react', 'es2015', 'stage-1'] 27 | } 28 | }, 29 | { 30 | test: /\.json$/, 31 | loader: 'json', 32 | }, 33 | { 34 | test: /\.woff\d?(\?.+)?$/, 35 | loader: 'url?limit=10000&minetype=application/font-woff', 36 | }, 37 | { 38 | test: /\.ttf(\?.+)?$/, 39 | loader: 'url?limit=10000&minetype=application/octet-stream', 40 | }, 41 | { 42 | test: /\.eot(\?.+)?$/, 43 | loader: 'url?limit=10000', 44 | }, 45 | { 46 | test: /\.svg(\?.+)?$/, 47 | loader: 'url?limit=10000&minetype=image/svg+xml', 48 | }, 49 | { 50 | test: /\.png$/, 51 | loader: 'url?limit=10000&mimetype=image/png', 52 | }, 53 | { 54 | test: /\.gif$/, 55 | loader: 'url?limit=10000&mimetype=image/gif' 56 | } 57 | ], 58 | } 59 | }; 60 | --------------------------------------------------------------------------------