├── .gitignore ├── .npmignore ├── index.js ├── web ├── logo │ ├── favicon.ico │ ├── dashflow-nav.png │ ├── dashflow-icon-16x16.png │ ├── dashflow-icon-32x32.png │ └── dashflow-icon-96x96.png ├── lib │ ├── Inconsolata-Regular.woff2 │ ├── react.production.min.js │ └── js-yaml.min.js ├── components │ ├── errors.js │ ├── dashboard.js │ ├── panel.js │ ├── pure.js │ ├── area.js │ ├── index.js │ └── header.js ├── dashboards │ ├── banner.js │ ├── index.js │ ├── system.js │ ├── log.js │ └── gauge.js ├── demo.html ├── index.html ├── production.js ├── utils.js ├── app.js ├── styles.css └── demo.js ├── guide_assets ├── architecture.png ├── web_dashboard.gif ├── interactive_shell.png └── dashflow-header.min.png ├── .eslintrc ├── test ├── events-test.js └── formatter-test.js ├── examples ├── interactive-streams.yml └── web-dev.dashflow.yml ├── lib ├── util.js ├── command.js ├── formatter.js ├── daemon.js ├── stream.js ├── server.js ├── events.js ├── cli.js ├── workflow.js ├── shell.js ├── config.js └── interactive_shell.js ├── package.json ├── bin └── dashflow ├── CONTRIBUTOR.md ├── pipeline-pull-request.yml ├── CHANGELOG.md ├── dashflow.yml ├── pipeline-master.yml ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | example/testlog 3 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | example/ 2 | test/ 3 | guide_assets/ 4 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { start } = require("./lib/cli"); 2 | 3 | module.exports = { start }; 4 | -------------------------------------------------------------------------------- /web/logo/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/logo/favicon.ico -------------------------------------------------------------------------------- /web/logo/dashflow-nav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/logo/dashflow-nav.png -------------------------------------------------------------------------------- /guide_assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/guide_assets/architecture.png -------------------------------------------------------------------------------- /guide_assets/web_dashboard.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/guide_assets/web_dashboard.gif -------------------------------------------------------------------------------- /web/logo/dashflow-icon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/logo/dashflow-icon-16x16.png -------------------------------------------------------------------------------- /web/logo/dashflow-icon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/logo/dashflow-icon-32x32.png -------------------------------------------------------------------------------- /web/logo/dashflow-icon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/logo/dashflow-icon-96x96.png -------------------------------------------------------------------------------- /guide_assets/interactive_shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/guide_assets/interactive_shell.png -------------------------------------------------------------------------------- /web/lib/Inconsolata-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/web/lib/Inconsolata-Regular.woff2 -------------------------------------------------------------------------------- /guide_assets/dashflow-header.min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewheel/dashflow/HEAD/guide_assets/dashflow-header.min.png -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "eslint:recommended", 3 | "parserOptions": { 4 | "ecmaVersion": 2017, 5 | "sourceType": "module" 6 | }, 7 | "env": { 8 | es6: true, 9 | node: true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/events-test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env node, mocha */ 2 | const { equal } = require("assert"); 3 | const events = require("../lib/events"); 4 | 5 | describe("events", () => { 6 | it("init", () => { 7 | const e = events.init(); 8 | 9 | equal(e.length, 0); 10 | equal(e.head, null); 11 | equal(e.tail, null); 12 | 13 | equal(e.capacity, 1000000); 14 | equal(Object.keys(e.subscribers).length, 0); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /web/components/errors.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | export const Errors = ({ messages, onClear }) => { 4 | if (messages.length > 0) { 5 | return React.createElement( 6 | "div", 7 | { className: "title error" }, 8 | [ 9 | React.createElement("button", { 10 | key: "clear", 11 | className: "btn btn-clear float-right", 12 | onClick: onClear, 13 | }), 14 | ].concat( 15 | messages.map(message => 16 | React.createElement("p", { key: message }, message) 17 | ) 18 | ) 19 | ); 20 | } else { 21 | return React.createElement("noscript"); 22 | } 23 | }; -------------------------------------------------------------------------------- /web/dashboards/banner.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | import { Components } from '../components/index.js'; 4 | import { Utils } from '../utils.js'; 5 | 6 | export function banner(positionSpec, content) { 7 | return Components.pure(function getBanner() { 8 | return React.createElement(Components.Panel, { 9 | style: Utils.positionSpecToStyle(positionSpec), 10 | content: React.createElement( 11 | "div", 12 | { 13 | style: { 14 | fontSize: "1.2em", 15 | padding: ".2em", 16 | fontWeight: 500, 17 | }, 18 | }, 19 | content 20 | ), 21 | }); 22 | }); 23 | } -------------------------------------------------------------------------------- /examples/interactive-streams.yml: -------------------------------------------------------------------------------- 1 | streams: 2 | irb: irb 3 | iex: iex 4 | python: 5 | shell: { cmd: python -i } 6 | vim: 7 | shell: { cmd: vim } 8 | 9 | dashboards: 10 | all-in-one: 11 | - log: 12 | position: quadrant/top-left 13 | title: VIM 14 | filter: stream:vim:.* 15 | - log: 16 | position: quadrant/top-right 17 | title: Python 18 | filter: stream:python:.* 19 | - log: 20 | position: quadrant/bottom-left 21 | title: IRB 22 | filter: stream:irb:.* 23 | - log: 24 | position: quadrant/bottom-right 25 | title: IEX 26 | filter: stream:iex:.* 27 | -------------------------------------------------------------------------------- /web/components/dashboard.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | export const Dashboard = ({ 4 | events, 5 | globalFilterValue, 6 | globalFilterValid, 7 | pannels, 8 | dashboardsYAML, 9 | updateDashboardsYAML, 10 | }) => 11 | React.createElement( 12 | "div", 13 | { className: "dashboard-container" }, 14 | React.createElement( 15 | "div", 16 | { className: "dashboard-wrapper" }, 17 | pannels.map(renderer => 18 | React.createElement(renderer, { 19 | events, 20 | globalFilterValue, 21 | globalFilterValid, 22 | dashboardsYAML, 23 | updateDashboardsYAML, 24 | }) 25 | ) 26 | ) 27 | ); -------------------------------------------------------------------------------- /web/components/panel.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | export const Panel = ({ title, pill, content, style, className }) => { 4 | const { top, left, width, height } = style; 5 | 6 | return React.createElement( 7 | "div", 8 | { 9 | className: "dash-panel", 10 | style: { top, left, width, height }, 11 | }, 12 | [ 13 | title && 14 | React.createElement( 15 | "div", 16 | { 17 | className: `${className} title`, 18 | }, 19 | [ 20 | title, 21 | pill && React.createElement( 22 | "span", 23 | { 24 | className: 'pill' 25 | }, 26 | pill 27 | ) 28 | ] 29 | ), 30 | React.createElement( 31 | "div", 32 | { 33 | className: "content", 34 | }, 35 | content 36 | ), 37 | ].filter(Boolean) 38 | ); 39 | }; -------------------------------------------------------------------------------- /web/components/pure.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | import { Header } from './header.js'; 4 | 5 | export const pure = (BaseComponent, keys = undefined) => { 6 | return class Decorated extends React.Component { 7 | shouldComponentUpdate(nextProps) { 8 | let changed = false; 9 | 10 | if (keys) { 11 | for (let p of keys) { 12 | if (this.props[p] !== nextProps[p]) { 13 | changed = true; 14 | } 15 | } 16 | } else { 17 | for (let p in this.props) { 18 | if (this.props[p] !== nextProps[p]) { 19 | changed = true; 20 | } 21 | } 22 | } 23 | 24 | return changed; 25 | } 26 | 27 | render() { 28 | return React.createElement( 29 | BaseComponent, 30 | this.props, 31 | this.props.children 32 | ); 33 | } 34 | }; 35 | }; 36 | 37 | export const PureHeader = pure(Header, [ 38 | "dashboards", 39 | "currentDashboardTitle", 40 | "globalFilterValue", 41 | "globalFilterValid", 42 | ]); -------------------------------------------------------------------------------- /web/demo.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 17 | 23 | 24 | Dashflow 25 | 26 | 27 | 28 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /web/dashboards/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | 3 | import { log } from './log.js'; 4 | import { gauge } from './gauge.js'; 5 | import { banner } from './banner.js'; 6 | import { SystemDashboard } from './system.js'; 7 | 8 | 9 | function parseDashboardFunction(title, spec) { 10 | const types = Object.keys(spec); 11 | 12 | const type = types[0]; 13 | const s = spec[type]; 14 | 15 | if (type === "log") { 16 | return log(s.title, s.position, s.filter, s.gauge, s.limit || 500); 17 | } else if (type === "gauge") { 18 | return gauge(s.title, s.position, s.filter, s.scan); 19 | } else if (type === "banner") { 20 | return banner(s.position, s.content); 21 | } else { 22 | return null; 23 | } 24 | } 25 | 26 | function parseDashboards(config) { 27 | const dashboardNames = Object.keys(config); 28 | 29 | return dashboardNames.map(dashboard => ({ 30 | title: dashboard.toUpperCase(), 31 | pannels: config[dashboard] 32 | .map(config => parseDashboardFunction(dashboard, config)) 33 | .filter(Boolean), 34 | })); 35 | } 36 | 37 | export const Dashboards = { 38 | SystemDashboard, 39 | parseDashboards, 40 | }; 41 | -------------------------------------------------------------------------------- /web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 17 | 23 | 24 | Dashflow 25 | 26 | 27 | 28 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /web/production.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | /* global io */ 3 | 4 | import { createStore, render } from "./app.js"; 5 | 6 | (function connect() { 7 | function connectToSocket(store) { 8 | const socket = io(); 9 | 10 | function init() { 11 | socket.emit("get_dashboards_config"); 12 | 13 | socket.emit("catchup"); 14 | } 15 | 16 | socket.on("get_dashboards_config", function(dashboards) { 17 | try { 18 | store.clearErrorMessages(); 19 | store.updateDashboards(dashboards); 20 | store.updateCurrentDashboardTitle(null); 21 | } catch (err) { 22 | store.addErrorMessage(err.message); 23 | } 24 | }); 25 | 26 | socket.on("catchup_batch", function(messages) { 27 | store.appendEvents(messages); 28 | }); 29 | 30 | socket.on("catchup_done", function() { 31 | socket.emit("delta"); 32 | }); 33 | 34 | socket.on("delta", function(message) { 35 | store.appendEvent(message); 36 | }); 37 | 38 | socket.on("reconnect", function() { 39 | store.resetEvents(); 40 | init(); 41 | }); 42 | 43 | init(); 44 | 45 | return socket; 46 | } 47 | 48 | const appStore = createStore(); 49 | connectToSocket(appStore); 50 | render(appStore); 51 | })(); 52 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | const BRIEF_LINE_LIMIT = 200; 2 | 3 | function brief(event) { 4 | const eventWithoutLineBreak = event.replace(/\n/g, "\\n"); 5 | 6 | if (eventWithoutLineBreak.length > BRIEF_LINE_LIMIT) { 7 | return eventWithoutLineBreak.substring(0, BRIEF_LINE_LIMIT) + "..."; 8 | } else { 9 | return eventWithoutLineBreak; 10 | } 11 | } 12 | 13 | function partition(event) { 14 | if (event === undefined) { 15 | return [undefined, undefined]; 16 | } 17 | 18 | const posOfFirstColon = event.indexOf(":"); 19 | 20 | if (posOfFirstColon === -1) { 21 | return [event, undefined]; 22 | } 23 | 24 | const before = event.substring(0, posOfFirstColon); 25 | const after = event.substring(posOfFirstColon + 1); 26 | 27 | return [before, after]; 28 | } 29 | 30 | // regex from https://github.com/chalk/ansi-regex 31 | const ANSI_REGEX = (() => { 32 | const pattern = [ 33 | "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)", 34 | "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))", 35 | ].join("|"); 36 | 37 | return new RegExp(pattern, "g"); 38 | })(); 39 | 40 | function stripAnsi(input) { 41 | return input.replace(ANSI_REGEX, ""); 42 | } 43 | 44 | const CONTROL_CHARACTERS = { 45 | ETX: Buffer.from([3]), 46 | }; 47 | 48 | module.exports = { brief, partition, stripAnsi, CONTROL_CHARACTERS }; 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dashflow", 3 | "version": "1.4.0", 4 | "description": "A modern makefile alternative with local dev workflow support and beautiful dashboard", 5 | "main": "index.js", 6 | "repository": "https://github.com/freewheel/dashflow", 7 | "author": "FreeWheel, A Comcast Company", 8 | "license": "Apache-2.0", 9 | "private": false, 10 | "bin": { 11 | "dashflow": "./bin/dashflow" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "lib", 16 | "web", 17 | "bin", 18 | "LICENSE", 19 | "README.md" 20 | ], 21 | "scripts": { 22 | "lint": "eslint lib test web/**/*.js --ignore-pattern web/lib/**/*.js && echo 'lint passed\n'", 23 | "start": "bin/dashflow", 24 | "format": "prettier --write \"lib/**/*.js\" && prettier --write \"web/**/*.js\" --ignore-pattern \"web/lib/**/*.js\"", 25 | "test": "mocha" 26 | }, 27 | "devDependencies": { 28 | "eslint": "^5.15.1", 29 | "mocha": "^6.0.2", 30 | "prettier": "^1.16.4" 31 | }, 32 | "dependencies": { 33 | "ansi-escapes": "^4.2.0", 34 | "boxen": "^4.1.0", 35 | "chalk": "^2.4.2", 36 | "chokidar": "^3.0.2", 37 | "columnify": "^1.5.4", 38 | "commander": "^3.0.0", 39 | "js-yaml": "^3.13.1", 40 | "node-pty": "https://github.com/microsoft/node-pty.git#04445ed76f90b4f56a190982ea2d4fcdd22a0ee7", 41 | "portfinder": "^1.0.23", 42 | "serve-handler": "^6.1.1", 43 | "socket.io": "^2.2.0", 44 | "vorpal": "^1.12.0" 45 | }, 46 | "prettier": { 47 | "trailingComma": "es5" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /web/components/area.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | const TEXTAREA_LINE_BREAK = String.fromCharCode(13, 10); 4 | 5 | export class ReadOnlyArea extends React.Component { 6 | constructor(props) { 7 | super(props); 8 | this.textareaRef = React.createRef(); 9 | this.autoScrolling = true; 10 | } 11 | 12 | componentDidMount() { 13 | const self = this; 14 | const textarea = this.textareaRef.current; 15 | textarea.scrollTop = textarea.scrollHeight - textarea.clientHeight; 16 | textarea.onscroll = function setAutoScrolling() { 17 | if (this.scrollTop + this.clientHeight === this.scrollHeight) { 18 | self.autoScrolling = true; 19 | } else { 20 | self.autoScrolling = false; 21 | } 22 | }; 23 | } 24 | 25 | componentDidUpdate() { 26 | if (this.autoScrolling) { 27 | const textarea = this.textareaRef.current; 28 | textarea.scrollTop = textarea.scrollHeight - textarea.clientHeight; 29 | } 30 | } 31 | 32 | render() { 33 | const { lines } = this.props; 34 | 35 | const textareaRef = this.textareaRef; 36 | return React.createElement("textarea", { 37 | ref: textareaRef, 38 | className: "read-only-area", 39 | readOnly: true, 40 | value: lines.join(TEXTAREA_LINE_BREAK), 41 | }); 42 | } 43 | } 44 | 45 | export const EditableArea = ({ value, onChange }) => 46 | React.createElement("textarea", { 47 | className: "editable-area", 48 | value, 49 | onChange: event => { 50 | onChange(event.target.value); 51 | }, 52 | }); 53 | -------------------------------------------------------------------------------- /lib/command.js: -------------------------------------------------------------------------------- 1 | const shell = require("./shell"); 2 | const debug = require("debug"); 3 | const debugCommand = commandId => debug(`command:${commandId}`); 4 | 5 | const PROVIDERS = { 6 | shell: function execShell(id, { cwd, cmd }, _, onEvent, attach) { 7 | debugCommand("run shell", id, cmd, cwd); 8 | 9 | const { promise, attachOneOff } = shell(cwd, cmd, onEvent); 10 | 11 | if (attach) { 12 | attachOneOff(process); 13 | } 14 | 15 | return promise; 16 | }, 17 | serial: async function execSerial( 18 | id, 19 | toExecuteCommands, 20 | commands, 21 | onEvent, 22 | attach 23 | ) { 24 | debugCommand("run serial", id); 25 | 26 | for (const command of toExecuteCommands) { 27 | await executeCommand(command, commands, onEvent, attach); 28 | } 29 | }, 30 | parallel: function execParallel(id, toExecuteCommands, commands, onEvent) { 31 | debugCommand("run parallel", id); 32 | 33 | // parallel doesn't support attach 34 | return Promise.all( 35 | toExecuteCommands.map(command => 36 | executeCommand(command, commands, onEvent, false) 37 | ) 38 | ); 39 | }, 40 | }; 41 | 42 | function executeCommand(id, commands, onEvent, attach = false) { 43 | const spec = commands[id]; 44 | const types = Object.keys(spec); 45 | 46 | if (types.length !== 1) { 47 | throw new Error(`Expect to have one single type for command ${id}`); 48 | } 49 | 50 | const type = types[0]; 51 | 52 | if (PROVIDERS[type]) { 53 | return PROVIDERS[type](id, spec[type], commands, onEvent, attach); 54 | } else { 55 | throw new Error(`Unsupported type ${type} for command ${id}`); 56 | } 57 | } 58 | 59 | module.exports = { 60 | executeCommand, 61 | }; 62 | -------------------------------------------------------------------------------- /bin/dashflow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const program = require("commander"); 4 | const packageJSON = require("../package.json"); 5 | const { start } = require("../lib/cli"); 6 | 7 | program 8 | .version(packageJSON.version) 9 | .usage("[options] [one-off-commands...]") 10 | .description(packageJSON.description) 11 | .option( 12 | "-c, --config ", 13 | "Specify one or multiple configuration files, separated by comma", 14 | "dashflow.yml" 15 | ) 16 | .option("-p, --port ", "Specify custom http port", null) 17 | .option("--verbose", "verbose output for errors", false) 18 | .on("--help", function() { 19 | process.stdout.write(` 20 | Examples 21 | 22 | by default dashflow reads from dashflow.yml in current folder 23 | 24 | execute one off commands, similar to what Makefile allows us to do 25 | $ dashflow lint 26 | $ dashflow lint test 27 | 28 | start a daemon which enables local dev workflow 29 | $ dashflow 30 | 31 | support multi config files, config files will be merged 32 | $ dashflow -c a_service/dashflow.yml,b_service/dashflow.yml 33 | 34 | custom http port 35 | $ dashflow -p 9527 36 | 37 | run with additional debug information 38 | $ DEBUG="daemon:*" dashflow 39 | $ DEBUG="events:*" dashflow 40 | $ DEBUG="stream:*" dashflow 41 | $ DEBUG="workflow:*" dashflow 42 | $ DEBUG="*" dashflow 43 | `); 44 | }) 45 | .parse(process.argv); 46 | 47 | function onError(err) { 48 | if (program.verbose) { 49 | process.stderr.write(`${err.stack}\n`); 50 | } else { 51 | process.stderr.write(err.message + "\n"); 52 | } 53 | 54 | process.exit(1); 55 | } 56 | 57 | start(program.config.split(","), program.port, program.args, onError).catch( 58 | onError 59 | ); 60 | -------------------------------------------------------------------------------- /CONTRIBUTOR.md: -------------------------------------------------------------------------------- 1 | # Dashflow Contributor's Guide 2 | 3 | ## How to Start Local Development 4 | 5 | 1. `yarn install` to install required packages 6 | 2. `yarn link` to link current project so `dashflow` command becomes globally available 7 | 3. local dev workflow 8 | a. We can definitely use dashflow itself for local dev workflow, just `dashflow` you'll be ready 9 | b. If you prefer the manual workflow, `yarn lint`, `yarn test`, and `yarn format` can be helpful 10 | 11 | ## How the Code is Organized 12 | 13 | 1. Directory Structure 14 | 15 | ``` 16 | ├── CHANGELOG.md 17 | ├── CONTRIBUTOR.md 18 | ├── LICENSE 19 | ├── README.md 20 | ├── bin/ # executables are located here 21 | ├── dashflow-sophisticated.yml # a dashflow config for test 22 | ├── dashflow.yml # the config for local development 23 | ├── guide_assets # images for README.md 24 | ├── index.js # export functions to allow using dashboard as a library 25 | ├── lib/ # source file for backend 26 | ├── package.json # dependencies and metadata 27 | ├── test/ # test cases 28 | ├── web/ # source file for web dashboard 29 | └── yarn.lock # lock down specific dependency package version 30 | ``` 31 | 32 | 2. Architecture 33 | 34 | ![architecture](./guide_assets/architecture.png) 35 | 36 | ## How to Debug Issues 37 | 38 | 1. Run with DEBUG environment variable 39 | 40 | ``` 41 | # run with additional debug information 42 | $ DEBUG="app:*" dashflow 43 | $ DEBUG="events:*" dashflow 44 | $ DEBUG="stream:*" dashflow 45 | $ DEBUG="workflow:*" dashflow 46 | $ DEBUG="*" dashflow 47 | ``` 48 | 49 | 2. Run with `--verbose` flag 50 | 51 | When we run dashflow with --verbose flag, we'll see full stacktrace for errors. 52 | -------------------------------------------------------------------------------- /test/formatter-test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env node, mocha */ 2 | const { equal } = require("assert"); 3 | const { 4 | shellFormatter, 5 | watchFormatter, 6 | commandFormatter, 7 | restartFormatter, 8 | smartUnformat, 9 | } = require("../lib/formatter"); 10 | 11 | describe("formatter", () => { 12 | it("format shell", () => { 13 | equal( 14 | shellFormatter.format("yarn lint", "stdout", "lint passing"), 15 | "shell:yarn lint:stdout:lint passing" 16 | ); 17 | }); 18 | 19 | it("unformat shell", () => { 20 | equal( 21 | shellFormatter.unformat("shell:yarn lint:stdout:lint passing"), 22 | "lint passing" 23 | ); 24 | }); 25 | 26 | it("format watch", () => { 27 | equal( 28 | watchFormatter.format("unlink", "/home/abc/def"), 29 | "watch:unlink:/home/abc/def" 30 | ); 31 | }); 32 | 33 | it("unformat watch", () => { 34 | equal( 35 | watchFormatter.unformat("watch:unlink:/home/abc/def"), 36 | "unlink:/home/abc/def" 37 | ); 38 | }); 39 | 40 | it("format restart", () => { 41 | equal(restartFormatter.format("irb"), "restart:irb"); 42 | }); 43 | 44 | it("unformat restart", () => { 45 | equal(restartFormatter.unformat("restart:irb"), "irb"); 46 | }); 47 | 48 | it("format command", () => { 49 | equal( 50 | commandFormatter.format("lint", "stdout", "lint passing"), 51 | "command:lint:stdout:lint passing" 52 | ); 53 | }); 54 | 55 | it("unformat command", () => { 56 | equal( 57 | commandFormatter.unformat("command:lint:stdout:lint passing"), 58 | "lint passing" 59 | ); 60 | }); 61 | 62 | it("smart unformat", () => { 63 | equal(smartUnformat("shell:yarn lint:stdout:lint passing"), "lint passing"); 64 | equal(smartUnformat("watch:unlink:/home/abc/def"), "unlink:/home/abc/def"); 65 | equal(smartUnformat("restart:irb"), "irb"); 66 | equal(smartUnformat("command:lint:stdout:lint passing"), "lint passing"); 67 | }); 68 | }); 69 | -------------------------------------------------------------------------------- /web/dashboards/system.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | import { Components } from "../components/index.js"; 4 | import { Utils } from "../utils.js"; 5 | 6 | const Events = ({ events, globalFilterValue, globalFilterValid }) => { 7 | const globalFilterPattern = globalFilterValid ? new RegExp(globalFilterValue) : new RegExp(''); 8 | 9 | const eventsAfterFilter = events 10 | .filter(evt => globalFilterPattern.test(evt.e)) 11 | .map(evt => `${evt.t} ${Utils.trimLineBreaks(Utils.stripAnsi(evt.e))}`); 12 | 13 | return React.createElement(Components.Panel, { 14 | title: React.createElement( 15 | "div", 16 | { 17 | className: "text" 18 | }, 19 | "Events" 20 | ), 21 | pill: React.createElement( 22 | "span", 23 | { 24 | className: "tooltip tooltip-left", 25 | "data-tooltip": "showing latest 200 events", 26 | }, 27 | `${events.length} Events` 28 | ), 29 | style: { 30 | top: 0, 31 | left: 0, 32 | width: "100%", 33 | height: "50%", 34 | }, 35 | content: React.createElement(Components.ReadOnlyArea, { 36 | lines: eventsAfterFilter.slice(-200), 37 | }), 38 | }); 39 | }; 40 | 41 | const DashboardsConfig = ({ dashboardsYAML, updateDashboardsYAML }) => 42 | React.createElement(Components.Panel, { 43 | title: React.createElement( 44 | "span", 45 | { 46 | className: "tooltip tooltip-right", 47 | "data-tooltip": "editable, change will be applied", 48 | }, 49 | React.createElement( 50 | "div", 51 | { 52 | className: "text" 53 | }, 54 | "Dashboards Config" 55 | ), 56 | ), 57 | style: { 58 | top: "50%", 59 | left: 0, 60 | width: "100%", 61 | height: "50%", 62 | }, 63 | content: React.createElement(Components.EditableArea, { 64 | value: dashboardsYAML, 65 | onChange: updateDashboardsYAML, 66 | }), 67 | }); 68 | 69 | export const SystemDashboard = { 70 | title: "SYSTEM", 71 | pannels: [Components.pure(Events), Components.pure(DashboardsConfig)], 72 | }; 73 | -------------------------------------------------------------------------------- /web/dashboards/log.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | import { Components } from '../components/index.js'; 4 | import { Utils } from '../utils.js'; 5 | import { getGaugeDisplay } from './gauge.js'; 6 | 7 | export function log(title, positionSpec, logFilter, gauge, limit = 500) { 8 | const logPattern = new RegExp(logFilter); 9 | 10 | return Components.pure( 11 | function getLogGauge({ events, globalFilterValue, globalFilterValid }) { 12 | const globalFilterPattern = globalFilterValid ? new RegExp(globalFilterValue) : new RegExp(''); 13 | 14 | let className = ""; 15 | let gaugeText = ""; 16 | 17 | if (gauge) { 18 | const gaugePattern = new RegExp(gauge.filter); 19 | 20 | const { level, text } = getGaugeDisplay( 21 | gauge.scan, 22 | events 23 | .filter(e => gaugePattern.test(Utils.stripAnsi(e.e))) 24 | .map(({ e }) => Utils.stripAnsi(e)) 25 | ); 26 | 27 | gaugeText = text; 28 | className = level; 29 | } 30 | 31 | return React.createElement(Components.Panel, { 32 | title: React.createElement( 33 | "span", 34 | { 35 | className: "tooltip tooltip-right", 36 | "data-tooltip": `filter: "${logFilter}"`, 37 | }, 38 | React.createElement( 39 | "div", 40 | { 41 | className: "text" 42 | }, 43 | title 44 | ) 45 | ), 46 | pill: gaugeText, 47 | style: Utils.positionSpecToStyle(positionSpec), 48 | className, 49 | content: React.createElement(Components.ReadOnlyArea, { 50 | lines: events 51 | .filter( 52 | e => 53 | logPattern.test(e.e) && 54 | e.h !== undefined && 55 | globalFilterPattern.test(e.h) 56 | ) 57 | .map(({ h }) => Utils.normalizeLineBreaks(Utils.stripAnsi(h))) 58 | .join("") 59 | .split("\n") 60 | .slice(-limit), 61 | }), 62 | }); 63 | }, 64 | ["events", "globalFilterValue", "globalFilterValid"] 65 | ); 66 | } -------------------------------------------------------------------------------- /web/dashboards/gauge.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | import { Components } from '../components/index.js'; 4 | import { Utils } from '../utils.js'; 5 | 6 | export function getGaugeDisplay(scan, lines) { 7 | const normalizedScan = { 8 | when: 9 | scan.when && 10 | scan.when.map(test => ({ 11 | pattern: new RegExp(test.pattern), 12 | text: test.text, 13 | level: test.level, 14 | })), 15 | default: { 16 | text: (scan.default && scan.default.text) || "Unknown", 17 | level: (scan.default && scan.default.level) || "", 18 | }, 19 | }; 20 | for (let i = lines.length - 1; i >= 0; i--) { 21 | for (let test of normalizedScan.when) { 22 | if (test.pattern.test(lines[i])) { 23 | return { 24 | text: test.text, 25 | level: test.level, 26 | }; 27 | } 28 | } 29 | } 30 | 31 | return normalizedScan.default; 32 | } 33 | 34 | export function gauge(title, positionSpec, filter, scan) { 35 | const pattern = new RegExp(filter); 36 | 37 | return Components.pure( 38 | function getGauge({ events }) { 39 | const { level, text } = getGaugeDisplay( 40 | scan, 41 | events 42 | .filter(e => pattern.test(Utils.stripAnsi(e.e))) 43 | .map(({ e }) => Utils.stripAnsi(e)) 44 | ); 45 | 46 | return React.createElement(Components.Panel, { 47 | title: React.createElement( 48 | "span", 49 | { 50 | className: "tooltip tooltip-bottom", 51 | "data-tooltip": `filter: "${filter}"`, 52 | }, 53 | title 54 | ), 55 | style: Utils.positionSpecToStyle(positionSpec), 56 | content: React.createElement( 57 | "span", 58 | { 59 | style: { 60 | padding: ".2em", 61 | cursor: "default", 62 | textTransform: "uppercase", 63 | }, 64 | }, 65 | React.createElement( 66 | "div", 67 | { className: `toast toast-${level}` }, 68 | text 69 | ) 70 | ), 71 | }); 72 | }, 73 | ["events"] 74 | ); 75 | } -------------------------------------------------------------------------------- /lib/formatter.js: -------------------------------------------------------------------------------- 1 | function substringAfter(string, keyword) { 2 | return string.substring(string.indexOf(keyword) + keyword.length); 3 | } 4 | 5 | const shellFormatter = { 6 | format: function format(command, device, content) { 7 | return `shell:${command}:${device}:${content}`; 8 | }, 9 | unformat: function unformat(event) { 10 | // possibleDevices are ['stdout', 'stderr', 'state'] 11 | // but state are invisible to human 12 | 13 | if (event.indexOf(":stdout:") !== -1) { 14 | return substringAfter(event, ":stdout:"); 15 | } else if (event.indexOf(":stderr:") !== -1) { 16 | return substringAfter(event, ":stderr:"); 17 | } else { 18 | return undefined; 19 | } 20 | }, 21 | }; 22 | 23 | const watchFormatter = { 24 | format: function format(event, path) { 25 | return `watch:${event}:${path}`; 26 | }, 27 | unformat: function unformat(event) { 28 | return substringAfter(event, "watch:"); 29 | }, 30 | }; 31 | 32 | const restartFormatter = { 33 | format: function format(streamId) { 34 | return `restart:${streamId}`; 35 | }, 36 | unformat: function unformat(event) { 37 | return substringAfter(event, "restart:"); 38 | }, 39 | }; 40 | 41 | const commandFormatter = { 42 | format: function format(name, device, content) { 43 | return `command:${name}:${device}:${content}`; 44 | }, 45 | unformat: function unformat(event) { 46 | // possibleDevices are ['stdout', 'state'] 47 | // but state are invisible to human 48 | 49 | if (event.indexOf(":stdout:") !== -1) { 50 | return substringAfter(event, ":stdout:"); 51 | } else { 52 | return undefined; 53 | } 54 | }, 55 | }; 56 | 57 | function smartUnformat(event) { 58 | if (event.indexOf("shell:") !== -1) { 59 | return shellFormatter.unformat(event); 60 | } else if (event.indexOf("watch:") !== -1) { 61 | return watchFormatter.unformat(event); 62 | } else if (event.indexOf("restart:") !== -1) { 63 | return restartFormatter.unformat(event); 64 | } else if (event.indexOf("command:") !== -1) { 65 | return commandFormatter.unformat(event); 66 | } else { 67 | // do not know how to unformat 68 | // return as is 69 | return event; 70 | } 71 | } 72 | 73 | module.exports = { 74 | shellFormatter, 75 | watchFormatter, 76 | restartFormatter, 77 | commandFormatter, 78 | smartUnformat, 79 | }; 80 | -------------------------------------------------------------------------------- /lib/daemon.js: -------------------------------------------------------------------------------- 1 | const { init, add, subscribe } = require("./events"); 2 | const { createStream } = require("./stream"); 3 | const { createWorkflow } = require("./workflow"); 4 | const debug = require("debug"); 5 | const debugDaemon = debug("daemon"); 6 | 7 | function startDaemon(config) { 8 | debugDaemon("starting daemon with config", config); 9 | 10 | const app = { 11 | events: init(), 12 | config, 13 | streams: {}, 14 | getStreamIds: function getStreamIds() { 15 | return Object.keys(this.config.streams); 16 | }, 17 | getWorkflowIds: function getWorkflowIds() { 18 | return Object.keys(this.config.workflows); 19 | }, 20 | restartStream: function restartStream(streamId) { 21 | const self = this; 22 | 23 | if (this.streams[streamId]) { 24 | this.streams[streamId].terminate(); 25 | } 26 | 27 | this.streams[streamId] = createStream( 28 | streamId, 29 | config.streams[streamId], 30 | function onEvent(newEvent) { 31 | self.addEvent(`stream:${streamId}:${newEvent}`); 32 | } 33 | ); 34 | }, 35 | attachToStream: function attach(streamId, attachToProcess) { 36 | return this.streams[streamId].attach(attachToProcess); 37 | }, 38 | writeToStream: function writeToStream(streamId, message) { 39 | this.streams[streamId].write(message); 40 | }, 41 | addEvent: function addEvent(event) { 42 | return add(event, this.events); 43 | }, 44 | }; 45 | 46 | Object.keys(config.workflows).forEach(id => { 47 | const workflow = createWorkflow( 48 | id, 49 | config.workflows[id], 50 | config.commands, 51 | function onEvent(newEvent) { 52 | app.addEvent(`workflow:${id}:${newEvent}`); 53 | }, 54 | function restartStream(streamId) { 55 | app.restartStream(streamId); 56 | } 57 | ); 58 | 59 | subscribe(app.events, id, workflow); 60 | }); 61 | 62 | app.addEvent("SYSTEM:started"); 63 | 64 | Object.keys(config.streams).forEach(id => { 65 | app.streams[id] = createStream(id, config.streams[id], function onEvent( 66 | newEvent 67 | ) { 68 | app.addEvent(`stream:${id}:${newEvent}`); 69 | }); 70 | }); 71 | 72 | debugDaemon("snapshot after initialization", app); 73 | 74 | process.on("exit", () => { 75 | // recycle stream processes 76 | Object.values(app.streams).forEach(stream => { 77 | stream.terminate(); 78 | }); 79 | }); 80 | 81 | return app; 82 | } 83 | 84 | module.exports = { startDaemon }; 85 | -------------------------------------------------------------------------------- /web/components/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | /* global React */ 3 | 4 | import { ReadOnlyArea, EditableArea } from './area.js' 5 | import { Panel } from './panel.js'; 6 | import { Dashboard } from './dashboard.js'; 7 | import { Errors } from './errors.js'; 8 | import { pure, PureHeader } from './pure.js'; 9 | 10 | class App extends React.Component { 11 | constructor(props) { 12 | super(props); 13 | 14 | props.store.subscribe(() => { 15 | this.forceUpdate(); 16 | }); 17 | } 18 | 19 | render() { 20 | const { store } = this.props; 21 | const { 22 | dashboards, 23 | dashboardsYAML, 24 | events, 25 | globalFilterValue, 26 | globalFilterValid, 27 | currentDashboardTitle, 28 | errorMessages, 29 | } = store; 30 | 31 | const updateGlobalFilter = store.updateGlobalFilter.bind(store); 32 | const updateCurrentDashboardTitle = store.updateCurrentDashboardTitle.bind( 33 | store 34 | ); 35 | const updateDashboardsYAML = store.updateDashboardsYAML.bind(store); 36 | const clearErrorMessages = store.clearErrorMessages.bind(store); 37 | 38 | if (dashboards.length === 0) { 39 | return [ 40 | React.createElement(Errors, { 41 | messages: errorMessages, 42 | onClear: clearErrorMessages, 43 | }), 44 | React.createElement( 45 | "div", 46 | { 47 | className: "loading loading-lg app", 48 | }, 49 | "Loading" 50 | ), 51 | ]; 52 | } 53 | 54 | let currentDashboard = 55 | dashboards.find( 56 | dashboard => dashboard.title === currentDashboardTitle 57 | ) || dashboards[0]; 58 | 59 | return [ 60 | React.createElement(Errors, { 61 | messages: errorMessages, 62 | onClear: clearErrorMessages, 63 | }), 64 | React.createElement(PureHeader, { 65 | dashboards, 66 | currentDashboardTitle: currentDashboard.title, 67 | updateCurrentDashboardTitle, 68 | globalFilterValue, 69 | globalFilterValid, 70 | updateGlobalFilter, 71 | }), 72 | React.createElement(Dashboard, { 73 | key: currentDashboard.title, 74 | pannels: currentDashboard.pannels, 75 | events, 76 | globalFilterValue, 77 | globalFilterValid, 78 | dashboardsYAML, 79 | updateDashboardsYAML, 80 | }), 81 | ]; 82 | } 83 | } 84 | 85 | export const Components = { 86 | ReadOnlyArea, 87 | EditableArea, 88 | Panel, 89 | Dashboard, 90 | App, 91 | pure, 92 | }; 93 | -------------------------------------------------------------------------------- /pipeline-pull-request.yml: -------------------------------------------------------------------------------- 1 | resource_types: 2 | - name: pull-request 3 | type: docker-image 4 | source: 5 | repository: teliaoss/github-pr-resource 6 | 7 | resources: 8 | - name: dashflow-pr 9 | type: pull-request 10 | check_every: 1m 11 | source: 12 | repository: freewheel/dashflow 13 | access_token: ((dashflow-pr-access-token)) 14 | 15 | jobs: 16 | - name: lint 17 | plan: 18 | - get: dashflow-pr 19 | trigger: true 20 | version: every 21 | - put: dashflow-pr 22 | params: 23 | path: dashflow-pr 24 | context: Lint 25 | status: PENDING 26 | 27 | - task: run-lint 28 | config: 29 | platform: linux 30 | image_resource: 31 | type: docker-image 32 | source: { repository: node, tag: 11 } 33 | inputs: 34 | - name: dashflow-pr 35 | run: 36 | path: sh 37 | args: 38 | - -exc 39 | - | 40 | cd dashflow-pr 41 | yarn install --network-concurrency 1 42 | yarn --silent lint 43 | on_success: 44 | put: dashflow-pr 45 | params: 46 | path: dashflow-pr 47 | context: Lint 48 | status: SUCCESS 49 | on_failure: 50 | put: dashflow-pr 51 | params: 52 | path: dashflow-pr 53 | context: Lint 54 | status: FAILURE 55 | 56 | - name: test 57 | plan: 58 | - get: dashflow-pr 59 | trigger: true 60 | version: every 61 | - put: dashflow-pr 62 | params: 63 | path: dashflow-pr 64 | context: Test 65 | status: PENDING 66 | 67 | - task: run-test 68 | config: 69 | platform: linux 70 | image_resource: 71 | type: docker-image 72 | source: { repository: node, tag: 11 } 73 | inputs: 74 | - name: dashflow-pr 75 | run: 76 | path: sh 77 | args: 78 | - -exc 79 | - | 80 | cd dashflow-pr 81 | yarn install --network-concurrency 1 82 | yarn --silent test --no-color --verbose 83 | on_success: 84 | put: dashflow-pr 85 | params: 86 | path: dashflow-pr 87 | context: Test 88 | status: SUCCESS 89 | on_failure: 90 | put: dashflow-pr 91 | params: 92 | path: dashflow-pr 93 | context: Test 94 | status: FAILURE 95 | 96 | -------------------------------------------------------------------------------- /web/components/header.js: -------------------------------------------------------------------------------- 1 | /* global React */ 2 | 3 | export const Header = ({ 4 | dashboards, 5 | currentDashboardTitle, 6 | updateCurrentDashboardTitle, 7 | globalFilterValue, 8 | globalFilterValid, 9 | updateGlobalFilter, 10 | }) => { 11 | return React.createElement("header", { className: "dash-header", }, 12 | [ 13 | React.createElement("section", { className: "header-left" }, 14 | [ 15 | React.createElement("a", 16 | { 17 | href: "https://github.com/freewheel/dashflow", 18 | target: "_blank" 19 | }, 20 | React.createElement("img", 21 | { 22 | className: "logo", 23 | src: "../logo/dashflow-nav.png" 24 | } 25 | ) 26 | ), 27 | React.createElement("div", { className: "dropdown" }, 28 | [ 29 | React.createElement("div", { className: "active-item item" }, 30 | [ 31 | React.createElement("span", { className: "text" }, currentDashboardTitle), 32 | React.createElement("div", { className: "arrow down" }) 33 | ] 34 | ), 35 | React.createElement("ul", { className: "inactive-items" }, 36 | dashboards.map(({ title }) => { 37 | const inactive = (title === currentDashboardTitle) ? 'inactive' : ''; 38 | return React.createElement( 39 | "li", 40 | { 41 | className: `item ${ inactive }`, 42 | key: title, 43 | onClick: event => { 44 | updateCurrentDashboardTitle(title); 45 | event.preventDefault(); 46 | }, 47 | }, 48 | title 49 | ); 50 | }) 51 | ) 52 | ] 53 | ) 54 | ] 55 | 56 | ), 57 | React.createElement("div", { className: "header-right" }, 58 | React.createElement("div", 59 | { 60 | className: `filter ${ !globalFilterValid ? 'invalid' : ''}` 61 | }, 62 | React.createElement("input", 63 | { 64 | className: 'input', 65 | type: "text", 66 | value: globalFilterValue, 67 | onChange: event => { 68 | updateGlobalFilter(event.target.value); 69 | }, 70 | placeholder: "REGEX FILTER", 71 | } 72 | ) 73 | ) 74 | ), 75 | ] 76 | ); 77 | }; -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 2019-09-06 2 | 3 | - removed spectre as dependency 4 | - redesigned header to be dropdown 5 | 6 | # 2019-08-23 7 | 8 | - supported alias for dashboard widget position 9 | - upgraded depended packages 10 | - added `ignore` option for watch stream 11 | 12 | # 2019-06-28 13 | 14 | - added node 12 support 15 | - fixed the issue that one off command cannot receive user input 16 | 17 | # 2019-06-25 18 | 19 | - fixed interactive shell bug: not printing out previous logs when attach/tail to a stream/workflow 20 | 21 | # 2019-06-24 22 | 23 | - reverted a bug fix of missing events which caused more issues. 24 | 25 | # 2019-06-18 26 | 27 | - improved error handling 28 | 29 | # 2019-06-17 30 | 31 | - updated dashboard look and feel 32 | 33 | # 2019-06-07 34 | 35 | - added option for log with gauge 36 | 37 | # 2019-05-23 38 | 39 | - updated favicon and logo 40 | 41 | # 2019-04-21 42 | 43 | - fixed interactive shell bug 44 | 45 | # 2019-04-20 46 | 47 | - added `run` commands for interative mode 48 | 49 | # 2019-04-19 50 | 51 | - supported automatic port allocation 52 | 53 | # 2019-04-16 54 | 55 | - fixed workflow parallel provider not triggering bug 56 | - fixed workflow command missing state:started event 57 | 58 | # 2019-04-12 59 | 60 | - suppressed stream exit promise error 61 | 62 | # 2019-03-11 63 | 64 | - redesigned dashflow.yml syntax 65 | 66 | # 2018-10-05 67 | 68 | - added support for `delay` command to workflow 69 | 70 | # 2018-07-31 71 | 72 | - fixed stream shell command bug when command contains double quotes 73 | 74 | # 2018-07-19 75 | 76 | - change web UI font 77 | - support $FILENAME environment variable in workflow shell function 78 | 79 | # 2018-07-06 80 | 81 | - refactor to provide a more usable API interface 82 | 83 | # 2018-07-02 84 | 85 | - improved web UI configuration error handling 86 | - removed package dependency on execa 87 | - added command autocomplete for interactive shell 88 | - support environment variable prefix in shell command 89 | - fixed workflow triggering order problem 90 | 91 | # 2018-06-30 92 | 93 | - added "config" command to interactive shell to dump config info 94 | 95 | # 2018-06-29 96 | 97 | - improved config file validation for streams/workflows 98 | - improved web UI to support graceful backend switch from one project to the other 99 | - added live config editing support 100 | 101 | # 2018-06-27 102 | 103 | - improved stream child process cleanup 104 | - added update notifier 105 | - improved shell output events, being more granular with additional state events 106 | - changed system dashboard to show latest events 107 | - refined web UI display 108 | 109 | # 2018-06-26 110 | 111 | - added gauge function for dashboard 112 | -------------------------------------------------------------------------------- /lib/stream.js: -------------------------------------------------------------------------------- 1 | const chokidar = require("chokidar"); 2 | const { watchFormatter, shellFormatter } = require("./formatter"); 3 | const shell = require("./shell"); 4 | const debug = require("debug"); 5 | const debugStream = streamId => debug(`stream:${streamId}`); 6 | 7 | const PROVIDERS = { 8 | watch: function execWatch(id, { cwd, glob, ignore }, onEvent) { 9 | debugStream(id)("run watch", glob, cwd); 10 | 11 | let watcherAlive = true; 12 | const watcher = chokidar 13 | .watch(glob, { 14 | cwd, 15 | ignored: ignore, 16 | atomic: true, 17 | ignoreInitial: true, 18 | }) 19 | .on("all", (event, path) => { 20 | debugStream(id)("watch", event, path); 21 | 22 | onEvent(watchFormatter.format(event, path)); 23 | }); 24 | 25 | return { 26 | promise: new Promise(function(resolve) { 27 | const intervalId = setInterval(function checkAlive() { 28 | debugStream(id)("watch check alive"); 29 | 30 | if (!watcherAlive) { 31 | clearInterval(intervalId); 32 | 33 | resolve(); 34 | } 35 | }, 500); 36 | }), 37 | terminate: function terminate() { 38 | watcher.close(); 39 | watcherAlive = false; 40 | }, 41 | attach: function attach() { 42 | return new Promise(function(resolve, reject) { 43 | reject(new Error("Cannot attach to a watch stream")); 44 | }); 45 | }, 46 | }; 47 | }, 48 | shell: function execShell(id, { cwd, cmd }, onEvent) { 49 | // state can be used to detect that a shell starts to run 50 | onEvent(shellFormatter.format(cmd, "state", "started")); 51 | 52 | const { promise, terminate, attach } = shell(cwd, cmd, (type, data) => { 53 | if (type === "data") { 54 | onEvent(shellFormatter.format(cmd, "stdout", data)); 55 | } else if (type === "exit") { 56 | onEvent(shellFormatter.format(cmd, "state", `exited with ${data}`)); 57 | } 58 | }); 59 | 60 | promise.catch(() => { 61 | // swallow exit error since we had it in event queue already 62 | }); 63 | 64 | return { promise, terminate, attach }; 65 | }, 66 | }; 67 | 68 | // each stream running in a child process 69 | // if the child process crashed 70 | // output errors but keeps main process running 71 | function createStream(id, spec, onEvent) { 72 | const types = Object.keys(spec); 73 | 74 | if (types.length !== 1) { 75 | throw new Error(`Expect to have one single type for stream ${id}`); 76 | } 77 | 78 | const type = types[0]; 79 | 80 | if (PROVIDERS[type]) { 81 | return PROVIDERS[type](id, spec[type], onEvent); 82 | } else { 83 | throw new Error(`Unsupported type ${type} for stream ${id}`); 84 | } 85 | } 86 | 87 | module.exports = { 88 | createStream, 89 | }; 90 | -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | const http = require("http"); 2 | const path = require("path"); 3 | const handler = require("serve-handler"); 4 | const debug = require("debug"); 5 | const socketIO = require("socket.io"); 6 | const { subscribe, unsubscribe, backwardsPage } = require("./events"); 7 | const { smartUnformat } = require("./formatter"); 8 | const debugServer = debug("server"); 9 | 10 | class PortUnavailableError extends Error {} 11 | 12 | const CATCHUP_BATCH_SIZE = 10000; 13 | 14 | function toBrowserEvent(event) { 15 | return { 16 | id: event.id, 17 | t: event.time, 18 | e: event.event, 19 | h: smartUnformat(event.event), 20 | }; 21 | } 22 | 23 | function serve(app, port, onError) { 24 | const server = http.createServer((req, res) => { 25 | debugServer(req); 26 | 27 | handler(req, res, { 28 | public: path.relative( 29 | process.cwd(), 30 | path.resolve(__dirname, "..", "web") 31 | ), 32 | directoryListing: false, 33 | }); 34 | }); 35 | 36 | let seq = 0; 37 | 38 | const io = socketIO(server); 39 | 40 | io.on("connection", function(socket) { 41 | debugServer("a user connected"); 42 | 43 | socket.on("get_dashboards_config", function() { 44 | socket.emit("get_dashboards_config", app.config.dashboards); 45 | }); 46 | 47 | socket.on("catchup", function() { 48 | let cursor = app.events.tail; 49 | 50 | const catchup = setInterval(() => { 51 | const page = backwardsPage(cursor, CATCHUP_BATCH_SIZE); 52 | cursor = page[0]; 53 | const batch = page[1]; 54 | 55 | if (batch.length > 0) { 56 | socket.emit("catchup_batch", batch.map(toBrowserEvent)); 57 | 58 | debugServer("catch"); 59 | } else { 60 | clearInterval(catchup); 61 | 62 | socket.emit("catchup_done"); 63 | 64 | debugServer("catch done"); 65 | } 66 | }, 1000); 67 | }); 68 | 69 | socket.on("delta", function() { 70 | let eventSubscriberId = `socket-agent-${seq++}`; 71 | 72 | subscribe(app.events, eventSubscriberId, (event, time, id) => { 73 | socket.emit("delta", toBrowserEvent({ event, time, id })); 74 | 75 | debugServer("delta"); 76 | }); 77 | 78 | socket.on("disconnect", function() { 79 | unsubscribe(app.events, eventSubscriberId); 80 | 81 | debugServer("user disconnected"); 82 | }); 83 | }); 84 | }); 85 | 86 | server.on("error", err => { 87 | if (err.code === "EADDRINUSE") { 88 | onError( 89 | new PortUnavailableError( 90 | `port ${port} is not available, please quit the other running job or specify a different port` 91 | ) 92 | ); 93 | } else { 94 | onError(err); 95 | } 96 | }); 97 | 98 | server.listen(port); 99 | } 100 | 101 | module.exports = { 102 | serve, 103 | }; 104 | -------------------------------------------------------------------------------- /web/utils.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | 3 | // regex from https://github.com/chalk/ansi-regex 4 | const ANSI_REGEX = (() => { 5 | const pattern = [ 6 | "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)", 7 | "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))", 8 | ].join("|"); 9 | 10 | return new RegExp(pattern, "g"); 11 | })(); 12 | 13 | function stripAnsi(input) { 14 | return input.replace(ANSI_REGEX, ""); 15 | } 16 | 17 | function trimLineBreaks(input) { 18 | return input.replace(/^[\n]+|[\n]+$/g, ""); 19 | } 20 | 21 | function normalizeLineBreaks(input) { 22 | return input.replace(/\r\n/g, "\n"); 23 | } 24 | 25 | const POSITION_ALIASES = { 26 | fullscreen: { top: "0", left: "0", width: "100%", height: "100%" }, 27 | "quadrant/top-left": { top: "0", left: "0", width: "50%", height: "50%" }, 28 | "quadrant/2": { top: "0", left: "0", width: "50%", height: "50%" }, 29 | "quadrant/top-right": { top: "0", left: "50%", width: "50%", height: "50%" }, 30 | "quadrant/1": { top: "0", left: "50%", width: "50%", height: "50%" }, 31 | "quadrant/bottom-left": { 32 | top: "50%", 33 | left: "0", 34 | width: "50%", 35 | height: "50%", 36 | }, 37 | "quadrant/3": { top: "50%", left: "0", width: "50%", height: "50%" }, 38 | "quadrant/bottom-right": { 39 | top: "50%", 40 | left: "50%", 41 | width: "50%", 42 | height: "50%", 43 | }, 44 | "quadrant/4": { top: "50%", left: "50%", width: "50%", height: "50%" }, 45 | "quadrant/top": { top: "0", left: "0", width: "100%", height: "50%" }, 46 | "quadrant/bottom": { top: "50%", left: "0", width: "100%", height: "50%" }, 47 | "quadrant/left": { top: "0", left: "0", width: "50%", height: "100%" }, 48 | "quadrant/right": { top: "0", left: "50%", width: "50%", height: "100%" }, 49 | }; 50 | 51 | function positionSpecToStyle(spec) { 52 | if (POSITION_ALIASES[spec]) { 53 | return POSITION_ALIASES[spec]; 54 | } 55 | 56 | function fromPercent(s) { 57 | if (s === "0") { 58 | return 0; 59 | } else { 60 | return parseInt(s.substring(0, s.length - 1), 10); 61 | } 62 | } 63 | 64 | function toPercent(i) { 65 | if (i === 0) { 66 | return 0; 67 | } else { 68 | return i + "%"; 69 | } 70 | } 71 | 72 | const [x1, y1, x2, y2] = spec 73 | .split(" ") 74 | .filter(s => s.length > 0) 75 | .map(fromPercent); 76 | 77 | return { 78 | top: toPercent(y1), 79 | left: toPercent(x1), 80 | width: toPercent(x2 - x1), 81 | height: toPercent(y2 - y1), 82 | }; 83 | } 84 | 85 | function throttleFactory(wait) { 86 | let queue = []; 87 | 88 | setInterval(() => { 89 | const callback = queue.pop(); 90 | 91 | if (callback instanceof Function) { 92 | callback(); 93 | } 94 | }, wait); 95 | 96 | return function(callback) { 97 | if (queue.length === 0) { 98 | queue.push(callback); 99 | } 100 | }; 101 | } 102 | 103 | export const Utils = { 104 | stripAnsi, 105 | trimLineBreaks, 106 | normalizeLineBreaks, 107 | positionSpecToStyle, 108 | throttleFactory, 109 | }; 110 | -------------------------------------------------------------------------------- /web/app.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | /* global React, ReactDOM, jsyaml */ 3 | 4 | import { Utils } from "./utils.js"; 5 | import { Components } from "./components/index.js"; 6 | import { Dashboards } from "./dashboards/index.js"; 7 | 8 | const notifyThrottle = Utils.throttleFactory(500); 9 | 10 | export function createStore() { 11 | return { 12 | dashboards: [], 13 | dashboardsYAML: null, 14 | currentDashboardTitle: null, 15 | globalFilterValue: "", 16 | globalFilterValid: true, 17 | events: [], 18 | errorMessages: [], 19 | subscribers: [], 20 | subscribe: function subscribe(subscriber) { 21 | this.subscribers.push(subscriber); 22 | }, 23 | notifySubscribers: function notifySubscribersWithThrottle( 24 | realtime = false 25 | ) { 26 | const self = this; 27 | 28 | if (realtime) { 29 | self.subscribers.forEach(subscriber => subscriber()); 30 | } else { 31 | notifyThrottle(function notifySubscribers() { 32 | self.subscribers.forEach(subscriber => subscriber()); 33 | }); 34 | } 35 | }, 36 | appendEvent: function appendEvent(event) { 37 | this.appendEvents([event]); 38 | }, 39 | appendEvents: function appendEvents(evts) { 40 | this.events = this.events.concat(evts); 41 | this.notifySubscribers(); 42 | }, 43 | resetEvents: function resetEvents() { 44 | this.events = []; 45 | this.notifySubscribers(); 46 | }, 47 | updateDashboards: function updateDashboards( 48 | dashboards, 49 | dashboardsYAML = null 50 | ) { 51 | this.dashboardsYAML = 52 | dashboardsYAML || jsyaml.safeDump(dashboards, { flowLevel: 2 }); 53 | this.dashboards = Dashboards.parseDashboards(dashboards).concat([ 54 | Dashboards.SystemDashboard, 55 | ]); 56 | 57 | this.notifySubscribers(true); 58 | }, 59 | updateGlobalFilter: function updateGlobalFilter(filter) { 60 | this.globalFilterValue = filter; 61 | 62 | try { 63 | new RegExp(filter); 64 | this.globalFilterValid = true; 65 | } catch (e) { 66 | this.globalFilterValid = false; 67 | } 68 | 69 | this.notifySubscribers(true); 70 | }, 71 | updateCurrentDashboardTitle: function updateCurrentDashboardTitle(title) { 72 | this.currentDashboardTitle = title; 73 | 74 | this.notifySubscribers(true); 75 | }, 76 | updateDashboardsYAML: function updateDashboardsYAML(dashboardsYAML) { 77 | try { 78 | const config = jsyaml.safeLoad(dashboardsYAML); 79 | 80 | this.updateDashboards(config, dashboardsYAML); 81 | } catch (err) { 82 | this.dashboardsYAML = dashboardsYAML; 83 | 84 | this.notifySubscribers(true); 85 | } 86 | }, 87 | addErrorMessage: function addErrorMessage(message) { 88 | this.errorMessages.push(message); 89 | 90 | this.notifySubscribers(true); 91 | }, 92 | clearErrorMessages: function clearErrorMessages() { 93 | this.errorMessages = []; 94 | 95 | this.notifySubscribers(true); 96 | }, 97 | }; 98 | } 99 | 100 | export function render(store) { 101 | ReactDOM.render( 102 | React.createElement(Components.App, { store }), 103 | document.getElementById("app") 104 | ); 105 | } 106 | -------------------------------------------------------------------------------- /lib/events.js: -------------------------------------------------------------------------------- 1 | // event is a time series 2 | // from latest to earliest 3 | // uses a FIFO linked list as the main structure 4 | 5 | const { brief } = require("./util"); 6 | const debug = require("debug"); 7 | const eventDebug = debug("events"); 8 | 9 | function now() { 10 | return Date.now(); 11 | } 12 | 13 | const DEFAULT_CAPACITY = 1000000; 14 | 15 | function init() { 16 | eventDebug("init events"); 17 | 18 | return { 19 | head: null, 20 | tail: null, 21 | length: 0, 22 | capacity: DEFAULT_CAPACITY, 23 | subscribers: {}, 24 | }; 25 | } 26 | 27 | function setCapacity(capacity, events) { 28 | events.capacity = capacity; 29 | 30 | eventDebug("set capacity for events", capacity); 31 | 32 | return events; 33 | } 34 | 35 | function length(events) { 36 | return events.length; 37 | } 38 | 39 | function add(event, events) { 40 | const timestamp = now(); 41 | 42 | if (events.head === null) { 43 | events.head = { id: 1, t: timestamp, e: event, n: null, p: null }; 44 | events.tail = events.head; 45 | events.length = 1; 46 | } else { 47 | const currentHead = events.head; 48 | events.head = { 49 | id: currentHead.id + 1, 50 | t: timestamp, 51 | e: event, 52 | n: currentHead, 53 | p: null, 54 | }; 55 | events.head.n.p = events.head; 56 | events.length = events.length + 1; 57 | } 58 | 59 | if (events.length > events.capacity) { 60 | const tp = events.tail.p; 61 | events.tail.p = null; 62 | tp.n = null; 63 | events.tail = tp; 64 | events.length = events.length - 1; 65 | } 66 | 67 | Object.values(events.subscribers).forEach(cb => { 68 | cb(events.head.e, events.head.t, events.head.id); 69 | }); 70 | 71 | eventDebug("added event", brief(event)); 72 | 73 | return events; 74 | } 75 | 76 | // scan events which matches given regx, less than number of limit lines, from latest to oldest 77 | function scan(regx, limit, events) { 78 | let results = []; 79 | let current = events.head; 80 | 81 | while (current !== null) { 82 | if (regx.test(current.e)) { 83 | results.push({ time: current.t, event: current.e, id: current.id }); 84 | } 85 | 86 | if (results.length >= limit) { 87 | break; 88 | } 89 | 90 | current = current.n; 91 | } 92 | 93 | return results; 94 | } 95 | 96 | // get a page from backwards, with given limit number of records 97 | function backwardsPage(cursor, limit) { 98 | let current = cursor; 99 | let results = []; 100 | 101 | while (current) { 102 | results.push({ time: current.t, event: current.e, id: current.id }); 103 | 104 | if (results.length >= limit) { 105 | break; 106 | } 107 | 108 | current = current.p; 109 | } 110 | 111 | return [current, results]; 112 | } 113 | 114 | // callback will be called with each new event 115 | function subscribe(events, subscriberId, callback) { 116 | events.subscribers[subscriberId] = callback; 117 | 118 | eventDebug("subscribed", subscriberId); 119 | 120 | return subscriberId; 121 | } 122 | 123 | function unsubscribe(events, subscriberId) { 124 | if (events.subscribers[subscriberId]) { 125 | delete events.subscribers[subscriberId]; 126 | 127 | eventDebug("unsubscribed", subscriberId); 128 | } 129 | } 130 | 131 | function toJSON(events) { 132 | return scan(/.*/, events.capacity, events).reverse(); 133 | } 134 | 135 | module.exports = { 136 | init, 137 | length, 138 | add, 139 | scan, 140 | backwardsPage, 141 | setCapacity, 142 | subscribe, 143 | unsubscribe, 144 | toJSON, 145 | }; 146 | -------------------------------------------------------------------------------- /dashflow.yml: -------------------------------------------------------------------------------- 1 | commands: 2 | lint: yarn --silent lint 3 | test: 4 | shell: { cmd: yarn --silent test --no-color --verbose } 5 | list-web-files: 6 | shell: { cmd: ls, cwd: web } 7 | lint-then-test: 8 | serial: 9 | - lint 10 | - test 11 | lint-and-test: 12 | parallel: 13 | - lint 14 | - test 15 | 16 | streams: 17 | watch-lib: 18 | watch: { glob: "lib/**/*.js" } 19 | watch-test: 20 | watch: { glob: "**/*.js", cwd: test } 21 | watch-web: 22 | watch: { glob: "web/**/*.js", cwd: web } 23 | 24 | workflows: 25 | initial-lint: 26 | match: SYSTEM:started 27 | command: lint 28 | initial-test: 29 | match: SYSTEM:started 30 | command: test 31 | lib-lint-then-test: 32 | match: watch-lib:.* 33 | serial: 34 | - command: lint 35 | - command: test 36 | test-lint-then-test: 37 | match: watch-test:.* 38 | parallel: 39 | - command: lint 40 | - command: test 41 | web-lint: 42 | match: watch-web:.* 43 | serial: 44 | - wait: 1000 45 | - command: lint 46 | 47 | dashboards: 48 | all-in-one: 49 | - log: 50 | position: quadrant/left 51 | title: Lint 52 | filter: command:lint:.* 53 | gauge: 54 | filter: command:lint:state:.* 55 | scan: 56 | when: 57 | - pattern: started 58 | text: Running 59 | level: warning 60 | - pattern: exited with 0 61 | text: Passed 62 | level: success 63 | - pattern: exited with 64 | text: Failed 65 | level: error 66 | default: 67 | text: Unknown 68 | - log: 69 | position: quadrant/right 70 | title: Test 71 | filter: command:test:.* 72 | gauge: 73 | filter: command:test:state:.* 74 | scan: 75 | when: 76 | - pattern: started 77 | text: Running 78 | level: warning 79 | - pattern: exited with 0 80 | text: Passed 81 | level: success 82 | - pattern: exited with 83 | text: Failed 84 | level: error 85 | default: 86 | text: Unknown 87 | lint-only: 88 | - log: 89 | position: fullscreen 90 | title: Lint 91 | filter: command:lint:.* 92 | gauge: 93 | filter: command:lint:state:.* 94 | scan: 95 | when: 96 | - pattern: started 97 | text: Running 98 | level: warning 99 | - pattern: exited with 0 100 | text: Passed 101 | level: success 102 | - pattern: exited with 103 | text: Failed 104 | level: error 105 | default: 106 | text: Unknown 107 | test-only: 108 | - log: 109 | position: fullscreen 110 | title: Test 111 | filter: command:test:.* 112 | gauge: 113 | filter: command:test:state:.* 114 | scan: 115 | when: 116 | - pattern: started 117 | text: Running 118 | level: warning 119 | - pattern: exited with 0 120 | text: Passed 121 | level: success 122 | - pattern: exited with 123 | text: Failed 124 | level: error 125 | default: 126 | text: Unknown 127 | -------------------------------------------------------------------------------- /pipeline-master.yml: -------------------------------------------------------------------------------- 1 | resource_types: 2 | - name: slack-notification 3 | type: docker-image 4 | source: 5 | repository: cfcommunity/slack-notification-resource 6 | 7 | resources: 8 | - name: dashflow 9 | type: git 10 | source: 11 | uri: https://github.com/freewheel/dashflow.git 12 | branch: master 13 | 14 | - name: slack 15 | type: slack-notification 16 | source: 17 | url: ((slack-webhook-url)) 18 | 19 | jobs: 20 | - name: lint 21 | plan: 22 | - get: dashflow 23 | trigger: true 24 | 25 | - task: run-lint 26 | config: 27 | platform: linux 28 | image_resource: 29 | type: docker-image 30 | source: { repository: node, tag: 11 } 31 | inputs: 32 | - name: dashflow 33 | run: 34 | path: sh 35 | args: 36 | - -exc 37 | - | 38 | cd dashflow 39 | yarn install --network-concurrency 1 40 | yarn --silent lint 41 | on_failure: 42 | put: slack 43 | params: 44 | text: | 45 | Lint failed for dashflow master branch(${ATC_EXTERNAL_URL}/builds/${BUILD_ID}), please investigate. 46 | 47 | - name: test 48 | plan: 49 | - get: dashflow 50 | trigger: true 51 | 52 | - task: run-test 53 | config: 54 | platform: linux 55 | image_resource: 56 | type: docker-image 57 | source: { repository: node, tag: 11 } 58 | inputs: 59 | - name: dashflow 60 | run: 61 | path: sh 62 | args: 63 | - -exc 64 | - | 65 | cd dashflow 66 | yarn install --network-concurrency 1 67 | yarn --silent test --no-color --verbose 68 | on_failure: 69 | put: slack 70 | params: 71 | text: | 72 | Test failed for dashflow master branch(${ATC_EXTERNAL_URL}/builds/${BUILD_ID}), please investigate. 73 | 74 | - name: publish-npm-package 75 | plan: 76 | - get: dashflow 77 | trigger: true 78 | passed: 79 | - lint 80 | - test 81 | 82 | - task: publish-dev 83 | config: 84 | platform: linux 85 | image_resource: 86 | type: docker-image 87 | source: { repository: node, tag: 11 } 88 | inputs: 89 | - name: dashflow 90 | params: 91 | NPM_PUBLISH_TOKEN: ((npm-publish-token)) 92 | run: 93 | path: sh 94 | args: 95 | - -ec 96 | - | 97 | NPM_CURRENT_VERSION=$(npm view dashflow dist-tags.latest) 98 | CODE_VERSION=$(cat dashflow/package.json | grep version | awk 'BEGIN { FS = "\"" }; { print $4 }') 99 | 100 | if ! [ "$CODE_VERSION" = "$NPM_CURRENT_VERSION" ]; then 101 | cd dashflow 102 | yarn install --network-concurrency 1 103 | echo "publishing dashflow package to npm" 104 | echo "//registry.npmjs.org/:_authToken=${NPM_PUBLISH_TOKEN}" > .npmrc 105 | npm publish 106 | else 107 | echo "skip publishing since the version is the same as in npm registry" 108 | fi 109 | on_failure: 110 | put: slack 111 | params: 112 | text: | 113 | Publish failed for dashflow(${ATC_EXTERNAL_URL}/builds/${BUILD_ID}), please investigate. 114 | -------------------------------------------------------------------------------- /lib/cli.js: -------------------------------------------------------------------------------- 1 | const boxen = require("boxen"); 2 | const chalk = require("chalk"); 3 | const path = require("path"); 4 | const fs = require("fs"); 5 | const { startDaemon } = require("./daemon"); 6 | const { serve } = require("./server"); 7 | const { createShell } = require("./interactive_shell"); 8 | const { parseMulti } = require("./config"); 9 | const { executeCommand } = require("./command"); 10 | 11 | const STARTING_PORT = 9527; 12 | 13 | function normalizeConfig(configs) { 14 | return configs.map(file => path.resolve(".", file)); 15 | } 16 | 17 | function checkExistence(configs) { 18 | configs.forEach(file => { 19 | const hasConfig = fs.existsSync(file); 20 | 21 | if (!hasConfig) { 22 | process.stderr.write( 23 | chalk.red(`Cannot find configuration file ${file}.\n`) 24 | ); 25 | 26 | process.exit(1); 27 | } 28 | }); 29 | } 30 | 31 | function printServerAddress(port) { 32 | const serverAddress = `http://localhost:${port}`; 33 | 34 | process.stdout.write( 35 | boxen( 36 | `${chalk.green("HTTP server is serving!")}` + 37 | "\n\n" + 38 | `Visit ${chalk.bold(serverAddress)} to view dashboards.`, 39 | { padding: 1, borderColor: "green" } 40 | ) + "\n\n" 41 | ); 42 | } 43 | 44 | function setTerminalTitle(title) { 45 | process.stdout.write( 46 | String.fromCharCode(27) + "]0;" + title + String.fromCharCode(7) 47 | ); 48 | } 49 | 50 | async function startOneOff(config, commands) { 51 | const supportedCommands = Object.keys(config.commands); 52 | const unknownCommands = commands.filter( 53 | command => !supportedCommands.includes(command) 54 | ); 55 | 56 | if (unknownCommands.length > 0) { 57 | throw new Error( 58 | `some commands are not found in configuration: ${commands.join(", ")}` 59 | ); 60 | } 61 | 62 | for (const command of commands) { 63 | // execute command 64 | setTerminalTitle(command); 65 | 66 | try { 67 | await executeCommand( 68 | command, 69 | config.commands, 70 | (type, data) => { 71 | if (type === "data") { 72 | process.stdout.write(data); 73 | } 74 | }, 75 | true 76 | ); 77 | } catch (exitCode) { 78 | process.exit(exitCode); 79 | } 80 | } 81 | } 82 | 83 | function findPort() { 84 | return new Promise((resolve, reject) => { 85 | require("portfinder").getPort({ port: STARTING_PORT }, function(err, port) { 86 | if (err) { 87 | reject(err); 88 | } else { 89 | resolve(port); 90 | } 91 | }); 92 | }); 93 | } 94 | 95 | function defaultOnServerError(err) { 96 | process.stderr.write("Quit due to Web Server Error: " + err.message + "\n"); 97 | 98 | process.exit(1); 99 | } 100 | 101 | async function start( 102 | configs, 103 | port = null, 104 | commands = [], 105 | onServerError = defaultOnServerError 106 | ) { 107 | checkExistence(configs); 108 | 109 | const configFiles = normalizeConfig(configs); 110 | 111 | const config = parseMulti(configFiles); 112 | 113 | if (commands.length > 0) { 114 | // one off mode 115 | await startOneOff(config, commands); 116 | } else { 117 | // daemon mode 118 | const app = startDaemon(config); 119 | // serve HTTP and websocket 120 | const finalPort = port || (await findPort()); 121 | 122 | serve(app, finalPort, onServerError); 123 | // do not print for debug mode 124 | if (process.env.DEBUG) { 125 | process.stdout.write("Debugging, skipping interactive shell..\n"); 126 | } else { 127 | printServerAddress(finalPort); 128 | createShell(app); 129 | } 130 | } 131 | } 132 | 133 | module.exports = { start }; 134 | -------------------------------------------------------------------------------- /examples/web-dev.dashflow.yml: -------------------------------------------------------------------------------- 1 | commands: 2 | lint: 3 | shell: 4 | cwd: "./code" 5 | cmd: "yarn --silent lint_min" 6 | test: 7 | shell: 8 | cwd: "./code" 9 | cmd: "yarn --silent test_min" 10 | 11 | streams: 12 | webpack: 13 | shell: 14 | cwd: "./code" 15 | cmd: "yarn --silent start" 16 | rails-dev: 17 | shell: 18 | cwd: "../.." 19 | cmd: 'rails s' 20 | watch-code: 21 | watch: 22 | cwd: "./code" 23 | glob: "**/*" 24 | file-sync: 25 | shell: 26 | cwd: "../.." 27 | cmd: "tail" 28 | 29 | workflows: 30 | restart-file-sync: 31 | match: "^stream:file-sync:.*:state:exited" 32 | restart: file-sync 33 | 34 | lint-then-test: 35 | match: "^stream:webpack:.*Webpack build success" 36 | parallel: 37 | - command: lint 38 | - command: test 39 | 40 | dashboards: 41 | all-in-one: 42 | - log: 43 | title: Webpack 44 | position: quadrant/top-left 45 | filter: "^stream:webpack:.*" 46 | gauge: 47 | filter: "^stream:webpack:.*" scan: 48 | when: 49 | - pattern: "Webpack. Starting" 50 | text: Running 51 | level: warning 52 | - pattern: "Webpack build success" 53 | text: Passed 54 | level: success 55 | - pattern: "Webpack build error" 56 | text: Failed 57 | level: error 58 | - pattern: "Failed to compile" 59 | text: Failed 60 | level: error 61 | default: 62 | text: Unknown 63 | - log: 64 | title: Lint 65 | position: 0 50% 25% 100% 66 | filter: "command:lint:stdout:.*" 67 | gauge: 68 | filter: "command:lint:state:.*" 69 | scan: 70 | when: 71 | - pattern: started 72 | text: Running 73 | level: warning 74 | - pattern: exited with 0 75 | text: Passed 76 | level: success 77 | - pattern: exited with 78 | text: Failed 79 | level: error 80 | default: 81 | text: Unknown 82 | - log: 83 | title: Test 84 | position: 25% 50% 50% 100% 85 | filter: "command:test:stdout:.*" 86 | gauge: 87 | filter: "command:test:state:.*" 88 | scan: 89 | when: 90 | - pattern: started 91 | text: Running 92 | level: warning 93 | - pattern: exited with 0 94 | text: Passed 95 | level: success 96 | - pattern: exited with 97 | text: Failed 98 | level: error 99 | default: 100 | text: Unknown 101 | - log: 102 | title: Rails Log 103 | position: 50% 0 100% 80% 104 | filter: "^stream:rails-dev:.*" 105 | - log: 106 | title: File Sync Log 107 | position: 50% 80% 100% 100% 108 | filter: "^stream:file-sync:.*" 109 | 110 | frontend: 111 | - log: 112 | title: Webpack 113 | position: fullscreen 114 | filter: "^stream:webpack:.*" 115 | 116 | lint: 117 | - log: 118 | title: Lint 119 | position: fullscreen 120 | filter: "command:lint:stdout:.*" 121 | 122 | test: 123 | - log: 124 | title: Test 125 | position: fullscreen 126 | filter: "command:test:stdout:.*" 127 | 128 | rails: 129 | - log: 130 | title: Rails Log 131 | position: fullscreen 132 | filter: "^stream:rails-dev:.*" 133 | 134 | file-sync: 135 | - log: 136 | title: File Sync Log 137 | position: fullscreen 138 | filter: "^stream:file-sync:.*" 139 | -------------------------------------------------------------------------------- /lib/workflow.js: -------------------------------------------------------------------------------- 1 | const { brief } = require("./util"); 2 | const { restartFormatter, commandFormatter } = require("./formatter"); 3 | const { executeCommand } = require("./command"); 4 | const debug = require("debug"); 5 | const debugWorkflow = workflowId => debug(`workflow:${workflowId}`); 6 | 7 | const PROVIDERS = { 8 | command: function execCommand(id, commandName, commands, onEvent) { 9 | debugWorkflow(id)("run command", commandName); 10 | 11 | onEvent(commandFormatter.format(commandName, "state", "started")); 12 | 13 | return executeCommand( 14 | commandName, 15 | commands, 16 | (type, data) => { 17 | if (type === "data") { 18 | onEvent(commandFormatter.format(commandName, "stdout", data)); 19 | } else if (type === "exit") { 20 | onEvent( 21 | commandFormatter.format(commandName, "state", `exited with ${data}`) 22 | ); 23 | } 24 | }, 25 | false 26 | ); 27 | }, 28 | restart: function execRestart( 29 | id, 30 | streamId, 31 | commands, 32 | onEvent, 33 | restartStream 34 | ) { 35 | return new Promise(function(resolve) { 36 | debugWorkflow(id)("run restart", streamId); 37 | 38 | restartStream(streamId); 39 | 40 | onEvent(restartFormatter.format(streamId)); 41 | 42 | resolve(); 43 | }); 44 | }, 45 | wait: function execWait(id, timeInMs) { 46 | debugWorkflow(id)("start wait", `${timeInMs}ms`); 47 | 48 | return new Promise(function(resolve) { 49 | setTimeout(() => { 50 | debugWorkflow(id)("done wait", `${timeInMs}ms`); 51 | 52 | resolve(); 53 | }, timeInMs); 54 | }); 55 | }, 56 | parallel: function execParallel( 57 | id, 58 | subTasks, 59 | commands, 60 | onEvent, 61 | restartStream 62 | ) { 63 | return Promise.all( 64 | subTasks.map(task => { 65 | const type = Object.keys(task)[0]; 66 | const spec = task[type]; 67 | 68 | return executeTask(id, type, spec, commands, onEvent, restartStream); 69 | }) 70 | ); 71 | }, 72 | serial: async function execSerial( 73 | id, 74 | subTasks, 75 | commands, 76 | onEvent, 77 | restartStream 78 | ) { 79 | for (const task of subTasks) { 80 | const type = Object.keys(task)[0]; 81 | const spec = task[type]; 82 | 83 | await executeTask(id, type, spec, commands, onEvent, restartStream); 84 | } 85 | }, 86 | }; 87 | 88 | function executeTask(id, type, spec, commands, onEvent, restartStream) { 89 | return PROVIDERS[type](id, spec, commands, onEvent, restartStream); 90 | } 91 | 92 | function createWorkflow(id, spec, commands, onEvent, restartStream) { 93 | const match = spec.match; 94 | 95 | const types = Object.keys(spec); 96 | 97 | if (types.length !== 2) { 98 | throw new Error(`Expect to have one single type for workflow ${id}`); 99 | } 100 | 101 | const type = types[0] === "match" ? types[1] : types[0]; 102 | 103 | if (!PROVIDERS[type]) { 104 | throw new Error(`Unsupported type ${type} for workflow ${id}`); 105 | } 106 | 107 | debugWorkflow(id)("match", match, "type", type); 108 | 109 | const matchPattern = new RegExp(match); 110 | 111 | return event => { 112 | debugWorkflow(id)("received an event", event, match); 113 | 114 | if (matchPattern.test(event)) { 115 | debugWorkflow(id)("found a match", brief(event)); 116 | 117 | // const filename = R.last(event.split(":")); 118 | 119 | setTimeout(() => { 120 | executeTask( 121 | id, 122 | type, 123 | spec[type], 124 | commands, 125 | onEvent, 126 | restartStream 127 | ).catch(() => { 128 | // do nothing for workflow errors 129 | }); 130 | }, 0); 131 | } 132 | }; 133 | } 134 | 135 | module.exports = { 136 | createWorkflow, 137 | }; 138 | -------------------------------------------------------------------------------- /lib/shell.js: -------------------------------------------------------------------------------- 1 | const { spawn } = require("node-pty"); 2 | const { brief, CONTROL_CHARACTERS } = require("./util"); 3 | const debug = require("debug"); 4 | const debugShell = debug("dashflow:shell"); 5 | 6 | function shell(cwd, command, on) { 7 | debugShell(`run shell command ${command} at ${cwd}`); 8 | 9 | let streamAlive = true; 10 | let exitCode = null; 11 | 12 | const stream = spawn("sh", ["-c", command], { cwd }); 13 | 14 | stream.on("data", chunk => { 15 | const content = chunk.toString("utf-8"); 16 | 17 | debugShell("shell data", command, brief(content)); 18 | 19 | on("data", content); 20 | }); 21 | 22 | stream.on("exit", (code, signal) => { 23 | streamAlive = false; 24 | exitCode = code; 25 | 26 | debugShell("exit", command, code, signal); 27 | 28 | on("exit", code); 29 | }); 30 | 31 | return { 32 | stream, 33 | terminate: function terminate(onErr) { 34 | if (streamAlive) { 35 | try { 36 | // PID range hack 37 | // https://azimi.me/2014/12/31/kill-child_process-node-js.html 38 | process.kill(process.platform === "win32" ? stream.pid : -stream.pid); 39 | } catch (err) { 40 | debugShell("terminate error", err.message); 41 | 42 | if (onErr) { 43 | onErr(err); 44 | } 45 | } 46 | } 47 | }, 48 | promise: new Promise(function(resolve, reject) { 49 | const intervalId = setInterval(function checkAlive() { 50 | debugShell("check alive"); 51 | 52 | if (!streamAlive) { 53 | clearInterval(intervalId); 54 | 55 | if (exitCode === 0) { 56 | resolve(); 57 | } else { 58 | reject(exitCode); 59 | } 60 | } 61 | }, 500); 62 | }), 63 | attachOneOff: function(parent) { 64 | function onUserInput(chunk) { 65 | stream.write(chunk); 66 | } 67 | 68 | function onDetach() { 69 | stream.removeListener("exit", onDetach); 70 | 71 | parent.stdin.removeListener("data", onUserInput); 72 | parent.stdin.unref(); 73 | parent.stdin.setRawMode(false); 74 | } 75 | 76 | if (streamAlive) { 77 | parent.stdin.setRawMode(true); 78 | parent.stdin.on("data", onUserInput); 79 | 80 | stream.on("exit", onDetach); 81 | stream.resize(parent.stdout.columns, parent.stdout.rows); 82 | } 83 | 84 | return onDetach; 85 | }, 86 | attach: function attach(parent) { 87 | return new Promise(function(resolve, reject) { 88 | function onStreamOut(chunk) { 89 | const content = chunk.toString("utf-8"); 90 | 91 | parent.stdout.write(content); 92 | } 93 | 94 | function onUserInput(chunk) { 95 | if (chunk.equals(CONTROL_CHARACTERS.ETX)) { 96 | // ctrl+c 97 | onDetach(); 98 | } else { 99 | stream.write(chunk); 100 | } 101 | } 102 | 103 | function onDetach() { 104 | stream.removeListener("data", onStreamOut); 105 | stream.removeListener("exit", onDetach); 106 | 107 | parent.stdin.removeListener("data", onUserInput); 108 | parent.stdin.unref(); 109 | parent.stdin.setRawMode(false); 110 | 111 | resolve(); 112 | } 113 | 114 | if (streamAlive) { 115 | parent.stdin.setRawMode(true); 116 | parent.stdin.on("data", onUserInput); 117 | 118 | stream.on("data", onStreamOut); 119 | stream.on("exit", onDetach); 120 | stream.resize(parent.stdout.columns, parent.stdout.rows); 121 | } else { 122 | reject( 123 | new Error( 124 | "Cannot attach to stream since the child process had quit" 125 | ) 126 | ); 127 | } 128 | }); 129 | }, 130 | }; 131 | } 132 | 133 | module.exports = shell; 134 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const path = require("path"); 3 | const yaml = require("js-yaml"); 4 | 5 | function readYAML(path) { 6 | const content = fs.readFileSync(path, "utf8"); 7 | 8 | return yaml.safeLoad(content); 9 | } 10 | 11 | function normalizeCommands(commands, cwd) { 12 | if (commands) { 13 | for (const command in commands) { 14 | const spec = commands[command]; 15 | 16 | if (spec === null) { 17 | throw new Error( 18 | `please check configuration of command "${command}", cannot be null` 19 | ); 20 | } else if (typeof spec === "string") { 21 | commands[command] = { 22 | shell: { 23 | cmd: spec, 24 | cwd, 25 | }, 26 | }; 27 | } else { 28 | if (spec.shell) { 29 | spec.shell = { 30 | cmd: spec.shell.cmd, 31 | cwd: path.resolve(cwd, spec.shell.cwd || "."), 32 | }; 33 | } 34 | } 35 | } 36 | 37 | return commands; 38 | } else { 39 | return {}; 40 | } 41 | } 42 | 43 | function normalizeStreams(streams, cwd) { 44 | if (streams) { 45 | for (const stream in streams) { 46 | const spec = streams[stream]; 47 | 48 | if (spec === null) { 49 | throw new Error( 50 | `please check configuration of stream "${stream}", cannot be null` 51 | ); 52 | } else if (typeof spec === "string") { 53 | streams[stream] = { 54 | shell: { 55 | cmd: spec, 56 | cwd, 57 | }, 58 | }; 59 | } else { 60 | if (spec.shell) { 61 | streams[stream] = { 62 | shell: { 63 | cmd: spec.shell.cmd, 64 | cwd: path.resolve(cwd, spec.shell.cwd || "."), 65 | }, 66 | }; 67 | } else if (spec.watch) { 68 | streams[stream] = { 69 | watch: { 70 | glob: spec.watch.glob || "**/*.js", 71 | cwd: path.resolve(cwd, spec.watch.cwd || "."), 72 | ignore: spec.watch.ignore || /node_modules/, 73 | }, 74 | }; 75 | } 76 | } 77 | } 78 | 79 | return streams; 80 | } else { 81 | return {}; 82 | } 83 | } 84 | 85 | function normalizeWorkflows(workflows) { 86 | if (workflows) { 87 | for (const workflow in workflows) { 88 | const spec = workflows[workflow]; 89 | 90 | if (spec === null) { 91 | throw new Error( 92 | `please check configuration of workflow "${workflow}", cannot be null` 93 | ); 94 | } else if (!workflow.match) { 95 | throw new Error( 96 | `please check configuration of workflow "${workflow}", match key must be present` 97 | ); 98 | } 99 | } 100 | 101 | return workflows; 102 | } else { 103 | return {}; 104 | } 105 | } 106 | 107 | function normalizeDashboards(dashboards) { 108 | if (dashboards) { 109 | for (const dashboard in dashboards) { 110 | const spec = dashboards[dashboard]; 111 | 112 | if (!Array.isArray(spec)) { 113 | throw new Error( 114 | `please check configuration of dashboard "${dashboard}", must be an array` 115 | ); 116 | } 117 | } 118 | 119 | return dashboards; 120 | } else { 121 | return {}; 122 | } 123 | } 124 | 125 | function parse(configFilePath) { 126 | const cwd = path.dirname(configFilePath); 127 | const config = readYAML(configFilePath); 128 | 129 | config.commands = normalizeCommands(config.commands, cwd); 130 | config.streams = normalizeStreams(config.streams, cwd); 131 | config.workflows = normalizeWorkflows(config.workflows); 132 | config.dashboards = normalizeDashboards(config.dashboards); 133 | 134 | return config; 135 | } 136 | 137 | function merge(config1, config2) { 138 | return { 139 | commands: Object.assign({}, config1.commands, config2.commands), 140 | streams: Object.assign({}, config1.streams, config2.streams), 141 | workflows: Object.assign({}, config1.workflows, config2.workflows), 142 | dashboards: Object.assign({}, config1.dashboards, config2.dashboards), 143 | }; 144 | } 145 | 146 | function parseMulti(arrayOfConfigFilePaths) { 147 | return arrayOfConfigFilePaths.reduce((acc, configFilePath) => { 148 | return merge(acc, parse(configFilePath)); 149 | }, {}); 150 | } 151 | 152 | module.exports = { 153 | parse, 154 | parseMulti, 155 | }; 156 | -------------------------------------------------------------------------------- /web/lib/react.production.min.js: -------------------------------------------------------------------------------- 1 | /** @license React v16.4.1 2 | * react.production.min.js 3 | * 4 | * Copyright (c) 2013-present, Facebook, Inc. 5 | * 6 | * This source code is licensed under the MIT license found in the 7 | * LICENSE file in the root directory of this source tree. 8 | */ 9 | 'use strict';(function(p,h){"object"===typeof exports&&"undefined"!==typeof module?module.exports=h():"function"===typeof define&&define.amd?define(h):p.React=h()})(this,function(){function p(a){for(var b=arguments.length-1,f="https://reactjs.org/docs/error-decoder.html?invariant="+a,d=0;du.length&&u.push(a)}function t(a,b,f,d){var e=typeof a;if("undefined"===e||"boolean"===e)a=null;var k=!1;if(null===a)k=!0;else switch(e){case "string":case "number":k=!0;break;case "object":switch(a.$$typeof){case r:case Q:k=!0}}if(k)return f(d,a,""===b?"."+y(a,0):b),1;k=0;b=""===b?".":b+":";if(Array.isArray(a))for(var c=0;ca;a++)b["_"+String.fromCharCode(a)]=a;if("0123456789"!==Object.getOwnPropertyNames(b).map(function(a){return b[a]}).join(""))return!1;var f={};"abcdefghijklmnopqrst".split("").forEach(function(a){f[a]=a});return"abcdefghijklmnopqrst"!==Object.keys(Object.assign({},f)).join("")?!1:!0}catch(d){return!1}}()?Object.assign:function(a,b){if(null===a||void 0===a)throw new TypeError("Object.assign cannot be called with null or undefined");var f=Object(a);for(var d,e=1;e { 9 | const { streamId } = args; 10 | const stream = app.streams[streamId]; 11 | 12 | if (!stream) { 13 | process.stdout.write(chalk.red(`Cannot find stream "${streamId}"\n`)); 14 | 15 | callback(); 16 | } else { 17 | handle(streamId, callback); 18 | } 19 | }; 20 | } 21 | 22 | async function withIOStack(callback) { 23 | const sigIntListeners = process.listeners("SIGINT"); 24 | 25 | sigIntListeners.forEach(listener => { 26 | process.removeListener("SIGINT", listener); 27 | }); 28 | 29 | const dataListeners = process.stdin.listeners("data"); 30 | 31 | dataListeners.forEach(listener => { 32 | process.stdin.removeListener("data", listener); 33 | }); 34 | 35 | process.stdout.write(ansiEscapes.clearScreen); 36 | 37 | process.stdin.resume(); 38 | 39 | try { 40 | await callback(); 41 | 42 | process.stdout.write(ansiEscapes.clearScreen); 43 | } catch (err) { 44 | process.stderr.write(err.message + "\n"); 45 | process.stderr.write(err.stack + "\n"); 46 | } 47 | 48 | sigIntListeners.forEach(listener => { 49 | process.on("SIGINT", listener); 50 | }); 51 | 52 | dataListeners.forEach(listener => { 53 | process.stdin.on("data", listener); 54 | }); 55 | } 56 | 57 | function createShell(app) { 58 | const shell = vorpal(); 59 | 60 | shell.delimiter("dashflow-shell~$"); 61 | 62 | shell 63 | .command("events [pattern]") 64 | .option("-l, --limit", "Show only specified number of records.") 65 | .description("Dump events") 66 | .action(function(args, done) { 67 | const columnify = require("columnify"); 68 | const { scan } = require("./events"); 69 | const { partition, stripAnsi } = require("./util"); 70 | const pattern = new RegExp(args.pattern || ".*"); 71 | const limit = parseInt((args.options && args.options.limit) || "100"); 72 | 73 | const matchingEvents = scan(pattern, limit, app.events).reverse(); 74 | const total = matchingEvents.length; 75 | 76 | this.log( 77 | columnify( 78 | matchingEvents.map(({ time, event }) => { 79 | const [type, typeRemain] = partition(stripAnsi(event)); 80 | 81 | const [id, idRemain] = partition(typeRemain); 82 | 83 | const [provider, providerRemain] = partition(idRemain); 84 | 85 | const [meta1, meta1Remain] = partition(providerRemain); 86 | 87 | const [meta2, meta2Remain] = partition(meta1Remain); 88 | 89 | const [meta3, meta4] = partition(meta2Remain); 90 | 91 | return { 92 | time, 93 | type, 94 | id, 95 | provider, 96 | meta1, 97 | meta2, 98 | meta3, 99 | meta4, 100 | }; 101 | }) 102 | ) 103 | ); 104 | 105 | if (limit < total) { 106 | this.log( 107 | `\nShowing last ${limit} of total ${total} events. Use --limit option if you would like to see more.` 108 | ); 109 | } 110 | 111 | done(); 112 | }); 113 | 114 | shell 115 | .command("emit ") 116 | .description("Emit a new event") 117 | .action(function(args, done) { 118 | app.addEvent(args.event); 119 | 120 | done(); 121 | }); 122 | 123 | shell 124 | .command("run ") 125 | .description("Run a command") 126 | .autocomplete(Object.keys(app.config.commands)) 127 | .action(function(args, done) { 128 | const supportedCommands = Object.keys(app.config.commands); 129 | const command = args.command; 130 | 131 | if (supportedCommands.includes(command)) { 132 | executeCommand( 133 | command, 134 | app.config.commands, 135 | (type, data) => { 136 | if (type === "data") { 137 | process.stdout.write(data); 138 | } 139 | }, 140 | true 141 | ) 142 | .then(() => { 143 | done(); 144 | }) 145 | .catch(() => { 146 | // swallow errors 147 | done(); 148 | }); 149 | } else { 150 | process.stderr.write( 151 | `unknown command ${command}, supported commands are:\n - ${supportedCommands.join( 152 | "\n - " 153 | )}\n` 154 | ); 155 | done(); 156 | } 157 | }); 158 | 159 | shell 160 | .command("tail ") 161 | .description("Tail to a stream or workflow") 162 | .alias("t") 163 | .autocomplete(app.getStreamIds().concat(app.getWorkflowIds())) 164 | .action(function(args, done) { 165 | const LINES_TO_RECALL = 100; 166 | const { scan } = require("./events"); 167 | 168 | const eventsRecalled = scan( 169 | new RegExp(`(stream|workflow):${args.streamOrWorkflowId}:`), 170 | LINES_TO_RECALL, 171 | app.events 172 | ); 173 | 174 | eventsRecalled.reverse().forEach(({ event }) => { 175 | const readable = smartUnformat(event); 176 | 177 | if (readable) { 178 | process.stdout.write(readable); 179 | } 180 | }); 181 | 182 | process.stdout.write("\n"); 183 | 184 | done(); 185 | }); 186 | 187 | shell 188 | .command("attach ") 189 | .description("Attach to a stream") 190 | .alias("a") 191 | .autocomplete(app.getStreamIds()) 192 | .action( 193 | guardStream(app, async (streamId, done) => { 194 | try { 195 | await withIOStack(() => { 196 | const LINES_TO_RECALL = 100; 197 | const { scan } = require("./events"); 198 | 199 | const eventsRecalled = scan( 200 | new RegExp(`^stream:${streamId}:`), 201 | LINES_TO_RECALL, 202 | app.events 203 | ); 204 | 205 | eventsRecalled.reverse().forEach(({ event }) => { 206 | const readable = smartUnformat(event); 207 | 208 | if (readable) { 209 | process.stdout.write(readable); 210 | } 211 | }); 212 | 213 | return app.attachToStream(streamId, process); 214 | }); 215 | 216 | done(); 217 | } catch (error) { 218 | process.stdout.write(chalk.red(error.message) + "\n"); 219 | 220 | done(); 221 | } 222 | }) 223 | ); 224 | 225 | shell 226 | .command("restart ") 227 | .description("Restart a stream") 228 | .autocomplete(app.getStreamIds()) 229 | .action( 230 | guardStream(app, (streamId, done) => { 231 | app.restartStream(streamId); 232 | 233 | done(); 234 | }) 235 | ); 236 | 237 | shell 238 | .command("stats") 239 | .description("Print stats information") 240 | .action(function(args, done) { 241 | const columnify = require("columnify"); 242 | const { length } = require("./events"); 243 | 244 | this.log( 245 | columnify({ 246 | "Number of Events": length(app.events), 247 | "Number of Streams": Object.keys(app.streams).length, 248 | }) 249 | ); 250 | 251 | done(); 252 | }); 253 | 254 | shell 255 | .command("config") 256 | .description("Print config information") 257 | .action(function(args, done) { 258 | const yaml = require("js-yaml"); 259 | 260 | this.log(yaml.safeDump(app.config, { flowLevel: 2 })); 261 | 262 | done(); 263 | }); 264 | 265 | shell 266 | .command("env") 267 | .description("Print environment variables") 268 | .action(function(args, done) { 269 | const columnify = require("columnify"); 270 | 271 | this.log(columnify(process.env)); 272 | 273 | done(); 274 | }); 275 | 276 | process.stdout.write( 277 | `Besides using browser the alternative way is using dashflow shell.\n` + 278 | `Type ${chalk.bold("help")} in the shell to list available commands.\n\n` 279 | ); 280 | 281 | shell.show(); 282 | } 283 | 284 | module.exports = { createShell }; 285 | -------------------------------------------------------------------------------- /web/styles.css: -------------------------------------------------------------------------------- 1 | /* GENERAL */ 2 | 3 | html, body { 4 | margin: 0 auto; 5 | top: 0; 6 | font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue" 7 | } 8 | 9 | /* vietnamese */ 10 | @font-face { 11 | font-family: 'Inconsolata'; 12 | font-style: normal; 13 | font-weight: 400; 14 | src: local('Inconsolata Regular'), local('Inconsolata-Regular'), url(lib/Inconsolata-Regular.woff2) format('woff2'); 15 | unicode-range: U+0102-0103, U+0110-0111, U+1EA0-1EF9, U+20AB; 16 | } 17 | /* latin-ext */ 18 | @font-face { 19 | font-family: 'Inconsolata'; 20 | font-style: normal; 21 | font-weight: 400; 22 | src: local('Inconsolata Regular'), local('Inconsolata-Regular'), url(lib/Inconsolata-Regular.woff2) format('woff2'); 23 | unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; 24 | } 25 | /* latin */ 26 | @font-face { 27 | font-family: 'Inconsolata'; 28 | font-style: normal; 29 | font-weight: 400; 30 | src: local('Inconsolata Regular'), local('Inconsolata-Regular'), url(lib/Inconsolata-Regular.woff2) format('woff2'); 31 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; 32 | } 33 | 34 | @font-face { 35 | font-family: 'Source Code Pro'; 36 | font-style: normal; 37 | font-weight: 400; 38 | src: local('SourceCodePro Light'), local('SourceCodePro-Light'), url(lib/SourceCodePro-Light.ttf) format('ttf'); 39 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; 40 | } 41 | 42 | ::-webkit-scrollbar { 43 | width: 4px; 44 | } 45 | 46 | ::-webkit-scrollbar-track { 47 | background: #DDD; 48 | } 49 | 50 | ::-webkit-scrollbar-thumb { 51 | background: #666; 52 | } 53 | 54 | /* APP */ 55 | 56 | .app { 57 | height: 100%; 58 | width: 100%; 59 | } 60 | 61 | /* HEADER */ 62 | 63 | .dash-header { 64 | height: 3em; 65 | width: 100%; 66 | display: flex; 67 | } 68 | 69 | /* HEADER LEFT */ 70 | 71 | .dash-header .header-left { 72 | width: 50%; 73 | height: 100%; 74 | display: flex; 75 | } 76 | 77 | .dash-header .header-left .logo { 78 | height: 36px; 79 | margin: 8px; 80 | } 81 | 82 | .dash-header .header-left .dropdown { 83 | position: relative; 84 | display: inline-block; 85 | cursor: pointer; 86 | height: 30px; 87 | margin: 9px; 88 | width: 160px; 89 | border: 2px solid #FFF0; 90 | border-bottom: none; 91 | border-radius: 3px; 92 | font-weight: bold; 93 | } 94 | 95 | .dash-header .header-left .dropdown:hover { 96 | border: 2px solid #BBB; 97 | } 98 | 99 | .dash-header .header-left .dropdown .item { 100 | padding: 6px 14px; 101 | color: #2b5fb4; 102 | } 103 | 104 | div.arrow { 105 | border: solid #DDD; 106 | border-width: 0 3px 3px 0; 107 | display: inline-block; 108 | padding: 3px; 109 | } 110 | 111 | div.arrow.down { 112 | transform: rotate(45deg); 113 | -webkit-transform: rotate(45deg); 114 | } 115 | 116 | .dash-header .header-left .dropdown .item.active-item div.arrow { 117 | margin: 0 0 3px 10px; 118 | -webkit-transition: all .6s; 119 | opacity: 1; 120 | } 121 | 122 | .dash-header .header-left .dropdown:hover .item.active-item div.arrow { 123 | opacity: 0; 124 | margin-bottom: -2px; 125 | } 126 | 127 | .dash-header .header-left .dropdown ul.inactive-items { 128 | display: none; 129 | position: absolute; 130 | background-color: #FFF; 131 | border: 2px solid #BBB; 132 | border-radius: 0 0 3px 3px; 133 | border-top: none; 134 | width: 160px; 135 | box-shadow: 0px 8px 8px 0px rgba(182, 128, 128, 0.2); 136 | z-index: 1; 137 | list-style-type: none; 138 | margin: -3px -2px; 139 | padding: 0; 140 | } 141 | 142 | .dash-header .header-left .dropdown ul.inactive-items li.item:hover { 143 | color: #1c3f78; 144 | background: #DDD; 145 | } 146 | 147 | .dash-header .header-left .dropdown ul.inactive-items li.item.inactive { 148 | color: #DDD; 149 | } 150 | 151 | .dash-header .header-left .dropdown ul.inactive-items li.item.inactive:hover { 152 | color: #DDD; 153 | background: #FFF; 154 | cursor: default; 155 | } 156 | 157 | .dash-header .header-left .dropdown:hover .inactive-items { 158 | display: block; 159 | } 160 | 161 | /* HEADER RIGHT */ 162 | 163 | .dash-header .header-right { 164 | width: 50%; 165 | height: 100%; 166 | } 167 | 168 | .dash-header .header-right .filter { 169 | float:right; 170 | outline: 0px; 171 | width: 300px; 172 | height: 6px; 173 | background: #DDD; 174 | margin: 34px 20px 0 0; 175 | -webkit-transition: all .6s; 176 | } 177 | 178 | .dash-header .header-right .filter .input { 179 | border: 0px; 180 | outline: none; 181 | width: 280px; 182 | background: #FFF; 183 | margin-left: 2px; 184 | margin-top: -19px; 185 | padding: 4px 8px; 186 | position: absolute; 187 | -webkit-transition: all .6s; 188 | font-family: 'Inconsolata', 'Lucida Console', Monaco, monospace; 189 | font-size: 14px; 190 | 191 | } 192 | 193 | .dash-header .header-right .filter:focus-within { 194 | background: #BBB; 195 | } 196 | 197 | .dash-header .header-right .filter.invalid { 198 | background: #db002f; 199 | } 200 | 201 | .dash-header .header-right .filter.invalid:focus-within { 202 | background: #db002f; 203 | } 204 | 205 | .dash-header .header-right .filter .input::placeholder { /* Chrome, Firefox, Opera, Safari 10.1+ */ 206 | color: #DDD; 207 | opacity: 1; /* Firefox */ 208 | -webkit-transition: all .6s; 209 | } 210 | 211 | .dash-header .header-right .filter .input:-ms-input-placeholder { /* Internet Explorer 10-11 */ 212 | color: #DDD; 213 | -webkit-transition: all .6s; 214 | } 215 | 216 | .dash-header .header-right .filter .input::-ms-input-placeholder { /* Microsoft Edge */ 217 | color: #DDD; 218 | -webkit-transition: all .6s; 219 | } 220 | 221 | .dash-header .header-right .filter .input:focus::placeholder { /* Chrome, Firefox, Opera, Safari 10.1+ */ 222 | color: #BBB; 223 | opacity: 1; /* Firefox */ 224 | } 225 | 226 | .dash-header .header-right .filter .input:focus:-ms-input-placeholder { /* Internet Explorer 10-11 */ 227 | color: #BBB; 228 | } 229 | 230 | .dash-header .header-right .filter .input:focus::-ms-input-placeholder { /* Microsoft Edge */ 231 | color: #BBB; 232 | } 233 | 234 | /* DASHBOARD */ 235 | 236 | .dashboard-container { 237 | padding: 2px; 238 | height: calc(100% - 3em); 239 | box-sizing: border-box; 240 | } 241 | 242 | .dashboard-wrapper { 243 | position: relative; 244 | height: 100%; 245 | } 246 | 247 | .read-only-area { 248 | flex-grow: 1; 249 | width: 100%; 250 | background: transparent; 251 | resize: none; 252 | outline: none; 253 | border: 0; 254 | padding: 0.5em; 255 | color: inherit; 256 | cursor: default; 257 | } 258 | 259 | .editable-area { 260 | flex-grow: 1; 261 | width: 100%; 262 | background: transparent; 263 | resize: none; 264 | outline: none; 265 | border: 0; 266 | padding: 0.5em; 267 | color: inherit; 268 | } 269 | 270 | /* PANEL */ 271 | 272 | .dash-panel { 273 | box-sizing: border-box; 274 | position: absolute; 275 | padding: 2px; 276 | display: flex; 277 | flex-direction: column; 278 | } 279 | 280 | .dash-panel .title { 281 | font-size: 1.2em; 282 | font-weight: 500; 283 | cursor: default; 284 | color: #fff; 285 | display: block; 286 | background: #2b5fb4; 287 | text-transform: uppercase; 288 | height: 1.8em; 289 | } 290 | 291 | .dash-panel .title.success { 292 | background: #00db87; 293 | } 294 | 295 | .dash-panel .title.warning { 296 | background: #dcd007; 297 | } 298 | 299 | .dash-panel .title.error { 300 | background: #db002f; 301 | } 302 | 303 | .dash-panel .title .text { 304 | float: left; 305 | padding: 6px 0 0 12px; 306 | } 307 | 308 | .dash-panel .title .pill { 309 | background: #0005; 310 | color: #FFF; 311 | font-size: .6em; 312 | margin: 0.4em .4em; 313 | padding: .6em 12px; 314 | vertical-align: middle; 315 | float: right; 316 | text-align: center; 317 | border-radius: 3px; 318 | width: 6em; 319 | } 320 | 321 | .dash-panel .content { 322 | flex-grow: 1; 323 | display: flex; 324 | flex-direction: column; 325 | background: #0000000D; 326 | color: #222; 327 | overflow: hidden; 328 | } 329 | 330 | .dash-panel .content textarea { 331 | font-size: 1em; 332 | font-family: 'Inconsolata', 'Lucida Console', Monaco, monospace; 333 | line-height: 1.3em; 334 | } 335 | -------------------------------------------------------------------------------- /web/demo.js: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | // this page is for demo site 3 | // new events will be generated every a few seconds 4 | // so the site will look dynamic 5 | 6 | import { createStore, render } from "./app.js"; 7 | 8 | (function connect() { 9 | function connectToDemo(store) { 10 | const dashboards = { 11 | "all-in-one": [ 12 | { 13 | log: { 14 | position: "0 0 100% 50%", 15 | title: "Build", 16 | filter: "command:build:.*", 17 | gauge: { 18 | filter: "command:build:state:.*", 19 | scan: { 20 | when: [ 21 | { 22 | pattern: "started", 23 | text: "Running", 24 | level: "warning", 25 | }, 26 | { 27 | pattern: "exited with 0", 28 | text: "Passed", 29 | level: "success", 30 | }, 31 | { 32 | pattern: "exited with", 33 | text: "Failed", 34 | level: "error", 35 | }, 36 | ], 37 | default: { 38 | text: "Unknown", 39 | }, 40 | }, 41 | }, 42 | }, 43 | }, 44 | { 45 | log: { 46 | position: "0 50% 50% 100%", 47 | title: "Lint", 48 | filter: "command:lint:.*", 49 | gauge: { 50 | filter: "command:lint:state:.*", 51 | scan: { 52 | when: [ 53 | { 54 | pattern: "started", 55 | text: "Running", 56 | level: "warning", 57 | }, 58 | { 59 | pattern: "exited with 0", 60 | text: "Passed", 61 | level: "success", 62 | }, 63 | { 64 | pattern: "exited with", 65 | text: "Failed", 66 | level: "error", 67 | }, 68 | ], 69 | default: { 70 | text: "Unknown", 71 | }, 72 | }, 73 | }, 74 | }, 75 | }, 76 | { 77 | log: { 78 | position: "50% 50% 100% 100%", 79 | title: "Test", 80 | filter: "command:test:.*", 81 | gauge: { 82 | filter: "command:test:state:.*", 83 | scan: { 84 | when: [ 85 | { 86 | pattern: "started", 87 | text: "Running", 88 | level: "warning", 89 | }, 90 | { 91 | pattern: "exited with 0", 92 | text: "Passed", 93 | level: "success", 94 | }, 95 | { 96 | pattern: "exited with", 97 | text: "Failed", 98 | level: "error", 99 | }, 100 | ], 101 | default: { 102 | text: "Unknown", 103 | }, 104 | }, 105 | }, 106 | }, 107 | }, 108 | ], 109 | "lint-only": [ 110 | { 111 | log: { 112 | position: "0 0 100% 100%", 113 | title: "Lint", 114 | filter: "command:lint:.*", 115 | gauge: { 116 | filter: "command:lint:state:.*", 117 | scan: { 118 | when: [ 119 | { 120 | pattern: "started", 121 | text: "Running", 122 | level: "warning", 123 | }, 124 | { 125 | pattern: "exited with 0", 126 | text: "Passed", 127 | level: "success", 128 | }, 129 | { 130 | pattern: "exited with", 131 | text: "Failed", 132 | level: "error", 133 | }, 134 | ], 135 | default: { 136 | text: "Unknown", 137 | }, 138 | }, 139 | }, 140 | }, 141 | }, 142 | ], 143 | "test-only": [ 144 | { 145 | log: { 146 | position: "0 0 100% 100%", 147 | title: "Test", 148 | filter: "command:test:.*", 149 | gauge: { 150 | filter: "command:test:state:.*", 151 | scan: { 152 | when: [ 153 | { 154 | pattern: "started", 155 | text: "Running", 156 | level: "warning", 157 | }, 158 | { 159 | pattern: "exited with 0", 160 | text: "Passed", 161 | level: "success", 162 | }, 163 | { 164 | pattern: "exited with", 165 | text: "Failed", 166 | level: "error", 167 | }, 168 | ], 169 | default: { 170 | text: "Unknown", 171 | }, 172 | }, 173 | }, 174 | }, 175 | }, 176 | ], 177 | }; 178 | 179 | try { 180 | store.clearErrorMessages(); 181 | store.updateDashboards(dashboards); 182 | store.updateCurrentDashboardTitle(null); 183 | } catch (err) { 184 | store.addErrorMessage(err.message); 185 | } 186 | 187 | setInterval(function() { 188 | // append a random event every 2 seconds 189 | store.appendEvents(generateRandonEvents()); 190 | }, 1500); 191 | } 192 | 193 | let globalEventId = 1; 194 | 195 | function getRandomChoice(choices) { 196 | const idx = Math.floor(Math.random() * choices.length); 197 | 198 | return choices[idx]; 199 | } 200 | 201 | function randomMessages() { 202 | const type = getRandomChoice(["build", "lint", "test"]); 203 | 204 | if (type === "lint") { 205 | return getRandomChoice([ 206 | [ 207 | "command:lint:state:started", 208 | "command:lint:stdout:lint passed\n\n", 209 | "command:lint:state:exited with 0", 210 | ], 211 | [ 212 | "command:lint:state:started", 213 | "command:lint:stdout:/tmp/web/dashflow_demo.js\n", 214 | "command:lint:stdout:5:7 error 'socket' is not defined no-undef\n", 215 | "command:lint:stdout:7:7 error 'socket' is not defined no-undef\n\n", 216 | "command:lint:stdout:✖ 2 problems (2 errors, 0 warnings)\n\n", 217 | "command:lint:stdout:error Command failed with exit code 1.\n\n", 218 | "command:lint:state:exited with 1", 219 | ], 220 | ]); 221 | } else if (type === "test") { 222 | return getRandomChoice([ 223 | [ 224 | "command:test:state:started", 225 | "command:test:stdout:formatter\n", 226 | "command:test:stdout: ✓ format shell\n", 227 | "command:test:stdout: ✓ unformat shell\n", 228 | "command:test:stdout: ✓ format watch\n", 229 | "command:test:stdout: ✓ unformat watch\n", 230 | "command:test:stdout: ✓ format restart\n", 231 | "command:test:stdout: ✓ unformat restart\n", 232 | "command:test:stdout: ✓ format command\n", 233 | "command:test:stdout: ✓ unformat command\n", 234 | "command:test:stdout: ✓ smart unformat\n\n", 235 | "command:test:stdout:10 passing (9ms)\n\n", 236 | "command:test:state:exited with 0", 237 | ], 238 | [ 239 | "command:test:state:started", 240 | "command:test:stdout:formatter\n", 241 | "command:test:stdout: ✓ format shell\n", 242 | "command:test:stdout: ✓ unformat shell\n", 243 | "command:test:stdout: ✓ format watch\n", 244 | "command:test:stdout: ✓ unformat watch\n", 245 | "command:test:stdout: ✓ format restart\n", 246 | "command:test:stdout: ✓ unformat restart\n", 247 | "command:test:stdout: ✓ format command\n", 248 | "command:test:stdout: ✓ unformat command\n", 249 | "command:test:stdout: ✖ smart unformat\n\n", 250 | "command:test:stdout:9 passing, 1 failed (19ms)\n\n", 251 | "command:test:state:exited with 1", 252 | "command:test:state:started", 253 | ], 254 | ]); 255 | } else { 256 | return getRandomChoice([ 257 | ["command:build:state:started"], 258 | [ 259 | "command:build:stdout:build passed\n\n", 260 | "command:build:state:exited with 0", 261 | ], 262 | [ 263 | "command:build:stdout:build failed\n\n", 264 | "command:build:state:exited with 1", 265 | ], 266 | ]); 267 | } 268 | } 269 | 270 | function humanize(message) { 271 | if (message.indexOf(":state:") !== -1) { 272 | return ""; 273 | } else { 274 | return message.substring(message.lastIndexOf(":stdout:") + 8); 275 | } 276 | } 277 | 278 | function generateRandonEvents() { 279 | const messages = randomMessages(); 280 | 281 | return messages.map(function(message) { 282 | return { 283 | id: globalEventId++, 284 | t: Date.now(), 285 | e: message, 286 | h: humanize(message), 287 | }; 288 | }); 289 | } 290 | 291 | const appStore = createStore(); 292 | connectToDemo(appStore); 293 | render(appStore); 294 | })(); 295 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dashflow 2 | 3 | ![logo](https://github.com/freewheel/dashflow/blob/master/guide_assets/dashflow-header.min.png) 4 | 5 | ![npm](https://img.shields.io/npm/v/dashflow.svg) 6 | 7 | [![Netlify Status](https://api.netlify.com/api/v1/badges/21173937-497e-4a6c-ba6f-0b8ffa94ca18/deploy-status)](https://app.netlify.com/sites/dashflow/deploys) 8 | 9 | Automate local development tasks. [See It In Action](https://dashflow.netlify.com/) 10 | 11 | ## What can Dashflow do for you 12 | 13 | - Run many commands in background and collect their stdout/stderr 14 | - Automatically trigger new commands when output from those commands matches certain pattern 15 | - Serve a web dashboard page to visualize the status of those commands 16 | - Replace Makefile using a easy readable dashflow.yml YAML file 17 | 18 | ![run processes](https://raw.githubusercontent.com/freewheel/dashflow/master/guide_assets/interactive_shell.png) 19 | ![web dashboard](https://raw.githubusercontent.com/freewheel/dashflow/master/guide_assets/web_dashboard.gif) 20 | 21 | ## Installation 22 | 23 | Make sure you have installed [NodeJS](https://nodejs.org/en/download/). 24 | 25 | ``` 26 | # install the package 27 | npm install -g dashflow 28 | ``` 29 | 30 | ## Usage 31 | 32 | ``` 33 | # print help 34 | $ dashflow --help 35 | 36 | # by default read from current folder dashflow.yml 37 | $ dashflow 38 | 39 | # support multi config files, config will be merged 40 | $ dashflow -c service1/dashflow.yml -c service2/dashflow.yml 41 | 42 | # custom http port 43 | $ dashflow -p 9528 44 | 45 | # use dashflow shell 46 | dashflow-shell~$ help 47 | 48 | Commands: 49 | 50 | help [command...] Provides help for a given command. 51 | exit Exits application. 52 | events [pattern] Dump events 53 | emit Emit a new event 54 | tail Tail to a stream or workflow 55 | attach Attach to a stream 56 | restart Restart a stream 57 | env Print environment variables 58 | 59 | ``` 60 | 61 | ## Concepts 62 | 63 | Dashflow is modeled around some key concepts. 64 | 65 | Please take some time to get familiar with those concepts so you can use the tool more effectively. 66 | 67 | ### Event 68 | 69 | An event is just a plain string, associated with its creation time. Usually events will have certain prefixes so that they can easily be distinguished from different sources of events. 70 | 71 | For example, here are some sample events: 72 | 73 | ``` 74 | stream:watch-src:watch:add:src/new_file 75 | stream:watch-test:watch:addDir:src/test/ 76 | command:lint:shell:yarn --silent lint:stdout:lint passed 77 | workflow:initial-lint:command:lint:shell:yarn --silent lint:stdout:lint passed 78 | ``` 79 | 80 | In dashflow, all events store in memory (which means all events will be lost once the dashflow process exited). 81 | 82 | The default view of events are ordered by their creation time, newer to older. 83 | 84 | ### Command 85 | 86 | A command is an alias for a shell function, commands can be composed together, executing one by one or concurrently. 87 | 88 | The following is an example of command configurations: 89 | 90 | ``` 91 | commands: 92 | // the following concise way of declaring a command is a shortcut for the formal one 93 | // which is demonstrated with "test" command 94 | lint: yarn --silent lint 95 | test: 96 | shell: { cmd: yarn --silent test --no-color --verbose } 97 | lint-then-test: 98 | // run those commands one by one 99 | serial: 100 | - lint 101 | - test 102 | lint-and-test: 103 | // run those commands concurrently 104 | parallel: 105 | - lint 106 | - test 107 | list-web-files: 108 | // by default we assume shell should executes from the same folder 109 | // where dashflow.yml file is located 110 | // but we can specify a sub folder by providing 'cwd' argument to shell 111 | shell: { cmd: ls, cwd: web } 112 | ``` 113 | 114 | ### Stream 115 | 116 | A stream is something that will emit events, it can be either a long running process like a web server, or a one off command like running test. 117 | 118 | The following is an example of stream configurations: 119 | 120 | ``` 121 | streams: 122 | watch-lib: 123 | // listen on current folder for file changes and report evens on matching files 124 | watch: { glob: "lib/**/*.js" } 125 | watch-test: 126 | // can specify a sub folder by providing 'cwd' argument to watch 127 | watch: { glob: "**/*.js", cwd: test } 128 | python: 129 | // can execute a shell command 130 | shell: { cmd: python -i } 131 | vim: 132 | // can specify a sub folder by providing 'cwd' argument to shell 133 | shell: { cmd: vim, cwd: test } 134 | ``` 135 | 136 | ### Workflow 137 | 138 | A workflow represents a rule, typically some execution logic like "when A happens, then B then C then D". 139 | 140 | A concrete example is that when a file changed in "src" folder, we would like lint and test to be triggered automatically. 141 | 142 | A workflow also emit events, from this point of view it is also a special stream. 143 | 144 | In dashflow, workflows are always triggered while a new event matches given pattern. 145 | 146 | The following is an example of workflow configurations: 147 | 148 | ``` 149 | workflows: 150 | initial-lint: 151 | // special event fires when dashflow starts 152 | match: SYSTEM:started 153 | command: lint 154 | initial-test: 155 | match: SYSTEM:started 156 | command: test 157 | lib-lint-then-test: 158 | match: watch-lib:.* 159 | // execute those commands serially 160 | serial: 161 | - command: lint 162 | - command: test 163 | test-lint-then-test: 164 | match: watch-test:.* 165 | // execute those commands parallelly 166 | parallel: 167 | - command: lint 168 | - command: test 169 | - restart: irb 170 | web-lint: 171 | match: watch-web:.* 172 | serial: 173 | - delay: 1000 174 | - command: lint 175 | ``` 176 | 177 | ### Dashboard 178 | 179 | Dashflow starts an HTTP server when running in daemon mode, and what being served is an special page that visualizes all those events dashflow has collected. 180 | The page has many tabs, each tab is a dashboard. A dashboard has many widgets, each occupies a rectangular area. 181 | 182 | Different widget type behaves very differently: 183 | 184 | - a "log" widget shows a subset of all events by applying a filter 185 | - a "gauge" widget shows status texts base on calculations on event streams, like "passing/failed/unknown" etc 186 | - a "banner" widget displays static text 187 | 188 | For example: 189 | 190 | ``` 191 | dashboards: 192 | all-in-one: 193 | - log: 194 | position: 0 0 45% 50% 195 | title: Lint 196 | filter: command:lint:.* 197 | - log: 198 | position: 45% 0 90% 50% 199 | title: Test 200 | filter: command:test:.* 201 | - banner: 202 | position: 90% 0 100% 16% 203 | content: This is Dashflow Dashboard 204 | - gauge: 205 | position: 90% 16% 100% 32% 206 | title: Lint Status 207 | filter: command:lint:state:.* 208 | scan: 209 | when: 210 | - pattern: started 211 | text: Running 212 | level: warning 213 | - pattern: exited with 0 214 | text: Passed 215 | level: success 216 | - pattern: exited with 217 | text: Failed 218 | level: error 219 | default: 220 | text: Unknown 221 | ``` 222 | 223 | ## Configuration 224 | 225 | A dashflow.yml is required in order to utilize dashflow as a local development workflow orchestrator. 226 | 227 | Basically what we need to do is to model the workflow we already have into those concepts in dashflow. 228 | 229 | Here're some example configurations for your inspiration. 230 | 231 | ### dashflow.yml for a frontend project 232 | 233 | ``` 234 | commands: 235 | lint: yarn --silent lint_min 236 | test: yarn --silent test_min 237 | 238 | streams: 239 | site: { shell: { cmd: yarn --silent site } } 240 | watch-src: { watch: { glob: "src/**/*.js*" } } 241 | 242 | workflows: 243 | initial-lint: 244 | match: SYSTEM:started 245 | command: lint 246 | initial-test: 247 | match: SYSTEM:started 248 | command: test 249 | on-src-update: | 250 | match: watch-src:.* 251 | parallel: 252 | - command: lint 253 | - command: test 254 | 255 | dashboards: 256 | spark-ui: 257 | - log: 258 | position: 0 0 90% 50% 259 | title: Webpack Logs 260 | filter: stream:site:.* 261 | - gauge: 262 | position: 90% 0 100% 25% 263 | title: Lint Status 264 | filter: command:lint:state:.* 265 | scan: 266 | when: 267 | - pattern: started 268 | text: Running 269 | level: warning 270 | - pattern: exited with 0 271 | text: Passed 272 | level: success 273 | - pattern: exited with 274 | text: Failed 275 | level: error 276 | default: 277 | text: Unknown 278 | - gauge: 279 | position: 90% 25% 100% 50% 280 | title: Test Status 281 | filter: command:test:state:.* 282 | scan: 283 | when: 284 | - pattern: started 285 | text: Running 286 | level: warning 287 | - pattern: exited with 0 288 | text: Passed 289 | level: success 290 | - pattern: exited with 291 | text: Failed 292 | level: error 293 | default: 294 | text: Unknown 295 | - log: 296 | position: quadrant/bottom-left 297 | title: Lint 298 | filter: command:lint:.* 299 | - log: 300 | position: quadrant/bottom-right 301 | title: Test 302 | filter: command:test:.* 303 | ``` 304 | 305 | ### dashflow.yml for dashflow project itself 306 | 307 | Click [here](./dashflow.yml) 308 | 309 | ## Reference 310 | 311 | commandID/streamID/workflowID is the key of a command/stream/workflow definition in the configuration file. 312 | 313 | ### Command 314 | 315 | ``` 316 | # execute shell command 317 | commandID: 318 | shell: 319 | cmd: 320 | cwd: "working folder, default to be where dashflow.yml is located" 321 | # produces events in following formats 322 | # command:commandID:shell::stdout: 323 | # command:commandID:shell::state:started 324 | # command:commandID:shell::state:exited with 325 | ``` 326 | 327 | ### Stream 328 | 329 | ``` 330 | # listen on file changes 331 | streamID: 332 | watch: 333 | glob: 334 | cwd: "working folder, default to be where dashflow.yml is located" 335 | ignore: /regex pattern for files to ignore/ 336 | # produces events in following formats 337 | # stream:streamID:watch:add: 338 | # stream:streamID:watch:addDir: 339 | # stream:streamID:watch:change: 340 | # stream:streamID:watch:unlink: 341 | # stream:streamID:watch:unlinkDir: 342 | 343 | # execute shell command 344 | streamID: 345 | shell: 346 | cmd: 347 | cwd: "working folder, default to be where dashflow.yml is located" 348 | # short form for the above if cwd is default 349 | streamID: 350 | # produces events in following formats 351 | # stream:streamID:shell::stdout: 352 | # stream:streamID:shell::state:started 353 | # stream:streamID:shell::state:exited with 354 | ``` 355 | 356 | ### Workflow 357 | 358 | ``` 359 | # execute command if there's a match 360 | workflowID: 361 | match: "event pattern that triggers this workflow" 362 | command: "command name to trigger" 363 | # produces events in following formats 364 | # workflow:workflowID: 365 | 366 | # restart stream if there's a match 367 | workflowID: 368 | match: "event pattern that triggers this workflow" 369 | restart: 370 | # produces events in following formats 371 | # workflow:workflowID:restart: 372 | 373 | # wait for a certain time if there's a match 374 | # usually used together with serial command 375 | workflowID: 376 | match: "event pattern that triggers this workflow" 377 | wait: