├── .editorconfig ├── .eslintrc.json ├── .github ├── dependabot.yml └── workflows │ ├── publish.yml │ └── test.yml ├── .gitignore ├── .npmignore ├── CHANGES.md ├── README.md ├── bin ├── flowtrace-replay └── flowtrace-show ├── doc ├── braindump.md └── flowtrace-replay-flowhub.png ├── examples └── flowtrace-without-events.json ├── package.json ├── spec ├── .eslintrc.json ├── Flowtrace.js ├── flowtrace-replay.js └── import.js ├── src ├── bin │ ├── performance.js │ ├── render.js │ ├── replay.js │ └── websocket.js ├── index.js └── lib │ ├── Flowtrace.js │ ├── common.js │ ├── record.js │ └── trace.js └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 2 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "airbnb-base", 3 | "globals": { 4 | "fetch": false 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | versioning-strategy: increase-if-necessary 8 | - package-ecosystem: github-actions 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Node.js Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2.3.4 13 | - uses: actions/setup-node@v2.1.4 14 | with: 15 | node-version: 12 16 | - run: npm install 17 | - run: npm test 18 | 19 | publish-npm: 20 | needs: build 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v2.3.4 24 | - uses: actions/setup-node@v2.1.4 25 | with: 26 | node-version: 12 27 | registry-url: https://registry.npmjs.org/ 28 | - run: npm install 29 | - run: npm run build 30 | - run: npm publish 31 | env: 32 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Node CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | name: Run test suite 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | node-version: [12.x] 12 | steps: 13 | - uses: actions/checkout@v2.3.4 14 | - name: Use Node.js ${{ matrix.node-version }} 15 | uses: actions/setup-node@v2.1.4 16 | with: 17 | node-version: ${{ matrix.node-version }} 18 | - run: npm install 19 | - run: npm test 20 | env: 21 | CI: true 22 | merge-me: 23 | name: Auto-merge dependency updates 24 | needs: test 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: ridedott/merge-me-action@v2.1.5 28 | with: 29 | GITHUB_LOGIN: 'dependabot[bot]' 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | components/ 3 | browser/ 4 | bower_components/ 5 | build/ 6 | dist/ 7 | package-lock.json 8 | .nyc_output/ 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .git* 2 | /coverage/ 3 | /.nyc_output/ 4 | /.github/ 5 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ## 0.1.10 (2020-12-29) 2 | 3 | * Fixed formatting of unserializable data packets 4 | 5 | ## 0.1.9 (2020-12-23) 6 | 7 | * Added safety when adding non-serializable packets to a Flowtrace 8 | * Fixed flowtrace-show command 9 | 10 | ## 0.1.8 (2020-12-03) 11 | 12 | * TypeScript definitions no longer rely on the esModuleInterop parameter 13 | 14 | ## 0.1.7 (2020-12-03) 15 | 16 | * Fixed EventEmitter typing in Flowtrace type defs 17 | 18 | ## 0.1.6 (2020-12-03) 19 | 20 | * Flowtrace source code now uses JS Modules, with CommonJS version distributed in the package 21 | 22 | ## 0.1.5 (2020-12-03) 23 | 24 | * This module now ships with basic TypeScript type definitions 25 | 26 | ## 0.1.4 (2020-12-03) 27 | 28 | * Flowtrace helper now includes methods for logging `stdout` and process error events 29 | 30 | ## 0.1.3 (2020-11-24) 31 | 32 | * Added a `clear` method to the Flowtracer for emptying the event buffer 33 | 34 | ## 0.1.2 (2020-11-11) 35 | 36 | * Added a fbp-protocol recorder to create Flowtraces client-side (for example in the fbp-spec tool or Flowhub) 37 | 38 | ## 0.1.1 (2020-10-12) 39 | 40 | * Data packets are now cloned to ensure they don't mutate later 41 | * Some cleanup of dependencies 42 | 43 | ## 0.1.0 (2020-10-12) 44 | 45 | * Ported from CoffeeScript tp modern ES6 46 | * Added a Flowtrace base library to make capturing traces easier to runtimes 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Flowtrace 2 | ========= 3 | 4 | A `flowtrace` is a persisted record of the execution of an Flow-based Programming (FBP) or dataflow program. 5 | It is used for retroactive (after-the-fact) debugging; to locate, understand and fix bugs. 6 | 7 | The concept is analogous to a 'stacktrace' or 'core dump' for imperative code. 8 | 9 | This project provides a data format to store traces in, and provide debugging tools for working with these traces, as well as JavaScript library for recording and producing them. 10 | 11 | ## Status 12 | 13 | In production 14 | 15 | * [NoFlo](https://github.com/noflo/noflo) has support for creating flowtraces from 1.3.0 onwards. Can be triggered programmatically, via fbp-protocol, or with the [noflo-nodejs](https://github.com/noflo/noflo-nodejs) command-line tool 16 | * [fbp-spec 0.8](https://github.com/flowbased/fbp-spec) has support for capturing flowtraces of test runs 17 | * Several commandline tools exist for working with flowtraces 18 | * Note: File format not 100% finalized 19 | 20 | See [braindump](./doc/braindump.md) for ideas/plans. 21 | 22 | ## Installing 23 | 24 | First make sure you have [Node.js](http://nodejs.org/) with NPM installed. 25 | 26 | To install locally in a project. Recommended. 27 | 28 | npm install flowtrace 29 | 30 | To install globally on your system 31 | 32 | npm install -g flowtrace 33 | 34 | ## Display flowtrace on commandline 35 | 36 | `flowtrace-show` reads a flowtrace, and renders a human-friendly log output from it. 37 | 38 | npx flowtrace-show mytrace.flowtrace.json 39 | 40 | Example output: 41 | 42 | ``` 43 | -> IN repeat CONN 44 | -> IN repeat DATA hello world 45 | -> IN stdout CONN 46 | -> IN stdout DATA hello world 47 | -> IN repeat DISC 48 | -> IN stdout DISC 49 | ``` 50 | 51 | When used in a terminal, supports colors. 52 | 53 | ## Show a flowtrace in Flowhub 54 | 55 | `flowtrace-replay` reads a flowtrace, and then acts as a live FBP runtime. That means it can be used with 56 | any FBP IDEs/client which support the [FBP runtime protocol](http://noflojs.org/documentation/protocol/). 57 | 58 | npx flowtrace-replay mytrace.flowtrace.json 59 | 60 | By default this will open [Flowhub](https://app.flowhub.io) in your browser, automatically connect and show you the graph. 61 | To replay the data press the play button. You should then see the data flowing through edges. 62 | 63 | ![Flowtrace replayed in Flowhub](./doc/flowtrace-replay-flowhub.png) 64 | 65 | You can specify which `--ide` to use, and disable automatic opening of browser with `-n`. 66 | 67 | npx flowtrace-replay --ide http://localhost:8888 -n 68 | 69 | You can also set the `--host` and `--port`. See `--help` for all options. 70 | 71 | ## Recording flowtraces in JavaScript 72 | 73 | It is possible to use this library for recording and serializing flowtraces. Quick example: 74 | 75 | ```javascript 76 | const { Flowtrace } = require('flowtrace'); 77 | 78 | const tracer = new Flowtrace({ 79 | // metadata about this run 80 | }); 81 | 82 | // Register the main graph you're tracing 83 | tracer.addGraph('example', myGraph, true); 84 | // You should also call addGraph for each subgraph that is running 85 | 86 | myProgram.on('packet', (packet) => { 87 | // Tell Flowtracer about each packet that arrives 88 | tracer.addNetworkpacket('network:data', packet.src, packet.tgt, 'example', packet.data); 89 | }); 90 | 91 | myProgram.on('end', () => { 92 | // Once your program is finished (or errors), you can dump the Flowtrace 93 | const myTrace = tracer.toJSON(); 94 | fs.writeFile('example.flowtrace.json', myTrace, (err) => { 95 | // ... 96 | }); 97 | }); 98 | ``` 99 | 100 | See the `src/lib/Flowtrace.js` file for more information. 101 | -------------------------------------------------------------------------------- /bin/flowtrace-replay: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require('../dist/bin/replay').main(); 3 | -------------------------------------------------------------------------------- /bin/flowtrace-show: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require('../dist/bin/render').default(); 3 | -------------------------------------------------------------------------------- /doc/braindump.md: -------------------------------------------------------------------------------- 1 | 2 | ## Trace dataformat 3 | 4 | * Should contain everything needed to reproduce an issue. 5 | Graph, data, components/runtime version identification. 6 | * Should contain information about what caused the dump? (trigger) 7 | * Should have a formal definition/schema 8 | * Should reuse exiting FBP dataformats as much as possible. They are mostly JSON-oriented. 9 | * Should be possible to stream 10 | * Should be easy to support with any FBP runtime. Same tools used by all runtimes. 11 | * Should be easy to manipulate in any language 12 | * Maybe support compression, to keep dump size down? 13 | 14 | Q 15 | 16 | * Should it be possible to store traces of multiple networks in one trace? 17 | Usecases: subgraphs, multiple top-level networks in one program 18 | 19 | ## Triggering mechanisms 20 | 21 | * Manual, local on system. Unix signal 22 | * Remote over network. FBP network protocol / messagequeue 23 | Can be used to trigger manually, or to implement external triggers, for instance by a service monitoring system. 24 | * Automatic. On exceptions, uncaught errors. 25 | * Batch. Set up that network execution will be dumped beforehand. 26 | * Data-breakpoints 27 | 28 | ## Data-breakpoints 29 | 30 | A functionality that imperative debuggers have is to set breakpoints, 31 | as a marker on a particular instruction/line, potentially with a predicate/expression 32 | for when to trigger. 33 | An analog for FBP/dataflow would be data-breakpoints, where one could set marker 34 | a particular connection/edge. The trigger predicates would be evaluated against. 35 | It may be possible to reuse [fbp-spec assertions](https://github.com/flowbased/fbp-spec) for that, 36 | or a similar format (should be declarative, runtime independent and serializable). 37 | 38 | The data-breakpoint could be used to trigger a flowtrace dump when ran non-interactively. 39 | When having opened a flowtrace dump, it should also be possible to set other breakpoints in the UI. 40 | Such interactive exploration might be seen more as searching/filtering. 41 | 42 | Triggering on breakpoints should be handled by the runtime, and setting them up 43 | part of the FBP protocol (as its own capability). 44 | 45 | ## Buffering 46 | 47 | For a long-running network, it is generally not feasible to keep the entire state 48 | since beginning of time around (it case it might be needed). 49 | So it should be possible to specify the (rotating) buffer which is kept. 50 | Ideally both as maximum size usage (in RAM, on disk) and in retention (in seconds/minutes/hours). 51 | 52 | Some networks will have huge data payloads traveling through edges. 53 | It might be desirable to drop the payloads first, but keep the information 54 | that something happened on the edge (at given time). 55 | 56 | When data has been truncated or abbreviated, it might be nice with some markers to denote this. 57 | 58 | ## Relation to stacktrace 59 | 60 | As the majority of FBP/dataflow programs have components implemented in 61 | imperative code. So flowtrace is not a substitute for stacktraces. 62 | 63 | Hopefully for debugging things on the FBP "level" which sits above, 64 | and what the FBP application developer deals with most of the time, 65 | it will mostly be enough. 66 | 67 | Sometimes a combination of flow and stacktraces/coredumps would be useful though. 68 | When debugging issues that occur *inside* a component, as opposed to in the FBP graph. 69 | And especially when components are fairly 'fat' (contains lots of code). 70 | 71 | Unlike typical stacktraces, flowtraces can keep 'history', that is to 72 | keep information about events which happened seconds or minutes before. 73 | 74 | ## Event store 75 | 76 | Flowtraces are currently serialized files of one or more network execution of a single runtime. 77 | However, in order to fully utilize the data in them, being able to store them all in a query-able 78 | database would be beneficial. 79 | This is especially for analysis which require data from different traces: 80 | across multiple network invocations, different machines and versions of the software. 81 | 82 | It should be possible to stream the data directly to such an event store from production machines. 83 | However the store should probably be the primary interface for analysis, with flowtrace files 84 | being imported in before doing non-trivial analysis on it. 85 | 86 | ### Usecases 87 | 88 | * Allow to lookup data for any production failure. Look at it, compare it against cases which did not fail. 89 | * Deduplicate production failures reported into a single bug, by finding which follow a common pattern. 90 | Also to estimate the impact of a single bug. 91 | * Do performance analytics. Both coarse-grained and fine-grained. Find hotspots and hickups. 92 | * (maybe) allow to perform invariant-based testing of components, by searching 93 | for input/output data in DB, and checking preconditions/postconditions. 94 | * (maybe) automatically generating minimal testcases 95 | 96 | Traces are *primarily meta-data* about the execution of an FBP system. 97 | However, such an event store may also be used to store *data* from the system. 98 | For instance to store sensor data, computation results and analytics. 99 | 100 | ### Possible SQL representation 101 | 102 | SQL has the advantage that many very scalable solutions and techniques exists for it. 103 | Thanks to IndexedDB one can also use it in the browser, which is interesting for interactive exploration and analysis. 104 | 105 | Events table 106 | 107 | id: UUID | identifier for this event 108 | time: DateTime | When the event was created 109 | event: string | Type of event. Typically "data" 110 | 111 | runtime: UUID | id of the FBP runtime this event comes from 112 | component: string | component (or subgraph) this event originated in 113 | networkpath: string | Path of the network (@component instance) from top-level of @runtime. Ex: "subsystem.mymodule.performfoo" 114 | sourcenode: string | Name of node @data was sent *from* 115 | sourceport: string | Port the @data was sent *from* 116 | targetnode: string | Name of node @data twas sent *to* 117 | targetport: string | Port the @data was sent *to* 118 | 119 | json: JSON | data in JSON representation 120 | blob: Blob | data as a binary blob, if it cannot be represented as JSON 121 | 122 | # TODO: specify versioning information 123 | 124 | There should also be some versioning information. 125 | Possibly this could be some hash (git SHA?) on the runtime level. 126 | Would then have to lookup details on component/dependency version 127 | 128 | -------------------------------------------------------------------------------- /doc/flowtrace-replay-flowhub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flowbased/flowtrace/fc6ab1b5f788c04f5dbb2d9b9657aa0d32c54b64/doc/flowtrace-replay-flowhub.png -------------------------------------------------------------------------------- /examples/flowtrace-without-events.json: -------------------------------------------------------------------------------- 1 | { 2 | "header": { 3 | "graphs": { 4 | "helloworld": { 5 | "caseSensitive": false, 6 | "properties": { 7 | "name": "helloworld" 8 | }, 9 | "inports": {}, 10 | "outports": {}, 11 | "groups": [], 12 | "processes": { 13 | "repeat": { 14 | "component": "core/Repeat", 15 | "metadata": {} 16 | }, 17 | "stdout": { 18 | "component": "core/Output", 19 | "metadata": {} 20 | } 21 | }, 22 | "connections": [ 23 | { 24 | "src": { 25 | "process": "repeat", 26 | "port": "out" 27 | }, 28 | "tgt": { 29 | "process": "stdout", 30 | "port": "in" 31 | } 32 | }, 33 | { 34 | "data": "hello world", 35 | "tgt": { 36 | "process": "repeat", 37 | "port": "in" 38 | } 39 | } 40 | ] 41 | } 42 | } 43 | } 44 | , "events": 45 | [] 46 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flowtrace", 3 | "version": "0.1.10", 4 | "description": "Tracing format tools for FBP", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "bin": { 8 | "flowtrace-show": "./bin/flowtrace-show", 9 | "flowtrace-replay": "./bin/flowtrace-replay" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/flowbased/flowtrace/issues" 13 | }, 14 | "author": "Jon Nordby ", 15 | "license": "MIT", 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/flowbased/flowtrace.git" 19 | }, 20 | "dependencies": { 21 | "circular-buffer": "^1.0.2", 22 | "cli-color": "~2.0.0", 23 | "clone": "^2.1.2", 24 | "commander": "^6.1.0", 25 | "debug": "^4.2.0", 26 | "fbp": "^1.7.0", 27 | "fbp-graph": "^0.7.0", 28 | "isomorphic-fetch": "^3.0.0", 29 | "open": "^7.2.1", 30 | "tv4": "^1.2.7", 31 | "websocket": "^1.0.21" 32 | }, 33 | "devDependencies": { 34 | "@types/clone": "^2.1.0", 35 | "@types/node": "^14.14.10", 36 | "chai": "^4.0.0", 37 | "eslint": "^7.10.0", 38 | "eslint-config-airbnb-base": "^14.2.0", 39 | "eslint-plugin-chai": "0.0.1", 40 | "eslint-plugin-import": "^2.22.1", 41 | "eslint-plugin-mocha": "^8.0.0", 42 | "fbp-client": "^0.4.3", 43 | "fbp-protocol-healthcheck": "^1.1.0", 44 | "mocha": "^8.1.3", 45 | "noflo-core": "^0.6.1", 46 | "noflo-nodejs": "^0.15.0", 47 | "nyc": "^15.1.0", 48 | "replace": "^1.2.0", 49 | "typescript": "^4.1.2" 50 | }, 51 | "scripts": { 52 | "lint": "eslint src spec", 53 | "pretest": "npm run build && npm run lint", 54 | "build": "tsc", 55 | "postbuild": "replace 'node/events' 'events' dist -r", 56 | "test": "nyc mocha spec/*.js" 57 | }, 58 | "nyc": { 59 | "include": [ 60 | "dist/*.js", 61 | "dist/lib/*.js", 62 | "dist/bin/*.js" 63 | ] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /spec/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "airbnb-base", 3 | "plugins": [ 4 | "chai", 5 | "mocha" 6 | ], 7 | "env": { 8 | "node": true, 9 | "mocha": true 10 | }, 11 | "rules": { 12 | "func-names": 0 13 | }, 14 | "globals": { 15 | "baseDir": false, 16 | "chai": false, 17 | "noflo": false, 18 | "window": false 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /spec/Flowtrace.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const { Flowtrace } = require('../dist/index'); 3 | 4 | describe('Flowtrace class', () => { 5 | let tracer; 6 | it('should be possible to instantiate with custom buffer size', () => { 7 | tracer = new Flowtrace({ 8 | type: 'example', 9 | }, 3); 10 | }); 11 | it('should be possible to serialize to JSON', () => { 12 | const json = tracer.toJSON(); 13 | chai.expect(json.header.metadata.type).to.equal('example'); 14 | chai.expect(json.events).to.eql([]); 15 | }); 16 | it('should be possible to register a packet', () => { 17 | tracer.addNetworkPacket('data', {}, {}, 'default/main', { 18 | data: true, 19 | }); 20 | }); 21 | it('should be possible to serialize to JSON', () => { 22 | const json = tracer.toJSON(); 23 | chai.expect(json.header.metadata.type).to.equal('example'); 24 | chai.expect(json.events.length).to.eql(1); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /spec/flowtrace-replay.js: -------------------------------------------------------------------------------- 1 | const { spawn } = require('child_process'); 2 | const { expect } = require('chai'); 3 | const path = require('path'); 4 | const fbpHealthCheck = require('fbp-protocol-healthcheck'); 5 | const fbpClient = require('fbp-client'); 6 | 7 | function healthCheck(address, callback) { 8 | fbpHealthCheck(address) 9 | .then(() => callback(), () => healthCheck(address, callback)); 10 | } 11 | 12 | describe('flowtrace-replay CLI', () => { 13 | const prog = path.resolve(__dirname, '../bin/flowtrace-replay'); 14 | describe('examples/flowtrace-without-events.json', () => { 15 | let runtimeProcess; 16 | let runtimeClient; 17 | after('stop runtime', (done) => { 18 | if (!runtimeProcess) { 19 | done(); 20 | return; 21 | } 22 | process.kill(runtimeProcess.pid); 23 | done(); 24 | }); 25 | it('should start a runtime', (done) => { 26 | runtimeProcess = spawn(prog, [ 27 | '--no-open', 28 | 'examples/flowtrace-without-events.json', 29 | ]); 30 | runtimeProcess.stdout.pipe(process.stdout); 31 | runtimeProcess.stderr.pipe(process.stderr); 32 | healthCheck('ws://localhost:3333', done); 33 | }); 34 | it('should be possible to connect', () => fbpClient({ 35 | address: 'ws://localhost:3333', 36 | protocol: 'websocket', 37 | }) 38 | .then((c) => { 39 | runtimeClient = c; 40 | return c.connect(); 41 | })); 42 | it('should have a known main graph', () => { 43 | expect(runtimeClient.definition.graph).to.equal('helloworld'); 44 | }); 45 | it('should be possible to get graph sources', () => runtimeClient 46 | .protocol.component.getsource({ 47 | name: runtimeClient.definition.graph, 48 | })); 49 | it('should be possible to get status of the traced network', () => runtimeClient 50 | .protocol.network.getstatus({ 51 | graph: runtimeClient.definition.graph, 52 | })); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /spec/import.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const flowtrace = require('../dist/index'); 3 | 4 | describe('Loading Flowtrace', () => { 5 | it('should contain a tracer', () => { 6 | chai.expect(flowtrace.trace).to.be.an('object'); 7 | }); 8 | }); 9 | -------------------------------------------------------------------------------- /src/bin/performance.js: -------------------------------------------------------------------------------- 1 | import debug from 'debug'; 2 | import { readGraphFile } from '../lib/common'; 3 | 4 | const debugPerf = debug('flowtrace:performance'); 5 | 6 | const svg = { 7 | node(type, attributes, children) { 8 | let attrs = ''; 9 | Object.keys(attributes).forEach((k) => { 10 | const v = attributes[k]; 11 | attrs += `${k}="${v}"\n`; 12 | }); 13 | return `<${type} \n${attrs} >${children}`; 14 | }, 15 | line(x0, y0, width, height) { 16 | return `m ${x0},${y0} ${width},${height}`; 17 | }, 18 | }; 19 | 20 | const extractFlows = function (graph) { 21 | // PERF: consider keeping around the process-sorted-connections 22 | const targetConnections = (name) => graph.connections.filter( 23 | (c) => c.src.process && c.tgt.process === name, 24 | ); 25 | const sourceConnections = (name) => graph.connections.filter( 26 | (c) => (c.src != null ? c.src.process : undefined) === name, 27 | ); 28 | 29 | // XXX: pretty sure this has serious bugs 30 | const walkConnectionGraph = function (start, collect, flow = []) { 31 | const outs = sourceConnections(start); 32 | debugPerf('w', start, outs.length, flow.length); 33 | let fl = flow; 34 | outs.forEach((c) => { 35 | const tgt = c.tgt.process; 36 | const subs = walkConnectionGraph(tgt, collect, fl); 37 | if (subs.length === 0) { 38 | // end of a flow, collect and reset 39 | if (collect) { collect(flow); } 40 | fl = []; 41 | } else { 42 | fl.unshift(c); 43 | } 44 | }); 45 | 46 | return outs; 47 | }; 48 | 49 | const flows = []; 50 | 51 | // find starting points 52 | Object.keys(graph.processes).forEach((process) => { 53 | const ins = targetConnections(process); 54 | // debugPerf 'c', process, ins.length, outs.length 55 | const startProcess = ins.length === 0; 56 | if (!startProcess) { return; } 57 | 58 | walkConnectionGraph(process, (flow) => flows.push(flow)); 59 | }); 60 | 61 | return flows; 62 | }; 63 | 64 | // Notes on graphs and color 65 | // http://www.perceptualedge.com/articles/visual_business_intelligence/rules_for_using_color.pdf 66 | 67 | const renderFlow = function (flow, times) { 68 | let objects = []; 69 | 70 | const style = { 71 | baseHeight: 20, 72 | dividerHeightFraction: 0.2, 73 | divider: 'stroke:#000000;stroke-opacity:1', 74 | rect: 'fill:#77cdb8;fill-opacity:1', 75 | unitSeconds: 100, 76 | }; 77 | style.dividerHeight = style.baseHeight * (1 + style.dividerHeightFraction); 78 | 79 | // XXX: should times be based on process.port combo instead of just process? 80 | 81 | // calculate total and weights 82 | const weights = {}; 83 | let totalTime = 0; 84 | flow.forEach((conn) => { 85 | totalTime += times[conn.src.process]; 86 | }); 87 | 88 | debugPerf('total time', totalTime); 89 | 90 | flow.forEach((conn) => { 91 | const p = conn.src.process; 92 | weights[p] = times[p] / totalTime; 93 | }); 94 | 95 | let xPos = 0; 96 | const yPos = 0; 97 | const rects = []; 98 | const dividers = []; 99 | const labels = []; 100 | flow.forEach((conn) => { 101 | const p = conn.src.process; 102 | const weight = weights[p]; 103 | debugPerf('weight', p, weight); 104 | 105 | const width = times[p] * style.unitSeconds; 106 | const height = style.baseHeight; 107 | const nextPos = xPos + width; 108 | const rect = svg.node('rect', { 109 | x: xPos, y: yPos, width, height, style: style.rect, 110 | }); 111 | const dHeight = style.dividerHeight; 112 | const dBase = yPos - ((style.baseHeight * style.dividerHeightFraction) / 2); 113 | const divider = svg.node('path', { d: svg.line(nextPos, dBase, 0, dHeight), style: style.divider }); 114 | 115 | const midX = (xPos + nextPos) / 2; 116 | 117 | const timeMs = (times[p] * 1000).toFixed(0); 118 | const percent = (weights[p] * 100).toFixed(1); 119 | const t = 'rotate(-270)'; // flips X and Y 120 | const label = svg.node('text', { transform: t, y: -midX, x: yPos + 10 }, `${p}: ${timeMs} ms (${percent}%)`); 121 | 122 | rects.push(rect); 123 | dividers.push(divider); 124 | labels.push(label); 125 | xPos = nextPos; 126 | }); 127 | 128 | // need to take care of order so clipping is right 129 | objects = objects.concat(rects); 130 | objects = objects.concat(dividers); 131 | objects = objects.concat(labels); 132 | 133 | return objects; 134 | }; 135 | 136 | // Render timelines of a FBP flow, with size of each process proportional to 137 | // the time spent in that process 138 | // TODO: also render secondary flows 139 | // MAYBE: render forks and joins 140 | // MAYBE: support average and variance? Or perhaps this should be done by the 141 | // tool before. Percentile is also interesting 142 | const renderTimeline = function (graph, times) { 143 | let objects = []; 144 | 145 | debugPerf('processes', Object.keys(graph.processes)); 146 | 147 | let flows = extractFlows(graph); 148 | flows = flows.sort((a, b) => { 149 | if (a.length > b.length) { 150 | return 1; 151 | } 152 | return 0; 153 | }); 154 | const flow = flows[0]; // longest 155 | 156 | const pretty = flow.map((c) => `${c.src.process}.${c.src.port} -> ${c.tgt.process}.${c.tgt.port}`); 157 | debugPerf('rendering flow', pretty); 158 | 159 | objects = objects.concat(renderFlow(flow, times)); 160 | 161 | const output = `${objects.join('\n')}`; 162 | 163 | return output; 164 | }; 165 | 166 | const main = function () { 167 | const [,, graphfile] = Array.from(process.argv); 168 | 169 | // TODO: accept times on commandline 170 | const times = { 171 | pre: 1.00, 172 | queue: 2.50, 173 | download: 0.50, 174 | scale: 0.50, 175 | calc: 2.50, 176 | load: 2.50, 177 | save: 0.20, 178 | uploadOutput: 0.2, 179 | }; 180 | 181 | const callback = function (err, result) { 182 | if (err) { 183 | console.error(err); 184 | if (err.stack) { console.error(err.stack); } 185 | return process.exit(2); 186 | } 187 | debugPerf('output\n', result.length); 188 | console.log(result); 189 | return process.exit(0); 190 | }; 191 | 192 | return readGraphFile(graphfile, {}, (err, graph) => { 193 | if (err) { return callback(err); } 194 | const out = renderTimeline(graph, times); 195 | return callback(null, out); 196 | }); 197 | }; 198 | 199 | if (!module.parent) { main(); } 200 | -------------------------------------------------------------------------------- /src/bin/render.js: -------------------------------------------------------------------------------- 1 | import * as clc from 'cli-color'; 2 | import * as ansiStrip from 'cli-color/strip'; 3 | import * as trace from '../lib/trace'; 4 | 5 | const connectionId = function (data) { 6 | const { src, tgt } = data; 7 | 8 | if (src && tgt) { 9 | return `${src.node} ${src.port.toUpperCase()} -> ${tgt.port.toUpperCase()} ${tgt.node}`; 10 | } 11 | if (src) { 12 | return `${src.node} ${src.port.toUpperCase()} ->`; 13 | } 14 | if (tgt) { 15 | return `-> ${tgt.port.toUpperCase()} ${tgt.node}`; 16 | } 17 | return 'UNKNOWN'; 18 | }; 19 | 20 | const renderText = function (msg, options = {}) { 21 | if (msg.protocol !== 'network') { return null; } 22 | 23 | const identifier = function (data) { 24 | const id = connectionId(data); 25 | let result = ''; 26 | if (data.subgraph) { result += `${clc.magenta.italic(data.subgraph.join(':'))} `; } 27 | result += clc.blue.italic(id); 28 | return result; 29 | }; 30 | 31 | if (msg.error) { 32 | return `TRACE ERROR: ${msg.error}`; 33 | } 34 | 35 | const data = msg.payload; 36 | let text = (() => { 37 | switch (msg.command) { 38 | case 'connect': return `${identifier(data)} ${clc.yellow('CONN')}`; 39 | case 'disconnect': return `${identifier(data)} ${clc.yellow('DISC')}`; 40 | case 'begingroup': return `${identifier(data)} ${clc.cyan(`< ${data.group}`)}`; 41 | case 'endgroup': return `${identifier(data)} ${clc.cyan(`> ${data.group}`)}`; 42 | case 'data': 43 | if (options.verbose) { 44 | return `${identifier(data)} ${clc.green('DATA')} ${JSON.stringify(data.data)}`; 45 | } 46 | return `${identifier(data)} ${clc.green('DATA')}`; 47 | 48 | default: return null; 49 | } 50 | })(); 51 | 52 | if (!(typeof process !== 'undefined' && process !== null ? process.stdout.isTTY : undefined)) { 53 | // in case we are redirected to a file or similar 54 | text = ansiStrip(text); 55 | } 56 | 57 | return text; 58 | }; 59 | 60 | export default function main() { 61 | const filepath = process.argv[2]; 62 | 63 | const options = { verbose: true }; 64 | 65 | return trace.loadFile(filepath, (err, tr) => { 66 | if (err) { throw err; } 67 | // TODO: Render graphs to FBP? 68 | const result = []; 69 | tr.events.forEach((e) => { 70 | const text = renderText(e, options); 71 | if (text) { result.push(console.log(text)); } else { 72 | result.push(undefined); 73 | } 74 | }); 75 | return result; 76 | }); 77 | } 78 | -------------------------------------------------------------------------------- /src/bin/replay.js: -------------------------------------------------------------------------------- 1 | import * as querystring from 'querystring'; 2 | import * as program from 'commander'; 3 | import * as os from 'os'; 4 | import * as open from 'open'; 5 | import { Server } from 'http'; 6 | import debug from 'debug'; 7 | import * as trace from '../lib/trace'; 8 | import websocket from './websocket'; 9 | 10 | const debugProtocol = debug('flowtrace:replay'); 11 | const debugReceive = debug('flowtrace:replay:receive'); 12 | const debugSend = debug('flowtrace:replay:send'); 13 | 14 | const connectionId = function (conn) { 15 | // FIXME: remove when https://github.com/noflo/noflo-ui/issues/293 is fixed 16 | let src; 17 | if (conn.src && conn.src.node) { 18 | src = `${conn.src.node}() ${conn.src.port.toUpperCase()}`; 19 | } else { 20 | src = 'DATA'; 21 | } 22 | let tgt; 23 | if (conn.tgt) { 24 | tgt = `${conn.tgt.port.toUpperCase()} ${conn.tgt.node}()`; 25 | } else { 26 | tgt = 'OUT'; 27 | } 28 | return `${src} -> ${tgt}`; 29 | }; 30 | 31 | function mainGraphName(flowtrace) { 32 | let mainGraph = 'default/main'; 33 | if (!flowtrace.header) { 34 | return mainGraph; 35 | } 36 | if (flowtrace.header.graphs && Object.keys(flowtrace.header.graphs).length) { 37 | return Object.keys(flowtrace.header.graphs)[0]; 38 | } 39 | if (flowtrace.header.metadata && flowtrace.header.metadata.main) { 40 | mainGraph = flowtrace.header.metadata.main; 41 | } 42 | return mainGraph; 43 | } 44 | 45 | /** 46 | * @param {import('../lib/Flowtrace').FlowtraceJson} flowtrace 47 | * @param {Function} sendFunc 48 | * @param {Function} callback 49 | * @returns {void} 50 | */ 51 | const replayEvents = function (flowtrace, sendFunc, callback) { 52 | flowtrace.events.forEach((event) => { 53 | sendFunc({ 54 | ...event, 55 | payload: { 56 | ...event.payload, 57 | id: connectionId(event.payload), 58 | graph: event.graph || mainGraphName(flowtrace), 59 | }, 60 | }); 61 | }); 62 | return callback(null); 63 | }; 64 | 65 | const sendError = function (protocol, error, sendFunc) { 66 | sendFunc({ 67 | protocol, 68 | command: 'error', 69 | payload: { 70 | message: error.message, 71 | }, 72 | }); 73 | }; 74 | 75 | function addInport(componentDef, portDef) { 76 | if (componentDef.inPorts.find((p) => p.id === portDef.id)) { 77 | return; 78 | } 79 | componentDef.inPorts.push(portDef); 80 | } 81 | 82 | function addOutport(componentDef, portDef) { 83 | if (componentDef.outPorts.find((p) => p.id === portDef.id)) { 84 | return; 85 | } 86 | componentDef.outPorts.push(portDef); 87 | } 88 | 89 | /** 90 | * @param {Object} components 91 | * @param {import("fbp-graph/src/Types").GraphJson} graph 92 | * @param {Object.} graphs 93 | * @returns {Object} 94 | */ 95 | function componentsFromGraph(components, graph, graphs) { 96 | const newComponents = { 97 | ...components, 98 | }; 99 | Object.keys(graph.processes).forEach((nodeId) => { 100 | const node = graph.processes[nodeId]; 101 | if (!node.component) { 102 | return; 103 | } 104 | const componentDef = newComponents[node.component] || { 105 | name: node.component, 106 | icon: 'cog', 107 | description: '', 108 | subgraph: (Object.keys(graphs).indexOf(node.component) !== -1), 109 | inPorts: [], 110 | outPorts: [], 111 | }; 112 | Object.keys(graph.inports).forEach((portName) => { 113 | const portDef = graph.inports[portName]; 114 | if (portDef.process !== nodeId) { 115 | return; 116 | } 117 | addInport(componentDef, { 118 | id: portDef.port, 119 | type: 'all', 120 | }); 121 | }); 122 | Object.keys(graph.outports).forEach((portName) => { 123 | const portDef = graph.outports[portName]; 124 | if (portDef.process !== nodeId) { 125 | return; 126 | } 127 | addOutport(componentDef, { 128 | id: portDef.port, 129 | type: 'all', 130 | }); 131 | }); 132 | graph.connections.forEach((edge) => { 133 | if (edge.src && edge.src.process === nodeId) { 134 | addOutport(componentDef, { 135 | id: edge.src.port, 136 | type: 'all', 137 | }); 138 | } 139 | if (edge.tgt && edge.tgt.process === nodeId) { 140 | addInport(componentDef, { 141 | id: edge.tgt.port, 142 | type: 'all', 143 | }); 144 | } 145 | }); 146 | newComponents[node.component] = componentDef; 147 | }); 148 | return newComponents; 149 | } 150 | /** 151 | * @param {import('../lib/Flowtrace').FlowtraceJson} flowtrace 152 | * @param {Function} sendFunc 153 | * @param {Function} callback 154 | * @returns {void} 155 | */ 156 | const sendComponents = function (flowtrace, sendFunc, callback) { 157 | // XXX: should the trace also store component info?? 158 | // maybe optional. 159 | const graphs = flowtrace.header != null ? flowtrace.header.graphs : {}; 160 | let components = {}; 161 | Object.keys(graphs).forEach((graph) => { 162 | components = componentsFromGraph(components, graphs[graph], graphs); 163 | }); 164 | 165 | Object.keys(components).forEach((componentName) => { 166 | sendFunc({ 167 | protocol: 'component', 168 | command: 'component', 169 | payload: components[componentName], 170 | }); 171 | }); 172 | 173 | // Send all graphs as components 174 | sendFunc({ 175 | protocol: 'component', 176 | command: 'componentsready', 177 | payload: Object.keys(components).length, 178 | }); 179 | callback(null); 180 | }; 181 | 182 | /** 183 | * @param {import('../lib/Flowtrace').FlowtraceJson} flowtrace 184 | * @param {string} graphName 185 | * @param {Function} sendFunc 186 | * @returns {void} 187 | */ 188 | const sendGraphSource = function (flowtrace, graphName, sendFunc) { 189 | // FIXME: get rid of this workaround for https://github.com/noflo/noflo-ui/issues/390 190 | 191 | const graphs = flowtrace.header != null ? flowtrace.header.graphs : {}; 192 | const currentGraph = graphs[graphName]; 193 | if (!currentGraph) { 194 | sendError('component', new Error(`Graph ${graphName} not found`), sendFunc); 195 | return; 196 | } 197 | let library; 198 | let name = graphName; 199 | if (graphName.indexOf('/') !== -1) { 200 | [library, name] = graphName.split('/'); 201 | } 202 | const payload = { 203 | name, 204 | library, 205 | language: 'json', 206 | code: JSON.stringify(currentGraph, null, 2), 207 | }; 208 | sendFunc({ 209 | protocol: 'component', 210 | command: 'source', 211 | payload, 212 | }); 213 | }; 214 | 215 | const flowhubLiveUrl = function (options) { 216 | const address = `ws://${options.host}:${options.port}`; 217 | const query = [ 218 | 'protocol=websocket', 219 | `address=${address}`, 220 | // TODO: ID 221 | ]; 222 | if (options.secret) { 223 | query.push(`secret=${options.secret}`); 224 | } 225 | 226 | return `${options.ide}#runtime/endpoint?${querystring.escape(query.join('&'))}`; 227 | }; 228 | 229 | const knownUnsupportedCommands = (p, c) => (p === 'network') && (c === 'debug'); 230 | 231 | /** 232 | * @param {string} [preferredInterface] 233 | * @returns {string} 234 | */ 235 | function discoverHost(preferredInterface) { 236 | const ifaces = os.networkInterfaces(); 237 | let address; 238 | let internalAddress; 239 | 240 | const filter = function (connection) { 241 | if (connection.family !== 'IPv4') { 242 | return; 243 | } 244 | if (connection.internal) { 245 | internalAddress = connection.address; 246 | } else { 247 | ({ 248 | address, 249 | } = connection); 250 | } 251 | }; 252 | 253 | if ((preferredInterface) && ifaces[preferredInterface]) { 254 | ifaces[preferredInterface].forEach(filter); 255 | } else { 256 | Object.keys(ifaces).forEach((device) => { 257 | ifaces[device].forEach(filter); 258 | }); 259 | } 260 | return address || internalAddress; 261 | } 262 | 263 | const normalizeOptions = function (options) { 264 | const opts = options; 265 | if (opts.host === 'autodetect') { 266 | opts.host = discoverHost(); 267 | } else if (/autodetect\(([a-z0-9]+)\)/.exec(options.host)) { 268 | const match = /autodetect\(([a-z0-9]+)\)/.exec(options.host); 269 | opts.host = discoverHost(match[1]); 270 | } 271 | 272 | return opts; 273 | }; 274 | 275 | const parse = function () { 276 | program 277 | .arguments('') 278 | .action((flowtrace) => { 279 | program.trace = flowtrace; 280 | }) 281 | .option('--ide ', 'FBP IDE to use for live-url', String, 'http://app.flowhub.io') 282 | .option('--host ', 'Hostname we serve on, for live-url', String, 'autodetect') 283 | .option('--port ', 'Command to launch runtime under test', Number, 3333) 284 | .option('-n --no-open', 'Automatically open replayed trace in browser', Boolean, true) 285 | .parse(process.argv); 286 | 287 | return program; 288 | }; 289 | 290 | exports.main = function () { 291 | let options = parse(process.argv); 292 | options = normalizeOptions(options); 293 | const filepath = options.trace; 294 | 295 | /** 296 | * @type {import('../lib/Flowtrace').FlowtraceJson | null} 297 | */ 298 | let mytrace = null; 299 | const httpServer = new Server(); 300 | const runtime = websocket(httpServer, {}); 301 | 302 | runtime.receive = (protocol, command, payload, context) => { 303 | debugReceive(protocol, command, payload); 304 | let status = { 305 | graph: mainGraphName(mytrace), 306 | started: false, 307 | running: false, 308 | }; 309 | const updateStatus = (news, event) => { 310 | status = { 311 | ...status, 312 | ...news, 313 | }; 314 | runtime.send('network', event, status, context); 315 | }; 316 | 317 | const send = (e) => { 318 | debugSend(e.protocol, e.command, e.payload); 319 | runtime.send(e.protocol, e.command, e.payload, context); 320 | }; 321 | 322 | switch (`${protocol}:${command}`) { 323 | case 'runtime:getruntime': { 324 | const capabilities = [ 325 | 'protocol:component', // read-only from client 326 | 'protocol:network', 327 | 'component:getsource', 328 | ]; 329 | const info = { 330 | type: 'flowtrace-replay', 331 | version: '0.5', 332 | capabilities, 333 | allCapabilities: capabilities, 334 | graph: mainGraphName(mytrace), 335 | }; 336 | send({ 337 | protocol: 'runtime', 338 | command: 'runtime', 339 | payload: info, 340 | }); 341 | return; 342 | // ignored 343 | } 344 | case 'network:getstatus': { 345 | send({ 346 | protocol: 'network', 347 | command: 'status', 348 | payload: status, 349 | }); 350 | return; 351 | } 352 | case 'network:start': { 353 | // replay our trace 354 | if (!mytrace) { return; } 355 | updateStatus({ 356 | started: true, 357 | running: true, 358 | }, 'started'); 359 | replayEvents(mytrace, send, () => updateStatus({ 360 | started: true, 361 | running: false, 362 | }, 'stopped')); 363 | return; 364 | } 365 | case 'network:edges': { 366 | send({ 367 | protocol: 'network', 368 | command: 'edges', 369 | payload: { 370 | edges: payload.edges, 371 | graph: payload.graph, 372 | }, 373 | }); 374 | return; 375 | } 376 | case 'component:list': { 377 | sendComponents(mytrace, send, () => { }); 378 | return; 379 | } 380 | case 'component:getsource': { 381 | sendGraphSource(mytrace, payload.name, send); 382 | return; 383 | } 384 | default: { 385 | if (!knownUnsupportedCommands(protocol, command)) { 386 | debugProtocol('Warning: Unknown FBP protocol message', protocol, command); 387 | } 388 | } 389 | } 390 | }; 391 | 392 | trace.loadFile(filepath, (err, tr) => { 393 | if (err) { throw err; } 394 | mytrace = tr; 395 | httpServer.listen(options.port, (listenErr) => { 396 | if (listenErr) { throw listenErr; } 397 | 398 | const liveUrl = flowhubLiveUrl(options); 399 | console.log('Trace live URL:', liveUrl); 400 | if (options.open) { 401 | open(liveUrl) 402 | .then( 403 | () => { 404 | console.log('Opened in browser'); 405 | }, 406 | (openErr) => { 407 | console.log('Failed to open live URL in browser:', openErr); 408 | }, 409 | ); 410 | } 411 | }); 412 | }); 413 | }; 414 | -------------------------------------------------------------------------------- /src/bin/websocket.js: -------------------------------------------------------------------------------- 1 | /* eslint class-methods-use-this: ["error", { "exceptMethods": ["send", "receive"] }] */ 2 | import { server as WebSocketServer } from 'websocket'; 3 | 4 | // XXX: Copied from https://github.com/noflo/noflo-runtime-websocket/blob/master/runtime/network.js 5 | // should probably reuse it as-is 6 | 7 | class WebSocketRuntime { 8 | constructor(options = {}) { 9 | this.options = options; 10 | this.connections = []; 11 | } 12 | 13 | /** 14 | * @param {string} protocol 15 | * @param {string} command 16 | * @param {Object} payload 17 | * @param {any} context 18 | * @abstract 19 | */ 20 | receive(protocol, command, payload, context) {} // eslint-disable-line no-unused-vars 21 | 22 | /** 23 | * @param {string} protocol 24 | * @param {string} topic 25 | * @param {Object} payload 26 | * @param {any} context 27 | */ 28 | send(protocol, topic, payload, context) { 29 | if (!context.connection || !context.connection.connected) { 30 | return; 31 | } 32 | context.connection.sendUTF(JSON.stringify({ 33 | protocol, 34 | command: topic, 35 | payload, 36 | })); 37 | } 38 | 39 | /** 40 | * @param {string} protocol 41 | * @param {string} topic 42 | * @param {Object} payload 43 | */ 44 | sendAll(protocol, topic, payload) { 45 | this.connections.forEach((connection) => { 46 | this.send(protocol, topic, payload, { 47 | connection, 48 | }); 49 | }); 50 | } 51 | } 52 | 53 | export default function createServer(httpServer, options) { 54 | const wsServer = new WebSocketServer({ httpServer }); 55 | const runtime = new WebSocketRuntime(options); 56 | 57 | const handleMessage = function (message, connection) { 58 | if (message.type === 'utf8') { 59 | let contents; 60 | try { 61 | contents = JSON.parse(message.utf8Data); 62 | } catch (e) { 63 | if (e.stack) { 64 | console.error(e.stack); 65 | } else { 66 | console.error(`Error: ${e.toString()}`); 67 | } 68 | return; 69 | } 70 | runtime.receive(contents.protocol, contents.command, contents.payload, { connection }); 71 | } 72 | }; 73 | 74 | wsServer.on('request', (request) => { 75 | const subProtocol = request.requestedProtocols.indexOf('noflo') !== -1 ? 'noflo' : null; 76 | const connection = request.accept(subProtocol, request.origin); 77 | runtime.connections.push(connection); 78 | connection.on('message', (message) => handleMessage(message, connection)); 79 | return connection.on('close', () => { 80 | if (runtime.connections.indexOf(connection) === -1) { return; } 81 | runtime.connections.splice(runtime.connections.indexOf(connection), 1); 82 | }); 83 | }); 84 | 85 | return runtime; 86 | } 87 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import * as trace from './lib/trace'; 2 | 3 | export { default as Flowtrace } from './lib/Flowtrace'; 4 | export { trace }; 5 | export { default as FlowtraceRecorder } from './lib/record'; 6 | -------------------------------------------------------------------------------- /src/lib/Flowtrace.js: -------------------------------------------------------------------------------- 1 | import * as CircularBuffer from 'circular-buffer'; 2 | import * as clone from 'clone'; 3 | import { EventEmitter } from 'events'; 4 | 5 | /** 6 | * @typedef {Object} PacketPort 7 | * @property {string} node 8 | * @property {string} port 9 | * @property {number} [index] 10 | */ 11 | 12 | /** 13 | * @typedef {Object} FlowtraceMetadata 14 | * @property {string} [label] 15 | * @property {string} [runtime] 16 | * @property {string} [type] 17 | * @property {string} [address] 18 | * @property {string} [namespace] 19 | * @property {string} [repository] 20 | * @property {string} [repositoryVersion] 21 | * @property {Date} [start] 22 | * @property {Date} [end] 23 | */ 24 | 25 | /** 26 | * @typedef {Object} FlowtraceJsonHeader 27 | * @property {FlowtraceMetadata} metadata 28 | * @property {Object.} graphs 29 | * @property {string} main 30 | */ 31 | 32 | /** 33 | * @typedef {Object} FlowtraceJsonEvent 34 | * @property {string} protocol 35 | * @property {string} command 36 | * @property {Object} payload 37 | * @property {string} graph 38 | * @property {Date} time 39 | */ 40 | 41 | /** 42 | * @typedef {Object} FlowtraceJson 43 | * @property {FlowtraceJsonHeader} header 44 | * @property {FlowtraceJsonEvent[]} events 45 | */ 46 | 47 | export default class Flowtrace extends EventEmitter { 48 | /** 49 | * @param {FlowtraceMetadata} metadata 50 | * @param {number} bufferSize 51 | */ 52 | constructor(metadata, bufferSize = 400) { 53 | super(); 54 | this.bufferSize = bufferSize; 55 | this.graphs = {}; 56 | this.metadata = { 57 | ...metadata, 58 | start: new Date(), 59 | }; 60 | /** 61 | * @type {string | null} 62 | */ 63 | this.mainGraph = null; 64 | this.clear(); 65 | this.subscribe(); 66 | } 67 | 68 | /** 69 | * @returns {void} 70 | */ 71 | clear() { 72 | this.events = new CircularBuffer(this.bufferSize); 73 | } 74 | 75 | subscribe() { 76 | this.on('event', (event, payload, graph) => { 77 | let clonedPayload; 78 | try { 79 | clonedPayload = clone(payload); 80 | } catch (e) { 81 | // Some data packets can't be serialized 82 | clonedPayload = { 83 | ...payload, 84 | data: `DATA ${typeof payload.data}`, 85 | }; 86 | } 87 | this.events.enq({ 88 | event, 89 | payload: clonedPayload, 90 | graph, 91 | time: new Date(), 92 | }); 93 | }); 94 | } 95 | 96 | /** 97 | * @param {string} graphName 98 | * @param {import("fbp-graph").Graph} graph 99 | * @param {boolean} [main] 100 | * @returns {void} 101 | */ 102 | addGraph(graphName, graph, main = false) { 103 | this.graphs[graphName] = graph; 104 | if (main) { 105 | this.mainGraph = graphName; 106 | } 107 | } 108 | 109 | /** 110 | * @param {string} type 111 | * @param {PacketPort | null} src 112 | * @param {PacketPort | null} tgt 113 | * @param {string} graph 114 | * @param {Object} payload 115 | * @returns {void} 116 | */ 117 | addNetworkPacket(type, src, tgt, graph, payload) { 118 | this.emit('event', type, { 119 | ...payload, 120 | src, 121 | tgt, 122 | }, graph); 123 | } 124 | 125 | /** 126 | * @param {string} graph 127 | * @returns {void} 128 | */ 129 | addNetworkStarted(graph) { 130 | this.emit('event', 'network:started', {}, graph); 131 | } 132 | 133 | /** 134 | * @param {string} graph 135 | * @returns {void} 136 | */ 137 | addNetworkStopped(graph) { 138 | this.emit('event', 'network:stopped', {}, graph); 139 | } 140 | 141 | /** 142 | * @param {string} graph 143 | * @param {Error} error 144 | * @returns {void} 145 | */ 146 | addNetworkError(graph, error) { 147 | this.emit('event', 'network:error', error, graph); 148 | } 149 | 150 | /** 151 | * @param {string} graph 152 | * @param {Error} error 153 | * @returns {void} 154 | */ 155 | addNetworkProcessError(graph, error) { 156 | this.emit('event', 'network:processerror', error, graph); 157 | } 158 | 159 | /** 160 | * @param {string} graph 161 | * @returns {void} 162 | */ 163 | addNetworkIcon(graph, node, icon) { 164 | this.emit('event', 'network:icon', { 165 | node, 166 | icon, 167 | }, graph); 168 | } 169 | 170 | /** 171 | * @param {string} graph 172 | * @param {Object} payload 173 | * @param {string} payload.message 174 | * @param {string} [payload.type] 175 | * @param {string} [payload.previewurl] 176 | * @returns {void} 177 | */ 178 | addNetworkOutput(graph, payload) { 179 | this.emit('event', 'network:output', payload, graph); 180 | } 181 | 182 | /** 183 | * @returns {FlowtraceJson} 184 | */ 185 | toJSON() { 186 | const tracedEvents = this.events.toarray().map((event) => { 187 | const [protocol, command] = event.event.split(':'); 188 | return { 189 | protocol, 190 | command, 191 | payload: event.payload, 192 | graph: event.graph, 193 | time: event.time, 194 | }; 195 | }); 196 | tracedEvents.reverse(); 197 | return { 198 | header: { 199 | metadata: { 200 | ...this.metadata, 201 | end: this.metadata.end || new Date(), 202 | }, 203 | graphs: this.graphs, 204 | main: this.mainGraph, 205 | }, 206 | events: tracedEvents, 207 | }; 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/lib/common.js: -------------------------------------------------------------------------------- 1 | import { parse } from 'fbp'; 2 | import { extname } from 'path'; 3 | import { readFile } from 'fs'; 4 | 5 | export function randomString(n) { 6 | let idx; 7 | let j; 8 | let ref; 9 | let text = ''; 10 | const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; 11 | 12 | for (j = 0, ref = n; ref >= 0 ? j < ref : j > ref; ref >= 0 ? j += 1 : j -= 1) { 13 | idx = Math.floor(Math.random() * possible.length); 14 | text += possible.charAt(idx); 15 | } 16 | 17 | return text; 18 | } 19 | 20 | export function isBrowser() { 21 | if ((typeof process !== 'undefined') && process.execPath && process.execPath.match(/node|iojs/)) { 22 | return false; 23 | } 24 | return true; 25 | } 26 | 27 | export function readGraph(contents, type, options) { 28 | let graph; 29 | if (type === 'fbp') { 30 | graph = parse(contents, { caseSensitive: options.caseSensitive }); 31 | } else if (type === 'object') { 32 | graph = contents; 33 | } else { 34 | graph = JSON.parse(contents); 35 | } 36 | 37 | // Normalize optional params 38 | if ((graph.inports == null)) { graph.inports = {}; } 39 | if ((graph.outports == null)) { graph.outports = {}; } 40 | 41 | return graph; 42 | } 43 | 44 | // node.js only 45 | export function readGraphFile(filepath, options, callback) { 46 | const type = extname(filepath).replace('.', ''); 47 | return readFile(filepath, { encoding: 'utf-8' }, (err, contents) => { 48 | let graph; 49 | if (err) { return callback(err); } 50 | try { 51 | graph = readGraph(contents, type, options); 52 | } catch (e) { 53 | return callback(e); 54 | } 55 | return callback(null, graph); 56 | }); 57 | } 58 | -------------------------------------------------------------------------------- /src/lib/record.js: -------------------------------------------------------------------------------- 1 | import { graph } from 'fbp-graph'; 2 | import Flowtrace from './Flowtrace'; 3 | 4 | function loadGraph(source) { 5 | return new Promise((resolve, reject) => { 6 | const method = (source.language === 'fbp') ? 'loadFBP' : 'loadJSON'; 7 | graph[method](source.code, (err, instance) => { 8 | if (err) { 9 | reject(err); 10 | return; 11 | } 12 | resolve(instance.toJSON()); 13 | }); 14 | }); 15 | } 16 | 17 | function filterGraphs(main, graphs) { 18 | let filtered = []; 19 | const mainGraph = graphs.find((g) => g.name === main); 20 | if (!mainGraph) { 21 | return filtered; 22 | } 23 | filtered.push(mainGraph); 24 | const graphNames = graphs.map((g) => g.name); 25 | Object.keys(mainGraph.processes).forEach((nodeId) => { 26 | const node = mainGraph.processes[nodeId]; 27 | const graphIdx = graphNames.indexOf(node.component); 28 | if (graphIdx === -1) { 29 | // Not a subgraph 30 | return; 31 | } 32 | filtered = filtered.concat(filterGraphs(node.component, graphs)); 33 | }); 34 | return filtered; 35 | } 36 | 37 | function loadGraphs(fbpClient, tracer, mainGraph) { 38 | // TODO: Instead of fetching all subgraphs, might be more 39 | // efficient to load the main graph, and then recurse 40 | return fbpClient.protocol.component.list() 41 | .then((components) => components.filter((c) => (c.subgraph && c.name !== 'Graph'))) 42 | .then((subgraphs) => { 43 | const graphNames = subgraphs.map((c) => c.name); 44 | if (graphNames.indexOf(mainGraph) === -1) { 45 | graphNames.push(mainGraph); 46 | } 47 | const graphsMissing = graphNames.filter((c) => { 48 | if (tracer.graphs[c]) { 49 | return false; 50 | } 51 | return true; 52 | }); 53 | return Promise.all(graphsMissing 54 | .map((graphName) => fbpClient 55 | .protocol.component.getsource({ 56 | name: graphName, 57 | }) 58 | .then(loadGraph) 59 | .then((graphDefinition) => ({ 60 | ...graphDefinition, 61 | name: graphName, 62 | })))) 63 | .then((graphs) => { 64 | const filtered = filterGraphs(mainGraph, graphs); 65 | filtered.forEach((graphDefinition) => { 66 | const main = (graphDefinition.name === mainGraph); 67 | tracer.addGraph(graphDefinition.name, graphDefinition, main); 68 | }); 69 | }); 70 | }); 71 | } 72 | 73 | export default class FlowtraceRecorder { 74 | constructor(fbpClient) { 75 | this.fbpClient = fbpClient; 76 | this.traces = {}; 77 | this.signalHandlers = {}; 78 | } 79 | 80 | handleSignal(graphName) { 81 | if (this.signalHandlers[graphName]) { 82 | return this.signalHandlers[graphName]; 83 | } 84 | const tracer = this.traces[graphName]; 85 | const handler = (signal) => { 86 | if (signal.payload.graph !== graphName) { 87 | return; 88 | } 89 | const event = `${signal.protocol}:${signal.command}`; 90 | switch (event) { 91 | case 'network:connect': 92 | case 'network:begingroup': 93 | case 'network:data': 94 | case 'network:endgroup': 95 | case 'network:disconnect': { 96 | tracer.addNetworkPacket( 97 | event, 98 | signal.payload.src, 99 | signal.payload.tgt, 100 | signal.payload.graph, 101 | signal.payload, 102 | ); 103 | break; 104 | } 105 | case 'network:error': { 106 | tracer.addNetworkError(signal.payload.graph, signal.payload); 107 | break; 108 | } 109 | case 'network:icon': { 110 | tracer.addNetworkIcon(signal.payload.graph, signal.payload.id, signal.payload.icon); 111 | break; 112 | } 113 | default: { 114 | // Ignore 115 | } 116 | } 117 | }; 118 | this.signalHandlers[graphName] = handler; 119 | return handler; 120 | } 121 | 122 | start(graphName) { 123 | return this.fbpClient.connect() 124 | .then(() => { 125 | // Prep trace with runtime metadata 126 | this.traces[graphName] = new Flowtrace({ 127 | runtime: this.fbpClient.definition.id, 128 | type: this.fbpClient.definition.type, 129 | address: this.fbpClient.definition.address, 130 | repository: this.fbpClient.definition.repository, 131 | repositoryVersion: this.fbpClient.definition.repositoryVersion, 132 | }); 133 | }) 134 | .then(() => { 135 | this.fbpClient.on('signal', this.handleSignal(graphName)); 136 | }) 137 | .then(() => loadGraphs(this.fbpClient, this.traces[graphName], graphName)); 138 | } 139 | 140 | stop(graphName) { 141 | if (!this.traces[graphName]) { 142 | return Promise.resolve(); 143 | } 144 | this.fbpClient.removeListener('signal', this.handleSignal(graphName)); 145 | this.traces[graphName].metadata.end = new Date(); 146 | return Promise.resolve(); 147 | } 148 | 149 | dump(graphName) { 150 | if (!this.traces[graphName]) { 151 | // No trace for this graph yet 152 | return {}; 153 | } 154 | return this.traces[graphName].toJSON(); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/lib/trace.js: -------------------------------------------------------------------------------- 1 | import { readFile } from 'fs'; 2 | import 'isomorphic-fetch'; 3 | 4 | exports.loadString = (str) => JSON.parse(str); 5 | 6 | /** 7 | * @callback FlowtraceCallback 8 | * @param {Error | null} err 9 | * @param {import("./Flowtrace").FlowtraceJson} [flowtrace] 10 | * @returns {void} 11 | */ 12 | 13 | /** 14 | * @param {string} filepath 15 | * @param {FlowtraceCallback} callback 16 | * @returns {void} 17 | */ 18 | export function loadFile(filepath, callback) { 19 | readFile(filepath, { encoding: 'utf-8' }, (err, contents) => { 20 | let trace; 21 | if (err) { 22 | callback(err); 23 | return; 24 | } 25 | try { 26 | trace = exports.loadString(contents); 27 | } catch (e) { 28 | callback(e); 29 | return; 30 | } 31 | callback(null, trace); 32 | }); 33 | } 34 | 35 | /** 36 | * @param {string} url 37 | * @param {FlowtraceCallback} callback 38 | * @returns {void} 39 | */ 40 | export function loadHttp(url, callback) { 41 | fetch(url) 42 | .then((response) => { 43 | if (response.status !== 200) { 44 | throw new Error(`Received HTTP error ${response.status}`); 45 | } 46 | return response.json(); 47 | }) 48 | .then( 49 | (content) => { 50 | callback(null, content); 51 | }, 52 | (err) => { 53 | callback(err); 54 | }, 55 | ); 56 | } 57 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "allowJs": true, 6 | "checkJs": true, 7 | "outDir": "./dist", 8 | "declaration": true, 9 | "strict": false, 10 | "rootDir": "./src" 11 | }, 12 | "include": [ 13 | "src/*.js", 14 | "src/**/*.js" 15 | ] 16 | } 17 | --------------------------------------------------------------------------------