├── demo ├── .gitignore ├── std_nodes │ ├── .gitignore │ ├── file_reader │ │ ├── .gitignore │ │ ├── data.csv │ │ ├── data.txt │ │ ├── data.ldjson │ │ ├── topology_long.json │ │ ├── demo_file_reader.js │ │ ├── topology.json │ │ └── demo_file_reader_long.js │ ├── file_append │ │ ├── .gitignore │ │ ├── demo_file_append.js │ │ └── topology.json │ ├── file_append_csv │ │ ├── .gitignore │ │ ├── demo_file_append_csv.js │ │ └── topology.json │ ├── file_append_ex │ │ ├── .gitignore │ │ ├── demo_file_append_ex.js │ │ └── topology.json │ ├── process-bolt │ │ ├── cpp │ │ │ ├── .gitignore │ │ │ └── demo.cpp │ │ ├── child.js │ │ ├── topology.json │ │ ├── topology_cpp.json │ │ └── demo_process_bolt.js │ ├── process │ │ ├── data.csv │ │ ├── data.txt │ │ ├── data.ldjson │ │ ├── demo_process.js │ │ └── topology.json │ ├── disabled │ │ ├── init_and_shutdown.js │ │ ├── init_and_shutdown2.js │ │ ├── demo_disabling.js │ │ └── topology.json │ ├── task_bolt_base │ │ ├── topology.json │ │ ├── custom_task.js │ │ └── demo_task_bolt_base.js │ ├── rss │ │ ├── topology.json │ │ └── demo_rss.js │ ├── dir_watcher │ │ ├── topology.json │ │ └── demo_dir_watcher.js │ ├── rest │ │ ├── demo_rest.js │ │ └── topology.json │ ├── telemetry │ │ ├── demo_telemetry.js │ │ └── topology.json │ ├── bomb │ │ ├── topology.json │ │ └── demo_bomb.js │ ├── process-continuous │ │ ├── emitter.js │ │ ├── demo_process_continuous.js │ │ └── topology.json │ ├── counter │ │ ├── topology.json │ │ └── demo_counter.js │ └── demo_std_nodes.js ├── qminer │ ├── .gitignore │ ├── bolt_qm.js │ ├── spout_1.js │ ├── topology.json │ ├── demo_qminer.js │ └── spouts.js ├── local_massive │ ├── bolt_inproc.js │ ├── spout_inproc.js │ ├── init_and_shutdown.js │ ├── topology.json │ ├── bolt_common.js │ ├── demo_local_massive.js │ └── spout_common.js ├── util │ ├── web_server │ │ ├── a.js │ │ ├── server.js │ │ └── a.html │ └── child_process_restarter │ │ ├── child.js │ │ ├── parent_fork.js │ │ ├── parent_spawn.js │ │ └── parent.js ├── async │ ├── top.js │ ├── my_bolt.js │ ├── topology.json │ └── my_spout.js ├── distributed_http_based │ ├── storage_test.js │ ├── topology.json │ └── worker_test.js ├── local │ ├── init_and_shutdown.js │ ├── demo_local.js │ ├── bolt_inproc.js │ └── spout_inproc.js ├── quick_start │ ├── topology.json │ ├── top.js │ ├── my_bolt.js │ └── my_spout.js ├── distributed_file_based │ ├── init_and_shutdown.js │ ├── worker_test.js │ └── topologies │ │ ├── topology1.json │ │ └── topology2.json ├── gui │ ├── topology.2.json │ ├── demo-express.js │ └── demo.js ├── cli │ └── demo-repl.js └── run_demos.sh ├── .jshintignore ├── docs ├── uml │ ├── .gitignore │ ├── readme.txt │ ├── state_worker.uml │ ├── sequence_register.uml │ ├── package.json │ ├── state_topology.uml │ ├── run.js │ ├── components.uml │ ├── sequence_worker.uml │ ├── sequence_leader.uml │ └── package-lock.json ├── _config.yml ├── presentation.pdf ├── release-procedures.md └── index.md ├── resources └── gui │ ├── logo.png │ ├── favicon.ico │ └── logo_transparent.png ├── src ├── distributed │ ├── topology_local_wrapper_main.ts │ ├── http_based │ │ └── rest_client.ts │ └── file_based │ │ └── file_storage.ts ├── util │ ├── callback_wrappers.ts │ ├── object_override.ts │ ├── freq_estimator.ts │ ├── schema_test.js │ ├── telemetry.ts │ ├── stream_helpers.ts │ ├── topology_config_example.json │ ├── crontab_parser.ts │ ├── pattern_matcher.ts │ └── strip_json_comments.ts ├── std_nodes │ ├── forward_bolt.ts │ ├── attacher_bolt.ts │ ├── console_bolt.ts │ ├── get_bolt.ts │ ├── filter_bolt.ts │ ├── bomb_bolt.ts │ ├── post_bolt.ts │ ├── router_bolt.ts │ ├── timer_spout.ts │ ├── task_bolt_base.ts │ ├── counter_bolt.ts │ ├── test_spout.ts │ ├── dir_watcher_spout.ts │ ├── get_spout.ts │ ├── rss_spout.ts │ ├── parsing_utils.ts │ └── process_bolt.ts ├── index.ts ├── topology_validation.ts └── topology_async_wrappers.ts ├── tests ├── distributed │ └── dummy_topology.json ├── std_nodes │ ├── simple_proc.js │ ├── console_bolt.tests.js │ ├── type_transform_bolt.tests.js │ └── attacher_bolt.tests.js ├── util │ ├── telemetry.tests.js │ ├── strip_json_comments.js │ └── freq_estimator.tests.js └── helpers │ ├── test_inproc.js │ ├── bad_bolt.js │ └── bad_spout.js ├── .npmignore ├── tasks.json ├── .github └── workflows │ └── node2.yml ├── .gitignore ├── tsconfig.json ├── tslint.json ├── LICENSE ├── package.json └── README.md /demo/.gitignore: -------------------------------------------------------------------------------- 1 | /debug 2 | -------------------------------------------------------------------------------- /.jshintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /demo/std_nodes/.gitignore: -------------------------------------------------------------------------------- 1 | ./logs 2 | -------------------------------------------------------------------------------- /docs/uml/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/.gitignore: -------------------------------------------------------------------------------- 1 | /data.long.txt 2 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append/.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.gzip 3 | *.gz 4 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_csv/.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.gzip 3 | *.gz 4 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_ex/.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.gzip 3 | *.gz 4 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/cpp/.gitignore: -------------------------------------------------------------------------------- 1 | *.out 2 | *.obj 3 | *.exe 4 | -------------------------------------------------------------------------------- /demo/std_nodes/process/data.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,6 4 | 7,8,9 5 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/data.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,6 4 | 7,8,9 5 | -------------------------------------------------------------------------------- /docs/presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qminer/qtopology/HEAD/docs/presentation.pdf -------------------------------------------------------------------------------- /resources/gui/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qminer/qtopology/HEAD/resources/gui/logo.png -------------------------------------------------------------------------------- /resources/gui/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qminer/qtopology/HEAD/resources/gui/favicon.ico -------------------------------------------------------------------------------- /resources/gui/logo_transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qminer/qtopology/HEAD/resources/gui/logo_transparent.png -------------------------------------------------------------------------------- /demo/qminer/.gitignore: -------------------------------------------------------------------------------- 1 | /db 2 | ./db1 3 | ./db2 4 | /models 5 | ./model1 6 | ./model2 7 | /db1 8 | /db2 9 | /model1 10 | /model2 11 | -------------------------------------------------------------------------------- /demo/std_nodes/process/data.txt: -------------------------------------------------------------------------------- 1 | This is the first line 2 | This is the second line 3 | 4 | We skip empty lines 5 | 6 | 7 | And the last line 8 | 9 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/data.txt: -------------------------------------------------------------------------------- 1 | This is the first line 2 | This is the second line 3 | 4 | We skip empty lines 5 | 6 | 7 | And the last line 8 | 9 | -------------------------------------------------------------------------------- /demo/qminer/bolt_qm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const bolt = require("./bolts"); 4 | 5 | exports.create = function () { 6 | return new bolt.QMinerBolt(); 7 | }; 8 | -------------------------------------------------------------------------------- /demo/qminer/spout_1.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let spt = require("./spouts"); 4 | 5 | exports.create = function () { 6 | return new spt.DummySpout(); 7 | }; 8 | -------------------------------------------------------------------------------- /src/distributed/topology_local_wrapper_main.ts: -------------------------------------------------------------------------------- 1 | import * as tlw from "./topology_local_wrapper"; 2 | const top = new tlw.TopologyLocalWrapper(); 3 | top.start(); 4 | -------------------------------------------------------------------------------- /docs/uml/readme.txt: -------------------------------------------------------------------------------- 1 | Open online editor for PlantUML at: 2 | 3 | http://www.plantuml.com/plantuml/uml/ 4 | 5 | Then download SVG file and put it into imgs directory. 6 | -------------------------------------------------------------------------------- /docs/uml/state_worker.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | [*] --> alive 3 | alive -> dead : by worker or timeout 4 | dead -> unloaded: by leader 5 | unloaded --> alive: by worker 6 | @enduml 7 | -------------------------------------------------------------------------------- /tests/distributed/dummy_topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 3000 4 | }, 5 | "spouts": [], 6 | "bolts": [], 7 | "variables": {} 8 | } 9 | -------------------------------------------------------------------------------- /demo/local_massive/bolt_inproc.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const bolt = require("./bolt_common"); 4 | 5 | exports.create = function () { 6 | return new bolt.MyBolt(); 7 | }; 8 | -------------------------------------------------------------------------------- /demo/local_massive/spout_inproc.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let spt = require("./spout_common"); 4 | 5 | exports.create = function () { 6 | return new spt.MySpout(); 7 | }; 8 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | demo 3 | docs 4 | node_modules 5 | src 6 | tests 7 | .gitignore 8 | .jshintignore 9 | .jshintrc 10 | .travis.yml 11 | tasks.json 12 | tsconfig.json 13 | -------------------------------------------------------------------------------- /demo/util/web_server/a.js: -------------------------------------------------------------------------------- 1 | let elem = document.getElementById("divTarget"); 2 | if (elem) { 3 | elem.innerHTML = "This text was created using the included javascript code :)" 4 | } 5 | -------------------------------------------------------------------------------- /demo/std_nodes/process/data.ldjson: -------------------------------------------------------------------------------- 1 | { "a": 12, "b": true } 2 | { "a": 16, "b": true } 3 | { "a": 22, "b": false } 4 | { "a": 7, "b": true } 5 | { "a": 452, "b": false, "ts": 1497894557000 } 6 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/data.ldjson: -------------------------------------------------------------------------------- 1 | { "a": 12, "b": true } 2 | { "a": 16, "b": true } 3 | { "a": 22, "b": false } 4 | { "a": 7, "b": true } 5 | { "a": 452, "b": false, "ts": 1497894557000 } 6 | -------------------------------------------------------------------------------- /docs/uml/sequence_register.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | Worker -> Storage: registerWorker 3 | activate Storage 4 | 5 | Storage ->Storage: enlist into worker list 6 | Storage --> Worker: 7 | deactivate Storage 8 | @enduml 9 | -------------------------------------------------------------------------------- /tests/std_nodes/simple_proc.js: -------------------------------------------------------------------------------- 1 | setTimeout(() => { 2 | console.error("bad error"); 3 | }, 50); 4 | setTimeout(() => { 5 | console.log(JSON.stringify( 6 | { a: 5 } 7 | )); 8 | console.log("bad json"); 9 | }, 100) -------------------------------------------------------------------------------- /demo/util/child_process_restarter/child.js: -------------------------------------------------------------------------------- 1 | // simple child that does nothing for 10 seconds 2 | console.log("Child process started - ", process.argv.slice(2)); 3 | setTimeout(function() { 4 | console.log("Child process shutting down"); 5 | }, 10000); 6 | -------------------------------------------------------------------------------- /demo/util/web_server/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../../.."); 4 | 5 | let server = new qtopology.MinimalHttpServer(); 6 | 7 | server.addRoute("a.html", "./a.html"); 8 | server.addRoute("a.js", "./a.js"); 9 | server.run(3000); 10 | -------------------------------------------------------------------------------- /demo/util/web_server/a.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

Hello

5 |

This page is server using minimal HTTP server from QTopology.

6 |
7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /demo/async/top.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../.."); 4 | const shutdown = qtopology.runLocalTopologyFromFile("./topology.json"); 5 | 6 | // let topology run for 5 seconds, then exit without error 7 | setTimeout(() => { 8 | shutdown(0); 9 | }, 5 * 1000); 10 | -------------------------------------------------------------------------------- /docs/uml/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "uml", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "run.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "async": "^2.6.1", 13 | "node-plantuml": "^0.8.1" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /demo/distributed_http_based/storage_test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../../"); 4 | 5 | ////////////////////////////////////////////////// 6 | 7 | let cmdln = new qtopology.CmdLineParser(); 8 | cmdln 9 | .define("p", "port", 3000, "Port"); 10 | let options = cmdln.process(process.argv); 11 | 12 | qtopology.runHttpServer(options); 13 | -------------------------------------------------------------------------------- /demo/std_nodes/disabled/init_and_shutdown.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | exports.init = function (config, context, callback) { 4 | console.log("Common initialization - this one is OK to be displayed"); 5 | callback(); 6 | }; 7 | 8 | exports.shutdown = function (callback) { 9 | console.log("Common shutdown - this one is OK to be displayed"); 10 | callback(); 11 | }; 12 | -------------------------------------------------------------------------------- /demo/std_nodes/disabled/init_and_shutdown2.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | exports.init = function (config, context, callback) { 4 | console.log("Common initialization - this one SHOULD NOT be displayed"); 5 | callback(); 6 | }; 7 | 8 | exports.shutdown = function (callback) { 9 | console.log("Common shutdown - this one SHOULD NOT be displayed"); 10 | callback(); 11 | }; 12 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/cpp/demo.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | using namespace std; 5 | 6 | int main() { 7 | int cntr = 0; 8 | while (true) { 9 | string s; 10 | cin >> s; 11 | if (++cntr % 3 == 0) { 12 | cout << "{ \"count\": " << cntr << "}" << endl; 13 | } 14 | } 15 | return 0; 16 | } 17 | -------------------------------------------------------------------------------- /docs/uml/state_topology.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | [*] -down-> unassigned 3 | unassigned -right-> waiting : by leader 4 | waiting --> running: by worker 5 | waiting -left-> unassigned : by leader (timeout) 6 | running -right-> stopped: by worker 7 | running -up-> unassigned: by leader 8 | stopped -up-> waiting : by leader 9 | running -left-> error: by worker 10 | error -up-> unassigned: manual 11 | @enduml 12 | -------------------------------------------------------------------------------- /tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "type": "typescript", 8 | "tsconfig": "tsconfig.json", 9 | "group": { 10 | "kind": "build", 11 | "isDefault": true 12 | } 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /demo/local_massive/init_and_shutdown.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let common_context = { 4 | cnt: 0 5 | }; 6 | 7 | exports.init = function (config, context, callback) { 8 | console.log("Common initialization"); 9 | common_context = context; 10 | common_context.cnt = 0; 11 | callback(); 12 | }; 13 | 14 | exports.shutdown = function (callback) { 15 | console.log("Common shutdown", common_context); 16 | callback(); 17 | }; 18 | -------------------------------------------------------------------------------- /demo/async/my_bolt.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MyAsyncBolt { 4 | 5 | constructor() { 6 | this._name = null; 7 | this._onEmit = null; 8 | } 9 | 10 | async init(name, config, context) { 11 | this._name = name; 12 | this._onEmit = config.onEmit; 13 | } 14 | 15 | heartbeat() { } 16 | async shutdown() { } 17 | 18 | async receive(data, stream_id) { 19 | console.log(data, stream_id); 20 | } 21 | } 22 | 23 | exports.create = function () { return new MyAsyncBolt(); }; 24 | -------------------------------------------------------------------------------- /demo/std_nodes/task_bolt_base/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [], 6 | "bolts": [ 7 | { 8 | "name": "task_bolt", 9 | "working_dir": ".", 10 | "type": "inproc", 11 | "cmd": "custom_task.js", 12 | "inputs": [], 13 | "init": { 14 | "repeat_after": 5000, 15 | "text": "Some custom text from config" 16 | } 17 | } 18 | ], 19 | "variables": {} 20 | } 21 | -------------------------------------------------------------------------------- /.github/workflows/node2.yml: -------------------------------------------------------------------------------- 1 | name: Node basic CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@master 12 | - name: Use Node.js 13 | uses: actions/setup-node@v1 14 | with: 15 | version: 10.x 16 | - name: npm install, build, and test 17 | run: | 18 | npm install -g typescript@2.9.2 19 | npm install -g mocha 20 | npm install 21 | npm run prepare 22 | npm run build --if-present 23 | npm run test-unit 24 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/child.js: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////// 2 | // Demo child process 3 | /////////////////////////////////////////////////////////////// 4 | 5 | var readline = require('readline'); 6 | var rl = readline.createInterface({ 7 | input: process.stdin, 8 | output: process.stdout, 9 | terminal: false 10 | }); 11 | 12 | let cntr = 0; 13 | 14 | // emit count after each 3 messages 15 | rl.on('line', (line) => { 16 | if (++cntr % 3 == 0) { 17 | console.log(JSON.stringify({ counter: cntr })); 18 | } 19 | }) 20 | -------------------------------------------------------------------------------- /demo/local/init_and_shutdown.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../.."); 4 | 5 | let common_context = { 6 | cnt: 0 7 | }; 8 | 9 | exports.init = function (config, context, callback) { 10 | qtopology.logger().important("Common initialization"); 11 | common_context = context; 12 | common_context.cnt = 0; 13 | callback(); 14 | }; 15 | 16 | exports.shutdown = function (callback) { 17 | qtopology.logger().important("Common shutdown " + common_context.cnt); 18 | callback(); 19 | }; 20 | 21 | exports.shutdown_hard = function () { 22 | qtopology.logger().important("Hard shutdown"); 23 | }; 24 | -------------------------------------------------------------------------------- /demo/util/child_process_restarter/parent_fork.js: -------------------------------------------------------------------------------- 1 | const qtopology = require("../../.."); 2 | 3 | let obj = new qtopology.ChildProcRestarterFork("child.js", []); 4 | obj.start(); 5 | 6 | 7 | setTimeout(() => { 8 | console.log("Parent will stop the child"); 9 | obj.stop(); 10 | setTimeout(() => { 11 | console.log("Parent will start the child"); 12 | obj.start(); 13 | setTimeout(() => { 14 | console.log("Parent will stop the child - 2"); 15 | obj.stop(() => { 16 | console.log("Parent stopped the child - 3"); 17 | }); 18 | }, 300); 19 | }, 5000); 20 | }, 17000); 21 | -------------------------------------------------------------------------------- /docs/uml/run.js: -------------------------------------------------------------------------------- 1 | const plantuml = require("node-plantuml"); 2 | const fs = require("fs"); 3 | const async = require("async"); 4 | 5 | function processFile(fname, cb) { 6 | console.log("Processing file", fname); 7 | const gen = plantuml.generate(fname, { format: "svg" }); 8 | const stream = gen.out.pipe(fs.createWriteStream(fname + ".svg")); 9 | stream.on("finish", () => { 10 | console.log("Finished file", fname); 11 | cb(); 12 | }); 13 | } 14 | 15 | const files = fs.readdirSync(".").filter(x => x.endsWith(".uml")); 16 | async.eachSeries(files, 17 | (fname, cb) => { 18 | processFile(fname, cb); 19 | } 20 | ) 21 | -------------------------------------------------------------------------------- /demo/util/child_process_restarter/parent_spawn.js: -------------------------------------------------------------------------------- 1 | const qtopology = require("../../.."); 2 | 3 | let obj = new qtopology.ChildProcRestarterSpawn("node", ["child.js"]); 4 | obj.start(); 5 | 6 | 7 | setTimeout(() => { 8 | console.log("Parent will stop the child"); 9 | obj.stop(); 10 | setTimeout(() => { 11 | console.log("Parent will start the child"); 12 | obj.start(); 13 | setTimeout(() => { 14 | console.log("Parent will stop the child - 2"); 15 | obj.stop(() => { 16 | console.log("Parent stopped the child - 3"); 17 | }); 18 | }, 300); 19 | }, 5000); 20 | }, 17000); 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # Transpiled code 24 | built 25 | 26 | # Dependency directories 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | .vscode 35 | *.stackdump 36 | 37 | tmp 38 | -------------------------------------------------------------------------------- /demo/async/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "inproc", 9 | "working_dir": ".", 10 | "cmd": "my_spout.js", 11 | "init": {} 12 | } 13 | ], 14 | "bolts": [ 15 | { 16 | "name": "bolt1", 17 | "working_dir": ".", 18 | "type": "inproc", 19 | "cmd": "my_bolt.js", 20 | "inputs": [ 21 | { "source": "pump1", "stream_id": "stream1" } 22 | ], 23 | "init": {} 24 | } 25 | ], 26 | "variables": {} 27 | } 28 | -------------------------------------------------------------------------------- /demo/quick_start/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "inproc", 9 | "working_dir": ".", 10 | "cmd": "my_spout.js", 11 | "init": {} 12 | } 13 | ], 14 | "bolts": [ 15 | { 16 | "name": "bolt1", 17 | "working_dir": ".", 18 | "type": "inproc", 19 | "cmd": "my_bolt.js", 20 | "inputs": [ 21 | { "source": "pump1", "stream_id": "stream1" } 22 | ], 23 | "init": {} 24 | } 25 | ], 26 | "variables": {} 27 | } -------------------------------------------------------------------------------- /demo/distributed_file_based/init_and_shutdown.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let common_context = { 4 | cnt: 0 5 | }; 6 | 7 | exports.init = function (config, context, callback) { 8 | console.log("Custom initialization"); 9 | common_context = context; 10 | common_context.cnt = 0; 11 | setTimeout(() => { 12 | // simulate some lengthy processing 13 | callback(); 14 | }, Math.floor(3000 * Math.random())); 15 | }; 16 | 17 | exports.shutdown = function (callback) { 18 | console.log("Custom shutdown", common_context); 19 | setTimeout(() => { 20 | // simulate some lengthy processing 21 | console.log("Exiting custom shutdown...", common_context); 22 | callback(); 23 | }, Math.floor(5000 * Math.random())); 24 | }; 25 | -------------------------------------------------------------------------------- /docs/uml/components.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | package "Worker - Main process" { 3 | [Worker] --> [Coordinator] 4 | [Coordinator] --> [Leader] 5 | [Worker] --o [TopologyProxy] 6 | [Coordinator] --> [CustomStorageImplementation] 7 | [Leader] --> [CustomStorageImplementation] 8 | } 9 | 10 | package "Child process" { 11 | [TopologyWrapper] --> [LocalTopology] 12 | } 13 | 14 | node "Worker 2" { 15 | [Worker2] 16 | } 17 | node "Worker 3" { 18 | [Worker3] 19 | } 20 | node "Worker 4" { 21 | [Worker4] 22 | } 23 | 24 | 25 | database "Common storage" { 26 | [Storage] 27 | } 28 | 29 | [TopologyProxy] -left-> [TopologyWrapper] 30 | [CustomStorageImplementation] --> [Storage] 31 | [Worker2] --> [Storage] 32 | [Worker3] --> [Storage] 33 | [Worker4] --> [Storage] 34 | @enduml 35 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./built", 4 | "allowJs": false, 5 | "declaration": true, 6 | "declarationMap": true, 7 | "sourceMap": true, 8 | "target": "es6", 9 | "lib":["es2016"], 10 | "module": "commonjs", 11 | "noImplicitReturns": true, 12 | "noFallthroughCasesInSwitch": true, 13 | "alwaysStrict": true, 14 | "diagnostics": false, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": false, 17 | "pretty": false, 18 | "types": [ 19 | "node" 20 | ], 21 | "typeRoots": [ 22 | "node_modules/@types" 23 | ] 24 | }, 25 | "include": [ 26 | "./src/**/*" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /demo/std_nodes/rss/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump_rss", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "rss", 11 | "init": { 12 | "repeat": 6000, 13 | "url": "http://rss.cnn.com/rss/cnn_topstories.rss" 14 | } 15 | } 16 | ], 17 | "bolts": [ 18 | { 19 | "name": "bolt2", 20 | "working_dir": ".", 21 | "type": "sys", 22 | "cmd": "console", 23 | "inputs": [ 24 | { "source": "pump_rss" } 25 | ], 26 | "init": {} 27 | } 28 | ], 29 | "variables": {} 30 | } 31 | -------------------------------------------------------------------------------- /demo/util/child_process_restarter/parent.js: -------------------------------------------------------------------------------- 1 | const qtopology = require("../../.."); 2 | 3 | let options = { 4 | cmd : "node", 5 | args: ["child.js"], 6 | args_restart: ["child.js", "-restart"], 7 | use_fork: false, 8 | stop_score: 5 9 | }; 10 | 11 | let obj = new qtopology.ChildProcRestarter(options); 12 | obj.start(); 13 | 14 | setTimeout(() => { 15 | console.log("Parent will stop the child"); 16 | obj.stop(); 17 | setTimeout(() => { 18 | console.log("Parent will start the child"); 19 | obj.start(); 20 | setTimeout(() => { 21 | console.log("Parent will stop the child - 2"); 22 | obj.stop(() => { 23 | console.log("Parent stopped the child - 3"); 24 | }); 25 | }, 300); 26 | }, 5000); 27 | }, 17000); 28 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/topology_long.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "file_reader", 11 | "init": { 12 | "file_name": "data.long.txt", 13 | "file_format": "json" 14 | } 15 | } 16 | ], 17 | "bolts": [ 18 | { 19 | "name": "bolt1", 20 | "working_dir": ".", 21 | "type": "sys", 22 | "cmd": "console", 23 | "disabled": false, 24 | "inputs": [ 25 | { "source": "pump" } 26 | ], 27 | "init": {} 28 | } 29 | ], 30 | "variables": {} 31 | } 32 | -------------------------------------------------------------------------------- /src/util/callback_wrappers.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as log from "../util/logger"; 3 | 4 | /** helper function that wraps a callback with try/catch and logs an error 5 | * if the callback threw an exception. 6 | */ 7 | export function tryCallback(callback: intf.SimpleCallback): intf.SimpleCallback { 8 | if (callback == undefined) { 9 | return (err?: Error) => { 10 | if (err) { 11 | log.logger().exception(err); 12 | } 13 | }; 14 | } 15 | return (err?: Error) => { 16 | try { 17 | return callback(err); 18 | } catch (e) { 19 | log.logger().error("THIS SHOULD NOT HAPPEN: exception THROWN in callback!"); 20 | log.logger().exception(e); 21 | } 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint:recommended"], 3 | "rules": { 4 | "no-console": [ "log" ], 5 | "ordered-imports": false, 6 | "triple-equals": false, 7 | "no-namespace": false, 8 | "max-classes-per-file": [false, 20], 9 | "no-consecutive-blank-lines": false, 10 | "arrow-parens": [true, "ban-single-arg-parens"], 11 | "trailing-comma": [true, {"multiline": "never", "singleline": "never"}], 12 | "variable-name": [ 13 | true, 14 | "ban-keywords", 15 | "check-format", 16 | "allow-leading-underscore", 17 | "allow-snake-case" 18 | ] 19 | }, 20 | "linterOptions": { 21 | "exclude": [ 22 | "public/**/*.js", 23 | "config/**/*.js", 24 | "node_modules/**/*.ts" 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /demo/std_nodes/dir_watcher/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 500 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "dir", 11 | "init": { 12 | "dir_name": ".", 13 | "stream_id": "some_stream" 14 | } 15 | } 16 | ], 17 | "bolts": [ 18 | { 19 | "name": "bolt1", 20 | "working_dir": ".", 21 | "type": "sys", 22 | "cmd": "console", 23 | "inputs": [ 24 | { 25 | "source": "pump1", 26 | "stream_id": "some_stream" 27 | } 28 | ], 29 | "init": {} 30 | } 31 | ], 32 | "variables": {} 33 | } 34 | -------------------------------------------------------------------------------- /docs/uml/sequence_worker.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | activate Worker 3 | Worker -> Storage: getMessages 4 | activate Storage 5 | 6 | Storage -> Storage: retrieve list of message for this worker 7 | Storage --> Worker: 8 | deactivate Storage 9 | 10 | Worker->LocalTopology: start 11 | activate LocalTopology 12 | LocalTopology->LocalTopology: run 13 | LocalTopology-->Worker: success 14 | 15 | Worker -> Storage: setTopologyStatus 16 | activate Storage 17 | Storage -> Storage: mark topology as running 18 | Storage --> Worker: 19 | deactivate Storage 20 | 21 | LocalTopology->LocalTopology: loop spouts 22 | 23 | LocalTopology->Worker: exit/error 24 | deactivate LocalTopology 25 | 26 | Worker -> Storage: setTopologyStatus 27 | activate Storage 28 | Storage -> Storage: mark topology as stopped/error 29 | Storage --> Worker: 30 | deactivate Storage 31 | 32 | deactivate Worker 33 | @enduml 34 | -------------------------------------------------------------------------------- /demo/async/my_spout.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MyAsyncSpout { 4 | 5 | constructor() { 6 | this._name = null; 7 | this._data = []; 8 | this._data_index = 0; 9 | } 10 | 11 | async init(name, config, context) { 12 | this._name = name; 13 | 14 | for (let i = 0; i < 100; i++) { 15 | this._data.push({ id: i }); 16 | } 17 | } 18 | 19 | heartbeat() { } 20 | async shutdown() { } 21 | run() { } 22 | pause() { } 23 | 24 | async next() { 25 | if (this._data_index >= this._data.length) { 26 | return null; 27 | } else { 28 | return { 29 | data: this._data[this._data_index++], 30 | stream_id: "stream1" 31 | }; 32 | } 33 | } 34 | } 35 | 36 | exports.create = function () { return new MyAsyncSpout(); }; 37 | -------------------------------------------------------------------------------- /demo/quick_start/top.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../../built/index.js"); 4 | 5 | // load configuration from file 6 | let config = require("./topology.json"); 7 | 8 | // compile it - injects variables and performs some checks 9 | let compiler = new qtopology.TopologyCompiler(config); 10 | compiler.compile(); 11 | config = compiler.getWholeConfig(); 12 | 13 | // ok, create topology 14 | let topology = new qtopology.TopologyLocal(); 15 | topology.init("uuid.1", config, (err) => { 16 | if (err) { console.log(err); return; } 17 | // let topology run for 20 seconds 18 | topology.run((e)=>{ 19 | if (e) { console.log(e) } 20 | setTimeout(() => { 21 | topology.shutdown((err) => { 22 | if (err) { console.log(err); } 23 | console.log("Finished."); 24 | }); 25 | }, 20 * 1000); 26 | }); 27 | }); -------------------------------------------------------------------------------- /demo/gui/topology.2.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": {} 12 | } 13 | ], 14 | "bolts": [ 15 | { 16 | "name": "boltp", 17 | "working_dir": ".", 18 | "type": "sys", 19 | "cmd": "process", 20 | "inputs": [ 21 | { "source": "pump1" } 22 | ], 23 | "init": {} 24 | }, 25 | { 26 | "name": "bolt1", 27 | "working_dir": ".", 28 | "type": "sys", 29 | "cmd": "console", 30 | "inputs": [ 31 | { "source": "boltp", "stream_id": "streamx" } 32 | ], 33 | "init": {} 34 | } 35 | ], 36 | "variables": {} 37 | } 38 | -------------------------------------------------------------------------------- /src/distributed/http_based/rest_client.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosInstance, AxiosRequestConfig } from "axios"; 2 | 3 | export interface IApiClient { 4 | get(path: string, params?: any): Promise; 5 | post(path: string, params: any): Promise; 6 | put(path: string, params: any): Promise; 7 | delete(path: string, params: any): Promise; 8 | } 9 | 10 | export type RestConfig = AxiosRequestConfig; 11 | export type RestClientInstance = AxiosInstance; 12 | export type RestCreateFunction = (config?: AxiosRequestConfig) => AxiosInstance; 13 | 14 | /** Factory function for REST client. Uses the Axios library internally. 15 | * This way we expose Axios interfaces but leave the implementation open to mocking. 16 | */ 17 | export function create(config: RestConfig): RestClientInstance { 18 | if (config) { 19 | config.maxContentLength = Infinity; 20 | config["Cache-Control"] = "no-cache"; 21 | } 22 | return axios.create(config); 23 | } 24 | -------------------------------------------------------------------------------- /demo/local_massive/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000, 4 | "initialization": [ 5 | { 6 | "working_dir": ".", 7 | "cmd": "init_and_shutdown.js" 8 | } 9 | ], 10 | "shutdown": [ 11 | { 12 | "working_dir": ".", 13 | "cmd": "init_and_shutdown.js" 14 | } 15 | ] 16 | }, 17 | "spouts": [ 18 | { 19 | "name": "pump1", 20 | "type": "inproc", 21 | "working_dir": ".", 22 | "cmd": "spout_inproc.js", 23 | "init": {} 24 | } 25 | ], 26 | "bolts": [ 27 | { 28 | "name": "bolt_console", 29 | "working_dir": ".", 30 | "type": "sys", 31 | "cmd": "console", 32 | "inputs": [{ "source": "pump1" }], 33 | "init": {} 34 | } 35 | ], 36 | "variables": {} 37 | } 38 | -------------------------------------------------------------------------------- /src/std_nodes/forward_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | 3 | ///////////////////////////////////////////////////////////////////////////// 4 | 5 | /** This bolt forwards all incoming messages. Can be used as stage separator. 6 | */ 7 | export class ForwardrBolt implements intf.IBolt { 8 | 9 | private onEmit: intf.BoltEmitCallback; 10 | 11 | constructor() { 12 | this.onEmit = null; 13 | } 14 | 15 | /** Initializes filtering pattern */ 16 | public init(_name: string, config: any, _context: any, callback: intf.SimpleCallback) { 17 | this.onEmit = config.onEmit; 18 | callback(); 19 | } 20 | 21 | public heartbeat() { 22 | // no-op 23 | } 24 | 25 | public shutdown(callback: intf.SimpleCallback) { 26 | callback(); 27 | } 28 | 29 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 30 | this.onEmit(data, stream_id, callback); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /demo/std_nodes/task_bolt_base/custom_task.js: -------------------------------------------------------------------------------- 1 | const qt = require("../../.."); 2 | 3 | class CustomTaskBolt extends qt.TaskBoltBase { 4 | 5 | constructor() { 6 | super(); 7 | this.custom_text = null; 8 | } 9 | 10 | init(name, config, context, callback) { 11 | let self = this; 12 | super.init(name, config, context, (err) => { 13 | if (err) return callback(err); 14 | self.custom_text = config.text; 15 | callback(); 16 | }) 17 | } 18 | 19 | runInternal(callback) { 20 | let self = this; 21 | console.log("Custom output from task bolt (1): " + self.custom_text); 22 | setTimeout(() => { 23 | console.log("Custom output from task bolt (2): " + self.custom_text); 24 | callback(); 25 | }, 700); 26 | } 27 | } 28 | 29 | ///////////////////////////////////////////////////////// 30 | exports.create = function () { 31 | return new CustomTaskBolt(); 32 | }; 33 | -------------------------------------------------------------------------------- /src/util/object_override.ts: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Function that overrides a base object. `createNew` defaults to false. 4 | * If `createNew` is true, a new object will be created, otherwise the 5 | * base object will be extended. 6 | */ 7 | 8 | export function overrideObject(baseObject: any, additional_data: any, createNew?: boolean) { 9 | if (!baseObject) { 10 | baseObject = {}; 11 | } 12 | if (createNew) { 13 | baseObject = JSON.parse(JSON.stringify(baseObject)); 14 | } 15 | Object.keys(additional_data).forEach(key => { 16 | if (isObjectAndNotArray(baseObject[key]) && isObjectAndNotArray(additional_data[key])) { 17 | overrideObject(baseObject[key], additional_data[key], false); 18 | } else { 19 | baseObject[key] = additional_data[key]; 20 | } 21 | }); 22 | return baseObject; 23 | } 24 | 25 | /** Helper function */ 26 | function isObjectAndNotArray(object) { 27 | return (typeof object === "object" && !Array.isArray(object)); 28 | } 29 | -------------------------------------------------------------------------------- /src/util/freq_estimator.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | export class EventFrequencyScore { 5 | 6 | private c: number; 7 | private prev_time: number; 8 | private prev_val: number; 9 | 10 | constructor(c: number) { 11 | if (c <= 0) { 12 | c = 1; 13 | } 14 | this.c = 1 / (9.5 * c); // this constant is based on experiments 15 | this.prev_time = 0; 16 | this.prev_val = 0; 17 | } 18 | 19 | public getEstimate(d: Date): number { 20 | return this.estimateFrequencyNum(this.prev_time, d.getTime(), this.prev_val, 0, this.c); 21 | } 22 | 23 | public add(d: Date): number { 24 | const dd = d.getTime(); 25 | const res = this.estimateFrequencyNum(this.prev_time, dd, this.prev_val, 1, this.c); 26 | this.prev_time = dd; 27 | this.prev_val = res; 28 | return res; 29 | } 30 | 31 | private estimateFrequencyNum(t1: number, t2: number, v1: number, v2: number, c: number): number { 32 | return v2 + v1 * Math.exp(-c * (t2 - t1)); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /demo/std_nodes/rest/demo_rest.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | -------------------------------------------------------------------------------- /demo/std_nodes/rss/demo_rss.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | -------------------------------------------------------------------------------- /demo/std_nodes/telemetry/demo_telemetry.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | -------------------------------------------------------------------------------- /src/util/schema_test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const Validator = require("jsonschema").Validator; 5 | const TopologyCompiler = require("../../built/topology_compiler").TopologyCompiler; 6 | 7 | //////////////////////////////////////////////////////////////////// 8 | 9 | let instance = JSON.parse(fs.readFileSync("./topology_config_example.json")); 10 | let schema = JSON.parse(fs.readFileSync("../../resources/topology_config_schema.json")); 11 | 12 | let v = new Validator(); 13 | let validation_result = v.validate(instance, schema); 14 | if (validation_result.errors.length > 0) { 15 | console.error("Errors while parsing topology schema:"); 16 | for (let error of validation_result.errors) { 17 | console.error(error.stack || error.property + " " + error.message); 18 | } 19 | process.exit(1); 20 | } else { 21 | console.log("Schema is valid."); 22 | console.log("Compiling"); 23 | let compiler = new TopologyCompiler(instance); 24 | compiler.compile(); 25 | console.log(JSON.stringify(compiler.getWholeConfig())); 26 | } 27 | -------------------------------------------------------------------------------- /docs/uml/sequence_leader.uml: -------------------------------------------------------------------------------- 1 | @startuml 2 | activate Worker 3 | Worker -> Storage: checkLeadershipStatus 4 | activate Storage 5 | Storage -> Storage: check current leadership status\nDisable leader if timeout 6 | Storage --> Worker: 7 | deactivate Storage 8 | 9 | Worker->Storage: (if no leader) sendLeadershipCandidacy 10 | activate Storage 11 | Storage->Storage: mark new candidate 12 | Storage-->Worker: 13 | 14 | Worker -> Storage: checkCandidacy 15 | Storage -> Storage: mark new leader if success 16 | Storage --> Worker: status if candidacy was successfull 17 | deactivate Storage 18 | 19 | Worker -> Storage: (if leader) getWorkerStatuses 20 | activate Storage 21 | Storage -> Storage: collect worker statuses 22 | Storage --> Worker: 23 | 24 | Worker -> Storage: setTopologyStatus (to unassigned if worker timed out) 25 | Storage -> Storage: update worker statuses 26 | Storage --> Worker: 27 | 28 | Worker -> Storage: updateWorkerStatus (if timed out) 29 | Storage -> Storage: update worker statuses 30 | Storage --> Worker: 31 | 32 | deactivate Storage 33 | 34 | deactivate Worker 35 | @enduml 36 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./topology_compiler"; 2 | export * from "./topology_local"; 3 | export { createSysBolt, createSysSpout } from "./topology_local_inprocess"; 4 | export * from "./topology_validation"; 5 | export * from "./topology_interfaces"; 6 | 7 | export * from "./distributed/topology_worker"; 8 | export * from "./distributed/memory/memory_storage"; 9 | export * from "./distributed/file_based/file_storage"; 10 | export * from "./distributed/http_based/http_storage_server"; 11 | export * from "./distributed/http_based/http_storage"; 12 | export * from "./distributed/gui/dashboard_server"; 13 | export * from "./distributed/cli/command_line"; 14 | 15 | export * from "./util/logger"; 16 | export * from "./util/cmdline"; 17 | export * from "./util/pattern_matcher"; 18 | export * from "./util/child_proc_restarter"; 19 | export * from "./util/http_server"; 20 | export * from "./util/crontab_parser"; 21 | export * from "./util/strip_json_comments"; 22 | 23 | export * from "./std_nodes/task_bolt_base"; 24 | export { TransformHelper, TransformHelperQewd } from "./std_nodes/transform_bolt"; 25 | -------------------------------------------------------------------------------- /demo/local_massive/bolt_common.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MyBolt { 4 | 5 | constructor() { 6 | this._context = null; 7 | this._sum = 0; 8 | this._forward = true; 9 | this._onEmit = null; 10 | } 11 | 12 | init(name, config, context, callback) { 13 | this._context = context; 14 | this._onEmit = config.onEmit; 15 | this._forward = config.forward; 16 | callback(); 17 | } 18 | 19 | heartbeat() { 20 | // this._onEmit({ sum: this._sum }, null, (err)=>{ 21 | // if (err) { 22 | // console.log(err); 23 | // } 24 | // }); 25 | } 26 | 27 | shutdown(callback) { 28 | callback(); 29 | } 30 | 31 | receive(data, stream_id, callback) { 32 | let self = this; 33 | if (self._context) { 34 | self._context.cnt++; 35 | } 36 | this._sum += data.a; 37 | callback(); 38 | } 39 | } 40 | 41 | //////////////////////////////////////////////////////////////////////////////// 42 | 43 | exports.MyBolt = MyBolt; 44 | -------------------------------------------------------------------------------- /demo/std_nodes/telemetry/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "spout1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "telemetry_timeout": 3000, 12 | "init": {} 13 | } 14 | ], 15 | "bolts": [ 16 | { 17 | "name": "bolt1", 18 | "working_dir": ".", 19 | "type": "sys", 20 | "cmd": "console", 21 | "telemetry_timeout": 2000, 22 | "inputs": [ 23 | { "source": "spout1" } 24 | ], 25 | "init": {} 26 | }, 27 | { 28 | "name": "bolt2", 29 | "working_dir": ".", 30 | "type": "sys", 31 | "cmd": "console", 32 | "inputs": [ 33 | { "source": "spout1", "stream_id": "$telemetry" }, 34 | { "source": "bolt1", "stream_id": "$telemetry" } 35 | ], 36 | "init": {} 37 | } 38 | ], 39 | "variables": {} 40 | } 41 | -------------------------------------------------------------------------------- /src/std_nodes/attacher_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as oo from "../util/object_override"; 3 | 4 | /** This bolt attaches fixed fields to incoming messages 5 | * and sends them forward. 6 | */ 7 | export class AttacherBolt implements intf.IBolt { 8 | 9 | private extra_fields: any; 10 | private onEmit: intf.BoltEmitCallback; 11 | 12 | constructor() { 13 | this.onEmit = null; 14 | this.extra_fields = null; 15 | } 16 | 17 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 18 | this.onEmit = config.onEmit; 19 | this.extra_fields = JSON.parse(JSON.stringify(config.extra_fields || {})); 20 | callback(); 21 | } 22 | 23 | public heartbeat() { 24 | // no-op 25 | } 26 | 27 | public shutdown(callback: intf.SimpleCallback) { 28 | callback(); 29 | } 30 | 31 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 32 | oo.overrideObject(data, this.extra_fields, false); 33 | this.onEmit(data, stream_id, callback); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/std_nodes/console_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as log from "../util/logger"; 3 | 4 | /** This bolt just writes all incoming data to console. */ 5 | export class ConsoleBolt implements intf.IBolt { 6 | 7 | private name: string; 8 | private prefix: string; 9 | private onEmit: intf.BoltEmitCallback; 10 | 11 | constructor() { 12 | this.name = null; 13 | this.prefix = ""; 14 | this.onEmit = null; 15 | } 16 | 17 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 18 | this.name = name; 19 | this.prefix = `[${this.name}]`; 20 | this.onEmit = config.onEmit; 21 | callback(); 22 | } 23 | 24 | public heartbeat() { 25 | // no-op 26 | } 27 | 28 | public shutdown(callback: intf.SimpleCallback) { 29 | callback(); 30 | } 31 | 32 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 33 | log.logger().log(`${this.prefix} [stream_id=${stream_id}] ${JSON.stringify(data)}`); 34 | this.onEmit(data, stream_id, callback); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/util/telemetry.tests.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const tel = require("../../built/util/telemetry"); 7 | 8 | describe('Telemetry', function () { 9 | it('constructable', function () { 10 | let target = new tel.Telemetry(); 11 | }); 12 | it('accepts data', function () { 13 | let n = "some name"; 14 | let target = new tel.Telemetry(n); 15 | 16 | assert.deepEqual(target.get(), { cnt: 0, avg: 0 }); 17 | assert.deepEqual(target.get(true), { name: n, cnt: 0, avg: 0 }); 18 | target.add(2); 19 | assert.deepEqual(target.get(), { cnt: 1, avg: 2 }); 20 | target.add(4); 21 | assert.deepEqual(target.get(), { cnt: 2, avg: 3 }); 22 | target.add(3); 23 | assert.deepEqual(target.get(), { cnt: 3, avg: 3 }); 24 | target.add(7); 25 | assert.deepEqual(target.get(), { cnt: 4, avg: 4 }); 26 | 27 | target.reset(); 28 | assert.deepEqual(target.get(), { cnt: 0, avg: 0 }); 29 | target.add(7); 30 | assert.deepEqual(target.get(), { cnt: 1, avg: 7 }); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /src/topology_validation.ts: -------------------------------------------------------------------------------- 1 | import * as jsch from "jsonschema"; 2 | import * as intf from "./topology_interfaces"; 3 | 4 | /** Utility function for validating given JSON */ 5 | export function validate(options: intf.IValidationOptions) { 6 | const { config, exitOnError, throwOnError } = options; 7 | const schema = require("../resources/topology_config_schema.json"); 8 | const v = new jsch.Validator(); 9 | const validation_result = v.validate(config, schema); 10 | if (validation_result.errors.length > 0) { 11 | if (exitOnError) { 12 | console.error("Errors while parsing topology schema:"); 13 | for (const error of validation_result.errors) { 14 | console.error(error.property + " " + error.message); 15 | } 16 | process.exit(1); 17 | } 18 | if (throwOnError) { 19 | let msg = ""; 20 | for (const error of validation_result.errors) { 21 | msg += (error.property + " " + error.message) + "\n"; 22 | } 23 | throw new Error(msg); 24 | } 25 | return validation_result.errors; 26 | } else { 27 | return false; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /demo/std_nodes/bomb/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | } 17 | ], 18 | "bolts": [ 19 | { 20 | "name": "bolt1", 21 | "working_dir": ".", 22 | "type": "sys", 23 | "cmd": "console", 24 | "inputs": [ 25 | { 26 | "source": "pump1" 27 | } 28 | ], 29 | "init": {} 30 | }, 31 | { 32 | "name": "bolt_bomb", 33 | "working_dir": ".", 34 | "type": "sys", 35 | "cmd": "bomb", 36 | "inputs": [ 37 | { 38 | "source": "pump1" 39 | } 40 | ], 41 | "init": { 42 | "explode_after": 4500 43 | } 44 | } 45 | ], 46 | "variables": {} 47 | } 48 | -------------------------------------------------------------------------------- /demo/distributed_http_based/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | } 17 | ], 18 | "bolts": [ 19 | { 20 | "name": "bolt", 21 | "working_dir": ".", 22 | "type": "sys", 23 | "cmd": "console", 24 | "inputs": [ 25 | { 26 | "source": "pump" 27 | } 28 | ], 29 | "init": {} 30 | }, 31 | { 32 | "name": "bolt_bomb", 33 | "working_dir": ".", 34 | "type": "sys", 35 | "cmd": "bomb", 36 | "disabled": true, 37 | "inputs": [ 38 | { 39 | "source": "pump" 40 | } 41 | ], 42 | "init": { 43 | "explode_after": 4500 44 | } 45 | } 46 | ], 47 | "variables": {} 48 | } 49 | -------------------------------------------------------------------------------- /src/std_nodes/get_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as rest from "../distributed/http_based/rest_client"; 3 | 4 | /** This bolt sends GET request to specified url 5 | * and forwards the result. 6 | */ 7 | export class GetBolt implements intf.IBolt { 8 | 9 | private fixed_url: string; 10 | private client: rest.IApiClient; 11 | private onEmit: intf.BoltEmitCallback; 12 | 13 | constructor() { 14 | this.onEmit = null; 15 | this.fixed_url = null; 16 | } 17 | 18 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 19 | this.onEmit = config.onEmit; 20 | this.fixed_url = config.url; 21 | this.client = rest.create({}); 22 | callback(); 23 | } 24 | 25 | public heartbeat() { 26 | // no-op 27 | } 28 | 29 | public shutdown(callback: intf.SimpleCallback) { 30 | callback(); 31 | } 32 | 33 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 34 | const url = this.fixed_url || data.url; 35 | this.client.get(url) 36 | .then(res => this.onEmit({ body: res.data.toString() }, null, callback)) 37 | .catch(err => callback(err)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /demo/distributed_http_based/worker_test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../../"); 4 | 5 | /////////////////////////////////////////////////////////////////////// 6 | let cmdln = new qtopology.CmdLineParser(); 7 | cmdln 8 | .define("n", "name", "worker1", "Logical name of the worker"); 9 | let opts = cmdln.process(process.argv); 10 | 11 | qtopology.logger().setLevel("debug"); 12 | 13 | let storage = new qtopology.HttpStorage(); 14 | let w = new qtopology.TopologyWorker({ 15 | name: opts.name, 16 | storage: storage 17 | }); 18 | w.run(); 19 | 20 | // after 5sec register new topology 21 | setTimeout(() => { 22 | let topo1 = require("./topology.json"); 23 | storage.registerTopology("topology.1", topo1, (err) => { 24 | if (err) { 25 | console.log("Topology was not registered:", err); 26 | } else { 27 | console.log("Topology sucessfully registered."); 28 | storage.enableTopology("topology.1", (err) => { 29 | if (err) { 30 | console.log("Error while enabling the topology:", err); 31 | } else { 32 | console.log("Topology sucessfully enabled."); 33 | } 34 | }); 35 | } 36 | }); 37 | }, 5000); 38 | -------------------------------------------------------------------------------- /demo/quick_start/my_bolt.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MyBolt { 4 | 5 | constructor() { 6 | this._name = null; 7 | this._onEmit = null; 8 | } 9 | 10 | init(name, config, context, callback) { 11 | this._name = name; 12 | this._onEmit = config.onEmit; 13 | // use other fields from config to control your execution 14 | callback(); 15 | } 16 | 17 | heartbeat() { 18 | // do something if needed 19 | } 20 | 21 | shutdown(callback) { 22 | // prepare for gracefull shutdown, e.g. save state 23 | callback(); 24 | } 25 | 26 | receive(data, stream_id, callback) { 27 | // process incoming data 28 | // possible emit new data, using this._onEmit 29 | console.log(data, stream_id); 30 | callback(); 31 | } 32 | } 33 | 34 | exports.create = function () { return new MyBolt(); }; 35 | /* 36 | // alternatively, one could have several bolts in single file. 37 | // in that case, "subtype" attribute of the bolt declaration would be 38 | // sent into create method and we could use it to choose appropriate implementation. 39 | exports.create = function (subtype) { 40 | if (subtype == "subtype1") return new MyOtherBolt(); 41 | return new MyBolt(); 42 | }; 43 | */ -------------------------------------------------------------------------------- /demo/quick_start/my_spout.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MySpout { 4 | 5 | constructor() { 6 | this._name = null; 7 | this._data = []; 8 | this._data_index = 0; 9 | } 10 | 11 | init(name, config, context, callback) { 12 | this._name = name; 13 | // use other fields from config to control your execution 14 | 15 | for (let i = 0; i < 100; i++) { 16 | this._data.push({ id: i}); 17 | } 18 | 19 | callback(); 20 | } 21 | 22 | heartbeat() { 23 | // do something if needed 24 | } 25 | 26 | shutdown(callback) { 27 | // prepare for gracefull shutdown, e.g. save state 28 | callback(); 29 | } 30 | 31 | run() { 32 | // enable this spout - by default it should be disabled 33 | } 34 | 35 | pause() { 36 | // disable this spout 37 | } 38 | 39 | next(callback) { 40 | // return new tuple or null. Third parameter is stream id. 41 | if (this._data_index >= this._data.length) { 42 | callback(null, null, null); // or just callback() 43 | } else { 44 | callback(null, this._data[this._data_index++], "stream1"); 45 | } 46 | } 47 | } 48 | 49 | exports.create = function () { return new MySpout(); }; -------------------------------------------------------------------------------- /src/std_nodes/filter_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as pm from "../util/pattern_matcher"; 3 | 4 | ///////////////////////////////////////////////////////////////////////////// 5 | 6 | /** This bolt filters incoming messages based on provided 7 | * filter and sends them forward. 8 | */ 9 | export class FilterBolt implements intf.IBolt { 10 | 11 | private matcher: pm.PaternMatcher; 12 | private onEmit: intf.BoltEmitCallback; 13 | 14 | constructor() { 15 | this.onEmit = null; 16 | this.matcher = null; 17 | } 18 | 19 | /** Initializes filtering pattern */ 20 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 21 | this.onEmit = config.onEmit; 22 | this.matcher = new pm.PaternMatcher(config.filter); 23 | callback(); 24 | } 25 | 26 | public heartbeat() { 27 | // no-op 28 | } 29 | 30 | public shutdown(callback: intf.SimpleCallback) { 31 | callback(); 32 | } 33 | 34 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 35 | if (this.matcher.isMatch(data)) { 36 | this.onEmit(data, stream_id, callback); 37 | } else { 38 | callback(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | } 17 | ], 18 | "bolts": [ 19 | { 20 | "name": "boltp", 21 | "working_dir": ".", 22 | "type": "sys", 23 | "cmd": "process", 24 | "inputs": [ 25 | { 26 | "source": "pump1" 27 | } 28 | ], 29 | "init": { 30 | "stream_id": "streamx", 31 | "cmd_line": "node child.js" 32 | } 33 | }, 34 | { 35 | "name": "bolt1", 36 | "working_dir": ".", 37 | "type": "sys", 38 | "cmd": "console", 39 | "inputs": [ 40 | { 41 | "source": "boltp", 42 | "stream_id": "streamx" 43 | } 44 | ], 45 | "init": {} 46 | } 47 | ], 48 | "variables": {} 49 | } 50 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/topology_cpp.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | } 17 | ], 18 | "bolts": [ 19 | { 20 | "name": "boltp", 21 | "working_dir": ".", 22 | "type": "sys", 23 | "cmd": "process", 24 | "inputs": [ 25 | { 26 | "source": "pump1" 27 | } 28 | ], 29 | "init": { 30 | "stream_id": "streamx", 31 | "cmd_line": "cpp/demo.exe" 32 | } 33 | }, 34 | { 35 | "name": "bolt1", 36 | "working_dir": ".", 37 | "type": "sys", 38 | "cmd": "console", 39 | "inputs": [ 40 | { 41 | "source": "boltp", 42 | "stream_id": "streamx" 43 | } 44 | ], 45 | "init": {} 46 | } 47 | ], 48 | "variables": {} 49 | } 50 | -------------------------------------------------------------------------------- /src/std_nodes/bomb_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as log from "../util/logger"; 3 | 4 | /** This bolt explodes after predefined time interval. 5 | * Primarily used for testing. 6 | */ 7 | export class BombBolt implements intf.IBolt { 8 | 9 | private explode_after: number; 10 | private started_at: number; 11 | private onEmit: intf.BoltEmitCallback; 12 | 13 | constructor() { 14 | this.onEmit = null; 15 | this.explode_after = null; 16 | this.started_at = null; 17 | } 18 | 19 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 20 | this.onEmit = config.onEmit; 21 | this.explode_after = config.explode_after || 10 * 1000; 22 | this.started_at = Date.now(); 23 | callback(); 24 | } 25 | 26 | public heartbeat() { 27 | if (Date.now() - this.started_at >= this.explode_after) { 28 | log.logger().log("Bomb about to explode"); 29 | process.kill(process.pid, "SIGTERM"); 30 | } 31 | } 32 | 33 | public shutdown(callback: intf.SimpleCallback) { 34 | callback(); 35 | } 36 | 37 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 38 | this.onEmit(data, stream_id, callback); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /demo/qminer/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 2000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "inproc", 9 | "working_dir": ".", 10 | "cmd": "spout_1.js", 11 | "init": {} 12 | } 13 | ], 14 | "bolts": [ 15 | { 16 | "name": "bolt_qm1", 17 | "working_dir": ".", 18 | "type": "inproc", 19 | "cmd": "bolt_qm.js", 20 | "inputs": [ 21 | { 22 | "source": "pump1" 23 | } 24 | ], 25 | "init": { 26 | "db_dir": "./db1", 27 | "model_dir": "./model1", 28 | "use_target": "target1" 29 | } 30 | }, 31 | { 32 | "name": "bolt_qm2", 33 | "working_dir": ".", 34 | "type": "inproc", 35 | "cmd": "bolt_qm.js", 36 | "inputs": [ 37 | { 38 | "source": "pump1" 39 | } 40 | ], 41 | "init": { 42 | "db_dir": "./db2", 43 | "model_dir": "./model2", 44 | "use_target": "target2" 45 | } 46 | } 47 | ], 48 | "variables": {} 49 | } 50 | -------------------------------------------------------------------------------- /demo/distributed_file_based/worker_test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const qtopology = require("../../"); 4 | 5 | /////////////////////////////////////////////////////////////////////// 6 | let cmdln = new qtopology.CmdLineParser(); 7 | cmdln 8 | .define('n', 'name', 'worker1', 'Logical name of the worker'); 9 | let opts = cmdln.process(process.argv); 10 | 11 | qtopology.logger().setLevel("debug"); 12 | let storage = new qtopology.FileStorage("./topologies", "*.json"); 13 | 14 | qtopology.logger().warn("***********************************************************************"); 15 | qtopology.logger().warn("** This worker will become dormant on each even minute (0, 2, 4, ...)"); 16 | qtopology.logger().warn("***********************************************************************"); 17 | 18 | let w = new qtopology.TopologyWorker({ 19 | name: opts.name, 20 | storage: storage, 21 | is_dormant_period: () => (new Date()).getMinutes() % 2 == 0 // alive only on odd minutes 22 | }); 23 | w.run(); 24 | 25 | function shutdown() { 26 | if (w) { 27 | w.shutdown((err) => { 28 | if (err) { 29 | console.log("Error while global shutdown:", err); 30 | } 31 | console.log("Shutdown complete"); 32 | process.exit(1); 33 | }); 34 | w = null; 35 | } 36 | } 37 | 38 | setTimeout(() => { shutdown(); }, 200000); 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2017, QMiner 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /src/std_nodes/post_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as rest from "../distributed/http_based/rest_client"; 3 | 4 | /** This bolt sends POST request to specified url (fixed or provided inside data) 5 | * and forwards the request. 6 | */ 7 | export class PostBolt implements intf.IBolt { 8 | 9 | private fixed_url: string; 10 | private client: rest.IApiClient; 11 | private onEmit: intf.BoltEmitCallback; 12 | 13 | constructor() { 14 | this.onEmit = null; 15 | this.fixed_url = null; 16 | } 17 | 18 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 19 | this.onEmit = config.onEmit; 20 | this.fixed_url = config.url; 21 | this.client = rest.create({}); 22 | callback(); 23 | } 24 | 25 | public heartbeat() { 26 | // no-op 27 | } 28 | 29 | public shutdown(callback: intf.SimpleCallback) { 30 | callback(); 31 | } 32 | 33 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 34 | let url = this.fixed_url; 35 | let args = data; 36 | if (!this.fixed_url) { 37 | url = data.url; 38 | args = data.body; 39 | } 40 | this.client.post(url, args) 41 | .then(res => this.onEmit({ body: res.data }, null, callback)) 42 | .catch(err => callback(err)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /demo/std_nodes/rest/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump_timer", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | }, 17 | { 18 | "name": "pump_rest", 19 | "type": "sys", 20 | "working_dir": "", 21 | "cmd": "rest", 22 | "init": { 23 | "port": 6789 24 | } 25 | } 26 | ], 27 | "bolts": [ 28 | { 29 | "name": "bolt1", 30 | "working_dir": ".", 31 | "type": "sys", 32 | "cmd": "post", 33 | "inputs": [ 34 | { 35 | "source": "pump_timer" 36 | } 37 | ], 38 | "init": { 39 | "url": "http://localhost:6789" 40 | } 41 | }, 42 | { 43 | "name": "bolt2", 44 | "working_dir": ".", 45 | "type": "sys", 46 | "cmd": "console", 47 | "inputs": [ 48 | { 49 | "source": "pump_rest" 50 | } 51 | ], 52 | "init": {} 53 | } 54 | ], 55 | "variables": {} 56 | } 57 | -------------------------------------------------------------------------------- /demo/std_nodes/process-continuous/emitter.js: -------------------------------------------------------------------------------- 1 | let type = "json" 2 | if (process.argv.length > 2) { 3 | type = process.argv[2]; 4 | } 5 | 6 | let content = []; 7 | 8 | if (type == "json") { 9 | content.push([50, JSON.stringify({ a: 2, b: true })]); 10 | content.push([130, JSON.stringify({ a: 1, b: false })]); 11 | content.push([2500, JSON.stringify({ a: 3, b: true })]); 12 | content.push([2550, JSON.stringify({ a: 8, b: false })]); 13 | content.push([4550, JSON.stringify({ a: 4, b: true })]); 14 | content.push([4650, JSON.stringify({ a: 55, b: true })]); 15 | content.push([4750, "fufeta"]); 16 | } else if (type == "csv") { 17 | content.push([70, "1,2,3"]); 18 | content.push([80, "1,2,5"]); 19 | content.push([3500, "g,h,j"]); 20 | content.push([3550, "pok,iuh,ug"]); 21 | content.push([5550, "žćč žćčžćč,ž,ć"]); 22 | content.push([5650, "00,rewer,true"]); 23 | } else { 24 | content.push([50, "jjji"]); 25 | content.push([130, "a b c d"]); 26 | content.push([1500, "098 098 09876 5"]); 27 | content.push([1550, "----------- - ----- ---- - -- "]); 28 | content.push([3550, "*******"]); 29 | content.push([3650, "tzu uzuztuz uizghbvbn klčnkj njk hhgk"]); 30 | 31 | for (let i = 0; i < 20; i++) { 32 | content.push([4500 + i, "content " + i]); 33 | } 34 | } 35 | 36 | content.forEach(x => { 37 | setTimeout(() => { console.log(x[1]); }, x[0]); 38 | }); 39 | setTimeout(() => { console.error('testin stderr')}, 5000); 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/util/telemetry.ts: -------------------------------------------------------------------------------- 1 | /** Simple class for collecting telemetry statistics for call durations */ 2 | export class Telemetry { 3 | 4 | private cnt: number; 5 | private min: number; 6 | private max: number; 7 | private avg: number; 8 | private name: string; 9 | 10 | constructor(name: string) { 11 | this.cnt = 0; 12 | this.avg = 0; 13 | this.min = 0; 14 | this.max = 0; 15 | this.name = name; 16 | } 17 | 18 | public add(duration: number) { 19 | if (this.cnt === 0) { 20 | this.avg = duration; 21 | this.cnt = 1; 22 | this.min = duration; 23 | this.max = duration; 24 | } else { 25 | const tc = this.cnt; 26 | const tc1 = this.cnt + 1; 27 | this.avg = this.avg * (tc / tc1) + duration / tc1; 28 | this.cnt++; 29 | this.min = Math.min(this.min, duration); 30 | this.max = Math.max(this.max, duration); 31 | } 32 | } 33 | 34 | public reset() { 35 | this.cnt = 0; 36 | this.avg = 0; 37 | this.min = 0; 38 | this.max = 0; 39 | } 40 | 41 | public get(add_name?: boolean) { 42 | if (add_name) { 43 | return { 44 | avg: this.avg, 45 | cnt: this.cnt, 46 | name: this.name 47 | }; 48 | } 49 | return { 50 | avg: this.avg, 51 | cnt: this.cnt 52 | }; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /demo/local_massive/demo_local_massive.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(() => { 19 | xcallback(); 20 | }, 10000); 21 | }, 22 | (xcallback) => { 23 | console.log("Starting shutdown sequence..."); 24 | topology.shutdown(xcallback); 25 | topology = null; 26 | } 27 | ], 28 | (err) => { 29 | if (err) { 30 | console.log("Error", err); 31 | } 32 | console.log("Finished."); 33 | } 34 | ); 35 | 36 | function shutdown(err) { 37 | if (topology) { 38 | topology.shutdown((err) => { 39 | if (err) { 40 | console.log("Error", err); 41 | } 42 | process.exit(1); 43 | }); 44 | topology = null; 45 | } 46 | } 47 | 48 | //do something when app is closing 49 | process.on('exit', shutdown); 50 | 51 | //catches ctrl+c event 52 | process.on('SIGINT', shutdown); 53 | 54 | //catches uncaught exceptions 55 | process.on('uncaughtException', shutdown); 56 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "qtopology", 3 | "version": "2.3.2", 4 | "description": "Distributed stream processing engine.", 5 | "main": "./built/index.js", 6 | "typings": "./built/index", 7 | "scripts": { 8 | "build": "tsc", 9 | "test": "npm run test-unit && cd demo && bash run_demos.sh", 10 | "test-unit": "mocha tests --recursive --timeout 10000", 11 | "prepare": "npm run format", 12 | "format": "./node_modules/.bin/tsfmt -r", 13 | "lint": "./node_modules/.bin/tslint --project ." 14 | }, 15 | "repository": "https://github.com/qminer/qtopology.git", 16 | "keywords": [ 17 | "node.js" 18 | ], 19 | "author": "Viktor Jovanoski", 20 | "contributors": [ 21 | { 22 | "name": "Viktor Jovanoski", 23 | "email": "viktor@carvic.si" 24 | }, 25 | { 26 | "name": "Jan Rupnik", 27 | "email": "jan.rupnik@ijs.si" 28 | } 29 | ], 30 | "license": "BSD-2-Clause", 31 | "readmeFilename": "README.md", 32 | "devDependencies": { 33 | "@types/async": "^2.4.2", 34 | "@types/node": "^8.10.59", 35 | "mocha": "^7.0.0", 36 | "tslint": "^5.20.1", 37 | "typescript": "^3.8.3", 38 | "typescript-formatter": "^7.2.2" 39 | }, 40 | "dependencies": { 41 | "async": "^2.6.4", 42 | "axios": "^0.21.1", 43 | "body-parser": "^1.19.0", 44 | "colors": "1.2.1", 45 | "deserialize-error": "0.0.3", 46 | "express": "^4.17.1", 47 | "jsonschema": "^1.2.5", 48 | "qewd-transform-json": "^1.11.0", 49 | "serialize-error": "^2.1.0" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /demo/std_nodes/counter/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "timer", 11 | "init": { 12 | "extra_fields": { 13 | "field1": "a" 14 | } 15 | } 16 | } 17 | ], 18 | "bolts": [ 19 | { 20 | "name": "bolt_counter", 21 | "working_dir": ".", 22 | "type": "sys", 23 | "cmd": "counter", 24 | "inputs": [ 25 | { "source": "pump1" } 26 | ], 27 | "init": { 28 | "timeout": 3500, 29 | "prefix": "Demo topology" 30 | } 31 | }, 32 | { 33 | "name": "bolt1", 34 | "working_dir": ".", 35 | "type": "sys", 36 | "cmd": "console", 37 | "inputs": [ 38 | { 39 | "source": "pump1" 40 | } 41 | ], 42 | "init": {} 43 | }, 44 | { 45 | "name": "bolt2", 46 | "working_dir": ".", 47 | "type": "sys", 48 | "cmd": "console", 49 | "inputs": [ 50 | { 51 | "source": "bolt_counter" 52 | } 53 | ], 54 | "init": {} 55 | } 56 | ], 57 | "variables": {} 58 | } 59 | -------------------------------------------------------------------------------- /demo/std_nodes/demo_std_nodes.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(() => { 19 | setTimeout(function () { 20 | xcallback(); 21 | }, 5000); 22 | }); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | 37 | 38 | function shutdown() { 39 | if (topology) { 40 | topology.shutdown((err) => { 41 | if (err) { 42 | console.log("Error", err); 43 | } 44 | process.exit(1); 45 | }); 46 | topology = null; 47 | } 48 | } 49 | 50 | //do something when app is closing 51 | process.on('exit', shutdown); 52 | 53 | //catches ctrl+c event 54 | process.on('SIGINT', shutdown); 55 | 56 | //catches uncaught exceptions 57 | process.on('uncaughtException', shutdown); 58 | -------------------------------------------------------------------------------- /demo/distributed_file_based/topologies/topology1.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000, 4 | "initialization": [ 5 | { 6 | "working_dir": ".", 7 | "cmd": "init_and_shutdown.js" 8 | } 9 | ], 10 | "shutdown": [ 11 | { 12 | "working_dir": ".", 13 | "cmd": "init_and_shutdown.js" 14 | } 15 | ] 16 | }, 17 | "spouts": [ 18 | { 19 | "name": "pump", 20 | "type": "sys", 21 | "working_dir": "", 22 | "cmd": "timer", 23 | "init": { 24 | "extra_fields": { 25 | "field1": "a" 26 | } 27 | } 28 | } 29 | ], 30 | "bolts": [ 31 | { 32 | "name": "bolt1", 33 | "working_dir": ".", 34 | "type": "sys", 35 | "cmd": "console", 36 | "inputs": [ 37 | { 38 | "source": "pump" 39 | } 40 | ], 41 | "init": {} 42 | }, 43 | { 44 | "name": "bolt_bomb", 45 | "working_dir": ".", 46 | "type": "sys", 47 | "cmd": "bomb", 48 | "disabled": true, 49 | "inputs": [ 50 | { 51 | "source": "pump" 52 | } 53 | ], 54 | "init": { 55 | "explode_after": 4500 56 | } 57 | } 58 | ], 59 | "variables": {} 60 | } 61 | -------------------------------------------------------------------------------- /demo/local/demo_local.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 10 sec"); 22 | setTimeout(() => { xcallback(); }, 10000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | topology = null; 28 | } 29 | ], 30 | (err) => { 31 | if (err) { 32 | console.log("Error", err); 33 | } 34 | console.log("Finished."); 35 | } 36 | ); 37 | 38 | function shutdown() { 39 | if (topology) { 40 | topology.shutdown((err) => { 41 | if (err) { 42 | console.log("Error", err); 43 | } 44 | process.exit(1); 45 | }); 46 | topology = null; 47 | } 48 | } 49 | 50 | //do something when app is closing 51 | process.on('exit', shutdown); 52 | 53 | //catches ctrl+c event 54 | process.on('SIGINT', shutdown); 55 | 56 | //catches uncaught exceptions 57 | process.on('uncaughtException', shutdown); 58 | -------------------------------------------------------------------------------- /demo/qminer/demo_qminer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(() => { 19 | setTimeout(function () { 20 | xcallback(); 21 | }, 4000); 22 | }); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | 37 | 38 | function shutdown() { 39 | if (topology) { 40 | topology.shutdown((err) => { 41 | if (err) { 42 | console.log("Error", err); 43 | } 44 | process.exit(1); 45 | }); 46 | topology = null; 47 | } 48 | } 49 | 50 | //do something when app is closing 51 | process.on('exit', shutdown); 52 | 53 | //catches ctrl+c event 54 | process.on('SIGINT', () => { 55 | console.log("CTRL+c"); 56 | shutdown(); 57 | }); 58 | 59 | //catches uncaught exceptions 60 | process.on('uncaughtException', shutdown); 61 | -------------------------------------------------------------------------------- /demo/distributed_file_based/topologies/topology2.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000, 4 | "pass_binary_messages": true, 5 | "initialization": [ 6 | { 7 | "working_dir": ".", 8 | "cmd": "init_and_shutdown.js" 9 | } 10 | ], 11 | "shutdown": [ 12 | { 13 | "working_dir": ".", 14 | "cmd": "init_and_shutdown.js" 15 | } 16 | ] 17 | }, 18 | "spouts": [ 19 | { 20 | "name": "pump", 21 | "type": "sys", 22 | "working_dir": "", 23 | "cmd": "timer", 24 | "init": { 25 | "extra_fields": { 26 | "field2": "b" 27 | } 28 | } 29 | } 30 | ], 31 | "bolts": [ 32 | { 33 | "name": "bolt2", 34 | "working_dir": ".", 35 | "type": "sys", 36 | "cmd": "console", 37 | "inputs": [ 38 | { 39 | "source": "pump" 40 | } 41 | ], 42 | "init": {} 43 | }, 44 | { 45 | "name": "bolt_bomb", 46 | "working_dir": ".", 47 | "type": "sys", 48 | "cmd": "bomb", 49 | "disabled": true, 50 | "inputs": [ 51 | { 52 | "source": "pump" 53 | } 54 | ], 55 | "init": { 56 | "explode_after": 11000 57 | } 58 | } 59 | ], 60 | "variables": {} 61 | } 62 | -------------------------------------------------------------------------------- /demo/std_nodes/bomb/demo_bomb.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | 37 | 38 | function shutdown() { 39 | if (topology) { 40 | topology.shutdown((err) => { 41 | if (err) { 42 | console.log("Error", err); 43 | } 44 | process.exit(1); 45 | }); 46 | topology = null; 47 | } 48 | } 49 | 50 | //do something when app is closing 51 | process.on('exit', shutdown); 52 | 53 | //catches ctrl+c event 54 | process.on('SIGINT', shutdown); 55 | 56 | //catches uncaught exceptions 57 | process.on('uncaughtException', 58 | (e) => { 59 | console.log(e); 60 | process.exit(1); 61 | } 62 | //shutdown 63 | ); 64 | -------------------------------------------------------------------------------- /demo/std_nodes/disabled/demo_disabling.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../.."); 5 | 6 | let config = require("./topology.json"); 7 | qtopology.validate({ config: config, exitOnError: true }); 8 | let topology = new qtopology.TopologyLocal(); 9 | 10 | async.series( 11 | [ 12 | (xcallback) => { 13 | topology.init("topology.1", config, xcallback); 14 | }, 15 | (xcallback) => { 16 | console.log("Init done"); 17 | topology.run(xcallback); 18 | }, 19 | (xcallback) => { 20 | console.log("Waiting - 5 sec"); 21 | setTimeout(() => { xcallback(); }, 5000); 22 | }, 23 | (xcallback) => { 24 | console.log("Starting shutdown sequence..."); 25 | topology.shutdown(xcallback); 26 | topology = null; 27 | } 28 | ], 29 | (err) => { 30 | if (err) { 31 | console.log("Error in shutdown", err); 32 | } 33 | console.log("Finished."); 34 | } 35 | ); 36 | 37 | 38 | function shutdown(err) { 39 | 40 | if (err) { 41 | console.log("Error", err); 42 | } 43 | if (topology) { 44 | topology.shutdown((err) => { 45 | if (err) { 46 | console.log("Error", err); 47 | } 48 | process.exit(1); 49 | }); 50 | topology = null; 51 | } 52 | } 53 | 54 | //do something when app is closing 55 | process.on('exit', shutdown); 56 | 57 | //catches ctrl+c event 58 | process.on('SIGINT', shutdown); 59 | 60 | //catches uncaught exceptions 61 | process.on('uncaughtException', (err) => { shutdown(err); }); 62 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append/demo_file_append.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const async = require("async"); 5 | const qtopology = require("../../.."); 6 | 7 | // demo configuration 8 | let config = require("./topology.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | async.series( 13 | [ 14 | (xcallback) => { 15 | console.log("Starting init"); 16 | topology.init("topology.1", config, xcallback); 17 | }, 18 | (xcallback) => { 19 | console.log("Init done"); 20 | topology.run(xcallback); 21 | }, 22 | (xcallback) => { 23 | console.log("Waiting - 10 sec"); 24 | setTimeout(() => { xcallback(); }, 10000); 25 | }, 26 | (xcallback) => { 27 | console.log("Starting shutdown sequence..."); 28 | topology.shutdown(xcallback); 29 | topology = null; 30 | } 31 | ], 32 | (err) => { 33 | if (err) { 34 | console.log("Error in shutdown", err); 35 | } 36 | console.log("Finished."); 37 | } 38 | ); 39 | 40 | 41 | function shutdown() { 42 | if (topology) { 43 | topology.shutdown((err) => { 44 | if (err) { 45 | console.log("Error", err); 46 | } 47 | process.exit(1); 48 | }); 49 | topology = null; 50 | } 51 | } 52 | 53 | //do something when app is closing 54 | process.on('exit', shutdown); 55 | 56 | //catches ctrl+c event 57 | process.on('SIGINT', shutdown); 58 | 59 | //catches uncaught exceptions 60 | process.on('uncaughtException', shutdown); 61 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_csv/demo_file_append_csv.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const async = require("async"); 5 | const qtopology = require("../../.."); 6 | 7 | // demo configuration 8 | let config = require("./topology.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | async.series( 13 | [ 14 | (xcallback) => { 15 | console.log("Starting init"); 16 | topology.init("topology.1", config, xcallback); 17 | }, 18 | (xcallback) => { 19 | console.log("Init done"); 20 | topology.run(xcallback); 21 | }, 22 | (xcallback) => { 23 | console.log("Waiting - 10 sec"); 24 | setTimeout(() => { xcallback(); }, 10000); 25 | }, 26 | (xcallback) => { 27 | console.log("Starting shutdown sequence..."); 28 | topology.shutdown(xcallback); 29 | topology = null; 30 | } 31 | ], 32 | (err) => { 33 | if (err) { 34 | console.log("Error in shutdown", err); 35 | } 36 | console.log("Finished."); 37 | } 38 | ); 39 | 40 | 41 | function shutdown() { 42 | if (topology) { 43 | topology.shutdown((err) => { 44 | if (err) { 45 | console.log("Error", err); 46 | } 47 | process.exit(1); 48 | }); 49 | topology = null; 50 | } 51 | } 52 | 53 | //do something when app is closing 54 | process.on('exit', shutdown); 55 | 56 | //catches ctrl+c event 57 | process.on('SIGINT', shutdown); 58 | 59 | //catches uncaught exceptions 60 | process.on('uncaughtException', shutdown); 61 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_ex/demo_file_append_ex.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const async = require("async"); 5 | const qtopology = require("../../.."); 6 | 7 | // demo configuration 8 | let config = require("./topology.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | async.series( 13 | [ 14 | (xcallback) => { 15 | console.log("Starting init"); 16 | topology.init("topology.1", config, xcallback); 17 | }, 18 | (xcallback) => { 19 | console.log("Init done"); 20 | topology.run(xcallback); 21 | }, 22 | (xcallback) => { 23 | console.log("Waiting - 10 sec"); 24 | setTimeout(() => { xcallback(); }, 10000); 25 | }, 26 | (xcallback) => { 27 | console.log("Starting shutdown sequence..."); 28 | topology.shutdown(xcallback); 29 | topology = null; 30 | } 31 | ], 32 | (err) => { 33 | if (err) { 34 | console.log("Error in shutdown", err); 35 | } 36 | console.log("Finished."); 37 | } 38 | ); 39 | 40 | 41 | function shutdown() { 42 | if (topology) { 43 | topology.shutdown((err) => { 44 | if (err) { 45 | console.log("Error", err); 46 | } 47 | process.exit(1); 48 | }); 49 | topology = null; 50 | } 51 | } 52 | 53 | //do something when app is closing 54 | process.on('exit', shutdown); 55 | 56 | //catches ctrl+c event 57 | process.on('SIGINT', shutdown); 58 | 59 | //catches uncaught exceptions 60 | process.on('uncaughtException', shutdown); 61 | -------------------------------------------------------------------------------- /demo/std_nodes/process/demo_process.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | topology = null; 28 | } 29 | ], 30 | (err) => { 31 | if (err) { 32 | console.log("Error in shutdown", err); 33 | } 34 | console.log("Finished."); 35 | } 36 | ); 37 | 38 | 39 | function shutdown() { 40 | if (topology) { 41 | topology.shutdown((err) => { 42 | if (err) { 43 | console.log("Error", err); 44 | } 45 | process.exit(1); 46 | }); 47 | topology = null; 48 | } 49 | } 50 | 51 | //do something when app is closing 52 | process.on('exit', shutdown); 53 | 54 | //catches ctrl+c event 55 | process.on('SIGINT', shutdown); 56 | 57 | //catches uncaught exceptions 58 | process.on('uncaughtException', 59 | (e) => { 60 | console.log(e); 61 | process.exit(1); 62 | } 63 | //shutdown 64 | ); 65 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/demo_file_reader.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | topology = null; 28 | } 29 | ], 30 | (err) => { 31 | if (err) { 32 | console.log("Error in shutdown", err); 33 | } 34 | console.log("Finished."); 35 | } 36 | ); 37 | 38 | 39 | function shutdown() { 40 | if (topology) { 41 | topology.shutdown((err) => { 42 | if (err) { 43 | console.log("Error", err); 44 | } 45 | process.exit(1); 46 | }); 47 | topology = null; 48 | } 49 | } 50 | 51 | //do something when app is closing 52 | process.on('exit', shutdown); 53 | 54 | //catches ctrl+c event 55 | process.on('SIGINT', shutdown); 56 | 57 | //catches uncaught exceptions 58 | process.on('uncaughtException', 59 | (e) => { 60 | console.log(e); 61 | process.exit(1); 62 | } 63 | //shutdown 64 | ); 65 | -------------------------------------------------------------------------------- /demo/std_nodes/task_bolt_base/demo_task_bolt_base.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 10 sec"); 22 | setTimeout(() => { xcallback(); }, 10000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | topology = null; 28 | } 29 | ], 30 | (err) => { 31 | if (err) { 32 | console.log("Error in shutdown", err); 33 | } 34 | console.log("Finished."); 35 | } 36 | ); 37 | 38 | 39 | function shutdown() { 40 | if (topology) { 41 | topology.shutdown((err) => { 42 | if (err) { 43 | console.log("Error", err); 44 | } 45 | process.exit(1); 46 | }); 47 | topology = null; 48 | } 49 | } 50 | 51 | //do something when app is closing 52 | process.on('exit', shutdown); 53 | 54 | //catches ctrl+c event 55 | process.on('SIGINT', shutdown); 56 | 57 | //catches uncaught exceptions 58 | process.on('uncaughtException', 59 | (e) => { 60 | console.log(e); 61 | process.exit(1); 62 | } 63 | //shutdown 64 | ); 65 | -------------------------------------------------------------------------------- /demo/std_nodes/process-continuous/demo_process_continuous.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | qtopology.validate({ config: config, exitOnError: true }); 9 | let topology = new qtopology.TopologyLocal(); 10 | 11 | async.series( 12 | [ 13 | (xcallback) => { 14 | topology.init("topology.1", config, xcallback); 15 | }, 16 | (xcallback) => { 17 | console.log("Init done"); 18 | topology.run(xcallback); 19 | }, 20 | (xcallback) => { 21 | console.log("Waiting - 20 sec"); 22 | setTimeout(() => { xcallback(); }, 20000); 23 | }, 24 | (xcallback) => { 25 | console.log("Starting shutdown sequence..."); 26 | topology.shutdown(xcallback); 27 | topology = null; 28 | } 29 | ], 30 | (err) => { 31 | if (err) { 32 | console.log("Error in shutdown", err); 33 | } 34 | console.log("Finished."); 35 | } 36 | ); 37 | 38 | 39 | function shutdown() { 40 | if (topology) { 41 | topology.shutdown((err) => { 42 | if (err) { 43 | console.log("Error", err); 44 | } 45 | process.exit(1); 46 | }); 47 | topology = null; 48 | } 49 | } 50 | 51 | //do something when app is closing 52 | process.on('exit', shutdown); 53 | 54 | //catches ctrl+c event 55 | process.on('SIGINT', shutdown); 56 | 57 | //catches uncaught exceptions 58 | process.on('uncaughtException', 59 | (e) => { 60 | console.log(e); 61 | process.exit(1); 62 | } 63 | //shutdown 64 | ); 65 | -------------------------------------------------------------------------------- /tests/std_nodes/console_bolt.tests.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const cb = require("../../built/std_nodes/console_bolt"); 7 | 8 | describe('ConsoleBolt', function () { 9 | it('constructable', function () { 10 | let target = new cb.ConsoleBolt(); 11 | }); 12 | it('init passes', function (done) { 13 | let emited = []; 14 | let name = "some_name"; 15 | let config = { 16 | onEmit: (data, stream_id, callback) => { 17 | emited.push({ data, stream_id }); 18 | callback(); 19 | } 20 | }; 21 | let target = new cb.ConsoleBolt(); 22 | target.init(name, config, null, (err) => { 23 | assert.ok(!err); 24 | done(); 25 | }); 26 | }); 27 | it('receive - must pass through', function (done) { 28 | let emited = []; 29 | let name = "some_name"; 30 | let xdata = { test: true }; 31 | let xstream_id = null; 32 | let config = { 33 | onEmit: (data, stream_id, callback) => { 34 | emited.push({ data, stream_id }); 35 | callback(); 36 | } 37 | }; 38 | let target = new cb.ConsoleBolt(); 39 | target.init(name, config, null, (err) => { 40 | assert.ok(!err); 41 | target.receive(xdata, xstream_id, (err) => { 42 | assert.ok(!err); 43 | assert.equal(emited.length, 1); 44 | assert.deepEqual(emited[0].data, xdata); 45 | assert.equal(emited[0].stream_id, xstream_id); 46 | done(); 47 | }); 48 | }); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /demo/std_nodes/counter/demo_counter.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const fs = require("fs"); 5 | const qtopology = require("../../../"); 6 | 7 | // demo configuration 8 | let config = qtopology.readJsonFileSync("./topology.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | async.series( 13 | [ 14 | (xcallback) => { 15 | topology.init("topology.1", config, xcallback); 16 | }, 17 | (xcallback) => { 18 | console.log("Init done"); 19 | topology.run(xcallback); 20 | }, 21 | (xcallback) => { 22 | console.log("Waiting - 10 sec"); 23 | setTimeout(() => { xcallback(); }, 10000); 24 | }, 25 | (xcallback) => { 26 | console.log("Starting shutdown sequence..."); 27 | topology.shutdown(xcallback); 28 | topology = null; 29 | } 30 | ], 31 | (err) => { 32 | if (err) { 33 | console.log("Error in shutdown", err); 34 | } 35 | console.log("Finished."); 36 | } 37 | ); 38 | 39 | 40 | function shutdown() { 41 | if (topology) { 42 | topology.shutdown((err) => { 43 | if (err) { 44 | console.log("Error", err); 45 | } 46 | process.exit(1); 47 | }); 48 | topology = null; 49 | } 50 | } 51 | 52 | //do something when app is closing 53 | process.on('exit', shutdown); 54 | 55 | //catches ctrl+c event 56 | process.on('SIGINT', shutdown); 57 | 58 | //catches uncaught exceptions 59 | process.on('uncaughtException', 60 | (e) => { 61 | console.log(e); 62 | process.exit(1); 63 | } 64 | //shutdown 65 | ); 66 | -------------------------------------------------------------------------------- /demo/local/bolt_inproc.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class MyBolt { 4 | 5 | constructor() { 6 | this._name = null; 7 | this._context = null; 8 | this._prefix = ""; 9 | this._sum = 0; 10 | this._forward = true; 11 | this._onEmit = null; 12 | } 13 | 14 | init(name, config, context, callback) { 15 | this._context = context; 16 | this._name = name; 17 | this._prefix = `[InprocBolt ${this._name}]`; 18 | console.log(this._prefix, "Inside init:", config); 19 | this._onEmit = config.onEmit; 20 | this._forward = config.forward; 21 | callback(); 22 | } 23 | 24 | heartbeat() { 25 | console.log(this._prefix, "Inside heartbeat. sum=" + this._sum, "Context", this._context); 26 | //this._onEmit({ sum: this._sum }, () => { }); 27 | } 28 | 29 | shutdown(callback) { 30 | console.log(this._prefix, "Shutting down gracefully. sum=" + this._sum); 31 | callback(); 32 | } 33 | 34 | receive(data, stream_id, callback) { 35 | let self = this; 36 | if (self._context) { 37 | self._context.cnt++; 38 | } 39 | console.log(this._prefix, "Inside receive", data, "$" + stream_id + "$"); 40 | this._sum += data.a; 41 | setTimeout(function () { 42 | if (self._forward) { 43 | data.sum = self._sum; 44 | let xstream_id = (data.sum % 2 === 0 ? "Even" : "Odd"); 45 | self._onEmit(data, xstream_id, callback); // emit same data, with addition of sum 46 | } else { 47 | callback(); 48 | } 49 | }, Math.round(80 * Math.random())); 50 | } 51 | } 52 | 53 | exports.create = function () { 54 | return new MyBolt(); 55 | }; 56 | -------------------------------------------------------------------------------- /src/std_nodes/router_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as async from "async"; 3 | import * as pm from "../util/pattern_matcher"; 4 | 5 | /** This bolt routs incoming messages based on provided 6 | * queries and sends them forward using mapped stream ids. 7 | */ 8 | export class RouterBolt implements intf.IBolt { 9 | 10 | private matchers: any[]; 11 | private onEmit: intf.BoltEmitCallback; 12 | 13 | /** Simple constructor */ 14 | constructor() { 15 | this.onEmit = null; 16 | this.matchers = []; 17 | } 18 | 19 | /** Initializes routing patterns */ 20 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 21 | this.onEmit = config.onEmit; 22 | for (const stream_id in config.routes) { 23 | if (config.routes.hasOwnProperty(stream_id)) { 24 | const filter = config.routes[stream_id]; 25 | this.matchers.push({ 26 | matcher: new pm.PaternMatcher(filter), 27 | stream_id 28 | }); 29 | } 30 | } 31 | callback(); 32 | } 33 | 34 | public heartbeat() { 35 | // no-op 36 | } 37 | 38 | public shutdown(callback: intf.SimpleCallback) { 39 | callback(); 40 | } 41 | 42 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 43 | const tasks = []; 44 | for (const item of this.matchers) { 45 | if (item.matcher.isMatch(data)) { 46 | /* jshint loopfunc:true */ 47 | tasks.push(xcallback => { 48 | this.onEmit(data, item.stream_id, xcallback); 49 | }); 50 | } 51 | } 52 | async.parallel(tasks, callback); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/util/stream_helpers.ts: -------------------------------------------------------------------------------- 1 | import * as stream from "stream"; 2 | import * as fs from "fs"; 3 | 4 | //////////////////////////////////////////////////////////////////////////// 5 | 6 | /** This class is stream-transforming object that can be piped. 7 | * It splits input data into lines and emits them one-by-one. 8 | */ 9 | export class Liner extends stream.Transform { 10 | 11 | private lastLineData: string; 12 | 13 | constructor() { 14 | super({ objectMode: true }); 15 | } 16 | 17 | // split lines and send them one-by-one 18 | public _transform(chunk, encoding, done) { 19 | let data = chunk.toString(); 20 | if (this.lastLineData) { 21 | data = this.lastLineData + data; 22 | } 23 | const lines = data.split("\n"); 24 | this.lastLineData = lines.splice(lines.length - 1, 1)[0]; 25 | 26 | lines.forEach(this.push.bind(this)); 27 | done(); 28 | } 29 | 30 | // flush any left-overs 31 | public _flush(done) { 32 | if (this.lastLineData) { 33 | this.push(this.lastLineData); 34 | } 35 | this.lastLineData = null; 36 | done(); 37 | } 38 | } 39 | 40 | export interface IParser { 41 | addLine(line: string); 42 | } 43 | 44 | export function importFileByLine(fname: string, line_parser: IParser, callback?: () => void) { 45 | const liner_obj = new Liner(); 46 | const source = fs.createReadStream(fname); 47 | source.pipe(liner_obj); 48 | liner_obj.on("readable", () => { 49 | let chunk = liner_obj.read(); 50 | while (chunk) { 51 | line_parser.addLine(chunk); 52 | chunk = liner_obj.read(); 53 | } 54 | }); 55 | source.on("close", () => { 56 | if (callback) { 57 | callback(); 58 | } 59 | }); 60 | } 61 | -------------------------------------------------------------------------------- /src/std_nodes/timer_spout.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as oo from "../util/object_override"; 3 | 4 | /** This spout emits single tuple each heartbeat */ 5 | export class TimerSpout implements intf.ISpout { 6 | 7 | private stream_id: string; 8 | private title: string; 9 | private should_run: boolean; 10 | private extra_fields: any; 11 | private next_tuple: any; 12 | 13 | constructor() { 14 | this.stream_id = null; 15 | this.title = null; 16 | this.extra_fields = null; 17 | 18 | this.next_tuple = null; 19 | this.should_run = false; 20 | } 21 | 22 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 23 | this.stream_id = config.stream_id; 24 | this.title = config.title || "heartbeat"; 25 | this.extra_fields = JSON.parse(JSON.stringify(config.extra_fields || {})); 26 | callback(); 27 | } 28 | 29 | public heartbeat() { 30 | if (!this.should_run) { return; } 31 | this.next_tuple = { 32 | title: this.title, 33 | ts: new Date().toISOString() 34 | }; 35 | oo.overrideObject(this.next_tuple, this.extra_fields, false); 36 | } 37 | 38 | public shutdown(callback: intf.SimpleCallback) { 39 | this.should_run = false; 40 | callback(); 41 | } 42 | 43 | public run() { 44 | this.should_run = true; 45 | } 46 | 47 | public pause() { 48 | this.should_run = false; 49 | } 50 | 51 | public next(callback: intf.SpoutNextCallback) { 52 | if (!this.should_run) { 53 | return callback(null, null, null); 54 | } 55 | const data = this.next_tuple; 56 | this.next_tuple = null; 57 | callback(null, data, this.stream_id); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/std_nodes/process-bolt/demo_process_bolt.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const qtopology = require("../../../"); 5 | 6 | // demo configuration 7 | let config = require("./topology.json"); 8 | // use this line instead to test C++ program 9 | //let config = require("./topology_cpp.json"); 10 | 11 | qtopology.validate({ config: config, exitOnError: true }); 12 | let topology = new qtopology.TopologyLocal(); 13 | 14 | async.series( 15 | [ 16 | (xcallback) => { 17 | topology.init("topology.1", config, xcallback); 18 | }, 19 | (xcallback) => { 20 | console.log("Init done"); 21 | topology.run(xcallback); 22 | }, 23 | (xcallback) => { 24 | console.log("Waiting - 20 sec"); 25 | setTimeout(() => { xcallback(); }, 20000); 26 | }, 27 | (xcallback) => { 28 | console.log("Starting shutdown sequence..."); 29 | topology.shutdown(xcallback); 30 | topology = null; 31 | } 32 | ], 33 | (err) => { 34 | if (err) { 35 | console.log("Error in shutdown", err); 36 | } 37 | console.log("Finished."); 38 | } 39 | ); 40 | 41 | 42 | function shutdown() { 43 | if (topology) { 44 | topology.shutdown((err) => { 45 | if (err) { 46 | console.log("Error", err); 47 | } 48 | process.exit(1); 49 | }); 50 | topology = null; 51 | } 52 | } 53 | 54 | //do something when app is closing 55 | process.on('exit', shutdown); 56 | 57 | //catches ctrl+c event 58 | process.on('SIGINT', shutdown); 59 | 60 | //catches uncaught exceptions 61 | process.on('uncaughtException', 62 | (e) => { 63 | console.log(e); 64 | process.exit(1); 65 | } 66 | //shutdown 67 | ); 68 | -------------------------------------------------------------------------------- /src/std_nodes/task_bolt_base.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | 3 | /** This bolt base object server as base class for simple tasks 4 | * that are repeated every X milliseconds. 5 | */ 6 | export class TaskBoltBase implements intf.IBolt { 7 | 8 | protected onEmit: intf.BoltEmitCallback; 9 | private is_running: boolean; 10 | private next_run: number; 11 | private repeat_after: number; 12 | private shutdown_cb: intf.SimpleCallback; 13 | 14 | constructor() { 15 | this.onEmit = null; 16 | this.is_running = false; 17 | this.next_run = 0; 18 | } 19 | 20 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 21 | this.onEmit = config.onEmit; 22 | this.repeat_after = config.repeat_after || 60000; // each minute 23 | callback(); 24 | } 25 | 26 | public heartbeat() { 27 | if (this.is_running) { 28 | return; 29 | } 30 | if (this.next_run > Date.now()) { 31 | return; 32 | } 33 | 34 | this.is_running = true; 35 | this.next_run = Date.now() + this.repeat_after; 36 | this.runInternal(err => { 37 | this.is_running = false; 38 | if (this.shutdown_cb) { 39 | const cb = this.shutdown_cb; 40 | this.shutdown_cb = null; 41 | cb(); 42 | } 43 | }); 44 | } 45 | 46 | public shutdown(callback: intf.SimpleCallback) { 47 | if (!this.is_running) { 48 | return callback(); 49 | } 50 | this.shutdown_cb = callback; 51 | } 52 | 53 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 54 | callback(); 55 | } 56 | 57 | protected runInternal(callback: intf.SimpleCallback) { 58 | callback(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /tests/util/strip_json_comments.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const qtopology = require("../.."); 7 | 8 | describe('stripJsonComments', function () { 9 | it('single-line comment', function () { 10 | let s = "{\n // abc \n \"a\": 12 \n }"; 11 | let s_stripped = qtopology.stripJsonComments(s); 12 | assert.deepEqual(JSON.parse(s_stripped), { a: 12}); 13 | }); 14 | it('open-close comment', function () { 15 | let s = "{\n /* abc \n asdasd */ \"a\": 12 \n }"; 16 | let s_stripped = qtopology.stripJsonComments(s); 17 | assert.deepEqual(JSON.parse(s_stripped), { a: 12}); 18 | }); 19 | 20 | 21 | it('mixed comment 1', function () { 22 | let s = "{\n // /* abc \n \"a\": 12 \n }"; 23 | let s_stripped = qtopology.stripJsonComments(s); 24 | assert.deepEqual(JSON.parse(s_stripped), { a: 12}); 25 | }); 26 | 27 | it('mixed comment 2', function () { 28 | let s = "{\n /* abc \n // abc \n */ \"a\": 12 \n }"; 29 | let s_stripped = qtopology.stripJsonComments(s); 30 | assert.deepEqual(JSON.parse(s_stripped), { a: 12}); 31 | }); 32 | 33 | describe('Comments inside a string', function () { 34 | it('single-line comment', function () { 35 | let s = "{ \"a\": \"as//d\" \n }"; 36 | let s_stripped = qtopology.stripJsonComments(s); 37 | assert.deepEqual(JSON.parse(s_stripped), { a: "as//d"}); 38 | }); 39 | it('open-close comment', function () { 40 | let s = "{ \"a\": \"as/*ww*/d\" \n }"; 41 | let s_stripped = qtopology.stripJsonComments(s); 42 | assert.deepEqual(JSON.parse(s_stripped), { a: "as/*ww*/d"}); 43 | }); 44 | }); 45 | }); 46 | 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # QTopology 2 | 3 | [![npm](https://img.shields.io/npm/v/qtopology.svg)]() 4 | 5 | QTopology is a distributed stream processing layer, written in `node.js`. 6 | 7 | NPM package: [https://www.npmjs.com/package/qtopology](https://www.npmjs.com/package/qtopology) 8 | 9 | Documentation [https://qminer.github.io/qtopology](https://qminer.github.io/qtopology) 10 | 11 | ## Installation 12 | 13 | `````````````bash 14 | npm install qtopology 15 | ````````````` 16 | 17 | ## Intro 18 | 19 | QTopology is a distributed stream processing layer, written in `node.js`. 20 | 21 | It uses the following terminology, originating in [Storm](http://storm.apache.org/) project: 22 | 23 | - **Topology** - Organization of nodes into a graph that determines paths where messages must travel. 24 | - **Bolt** - Node in topology that receives input data from other nodes and emits new data into the topology. 25 | - **Spout** - Node in topology that reads data from external sources and emits the data into the topology. 26 | - **Stream** - When data flows through the topology, it is optionaly tagged with stream ID. This can be used for routing. 27 | 28 | When running in distributed mode, `qtopology` also uses the following: 29 | 30 | - **Coordination storage** - must be resilient, receives worker registrations and sends them the initialization data. Also sends shutdown signals. Implementation is custom. `QTopology` provides `REST`-based service out-of-the-box, but the design is similar for other options like `MySQL` storage etc. 31 | - **Worker** - Runs on single server. Registers with coordination storage, receives initialization data and instantiates local topologies in separate subprocesses. 32 | - **Leader** - one of the active workers is announced leader and it performs leadership tasks such as assigning of topologies to workers, detection of dead or inactive workers. 33 | 34 | ## Quick start 35 | 36 | See [documentation](https://qminer.github.io/qtopology/) 37 | -------------------------------------------------------------------------------- /demo/local_massive/spout_common.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const batch_size = 1100; 4 | 5 | class DataGenerator { 6 | constructor() { 7 | this._enabled = false; 8 | this._data = []; 9 | } 10 | enable() { 11 | this._enabled = true; 12 | } 13 | disable() { 14 | this._enabled = false; 15 | } 16 | next() { 17 | if (!this._enabled) { 18 | return false; 19 | } 20 | if (this._data.length === 0) { 21 | this._data = []; 22 | for (let i = 0; i < batch_size; i++) { 23 | this._data.push({ a: i }); 24 | } 25 | return null; 26 | } else { 27 | return this._data.pop(); 28 | } 29 | } 30 | } 31 | 32 | class MySpout { 33 | 34 | constructor() { 35 | this._name = null; 36 | this._prefix = ""; 37 | this._generator = new DataGenerator(); 38 | //this._waiting_for_ack = false; 39 | } 40 | 41 | init(name, config, context, callback) { 42 | this._context = context; 43 | this._name = name; 44 | this._prefix = `[InprocSpout ${this._name}]`; 45 | console.log(this._prefix, "Inside init:", config); 46 | callback(); 47 | } 48 | 49 | heartbeat() { } 50 | 51 | shutdown(callback) { 52 | callback(); 53 | } 54 | 55 | run() { 56 | this._generator.enable(); 57 | } 58 | 59 | pause() { 60 | this._generator.disable(); 61 | } 62 | 63 | next(callback) { 64 | let data = this._generator.next(); 65 | callback(null, data, null/*, (err, xcallback) => { 66 | this._waiting_for_ack = false; 67 | if (xcallback) { 68 | xcallback(); 69 | } 70 | }*/); 71 | } 72 | } 73 | 74 | //////////////////////////////////////////////////////////////////////////////// 75 | 76 | exports.MySpout = MySpout; 77 | -------------------------------------------------------------------------------- /tests/helpers/test_inproc.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | 5 | class InprocHelper { 6 | 7 | constructor() { 8 | this._name = null; 9 | this._init = null; 10 | this._onEmit = null; 11 | this._receive_list = []; 12 | this._emit_list = []; 13 | 14 | this._init_called = 0; 15 | this._heartbeat_called = 0; 16 | this._shutdown_called = 0; 17 | } 18 | 19 | init(name, init, context, callback) { 20 | this._init_called++; 21 | this._name = name; 22 | this._init = init; 23 | this._onEmit = init.onEmit; 24 | this._context = context; 25 | callback(); 26 | } 27 | 28 | heartbeat() { 29 | this._heartbeat_called++; 30 | } 31 | 32 | shutdown() { 33 | this._shutdown_called++; 34 | } 35 | 36 | receive(data, stream_id, callback) { 37 | let self = this; 38 | self._receive_list.push({ data, stream_id }); 39 | async.eachSeries( 40 | self._emit_list, 41 | (item, xcallback) => { 42 | self._onEmit(item.data, item.stream_id, xcallback); 43 | }, 44 | callback 45 | ); 46 | } 47 | ///////////////////////////////////// 48 | 49 | _setupEmit(data, stream_id) { 50 | this._emit_list.push({ data, stream_id }); 51 | } 52 | } 53 | 54 | 55 | class InprocHelper2 extends InprocHelper { 56 | constructor() { 57 | super(); 58 | } 59 | 60 | receive(data, stream_id, callback) { 61 | super.receive(data, stream_id, (err) => { 62 | // confirm after 1.5 sec 63 | setTimeout(() => { 64 | callback(err); 65 | }, 1500); 66 | }); 67 | } 68 | } 69 | 70 | exports.create = function (subtype) { 71 | if (subtype == "derived") { 72 | return new InprocHelper2(); 73 | } 74 | return new InprocHelper(); 75 | }; 76 | -------------------------------------------------------------------------------- /demo/cli/demo-repl.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let qtopology = require("../.."); 4 | 5 | let dummy_topology_config = { 6 | general: { heartbeat: 1000 }, 7 | spouts: [], 8 | bolts: [], 9 | variables: {} 10 | }; 11 | 12 | let storage = new qtopology.MemoryStorage(); 13 | 14 | storage.registerWorker("worker1", () => { }); 15 | storage.registerWorker("worker2", () => { }); 16 | storage.registerWorker("worker3", () => { }); 17 | storage.registerWorker("worker4", () => { }); 18 | 19 | storage.setWorkerStatus("worker3", "dead", () => { }); 20 | storage.setWorkerStatus("worker4", "unloaded", () => { }); 21 | 22 | storage.registerTopology("topology.test.1", dummy_topology_config, () => { }); 23 | storage.registerTopology("topology.test.2", dummy_topology_config, () => { }); 24 | storage.registerTopology("topology.test.x", dummy_topology_config, () => { }); 25 | storage.registerTopology("topology.test.y", dummy_topology_config, () => { }); 26 | storage.registerTopology("topology.test.z", dummy_topology_config, () => { }); 27 | 28 | storage.enableTopology("topology.test.1", () => { }); 29 | storage.enableTopology("topology.test.2", () => { }); 30 | storage.disableTopology("topology.test.x", () => { }); 31 | storage.disableTopology("topology.test.y", () => { }); 32 | storage.enableTopology("topology.test.z", () => { }); 33 | 34 | storage.assignTopology("topology.test.1", "worker1", () => { }); 35 | storage.assignTopology("topology.test.2", "worker2", () => { }); 36 | storage.assignTopology("topology.test.z", "worker1", () => { }); 37 | 38 | storage.setTopologyStatus("topology.test.1", "waiting", "", () => { }); 39 | storage.setTopologyStatus("topology.test.2", "running", "", () => { }); 40 | storage.setTopologyStatus("topology.test.x", "unassigned", "", () => { }); 41 | storage.setTopologyStatus("topology.test.y", "error", "Stopped manually", () => { }); 42 | storage.setTopologyStatus("topology.test.z", "running", "", () => { }); 43 | 44 | // run CLI tool on it 45 | qtopology.runRepl(storage); 46 | -------------------------------------------------------------------------------- /demo/qminer/spouts.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class DataGenerator { 4 | constructor() { 5 | this._enabled = false; 6 | this._data = []; 7 | } 8 | enable() { 9 | this._enabled = true; 10 | } 11 | disable() { 12 | this._enabled = false; 13 | } 14 | next() { 15 | if (!this._enabled) { 16 | return false; 17 | } 18 | if (this._data.length === 0) { 19 | this._data = []; 20 | for (let i = 0; i < 500; i++) { 21 | this._data.push({ 22 | a: Math.sin(i), 23 | b: Math.cos(i) 24 | }); 25 | } 26 | return null; 27 | } else { 28 | return this._data.pop(); 29 | } 30 | } 31 | } 32 | 33 | class DummySpout { 34 | 35 | constructor() { 36 | this._name = null; 37 | this._prefix = ""; 38 | this._generator = new DataGenerator(); 39 | } 40 | 41 | init(name, config, context, callback) { 42 | this._name = name; 43 | this._prefix = `[DummySpout ${this._name}]`; 44 | console.log(this._prefix, "Inside init:", config); 45 | callback(); 46 | } 47 | 48 | heartbeat() { 49 | console.log(this._prefix, "Inside heartbeat."); 50 | } 51 | 52 | shutdown(callback) { 53 | console.log(this._prefix, "Shutting down gracefully."); 54 | callback(); 55 | } 56 | 57 | run() { 58 | //console.log(this._prefix, "Inside run"); 59 | this._generator.enable(); 60 | } 61 | 62 | pause() { 63 | //console.log(this._prefix, "Inside pause"); 64 | this._generator.disable(); 65 | } 66 | 67 | next(callback) { 68 | //console.log(this._prefix, "Inside next"); 69 | let data = this._generator.next(); 70 | callback(null, data, null); 71 | } 72 | } 73 | 74 | //////////////////////////////////////////////////////////////////////////////// 75 | 76 | exports.DummySpout = DummySpout; 77 | -------------------------------------------------------------------------------- /src/std_nodes/counter_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as log from "../util/logger"; 3 | 4 | /** This bolt counts incoming data and outputs statistics 5 | * to console and emits it as message to listeners. 6 | */ 7 | export class CounterBolt implements intf.IBolt { 8 | 9 | private name: string; 10 | private prefix: string; 11 | private timeout: number; 12 | private last_output: number; 13 | private counter: number; 14 | private onEmit: intf.BoltEmitCallback; 15 | 16 | constructor() { 17 | this.name = null; 18 | this.onEmit = null; 19 | this.prefix = ""; 20 | this.counter = 0; 21 | this.last_output = Date.now(); 22 | } 23 | 24 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 25 | this.name = name; 26 | this.onEmit = config.onEmit; 27 | this.prefix = `[${this.name}]`; 28 | if (config.prefix) { 29 | this.prefix += ` ${config.prefix}`; 30 | } 31 | this.timeout = config.timeout; 32 | callback(); 33 | } 34 | 35 | public heartbeat() { 36 | const d = Date.now(); 37 | if (d >= this.last_output + this.timeout) { 38 | const sec = Math.round(d - this.last_output) / 1000; 39 | log.logger().log(`${this.prefix} processed ${this.counter} in ${sec} sec`); 40 | const msg = { 41 | counter: this.counter, 42 | period: sec, 43 | ts: new Date() 44 | }; 45 | this.counter = 0; 46 | this.last_output = d; 47 | this.onEmit(msg, null, () => { 48 | // no-op 49 | }); 50 | } 51 | } 52 | 53 | public shutdown(callback: intf.SimpleCallback) { 54 | callback(); 55 | } 56 | 57 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 58 | this.counter++; 59 | this.onEmit(data, stream_id, callback); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/std_nodes/test_spout.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | 3 | /** This spout emits pre-defined tuples. Mainly used for testing. */ 4 | export class TestSpout implements intf.ISpout { 5 | 6 | private stream_id: string; 7 | private tuples: any[]; 8 | private delay_start: number; 9 | private delay_between: number; 10 | private ts_next_emit: number; 11 | private should_run: boolean; 12 | 13 | constructor() { 14 | this.stream_id = null; 15 | this.tuples = null; 16 | this.should_run = false; 17 | this.delay_start = 0; 18 | this.delay_between = 0; 19 | this.ts_next_emit = 0; 20 | } 21 | 22 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 23 | this.stream_id = config.stream_id; 24 | this.tuples = config.tuples || []; 25 | this.delay_between = config.delay_between || 0; 26 | this.delay_start = config.delay_start || 0; 27 | if (this.delay_start > 0) { 28 | this.ts_next_emit = Date.now() + this.delay_start; 29 | } 30 | callback(); 31 | } 32 | 33 | public heartbeat() { 34 | // no-op 35 | } 36 | 37 | public shutdown(callback: intf.SimpleCallback) { 38 | callback(); 39 | } 40 | 41 | public run() { 42 | this.should_run = true; 43 | } 44 | 45 | public pause() { 46 | this.should_run = false; 47 | } 48 | 49 | public next(callback: intf.SpoutNextCallback) { 50 | if (!this.should_run) { 51 | return callback(null, null, null); 52 | } 53 | if (this.tuples.length === 0) { 54 | return callback(null, null, null); 55 | } 56 | if (this.ts_next_emit > Date.now()) { 57 | return callback(null, null, null); 58 | } 59 | const data = this.tuples[0]; 60 | this.tuples = this.tuples.slice(1); 61 | this.ts_next_emit = Date.now() + this.delay_between; 62 | callback(null, data, this.stream_id); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tests/util/freq_estimator.tests.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const fe = require("../../built/util/freq_estimator"); 7 | 8 | describe('EventFrequencyScore', function () { 9 | describe('simple tests', function () { 10 | it('no data', function () { 11 | let d = new Date(); 12 | let obj = new fe.EventFrequencyScore(1); 13 | assert.equal(obj.getEstimate(d), 0); 14 | }); 15 | it('single data point', function () { 16 | let d = new Date(); 17 | let obj = new fe.EventFrequencyScore(1); 18 | assert.equal(obj.add(d), 1); 19 | }); 20 | it('two simutaneous data points', function () { 21 | let d = new Date(); 22 | let obj = new fe.EventFrequencyScore(1); 23 | assert.equal(obj.add(d), 1); 24 | assert.equal(obj.add(d), 2); 25 | }); 26 | }); 27 | describe('no-so-simple tests', function () { 28 | it('constant influx - above', function () { 29 | let c = 10 * 60 * 1000; 30 | let obj = new fe.EventFrequencyScore(c); 31 | let dx = Date.now(); 32 | for (let i = 0; i < 100; i++) { 33 | dx += c; 34 | let d = new Date(dx); 35 | //console.log(d, obj.add(d)); 36 | obj.add(d); 37 | } 38 | assert.ok(obj.getEstimate(new Date(dx)) > 10); 39 | }); 40 | it('constant influx - below', function () { 41 | let c = 11 * 60 * 1000; 42 | let obj = new fe.EventFrequencyScore(c - 1 * 60 * 1000); 43 | let dx = Date.now(); 44 | for (let i = 0; i < 100; i++) { 45 | dx += c; 46 | let d = new Date(dx); 47 | //console.log(d, obj.add(d)); 48 | obj.add(d); 49 | } 50 | assert.ok(obj.getEstimate(new Date(dx)) < 10); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/std_nodes/dir_watcher_spout.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as fs from "fs"; 3 | import * as path from "path"; 4 | 5 | class FileChangeRec { 6 | public target_dir: string; 7 | public file_name: string; 8 | public change_type: string; 9 | public ts: Date; 10 | } 11 | 12 | /** This spout monitors directory for changes. */ 13 | export class DirWatcherSpout implements intf.ISpout { 14 | private dir_name: string; 15 | private queue: FileChangeRec[]; 16 | private should_run: boolean; 17 | private stream_id: string; 18 | 19 | constructor() { 20 | this.should_run = true; 21 | this.dir_name = null; 22 | this.queue = []; 23 | this.stream_id = null; 24 | } 25 | 26 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 27 | this.dir_name = path.resolve(config.dir_name); 28 | this.stream_id = config.stream_id; 29 | 30 | fs.watch(this.dir_name, { persistent: false }, (eventType, filename) => { 31 | if (filename) { 32 | const rec = new FileChangeRec(); 33 | rec.change_type = eventType; 34 | rec.file_name = "" + filename; 35 | rec.target_dir = this.dir_name; 36 | rec.ts = new Date(); 37 | this.queue.push(rec); 38 | } 39 | }); 40 | callback(); 41 | } 42 | 43 | public heartbeat() { 44 | // no-op 45 | } 46 | 47 | public shutdown(callback: intf.SimpleCallback) { 48 | callback(); 49 | } 50 | 51 | public run() { 52 | this.should_run = true; 53 | } 54 | 55 | public pause() { 56 | this.should_run = false; 57 | } 58 | 59 | public next(callback: intf.SpoutNextCallback) { 60 | if (!this.should_run || this.queue.length === 0) { 61 | return callback(null, null, null); 62 | } 63 | const data = this.queue[0]; 64 | this.queue = this.queue.slice(1); 65 | callback(null, data, this.stream_id); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/std_nodes/get_spout.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as rest from "../distributed/http_based/rest_client"; 3 | 4 | /** This spout sends GET request to the specified url in regular 5 | * time intervals and forwards the result. 6 | */ 7 | export class GetSpout implements intf.ISpout { 8 | 9 | private stream_id: string; 10 | private url: string; 11 | private repeat: number; 12 | private should_run: boolean; 13 | private next_tuple: any; 14 | private next_ts: number; 15 | private client: rest.IApiClient; 16 | 17 | constructor() { 18 | this.url = null; 19 | this.stream_id = null; 20 | this.repeat = null; 21 | 22 | this.should_run = false; 23 | this.next_tuple = null; 24 | this.next_ts = Date.now(); 25 | } 26 | 27 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 28 | this.url = config.url; 29 | this.repeat = config.repeat; 30 | this.stream_id = config.stream_id; 31 | this.client = rest.create({}); 32 | callback(); 33 | } 34 | 35 | public heartbeat() { 36 | if (!this.should_run) { 37 | return; 38 | } 39 | if (this.next_ts < Date.now()) { 40 | this.client.get(this.url) 41 | .then(res => { 42 | this.next_tuple = { body: res.data.toString() }; 43 | this.next_ts = Date.now() + this.repeat; 44 | }) 45 | .catch(() => { 46 | // do nothing 47 | }); 48 | } 49 | } 50 | 51 | public shutdown(callback: intf.SimpleCallback) { 52 | callback(); 53 | } 54 | 55 | public run() { 56 | this.should_run = true; 57 | } 58 | 59 | public pause() { 60 | this.should_run = false; 61 | } 62 | 63 | public next(callback: intf.SpoutNextCallback) { 64 | const data = this.next_tuple; 65 | this.next_tuple = null; 66 | callback(null, data, this.stream_id); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /demo/std_nodes/process-continuous/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "process-continuous", 11 | "init": { 12 | "cmd_line": "node emitter.js json", 13 | "stream_id": "stream1", 14 | "emit_parse_errors" : false, 15 | "emit_stderr_errors": false, 16 | "emit_error_on_exit" : false, 17 | "date_transform_fields": ["ts"] 18 | } 19 | }, 20 | { 21 | "name": "pump2", 22 | "type": "sys", 23 | "working_dir": "", 24 | "cmd": "process-continuous", 25 | "init": { 26 | "cmd_line": "node emitter.js csv", 27 | "file_format": "csv", 28 | "csv_has_header": true, 29 | "stream_id": "stream2" 30 | } 31 | }, 32 | { 33 | "name": "pump3", 34 | "type": "sys", 35 | "working_dir": "", 36 | "cmd": "process-continuous", 37 | "init": { 38 | "cmd_line": "node emitter.js raw", 39 | "file_format": "raw", 40 | "stream_id": "stream3" 41 | } 42 | } 43 | ], 44 | "bolts": [ 45 | { 46 | "name": "bolt1", 47 | "working_dir": ".", 48 | "type": "sys", 49 | "cmd": "console", 50 | "inputs": [ 51 | { 52 | "source": "pump1", 53 | "stream_id": "stream1" 54 | }, 55 | { 56 | "source": "pump2", 57 | "stream_id": "stream2" 58 | }, 59 | { 60 | "source": "pump3", 61 | "stream_id": "stream3" 62 | } 63 | ], 64 | "init": {} 65 | } 66 | ], 67 | "variables": {} 68 | } 69 | -------------------------------------------------------------------------------- /docs/release-procedures.md: -------------------------------------------------------------------------------- 1 | # Release procedures 2 | 3 | This document describes versioning methodology for `QTopology` project. 4 | 5 | ## Semantic versioning 6 | 7 | We use [semantic versioning](https://docs.npmjs.com/getting-started/semantic-versioning), thus supporting easier `npm` dependency tracking and auto-upgrading. 8 | 9 | ## Git organization 10 | 11 | Code is being accumulated in Github on `master` branch. Developers should fork the repository and develop in their forks. When change is ready to be included into `master`, a pull-request should be created and reviewed. 12 | 13 | Github also contains two additional branches, used for versioning and patching: 14 | 15 | - `release` branch - from here the official version are created. 16 | - `frozen` branch - "code-freeze" branch, used also for creating patches. 17 | 18 | ## Github steps for new release 19 | 20 | These steps must be taken for **major or minor version**. When code is not under heavy concurrent development, these steps can **also** be taken for **patch version**. 21 | 22 | 1. Commit and merge all relevant code into `master` branch on Github. 23 | 1. Increase version number, either manually or using `npm version` command. This must also be done on `master` branch. 24 | 1. Merge `master` branch into `frozen` branch 25 | 1. Merge `frozen` branch into `release` branch 26 | 1. Enter new release into Github - using the version number created in the first step. Tag name should be `vX.Y.Z` and release name should be `X.Y.Z`. 27 | 1. Publish new code to npm using `npm publish` command. 28 | 29 | ## Github steps for new patch 30 | 31 | These steps are **recommended for patching current version** when master code already heavily changed, possibly with breaking or non-tested changes. 32 | 33 | 1. Open `frozen` branch 34 | 1. Included the patched code to this branch 35 | 1. Increase version number, either manually or using `npm version` command. 36 | 1. Merge `frozen` branch into `master` branch 37 | 1. Merge `frozen` branch into `release` branch 38 | 1. Enter new release into Github - using the version number created in the third step. Tag name should be `vX.Y.Z` and release name should be `X.Y.Z`. 39 | 1. Publish new code to npm using `npm publish` command. 40 | -------------------------------------------------------------------------------- /src/util/topology_config_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 3000, 4 | "initialization": [ 5 | { 6 | "working_dir": "${CODE_DIR}", 7 | "cmd": "custom_init_and_shutdown.js", 8 | "init": { 9 | "main_config": "${CONFIG_DIR}/main_config_dev.json" 10 | } 11 | } 12 | ], 13 | "shutdown": [ 14 | { 15 | "working_dir": "${CODE_DIR}", 16 | "cmd": "custom_init_and_shutdown.js" 17 | } 18 | ] 19 | }, 20 | "spouts": [ 21 | { 22 | "name": "custom_spout", 23 | "type": "inproc", 24 | "working_dir": "${CODE_DIR}", 25 | "cmd": "custom_spout.js", 26 | "init": { 27 | "field1": "normal", 28 | "field2": "normal" 29 | } 30 | } 31 | ], 32 | "bolts": [ 33 | { 34 | "name": "bolt_enricher", 35 | "type": "inproc", 36 | "working_dir": "${CODE_DIR}", 37 | "cmd": "bolt_enricher.js", 38 | "inputs": [ 39 | { "source": "custom_spout" } 40 | ], 41 | "init": { 42 | "fieldx": true 43 | } 44 | }, 45 | { 46 | "name": "bolt_processor", 47 | "type": "inproc", 48 | "working_dir": "${CODE_DIR}", 49 | "cmd": "bolt_processor.js", 50 | "inputs": [ 51 | { "source": "bolt_enricher" } 52 | ], 53 | "init": { 54 | "field1": "${SOME_DIR}/some.file.json" 55 | } 56 | }, 57 | { 58 | "name": "bolt_console", 59 | "type": "sys", 60 | "working_dir": ".", 61 | "cmd": "console", 62 | "inputs": [ 63 | { "source": "bolt_processor" }, 64 | { "source": "bolt_processor", "stream_id": "errors" } 65 | ], 66 | "init": {} 67 | } 68 | ], 69 | "variables": { 70 | "CODE_DIR": "/path/to/custom/bolts", 71 | "CONFIG_DIR": "../../configs", 72 | "SOME_DIR": "../../configs/pipelines" 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # qtopology 2 | 3 | ![Build status](https://travis-ci.org/qminer/qtopology.svg?branch=master "Travis CI status") 4 | ![npm version](https://badge.fury.io/js/qtopology.svg "NPM version") 5 | 6 | NPM package: [https://www.npmjs.com/package/qtopology](https://www.npmjs.com/package/qtopology) 7 | 8 | Travis CI: [https://travis-ci.org/qminer/qtopology](https://travis-ci.org/qminer/qtopology) 9 | 10 | ### Installation 11 | 12 | `````````````bash 13 | npm install qtopology 14 | ````````````` 15 | 16 | ## Intro 17 | 18 | QTopology is a distributed stream processing layer, written in `node.js`. 19 | 20 | It uses the following terminology, originating in [Storm](http://storm.apache.org/) project: 21 | 22 | - **Topology** - Organization of nodes into a graph that determines paths where messages must travel. 23 | - **Bolt** - Node in topology that receives input data from other nodes and emits new data into the topology. 24 | - **Spout** - Node in topology that reads data from external sources and emits the data into the topology. 25 | - **Stream** - When data flows through the topology, it is optionaly tagged with stream ID. This can be used for routing and filtering. 26 | 27 | When running in distributed mode, `qtopology` also uses the following: 28 | 29 | - **Coordination storage** - must be resilient, receives worker registrations and sends them the initialization data. Also sends shutdown signals. Implementation is custom. `QTopology` provides `REST`-based service out-of-the-box, but the design is similar for other options like `MySQL` storage etc. 30 | - **Worker** - Runs on single server. Registers with coordination storage, receives initialization data and instantiates local topologies in separate subprocesses. 31 | - **Leader** - one of the active workers is announced leader and it performs leadership tasks such as assigning of topologies to workers, detection of dead or inactive workers. 32 | 33 | ## Quick start 34 | 35 | Read [this quick-start guide](quick_start.md) to quickly create your own topology. 36 | 37 | ## Further reading 38 | 39 | - [Topology definition](topology-definition.md) 40 | - [Standard nodes](std-nodes.md) 41 | - [Internal protocols](protocols.md) 42 | - [Exposed utilities](utilities.md) 43 | - [Administration](administration.md) 44 | - [Release procedures](release-procedures.md) 45 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "file_reader", 11 | "init": { 12 | "file_name": "data.ldjson", 13 | "stream_id": "stream1" 14 | } 15 | }, 16 | { 17 | "name": "pump2", 18 | "type": "sys", 19 | "working_dir": "", 20 | "cmd": "file_reader", 21 | "init": { 22 | "file_name": "data.csv", 23 | "file_format": "csv", 24 | "stream_id": "stream2" 25 | } 26 | }, 27 | { 28 | "name": "pump3", 29 | "type": "sys", 30 | "working_dir": "", 31 | "cmd": "file_reader", 32 | "init": { 33 | "file_name": "data.txt", 34 | "file_format": "raw", 35 | "stream_id": "stream3" 36 | } 37 | } 38 | ], 39 | "bolts": [ 40 | { 41 | "name": "bolt_date", 42 | "working_dir": ".", 43 | "type": "sys", 44 | "cmd": "date_transform", 45 | "inputs": [ 46 | { 47 | "source": "pump1", 48 | "stream_id": "stream1" 49 | } 50 | ], 51 | "init": { 52 | "date_transform_fields": ["ts"], 53 | "reuse_stream_id": true 54 | } 55 | }, 56 | { 57 | "name": "bolt1", 58 | "working_dir": ".", 59 | "type": "sys", 60 | "cmd": "console", 61 | "inputs": [ 62 | { 63 | "source": "bolt_date", 64 | "stream_id": "stream1" 65 | }, 66 | { 67 | "source": "pump2", 68 | "stream_id": "stream2" 69 | }, 70 | { 71 | "source": "pump3", 72 | "stream_id": "stream3" 73 | } 74 | ], 75 | "init": {} 76 | } 77 | ], 78 | "variables": {} 79 | } 80 | -------------------------------------------------------------------------------- /src/util/crontab_parser.ts: -------------------------------------------------------------------------------- 1 | 2 | export class CronTabParser { 3 | 4 | private parts: number[][]; 5 | 6 | constructor(s: string) { 7 | const simple_regex = /^\*|\d\d?(\-\d\d?)?$/; 8 | const dow_regex = /^(sun|mon|tue|wed|thu|fri|sat)(\-(sun|mon|tue|wed|thu|fri|sat))?$/; 9 | const parts = s.split(" "); 10 | if (parts.length != 6) { 11 | throw new Error(`Invalid CRON string: ${s} - ${JSON.stringify(parts)}`); 12 | } 13 | this.parts = []; 14 | for (let i = 0; i < parts.length; i++) { 15 | let p = parts[i].toLowerCase(); 16 | if (i == 5 && dow_regex.test(p)) { 17 | // support for day-of-week enumeration 18 | ["sun", "mon", "tue", "wed", "thu", "fri", "sat"].forEach((x, index) => { 19 | // do this twice, just in case someone 20 | // sent in same-day- range, e.g. wed-wed 21 | p = p 22 | .replace(x, "" + index) 23 | .replace(x, "" + index); 24 | }); 25 | } 26 | if (!simple_regex.test(p)) { 27 | throw new Error(`Invalid CRON string: ${p}`); 28 | } 29 | if (p == "*") { 30 | this.parts.push([]); 31 | } else if (p.indexOf("-") > 0) { 32 | const tmp = p.split("-"); 33 | this.parts.push([+tmp[0], +tmp[1]]); 34 | } else { 35 | this.parts.push([+p, +p]); 36 | } 37 | } 38 | } 39 | 40 | public isIncluded(target: Date): boolean { 41 | const res = 42 | this.miniTest(target.getSeconds(), this.parts[0]) && 43 | this.miniTest(target.getMinutes(), this.parts[1]) && 44 | this.miniTest(target.getHours(), this.parts[2]) && 45 | this.miniTest(target.getDate(), this.parts[3]) && 46 | this.miniTest(target.getMonth() + 1, this.parts[4]) && 47 | this.miniTest(target.getDay(), this.parts[5]); 48 | return res; 49 | } 50 | 51 | private miniTest(val: number, bounds: number[]): boolean { 52 | if (bounds.length > 0) { 53 | if (val < bounds[0] || val > bounds[1]) { 54 | return false; 55 | } 56 | } 57 | return true; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/std_nodes/file_reader/demo_file_reader_long.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const async = require("async"); 4 | const fs = require("fs"); 5 | const qtopology = require("../../../"); 6 | 7 | // demo configuration 8 | let config = require("./topology_long.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | let data_file_name = "./data.long.txt"; 13 | async.series( 14 | [ 15 | (xcallback) => { 16 | console.log("Writing file..."); 17 | createLongFile(data_file_name, xcallback); 18 | }, 19 | (xcallback) => { 20 | topology.init("topology.1", config, xcallback); 21 | }, 22 | (xcallback) => { 23 | console.log("Init done"); 24 | topology.run(() => { 25 | setTimeout(function () { 26 | xcallback(); 27 | }, 200000); 28 | }); 29 | }, 30 | (xcallback) => { 31 | console.log("Starting shutdown sequence..."); 32 | topology.shutdown(xcallback); 33 | } 34 | ], 35 | (err) => { 36 | if (err) { 37 | console.log("Error in shutdown", err); 38 | } 39 | console.log("Finished."); 40 | } 41 | ); 42 | 43 | 44 | function shutdown() { 45 | if (topology) { 46 | topology.shutdown((err) => { 47 | if (err) { 48 | console.log("Error", err); 49 | } 50 | process.exit(1); 51 | }); 52 | topology = null; 53 | } 54 | } 55 | 56 | //do something when app is closing 57 | process.on('exit', shutdown); 58 | 59 | //catches ctrl+c event 60 | process.on('SIGINT', shutdown); 61 | 62 | //catches uncaught exceptions 63 | process.on('uncaughtException', 64 | (e) => { 65 | console.log(e); 66 | process.exit(1); 67 | } 68 | //shutdown 69 | ); 70 | 71 | function createLongFile(name, callback) { 72 | if (fs.existsSync(name)){ 73 | return callback(); 74 | } 75 | let counter = 1; 76 | async.whilst( 77 | () => (counter < 100000), 78 | (xcallback) => { 79 | let obj = { val: counter++ }; 80 | fs.appendFile(name, JSON.stringify(obj) + "\n", xcallback); 81 | }, 82 | callback 83 | ); 84 | fs.appendFile(name) 85 | } 86 | -------------------------------------------------------------------------------- /demo/std_nodes/dir_watcher/demo_dir_watcher.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const async = require("async"); 5 | const qtopology = require("../../.."); 6 | 7 | // demo configuration 8 | let config = require("./topology.json"); 9 | qtopology.validate({ config: config, exitOnError: true }); 10 | let topology = new qtopology.TopologyLocal(); 11 | 12 | async.series( 13 | [ 14 | (xcallback) => { 15 | console.log("Starting init"); 16 | topology.init("topology.1", config, xcallback); 17 | }, 18 | (xcallback) => { 19 | console.log("Init done"); 20 | topology.run(xcallback); 21 | }, 22 | (xcallback) => { 23 | console.log("Waiting - 2 sec"); 24 | setTimeout(() => { xcallback(); }, 2000); 25 | }, 26 | (xcallback) => { 27 | fs.appendFileSync("./temp_file.tmp", "Some content\n", { encoding: "utf8" }); 28 | setTimeout(function () { 29 | xcallback(); 30 | }, 2000); 31 | }, 32 | (xcallback) => { 33 | fs.appendFileSync("./temp_file.tmp", "Another content\n", { encoding: "utf8" }); 34 | setTimeout(function () { 35 | xcallback(); 36 | }, 2000); 37 | }, 38 | (xcallback) => { 39 | fs.unlinkSync("./temp_file.tmp"); 40 | setTimeout(function () { 41 | xcallback(); 42 | }, 2000); 43 | }, 44 | (xcallback) => { 45 | console.log("Starting shutdown sequence..."); 46 | topology.shutdown(xcallback); 47 | topology = null; 48 | } 49 | ], 50 | (err) => { 51 | if (err) { 52 | console.log("Error in shutdown", err); 53 | } 54 | console.log("Finished."); 55 | } 56 | ); 57 | 58 | 59 | function shutdown() { 60 | if (topology) { 61 | topology.shutdown((err) => { 62 | if (err) { 63 | console.log("Error", err); 64 | } 65 | process.exit(1); 66 | }); 67 | topology = null; 68 | } 69 | } 70 | 71 | //do something when app is closing 72 | process.on('exit', shutdown); 73 | 74 | //catches ctrl+c event 75 | process.on('SIGINT', shutdown); 76 | 77 | //catches uncaught exceptions 78 | process.on('uncaughtException', shutdown); 79 | -------------------------------------------------------------------------------- /demo/local/spout_inproc.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | class DataGenerator { 4 | constructor() { 5 | this._enabled = false; 6 | this._data = []; 7 | } 8 | enable() { 9 | this._enabled = true; 10 | } 11 | disable() { 12 | this._enabled = false; 13 | } 14 | next() { 15 | if (!this._enabled) { 16 | return false; 17 | } 18 | if (this._data.length === 0) { 19 | this._data = []; 20 | for (let i = 0; i < 15; i++) { 21 | this._data.push({ a: i }); 22 | } 23 | return null; 24 | } else { 25 | return this._data.pop(); 26 | } 27 | } 28 | } 29 | 30 | class MySpout { 31 | 32 | constructor() { 33 | this._name = null; 34 | this._context = null; 35 | this._prefix = ""; 36 | this._generator = new DataGenerator(); 37 | this._waiting_for_ack = false; 38 | } 39 | 40 | init(name, config, context, callback) { 41 | this._name = name; 42 | this._context = context; 43 | this._prefix = `[InprocSpout ${this._name}]`; 44 | console.log(this._prefix, "Inside init:", config); 45 | callback(); 46 | } 47 | 48 | heartbeat() { 49 | console.log(this._prefix, "Inside heartbeat. context=", this._context); 50 | } 51 | 52 | shutdown(callback) { 53 | console.log(this._prefix, "Shutting down gracefully."); 54 | callback(); 55 | } 56 | 57 | run() { 58 | console.log(this._prefix, "Inside run"); 59 | this._generator.enable(); 60 | } 61 | 62 | pause() { 63 | console.log(this._prefix, "Inside pause"); 64 | this._generator.disable(); 65 | } 66 | 67 | next(callback) { 68 | console.log(this._prefix, "Inside next"); 69 | if (this._waiting_for_ack) { 70 | return callback(null, null, null); // no data 71 | } 72 | let data = this._generator.next(); 73 | if (data) { 74 | data.ts_tag = new Date(); 75 | } 76 | this._waiting_for_ack = (data !== null); 77 | callback(null, data, null, (err, xcallback) => { 78 | this._waiting_for_ack = false; 79 | if (xcallback) { 80 | xcallback(); 81 | } 82 | }); 83 | } 84 | } 85 | 86 | exports.create = function () { 87 | return new MySpout(); 88 | }; 89 | -------------------------------------------------------------------------------- /demo/std_nodes/process/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "type": "sys", 9 | "working_dir": "", 10 | "cmd": "process", 11 | "init": { 12 | "cmd_line": "less ./data.ldjson", 13 | "stream_id": "stream1", 14 | "run_interval": 4000, 15 | "date_transform_fields": ["ts"] 16 | } 17 | }, 18 | { 19 | "name": "pump2", 20 | "type": "sys", 21 | "working_dir": "", 22 | "cmd": "process", 23 | "init": { 24 | "cmd_line": "less ./data.csv", 25 | "file_format": "csv", 26 | "csv_has_header": true, 27 | "run_interval": 8000, 28 | "stream_id": "stream2" 29 | } 30 | }, 31 | { 32 | "name": "pump3", 33 | "type": "sys", 34 | "working_dir": "", 35 | "cmd": "process", 36 | "init": { 37 | "cmd_line": "less ./data.txt", 38 | "file_format": "raw", 39 | "run_interval": 3000, 40 | "stream_id": "stream3" 41 | } 42 | } 43 | ], 44 | "bolts": [ 45 | { 46 | "name": "bolt_date", 47 | "working_dir": ".", 48 | "type": "sys", 49 | "cmd": "date_transform", 50 | "inputs": [ 51 | { 52 | "source": "pump1", 53 | "stream_id": "stream1" 54 | } 55 | ], 56 | "init": { 57 | "date_transform_fields": ["ts"], 58 | "reuse_stream_id": true 59 | } 60 | }, 61 | { 62 | "name": "bolt1", 63 | "working_dir": ".", 64 | "type": "sys", 65 | "cmd": "console", 66 | "inputs": [ 67 | { 68 | "source": "bolt_date", 69 | "stream_id": "stream1" 70 | }, 71 | { 72 | "source": "pump2", 73 | "stream_id": "stream2" 74 | }, 75 | { 76 | "source": "pump3", 77 | "stream_id": "stream3" 78 | } 79 | ], 80 | "init": {} 81 | } 82 | ], 83 | "variables": {} 84 | } 85 | -------------------------------------------------------------------------------- /src/std_nodes/rss_spout.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as rest from "../distributed/http_based/rest_client"; 3 | import * as log from "../util/logger"; 4 | 5 | /** This spout periodically checks specified RSS feed and emits all items. */ 6 | export class RssSpout implements intf.ISpout { 7 | 8 | private name: string; 9 | private stream_id: string; 10 | private url: string; 11 | private logging_prefix: string; 12 | private repeat: number; 13 | private should_run: boolean; 14 | private tuples: any[]; 15 | private next_call_after: number; 16 | private client: rest.IApiClient; 17 | 18 | constructor() { 19 | this.tuples = []; 20 | this.should_run = false; 21 | } 22 | 23 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 24 | this.name = name; 25 | this.logging_prefix = `[RssSpout ${this.name}] `; 26 | this.stream_id = config.stream_id; 27 | this.url = config.url; 28 | this.repeat = config.repeat || 10 * 60 * 1000; 29 | this.next_call_after = Date.now() - 10; 30 | this.client = rest.create({}); 31 | callback(); 32 | } 33 | 34 | public heartbeat() { 35 | if (Date.now() >= this.next_call_after && this.should_run) { 36 | log.logger().debug(this.logging_prefix + "Starting RSS crawl: " + this.url); 37 | this.client.get(this.url) 38 | .then(res => { 39 | for (const item of res.data.rss.channel.item) { 40 | this.tuples.push(item); 41 | } 42 | this.next_call_after = Date.now() + this.repeat; 43 | }) 44 | .catch(() => { 45 | // do nothing 46 | }); 47 | } 48 | } 49 | 50 | public shutdown(callback: intf.SimpleCallback) { 51 | this.should_run = false; 52 | callback(); 53 | } 54 | 55 | public run() { 56 | this.should_run = true; 57 | } 58 | 59 | public pause() { 60 | this.should_run = false; 61 | } 62 | 63 | public next(callback: intf.SpoutNextCallback) { 64 | if (!this.should_run) { 65 | return callback(null, null, null); 66 | } 67 | if (this.tuples.length == 0) { 68 | return callback(null, null, null); 69 | } 70 | const data = this.tuples[0]; 71 | this.tuples = this.tuples.slice(1); 72 | callback(null, data, this.stream_id); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /tests/std_nodes/type_transform_bolt.tests.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const ttb = require("../../built/std_nodes/type_transform_bolt"); 7 | 8 | 9 | describe('TypeTransformBolt', function () { 10 | it('constructable', function () { 11 | let target = new ttb.TypeTransformBolt(); 12 | }); 13 | it('init', function (done) { 14 | let emited = []; 15 | let name = "some_name"; 16 | let config = { 17 | onEmit: (data, stream_id, callback) => { 18 | emited.push({ data, stream_id }); 19 | callback(); 20 | }, 21 | date_transform_fields: ["field1", "field2"], 22 | numeric_transform_fields: ["field3"], 23 | bool_transform_fields: ["field4"] 24 | }; 25 | let target = new ttb.TypeTransformBolt(); 26 | target.init(name, config, null, (err) => { 27 | assert.ok(!err); 28 | done(); 29 | }); 30 | }); 31 | it('receive', function (done) { 32 | let emited = []; 33 | let name = "some_name"; 34 | let xdata = { 35 | field1: "2010-03-23T12:23:34Z", 36 | field2: "2010-03-23T12:23:34Z", 37 | field3: "3214", 38 | field4: "true", 39 | field5: "false" 40 | }; 41 | let xdata_out = { 42 | field1: new Date("2010-03-23T12:23:34Z"), 43 | field2: new Date("2010-03-23T12:23:34Z"), 44 | field3: 3214, 45 | field4: true, 46 | field5: false 47 | }; 48 | let xstream_id = null; 49 | let config = { 50 | onEmit: (data, stream_id, callback) => { 51 | emited.push({ data, stream_id }); 52 | callback(); 53 | }, 54 | date_transform_fields: ["field1", "field2"], 55 | numeric_transform_fields: ["field3"], 56 | bool_transform_fields: ["field4", "field5"] 57 | }; 58 | let target = new ttb.TypeTransformBolt(); 59 | target.init(name, config, null, (err) => { 60 | assert.ok(!err); 61 | target.receive(xdata, xstream_id, (err) => { 62 | assert.ok(!err); 63 | assert.equal(emited.length, 1); 64 | assert.deepEqual(emited[0].data, xdata_out); 65 | assert.equal(emited[0].stream_id, xstream_id); 66 | done(); 67 | }); 68 | }); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /demo/run_demos.sh: -------------------------------------------------------------------------------- 1 | ################################################################ 2 | ## This script runs all (most?) of demo projects to check 3 | ## that they execute without error. 4 | ################################################################ 5 | 6 | set -e 7 | start=`date +%s` 8 | 9 | function title { 10 | echo "" 11 | echo "#######################################################" 12 | echo "## "$1 13 | echo "#######################################################" 14 | echo "" 15 | } 16 | 17 | #title "Running bomb demo" 18 | #cd std_nodes/bomb 19 | #node demo_bomb.js 20 | #cd ../.. 21 | 22 | title "Running rest telemetry" 23 | cd local 24 | node demo_local.js 25 | cd .. 26 | 27 | title "Running rest telemetry" 28 | cd std_nodes/telemetry 29 | node demo_telemetry.js 30 | cd ../.. 31 | 32 | title "Running rest demo" 33 | cd std_nodes/rest 34 | node demo_rest.js 35 | cd ../.. 36 | 37 | title "Running rss demo" 38 | cd std_nodes/rss 39 | node demo_rss.js 40 | cd ../.. 41 | 42 | title "Running process demo" 43 | cd std_nodes/process 44 | node demo_process.js 45 | cd ../.. 46 | 47 | title "Running process-continuous demo" 48 | cd std_nodes/process-continuous 49 | node demo_process_continuous.js 50 | cd ../.. 51 | 52 | title "Running process-bolt demo" 53 | cd std_nodes/process-bolt 54 | node demo_process_bolt.js 55 | cd ../.. 56 | 57 | title "Running file_reader demo" 58 | cd std_nodes/file_reader 59 | node demo_file_reader.js 60 | cd ../.. 61 | 62 | title "Running disabled demo" 63 | cd std_nodes/disabled 64 | node demo_disabling.js 65 | cd ../.. 66 | 67 | title "Running file_append demo" 68 | cd std_nodes/file_append 69 | node demo_file_append.js 70 | rm *.gz # cleanup 71 | cd ../.. 72 | 73 | title "Running file_append_ex demo" 74 | cd std_nodes/file_append_ex 75 | node demo_file_append_ex.js 76 | rm *.gz # cleanup 77 | cd ../.. 78 | 79 | title "Running dir_watcher demo" 80 | cd std_nodes/dir_watcher 81 | node demo_dir_watcher.js 82 | cd ../.. 83 | 84 | title "Running counter demo" 85 | cd std_nodes/counter 86 | node demo_counter.js 87 | cd ../.. 88 | 89 | title "Running custom_task_base demo" 90 | cd std_nodes/task_bolt_base 91 | node demo_task_bolt_base.js 92 | cd ../.. 93 | 94 | title "Running quick start demo" 95 | cd quick_start 96 | node top.js 97 | cd .. 98 | 99 | title "Running async demo" 100 | cd async 101 | node top.js 102 | cd .. 103 | 104 | title "Demos finished" 105 | end=`date +%s` 106 | 107 | runtime=$((end-start)) 108 | echo "Time needed: "$runtime" sec" 109 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_csv/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "working_dir": ".", 9 | "type": "sys", 10 | "cmd": "timer", 11 | "init": { 12 | "title": "some title", 13 | "extra_fields": { 14 | "field1": "a" 15 | } 16 | } 17 | }, 18 | { 19 | "name": "pump_test", 20 | "type": "sys", 21 | "working_dir": "", 22 | "cmd": "test", 23 | "init": { 24 | "delay_between": 2000, 25 | "tuples": [ 26 | { 27 | "server": "server1" 28 | }, 29 | { 30 | "server": "server1" 31 | }, 32 | { 33 | "server": "server1" 34 | }, 35 | { 36 | "server": "server2" 37 | }, 38 | { 39 | "server": "server1" 40 | }, 41 | { 42 | "server": "server1" 43 | }, 44 | { 45 | "server": "server2" 46 | }, 47 | { 48 | "server": "server1" 49 | } 50 | ] 51 | } 52 | } 53 | ], 54 | "bolts": [ 55 | { 56 | "name": "bolt1", 57 | "working_dir": ".", 58 | "type": "sys", 59 | "cmd": "file_append_csv", 60 | "inputs": [{ "source": "pump1" }], 61 | "init": { 62 | "file_name": "./log.txt", 63 | "delete_existing": true, 64 | "delimiter": ",", 65 | "fields": ["ts", "title", "field1"], 66 | "header": "ts,name,value" 67 | } 68 | }, 69 | { 70 | "name": "bolt2", 71 | "working_dir": ".", 72 | "type": "sys", 73 | "cmd": "file_append_csv", 74 | "inputs": [{ "source": "pump_test" }], 75 | "init": { 76 | "file_name": "./log2.txt", 77 | "delete_existing": true, 78 | "delimiter": ",", 79 | "fields": ["server"], 80 | "header": "name" 81 | } 82 | } 83 | ], 84 | "variables": {} 85 | } 86 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append_ex/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "working_dir": ".", 9 | "type": "sys", 10 | "cmd": "timer", 11 | "init": { 12 | "title": "some title", 13 | "extra_fields": { 14 | "field1": "a" 15 | } 16 | } 17 | }, 18 | { 19 | "name": "pump_test", 20 | "type": "sys", 21 | "working_dir": "", 22 | "cmd": "test", 23 | "init": { 24 | "delay_between": 500, 25 | "tuples": [ 26 | { 27 | "server": "server1", 28 | "ts": "2017-01-01T00:00:01" 29 | }, 30 | { 31 | "server": "server1", 32 | "ts": "2017-01-01T00:00:45" 33 | }, 34 | { 35 | "server": "server1", 36 | "ts": "2017-01-01T00:02:14" 37 | }, 38 | { 39 | "server": "server2", 40 | "ts": "2017-01-01T00:02:28" 41 | }, 42 | { 43 | "server": "server1", 44 | "ts": "2017-01-01T00:03:45" 45 | }, 46 | { 47 | "server": "server1", 48 | "ts": "2017-01-01T00:04:31" 49 | }, 50 | { 51 | "server": "server2", 52 | "ts": "2017-01-01T00:04:33" 53 | }, 54 | { 55 | "server": "server1", 56 | "ts": "2017-01-01T00:04:34" 57 | } 58 | ] 59 | } 60 | } 61 | ], 62 | "bolts": [ 63 | { 64 | "name": "bolt2", 65 | "working_dir": ".", 66 | "type": "sys", 67 | "cmd": "file_append_ex", 68 | "inputs": [ 69 | { 70 | "source": "pump_test" 71 | } 72 | ], 73 | "init": { 74 | "file_name_template": "./log.txt", 75 | "split_period": 60000, 76 | "split_by_field": "server", 77 | "timestamp_field": "ts" 78 | } 79 | } 80 | ], 81 | "variables": {} 82 | } 83 | -------------------------------------------------------------------------------- /tests/helpers/bad_bolt.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let badLocations = { 4 | heartbeat: "heartbeat", 5 | shutdown: "shutdown", 6 | receive: "receive", 7 | init: "init", 8 | } 9 | let badActions = { 10 | throw: "throw", 11 | callbackException: "callbackException" 12 | } 13 | 14 | class BadBolt { 15 | constructor(subtype) { 16 | this._init_called = 0; 17 | this._heartbeat_called = 0; 18 | this._shutdown_called = 0; 19 | this._receive_called = 0; 20 | this._timeout = 0; 21 | 22 | if (subtype == badActions.throw) { 23 | this.action = badActions.throw; 24 | this.doAction(); 25 | } 26 | } 27 | 28 | doAction(callback) { 29 | if (this.action == badActions.throw) { 30 | throw new Error(); 31 | } else if (this.action == badActions.callbackException){ 32 | setTimeout(()=> { 33 | return callback(new Error()); 34 | }, this._timeout); 35 | return; 36 | } 37 | setTimeout(callback, this._timeout); 38 | } 39 | 40 | init(name, config, context, callback) { 41 | this._init_called++; 42 | this.name = name; 43 | this.onEmit = config.onEmit || (() => { }); 44 | this.action = config.action; 45 | this.location = config.location; 46 | this._timeout = config.timeout || 0; 47 | 48 | if (this.location == badLocations.init) { 49 | this.doAction(callback); 50 | } else { 51 | setTimeout(callback, this._timeout); 52 | } 53 | } 54 | 55 | heartbeat() { 56 | this._heartbeat_called++; 57 | if (this.location == badLocations.heartbeat && this.action != badActions.callbackException) { 58 | this.doAction(); 59 | } 60 | } 61 | 62 | shutdown(callback) { 63 | this._shutdown_called++; 64 | if (this.location == badLocations.shutdown) { 65 | this.doAction(callback); 66 | } else { 67 | setTimeout(callback, this._timeout); 68 | } 69 | } 70 | 71 | receive(data, stream_id, callback) { 72 | this._receive_called++; 73 | let self = this; 74 | if (this.location == badLocations.receive) { 75 | this.doAction(callback); 76 | } else { 77 | setTimeout(()=>{ 78 | self.onEmit(data, stream_id, callback); 79 | }, this._timeout); 80 | } 81 | } 82 | } 83 | 84 | exports.badLocations = badLocations; 85 | exports.badActions = badActions; 86 | 87 | exports.create = function (subtype) { 88 | return new BadBolt(subtype); 89 | } -------------------------------------------------------------------------------- /demo/std_nodes/disabled/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000, 4 | "initialization": [ 5 | { 6 | "working_dir": ".", 7 | "cmd": "init_and_shutdown.js" 8 | },{ 9 | "working_dir": ".", 10 | "cmd": "init_and_shutdown2.js", 11 | "disabled": true 12 | } 13 | ], 14 | "shutdown": [ 15 | { 16 | "working_dir": ".", 17 | "cmd": "init_and_shutdown.js" 18 | }, 19 | { 20 | "working_dir": ".", 21 | "cmd": "init_and_shutdown.js", 22 | "disabled": true 23 | } 24 | ] 25 | }, 26 | "spouts": [ 27 | { 28 | "name": "pump1", 29 | "type": "sys", 30 | "working_dir": "", 31 | "cmd": "timer", 32 | "init": { 33 | "extra_fields": { 34 | "field1": "This data is OK to pass" 35 | } 36 | } 37 | }, 38 | { 39 | "name": "pump2", 40 | "type": "sys", 41 | "working_dir": "", 42 | "disabled": true, 43 | "cmd": "timer", 44 | "init": { 45 | "extra_fields": { 46 | "field1": "This data should not be passed anywhere" 47 | } 48 | } 49 | } 50 | ], 51 | "bolts": [ 52 | { 53 | "name": "bolt1", 54 | "working_dir": ".", 55 | "type": "sys", 56 | "cmd": "console", 57 | "inputs": [ 58 | { 59 | "source": "pump1" 60 | }, 61 | { 62 | "source": "pump2" 63 | } 64 | ], 65 | "init": {} 66 | }, 67 | { 68 | "name": "bolt2", 69 | "working_dir": ".", 70 | "type": "sys", 71 | "disabled": true, 72 | "cmd": "console", 73 | "inputs": [ 74 | { 75 | "source": "bolt1" 76 | } 77 | ], 78 | "init": {} 79 | }, 80 | { 81 | "name": "bolt3", 82 | "working_dir": ".", 83 | "type": "sys", 84 | "cmd": "console", 85 | "inputs": [ 86 | { 87 | "source": "bolt1", 88 | "disabled": true 89 | } 90 | ], 91 | "init": {} 92 | } 93 | ], 94 | "variables": {} 95 | } 96 | -------------------------------------------------------------------------------- /src/distributed/file_based/file_storage.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import * as path from "path"; 3 | import * as intf from "../../topology_interfaces"; 4 | import * as log from "../../util/logger"; 5 | import * as mem from "../memory/memory_storage"; 6 | 7 | ////////////////////////////////////////////////////////////////////// 8 | 9 | export class FileStorage extends mem.MemoryStorage { 10 | 11 | private dir_name: string; 12 | private file_patterns: string[]; 13 | private file_patterns_regex: RegExp[]; 14 | 15 | constructor(dir_name: string, file_pattern: string | string[]) { 16 | super(); 17 | this.dir_name = dir_name; 18 | this.dir_name = path.resolve(this.dir_name); 19 | this.file_patterns = (typeof file_pattern === "string" ? [file_pattern as string] : file_pattern as string[]); 20 | this.file_patterns_regex = this.file_patterns 21 | .map(x => this.createRegexpForPattern(x)); 22 | 23 | const items = fs.readdirSync(this.dir_name); 24 | log.logger().log("[FileStorage] Starting file-based coordination, from directory " + this.dir_name); 25 | for (const item of items) { 26 | let is_ok = false; 27 | for (const pattern of this.file_patterns_regex) { 28 | if (item.match(pattern)) { 29 | is_ok = true; 30 | continue; 31 | } 32 | } 33 | if (!is_ok) { 34 | continue; 35 | } 36 | 37 | const topology_uuid = item.slice(0, -path.extname(item).length); // file name without extension 38 | log.logger().log("[FileStorage] Found topology file " + item); 39 | const config = require(path.join(this.dir_name, item)); 40 | 41 | this.registerTopology(topology_uuid, config, err => { 42 | // no-op 43 | }); 44 | this.enableTopology(topology_uuid, err => { 45 | // no-op 46 | }); 47 | } 48 | } 49 | 50 | public getProperties(callback: intf.SimpleResultCallback) { 51 | const res = []; 52 | res.push({ key: "type", value: "FileStorage" }); 53 | res.push({ key: "directory", value: this.dir_name }); 54 | res.push({ key: "file_patterns", value: this.file_patterns }); 55 | res.push({ key: "file_patterns_regex", value: this.file_patterns_regex }); 56 | callback(null, res); 57 | } 58 | 59 | private createRegexpForPattern(str: string): RegExp { 60 | if (!str) { return /.*/g; } 61 | str = str 62 | .replace(/\./g, "\.") 63 | .replace(/\*/g, ".*"); 64 | return new RegExp("^" + str + "$", "gi"); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/util/pattern_matcher.ts: -------------------------------------------------------------------------------- 1 | class SingleFilter { 2 | public $like: RegExp[]; 3 | public values: any[]; 4 | public fields: string[]; 5 | } 6 | 7 | /** Simple class for pattern matching */ 8 | export class PaternMatcher { 9 | 10 | private pattern: any; 11 | private filters: SingleFilter[]; 12 | 13 | /** Constructor that receives pattern as object */ 14 | constructor(pattern: any) { 15 | this.pattern = JSON.parse(JSON.stringify(pattern)); 16 | this.filters = []; 17 | // prepare RegEx objects in advance 18 | for (const filter in this.pattern) { 19 | if (this.pattern.hasOwnProperty(filter)) { 20 | const curr = this.pattern[filter]; 21 | const rec = new SingleFilter(); 22 | rec.fields = filter.split("."); 23 | if (typeof (curr) == "object" && curr.$like) { 24 | rec.$like = []; 25 | if (Array.isArray(curr.$like)) { 26 | for (const like of curr.$like) { 27 | rec.$like.push(new RegExp(like)); 28 | } 29 | } else if (typeof (curr.$like) == "string") { 30 | rec.$like.push(new RegExp(curr.$like)); 31 | } 32 | } else { 33 | if (Array.isArray(curr)) { 34 | rec.values = curr; 35 | } else { 36 | rec.values = [curr]; 37 | } 38 | } 39 | this.filters.push(rec); 40 | } 41 | } 42 | } 43 | 44 | /** Simple procedure for checking if given item 45 | * matches the pattern. 46 | */ 47 | public isMatch(item: any) { 48 | for (const filter of this.filters) { 49 | if (!this.matchSingleFilter(item, filter)) { 50 | return false; 51 | } 52 | } 53 | return true; 54 | } 55 | 56 | private matchSingleFilter(item: any, filter: SingleFilter): boolean { 57 | let target_value = item; 58 | for (const field of filter.fields) { 59 | if (target_value[field] === undefined) { 60 | return false; 61 | } 62 | target_value = target_value[field]; 63 | } 64 | 65 | if (filter.values) { 66 | for (const val of filter.values) { 67 | if (target_value === val) { 68 | return true; 69 | } 70 | } 71 | } else { 72 | for (const like of filter.$like) { 73 | if (like.test(target_value)) { 74 | return true; 75 | } 76 | } 77 | } 78 | return false; 79 | } 80 | } 81 | 82 | -------------------------------------------------------------------------------- /demo/std_nodes/file_append/topology.json: -------------------------------------------------------------------------------- 1 | { 2 | "general": { 3 | "heartbeat": 1000 4 | }, 5 | "spouts": [ 6 | { 7 | "name": "pump1", 8 | "working_dir": ".", 9 | "type": "sys", 10 | "cmd": "timer", 11 | "init": { 12 | "title": "some title", 13 | "extra_fields": { 14 | "field1": "a" 15 | } 16 | } 17 | }, 18 | { 19 | "name": "pump_test", 20 | "type": "sys", 21 | "working_dir": "", 22 | "cmd": "test", 23 | "init": { 24 | "delay_between": 2000, 25 | "tuples": [ 26 | { 27 | "server": "server1" 28 | }, 29 | { 30 | "server": "server1" 31 | }, 32 | { 33 | "server": "server1" 34 | }, 35 | { 36 | "server": "server2" 37 | }, 38 | { 39 | "server": "server1" 40 | }, 41 | { 42 | "server": "server1" 43 | }, 44 | { 45 | "server": "server2" 46 | }, 47 | { 48 | "server": "server1" 49 | } 50 | ] 51 | } 52 | } 53 | ], 54 | "bolts": [ 55 | { 56 | "name": "bolt1", 57 | "working_dir": ".", 58 | "type": "sys", 59 | "cmd": "file_append", 60 | "inputs": [ 61 | { 62 | "source": "pump1" 63 | } 64 | ], 65 | "init": { 66 | "prepend_timestamp": true, 67 | "file_name_template": "./log.txt", 68 | "split_over_time": true, 69 | "split_period": 3000, 70 | "compress": true 71 | } 72 | }, 73 | { 74 | "name": "bolt2", 75 | "working_dir": ".", 76 | "type": "sys", 77 | "cmd": "file_append", 78 | "inputs": [ 79 | { 80 | "source": "pump_test" 81 | } 82 | ], 83 | "init": { 84 | "prepend_timestamp": true, 85 | "file_name_template": "./log2.txt", 86 | "split_over_time": true, 87 | "split_period": 3000, 88 | "split_by_field": "server", 89 | "compress": true 90 | } 91 | } 92 | ], 93 | "variables": {} 94 | } 95 | -------------------------------------------------------------------------------- /src/std_nodes/parsing_utils.ts: -------------------------------------------------------------------------------- 1 | /** Utility class with static methods for parsing */ 2 | export class Utils { 3 | 4 | /** Reads and parses JSON data, one object per line. */ 5 | public static readJsonFile(content: string, tuples: any[], pushError: boolean = true) { 6 | const lines = content.split("\n"); 7 | for (let line of lines) { 8 | line = line.trim(); 9 | if (line.length == 0) { 10 | continue; 11 | } 12 | try { 13 | const json = JSON.parse(line); 14 | tuples.push(json); 15 | } catch (e) { 16 | if (pushError) { 17 | tuples.push(e); 18 | } 19 | } 20 | } 21 | } 22 | 23 | /** Reads raw text data, one line at the time. */ 24 | public static readRawFile(content: string, tuples: any[]) { 25 | const lines = content.split("\n"); 26 | for (let line of lines) { 27 | line = line.trim().replace("\r", ""); 28 | if (line.length == 0) { 29 | continue; 30 | } 31 | tuples.push({ content: line }); 32 | } 33 | } 34 | } 35 | 36 | /** Utility class for parsing CSV. Reads settings int constructor */ 37 | export class CsvParser { 38 | 39 | private csv_separator: string; 40 | private csv_fields: string[]; 41 | private csv_has_header: boolean; 42 | 43 | private header_read: boolean; 44 | 45 | constructor(config: any) { 46 | this.csv_separator = config.csv_separator || ","; 47 | this.csv_fields = config.csv_fields; 48 | this.csv_has_header = (config.csv_fields == null); 49 | this.header_read = false; 50 | } 51 | 52 | /** Main method of this class - processes multi-line CSV input */ 53 | public process(content: string, tuples: any[]) { 54 | let lines = content.split("\n"); 55 | 56 | // if CSV input contains header, use it. 57 | // otherwise, the first line already contains data 58 | if (!this.header_read && this.csv_has_header) { 59 | // read first list and parse fields names 60 | const header = lines[0].replace("\r", ""); 61 | this.csv_fields = header.split(this.csv_separator); 62 | lines = lines.slice(1); 63 | this.header_read = true; 64 | } 65 | 66 | for (let line of lines) { 67 | line = line.trim().replace("\r", ""); 68 | if (line.length == 0) { 69 | continue; 70 | } 71 | const values = line.split(this.csv_separator); 72 | const result = {}; 73 | for (let i = 0; i < this.csv_fields.length; i++) { 74 | result[this.csv_fields[i]] = values[i]; 75 | } 76 | tuples.push(result); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /demo/gui/demo-express.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let qtopology = require("../.."); 4 | let express = require("express"); 5 | let bodyParser = require("body-parser"); 6 | 7 | let dummy_topology_config = { 8 | general: { heartbeat: 1000 }, 9 | spouts: [], 10 | bolts: [], 11 | variables: {} 12 | }; 13 | 14 | let storage = new qtopology.MemoryStorage(); 15 | 16 | storage.registerWorker("worker1", () => { }); 17 | storage.registerWorker("worker2", () => { }); 18 | storage.registerWorker("worker3", () => { }); 19 | storage.registerWorker("worker4", () => { }); 20 | 21 | storage.setWorkerStatus("worker3", "dead", () => { }); 22 | storage.setWorkerStatus("worker4", "unloaded", () => { }); 23 | 24 | storage.registerTopology("topology.test.1", dummy_topology_config, () => { }); 25 | storage.registerTopology("topology.test.2", dummy_topology_config, () => { }); 26 | storage.registerTopology("topology.test.x", dummy_topology_config, () => { }); 27 | storage.registerTopology("topology.test.y", dummy_topology_config, () => { }); 28 | storage.registerTopology("topology.test.z", dummy_topology_config, () => { }); 29 | 30 | storage.enableTopology("topology.test.1", () => { }); 31 | storage.enableTopology("topology.test.2", () => { }); 32 | storage.disableTopology("topology.test.x", () => { }); 33 | storage.disableTopology("topology.test.y", () => { }); 34 | storage.enableTopology("topology.test.z", () => { }); 35 | 36 | storage.assignTopology("topology.test.1", "worker1", () => { }); 37 | storage.assignTopology("topology.test.2", "worker2", () => { }); 38 | storage.assignTopology("topology.test.z", "worker1", () => { }); 39 | 40 | storage.setTopologyStatus("topology.test.1", "worker1", "waiting", "", () => { }); 41 | storage.setTopologyStatus("topology.test.2", "worker2", "running", "", () => { }); 42 | storage.setTopologyStatus("topology.test.x", "", "unassigned", "", () => { }); 43 | storage.setTopologyStatus("topology.test.y", "", "error", "Stopped manually", () => { }); 44 | storage.setTopologyStatus("topology.test.z", "worker1", "running", "", () => { }); 45 | 46 | //////////////////////////////////////////////////////// 47 | 48 | let app = express(); 49 | app.use(bodyParser.json()); 50 | 51 | app.get('/a', function (req, res) { 52 | res.send('Hello World!') 53 | }) 54 | 55 | let server = new qtopology.DashboardServer(); 56 | server.initComplex( 57 | { 58 | app: app, 59 | prefix: "qtopology", 60 | back_title: "Back to main page", 61 | back_url: "/abc", 62 | storage: storage, 63 | title: "Custom dashboard title" 64 | }, 65 | (err) => { 66 | if (err) { 67 | console.log(err); 68 | process.exit(1); 69 | } 70 | let port = 3000; 71 | app.listen(port, () => { 72 | console.log("Express running on port " + port); 73 | console.log(`Open http://localhost:${port}/dashboard`); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /src/topology_async_wrappers.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "./topology_interfaces"; 2 | 3 | /** Wrapper for async bolt that transforms it into normal, callback-based bolt */ 4 | export class BoltAsyncWrapper implements intf.IBolt { 5 | 6 | private inner: intf.IBoltAsync; 7 | 8 | constructor(obj: intf.IBoltAsync) { 9 | this.inner = obj; 10 | } 11 | 12 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback): void { 13 | const new_config: intf.IBoltAsyncConfig = Object.assign({}, config); 14 | new_config.onEmit = (data: any, stream_id: string): Promise => { 15 | return new Promise((resolve, reject) => { 16 | config.emit(data, stream_id, (err: Error) => { 17 | if (err) { 18 | reject(err); 19 | } else { 20 | resolve(); 21 | } 22 | }); 23 | }); 24 | }; 25 | this.inner.init(name, config, context) 26 | .then(() => { callback(); }) 27 | .catch(callback); 28 | } 29 | 30 | public heartbeat(): void { this.inner.heartbeat(); } 31 | 32 | public shutdown(callback: intf.SimpleCallback): void { 33 | this.inner.shutdown() 34 | .then(() => { callback(); }) 35 | .catch(callback); 36 | } 37 | 38 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback): void { 39 | this.inner.receive(data, stream_id) 40 | .then(() => { callback(); }) 41 | .catch(callback); 42 | } 43 | } 44 | 45 | /** Wrapper for async spout that transforms it into normal, callback-based spout */ 46 | export class SpoutAsyncWrapper implements intf.ISpout { 47 | 48 | private inner: intf.ISpoutAsync; 49 | 50 | constructor(obj: intf.ISpoutAsync) { 51 | this.inner = obj; 52 | } 53 | 54 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback): void { 55 | this.inner.init(name, config, context) 56 | .then(() => { callback(); }) 57 | .catch(callback); 58 | } 59 | 60 | public shutdown(callback: intf.SimpleCallback): void { 61 | this.inner.shutdown() 62 | .then(() => { callback(); }) 63 | .catch(callback); 64 | } 65 | 66 | public heartbeat(): void { this.inner.heartbeat(); } 67 | public run(): void { this.inner.run(); } 68 | public pause(): void { this.inner.pause(); } 69 | 70 | public next(callback: intf.SpoutNextCallback): void { 71 | this.inner.next() 72 | .then(res => { 73 | if (res) { 74 | callback(null, res.data, res.stream_id); 75 | } else { 76 | callback(null, null, null); 77 | } 78 | }) 79 | .catch(err => { callback(err, null, null); }); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/std_nodes/attacher_bolt.tests.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /*global describe, it, before, beforeEach, after, afterEach */ 4 | 5 | const assert = require("assert"); 6 | const ab = require("../../built/std_nodes/attacher_bolt"); 7 | 8 | describe('AttacherBolt', function () { 9 | it('constructable', function () { 10 | let target = new ab.AttacherBolt(); 11 | }); 12 | it('init', function (done) { 13 | let emited = []; 14 | let name = "some_name"; 15 | let config = { 16 | onEmit: (data, stream_id, callback) => { 17 | emited.push({ data, stream_id }); 18 | callback(); 19 | }, 20 | extra_fields: { a: true } 21 | }; 22 | let target = new ab.AttacherBolt(); 23 | target.init(name, config, null, (err) => { 24 | assert.ok(!err); 25 | done(); 26 | }); 27 | }); 28 | it('receive', function (done) { 29 | let emited = []; 30 | let name = "some_name"; 31 | let xdata = { test: true }; 32 | let xdata_out = { test: true, a: true }; 33 | let xstream_id = null; 34 | let config = { 35 | onEmit: (data, stream_id, callback) => { 36 | emited.push({ data, stream_id }); 37 | callback(); 38 | }, 39 | extra_fields: { a: true } 40 | }; 41 | let target = new ab.AttacherBolt(); 42 | target.init(name, config, null, (err) => { 43 | assert.ok(!err); 44 | target.receive(xdata, xstream_id, (err) => { 45 | assert.ok(!err); 46 | assert.equal(emited.length, 1); 47 | assert.deepEqual(emited[0].data, xdata_out); 48 | assert.equal(emited[0].stream_id, xstream_id); 49 | done(); 50 | }); 51 | }); 52 | }); 53 | it('receive + nested extra fields', function (done) { 54 | let emited = []; 55 | let name = "some_name"; 56 | let xdata = { test: true, tags: { a: "top" } }; 57 | let xdata_out = { test: true, tags: { a: "top", b: "ok" }, values: { val1: 12 } }; 58 | let xstream_id = null; 59 | let config = { 60 | onEmit: (data, stream_id, callback) => { 61 | emited.push({ data, stream_id }); 62 | callback(); 63 | }, 64 | extra_fields: { tags: { b: "ok" }, values: { val1: 12 } } 65 | }; 66 | let target = new ab.AttacherBolt(); 67 | target.init(name, config, null, (err) => { 68 | assert.ok(!err); 69 | target.receive(xdata, xstream_id, (err) => { 70 | assert.ok(!err); 71 | assert.equal(emited.length, 1); 72 | assert.deepEqual(emited[0].data, xdata_out); 73 | assert.equal(emited[0].stream_id, xstream_id); 74 | done(); 75 | }); 76 | }); 77 | }); 78 | }); 79 | -------------------------------------------------------------------------------- /docs/uml/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "uml", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "async": { 8 | "version": "2.6.4", 9 | "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", 10 | "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", 11 | "requires": { 12 | "lodash": "^4.17.14" 13 | } 14 | }, 15 | "commander": { 16 | "version": "2.19.0", 17 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", 18 | "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==" 19 | }, 20 | "lodash": { 21 | "version": "4.17.21", 22 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 23 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" 24 | }, 25 | "node-nailgun-client": { 26 | "version": "0.1.2", 27 | "resolved": "https://registry.npmjs.org/node-nailgun-client/-/node-nailgun-client-0.1.2.tgz", 28 | "integrity": "sha512-OC611lR0fsDUSptwnhBf8d3sj4DZ5fiRKfS2QaGPe0kR3Dt9YoZr1MY7utK0scFPTbXuQdSBBbeoKYVbME1q5g==", 29 | "requires": { 30 | "commander": "^2.8.1" 31 | } 32 | }, 33 | "node-nailgun-server": { 34 | "version": "0.1.4", 35 | "resolved": "https://registry.npmjs.org/node-nailgun-server/-/node-nailgun-server-0.1.4.tgz", 36 | "integrity": "sha512-e0Hbh6XPb/7GqATJ45BePaUEO5AwR7InRW/pGeMKHH1cqPMBFCeqdBNfvi+bkVLnsbYOOQE+pAek9nmNoD8sYw==", 37 | "requires": { 38 | "commander": "^2.8.1" 39 | } 40 | }, 41 | "node-plantuml": { 42 | "version": "0.8.1", 43 | "resolved": "https://registry.npmjs.org/node-plantuml/-/node-plantuml-0.8.1.tgz", 44 | "integrity": "sha512-sEI9j61MLunxkV0QTUyycanLhk9qP23iEkv4IBT+bcndR8qJ1hm9TCD21I0cK5XyBCXNQ3gywXKujHWDojwcQg==", 45 | "requires": { 46 | "commander": "^2.8.1", 47 | "node-nailgun-client": "^0.1.0", 48 | "node-nailgun-server": "^0.1.4", 49 | "plantuml-encoder": "^1.2.5" 50 | } 51 | }, 52 | "pako": { 53 | "version": "1.0.3", 54 | "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.3.tgz", 55 | "integrity": "sha1-X1FbDGci4ZgpIK6ABerLC3ynPM8=" 56 | }, 57 | "plantuml-encoder": { 58 | "version": "1.2.5", 59 | "resolved": "https://registry.npmjs.org/plantuml-encoder/-/plantuml-encoder-1.2.5.tgz", 60 | "integrity": "sha512-viV7Sz+BJNX/sC3iyebh2VfLyAZKuu3+JuBs2ISms8+zoTGwPqwk3/WEDw/zROmGAJ/xD4sNd8zsBw/YmTo7ng==", 61 | "requires": { 62 | "pako": "1.0.3", 63 | "utf8-bytes": "0.0.1" 64 | } 65 | }, 66 | "utf8-bytes": { 67 | "version": "0.0.1", 68 | "resolved": "https://registry.npmjs.org/utf8-bytes/-/utf8-bytes-0.0.1.tgz", 69 | "integrity": "sha1-EWsCVEjJtQAIHN+/H01sbDfYg30=" 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/std_nodes/process_bolt.ts: -------------------------------------------------------------------------------- 1 | import * as intf from "../topology_interfaces"; 2 | import * as cp from "child_process"; 3 | import * as async from "async"; 4 | import { Utils } from "./parsing_utils"; 5 | import { logger } from "../index"; 6 | 7 | /** This bolt spawns specified process and communicates with it using stdin and stdout. 8 | * Tuples are serialized into JSON. 9 | */ 10 | export class ProcessBoltContinuous implements intf.IBolt { 11 | 12 | private stream_id: string; 13 | private cmd_line: string; 14 | private tuples: any[]; 15 | private onEmit: intf.BoltEmitCallback; 16 | private child_process: cp.ChildProcess; 17 | 18 | constructor() { 19 | this.stream_id = null; 20 | this.tuples = null; 21 | } 22 | 23 | public init(name: string, config: any, context: any, callback: intf.SimpleCallback) { 24 | this.stream_id = config.stream_id; 25 | this.onEmit = config.onEmit; 26 | this.cmd_line = config.cmd_line; 27 | this.tuples = []; 28 | 29 | let args = this.cmd_line.split(" "); 30 | const cmd = args[0]; 31 | args = args.slice(1); 32 | this.child_process = cp.spawn(cmd, args); 33 | this.child_process.stdout.on("data", data => { 34 | this.handleNewData(data.toString()); 35 | }); 36 | this.child_process.on("exit", () => { 37 | this.child_process = null; 38 | logger().log("child process closed"); 39 | }); 40 | callback(); 41 | } 42 | 43 | public heartbeat() { 44 | // no-op 45 | } 46 | 47 | public shutdown(callback: intf.SimpleCallback) { 48 | this.child_process.kill("SIGTERM"); 49 | callback(); 50 | } 51 | 52 | public receive(data: any, stream_id: string, callback: intf.SimpleCallback) { 53 | if (!this.child_process) { 54 | return callback(new Error("Child process died, cannot receive new data")); 55 | } 56 | this.child_process.stdin.write(JSON.stringify(data) + "\n"); 57 | callback(); 58 | } 59 | 60 | private handleNewData(content: string) { 61 | Utils.readJsonFile(content, this.tuples); 62 | const tmp_tuples = this.tuples; 63 | this.tuples = []; 64 | async.eachSeries( 65 | tmp_tuples, 66 | (tuple, callback) => { 67 | try { 68 | this.onEmit(tuple, this.stream_id, err => { 69 | if (err) { 70 | logger().error("Error in process-bolt emit (1)"); 71 | logger().exception(err); 72 | } 73 | callback(); 74 | }); 75 | } catch (err) { 76 | logger().error("Error in process-bolt emit (2)"); 77 | logger().exception(err); 78 | callback(); 79 | } 80 | }, 81 | () => { 82 | // no-op 83 | } 84 | ); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /demo/gui/demo.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let fs = require("fs"); 4 | let qtopology = require("../.."); 5 | 6 | let dummy_topology_config = { 7 | general: { heartbeat: 1000 }, 8 | spouts: [], 9 | bolts: [], 10 | variables: {} 11 | }; 12 | let dummy_topology_config2 = JSON.parse(fs.readFileSync("topology.1.json", { encoding: "utf8" })); 13 | let dummy_topology_config3 = JSON.parse(fs.readFileSync("topology.2.json", { encoding: "utf8" })); 14 | 15 | let storage = new qtopology.MemoryStorage(); 16 | 17 | storage.registerWorker("worker1", () => { }); 18 | storage.registerWorker("worker2", () => { }); 19 | storage.registerWorker("worker3", () => { }); 20 | storage.registerWorker("worker4", () => { }); 21 | storage.registerWorker("worker5", () => { }); 22 | 23 | storage.announceLeaderCandidacy("worker1", () => { 24 | storage.checkLeaderCandidacy("worker1", () => { }); 25 | }); 26 | storage.setWorkerStatus("worker3", "dead", () => { }); 27 | storage.setWorkerStatus("worker4", "unloaded", () => { }); 28 | storage.setWorkerStatus("worker4", "disabled", () => { }); 29 | 30 | storage.registerTopology("topology.test.1", dummy_topology_config2, () => { }); 31 | storage.registerTopology("topology.test.2", dummy_topology_config3, () => { }); 32 | storage.registerTopology("topology.test.x", dummy_topology_config, () => { }); 33 | storage.registerTopology("topology.test.y", dummy_topology_config, () => { }); 34 | storage.registerTopology("topology.test.z", dummy_topology_config, () => { }); 35 | 36 | storage.enableTopology("topology.test.1", () => { }); 37 | storage.enableTopology("topology.test.2", () => { }); 38 | storage.disableTopology("topology.test.x", () => { }); 39 | storage.disableTopology("topology.test.y", () => { }); 40 | storage.enableTopology("topology.test.z", () => { }); 41 | 42 | storage.assignTopology("topology.test.1", "worker1", () => { }); 43 | storage.assignTopology("topology.test.2", "worker2", () => { }); 44 | storage.assignTopology("topology.test.z", "worker1", () => { }); 45 | 46 | storage.setTopologyStatus("topology.test.1", "worker1", "waiting", "", () => { }); 47 | storage.setTopologyStatus("topology.test.2", "worker2", "running", "", () => { }); 48 | storage.setTopologyPid("topology.test.2", 3212, () => { }); 49 | storage.setTopologyStatus("topology.test.x", "", "unassigned", "", () => { }); 50 | storage.setTopologyStatus("topology.test.y", "", "error", "Stopped manually", () => { }); 51 | storage.setTopologyStatus("topology.test.z", "worker1", "running", "", () => { }); 52 | storage.setTopologyPid("topology.test.z", 16343, () => { }); 53 | 54 | let server = new qtopology.DashboardServer(); 55 | 56 | server.initComplex( 57 | { 58 | port: 3000, 59 | back_title: "Back to main page", 60 | back_url: "/abc", 61 | storage: storage, 62 | title: "Custom dashboard title", 63 | custom_props: [ 64 | { key: "Custom property 1", value: true }, 65 | { key: "Custom property 2", value: "Some value" }, 66 | { key: "Custom property 3", value: 542312 } 67 | ] 68 | }, 69 | function () { 70 | server.run(); 71 | }); 72 | -------------------------------------------------------------------------------- /src/util/strip_json_comments.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Utility function that strips comments from JSON. 3 | Though they are not part of JSON standard, they are very handy 4 | in config files. 5 | 6 | Code taken and adopted from: 7 | 8 | https://github.com/sindresorhus/strip-json-comments 9 | */ 10 | 11 | import * as fs from "fs"; 12 | 13 | const singleComment = 1; 14 | const multiComment = 2; 15 | const stripWithoutWhitespace = () => ""; 16 | const stripWithWhitespace = (str, start?, end?) => str.slice(start, end).replace(/\S/g, " "); 17 | 18 | /** 19 | * Reads given file and transforms it into object. 20 | * It allows non-standard JSON comments. 21 | */ 22 | export function readJsonFileSync(fname: string): any { 23 | const s = fs.readFileSync(fname, "utf8"); 24 | return JSON.parse(stripJsonComments(s)); 25 | } 26 | 27 | /** 28 | * Utility function that removes comments from given JSON. Non-standard feature. 29 | * @param str - string containing JSON data with comments 30 | * @param opts - optional options object 31 | * @param opts.whitespace - should whitespaces also be removed 32 | */ 33 | export function stripJsonComments(str: string, opts?: any): string { 34 | opts = opts || {}; 35 | 36 | const strip = opts.whitespace === false ? stripWithoutWhitespace : stripWithWhitespace; 37 | 38 | let insideString = false; 39 | let insideComment = 0; 40 | let offset = 0; 41 | let ret = ""; 42 | 43 | for (let i = 0; i < str.length; i++) { 44 | const currentChar = str[i]; 45 | const nextChar = str[i + 1]; 46 | 47 | if (!insideComment && currentChar === "\"") { 48 | const escaped = str[i - 1] === "\\" && str[i - 2] !== "\\"; 49 | if (!escaped) { 50 | insideString = !insideString; 51 | } 52 | } 53 | 54 | if (insideString) { 55 | continue; 56 | } 57 | 58 | if (!insideComment && currentChar + nextChar === "//") { 59 | ret += str.slice(offset, i); 60 | offset = i; 61 | insideComment = singleComment; 62 | i++; 63 | } else if (insideComment === singleComment && currentChar + nextChar === "\r\n") { 64 | i++; 65 | insideComment = 0; 66 | ret += strip(str, offset, i); 67 | offset = i; 68 | continue; 69 | } else if (insideComment === singleComment && currentChar === "\n") { 70 | insideComment = 0; 71 | ret += strip(str, offset, i); 72 | offset = i; 73 | } else if (!insideComment && currentChar + nextChar === "/*") { 74 | ret += str.slice(offset, i); 75 | offset = i; 76 | insideComment = multiComment; 77 | i++; 78 | continue; 79 | } else if (insideComment === multiComment && currentChar + nextChar === "*/") { 80 | i++; 81 | insideComment = 0; 82 | ret += strip(str, offset, i + 1); 83 | offset = i + 1; 84 | continue; 85 | } 86 | } 87 | 88 | return ret + (insideComment ? strip(str.substr(offset)) : str.substr(offset)); 89 | } 90 | -------------------------------------------------------------------------------- /tests/helpers/bad_spout.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | let badLocations = { 4 | heartbeat: "heartbeat", 5 | shutdown: "shutdown", 6 | run: "run", 7 | pause: "pause", 8 | next: "next", 9 | init: "init", 10 | } 11 | let badActions = { 12 | throw: "throw", 13 | callbackException: "callbackException" 14 | } 15 | 16 | class BadSpout { 17 | constructor(subtype) { 18 | this._init_called = 0; 19 | this._heartbeat_called = 0; 20 | this._shutdown_called = 0; 21 | this._run_called = 0; 22 | this._pause_called = 0; 23 | this._next_called = 0; 24 | this._timeout = 0; 25 | 26 | if (subtype == badActions.throw) { 27 | this.action = badActions.throw; 28 | this.doAction(); 29 | } 30 | } 31 | 32 | doAction(callback) { 33 | if (this.action == badActions.throw) { 34 | throw new Error(); 35 | } else if (this.action == badActions.callbackException) { 36 | setTimeout(() => { 37 | return callback(new Error()); 38 | }, this._timeout); 39 | return; 40 | } 41 | setTimeout(callback, this._timeout); 42 | } 43 | 44 | init(name, config, context, callback) { 45 | this._init_called++; 46 | this.name = name; 47 | this.onEmit = config.onEmit || (() => { }); 48 | this.action = config.action; 49 | this.location = config.location; 50 | this._timeout = config.timeout || 0; 51 | 52 | if (this.location == badLocations.init) { 53 | this.doAction(callback); 54 | } else { 55 | setTimeout(callback, this._timeout); 56 | } 57 | } 58 | 59 | heartbeat() { 60 | this._heartbeat_called++; 61 | if (this.location == badLocations.heartbeat && this.action != badActions.callbackException) { 62 | this.doAction(); 63 | } 64 | } 65 | 66 | shutdown(callback) { 67 | this._shutdown_called++; 68 | if (this.location == badLocations.shutdown) { 69 | this.doAction(callback); 70 | } else { 71 | setTimeout(callback, this._timeout); 72 | } 73 | } 74 | 75 | run() { 76 | this._run_called++; 77 | if (this.location == badLocations.run && this.action == badActions.throw) { 78 | this.doAction(); 79 | } 80 | } 81 | 82 | pause() { 83 | this._pause_called++; 84 | if (this.location == badLocations.pause && this.action == badActions.throw) { 85 | this.doAction(); 86 | } 87 | } 88 | next(callback) { 89 | this._next_called++; 90 | if (this.location == badLocations.next) { 91 | this.doAction(callback); 92 | } else { 93 | setTimeout(() => { 94 | return callback(null, {}, null); 95 | }, this._timeout); 96 | } 97 | } 98 | } 99 | 100 | 101 | exports.badLocations = badLocations; 102 | exports.badActions = badActions; 103 | 104 | exports.create = function (subtype) { 105 | return new BadSpout(subtype); 106 | } --------------------------------------------------------------------------------