├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── appveyor.yml ├── bin └── main.js ├── config.json ├── docs ├── README.md └── sections │ ├── api-doc.md │ ├── cli-doc.md │ ├── getting-started.md │ └── tests-benchmarks.md ├── gulpfile.js ├── index.js ├── lib ├── api-utilis.js ├── beacon-configs.js ├── beacon-controller.js ├── beacon.js ├── cli-controller.js ├── command-line-parameters.js ├── crypto-service.js ├── job-manager.js ├── router.js └── tcp.js ├── package.json └── tests ├── benchmarks ├── run.js ├── tasks.js ├── tasks │ ├── pi │ │ ├── klyng.js │ │ └── mpi.cpp │ └── primes │ │ ├── klyng.js │ │ └── mpi.cpp └── utilis │ └── cputime.h ├── fixtures ├── beacon │ ├── fake-job.js │ ├── fake-tcp-server.js │ ├── fake_app │ │ ├── local-dep.js │ │ └── main.js │ ├── fake_functional_app │ │ └── main.js │ ├── router-ipc-client.js │ ├── router-local-process.js │ └── runner-fake-job.js ├── cli │ ├── fake-hosts.invalid-1.json │ ├── fake-hosts.invalid-2.json │ ├── fake-hosts.invalid-3.json │ ├── fake-hosts.invalid-4.json │ ├── fake-hosts.invalid-5.json │ └── fake-hosts.valid.json └── klyng-apis │ ├── sync-init.js │ ├── sync-rank-size.js │ ├── sync-recv-from.js │ ├── sync-recv-full.js │ ├── sync-recv-no-criteria.js │ ├── sync-recv-subject.js │ ├── sync-send-invalid-1.js │ ├── sync-send-invalid-2.js │ └── sync-send.js ├── integration ├── assets │ ├── bad.job.js │ ├── job.js │ ├── machines.with.local.json │ ├── machines.with.wrong.pass.json │ ├── machines.without.local.json │ └── tcp.only-beacon.js └── run.js └── specs ├── beacon ├── beacon-process.specs.js ├── beacon-remote.specs.js ├── controller.specs.js ├── job-manager.specs.js └── router.specs.js ├── cli ├── cli.specs.js └── controller.specs.js └── klyng-apis └── sync-primitives.specs.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | 35 | # klyng's unpacks directory 36 | .unpacks 37 | 38 | # fake_app.zip asset as it changes every test run 39 | tests/fixtures/beacon/fake_app.zip 40 | 41 | # compiled mpi programs 42 | mpi.out 43 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4.2" 4 | - "5" 5 | - "5.1" 6 | - "6.1" 7 | env: 8 | - CXX=g++-4.8 9 | addons: 10 | apt: 11 | sources: 12 | - ubuntu-toolchain-r-test 13 | packages: 14 | - g++-4.8 15 | before_script: 16 | - npm install gulp -g 17 | script: gulp 18 | notifications: 19 | email: false 20 | branches: 21 | only: 22 | - master 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Mostafa-Samir 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | klyng 3 |

4 | 5 |

6 | 7 | Build Status 8 | 9 | 10 | 11 | Build Status 12 | 13 | 14 | Node Version >= 4.2.3 15 |

16 | 17 |

18 | Write and execute distributed code on any* platform that can run node.js 19 |

20 | 21 | # Distributed Hello World! 22 | 23 | ```javascript 24 | var klyng = require('klyng'); 25 | 26 | function main() { 27 | var size = klyng.size(); 28 | var rank = klyng.rank(); 29 | 30 | console.log("Hello World! I'm Process %d-%d", rank, size); 31 | 32 | klyng.end(); 33 | } 34 | 35 | klyng.init(main); 36 | ``` 37 |

38 | Distributed Hello World 39 |

40 | 41 | # System Requirements 42 | 43 | You can use klyng anywhere; provided that you have node.js **v4.2.3 or later** installed. Also, because klyng is built on [node-fibers](https://github.com/laverdet/node-fibers), your need to be able to run or build fibers on your system. Fibers is naturally available via npm for Linux, OS X and Windows 7 (or later), for other operating systems you will probably need to compile fibers form its C/C++ source. 44 | 45 | # Security 46 | When it comes to running jobs on remote devices, klyng implements several measures to ensure a secure communication channel. 47 | 48 | 1. A key-exchange algorithm (Diffie-Hellman) is used to establish a shared secret key between the two communicating devices. This key is used later to encrypt the messages carrying sensitive data (with AES-256) and stamp them with an HMAC. 49 | 50 | 2. For now, each node (in klyng's jargon, a **beacon**) is protected with a password. Sending a password between two nodes for authorization is done through the secure channel established is step 1. 51 | 52 | 3. Once a node is authorized, it can pack the job's source in one file a sends it through the secure channel for the other node to run. Any control signals is also sent through this channel. Only the messages sent between the processes during the job is not secured. 53 | 54 | # Performance 55 | 56 | In order to evaluate klyng's performance and how well it scales with the number of processors, A benchmarking script was designed to compare klyng's performance to that of MPICH2 (with C/C++) on the same tasks. The benchmarks focused on two metrics: 57 | * **Runner Total Execution Time (RTET):** which is actual time taken by the job, form the moment the runner is executed with the job to the moment it exits. 58 | 59 | * **Max Process CPU Time (MPCT):** which is the maximum of the CPU time of all the participating processes in the job. 60 | 61 | As the MPCT metric uses the actual time spent by an individual process on the CPU, it measure how well is the framework scaling as if each process is running on its own CPU, which is not the real case. The real case, in which there are multiple processes and a limited number of processors so processes will probably share time on a single processor, is captured with RTET metric. 62 | 63 | Compared to MPICH2 with C/C++, the data shows that klyng and javascript scales with the number of processors just as well (and in some cases, even better). 64 | 65 | ![Pi-MPCT](https://drive.google.com/uc?export=view&id=0BwJ57iK3uPsVNXBiUVB1SEE1VHc) 66 | 67 | ![Primes-MPCT](https://drive.google.com/uc?export=view&id=0BwJ57iK3uPsVUGJTYWQ2Q2pZRzQ) 68 | 69 | ![Pi-RTET](https://drive.google.com/uc?export=view&id=0BwJ57iK3uPsVTnBMLWd1Tmgwd0k) 70 | 71 | ![Primes-RTET](https://drive.google.com/uc?export=view&id=0BwJ57iK3uPsVcFVVVHBzNDI2bWM) 72 | 73 | The data represents how both MPICH2 and klyng scale with the number of processes on two computationally-intensive tasks: 74 | 75 | * **Pi Approximation:** which approximates the value of π with the [arctan integral formula](https://en.wikipedia.org/wiki/Inverse_trigonometric_functions#Expression_as_definite_integrals) using the a [Reimann sum](http://mathworld.wolfram.com/RiemannSum.html) with Δx = 2x10⁻⁹. 76 | 77 | * **Counting Primes:** which counts the number of prime numbers between 1 and 10⁷ using the naive primality test of trial division. 78 | 79 | These data were collected on a machine with an Intel Core i5 2410M CPU @ 2.30GHz (2 physical cores, with hyper-threading disabled), running node v5.4.1 on Ubuntu14.04. Each task of the two ran 100 times and the RTET and MPCT were collected for each run and averaged in the end into the data depicted in the charts. This process was repeated for each framework (MPICH2 and klyng) on each process count (1, 2, and 4). 80 | 81 | To make these data reproducible, the benchmarking script along with the source code for the tasks in question are shipped with the framework. The benchmarking script is also customizable for different environment parameters and extensible for more tasks. 82 | 83 | For more information on running the benchmarks, please refer to the [documentations](./docs/sections/tests-benchmarks.md#benchmarks). 84 | 85 | # Relation to MPI Standards 86 | 87 | The project was originally motivated by MPI, so it's greatly influenced by the MPI standards, and the standards was used more than one time as a reference in the implementations of some aspects of the project. However, the project in its current state cannot be considered as an implementation of the MPI standards; it can be considered, for now, as a *weak implementation of the MPI standards*. 88 | 89 | # Dive In! 90 | 91 | 1. [Getting Started](./docs/sections/getting-started.md) 92 | 1. [Installation](./docs/sections/getting-started.md#installation) 93 | 2. [Program Structure](./docs/sections/getting-started.md#program-structure) 94 | 3. [Dividing the Job](./docs/sections/getting-started.md#dividing-the-job) 95 | 4. [Sending and Receiving Messages](./docs/sections/getting-started.md#sending-and-receiving-messages) 96 | 5. [Running Locally](./docs/sections/getting-started.md#running-locally) 97 | 6. [Configuring your Device for Remote Jobs](./docs/sections/getting-started.md#configuring-your-device-for-remote-jobs) 98 | 5. [Running on Remote Devices](./docs/sections/getting-started.md#running-on-remote-devices) 99 | 2. [API Documentation](./docs/sections/api-doc.md) 100 | 1. [Environment Methods](./docs/sections/api-doc.md#environment-methods) 101 | 2. [Communication Methods](./docs/sections/api-doc.md#communication-methods) 102 | 3. [CLI Documentation](./docs/sections/cli-doc.md) 103 | 4. [Tests and Benchmarks](./docs/sections/tests-benchmarks.md) 104 | 1. [Unit and Integration Tests](./docs/sections/tests-benchmarks.md#unit-and-integration-tests) 105 | 2. [Benchmarks](./docs/sections/tests-benchmarks.md#benchmarks) 106 | 107 | # License 108 | 109 | ### MIT 110 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | branches: 2 | only: 3 | - master 4 | 5 | init: 6 | - git config --global core.autocrlf true 7 | 8 | environment: 9 | MSVS_VERSION: 2013 10 | 11 | matrix: 12 | - nodejs_version: "4.2" 13 | - nodejs_version: "5.4" 14 | 15 | platform: 16 | - x86 17 | - x64 18 | 19 | install: 20 | - ps: Install-Product node $env:nodejs_version 21 | - npm install gulp -g 22 | - npm install --msvs_version=%MSVS_VERSION% 23 | 24 | test_script: 25 | - node --version 26 | - npm --version 27 | - cmd: npm test 28 | 29 | build: off 30 | -------------------------------------------------------------------------------- /bin/main.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var parameters = require('../lib/command-line-parameters.js'); 4 | var cli = require('../lib/cli-controller.js'); 5 | var path = require('path'); 6 | 7 | // parse the command-line arguments 8 | var args = parameters.parse(); 9 | 10 | if (Object.keys(args).length === 0) { 11 | cli.noOptionsAlert(); 12 | } 13 | else if (args.help) { 14 | cli.help(); 15 | } 16 | else if (!!args["beacon-up"]) { 17 | cli.beaconUp(); 18 | } 19 | else if(!!args["beacon-down"]) { 20 | cli.beaconDown(); 21 | } 22 | else if(!!args['num-processes'] && !!args['app']) { 23 | 24 | var job_descriptor = { 25 | size: args['num-processes'], 26 | app: path.resolve(process.cwd(), args['app']) 27 | }; 28 | 29 | if(!!args['machines']) { 30 | var hosts = cli.parseHosts(args['machines']); 31 | if(!hosts.error) { 32 | job_descriptor.hosts = hosts; 33 | } 34 | else { 35 | console.error(hosts.error); 36 | process.exit(); 37 | } 38 | } 39 | 40 | cli.run(job_descriptor); 41 | } 42 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "port": 2222, 3 | "password": "" 4 | } 5 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Klyng Docs 2 | 3 | 1. [Getting Started](./sections/getting-started.md) 4 | 1. [Installation](./sections/getting-started.md#installation) 5 | 2. [Program Structure](./sections/getting-started.md#program-structure) 6 | 3. [Dividing the Job](./sections/getting-started.md#dividing-the-job) 7 | 4. [Sending and Receiving Messages](./sections/getting-started.md#sending-and-receiving-messages) 8 | 5. [Running Locally](./sections/getting-started.md#running-locally) 9 | 6. [Configuring your Device for Remote Jobs](./sections/getting-started.md#configuring-your-device-for-remote-jobs) 10 | 5. [Running on Remote Devices](./sections/getting-started.md#running-on-remote-devices) 11 | 2. [API Documentation](./sections/api-doc.md) 12 | 1. [Environment Methods](./sections/api-doc.md#environment-methods) 13 | 2. [Communication Methods](./sections/api-doc.md#communication-methods) 14 | 3. [CLI Documentation](./sections/cli-doc.md) 15 | 4. [Tests and Benchmarks](./sections/tests-benchmarks.md) 16 | 1. [Unit and Integration Tests](./sections/tests-benchmarks.md#unit-and-integration-tests) 17 | 2. [Benchmarks](./sections/tests-benchmarks.md#benchmarks) 18 | -------------------------------------------------------------------------------- /docs/sections/api-doc.md: -------------------------------------------------------------------------------- 1 | # API Documentation 2 | 3 | * [Environment Methods](#environment-methods) 4 | * [klyng.init(app_main)](#klynginitapp-main) 5 | * [klyng.size()](#klyngsize) 6 | * [klyng.rank()](#klyngrank) 7 | * [klyng.end()](#klyngend) 8 | * [Communication Methods](#communicationmethods) 9 | * [klyng.send(message)](#klyngsendmessage) 10 | * [klyng.recv([criteria])](#klyngrecvcriteria) 11 | 12 | ## Environment Methods 13 | 14 | ### klyng.init(app_main) 15 | Starts and initializes the klyng program. 16 | 17 | **Parameter** `app_main` *{Function}*: The entry point function of the program. 18 | 19 | --- 20 | 21 | ### klyng.size() 22 | Gets the size of the current job, which is the number of processes the current job was started with (the value of the `-n` option in the running command) 23 | 24 | **Returns** *{Number}*: The job's size. 25 | 26 | --- 27 | 28 | ### klyng.rank() 29 | Gets the rank of the calling process, which is a unique identifier given to the process form 0 to n-1 where n is the job's size. 30 | 31 | **Returns** *{Number}*: The process' rank. 32 | 33 | --- 34 | 35 | ### klyng.end() 36 | Marks the end of the klyng program and allows the process to exit. 37 | 38 | ## Communication Methods 39 | 40 | ### klyng.send(message) 41 | Sends a message to another process in the job. Blocks the execution until the message is out of the process, so sending large messages would affect the performance. 42 | 43 | **Parameter** `message` *{object}*: An object representing the message to be sent. The effective fields that would exist are: 44 | * `to`: the rank of the process to which the message will be sent. **[required]** 45 | * `data`: the data to be sent in the message. This can be any serializable data. **[required]** 46 | * `subject`: An extra identifier for the message. This can be any data type that can be checked for equality in a shallow manner. *[optional]* 47 | 48 | **Throws** `Error`: if one or more of the two required fields in the `message` argument is missing. 49 | 50 | --- 51 | 52 | ### klyng.recv([criteria]) 53 | Receives a message form another process in the job. Blocks the execution until a message that matches the given criteria is received. 54 | 55 | **Parameter** `criteria` *{object}*: A representation of the message the process will be waiting for, it only has two effective fields: 56 | * `from`: the rank of the process to wait for the message from. 57 | * `subject`: the subject of the message to be waiting for. 58 | 59 | If any of the fields of `criteria` is missing, klyng will wait for a message with **any** value of that field. For example, if `from` is missing, klyng will wait for a message form any source, the same is for `subject`. If nothing is passed to the method, klyng will wait for a message form any source with any subject. 60 | 61 | **Returns** *{Object}*: the contents of the `data` field in the received message. 62 | 63 | *Note*: While the execution is blocked waiting for a message, if a message that doesn't match the criteria of the current wait is received, this message will be queued for later use. Later, when another call to `klyng.recv` is made, it'll first check that queue for the message it wants, if the message exist in the queue it will be immediately consumed, otherwise `klyng.recv` will block and wait. 64 | -------------------------------------------------------------------------------- /docs/sections/cli-doc.md: -------------------------------------------------------------------------------- 1 | # CLI Documentation 2 | 3 | ## Usage 4 | 5 | ``` 6 | klyng --help 7 | klyng --beacon-up 8 | klyng --beacon-down 9 | klyng -n 10 | klyng -n -m 11 | ``` 12 | 13 | ## Options 14 | 15 | | Alias| Full Name | Description | 16 | |------|----------------------------|-----------------------------------------------------| 17 | | -h | --help | shows the help guide and exits | 18 | | -u | --beacon-up | starts the beacon daemon in the background | 19 | | -d | --beacon-down | takes the beacon process down if it's running | 20 | | -n | --num-processes | the number of processes to start a job with | 21 | | -m | --machines | the path to the machines file to start a job across | 22 | -------------------------------------------------------------------------------- /docs/sections/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | In this section we'll get started on working with klyng through the simple problem of summing a large list of numbers. We'll attempt to write a klyng program to divide this summation over a number of processes and collect back the result to the user. 4 | 5 | We'll start off by installing klyng on our system. 6 | 7 | ### Installation 8 | The klyng framework is divided into two parts: the API that you're going to import in your program, and command-line interface (the CLI) that you're going to use to run your program. 9 | 10 | So first of all, your going to have[¹](#foot-note1) to install klyng globally on your system to easily access the CLI when you attempt to run your programs. Then for each klyng program you going to write you'll need[²](#foot-note2) to install klyng as a dependency to import the API into your program. 11 | 12 | Installing klyng is not a fancy complicated process, it's just an npm install: 13 | * `npm install -g klyng` for the initial global installation 14 | * `npm install klyng` for the dependency installation. 15 | 16 | However, on Unix systems there is a little detail that you need to consider. On such system, npm by default installs global packages in `/usr/local/lib/node_modules`. This is probably going to cause security and permission problems as klyng would need to write files in its installation folder (The packed job files form remote nodes). 17 | 18 | To avoid these possible issues in a secure way, it's recommended that you install klyng in another location than the npm's default. For example, you can create the following directory `~/.npm-modules/` and add the following line in your `~/.profile` file: 19 | 20 | ``` 21 | export PATH=~/.npm-modules/bin:$PATH 22 | ``` 23 | 24 | This line ensures that you'll end up with klyng's CLI in your PATH so that you can run it from anywhere. Now when you attempt to install klyng globally you do: 25 | 26 | ``` 27 | NPM_CONFIG_PREFIX=~/.npm-modules npm install -g klyng 28 | ``` 29 | 30 | ### Program Structure 31 | 32 | Klyng doesn't enforce any particular organizational structure for your code, it just requires two things: 33 | * That your code have an entry function to initialize your klyng progarm with. 34 | * That before your program exits, you explicitly say that the klyng program has ended. 35 | 36 | So you just have to pass an entry function to `klyng.init` and put a call to `klyng.end` at the end of your program logic, after that you're totally free to organize your code the way you want. 37 | 38 | So for example, here's how our code for the summation problem would start off: 39 | 40 | ```javascript 41 | var klyng = require('klyng'); 42 | 43 | // This is the entry function 44 | function main() { 45 | 46 | // <--- 47 | // Here goes any logic you want to put in the entry 48 | // ---> 49 | 50 | // This says that the program ended 51 | // without it, it won't exit 52 | klyng.end(); 53 | } 54 | 55 | // here we start the program from the entry function 56 | // without it, it won't start 57 | klyng.init(main); 58 | ``` 59 | Note that the call to `klyng.end` doesn't have to be at the end of the entry function, it's just have to be in the **logical** end of the program, not necessarily the **physical**. Say that you have a function in your code that at the end cleans any side-effect that might have happed by the process, the call to `klyng.end` could go there. 60 | 61 | ```javascript 62 | var klyng = require('klyng'); 63 | 64 | function cleanAndExit() { 65 | 66 | // Do some cleaning 67 | klyng.end(); // then exit 68 | } 69 | 70 | // This is the entry function 71 | function main() { 72 | 73 | // <--- 74 | // Here goes any logic you want to put in the entry 75 | // ---> 76 | 77 | cleanAndExit(); 78 | } 79 | 80 | // here we start the program from the entry function 81 | // without it, it won't start 82 | klyng.init(main); 83 | ``` 84 | The same thing goes for `klyng.init`, it has to be at the logical start of your program which doesn't necessarily align with the physical start. 85 | 86 | In order to generate the data on which the program will work, we're gonna have a function that generates an array of 1 million random number. It's your choice where you're gonna put this function. You may put in the same file as the entry function, you could possibly put in a separate module, or you can even put it on a remote HTTP server and call it via a GET request; whatever floats your boat. Here we'll treat it as if it's in the same file as the entry function. 87 | 88 | ```javascript 89 | function generateRandomData() { 90 | var min = -10, max = 10; 91 | var list = new Array(1000000); 92 | for(var i = 0; i < 1000000; ++i) { 93 | list[i] = Math.random() * (max - min) + min; 94 | } 95 | 96 | return list; 97 | } 98 | ``` 99 | 100 | ### Dividing the Job 101 | 102 | In order to be able to divide the job between the participating processes, we need to know two things: 103 | * The number of processes participating in the job (aka the job size). 104 | * How to tell processes apart. For this a unique identifier is needed for each process. 105 | 106 | The first information can be retrieved by calling `klyng.size` which returns the number of the processes the job was started with. We can get the second piece of info by calling `klyng.rank`, which returns the unique identifier of the calling process. If a job started with *n* processes, each process gets a unique identifier from *0,...,n-1*. 107 | 108 | After acquiring these information, we can easily tell the rank 0 process (aka the root process), which is usually responsible for distributing the job over the other processes, how the job can be divided. 109 | 110 | 111 | ```javascript 112 | var klyng = require('klyng'); 113 | 114 | // This is the entry function 115 | function main() { 116 | 117 | var size = klyng.size(); 118 | var rank = klyng.rank(); 119 | 120 | if(rank === 0) { 121 | // This the whole data 122 | var list = generateRandomData(); 123 | 124 | // This is how much of the data each process would at least get 125 | var portionSize = Math.floor(1000000 / size); 126 | 127 | for(var p = 1; p < size; ++p) { 128 | // This is the portion of the data to be sent to process p 129 | var portion = list.slice((p - 1) * portionSize, p * portionSize); 130 | 131 | // we still need to send the portion 132 | } 133 | 134 | // After the root has given each process its job, it's time to do its own 135 | // its portion the rest of the remaining data 136 | var rootPortion = list.slice((size - 1) * portionSize, 1000000); 137 | 138 | var localSum = rootPortion.reduce((prev, next) => prev + next); 139 | } 140 | else { 141 | // other processes need to wait to receive their portions from the root 142 | } 143 | 144 | klyng.end(); 145 | } 146 | 147 | klyng.init(main); 148 | ``` 149 | 150 | ### Sending and Receiving Messages 151 | 152 | After the job was partitioned over the participating processes, the root needs to send each other process its partition and each other process needs to receive it. 153 | 154 | For sending messages, the `klyng.send` method is used. The method takes a simple object that describes the message to be sent. This object must contain two fields: `to` which is the rank of the receiving process, and `data` which is the data to be sent (any serializable data can be sent). An optional `subject` field may be provided as an extra info to identify a message, it can be any data type that can be tested for equality in a shallow manner. 155 | 156 | For example, this call `klyng.send({to: 5, data: "Hello", subject:"Greetings"})` will send to the process ranked 5 a message carrying "Hello" with a subject "Greetings". 157 | 158 | In our summation problem, the following demonstrates how the root process would send each other process its data portion: 159 | 160 | ```javascript 161 | 162 | ... 163 | 164 | for(var p = 1; p < size; ++p) { 165 | // This is the portion of the data to be sent to process p 166 | var portion = list.slice((p - 1) * portionSize, p * portionSize); 167 | 168 | // sending the portion to process p 169 | klyng.send({to: p, data: portion}); 170 | } 171 | 172 | ... 173 | ``` 174 | To receive messages on the other end, the `klyng.recv` method is used. It takes an object that describes the criteria of the message it's waiting for. The criteria object can have a `from` field which indicates the rank of the source process it's waiting a message from. It can also have a `subject` field indicating the subject of the message it's waiting. 175 | 176 | Note that after calling `klyng.recv` the execution blocks until a message with the desired criteria is received; and when it returns, it returns the `data` field that was sent in the message. If other messages come to the process, they will be queued for later calls of `klyng.recv`. 177 | 178 | If any one of the criteria fields is missing, the process will wait for any message that has any value of that missing criteria. For example, `klyng.recv({from: 0})` will accept messages form process 0 with any subject. On the other hand, `klyng.recv({subject: "Greetings"})` will accept messages with subject "Greetings" from any source. If both criteria are missing (aka an empty object is passed) or nothing is passed at all, `klyng.recv` would wait for a message from any source with any subject. 179 | 180 | The following shows how each of the other processes would receive its data portion from root and compute its sum: 181 | 182 | ```javascript 183 | 184 | ... 185 | 186 | else { 187 | var portion = klyng.recv({from: 0}); 188 | var partialSum = portion.reduce((prev, next) => prev + next); 189 | } 190 | 191 | ... 192 | ``` 193 | The only thing that is remaining now is for each process to send its partial sum to the root process and for the root process to collect those. This is shown in the following full version of the code. 194 | 195 | ```javascript 196 | var klyng = require('klyng'); 197 | 198 | // This is the entry function 199 | function main() { 200 | 201 | var size = klyng.size(); 202 | var rank = klyng.rank(); 203 | 204 | if(rank === 0) { 205 | var list = generateRandomData(); 206 | var portionSize = Math.floor(1000000 / size); 207 | 208 | for(var p = 1; p < size; ++p) { 209 | var portion = list.slice((p - 1) * portionSize, p * portionSize); 210 | 211 | klyng.send({to: p, data: portion}); 212 | } 213 | 214 | var rootPortion = list.slice((size - 1) * portionSize, 1000000); 215 | 216 | var localSum = rootPortion.reduce((prev, next) => prev + next); 217 | 218 | // here the root will wait for other processes partial sums 219 | for(var p = 1; p < size; ++p) { 220 | // it doesn't matter from where the partial sum is coming 221 | // we'll collect them all anyway, so no need to pass criteria 222 | var partialSum = klyng.recv(); 223 | localSum += partialSum; 224 | } 225 | 226 | // report back the total sum to the user 227 | console.log("The Total Sum is %d", localSum); 228 | } 229 | else { 230 | var portion = klyng.recv({from: 0}); 231 | var partialSum = portion.reduce((prev, next) => prev + next); 232 | 233 | // report back the partial sum to the root process 234 | klyng.send({to:0 , data: partialSum}); 235 | } 236 | 237 | klyng.end(); 238 | } 239 | 240 | klyng.init(main); 241 | ``` 242 | 243 | ### Running Locally 244 | 245 | Now the program is ready to be run. To run it on your local machine, you just write in your terminal: 246 | 247 | ``` 248 | klyng -n 249 | ``` 250 | 251 | So assuming that the file containing the code above is named **distributed.summation.js** and it's in the terminal's current directory. And assuming that you want to run your code across 8 processes, you would write: 252 | 253 | ``` 254 | klyng -n 8 distributed.summation.js 255 | ``` 256 | And the program will run on your local machines across 8 processes divided over your available CPU cores. 257 | 258 | ### Configuring your Device for Remote Jobs 259 | 260 | In case you want to distribute your job across remote devices, you first need to make sure that your device is ready to listen for and accept such jobs. 261 | 262 | The first thing is to make sure that the klyng's **beacon** is up and running on the device. So what is that beacon? 263 | 264 | Simply and without going into much details, the beacon is a daemon process that runs in the background of your system and is responsible for running your klyng jobs. It needs to be up and running for any klyng job to run on the machine, even in the case of running locally. You may have noticed that we didn't make explicit action to start the beacon when we ran the job locally, that's because it was automatically taken care of by the klyng command we wrote to run the job. It checks first if the beacon is running and starts it if it's not. 265 | 266 | However, in the case of running a job on remote devices where the run command will be on a different machine, this automatic check cannot be carried out and you need to make sure that the beacon is running yourself. 267 | 268 | This can be easily done by running this command on the machine's terminal: 269 | 270 | ``` 271 | klyng --beacon-up 272 | ``` 273 | This will start the beacon in the background if it's not already started. The beacon by default starts to listen for remote requests on port 2222 and is not protected by any password. To change that you can edit the `config.json` file with the desired port and password. 274 | 275 | You'll find the `config.json` in the directory in which klyng is installed. On Unix systems, assuming that you have followed the installation procedure described in the beginning, you would find it in `~/.npm-modules/lib/node_modules/klyng`. On Windows, assuming you didn't change npm global prefix, you would find it in `%AppData%\Roaming\npm\node_modules\klyng`. 276 | 277 | Note that any changes in the `config.json` won't take effect until the beacon is started. If the beacon was already running, you'll need to take it down and then start it again. To stop the beacon, you just write: 278 | 279 | ``` 280 | klyng --beacon-down 281 | ``` 282 | 283 | ### Running on Remote Devices 284 | 285 | Once you have your remote devices ready, you'll need to prepare a ***machines file*** that lists the information about the devices you want to run your job across. 286 | 287 | A machines file is a simple JSON file where each key is the IP adderess (or the host name if your hosts file or DNS can resolve it) of a remote machine you want to distribute part of your job to. The value of that key is an object that can possibly take three fields: 288 | * `max_procs`: The maximum number of processes that you want to run on that machine. Klyng will attempt to respect that limit as far the environment allows[³](#foot-note3). If this field is missing, it defaults to infinity. 289 | 290 | * `port`: The port number this machine would be listening on. If it's missing, the default port number 2222 is assumed. 291 | 292 | * `passwd`: The password of that machine's beacon. If missing, it defaults to an empty string. 293 | 294 | If you wish to include the local machine in the job, you'll need to add another entry to the machines file keyed by `"local"` that takes an object value optionally containing the `max_procs` field. 295 | 296 | The following is an example of a machines file to run a job across the local machine and two other machines on LAN: 297 | 298 | ```javascript 299 | { 300 | "local": { 301 | "max_procs": 2 302 | }, 303 | 304 | "192.168.1.104": { 305 | "max_procs": 4, 306 | "passwd": "dummy", 307 | "port": 9865 308 | }, 309 | 310 | "192.168.1.109": { 311 | "max_procs": 4, 312 | } 313 | } 314 | ``` 315 | 316 | Now to run the program across the specified machines in your machines file, you just need to write in your terminal: 317 | 318 | ``` 319 | klyng -n -m 320 | ``` 321 | 322 | Assuming that the machines file is called **machines.json** and is located in the same directory where the program code and the terminal is, you would write: 323 | 324 | ``` 325 | klyng -n 8 distributed.summation.js -m machines.json 326 | ``` 327 | 328 | Klyng will handle every thing from here and logs or errors from the remote processes will appear at your local terminal. 329 | 330 | --- 331 | 1: You don't necessarily have to do that. You can run the CLI from the local dependency, but this may lead into problems when you have more than one program if you're not careful. So it's not recommended. 332 | 333 | 2: Again, you don't necessarily need to do that, you can import the API directly from your globally installed version, but that's also not recommended especially if you intend to package and distribute your program. 334 | 335 | 3: Klyng divides the job over the nodes in a round-robin fashion to ensure the most balanced division possible. If a node reached its specified maximum number of processes during the division, klyng will attempt to allocate the remaining processes to other nodes that didn't reach the maximum yet. If all nodes reached the maximum, klyng will have to cross the `max_procs` limit. 336 | -------------------------------------------------------------------------------- /docs/sections/tests-benchmarks.md: -------------------------------------------------------------------------------- 1 | # Tests and Benchmarks 2 | 3 | ## Unit and Integration Tests 4 | 5 | **Unit tests** can be found under the directory `tests/specs` and it's organized into three directories: one for the beacon components, the other for the CLI components, and the last is for the API components. Each component in these directory has its own `*.specs.js` file where tests for each functional unit in the component are listed. 6 | 7 | The tests are written in BDD style with `chai.expect` and run under Mocha testing framework. You can run the unit tests either with `glup` or `npm` scripts as follows: 8 | 9 | ``` 10 | gulp unit-tests 11 | npm run unit-tests 12 | ``` 13 | 14 | **Integration tests** on the other hand can be found under the `tests/integration` directory and the tests are listed in the `run.js` file there. The tests ensures that the local and remote components communicate correctly and recover gracefully form errors that might occur during the job so that they remain available for later jobs. 15 | 16 | Integration tests are also written in BDD style and can be run using either `gulp` or `npm`: 17 | 18 | ``` 19 | gulp integration-tests 20 | npm run integration-tests 21 | ``` 22 | 23 | To run both the unit and integration tests sequentially (this is what is used in the ci services), you can use `gulp test` or `npm test`. 24 | 25 | *Note 1*: The seemingly large amount of time that the tests take are due to intentional waits and do not reflect the framework's performance. 26 | 27 | *Note 2*: There could be some instability in the tests (specially in the integration tests) due to the fact delays in network communications may not align with the waits mentioned above. So please make sure that if the tests fail, they fail all the time in a consistent manner before reporting an issue. 28 | 29 | ## Benchmarks 30 | ***Important Note: For now, the benchmarks cannot be run on Windows*** 31 | 32 | The benchmarks can be found under the `tests/benchmarks` directory, it's organized in the following manner: 33 | 34 | * `tasks/` **directory**: which contains a list of directories, one for each task that can be run by the benchmarking script. Each task folder need to have 2 files: 35 | * `mpi.cpp`: This is the task written in C/C++ with MPI. If one task is missing this file, MPI runs will not be carried out throughout all the tasks. 36 | * `klyng.js`: This is the task written in javascript with klyng. This file is necessary! 37 | * `utilis/` **directory**: this contains C/C++ utility function to retrieve a CPU time. It's only supports Unix-based systems for now. 38 | 39 | * `tasks.js` **file**: this is like a registry index for the tasks in the `tasks/` directory where each task gets a name, an alias (which must be the same as its directory name), and description. This helps the bench locate the task sources and display descriptive information of the task while running. 40 | 41 | * `run.js` **file**: which is the benchmarking script itself. This script runs each of the given tasks on each framework across each specified number of processes for a specific number of times and collects the specified metrics. By default: the script runs all tasks in the `tasks.js` file on each framework across 1, 2, and 4 processes for 100 times and collects the ***Maximum Process CPU Time*** [¹](#foot-note1) (MPCT) and the ***Runner total execution time*** (RTET) metrics. However these parameters are configurable through command line arguments. 42 | 43 | ### Running the Benchmarks 44 | 45 | ``` 46 | npm run benchmarks -- [options] 47 | ``` 48 | 49 | ### Options 50 | 51 | | Option | Description | 52 | |-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| 53 | | --process-counts | Takes a list of numbers to be process counts to run the benchmarks across
**Default**: [1, 2, 4]
**Example**: `npm run benchmarks -- --process-counts 2 4 8` 54 | | --iterations | Takes a number to define how many times each task should be run
**Default**: 100
**Example**: `npm run benchmarks -- --iterations 1000` | 55 | | --tasks | Takes a list of strings that represent aliases of the tasks to run by the benchmark
**Default**: ['pi', 'primes']
**Example**: `npm run benchmarks -- --tasks pi` | 56 | | --no-mpi | If present, stops the execution of the MPI version of the tasks | 57 | | --metrics | Takes a list of strings representing which metrics to be reported by the bechmark
**Default**: ['mpct', 'rtet']
**Example**: `npm run benchmarks -- --metrics rtet` | 58 | 59 | --- 60 | 1 The benchmarks collects each process CPU time through the process's stdout. So each process, before exiting must report its total CPU time to stdout (check the tasks source code to have a better understanding) 61 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | var gulp = require('gulp'); 2 | var mocha = require('gulp-mocha'); 3 | var sequence = require('gulp-sequence'); 4 | 5 | gulp.task('unit-tests', () => { 6 | return gulp.src("tests/specs/**/*.js") 7 | .pipe(mocha()) 8 | .on('error', (error) => { 9 | console.error(error.message); 10 | console.error(error.stack); 11 | process.exit(1); 12 | }) 13 | .on('end', () => process.exit(0)) 14 | }); 15 | 16 | gulp.task('non-terminating-unit-tests', () => { 17 | return gulp.src("tests/specs/**/*.js") 18 | .pipe(mocha()) 19 | .on('error', (error) => { 20 | console.error(error.message); 21 | console.error(error.stack); 22 | process.exit(1); 23 | }) 24 | }); 25 | 26 | gulp.task('integration-tests', () => { 27 | return gulp.src("tests/integration/run.js") 28 | .pipe(mocha()) 29 | .on('error', (error) => { 30 | console.error(error.message); 31 | console.error(error.stack); 32 | process.exit(1); 33 | }) 34 | .on('end', () => process.exit(0)) 35 | }); 36 | gulp.task('test', sequence('non-terminating-unit-tests', 'integration-tests')); 37 | 38 | gulp.task('default', ['test']); 39 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var fiber = require('fibers'); 2 | var utilis = require('./lib/api-utilis'); 3 | 4 | // holds the size of the job the process is participating in 5 | var job_size = -1; 6 | 7 | // holds the rank of the process inside the job 8 | var process_rank = -1; 9 | 10 | // holds the criteria of latest unmatched recieve request 11 | var registered_sync_request = null; 12 | 13 | // holds unexpected messages as they arrive to the process 14 | var queue = []; 15 | 16 | // the holds the fiber in which the klyng code runs 17 | var process_fiber = undefined; 18 | 19 | process.on('message', function(message) { 20 | if(message.type === 'klyng:msg') { 21 | if(utilis.match(registered_sync_request, message.header)) { 22 | // reset and resume 23 | registered_sync_request = null; 24 | process_fiber.run(message.data); 25 | } 26 | else { 27 | queue.push(message); 28 | } 29 | } 30 | }); 31 | 32 | /* 33 | * initiates the klyng app parameters and runs the app in a fiber 34 | * @param app_main {Function}: the entry function of the app 35 | */ 36 | function init(app_main) { 37 | 38 | // first check if the process is running as a child process with an open ipc 39 | if(!process.send) { 40 | console.log("klyng apps cannot be run directly with node"); 41 | console.log("You must run it using klyng"); 42 | process.exit(1); 43 | } 44 | else { 45 | // read the app parameters from the process arguments 46 | job_size = parseInt(process.argv[2]); 47 | process_rank = parseInt(process.argv[3]); 48 | 49 | process_fiber = fiber(app_main); 50 | process_fiber.run(); 51 | } 52 | } 53 | 54 | /* 55 | * returns the size of the job the process is participating in 56 | * @retrun {Number} 57 | */ 58 | function size() { 59 | return job_size; 60 | } 61 | 62 | /* 63 | * returns the rank of the process in the job 64 | * @return {Number} 65 | */ 66 | function rank() { 67 | return process_rank; 68 | } 69 | 70 | /* 71 | * dispatches a message to the beacon process to be routed to the intended process 72 | * @param message {Object}: the message object to be sent 73 | */ 74 | function send(message) { 75 | /* 76 | * the message object contains the following keys 77 | * to {Number}: the rank of the process to recive that message (required) 78 | * data {Any JSON serializable data}: the data to be sent (required) 79 | * subject {String}: a mark for the message (optional) 80 | */ 81 | if((!message.to && message.to !== 0) || (!message.data && message.data !== 0)) { 82 | throw new Error("'to' or 'data' fields are missing from the message"); 83 | } 84 | 85 | var enveloped_message = { 86 | type: 'klyng:msg', 87 | header: { 88 | from: process_rank, 89 | to: message.to 90 | }, 91 | data : message.data 92 | }; 93 | 94 | if(!!message.subject) 95 | enveloped_message.header.subject = message.subject; 96 | 97 | process.send(enveloped_message); 98 | } 99 | 100 | /* 101 | * consumes a matching message from the queue or blocks and waits for the message to arrive 102 | * @param criteria {Object}: contains the criteria of the message to be recieved (optional) 103 | * @return {Any JSON serializable data}: the message's data 104 | */ 105 | function recv(criteria) { 106 | /* 107 | * the criteria object may containe: 108 | * @key from {Number}: the source of the message to be received (any source if ommitted or -1) 109 | * @key subject {String}: the subject of the message to be received (any subject if omitted or "-1") 110 | * if the criteria object wasn't provided, a default criteria of any source, any subject will be used 111 | */ 112 | var default_criteria = { 113 | from : -1, // from any source 114 | subject: "-1", // with any subject 115 | }; 116 | 117 | var used_criteria = utilis.blend(default_criteria, criteria); 118 | 119 | // first check the queue for a message matching the criteria 120 | // TODO: come up with a more effcient and scalable implementation for the queue checking 121 | var queue_size = queue.length; 122 | for(var i = 0; i < queue_size; i++) { 123 | var message = queue[i]; 124 | if(utilis.match(used_criteria, message.header)) { 125 | queue.splice(i, 1); // removes the message from the queue 126 | return message.data; 127 | } 128 | } 129 | 130 | // if no matching messsage was found in the queue 131 | // register the criteria for upcoming messages 132 | registered_sync_request = used_criteria; 133 | 134 | // block the execution 135 | return fiber.yield(); 136 | } 137 | 138 | /* 139 | * marks the end of the klyng app 140 | */ 141 | function end() { 142 | process.exit(0); 143 | } 144 | 145 | module.exports = { 146 | init: init, 147 | size: size, 148 | rank: rank, 149 | send: send, 150 | recv: recv, 151 | end: end 152 | }; 153 | -------------------------------------------------------------------------------- /lib/api-utilis.js: -------------------------------------------------------------------------------- 1 | /* 2 | * blends two objects together, with second overriding the first 3 | * @param first {Object} 4 | * @param second {Object} 5 | */ 6 | function blend(first, second) { 7 | var mix = {}; 8 | 9 | for(var key in first) 10 | mix[key] = first[key]; 11 | for(var key in second) 12 | mix[key] = second[key]; 13 | 14 | return mix; 15 | } 16 | 17 | /* 18 | * matches a criteria object with a message header 19 | * @param criteria {Object} 20 | * @param header {Object} 21 | * @return {Boolean}: true if they match, false otherwise 22 | */ 23 | function match(criteria, header) { 24 | var matches_source = (criteria.from === header.from || criteria.from === -1); 25 | var matches_subject = (criteria.subject === header.subject || criteria.subject === "-1"); 26 | 27 | return matches_source && matches_subject; 28 | } 29 | 30 | module.exports = { 31 | blend: blend, 32 | match: match 33 | }; 34 | -------------------------------------------------------------------------------- /lib/beacon-configs.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | /* 4 | * configures an ipc with the configurations specific to remote communincation 5 | * @param ipc {IPC}: the ipc to be configured 6 | */ 7 | function configureRemoteIPC(ipc) { 8 | ipc.config.maxRetries = 3; 9 | ipc.config.retry = 600; 10 | //ipc.config.silent = true; 11 | } 12 | 13 | function configureLocalIPC(ipc) { 14 | ipc.config.sync = true; 15 | //ipc.config.silent = true; 16 | ipc.config.id = 'klyng_beacon'; 17 | ipc.config.maxRetries = 0; 18 | } 19 | 20 | // read and parse the configurations file 21 | var configs_data = fs.readFileSync(__dirname + "/../config.json", "utf8"); 22 | var configs = JSON.parse(configs_data); 23 | 24 | module.exports = { 25 | klyngConfigs: configs, 26 | configureRemoteIPC: configureRemoteIPC, 27 | configureLocalIPC: configureLocalIPC 28 | }; 29 | -------------------------------------------------------------------------------- /lib/beacon-controller.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var cp = require('child_process'); 3 | 4 | /* 5 | * checks if the beacon is already running or not and opens an ipc socket to it 6 | * @return {Promise}: a promise of the boolean status value - true if running, false otherwise 7 | */ 8 | function checkIfRunning() { 9 | 10 | return new Promise(function(resolve, reject) { 11 | 12 | // this flag indicates if the promise enclosing the connectTo call is fullfiled 13 | // and is used to determine if a connection faliure happened on the initial attempt 14 | // (aka before the promise is fullfiled) or during the process (after fullfillment) 15 | var promise_fullfiled = false; 16 | 17 | ipc.connectTo('klyng_beacon', function() { 18 | 19 | ipc.of.klyng_beacon.on('connect', function() { 20 | promise_fullfiled = true; 21 | resolve(true); 22 | }); 23 | 24 | ipc.of.klyng_beacon.on('error', function(err) { 25 | if(ipc.of.klyng_beacon.retriesRemaining === 0) { 26 | if(!promise_fullfiled) { 27 | promise_fullfiled = true; 28 | resolve(false); 29 | } 30 | else { 31 | console.error('[Aborted]: Lost connection to local beacon'); 32 | } 33 | } 34 | }); 35 | }); 36 | }); 37 | } 38 | 39 | /* 40 | * starts the beacon process in the background 41 | * @return {Promise}: a promise of a boolean status value - true if the beacon started successfully, false otherwise 42 | */ 43 | function start() { 44 | 45 | // close open socket if exists 46 | if(!!ipc.of.klyng_beacon) 47 | ipc.disconnect('klyng_beacon'); 48 | 49 | var beacon_daemon = cp.spawn('node', [__dirname + '/beacon.js'], {detached: true, stdio: 'ignore'}); 50 | beacon_daemon.unref(); 51 | 52 | return checkIfRunning() ; 53 | } 54 | 55 | /* 56 | * stops the running beacon process 57 | * @return {Promise}: a promise for a boolean status vale - true if the beacon stopped successfully, false otherwise 58 | */ 59 | function stop() { 60 | 61 | var stop = new Promise(function(resolve, reject) { 62 | ipc.of.klyng_beacon.emit('STOP:MSG', {}); 63 | ipc.of.klyng_beacon.socket.on('end', function() { 64 | ipc.disconnect('klyng_beacon'); 65 | setTimeout(resolve, 100); // wait for the beacon to fully close 66 | }); 67 | }) 68 | .then(function() { 69 | return checkIfRunning(); 70 | }) 71 | .then(function(running) { 72 | return !running; 73 | }); 74 | 75 | return stop; 76 | } 77 | 78 | /* 79 | * signals a started beacon to run a given job 80 | * @param job {Object}: the job's description 81 | * @return {Promise}: a promise of boolean/string status value : {'available', 'busy', false (indicates network error)} 82 | */ 83 | function signalToRun(job) { 84 | return new Promise(function(resolve, reject) { 85 | 86 | ipc.of.klyng_beacon.emit('SIGNAL:RUN', {job_specs:job}); 87 | 88 | ipc.of.klyng_beacon.on('SIGNAL:CONFIRM', function(data) { 89 | resolve(data.status); 90 | }); 91 | }); 92 | } 93 | 94 | /* 95 | * signals the beacon indicating the job is done 96 | * so that the beacon can clear the data structures for the next job 97 | * @return {Promise}: a promise for a boolean status value 98 | */ 99 | function signalDone() { 100 | return new Promise(function(resolve, reject) { 101 | ipc.of.klyng_beacon.emit('SIGNAL:DONE', {}); 102 | 103 | ipc.of.klyng_beacon.socket.on('end', function() { 104 | ipc.disconnect('klyng_beacon'); 105 | resolve(true); 106 | }); 107 | }); 108 | } 109 | 110 | /* 111 | * sends the beacon with an abort signal 112 | */ 113 | function signalAbort() { 114 | ipc.of.klyng_beacon.emit('SIGNAL:ABORT', {}); 115 | } 116 | 117 | // exporting the controller methods 118 | module.exports = { 119 | checkIfRunning: checkIfRunning, 120 | start: start, 121 | stop: stop, 122 | signalToRun: signalToRun, 123 | signalDone: signalDone, 124 | signalAbort: signalAbort 125 | }; 126 | -------------------------------------------------------------------------------- /lib/beacon.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var router = require('./router.js'); 3 | var jobman = require('./job-manager.js'); 4 | var tcp = require('./tcp'); 5 | 6 | var configs = require('./beacon-configs'); 7 | 8 | var localIPC = new ipc.IPC(); 9 | configs.configureLocalIPC(localIPC); 10 | ipc.klyngLocal = localIPC; 11 | 12 | localIPC.serve(function() { 13 | 14 | localIPC.server.on('STOP:MSG', function(data, socket) { 15 | socket.end(); 16 | localIPC.server.stop(); 17 | tcp.stop(); 18 | }); 19 | 20 | localIPC.server.on('SIGNAL:RUN', function(data, socket) { 21 | 22 | if(router.isClean()) { 23 | localIPC.server.emit(socket, 'SIGNAL:CONFIRM', {status: 'available'}); 24 | router.setMonitorSocket(socket); 25 | 26 | // handle abrupt monitor_socket closure 27 | socket.on('close', () => { 28 | if(!socket.ignore_closure) { 29 | // send ABORT signals to all remote beacons 30 | var remoteHosts = router.getRemoteChannels(); 31 | for(var host in remoteHosts) { 32 | var connection = remoteHosts[host]; 33 | if(!connection.socket.destroyed) 34 | tcp.signalAbortOver(connection); 35 | } 36 | 37 | // abort everything on this machine 38 | jobman.abort("Lost connection to runner process"); 39 | } 40 | }); 41 | 42 | var job = data.job_specs; 43 | 44 | if(!job.hosts) { 45 | var routingTable = router.buildPureLocalTable(job.size); 46 | router.setMetaTable(routingTable); 47 | job.start = 0; 48 | job.subsize = job.size; 49 | 50 | jobman.runLocally(job); 51 | } 52 | else { 53 | var rDescriptor; 54 | var plan = jobman.divide(job); 55 | 56 | // ensure that local exists in the plan to refer to the parent 57 | plan.local = plan.local || {count: 0}; 58 | 59 | var table = router.buildTableFromPlan(plan, true); 60 | router.setMetaTable(table); 61 | 62 | jobman.pack(job) 63 | .then((appPckg) => { 64 | 65 | // the job descriptor for remote beacons 66 | rDescriptor = { 67 | app: appPckg, 68 | size: job.size, 69 | plan: plan 70 | }; 71 | 72 | var remoteHosts = Object.keys(job.hosts).filter((host) => host !== "local"); 73 | 74 | var connectPromises = remoteHosts.map((host) => { 75 | var creds = host.split(":"); 76 | return tcp.connectTo(creds[0], parseInt(creds[1])); 77 | }); 78 | 79 | return Promise.all(connectPromises); 80 | }) 81 | .then((connections) => { 82 | connections.forEach((connection) => { 83 | // save id and password for later use 84 | connection.id = connection.path + ":" + connection.port; 85 | connection.password = job.hosts[connection.id].password; 86 | 87 | router.setRemoteChannel(connection.id, connection); 88 | }); 89 | 90 | return connections; 91 | }) 92 | .then((connections) => { 93 | var keyPromises = connections.map((connection) => tcp.exchangeKeyOver(connection)); 94 | return Promise.all(keyPromises); 95 | }) 96 | .then((paramsList) => { 97 | var authPromises = paramsList.map( 98 | (params) => tcp.authOver(params.connection, params.secret, params.connection.password) 99 | ); 100 | return Promise.all(authPromises); 101 | }) 102 | .then((paramsList) => { 103 | var jobPromises = paramsList.map( 104 | (params) => tcp.sendJobOver(params.connection, params.secret, rDescriptor, plan) 105 | ); 106 | return Promise.all(jobPromises); 107 | }) 108 | .then(() => { 109 | if(!!plan.local.count) { 110 | jobman.runLocally({ 111 | app: job.app, 112 | size: job.size, 113 | start: plan.local.start, 114 | subsize: plan.local.count 115 | }); 116 | } 117 | }) 118 | .catch((error) => { 119 | // send ABORT signals to all remote beacons 120 | var remoteHosts = router.getRemoteChannels(); 121 | for(var host in remoteHosts) { 122 | var connection = remoteHosts[host]; 123 | if(!connection.socket.destroyed) 124 | tcp.signalAbortOver(connection); 125 | } 126 | 127 | // abort everything on this machine 128 | jobman.abort(error.message); 129 | }) 130 | } 131 | } 132 | else { 133 | localIPC.server.emit(socket, 'SIGNAL:CONFIRM', {status: 'busy'}); 134 | } 135 | }); 136 | 137 | localIPC.server.on('SIGNAL:DONE', function(data, socket) { 138 | 139 | // by now the job should have completed successfully 140 | // and the nodes are about to disconnect form each other 141 | // and clear their data structures (either by the DONE signals or 142 | // by ABORT signals if any error occured in the porcess) 143 | // so now, aborting to monitor_socket errors is redundent 144 | router.ignoreMonitorSocketErrors(); 145 | 146 | var remoteHosts = router.getRemoteChannels(); 147 | var donePromises = Object.keys(remoteHosts).map((host) => { 148 | var connection = remoteHosts[host]; 149 | 150 | return tcp.signalDoneOver(connection) 151 | }); 152 | 153 | Promise.all(donePromises) 154 | .then(() => { 155 | Object.keys(remoteHosts).forEach((host) => { 156 | var connection = remoteHosts[host]; 157 | tcp.disconnectFrom(connection.path, connection.port); 158 | }); 159 | }) 160 | .then(() => { 161 | router.clear(); 162 | socket.end(); 163 | }) 164 | .catch((error) => { 165 | // send ABORT signals to all remote beacons 166 | var remoteHosts = router.getRemoteChannels(); 167 | for(var host in remoteHosts) { 168 | var connection = remoteHosts[host]; 169 | if(!connection.socket.destroyed) 170 | tcp.signalAbortOver(connection); 171 | } 172 | 173 | // abort everything on this machine 174 | jobman.abort(error.message); 175 | }); 176 | }); 177 | 178 | localIPC.server.on('SIGNAL:ABORT', function(data, socket) { 179 | // send ABORT signals to all remote beacons 180 | var remoteHosts = router.getRemoteChannels(); 181 | for(var host in remoteHosts) { 182 | var connection = remoteHosts[host]; 183 | if(!connection.socket.destroyed) 184 | tcp.signalAbortOver(connection); 185 | } 186 | 187 | // abort everything on this machine 188 | jobman.abort("Keyboard Interrupt"); 189 | }); 190 | 191 | }); 192 | 193 | localIPC.server.start(); 194 | tcp.start(); 195 | -------------------------------------------------------------------------------- /lib/cli-controller.js: -------------------------------------------------------------------------------- 1 | var beacon = require('../lib/beacon-controller.js'); 2 | var parameters = require('../lib/command-line-parameters.js'); 3 | var path = require('path'); 4 | var fs = require('fs'); 5 | var os = require('os').platform(); 6 | var ipc = require('node-ipc'); 7 | 8 | // ipc configurations 9 | ipc.config.retry = 600; 10 | ipc.config.maxRetries = 3; 11 | ipc.config.sync = true; 12 | ipc.config.silent = true; 13 | 14 | 15 | /* 16 | * prints an alert to consult help guide in no options are provided 17 | */ 18 | function noOptionsAlert() { 19 | console.log("You didn't specify any options.") 20 | console.log("Run (klyng --help) for more info."); 21 | } 22 | 23 | /* 24 | * prints the help guide of the cli tool 25 | */ 26 | function help() { 27 | 28 | var guide = parameters.getUsage({ 29 | title: "klyng", 30 | hide: ["app"], 31 | description: [ 32 | "A command line tool to run klyng apps" 33 | ], 34 | synopsis: [ 35 | "klyng --help", 36 | "klyng --beacon-up", 37 | "klyng --beacon-down", 38 | "klyng -n ", 39 | "klyng -n -m " 40 | ] 41 | }); 42 | 43 | console.log(guide); 44 | } 45 | 46 | /* 47 | * starts the beacon process in the background 48 | */ 49 | function beaconUp() { 50 | 51 | beacon.checkIfRunning() 52 | .then(function(running) { 53 | if(running) 54 | console.log('The beacon is already up and listening.'); 55 | else 56 | return beacon.start(); 57 | }) 58 | .then(function(started) { 59 | if(started) 60 | console.log('The beacon is now up and listening.'); 61 | else if(started === false) 62 | console.error("Failed to start or connect to beacon."); 63 | }) 64 | .then(function() { 65 | ipc.disconnect('klyng_beacon'); 66 | }); 67 | } 68 | 69 | /* 70 | * shuts the beacon process down if running 71 | */ 72 | function beaconDown() { 73 | 74 | beacon.checkIfRunning() 75 | .then(function(running) { 76 | if(running) { 77 | return beacon.stop(); 78 | } 79 | else { 80 | console.log("The beacon is not up."); 81 | } 82 | }) 83 | .then(function(stopped) { 84 | if(stopped) { 85 | console.log("The beacon is now down."); 86 | } 87 | else if(stopped === false) { 88 | console.error("Failed to take beacon down."); 89 | } 90 | }); 91 | } 92 | 93 | /* 94 | * initaites a specified klyng job 95 | * signals the beacon process to spawn the parallel process 96 | * starts listening for control and monitoring messages from the beacon 97 | * @param job {Object}: the klyng job description 98 | */ 99 | function run(job) { 100 | 101 | beacon.checkIfRunning() 102 | .then(function(running) { 103 | if(running) { 104 | return true; 105 | } 106 | else { 107 | return beacon.start(); 108 | } 109 | }) 110 | .then(function(started) { 111 | if(started) { 112 | return beacon.signalToRun(job); 113 | } 114 | else { 115 | console.error("Failed to start or connect to beacon."); 116 | } 117 | }) 118 | .then(function(signaled) { 119 | if(signaled) { 120 | if(signaled === 'busy') { 121 | console.log('The beacon is busy.'); 122 | ipc.disconnect('klyng_beacon'); 123 | } 124 | else { 125 | 126 | var remainingProcessesCount = job.size; 127 | 128 | ipc.of.klyng_beacon.on('MONITOR:MSG', function(message, socket) { 129 | 130 | if(message.type === 'process:stdout') { 131 | console.log(message.data.line); 132 | } 133 | if(message.type === 'process:stderr') { 134 | console.error(message.data.line); 135 | } 136 | else if(message.type === 'process:exit') { 137 | remainingProcessesCount--; 138 | if(!remainingProcessesCount) 139 | beacon.signalDone() 140 | } 141 | else if(message.type === 'job:aborted') { 142 | console.error("[Aborted]: %s", message.reason); 143 | ipc.disconnect('klyng_beacon'); 144 | } 145 | }); 146 | } 147 | } 148 | }); 149 | } 150 | 151 | /* 152 | * parses a hosts file into an object 153 | * @param path {String}: the path to the hosts file 154 | * @return {Object|Boolean}: the parsed hosts object, or error object if any occured 155 | */ 156 | function parseHosts(path) { 157 | try { 158 | var hFile = fs.readFileSync(path, {encoding: 'utf8'}); 159 | var _hosts = JSON.parse(hFile); 160 | var hosts = {}; // the parsed hosts object 161 | 162 | // iterate over the entries, check for errors, fill in missing data, and parse 163 | for(var host in _hosts) { 164 | var current = _hosts[host]; 165 | 166 | if(typeof current !== "object") { 167 | throw new Error("INVALID: an entry in hosts file is not an object"); 168 | } 169 | 170 | // fill in data if missing 171 | current.max_procs = current.max_procs || Infinity; 172 | if(host !== "local") { 173 | current.port = current.port || 2222; 174 | current.passwd = current.passwd || ""; 175 | } 176 | 177 | // check for errors in entry's data 178 | if(typeof current.max_procs !== "number") { 179 | throw new Error("INVALID: an entry's processes count is not a number"); 180 | } 181 | if(current.max_procs < 1) { 182 | throw new Error("INVALID: an entry's processes count is less than one"); 183 | } 184 | if(host !== "local") { 185 | if(typeof current.port !== "number") { 186 | throw new Error("INVALID: an entry's port number is not a number!"); 187 | } 188 | if(typeof current.passwd !== "string") { 189 | throw new Error("INVALID: an entry's password is not a string"); 190 | } 191 | } 192 | 193 | if(host === "local") { 194 | hosts.local = {max_procs: current.max_procs}; 195 | } 196 | else { 197 | hosts[host + ":" + current.port] = { 198 | max_procs: current.max_procs, 199 | password: current.passwd 200 | } 201 | } 202 | } 203 | 204 | return hosts; 205 | } 206 | catch(error) { 207 | return {error: error.message}; 208 | } 209 | } 210 | 211 | // listner for a SIGINT message for windows as windows doesn't support 212 | // kill('SIGINT'). This is only for testing purpose 213 | if(os === "win32") { 214 | process.on('message', (data) => { 215 | if(data === 'SIGINT') { 216 | process.emit('SIGINT'); 217 | process.disconnect(); 218 | } 219 | }); 220 | } 221 | 222 | process.on('SIGINT', () => { beacon.signalAbort() }); 223 | 224 | module.exports = { 225 | noOptionsAlert: noOptionsAlert, 226 | help: help, 227 | beaconUp: beaconUp, 228 | beaconDown: beaconDown, 229 | run: run, 230 | parseHosts: parseHosts 231 | }; 232 | -------------------------------------------------------------------------------- /lib/command-line-parameters.js: -------------------------------------------------------------------------------- 1 | var commandLineArgs = require('command-line-args'); 2 | 3 | var parameters = new commandLineArgs([ 4 | 5 | { 6 | name: "help", 7 | alias: "h", 8 | type: Boolean, 9 | description: "shows this help guide and exits" 10 | }, 11 | 12 | { 13 | name: "num-processes", 14 | alias: "n", 15 | type: Number, 16 | description: "sets the number of processes to run" 17 | }, 18 | 19 | { 20 | name: "machines", 21 | alias: "m", 22 | type: String, 23 | description: "path to the machines file" 24 | }, 25 | 26 | { 27 | name: "app", 28 | alias: "a", 29 | type: String, 30 | defaultOption: true, 31 | description: "sets the path to the klyng app entry point" 32 | }, 33 | 34 | { 35 | name: "beacon-up", 36 | alias: "u", 37 | type: Boolean, 38 | description: "starts the beacon" 39 | }, 40 | 41 | { 42 | name: "beacon-down", 43 | alias: "d", 44 | type: Boolean, 45 | description: "stops the running beacon" 46 | } 47 | ]); 48 | 49 | module.exports = parameters; 50 | -------------------------------------------------------------------------------- /lib/crypto-service.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto'); 2 | 3 | /* 4 | * encrypts a message and signs it with HMAC 5 | * @param msg {Object}: the message to be secured 6 | * @param key {String}: the secret key to be used 7 | * @return {Object}: the secured message 8 | */ 9 | function secure(msg, key) { 10 | var cipher = crypto.createCipher('aes-256-ctr', key); 11 | var hmac = crypto.createHmac('sha256', key); 12 | var plaintext = (typeof msg === "object") ? JSON.stringify(msg) : msg.toString(); 13 | 14 | var ciphertext = cipher.update(plaintext, 'utf8', 'base64') + cipher.final('base64'); 15 | 16 | hmac.update(ciphertext); 17 | var mac = hmac.digest('base64'); 18 | 19 | return {payload: ciphertext, mac: mac}; 20 | } 21 | 22 | /* verifies an encrypted message and decryptes it if verified 23 | * @param msg {Object}: the encrypted message 24 | * @param key {String}: the secret key to be used 25 | * @return {Object|Boolean}: the decrypted message if verified, false otherwise 26 | */ 27 | function verify(msg, key) { 28 | var hmac = crypto.createHmac('sha256', key); 29 | 30 | hmac.update(msg.payload); 31 | var mac = hmac.digest('base64'); 32 | 33 | if(mac === msg.mac) { 34 | var decipher = crypto.createDecipher('aes-256-ctr', key); 35 | var plaintext = decipher.update(msg.payload, 'base64', 'utf8') + decipher.final('utf8'); 36 | 37 | // attempt to parse as json 38 | try { 39 | return JSON.parse(plaintext); 40 | } 41 | catch(err) { 42 | // if failed, return plaintext directly 43 | return plaintext; 44 | } 45 | } 46 | 47 | return false; 48 | } 49 | 50 | /* 51 | * returns a wrapper over crypto.DiffieHellman 52 | * @param prime {String}: a prime number to initialize with (optional) 53 | * @return {Object} 54 | */ 55 | function diffieHellman(prime) { 56 | var dhObj = null; 57 | var wrapper = {}; 58 | 59 | if(!!prime) 60 | dhObj = crypto.createDiffieHellman(prime, 'base64'); 61 | else 62 | dhObj = crypto.createDiffieHellman(512); 63 | 64 | wrapper.prime = dhObj.getPrime('base64'); 65 | wrapper.publicKey = dhObj.generateKeys('base64'); 66 | 67 | wrapper.computeSecret = function(remotePublicKey) { 68 | return dhObj.computeSecret(remotePublicKey, 'base64', 'base64'); 69 | }; 70 | 71 | return wrapper; 72 | } 73 | 74 | module.exports = { 75 | secure: secure, 76 | verify: verify, 77 | diffieHellman: diffieHellman 78 | }; 79 | -------------------------------------------------------------------------------- /lib/job-manager.js: -------------------------------------------------------------------------------- 1 | var cp = require('child_process'); 2 | var path = require('path'); 3 | var fs = require('fs'); 4 | var readline = require('readline'); 5 | var zipper = require('zip-local'); 6 | var browserify = require('browserify'); 7 | var router = require('./router.js'); 8 | var ipc = require('node-ipc'); 9 | 10 | /* 11 | * packs the app into a zip file 12 | * @param job {Object}: the job specs 13 | * @return {Promise}: of the app descriptor {pckg, id} 14 | */ 15 | function pack(job) { 16 | return new Promise(function(resolve, reject) { 17 | 18 | browserify(job.app, { 19 | builtins: false, 20 | browserField: false, 21 | insertGlobalVars: { 22 | 'process': undefined 23 | } 24 | }) 25 | .external('fibers') 26 | .bundle(function (err, buffer) { 27 | if(err) { 28 | return reject(err); 29 | } 30 | 31 | var app_id = Date.now().toString(); 32 | 33 | zipper.zip(buffer, "app_" + app_id + ".js", function(err, zipped) { 34 | if(err) { 35 | return reject(err); 36 | } 37 | 38 | var packed_buff = zipped.compress().memory(); 39 | resolve({ 40 | id: app_id, 41 | pckg: packed_buff.toString('base64') 42 | }); 43 | }); 44 | }); 45 | }); 46 | } 47 | 48 | /* 49 | * unpacks a packed app from remote root 50 | * @param app {Object}: the app descriptor recived from remote root 51 | */ 52 | function unpack(app) { 53 | return new Promise(function(resolve, reject) { 54 | // check if the unpaking directory exists 55 | var unpacks_dir = __dirname + '/../.unpacks/'; 56 | fs.stat(unpacks_dir, function(err, stats) { 57 | if(err) { 58 | try { 59 | fs.mkdirSync(unpacks_dir); 60 | } 61 | catch(err) { return reject(err); } 62 | } 63 | 64 | var data = new Buffer(app.pckg, 'base64'); 65 | 66 | zipper.unzip(data, function(err, unzipped) { 67 | if(err) { 68 | return reject(err); 69 | } 70 | 71 | unzipped.save(unpacks_dir, function(err) { 72 | if(err) { 73 | return reject(err); 74 | } 75 | 76 | resolve(unpacks_dir + "app_" + app.id + ".js"); 77 | }); 78 | }); 79 | }); 80 | }); 81 | } 82 | 83 | /* 84 | * divides the job processes over the specified hosts using round-robin divison 85 | * @param job {Object}: the job specs 86 | * @return {Object}: a job distribution plan 87 | */ 88 | function divide(job) { 89 | 90 | var hosts = {}; 91 | 92 | for(var host in job.hosts) { 93 | hosts[host] = {count: 0, max: job.hosts[host].max_procs} 94 | } 95 | 96 | // round-robin over the hosts and allocated processes numbers 97 | var hosts_list = Object.keys(hosts); 98 | var allocated = 0; 99 | var next = 0; 100 | var counter = 0; 101 | var overallocate = false; // if true: hosts will be overallocated with processes to meet job.size 102 | while(allocated < job.size) { 103 | var host = hosts[hosts_list[next++ % hosts_list.length]]; 104 | if(host.count != host.max || overallocate) { 105 | host.count++; 106 | allocated++; 107 | if(!overallocate) { 108 | // if not in overallocate mode already, reset the counter to indicate allocation 109 | counter = 0; 110 | } 111 | } 112 | else { 113 | counter++; 114 | // set the overallocate to true if gone through the list without allocation 115 | overallocate = (counter === hosts_list.length); 116 | } 117 | } 118 | 119 | var start_index = 0; 120 | // register the 0 index process to the local host if exists 121 | if(!!hosts["local"]) { 122 | hosts.local.start = 0; 123 | start_index += hosts.local.count; 124 | } 125 | // assign the rest of indecies to the rest of hosts 126 | for(var host in hosts) { 127 | if(host !== "local") { 128 | hosts[host].start = start_index; 129 | start_index += hosts[host].count; 130 | } 131 | delete hosts[host]["max"]; 132 | } 133 | 134 | return hosts; 135 | } 136 | 137 | /* 138 | * runs the given subjob on the local machine 139 | * @param subjob {Object}: subjob description 140 | */ 141 | function runLocally(subjob) { 142 | 143 | for(var i = 0; i < subjob.subsize; i++) { 144 | 145 | var rank = subjob.start + i; 146 | var jobInstance = cp.spawn('node', [subjob.app, subjob.size, rank], { 147 | stdio: [null, null, null, 'ipc'] 148 | }); 149 | 150 | var stdoutRlInterface = readline.createInterface({input: jobInstance.stdout}); 151 | var stderrRlInterface = readline.createInterface({input: jobInstance.stderr}); 152 | 153 | // register a handler for instance's stdout data 154 | // reports back to the parent klyng process 155 | stdoutRlInterface.on('line', function(log) { 156 | router.routeToParent({type: "process:stdout", data: {line: log}}); 157 | }); 158 | 159 | // register a handler for instance's stderr data 160 | // reports back to the parent klyng process 161 | stderrRlInterface.on('line', function(elog) { 162 | router.routeToParent({type: "process:stderr", data: {line: elog}}); 163 | }) 164 | 165 | // register a handler for instance exit event 166 | // reports back to the parent klyng process 167 | jobInstance.on('exit', function(code) { 168 | router.routeToParent({type: "process:exit", data: {code: code}}); 169 | }); 170 | 171 | // register a handler for instance's messages 172 | jobInstance.on('message', function(msg) { 173 | if(msg.type === 'klyng:msg') { 174 | router.routeTo(msg.header.to, msg); 175 | } 176 | }); 177 | 178 | jobInstance.unref(); 179 | 180 | router.setLocalChannel(rank, jobInstance); 181 | } 182 | }; 183 | 184 | /* 185 | * aborts the job running on the current host 186 | * @param reason {String}: the reason for abortion 187 | */ 188 | function abort(reason) { 189 | 190 | // the nodes by now already recieved a signal to abort the job 191 | // handeling any monitor_socket error now (aka aborting the job) 192 | // is redundent and useless 193 | router.ignoreMonitorSocketErrors(); 194 | 195 | router.routeToParent({type: 'job:aborted', reason: reason}); 196 | 197 | // kill all local processes 198 | var locals = router.getLocalChannels(); 199 | for(var rank in locals) { 200 | var proc = locals[rank]; 201 | proc.kill(); 202 | } 203 | 204 | // wait for a second to ensure that job:aborted is on the network if possible 205 | setTimeout(() => { 206 | // disconnect from all remote beacon 207 | console.log(); 208 | for(var socketId in ipc.klyngRemote.of) { 209 | ipc.klyngRemote.disconnect(socketId); 210 | } 211 | router.clear(); 212 | }, 1000); 213 | } 214 | 215 | module.exports = { 216 | pack: pack, 217 | unpack: unpack, 218 | divide, 219 | runLocally: runLocally, 220 | abort: abort 221 | }; 222 | -------------------------------------------------------------------------------- /lib/router.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | 3 | /* 4 | * stores information about the location of participating processes in the job 5 | * each process key has a value of either 'local' or 'remote' 6 | * a special 'parent' key is present for the location of the klyng process 7 | */ 8 | var meta_routing_table = {}; 9 | 10 | /* 11 | * stores the ipc channels of each local process participating in the job 12 | * @key {String}: 'proc' + process rank 13 | * @value {ChildProcess}: a refernce to the process 14 | */ 15 | var locals_routiung_channels = {}; 16 | 17 | /* 18 | * stores the ipc tcp channels of each remote beacon participating in the job 19 | * @key {String}: 'sock_' + ip:port 20 | * @value {Socket} 21 | */ 22 | var remote_routing_channels = {}; 23 | 24 | // holds a reference to the socket connected to klyng process 25 | // to send monitoring messages across 26 | var monitor_socket = null; 27 | 28 | /* 29 | * stores messages for local processes that arrived early before process creation 30 | * @key {String}: the process id 31 | * @value {Array}: the queue of the early messages 32 | */ 33 | var waiting_queues = {}; 34 | 35 | /* 36 | * checks if the routing data structures are empty 37 | * @return {Boolean}: true if empty, false otherwise 38 | */ 39 | function isClean() { 40 | var clean_meta = !Object.keys(meta_routing_table).length; 41 | var clean_locals = !Object.keys(locals_routiung_channels).length; 42 | var clean_remotes = !Object.keys(remote_routing_channels).length; 43 | var clean_monitor = (monitor_socket === null); 44 | 45 | return clean_meta && clean_locals && clean_remotes && clean_monitor; 46 | } 47 | 48 | /* 49 | * checks if the routing table contains remote members 50 | * @return {Boolean}: true if it has remotes, false otherwise 51 | */ 52 | function hasRemotes() { 53 | return !Object.keys(remote_routing_channels).length; 54 | } 55 | 56 | /* 57 | * builds a pure local routing table for jobs with no remotes 58 | * @param size {Number}: the size of the job 59 | * @return {Object}: the built table; 60 | */ 61 | function buildPureLocalTable(size) { 62 | var pureLocal = {}; 63 | for(var id = 0; id < size; id++) 64 | pureLocal['proc' + id] = 'local'; 65 | pureLocal.parent = 'local'; 66 | 67 | return pureLocal; 68 | } 69 | 70 | /* 71 | * builds a routing table using a given job distribution plan 72 | * @param plan {Object}: job plan 73 | * @param local_parent {Boolean}: a flag to indicate whether the parent is local or remote 74 | * @return {Object}: the built table 75 | */ 76 | function buildTableFromPlan(plan, local_parent) { 77 | var table = {}; 78 | for(var host in plan) { 79 | var hostinfo = plan[host]; 80 | for(var i = 0; i < hostinfo.count; i++) { 81 | table['proc' + (hostinfo.start + i)] = host; 82 | } 83 | } 84 | 85 | table.parent = local_parent ? "local" : "remote"; 86 | 87 | return table; 88 | } 89 | 90 | /* 91 | * sets the meta routing table 92 | * @param table {Object}: the new job's table 93 | */ 94 | function setMetaTable(table) { 95 | meta_routing_table = table; 96 | } 97 | 98 | /* 99 | * sets the routing socket of the table 100 | * @param socket {Socket} 101 | */ 102 | function setMonitorSocket(socket) { 103 | monitor_socket = socket; 104 | } 105 | 106 | /* 107 | * sets a flag on monitor_socket to ignore any abrupt closure 108 | */ 109 | function ignoreMonitorSocketErrors() { 110 | if(!!monitor_socket) { 111 | monitor_socket.ignore_closure = true; 112 | } 113 | } 114 | 115 | /* 116 | * sets a local channel of a process, and consumes waiting messages if any 117 | * @param rank {Number} 118 | * @param proc {ChildProcess} 119 | */ 120 | function setLocalChannel(rank, proc) { 121 | locals_routiung_channels['proc' + rank] = proc; 122 | if(!!waiting_queues['proc' + rank]) { 123 | waiting_queues['proc' + rank].forEach((msg) => proc.send(msg)); 124 | delete waiting_queues['proc' + rank]; 125 | } 126 | } 127 | 128 | /* 129 | * returns a refernce to the local channels table 130 | * @return {Object} 131 | */ 132 | function getLocalChannels() { 133 | return locals_routiung_channels; 134 | } 135 | 136 | /* 137 | * sets a remote channel of a beacon 138 | * @param id {String}: the identifing 'ip:port' connection string 139 | * @param sock {Socket}: the socket to the beacon 140 | */ 141 | function setRemoteChannel(id, sock) { 142 | remote_routing_channels['sock_' + id] = sock; 143 | if(!!waiting_queues['sock_' + id]) { 144 | waiting_queues['sock_' + id].forEach((msg) => sock.emit('KLYNG:MSG', msg)); 145 | delete waiting_queues['sock_' + id]; 146 | } 147 | } 148 | 149 | /* 150 | * returns a refernce to the remote channels table 151 | * @return {Object} 152 | */ 153 | function getRemoteChannels() { 154 | return remote_routing_channels; 155 | } 156 | 157 | /* 158 | * clears the router data structures 159 | */ 160 | function clear() { 161 | waiting_queues = {}; 162 | meta_routing_table = {}; 163 | locals_routiung_channels = {}; 164 | remote_routing_channels = {}; 165 | monitor_socket = null; 166 | } 167 | 168 | /* 169 | * routes a message to the parent process 170 | * @param id {Number}: the job's id 171 | * @param message {Object}: the message to be routed 172 | */ 173 | function routeToParent(message) { 174 | if(meta_routing_table.parent === 'local') { 175 | // fall to common ipc instance if klyng's is not there (happens in unit tests) 176 | ipc = ipc.klyngLocal || ipc; 177 | if(!monitor_socket.destroyed) { 178 | ipc.server.emit(monitor_socket, 'MONITOR:MSG', message); 179 | } 180 | } 181 | else { 182 | if(!!monitor_socket && !monitor_socket.socket.destroyed) { 183 | monitor_socket.emit('MONITOR:MSG', message); 184 | } 185 | } 186 | } 187 | 188 | /* 189 | * routes a message to a process by rank, and stores messsages to local 190 | * processes if they haven't been craeted yet 191 | * @param to {Number}: the recepient's rank 192 | * @param msg {Object}: the message to route 193 | */ 194 | function routeTo(to, msg) { 195 | if(meta_routing_table['proc' + to] === 'local') { 196 | if(!!locals_routiung_channels['proc' + to]) { 197 | try { 198 | locals_routiung_channels['proc' + to].send(msg); 199 | } 200 | catch(err) { 201 | // swallow the error to prevent the beacon from failing 202 | // NOTE: This is a lousy quick fix, think of something better 203 | } 204 | } 205 | else { 206 | if(!!waiting_queues['proc' + to]) { 207 | waiting_queues['proc' + to] = []; 208 | } 209 | waiting_queues['proc' + to].push(msg); 210 | } 211 | } 212 | else { 213 | var id = meta_routing_table['proc' + to]; 214 | if(!!remote_routing_channels['sock_' + id]) { 215 | remote_routing_channels['sock_' + id].emit('KLYNG:MSG', msg); 216 | } 217 | else { 218 | if(!!waiting_queues['sock_' + id]) { 219 | waiting_queues['sock_' + id] = []; 220 | } 221 | waiting_queues['sock_' + id].push(msg); 222 | } 223 | } 224 | } 225 | 226 | module.exports = { 227 | isClean: isClean, 228 | hasRemotes: hasRemotes, 229 | buildPureLocalTable: buildPureLocalTable, 230 | buildTableFromPlan: buildTableFromPlan, 231 | setMetaTable: setMetaTable, 232 | setMonitorSocket: setMonitorSocket, 233 | ignoreMonitorSocketErrors: ignoreMonitorSocketErrors, 234 | setLocalChannel: setLocalChannel, 235 | getLocalChannels: getLocalChannels, 236 | setRemoteChannel: setRemoteChannel, 237 | getRemoteChannels: getRemoteChannels, 238 | clear: clear, 239 | routeToParent: routeToParent, 240 | routeTo: routeTo 241 | }; 242 | -------------------------------------------------------------------------------- /lib/tcp.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var cs = require('./crypto-service'); 3 | var configs = require('./beacon-configs'); 4 | var router = require('./router'); 5 | var jobman = require('./job-manager'); 6 | 7 | var remoteIPC = new ipc.IPC(); 8 | configs.configureRemoteIPC(remoteIPC); 9 | ipc.klyngRemote = remoteIPC; 10 | 11 | 12 | /* 13 | * connects to a remote beacon via ip:port 14 | * @param ip {String} 15 | * @param port {Number} 16 | * @return {Promise}: promise of the connection to be made 17 | */ 18 | function connectTo(ip, port) { 19 | return new Promise(function(resolve, reject) { 20 | var id = "sock_" + ip + ":" + port; 21 | 22 | remoteIPC.connectToNet(id, ip, port, function() { 23 | remoteIPC.of[id].on('connect', function() { 24 | resolve(remoteIPC.of[id]); 25 | }); 26 | 27 | remoteIPC.of[id].on('error', function(error) { 28 | if(remoteIPC.of[id].retriesRemaining === 0) { 29 | var remoteHosts = router.getRemoteChannels(); 30 | if(!!remoteHosts[id]) { 31 | jobman.abort(error.message); 32 | } 33 | reject(error); 34 | } 35 | }); 36 | }); 37 | }); 38 | } 39 | 40 | /* 41 | * disconnects from a remote beacon identified by ip:port 42 | * @param ip {String} 43 | * @param port {Number} 44 | */ 45 | function disconnectFrom(ip, port) { 46 | var id = "sock_" + ip + ":" + port; 47 | remoteIPC.disconnect(id); 48 | } 49 | 50 | /* 51 | * carries the key exchange process between the communicating nodes 52 | * @param connection {Socket}: the socket to between the nodes 53 | * @return {Promise}: a promise of the shared secret 54 | */ 55 | function exchangeKeyOver(connection) { 56 | return new Promise(function(resolve, reject) { 57 | var hostId = connection.path + ":" + connection.port; 58 | try { 59 | var dhObj = cs.diffieHellman(); 60 | var sharedPrime = dhObj.prime; 61 | var publicKey = dhObj.publicKey; 62 | 63 | connection.emit('KEY-EXT:PARAMS', {prime: sharedPrime, key: publicKey}); 64 | connection.on('KEY-EXT:PUBLIC', function(data) { 65 | if(!data.error) { 66 | var sharedSecret = dhObj.computeSecret(data.key); 67 | resolve({connection: connection, secret: sharedSecret}); 68 | } 69 | else { 70 | reject(new Error(hostId + " " + data.error)); 71 | } 72 | }); 73 | } 74 | catch(error) { reject(new Error(hostId + " " + error.message)); } 75 | }); 76 | } 77 | 78 | /* 79 | * authenticates and authorize access to remote node with password 80 | * @param connection {Socket}: the socket connecting to the remote node 81 | * @param secret {String}: the encryption seceret key 82 | * @param passwd {String}: the remote node password 83 | * @return {Promise} 84 | */ 85 | function authOver(connection, secret, passwd) { 86 | var hostId = connection.path + ":" + connection.port; 87 | return new Promise(function(resolve, reject) { 88 | try { 89 | connection.emit('AUTH', cs.secure({data: passwd}, secret)); 90 | connection.on('AUTH:STATUS', function(data) { 91 | if(data.status) { 92 | resolve({connection: connection, secret: secret}); 93 | } 94 | else { 95 | reject(new Error(hostId + " " + data.error)); 96 | } 97 | }); 98 | } 99 | catch(error) { reject(new Error(hostId + " " + error.message)); } 100 | }); 101 | } 102 | 103 | /* 104 | * sends a job descriptor securly to a remote beacon over a given connection 105 | * @param connection {Socket}: the socket connection to the remote node 106 | * @param secret {String}: the encryption secret key 107 | * @param job {Object}: the job descriptor object 108 | * @param plan {Object}: the job plan object 109 | * @return {Promise} 110 | */ 111 | function sendJobOver(connection, secret, job, plan) { 112 | var hostId = connection.path + ":" + connection.port; 113 | return new Promise(function(resolve, reject) { 114 | 115 | var target_params = connection.path + ":" + connection.port; 116 | var modified_plan = {}; 117 | 118 | for(var host in plan) { 119 | if(host === "local") { 120 | // replace local with parent to refer to the parent node in target node 121 | var boundPort = !!remoteIPC.server ? remoteIPC.server.port : configs.klyngConfigs.port; 122 | modified_plan.parent = plan.local; 123 | modified_plan.parent.port = boundPort; 124 | } 125 | else if(host === target_params) { 126 | // replace the target node params in the plan with local 127 | modified_plan.local = plan[host]; 128 | } 129 | else { 130 | // keep irrelevant hosts intact 131 | modified_plan[host] = plan[host]; 132 | } 133 | } 134 | 135 | job.plan = modified_plan; 136 | 137 | try { 138 | connection.emit('KLYNG:JOB', cs.secure({data: job}, secret)); 139 | connection.on('JOB:ACK', function(data) { 140 | if(!data.status) { 141 | reject(new Error(hostId + " " + data.error)); 142 | } 143 | else { 144 | resolve(true); 145 | } 146 | }); 147 | } 148 | catch(error) { reject(new Error(hostId + " " + error.message)); } 149 | }); 150 | } 151 | 152 | /* 153 | * sends a DONE signal to a remote beacon over a given connection 154 | * @param connection {Socket}: the socket connecting to the remote beacon 155 | * @return {Promise} 156 | */ 157 | function signalDoneOver(connection) { 158 | var hostId = connection.path + ":" + connection.port; 159 | return new Promise(function(resolve, reject) { 160 | connection.emit('SIGNAL:DONE', {}); 161 | connection.on('DONE:ACK', function(data) { 162 | if(data.status) { 163 | resolve(true); 164 | } 165 | else { 166 | reject(new Error(hostId + " " + data.error)); 167 | } 168 | }); 169 | }); 170 | } 171 | 172 | /* 173 | * sends an ABORT signal to a remote beacon over a given connection 174 | * @param connection {Socket}: the socket connecting to the remote beacon 175 | * @param reason {String}: the abortion reason 176 | */ 177 | function signalAbortOver(connection) { 178 | connection.emit('SIGNAL:ABORT', {}); 179 | } 180 | 181 | /* 182 | * starts the beacon's tcp server 183 | * @param _configs {Object}: An optional configurations object to override configs.json 184 | */ 185 | function start(_configs) { 186 | 187 | var configurations = _configs || configs.klyngConfigs; 188 | 189 | remoteIPC.serveNet('0.0.0.0', configurations.port, function() { 190 | 191 | // listner for key exchange process initiation 192 | remoteIPC.server.on('KEY-EXT:PARAMS', function(data, socket) { 193 | try { 194 | 195 | if(!router.isClean()) { 196 | throw new Error("Beacon is Busy."); 197 | } 198 | 199 | var dhObj = cs.diffieHellman(data.prime); 200 | var publicKey = dhObj.publicKey; 201 | 202 | // store the secret in the socket object 203 | socket.klyng_secret = dhObj.computeSecret(data.key); 204 | 205 | remoteIPC.server.emit(socket, 'KEY-EXT:PUBLIC', { 206 | key: publicKey, 207 | cipherWelcome: cs.secure("Hello from Beacon's TCP Server!", socket.klyng_secret) 208 | }); 209 | } 210 | catch(error) { 211 | remoteIPC.server.emit(socket, 'KEY-EXT:PUBLIC', {error: error.message}); 212 | } 213 | }); 214 | 215 | // listner for password auth messages 216 | remoteIPC.server.on('AUTH', function(data, socket) { 217 | if(!!socket.klyng_secret) { 218 | var errorMessage = ""; 219 | var attemptPassword; 220 | 221 | try { 222 | attemptPassword = cs.verify(data, socket.klyng_secret); 223 | } 224 | catch(error) {errorMessage = error.message;} 225 | 226 | if(!!attemptPassword && !errorMessage) { 227 | if(attemptPassword.data === configurations.password) { 228 | socket.klyng_authorized = true; 229 | remoteIPC.server.emit(socket, 'AUTH:STATUS', {status: true}); 230 | return; 231 | } 232 | else { 233 | errorMessage = "incorrect password"; 234 | } 235 | } 236 | else { 237 | errorMessage = "corrupted data"; 238 | } 239 | } 240 | else { 241 | errorMessage = "unsecure channel"; 242 | } 243 | 244 | remoteIPC.server.emit(socket, 'AUTH:STATUS', {status: false, error: errorMessage}); 245 | }); 246 | 247 | // listner for a job 248 | remoteIPC.server.on('KLYNG:JOB', function(data, socket) { 249 | var errorMessage = ""; 250 | if(!!socket.klyng_secret && !!socket.klyng_authorized) { 251 | if(router.isClean()) { 252 | var decrypted; 253 | try { 254 | decrypted = cs.verify(data, socket.klyng_secret); 255 | } 256 | catch(error) { errorMessage = error.message; } 257 | 258 | if(!!decrypted && !errorMessage) { 259 | 260 | // this call to clear is to ensure that the waiting_queues are empty 261 | // as some messages could have come after a past job was aborted and 262 | // were stored in the waiting_queues 263 | router.clear(); 264 | 265 | remoteIPC.server.emit(socket, 'JOB:ACK', {status: true}); 266 | 267 | var job = decrypted.data; 268 | 269 | // save the job in the socket object to be able to mark 270 | // it as aborted later if neccessary 271 | socket.klyng_job = job; 272 | 273 | // modify the parent field in job's plan with the remoteAddress 274 | job.plan.parent.isParent = true; 275 | var parentPort = job.plan.parent.port; 276 | job.plan[socket.remoteAddress + ":" + parentPort] = job.plan.parent; 277 | delete job.plan.parent; 278 | 279 | // connect early to the monitor_socket 280 | connectTo(socket.remoteAddress, parentPort) 281 | .then((connection) => { 282 | var id = socket.remoteAddress + ":" + parentPort; 283 | router.setMonitorSocket(connection); 284 | router.setRemoteChannel(id, connection); 285 | }) 286 | .then(() => { 287 | var hosts = Object.keys(job.plan).filter( 288 | (host) => host !== "local" && !job.plan[host].isParent 289 | ); 290 | 291 | // attempt to connect to all listed hosts 292 | var connectionPromises = hosts.map((host) => { 293 | var creds = host.split(":"); 294 | return connectTo(creds[0], parseInt(creds[1])); 295 | }); 296 | 297 | return Promise.all(connectionPromises) 298 | }) 299 | .then((connections) => { 300 | connections.forEach((connection) => { 301 | var hostid = connection.path + ":" + connection.port; 302 | router.setRemoteChannel(hostid, connection); 303 | if(job.plan[hostid].isParent) { 304 | router.setMonitorSocket(connection); 305 | } 306 | }); 307 | }) 308 | .then(() => { 309 | var metatable = router.buildTableFromPlan(job.plan, false); 310 | router.setMetaTable(metatable); 311 | 312 | if(!job.aborted) { 313 | return jobman.unpack(job.app); 314 | } 315 | }) 316 | .then((app_path) => { 317 | var localWork = { 318 | app: app_path, 319 | size: job.size, 320 | subsize: job.plan.local.count, 321 | start: job.plan.local.start 322 | }; 323 | 324 | if(!job.aborted) { 325 | jobman.runLocally(localWork); 326 | } 327 | }) 328 | .catch((error) => jobman.abort(error.message)); 329 | 330 | return; 331 | } 332 | else { 333 | errorMessage = "corrupted data"; 334 | } 335 | } 336 | else { 337 | errorMessage = "The Beacon is busy"; 338 | } 339 | } 340 | else { 341 | errorMessage = "Unauthorized"; 342 | } 343 | 344 | // in case of error 345 | remoteIPC.server.emit(socket, 'JOB:ACK', {status: false, error: errorMessage}); 346 | }); 347 | 348 | // listner for klyng messages 349 | remoteIPC.server.on('KLYNG:MSG', function(msg, socket) { 350 | router.routeTo(msg.header.to, msg); 351 | }); 352 | 353 | // listner for monitor messages 354 | remoteIPC.server.on('MONITOR:MSG', function(msg, socket) { 355 | var hostId = socket.remoteAddress; 356 | if(msg.type !== "job:aborted") { 357 | router.routeToParent(msg); 358 | } 359 | else if(msg.reason !== "SIGNALED") { 360 | // in case parent got a job:aborted mesaage and the reason is not 361 | // that it was signaled by the parent to abort, this would mean that 362 | // some node failed and the parent should notify all others nodes 363 | // and abort the whole job 364 | var remoteHosts = router.getRemoteChannels(); 365 | for(var host in remoteHosts) { 366 | var connection = remoteHosts[host]; 367 | if(!connection.socket.destroyed) 368 | signalAbortOver(connection); 369 | } 370 | 371 | // then abort its own and report back to the runner 372 | jobman.abort(hostId + " " + msg.reason); 373 | } 374 | }); 375 | 376 | // listner for done signal 377 | remoteIPC.server.on('SIGNAL:DONE', function(data, socket) { 378 | var errorMessage = ""; 379 | if(!!socket.klyng_secret && !!socket.klyng_authorized) { 380 | var connectedHosts = Object.keys(router.getRemoteChannels()); 381 | 382 | connectedHosts.forEach((hostid) => remoteIPC.disconnect(hostid)); 383 | 384 | router.clear(); 385 | remoteIPC.server.emit(socket, 'DONE:ACK', {status: true}); 386 | return; 387 | } 388 | else { 389 | errorMessage = "Unauthorized"; 390 | } 391 | 392 | remoteIPC.server.emit(socket, 'DONE:ACK', {status: false, error: errorMessage}); 393 | }); 394 | 395 | // listner for abort signal 396 | remoteIPC.server.on('SIGNAL:ABORT', function(data, socket) { 397 | if(!!socket.klyng_secret && !! socket.klyng_authorized) { 398 | if(!!socket.klyng_job) { 399 | socket.klyng_job.aborted = true; 400 | } 401 | 402 | jobman.abort('SIGNALED'); 403 | } 404 | }); 405 | 406 | }); 407 | remoteIPC.server.start(); 408 | } 409 | 410 | /* 411 | * stops the beacon's tcp server 412 | */ 413 | function stop() { 414 | remoteIPC.server.stop(); 415 | } 416 | 417 | module.exports = { 418 | connectTo: connectTo, 419 | disconnectFrom: disconnectFrom, 420 | exchangeKeyOver: exchangeKeyOver, 421 | authOver: authOver, 422 | sendJobOver: sendJobOver, 423 | signalDoneOver: signalDoneOver, 424 | signalAbortOver: signalAbortOver, 425 | start: start, 426 | stop: stop 427 | }; 428 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "klyng", 3 | "version": "1.0.5", 4 | "description": "A message-passing distributed computing framework for node.js", 5 | "bin": "bin/main.js", 6 | "scripts": { 7 | "test": "gulp test", 8 | "unit-tests": "gulp unit-tests", 9 | "integration-tests": "gulp integration-tests", 10 | "benchmarks": "node tests/benchmarks/run.js" 11 | }, 12 | "author": { 13 | "name": "Mostafa Samir", 14 | "email": "mostafa.3210@gmail.com" 15 | }, 16 | "license": "MIT", 17 | "keywords": [ 18 | "distributed", 19 | "message-passing", 20 | "parallel" 21 | ], 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/Mostafa-Samir/klyng.git" 25 | }, 26 | "dependencies": { 27 | "browserify": "^13.0.0", 28 | "command-line-args": "^2.1.4", 29 | "fibers": "^1.0.9", 30 | "node-ipc": "https://github.com/RIAEvangelist/node-ipc/tarball/master", 31 | "zip-local": "^0.3.2" 32 | }, 33 | "devDependencies": { 34 | "chai": "^3.5.0", 35 | "colors": "^1.1.2", 36 | "gulp": "^3.9.1", 37 | "gulp-mocha": "^2.2.0", 38 | "gulp-sequence": "^0.4.5", 39 | "mocha": "^2.4.5", 40 | "rimraf": "^2.5.2" 41 | }, 42 | "engines": { 43 | "node": ">=4.2.3" 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /tests/benchmarks/run.js: -------------------------------------------------------------------------------- 1 | var colors = require('colors'); 2 | var os = require('os').platform(); 3 | var spawn = require('child_process').spawnSync; 4 | var cmdargs = require('command-line-args'); 5 | var tasks = require('./tasks'); 6 | 7 | if(os === "win32") { 8 | console.error("Sorry, Benchmarking on Windows is not supported for now!".bold.red); 9 | prcocess.exit(); 10 | } 11 | 12 | var params = cmdargs([ 13 | { 14 | name: "process-counts", 15 | type: Number, 16 | multiple: true, 17 | defaultValue: [1, 2, 4] 18 | }, 19 | 20 | { 21 | name: "iterations", 22 | type: Number, 23 | defaultValue: 100 24 | }, 25 | 26 | { 27 | name: "tasks", 28 | type: String, 29 | multiple: true, 30 | defaultValue: tasks.map(function(task) { return task.alias; }) 31 | }, 32 | 33 | { 34 | name: "no-mpi", 35 | type: Boolean, 36 | defaultValue: false 37 | }, 38 | 39 | { 40 | name: "metrics", 41 | type: String, 42 | multiple: true, 43 | defaultValue: ['rtet', 'mpct'] 44 | } 45 | ]); 46 | 47 | var args = params.parse(); 48 | 49 | var pcounts = args['process-counts']; 50 | var iterations = args['iterations']; 51 | var executed_tasks = args['tasks'].map(function(task) { return task.toLowerCase(); }); 52 | args['metrics'] = args['metrics'].map(function(metric) { return metric.toLowerCase(); }); 53 | 54 | /* 55 | * checks if getrusage package is installed 56 | * @return {Boolean} 57 | */ 58 | function check_getrusage() { 59 | try { 60 | require('getrusage'); 61 | return true; 62 | } 63 | catch(err) { 64 | return false; 65 | } 66 | } 67 | 68 | /* 69 | * installs getrusage package 70 | * @return {Boolean}: true if installed, false otherwise 71 | */ 72 | function install_getrusage() { 73 | var install = spawn('npm', ['install', 'getrusage']); 74 | if(install.stderr.length) { 75 | return false; 76 | } 77 | else { 78 | return check_getrusage(); 79 | } 80 | } 81 | 82 | /* 83 | * checks if mpi binaries {mpicc, mpiexec/mpirun} exist on the system 84 | * @return {Object}: {'exists': true/false, 'runner': 'mpirun'/'mpiexec'/undefined} 85 | */ 86 | function checkmpi() { 87 | var compiler = spawn('mpicc'); 88 | if(!compiler.error) { 89 | var run = spawn('mpirun'); 90 | if(!run.error) { 91 | return {exits: true, runner: 'mpirun'}; 92 | } 93 | else { 94 | var exec = spawn('mpiexec'); 95 | if(!exec.error) { 96 | return {exists: true, runner: 'mpiexec'}; 97 | } 98 | } 99 | } 100 | 101 | return {exists: false}; 102 | } 103 | 104 | /* 105 | * compiles the mpi.cpp file in the task_dir to mpi.out file 106 | * @param task_dir {String}: the directory of the task 107 | * @return {Object}: the result of the operation (errored or not) 108 | */ 109 | function compile(task_dir) { 110 | var mpi_src = task_dir + '/mpi.cpp'; 111 | var mpi_exe = task_dir + '/mpi.out'; 112 | 113 | var compilation = spawn('mpicc', [mpi_src, '-o', mpi_exe]); 114 | var status = { success: true }; 115 | 116 | if(compilation.error || compilation.stderr.length > 0) { 117 | status.success = false; 118 | status.error = compilation.stderr.toString(); 119 | } 120 | 121 | return status; 122 | } 123 | 124 | /* 125 | * reports on the current line of stdout the progress of the running task 126 | * @param msg {String}: a descriptive message 127 | * @param done {Number}: the portion done of the task 128 | * @param size {Number}: task size 129 | */ 130 | function report_progress(msg, done, size) { 131 | process.stdout.clearLine(); 132 | process.stdout.cursorTo(0); 133 | process.stdout.write(msg + " " + done + "/" + size); 134 | } 135 | 136 | /* 137 | * gets the max cpu time of all n process as reported on processes stdout 138 | * @param stdout {String}: string represntation of the runner stdout 139 | * @param n {Number}: the number of processes 140 | * @return {Number}: the max of all cpu times 141 | */ 142 | function getmaxcputime(stdout, n) { 143 | var regex = /cputime:([0-9.]*)/; 144 | var max_cputime = -1; 145 | 146 | for(var p = 0; p < n; ++p) { 147 | var parsed = regex.exec(stdout); 148 | if(parsed === null) { 149 | return -1; 150 | } 151 | 152 | var pcputime = parseFloat(parsed[1]); 153 | max_cputime = (pcputime > max_cputime) ? pcputime: max_cputime; 154 | } 155 | 156 | return max_cputime; 157 | } 158 | 159 | console.log("Klyng Benchmarks".cyan.bold); 160 | console.log(""); 161 | 162 | var frameworks = ['MPI', 'klyng']; 163 | 164 | var runmpi = !args['no-mpi']; 165 | var mpirunner = ""; 166 | 167 | if(!args['no-mpi']) { 168 | process.stdout.write("Checking MPI Binaries ..."); 169 | var mpicheck = checkmpi(); 170 | if(mpicheck.exits) { 171 | process.stdout.write(" Found!\n".green); 172 | mpirunner = mpicheck.runner; 173 | report_progress("Compiling MPI Files:", 0, tasks.length); 174 | 175 | for(var i = 0; i < tasks.length; ++i) { 176 | var task_dir = __dirname + '/tasks/' + tasks[i].alias; 177 | var compilation = compile(task_dir); 178 | 179 | if(compilation.success) { 180 | report_progress("Compiling MPI Files:", i + 1, tasks.length); 181 | } 182 | else { 183 | console.error("\nError.".red.bold); 184 | console.error(compilation.error.italic); 185 | runmpi = false; 186 | 187 | break; 188 | } 189 | } 190 | } 191 | else { 192 | process.stdout.write(" Not Found!\n".red); 193 | process.stdout.write("Skipping MPI Compilation".yellow); 194 | runmpi = false; 195 | } 196 | 197 | process.stdout.write("\n"); 198 | } 199 | 200 | process.stdout.write("Checking getrusage package ..."); 201 | if(check_getrusage()) { 202 | process.stdout.write(" Found!\n".green); 203 | } 204 | else { 205 | process.stdout.write(" Not Found!\n".yellow); 206 | process.stdout.write("Installing getrusage package ..."); 207 | 208 | var installation = install_getrusage(); 209 | if(installation) { 210 | process.stdout.write(" Done!\n".green); 211 | } 212 | else { 213 | process.stdout.write(" Failed!\n".red); 214 | process.exit(); 215 | } 216 | } 217 | 218 | process.stdout.write("Retarting klyng's Beacon ..."); 219 | var status = spawn("node", [__dirname + "/../../bin/main.js", '-d']); 220 | var status = spawn("node", [__dirname + "/../../bin/main.js", '-u']); 221 | if(!status.err && !status.stderr.length) { 222 | process.stdout.write(" Done!\n".green); 223 | } 224 | else { 225 | process.stderr.write(" Error!\n".red); 226 | console.error(status.stderr.toString().italic); 227 | process.exit(); 228 | } 229 | 230 | process.stdout.write("\n"); 231 | 232 | for(var i = 0 ; i < tasks.length ; ++i) { 233 | var task = tasks[i]; 234 | var mpct_parse_error = false; 235 | var reported_metrics = args['metrics'].slice(0, args['metrics'].length); 236 | 237 | if(executed_tasks.indexOf(task.alias) === -1) { 238 | continue; 239 | } 240 | 241 | console.log(("Task: " + task.name).red.bold); 242 | console.log(task.description); 243 | console.log(""); 244 | 245 | var baseline = {MPI : 0, klyng: 0}; 246 | var i_baseline = {MPI: 0, klyng: 0}; 247 | 248 | for(var j = 0 ; j < pcounts.length; ++j) { 249 | 250 | var np = pcounts[j]; 251 | 252 | console.log(" " + "Running on %d process(es)".blue.underline, np); 253 | 254 | for(var k = 0 ; k < 2 ; k++) { 255 | var framework = frameworks[k]; 256 | 257 | if(framework === "MPI" && !runmpi) { 258 | console.log(" MPI:".bold + " Skipped"); 259 | continue; 260 | } 261 | 262 | var runner = (framework === "klyng") ? __dirname + "/../../bin/main.js" : mpirunner; 263 | var task_path = __dirname + "/tasks/" + task.alias; 264 | task_path += (framework === "klyng") ? "/klyng.js" : "/mpi.out"; 265 | 266 | var min = Infinity, i_min = Infinity; 267 | var max = -1, i_max = -1; 268 | var avg = 0, i_avg = 0; 269 | 270 | for(var itr = 0; itr < iterations; itr++) { 271 | report_progress(" " + (framework + ":").bold.underline + " Iteration", itr + 1, iterations); 272 | 273 | var output; 274 | var start = process.hrtime(); 275 | if(framework === "MPI") { 276 | output = spawn(runner, ['-n', np, task_path]); 277 | } 278 | else { 279 | output = spawn('node', [runner, '-n', np, task_path]); 280 | } 281 | var diff = process.hrtime(start); 282 | 283 | var duration = diff[0] + diff[1] * 1e-9; 284 | min = (duration <= min) ? duration : min; 285 | max = (duration >= max) ? duration : max; 286 | avg += duration; 287 | 288 | var i_duration = getmaxcputime(output.stdout.toString(), np); 289 | if(i_duration === -1) { 290 | var mpct_index = reported_metrics.indexOf('mpct'); 291 | if(mpct_index !== -1) { 292 | mpct_parse_error = true; 293 | reported_metrics.splice(mpct_index, 1); 294 | } 295 | } 296 | i_min = (i_duration <= i_min) ? i_duration : i_min; 297 | i_max = (i_duration >= i_max) ? i_duration : i_max; 298 | i_avg += i_duration; 299 | } 300 | 301 | avg /= iterations; 302 | i_avg /= iterations; 303 | 304 | var report = "min = " + min.toFixed(3) + "s, max = " + max.toFixed(3) + "s, avg = " + avg.toFixed(3) + "s"; 305 | var i_report = "min = " + i_min.toFixed(3) + "s, max = " + i_max.toFixed(3) + "s, avg = " + i_avg.toFixed(3) + "s"; 306 | 307 | if(np === pcounts[0]) { 308 | baseline[framework] = avg; 309 | i_baseline[framework] = i_avg; 310 | } 311 | else { 312 | var speedup = (baseline[framework] - avg) * 100 / baseline[framework]; 313 | var i_speedup = (i_baseline[framework] - i_avg) * 100 / i_baseline[framework]; 314 | 315 | var speedup_str = speedup.toFixed(3) + "%"; 316 | var i_speedup_str = i_speedup.toFixed(3) + "%"; 317 | 318 | speedup_str = (speedup > 0) ? ("+" + speedup_str).green : (speedup_str).red; 319 | i_speedup_str = (i_speedup > 0) ? ("+" + i_speedup_str).green : (i_speedup_str).red; 320 | 321 | report += " (" + speedup_str + ")"; 322 | i_report += " (" + i_speedup_str + ")"; 323 | } 324 | 325 | var compined_report = (reported_metrics.indexOf('rtet') !== -1) ? "\n RTET: ".gray.bold + report : ""; 326 | compined_report += (reported_metrics.indexOf('mpct') !== -1) ? "\n MPCT: ".green.bold + i_report: ""; 327 | 328 | process.stdout.clearLine(); 329 | process.stdout.cursorTo(0); 330 | console.log(" " + (framework + ":").bold.underline + compined_report); 331 | } 332 | } 333 | 334 | if(mpct_parse_error) { 335 | console.error("\nMPCT Report Skipped due to Error in Parsing CPU times.\n".bold.red); 336 | } 337 | else { 338 | console.log(""); 339 | } 340 | } 341 | -------------------------------------------------------------------------------- /tests/benchmarks/tasks.js: -------------------------------------------------------------------------------- 1 | var tasks = [ 2 | { 3 | name: "Pi Approximation", 4 | alias: "pi", 5 | description: "Approximates pi using arctan integral formula (dx = 2e-9)" 6 | }, 7 | 8 | { 9 | name: "Count Primes", 10 | alias: "primes", 11 | description: "Counts the number of prime integers between 1 and 10e6" 12 | } 13 | ]; 14 | 15 | module.exports = tasks; 16 | -------------------------------------------------------------------------------- /tests/benchmarks/tasks/pi/klyng.js: -------------------------------------------------------------------------------- 1 | var klyng = require('./../../../../index'); 2 | var getrusage = require('getrusage'); 3 | 4 | function approx_pi(from, to) { 5 | var pi = 0; 6 | var dx = 0.000000002; 7 | for(var x = from; x < to; x += dx) { 8 | pi += 4 / (1 + x * x); 9 | } 10 | 11 | return pi * dx; 12 | } 13 | 14 | function main() { 15 | 16 | var size = klyng.size(); 17 | var rank = klyng.rank(); 18 | 19 | if(rank === 0) { 20 | 21 | var interval_size = 1 / size; 22 | 23 | // distribute the range over the other processes 24 | for(var proc = 1; proc < size; proc++) { 25 | var range = [proc * interval_size, (proc + 1) * interval_size]; 26 | klyng.send({to: proc, data: range}); 27 | } 28 | 29 | // process own's range 30 | var local_pi = approx_pi(0, interval_size); 31 | 32 | // wait for the others 33 | for(var proc = 1; proc < size; proc++) { 34 | var other_pi = klyng.recv(); 35 | local_pi += other_pi; 36 | } 37 | } 38 | else { 39 | var range = klyng.recv({from: 0}); 40 | var local_pi = approx_pi(range[0], range[1]); 41 | klyng.send({to: 0, data: local_pi}); 42 | } 43 | 44 | console.log("cputime:%s", getrusage.getcputime().toFixed(3)); 45 | 46 | klyng.end(); 47 | } 48 | 49 | klyng.init(main); 50 | -------------------------------------------------------------------------------- /tests/benchmarks/tasks/pi/mpi.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "../../utilis/cputime.h" 5 | 6 | double approx_pi(double from, double to) { 7 | double pi = 0.0; 8 | double dx = 0.000000002; 9 | 10 | for(double x = from; x < to ; x += dx) { 11 | pi += 4.0 / (1 + x * x); 12 | } 13 | 14 | return pi * dx; 15 | } 16 | 17 | int main(int argc, char* argv[]) { 18 | 19 | int size, rank; 20 | MPI_Status status; 21 | 22 | MPI_Init(&argc, &argv); 23 | MPI_Comm_size(MPI_COMM_WORLD, &size); 24 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 25 | 26 | if(rank == 0) { 27 | 28 | double interval_size = 1.0 / size; 29 | for(int p = 1; p < size; ++p) { 30 | double range[2]; 31 | range[0] = p * interval_size; 32 | range[1] = (p + 1) * interval_size; 33 | MPI_Send(&range, 2, MPI_DOUBLE, p, 0, MPI_COMM_WORLD); 34 | } 35 | 36 | double local_pi = approx_pi(0, interval_size); 37 | 38 | for(int p = 1; p < size; ++p) { 39 | double other_pi = 0.0; 40 | MPI_Recv(&other_pi, 1, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); 41 | local_pi += other_pi; 42 | } 43 | 44 | printf("%.3f\n", local_pi); 45 | } 46 | else { 47 | double range[2]; 48 | MPI_Recv(&range, 2, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, &status); 49 | double local_pi = approx_pi(range[0], range[1]); 50 | MPI_Send(&local_pi, 1, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD); 51 | } 52 | 53 | printf("cputime:%.3f\n", getcputime()); 54 | 55 | MPI_Finalize(); 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /tests/benchmarks/tasks/primes/klyng.js: -------------------------------------------------------------------------------- 1 | var klyng = require('./../../../../index'); 2 | var getrusage = require('getrusage'); 3 | 4 | function isprime(number) { 5 | if(number === 2) { return true;} 6 | 7 | if(number % 2 === 0) { return false; } 8 | 9 | var k = 3; 10 | while(k * k <= number) { 11 | if(number % k === 0) { return false; } 12 | k += 2; 13 | } 14 | 15 | return true; 16 | } 17 | 18 | function main() { 19 | 20 | var size = klyng.size(); 21 | var rank = klyng.rank(); 22 | 23 | var max = 10000000; 24 | 25 | var counter = 0; 26 | var start = rank * (max / size); 27 | var end = (rank + 1) * (max/ size); 28 | 29 | for(var num = start; num < end; ++num) { 30 | if(isprime(num)) { ++counter; } 31 | } 32 | 33 | if(rank !== 0) { 34 | klyng.send({to: 0, data: counter}); 35 | } 36 | else { 37 | for(var p = 1; p < size; ++p) { 38 | var other_count = klyng.recv(); 39 | counter += other_count; 40 | } 41 | 42 | console.log(counter); 43 | } 44 | 45 | console.log("cputime:%s", getrusage.getcputime().toFixed(3)); 46 | 47 | klyng.end(); 48 | 49 | } 50 | 51 | klyng.init(main); 52 | -------------------------------------------------------------------------------- /tests/benchmarks/tasks/primes/mpi.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "../../utilis/cputime.h" 4 | 5 | bool isprime(int number) { 6 | if(number == 2) { return true;} 7 | 8 | if(number % 2 == 0) { return false; } 9 | 10 | int k = 3; 11 | while(k * k <= number) { 12 | if(number % k == 0) { return false; } 13 | k += 2; 14 | } 15 | 16 | return true; 17 | } 18 | 19 | int main(int argc, char* argv[]) { 20 | 21 | int size, rank; 22 | MPI_Status status; 23 | 24 | MPI_Init(&argc, &argv); 25 | MPI_Comm_size(MPI_COMM_WORLD, &size); 26 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 27 | 28 | int max = 10000000; 29 | 30 | int counter = 0; 31 | int start = rank * (max / size); 32 | int end = (rank + 1) * (max/ size); 33 | 34 | for(int num = start; num < end; ++num) { 35 | if(isprime(num)) { ++counter; } 36 | } 37 | 38 | if(rank != 0) { 39 | MPI_Send(&counter, 1, MPI_INT, 0, 0, MPI_COMM_WORLD); 40 | } 41 | else { 42 | for(int p = 1; p < size; ++p) { 43 | 44 | int other_count; 45 | MPI_Recv(&other_count, 1, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); 46 | counter += other_count; 47 | } 48 | 49 | printf("%d\n", counter); 50 | } 51 | 52 | printf("cputime:%.3f\n", getcputime()); 53 | 54 | MPI_Finalize(); 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /tests/benchmarks/utilis/cputime.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | double getcputime() { 5 | struct rusage r; 6 | 7 | getrusage(RUSAGE_SELF, &r); 8 | double cputime = (double)r.ru_utime.tv_sec + (double)r.ru_utime.tv_usec * 1e-6; 9 | cputime += (double)r.ru_stime.tv_sec + (double)r.ru_stime.tv_usec * 1e-6; 10 | 11 | return cputime; 12 | } 13 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/fake-job.js: -------------------------------------------------------------------------------- 1 | console.log("FAKE!"); 2 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/fake-tcp-server.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var crypto = require('crypto'); 3 | var cs = require('../../../lib/crypto-service'); 4 | 5 | ipc.config.silent = true; 6 | 7 | var password = "a1b2c3d4"; 8 | var secret = ""; 9 | 10 | ipc.serveNet('127.0.0.1', 4895, function() { 11 | 12 | ipc.server.on('PROBE:MSG', function(data, socket) { 13 | ipc.server.emit(socket, "ALIVE:MSG", {}); 14 | }); 15 | 16 | ipc.server.on('MONITOR:MSG', function(msg, socket) { 17 | console.log(msg.data.line); 18 | socket.destroy(); 19 | ipc.server.stop(); 20 | }); 21 | 22 | ipc.server.on('KLYNG:MSG', function(msg, socket) { 23 | console.log(msg.data); 24 | socket.destroy(); 25 | ipc.server.stop(); 26 | }); 27 | 28 | ipc.server.on('KEY-EXT:PARAMS', function(data, socket) { 29 | var dhObj = cs.diffieHellman(data.prime); 30 | var publicKey = dhObj.publicKey; 31 | 32 | var sharedSecret = dhObj.computeSecret(data.key); 33 | secret = sharedSecret; 34 | 35 | console.log(sharedSecret); 36 | 37 | ipc.server.emit(socket, 'KEY-EXT:PUBLIC', {key: publicKey}); 38 | }); 39 | 40 | ipc.server.on('AUTH', function(data, socket) { 41 | var decrypted = cs.verify(data, secret); 42 | if(decrypted.data === password) 43 | ipc.server.emit(socket, 'AUTH:STATUS', {status: true}); 44 | else 45 | ipc.server.emit(socket, 'AUTH:STATUS', {status: false, error:"incorrect password"}); 46 | }); 47 | 48 | ipc.server.on('KLYNG:JOB', function(data, socket) { 49 | var decrypted = cs.verify(data, secret); 50 | var ack = {status: false, error: "REMOTE: BAD JOB"}; 51 | if(!!decrypted) { 52 | var correctId = (decrypted.data.id === 1); 53 | var correctData = (decrypted.data.pckg === "packed.app"); 54 | var correctSize = (decrypted.data.size === 11); 55 | var correctParent = (decrypted.data.plan.parent.count === 5); 56 | var correctParentPort = (decrypted.data.plan.parent.port === 7777); 57 | var correctLocal = (decrypted.data.plan.local.count === 4); 58 | var correctOther = (decrypted.data.plan["127.0.0.2:2222"].count === 2); 59 | 60 | ack.status = correctId && correctData && correctSize && correctParent && correctLocal && correctOther && correctParentPort; 61 | if(!ack.status) { 62 | ack.error = "REMOTE: ASSERTION FAILED"; 63 | } 64 | else { 65 | delete ack["error"]; 66 | } 67 | } 68 | 69 | ipc.server.emit(socket, 'JOB:ACK', ack); 70 | }) 71 | 72 | ipc.server.on('SIGNAL:DONE', function(data, socket) { 73 | ipc.server.emit(socket, 'DONE:ACK', {status: true}); 74 | }) 75 | }); 76 | 77 | ipc.server.start(); 78 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/fake_app/local-dep.js: -------------------------------------------------------------------------------- 1 | console.log("Local Dep"); 2 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/fake_app/main.js: -------------------------------------------------------------------------------- 1 | var fiber = require('fibers'); 2 | var localDep = require('./local-dep') 3 | 4 | console.log(!!fiber.yield ? "endency": "?!"); 5 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/fake_functional_app/main.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../../index'); 2 | 3 | function main() { 4 | 5 | klyng.send({to: 0, data: "Weee!"}); 6 | 7 | klyng.end(); 8 | } 9 | 10 | klyng.init(main); 11 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/router-ipc-client.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | 3 | ipc.config.silent = true; 4 | 5 | ipc.connectTo('FAKE', function() { 6 | 7 | // emit a message for the server to recoginze the socket 8 | ipc.of.FAKE.emit('SOCKET:PUB', {}); 9 | 10 | ipc.of.FAKE.on('MONITOR:MSG', function(msg) { 11 | if(!!msg.data && !!msg.data.line) 12 | console.log(msg.data.line); 13 | if(msg.type === "process:exit") 14 | ipc.disconnect('FAKE'); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/router-local-process.js: -------------------------------------------------------------------------------- 1 | process.on('message', function(msg) { 2 | console.log(msg.data); 3 | process.exit(0); 4 | }); 5 | 6 | process.stdin.resume(); 7 | -------------------------------------------------------------------------------- /tests/fixtures/beacon/runner-fake-job.js: -------------------------------------------------------------------------------- 1 | process.send({type: "klyng:msg", header: {from: 1, to: 0}, data:"Fake Hello"}); 2 | 3 | console.log("Hello from Fake"); 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.invalid-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "192.168.0.4": "Hello" 3 | } 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.invalid-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "192.168.5.6": {"max_procs": true} 3 | } 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.invalid-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "192.169.58.10": {"port": "2222"} 3 | } 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.invalid-4.json: -------------------------------------------------------------------------------- 1 | { 2 | "192.68.75.85": {"passwd": 12345} 3 | } 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.invalid-5.json: -------------------------------------------------------------------------------- 1 | { 2 | "machine-10": {"max_procs": -8} 3 | } 4 | -------------------------------------------------------------------------------- /tests/fixtures/cli/fake-hosts.valid.json: -------------------------------------------------------------------------------- 1 | { 2 | "local" : {"max_procs": 2}, 3 | "192.168.0.10": {}, 4 | "192.168.0.2": {"port": 4578, "max_procs": 3}, 5 | "192.168.0.103": {"max_procs": 5, "passwd": "h3llo"}, 6 | "192.168.0.20": {"max_procs": 1, "port": 5986, "passwd": "oll3h"} 7 | } 8 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-init.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | klyng.end(); 5 | } 6 | 7 | klyng.init(main); 8 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-rank-size.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | console.log(klyng.rank() + ":" + klyng.size()); 5 | klyng.end(); 6 | } 7 | 8 | klyng.init(main); 9 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-recv-from.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | 5 | var msg = klyng.recv({from: 5}); 6 | 7 | console.log(msg); 8 | 9 | klyng.end(); 10 | } 11 | 12 | klyng.init(main); 13 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-recv-full.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | 5 | var msg = klyng.recv({from: 5, subject: "sub2"}); 6 | 7 | console.log(msg); 8 | 9 | klyng.end(); 10 | } 11 | 12 | klyng.init(main); 13 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-recv-no-criteria.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | 5 | var msg = klyng.recv(); 6 | 7 | console.log(msg); 8 | 9 | klyng.end(); 10 | } 11 | 12 | klyng.init(main); 13 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-recv-subject.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | 5 | var msg = klyng.recv({subject: "sub2"}); 6 | 7 | console.log(msg); 8 | 9 | klyng.end(); 10 | } 11 | 12 | klyng.init(main); 13 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-send-invalid-1.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | klyng.send({data: "Hello"}); 5 | klyng.end(); 6 | } 7 | 8 | klyng.init(main); 9 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-send-invalid-2.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | klyng.send({to: 1}); 5 | klyng.end(); 6 | } 7 | 8 | klyng.init(main); 9 | -------------------------------------------------------------------------------- /tests/fixtures/klyng-apis/sync-send.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index.js'); 2 | 3 | function main() { 4 | klyng.send({to: 1, data: "Hello"}); 5 | klyng.end(); 6 | } 7 | 8 | klyng.init(main); 9 | -------------------------------------------------------------------------------- /tests/integration/assets/bad.job.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index'); 2 | 3 | function main() { 4 | 5 | var size = klyng.size(); 6 | var rank = klyng.rank(); 7 | 8 | // send greeting messages to all other processes 9 | for(var p = 0; p < size; ++p) { 10 | if(p !== rank) { 11 | var greetings = "Greetings P" + p + " from P" + rank; 12 | klyng.send({to: p, data: greetings}); 13 | } 14 | } 15 | 16 | for(var p = 0; p < size - 1; ++p) { 17 | var othersGreetings = klyng.recv(); 18 | console.log(othersGreetings); 19 | } 20 | } 21 | 22 | klyng.init(main); 23 | -------------------------------------------------------------------------------- /tests/integration/assets/job.js: -------------------------------------------------------------------------------- 1 | var klyng = require('../../../index'); 2 | 3 | function main() { 4 | 5 | var size = klyng.size(); 6 | var rank = klyng.rank(); 7 | 8 | // send greeting messages to all other processes 9 | for(var p = 0; p < size; ++p) { 10 | if(p !== rank) { 11 | var greetings = "Greetings P" + p + " from P" + rank; 12 | klyng.send({to: p, data: greetings}); 13 | } 14 | } 15 | 16 | for(var p = 0; p < size - 1; ++p) { 17 | var othersGreetings = klyng.recv(); 18 | console.log(othersGreetings); 19 | } 20 | 21 | klyng.end(); 22 | } 23 | 24 | klyng.init(main); 25 | -------------------------------------------------------------------------------- /tests/integration/assets/machines.with.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "local": { 3 | "max_procs": 2 4 | }, 5 | 6 | "127.0.0.1": { 7 | "port": 8000, 8 | "passwd": "dummy", 9 | "max_procs": 2 10 | }, 11 | 12 | "localhost": { 13 | "port": 8001, 14 | "passwd": "dummy", 15 | "max_procs": 2 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/integration/assets/machines.with.wrong.pass.json: -------------------------------------------------------------------------------- 1 | { 2 | "local": { 3 | "max_procs": 2 4 | }, 5 | 6 | "127.0.0.1": { 7 | "port": 8000, 8 | "passwd": "", 9 | "max_procs": 2 10 | }, 11 | 12 | "localhost": { 13 | "port": 8001, 14 | "passwd": "dummy", 15 | "max_procs": 2 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/integration/assets/machines.without.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "127.0.0.1": { 3 | "port": 8000, 4 | "passwd": "dummy", 5 | "max_procs": 2 6 | }, 7 | 8 | "localhost": { 9 | "port": 8001, 10 | "passwd": "dummy", 11 | "max_procs": 2 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/integration/assets/tcp.only-beacon.js: -------------------------------------------------------------------------------- 1 | var tcp = require('../../../lib/tcp'); 2 | console.log(process.argv[2]); 3 | var customConfig = { 4 | port: parseInt(process.argv[2]), 5 | password: "dummy" 6 | }; 7 | 8 | tcp.start(customConfig); 9 | -------------------------------------------------------------------------------- /tests/integration/run.js: -------------------------------------------------------------------------------- 1 | var readline = require('readline'); 2 | var spawn = require('child_process').spawn; 3 | var os = require('os').platform(); 4 | var spawnSync = require('child_process').spawnSync; 5 | var expect = require('chai').expect; 6 | 7 | var startingPort = 8000; 8 | var tcpBeaconsPath = __dirname + "/assets/tcp.only-beacon.js"; 9 | var jobPath = __dirname + "/assets/job.js"; 10 | var badJobPath = __dirname + "/assets/bad.job.js"; 11 | var machinesWithLocalPath = __dirname + "/assets/machines.with.local.json"; 12 | var machinesWithoutLocalPath = __dirname + "/assets/machines.without.local.json"; 13 | var machinesWithWrongPass = __dirname + "/assets/machines.with.wrong.pass.json"; 14 | var runnerPath = __dirname + "/../../bin/main.js"; 15 | var beaconPath = __dirname + "/../../lib/beacon.js"; 16 | var tcpBeacons = []; 17 | var localBeacon; 18 | 19 | before(function() { 20 | 21 | this.timeout(10000); 22 | 23 | process.stdout.write(' Running Remote Beacons ...'); 24 | 25 | for(var i = 0; i < 2; ++i) { 26 | var semiRemoteBeacon = spawn('node', [tcpBeaconsPath, startingPort + i]); 27 | tcpBeacons.push(semiRemoteBeacon); 28 | } 29 | 30 | process.stdout.write(' Done!\n'); 31 | process.stdout.write(' Restarting Local Beacon ...'); 32 | 33 | spawnSync('node', [runnerPath, '-d']); 34 | localBeacon = spawn('node', [beaconPath]); 35 | 36 | process.stdout.write(' Done!\n\n'); 37 | }); 38 | 39 | after(function() { 40 | 41 | this.timeout(10000); 42 | 43 | process.stdout.write(' Stopping Remote Beacons ...'); 44 | 45 | tcpBeacons.forEach((beacon) => beacon.kill()); 46 | 47 | process.stdout.write(' Done!\n'); 48 | process.stdout.write(' Stopping Local Beacon ...'); 49 | 50 | //spawnSync('node', [runnerPath, '-d']); 51 | localBeacon.kill(); 52 | 53 | process.stdout.write(' Done!\n\n'); 54 | }); 55 | 56 | describe("Klyng's Integartion tests", function() { 57 | 58 | this.timeout(15000); 59 | 60 | it('runs a job of size 6 on local and 2 remote hosts', function(done) { 61 | 62 | var parent = spawn('node', [runnerPath, '-n', 6, '-m', machinesWithLocalPath, jobPath]); 63 | var parentStdout = []; 64 | 65 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 66 | 67 | stdoutRlInterface.on('line', (line) => { 68 | parentStdout.push(line); 69 | }); 70 | 71 | parent.stderr.on('data', (chunk) => { 72 | console.log(chunk.toString().trim()); 73 | }); 74 | 75 | parent.on('exit', () => { 76 | for(var i = 0; i < 6; ++i) { 77 | for(var j = 0; j < 6; j++) { 78 | if(i !== j) { 79 | var msg = "Greetings P" + i + " from P" + j; 80 | expect(parentStdout).to.include(msg); 81 | } 82 | } 83 | } 84 | done(); 85 | }); 86 | }); 87 | 88 | it('runs a job of size 6 on 2 remote hosts', function(done) { 89 | 90 | var parent = spawn('node', [runnerPath, '-n', 6, '-m', machinesWithoutLocalPath, jobPath]); 91 | var parentStdout = []; 92 | 93 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 94 | 95 | stdoutRlInterface.on('line', (line) => { 96 | parentStdout.push(line); 97 | }); 98 | 99 | parent.stderr.on('data', (chunk) => { 100 | console.log(chunk.toString().trim()); 101 | }); 102 | 103 | parent.on('exit', () => { 104 | for(var i = 0; i < 6; ++i) { 105 | for(var j = 0; j < 6; j++) { 106 | if(i !== j) { 107 | var msg = "Greetings P" + i + " from P" + j; 108 | expect(parentStdout).to.include(msg); 109 | } 110 | } 111 | } 112 | done(); 113 | }); 114 | }); 115 | 116 | it('aborts when an error occurs during the job', function(done) { 117 | var parent = spawn('node', [runnerPath, '-n', 6, '-m', machinesWithWrongPass, jobPath]); 118 | var parentStderr = []; 119 | 120 | var stderrRlInterface = readline.createInterface({input: parent.stderr}); 121 | stderrRlInterface.on('line', (line) => { 122 | parentStderr.push(line); 123 | }); 124 | 125 | parent.on('exit', () => { 126 | expect(parentStderr).to.include('[Aborted]: 127.0.0.1:8000 incorrect password'); 127 | 128 | // wait for a second for the beacons to clear there data structures 129 | setTimeout(done, 1500); 130 | }); 131 | }); 132 | 133 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 134 | 135 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 136 | var parentStdout = []; 137 | 138 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 139 | 140 | stdoutRlInterface.on('line', (line) => { 141 | parentStdout.push(line); 142 | }); 143 | 144 | parent.stderr.on('data', (chunk) => { 145 | console.log(chunk.toString().trim()); 146 | }); 147 | 148 | parent.on('exit', () => { 149 | for(var i = 0; i < 3; ++i) { 150 | for(var j = 0; j < 3; j++) { 151 | if(i !== j) { 152 | var msg = "Greetings P" + i + " from P" + j; 153 | expect(parentStdout).to.include(msg); 154 | } 155 | } 156 | } 157 | done(); 158 | }); 159 | }); 160 | 161 | it('aborts on keyboard interrupt (SIGINT)', function(done) { 162 | 163 | var spawnOptions = (os === "win32") ? {stdio: [null, null, null, 'ipc']} : {}; 164 | 165 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, badJobPath], spawnOptions); 166 | var parentStdout = []; 167 | var parentStderr = []; 168 | 169 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 170 | var stderrRlInterface = readline.createInterface({input: parent.stderr}); 171 | 172 | stderrRlInterface.on('line', (line) => { 173 | parentStderr.push(line); 174 | }); 175 | 176 | stdoutRlInterface.on('line', (line) => { 177 | parentStdout.push(line); 178 | }); 179 | 180 | 181 | setTimeout(() => os === "win32" ? parent.send('SIGINT') : parent.kill('SIGINT'), 5000); 182 | 183 | parent.on('exit', () => { 184 | for(var i = 0; i < 3; ++i) { 185 | for(var j = 0; j < 3; j++) { 186 | if(i !== j) { 187 | var msg = "Greetings P" + i + " from P" + j; 188 | expect(parentStdout).to.include(msg); 189 | } 190 | } 191 | } 192 | 193 | expect(parentStderr).to.include('[Aborted]: Keyboard Interrupt'); 194 | 195 | // wait for a second for the beacons to clear there data structures 196 | setTimeout(done, 1500); 197 | }); 198 | }); 199 | 200 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 201 | 202 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 203 | var parentStdout = []; 204 | 205 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 206 | 207 | stdoutRlInterface.on('line', (line) => { 208 | parentStdout.push(line); 209 | }); 210 | 211 | parent.stderr.on('data', (chunk) => { 212 | console.log(chunk.toString().trim()); 213 | }); 214 | 215 | parent.on('exit', () => { 216 | for(var i = 0; i < 3; ++i) { 217 | for(var j = 0; j < 3; j++) { 218 | if(i !== j) { 219 | var msg = "Greetings P" + i + " from P" + j; 220 | expect(parentStdout).to.include(msg); 221 | } 222 | } 223 | } 224 | done(); 225 | }); 226 | }); 227 | 228 | it('aborts when one of the remote node is not up', function(done) { 229 | // kill node at :8001 230 | tcpBeacons[1].kill(); 231 | 232 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 233 | var parentStderr = []; 234 | 235 | var stderrRlInterface = readline.createInterface({input: parent.stderr}); 236 | stderrRlInterface.on('line', (line) => { 237 | parentStderr.push(line); 238 | }); 239 | 240 | parent.on('exit', () => { 241 | expect(parentStderr).to.include('[Aborted]: connect ECONNREFUSED 127.0.0.1:8001'); 242 | 243 | // wait for a second for the beacons to clear there data structures 244 | setTimeout(done, 1500); 245 | }); 246 | }); 247 | 248 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 249 | 250 | tcpBeacons[1] = spawn('node', [tcpBeaconsPath, 8001]); 251 | 252 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 253 | var parentStdout = []; 254 | 255 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 256 | 257 | stdoutRlInterface.on('line', (line) => { 258 | parentStdout.push(line); 259 | }); 260 | 261 | parent.stderr.on('data', (chunk) => { 262 | console.log(chunk.toString().trim()); 263 | }); 264 | 265 | parent.on('exit', () => { 266 | for(var i = 0; i < 3; ++i) { 267 | for(var j = 0; j < 3; j++) { 268 | if(i !== j) { 269 | var msg = "Greetings P" + i + " from P" + j; 270 | expect(parentStdout).to.include(msg); 271 | } 272 | } 273 | } 274 | done(); 275 | }); 276 | }); 277 | 278 | it('aborts when one of the remote node fails during the job', function(done) { 279 | 280 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, badJobPath]); 281 | var parentStdout = []; 282 | var parentStderr = []; 283 | 284 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 285 | var stderrRlInterface = readline.createInterface({input: parent.stderr}); 286 | 287 | stderrRlInterface.on('line', (line) => { 288 | parentStderr.push(line); 289 | }); 290 | 291 | stdoutRlInterface.on('line', (line) => { 292 | parentStdout.push(line); 293 | }); 294 | 295 | setTimeout(() => tcpBeacons[1].kill(), 5000); 296 | 297 | parent.on('exit', () => { 298 | 299 | for(var i = 0; i < 3; ++i) { 300 | for(var j = 0; j < 3; j++) { 301 | if(i !== j) { 302 | var msg = "Greetings P" + i + " from P" + j; 303 | expect(parentStdout).to.include(msg); 304 | } 305 | } 306 | } 307 | 308 | expect(parentStderr.length).to.be.above(0); 309 | expect(parentStderr[0]).to.have.string('connect ECONNREFUSED 127.0.0.1:8001'); 310 | 311 | // wait for a second for the beacons to clear there data structures 312 | setTimeout(done, 1500); 313 | }); 314 | }); 315 | 316 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 317 | 318 | tcpBeacons[1] = spawn('node', [tcpBeaconsPath, 8001]); 319 | 320 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 321 | var parentStdout = []; 322 | 323 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 324 | 325 | stdoutRlInterface.on('line', (line) => { 326 | parentStdout.push(line); 327 | }); 328 | 329 | parent.stderr.on('data', (chunk) => { 330 | console.log(chunk.toString().trim()); 331 | }); 332 | 333 | parent.on('exit', () => { 334 | for(var i = 0; i < 3; ++i) { 335 | for(var j = 0; j < 3; j++) { 336 | if(i !== j) { 337 | var msg = "Greetings P" + i + " from P" + j; 338 | expect(parentStdout).to.include(msg); 339 | } 340 | } 341 | } 342 | done(); 343 | }); 344 | }); 345 | 346 | it('aborts when the local beacon fails during the job', function(done) { 347 | 348 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, badJobPath]); 349 | var parentStdout = []; 350 | var parentStderr = []; 351 | 352 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 353 | var stderrRlInterface = readline.createInterface({input: parent.stderr}); 354 | 355 | stderrRlInterface.on('line', (line) => { 356 | parentStderr.push(line); 357 | }); 358 | 359 | stdoutRlInterface.on('line', (line) => { 360 | parentStdout.push(line); 361 | }); 362 | 363 | setTimeout(() => localBeacon.kill(), 5000); 364 | 365 | parent.on('exit', () => { 366 | 367 | for(var i = 0; i < 3; ++i) { 368 | for(var j = 0; j < 3; j++) { 369 | if(i !== j) { 370 | var msg = "Greetings P" + i + " from P" + j; 371 | expect(parentStdout).to.include(msg); 372 | } 373 | } 374 | } 375 | 376 | expect(parentStderr).to.include('[Aborted]: Lost connection to local beacon'); 377 | 378 | // wait for a second for the beacons to clear there data structures 379 | setTimeout(done, 3000); 380 | }); 381 | }); 382 | 383 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 384 | 385 | localBeacon = spawn('node', [beaconPath]); 386 | 387 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 388 | var parentStdout = []; 389 | 390 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 391 | 392 | stdoutRlInterface.on('line', (line) => { 393 | parentStdout.push(line); 394 | }); 395 | 396 | parent.stderr.on('data', (chunk) => { 397 | console.log(chunk.toString().trim()); 398 | }); 399 | 400 | parent.on('exit', () => { 401 | for(var i = 0; i < 3; ++i) { 402 | for(var j = 0; j < 3; j++) { 403 | if(i !== j) { 404 | var msg = "Greetings P" + i + " from P" + j; 405 | expect(parentStdout).to.include(msg); 406 | } 407 | } 408 | } 409 | done(); 410 | }); 411 | }); 412 | 413 | it('aborts when the runner closes abruptly during a job', function(done) { 414 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, badJobPath]); 415 | var parentStdout = []; 416 | 417 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 418 | 419 | stdoutRlInterface.on('line', (line) => { 420 | parentStdout.push(line); 421 | }); 422 | 423 | parent.stderr.on('data', (chunk) => { 424 | console.log(chunk.toString().trim()); 425 | }); 426 | 427 | setTimeout(() => parent.kill(), 5000); 428 | 429 | parent.on('exit', () => { 430 | for(var i = 0; i < 3; ++i) { 431 | for(var j = 0; j < 3; j++) { 432 | if(i !== j) { 433 | var msg = "Greetings P" + i + " from P" + j; 434 | expect(parentStdout).to.include(msg); 435 | } 436 | } 437 | } 438 | setTimeout(done, 3000); 439 | }); 440 | }); 441 | 442 | it('ensures that the beacons aborted gracefully (by running a job)', function(done) { 443 | 444 | var parent = spawn('node', [runnerPath, '-n', 3, '-m', machinesWithLocalPath, jobPath]); 445 | var parentStdout = []; 446 | 447 | var stdoutRlInterface = readline.createInterface({input: parent.stdout}); 448 | 449 | stdoutRlInterface.on('line', (line) => { 450 | parentStdout.push(line); 451 | }); 452 | 453 | parent.stderr.on('data', (chunk) => { 454 | console.log(chunk.toString().trim()); 455 | }); 456 | 457 | parent.on('exit', () => { 458 | for(var i = 0; i < 3; ++i) { 459 | for(var j = 0; j < 3; j++) { 460 | if(i !== j) { 461 | var msg = "Greetings P" + i + " from P" + j; 462 | expect(parentStdout).to.include(msg); 463 | } 464 | } 465 | } 466 | done(); 467 | }); 468 | }); 469 | }); 470 | -------------------------------------------------------------------------------- /tests/specs/beacon/beacon-process.specs.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var spawn = require('child_process').spawn; 3 | var expect = require('chai').expect; 4 | 5 | ipc.config.silent = true; 6 | 7 | describe('Beacon\'s Process', function() { 8 | 9 | this.timeout(6000); 10 | 11 | var beacon_process = spawn('node', ['./lib/beacon.js']); 12 | var beacon_process_exit = new Promise(function(resolve, reject) { 13 | beacon_process.on('exit', function() {resolve();}); 14 | }); 15 | 16 | afterEach(function() { ipc.disconnect('klyng_beacon'); }); 17 | after(function() { beacon_process.kill(); }); 18 | 19 | it('responds with available to \'SIGNAL:RUN\' message', function(done) { 20 | ipc.connectTo('klyng_beacon', function() { 21 | ipc.of.klyng_beacon.on('connect', function() { 22 | ipc.of.klyng_beacon.emit('SIGNAL:RUN', {job_specs: {size: 1, app: './tests/fixtures/beacon/fake-job.js'}}); 23 | ipc.of.klyng_beacon.on('SIGNAL:CONFIRM', function(msg) { 24 | expect(msg.status).to.equal('available'); 25 | done(); 26 | }); 27 | }) 28 | }); 29 | }); 30 | 31 | it('responds with busy to second \'SIGNAL:RUN\' message', function(done) { 32 | ipc.connectTo('klyng_beacon', function() { 33 | ipc.of.klyng_beacon.on('connect', function() { 34 | ipc.of.klyng_beacon.emit('SIGNAL:RUN', {job_specs: {size: 1, app: './tests/fixtures/beacon/runner-fake-job.js'}}); 35 | ipc.of.klyng_beacon.on('SIGNAL:CONFIRM', function(msg) { 36 | expect(msg.status).to.equal('busy'); 37 | done(); 38 | }); 39 | }) 40 | }); 41 | }); 42 | 43 | it('responds to a \'SIGNAL:DONE\' message', function(done) { 44 | ipc.connectTo('klyng_beacon', function() { 45 | ipc.of.klyng_beacon.on('connect', function() { 46 | ipc.of.klyng_beacon.emit('SIGNAL:DONE', {}); 47 | ipc.of.klyng_beacon.socket.on('end', function() { 48 | done(); 49 | }); 50 | }); 51 | }); 52 | }); 53 | 54 | it('responds to a \'STOP:MSG\'', function(done) { 55 | ipc.connectTo('klyng_beacon', function() { 56 | ipc.of.klyng_beacon.on('connect', function() { 57 | ipc.of.klyng_beacon.emit('STOP:MSG', {}); 58 | var socket_close = new Promise(function(resolve, reject) { 59 | ipc.of.klyng_beacon.socket.on('end', function() { 60 | resolve(); 61 | }); 62 | }); 63 | 64 | Promise.all([socket_close, beacon_process_exit]).then(function(){ 65 | done(); 66 | }); 67 | }); 68 | }); 69 | }); 70 | 71 | }); 72 | -------------------------------------------------------------------------------- /tests/specs/beacon/beacon-remote.specs.js: -------------------------------------------------------------------------------- 1 | var tcp = require('../../../lib/tcp'); 2 | var cs = require('../../../lib/crypto-service'); 3 | var jobman = require('../../../lib/job-manager'); 4 | var router = require('../../../lib/router'); 5 | var expect = require('chai').expect; 6 | var spawn = require('child_process').spawn; 7 | var ipc = require('node-ipc'); 8 | 9 | var configs = require('../../../lib/beacon-configs'); 10 | 11 | var fake_server; 12 | 13 | configs.configureRemoteIPC(ipc); 14 | 15 | describe("Beacon Remote Communincation", function() { 16 | 17 | this.timeout(7000); 18 | 19 | before(function() {tcp.start({port: 7777, password: "dummy"});}); 20 | after(function() { 21 | ipc.disconnect('auth_socket'); 22 | ipc.disconnect('nauth_socket'); 23 | tcp.stop(); 24 | }); 25 | 26 | beforeEach(function() { 27 | if(!!ipc.of.auth_socket) { 28 | ipc.of.auth_socket.off('AUTH:STATUS', '*'); 29 | } 30 | 31 | if(!!ipc.of.nauth_socket) { 32 | ipc.of.nauth_socket.off('AUTH:STATUS', '*'); 33 | } 34 | }); 35 | 36 | afterEach(function(done) { 37 | tcp.disconnectFrom('127.0.0.1', 4895); 38 | fake_server.kill(); 39 | setTimeout(done, 500); 40 | }); 41 | 42 | it('connects/disconnects to/from a running tcp server', function(done) { 43 | 44 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 45 | 46 | tcp.connectTo('127.0.0.1', 4895) 47 | .then(function(connection) { 48 | expect(!!connection).to.equal(true); 49 | expect(connection.socket.destroyed).to.equal(false); 50 | tcp.disconnectFrom('127.0.0.1', 4895); 51 | expect(connection.socket.destroyed).to.equal(true); 52 | 53 | done(); 54 | }) 55 | .catch(done); 56 | }); 57 | 58 | it('fails to connect to non-existing tcp server', function(done) { 59 | 60 | tcp.connectTo('127.0.0.1', 4895) 61 | .then(function(connection) { 62 | expect(!!connection).to.equal(true); 63 | done(new Error("This should never happen")); 64 | }) 65 | .catch(function(err) { 66 | done(); 67 | }); 68 | }); 69 | 70 | it('exchanges a shared secret key with tcp server', function(done) { 71 | 72 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 73 | var fake_server_stdout = ""; 74 | fake_server.stdout.on('data', function(chunck) { fake_server_stdout += chunck.toString().trim(); }); 75 | 76 | tcp.connectTo('127.0.0.1', 4895) 77 | .then(function(connection) { 78 | return tcp.exchangeKeyOver(connection); 79 | }) 80 | .then(function(params) { 81 | expect(params.secret).to.equal(fake_server_stdout); 82 | done(); 83 | }) 84 | .catch(done); 85 | }); 86 | 87 | it('authorizes access to remote address with correct password', function(done) { 88 | 89 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 90 | 91 | var con = null; 92 | 93 | tcp.connectTo('127.0.0.1', 4895) 94 | .then(function(connection) { 95 | con = connection; 96 | return tcp.exchangeKeyOver(connection); 97 | }) 98 | .then(function(params) { 99 | return tcp.authOver(params.connection, params.secret, 'a1b2c3d4'); 100 | }) 101 | .then(function(params) { 102 | done(); 103 | }) 104 | .catch(done); 105 | }); 106 | 107 | it('fails to authorize access to remote address due to wrong password', function(done) { 108 | 109 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 110 | 111 | var con = null; 112 | 113 | tcp.connectTo('127.0.0.1', 4895) 114 | .then(function(connection) { 115 | con = connection; 116 | return tcp.exchangeKeyOver(connection); 117 | }) 118 | .then(function(params) { 119 | return tcp.authOver(params.connection, params.secret, '12345678'); 120 | }) 121 | .then(function(params) { 122 | done(new Error("This should never happen")); 123 | }) 124 | .catch(function(err) { 125 | expect(err.message).to.equal("127.0.0.1:4895 incorrect password"); 126 | done(); 127 | }); 128 | }); 129 | 130 | it('sends a job to a remote beacon', function(done) { 131 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 132 | 133 | tcp.connectTo('127.0.0.1', 4895) 134 | .then(function(connection) { 135 | return tcp.exchangeKeyOver(connection); 136 | }) 137 | .then(function(params) { 138 | return tcp.authOver(params.connection, params.secret, 'a1b2c3d4'); 139 | }) 140 | .then(function(params) { 141 | var job = { 142 | id: 1, 143 | pckg: 'packed.app', 144 | size: 11, 145 | }; 146 | var plan = { 147 | "local": {count: 5, start: 0}, 148 | "127.0.0.1:4895": {count: 4, start: 5}, 149 | "127.0.0.2:2222": {count: 2, start: 9} 150 | }; 151 | 152 | return tcp.sendJobOver(params.connection, params.secret, job, plan); 153 | }) 154 | .then(function(sent) { 155 | if(sent) { 156 | done(); 157 | } 158 | }) 159 | .catch(done); 160 | }); 161 | 162 | it('sends a DONE signal to a remote beacon', function(done) { 163 | fake_server= spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 164 | 165 | tcp.connectTo('127.0.0.1', 4895) 166 | .then(function(connection) { 167 | return tcp.exchangeKeyOver(connection); 168 | }) 169 | .then(function(params) { 170 | return tcp.authOver(params.connection, params.secret, 'a1b2c3d4'); 171 | }) 172 | .then(function(params) { 173 | return tcp.signalDoneOver(params.connection); 174 | }) 175 | .then(function() { 176 | done(); 177 | }) 178 | .catch(done); 179 | }) 180 | 181 | it('responds to KEY-EXT:PARAMS and creates a shared secret', function(done) { 182 | ipc.connectToNet('auth_socket', '127.0.0.1', 7777, function() { 183 | var dhObj = cs.diffieHellman(); 184 | ipc.of.auth_socket.emit('KEY-EXT:PARAMS', { 185 | prime: dhObj.prime, 186 | key: dhObj.publicKey 187 | }); 188 | 189 | ipc.of.auth_socket.on('KEY-EXT:PUBLIC', function(data) { 190 | ipc.of.auth_socket.klyng_secret = dhObj.computeSecret(data.key); 191 | var welcomeMsg = cs.verify(data.cipherWelcome, ipc.of.auth_socket.klyng_secret); 192 | 193 | expect(welcomeMsg).to.equal("Hello from Beacon's TCP Server!"); 194 | done(); 195 | }) 196 | }); 197 | }); 198 | 199 | it('responds to AUTH message with an incorrect password', function(done) { 200 | var secret = ipc.of.auth_socket.klyng_secret; 201 | ipc.of.auth_socket.emit('AUTH', cs.secure({data: "12345"}, secret)); 202 | ipc.of.auth_socket.on('AUTH:STATUS', function(data) { 203 | expect(data.status).to.be.false; 204 | expect(data.error).to.equal("incorrect password"); 205 | done(); 206 | }); 207 | }); 208 | 209 | it('responds to AUTH message with a correct password', function(done) { 210 | var secret = ipc.of.auth_socket.klyng_secret; 211 | ipc.of.auth_socket.emit('AUTH', cs.secure({data: "dummy"}, secret)); 212 | ipc.of.auth_socket.on('AUTH:STATUS', function(data) { 213 | expect(data.status).to.be.true; 214 | done(); 215 | }); 216 | }); 217 | 218 | it('responds to KLYNG:JOB message and runs the job', function(done) { 219 | var secret = ipc.of.auth_socket.klyng_secret; 220 | var job = { 221 | app: __dirname + '/../../fixtures/beacon/fake_functional_app/main.js', 222 | size: 2, 223 | plan: { 224 | "parent": {port: 9876, start: 0, count: 1}, 225 | "local": {start: 1, count: 1} 226 | } 227 | } 228 | 229 | var klyngMsgPromise = new Promise(function(resolve, reject) { 230 | ipc.serveNet('127.0.0.1', 9876, function() { 231 | ipc.server.on('KLYNG:MSG', function(msg, socket) { 232 | try { 233 | expect(msg.header.from).to.equal(1); 234 | expect(msg.header.to).to.equal(0); 235 | expect(msg.data).to.equal("Weee!"); 236 | resolve(); 237 | } 238 | catch(err) { reject(err); } 239 | }); 240 | }); 241 | ipc.server.start(); 242 | }); 243 | 244 | var jobAckPromise = jobman.pack(job) 245 | .then(function(app) { 246 | return new Promise(function(resolve, reject) { 247 | job.app = app; 248 | ipc.of.auth_socket.emit('KLYNG:JOB', cs.secure({data: job}, secret)); 249 | ipc.of.auth_socket.on('JOB:ACK', function(data) { 250 | if(!data.status) { 251 | reject(new Error(data.error)); 252 | } 253 | else { 254 | resolve(true); 255 | } 256 | }); 257 | }); 258 | }); 259 | 260 | Promise.all([klyngMsgPromise, jobAckPromise]) 261 | .then(function() { done(); }) 262 | .catch(done); 263 | }); 264 | 265 | it('responds to KLYNG:JOB indicating that the beacon is busy', function(done) { 266 | ipc.of.auth_socket.emit('KLYNG:JOB', {}); 267 | ipc.of.auth_socket.on('JOB:ACK', function(data) { 268 | expect(data.status).to.be.false; 269 | expect(data.error).to.equal("The Beacon is busy"); 270 | done(); 271 | }); 272 | }); 273 | 274 | it('responds to SIGNAL:DONE message', function(done) { 275 | ipc.of.auth_socket.emit('SIGNAL:DONE', {}); 276 | 277 | var disconnectPromise = new Promise(function(resolve, reject) { 278 | ipc.server.on('close', function(socket) { 279 | resolve(); 280 | }); 281 | }); 282 | 283 | var ackPromise = new Promise(function(resolve, reject) { 284 | ipc.of.auth_socket.on('DONE:ACK', function(data) { 285 | try { 286 | expect(data.status).to.be.true; 287 | } 288 | catch(err) { reject(err); } 289 | ipc.disconnect('auth_socket'); 290 | resolve(); 291 | }); 292 | }); 293 | 294 | Promise.all([disconnectPromise, ackPromise]) 295 | .then(function() { 296 | expect(router.isClean()).to.be.true; 297 | done(); 298 | }) 299 | .catch(done); 300 | }); 301 | 302 | it('refuses a SIGNAL:DONE message on an unothorized socket', function(done) { 303 | ipc.connectToNet('nauth_socket', '127.0.0.1', 7777, function() { 304 | ipc.of.nauth_socket.emit('SIGNAL:DONE'); 305 | ipc.of.nauth_socket.on('DONE:ACK', function(data) { 306 | expect(data.status).to.be.false; 307 | expect(data.error).to.equal("Unauthorized"); 308 | done(); 309 | }); 310 | }); 311 | }); 312 | 313 | it('refuses a KLYNG:JOB message on an unothorized socket', function(done) { 314 | ipc.of.nauth_socket.emit('KLYNG:JOB', {}); 315 | ipc.of.nauth_socket.on('JOB:ACK', function(data) { 316 | expect(data.status).to.be.false; 317 | expect(data.error).to.equal("Unauthorized"); 318 | done(); 319 | }); 320 | }); 321 | 322 | it('refuses an AUTH attempt on an unsecure socket', function(done) { 323 | ipc.of.nauth_socket.emit('AUTH', {}); 324 | ipc.of.nauth_socket.on('AUTH:STATUS', function(data) { 325 | expect(data.status).to.be.false; 326 | expect(data.error).to.equal("unsecure channel"); 327 | done(); 328 | }); 329 | }); 330 | }); 331 | -------------------------------------------------------------------------------- /tests/specs/beacon/controller.specs.js: -------------------------------------------------------------------------------- 1 | var controller = require('../../../lib/beacon-controller.js'); 2 | var expect = require('chai').expect; 3 | 4 | describe('Beacon\'s Controller', function() { 5 | 6 | this.timeout(5000); 7 | 8 | it('checks and finds the beacon not running', function(done) { 9 | controller.checkIfRunning() 10 | .then(function(running) { 11 | expect(running).to.equal(false); 12 | done(); 13 | }); 14 | }); 15 | 16 | it('starts the beacon process', function(done) { 17 | controller.start() 18 | .then(function(started) { 19 | expect(started).to.equal(true); 20 | done(); 21 | }); 22 | }); 23 | 24 | it('signals the beacon to run a job', function(done) { 25 | controller.signalToRun({size:1, app: './tests/fixtures/beacon/runner-fake-job.js'}) 26 | .then(function(status) { 27 | expect(!!status).to.equal(true); 28 | done(); 29 | }); 30 | }); 31 | 32 | it('signals the beacon that it\'s done with a job', function(done) { 33 | controller.signalDone() 34 | .then(function(signaled) { 35 | expect(signaled).to.equal(true); 36 | done(); 37 | }); 38 | }); 39 | 40 | it('stops the beacon process', function(done) { 41 | controller.checkIfRunning() // to re-open the socket closed in the last test case 42 | .then(function() { 43 | return controller.stop(); 44 | }) 45 | .then(function(stopped) { 46 | expect(stopped).to.equal(true); 47 | done(); 48 | }); 49 | }); 50 | 51 | }); 52 | -------------------------------------------------------------------------------- /tests/specs/beacon/job-manager.specs.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var router = require('../../../lib/router.js'); 3 | var jobman = require('../../../lib/job-manager.js'); 4 | var expect = require('chai').expect; 5 | var spawn = require('child_process').spawn; 6 | var fs = require('fs'); 7 | var path = require('path'); 8 | var zipper = require('zip-local'); 9 | var rmrf = require('rimraf'); 10 | 11 | // FAKE: ipc socket server 12 | ipc.config.silent = true; 13 | ipc.config.id = 'FAKE'; 14 | ipc.config.retry = 1500; 15 | 16 | describe('Beacon\'s Job Manager', function() { 17 | 18 | var globals = {}; 19 | 20 | before(function(done) { 21 | ipc.serve(function(){ 22 | ipc.server.on('message', function(msg) {}); 23 | }); 24 | ipc.server.start(); 25 | 26 | rmrf('.unpacks', function(err) { 27 | done(); 28 | }); 29 | }); 30 | 31 | after(function() { 32 | ipc.server.stop(); 33 | }); 34 | 35 | it('runs a job locally and mediates communication between processes', function(done) { 36 | var fake_parent = spawn('node', ['./tests/fixtures/beacon/router-ipc-client.js']); 37 | var fake_parent_stdout = ""; 38 | fake_parent.stdout.on('data', function(chunck) { fake_parent_stdout += chunck.toString().trim(); }); 39 | var fake_parent_exit = new Promise(function(resolve, reject) { 40 | fake_parent.on('exit', function(){resolve();}); 41 | }); 42 | 43 | // wait for a message from the fake client announcing its ipc socket 44 | ipc.server.on('SOCKET:PUB', function(data, socket) { 45 | // set the meta table and monitor_socket 46 | router.setMetaTable(router.buildPureLocalTable(2)); 47 | router.setMonitorSocket(socket); 48 | 49 | var fake_instance = spawn('node', ['./tests/fixtures/beacon/router-local-process.js'], {stdio: [null, null, null, 'ipc']}); 50 | var fake_instance_stdout = ""; 51 | fake_instance.stdout.on('data', function(chunck) { fake_instance_stdout += chunck.toString().trim(); }); 52 | var fake_instance_exit = new Promise(function(resolve, reject) { 53 | fake_instance.on('exit', function(){resolve();}); 54 | }); 55 | router.setLocalChannel(0, fake_instance); 56 | 57 | var job = {size: 2, subsize: 1, start: 1, app: './tests/fixtures/beacon/runner-fake-job.js'}; 58 | 59 | jobman.runLocally(job); 60 | 61 | Promise.all([fake_parent_exit, fake_instance_exit]).then(function() { 62 | expect(fake_parent_stdout).to.equal("Hello from Fake"); 63 | expect(fake_instance_stdout).to.equal("Fake Hello"); 64 | done(); 65 | }); 66 | }); 67 | }); 68 | 69 | it('packs an app correctly', function(done) { 70 | jobman.pack({app: './tests/fixtures/beacon/fake_app/main.js'}) 71 | .then(function(app) { 72 | var packg = zipper.sync.unzip(new Buffer(app.pckg, "base64")).memory(); 73 | var packg_contents = packg.contents(); 74 | expect(packg_contents.length).to.equal(1); 75 | expect(packg_contents).to.include('app_' + app.id +'.js'); 76 | 77 | // save data for the next test 78 | globals.id = app.id; 79 | fs.writeFileSync("./tests/fixtures/beacon/fake_app.zip", app.pckg, "base64"); 80 | 81 | done(); 82 | }) 83 | .catch(done); 84 | }); 85 | 86 | it('unpacks an app correctly', function(done) { 87 | jobman.unpack({ 88 | id: globals.id, 89 | pckg: fs.readFileSync("./tests/fixtures/beacon/fake_app.zip", {encoding: "base64"}) 90 | }) 91 | .then(function(app) { 92 | var unpacked_app = spawn('node', [app]); 93 | var unpacked_app_stdout = ""; 94 | unpacked_app.stdout.on('data', function(chunck) { 95 | unpacked_app_stdout += chunck.toString().replace(/[\n\r]/g, ''); 96 | }); 97 | 98 | unpacked_app.on('exit', function() { 99 | expect(unpacked_app_stdout).to.equal("Local Dependency"); 100 | done(); 101 | }) 102 | }) 103 | .catch(done); 104 | }); 105 | 106 | it('divides a job over given hosts correctly (perfect dist, no infinity host)', function() { 107 | // perfect dist means that job.size <= sum of hosts max processes 108 | var fake_job = { 109 | hosts: { 110 | "local": {max_procs: 2}, 111 | "192.168.0.100:2222": {max_procs: 5, password: ""}, 112 | "192.168.0.58:2222": {max_procs: 4, password: ""} 113 | }, 114 | size: 11 115 | }; 116 | 117 | var plan = jobman.divide(fake_job); 118 | expect(plan.local).to.exist; 119 | expect(plan.local.count).to.equal(2); 120 | expect(plan.local.start).to.equal(0); 121 | expect(plan["192.168.0.100:2222"]).to.exist; 122 | expect(plan["192.168.0.100:2222"].start).to.equal(2); 123 | expect(plan["192.168.0.100:2222"].count).to.equal(5); 124 | expect(plan["192.168.0.58:2222"]).to.exist; 125 | expect(plan["192.168.0.58:2222"].start).to.equal(7); 126 | expect(plan["192.168.0.58:2222"].count).to.equal(4); 127 | }); 128 | 129 | it('divides a job over given hosts correctly (perfect dist, infinity host)', function() { 130 | var fake_job = { 131 | hosts: { 132 | "local": {max_procs: 2}, 133 | "192.168.0.100:2222": {max_procs: 1, password: ""}, 134 | "192.168.0.58:2222": {max_procs: Infinity, password: ""} 135 | }, 136 | size: 11 137 | }; 138 | 139 | var plan = jobman.divide(fake_job); 140 | expect(plan.local.count).to.equal(2); 141 | expect(plan["192.168.0.100:2222"].count).to.equal(1); 142 | expect(plan["192.168.0.58:2222"].count).to.equal(8); 143 | }); 144 | 145 | it('divides a job over given hosts correctly (overallocation)', function() { 146 | var fake_job = { 147 | hosts: { 148 | "local": {max_procs: 2}, 149 | "192.168.0.100:2222": {max_procs: 1, password: ""}, 150 | "192.168.0.58:2222": {max_procs: 3, password: ""} 151 | }, 152 | size: 9 153 | }; 154 | 155 | var plan = jobman.divide(fake_job); 156 | expect(plan.local.count).to.equal(3); 157 | expect(plan["192.168.0.100:2222"].count).to.equal(2); 158 | expect(plan["192.168.0.58:2222"].count).to.equal(4); 159 | }); 160 | 161 | }); 162 | -------------------------------------------------------------------------------- /tests/specs/beacon/router.specs.js: -------------------------------------------------------------------------------- 1 | var ipc = require('node-ipc'); 2 | var router = require('../../../lib/router.js'); 3 | var expect = require('chai').expect; 4 | var spawn = require('child_process').spawn; 5 | 6 | // FAKE: ipc socket server 7 | ipc.config.silent = true; 8 | ipc.config.id = 'FAKE'; 9 | ipc.config.retry = 1500; 10 | 11 | describe('Beacon\'s Router', function() { 12 | 13 | before(function() { 14 | ipc.serve(function(){ 15 | ipc.server.on('message', function(msg) {}); 16 | }); 17 | ipc.server.start(); 18 | }); 19 | 20 | after(function() { 21 | ipc.server.stop(); 22 | }); 23 | 24 | it('builds a pure local meta routing table', function() { 25 | var table = router.buildPureLocalTable(1); 26 | expect(table.proc0).to.equal('local'); 27 | expect(table.parent).to.equal('local'); 28 | }); 29 | 30 | it('builds a meta routing table from a given plan', function() { 31 | 32 | var plan = { 33 | local: { count: 2, start: 0 }, 34 | '192.168.0.100:2222': { count: 2, start: 2 }, 35 | '192.168.0.58:2222': { count: 1, start: 4 } 36 | }; 37 | 38 | var table = router.buildTableFromPlan(plan, true); 39 | 40 | expect(table.parent).to.equal('local'); 41 | expect(table['proc0']).to.equal('local'); 42 | expect(table['proc1']).to.equal('local'); 43 | expect(table['proc2']).to.equal('192.168.0.100:2222'); 44 | expect(table['proc3']).to.equal('192.168.0.100:2222'); 45 | expect(table['proc4']).to.equal('192.168.0.58:2222'); 46 | 47 | }); 48 | 49 | it('routes a message correctly to local parent', function(done) { 50 | var fake_client = spawn('node', ['./tests/fixtures/beacon/router-ipc-client.js']); 51 | var fake_client_stdout = ""; 52 | fake_client.stdout.on('data', function(chunck) { fake_client_stdout += chunck.toString().trim(); }); 53 | 54 | // wait for a message from the fake client announcing its ipc socket 55 | ipc.server.on('SOCKET:PUB', function(data, socket) { 56 | // set the meta table and monitor_socket 57 | router.setMetaTable(router.buildPureLocalTable(1)); 58 | router.setMonitorSocket(socket); 59 | 60 | router.routeToParent({type: 'process:exit', data: {line: "Hello from router"}}); 61 | 62 | fake_client.on('exit', function() { 63 | expect(fake_client_stdout).to.equal("Hello from router"); 64 | done(); 65 | }); 66 | }) 67 | }); 68 | 69 | it('routes a monitor message correctly to a remote beacon', function(done) { 70 | var fake_parent = spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 71 | var fake_parent_stdout = ""; 72 | fake_parent.stdout.on('data', function(chunck) { fake_parent_stdout += chunck.toString().trim(); }); 73 | 74 | setTimeout(function() { 75 | // clear router from past test and set metadata to refer to a tcp parent 76 | router.clear(); 77 | router.setMetaTable({parent: "127.0.0.1:4895"}); 78 | 79 | ipc.connectToNet('remote_parent', '127.0.0.1', 4895, function() { 80 | router.setMonitorSocket(ipc.of.remote_parent); 81 | 82 | router.routeToParent({type: 'process:exit', data: {line: "Hello remote from router"}}); 83 | 84 | ipc.of.remote_parent.socket.on('end', function() { ipc.disconnect('remote_parent'); }); 85 | }); 86 | 87 | ipc.of.remote_parent.on('error', function(err) { 88 | ipc.disconnect('remote_parent'); 89 | done(err); 90 | }); 91 | 92 | fake_parent.on('exit', function() { 93 | expect(fake_parent_stdout).to.equal("Hello remote from router"); 94 | done(); 95 | }); 96 | }, 1000); 97 | }); 98 | 99 | it('rouets a message correctly to local job instance', function(done) { 100 | var fake_instance = spawn('node', ['./tests/fixtures/beacon/router-local-process.js'], {stdio: [null, null, null, 'ipc']}); 101 | var fake_instance_stdout = ""; 102 | fake_instance.stdout.on('data', function(chunck) { fake_instance_stdout += chunck.toString().trim() }); 103 | 104 | // set the meta table and local channel 105 | router.setMetaTable(router.buildPureLocalTable(1)); 106 | router.setLocalChannel(0, fake_instance); 107 | 108 | router.routeTo(0, {data: "Hello from router"}); 109 | 110 | fake_instance.on('exit', function() { 111 | expect(fake_instance_stdout).to.equal("Hello from router"); 112 | done(); 113 | }); 114 | }); 115 | 116 | it('routes a message correctly to a remote beacon', function(done) { 117 | var fake_beacon = spawn('node', ['./tests/fixtures/beacon/fake-tcp-server.js']); 118 | var fake_beacon_stdout = ""; 119 | fake_beacon.stdout.on('data', function(chunck){ fake_beacon_stdout += chunck.toString().trim(); }); 120 | 121 | setTimeout(function() { 122 | router.clear(); 123 | router.setMetaTable({ proc0: '127.0.0.1:4895' }); 124 | 125 | ipc.connectToNet('remote_beacon', '127.0.0.1', 4895, function() { 126 | router.setRemoteChannel("127.0.0.1:4895", ipc.of.remote_beacon); 127 | 128 | router.routeTo(0, {data: "Hello remote from router"}); 129 | 130 | ipc.of.remote_beacon.socket.on('end', function() { ipc.disconnect('remote_beacon'); }); 131 | }); 132 | 133 | ipc.of.remote_beacon.on('error', function(err) { 134 | ipc.disconnect('remote_beacon'); 135 | done(err); 136 | }); 137 | 138 | fake_beacon.on('exit', function() { 139 | expect(fake_beacon_stdout).to.equal("Hello remote from router"); 140 | done(); 141 | }); 142 | }, 1000); 143 | }); 144 | 145 | it('cleans the router\'s data structures', function() { 146 | expect(router.isClean()).to.equal(false); 147 | router.clear(); 148 | expect(router.isClean()).to.equal(true); 149 | }); 150 | 151 | }); 152 | -------------------------------------------------------------------------------- /tests/specs/cli/cli.specs.js: -------------------------------------------------------------------------------- 1 | var spawn = require('child_process').spawn; 2 | var expect = require('chai').expect; 3 | var readline = require('readline'); 4 | var beacon = require('../../../lib/beacon-controller.js'); 5 | 6 | describe('Command Line Interface', function() { 7 | 8 | this.timeout(5000); 9 | 10 | it('runs the cli with no options', function(done) { 11 | var cli = spawn('node', ['./bin/main.js']); 12 | var cli_stdout = []; 13 | var rli = readline.createInterface({input: cli.stdout}); 14 | rli.on('line', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 15 | 16 | cli.on('exit', function() { 17 | expect(cli_stdout).to.deep.equal([ 18 | 'You didn\'t specify any options.', 19 | 'Run (klyng --help) for more info.' 20 | ]); 21 | done(); 22 | }); 23 | }); 24 | 25 | it('runs the cli with -u option while the beacon is down', function(done) { 26 | var cli = spawn('node', ['./bin/main.js', '-u']); 27 | var cli_stdout = []; 28 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 29 | 30 | cli.on('exit', function() { 31 | expect(cli_stdout).to.deep.equal(['The beacon is now up and listening.']); 32 | done(); 33 | }); 34 | }); 35 | 36 | it('runs the cli with -u option while the beacon is up', function(done) { 37 | var cli = spawn('node', ['./bin/main.js', '-u']); 38 | var cli_stdout = []; 39 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 40 | 41 | cli.on('exit', function() { 42 | expect(cli_stdout).to.deep.equal(['The beacon is already up and listening.']); 43 | done(); 44 | }); 45 | }); 46 | 47 | it('runs the cli with -d option while the beacon is up', function(done) { 48 | var cli = spawn('node', ['./bin/main.js', '-d']); 49 | var cli_stdout = []; 50 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 51 | 52 | cli.on('exit', function() { 53 | expect(cli_stdout).to.deep.equal(['The beacon is now down.']); 54 | done(); 55 | }); 56 | }); 57 | 58 | it('runs the cli with -d option while the beacon is down', function(done) { 59 | var cli = spawn('node', ['./bin/main.js', '-d']); 60 | var cli_stdout = []; 61 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 62 | 63 | cli.on('exit', function() { 64 | expect(cli_stdout).to.deep.equal(['The beacon is not up.']); 65 | done(); 66 | }); 67 | }); 68 | 69 | it('runs the cli with a job while the beacon is down', function(done) { 70 | var cli = spawn('node', ['./bin/main.js', '-n', 1, './tests/fixtures/beacon/runner-fake-job.js']); 71 | var cli_stdout = []; 72 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 73 | 74 | cli.on('exit', function() { 75 | expect(cli_stdout).to.deep.equal(['Hello from Fake']); 76 | done(); 77 | }); 78 | }); 79 | 80 | it('runs the cli with a job while the beacon is up', function(done) { 81 | var cli = spawn('node', ['./bin/main.js', '-n', 1, './tests/fixtures/beacon/runner-fake-job.js']); 82 | var cli_stdout = []; 83 | cli.stdout.on('data', function(chunck){ cli_stdout.push(chunck.toString().trim()); }); 84 | 85 | cli.on('exit', function() { 86 | expect(cli_stdout).to.deep.equal(['Hello from Fake']); 87 | done(); 88 | }); 89 | }); 90 | 91 | after(function(done) { 92 | beacon.checkIfRunning() 93 | .then(function() {return beacon.stop();}) 94 | .then(function() { 95 | done(); 96 | }); 97 | }); 98 | 99 | }); 100 | -------------------------------------------------------------------------------- /tests/specs/cli/controller.specs.js: -------------------------------------------------------------------------------- 1 | var spawn = require('child_process').spawn; 2 | var expect = require('chai').expect; 3 | var cli = require('../../../lib/cli-controller'); 4 | 5 | describe('CLI Controller', function() { 6 | 7 | it("parses a valid hosts file correctly", function() { 8 | 9 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.valid.json"); 10 | 11 | expect(hosts.error).to.not.exist; 12 | expect(hosts.local.max_procs).to.equal(2); 13 | expect(hosts["192.168.0.10:2222"].max_procs).to.equal(Infinity); 14 | expect(hosts["192.168.0.10:2222"].password).to.equal(""); 15 | expect(hosts["192.168.0.2:4578"].max_procs).to.equal(3); 16 | expect(hosts["192.168.0.2:4578"].password).to.equal(""); 17 | expect(hosts["192.168.0.103:2222"].max_procs).to.equal(5); 18 | expect(hosts["192.168.0.103:2222"].password).to.equal("h3llo"); 19 | expect(hosts["192.168.0.20:5986"].max_procs).to.equal(1); 20 | expect(hosts["192.168.0.20:5986"].password).to.equal("oll3h"); 21 | 22 | }); 23 | 24 | it('fails to parse a non-existing hosts file', function() { 25 | var hosts = cli.parseHosts("fake-hosts.that.is.not.there.json"); 26 | 27 | expect(hosts.error).to.exist; 28 | expect(hosts.error).to.have.string('ENOENT'); 29 | }); 30 | 31 | it('fails to parse a non-json hosts file', function() { 32 | var hosts = cli.parseHosts(__dirname + "/cli.specs.js"); 33 | 34 | expect(hosts.error).to.exist; 35 | expect(hosts.error).to.have.string('Unexpected token'); 36 | }); 37 | 38 | it('fails to parse a hosts file with non-object entry', function() { 39 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.invalid-1.json"); 40 | 41 | expect(hosts.error).to.exist; 42 | expect(hosts.error).to.have.string('INVALID: an entry in hosts file is not an object'); 43 | }); 44 | 45 | it('fails to parse a hosts file with an entry of a non-number pCount', function() { 46 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.invalid-2.json"); 47 | 48 | expect(hosts.error).to.exist; 49 | expect(hosts.error).to.have.string("INVALID: an entry's processes count is not a number"); 50 | }); 51 | 52 | it('fails to parse a hosts file with an entry of a non-positive pCount', function() { 53 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.invalid-5.json"); 54 | 55 | expect(hosts.error).to.exist; 56 | expect(hosts.error).to.have.string("INVALID: an entry's processes count is less than one"); 57 | }); 58 | 59 | it('fails to parse a hosts file with an entry of a non-number port', function() { 60 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.invalid-3.json"); 61 | 62 | expect(hosts.error).to.exist; 63 | expect(hosts.error).to.have.string("INVALID: an entry's port number is not a number!"); 64 | }); 65 | 66 | it('fails to parse a hosts file with an entry of a non-string password', function() { 67 | var hosts = cli.parseHosts(__dirname + "/../../fixtures/cli/fake-hosts.invalid-4.json"); 68 | 69 | expect(hosts.error).to.exist; 70 | expect(hosts.error).to.have.string("INVALID: an entry's password is not a string"); 71 | }); 72 | 73 | }); 74 | -------------------------------------------------------------------------------- /tests/specs/klyng-apis/sync-primitives.specs.js: -------------------------------------------------------------------------------- 1 | var spawn = require('child_process').spawn; 2 | var expect = require('chai').expect; 3 | 4 | describe("klyng sync-primitives", function() { 5 | 6 | it('initializes and ends a klyng app without errors', function(done) { 7 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-init.js', 1, 2], {stdio: [null,null,null,'ipc']}); 8 | app.on('exit', function(code) { 9 | expect(code).to.equal(0); 10 | done(); 11 | }); 12 | }); 13 | 14 | it('fails to initialize a klyng app with no ipc', function(done) { 15 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-init.js', 1, 2]); 16 | app.on('exit', function(code) { 17 | expect(code).to.equal(1); 18 | done(); 19 | }); 20 | }) 21 | 22 | it('gets the rank and size of the process correctly', function(done) { 23 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-rank-size.js', 2, 0], {stdio: [null,null,null,'ipc']}); 24 | var app_stdout = ""; 25 | app.stdout.on('data', function(chunck) { app_stdout += chunck.toString().trim(); }); 26 | app.on('exit', function(code) { 27 | expect(code).to.equal(0); 28 | expect(app_stdout).to.equal("0:2"); 29 | done(); 30 | }); 31 | }); 32 | 33 | it('sends a message correctly', function(done) { 34 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-send.js', 2, 0], {stdio: [null,null,null,'ipc']}); 35 | var recieved_msg = null; 36 | app.on('message', function(msg) { recieved_msg = msg; }); 37 | app.on('exit', function(code) { 38 | expect(code).to.equal(0); 39 | expect(recieved_msg).to.not.equal(null); 40 | expect(recieved_msg.type).to.equal('klyng:msg'); 41 | expect(recieved_msg.header.from).to.equal(0); 42 | expect(recieved_msg.header.to).to.equal(1); 43 | expect(recieved_msg.data).to.equal("Hello"); 44 | done(); 45 | }); 46 | }); 47 | 48 | it('raises an error when send is called with no \'to\' field', function(done) { 49 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-send-invalid-1.js', 2, 0], {stdio: [null,null,null,'ipc']}); 50 | app.on('exit', function(code) { 51 | expect(code).to.equal(1); 52 | done() 53 | }); 54 | }); 55 | 56 | it('raises an error when send is called with no \'data\' field', function(done) { 57 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-send-invalid-2.js', 2, 0], {stdio: [null,null,null,'ipc']}); 58 | app.on('exit', function(code) { 59 | expect(code).to.equal(1); 60 | done() 61 | }); 62 | }); 63 | 64 | it('recieves a message correctly with no criteria specified', function(done) { 65 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-recv-no-criteria.js', 2, 0], {stdio: [null,null,null,'ipc']}); 66 | var app_stdout = ""; 67 | app.stdout.on('data', function(chunck){ app_stdout += chunck.toString().trim(); }); 68 | app.send({type: 'klyng:msg',header: {to:0,from: 1,},data: "Hello"}); 69 | app.on('exit', function(code) { 70 | expect(code).to.equal(0); 71 | expect(app_stdout).to.equal("Hello"); 72 | done(); 73 | }); 74 | }); 75 | 76 | it('recieves a message correctly with source specified', function(done) { 77 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-recv-from.js', 2, 0], {stdio: [null,null,null,'ipc']}); 78 | var app_stdout = ""; 79 | app.stdout.on('data', function(chunck){ app_stdout += chunck.toString().trim(); }); 80 | app.send({type: 'klyng:msg',header: {to:0,from: 2,},data: "Hello from 2"}); 81 | app.send({type: 'klyng:msg',header: {to:0,from: 5,},data: "Hello from 5"}); 82 | app.on('exit', function(code) { 83 | expect(code).to.equal(0); 84 | expect(app_stdout).to.equal("Hello from 5"); 85 | done(); 86 | }); 87 | }); 88 | 89 | it('recieves a message correctly with subject specified', function(done) { 90 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-recv-subject.js', 2, 0], {stdio: [null,null,null,'ipc']}); 91 | var app_stdout = ""; 92 | app.stdout.on('data', function(chunck){ app_stdout += chunck.toString().trim(); }); 93 | app.send({type: 'klyng:msg',header: {to:0,from: 2,subject: "sub1"},data: "Hello with sub1"}); 94 | app.send({type: 'klyng:msg',header: {to:0,from: 5,subject: "sub2"},data: "Hello with sub2"}); 95 | app.on('exit', function(code) { 96 | expect(code).to.equal(0); 97 | expect(app_stdout).to.equal("Hello with sub2"); 98 | done(); 99 | }); 100 | }); 101 | 102 | it('recieves a message correctly with full criteria', function(done) { 103 | var app = spawn('node', ['./tests/fixtures/klyng-apis/sync-recv-full.js', 2, 0], {stdio: [null,null,null,'ipc']}); 104 | var app_stdout = ""; 105 | app.stdout.on('data', function(chunck){ app_stdout += chunck.toString().trim(); }); 106 | app.send({type: 'klyng:msg',header: {to:0,from: 2,subject: "sub1"},data: "Hello with sub1 form 2"}); 107 | app.send({type: 'klyng:msg',header: {to:0,from: 5,subject: "sub2"},data: "Hello with sub2 form 5"}); 108 | app.on('exit', function(code) { 109 | expect(code).to.equal(0); 110 | expect(app_stdout).to.equal("Hello with sub2 form 5"); 111 | done(); 112 | }); 113 | }); 114 | 115 | }); 116 | --------------------------------------------------------------------------------