├── LICENSE ├── README.md ├── owperf.js ├── owperf.sh ├── owperf_data.odg ├── owperf_data.png ├── package.json ├── setup.sh └── testAction.js /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 19 | # :electric_plug: owperf - a performance test tool for Apache OpenWhisk 20 | 21 | ## General Info 22 | This test tool benchmarks an OpenWhisk deployment for (warm) latency and throughput, with several new capabilities: 23 | 1. Measure performance of rules (trigger-to-action) in addition to actions 24 | 1. Deeper profiling without instrumentation (e.g., Kamino) by leveraging the activation records in addition to the client's timing data. This avoids special setups, and can help gain performance insights on third-party deployments of OpenWhisk. 25 | 1. New tunables that can affect performance: 26 | 1. Parameter size - controls the size of the parameter passed to the action or event 27 | 1. Actions per iteration (a.k.a. _ratio_) - controls how many rules are associated with a trigger [for rules] or how many actions are asynchronously invoked (burst size) at each iteration of a test worker [for actions]. 28 | 1. "Master apart" mode - Allow the master client to perform latency measurements while the worker clients stress OpenWhisk using a specific invocation pattern in the background. Useful for measuring latency under load, and for comparing latencies of rules and actions under load. 29 | The tool is written in node.js, using mainly the modules of OpenWhisk client, cluster for concurrency, and commander for CLI procssing. 30 | 31 | ### Operation 32 | The general operation of a test is simple: 33 | 1. **Setup**: the tool creates the test action, test trigger, and a number of rules that matches the ratio tunable above. 34 | 1. **Test**: the tool fires up a specified number of concurrent clients - a master and workers. 35 | 1. Each client wakes up once every _delta_ msec (iteration) and invokes the specified activity: either the trigger (for rule testing) or multiple concurrent actions - matching the ratio tunable. Action invocations can be blocking. 36 | 1. After each client has completed a number of initial iterations (warmup), measurement begins, controlled by the master client, for either a specified number of iterations or specified time. 37 | 1. At the end of the measurement, each client retrieves the activation records of its triggers and/or actions, and generates summary data that is sent to the master, which generates and prints the final results. 38 | 1. **Teardown**: clean up the OpenWhisk assets created during setup 39 | 40 | Final results are written to the standard output stream (so can be redirected to a file) as a single highly-detailed CSV record containing all the input settings and the output measurements (see below). There is additional control information that is written to the standard error stream and can be silenced in CLI. The control information also contains the CSV header, so it can be copied into a spreadsheet if needed. 41 | 42 | It is possible to invoke the tool in "Master apart" mode, where the master client is invoking a diffrent activity than the workers, and at possibly a different (very likely, much slower) rate. In this mode, latency statsitics are computed based solely on the master's data, since the worker's activity is used only as background to stress the OpenWhisk deployment. So one experiment can have the master client invoke rules and another one can have the master client invoke actions, while in both experiments the worker clients perform the same background activity. 43 | 44 | The tool is highly customizable via CLI options. All the independent test variables are controlled via CLI. This includes number of workers, invocation pattern, OW client configuration, test action sleep time, etc. 45 | 46 | Test setup and teardown can be independently skipped via CLI, and/or be directly invoked from the external setup script (```setup.sh```), so that setup can be shared between multiple tests. More advanced users can set up a custom test action, a custom event trigger and custom payload for events or action parameters. This can be used to benchmark specific applications, or to combine multiple tests together concurrently for a simulation of a mix of serverless applications. 47 | 48 | **Clock skew**: OpenWhisk is a distributed system, which means that clock skew is expected between the client machine computing invocation timestamps and the controllers or invokers that generate the timestamps in the activation records. However, this tool assumes that clock skew is bound at few msec range, due to having all machines clocks synchronized, typically using NTP. At such a scale, clock skew is quite small compared to the measured time periods. Some of the time periods are measured using the same clock (see below) and are therefore oblivious to clock skew issues. 49 | 50 | ## Initial Setup 51 | The tool requires very little setup. You need to have node.js (v8+) and the wsk CLI client installed (on $PATH). Before the first run, execute ```npm install``` in the tool folder to install the dependencies. 52 | **Throttling**: By default, OW performance is throttled according to some [limits](https://github.com/apache/incubator-openwhisk/blob/master/docs/reference.md#system-limits), such as maximum number of concurrent requests, or maximum invocations per minute. If your benchmark stresses OpenWhisk beyond the limit value, you might want to relax those limits. If it's an OpenWhisk deployment that you control, you can set the limits to 999999, thereby effectively cancelling the limits. If it's a third-party service, you may want to consult the service documentation and/or support to see what limits can be relaxed and by how much. 53 | 54 | ## Usage 55 | To use the tool, run ```./owperf.sh ``` to perform a test. To see all the available options and defaults run ```./owperf.sh -h```. 56 | 57 | The default for ratio is 1. If using a different ratio, be sure to specify the same ratio value for all steps. 58 | 59 | For example, let's perform a test of rule performance with 3 clients, using the default delta of 200 msec, for 100 iterations (counted at the master client, excluding the warmup), ratio of 4. Each client performs 5 iterations per second, each iteration firing a trigger that invokes 4 rules, yielding a total of 3x5x4=60 rule invocations per second. The command to run this test: ```./owperf.sh -a rule -w 3 -i 100 -r 4``` 60 | 61 | ## Measurements 62 | As explained above, the owperf tool collects both latency and throughput data at each experiment. 63 | 64 | ### Latency 65 | The following time-stamps are collected for each invocation, of either action, or rule (containing an action): 66 | * **BI** (Before Invocation) - taken by a client immediately before invoking - either the trigger fire (for rules), or an action invocation. 67 | * **TS** (Trigger Start) - taken from the activation record of the trigger linked to the rules, so applies only to rule tests. All actions invoked by the rules of the same trigger have the same TS value. 68 | * **AS** (Action Start) - taken from the activation record of the action. 69 | * **AE** (Action End) - taken from the activation record of the action. 70 | * **AI** (After Invocation) - taken by the client immmediately after the invocation, for blocking action invocation tests only. 71 | 72 | Based on these timestamps, the following measurements are taken: 73 | * **OEA** (Overhead of Entering Action) - OpenWhisk processing overhead from sending the action invocation or trigger fire to the beginning of the action execution. OEA = AS-BI 74 | * **D** - the duration of the test action - as reported by the action itself in the return value. 75 | * **AD** - Action Duration - as measured by OpenWhisk invoker. AD = AE - AS. Always expect that AD >= D. 76 | * **OER** (Overhead of Executing Request) - OpenWhisk processing overhead from sending the action invocation or trigger fire to the completion of the action execution in the OpenWhisk Invoker. OER = AE-BI-D 77 | * **TA** (Trigger to Answer) - the processing time from the start of the trigger process to the start of the action (rule tests only). TA = AS-TS 78 | * **ORA** (Overhead of Returning from Action) - time from action end till being received by the client (blocking action tests only). ORA = AI - AE 79 | * **RTT** (Round Trip Time) - time at the client from action invocation till reply received (blocking action tests only). RTT = AI - BI 80 | * **ORTT** (Overhead of RTT) - RTT at the client exclugin the net action computation time. ORTT = RTT - D 81 | 82 | For each measurement, the tool computes average (_avg_), standard deviation (_std_), extremes (_min_ and _max_) and number of samples (_cnt_). For the current implementation, _cnt_ of any of the above mmeasurements should be the same as number of invocations. 83 | 84 | The following chart depicts the relationship between the various measurements and the action invocation and rule invocation flows. 85 | 86 | ![](owperf_data.png) 87 | 88 | ### Throughput 89 | Throughput is measured w.r.t. several different counters. During post-processing of an experiment, each counter value is divided by the measurement time period to compute a respective throughput. 90 | * **Attempts** - number of invocation attempts performed inside the time frame (according to their BI). This is the "arrival rate" of invocations, should be close to _clients * ratio / delta_ . 91 | * **Requests** - number of requests sent to OpenWhisk inside the time frame. Each action invocation is one request, and each trigger fire is also one request (so a client invoking rules at ratio _k_ generates _k+1_ requests). 92 | * **Activations** - number of completed activations inside the time frame, counting both trigger activations (based on TS), and action activations (based on AS and AE). 93 | * **Invocations** - number of successful invocations of complete rules or actions (depending on the activity). This is the "service rate" of invocations (assuming errors happen only because OW is overloaded). 94 | 95 | For each counter, the tool reports the total counter value (_abs_), total throughput per second (_tp_), througput of the worker clients without the master (_tpw_) and the master's percentage of throughput relative to workers (_tpd_). The last two values are important mostly for master apart mode. 96 | 97 | ### Errors 98 | Aside from latency and throughput, the tool also counts **errors**. Failed invocations - of actions, of triggers, or of actions from triggers (via rules) are counted each as an error. The tool reports both absolute error count (_abs_) and percent out of requests (_percent_). 99 | 100 | Another specific latency metric in errors is _cim_ (Client Invocation Miss). This metric records how long it takes the client to perform an invocation of an action (directly) or of a rule (via trigger), beyond the specified _delta_. Consequently, this metric is recorded only when the API invocation at the client takes more than _delta_, which may happen in a slow connections between the test machine (e.g., the user's laptop) and the OpenWhisk service. The user should increase _delta_ to make the _cim.cnt_ and _cim.pct_ (percent of total attempts) values as low as possible (ideally, zero), denoting a long-enough delta to acoommodate the quality of the connection. This allows the user to set delta such that the attempts throughput (the arrival rate - see above) is indeed controlled, according to the formula above. 101 | 102 | ## Acknowledgements 103 | The owperf tool has been developed by IBM Research as part of the [CLASS](https://class-project.eu/) EU project. CLASS aims to integrate OpenWhisk as a foundation for latency-sensitive polyglot event-driven big-data analytics platform running on a compute continuum from the cloud to the edge. CLASS is funded by the European Union's Horizon 2020 Programme grant agreement No. 780622. 104 | 105 | -------------------------------------------------------------------------------- /owperf.js: -------------------------------------------------------------------------------- 1 | /** 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | */ 19 | 20 | 21 | /** 22 | * This is a test tool for measuring the performance of OpenWhisk actions and rules. 23 | * The full documentation of the tool is available in README.md . 24 | */ 25 | 26 | const fs = require('fs'); 27 | const ini = require('ini'); 28 | const cluster = require('cluster'); 29 | const openwhisk = require('openwhisk'); 30 | const program = require('commander'); 31 | const exec = require('node-exec-promise').exec; 32 | 33 | const ACTION = "action"; 34 | const RULE = "rule"; 35 | const RESULT = "result"; 36 | const ACTIVATION = "activation"; 37 | const NONE = "none"; 38 | 39 | function parseIntDef(strval, defval) { 40 | return parseInt(strval); 41 | } 42 | 43 | program 44 | .description('Latency and throughput measurement of OpenWhisk actions and rules') 45 | .version('0.0.1') 46 | .option('-a, --activity ', "Activity to measure", /^(action|rule)$/i, "action") 47 | .option('-b, --blocking ', "For actions, wait until result or activation, or don't wait", /^(result|activation|none)$/i, "none") 48 | .option('-d, --delta ', "Time diff between consequent invocations of the same worker, in msec", parseIntDef, 200) 49 | .option('-i, --iterations ', "Number of measurement iterations", parseInt) 50 | .option('-p, --period ', "Period of measurement in msec", parseInt) 51 | .option('-r, --ratio ', "How many actions per iteration (or rules per trigger)", parseIntDef, 1) 52 | .option('-s, --parameter_size ', "Size of string parameter passed to trigger or actions", parseIntDef, 1000) 53 | .option('-w, --workers ', "Total number of concurrent workers incl. master", parseIntDef, 1) 54 | .option('-A, --master_activity ', "Set master activity apart from other workerss", /^(action|rule)$/i) 55 | .option('-B, --master_blocking ', "Set master blocking apart from other workers", /^(result|activation|none)$/i) 56 | .option('-D, --master_delta ', "Set master delta apart from other workers", parseInt) 57 | .option('-u, --warmup ', "How many invocations to perform at each worker as warmup", parseIntDef, 5) 58 | .option('-l, --delay ', "How many msec to delay at each action", parseIntDef, 50) 59 | .option('-P, --pp_delay ', "Wait for remaining activations to finalize before post-processing", parseIntDef, 60000) 60 | .option('-G, --burst_timing', "For actions, use the same invocation timing (BI) for all actions in a burst") 61 | .option('-S, --no-setup', "Skip test setup (so use existing setup)") 62 | .option('-T, --no-teardown', "Skip test teardown (to allow setup reuse)") 63 | .option('-f, --config_file ', "Specify a wskprops configuration file to use", `${process.env.HOME}/.wskprops`) 64 | .option('-e, --event_trigger ', "Specify a custom event trigger to use in tests", "testTrigger") 65 | .option('-t, --test_action ', "Specify a custom action to use in tests", "testAction") 66 | .option('-L, --payload_file ', "Specify a custom payload for rule or action in a JSON file") 67 | .option('-q, --quiet', "Suppress progress information on stderr"); 68 | 69 | program.parse(process.argv); 70 | 71 | var testRecord = {input: {}, output: {}}; // holds the final test data 72 | 73 | for (var opt in program.opts()) 74 | if (typeof program[opt] != 'function') 75 | testRecord.input[opt] = program[opt]; 76 | 77 | // If neither period nor iterations are set, then period is set by default to 1000 msec 78 | if (!testRecord.input.iterations && !testRecord.input.period) 79 | testRecord.input.period = 1000; 80 | 81 | // If either master_activity, master_blocking or master_delta are set, then test is in 'master apart' mode 82 | testRecord.input.master_apart = ((testRecord.input.master_activity || testRecord.input.master_blocking || testRecord.input.master_delta) && true); 83 | 84 | mLog("Parameter Configuration:"); 85 | for (var opt in testRecord.input) 86 | mLog(`${opt} = ${testRecord.input[opt]}`); 87 | mLog("-----\n"); 88 | 89 | mLog("Generating invocation parameters"); 90 | const params = (testRecord.input.payload_file ? JSON.parse(fs.readFileSync(testRecord.input.payload_file)) : {}); 91 | var inputMessage = "A".repeat(testRecord.input.parameter_size); 92 | params.sleep = testRecord.input.delay; 93 | params.message = inputMessage; 94 | mLog(`Parameters: ${JSON.stringify(params, null, 4)}`); 95 | 96 | mLog("Loading wskprops"); 97 | const config = ini.parse(fs.readFileSync(testRecord.input.config_file, "utf-8")); 98 | mLog("APIHOST = " + config.APIHOST); 99 | mLog("AUTH = " + config.AUTH); 100 | mLog("-----\n"); 101 | const wskParams = `--apihost ${config.APIHOST} --auth ${config.AUTH} -i`; // to be used when invoking setup and teardown via external wsk 102 | 103 | // openwhisk client used for invocations 104 | const ow = openwhisk({apihost: config.APIHOST, api_key: config.AUTH, ignore_certs: true}); 105 | 106 | // counters for throughput computation (all) 107 | const tpCounters = {attempts: 0, invocations: 0, activations: 0, requests: 0, errors: 0, iterations: 0}; 108 | 109 | // counters for latency computation 110 | const latCounters = { 111 | cim: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 112 | ta: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 113 | oea: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 114 | oer: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 115 | d: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 116 | ad: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 117 | ora: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 118 | rtt: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0}, 119 | ortt: {sum: undefined, sumSqr: undefined, min: undefined, max: undefined, cnt: 0} 120 | }; 121 | 122 | const measurementTime = {start: -1, stop: -1}; 123 | 124 | const sampleData = []; // array of samples (tuples of collected invocation data, for rule or for action, depending on the activity) 125 | 126 | var loopSleeper; // used to abort sleep in mainLoop() 127 | var abort = false; // used to abort the loop in mainLoop() 128 | 129 | // Used only at the master 130 | var workerData = []; // holds data for each worker, at [1..#workers]. Master's entry is 0. 131 | 132 | const activity = ((cluster.isWorker || !testRecord.input.master_activity) ? testRecord.input.activity : testRecord.input.master_activity); 133 | 134 | if (cluster.isMaster) 135 | runMaster(); 136 | else 137 | runWorker(); 138 | 139 | // -------- END OF MAIN ------------- 140 | 141 | /** 142 | * Master operation 143 | */ 144 | function runMaster() { 145 | 146 | // Setup OpenWhisk assets for the test 147 | testSetup().then(() => { 148 | 149 | // Start workers, configure interaction 150 | for(var i = 0; i < testRecord.input.workers; i++) { 151 | if (i > 0) // fork only (workers - 1) times 152 | cluster.fork(); 153 | } 154 | 155 | for (const id in cluster.workers) { 156 | 157 | // Exit handler for each worker 158 | cluster.workers[id].on('exit', (code, signal) => { 159 | if (signal) 160 | mLog(`Worker ${id} was killed by signal: ${signal}`); 161 | else 162 | if (code !== 0) 163 | mLog(`Worker ${id} exited with error code: ${code}`); 164 | checkExit(); 165 | }); 166 | 167 | // Message handler for each worker 168 | cluster.workers[id].on('message', (msg) => { 169 | if (msg.init) 170 | // Initialization barrier for workers. Makes sure they are all fully engaged when the measurement start 171 | checkInit(); 172 | 173 | if (msg.summary) { 174 | workerData[id] = msg.summary; 175 | checkSummary(); 176 | } 177 | }); 178 | } 179 | 180 | mainLoop().then(() => { 181 | 182 | // set finish of measurement and notify all other workers 183 | measurementTime.stop = new Date().getTime(); 184 | testRecord.output.measure_time = (measurementTime.stop - measurementTime.start) / 1000.0; // measurement duration converted to seconds 185 | mLog(`Stop measurement. Start post-processing after ${testRecord.input.pp_delay} msec`); 186 | mLogSampleHeader(); 187 | for (const j in cluster.workers) 188 | cluster.workers[j].send({abort: measurementTime}); 189 | 190 | // The master's post-processing to generate its workerData 191 | sleep(testRecord.input.pp_delay) 192 | .then(() => { 193 | postProcess() 194 | .then(() => { 195 | // The master's workerData 196 | workerData[0] = {lat: latCounters, tp: tpCounters}; 197 | checkSummary(); 198 | }) 199 | .catch(err => { // FATAL - shouldn't happen unless BUG 200 | mLog(`Post-process ERROR in MASTER: ${err}`); 201 | throw err; 202 | }); 203 | }); 204 | }); 205 | 206 | }); 207 | 208 | } 209 | 210 | 211 | /** 212 | * Setup assets before the test depending on configuration 213 | */ 214 | async function testSetup() { 215 | 216 | if (!testRecord.input.setup) 217 | return; 218 | 219 | const cmd = `./setup.sh s ${testRecord.input.ratio} ${testRecord.input.event_trigger} ${testRecord.input.test_action} ${wskParams}`; 220 | mLog(`SETUP: ${cmd}`); 221 | 222 | try { 223 | await exec(cmd); 224 | } 225 | catch (error) { 226 | mLog(`FATAL: setup failure - ${error}`); 227 | process.exit(-2); 228 | } 229 | } 230 | 231 | 232 | /** 233 | * Teardown assets after the test depending on configuration 234 | */ 235 | async function testTeardown() { 236 | 237 | if (!testRecord.input.teardown) 238 | return; 239 | 240 | const cmd = `./setup.sh t ${testRecord.input.ratio} ${testRecord.input.event_trigger} ${testRecord.input.test_action} ${wskParams}`; 241 | mLog(`TEARDOWN: ${cmd}`); 242 | 243 | try { 244 | await exec(cmd); 245 | } 246 | catch (error) { 247 | mLog(`WARNING: teardown error - ${error}`); 248 | process.exit(-3); 249 | } 250 | } 251 | 252 | 253 | /** 254 | * Print table header for samples to the runtime log on stderr 255 | */ 256 | function mLogSampleHeader() { 257 | mLog("bi,\tas,\tae,\tts,\tta,\toea,\toer,\td,\tad,\tai,\tora,\trtt,\tortt"); 258 | } 259 | 260 | /** 261 | * Worker operation 262 | */ 263 | function runWorker() { 264 | 265 | // abort message from master will set the measurement time frame and abort the loop 266 | process.on('message', (msg) => { 267 | if (msg.abort) { 268 | // Set the measurement time frame at the worker - required for post-processing 269 | measurementTime.start = msg.abort.start; 270 | measurementTime.stop = msg.abort.stop; 271 | abortLoop(); 272 | } 273 | }); 274 | 275 | mainLoop().then(() => { 276 | sleep(testRecord.input.pp_delay) 277 | .then(() => { 278 | postProcess() 279 | .then(() => { 280 | process.send({summary:{lat: latCounters, tp:tpCounters}}); 281 | process.exit(); 282 | }) 283 | .catch(err => { // shouldn't happen unless BUG 284 | mLog(`Post-process ERROR in WORKER: ${err}`); 285 | throw err; 286 | }); 287 | }); 288 | }); 289 | } 290 | 291 | 292 | // Barrier for checking all workers have initialized and then start measurement 293 | 294 | var remainingInits = testRecord.input.workers; 295 | var remainingIterations = -1; 296 | 297 | function checkInit() { 298 | remainingInits--; 299 | if (remainingInits == 0) { // all workers are engaged (incl. master) - can start measurement 300 | mLog("All clients finished warmup. Start measurement."); 301 | measurementTime.start = new Date().getTime(); 302 | 303 | if (testRecord.input.period) 304 | setTimeout(abortLoop, testRecord.input.period); 305 | 306 | if (testRecord.input.iterations) 307 | remainingIterations = testRecord.input.iterations; 308 | } 309 | } 310 | 311 | // Barrier for checking all workers have finished, generate output and exit 312 | 313 | var remainingExits = testRecord.input.workers; 314 | 315 | function checkExit() { 316 | remainingExits--; 317 | if (remainingExits == 0) { 318 | mLog("All workers finished - generating output and exiting."); 319 | generateOutput(); 320 | // Cleanup test assets from OW and then exit 321 | testTeardown().then(() => { 322 | mLog("Done"); 323 | process.exit(); 324 | }); 325 | } 326 | } 327 | 328 | // Barrier for receiving post-processing results from all workers before computing final results 329 | 330 | var remainingSummaries = testRecord.input.workers; 331 | 332 | function checkSummary() { 333 | remainingSummaries--; 334 | if (remainingSummaries == 0) { 335 | mLogSampleHeader(); 336 | mLog("All clients post-processing completed - computing output.") 337 | computeOutputRecord(); 338 | checkExit(); 339 | } 340 | } 341 | 342 | 343 | /** 344 | * Main loop for invocations - invoke activity asynchronously once every (delta) msec until aborted 345 | */ 346 | async function mainLoop() { 347 | 348 | var warmupCounter = testRecord.input.warmup; 349 | const delta = ((cluster.isWorker || !testRecord.input.master_delta) ? testRecord.input.delta : testRecord.input.master_delta); 350 | const blocking = ((cluster.isWorker || !testRecord.input.master_blocking) ? testRecord.input.blocking : testRecord.input.master_blocking); 351 | const doBlocking = (blocking != NONE); 352 | const getResult = (blocking == RESULT); 353 | 354 | while (!abort) { 355 | 356 | // ---- 357 | // Pass init (worker - send message) after iterations 358 | if (warmupCounter == 0) { 359 | if (cluster.isMaster) 360 | checkInit(); 361 | else // worker - send init 362 | process.send({init: 1}); 363 | } 364 | 365 | if (warmupCounter >= 0) // take 0 down to -1 to make sure it does not trigger another init message 366 | warmupCounter--; 367 | // ---- 368 | 369 | // If iterations limit set, abort loop when finished iterations 370 | if (remainingIterations == 0) { 371 | abortLoop(); 372 | continue; 373 | } 374 | 375 | if (remainingIterations > 0) 376 | remainingIterations--; 377 | 378 | const si = new Date().getTime(); // SI = Start of Iteration timestamp 379 | 380 | var samples; 381 | 382 | if (activity == ACTION) 383 | samples = await invokeActions(testRecord.input.ratio, doBlocking, getResult); 384 | else 385 | samples = await invokeRules(); 386 | 387 | samples.forEach(sample => { 388 | sampleData.push(sample); 389 | }); 390 | 391 | const ei = new Date().getTime(); // EI = End of Iteration timestamp 392 | const duration = ei - si; 393 | if (duration > delta) 394 | updateLatSample("cim", (duration - delta)); // client iteration miss - to allow a user to select a reasonable delta between iterations 395 | tpCounters.iterations++; // count another completed iteration 396 | if (delta > duration) { 397 | loopSleeper = sleep(delta - duration); 398 | if (!abort) // check again to avoid race condition on loopSleeper 399 | await loopSleeper; 400 | } 401 | } 402 | } 403 | 404 | 405 | /** 406 | * Used to abort the mainLoop() function 407 | */ 408 | function abortLoop() { 409 | abort = true; 410 | if (loopSleeper) 411 | loopSleeper.resolve(); 412 | } 413 | 414 | 415 | /** 416 | * Invoke the predefined OW action a specified number of times without waiting using Promises (burst). 417 | * Returns a promise that resolves to an array of {id, isError}. 418 | */ 419 | function invokeActions(count, doBlocking, getResult) { 420 | return new Promise( function (resolve, reject) { 421 | const burst_bi = new Date().getTime(); 422 | var ipa = []; // array of invocation promises; 423 | for(var i = 0; i< count; i++) { 424 | ipa[i] = new Promise((resolve, reject) => { 425 | const bi = (testRecord.input.burst_timing ? burst_bi : new Date().getTime()); // default is BI per invocation 426 | ow.actions.invoke({name: testRecord.input.test_action, blocking: doBlocking, result: getResult, params: params}) 427 | // If returnedJSON is full activation or just activation ID then activation ID should be in "activationId" field 428 | // If returnedJSON is the result of the test action, then "activationId" is part of the returned result of the test action 429 | .then(returnedJSON => { 430 | var ai; // after invocation 431 | if (doBlocking) 432 | ai = new Date().getTime(); // only for blocking invocations, AI is meaningful 433 | resolve({aaid: returnedJSON.activationId, bi: bi, ai: ai}); 434 | }) 435 | .catch(err => { 436 | resolve({aaidError: err}); 437 | }); 438 | }); 439 | } 440 | 441 | Promise.all(ipa).then(ipArray => { 442 | resolve(ipArray); 443 | }).catch(err => { // Impossible to reach since no contained promise rejects 444 | reject(err); 445 | }); 446 | 447 | }); 448 | } 449 | 450 | 451 | /** 452 | * Invoke the predefined OW rules asynchronously and return a promise of an array with a single element of {id, isError} 453 | */ 454 | function invokeRules() { 455 | return new Promise( function (resolve, reject) { 456 | const bi = new Date().getTime(); 457 | const triggerSamples = []; 458 | // Fire trigger to invoke the rule 459 | ow.triggers.invoke({name: testRecord.input.event_trigger, params: params}) 460 | .then(triggerActivationIdJSON => { 461 | const triggerActivationId = triggerActivationIdJSON.activationId; 462 | triggerSamples.push({taid: triggerActivationId, bi: bi}); 463 | resolve(triggerSamples); 464 | }) 465 | .catch (err => { 466 | triggerSamples.push({taidError: err}); 467 | resolve(triggerSamples); 468 | }); 469 | }); 470 | } 471 | 472 | 473 | /** 474 | * This function processes the sampleData. Each sample is processed as following: 475 | * 1. A sample with error (TAID or AAID) is processed directly (not much to do beyond counting errors) 476 | * 2. An action sample - first attempt to retrieve activation, then process with it 477 | * 3. A rule sample - first convert to set of bound action samples (by processing the trigger activation), then process each action sample in step 2 above 478 | */ 479 | async function postProcess() { 480 | for(var i in sampleData) { 481 | const sample = sampleData[i]; 482 | if (activity == ACTION) { 483 | await processSampleWithAction(sample); 484 | } 485 | else { // activity == RULE 486 | if (sample.taidError) // TAID error - no need to retrieve bound actions - move to process the sample directly 487 | processSample(sample); 488 | else { // have valid TAID - retrieve bound action ids and then process 489 | const actionSamples = await getActionSamplesOfRules(sample); 490 | for(var j in actionSamples) 491 | await processSampleWithAction(actionSamples[j]); 492 | } 493 | } 494 | } 495 | } 496 | 497 | 498 | /** 499 | * Retrieve the activation ids of the actions bound to the trigger activation provided by id. 500 | * Failure to retrieve trigger activation for a valid activation id is considered a fatal error, since the activation must exist. 501 | * @param {*} triggerActivation 502 | */ 503 | function getActionSamplesOfRules(triggerSample) { 504 | return new Promise((resolve, reject) => { 505 | ow.activations.get({name: triggerSample.taid}) 506 | .then(triggerActivation => { 507 | triggerSample.ts = triggerActivation.start; 508 | var actionSamples = []; 509 | for(var i = 0; i < triggerActivation.logs.length; i++) { 510 | const boundActionRecord = JSON.parse(triggerActivation.logs[i]); 511 | const actionSample = Object.assign({}, triggerSample); 512 | if (boundActionRecord.success) 513 | actionSample.aaid = boundActionRecord.activationId; 514 | else 515 | actionSample.aaidError = boundActionRecord.error; 516 | actionSamples.push(actionSample); 517 | } 518 | resolve(actionSamples); 519 | }) 520 | .catch (err => { // FATAL: failed to retrieve trigger activation for a valid id 521 | mLog(`getActionSamplesOfRules returned ERROR: ${err}`); 522 | reject(err); 523 | }); 524 | }); 525 | } 526 | 527 | 528 | /** 529 | * Processing each action sample sequentially, i.e., wait until activation is retrieved before retrieving the next one. 530 | * Otherwise, concurrent retrieval of possibly thousands of activations and more, may cause issues. 531 | * Failure to retrieve activation record for a valid id is ok, assuming the action may have not completed yet. 532 | * @param {*} actionSample 533 | */ 534 | async function processSampleWithAction(actionSample) { 535 | if (actionSample.aaidError) // no activation, move on to processing sample with error 536 | processSample(actionSample); 537 | else { // have activation, try to get record 538 | var activation; 539 | try { 540 | activation = await ow.activations.get({name: actionSample.aaid}); 541 | } 542 | catch (err) { 543 | mLog(`Failed to retrieve activation for id: ${actionSample.aaid} for reason: ${err}`); 544 | } 545 | processSample(actionSample, activation); 546 | } 547 | } 548 | 549 | 550 | /** 551 | * Process a single sample + optional related action activation, updating latency and throughput counters 552 | * @param {*} sample 553 | */ 554 | function processSample(sample, activation) { 555 | 556 | const bi = sample.bi; 557 | 558 | if (bi < measurementTime.start || bi > measurementTime.stop) { // BI outside time frame. No further processing. 559 | mLog(`Sample discarded. BI exceeds measurement time frame`); 560 | return; 561 | } 562 | 563 | tpCounters.attempts++; // each sample invoked in the time frame counts as one invocation attempt 564 | 565 | if (sample.taidError) { // trigger activation failed - count one request, one error. No further processing. 566 | tpCounters.requests++; 567 | tpCounters.errors++; 568 | mLog(`Sample discarded. Trigger activation error: ${sample.taidError}`); 569 | return; 570 | } 571 | 572 | var ts; 573 | if (sample.ts) { 574 | ts = parseInt(sample.ts); 575 | 576 | if (ts >= measurementTime.start && ts <= measurementTime.stop) { // trigger activation in time frame - count one activation, one request 577 | tpCounters.activations++; 578 | tpCounters.requests++; 579 | } 580 | } 581 | else 582 | ts = undefined; 583 | 584 | if (sample.aaidError) { // action activation failed - count one request, one error. No further processing. 585 | tpCounters.requests++; 586 | tpCounters.errors++; 587 | mLog(`Sample discarded. Action activation error: ${sample.aaidError}`); 588 | return; 589 | } 590 | 591 | if (!activation) { // no activation, so assumed incomplete. No further processing. 592 | mLog(`Sample discarded. Activation was not retrieved.`) 593 | return; 594 | } 595 | 596 | const as = parseInt(activation.start); 597 | const ae = parseInt(activation.end); 598 | const d = parseInt(activation.response.result.duration); 599 | 600 | if (as < measurementTime.start || ae > measurementTime.stop) { // got activation, but it exceeds the time frame. No further processing. 601 | mLog(`Sample discarded. Action activation exceeded measurement time frame.`) 602 | return; 603 | } 604 | 605 | // Activation is in time frame, so count one activation, one request and one full invocation 606 | tpCounters.activations++; 607 | tpCounters.requests++; 608 | tpCounters.invocations++; 609 | 610 | // For full invocations, update latency counters 611 | 612 | const ta = (ts ? as - ts : undefined); 613 | const ad = ae - as; 614 | const oea = as - bi; 615 | const oer = ae - bi - d; 616 | 617 | updateLatSample("d", d); 618 | updateLatSample("ta", ta); 619 | updateLatSample("ad", ad); 620 | updateLatSample("oea", oea); 621 | updateLatSample("oer", oer); 622 | 623 | // for blocking action invocations - will be "undefined" otherwise 624 | const ai = sample.ai; 625 | 626 | const ora = (ai ? ai - ae : undefined); 627 | const rtt = (ai ? ai - bi : undefined); 628 | const ortt = (rtt ? rtt - d : undefined); 629 | 630 | updateLatSample("ora", ora); 631 | updateLatSample("rtt", rtt); 632 | updateLatSample("ortt", ortt); 633 | 634 | mLog(`${bi},\t${as},\t${ae},\t${ts},\t${ta},\t${oea},\t${oer},\t${d},\t${ad},\t${ai},\t${ora},\t${rtt},\t${ortt}`); 635 | } 636 | 637 | /** 638 | * Update counters of one latency statistic of a worker with value data from one sample 639 | */ 640 | function updateLatSample(statName, value) { 641 | 642 | if (!value) // value == undefined => skip it 643 | return; 644 | 645 | // increase sample counter 646 | latCounters[statName].cnt++; 647 | 648 | // Update sum for avg 649 | if (!latCounters[statName].sum) 650 | latCounters[statName].sum = 0; 651 | latCounters[statName].sum += value; 652 | 653 | // Update sumSqr for std 654 | if (!latCounters[statName].sumSqr) 655 | latCounters[statName].sumSqr = 0; 656 | latCounters[statName].sumSqr += value * value; 657 | 658 | // Update min value 659 | if (!latCounters[statName].min || latCounters[statName].min > value) 660 | latCounters[statName].min = value; 661 | 662 | // Update max value 663 | if (!latCounters[statName].max || latCounters[statName].max < value) 664 | latCounters[statName].max = value; 665 | } 666 | 667 | 668 | /** 669 | * Compute the final output record based on the workerData records. 670 | * The output of the program is a single CSV row of data consisting of the input parameters, 671 | * then latencies computed above - avg (average) and std (std. dev.), then throughput. 672 | */ 673 | function computeOutputRecord() { 674 | 675 | // Latency stats: avg, std, min, max 676 | ["ta", "oea", "oer", "d", "ad", "ora", "rtt", "ortt"].forEach(statName => { 677 | testRecord.output[statName] = computeLatStats(statName); 678 | }); 679 | 680 | // Tp stats: abs, tp, tpw, tpd 681 | ["attempts", "invocations", "activations", "requests"].forEach(statName => { 682 | testRecord.output[statName] = computeTpStats(statName); 683 | }); 684 | 685 | // Error stats: abs, percent 686 | testRecord.output.errors = computErrorStats(); 687 | } 688 | 689 | 690 | /** 691 | * Based on workerData, compute average and standard deviation of a given latency statistic. 692 | * @param {*} statName 693 | */ 694 | function computeLatStats(statName) { 695 | var totalSum = 0; 696 | var totalSumSqr = 0; 697 | var totalCount = 0; 698 | var min = undefined; 699 | var max = undefined; 700 | if (testRecord.input.master_apart) { // in master_apart mode, only master performs latency measurements 701 | totalSum = workerData[0].lat[statName].sum; 702 | totalSumSqr = workerData[0].lat[statName].sumSqr; 703 | min = workerData[0].lat[statName].min; 704 | max = workerData[0].lat[statName].max; 705 | totalCount = workerData[0].lat[statName].cnt; 706 | } 707 | else // in regular mode, all workers participate in latency measurements 708 | workerData.forEach(wd => { 709 | if (wd.lat[statName].cnt > 0) { // If this worker has valid latency samples 710 | totalSum += wd.lat[statName].sum; 711 | totalSumSqr += wd.lat[statName].sumSqr; 712 | if (!min || min > wd.lat[statName].min) 713 | min = wd.lat[statName].min; 714 | if (!max || max < wd.lat[statName].max) 715 | max = wd.lat[statName].max; 716 | } 717 | totalCount += wd.lat[statName].cnt; 718 | }); 719 | 720 | const avg = (totalCount > 0 ? totalSum / totalCount : undefined); 721 | const std = (totalCount > 0 ? Math.sqrt(totalSumSqr / totalCount - avg * avg) : undefined); 722 | 723 | return ({avg: avg, std: std, min: min, max: max, cnt: totalCount}); 724 | } 725 | 726 | 727 | /** 728 | * Based on workerData, compute throughput of a given counter, with (tp) and without (tpw) the master, and the percent difference (tpd) 729 | * @param {*} statName 730 | */ 731 | function computeTpStats(statName) { 732 | var masterCount = workerData[0].tp[statName]; 733 | var totalCount = 0; 734 | workerData.forEach(wd => {totalCount += wd.tp[statName];}); 735 | const tp = totalCount / testRecord.output.measure_time; // throughput 736 | const tpw = (totalCount - masterCount) / testRecord.output.measure_time; // throughput without master 737 | const tpd = (tp - tpw) * 100.0 / tp; // percent difference relative to TP 738 | 739 | return ({abs: totalCount, tp: tp, tpw: tpw, tpd: tpd}); 740 | } 741 | 742 | 743 | /** 744 | * Based on workerData, compute the relative portion of total errors out of total requests 745 | */ 746 | function computErrorStats() { 747 | var totalErrors = 0; 748 | var totalRequests = 0; 749 | 750 | workerData.forEach(wd => { 751 | totalErrors += wd.tp.errors; 752 | totalRequests += wd.tp.requests; 753 | }); 754 | 755 | const errAbs = totalErrors; 756 | const errPer = totalErrors * 100.0 / totalRequests; 757 | const cim = computeLatStats("cim"); 758 | 759 | // Compute the number of iterations that cim.cnt is relative to 760 | var iterations = 0; 761 | if (testRecord.input.master_apart) 762 | iterations = workerData[0].tp.iterations; 763 | else 764 | workerData.forEach(wd => {iterations += wd.tp.iterations;}); 765 | 766 | cim.pct = cim.cnt * 100.0 / iterations; // percent of delta miss out of total iterations 767 | 768 | return ({abs: errAbs, percent: errPer, cim: cim}); 769 | } 770 | 771 | 772 | /** 773 | * Generate a properly formatted output record to stdout. The header is also printed, but via mDump to stderr and can be 774 | * silenced. 775 | */ 776 | function generateOutput() { 777 | var first = true; 778 | 779 | // First, print header to stderr 780 | dfsObject(testRecord, (name, data, isRoot, isObj) => { 781 | if (!isObj) { // print leaf nodes 782 | if (!first) 783 | mWrite(",\t"); 784 | first = false; 785 | mWrite(`${name}`); 786 | } 787 | }); 788 | mWrite("\n"); 789 | 790 | first = true; 791 | 792 | // Now, print data to stdout 793 | dfsObject(testRecord, (name, data, isRoot, isObj) => { 794 | if (!isObj) { // print leaf nodes 795 | if (!first) 796 | process.stdout.write(",\t"); 797 | first = false; 798 | if (typeof data == 'number') // round each number to 3 decimal digits 799 | data = round(data, 3); 800 | process.stdout.write(`${data}`); 801 | } 802 | }); 803 | process.stdout.write("\n"); 804 | } 805 | 806 | 807 | /** 808 | * Sleep for a given time. Useful mostly with await from an async function 809 | * resolve and reject are externalized as properties to allow early abortion 810 | * @param {*} ms 811 | */ 812 | function sleep(ms) { 813 | var res, rej; 814 | var p = new Promise((resolve, reject) => { 815 | setTimeout(resolve, ms); 816 | res = resolve; 817 | rej = reject; 818 | }); 819 | p.resolve = res; 820 | p.reject = rej; 821 | 822 | return p; 823 | } 824 | 825 | 826 | /** 827 | * Generate a random integer in the range of [1..max] 828 | * @param {*} max 829 | */ 830 | function getRandomInt(max) { 831 | return Math.floor(Math.random() * Math.floor(max) + 1); 832 | } 833 | 834 | 835 | /** 836 | * Round a number after specified decimal digits 837 | * @param {*} num 838 | * @param {*} digits 839 | */ 840 | function round(num, digits = 0) { 841 | const factor = Math.pow(10, digits); 842 | return Math.round(num * factor) / factor; 843 | } 844 | 845 | 846 | // If not quiet, emit control messages on stderr (with newline) 847 | function mLog(text) { 848 | if (!testRecord.input.quiet) 849 | console.error(`${clientId()}:\t${text}`); 850 | } 851 | 852 | 853 | /** 854 | * Return the id of the client - MASTER-0 or WORKER-k (k=1..w-1) 855 | */ 856 | function clientId() { 857 | return (cluster.isMaster ? "MASTER-0" : `WORKER-${cluster.worker.id}`); 858 | } 859 | 860 | 861 | // If not quiet, write strings on stderr (w/o newline) 862 | function mWrite(text) { 863 | if (!testRecord.input.quiet) 864 | process.stderr.write(text); 865 | } 866 | 867 | /** 868 | * Traverse a (potentially deep) object in DFS, visiting each non-function node with function f 869 | * @param {*} data 870 | * @param {*} func 871 | */ 872 | function dfsObject(data, func, allowInherited = false) { 873 | var isRoot = true; 874 | var rootObj = data; 875 | crawlObj("", data, func, allowInherited); 876 | 877 | function crawlObj(name, data, f, allowInherited) { 878 | var isObj = (typeof data == 'object'); 879 | var isFunc = (typeof data == 'function'); 880 | if (!isFunc) 881 | f(name, data, isRoot, isObj); // visit the current node 882 | isRoot = false; 883 | if (isObj) 884 | for (var child in data) { 885 | if (allowInherited || data.hasOwnProperty(child)) { 886 | const childName = (name == "" ? child : name + "." + child); 887 | crawlObj(childName, data[child], f, true); // After root level no need to check inheritance 888 | } 889 | } 890 | } 891 | } 892 | -------------------------------------------------------------------------------- /owperf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # This is a simple launch script for owperf 21 | 22 | node owperf.js $@ 23 | -------------------------------------------------------------------------------- /owperf_data.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/owperf/8ece0164841ddec6a6f19f4a7bb7d525afca4b60/owperf_data.odg -------------------------------------------------------------------------------- /owperf_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/owperf/8ece0164841ddec6a6f19f4a7bb7d525afca4b60/owperf_data.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "owperf", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "owperf.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "btoa": "^1.2.1", 13 | "child-process-promise": "^2.2.1", 14 | "cluster": "^0.7.7", 15 | "commander": "^2.19.0", 16 | "ini": "^1.3.5", 17 | "node-exec-promise": "^1.0.2", 18 | "openwhisk": "^3.18.0", 19 | "xmlhttprequest": "^1.8.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | set -x 21 | 22 | # Setup for overhead test: create action, create trigger, and create a number of rules as required. 23 | # This script performs both setup and teardown 24 | # Designed to be an idempotent operation (can be applied repeatedly for the same result) 25 | # Usage: setup.sh [op] [ratio] 26 | # op - MANDATORY. "s" for setup, "t" for teardown 27 | # ratio - MANDATORY. ratio as defined in README.md 28 | # wsk global flags - OPTIONAL. Global flags for the wsk command (e.g. for specifying non-default wsk API host, auth, etc) 29 | 30 | MAXRULES=30 # assume no more than 30 rules per trigger max 31 | op=$1 # s for setup, t for teardown 32 | count=$2 # ratio for rules 33 | trigger=$3 # name of trigger 34 | action=$4 # name of action 35 | rulepfx="rule-$trigger-$action" 36 | delcount=$count # For teardown, delete ratio rules. For setup, delete MAXRULES 37 | if [ "$op" = "s" ]; then 38 | delcount=$MAXRULES 39 | fi 40 | 41 | shift 4 42 | wskparams="$@" # All other parameters are assumed to be OW-specific 43 | 44 | 45 | function remove_assets() { 46 | 47 | # Delete rules 48 | for i in $(seq 1 $delcount); do 49 | wsk rule delete $rulepfx-$i $wskparams; 50 | done 51 | 52 | # Delete trigger 53 | wsk trigger delete $trigger $wskparams 54 | 55 | # Delete action 56 | wsk action delete $action $wskparams 57 | 58 | } 59 | 60 | 61 | function deploy_assets() { 62 | 63 | # Create action 64 | wsk action create $action testAction.js --kind nodejs:8 $wskparams 65 | 66 | # Create trigger after deleting it 67 | wsk trigger create $trigger $wskparams 68 | 69 | # Create rules 70 | for i in $(seq 1 $count); do 71 | wsk rule create $rulepfx-$i $trigger $action $wskparams; 72 | done 73 | 74 | } 75 | 76 | 77 | # Always start with removal of existing assets 78 | remove_assets 79 | 80 | # If setup requested, deploy new assets 81 | if [ "$op" = "s" ]; then 82 | deploy_assets 83 | fi 84 | 85 | -------------------------------------------------------------------------------- /testAction.js: -------------------------------------------------------------------------------- 1 | /** 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | */ 19 | 20 | 21 | /** 22 | * Default test action for owperf. Sleeps specified time. 23 | * All test actions should return the invocation parameters (to stress the return path), but augmented with the execution duration and the activation id. 24 | * Use this code as reference if you want to create a custom test action. 25 | */ 26 | 27 | function sleep(ms) { 28 | return new Promise(resolve => setTimeout(resolve, ms)); 29 | } 30 | 31 | async function main(params) { 32 | var start = new Date().getTime(); 33 | params.activationId = process.env.__OW_ACTIVATION_ID; 34 | await sleep(parseInt(params.sleep)); 35 | var end = new Date().getTime(); 36 | params.duration = end - start; 37 | return params; 38 | } 39 | 40 | // main().then(dur => console.log(dur)); 41 | 42 | --------------------------------------------------------------------------------