├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── buddhi ├── .gitignore ├── .rspec ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── Makefile ├── README.md ├── Rakefile ├── exe │ └── perftest-toolkit-buddhi ├── lib │ └── amp │ │ ├── toolkit.rb │ │ └── toolkit │ │ ├── 3scale.rb │ │ ├── 3scale │ │ ├── client_factory.rb │ │ └── helper.rb │ │ ├── buddhi.rb │ │ └── buddhi │ │ ├── cli.rb │ │ ├── factory.rb │ │ ├── main.rb │ │ ├── profiles.rb │ │ ├── profiles │ │ ├── backend.rb │ │ ├── medium.rb │ │ ├── multiservice.rb │ │ ├── register.rb │ │ ├── simple.rb │ │ └── standard.rb │ │ ├── service.rb │ │ └── version.rb ├── perftest-toolkit-buddhi.gemspec └── spec │ └── spec_helper.rb ├── deployment ├── ansible.cfg ├── benchmarks │ └── 3scale-benchmark.yaml.j2 ├── doc │ ├── deploy-upstream-api.md │ ├── infrastructure.odp │ └── infrastructure.png ├── group_vars │ └── all.yml ├── hosts ├── injector.yml ├── inventory │ └── 3scale_inventory_plugin.py ├── profiled-injector.yml ├── requirements.yaml ├── roles │ ├── hyperfoil_generate_report │ │ └── tasks │ │ │ └── main.yml │ ├── injector-setup │ │ └── tasks │ │ │ └── main.yml │ ├── platform-setup │ │ ├── tasks │ │ │ └── main.yml │ │ └── vars │ │ │ ├── CentOS.yml │ │ │ ├── Fedora.yml │ │ │ ├── RedHat.yml │ │ │ └── Ubuntu.yml │ ├── profiled-traffic-generator │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ ├── traffic-distributor │ │ └── tasks │ │ │ └── main.yml │ ├── traffic-retriever │ │ └── tasks │ │ │ └── main.yml │ ├── upstream-configurator │ │ ├── README.md │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── etc │ │ │ │ └── apicast_config.json │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ ├── tests │ │ │ ├── inventory │ │ │ └── test.yml │ │ └── vars │ │ │ └── main.yml │ └── user-traffic-reader │ │ ├── README.md │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── meta │ │ └── main.yml │ │ ├── tasks │ │ └── main.yml │ │ ├── tests │ │ ├── inventory │ │ └── test.yml │ │ └── vars │ │ └── main.yml ├── run.yml └── upstream.yml └── locust ├── README.md ├── kill.sh ├── locustfile.py └── start.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /test/tmp/ 10 | /test/version_tmp/ 11 | /tmp/ 12 | 13 | # Used by dotenv library to load environment variables. 14 | # .env 15 | 16 | ## Specific to RubyMotion: 17 | .dat* 18 | .repl_history 19 | build/ 20 | *.bridgesupport 21 | build-iPhoneOS/ 22 | build-iPhoneSimulator/ 23 | 24 | ## Specific to RubyMotion (use of CocoaPods): 25 | # 26 | # We recommend against adding the Pods directory to your .gitignore. However 27 | # you should judge for yourself, the pros and cons are mentioned at: 28 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 29 | # 30 | # vendor/Pods/ 31 | 32 | ## Documentation cache and generated files: 33 | /.yardoc/ 34 | /_yardoc/ 35 | /doc/ 36 | /rdoc/ 37 | 38 | ## Environment normalization: 39 | /.bundle/ 40 | /vendor/bundle 41 | /lib/bundler/man/ 42 | 43 | # for a library or gem, you might want to ignore these files since the code is 44 | # intended to run in multiple environments; otherwise, check them in: 45 | # Gemfile.lock 46 | # .ruby-version 47 | # .ruby-gemset 48 | 49 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 50 | .rvmrc 51 | 52 | # IDE generated files: 53 | .idea/ 54 | 55 | # Benchmarks: 56 | deployment/benchmarks/*.html 57 | deployment/benchmarks/*.csv 58 | deployment/inventory/__pycache__/ 59 | 60 | # Locust 61 | locust/*.csv 62 | locust/__pycache__/ 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/). 6 | 7 | ## [2.4.0] - 2020-10-15 8 | 9 | ### Added 10 | 11 | - Added Medium profile [PR #72](https://github.com/3scale/perftest-toolkit/pull/72) 12 | - Hyperfoil 0.15.0 [PR #34](https://github.com/3scale/perftest-toolkit/pull/34) 13 | - Traffic profiles with self-managed deployment option [PR #67](https://github.com/3scale/perftest-toolkit/pull/67) 14 | 15 | ## [2.2.2] - 2020-10-15 16 | 17 | ### Fixed 18 | 19 | - fix: old 3scale releases returned 404 Not found on backend endpoints [PR #55](https://github.com/3scale/perftest-toolkit/pull/55) 20 | 21 | ## [2.2.1] - 2020-06-02 22 | 23 | ### Fixed 24 | 25 | - fix standard concurrency [PR #49](https://github.com/3scale/perftest-toolkit/pull/49) 26 | 27 | ## [2.2.0] - 2020-06-02 28 | 29 | ### Added 30 | 31 | - Provision standard profile concurrently [PR #46](https://github.com/3scale/perftest-toolkit/pull/46) 32 | 33 | ## [2.1.0] - 2020-05-26 34 | 35 | ### Added 36 | 37 | - Added Simple and Backend profile [PR #36](https://github.com/3scale/perftest-toolkit/pull/36) 38 | - Added Standard profile [PR #40](https://github.com/3scale/perftest-toolkit/pull/40) 39 | 40 | ## [2.0.0] - 2020-04-03 41 | 42 | ### Added 43 | 44 | - Testing setup fetched from 3scale API [PR #31](https://github.com/3scale/perftest-toolkit/pull/31) 45 | - Jmeter upgraded to 5.2 [PR #31](https://github.com/3scale/perftest-toolkit/pull/31) 46 | 47 | ### Fixes 48 | 49 | - [Deployment] do not require interactive mode for 'docker run' command [PR #24](https://github.com/3scale/perftest-toolkit/pull/24) 50 | 51 | ## [1.1.0] - 2018-06-07 52 | 53 | ### Added 54 | 55 | - [Buddhi] metric reporter service api [PR #6](https://github.com/3scale/perftest-toolkit/pull/6) 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # perftest-toolkit 2 | 3 | [![Docker Repository on Quay](https://quay.io/repository/3scale/perftest-toolkit/status "Docker Repository on Quay")](https://quay.io/repository/3scale/perftest-toolkit) 4 | 5 | This repo has tools and deployment configs for a performance testing environment to be able to run performance tests of a 3scale API Management solution, focusing on the traffic intensive parts of the solution (the API Gateway and the Service Management API). 6 | 7 | We have open sourced it to enable partners, customers and support engineers to run their own performance tests on "self-managed" (i.e. Not SaaS) installations of the 3scale API Management solution. 8 | 9 | By running performance test with the same tools, scripts, traffic patterns and measurements as we at 3scale do, we hope it will help produce results that can be more easily compared with the results we achieve in our regular in-house performance testing and that we can run internally. 10 | 11 | The goal is to help to resolve doubts or issues related to scalability or performance more quickly and easily - allowing you to achieve the high levels of low-latency performance we strive for and ensure in our own internal testing. 12 | 13 | ## Table of Contents 14 | 15 | * [High level overview](#high-level-overview) 16 | * [Prerequisites](#prerequisites) 17 | * [Deploy injector](#deploy-injector) 18 | * [Common settings](#common-settings) 19 | * [Test your 3scale services](#test-your-3scale-services) 20 | * [Setup traffic profiles](#setup-traffic-profiles) 21 | * [Run tests](#run-tests) 22 | * [Sustained load](#sustained-load) 23 | * [Troubleshooting](#troubleshooting) 24 | * [Check apicast gateway configuration](#check-apicast-gateway-configuration) 25 | * [Check backend listener traffic](#check-backend-listener-traffic) 26 | * [Check upstream service traffic](#check-upstream-service-traffic) 27 | 28 | Generated using [github-markdown-toc](https://github.com/ekalinin/github-markdown-toc) 29 | 30 | ## High level overview 31 | 32 | High level overview is quite simple. Main components are represented in the diagram below. 33 | 34 | * Injector: Source of HTTP/HTTPS traffic requests 35 | * Openshift Container Platform with 3scale installed 36 | * Upstream API: Also named as backend API, this is the final API service as an HTTP traffic endpoint. Optionally, for testing purposes, [deploy Upstream API](deployment/doc/deploy-upstream-api.md). 37 | * Test Configurator (Buddhi): 3scale setup and traffic generation tool 38 | 39 | ![Test setup](deployment/doc/infrastructure.png "Infrastructure") 40 | 41 | ## Prerequisites 42 | * An OpenShift cluster with 3scale installed. You can also use the OpenShift cluster to host your Upstream API (or you can host it elsewhere). 43 | * A machine to run the hyperfoil injector (for example, an AWS EC2 Instance). Keep in mind this machine will be running performance tests so make sure it has sufficient compute resources to not be the bottleneck. 44 | * This machine can serve as both the control node and the managed node for the injector _**or**_ it can just be the managed node. For example, you could use your local machine as the injector's control node and the remote machine as the injector's managed node. 45 | 46 | The perftest-toolkit will take care of: 47 | * Installing hyperfoil on the remote machine (e.g. the AWS EC2 instance) 48 | * Running [Buddhi](buddhi/README.md) 49 | * Creating 3scale products, backends, etc. (**only** if using [traffic profiles](buddhi/README.md#profiles)) 50 | 51 | ## Deploy injector 52 | 53 | There are **two** ways of running your tests and the injector has to be configured accordingly. 54 | 55 | * [Test your own 3scale services](#test-your-3scale-services): The injector will be custom configured to use your 3scale products (a.k.a. services). 56 | 57 | * [Setup traffic profiles](#setup-traffic-profiles): Configure your performance tests to use synthetically generated traffic based on traffic models. 58 | 59 | **Requirements**: 60 | 61 | Control node: 62 | * ansible >= 2.9.14 63 | * python >= 3.0 64 | * Install ansible requirements 65 | ```bash 66 | cd deployment 67 | ansible-galaxy install -r requirements.yaml 68 | ``` 69 | 70 | Managed node host: 71 | * Docker >= 1.12 72 | * python >= 2.7 73 | * docker-py >= 1.7.0 74 | 75 | Make sure that the injector host’s hardware resources is not the performance tests bottleneck. Enough cpu, memory and network resources should be available. 76 | 77 | ### Common settings 78 | 79 | **1.** Provide **hyperfoil controller** host IP address or DNS name and at least one **hyperfoil agent** host IP address or DNS name to the [deployment/hosts](deployment/hosts) file. For example: 80 | 81 | ``` 82 | upstream ansible_host=controllerhost.example.com ansible_user=root 83 | 84 | [hyperfoil_controller] 85 | controllerhost.example.com ansible_host=controllerhost.example.com ansible_user=root 86 | 87 | [hyperfoil_agent] 88 | agent1 ansible_host=agenthost.example.com ansible_user=root 89 | ``` 90 | 91 | **Note 01**: make sure defined ansible user has ssh login access to the host without password. 92 | 93 | **Note 02**: make sure hyperfoil controller host has ssh login access to the agent host without password. 94 | 95 | More than one hyperfoil agent can be configured. Useful when the injector becomes a bottleneck. For example to configure two agents: 96 | 97 | ``` 98 | upstream ansible_host=controllerhost.example.com ansible_user=root 99 | 100 | [hyperfoil_controller] 101 | controllerhost.example.com ansible_host=controllerhost.example.com ansible_user=root 102 | 103 | [hyperfoil_agent] 104 | agent1 ansible_host=agenthost01.example.com ansible_role=root 105 | agent2 ansible_host=agenthost02.example.com ansible_role=root 106 | ``` 107 | 108 | **2.** By default, the injector will generate HTTPS traffic on the port number 443. You can change this setting editing the `injector_hyperfoil_target_protocol` and `injector_hyperfoil_target_port` parameters in the [deployment/group_vars/all.yml](deployment/group_vars/all.yml) file. 109 | 110 | **3.** If you're having ssh issues when running ansible playbooks, try adding an ssh certificate to [deployment/ansible.cfg](deployment/ansible.cfg). Otherwise, remove the `-i "/path/to/ssh/certificate"` from this [line](https://github.com/3scale-labs/perftest-toolkit/blob/c906ca3349a34d9fe6e72d9b28570268387257fd/deployment/ansible.cfg#L11). 111 | 112 | ### Test your 3scale services 113 | 114 | Skip these steps if using traffic profiles. These steps will configure the injector to use your 3scale services. 115 | 116 | **1.** Configure the following settings in [deployment/roles/user-traffic-reader/defaults/main.yml](deployment/roles/user-traffic-reader/defaults/main.yml) file: 117 | * `threescale_portal_endpoint`: 3scale portal endpoint 118 | * `threescale_services`: Select the 3scale services you want to use for the tests. Leave it empty to use them all. 119 | 120 | ``` 121 | --- 122 | # defaults file for user-traffic-reader 123 | 124 | # URI that includes your password and portal endpoint in the following format: ://@. 125 | # The can be either the provider key or an access token for the 3scale Account Management API. 126 | # is the URL used to log into the admin portal. 127 | # Example: https://access-token@account-admin.3scale.net 128 | threescale_portal_endpoint: 129 | 130 | # Comma separated list of services (Id's or system names) 131 | # If empty, all available services will be used 132 | threescale_services: "" 133 | ``` 134 | 135 | **2.** Execute the playbook `injector.yml` to deploy injector. 136 | ```bash 137 | cd deployment/ 138 | ansible-playbook -i hosts injector.yml 139 | ``` 140 | 141 | ### Setup traffic profiles 142 | 143 | Skip these steps if testing your own 3scale services. These steps will set up 3scale services for performance testing. 144 | 145 | **1.** Configure the following settings in [deployment/roles/profiled-traffic-generator/defaults/main.yml](deployment/roles/profiled-traffic-generator/defaults/main.yml): 146 | * `threescale_portal_endpoint`: 3scale portal endpoint 147 | * `traffic_profile`: Currently [available profiles](buddhi/README.md#profiles): `simple, backend, medium, standard` 148 | * `private_base_url`: Private Base URL used for the tests. Make sure your private application behaves like an echo api service. 149 | * `public_base_url`: Optionally, configure the `Public Base URL` used for the tests for self-managed apicast environments. Otherwise, leave it empty. 150 | 151 | ``` 152 | --- 153 | # defaults file for profiled-traffic-generator 154 | 155 | # URI that includes your password and portal endpoint in the following format: ://@. 156 | # The can be either the provider key or an access token for the 3scale Account Management API. 157 | # is the URL used to log into the admin portal. 158 | # Example: https://access-token@account-admin.3scale.net 159 | threescale_portal_endpoint: 160 | 161 | # Used traffic for performance testing is not real traffic. 162 | # It is synthetically generated traffic based on traffic models. 163 | # Information about available traffic profiles (or test plans) can be found here: 164 | # https://github.com/3scale/perftest-toolkit/blob/master/buddhi/README.md#profiles 165 | # Currently available profiles: [ simple | backend | medium | standard ] 166 | traffic_profile: 167 | 168 | # Private Base URL 169 | # Make sure your private application behaves like an echo api service 170 | # example: https://echo-api.3scale.net:443 171 | private_base_url: 172 | 173 | # Public Base URL 174 | # Public address of your API gateway in the production environment. 175 | # Optional. When it is left empty, public base url will be the hosted gateway url 176 | # example: https://gw.example.com:443 177 | public_base_url: 178 | ``` 179 | 180 | **2.** Execute the playbook `profiled-injector.yml` to deploy injector. 181 | ```bash 182 | cd deployment/ 183 | ansible-playbook -i hosts profiled-injector.yml 184 | ``` 185 | 186 | ## Run tests 187 | 188 | **Note**: If you'd prefer to run the tests using [Locust](https://locust.io/), refer to [this guide](locust/README.md) and skip the below steps. 189 | 190 | **1.** Configure testing settings in [deployment/run.yml](https://github.com/3scale-labs/perftest-toolkit/blob/c906ca3349a34d9fe6e72d9b28570268387257fd/deployment/run.yml#L9-L11): 191 | 192 | ``` 193 | USERS_PER_SEC: Requests per second 194 | DURATION_SEC: Duration of the performance test in seconds 195 | SHARED_CONNECTIONS: Number of connections open per target HOST 196 | ``` 197 | 198 | **2.** Run tests 199 | 200 | ```bash 201 | ansible-playbook -i hosts -i benchmarks/3scale.csv run.yml 202 | ``` 203 | 204 | **3.** View Report 205 | 206 | The test results of the last execution are automatically stored in **deployment/benchmarks/.html**. 207 | The html file can be directly opened with your favorite web browser. 208 | 209 | ## Sustained load 210 | 211 | Some performance test are looking for *peak* and *sustained* traffic maximum performance. 212 | *Sustained* traffic is defined as traffic load where *Job Queue* size is always at low levels, or even empty. 213 | For *sustained* traffic performance benchmark, *Job Queue* must be monitorized. 214 | 215 | This is a small guideline to monitor *Job Queue* size: 216 | 217 | - Get backend redis pod 218 | 219 | ```bash 220 | $ oc get pods | grep redis 221 | backend-redis-2-nkrkk 1/1 Running 0 14d 222 | ``` 223 | 224 | - Get Job Queue size 225 | 226 | ```bash 227 | $ oc rsh backend-redis-2-nkrkk /bin/sh -i -c 'redis-cli -n 1 llen resque:queue:priority' 228 | (integer) 0 229 | ``` 230 | 231 | ## Troubleshooting 232 | 233 | Sometimes, even though all deployment commands run successfully, performance traffic may be broken. 234 | This might be due to a misconfiguration in any stage of the deployment process. 235 | When performance HTTP traffic response codes are not as expected, i.e. **200 OK**, 236 | there are few checks that can be very handy to find out configuration mistakes. 237 | 238 | ### Check apicast gateway configuration 239 | 240 | First, scale down *apicast-production* service to just one pod. 241 | 242 | Monitor pod's logs for traffic accesslog. 243 | 244 | ```bash 245 | oc logs -f apicast-production-X-podId 246 | ``` 247 | 248 | [Run tests](#run-tests) and check for logs. 249 | 250 | Check response codes on accesslog. 251 | 252 | If accesslog shows *could not find service for host* error, then the configured virtual hosts do not match traffic *Host* header. For example: 253 | ``` 254 | 2018/06/05 13:32:41 [warn] 25#25: *883 [lua] errors.lua:43: get_upstream(): could not find service for host: 9ccd143c-dbe4-471c-9bce-41df7dde8d99.benchmark.perftest.3sca.net, client: 10.130.4.1, server: _, request: "GET /855aaf5c-a199-4145-a3ab-ea9402cc35db/some-request?user_key=32313d20d99780a5 HTTP/1.1", host: "9ccd143c-dbe4-471c-9bce-41df7dde8d99.benchmark.perftest.3sca.net" 255 | ``` 256 | 257 | Another issue might be when response codes are *404 Not Found*. Then proxy-rules do not match traffic path. 258 | 259 | In anyone of the previous cases, it seems that *apicast gateway* does not have the latest configuration. 260 | Either restart the pod(s) or wait until process fetches the new configuration based on 261 | *APICAST_CONFIGURATION_CACHE* apicast configuration parameter. 262 | 263 | The pods can be restarted by scaling the Deployment to 0 and then scaling back to the desired number of pods. 264 | 265 | ```bash 266 | oc scale deployment apicast-production --replicas=0 267 | oc scale deployment apicast-production --replicas=2 268 | ``` 269 | 270 | ### Check backend listener traffic 271 | 272 | First, scale down *backend-listener* service to just one pod. 273 | ```bash 274 | oc scale deployment backend-listener --replicas=1 275 | ``` 276 | 277 | Then monitor the pod's logs for traffic accesslog. 278 | ```bash 279 | oc logs -f backend-listener-X-podId 280 | ``` 281 | 282 | [Run tests](#run-tests) and check for logs. 283 | 284 | If no logs are shown, check [gateway troubleshooting section](#check-apicast-gateway-configuration) 285 | 286 | If logs are shown, check response codes on accesslog. Other than *200 OK* means 287 | - *redis* is down, 288 | - *redis* address is misconfigured in *backend-listener* 289 | - redis does not have required data to authenticate requests 290 | 291 | ### Check upstream service traffic 292 | 293 | When *backend-listener* accesslog shows requests are being answered with *200 OK* response codes, 294 | the last usual suspect is upstream or upstream configuration. 295 | 296 | Check *upstream* uri is correctly configured in your 3scale configuration. 297 | -------------------------------------------------------------------------------- /buddhi/.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /pkg/ 6 | /spec/reports/ 7 | /tmp/ 8 | /spec/examples.txt 9 | /vendor/cache/ 10 | /vendor/bundle/ 11 | -------------------------------------------------------------------------------- /buddhi/.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /buddhi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/3scale/perftest-toolkit:ruby2.7 2 | MAINTAINER Eguzki Astiz Lezaun 3 | 4 | WORKDIR /usr/src/app 5 | COPY . . 6 | RUN gem build perftest-toolkit-buddhi.gemspec 7 | RUN gem install perftest-toolkit-buddhi-*.gem --no-document 8 | RUN adduser --home /home/buddhiuser buddhiuser 9 | WORKDIR /home/buddhiuser 10 | 11 | # clean up 12 | RUN rm -rf /usr/src/app 13 | 14 | # Drop privileges 15 | USER buddhiuser 16 | -------------------------------------------------------------------------------- /buddhi/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in perftest-toolkit-buddhi.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /buddhi/Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | perftest-toolkit-buddhi (2.3.0) 5 | 3scale-api (~> 1.4) 6 | concurrent-ruby (~> 1.1) 7 | slop (~> 4.4) 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | 3scale-api (1.4.0) 13 | concurrent-ruby (1.1.9) 14 | diff-lcs (1.3) 15 | rake (13.0.1) 16 | rspec (3.7.0) 17 | rspec-core (~> 3.7.0) 18 | rspec-expectations (~> 3.7.0) 19 | rspec-mocks (~> 3.7.0) 20 | rspec-collection_matchers (1.1.3) 21 | rspec-expectations (>= 2.99.0.beta1) 22 | rspec-core (3.7.1) 23 | rspec-support (~> 3.7.0) 24 | rspec-expectations (3.7.0) 25 | diff-lcs (>= 1.2.0, < 2.0) 26 | rspec-support (~> 3.7.0) 27 | rspec-mocks (3.7.0) 28 | diff-lcs (>= 1.2.0, < 2.0) 29 | rspec-support (~> 3.7.0) 30 | rspec-support (3.7.1) 31 | slop (4.9.1) 32 | 33 | PLATFORMS 34 | ruby 35 | 36 | DEPENDENCIES 37 | bundler 38 | perftest-toolkit-buddhi! 39 | rake (~> 13.0) 40 | rspec (~> 3.5) 41 | rspec-collection_matchers 42 | 43 | BUNDLED WITH 44 | 2.1.4 45 | -------------------------------------------------------------------------------- /buddhi/Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_TAG ?= master 2 | IMAGE_NAME ?= perftest-toolkit:$(IMAGE_TAG) 3 | LATEST_IMAGE ?= perftest-toolkit:latest 4 | REGISTRY ?= quay.io/3scale 5 | 6 | .PHONY: build, push, bash, deps, test, clean 7 | 8 | build: 9 | docker build -t $(IMAGE_NAME) . 10 | 11 | push: 12 | docker tag $(IMAGE_NAME) $(REGISTRY)/$(IMAGE_NAME) 13 | docker push $(REGISTRY)/$(IMAGE_NAME) 14 | docker tag $(IMAGE_NAME) $(REGISTRY)/$(LATEST_IMAGE) 15 | docker push $(REGISTRY)/$(LATEST_IMAGE) 16 | 17 | bash: 18 | docker run --rm -it $(IMAGE_NAME) bash 19 | 20 | deps: 21 | bundle install --path vendor/bundle 22 | 23 | test: deps 24 | bundle exec rake spec 25 | 26 | clean: 27 | rm -rf vendor 28 | rm -rf pkg 29 | rm -rf coverage 30 | -------------------------------------------------------------------------------- /buddhi/README.md: -------------------------------------------------------------------------------- 1 | # Buddhi - 3scale traffic file generation tool 2 | 3 | Responsibilities: 4 | 5 | * Generate CSV formatted file with **Host, Path** columns. 6 | ```bash 7 | $ cat traffic.csv 8 | "53f07c14-e35e-4bfa-b0b1-9d3a993fad14.benchmark.3sca.net","/1?app_id=ddfa9a8842a3822e&app_key=73418183a69b027a" 9 | "e75ef4f7-54da-4ec6-a4b2-33a163764385.benchmark.3sca.net","/1?app_id=5e4618aa57d801cd&app_key=fe4db52e5e86668f" 10 | "e75ef4f7-54da-4ec6-a4b2-33a163764385.benchmark.3sca.net","/11?app_id=ceeeb23abfd0adfd&app_key=fbdfae99a587811e" 11 | "31b75b9b-fbb4-4223-8736-b93c34676f04.benchmark.3sca.net","/1?user_key=aa5736e41a3888db" 12 | "e75ef4f7-54da-4ec6-a4b2-33a163764385.benchmark.3sca.net","/111?app_id=ca2f8ff8b0a8707c&app_key=4b349db5bb77b9db" 13 | ``` 14 | 15 | ## Usage 16 | 17 | ```shell 18 | docker run --rm quay.io/3scale/perftest-toolkit:v2.4.0 -h 19 | usage: buddhi [options] 20 | -P, --portal Admin portal endpoint 21 | -s, --services 3scale service list 22 | -e, --endpoint API upstream endpoint 23 | -p, --profile 3scale product profile. Valid profiles ["simple", "backend", "medium", "standard"] 24 | -o, --output output file 25 | -h, --help 26 | -v, --version print the version 27 | ``` 28 | 29 | `--services` and `--profile` are mutually exclusive options. 30 | 31 | * If `--services` is provided, the tool will inspect those services and generate traffic tool from them. 32 | * If `--profile` is provided, the tool will create a 3scale product with the given profile. Currently valid profiles are `simple, backend, medium, standard`. `--profile` option requires `--endpoint` option to be provided. 33 | 34 | ### Profiles 35 | 36 | * The **simple** profile defines: 37 | * One product 38 | * One mapping rule (for hits metric) 39 | * One application plan 40 | * One application plan limit (big enough to not be reached) 41 | * One application 42 | * One backend 43 | * The **backend** profile defines: 44 | * One product 45 | * One application plan 46 | * One application plan limit (big enough to not be reached) 47 | * One application 48 | * One backend 49 | * One method 50 | * One mapping rule (for the previous method) 51 | * The **medium** profile defines: 52 | * 1 Account 53 | * 500 Applications 54 | * 10 products 55 | * 1 application plan per product 56 | * 10 application plan limits per product 57 | * 50 application per plan 58 | * 5 backend usages per product 59 | * 50 backend (each product will be using 5 backends) 60 | * 10 methods 61 | * 10 mapping rules 62 | * The **standard** profile defines: 63 | * 1 Account 64 | * 10000 Applications 65 | * 100 products 66 | * 1 application plan per product 67 | * 10 application plan limits per product 68 | * 100 application per plan 69 | * 10 backend usages per product 70 | * 1000 backend (each product will be using 10 backends) 71 | * 50 methods 72 | * 50 mapping rules 73 | 74 | ## Development 75 | 76 | ## Build docker image 77 | 78 | ```shell 79 | make clean 80 | make build' 81 | ``` 82 | 83 | ## Releasing 84 | 85 | ```shell 86 | make push 87 | ``` 88 | -------------------------------------------------------------------------------- /buddhi/Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | 3 | begin 4 | require 'rspec/core/rake_task' 5 | RSpec::Core::RakeTask.new(:spec) 6 | task default: :spec 7 | rescue LoadError 8 | # no rspec for you 9 | end 10 | -------------------------------------------------------------------------------- /buddhi/exe/perftest-toolkit-buddhi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'amp/toolkit' 4 | 5 | AMP::Toolkit::Buddhi.main 6 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'slop' 3 | require 'yaml' 4 | require 'uri' 5 | require 'time' 6 | require 'csv' 7 | require 'net/http' 8 | require 'pathname' 9 | require 'concurrent' 10 | require '3scale/api' 11 | 12 | require 'amp/toolkit/3scale' 13 | require 'amp/toolkit/buddhi' 14 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/3scale.rb: -------------------------------------------------------------------------------- 1 | require 'amp/toolkit/3scale/helper' 2 | require 'amp/toolkit/3scale/client_factory' 3 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/3scale/client_factory.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module ThreeScale 4 | ## 5 | # Input param can be endpoint url or remote name 6 | # 7 | def self.client(portal_url) 8 | remote_client(**remote(portal_url)) 9 | end 10 | 11 | def self.remote(uri_str) 12 | uri = Helper.parse_uri(uri_str) 13 | 14 | authentication = uri.user 15 | uri.user = '' 16 | { authentication: authentication, endpoint: uri.to_s } 17 | end 18 | 19 | def self.remote_client(endpoint:, authentication:) 20 | ::ThreeScale::API.new(endpoint: endpoint, provider_key: authentication, verify_ssl: false) 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/3scale/helper.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module ThreeScale 4 | module Helper 5 | def self.random_lowercase_name 6 | [*('a'..'z')].sample(8).join 7 | end 8 | 9 | def self.parse_uri(uri) 10 | # raises error when remote_str is not string, but object or something else. 11 | uri_obj = URI(uri) 12 | # URI::HTTP is parent of URI::HTTPS 13 | # with single check both types are checked 14 | raise "invalid url: #{uri}" unless uri_obj.kind_of?(URI::HTTP) 15 | 16 | uri_obj 17 | end 18 | 19 | def self.create_service(client, public_base_url) 20 | service_name = "PERF_TEST_#{random_lowercase_name}" 21 | system_name = service_name.delete("\s").downcase 22 | deployment_option = if public_base_url.nil? 23 | 'hosted' 24 | else 25 | 'self_managed' 26 | end 27 | svc_params = { 'name' => service_name, 'system_name' => system_name, 28 | 'deployment_option' => deployment_option } 29 | svc_obj = client.create_service svc_params 30 | 31 | if (errors = svc_obj['errors']) 32 | raise "Service has not been created: #{errors}" 33 | end 34 | 35 | svc_obj 36 | end 37 | 38 | def self.create_application_plan(client, service) 39 | name = random_lowercase_name 40 | plan_params = { 41 | 'name' => name, 'state' => 'published', 'default' => false, 42 | 'custom' => false, 'system_name' => name 43 | } 44 | 45 | plan_obj = client.create_application_plan service.fetch('id'), plan_params 46 | if (errors = plan_obj['errors']) 47 | raise "Application plan has not been created: #{errors}" 48 | end 49 | 50 | plan_obj 51 | end 52 | 53 | def self.hits_metric(client, service) 54 | metrics = client.list_metrics service.fetch('id') 55 | if metrics.respond_to?(:has_key?) && (errors = metrics['errors']) 56 | raise "Service metrics not read: #{errors}" 57 | end 58 | 59 | hits_metric_obj = metrics.find { |metric| metric['system_name'] == 'hits' } 60 | raise "Missing hits metric in service #{service.fetch('id')}" if hits_metric_obj.nil? 61 | 62 | hits_metric_obj 63 | end 64 | 65 | def self.create_application_plan_limit(client, service, plan, metric_id) 66 | # Very high limit: 4294967295 / 3600 => 1.2 M req/second during one hour to go over limit 67 | limit_params = { 'period' => 'hour', 'value' => 2**32 - 1 } 68 | limit_obj = client.create_application_plan_limit( 69 | plan.fetch('id'), metric_id, limit_params 70 | ) 71 | if (errors = limit_obj['errors']) 72 | raise "Limit has not been created: #{errors}" 73 | end 74 | 75 | limit_obj 76 | end 77 | 78 | def self.delete_mapping_rules(client, service) 79 | mapping_rules = client.list_mapping_rules service.fetch('id') 80 | if mapping_rules.respond_to?(:has_key?) && (errors = mapping_rules['errors']) 81 | raise "Service mapping rules not read: #{errors}" 82 | end 83 | 84 | mapping_rules.each do |mapping_rule| 85 | client.delete_mapping_rule service.fetch('id'), mapping_rule.fetch('id') 86 | end 87 | end 88 | 89 | def self.create_mapping_rule(client, service, path) 90 | hits_metric_obj = hits_metric(client, service) 91 | mapping_rule_params = { 92 | 'metric_id' => hits_metric_obj.fetch('id'), 'pattern' => path, 93 | 'http_method' => 'GET', 94 | 'delta' => 1 95 | } 96 | 97 | mapping_rule_obj = client.create_mapping_rule service.fetch('id'), mapping_rule_params 98 | if (errors = mapping_rule_obj['errors']) 99 | raise "MappingRule has not been created: #{errors}" 100 | end 101 | 102 | mapping_rule_obj 103 | end 104 | 105 | def self.create_application(client, plan, account) 106 | app_params = { 107 | 'name' => "app_#{random_lowercase_name}", 108 | 'description' => "app #{random_lowercase_name}" 109 | } 110 | 111 | app_obj = client.create_application(account.fetch('id'), app_params, plan_id: plan.fetch('id')) 112 | if (errors = app_obj['errors']) 113 | raise "Application has not been created: #{errors}" 114 | end 115 | 116 | app_obj 117 | end 118 | 119 | def self.account(client) 120 | accounts = client.list_accounts 121 | if accounts.respond_to?(:has_key?) && (errors = accounts['errors']) 122 | raise "Accounts not read: #{errors}" 123 | end 124 | 125 | raise 'No accounts available' if accounts.length.zero? 126 | 127 | accounts[0] 128 | end 129 | 130 | def self.create_account(client) 131 | account_name = "account_#{random_lowercase_name}" 132 | account_obj = client.signup(name: account_name, username: account_name) 133 | if account_obj.respond_to?(:has_key?) && (errors = account_obj['errors']) 134 | raise "Account not created: #{errors}" 135 | end 136 | 137 | account_obj 138 | end 139 | 140 | def self.create_backend(client, private_base_url) 141 | attrs = { 142 | name: random_lowercase_name, 143 | private_endpoint: private_base_url, 144 | } 145 | 146 | backend_obj = client.create_backend(attrs) 147 | if backend_obj.respond_to?(:has_key?) && (errors = backend_obj['errors']) 148 | raise "Backend not created: #{errors}" 149 | end 150 | 151 | backend_obj 152 | end 153 | 154 | def self.create_backend_usage(client, product, backend, path) 155 | attrs = { 156 | backend_api_id: backend.fetch('id'), 157 | path: path 158 | } 159 | 160 | backend_usage_obj = client.create_backend_usage(product.fetch('id'), attrs) 161 | if backend_usage_obj.respond_to?(:has_key?) && (errors = backend_usage_obj['errors']) 162 | raise "Backend usage not created: #{errors}" 163 | end 164 | 165 | backend_usage_obj 166 | end 167 | 168 | def self.update_service_proxy(client, service, public_base_url) 169 | proxy = { 'endpoint' => public_base_url } 170 | 171 | proxy.compact! 172 | 173 | unless proxy.empty? 174 | new_proxy_attrs = client.update_proxy service.fetch('id'), proxy 175 | 176 | if (errors = new_proxy_attrs['errors']) 177 | raise "Service proxy not updated: #{errors}" 178 | end 179 | 180 | new_proxy_attrs 181 | end 182 | end 183 | 184 | def self.bump_proxy_conf(client, service) 185 | client.proxy_deploy service.fetch('id') 186 | end 187 | 188 | def self.promote_proxy_conf(client, service) 189 | sandbox_proxy_cfg = client.proxy_config_latest(service.fetch('id'), 'sandbox') 190 | if (errors = sandbox_proxy_cfg['errors']) 191 | raise "Sandbox Proxy config not read: #{errors}" 192 | end 193 | 194 | res = client.promote_proxy_config( 195 | service.fetch('id'), 196 | 'sandbox', 197 | sandbox_proxy_cfg.fetch('version'), 198 | 'production' 199 | ) 200 | if (errors = res['errors']) 201 | raise "Proxy not promoted: #{errors}" 202 | end 203 | 204 | res 205 | end 206 | 207 | def self.backend_hits_metric(client, backend) 208 | metrics = client.list_backend_metrics backend.fetch('id') 209 | if metrics.respond_to?(:has_key?) && (errors = metrics['errors']) 210 | raise "Backend metrics not read: #{errors}" 211 | end 212 | 213 | hits_metric_obj = metrics.find { |metric| metric['system_name'].include? 'hits' } 214 | raise "Missing hits metric in backend #{backend.fetch('id')}" if hits_metric_obj.nil? 215 | 216 | hits_metric_obj 217 | end 218 | 219 | def self.create_backend_mapping_rule(client, backend, method, path) 220 | mapping_rule_params = { 221 | 'metric_id' => method.fetch('id'), 'pattern' => path, 222 | 'http_method' => 'GET', 223 | 'delta' => 1 224 | } 225 | 226 | mapping_rule_obj = client.create_backend_mapping_rule backend.fetch('id'), mapping_rule_params 227 | if (errors = mapping_rule_obj['errors']) 228 | raise "Backend MappingRule has not been created: #{errors}" 229 | end 230 | 231 | mapping_rule_obj 232 | end 233 | 234 | def self.create_backend_method(client, backend) 235 | hits_metric_obj = backend_hits_metric(client, backend) 236 | attrs = { 237 | 'system_name' => random_lowercase_name, 238 | 'friendly_name' => random_lowercase_name, 239 | 'description' => random_lowercase_name 240 | } 241 | method_obj = client.create_backend_method(backend.fetch('id'), hits_metric_obj.fetch('id'), attrs) 242 | 243 | if (errors = method_obj['errors']) 244 | raise "Method has not been created: #{errors}" 245 | end 246 | 247 | method_obj 248 | end 249 | 250 | def self.update_private_endpoint(client, service, private_base_url) 251 | proxy = { api_backend: private_base_url } 252 | new_proxy_attrs = client.update_proxy service.fetch('id'), proxy 253 | 254 | if (errors = new_proxy_attrs['errors']) 255 | raise "Service proxy not updated: #{errors}" 256 | end 257 | 258 | new_proxy_attrs 259 | end 260 | 261 | # wait tries a block of code until it returns true, or the timeout is reached. 262 | # timeout give an upper limit to the amount of time this method will run 263 | # Some intervals may be missed if the block takes too long or the time window is too short. 264 | def self.wait(interval = 1.5, timeout = 30) 265 | raise 'wait expects block' unless block_given? 266 | 267 | end_time = Time.now + timeout 268 | until Time.now > end_time 269 | result = yield 270 | return if result == true 271 | 272 | sleep interval 273 | end 274 | 275 | raise "timed out after #{timeout} seconds" 276 | end 277 | 278 | def self.backends(client) 279 | b_list = client.list_backends 280 | if b_list.respond_to?(:has_key?) && (errors = b_list['errors']) 281 | raise "Backend list not read: #{errors}" 282 | end 283 | 284 | b_list 285 | end 286 | 287 | def self.backend_methods(client, backend) 288 | hits_metric_obj = backend_hits_metric(client, backend) 289 | m_list = client.list_backend_methods(backend.fetch('id'), hits_metric_obj.fetch('id')) 290 | if m_list.respond_to?(:has_key?) && (errors = m_list['errors']) 291 | raise "Backend list not read: #{errors}" 292 | end 293 | 294 | m_list 295 | end 296 | end 297 | end 298 | end 299 | end 300 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi.rb: -------------------------------------------------------------------------------- 1 | require 'amp/toolkit/buddhi/profiles' 2 | require 'amp/toolkit/buddhi/service' 3 | require 'amp/toolkit/buddhi/factory' 4 | require 'amp/toolkit/buddhi/cli' 5 | require 'amp/toolkit/buddhi/main' 6 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/cli.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | class CLI 5 | def self.cli_flags 6 | options = ::Slop::Options.new 7 | options.banner = 'usage: buddhi [options]' 8 | options.string '-P', '--portal', 'Admin portal endpoint', required: true 9 | options.string '-s', '--services', '3scale service list' 10 | options.string '-e', '--private-base-url', 'Private base URL' 11 | options.string '-b', '--public-base-url', 'Public base URL' 12 | options.string '-p', '--profile', "3scale product profile. Valid profiles #{Profiles::Register.profile_keys.map(&:to_s)}" do |profile| 13 | unless Profiles::Register.profile_keys.include? profile.to_sym 14 | raise Slop::Error, "Invalid profile: #{profile}" 15 | end 16 | end 17 | options.string '-o', '--output', 'output file', required: true 18 | 19 | options.on '-h', '--help' do 20 | help!(options) 21 | end 22 | 23 | options.on '-v', '--version', 'print the version' do 24 | puts Buddhi::VERSION 25 | exit 26 | end 27 | 28 | options 29 | end 30 | 31 | def self.run(args = ARGV) 32 | parser = ::Slop::Parser.new cli_flags 33 | begin 34 | result = parser.parse(args) 35 | result.to_hash.tap { |pargs| validate_args(pargs) } 36 | rescue Slop::Error => error 37 | error!(error, cli_flags) 38 | end 39 | end 40 | 41 | def self.error!(error, options) 42 | warn "ERROR: #{error.message}" 43 | warn 44 | warn options 45 | exit 1 46 | end 47 | 48 | def self.help!(options) 49 | puts options 50 | exit 51 | end 52 | 53 | def self.validate_args(pargs) 54 | raise Slop::Error, 'services or profile parameter is required' if pargs.fetch(:services).nil? && pargs.fetch(:profile).nil? 55 | 56 | raise Slop::Error, 'services and profile parameters are mutually exclusive' unless pargs.fetch(:services).nil? || pargs.fetch(:profile).nil? 57 | 58 | raise Slop::Error, 'admin portal not valid' unless Factory.validate_portal pargs.fetch(:portal) 59 | 60 | # if profile specified, private-base-url is required 61 | raise Slop::Error, 'private-base-url is required' if !pargs.fetch(:profile).nil? && pargs.fetch(:private_base_url).nil? 62 | end 63 | end 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/factory.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | class Factory 5 | def self.call(portal:, services:, output:, **_opts) 6 | client = ThreeScale.client(portal) 7 | File.open(output, 'w') do |file| 8 | service_ary(client, services).each do |service_id| 9 | Service.new(client, service_id).items.each do |host, path| 10 | file.puts %("#{host}","#{path}") 11 | end 12 | end 13 | end 14 | rescue StandardError => e 15 | warn "\e[1m\e[31m#{e.class}: #{e.message}\e[0m" 16 | exit(false) 17 | end 18 | 19 | def self.service_ary(client, services) 20 | if services.empty? 21 | client.list_services.map { |service| service.fetch('id') } 22 | else 23 | services.split(',') 24 | end 25 | end 26 | 27 | def self.validate_portal(portal_url) 28 | # parsing url before trying to create client 29 | # raises Invalid URL when syntax is incorrect 30 | ThreeScale::Helper.parse_uri(portal_url) 31 | ThreeScale.client(portal_url).list_accounts 32 | true 33 | rescue StandardError => e 34 | warn "\e[1m\e[31m#{e.class}: #{e.message}\e[0m" 35 | false 36 | end 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/main.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | def self.main 5 | opts = Buddhi::CLI.run 6 | 7 | unless opts.fetch(:profile).nil? 8 | service_id_list = Profiles::Register.call **opts 9 | opts[:services] = service_id_list.join(',') 10 | end 11 | 12 | puts "================== provisioning done, reading services" 13 | 14 | Buddhi::Factory.call **opts 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles.rb: -------------------------------------------------------------------------------- 1 | require 'amp/toolkit/buddhi/profiles/register' 2 | require 'amp/toolkit/buddhi/profiles/simple' 3 | require 'amp/toolkit/buddhi/profiles/backend' 4 | require 'amp/toolkit/buddhi/profiles/multiservice' 5 | require 'amp/toolkit/buddhi/profiles/medium' 6 | require 'amp/toolkit/buddhi/profiles/standard' 7 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/backend.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | 6 | # Backend profile defintion 7 | # 1 account 8 | # # 1 applications 9 | # 10 | # 1 product 11 | # # 1 Application plan per product 12 | # # 1 application plan limits per product 13 | # # 1 backends used per product 14 | # # 0 Method per product 15 | # # 0 MappingRules per product 16 | # 17 | # 1 backend 18 | # # 1 Method per backend 19 | # # 1 MappingRule per backend 20 | class Backend 21 | def self.call(portal:, private_base_url:, public_base_url:, **_opts) 22 | client = ThreeScale.client(portal) 23 | service = ThreeScale::Helper.create_service(client, public_base_url) 24 | ThreeScale::Helper.update_service_proxy(client, service, public_base_url) 25 | plan = ThreeScale::Helper.create_application_plan(client, service) 26 | account = ThreeScale::Helper.account(client) 27 | ThreeScale::Helper.create_application(client, plan, account) 28 | ThreeScale::Helper.delete_mapping_rules(client, service) 29 | begin 30 | backend = ThreeScale::Helper.create_backend(client, private_base_url) 31 | ThreeScale::Helper.create_backend_usage(client, service, backend, '/') 32 | backend_method = ThreeScale::Helper.create_backend_method(client, backend) 33 | ThreeScale::Helper.create_application_plan_limit(client, service, plan, backend_method.fetch('id')) 34 | ThreeScale::Helper.create_backend_mapping_rule(client, backend, backend_method, '/pets') 35 | rescue ::ThreeScale::API::HttpClient::ForbiddenError 36 | raise 'Provider account does not support backend profile. ' \ 37 | 'Upgrade account to API as Product model or choose another profile.' 38 | end 39 | ThreeScale::Helper.bump_proxy_conf(client, service) 40 | ThreeScale::Helper.promote_proxy_conf(client, service) 41 | return [service.fetch('id')] 42 | end 43 | end 44 | 45 | Register.register_profile(:backend) { |**opts| Backend.call(**opts) } 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/medium.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | # Medium profile defintion 6 | # 1 account 7 | # # 500 applications 8 | # 9 | # 10 products 10 | # # 1 Application plan per product 11 | # # 10 application plan limits per product 12 | # # 50 applications under each plan 13 | # # 5 backends used per product 14 | # 15 | # 50 backends 16 | # # 10 Methods per backend 17 | # # 10 MappingRules per backend 18 | Register.register_profile(:medium) do |**opts| 19 | opts[:services_n] = 10 20 | opts[:app_per_plan_n] = 50 21 | opts[:backend_per_svc_n] = 5 22 | opts[:methods_per_backend_n] = 10 23 | opts[:limits_per_product] = 10 24 | 25 | MultiService.call(**opts) 26 | end 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/multiservice.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | class MultiService 6 | THREADS_N = Integer(ENV.fetch("THREADS_N", "10")) 7 | 8 | def self.call(portal:, services_n:, **opts) 9 | client = ThreeScale.client(portal) 10 | account = ThreeScale::Helper.create_account(client) 11 | services = Concurrent::Array.new 12 | # array with the number of services for each thread 13 | thread_tasks = [services_n/THREADS_N]*THREADS_N 14 | # Remaining shared between threads 15 | thread_tasks[0, services_n%THREADS_N] = thread_tasks[0, services_n%THREADS_N].map { |x| x + 1 } 16 | 17 | # threads array 18 | threads = thread_tasks.each_with_index.map do |n_tasks, idx| 19 | Thread.new(idx, account, services, n_tasks, portal, opts) do |i, acc, s_list, n, p, opts| 20 | n.times do 21 | multi_service = MultiService.new(i, p, acc, opts) 22 | multi_service.run 23 | puts "Service #{multi_service.service_id}" 24 | s_list << multi_service.service_id 25 | rescue => e 26 | STDERR.puts e 27 | end 28 | end 29 | end 30 | 31 | # run all threads 32 | threads.each(&:join) 33 | 34 | services 35 | end 36 | 37 | attr_reader :client, :account, :service, :private_base_url, :public_base_url, :idx, 38 | :backend_per_svc_n, :app_per_plan_n, :methods_per_backend_n, :limits_per_product 39 | 40 | def initialize(idx, portal, account, opts) 41 | @idx = idx 42 | @client = ThreeScale.client(portal) 43 | @private_base_url = opts.fetch(:private_base_url) 44 | @public_base_url = opts.fetch(:public_base_url) 45 | @backend_per_svc_n = opts.fetch(:backend_per_svc_n) 46 | @app_per_plan_n = opts.fetch(:app_per_plan_n) 47 | @methods_per_backend_n = opts.fetch(:methods_per_backend_n) 48 | @limits_per_product = opts.fetch(:limits_per_product) 49 | @service = ThreeScale::Helper.create_service(client, public_base_url) 50 | @account = account 51 | end 52 | 53 | def service_id 54 | service.fetch('id') 55 | end 56 | 57 | def run 58 | ThreeScale::Helper.update_service_proxy(client, service, public_base_url) 59 | ThreeScale::Helper.delete_mapping_rules(client, service) 60 | plan = ThreeScale::Helper.create_application_plan(client, service) 61 | 62 | app_per_plan_n.times do |app_idx| 63 | ThreeScale::Helper.create_application(client, plan, account) 64 | end 65 | 66 | backends = nil 67 | begin 68 | backends = Array.new(backend_per_svc_n) do |backend_idx| 69 | new_backend(client, private_base_url, service, backend_idx) 70 | end 71 | rescue ::ThreeScale::API::HttpClient::ForbiddenError 72 | raise 'Provider account does not support backend profile. ' \ 73 | 'Upgrade account to API as Product model or choose another profile.' 74 | end 75 | 76 | method_iter = backends.lazy.flat_map do |backend| 77 | ThreeScale::Helper.backend_methods(client, backend) 78 | end 79 | 80 | method_iter.take(limits_per_product).each do |method| 81 | ThreeScale::Helper.create_application_plan_limit(client, service, plan, method.fetch('id')) 82 | end 83 | 84 | ThreeScale::Helper.bump_proxy_conf(client, service) 85 | ThreeScale::Helper.promote_proxy_conf(client, service) 86 | end 87 | 88 | private 89 | 90 | def new_backend(client, private_base_url, service, backend_idx) 91 | backend = ThreeScale::Helper.create_backend(client, private_base_url) 92 | ThreeScale::Helper.create_backend_usage(client, service, backend, "/v#{idx}/#{format('v%04d', backend_idx)}") 93 | methods_per_backend_n.times do |method_idx| 94 | backend_method = ThreeScale::Helper.create_backend_method(client, backend) 95 | ThreeScale::Helper.create_backend_mapping_rule(client, backend, backend_method, format('/v%04d', method_idx)) 96 | end 97 | 98 | backend 99 | end 100 | end 101 | end 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/register.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | class Register 6 | @profiles = {} 7 | def self.register_profile(key, &block) 8 | @profiles[key] = block 9 | end 10 | 11 | def self.profile_keys 12 | @profiles.keys 13 | end 14 | 15 | def self.call(profile:, public_base_url:, **options) 16 | public_base_url = nil if !public_base_url.nil? && public_base_url.empty? 17 | # profile is valid 18 | @profiles[profile.to_sym].call(public_base_url: public_base_url, **options) 19 | end 20 | end 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/simple.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | 6 | # Simple profile defintion 7 | # 1 account 8 | # # 1 applications 9 | # 10 | # 1 product 11 | # # 1 Application plan per product 12 | # # 1 application plan limits per product 13 | # # 1 backends used per product 14 | # # 1 Method per product 15 | # # 1 MappingRule per product 16 | # 17 | # 1 backend 18 | # # 0 Method per backend 19 | # # 0 MappingRule per backend 20 | class Simple 21 | def self.call(portal:, private_base_url:, public_base_url:, **_opts) 22 | client = ThreeScale.client(portal) 23 | service = ThreeScale::Helper.create_service(client, public_base_url) 24 | ThreeScale::Helper.update_service_proxy(client, service, public_base_url) 25 | plan = ThreeScale::Helper.create_application_plan(client, service) 26 | hits_metric_obj = ThreeScale::Helper.hits_metric(client, service) 27 | ThreeScale::Helper.create_application_plan_limit(client, service, plan, hits_metric_obj.fetch('id')) 28 | ThreeScale::Helper.delete_mapping_rules(client, service) 29 | ThreeScale::Helper.create_mapping_rule(client, service, '/pets') 30 | account = ThreeScale::Helper.account(client) 31 | ThreeScale::Helper.create_application(client, plan, account) 32 | begin 33 | backend = ThreeScale::Helper.create_backend(client, private_base_url) 34 | ThreeScale::Helper.create_backend_usage(client, service, backend, '/') 35 | rescue ::ThreeScale::API::HttpClient::ForbiddenError, ::ThreeScale::API::HttpClient::NotFoundError 36 | # 3scale Backends not supported 37 | ThreeScale::Helper.update_private_endpoint(client, service, private_base_url) 38 | end 39 | ThreeScale::Helper.bump_proxy_conf(client, service) 40 | ThreeScale::Helper.promote_proxy_conf(client, service) 41 | return [service.fetch('id')] 42 | end 43 | end 44 | 45 | Register.register_profile(:simple) { |**opts| Simple.call(**opts) } 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/profiles/standard.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | module Profiles 5 | # Standard profile defintion 6 | # 1 account 7 | # # 10000 applications 8 | # 9 | # 100 products 10 | # # 1 Application plan per product 11 | # # 10 application plan limits per product 12 | # # 100 applications under each plan 13 | # # 10 backends used per product 14 | # 15 | # 1000 backends 16 | # # 50 Methods per backend 17 | # # 50 MappingRules per backend 18 | Register.register_profile(:standard) do |**opts| 19 | opts[:services_n] = 100 20 | opts[:app_per_plan_n] = 100 21 | opts[:backend_per_svc_n] = 10 22 | opts[:methods_per_backend_n] = 50 23 | opts[:limits_per_product] = 10 24 | 25 | MultiService.call(**opts) 26 | end 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/service.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | class Service 5 | attr_reader :client, :service_id 6 | 7 | def initialize(client, service_id) 8 | @client = client 9 | @service_id = service_id 10 | end 11 | 12 | def items 13 | return [] if service_host.nil? 14 | 15 | # When multiple applications exist, each mapping rule will be authorized by a random app 16 | # For services with app_id && app_key (backend_version 2) 17 | # When multiple applications key exist, 18 | # each mapping rule will be authorized by a random app key 19 | get_mapping_rules = mapping_rules.select { |mr| mr.fetch('http_method') == 'GET' } 20 | url_ary = get_mapping_rules.map(&method(:build_url)) 21 | url_ary.map { |u| [u.host, u.request_uri] } 22 | end 23 | 24 | private 25 | 26 | def build_url(mapping_rule) 27 | URI::HTTP.build( 28 | host: service_host, 29 | path: cleaned_pattern(mapping_rule.fetch('pattern')), 30 | query: application_key_sample 31 | ) 32 | end 33 | 34 | def cleaned_pattern(pattern) 35 | pattern.chomp('$') 36 | end 37 | 38 | def mapping_rules 39 | product_mapping_rules + backend_mapping_rules 40 | end 41 | 42 | def product_mapping_rules 43 | client.list_mapping_rules service_id 44 | end 45 | 46 | def backend_mapping_rules 47 | backend_usages.flat_map do |backend_usage| 48 | client.list_backend_mapping_rules(backend_usage.fetch('backend_id')).map do |mp_rule| 49 | mp_rule.merge('pattern' => "#{backend_usage.fetch('path').chomp('/')}#{mp_rule['pattern']}") 50 | end 51 | end 52 | end 53 | 54 | def backend_usages 55 | client.list_backend_usages(service_id) 56 | rescue ::ThreeScale::API::HttpClient::ForbiddenError, ::ThreeScale::API::HttpClient::NotFoundError 57 | # 3scale Backends not supported 58 | [] 59 | end 60 | 61 | def applications 62 | @applications ||= fetch_service_applications 63 | end 64 | 65 | def service_host 66 | @service_host ||= parse_service_host 67 | end 68 | 69 | def parse_service_host 70 | endpoint_url = client.show_proxy(service_id).fetch('endpoint') 71 | if endpoint_url.empty? 72 | warn "service_id: #{service_id}: Production Public Base URL is empty" 73 | return nil 74 | end 75 | 76 | endpoint = ThreeScale::Helper.parse_uri(endpoint_url) 77 | endpoint.host 78 | end 79 | 80 | def fetch_service_applications 81 | client.list_applications(service_id: service_id) 82 | end 83 | 84 | def application_key_sample 85 | return nil if applications.empty? 86 | 87 | URI.encode_www_form(app_auth_params(applications.sample)) 88 | end 89 | 90 | def app_auth_params(app) 91 | if app['application_id'].nil? 92 | { 93 | user_key: app['user_key'] 94 | } 95 | else 96 | { 97 | app_id: app['application_id'], 98 | app_key: application_key(app) 99 | } 100 | end 101 | end 102 | 103 | def application_key(app) 104 | application_keys = client.list_application_keys(app['account_id'], app['id']) 105 | return nil if application_keys.empty? 106 | 107 | application_keys.sample['value'] 108 | end 109 | end 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /buddhi/lib/amp/toolkit/buddhi/version.rb: -------------------------------------------------------------------------------- 1 | module AMP 2 | module Toolkit 3 | module Buddhi 4 | VERSION = '2.4.0'.freeze 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /buddhi/perftest-toolkit-buddhi.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib/', __FILE__) 2 | $LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) 3 | 4 | require 'amp/toolkit/buddhi/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'perftest-toolkit-buddhi' 8 | spec.version = AMP::Toolkit::Buddhi::VERSION 9 | spec.authors = ['Eguzki Astiz Lezaun'] 10 | spec.email = ['eastizle@redhat.com'] 11 | 12 | spec.summary = '3scale AMP setup tool for testing' 13 | spec.description = 'Helper tool for 3scale AMP testing' 14 | spec.homepage = 'https://github.com/3scale/perftest-toolkit' 15 | spec.license = 'Apache-2.0' 16 | 17 | spec.files = Dir['{lib}/**/*.rb'] 18 | spec.files += Dir['{exe,resources}/*'] 19 | spec.files << 'README.md' 20 | # There is a bug in gem 2.7.6 and __FILE__ cannot be used. 21 | # It is expanded in rake release task with full path on the building host 22 | spec.files << 'perftest-toolkit-buddhi.gemspec' 23 | 24 | spec.bindir = 'exe' 25 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 26 | spec.require_paths = ['lib'] 27 | 28 | spec.add_development_dependency 'bundler' 29 | spec.add_development_dependency 'rake', '~> 13.0' 30 | spec.add_development_dependency 'rspec', '~> 3.5' 31 | spec.add_development_dependency 'rspec-collection_matchers' 32 | spec.add_dependency 'concurrent-ruby', '~> 1.1' 33 | spec.add_dependency '3scale-api', '~> 1.4' 34 | spec.add_dependency 'slop', '~> 4.4' 35 | spec.required_ruby_version = '>= 2.7' 36 | end 37 | -------------------------------------------------------------------------------- /buddhi/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # This file was generated by the `rspec --init` command. Conventionally, all 2 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 3 | # The generated `.rspec` file contains `--require spec_helper` which will cause 4 | # this file to always be loaded, without a need to explicitly require it in any 5 | # files. 6 | # 7 | # Given that it is always loaded, you are encouraged to keep this file as 8 | # light-weight as possible. Requiring heavyweight dependencies from this file 9 | # will add to the boot time of your test suite on EVERY test run, even for an 10 | # individual file that may not need all of that loaded. Instead, consider making 11 | # a separate helper file that requires the additional dependencies and performs 12 | # the additional setup, and require it from the spec files that actually need 13 | # it. 14 | # 15 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 16 | RSpec.configure do |config| 17 | # rspec-expectations config goes here. You can use an alternate 18 | # assertion/expectation library such as wrong or the stdlib/minitest 19 | # assertions if you prefer. 20 | config.expect_with :rspec do |expectations| 21 | # This option will default to `true` in RSpec 4. It makes the `description` 22 | # and `failure_message` of custom matchers include text for helper methods 23 | # defined using `chain`, e.g.: 24 | # be_bigger_than(2).and_smaller_than(4).description 25 | # # => "be bigger than 2 and smaller than 4" 26 | # ...rather than: 27 | # # => "be bigger than 2" 28 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 29 | end 30 | 31 | # rspec-mocks config goes here. You can use an alternate test double 32 | # library (such as bogus or mocha) by changing the `mock_with` option here. 33 | config.mock_with :rspec do |mocks| 34 | # Prevents you from mocking or stubbing a method that does not exist on 35 | # a real object. This is generally recommended, and will default to 36 | # `true` in RSpec 4. 37 | mocks.verify_partial_doubles = true 38 | end 39 | 40 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 41 | # have no way to turn it off -- the option exists only for backwards 42 | # compatibility in RSpec 3). It causes shared context metadata to be 43 | # inherited by the metadata hash of host groups and examples, rather than 44 | # triggering implicit auto-inclusion in groups with matching metadata. 45 | config.shared_context_metadata_behavior = :apply_to_host_groups 46 | 47 | # The settings below are suggested to provide a good initial experience 48 | # with RSpec, but feel free to customize to your heart's content. 49 | =begin 50 | # This allows you to limit a spec run to individual examples or groups 51 | # you care about by tagging them with `:focus` metadata. When nothing 52 | # is tagged with `:focus`, all examples get run. RSpec also provides 53 | # aliases for `it`, `describe`, and `context` that include `:focus` 54 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 55 | config.filter_run_when_matching :focus 56 | 57 | # Allows RSpec to persist some state between runs in order to support 58 | # the `--only-failures` and `--next-failure` CLI options. We recommend 59 | # you configure your source control system to ignore this file. 60 | config.example_status_persistence_file_path = "spec/examples.txt" 61 | 62 | # Limits the available syntax to the non-monkey patched syntax that is 63 | # recommended. For more details, see: 64 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 65 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 66 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 67 | config.disable_monkey_patching! 68 | 69 | # This setting enables warnings. It's recommended, but in some cases may 70 | # be too noisy due to issues in dependencies. 71 | config.warnings = true 72 | 73 | # Many RSpec users commonly either run the entire suite or an individual 74 | # file, and it's useful to allow more verbose output when running an 75 | # individual spec file. 76 | if config.files_to_run.one? 77 | # Use the documentation formatter for detailed output, 78 | # unless a formatter has already been configured 79 | # (e.g. via a command-line flag). 80 | config.default_formatter = "doc" 81 | end 82 | 83 | # Print the 10 slowest examples and example groups at the 84 | # end of the spec run, to help surface which specs are running 85 | # particularly slow. 86 | config.profile_examples = 10 87 | 88 | # Run specs in random order to surface order dependencies. If you find an 89 | # order dependency and want to debug it, you can fix the order by providing 90 | # the seed, which is printed after each run. 91 | # --seed 1234 92 | config.order = :random 93 | 94 | # Seed global randomization in this process using the `--seed` CLI option. 95 | # Setting this allows you to use `--seed` to deterministically reproduce 96 | # test failures related to randomization by passing the same `--seed` value 97 | # as the one that triggered the failure. 98 | Kernel.srand config.seed 99 | =end 100 | end 101 | -------------------------------------------------------------------------------- /deployment/ansible.cfg: -------------------------------------------------------------------------------- 1 | # project specific configuration for Ansible 2 | 3 | [defaults] 4 | inventory_plugins=inventory 5 | command_warnings=False 6 | 7 | [inventory] 8 | enable_plugins=3scale_inventory_plugin,ini 9 | 10 | [ssh_connection] 11 | ssh_args = -o ServerAliveInterval=30 -i "/path/to/ssh/certificate" 12 | pipelining = True -------------------------------------------------------------------------------- /deployment/benchmarks/3scale-benchmark.yaml.j2: -------------------------------------------------------------------------------- 1 | name: 3scale-benchmark 2 | agents: 3 | {% for agent in groups[hyperfoil_agent_group] %} 4 | {{ agent }}: {{ hostvars[agent]['ansible_host'] }}:{{ hyperfoil_agent_port }} 5 | {% endfor %} 6 | http: 7 | {% for host in groups[injector_target_group] %} 8 | - host: {{ injector_hyperfoil_target_protocol }}://{{ host }}:{{ injector_hyperfoil_target_port }} 9 | sharedConnections: {{ shared_connections }} 10 | {% endfor %} 11 | usersPerSec: {{ users_per_sec }} 12 | duration: {{ duration_sec }}s 13 | maxDuration: {{ duration_sec }}s 14 | scenario: 15 | - testSequence: 16 | - randomCsvRow: 17 | file: {{ csv_dest_file_path }} 18 | skipComments: 'True' 19 | removeQuotes: 'True' 20 | columns: 21 | 0: target-host 22 | 1: uri 23 | - template: 24 | pattern: ${target-host}:{{ injector_hyperfoil_target_port }} 25 | toVar: target-authority 26 | - httpRequest: 27 | authority: 28 | fromVar: target-authority 29 | GET: 30 | fromVar: uri 31 | headers: 32 | HOST: 33 | fromVar: target-host 34 | 35 | threads: {{ injector_hyperfoil_agent_threads }} 36 | -------------------------------------------------------------------------------- /deployment/doc/deploy-upstream-api.md: -------------------------------------------------------------------------------- 1 | ## Deploy Upstream API 2 | 3 | If you don’t want to use your own Upstream API, the following steps show how to deploy the *echo-api* Upstream test API. 4 | On 3scale, tests were carried out using an *echo-api* as backend api endpoint. 5 | The service will answer to http requests with response body including information from http requests. 6 | It is very very fast and response body tend to be very small. 7 | 8 | **Requirements**: 9 | 10 | Control node: 11 | * ansible >= 2.3.1.0 12 | 13 | Managed node host: 14 | * Docker >= 1.12 15 | * python >= 2.6 16 | * docker-py >= 1.7.0 17 | 18 | **Steps**: 19 | 20 | Checkout playbooks 21 | 22 | ```bash 23 | $ git clone git@github.com:3scale/perftest-toolkit.git 24 | $ cd deployment 25 | ``` 26 | 27 | Edit the *ansible_host* parameter of the ‘upstream’ entry in the ‘hosts’ file located at the root of the repository by replacing ** with the host IP address/DNS name of the machine where you want to install the *echo-api* test. 28 | 29 | For example: 30 | ``` 31 | upstream ansible_host=myupstreamhost.addr.com ansible_user=centos 32 | ``` 33 | 34 | Execute the playbook that installs and configures the *echo-api* upstream test API via Ansible. 35 | 36 | ```bash 37 | ansible-playbook -i hosts upstream.yml 38 | ``` 39 | 40 | After this, the *echo-api* service should be listening on port **8081** 41 | 42 | Test that the *echo-api* upstream test API has been installed and configured correctly. 43 | To do this you can test that the service responds correctly to HTTP requests: 44 | 45 | ```bash 46 | $ curl -v http://127.0.0.1:8081 47 | * About to connect() to 127.0.0.1 port 8081 (#0) 48 | * Trying 127.0.0.1… 49 | * Connected to 127.0.0.1 (127.0.0.1) port 8081 (#0) 50 | > GET / HTTP/1.1 51 | > User-Agent: curl/7.29.0 52 | > Host: 127.0.0.1:8081 53 | > Accept: */* 54 | > 55 | < HTTP/1.1 200 OK 56 | < Server: openresty/1.13.6.1 57 | < Date: Mon, 14 May 2018 15:38:38 GMT 58 | < Content-Type: text/plain 59 | < Transfer-Encoding: chunked 60 | < Connection: keep-alive 61 | < 62 | GET / HTTP/1.1 63 | User-Agent: curl/7.29.0 64 | Host: 127.0.0.1:8081 65 | Accept: */* 66 | * Connection #0 to host 127.0.0.1 left intact 67 | ``` 68 | 69 | Some considerations worth to be noted. Upstream API host’s hardware resources should not be performance tests bottleneck. 70 | Enough cpu, memory and network resources should be available. Upstream API endpoint can be any HTTP endpoint service. Some constraints: 71 | * Must be fast. Must never be performance bottleneck 72 | * Generated response body should be small. Network should not be bottleneck, unless this effect is what is being tested 73 | -------------------------------------------------------------------------------- /deployment/doc/infrastructure.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3scale-labs/perftest-toolkit/498cc8d6c8d1b205450046c767503356c377cced/deployment/doc/infrastructure.odp -------------------------------------------------------------------------------- /deployment/doc/infrastructure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3scale-labs/perftest-toolkit/498cc8d6c8d1b205450046c767503356c377cced/deployment/doc/infrastructure.png -------------------------------------------------------------------------------- /deployment/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | # variables file for group_vars 2 | --- 3 | csv_file_path: /tmp 4 | csv_file_name: 3scale.csv 5 | reports_path: /tmp/3scale-perftest/reports 6 | csv_dest_file_path: "{{ csv_file_path }}/{{ csv_file_name }}" 7 | toolkit_csv_file_path: benchmarks 8 | locust_csv_file_path: locust 9 | injector_target_group: injector_target 10 | hyperfoil_agent_port: 22 11 | hyperfoil_controller_port: 8090 12 | hyperfoil_agent_group: hyperfoil_agent 13 | hyperfoil_controller_group: hyperfoil_controller 14 | injector_hyperfoil_agent_threads: 4 15 | injector_hyperfoil_target_protocol: https 16 | injector_hyperfoil_target_port: 443 17 | hyperfoil_version: 0.23 18 | -------------------------------------------------------------------------------- /deployment/hosts: -------------------------------------------------------------------------------- 1 | upstream ansible_host= ansible_user= 2 | 3 | [hyperfoil_controller] 4 | ansible_host= ansible_user= 5 | 6 | [hyperfoil_agent] 7 | agent1 ansible_host= ansible_user= 8 | -------------------------------------------------------------------------------- /deployment/injector.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 3 | - hyperfoil_controller 4 | - hyperfoil_agent 5 | roles: 6 | - platform-setup 7 | become: yes 8 | - hosts: 9 | - hyperfoil_controller 10 | roles: 11 | - user-traffic-reader 12 | - hosts: 13 | - hyperfoil_controller 14 | roles: 15 | - traffic-retriever 16 | - hosts: 17 | - hyperfoil_controller 18 | - hyperfoil_agent 19 | roles: 20 | - traffic-distributor 21 | - hosts: hyperfoil_controller 22 | roles: 23 | - injector-setup 24 | -------------------------------------------------------------------------------- /deployment/inventory/3scale_inventory_plugin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | from __future__ import (absolute_import, division, print_function) 4 | __metaclass__ = type 5 | 6 | DOCUMENTATION = ''' 7 | name: 3scale_inventory_plugin 8 | author: Jeremy Whiting (jwhiting@r e d h a t . c o m) 9 | short_description: CSV inventory source 10 | description: 11 | - Get inventory hosts from a CSV file 12 | options: 13 | csv_data_file: 14 | description: path of CSV file to read 15 | required: True 16 | notes: 17 | - None 18 | ''' 19 | 20 | EXAMPLES = ''' 21 | ansible-playbook -i inventory -i hosts injector.yml 22 | ''' 23 | 24 | from ansible.errors import AnsibleError 25 | from ansible.plugins.inventory import BaseInventoryPlugin 26 | from ansible.plugins.inventory import Constructable 27 | from ansible.utils.display import Display 28 | from ansible.inventory.data import InventoryData 29 | 30 | display = Display() 31 | 32 | class InventoryModule(BaseInventoryPlugin, Constructable): 33 | ''' Host inventory parser for ansible using 3Scale data file as source. ''' 34 | 35 | NAME = '3Scale_inventory_plugin' 36 | 37 | def verify_file(self, path): 38 | """Return the possibility of a configuration file being consumable by this plugin.""" 39 | valid = path.endswith('.csv') or path.endswith('.CSV') 40 | return valid 41 | 42 | def parse(self, inventory, loader, path, cache=True): 43 | super(InventoryModule, self).parse(inventory, loader, path, cache) 44 | display.debug("File lookup path: %s" % path) 45 | data_file = open( path, "r") 46 | hostsset = set() 47 | for line in data_file: 48 | hostsset.add(line.split(',')[0].lstrip('"').rstrip('"')) 49 | 50 | self.inventory.add_group('injector_target') 51 | for host in hostsset: 52 | self.inventory.add_host(host, 'injector_target') 53 | self.inventory.set_variable(host, 'ansible_host', host) 54 | -------------------------------------------------------------------------------- /deployment/profiled-injector.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 3 | - hyperfoil_controller 4 | - hyperfoil_agent 5 | roles: 6 | - platform-setup 7 | become: yes 8 | - hosts: 9 | - hyperfoil_controller 10 | roles: 11 | - profiled-traffic-generator 12 | - hosts: 13 | - hyperfoil_controller 14 | roles: 15 | - traffic-retriever 16 | - hosts: 17 | - hyperfoil_controller 18 | - hyperfoil_agent 19 | roles: 20 | - traffic-distributor 21 | - hosts: hyperfoil_controller 22 | roles: 23 | - injector-setup 24 | -------------------------------------------------------------------------------- /deployment/requirements.yaml: -------------------------------------------------------------------------------- 1 | - src: hyperfoil.hyperfoil_setup 2 | version: 0.19.0 3 | - src: hyperfoil.hyperfoil_shutdown 4 | version: 0.19.0 5 | - src: hyperfoil.hyperfoil_test 6 | version: 0.19.0 7 | -------------------------------------------------------------------------------- /deployment/roles/hyperfoil_generate_report/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # role task file for hyperfoil_generate_report 2 | --- 3 | - name: Reports directory 4 | file: 5 | state: directory 6 | path: "{{ reports_path }}" 7 | recurse: yes 8 | mode: '0755' 9 | - name: Retrieve the report 10 | get_url: 11 | url: "http://{{ hyperfoil_controller_host }}:{{ hyperfoil_controller_port }}/run/{{ test_runid }}/report" 12 | dest: "{{ reports_path }}/{{ test_runid }}.html" 13 | -------------------------------------------------------------------------------- /deployment/roles/injector-setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # role tasks file for injector-setup 2 | --- 3 | - name: Set the hyperfoil controller host variable 4 | set_fact: 5 | hyperfoil_controller_host: "{{ groups[hyperfoil_controller_group][0] }}" 6 | - name: Check Controller status 7 | uri: 8 | url: "http://{{ hyperfoil_controller_host }}:{{ hyperfoil_controller_port }}/" 9 | register: hfc_status 10 | ignore_errors: True 11 | - name: Start Controller 12 | when: hfc_status.status == -1 13 | include_role: 14 | name: hyperfoil.hyperfoil_setup 15 | vars: 16 | hyperfoil_controller_args: "-Dio.hyperfoil.deploy.timeout=30000 -Dio.hyperfoil.controller.cluster.ip={{ hyperfoil_controller_host }}" 17 | -------------------------------------------------------------------------------- /deployment/roles/platform-setup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # role tasks file for platform-setup 2 | --- 3 | - name: Load platform package names 4 | include_vars: "{{ item }}" 5 | with_first_found: 6 | - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml" 7 | - "{{ ansible_distribution }}.yml" 8 | - name: Install Java 9 | package: 10 | name: "{{ java_package }}" 11 | state: present 12 | - name: Install unzip 13 | package: 14 | name: "{{ unzip_package }}" 15 | state: present 16 | -------------------------------------------------------------------------------- /deployment/roles/platform-setup/vars/CentOS.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java_package: 3 | - java-11-openjdk 4 | unzip_package: 5 | - unzip 6 | -------------------------------------------------------------------------------- /deployment/roles/platform-setup/vars/Fedora.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java_package: 3 | - java-11-openjdk 4 | unzip_package: 5 | - unzip 6 | -------------------------------------------------------------------------------- /deployment/roles/platform-setup/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java_package: 3 | - java-11-openjdk 4 | unzip_package: 5 | - unzip 6 | -------------------------------------------------------------------------------- /deployment/roles/platform-setup/vars/Ubuntu.yml: -------------------------------------------------------------------------------- 1 | --- 2 | java_package: 3 | - openjdk-11-jre 4 | unzip_package: 5 | - unzip 6 | -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for profiled-traffic-generator 3 | 4 | # URI that includes your password and portal endpoint in the following format: ://@. 5 | # The can be either the provider key or an access token for the 3scale Account Management API. 6 | # is the URL used to log into the admin portal. 7 | # Example: https://access-token@account-admin.3scale.net 8 | threescale_portal_endpoint: 9 | 10 | # Used traffic for performance testing is not real traffic. 11 | # It is synthetically generated traffic based on traffic models. 12 | # Information about available traffic profiles (or test plans) can be found here: 13 | # https://github.com/3scale/perftest-toolkit/blob/master/buddhi/README.md#profiles 14 | # Currently available profiles: [ simple | backend | medium | standard ] 15 | traffic_profile: 16 | 17 | # Private Base URL 18 | # Make sure your private application behaves like an echo api service 19 | # example: https://echo-api.3scale.net:443 20 | private_base_url: 21 | 22 | # Public Base URL 23 | # Public address of your API gateway in the production environment. 24 | # Optional. When it is left empty, public base url will be the hosted gateway url 25 | # example: https://gw.example.com:443 26 | public_base_url: 27 | -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for profiled-traffic-generator -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # tasks file for profiled-traffic-generator 2 | --- 3 | - name: Creates tmp traffic directory 4 | tempfile: 5 | state: directory 6 | register: tempdir_1 7 | 8 | - name: Run Buddhi to create traffic file 9 | docker_container: 10 | name: buddhi 11 | state: started 12 | user: root 13 | tty: True 14 | detach: no 15 | image: quay.io/3scale/perftest-toolkit:v2.4.0 16 | command: "perftest-toolkit-buddhi --portal {{ threescale_portal_endpoint }} --profile \"{{ traffic_profile }}\" --public-base-url \"{{ public_base_url }}\" --private-base-url \"{{ private_base_url }}\" -o /traffic/traffic.csv" 17 | volumes: 18 | - "{{ tempdir_1.path }}:/traffic:z" 19 | env: 20 | THREADS_N: "5" 21 | 22 | - name: Copy traffic file 23 | copy: 24 | src: "{{ tempdir_1.path }}/traffic.csv" 25 | dest: "{{csv_file_path}}/{{csv_file_name}}" 26 | remote_src: true 27 | 28 | - name: Remove the temporary file 29 | file: 30 | path: "{{ tempdir_1.path }}" 31 | state: absent 32 | when: tempdir_1.path is defined 33 | -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - profiled-traffic-generator -------------------------------------------------------------------------------- /deployment/roles/profiled-traffic-generator/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for profiled-traffic-generator -------------------------------------------------------------------------------- /deployment/roles/traffic-distributor/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # role tasks file for traffic-distribution 2 | --- 3 | - name: Distribute CSV inventory source file 4 | copy: 5 | src: "{{ playbook_dir }}/{{ toolkit_csv_file_path }}/{{ csv_file_name }}" 6 | dest: "{{ csv_file_path }}/" 7 | force: yes 8 | -------------------------------------------------------------------------------- /deployment/roles/traffic-retriever/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # role tasks file for traffic-retriever 2 | --- 3 | - name: Retrieve the CSV inventory file 4 | fetch: 5 | src: "{{ csv_file_path }}/{{ csv_file_name }}" 6 | dest: "{{ playbook_dir }}/{{ toolkit_csv_file_path }}/" 7 | flat: yes 8 | - name: Retrieve the CSV inventory file for locust 9 | fetch: 10 | src: "{{ csv_file_path }}/{{ csv_file_name }}" 11 | dest: "{{ playbook_dir | dirname }}/{{ locust_csv_file_path }}/" 12 | flat: yes 13 | -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3scale-labs/perftest-toolkit/498cc8d6c8d1b205450046c767503356c377cced/deployment/roles/upstream-configurator/README.md -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for upstream-configurator 3 | 4 | upstream_apicast_registry_name: "quay.io" 5 | upstream_apicast_image_tag: "3.2-stable" 6 | upstream_apicast_repository_name: "3scale/apicast" 7 | upstream_apicast_image: "{{ upstream_apicast_registry_name }}/{{ upstream_apicast_repository_name }}:{{ upstream_apicast_image_tag }}" 8 | upstream_apicast_workers: 4 9 | upstream_apicast_config_file_origin: "etc/apicast_config.json" 10 | upstream_apicast_config_file_destination: "/etc/apicast_config.json" 11 | -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/files/etc/apicast_config.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for upstream-configurator -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set apicast configuration file 3 | copy: 4 | src: "{{ upstream_apicast_config_file_origin }}" 5 | dest: "{{ upstream_apicast_config_file_destination }}" 6 | register: copy_result 7 | 8 | - name: Run echo api container 9 | docker_container: 10 | name: echo-api 11 | state: started 12 | tty: yes 13 | network_mode: host 14 | privileged: yes 15 | restart: "{{ copy_result.changed }}" 16 | volumes: 17 | - "{{ upstream_apicast_config_file_destination }}:{{ upstream_apicast_config_file_destination }}" 18 | env: 19 | THREESCALE_CONFIG_FILE: "{{ upstream_apicast_config_file_destination }}" 20 | APICAST_WORKERS: "{{ upstream_apicast_workers }}" 21 | image: "{{ upstream_apicast_image }}" 22 | -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - upstream-configurator -------------------------------------------------------------------------------- /deployment/roles/upstream-configurator/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for upstream-configurator -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for user-traffic-reader 3 | 4 | # URI that includes your password and portal endpoint in the following format: ://@. 5 | # The can be either the provider key or an access token for the 3scale Account Management API. 6 | # is the URL used to log into the admin portal. 7 | # Example: https://access-token@account-admin.3scale.net 8 | threescale_portal_endpoint: 9 | 10 | # Comma separated list of services (Id's or system names) 11 | # If empty, all available services will be used 12 | threescale_services: "" 13 | -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for user-traffic-reader -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # Optionally specify the branch Galaxy will use when accessing the GitHub 25 | # repo for this role. During role install, if no tags are available, 26 | # Galaxy will use this branch. During import Galaxy will access files on 27 | # this branch. If Travis integration is configured, only notifications for this 28 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 29 | # (usually master) will be used. 30 | #github_branch: 31 | 32 | # 33 | # platforms is a list of platforms, and each platform has a name and a list of versions. 34 | # 35 | # platforms: 36 | # - name: Fedora 37 | # versions: 38 | # - all 39 | # - 25 40 | # - name: SomePlatform 41 | # versions: 42 | # - all 43 | # - 1.0 44 | # - 7 45 | # - 99.99 46 | 47 | galaxy_tags: [] 48 | # List tags for your role here, one per line. A tag is a keyword that describes 49 | # and categorizes the role. Users find roles by searching for tags. Be sure to 50 | # remove the '[]' above, if you add tags to this list. 51 | # 52 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 53 | # Maximum 20 tags per role. 54 | 55 | dependencies: [] 56 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 57 | # if you add dependencies to this list. -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # tasks file for user-traffic-reader 3 | - name: Creates tmp traffic directory 4 | tempfile: 5 | state: directory 6 | register: tempdir_1 7 | 8 | - name: Run Buddhi to create traffic file 9 | docker_container: 10 | name: buddhi 11 | state: started 12 | user: root 13 | tty: True 14 | detach: no 15 | image: quay.io/3scale/perftest-toolkit:v2.4.0 16 | command: "perftest-toolkit-buddhi --portal {{ threescale_portal_endpoint }} --services \"{{ threescale_services }}\" -o /traffic/traffic.csv" 17 | volumes: 18 | - "{{ tempdir_1.path }}:/traffic:z" 19 | 20 | - name: Copy traffic file 21 | copy: 22 | src: "{{ tempdir_1.path }}/traffic.csv" 23 | dest: "{{csv_file_path}}/{{csv_file_name}}" 24 | remote_src: true 25 | 26 | - name: Remove the temporary file 27 | file: 28 | path: "{{ tempdir_1.path }}" 29 | state: absent 30 | when: tempdir_1.path is defined 31 | -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - user-traffic-reader -------------------------------------------------------------------------------- /deployment/roles/user-traffic-reader/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for user-traffic-reader -------------------------------------------------------------------------------- /deployment/run.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: hyperfoil_controller 3 | roles: 4 | - hyperfoil.hyperfoil_test 5 | vars: 6 | test_name: 3scale-benchmark 7 | test_files: 8 | - "{{ csv_dest_file_path }}" 9 | shared_connections: 10 | users_per_sec: 11 | duration_sec: 12 | - hosts: hyperfoil_controller 13 | become: yes 14 | roles: 15 | - hyperfoil_generate_report 16 | - hosts: hyperfoil_controller 17 | tasks: 18 | - name: Retrieve the report 19 | fetch: 20 | src: "{{ reports_path }}/{{ test_runid }}.html" 21 | dest: "{{ playbook_dir }}/{{ toolkit_csv_file_path }}/" 22 | flat: yes 23 | -------------------------------------------------------------------------------- /deployment/upstream.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: upstream 3 | become: yes 4 | roles: 5 | - upstream-configurator 6 | -------------------------------------------------------------------------------- /locust/README.md: -------------------------------------------------------------------------------- 1 | # Locust 2 | 3 | If you’re running into memory/garbage collection issues with hyperfoil that are preventing a full run on the `standard` [profile](https://github.com/3scale-labs/perftest-toolkit/blob/main/buddhi/README.md#profiles) then try using locust to run the actual performance test. 4 | 5 | **NOTE:** The files in this directory (except for this README) were copied from the [3scale-2.15-injector branch](https://github.com/integr8ly/locust-integreatly-operator/tree/3scale-2.15-injector) in the [integr8ly/locust-integreatly-operator](https://github.com/integr8ly/locust-integreatly-operator) project. 6 | 7 | ## Prerequisites 8 | * Successfully ran the [injector](https://github.com/3scale-labs/perftest-toolkit/tree/main?tab=readme-ov-file#deploy-injector) to generate the `3scale.csv` traffic file (the ansible playbook will automatically copy the file to this directory) 9 | * Installed the Locust CLI - installation instructions can be found [here](https://docs.locust.io/en/stable/installation.html) 10 | 11 | ## Run tests 12 | **1.** From the `locust` directory start locust. 13 | ```bash 14 | ./start.sh 15 | ``` 16 | 17 | **NOTE:** If locust is complaining about the port `8089` being blocked, try specifying a different port in [start.sh](start.sh) using the `--web-port` flag like this: 18 | ``` 19 | cores=$(grep -c ^processor /proc/cpuinfo) 20 | ulimit -n 10000 21 | 22 | echo "starting locust master" 23 | locust --master --web-port 8888 & 24 | 25 | echo "creating worker nodes for other cores" 26 | for (( c=2; c<=cores; c++ )) 27 | do 28 | echo "starting locust worker" 29 | locust --worker --web-port 8888 & 30 | done 31 | ``` 32 | 33 | **2.** Access the locust UI in your browser at http://localhost:8089/ (make sure to specify the correct port if you changed it above). 34 | 35 | **3.** Set the `Number of users` and `Ramp up` but clear the `Host` field since the host will be randomly fetched from the CSV traffic file. You can let the test run indefinitely or specify the `Run time` in the Advanced options drop down menu. 36 | 37 | **4.** When the run is complete you can download the load test report from the `DOWNLOAD DATA` tab. 38 | 39 | **5.** After saving the results, you can shut down the workers by running the kill script. 40 | ```bash 41 | ./kill.sh 42 | ``` -------------------------------------------------------------------------------- /locust/kill.sh: -------------------------------------------------------------------------------- 1 | killall locust -------------------------------------------------------------------------------- /locust/locustfile.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import string 3 | import random 4 | import sys 5 | from dataclasses import dataclass 6 | from pathlib import Path 7 | 8 | from locust import HttpUser, task, run_single_user, constant_throughput, constant_pacing 9 | 10 | logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO) 11 | 12 | @dataclass 13 | class HostData: 14 | host: str 15 | token_url: str 16 | 17 | 18 | def get_host(url: str): 19 | data = url.strip("\"") 20 | if data.endswith(":443"): 21 | data = data.strip(":443") 22 | data = f"{data}" 23 | else: 24 | data = f"{data}" 25 | 26 | return data 27 | 28 | 29 | def get_path(path: str): 30 | data = path.strip("\"") 31 | return data 32 | 33 | def parse_csv(csv_file: Path): 34 | result = HostData 35 | 36 | logging.info(f"loading csv file {csv_file}") 37 | 38 | with open(csv_file) as f: 39 | lines = f.readlines() 40 | random_line = random.choice(lines) # Select a random line from the CSV file 41 | data = random_line.strip().split(",") 42 | result.host = get_host(data[0]) 43 | result.param = data[1].strip('"') 44 | 45 | return result 46 | 47 | 48 | def python_version_check(): 49 | if sys.version_info < (3, 11): 50 | logging.info("recommended to use python 3.11 or above and the toml configuration file") 51 | else: 52 | logging.info("recommended to use the toml configuration file format for more features") 53 | 54 | 55 | def parse_json(json_auth: Path): 56 | import json 57 | logging.info("loading json configuration file") 58 | result = HostData 59 | with open(json_auth) as f: 60 | data = f.read() 61 | data = json.loads(data) 62 | result.host = get_host(data['host']) 63 | result.param = data['param'] 64 | 65 | return result 66 | 67 | 68 | def load_data(): 69 | 70 | auth_csv = Path("3scale.csv") 71 | if auth_csv.is_file(): 72 | return parse_csv(auth_csv) 73 | 74 | logging.error("no configuration file found, please create one") 75 | exit(1) 76 | 77 | 78 | def generate_payload(payload_size): 79 | return ''.join([random.choice(string.ascii_letters) for _ in range(payload_size)]) 80 | 81 | 82 | auth_data = load_data() 83 | 84 | 85 | class RhoamUser(HttpUser): 86 | host = auth_data.host 87 | param = auth_data.param 88 | wait_time = constant_pacing(0.207) 89 | request_headers = "" 90 | 91 | @task(40) 92 | def get_data(self): 93 | logging.warning("route: https://%s%s",self.host,self.param) 94 | self.client.get(f"https://{self.host}{self.param}", headers=self.request_headers, name="Get Data") 95 | 96 | 97 | if __name__ == "__main__": 98 | run_single_user(RhoamUser) 99 | -------------------------------------------------------------------------------- /locust/start.sh: -------------------------------------------------------------------------------- 1 | cores=$(grep -c ^processor /proc/cpuinfo) 2 | ulimit -n 10000 3 | 4 | echo "starting locust master" 5 | locust --master & 6 | 7 | echo "creating worker nodes for other cores" 8 | for (( c=2; c<=cores; c++ )) 9 | do 10 | echo "starting locust worker" 11 | locust --worker & 12 | done --------------------------------------------------------------------------------