├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── elvis.config ├── guides ├── amoc_livebook.livemd ├── assets │ └── amoc_throttle_dist.svg ├── configuration.md ├── coordinator.md ├── distributed-run.md ├── distributed.md ├── local-run.md ├── scenario.md ├── telemetry.md └── throttle.md ├── integration_test ├── README.md ├── build_docker_image.sh ├── docker-compose.yml ├── extra_code_paths │ ├── path1 │ │ └── dummy_helper.erl │ └── path2 │ │ └── dummy_scenario.erl ├── helper.sh ├── start_test_cluster.sh ├── stop_test_cluster.sh ├── test_add_new_node.sh ├── test_amoc_cluster.sh ├── test_distribute_scenario.sh └── test_run_scenario.sh ├── rebar.config ├── rebar.lock ├── rel └── app.config ├── src ├── amoc.app.src ├── amoc.erl ├── amoc_app.erl ├── amoc_code_server.erl ├── amoc_controller.erl ├── amoc_scenario.erl ├── amoc_sup.erl ├── amoc_telemetry.erl ├── config │ ├── amoc_config.erl │ ├── amoc_config.hrl │ ├── amoc_config_attributes.erl │ ├── amoc_config_env.erl │ ├── amoc_config_parser.erl │ ├── amoc_config_scenario.erl │ ├── amoc_config_utils.erl │ └── amoc_config_verification.erl ├── coordinator │ ├── amoc_coordinator.erl │ ├── amoc_coordinator_sup.erl │ ├── amoc_coordinator_timeout.erl │ ├── amoc_coordinator_worker.erl │ └── amoc_coordinator_worker_sup.erl ├── dist │ ├── amoc_cluster.erl │ └── amoc_dist.erl ├── throttle │ ├── amoc_throttle.erl │ ├── amoc_throttle_config.erl │ ├── amoc_throttle_controller.erl │ ├── amoc_throttle_pool.erl │ ├── amoc_throttle_pooler.erl │ ├── amoc_throttle_process.erl │ ├── amoc_throttle_runner.erl │ └── amoc_throttle_sup.erl └── users │ ├── amoc_user.erl │ ├── amoc_users_sup.erl │ └── amoc_users_worker_sup.erl └── test ├── amoc_SUITE.erl ├── amoc_code_server_SUITE.erl ├── amoc_code_server_SUITE_data └── module.mustache ├── amoc_config_attributes_SUITE.erl ├── amoc_config_env_SUITE.erl ├── amoc_config_helper.erl ├── amoc_config_scenario_SUITE.erl ├── amoc_config_verification_SUITE.erl ├── amoc_coordinator_SUITE.erl ├── controller_SUITE.erl ├── telemetry_helpers.erl ├── test_helpers.erl ├── testing_scenario.erl ├── testing_scenario_with_error_in_init.erl ├── testing_scenario_with_state.erl ├── testing_scenario_with_throttle.erl ├── testing_scenario_without_callbacks.erl └── throttle_SUITE.erl /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | 10 | jobs: 11 | test: 12 | name: ${{ matrix.test-type }} test on OTP ${{matrix.otp_vsn}} 13 | strategy: 14 | matrix: 15 | otp_vsn: ['28', '27', '26'] 16 | rebar_vsn: ['3.25'] 17 | test-type: ['regular', 'integration'] 18 | runs-on: 'ubuntu-24.04' 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: erlef/setup-beam@v1 22 | with: 23 | otp-version: ${{ matrix.otp_vsn }} 24 | rebar3-version: ${{ matrix.rebar_vsn }} 25 | - if: matrix.test-type == 'regular' 26 | run: make test 27 | - if: matrix.test-type == 'regular' 28 | name: Upload coverage reports to Codecov 29 | uses: codecov/codecov-action@v3 30 | env: 31 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 32 | - if: matrix.test-type == 'integration' 33 | run: make integration_test 34 | env: 35 | OTP_RELEASE: ${{ matrix.otp_vsn }} 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## default .gitignore for rebar3 app project 2 | .rebar3 3 | _* 4 | .eunit 5 | *.o 6 | *.beam 7 | *.plt 8 | *.swp 9 | *.swo 10 | .erlang.cookie 11 | ebin 12 | log 13 | erl_crash.dump 14 | .rebar 15 | logs 16 | _build 17 | .idea 18 | *.iml 19 | rebar3.crashdump 20 | *~ 21 | doc/ 22 | codecov.json 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | This project adheres to [Semantic Versioning](http://semver.org/) 5 | 6 | ## [2.2.1](https://github.com/esl/amoc/compare/2.2.0...2.2.1) - 2021-07-19 7 | 8 | * [PR#138](https://github.com/esl/amoc/pull/138) - Fixing docsh dependency 9 | 10 | ## [2.2.0](https://github.com/esl/amoc/compare/2.1.0...2.2.0) - 2021-07-15 11 | 12 | This release enables OTP24, and migrates CI to Github Actions. 13 | 14 | ## The main changes: 15 | 16 | * [PR#135](https://github.com/esl/amoc/pull/135) - Switching from TravisCI to GH Actions 17 | * [PR#136](https://github.com/esl/amoc/pull/136) - Enabling OTP24 compatibility. Only OTP releases since 23.0 are supported now. 18 | 19 | ## [2.1.0](https://github.com/esl/amoc/compare/2.0.1...2.1.0) - 2020-08-19 20 | 21 | This release focuses on the REST API, which is now powered by OpenAPI Specifications generated by [openapi-generator](https://openapi-generator.tech/). 22 | 23 | ## The main changes: 24 | 25 | * [PR#123](https://github.com/esl/amoc/pull/123) - simplification of scenario uploading, now it can be done using the following command: `curl -s -H "Content-Type: text/plain" -T scenario.erl 'http://localhost:4000/upload'` 26 | * [PR#124](https://github.com/esl/amoc/pull/124) - switch from [cowboy-swagger](https://github.com/inaka/cowboy_swagger) to [amoc_rest](https://github.com/esl/amoc_rest) (framework generated using [openapi-generator](https://openapi-generator.tech/)). Online Swagger UI documentation for the current release can be found [here](https://esl.github.io/amoc_rest/?v=1.1.0). 27 | * [PR#125](https://github.com/esl/amoc/pull/125) - possibility to update scenario settings at runtime. Support of the settings for helper modules. Update of the `-required_variable(...)` module attribute format: 28 | ``` 29 | -type module_attribute() :: #{name := name(), 30 | description := string(), 31 | default_value => value(), 32 | verification => verification_method(), 33 | update => update_method()}. 34 | ``` 35 | * [PR#130](https://github.com/esl/amoc/pull/130) - added new `/scenarios/info/{scenario_name}` REST API, it returns edoc description of the scenario module and all the relevant settings declared using `-required_variable(...)` attribute. 36 | * [PR#131](https://github.com/esl/amoc/pull/131) - implementation of the `/execution/*` REST APIs: 37 | * `/execution/start` - starts scenario on all the nodes in the cluster 38 | * `/execution/stop` - stops scenario execution on all the nodes in the cluster 39 | * `/execution/add_users` - adds new users on all or specific nodes in the cluster 40 | * `/execution/remove_users` - removes users on all or specific nodes in the cluster 41 | * `/execution/update_settings` - updates scenario settings on all or specific nodes in the cluster 42 | * [PR#132](https://github.com/esl/amoc/pull/132) - remove the legacy way of providing configuration through erlang app environment variables 43 | * [PR#133](https://github.com/esl/amoc/pull/133) - significant improvement of the `/status` REST API, introduction of the `/status/{node_name}` REST API (which can be used to check the status of other nodes in the cluster). The following thing are reported: 44 | * Amoc application status (up/down) 45 | * Amoc specific env. variables 46 | * Amoc controller status + runtime scenario settings for running/terminating/finished states 47 | 48 | ## [2.0.1](https://github.com/esl/amoc/compare/2.0.0...2.0.1) - 2019-12-05 49 | 50 | ### Changed: 51 | - add some fixes in the documentation 52 | - set the release version automatically 53 | 54 | ## [2.0.0](https://github.com/esl/amoc/compare/2.0.0-beta...2.0.0) - 2019-12-03 55 | 56 | ### Changed: 57 | - extended amoc documentation 58 | - automatic distribution of the uploaded scenarios to all the nodes in the amoc cluster 59 | - amoc configuration: 60 | - mandatory declaration of the required parameters for the scenario 61 | - ets based ``amoc_config:get/2`` interface 62 | - ``amoc_controller`` - module doesn't hold any information about the cluster any more, it's now a responsibility of the ``amoc_dist`` module 63 | 64 | 65 | ### Added: 66 | - integration tests for docker based amoc cluster 67 | - possibility to dynamically add the new amoc node to the cluster 68 | 69 | ### Removed: 70 | - unused rebar dependencies 71 | 72 | ## [2.0.0-beta](https://github.com/esl/amoc/compare/1.3.0...2.0.0-beta) - 2019-10-29 73 | 74 | ### Changed: 75 | - exometer - now Amoc uses only exometer_core and 2 reporters graphite and statsd 76 | - scenario for GDPR removal 77 | 78 | ### Removed: 79 | - scenarios and helpers related to XMPP - they were moved to https://github.com/esl/amoc-arsenal-xmpp 80 | - dependency on escalus, amqp_client and other libraries not directly used by Amoc 81 | 82 | ## [1.3.0](https://github.com/esl/amoc/compare/1.2.1...1.3.0) - 2019-10-28 83 | 84 | ### Changed: 85 | - escalus to esl/escalus@58c2bf8 86 | - `amoc_xmpp` new helper function `send_request_and_get_response` 87 | - `amoc_xmpp_handlers` new function for constructing handlers 88 | - `amoc_scenario` behavior was extended with optional callbacks `continue`, `terminate`, `next_user_batch` more details in #90 89 | - Amoc's docker container allows to pass `AMOC_EXTRA_CODE_PATHS` env var with path to additional beam files 90 | - Amoc's REST API allows to start a scenario which is outside of Amoc's `scenario` directory 91 | - `amoc_config` 92 | - allows to pass env vars containing value `false` 93 | - allows to parse and validate scenario variables passed as env vars 94 | - `amoc_dist` and `amoc_slave` now master node is the one a scenario is started on 95 | - `amoc_throttle` works in distributed mode now 96 | - documentation - the structure of the documentation was reworked and the content was updated and extended 97 | 98 | 99 | ### Added: 100 | - `amoc_xmpp_muc` new module with helper function for building MUC scenarios 101 | - scripts and documentation showing how to setup and run load tests with multiple Amoc nodes in docker containers 102 | - `iq_metrics` helper for generic metrics related to IQ stanzas 103 | - `MUC` and `MUC_light` load test scenarios 104 | - `amoc_coordinator` to coordinate sessions 105 | 106 | 107 | ### Removed: 108 | - ansible scripts for deploying amoc 109 | - `amoc_annotations` module 110 | - `config` helper module - functionality moved to `amoc_config` 111 | 112 | ## [1.2.1](https://github.com/esl/amoc/compare/1.2.0...1.2.1) - 2019-10-18 113 | 114 | ### Changed: 115 | 116 | - `amoc_controller` to allow passing stated from scenario's `init` callback to `start` callback 117 | - `amoc_metrics` support gauge metric type 118 | - `amoc_scenario` behavior to allow passing state from `init` to `start` callbacks 119 | - `amoc_xmpp` 120 | - `connect_or_exit` function was extended to allow passing extra user/connection options 121 | - new function `pick_server/1`which picks random server from config var `xmpp_servers` 122 | - supported Erlang/OTP versions - now the oldest supported is Erlang/OTP 21.2 123 | - escalus updated to esl/escalus@f20bee4 124 | - scenarios where adjusted to be compatible with the updated escalus 125 | 126 | ### Added: 127 | 128 | - helper module for scenario configuration 129 | - `amoc_throtlle` module 130 | - `amoc_xmpp_user` module for unified user and password generation 131 | 132 | ## [1.2.0](https://github.com/esl/amoc/compare/1.1.0...1.2.0) - 2019-01-16 133 | 134 | ### Changed: 135 | - `amoc_metrics` - now only 2 type of metrics are available via the `amoc_metrics` API 136 | - `counters` - counting occurrences of an event in total and in last minute. This is exometer's spiral metric. Its name is prefixed with `[amoc, counters]`. 137 | - `times` - provides statistic of given action execution time. This is exometer's histogram metric. Its name is prefixed with `[amoc, times]`. 138 | 139 | ### Added: 140 | - `amoc_xmpp` - a new module, currently with only one function simplifying connection to the XMPP server 141 | - `amoc_xmpp_handlers` - a new module with 2 handlers which can be used with escalus's stnaza handler feature. See sample scenarios (mongoose_simple_with_metrics) for examples. 142 | 143 | ## [1.1.0](https://github.com/esl/amoc/compare/1.0.0...1.1.0) - 2019-01-04 144 | 145 | ### Changed: 146 | - updated deps #67: 147 | - escalus 148 | - lager to `3.6.8` 149 | - jiffy to `0.15.2` 150 | - trails to `2.1.0` 151 | - cowboy_trails to `2.1.0` 152 | - recon to `2.4.0` 153 | - cowboy to `2.3.0` 154 | 155 | ### Added: 156 | - amqp_client `3.7.9` 157 | 158 | ### Removed: 159 | - mochijson2 160 | - lhttpc 161 | 162 | ## [1.0.0](https://github.com/esl/amoc/compare/0.9.1...1.0.0) - 2019-01-04 163 | 164 | ### Changed: 165 | - use rebar3 to build the project #62 166 | - `Dockerfile` to build local version of Amoc - enables automated builds #66 167 | 168 | ## [0.9.1](https://github.com/esl/amoc/compare/0.9.0...0.9.1) - 2018-04-09 169 | 170 | ### Changed: 171 | - esl/escalus updated to esl/escalus@47848b5 172 | 173 | ### Added: 174 | - `iproute2` pkg to amoc's docker 175 | - ability to pass graphite's port number to amoc's docker container 176 | 177 | 178 | ## [0.9.0] - 2017-01-09 179 | 180 | ### This release includes: 181 | 182 | - Core functionality of parallel scenario execution (in either local or distributed environment) 183 | - Possibility of controlling the scenario during execution (via the Erlang API) 184 | - Basic instrumentation with Graphite metrics and annotations (via exometer reporter) 185 | - An [HTTP API](https://github.com/esl/amoc/blob/master/REST_API_DOCS.md) for controlling scenario execution remotely 186 | - Ansible automation for deploying and configuring the release on many nodes 187 | - Set of Docker files in order to facilitate automation, but without Ansible 188 | - Example XMPP and HTTP scenarios 189 | - Very basic documentation 190 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG otp_vsn=25.3 2 | FROM erlang:${otp_vsn} 3 | MAINTAINER Erlang Solutions 4 | 5 | WORKDIR /amoc 6 | COPY ./ ./ 7 | 8 | RUN make clean 9 | RUN make rel 10 | 11 | ENV PATH "/amoc/_build/default/rel/amoc/bin:${PATH}" 12 | 13 | CMD ["amoc", "foreground"] 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default rel deps compile clean ct lint dialyzer xref console 2 | .PHONY: test integration_test rerun_integration_test 3 | 4 | ifdef SUITE 5 | SUITE_OPTS = --suite $$SUITE 6 | endif 7 | 8 | default: compile 9 | 10 | rel: 11 | rebar3 release 12 | 13 | deps: 14 | rebar3 deps 15 | rebar3 compile --deps_only 16 | 17 | compile: 18 | rebar3 compile 19 | 20 | clean: 21 | rm -rf _build 22 | 23 | ct: 24 | ## in order to run some specific test suite(s) you can override 25 | ## the SUITE variable from the command line or as env variable: 26 | ## make ct SUITE=some_test_SUITE 27 | ## make ct SUITE=some_test_SUITE,another_test_SUITE 28 | ## SUITE=some_test_SUITE make ct 29 | ## SUITE=some_test_SUITE,another_test_SUITE make ct 30 | @ echo rebar3 ct -c --verbose $(SUITE_OPTS) 31 | @ rebar3 ct -c --verbose $(SUITE_OPTS) 32 | 33 | lint: 34 | rebar3 lint 35 | 36 | test: compile xref dialyzer ct lint codecov 37 | 38 | codecov: 39 | rebar3 as test codecov analyze 40 | 41 | integration_test: 42 | ./integration_test/stop_test_cluster.sh 43 | ./integration_test/build_docker_image.sh 44 | ./integration_test/start_test_cluster.sh 45 | ./integration_test/test_amoc_cluster.sh 46 | ./integration_test/test_distribute_scenario.sh 47 | ./integration_test/test_run_scenario.sh 48 | ./integration_test/test_add_new_node.sh 49 | 50 | rerun_integration_test: 51 | ./integration_test/stop_test_cluster.sh 52 | ./integration_test/start_test_cluster.sh 53 | ./integration_test/test_amoc_cluster.sh 54 | ./integration_test/test_distribute_scenario.sh 55 | ./integration_test/test_run_scenario.sh 56 | ./integration_test/test_add_new_node.sh 57 | 58 | dialyzer: 59 | rebar3 dialyzer 60 | 61 | xref: 62 | rebar3 xref 63 | 64 | console: 65 | @echo "tests can be executed manually using ct:run/1 function:\n" \ 66 | ' ct:run("test").' 67 | rebar3 as test shell 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Murder of Crows 2 | [![](https://github.com/esl/amoc/workflows/CI/badge.svg)](https://github.com/esl/amoc/actions?query=workflow%3ACI) 3 | [![Hex](http://img.shields.io/hexpm/v/amoc.svg)](https://hex.pm/packages/amoc) 4 | [![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/amoc/) 5 | [![codecov](https://codecov.io/github/esl/amoc/graph/badge.svg?token=R1zXAjO7H7)](https://codecov.io/github/esl/amoc) 6 | 7 | --- 8 | 9 | A Murder of Crows, aka amoc, is a simple framework for running massively parallel tests in a distributed environment. 10 | 11 | It can be used as a `rebar3` dependency: 12 | 13 | ```erlang 14 | {deps, [ 15 | {amoc, "3.2.0"} 16 | ]}. 17 | ``` 18 | 19 | or in `mix`: 20 | 21 | ```elixir 22 | defp deps() do 23 | [ 24 | {:amoc, "~> 3.2"} 25 | ] 26 | end 27 | ``` 28 | 29 | [MongooseIM](https://github.com/esl/MongooseIM) is continuously being load tested with Amoc. 30 | All the XMPP scenarios can be found [here](https://github.com/esl/amoc-arsenal-xmpp). 31 | 32 | --- 33 | 34 | In order to implement and run locally your scenarios, follow the chapters about 35 | [developing](https://hexdocs.pm/amoc/scenario.html) and [running](https://hexdocs.pm/amoc/local-run.html) 36 | a scenario locally. 37 | Before [setting up the distributed environment](https://hexdocs.pm/amoc/distributed.html), 38 | please read through the configuration overview. 39 | 40 | To see the full documentation, see [hexdocs](https://hexdocs.pm/amoc). 41 | 42 | You can also try with the livebook demo here: 43 | 44 | [![Run in Livebook](https://livebook.dev/badge/v1/blue.svg)](https://livebook.dev/run?url=https%3A%2F%2Fgithub.com%2Fesl%2Famoc%2Fblob%2Fmaster%2Fguides%2Famoc_livebook.livemd) 45 | -------------------------------------------------------------------------------- /elvis.config: -------------------------------------------------------------------------------- 1 | [{elvis, [ 2 | {config, [ 3 | #{dirs => ["src", "src/*", "scenarios"], 4 | filter => "*.erl", 5 | ruleset => erl_files, 6 | rules => [ 7 | {elvis_style, invalid_dynamic_call, 8 | #{ignore => [amoc_user, {amoc_code_server, get_md5, 1}]}}, 9 | {elvis_style, export_used_types, disable}, 10 | {elvis_style, no_throw, #{ignore => [{amoc_config, get, 2}] }}, 11 | {elvis_text_style, line_length, #{skip_comments => whole_line }}, 12 | {elvis_style, no_block_expressions, disable} 13 | ]}, 14 | #{dirs => ["test"], 15 | filter => "*.erl", 16 | ruleset => erl_files, 17 | rules => [ 18 | {elvis_style, function_naming_convention, 19 | #{regex => "^[a-z]([a-z0-9]*_?)*$"}}, 20 | {elvis_style, atom_naming_convention, 21 | #{regex => "^[a-z]([a-z0-9]*_?)*(_SUITE)?$"}}, 22 | {elvis_style, invalid_dynamic_call, #{ignore => [amoc_code_server_SUITE]}}, 23 | {elvis_style, dont_repeat_yourself, #{min_complexity => 50}}, 24 | {elvis_style, no_debug_call, disable}, 25 | {elvis_style, no_block_expressions, #{ignore => [amoc_code_server_SUITE, controller_SUITE]}}, 26 | {elvis_style, no_throw, disable}, 27 | {elvis_style, no_import, disable} 28 | ]}, 29 | #{dirs => ["."], 30 | filter => "rebar.config", 31 | ruleset => rebar_config} 32 | ]} 33 | ]}]. 34 | -------------------------------------------------------------------------------- /guides/configuration.md: -------------------------------------------------------------------------------- 1 | ## Configuration 2 | 3 | Amoc is configured through environment variables (uppercase with prefix `AMOC_`). 4 | Note that by default, environment variables values are deserialized as Erlang terms. This behavior can be customized by providing an alternative parsing module in the `config_parser_mod` configuration parameter for the amoc application. It can be done using the [application:set_env/4](https://www.erlang.org/doc/man/application#set_env-4) interface or via a [config file](https://www.erlang.org/doc/man/config). The custom parsing module must implement the `amoc_config_env` behavior. 5 | 6 | Amoc supports the following generic configuration parameters: 7 | 8 | * `nodes` - required for the distributed scenario execution, a list of nodes that should be clustered together: 9 | * default value - empty list (`[]`) 10 | * example: `AMOC_NODES="['amoc@amoc-1', 'amoc@amoc-2']"` 11 | 12 | 13 | * `interarrival` - a delay (in ms, for each node in the cluster independently) between creating the processes 14 | for two consecutive users: 15 | * default value - 50 ms. 16 | * example: `AMOC_INTERARRIVAL="50"` 17 | * this parameter can be updated at runtime (in the same way as scenario configuration). 18 | 19 | * `extra_code_paths` - a list of paths that should be included using `code:add_pathsz/1` interface 20 | * default value - empty list (`[]`) 21 | * example: `AMOC_EXTRA_CODE_PATHS='["/some/path", "/another/path"]'` 22 | 23 | In the same manner you can also define your own entries to configure the scenario. 24 | 25 | ## Required Variables 26 | 27 | The `amoc_config:get/1` and `amoc_config:get/2` interfaces can be used to get 28 | parameters required for your scenario, however every scenario must declare (using 29 | `-required_variable(...)`/`@required_variable ...` attributes) all the required parameters in advance. 30 | For more information, see the example `dummy_scenario` in integration tests. 31 | 32 | Scenario configuration also can be set/updated at runtime using an API. 33 | 34 | ```erlang 35 | -required_variable( 36 | #{name => Name, description => Description, 37 | default_value => Value, 38 | update => UpdateMfa, 39 | verification => VerificationMfa} 40 | ). 41 | ``` 42 | 43 | ```elixir 44 | ## Note that the attribute needs to be marked as persisted 45 | ## for the Elixir compiler to store it in the generated BEAM file. 46 | Module.register_attribute(__MODULE__, :required_variable, accumulate: true, persist: true) 47 | 48 | @required_variable %{ 49 | name: name, 50 | description: description, 51 | default_value: 6, 52 | update: updated_mfa, 53 | verification: verification_mfa 54 | } 55 | ``` 56 | 57 | where 58 | 59 | ### `name` 60 | 61 | * **Syntax:** atom 62 | * **Example:** `var1` 63 | * **Default:** this field is mandatory 64 | 65 | ### `description` 66 | 67 | * **Syntax:** A string describing how this variable is used, can be extracted by APIs to document the behavior 68 | * **Example:** `"a description of this variable"` 69 | * **Default:** this field is mandatory 70 | 71 | ### `default_value` 72 | 73 | * **Syntax:** value of the expected type 74 | * **Example:** `10` 75 | * **Default:** `undefined` 76 | 77 | ### `verification` 78 | 79 | * **Syntax:** `none`, a list of allowed values, or an `mfa` of arity `1` 80 | * **Example:** `{?MODULE, is_binary, 1}` 81 | * **Default:** `none` 82 | 83 | A verification function that will check the given value is correct. It is trigger for verifying the initial values, including the default value, and before updated values are applied. 84 | - If it is set to `none`, all values are allowed. 85 | - If it is set to a list of values, any given value checks that the new value is in such allowlist. 86 | - If it is an `mfa`, the given function will be called on the given value. This function 87 | must be pure and return a boolean or a `{true, NewValue} | {false, Reason}`. It can also be used for preprocessing of the input value by returning `{true, NewValue}`. 88 | 89 | ### `update` 90 | 91 | * **Syntax:** `read_only`, `none`, or an `mfa` of arity 2 92 | * **Example:** `{?MODULE, update, 2}` 93 | * **Default:** `read_only` 94 | 95 | An action to take when the value of this variable is updated. It is triggered at runtime when updates to the value are applied. 96 | - If it is set to `read_only`, updates will fail. 97 | - If it is set to `none`, all updates are allowed. 98 | - If it is an `mfa`, the given function will be called on the old and new value. 99 | 100 | ## Rationale 101 | 102 | The reason why the `-required_variable(...)` is preferred over the usual behavior 103 | callback is because the orchestration tools can easily extract the attributes even 104 | without the compilation, while configuring via a callback, requires a successful 105 | compilation of the module. As an example, a module: 106 | 107 | ```erlang 108 | -module(example). 109 | -include("some_unavailable_header.hrl"). 110 | -some_attr({"some", value}). 111 | -some_attr([{another, "value"}, 112 | {yet, <<"another">>, "value"}]). 113 | ``` 114 | 115 | cannot be compiled without the `some_unavailable_header.hrl` file, but we still 116 | can parse it and extract the attributes: 117 | 118 | ``` 119 | Eshell V14.0 (press Ctrl+G to abort, type help(). for help) 120 | 1> c(example). 121 | example.erl:2: can't find include file "some_unavailable_header.hrl" 122 | error 123 | 2> {ok, AbstractForm} = epp:parse_file("example.erl", []). 124 | {ok,[{attribute,1,file,{"example.erl",1}}, 125 | {attribute,1,module,example}, 126 | {error,{2,epp,{include,file,"some_unavailable_header.hrl"}}}, 127 | {attribute,3,some_attr,{"some",value}}, 128 | {attribute,4,some_attr, 129 | [{another,"value"},{yet,<<"another">>,"value"}]}, 130 | {eof,6}]} 131 | 3> lists:flatten([Value || {attribute, _, some_attr, Value} <- AbstractForm]). 132 | [{"some",value}, 133 | {another,"value"}, 134 | {yet,<<"another">>,"value"}] 135 | ``` 136 | -------------------------------------------------------------------------------- /guides/coordinator.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | See `m:amoc_coordinator`. 4 | 5 | ## Description 6 | 7 | This module allows to synchronize the users and act on groups of them. 8 | 9 | The coordinator reacts to new users showing up in a system, according to the *Coordination Plan*. 10 | The *Coordination Plan* consists of *Coordination Items*, and each of them is defined as one of the following: `{NumberOfUsers, CoordinationActions}`. 11 | - When the `NumberOfUsers` is set to `all`, then only *Coordination Actions* with the arities `/1, /2` are handled. 12 | The *Coordination Items* with `all` are triggered by the `timeout` event type. 13 | - When the `NumberOfUsers` is set to a positive integer or a range, all *Coordination Actions* with arities `/1, /2` and `/3` are handled. 14 | 15 | Note that `NumberOfUsers` can be a range, in which case a new integer within the range will be randomly selected every time the coordinator fills a batch, to ensure a non-equal but uniform distribution of coordination. 16 | 17 | The timeout timer is reset by calling the `add` function. 18 | A new batch size is set in the `NumberOfUsers`. Each user in the batch calls the `add` function registering to the coordinator and triggering the *Coordination Plan*. 19 | If more than one of the *Coordination Items* matching the `NumberOfUsers` is triggered, each of them will be passed a respective number of users. 20 | For example if the *Coordination Plan* is `[{2, Act1}, {3, Act2}]` then on the 6th user calling `add`, `Act1` will be called with 2 users passed and `Act2` will be called with 3 users passed. 21 | 22 | *Coordination Actions* may be one of the following: 23 | - `fun(Event) -> any()` - this type of action does not care about particular users, but only about the number of them; 24 | - `fun(Event, ListOfUsersData) -> any()` - this type of action gets `ListOfUsersData` which is a list of `{Pid, Data}` tuples with `Pid`s passed by users calling `amoc_coordinator:add/2` or `amoc_coordinator:add/3`; 25 | - `fun(Event, User1, User2) -> any()` - this type of action gets `distinct pairs` from the batch of users `User1` and `User2` which are `{Pid, Data}` tuples with `Pid`s passed by users calling `amoc_coordinator:add/2` or `amoc_coordinator:add/3`; 26 | 27 | where an `Event` is a `{EventType, NumOfUsers}` tuple, in which `NumOfUsers` is the number of users passed to the event. 28 | 29 | The order of *Coordination Actions* execution is not guaranteed. 30 | It’s guaranteed that all the *Coordination Actions* with `all` are executed after all numbered *Coordination Actions* are done. 31 | 32 | `distinct pairs` - in the context, these are pairs from a given collection of users, where: 33 | - when `{A, B}` is in the `distinct pairs` then `{B, A}` is not; 34 | - `{A, A}` is not in the `distinct pairs`; 35 | - all pairs are distinct; 36 | - Eg. for `[a]`, the `distinct pairs` collection is `[{a, undefined}]`; 37 | - Eg. for `[a, b]`, the `distinct pairs` collection is `[{a, b}]`; 38 | - Eg. for `[a, b, c]`, the `distinct pairs` collection is `[{a, b}, {a, c}, {b, c}]`. 39 | 40 | ## Example 41 | 42 | This scenario shows how the `users` interact with `amoc_coordinator`: 43 | 44 | ```erlang 45 | -module(example). 46 | 47 | -behaviour(amoc_scenario). 48 | 49 | -export([init/0]). 50 | -export([start/2]). 51 | 52 | init() -> 53 | Plan = [ 54 | {2, fun(Event) -> 55 | io:fwrite("Two new users showed up: Event = ~p\n", [Event]) 56 | end}, 57 | {2, fun(Event, ListOfUsers) -> 58 | io:fwrite("Two new users showed up: Event = ~p; ListOfUsers = ~p\n", [Event, ListOfUsers]), 59 | [ Pid ! {hello, Data} || {Pid, Data} <- ListOfUsers] 60 | end}, 61 | {2, fun(Event, User1, User2) -> 62 | io:fwrite("Two new users showed up: Event = ~p; User1 = ~p; User2 = ~p\n", [Event, User1, User2]) 63 | end}, 64 | 65 | {3, fun(_Event) -> 66 | io:fwrite("Three new users showed up\n", []) 67 | end}, 68 | {all, fun(Event) -> 69 | io:fwrite("All users have called amoc_coordinator:add in Event = ~p\n", [Event]) 70 | end} 71 | 72 | ], 73 | Settings = [setting1, {setting2, something}], 74 | amoc_coordinator:start(?MODULE, Plan), 75 | {ok, Settings}. 76 | 77 | start(Id, _Settings) -> 78 | io:fwrite("User = ~p\n", [Id]), 79 | amoc_coordinator:add(?MODULE, Id), 80 | receive 81 | Msg -> 82 | io:fwrite("{Msg = ~p, Id = ~p\n", [Msg, Id]) 83 | end, 84 | ok. 85 | ``` 86 | 87 | To run it: 88 | 89 | ```bash 90 | $ make rel 91 | $ _build/default/rel/amoc/bin/amoc console 92 | 93 | 1> amoc:do(example, 5, []). 94 | ``` 95 | 96 | Filtered, formatted and explained output: 97 | 98 | ```erlang 99 | User = 1 % First user is started 100 | 101 | ok % result of calling amoc:do/3 102 | 103 | User = 2 % First user is started 104 | 105 | % We have 2 users added to amoc_coordinator so all of actions {2, _} are triggered: 106 | Two new users showed up: Event = {coordinate,2}; User1 = {<0.1142.0>,2}; User2 = {<0.1140.0>,1} 107 | % This action triggers sending {hello,Id} to the users 1 and 2 108 | Two new users showed up: Event = {coordinate,2}; ListOfUsers = [{<0.1142.0>,2},{<0.1140.0>,1}] 109 | Two new users showed up: Event = {coordinate,2} 110 | 111 | % Users 1 and 2 received messages and print them: 112 | {Msg = {hello,2}, Id = 2 113 | {Msg = {hello,1}, Id = 1 114 | 115 | User = 3 116 | % We have 3 users added to amoc_coordinator so all of the {3, _} actions are triggered: 117 | Three new users showed up 118 | 119 | User = 4 120 | % We have 4 and 4 rem 2 == 0 therefore users added to amoc_coordinator so all of the {2, _} actions are triggered: 121 | Two new users showed up: Event = {coordinate,2}; User1 = {<0.1144.0>,4}; User2 = {<0.1143.0>,3} 122 | Two new users showed up: Event = {coordinate,2}; ListOfUsers = [{<0.1144.0>,4},{<0.1143.0>,3}] 123 | Two new users showed up: Event = {coordinate,2} 124 | 125 | {Msg = {hello,4}, Id = 4 126 | {Msg = {hello,3}, Id = 3 127 | User = 5 128 | 129 | % You need to wait for a while, and ... 130 | % Timeout has been reached, which triggers all of the Coordination Actions with the remaining number of users. 131 | Three new users showed up 132 | Two new users showed up: Event = {timeout,1}; User1 = {<0.1139.0>,5}; User2 = undefined 133 | Two new users showed up: Event = {timeout,1}; ListOfUsers = [{<0.1139.0>,5}] 134 | {Msg = {hello,5}, Id = 5 135 | Two new users showed up: Event = {timeout,1} 136 | All users have called amoc_coordinator:add in Event = {timeout,5} 137 | ``` 138 | -------------------------------------------------------------------------------- /guides/distributed-run.md: -------------------------------------------------------------------------------- 1 | ## Running load test 2 | 3 | Starting a scenario on multiple nodes is slightly more difficult. 4 | You need a number of machines that will be actually running the test 5 | (*slaves*) and one controller machine (*master*, which might be one of the test nodes). 6 | Another aproach to do it is to use docker containers. 7 | 8 | Now instead of `amoc` use `amoc_dist` - this will tell amoc to distribute 9 | and start scenarios on all known nodes (except master). 10 | 11 | ```erlang 12 | amoc_dist:do(my_scenario, 100, Settings). 13 | ``` 14 | 15 | ```elixir 16 | :amoc_dist.do(:my_scenario, 100, settings) 17 | ``` 18 | 19 | Start `my_scenario` spawning 100 amoc users with IDs from the range `[1, 100]` inclusive. 20 | In this case sessions are going to be distributed across all nodes except master. 21 | 22 | `Settings` is an optional proplist with scenario options that can be extracted using amoc_config module. 23 | The values provided in this list shadow OS and APP environment variables. 24 | Note that these settings will be propagated automatically among all the nodes in the amoc cluster. 25 | 26 | ```erlang 27 | amoc_dist:add(50). 28 | ``` 29 | 30 | ```elixir 31 | :amoc_dist.add(50) 32 | ``` 33 | 34 | Add 50 more users to the currently started scenario. 35 | 36 | ```erlang 37 | amoc_dist:remove(50, Force). 38 | ``` 39 | 40 | ```elixir 41 | :amoc_dist.remove(50, force) 42 | ``` 43 | 44 | Remove 50 sessions. 45 | 46 | Where `Force` is a boolean of value: 47 | 48 | * `true` - to kill the user processes using `supervisor:terminate_child/2` function 49 | * `false` - to send `exit(User, shutdown)` signal to the user process (can be ignored by the user) 50 | 51 | All the users are `temporary` children of a `simple_one_for_one` supervisor with the `shutdown` key set to `2000`. 52 | 53 | Note that removal operation is asynchronous, and if we call `amoc_controller:remove_users/2` two times in a row, it may select the same users for removal. 54 | 55 | Also all the user processes trap exits. 56 | 57 | ## Don't stop scenario on exit 58 | 59 | There is one problem with the `bin/amoc console` command. When you exit the Erlang 60 | shell the scenario is stopped (in fact the erlang nodes are killed). 61 | To prevent that start the amoc node in the background using `bin/amoc start`. 62 | Now you can run commands by attaching to the amoc's node with `bin/amoc attach`, 63 | typing a command and then pressing Ctrl+D to exit the shell. 64 | After that the scenario will keep running. 65 | -------------------------------------------------------------------------------- /guides/distributed.md: -------------------------------------------------------------------------------- 1 | To build a Docker image with Amoc, run the following command from the root of 2 | the repository: 3 | 4 | ``` 5 | docker build -t amoc_image:tag . 6 | ``` 7 | 8 | It is important to start building at project root 9 | (it is indicated with dot `.` at the end of command). 10 | It will set build context at the project root. 11 | Dockerfile commands expects a context to be set like that: 12 | - it copies the **current** source code into the container to compile it. 13 | - It looks for files in `docker/` relative path. 14 | 15 | When the image is ready you can start either a single instance of Amoc or configure a distributed environment, 16 | for which you should follow the steps described below. 17 | 18 | Before running Amoc containers, create a network and start a Graphite instance to collect and visualize some metrics. 19 | 20 | ``` 21 | docker network create amoc-test-network 22 | 23 | docker run --rm -d --name=graphite --network amoc-test-network \ 24 | -p 2003:2003 -p 8080:80 graphiteapp/graphite-statsd 25 | ``` 26 | 27 | Start two Amoc containers, export all of the necessary environmental variables so that the nodes can communicate with each other and send metrics to Graphite. 28 | In order to use Amoc HTTP API for uploading and starting scenarios, port 4000 should be published. 29 | 30 | ``` 31 | docker run --rm -t -d --name amoc-1 -h amoc-1 \ 32 | --network amoc-test-network \ 33 | -e AMOC_NODES="['amoc@amoc-1','amoc@amoc-2']" \ 34 | -e AMOC_GRAPHITE_HOST='"graphite"' \ 35 | -e AMOC_GRAPHITE_PORT=2003 \ 36 | -e AMOC_GRAPHITE_PREFIX='"amoc1"' \ 37 | --health-cmd="/home/amoc/amoc/bin/amoc status" \ 38 | -p 8081:4000 \ 39 | amoc_image:tag 40 | 41 | docker run --rm -t -d --name amoc-2 -h amoc-2 \ 42 | --network amoc-test-network \ 43 | -e AMOC_NODES="['amoc@amoc-1','amoc@amoc-2']" \ 44 | -e AMOC_GRAPHITE_HOST='"graphite"' \ 45 | -e AMOC_GRAPHITE_PORT=2003 \ 46 | -e AMOC_GRAPHITE_PREFIX='"amoc2"' \ 47 | --health-cmd="/home/amoc/amoc/bin/amoc status" \ 48 | -p 8082:4000 \ 49 | amoc_image:tag 50 | ``` 51 | 52 | Connect to Amoc console and go to the [next](distributed-run.md) section. 53 | 54 | ``` 55 | docker exec -it amoc-1 /home/amoc/amoc/bin/amoc remote_console 56 | ``` 57 | -------------------------------------------------------------------------------- /guides/local-run.md: -------------------------------------------------------------------------------- 1 | ### Test your scenario locally 2 | 3 | Everything you need to do is to create the release. To achieve that run: 4 | `make rel`. Now you are ready to test our scenario locally with one Amoc node; 5 | to start the node run `_build/default/rel/amoc/bin/amoc console`. 6 | 7 | Start `my_scenario` spawning 10 amoc users with IDs from range (1,10) inclusive. 8 | ```erlang 9 | amoc:do(my_scenario, 10, []). 10 | ``` 11 | ```elixir 12 | :amoc.do(:my_scenario, 10, []) 13 | ``` 14 | 15 | Add 10 more user sessions. 16 | ```erlang 17 | amoc:add(10). 18 | ``` 19 | ```elixir 20 | :amoc.add(10) 21 | ``` 22 | 23 | Remove 10 users. 24 | ```erlang 25 | amoc:remove(10, true). 26 | ``` 27 | ```elixir 28 | :amoc.remove(10, true) 29 | ``` 30 | 31 | Note that removal operation is asynchronous, and if we call `amoc_controller:remove_users/2` two times in a row, it may select the same users for removal. 32 | 33 | Also note that all the user processes trap exits. 34 | 35 | #### Many independent Amoc nodes 36 | 37 | Sometimes a need arises to run several Amoc nodes independently from each other. 38 | In this case we would like to be able to run different ranges of user ids on every node. 39 | To do so, the following trick could be applied: 40 | 41 | 1. `amoc:start(my_scenario, 0, []).` 42 | 2. `amoc_controller:add_users(StartId, StopId).` 43 | 44 | NODE: in case of independent Amoc nodes, it's also possible to run different scenarios on different nodes. 45 | -------------------------------------------------------------------------------- /guides/scenario.md: -------------------------------------------------------------------------------- 1 | ## Developing a scenario 2 | 3 | A scenario specification is an [Erlang](https://www.erlang.org/) or [Elixir](https://elixir-lang.org/) module that implements the `amoc_scenario` behaviour. 4 | It has to export two callback functions: 5 | - `init/0` - called only once per test run on every node, at the very beginning. 6 | It can be used for setting up initial (global) state: metrics, database connections, etc. 7 | It can return an `ok` or a tuple `{ok, State}` from which the `State` may be passed to every user. 8 | - `start/1` or `start/2` - implements the actual scenario and is executed for 9 | each user, in the context of that user's process. 10 | The first argument is the given user's unique integer id. 11 | The second, which is optional, is the state, as returned from the `init/0` function. 12 | 13 | In addition to that, the scenario module can implement a `terminate/0,1` callback; 14 | it has one optional argument – the scenario state, as returned from the `init/0` function. 15 | 16 | A typical scenario file will look like this: 17 | 18 | ```erlang 19 | -module(my_scenario). 20 | 21 | -behaviour(amoc_scenario). 22 | 23 | -export([init/0]). 24 | -export([start/1]). 25 | 26 | init() -> 27 | %% initialize some metrics 28 | ok. 29 | 30 | start(Id) -> 31 | %% connect user 32 | %% fetch user's history 33 | %% send some messages 34 | %% wait a little bit 35 | %% send some messages again 36 | ok. 37 | ``` 38 | 39 | ```elixir 40 | defmodule LoadTest do 41 | @behaviour :amoc_scenario 42 | 43 | def init do 44 | ## initialize some metrics 45 | :ok 46 | end 47 | 48 | def start(id) do 49 | ## connect user 50 | ## fetch user's history 51 | ## send some messages 52 | ## wait a little bit 53 | ## send some messages again 54 | :ok 55 | end 56 | 57 | end 58 | ``` 59 | 60 | or, using the `start/2` function: 61 | 62 | ```erlang 63 | -module(my_scenario). 64 | 65 | -behaviour(amoc_scenario). 66 | 67 | -export([init/0]). 68 | -export([start/2]). 69 | 70 | init() -> 71 | %% initialize some metrics 72 | Settings = get_settings(), 73 | {ok, Settings}. 74 | 75 | start(Id, Settings) -> 76 | %% connect user using Settings 77 | %% fetch user's history 78 | %% send some messages 79 | %% wait a little bit 80 | %% send some messages again 81 | ok. 82 | ``` 83 | 84 | ```elixir 85 | defmodule LoadTest do 86 | @behaviour :amoc_scenario 87 | 88 | def init do 89 | ## initialize some metrics 90 | settings = get_settings() 91 | {:ok, settings} 92 | end 93 | 94 | def start(id, settings) do 95 | ## connect user using Settings 96 | ## fetch user's history 97 | ## send some messages 98 | ## wait a little bit 99 | ## send some messages again 100 | :ok 101 | end 102 | 103 | end 104 | ``` 105 | 106 | For developing XMPP scenarios, we recommend the 107 | [esl/escalus](https://github.com/esl/escalus) library. 108 | If additional dependencies are required by your scenario, 109 | a `rebar.config` file can be created inside the `scenario` dir 110 | and `deps` from that file will be merged with Amoc's dependencies. 111 | 112 | ## Coordinate users 113 | 114 | See [Amoc coordinator](coordinator.html). 115 | 116 | ## Throttle actions 117 | 118 | See [Amoc throttle](throttle.md). 119 | -------------------------------------------------------------------------------- /guides/telemetry.md: -------------------------------------------------------------------------------- 1 | Amoc also exposes the following telemetry events: 2 | 3 | ## Scenario 4 | 5 | All telemetry spans below contain an extra key `return` in the metadata for the `stop` event with the return value of the given callback. 6 | 7 | A telemetry span of a scenario initialisation (i.e. the exported `init/0` function): 8 | 9 | ```erlang 10 | event_name: [amoc, scenario, init, _] 11 | measurements: #{} %% As described in `telemetry:span/3` 12 | metadata: #{scenario := module()} %% Plus as described in `telemetry:span/3` 13 | ``` 14 | 15 | A telemetry span of a full scenario execution for a user (i.e. the exported `start/1,2` function): 16 | 17 | ```erlang 18 | event_name: [amoc, scenario, start, _] 19 | measurements: #{} %% As described in `telemetry:span/3` 20 | metadata: #{scenario := module(), %% Running scenario 21 | state := term(), %% The state as returned by `init/0` 22 | user_id := non_neg_integer() %% User ID assigned to the running process 23 | } %% Plus as described in `telemetry:span/3` 24 | ``` 25 | 26 | A telemetry span of a full scenario execution for a user (i.e. the exported `terminate/1,2` function): 27 | 28 | ```erlang 29 | event_name: [amoc, scenario, terminate, _] 30 | measurements: #{} %% As described in `telemetry:span/3` 31 | metadata: #{scenario := module(), %% Running scenario 32 | state := term() %% The state as returned by `init/0` 33 | } %% Plus as described in `telemetry:span/3` 34 | ``` 35 | 36 | ## Controller 37 | 38 | Indicates the number of users manually added or removed 39 | 40 | ```erlang 41 | event_name: [amoc, controller, users] 42 | measurements: #{count := non_neg_integer()} 43 | metadata: #{monotonic_time := integer(), scenario := module(), type := add | remove} 44 | ``` 45 | 46 | ## Throttle 47 | 48 | ### Init 49 | 50 | Raised when a throttle mechanism is initialised. 51 | 52 | ```erlang 53 | event_name: [amoc, throttle, init] 54 | measurements: #{count := 1} 55 | metadata: #{monotonic_time := integer(), name := atom()} 56 | ``` 57 | 58 | ### Rate 59 | 60 | Raised when a throttle mechanism is initialised or its configured rate is changed. 61 | This event is raised only on the master node. 62 | 63 | ```erlang 64 | event_name: [amoc, throttle, rate] 65 | measurements: #{rate := rate(), interval := interval()} 66 | metadata: #{monotonic_time := integer(), name := atom(), msg => binary()} 67 | ``` 68 | 69 | ### Request 70 | 71 | Raised when a process client requests to be allowed pass through a throttled mechanism. 72 | 73 | ```erlang 74 | event_name: [amoc, throttle, request] 75 | measurements: #{count := 1} 76 | metadata: #{monotonic_time := integer(), name := atom()} 77 | ``` 78 | 79 | ### Execute 80 | 81 | Raised when a process client is allowed to execute after a throttled mechanism. 82 | 83 | ```erlang 84 | event_name: [amoc, throttle, execute] 85 | measurements: #{count := 1} 86 | metadata: #{monotonic_time := integer(), name := atom()} 87 | ``` 88 | 89 | ### Throttle process internals 90 | 91 | Events related to internals of the throttle processes, these might expose unstable conditions you 92 | might want to log or reconfigure: 93 | 94 | ```erlang 95 | event_name: [amoc, throttle, process] 96 | measurements: #{logger:level() => 1} 97 | metadata: #{monotonic_time := integer(), 98 | log_level := logger:level(), 99 | msg := binary(), 100 | rate => non_neg_integer(), 101 | interval => non_neg_integer(), 102 | state => map(), 103 | _ => _} 104 | ``` 105 | 106 | ## Coordinator 107 | 108 | ### Event 109 | Indicates when a coordinating event was raised, like a process being added for coordination or a timeout being triggered 110 | 111 | ```erlang 112 | event_name: [amoc, coordinator, start | stop | add | reset | timeout] 113 | measurements: #{count := 1} 114 | metadata: #{monotonic_time := integer(), name := atom()} 115 | ``` 116 | 117 | ### Action triggered 118 | Indicates an action is about to be triggered, either by enough users in the group or by timeout 119 | 120 | ```erlang 121 | event_name: [amoc, coordinator, execute] 122 | measurements: #{count := num_of_users()} 123 | metadata: #{monotonic_time := integer(), event := coordinate | reset | timeout | stop} 124 | ``` 125 | 126 | ## Config 127 | 128 | ### Internal events 129 | There are related to bad configuration events, they might deserve logging 130 | 131 | ```erlang 132 | event_name: [amoc, config, get | verify | env] 133 | measurements: #{logger:level() => 1} 134 | metadata: #{monotonic_time := integer(), 135 | log_level => logger:level(), 136 | setting => atom(), 137 | msg => binary(), _ => _} 138 | ``` 139 | 140 | ## Cluster 141 | 142 | ### Internal events 143 | There are related to clustering events 144 | 145 | ```erlang 146 | event_name: [amoc, cluster, connect_nodes | nodedown | master_node_down] 147 | measurements: #{count => non_neg_integer()}, 148 | metadata: #{nodes => nodes(), state => map()} 149 | ``` 150 | -------------------------------------------------------------------------------- /guides/throttle.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | See `m:amoc_throttle`. 4 | 5 | ## Overview 6 | 7 | Amoc throttle is a module that allows limiting the number of users' actions per given interval, no matter how many users there are in a test. 8 | It works in both local and distributed environments, allows for dynamic rate changes during a test and exposes telemetry events showing the number of requests and executions. 9 | 10 | Amoc throttle allows to: 11 | 12 | - Setting the execution `Rate` per `Interval`, or inversely, the `Interarrival` time between actions. 13 | - Limiting the number of parallel executions when `interval` is set to `0`. 14 | 15 | Each throttle is identified with a `Name`. 16 | The rate limiting mechanism allows responding to a request only when it does not exceed the given throttle. 17 | Amoc throttle makes sure that the given throttle is maintained on a constant level. 18 | It prevents bursts of executions which could blurry the results, as they technically produce a desired rate in a given interval. 19 | Because of that, it may happen that the actual throttle rate would be slightly below the demanded rate. However, it will never be exceeded. 20 | 21 | ## Examples 22 | 23 | A typical use of Amoc throttle will look something like this: 24 | 25 | ```erlang 26 | -module(scenario_with_throttle). 27 | 28 | -behaviour(amoc_scenario). 29 | 30 | -export([init/0]). 31 | -export([start/1]). 32 | 33 | init() -> 34 | amoc_throttle:start(messages_rate, 100), %% 100 messages per minute 35 | %% continue initialization 36 | ok. 37 | 38 | start(Id) -> 39 | %% initialize user 40 | user_loop(Id), 41 | ok. 42 | 43 | user_loop(Id) -> 44 | amoc_throttle:send_and_wait(messages_rate, some_message), 45 | send_message(Id), 46 | user_loop(Id). 47 | ``` 48 | Here a system should be under a continuous load of 100 messages per minute. 49 | Note that if we used something like `amoc_throttle:run(messages_rate, fun() -> send_message(Id) end)` instead of `amoc_throttle:wait/1` the system would be flooded with requests. 50 | 51 | A test may of course be much more complicated. 52 | For example it can have the load changing in time. 53 | A plan for that can be set for the whole test in `init/1`: 54 | ```erlang 55 | init() -> 56 | amoc_throttle:start(messages_rate, 100), 57 | %% 9 steps of 100 increases in Rate, each lasting one minute 58 | Gradual = #{from_rate => 100, 59 | to_rate => 1000, 60 | step_count => 9, 61 | step_size => 100, 62 | step_interval => timer:minutes(1)}, 63 | amoc_throttle:change_rate_gradually(messages_rate, Gradual). 64 | ``` 65 | 66 | Normal Erlang messages can be used to schedule tasks for users by themselves or by some controller process. 67 | Below is a sketch of a user's behaviour during a test. 68 | It waits for messages in a loop, and sends one after receiving the message `{send_message, To}`. 69 | The rate of messages sent during a test will not exceed the one set in the `message_rate`. 70 | Sending messages is scheduled in the `set_up/1` function and in the user loop if some arbitrary condition is met. 71 | This models the behaviour common across load tests, when users respond only to some messages. 72 | 73 | ```erlang 74 | set_up(Users) -> 75 | [User ! {send_message, To} || User <- Users, To <- Users]. 76 | 77 | user_loop() -> 78 | receive 79 | {send_message, To} -> 80 | send_xmpp_message(To), 81 | user_loop(); 82 | Message -> 83 | process_message(Message), 84 | user_loop() 85 | end. 86 | 87 | process_message(Message) -> 88 | case some_condition(Message) of 89 | true -> 90 | To = get_sender(Message), 91 | amoc_throttle:send(message_rate, {send_message, To}); 92 | false -> 93 | ok 94 | end. 95 | ``` 96 | 97 | For a more comprehensive example please refer to the `throttle_test` scenario, which shows possible usages of the Amoc throttle. 98 | 99 | ## How it works 100 | 101 | ### Module overview 102 | 103 | - `amoc_throttle.erl` - provides an API for `amoc_throttle_controller`. 104 | - `amoc_throttle_controller.erl` - a gen_server which is responsible for reacting to requests, and managing `throttle_processes`. 105 | In a distributed environment an instance of `throttle_controller` runs on every node, and the one running on the master Amoc node stores the state for all nodes. 106 | - `amoc_throttle_process.erl` - gen_server module, implements the logic responsible for limiting the rate. 107 | For every `Name`, a number of processes are created, each responsible for keeping executions at a level proportional to their part of the throttle. 108 | 109 | ### Distributed environment 110 | 111 | #### Metrics 112 | In a distributed environment every Amoc node with a throttle started, exposes telemetry events showing the numbers of requests and executions. 113 | Those exposed by the master node show the aggregate of all telemetry events from all nodes. 114 | This allows to quickly see the real rates across the whole system. 115 | 116 | #### Workflow 117 | When a user executes `amoc_throttle:run/2`, a request is reported to a metric that runs on the user's node. 118 | Then a runner process is spawned on the same node. 119 | Its task will be to execute `Fun` asynchronously. 120 | A random throttle process which is assigned to the `Name` is asked for a permission for asynchronous runner to execute `Fun`. 121 | When the request reaches the master node, where throttle processes reside, the request metric on the master node is updated and the throttle process which got the request starts monitoring the asynchronous runner process. 122 | Then, depending on the system's load and the current rate of executions, the asynchronous runner is allowed to run the `Fun` or compelled to wait, because executing the function would exceed the calculated throttle. 123 | When the rate finally allows it, the asynchronous runner gets the permission to run the function from the throttle process. 124 | Both processes increase the metrics which count executions, but for each the metric is assigned to their own node. 125 | Then the asynchronous runner tries to execute `Fun`. 126 | It may succeed or fail, either way it dies and an `'EXIT'` signal is sent to the throttle process. 127 | This way it knows that the execution of a task has ended, and can allow a different process to run its task connected to the same `Name` if the current throttle allows it. 128 | 129 | Below is a graph showing the communication between processes on different nodes described above. 130 | ![amoc_throttle_dist](assets/amoc_throttle_dist.svg) 131 | -------------------------------------------------------------------------------- /integration_test/README.md: -------------------------------------------------------------------------------- 1 | ## Integration tests 2 | 3 | All shell scripts should conform to this 4 | [code style](https://google.github.io/styleguide/shellguide.html) 5 | 6 | ### 1. Build amoc docker image 7 | 8 | In the Amoc repo root directory run: 9 | 10 | `./integration_test/build_docker_image.sh` 11 | 12 | This command builds the `amoc:latest` docker image. 13 | 14 | ### 2. Start amoc test cluster 15 | 16 | `./integration_test/start_test_cluster.sh` 17 | 18 | This command requires the `amoc:latest` docker image to exist. More information about the test cluster can be found further in this document. 19 | 20 | ### 3. Check that clustering is done properly 21 | 22 | `./integration_test/test_amoc_cluster.sh` 23 | 24 | This command verifies that clustering is done properly. 25 | 26 | ### 4. Test distribution of a custom scenario in amoc cluster 27 | 28 | `./integration_test/test_distribute_scenario.sh` 29 | 30 | This command checks distribution of the sample `dummy_scenario.erl` from the `amoc-master` node 31 | to the worker nodes. 32 | 33 | ### 5. Run the distributed scenario. 34 | 35 | `./integration_test/test_run_scenario.sh` 36 | 37 | This command starts execution of `dummy_scenario.erl` scenario (it must be distributed 38 | prior to this action) 39 | 40 | ### 6. Add additional node to the cluster 41 | 42 | `./integration_test/test_add_new_node.sh` 43 | 44 | This command verifies that joining of the new node to the cluster is done properly. 45 | It is expected that cluster is running `dummy_scenario.erl` scenario at the moment 46 | when the new amoc node joins. 47 | 48 | ### 7. Cleanup 49 | 50 | To stop Amoc test cluster run: 51 | 52 | `./integration_test/stop_test_cluster.sh` 53 | 54 | ## Test cluster 55 | 56 | * To start the test cluster you can run these commands: 57 | 58 | ``` 59 | ./integration_test/build_docker_image.sh 60 | ./integration_test/start_test_cluster.sh 61 | ``` 62 | 63 | * To get the list of nodes in the amoc test cluster use the following command: 64 | 65 | `docker compose -p "amoc-test-cluster" ps` 66 | 67 | * To check the most recent `amoc-master` logs you can run this command: 68 | 69 | `docker compose -p "amoc-test-cluster" logs --tail=100 amoc-master` 70 | 71 | * In order to attach to the `amoc-master` erlang node run the following command: 72 | 73 | `docker compose -p "amoc-test-cluster" exec amoc-master amoc remote_console` 74 | 75 | * To open a shell inside the `amoc-master` container use this command: 76 | 77 | `docker compose -p "amoc-test-cluster" exec amoc-master bash` 78 | -------------------------------------------------------------------------------- /integration_test/build_docker_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | cd "$git_root" 6 | 7 | otp_vsn="${OTP_RELEASE:-25.3}" 8 | echo "ERLANG/OTP '${otp_vsn}'" 9 | 10 | docker build \ 11 | -f Dockerfile \ 12 | -t amoc:latest \ 13 | --build-arg "otp_vsn=${otp_vsn}" \ 14 | . 15 | -------------------------------------------------------------------------------- /integration_test/docker-compose.yml: -------------------------------------------------------------------------------- 1 | x-amoc-defaults: &amoc-defaults 2 | image: "amoc:latest" 3 | networks: 4 | - amoc-test-network 5 | volumes: 6 | - type: bind 7 | source: ./extra_code_paths 8 | target: /extra_code_paths 9 | environment: 10 | AMOC_NODES: "['amoc@amoc-master']" 11 | healthcheck: 12 | test: "amoc status" 13 | 14 | services: 15 | amoc-master: 16 | <<: *amoc-defaults 17 | hostname: "amoc-master" 18 | environment: 19 | AMOC_EXTRA_CODE_PATHS: '["/extra_code_paths/path1", "/extra_code_paths/path2"]' 20 | amoc-worker-1: &amoc-worker 21 | <<: *amoc-defaults 22 | hostname: "amoc-worker-1" 23 | amoc-worker-2: 24 | <<: *amoc-defaults 25 | hostname: "amoc-worker-2" 26 | amoc-worker-3: 27 | <<: *amoc-defaults 28 | hostname: "amoc-worker-3" 29 | networks: 30 | amoc-test-network: 31 | -------------------------------------------------------------------------------- /integration_test/extra_code_paths/path1/dummy_helper.erl: -------------------------------------------------------------------------------- 1 | -module(dummy_helper). 2 | 3 | -include_lib("stdlib/include/assert.hrl"). 4 | 5 | -required_variable(#{name => dummy_var, 6 | description => "dummy_var", 7 | default_value => default_value}). 8 | 9 | -define(comment(U), io_lib:format("Condition failed with last users distribution ~n~p", [U])). 10 | 11 | %% amoc_dist testing function 12 | -export([test_amoc_dist/0]). 13 | 14 | test_amoc_dist() -> 15 | try 16 | Master = amoc_cluster:master_node(), 17 | Slaves = amoc_cluster:slave_nodes(), 18 | %% check the status of the nodes 19 | ?assertEqual(disabled, get_status(Master)), 20 | [ ?assertMatch({running, #{scenario := dummy_scenario}}, get_status(Node)) || Node <- Slaves], 21 | %% check user ids, users have all been started at the first two nodes 22 | {N1, Max1, Nodes1, Ids1, Users1} = get_users_info(Slaves), 23 | ?assert(N1 > 0), 24 | ?assertEqual(N1, Max1, ?comment(Users1)), 25 | ?assertEqual(Ids1, lists:seq(1, N1), ?comment(Users1)), 26 | [AddedNode] = Slaves -- Nodes1, 27 | %% add 20 users 28 | add_and_wait(Master, 20), 29 | {N2, Max2, Nodes2, Ids2, Users2} = get_users_info(Slaves), 30 | ?assertEqual(N2, Max2, ?comment(Users2)), 31 | ?assertEqual(Ids2, lists:seq(1, N2), ?comment(Users2)), 32 | ?assertEqual([AddedNode], Nodes2 -- Nodes1, ?comment(Users2)), 33 | ?assertEqual(N2, N1 + 20, ?comment(Users2)), 34 | %% remove 10 users 35 | remove_and_wait(Master, 10), 36 | {N3, Max3, Nodes3, _Ids3, Users3} = get_users_info(Slaves), 37 | ?assertEqual(N2 - 10, N3, ?comment(Users3)), 38 | ?assertEqual(Max2, Max3, ?comment(Users3)), 39 | ?assertEqual(Nodes2, Nodes3, ?comment(Users3)), 40 | %% try to remove N3 users 41 | Ret = remove_and_wait(Master, N3), 42 | RemovedN = lists:sum([N || {_, N} <- Ret]), 43 | {N4, _Max4, Nodes4, Ids4, Users4} = get_users_info(Slaves), 44 | ?assertEqual(N3 - RemovedN, N4, ?comment(Users4)), 45 | ?assertEqual(Nodes1, Nodes4, ?comment(Users4)), 46 | ?assert(RemovedN < N3), 47 | %% add 20 users 48 | add_and_wait(Master, 20), 49 | {N5, Max5, Nodes5, Ids5, Users5} = get_users_info(Slaves), 50 | ?assertEqual(Nodes2, Nodes5, ?comment(Users5)), 51 | ?assertEqual(Max5, Max2 + 20, ?comment(Users5)), 52 | ?assertEqual(N5, N4 + 20, ?comment(Users5)), 53 | ?assertEqual(Ids5 -- Ids4, lists:seq(Max2 + 1, Max5), ?comment(Users5)), 54 | %% terminate scenario 55 | stop(Master), 56 | [ ?assertEqual({finished, dummy_scenario}, get_status(Node)) || Node <- Slaves], 57 | %% return expected value 58 | amoc_dist_works_as_expected 59 | catch 60 | C:E:S -> 61 | {C, E, S} 62 | end. 63 | 64 | get_users_info(SlaveNodes) -> 65 | Distrib = [ {Node, erpc:call(Node, amoc_users_sup, get_all_children, [])} || Node <- SlaveNodes ], 66 | Ids = lists:usort([Id || {_Node, Users} <- Distrib, {_, Id} <- Users]), 67 | Nodes = lists:usort([Node || {Node, Users} <- Distrib, [] =/= Users]), 68 | N = length(Ids), 69 | MaxId = lists:max(Ids), 70 | {N, MaxId, Nodes, Ids, Distrib}. 71 | 72 | add_and_wait(Master, Num) -> 73 | {ok, Ret} = erpc:call(Master, amoc_dist, add, [Num]), 74 | timer:sleep(3000), 75 | Ret. 76 | 77 | remove_and_wait(Master, Num) -> 78 | {ok, Ret} = erpc:call(Master, amoc_dist, remove, [Num, true]), 79 | timer:sleep(3000), 80 | Ret. 81 | 82 | stop(Master) -> 83 | {ok, Ret} = erpc:call(Master, amoc_dist, stop, []), 84 | timer:sleep(3000), 85 | Ret. 86 | 87 | get_status(Node) -> 88 | erpc:call(Node, amoc_controller, get_status, []). 89 | -------------------------------------------------------------------------------- /integration_test/extra_code_paths/path2/dummy_scenario.erl: -------------------------------------------------------------------------------- 1 | -module(dummy_scenario). 2 | -behaviour(amoc_scenario). 3 | 4 | %% var1 must be equal to one of the values in the verification list 5 | %% and the default value is def1 6 | -required_variable(#{name => var1, description => "description1", 7 | default_value => def1, 8 | verification => [def1, another_value]}). 9 | 10 | -required_variable([ 11 | %% var2 must be positively verified by the test_verification_function/1 function. 12 | %% verification function must be supplied as an mfa of the format 13 | %% '{module, function, arity}', or a fun of the format `fun module:function/arity`. 14 | %% Note that it must be an exported function, otherwise it will not pass compilation. 15 | #{name => var2, description => "description2", default_value => def2, 16 | verification => {?MODULE, test_verification_function, 1}}, 17 | #{name => var3, description => "description3", default_value => def3, 18 | verification => {?MODULE, test_verification_function, 1}}]). 19 | 20 | %% 'none' is a predefined verification function which accepts all the values 21 | -required_variable(#{name => var4, description => "description4", 22 | default_value => def4, 23 | verification => none}). 24 | 25 | -required_variable([ 26 | %% when verification method is not set, it defaults to `none` 27 | #{name => var5, description => "description5", default_value => def5}, 28 | %% when value is not set, it defaults to `undefined` 29 | #{name => var6, description => "description6"}, 30 | #{name => nodes, description => "this variable is set for docker " 31 | "container via AMOC_NODES env"}, 32 | #{name => test, description => "this one to be set via REST API"}]). 33 | 34 | %% parameter verification method 35 | -export([test_verification_function/1]). 36 | 37 | %% amoc_scenario behaviour 38 | -export([init/0, start/1]). 39 | 40 | test_verification_function(def2) -> true; 41 | test_verification_function(_) -> {true, new_value}. 42 | 43 | -spec init() -> ok. 44 | init() -> 45 | %% amoc follows a couple of rules during the scenario initialisation: 46 | %% - if any parameter verification fails, amoc will not start 47 | %% the scenario and the init/0 function is not triggered. 48 | %% - if the init/0 function fails, amoc will not start any users (by 49 | %% calling a start/1 or start2 function) 50 | %% if the REST API reports that scenario is executed, than all the 51 | %% initialisation steps described above have passed successfully 52 | def1 = amoc_config:get(var1, def_value), 53 | def2 = amoc_config:get(var2, def_value), 54 | new_value = amoc_config:get(var3, def_value), 55 | def4 = amoc_config:get(var4, def_value), 56 | def5 = amoc_config:get(var5, def_value), 57 | def_value = amoc_config:get(var6, def_value), 58 | undefined = amoc_config:get(var6), 59 | [_ | _] = amoc_config:get(nodes), 60 | %% it doesn't matter if an undeclared_variable is passed through the 61 | %% os or erlang app environment variable. if it's not declared using 62 | %% the -required_variable(...) attribute, then any attempt to get it 63 | %% results in exception. 64 | {invalid_setting, undeclared_variable} = 65 | (catch amoc_config:get(undeclared_variable)), 66 | %% this variable is set via REST API 67 | <<"test_value">> = amoc_config:get(test), 68 | %% dummy_var variable is defined in the dummy_helper module. 69 | %% if dummy_helper is not propagated, then this call crashes 70 | default_value = amoc_config:get(dummy_var), 71 | ok. 72 | 73 | -spec start(amoc_scenario:user_id()) -> any(). 74 | start(_Id) -> 75 | %%sleep 15 minutes 76 | timer:sleep(timer:minutes(15)), 77 | amoc_user:stop(). 78 | -------------------------------------------------------------------------------- /integration_test/helper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################### 4 | ## general purpose functions ## 5 | ############################### 6 | function enable_strict_mode() { 7 | # the below settings are based on: 8 | # http://redsymbol.net/articles/unofficial-bash-strict-mode/ 9 | set -euo pipefail 10 | IFS=$'\n\t' 11 | } 12 | 13 | function compile_file() { 14 | local erl_file="$(realpath "$1")" 15 | local output_dir="$(dirname "$erl_file")" 16 | erlc -o "$output_dir" "$erl_file" 17 | } 18 | 19 | function contains_all() { 20 | local output="$(cat -)" 21 | local ret= acc=0 22 | for pattern in "$@"; do 23 | ret="$(echo "$output" | grep -L -e "$pattern" | wc -l)" 24 | if [ "$ret" -ne "0" ]; then 25 | [ "$(($acc))" -eq "0" ] && { 26 | echo "contains_all FAILED" 27 | echo "stdin: '${output}'"; } 28 | echo "pattern is missing: '${pattern}'" 29 | fi >&2 30 | acc+="+${ret}" 31 | done 32 | test "$(($acc))" -eq 0 33 | } 34 | 35 | function doesnt_contain_any() { 36 | local output="$(cat -)" 37 | local ret= acc=0 38 | for pattern in "$@"; do 39 | ret="$(echo "$output" | grep -l -e "$pattern" | wc -l || true)" 40 | if [ "$ret" -ne "0" ]; then 41 | [ "$(($acc))" -eq "0" ] && { 42 | echo "doesnt_contain_any FAILED" 43 | echo "stdin: '${output}'"; } 44 | echo "pattern is present: '${pattern}'" 45 | fi >&2 46 | acc+="+${ret}" 47 | done 48 | test "$(($acc))" -eq 0 49 | } 50 | 51 | ###################### 52 | ## docker functions ## 53 | ###################### 54 | docker_compose() { 55 | local compose_file="${git_root}/integration_test/docker-compose.yml" 56 | docker compose -p "amoc-test-cluster" -f "$compose_file" "$@" 57 | } 58 | 59 | function amoc_eval() { 60 | local exec_path="amoc" 61 | local service="$1" 62 | shift 1 63 | docker_compose exec -T "$service" "$exec_path" eval "$@" 64 | } 65 | 66 | ###################### 67 | ## common variables ## 68 | ###################### 69 | git_root="$(git rev-parse --show-toplevel)" 70 | -------------------------------------------------------------------------------- /integration_test/start_test_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | 6 | compile_file integration_test/extra_code_paths/path1/dummy_helper.erl 7 | compile_file integration_test/extra_code_paths/path2/dummy_scenario.erl 8 | 9 | docker_compose up --wait --wait-timeout 100 amoc-{master,worker-1,worker-2} 10 | -------------------------------------------------------------------------------- /integration_test/stop_test_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | 6 | docker_compose down 7 | -------------------------------------------------------------------------------- /integration_test/test_add_new_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | 6 | docker_compose up --wait --wait-timeout 100 amoc-worker-3 7 | 8 | amoc_eval amoc-worker-3 "amoc_controller:get_status()." | contains_all dummy_scenario running 9 | amoc_eval amoc-worker-3 "binary_to_list(amoc_config:get(test))." | contains_all "test_value" 10 | amoc_eval amoc-worker-3 "dummy_helper:test_amoc_dist()." | contains_all 'amoc_dist_works_as_expected' 11 | echo "amoc_dist_works_as_expected" 12 | -------------------------------------------------------------------------------- /integration_test/test_amoc_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | 6 | echo "checking that clustering is done properly" 7 | amoc_eval amoc-master "nodes()." | contains_all amoc-worker-1 amoc-worker-2 8 | amoc_eval amoc-worker-1 "nodes()." | contains_all amoc-master amoc-worker-2 9 | amoc_eval amoc-worker-2 "nodes()." | contains_all amoc-master amoc-worker-1 10 | 11 | echo "checking that setting AMOC_EXTRA_CODE_PATHS env works as expected" 12 | amoc_eval amoc-master "amoc_code_server:list_scenario_modules()." | contains_all dummy_scenario 13 | amoc_eval amoc-master "amoc_code_server:list_configurable_modules()." | contains_all dummy_helper 14 | amoc_eval amoc-worker-1 "amoc_code_server:list_scenario_modules()." | doesnt_contain_any dummy_scenario 15 | amoc_eval amoc-worker-1 "amoc_code_server:list_configurable_modules()." | doesnt_contain_any dummy_helper 16 | amoc_eval amoc-worker-2 "amoc_code_server:list_scenario_modules()." | doesnt_contain_any dummy_scenario 17 | amoc_eval amoc-worker-2 "amoc_code_server:list_configurable_modules()." | doesnt_contain_any dummy_helper 18 | -------------------------------------------------------------------------------- /integration_test/test_distribute_scenario.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | cd "${git_root}/integration_test" 6 | 7 | modules=( "dummy_scenario" "dummy_helper" ) 8 | 9 | function get_scenarios() { 10 | amoc_eval "$1" "amoc_code_server:list_scenario_modules()." 11 | } 12 | 13 | function get_helpers() { 14 | amoc_eval "$1" "amoc_code_server:list_configurable_modules()." 15 | } 16 | 17 | function list_scenarios_and_helpers() { 18 | local scenarios="$(get_scenarios "$1")" 19 | local helpers="$(get_helpers "$1")" 20 | echo "Scenarios on the ${1} node: ${scenarios}" 21 | echo "Configurable helpers on the ${1} node: ${helpers}" 22 | } 23 | 24 | function erlang_list() { 25 | local ret=( "[" ) 26 | local element 27 | if [ "$#" -gt 0 ]; then 28 | ret+=( "$1" ) 29 | shift 1 30 | for element in "$@"; do 31 | ret+=( "," "$element" ) 32 | done 33 | fi 34 | ret+=( "]" ) 35 | echo "${ret[@]}" 36 | } 37 | 38 | function ensure_modules_loaded() { 39 | local node="$1" 40 | shift 1 41 | local modules="$(erlang_list "$@")" 42 | amoc_eval "$node" "[code:ensure_loaded(M) || M <- ${modules}]." 43 | } 44 | 45 | function add_module() { 46 | local node="${1}" 47 | local module 48 | shift 1 49 | for module in "$@"; do 50 | echo "adding module '${module}' for distribution from the node '${node}'" 51 | amoc_eval "${node}" "amoc_code_server:add_module(${module})." 52 | done 53 | } 54 | 55 | function distribute_modules() { 56 | amoc_eval "${1}" "amoc_code_server:distribute_modules('amoc@${2}')." 57 | } 58 | 59 | ensure_modules_loaded amoc-master "${modules[@]}" | contains_all "${modules[@]}" 60 | ensure_modules_loaded amoc-worker-1 "${modules[@]}" | doesnt_contain_any "${modules[@]}" 61 | ensure_modules_loaded amoc-worker-2 "${modules[@]}" | doesnt_contain_any "${modules[@]}" 62 | 63 | list_scenarios_and_helpers amoc-worker-2 | doesnt_contain_any "${modules[@]}" 64 | list_scenarios_and_helpers amoc-worker-1 | doesnt_contain_any "${modules[@]}" 65 | 66 | echo "Distributing scenario and helper module from the amoc-master node" 67 | ## amoc_controller is added to the list as an example of module 68 | ## that already exists on all the slave amoc nodes 69 | add_module amoc-master "${modules[@]}" amoc_controller 70 | distribute_modules amoc-master amoc-worker-1 | contains_all "${modules[@]}" amoc_controller 71 | 72 | ensure_modules_loaded amoc-worker-1 "${modules[@]}" | contains_all "${modules[@]}" 73 | ensure_modules_loaded amoc-worker-2 "${modules[@]}" | doesnt_contain_any "${modules[@]}" 74 | 75 | list_scenarios_and_helpers amoc-worker-1 | contains_all "${modules[@]}" 76 | list_scenarios_and_helpers amoc-worker-2 | doesnt_contain_any "${modules[@]}" 77 | -------------------------------------------------------------------------------- /integration_test/test_run_scenario.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source "$(dirname "$0")/helper.sh" 4 | enable_strict_mode 5 | 6 | ############################# 7 | ## amoc REST API functions ## 8 | ############################# 9 | run_scenario() { 10 | amoc_eval "$1" "amoc_dist:do(${2}, ${3}, [{test, <<\"test_value\">>}])." 11 | } 12 | 13 | result="$(run_scenario amoc-master dummy_scenario 10)" 14 | 15 | echo "$result" 16 | 17 | if echo "$result" | contains_all "ok" "'amoc@amoc-worker-1'" "'amoc@amoc-worker-2'" ; then 18 | echo "Scenario executed" 19 | exit 0 20 | else 21 | echo "Scenario failed" 22 | exit -1 23 | fi 24 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [ 2 | debug_info, 3 | warn_missing_spec 4 | ]}. 5 | 6 | {deps, [ 7 | {telemetry, "~> 1.3"} 8 | ]}. 9 | 10 | {profiles, [ 11 | {test, [ 12 | {deps, [ 13 | {meck, "1.0.0"}, 14 | {proper, "1.5.0"}, 15 | {bbmustache, "1.12.2"}, 16 | {wait_helper, "0.2.1"} 17 | ]} 18 | ]} 19 | ]}. 20 | 21 | {relx, [ 22 | {release, {amoc, git}, [amoc, runtime_tools]}, 23 | {debug_info, keep}, 24 | {include_src, true}, 25 | {include_erts, true}, 26 | {dev_mode, false}, 27 | {extended_start_script, true}, 28 | {sys_config, "rel/app.config"} 29 | ]}. 30 | 31 | {xref_checks, [ 32 | undefined_function_calls, 33 | undefined_functions, 34 | locals_not_used, 35 | deprecated_function_calls, 36 | deprecated_functions 37 | ]}. 38 | 39 | {dialyzer, [ 40 | {warnings, [unknown]} 41 | ]}. 42 | 43 | {project_plugins, [ 44 | {rebar3_hex, "~> 7.0"}, 45 | {rebar3_ex_doc, "~> 0.2"}, 46 | {rebar3_lint, "~> 4.0"}, 47 | {rebar3_codecov, "~> 0.7"} 48 | ]}. 49 | 50 | {ex_doc, [ 51 | {source_url, <<"https://github.com/esl/amoc">>}, 52 | {extras, [ 53 | {'README.md', #{title => <<"A Murder of Crows">>}}, 54 | {'guides/scenario.md', #{title => <<"Developing a scenario">>}}, 55 | {'guides/local-run.md', #{title => <<"Running locally">>}}, 56 | {'guides/configuration.md', #{title => <<"Configuration">>}}, 57 | {'guides/distributed.md', #{title => <<"Setting up a distributed environment">>}}, 58 | {'guides/distributed-run.md', #{title => <<"Running a load test">>}}, 59 | {'guides/telemetry.md', #{title => <<"Telemetry events">>}}, 60 | {'guides/throttle.md', #{title => <<"Amoc throttle">>}}, 61 | {'guides/coordinator.md', #{title => <<"Amoc coordinator">>}}, 62 | {'guides/amoc_livebook.livemd', #{title => <<"Livebook tutorial">>}}, 63 | {'LICENSE', #{title => <<"License">>}} 64 | ]}, 65 | {assets, #{<<"guides/assets">> => <<"assets">>}}, 66 | {main, <<"readme">>} 67 | ]}. 68 | 69 | {hex, [{doc, #{provider => ex_doc}}]}. 70 | 71 | {elvis, [ 72 | #{ 73 | dirs => ["src/**"], 74 | filter => "*.erl", 75 | ruleset => erl_files, 76 | rules => [ 77 | {elvis_text_style, line_length, #{skip_comments => whole_line}}, 78 | {elvis_style, export_used_types, disable}, 79 | {elvis_style, invalid_dynamic_call, #{ignore => [{amoc_code_server, get_md5}]}}, 80 | {elvis_style, no_block_expressions, #{ignore => [amoc_cluster]}}, 81 | {elvis_style, no_throw, #{ignore => [amoc_config]}} 82 | ] 83 | }, 84 | #{ 85 | dirs => ["."], 86 | filter => "rebar.config", 87 | ruleset => rebar_config 88 | } 89 | ]}. 90 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.2.0", 2 | [{<<"telemetry">>,{pkg,<<"telemetry">>,<<"1.3.0">>},0}]}. 3 | [ 4 | {pkg_hash,[ 5 | {<<"telemetry">>, <<"FEDEBBAE410D715CF8E7062C96A1EF32EC22E764197F70CDA73D82778D61E7A2">>}]}, 6 | {pkg_hash_ext,[ 7 | {<<"telemetry">>, <<"7015FC8919DBE63764F4B4B87A95B7C0996BD539E0D499BE6EC9D7F3875B79E6">>}]} 8 | ]. 9 | -------------------------------------------------------------------------------- /rel/app.config: -------------------------------------------------------------------------------- 1 | %% -*-erlang-*- 2 | 3 | [ 4 | {kernel, [ 5 | {logger, 6 | [{handler, log_to_console, logger_std_h, 7 | #{formatter => {logger_formatter, #{template => [time," [",level,"] ",msg,"\n"]}}} 8 | }, 9 | {handler, log_to_file, logger_std_h, 10 | #{config => #{type => {file, "log/erlang.log"}}, 11 | formatter => {logger_formatter, #{template => [time," [",level,"] ",msg,"\n"]}} 12 | }} 13 | ]}]} 14 | ]. 15 | 16 | -------------------------------------------------------------------------------- /src/amoc.app.src: -------------------------------------------------------------------------------- 1 | {application, amoc, [ 2 | {description, "A Murder of Crows"}, 3 | {vsn, git}, 4 | {registered, []}, 5 | {applications, [ 6 | kernel, 7 | stdlib, 8 | compiler, 9 | telemetry 10 | ]}, 11 | {mod, {amoc_app, []}}, 12 | {env, []}, 13 | {licenses, ["Apache-2.0"]}, 14 | {links, [{"GitHub", "https://github.com/esl/amoc"}]} 15 | ]}. 16 | -------------------------------------------------------------------------------- /src/amoc.erl: -------------------------------------------------------------------------------- 1 | %% @copyright 2023 Erlang Solutions Ltd. 2 | %% @doc API module for running locally, in a non-distributed environment 3 | %% 4 | %% Use `amoc_dist' module to run scenarios in a distributed environment 5 | %% @end 6 | -module(amoc). 7 | 8 | -export([do/3, 9 | add/1, 10 | remove/2, 11 | stop/0, 12 | reset/0]). 13 | 14 | -type scenario() :: module(). 15 | -export_type([scenario/0]). 16 | 17 | %% @doc Start a scenario with the given number of users and configuration 18 | -spec do(scenario(), non_neg_integer(), amoc_config:settings()) -> 19 | ok | {error, term()}. 20 | do(Scenario, Count, Settings) -> 21 | case amoc_cluster:set_master_node(node()) of 22 | ok -> 23 | %% amoc_controller:start_scenario/2 will fail, 24 | %% if amoc is running in a distributed mode 25 | case {amoc_controller:start_scenario(Scenario, Settings), Count} of 26 | {ok, 0} -> ok; 27 | {ok, Count} -> amoc_controller:add_users(1, Count); 28 | Error -> Error 29 | end; 30 | Error -> Error 31 | end. 32 | 33 | %% @doc Dynamically add more users to a currently running scenario 34 | -spec add(pos_integer()) -> ok | {error, any()}. 35 | add(Count) when is_integer(Count), Count > 0 -> 36 | case is_running_locally() of 37 | ok -> 38 | {running, #{highest_user_id := LastUserId}} = amoc_controller:get_status(), 39 | amoc_controller:add_users(LastUserId + 1, LastUserId + Count); 40 | Error -> Error 41 | end. 42 | 43 | %% @doc Dynamically remove more users from a currently running scenario, optionally forcibly 44 | %% 45 | %% Forcing user removal means that all users will be signal to exit in parallel, 46 | %% and will forcibly be killed after a short timeout (2 seconds), 47 | %% whether they have exited already or not. 48 | -spec remove(pos_integer(), boolean()) -> {ok, non_neg_integer()} | {error, any()}. 49 | remove(Count, ForceRemove) when is_integer(Count), Count > 0 -> 50 | case is_running_locally() of 51 | ok -> 52 | amoc_controller:remove_users(Count, ForceRemove); 53 | Error -> Error 54 | end. 55 | 56 | %% @doc Stop a running scenario 57 | -spec stop() -> ok | {error, any()}. 58 | stop() -> 59 | case is_running_locally() of 60 | ok -> 61 | amoc_controller:stop_scenario(); 62 | Error -> Error 63 | end. 64 | 65 | %% @doc Restart the whole amoc supervision tree 66 | -spec reset() -> ok | {error, term()}. 67 | reset() -> 68 | case is_running_locally() of 69 | ok -> 70 | application:stop(?MODULE), 71 | application:ensure_all_started(?MODULE), 72 | ok; 73 | Error -> 74 | Error 75 | end. 76 | 77 | %% ------------------------------------------------------------------ 78 | %% Local functions 79 | %% ------------------------------------------------------------------ 80 | 81 | -spec is_running_locally() -> ok | {error, any()}. 82 | is_running_locally() -> 83 | Node = node(), 84 | case {amoc_cluster:master_node(), amoc_controller:get_status()} of 85 | {undefined, _} -> {error, master_node_is_not_set}; 86 | {Node, disabled} -> {error, node_is_clustered}; 87 | {Node, _} -> ok; 88 | {_, _} -> {error, slave_node} 89 | end. 90 | -------------------------------------------------------------------------------- /src/amoc_app.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2023 Erlang Solutions Ltd. 3 | -module(amoc_app). 4 | 5 | -behaviour(application). 6 | 7 | %% Application callbacks 8 | -export([start/2, stop/1]). 9 | 10 | %% =================================================================== 11 | %% Application callbacks 12 | %% =================================================================== 13 | 14 | -spec start(application:start_type(), term()) -> {ok, pid()}. 15 | start(_StartType, _StartArgs) -> 16 | amoc_sup:start_link(). 17 | 18 | -spec stop(term()) -> ok. 19 | stop(_State) -> 20 | ok. 21 | -------------------------------------------------------------------------------- /src/amoc_scenario.erl: -------------------------------------------------------------------------------- 1 | %% @copyright 2024 Erlang Solutions Ltd. 2 | %% @doc Wrapper around the defined scenario 3 | -module(amoc_scenario). 4 | 5 | -export([init/1, terminate/2, start/3]). 6 | 7 | %%------------------------------------------------------------------------- 8 | %% behaviour definition 9 | %%------------------------------------------------------------------------- 10 | -export_type([user_id/0, state/0]). 11 | 12 | -type user_id() :: pos_integer(). %% Unique integer ID of every actor spawned 13 | -type state() :: term(). %% The state of the scenario as returned by `init/0' 14 | 15 | -callback init() -> ok | {ok, state()} | {error, Reason :: term()}. 16 | -callback start(user_id(), state()) -> any(). 17 | %% `start/2' is preferred over `start/1'. At least one of them is required. 18 | -callback start(user_id()) -> any(). 19 | -callback terminate(state()) -> any(). 20 | %% `terminate/1' is preferred over `terminate/0' 21 | -callback terminate() -> any(). 22 | 23 | -optional_callbacks([start/1, start/2]). 24 | -optional_callbacks([terminate/0, terminate/1]). 25 | 26 | %%------------------------------------------------------------------------- 27 | %% API 28 | %%------------------------------------------------------------------------- 29 | 30 | %% @doc Applies the `Scenario:init/0' callback 31 | %% 32 | %% Runs on the controller process and spans a `[amoc, scenario, init, _]' telemetry event. 33 | -spec init(amoc:scenario()) -> {ok, state()} | {error, Reason :: term()}. 34 | init(Scenario) -> 35 | case verify_exports_callbacks(Scenario) of 36 | true -> 37 | apply_safely(Scenario, init, [], #{scenario => Scenario}); 38 | false -> 39 | {error, invalid_scenario} 40 | end. 41 | 42 | verify_exports_callbacks(Scenario) -> 43 | erlang:function_exported(Scenario, init, 0) 44 | andalso erlang:function_exported(Scenario, start, 2) 45 | orelse erlang:function_exported(Scenario, start, 1). 46 | 47 | %% @doc Applies the `Scenario:terminate/0,1' callback 48 | %% 49 | %% `Scenario:terminate/0' and `Scenario:terminate/1' callbacks are optional. 50 | %% If the scenario module exports both functions, `Scenario:terminate/1' is used. 51 | %% 52 | %% Runs on the controller process and spans a `[amoc, scenario, terminate, _]' telemetry event. 53 | -spec terminate(amoc:scenario(), state()) -> ok | {ok, any()} | {error, Reason :: term()}. 54 | terminate(Scenario, State) -> 55 | Metadata = #{scenario => Scenario, state => State}, 56 | case {erlang:function_exported(Scenario, terminate, 1), 57 | erlang:function_exported(Scenario, terminate, 0)} of 58 | {true, _} -> 59 | %% since we ignore Scenario:terminate/1 return value 60 | %% we can use apply_safely/3 function 61 | apply_safely(Scenario, terminate, [State], Metadata); 62 | {_, true} -> 63 | %% since we ignore Scenario:terminate/0 return value 64 | %% we can use apply_safely/3 function 65 | apply_safely(Scenario, terminate, [], Metadata); 66 | _ -> 67 | ok 68 | end. 69 | 70 | %% @doc Applies the `Scenario:start/1,2' callback 71 | %% 72 | %% Either `Scenario:start/1' or `Scenario:start/2' must be exported from the behaviour module. 73 | %% if scenario module exports both functions, `Scenario:start/2' is used. 74 | %% 75 | %% Runs on the user process and spans a `[amoc, scenario, user, _]' telemetry event. 76 | -spec start(amoc:scenario(), user_id(), state()) -> term(). 77 | start(Scenario, Id, State) -> 78 | Metadata = #{scenario => Scenario, state => State, user_id => Id}, 79 | Span = case {erlang:function_exported(Scenario, start, 2), 80 | erlang:function_exported(Scenario, start, 1)} of 81 | {true, _} -> 82 | fun() -> 83 | Ret = Scenario:start(Id, State), 84 | {Ret, Metadata#{return => Ret}} 85 | end; 86 | {_, true} -> 87 | fun() -> 88 | Ret = Scenario:start(Id), 89 | {Ret, Metadata#{return => Ret}} 90 | end; 91 | {false, false} -> 92 | exit("the scenario module must export either start/2 or start/1 function") 93 | end, 94 | telemetry:span([amoc, scenario, start], Metadata, Span). 95 | 96 | %% ------------------------------------------------------------------ 97 | %% internal functions 98 | %% ------------------------------------------------------------------ 99 | 100 | -spec apply_safely(atom(), atom(), [term()], map()) -> {ok | error, term()}. 101 | apply_safely(M, F, A, Metadata) -> 102 | Span = fun() -> 103 | Ret = erlang:apply(M, F, A), 104 | {Ret, Metadata#{return => Ret}} 105 | end, 106 | try telemetry:span([amoc, scenario, F], Metadata, Span) of 107 | {ok, RetVal} -> {ok, RetVal}; 108 | {error, Error} -> {error, Error}; 109 | Result -> {ok, Result} 110 | catch 111 | Class:Exception:Stacktrace -> 112 | {error, {Class, Exception, Stacktrace}} 113 | end. 114 | -------------------------------------------------------------------------------- /src/amoc_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2024 Erlang Solutions Ltd. 3 | -module(amoc_sup). 4 | 5 | -behaviour(supervisor). 6 | 7 | %% API 8 | -export([start_link/0]). 9 | 10 | %% Supervisor callbacks 11 | -export([init/1]). 12 | 13 | %% Helper macro for declaring children of supervisor 14 | -define(WORKER(I), {I, {I, start_link, []}, permanent, 5000, worker, [I]}). 15 | -define(SUP(I), {I, {I, start_link, []}, permanent, infinity, supervisor, [I]}). 16 | 17 | %% =================================================================== 18 | %% API functions 19 | %% =================================================================== 20 | 21 | -spec start_link() -> supervisor:startlink_ret(). 22 | start_link() -> 23 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 24 | 25 | %% =================================================================== 26 | %% Supervisor callbacks 27 | %% =================================================================== 28 | -spec init(term()) -> 29 | {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 30 | init([]) -> 31 | {ok, {#{strategy => one_for_all, intensity => 0}, 32 | [ 33 | ?SUP(amoc_users_sup), 34 | ?SUP(amoc_throttle_sup), 35 | ?SUP(amoc_coordinator_sup), 36 | ?WORKER(amoc_controller), 37 | ?WORKER(amoc_cluster), 38 | ?WORKER(amoc_code_server) 39 | ]}}. 40 | -------------------------------------------------------------------------------- /src/amoc_telemetry.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2023 Erlang Solutions Ltd. 3 | -module(amoc_telemetry). 4 | 5 | -export([execute/3, execute_log/4]). 6 | 7 | -spec execute(EventName, Measurements, Metadata) -> ok when 8 | EventName :: telemetry:event_name(), 9 | Measurements :: telemetry:event_measurements(), 10 | Metadata :: telemetry:event_metadata(). 11 | execute(Name, Measurements, Metadata) -> 12 | TimeStamp = erlang:monotonic_time(), 13 | PrefixedName = [amoc | Name], 14 | MetadataWithTS = Metadata#{monotonic_time => TimeStamp}, 15 | telemetry:execute(PrefixedName, Measurements, MetadataWithTS). 16 | 17 | -spec execute_log(Level, EventName, Metadata, Msg) -> ok when 18 | Level :: logger:level(), 19 | EventName :: telemetry:event_name(), 20 | Metadata :: telemetry:event_metadata(), 21 | Msg :: binary(). 22 | execute_log(Level, Name, Metadata, Message) -> 23 | MetadataWithLog = Metadata#{log_level => Level, msg => Message}, 24 | execute(Name, #{Level => 1}, MetadataWithLog). 25 | -------------------------------------------------------------------------------- /src/config/amoc_config.erl: -------------------------------------------------------------------------------- 1 | %% @see amoc_config 2 | %% @copyright 2023 Erlang Solutions Ltd. 3 | %% @doc TODO 4 | -module(amoc_config). 5 | 6 | -include("amoc_config.hrl"). 7 | 8 | -export([get/1, get/2]). 9 | -export_type([name/0, value/0, settings/0, maybe_module_config/0]). 10 | 11 | %% ------------------------------------------------------------------ 12 | %% API 13 | %% ------------------------------------------------------------------ 14 | -spec get(name()) -> any(). 15 | get(Name) -> 16 | get(Name, undefined). 17 | 18 | -spec get(name(), value()) -> value(). 19 | get(Name, Default) when is_atom(Name) -> 20 | case ets:lookup(amoc_config, Name) of 21 | [] -> 22 | amoc_telemetry:execute_log( 23 | error, [config, get], #{setting => Name}, <<"no scenario setting">>), 24 | throw({invalid_setting, Name}); 25 | [#module_parameter{name = Name, value = undefined}] -> 26 | Default; 27 | [#module_parameter{name = Name, value = Value}] -> 28 | Value; 29 | InvalidLookupRet -> 30 | amoc_telemetry:execute_log( 31 | error, [config, get], #{setting => Name, return => InvalidLookupRet}, 32 | <<"invalid lookup return value">>), 33 | throw({invalid_lookup_ret_value, InvalidLookupRet}) 34 | end. 35 | -------------------------------------------------------------------------------- /src/config/amoc_config.hrl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2023 Erlang Solutions Ltd. 3 | %% Licensed under the Apache License, Version 2.0 (see LICENSE file) 4 | %%============================================================================== 5 | 6 | 7 | %%------------------------------------------------------------------------------ 8 | %% basic types 9 | %%------------------------------------------------------------------------------ 10 | -type name() :: atom(). 11 | -type value() :: any(). 12 | 13 | %%------------------------------------------------------------------------------ 14 | %% runtime supplied settings 15 | %%------------------------------------------------------------------------------ 16 | -type settings() :: [{name(), value()}]. 17 | 18 | %%------------------------------------------------------------------------------ 19 | %% error types 20 | %%------------------------------------------------------------------------------ 21 | -type reason() :: any(). 22 | -type error_type() :: atom(). 23 | -type error() :: {error, error_type(), reason()}. 24 | 25 | %%------------------------------------------------------------------------------ 26 | %% module attributes definition 27 | %%------------------------------------------------------------------------------ 28 | -type verification_fun() :: fun((Value :: value()) -> boolean() | 29 | {true, NewValue :: value()} | 30 | {false, reason()}). 31 | 32 | -type update_fun() :: fun((ParamName :: name(), NewValue :: value()) -> any()). 33 | 34 | -type maybe_verification_fun() :: verification_fun() | fun((_)-> any()). 35 | -type maybe_update_fun() :: update_fun() | fun((_,_)-> any()). 36 | 37 | -record(module_parameter, {name :: name(), 38 | mod :: module(), 39 | value :: value(), 40 | description :: string(), 41 | verification_fn :: maybe_verification_fun(), 42 | update_fn = read_only :: maybe_update_fun() | read_only}). 43 | 44 | -type module_parameter() :: #module_parameter{}. 45 | -type module_configuration() :: [module_parameter()]. 46 | -type maybe_module_config() :: {ok, [module_parameter()]} | error(). 47 | 48 | -type one_of() :: [value(), ...]. 49 | -type mfa(Arity) :: {module(), atom(), Arity}. 50 | -type verification_method() :: none | one_of() | mfa(1) | verification_fun(). 51 | -type update_method() :: read_only | none | mfa(2) | update_fun(). 52 | 53 | -type module_attribute() :: #{ name := name(), 54 | description := string(), 55 | default_value => value(), 56 | verification => verification_method(), 57 | update => update_method()}. 58 | -------------------------------------------------------------------------------- /src/config/amoc_config_attributes.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2023 Erlang Solutions Ltd. 3 | %% Licensed under the Apache License, Version 2.0 (see LICENSE file) 4 | %%============================================================================== 5 | %% this module is responsible for parsing module attributes 6 | %%============================================================================== 7 | -module(amoc_config_attributes). 8 | 9 | -include("amoc_config.hrl"). 10 | 11 | %% API 12 | -export([get_module_configuration/2]). 13 | -export([none/1, none/2]). 14 | -ifdef(TEST). 15 | -export([%% exported for testing only 16 | get_module_attributes/2, 17 | process_module_attributes/2]). 18 | -endif. 19 | 20 | -type maybe_module_attribute() :: module_attribute() | term(). 21 | -type maybe_verification_method() :: verification_method() | term(). 22 | -type maybe_update_method() :: update_method() | term(). 23 | 24 | -type attribute_name() :: required_variable | override_variable. 25 | 26 | %% ------------------------------------------------------------------ 27 | %% API 28 | %% ------------------------------------------------------------------ 29 | -spec get_module_configuration(attribute_name(), module()) -> 30 | maybe_module_config(). 31 | get_module_configuration(AttrName, Module) -> 32 | ScenarioAttributes = get_module_attributes(AttrName, Module), 33 | process_module_attributes(Module, ScenarioAttributes). 34 | 35 | -spec none(any()) -> true. 36 | none(_) -> true. 37 | 38 | -spec none(any(), any()) -> ok. 39 | none(_, _) -> ok. 40 | %% ------------------------------------------------------------------ 41 | %% Internal Function Definitions 42 | %% ------------------------------------------------------------------ 43 | -spec get_module_attributes(attribute_name(), module()) -> [maybe_module_attribute()]. 44 | get_module_attributes(AttrName, Module) -> 45 | ModuleAttributes = apply(Module, module_info, [attributes]), 46 | RequiredVariables = proplists:get_all_values(AttrName, ModuleAttributes), 47 | lists:append(RequiredVariables). 48 | 49 | -spec process_module_attributes(module(), [maybe_module_attribute()]) -> 50 | maybe_module_config(). 51 | process_module_attributes(Module, ScenarioAttributes) -> 52 | Config = [process_var_attr(Module, Attr) 53 | || Attr <- ScenarioAttributes], 54 | amoc_config_utils:maybe_error(invalid_attribute_format, Config). 55 | 56 | -spec process_var_attr(module(), maybe_module_attribute()) -> 57 | {ok, module_parameter()} | {error, reason()}. 58 | process_var_attr(Module, Attr) -> 59 | PipelineActions = [ 60 | {fun check_mandatory_fields/1, []}, 61 | {fun check_default_value/1, []}, 62 | {fun check_verification_method/1, []}, 63 | {fun check_update_method/1, []}, 64 | {fun make_module_parameter/2, [Module]}], 65 | case amoc_config_utils:pipeline(PipelineActions, {ok, Attr}) of 66 | {error, Reason} -> {error, add_original_attribute(Reason, Attr)}; 67 | {ok, Param} -> {ok, Param} 68 | end. 69 | 70 | -spec check_mandatory_fields(maybe_module_attribute()) -> 71 | {ok, #{name := name(), description := string(), any() => any()}} | {error, reason()}. 72 | check_mandatory_fields(#{description := List, name := Atom} = Attr) when is_atom(Atom), 73 | is_list(List) -> 74 | case io_lib:char_list(List) of 75 | true -> {ok, Attr}; 76 | false -> {error, invalid_attribute} 77 | end; 78 | check_mandatory_fields(_Attr) -> 79 | {error, invalid_attribute}. 80 | 81 | -spec check_default_value(#{any() => any()}) -> 82 | {ok, #{default_value := value(), any() => any()}}. 83 | check_default_value(Attr) -> 84 | DefaultValue = maps:get(default_value, Attr, undefined), 85 | {ok, Attr#{default_value => DefaultValue}}. 86 | 87 | -spec check_verification_method(#{any() => any()}) -> 88 | {ok, #{verification := maybe_verification_fun(), any() => any()}} | {error, reason()}. 89 | check_verification_method(Attr) -> 90 | VerificationMethod = maps:get(verification, Attr, none), 91 | case verification_fn(VerificationMethod) of 92 | not_exported -> 93 | {error, verification_method_not_exported}; 94 | invalid_method -> 95 | {error, invalid_verification_method}; 96 | VerificationFn -> 97 | {ok, Attr#{verification => VerificationFn}} 98 | end. 99 | 100 | -spec check_update_method(#{any() => any()}) -> 101 | {ok, #{update := maybe_update_fun(), any() => any()}} | {error, reason()}. 102 | check_update_method(Attr) -> 103 | UpdateMethod = maps:get(update, Attr, read_only), 104 | case update_fn(UpdateMethod) of 105 | not_exported -> 106 | {error, update_method_not_exported}; 107 | invalid_method -> 108 | {error, invalid_update_method}; 109 | UpdateFn -> 110 | {ok, Attr#{update => UpdateFn}} 111 | end. 112 | 113 | -spec make_module_parameter(#{name := name(), 114 | description := string(), 115 | default_value := value(), 116 | verification := maybe_verification_fun(), 117 | update := maybe_update_fun(), 118 | any() => any()}, 119 | module()) -> 120 | {ok, module_parameter()}. 121 | make_module_parameter(#{name := Name, description := Description, default_value := Value, 122 | verification := VerificationFn, update := UpdateFn}, Module) -> 123 | {ok, #module_parameter{name = Name, mod = Module, description = Description, value = Value, 124 | verification_fn = VerificationFn, update_fn = UpdateFn}}. 125 | 126 | -spec verification_fn(maybe_verification_method()) -> 127 | maybe_verification_fun() | not_exported | invalid_method. 128 | verification_fn(none) -> 129 | fun ?MODULE:none/1; 130 | verification_fn([_ | _] = OneOf) -> 131 | one_of_fun(OneOf); 132 | verification_fn(Fun) when is_function(Fun, 1) -> 133 | is_exported(Fun); 134 | verification_fn({Module, Function, 1}) -> 135 | is_exported(fun Module:Function/1); 136 | verification_fn(_) -> 137 | invalid_method. 138 | 139 | -spec update_fn(maybe_update_method()) -> 140 | maybe_update_fun() | not_exported | invalid_method | read_only. 141 | update_fn(read_only) -> 142 | read_only; 143 | update_fn(none) -> 144 | fun ?MODULE:none/2; 145 | update_fn(Fun) when is_function(Fun, 2) -> 146 | is_exported(Fun); 147 | update_fn({Module, Function, 2}) -> 148 | is_exported(fun Module:Function/2); 149 | update_fn(_) -> 150 | invalid_method. 151 | 152 | -spec is_exported(function()) -> 153 | function() | not_exported. 154 | is_exported(Fn) -> 155 | [{type, T}, {module, M}, {name, F}, {arity, A}] = 156 | [erlang:fun_info(Fn, I) || I <- [type, module, name, arity]], 157 | case {T, code:ensure_loaded(M)} of 158 | {external, {module, M}} -> 159 | case erlang:function_exported(M, F, A) of 160 | true -> Fn; 161 | false -> not_exported 162 | end; 163 | _ -> not_exported 164 | end. 165 | 166 | -spec one_of_fun(one_of()) -> verification_fun(). 167 | one_of_fun(OneOf) -> 168 | fun(X) -> 169 | case lists:member(X, OneOf) of 170 | true -> true; 171 | false -> {false, {not_one_of, OneOf}} 172 | end 173 | end. 174 | 175 | add_original_attribute(Reason, Attr) when is_tuple(Reason) -> 176 | list_to_tuple([Attr | tuple_to_list(Reason)]); 177 | add_original_attribute(Reason, Attr) -> 178 | {Attr, Reason}. 179 | -------------------------------------------------------------------------------- /src/config/amoc_config_env.erl: -------------------------------------------------------------------------------- 1 | %% @see amoc_config 2 | %% @copyright 2023 Erlang Solutions Ltd. 3 | %% @doc This module defines a behaviour to parse values as extracted from environment variables. 4 | %% 5 | %% The default implementation is `amoc_config_parser', which implements Erlang parsing. 6 | %% This way plain strings in valid Erlang syntax can be passed by env-vars 7 | %% and transformed into full Erlang terms. 8 | %% 9 | %% This module is to be used directly only for the read-only env init parameters, 10 | %% do not use it for the scenarios/helpers configuration, the amoc_config module 11 | %% must be used instead! This allows to provide configuration via REST API in a JSON format 12 | %% @end 13 | -module(amoc_config_env). 14 | 15 | -export([get/2]). 16 | 17 | -define(DEFAULT_PARSER_MODULE, amoc_config_parser). 18 | 19 | -callback(parse_value(string()) -> {ok, amoc_config:value()} | {error, any()}). 20 | 21 | %% ------------------------------------------------------------------ 22 | %% API 23 | %% ------------------------------------------------------------------ 24 | -spec get(amoc_config:name(), amoc_config:value()) -> amoc_config:value(). 25 | get(Name, Default) when is_atom(Name) -> 26 | EnvName = os_env_name(Name), 27 | Value = os:getenv(EnvName), 28 | case parse_value(Value, Default) of 29 | {ok, Term} -> Term; 30 | {error, Error} -> 31 | amoc_telemetry:execute_log( 32 | error, [config, env], 33 | #{error => Error, variable_name => EnvName, 34 | variable_value => Value, default_value => Default}, 35 | <<"cannot parse environment variable, using default value">>), 36 | Default 37 | end. 38 | 39 | %% ------------------------------------------------------------------ 40 | %% Internal Function Definitions 41 | %% ------------------------------------------------------------------ 42 | -spec os_env_name(amoc_config:name()) -> string(). 43 | os_env_name(Name) when is_atom(Name) -> 44 | "AMOC_" ++ string:uppercase(erlang:atom_to_list(Name)). 45 | 46 | -spec parse_value(string() | false, any()) -> {ok, amoc_config:value()} | {error, any()}. 47 | parse_value(false, Default) -> {ok, Default}; 48 | parse_value("", Default) -> {ok, Default}; 49 | parse_value(String, _) -> 50 | Mod = application:get_env(amoc, config_parser_mod, ?DEFAULT_PARSER_MODULE), 51 | try Mod:parse_value(String) of 52 | {ok, Value} -> {ok, Value}; 53 | {error, Error} -> {error, Error}; 54 | InvalidRetValue -> {error, {parser_returned_invalid_value, InvalidRetValue}} 55 | catch 56 | Class:Error:Stacktrace -> 57 | {error, {parser_crashed, {Class, Error, Stacktrace}}} 58 | end. 59 | -------------------------------------------------------------------------------- /src/config/amoc_config_parser.erl: -------------------------------------------------------------------------------- 1 | %% @see amoc_config 2 | %% @see amoc_config_env 3 | %% @copyright 2023 Erlang Solutions Ltd. 4 | %% @doc This module defines a behaviour to parse values as extracted from environment variables. 5 | %% This module implements the default parser for the `amoc_config_env' module 6 | %% @end 7 | -module(amoc_config_parser). 8 | -behaviour(amoc_config_env). 9 | 10 | -export([parse_value/1]). 11 | 12 | %% format/2 is exported for testing purposes 13 | %% it is also re-used by amoc-arsenal 14 | -export([format/2]). 15 | 16 | %% ------------------------------------------------------------------ 17 | %% API 18 | %% ------------------------------------------------------------------ 19 | 20 | -spec parse_value(string() | binary()) -> {ok, amoc_config:value()} | {error, any()}. 21 | parse_value(Binary) when is_binary(Binary) -> 22 | parse_value(binary_to_list(Binary)); 23 | parse_value(String) when is_list(String) -> 24 | try 25 | {ok, Tokens, _} = erl_scan:string(String ++ "."), 26 | {ok, _} = erl_parse:parse_term(Tokens) 27 | catch 28 | _:E -> {error, E} 29 | end. 30 | 31 | -spec format(any(), binary) -> binary(); 32 | (any(), string) -> string(). 33 | format(Value, binary) -> 34 | list_to_binary(format(Value, string)); 35 | format(Value, string) -> 36 | lists:flatten(io_lib:format("~tp", [Value])). 37 | -------------------------------------------------------------------------------- /src/config/amoc_config_scenario.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2023 Erlang Solutions Ltd. 3 | %% Licensed under the Apache License, Version 2.0 (see LICENSE file) 4 | %%============================================================================== 5 | -module(amoc_config_scenario). 6 | 7 | %% API 8 | -export([parse_scenario_settings/2, 9 | update_settings/1, 10 | get_default_configuration/1, 11 | get_current_configuration/0]). 12 | 13 | -include("amoc_config.hrl"). 14 | 15 | -type module_configuration_map() :: #{name() => #{value := any(), 16 | any() => any()}}. 17 | 18 | %% ------------------------------------------------------------------ 19 | %% API 20 | %% ------------------------------------------------------------------ 21 | -spec parse_scenario_settings(module(), settings()) -> ok | error(). 22 | parse_scenario_settings(Module, Settings) when is_atom(Module) -> 23 | PipelineActions = [ 24 | {fun get_configuration/1, [Module]}, 25 | {fun verify_settings/2, [Settings]}, 26 | {fun amoc_config_verification:process_scenario_config/2, [Settings]}, 27 | {fun amoc_config_utils:store_scenario_config/1, []}], 28 | case amoc_config_utils:pipeline(PipelineActions, ok) of 29 | ok -> ok; 30 | {error, _, _} = Error -> Error; 31 | UnexpectedReturnValue -> 32 | {error, invalid_return_value, UnexpectedReturnValue} 33 | end. 34 | 35 | -spec update_settings(settings()) -> ok | error(). 36 | update_settings(Settings) -> 37 | PipelineActions = [ 38 | {fun get_existing_configuration/0, []}, 39 | {fun verify_settings/2, [Settings]}, 40 | {fun filter_configuration/2, [Settings]}, 41 | {fun amoc_config_verification:process_scenario_config/2, [Settings]}, 42 | {fun store_scenario_config_and_run_update_functions/1, []}], 43 | case amoc_config_utils:pipeline(PipelineActions, ok) of 44 | ok -> ok; 45 | {error, _, _} = Error -> Error; 46 | UnexpectedReturnValue -> 47 | {error, invalid_return_value, UnexpectedReturnValue} 48 | end. 49 | 50 | -spec get_default_configuration(module()) -> {ok, module_configuration_map()} | error(). 51 | get_default_configuration(Module) -> 52 | PipelineActions = [ 53 | {fun get_configuration/1, []}, 54 | {fun convert_to_config_map/1, []}], 55 | amoc_config_utils:pipeline(PipelineActions, {ok, Module}). 56 | 57 | -spec get_current_configuration() -> {ok, module_configuration_map()}. 58 | get_current_configuration() -> 59 | PipelineActions = [ 60 | {fun get_existing_configuration/0, []}, 61 | {fun convert_to_config_map/1, []}], 62 | amoc_config_utils:pipeline(PipelineActions, ok). 63 | 64 | %% ------------------------------------------------------------------ 65 | %% Internal Function Definitions 66 | %% ------------------------------------------------------------------ 67 | -spec get_configuration(module()) -> 68 | {ok, module_configuration()} | error(). 69 | get_configuration(Module) -> 70 | ConfigurableModules = amoc_code_server:list_configurable_modules(), 71 | AllConfigurableModules = [Module | ConfigurableModules], 72 | PipelineActions = [ 73 | {fun compose_configuration/1, [AllConfigurableModules]}, 74 | {fun override_configuration/2, [Module]}], 75 | amoc_config_utils:pipeline(PipelineActions, ok). 76 | 77 | -spec compose_configuration([module()]) -> 78 | {ok, module_configuration()} | error(). 79 | compose_configuration(AllConfigurableModules) -> 80 | compose_configuration([], AllConfigurableModules). 81 | 82 | -spec override_configuration(module_configuration(), module()) -> 83 | {ok, module_configuration()} | error(). 84 | override_configuration(OldConfig, Module) -> 85 | AttrName = override_variable, 86 | case amoc_config_attributes:get_module_configuration(AttrName, Module) of 87 | {ok, NewConfig} -> 88 | amoc_config_utils:override_config(OldConfig, NewConfig); 89 | {error, _, _} = Error -> Error 90 | end. 91 | 92 | -spec compose_configuration(module_configuration(), [module()]) -> 93 | {ok, module_configuration()} | error(). 94 | compose_configuration(Config, []) -> 95 | {ok, Config}; 96 | compose_configuration(Config, [M | L]) -> 97 | AttrName = required_variable, 98 | case amoc_config_attributes:get_module_configuration(AttrName, M) of 99 | {ok, NewConfig} -> 100 | case amoc_config_utils:merge_config(Config, NewConfig) of 101 | {ok, MergedConfig} -> 102 | compose_configuration(MergedConfig, L); 103 | {error, _, _} = Error -> Error 104 | end; 105 | {error, _, _} = Error -> Error 106 | end. 107 | 108 | verify_settings(Config, Settings) -> 109 | verify_settings([], Config, Settings). 110 | 111 | verify_settings([], Config, []) -> 112 | {ok, Config}; 113 | verify_settings(UndefinedParameters, _Config, []) -> 114 | {error, undefined_parameters, UndefinedParameters}; 115 | verify_settings(UndefinedParameters, Config, [{Name, _} | T]) -> 116 | case lists:keyfind(Name, #module_parameter.name, Config) of 117 | false -> 118 | verify_settings([Name | UndefinedParameters], Config, T); 119 | Tuple when is_tuple(Tuple) -> 120 | verify_settings(UndefinedParameters, Config, T) 121 | end. 122 | 123 | get_existing_configuration() -> 124 | {ok, ets:tab2list(amoc_config)}. 125 | 126 | filter_configuration(Config, Settings) -> 127 | Keys = proplists:get_keys(Settings), 128 | KeyPos = #module_parameter.name, 129 | FilteredConfig = [lists:keyfind(Name, KeyPos, Config) || Name <- Keys], 130 | case [{N, M} || #module_parameter{name = N, mod = M, 131 | update_fn = read_only} <- FilteredConfig] of 132 | [] -> 133 | %% filter out unchanged parameters 134 | ChangedParameters = 135 | [P || #module_parameter{name = N, value = V} = P <- FilteredConfig, 136 | V =/= proplists:get_value(N, Settings)], 137 | {ok, ChangedParameters}; 138 | ReadOnlyParameters -> 139 | {error, readonly_parameters, ReadOnlyParameters} 140 | end. 141 | 142 | store_scenario_config_and_run_update_functions(Config) -> 143 | amoc_config_utils:store_scenario_config(Config), 144 | [spawn(fun() -> apply(Fn, [Name, Value]) end) 145 | || #module_parameter{name = Name, value = Value, update_fn = Fn} <- Config], 146 | ok. 147 | 148 | convert_to_config_map(Config) -> 149 | PropList = [{Name, parameter_to_map(P)} 150 | || #module_parameter{name = Name} = P <- Config], 151 | {ok, maps:from_list(PropList)}. 152 | 153 | parameter_to_map(#module_parameter{} = Param) -> 154 | RecordFields = record_info(fields, module_parameter), 155 | RecordSize = record_info(size, module_parameter), 156 | FieldsWithPosition = lists:zip(lists:seq(2, RecordSize), RecordFields), 157 | PropList = [{Field, element(Pos, Param)} || {Pos, Field} <- FieldsWithPosition, 158 | filter_parameter_fields(Field)], 159 | maps:from_list(PropList). 160 | 161 | filter_parameter_fields(name) -> false; 162 | filter_parameter_fields(_) -> true. 163 | -------------------------------------------------------------------------------- /src/config/amoc_config_utils.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_config 3 | %% @copyright 2023 Erlang Solutions Ltd. 4 | -module(amoc_config_utils). 5 | 6 | -include("amoc_config.hrl"). 7 | 8 | %% API 9 | -export([maybe_error/2, 10 | pipeline/2, 11 | merge_config/2, 12 | override_config/2, 13 | store_scenario_config/1, 14 | create_amoc_config_ets/0]). 15 | 16 | -spec maybe_error(error_type(), [{error, reason()} | {ok, any()}]) -> 17 | error() | {ok, [any()]}. 18 | maybe_error(ErrorType, List) -> 19 | case [Reason || {error, Reason} <- List] of 20 | [] -> 21 | {ok, [CorrectValue || {ok, CorrectValue} <- List]}; 22 | Errors -> 23 | {error, ErrorType, Errors} 24 | end. 25 | 26 | -spec pipeline([{function(), Args :: [any()]}], any()) -> any(). 27 | pipeline(Actions, InitValue) -> 28 | lists:foldl(fun run_action/2, InitValue, Actions). 29 | 30 | run_action({Fun, Args}, {ok, FirstArg}) -> 31 | try_apply_fn(Fun, [FirstArg | Args]); 32 | run_action({Fun, Args}, ok) -> 33 | try_apply_fn(Fun, Args); 34 | run_action(_, Error) -> Error. 35 | 36 | try_apply_fn(Fun, Args) -> 37 | try 38 | apply(Fun, Args) 39 | catch 40 | C:E:S -> {error, pipeline_action_crashed, {C, E, S}} 41 | end. 42 | 43 | -spec merge_config(module_configuration(), module_configuration()) -> 44 | maybe_module_config(). 45 | merge_config(MergedConfig, []) -> 46 | {ok, MergedConfig}; 47 | merge_config(OldConfig, [#module_parameter{name = Name, mod = Module} = Param | L]) -> 48 | case lists:keyfind(Name, #module_parameter.name, OldConfig) of 49 | false -> 50 | merge_config([Param | OldConfig], L); 51 | #module_parameter{name = Name, mod = AnotherModule} -> 52 | {error, parameter_overriding, {Name, Module, AnotherModule}} 53 | end. 54 | 55 | -spec override_config(module_configuration(), module_configuration()) -> 56 | {ok, module_configuration()}. 57 | override_config(OldConfig, NewConfig) -> 58 | ReversedNewConfig = lists:reverse(NewConfig), 59 | FullConfig = ReversedNewConfig ++ OldConfig, 60 | {ok, lists:ukeysort(#module_parameter.name, FullConfig)}. 61 | 62 | -spec store_scenario_config(module_configuration()) -> ok. 63 | store_scenario_config(Config) -> 64 | true = ets:insert(amoc_config, Config), 65 | ok. 66 | 67 | -spec create_amoc_config_ets() -> any(). 68 | create_amoc_config_ets() -> 69 | amoc_config = ets:new(amoc_config, [named_table, 70 | protected, 71 | {keypos, #module_parameter.name}, 72 | {read_concurrency, true}]). 73 | -------------------------------------------------------------------------------- /src/config/amoc_config_verification.erl: -------------------------------------------------------------------------------- 1 | %% @see amoc_config 2 | %% @copyright 2023 Erlang Solutions Ltd. 3 | %% @doc This module is responsible for processing scenario configuration. 4 | %% It applies the verification function provided in the `required_variable' parameter to the respective value 5 | %% @end 6 | -module(amoc_config_verification). 7 | 8 | %% API 9 | -export([process_scenario_config/2]). 10 | 11 | -include("amoc_config.hrl"). 12 | 13 | %% @doc Applies the processing as provided by the `required_variable' list to the provided scenario config 14 | -spec process_scenario_config(module_configuration(), settings()) -> 15 | {ok, module_configuration()} | error(). 16 | process_scenario_config(Config, Settings) -> 17 | ParametersVerification = [get_value_and_verify(Param, Settings) || Param <- Config], 18 | amoc_config_utils:maybe_error(parameters_verification_failed, ParametersVerification). 19 | 20 | -spec get_value_and_verify(module_parameter(), settings()) -> 21 | {ok, module_parameter()} | {error, reason()}. 22 | get_value_and_verify(#module_parameter{name = Name, value = Default, 23 | verification_fn = VerificationFn} = Param, 24 | Settings) -> 25 | DefaultValue = amoc_config_env:get(Name, Default), 26 | Value = proplists:get_value(Name, Settings, DefaultValue), 27 | case verify(VerificationFn, Value) of 28 | {true, NewValue} -> 29 | {ok, Param#module_parameter{value = NewValue}}; 30 | {false, Reason} -> 31 | {error, {Name, Value, Reason}} 32 | end. 33 | 34 | -spec verify(maybe_verification_fun(), value()) -> {true, value()} | {false, reason()}. 35 | verify(Fun, Value) -> 36 | try apply(Fun, [Value]) of 37 | true -> {true, Value}; 38 | false -> {false, verification_failed}; 39 | {true, NewValue} -> {true, NewValue}; 40 | {false, Reason} -> {false, {verification_failed, Reason}}; 41 | Ret -> 42 | amoc_telemetry:execute_log( 43 | error, [config, verify], 44 | #{verification_method => Fun, verification_arg => Value, verification_return => Ret}, 45 | <<"invalid verification method">>), 46 | {false, {invalid_verification_return_value, Ret}} 47 | catch 48 | C:E:S -> 49 | amoc_telemetry:execute_log( 50 | error, [config, verify], 51 | #{verification_method => Fun, verification_arg => Value, 52 | kind => C, reason => E, stacktrace => S}, 53 | <<"invalid verification method">>), 54 | {false, {exception_during_verification, {C, E, S}}} 55 | end. 56 | -------------------------------------------------------------------------------- /src/coordinator/amoc_coordinator.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @doc This module allows to synchronize the users and act on groups of them. 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %%============================================================================== 5 | -module(amoc_coordinator). 6 | 7 | %% API 8 | -export([start/3, start/2, 9 | add/2, add/3, 10 | stop/1, reset/1, 11 | notify/2]). 12 | -ifdef(TEST). 13 | -export([normalize_coordination_plan/1, order_plan/1]). 14 | -endif. 15 | 16 | -define(DEFAULT_TIMEOUT, 30). %% 30 seconds 17 | 18 | -define(IS_POS_INT(Integer), (is_integer(Integer) andalso Integer > 0)). 19 | -define(IS_N_OF_USERS(N), (?IS_POS_INT(N) orelse N =:= all)). 20 | -define(IS_TIMEOUT(Timeout), (?IS_POS_INT(Timeout) orelse Timeout =:= infinity)). 21 | 22 | -type name() :: term(). 23 | 24 | -type data() :: {pid(), Data :: any()}. 25 | 26 | -type maybe_coordination_data() :: data() | undefined. 27 | 28 | -type event() :: coordinator_timeout | reset_coordinator | {coordinate, {pid(), term()}}. 29 | -type event_type() :: coordinate | timeout | stop | reset. 30 | 31 | -type coordination_event() :: {event_type(), non_neg_integer()}. 32 | 33 | -type action() :: 34 | fun((coordination_event(), [data()]) -> any()) | 35 | fun((coordination_event(), maybe_coordination_data(), maybe_coordination_data()) -> any()) | 36 | fun((coordination_event()) -> any()). 37 | 38 | -type coordination_actions() :: [action()] | action(). 39 | 40 | -type num_of_users() :: pos_integer() | {Min :: pos_integer(), Max :: pos_integer()} | all. 41 | 42 | -type coordination_item() :: {num_of_users(), coordination_actions()}. 43 | 44 | -type normalized_coordination_item() :: {NoOfUsers :: pos_integer() | all, 45 | [action()]}. 46 | 47 | -type plan() :: [coordination_item()] | coordination_item(). 48 | 49 | %% timeout in seconds 50 | -type coordination_timeout_in_sec() :: pos_integer() | infinity. 51 | 52 | -export_type([name/0, 53 | event/0, 54 | plan/0, 55 | event_type/0, 56 | action/0, 57 | data/0, 58 | num_of_users/0, 59 | coordination_event/0, 60 | normalized_coordination_item/0]). 61 | 62 | %%%=================================================================== 63 | %%% Api 64 | %%%=================================================================== 65 | 66 | %% @see start/3 67 | -spec start(name(), plan()) -> ok | error. 68 | start(Name, CoordinationPlan) -> 69 | start(Name, CoordinationPlan, ?DEFAULT_TIMEOUT). 70 | 71 | %% @doc Starts a coordinator. Usually called in the init callback of an amoc scenario. 72 | -spec start(name(), plan(), coordination_timeout_in_sec()) -> ok | error. 73 | start(Name, CoordinationPlan, Timeout) when ?IS_TIMEOUT(Timeout) -> 74 | Plan = normalize_coordination_plan(CoordinationPlan), 75 | OrderedPlan = order_plan(Plan), 76 | case amoc_coordinator_sup:start_coordinator(Name, OrderedPlan, Timeout) of 77 | {ok, _} -> 78 | amoc_telemetry:execute([coordinator, start], #{count => 1}, #{name => Name}), 79 | ok; 80 | {error, _} -> error 81 | end. 82 | 83 | %% @doc Stops a coordinator. 84 | -spec stop(name()) -> ok. 85 | stop(Name) -> 86 | amoc_coordinator_sup:stop_coordinator(Name), 87 | amoc_telemetry:execute([coordinator, stop], #{count => 1}, #{name => Name}). 88 | 89 | %% @see add/3 90 | -spec add(name(), any()) -> ok. 91 | add(Name, Data) -> 92 | add(Name, self(), Data). 93 | 94 | %% @doc Adds the current process data. Usually called in the `start/2' callback of an amoc scenario. 95 | -spec add(name(), pid(), any()) -> ok. 96 | add(Name, Pid, Data) -> 97 | notify(Name, {coordinate, {Pid, Data}}). 98 | 99 | %% @doc Resets a coordinator, that is, calls all coordination actions with `reset' as the coordination data. 100 | -spec reset(name()) -> ok. 101 | reset(Name) -> 102 | notify(Name, reset_coordinator). 103 | 104 | -spec notify(name(), coordinator_timeout | reset_coordinator | {coordinate, {pid(), term()}}) -> ok. 105 | notify(Name, coordinator_timeout) -> 106 | do_notify(Name, coordinator_timeout); 107 | notify(Name, reset_coordinator) -> 108 | do_notify(Name, reset_coordinator); 109 | notify(Name, {coordinate, _} = Event) -> 110 | do_notify(Name, Event). 111 | 112 | do_notify(Name, Event) -> 113 | raise_telemetry_event(Name, Event), 114 | case amoc_coordinator_sup:get_workers(Name) of 115 | {ok, TimeoutWorker, Workers} -> 116 | amoc_coordinator_timeout:notify(TimeoutWorker, Event), 117 | [ notify_worker(Worker, Event) || Worker <- Workers ], 118 | ok; 119 | {error, not_found} -> 120 | ok 121 | end. 122 | 123 | %%%=================================================================== 124 | %%% Internal functions 125 | %%%=================================================================== 126 | notify_worker(WorkerPid, coordinator_timeout) -> %% synchronous notification 127 | amoc_coordinator_worker:timeout(WorkerPid); 128 | notify_worker(WorkerPid, reset_coordinator) -> %% synchronous notification 129 | amoc_coordinator_worker:reset(WorkerPid); 130 | notify_worker(WorkerPid, {coordinate, {Pid, Data}}) when is_pid(Pid) -> %% async notification 131 | amoc_coordinator_worker:add(WorkerPid, {Pid, Data}). 132 | 133 | raise_telemetry_event(Name, Event) -> 134 | TelemetryEvent = case Event of 135 | {coordinate, _} -> add; 136 | reset_coordinator -> reset; 137 | coordinator_timeout -> timeout 138 | end, 139 | amoc_telemetry:execute([coordinator, TelemetryEvent], #{count => 1}, #{name => Name}). 140 | 141 | %% This function defines the execution order of the events that must be processed synchronously (on 142 | %% reset/timeout/stop). We need to ensure that 'all' items are executed after 'non-all'. Note that 143 | %% '{coordinate, _}' events are exceptional and their processing is done asynchronously. The order 144 | %% of All plan items must be preserved as in the original plan, the same applies to NonAll items. 145 | -spec order_plan([normalized_coordination_item()]) -> [normalized_coordination_item()]. 146 | order_plan(Items) -> 147 | {All, NotAll} = lists:partition(fun partitioner/1, Items), 148 | NotAll ++ All. 149 | 150 | -spec partitioner(normalized_coordination_item()) -> boolean(). 151 | partitioner({all, _}) -> 152 | true; 153 | partitioner(_) -> 154 | false. 155 | 156 | -spec normalize_coordination_plan(plan()) -> [normalized_coordination_item()]. 157 | normalize_coordination_plan(CoordinationPlan) when is_tuple(CoordinationPlan) -> 158 | normalize_coordination_plan([CoordinationPlan]); 159 | normalize_coordination_plan(CoordinationPlan) -> 160 | [normalize_coordination_item(I) || I <- CoordinationPlan]. 161 | 162 | normalize_coordination_item({NoOfUsers, Action}) when is_function(Action) -> 163 | normalize_coordination_item({NoOfUsers, [Action]}); 164 | normalize_coordination_item({NoOfUsers, Actions}) when ?IS_N_OF_USERS(NoOfUsers), 165 | is_list(Actions) -> 166 | [assert_action(NoOfUsers, A) || A <- Actions], 167 | {NoOfUsers, Actions}; 168 | normalize_coordination_item({{Min, Max}, Actions}) when ?IS_POS_INT(Min), ?IS_POS_INT(Max), 169 | Max > Min, is_list(Actions) -> 170 | [assert_action({Min, Max}, A) || A <- Actions], 171 | {{Min, Max}, Actions}. 172 | 173 | assert_action(all, Action) when is_function(Action, 1); 174 | is_function(Action, 2) -> 175 | ok; 176 | assert_action(N, Action) when is_integer(N), 177 | (is_function(Action, 1) orelse 178 | is_function(Action, 2) orelse 179 | is_function(Action, 3)) -> 180 | ok; 181 | assert_action({Min, Max}, Action) when is_integer(Min), is_integer(Max), 182 | (is_function(Action, 1) orelse 183 | is_function(Action, 2) orelse 184 | is_function(Action, 3)) -> 185 | ok. 186 | -------------------------------------------------------------------------------- /src/coordinator/amoc_coordinator_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_coordinator 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %% @doc Global supervisor for all started coordination actions 5 | -module(amoc_coordinator_sup). 6 | 7 | -behaviour(supervisor). 8 | 9 | -export([start_coordinator/3, stop_coordinator/1, get_workers/1]). 10 | 11 | -export([start_link/0, init/1]). 12 | 13 | -spec start_coordinator(amoc_coordinator:name(), amoc_coordinator:plan(), timeout()) -> 14 | {ok, pid()} | {error, term()}. 15 | start_coordinator(Name, OrderedPlan, Timeout) -> 16 | case supervisor:start_child(?MODULE, [{Name, OrderedPlan, Timeout}]) of 17 | {ok, Coordinator} -> 18 | Children = supervisor:which_children(Coordinator), 19 | Workers = get_workers_in_order(Name, OrderedPlan, Children), 20 | TimeoutWorker = get_timeout_pid(Name, Timeout, Children), 21 | store_coordinator(Name, Coordinator, TimeoutWorker, Workers), 22 | {ok, Coordinator}; 23 | Other -> 24 | Other 25 | end. 26 | 27 | -spec get_workers_in_order(amoc_coordinator:name(), amoc_coordinator:plan(), []) -> [pid()]. 28 | get_workers_in_order(Name, OrderedPlan, Children) -> 29 | [ get_child_pid({amoc_coordinator_worker, Name, Item}, Children) || Item <- OrderedPlan ]. 30 | 31 | -spec get_timeout_pid(amoc_coordinator:name(), amoc_coordinator:plan(), []) -> pid(). 32 | get_timeout_pid(Name, Timeout, Children) -> 33 | get_child_pid({amoc_coordinator_timeout, Name, Timeout}, Children). 34 | 35 | get_child_pid(ChildId, Children) -> 36 | [Pid] = [ Pid || {Id, Pid, _, _} <- Children, Id =:= ChildId ], 37 | Pid. 38 | 39 | -spec stop_coordinator(amoc_coordinator:name()) -> 40 | ok | {error, term()}. 41 | stop_coordinator(Name) -> 42 | case persistent_term:get({?MODULE, Name}) of 43 | #{coordinator := Coordinator, timeout_worker := TimeoutWorker, workers := Workers} -> 44 | amoc_coordinator_timeout:stop(TimeoutWorker), 45 | [ amoc_coordinator_worker:stop(Worker) || Worker <- Workers ], 46 | supervisor:terminate_child(?MODULE, Coordinator); 47 | not_found -> 48 | {error, not_found} 49 | end. 50 | 51 | -spec get_workers(amoc_coordinator:name()) -> 52 | {ok, pid(), [pid()]} | {error, term()}. 53 | get_workers(Name) -> 54 | case persistent_term:get({?MODULE, Name}, not_found) of 55 | #{timeout_worker := TimeoutWorker, workers := Workers} -> 56 | {ok, TimeoutWorker, Workers}; 57 | not_found -> 58 | {error, not_found} 59 | end. 60 | 61 | store_coordinator(Name, Coordinator, TimeoutWorker, Workers) -> 62 | Item = #{coordinator => Coordinator, timeout_worker => TimeoutWorker, workers => Workers}, 63 | persistent_term:put({?MODULE, Name}, Item). 64 | 65 | -spec start_link() -> {ok, Pid :: pid()}. 66 | start_link() -> 67 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 68 | 69 | -spec init(term()) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 70 | init([]) -> 71 | AChild = #{id => amoc_coordinator_worker_sup, 72 | start => {amoc_coordinator_worker_sup, start_link, []}, 73 | restart => transient, 74 | shutdown => infinity, 75 | type => supervisor, 76 | modules => [amoc_coordinator_worker_sup]}, 77 | SupFlags = #{strategy => simple_one_for_one, intensity => 0}, 78 | {ok, {SupFlags, [AChild]}}. 79 | -------------------------------------------------------------------------------- /src/coordinator/amoc_coordinator_timeout.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_coordinator 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | -module(amoc_coordinator_timeout). 5 | 6 | -behaviour(gen_server). 7 | 8 | -export([start_link/2, notify/2, stop/1]). 9 | 10 | -export([init/1, 11 | handle_call/3, 12 | handle_cast/2, 13 | handle_info/2]). 14 | 15 | -record(timeouts, { 16 | name :: amoc_coordinator:name(), 17 | timeout :: timeout() 18 | }). 19 | -type state() :: #timeouts{}. 20 | 21 | -spec notify(pid(), amoc_coordinator:event()) -> ok. 22 | notify(Pid, Event) -> 23 | gen_server:cast(Pid, Event). 24 | 25 | -spec stop(pid()) -> ok. 26 | stop(Pid) -> 27 | gen_server:call(Pid, stop). 28 | 29 | -spec start_link(amoc_coordinator:name(), timeout()) -> 30 | {ok, pid()}. 31 | start_link(Name, Timeout) -> 32 | gen_server:start_link(?MODULE, {Name, Timeout}, []). 33 | 34 | -spec init({amoc_coordinator:name(), timeout()}) -> 35 | {ok, state()}. 36 | init({Name, Timeout}) -> 37 | case Timeout of 38 | infinity -> 39 | State = #timeouts{name = Name, timeout = infinity}, 40 | {ok, State}; 41 | Int when is_integer(Int), Int > 0 -> 42 | State = #timeouts{name = Name, timeout = timer:seconds(Timeout)}, 43 | {ok, State} 44 | end. 45 | 46 | -spec handle_call(any(), term(), state()) -> 47 | {stop, normal, ok, state()}. 48 | handle_call(stop, _From, State) -> 49 | {stop, normal, ok, State}. 50 | 51 | -spec handle_cast(term(), state()) -> {noreply, state(), timeout()}. 52 | handle_cast({coordinate, _}, #timeouts{timeout = Timeout} = State) -> 53 | {noreply, State, Timeout}; 54 | handle_cast(reset_coordinator, State) -> 55 | {noreply, State, infinity}; 56 | handle_cast(coordinator_timeout, State) -> 57 | {noreply, State, infinity}. 58 | 59 | -spec handle_info(term(), state()) -> {noreply, state(), timeout()}. 60 | handle_info(timeout, #timeouts{name = Name} = State) -> 61 | amoc_coordinator:notify(Name, coordinator_timeout), 62 | {noreply, State, infinity}. 63 | -------------------------------------------------------------------------------- /src/coordinator/amoc_coordinator_worker.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_coordinator 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | -module(amoc_coordinator_worker). 5 | 6 | -behaviour(gen_server). 7 | 8 | %% API 9 | -export([start_link/1, add/2, 10 | stop/1, reset/1, 11 | timeout/1]). 12 | 13 | %% gen_server callbacks 14 | -export([init/1, 15 | handle_call/3, 16 | handle_cast/2]). 17 | 18 | -type event_type() :: amoc_coordinator:event_type(). 19 | -type event() :: amoc_coordinator:coordination_event(). 20 | -type action() :: amoc_coordinator:action(). 21 | -type data() :: amoc_coordinator:data(). 22 | 23 | -record(state, {configured = all :: amoc_coordinator:num_of_users(), 24 | required_n = all :: pos_integer() | all, 25 | n = 0 :: non_neg_integer(), 26 | actions = [] :: [action()], 27 | collect_data = true :: boolean(), 28 | acc = [] :: [data()]}). 29 | 30 | -type state() :: #state{}. 31 | 32 | %%%=================================================================== 33 | %%% API 34 | %%%=================================================================== 35 | 36 | -spec start_link(amoc_coordinator:normalized_coordination_item()) -> {ok, Pid :: pid()}. 37 | start_link(CoordinationItem) -> 38 | gen_server:start_link(?MODULE, CoordinationItem, []). 39 | 40 | -spec reset(pid()) -> ok. 41 | reset(Pid) -> 42 | gen_server:call(Pid, {reset, reset}). 43 | 44 | -spec timeout(pid()) -> ok. 45 | timeout(Pid) -> 46 | gen_server:call(Pid, {reset, timeout}). 47 | 48 | -spec stop(pid()) -> ok. 49 | stop(Pid) -> 50 | gen_server:call(Pid, {reset, stop}). 51 | 52 | -spec add(pid(), data()) -> ok. 53 | add(Pid, Data) -> 54 | gen_server:cast(Pid, {add, Data}). 55 | 56 | %%%=================================================================== 57 | %%% gen_server callbacks 58 | %%%=================================================================== 59 | 60 | -spec init(amoc_coordinator:normalized_coordination_item()) -> {ok, state()}. 61 | init({NoOfUsers, Actions}) -> 62 | State = #state{configured = NoOfUsers, 63 | required_n = calculate_n(NoOfUsers), 64 | actions = Actions, 65 | collect_data = is_acc_required(Actions)}, 66 | {ok, State}. 67 | 68 | -spec handle_call({reset, reset | timeout | stop}, term(), state()) -> 69 | {reply, ok, state()} | {stop, normal, ok, state()}. 70 | handle_call({reset, stop}, _, State) -> 71 | {stop, normal, ok, reset_state(stop, State)}; 72 | handle_call({reset, Event}, _, State) -> 73 | {reply, ok, reset_state(Event, State)}. 74 | 75 | -spec handle_cast({add, data()}, state()) -> {noreply, state()}. 76 | handle_cast({add, Data}, State) -> 77 | NewState = add_data(Data, State), 78 | {noreply, NewState}. 79 | 80 | %%%=================================================================== 81 | %%% Internal functions 82 | %%%=================================================================== 83 | -spec is_acc_required([action()]) -> boolean(). 84 | is_acc_required(Actions) -> 85 | lists:any(fun(F) when is_function(F, 1) -> false; 86 | (_) -> true 87 | end, Actions). 88 | 89 | -spec add_data(data(), state()) -> state(). 90 | add_data(Data, #state{n = N, acc = Acc} = State) -> 91 | NewState = case State#state.collect_data of 92 | false -> 93 | State#state{n = N + 1}; 94 | true -> 95 | State#state{n = N + 1, acc = [Data | Acc]} 96 | end, 97 | maybe_reset_state(NewState). 98 | 99 | -spec maybe_reset_state(state()) -> state(). 100 | maybe_reset_state(#state{n = N, required_n = N} = State) -> 101 | reset_state(coordinate, State); 102 | maybe_reset_state(State) -> 103 | State. 104 | 105 | -spec reset_state(event_type(), state()) -> state(). 106 | reset_state(Event, #state{configured = Config, 107 | actions = Actions, 108 | acc = Acc, 109 | n = N, required_n = ReqN} = State) -> 110 | amoc_telemetry:execute([coordinator, execute], #{count => N}, 111 | #{event => Event, configured => ReqN}), 112 | [execute_action(Action, {Event, N}, Acc) || Action <- Actions], 113 | NewN = calculate_n(Config), 114 | State#state{required_n = NewN, n = 0, acc = []}. 115 | 116 | -spec execute_action(action(), event(), [data()]) -> any(). 117 | execute_action(Action, Event, _) when is_function(Action, 1) -> 118 | safe_executions(Action, [Event]); 119 | execute_action(Action, Event, Acc) when is_function(Action, 2) -> 120 | safe_executions(Action, [Event, Acc]); 121 | execute_action(Action, Event, Acc) when is_function(Action, 3) -> 122 | Fun = fun(A, B) -> safe_executions(Action, [Event, A, B]) end, 123 | distinct_pairs(Fun, Acc). 124 | 125 | -spec safe_executions(function(), [any()]) -> any(). 126 | safe_executions(Fun, Args) -> 127 | try 128 | erlang:apply(Fun, Args) 129 | catch 130 | _:_ -> ok 131 | end. 132 | 133 | -spec calculate_n(amoc_coordinator:num_of_users()) -> all | pos_integer(). 134 | calculate_n({Min, Max}) -> 135 | Min - 1 + rand:uniform(Max - Min); 136 | calculate_n(Value) -> 137 | Value. 138 | 139 | -spec distinct_pairs(fun((data(), data()) -> any()), [data()]) -> any(). 140 | distinct_pairs(Fun, []) -> 141 | Fun(undefined, undefined); 142 | distinct_pairs(Fun, [OneElement]) -> 143 | Fun(OneElement, undefined); 144 | distinct_pairs(Fun, [Element1, Element2]) -> 145 | Fun(Element1, Element2); 146 | distinct_pairs(Fun, [Element1 | Tail]) -> %% length(Tail) >= 2 147 | [Fun(Element1, Element2) || Element2 <- Tail], 148 | distinct_pairs(Fun, Tail). 149 | -------------------------------------------------------------------------------- /src/coordinator/amoc_coordinator_worker_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_coordinator 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %% @doc Supervisor for a pool of handlers for a new supervision tree 5 | -module(amoc_coordinator_worker_sup). 6 | 7 | -behaviour(supervisor). 8 | 9 | -export([start_link/1, init/1]). 10 | 11 | -spec start_link({amoc_coordinator:name(), amoc_coordinator:plan(), timeout()}) -> 12 | supervisor:startlink_ret(). 13 | start_link({Name, OrderedPlan, Timeout}) -> 14 | supervisor:start_link(?MODULE, {Name, OrderedPlan, Timeout}). 15 | 16 | -spec init({amoc_coordinator:name(), amoc_coordinator:plan(), timeout()}) -> 17 | {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 18 | init({Name, OrderedPlan, Timeout}) -> 19 | OrderedChilds = [ worker_spec(Name, Item) || Item <- OrderedPlan ], 20 | TimeoutChild = timeout_child(Name, Timeout), 21 | Childs = [TimeoutChild | OrderedChilds], 22 | SupFlags = #{strategy => one_for_one, intensity => 0}, 23 | {ok, {SupFlags, Childs}}. 24 | 25 | %% Helpers 26 | timeout_child(Name, Timeout) -> 27 | #{id => {amoc_coordinator_timeout, Name, Timeout}, 28 | start => {amoc_coordinator_timeout, start_link, [Name, Timeout]}, 29 | restart => transient, 30 | shutdown => timer:seconds(5), 31 | type => worker, 32 | modules => [amoc_coordinator_timeout]}. 33 | 34 | worker_spec(Name, Item) -> 35 | #{id => {amoc_coordinator_worker, Name, Item}, 36 | start => {amoc_coordinator_worker, start_link, [Item]}, 37 | restart => transient, 38 | shutdown => timer:seconds(5), 39 | type => worker, 40 | modules => [amoc_coordinator_worker]}. 41 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle.erl: -------------------------------------------------------------------------------- 1 | %% @copyright 2024 Erlang Solutions Ltd. 2 | %% @doc Allows limiting the number of users' actions per interval. 3 | -module(amoc_throttle). 4 | 5 | %% API 6 | -export([start/2, stop/1, 7 | send/2, send/3, wait/1, 8 | run/2, pause/1, resume/1, unlock/1, 9 | change_rate/2, change_rate_gradually/2]). 10 | 11 | -type name() :: atom(). 12 | %% Atom representing the name of the throttle. 13 | 14 | -type rate() :: infinity | non_neg_integer(). 15 | %% Number of events per given `t:interval/0'. 16 | %% 17 | %% It can also be: 18 | %% 22 | 23 | -type interarrival() :: infinity | non_neg_integer(). 24 | %% Time in milliseconds between two events. 25 | %% 26 | %% It can also be: 27 | %% 31 | 32 | -type interval() :: non_neg_integer(). 33 | %% In milliseconds, defaults to 60000 (one minute). 34 | %% 35 | %% Note that an interval of zero means effectively allowing `t:rate/0' number of executions in 36 | %% parallel. It might be expected for this to be always `infinity' as a result of the limit when 37 | %% dividing by zero, but this needs to be made explicit in the `t:rate/0' by setting it to infinity. 38 | 39 | -type t() :: #{rate := rate(), interval => interval()} | 40 | #{interarrival := interarrival()}. 41 | %% Throttle unit of measurement 42 | 43 | -type gradual() :: #{from_rate := non_neg_integer(), 44 | to_rate := non_neg_integer(), 45 | interval => interval()} | 46 | #{from_interarrival := non_neg_integer(), 47 | to_interarrival := non_neg_integer()}. 48 | %% Configuration throttle for a gradual rate change. 49 | %% 50 | %% "from" and "to" prefixed parameters, whether rates or interarrivals, are required. 51 | %% `interval' applies only to rate and defaults to 1s. 52 | 53 | -type plan() :: #{step_interval := pos_integer(), 54 | step_count := pos_integer()} | 55 | #{duration := pos_integer()}. 56 | %% Configuration plan for a gradual rate change. 57 | %% 58 | %% The throttle mechanism will take a series of discrete steps, 59 | %% for as long as the duration given, 60 | %% or in the shape of the `step_interval' and `step_count'. 61 | 62 | -type gradual_plan() :: #{throttle := gradual(), 63 | plan := plan()}. 64 | %% Gradual plan details. Must specify a `t:gradual/0', and a `t:plan/0'. 65 | 66 | -export_type([t/0, name/0, rate/0, interval/0, gradual_plan/0]). 67 | 68 | %% @doc Starts the throttle mechanism for a given `Name' with a given config. 69 | %% 70 | %% `Name' is needed to identify the rate as a single test can have different rates for different tasks. 71 | -spec start(name(), t() | rate()) -> {ok, started | already_started} | {error, any()}. 72 | start(Name, #{} = Config) -> 73 | case amoc_throttle_config:verify_config(Config) of 74 | {error, Error} -> 75 | {error, Error}; 76 | VerifiedConfig -> 77 | amoc_throttle_controller:ensure_throttle_processes_started(Name, VerifiedConfig) 78 | end; 79 | start(Name, Rate) when is_integer(Rate) -> 80 | start(Name, #{rate => Rate}). 81 | 82 | %% @doc Pauses executions for the given `Name' as if `Rate' was set to `0'. 83 | %% 84 | %% Does not stop the scheduled rate changes. `resume/1' undoes the pausing. 85 | -spec pause(name()) -> ok | {error, any()}. 86 | pause(Name) -> 87 | amoc_throttle_controller:pause(Name). 88 | 89 | %% @doc Resumes the executions for the given `Name', to their original configuration value. 90 | %% 91 | %% It is the counterpart to the `pause/1' API, resuming the execution of what that mechanism paused. 92 | -spec resume(name()) -> ok | {error, any()}. 93 | resume(Name) -> 94 | amoc_throttle_controller:resume(Name). 95 | 96 | %% @doc Unlocks executions for the given `Name' by setting `Rate' to `infinity'. 97 | -spec unlock(name()) -> ok | {error, any()}. 98 | unlock(Name) -> 99 | change_rate(Name, #{rate => infinity, interval => 0}). 100 | 101 | %% @doc Sets the throttle `Config' for `Name' according to the given values. 102 | -spec change_rate(name(), t() | rate()) -> ok | {error, any()}. 103 | change_rate(Name, #{} = Config) -> 104 | case amoc_throttle_config:verify_config(Config) of 105 | {error, Error} -> 106 | {error, Error}; 107 | VerifiedConfig -> 108 | amoc_throttle_controller:change_rate(Name, VerifiedConfig) 109 | end; 110 | change_rate(Name, Rate) when is_integer(Rate) -> 111 | change_rate(Name, #{rate => Rate}). 112 | 113 | %% @doc Allows to set a plan of gradual rate changes for a given `Name'. 114 | %% 115 | %% The configuration will be changed in a series of consecutive steps. 116 | %% Rates can be changed upwards as well as downwards. 117 | %% See the documentation for `t:gradual_plan/0' for more info. 118 | %% 119 | %% Be aware that, at first, the rate will be changed to the initial point given 120 | %% in the configuration, and this is not considered a step. 121 | -spec change_rate_gradually(name(), gradual_plan()) -> 122 | ok | {error, any()}. 123 | change_rate_gradually(Name, Config) -> 124 | case amoc_throttle_config:verify_gradual_config(Config) of 125 | {error, _} = Error -> 126 | Error; 127 | VerifiedConfig -> 128 | amoc_throttle_controller:change_rate_gradually(Name, VerifiedConfig) 129 | end. 130 | 131 | %% @doc Executes a given function `Fn' when it does not exceed the rate for `Name'. 132 | %% 133 | %% `Fn' is executed in the context of a new process spawned on the same node on which 134 | %% the process executing `run/2' runs, so a call to `run/2' is non-blocking. 135 | %% 136 | %% Diagram 137 | %% showing function execution flow in distributed environment. 138 | %% ``` 139 | %% title Amoc distributed 140 | %% participantgroup **Slave node** 141 | %% participant User 142 | %% participant Async runner 143 | %% end 144 | %% participantgroup **Master node** 145 | %% participant Throttle process 146 | %% end 147 | %% box left of User: request telemetry event 148 | %% 149 | %% User -> *Async runner : Fun 150 | %% 151 | %% User -> Throttle process : {schedule, Async runner PID} 152 | %% box right of Throttle process : request telemetry event 153 | %% 154 | %% ==throtlling delay== 155 | %% 156 | %% Throttle process -> Async runner: scheduled 157 | %% box right of Throttle process : execution telemetry event 158 | %% space -5 159 | %% box left of Async runner : execution telemetry event 160 | %% abox over Async runner : scheduled action 161 | %% activate Async runner 162 | %% space 163 | %% deactivate Async runner 164 | %% Async runner ->Throttle process:'DOWN' 165 | %% destroy Async runner 166 | %% ''' 167 | -spec run(name(), fun(() -> any())) -> ok | {error, any()}. 168 | run(Name, Fn) -> 169 | amoc_throttle_runner:throttle(Name, Fn). 170 | 171 | %% @see send/3 172 | %% @doc Sends a given message to `erlang:self()' 173 | -spec send(name(), any()) -> ok | {error, any()}. 174 | send(Name, Msg) -> 175 | amoc_throttle_runner:throttle(Name, {self(), Msg}). 176 | 177 | %% @doc Sends a given message `Msg' to a given `Pid' when the rate for `Name' allows for that. 178 | %% 179 | %% May be used to schedule tasks. 180 | -spec send(name(), pid(), any()) -> ok | {error, any()}. 181 | send(Name, Pid, Msg) -> 182 | amoc_throttle_runner:throttle(Name, {Pid, Msg}). 183 | 184 | %% @doc Blocks the caller until the throttle mechanism allows. 185 | -spec wait(name()) -> ok | {error, any()}. 186 | wait(Name) -> 187 | amoc_throttle_runner:throttle(Name, wait). 188 | 189 | %% @doc Stops the throttle mechanism for the given `Name'. 190 | -spec stop(name()) -> ok | {error, any()}. 191 | stop(Name) -> 192 | amoc_throttle_controller:stop(Name). 193 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_config.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | -module(amoc_throttle_config). 5 | 6 | -include_lib("kernel/include/logger.hrl"). 7 | 8 | -define(TIMEOUT(N), (infinity =:= N orelse is_integer(N) andalso N >= 0)). 9 | -define(NON_NEG_INT(N), (is_integer(N) andalso N >= 0)). 10 | -define(POS_INT(N), (is_integer(N) andalso N > 0)). 11 | -define(DEFAULT_INTERVAL, 60000). %% one minute 12 | -define(DEFAULT_STEP_INTERVAL, 100). %% every 100ms 13 | 14 | -export([verify_config/1, verify_gradual_config/1, pool_config/2, process_pool_config/2]). 15 | -export([no_of_processes/0]). 16 | -export_type([config/0, gradual_plan/0, pool_config/0]). 17 | 18 | -type process_number() :: non_neg_integer(). 19 | -type config() :: #{rate := amoc_throttle:rate(), 20 | interval := amoc_throttle:interval()}. 21 | -type gradual_plan() :: #{rates := [non_neg_integer()], 22 | interval := amoc_throttle:interval(), 23 | step_interval := non_neg_integer()}. 24 | -type pool_config() :: #{process_number() := 25 | #{max_n := infinity | non_neg_integer(), 26 | delay := non_neg_integer(), 27 | status := active | inactive, 28 | pid := undefined | pid()}}. 29 | 30 | -spec verify_config(amoc_throttle:t()) -> config() | {error, any()}. 31 | verify_config(#{interarrival := infinity} = Config) 32 | when 1 =:= map_size(Config) -> 33 | #{rate => 0, interval => ?DEFAULT_INTERVAL}; 34 | verify_config(#{interarrival := 0} = Config) 35 | when 1 =:= map_size(Config) -> 36 | #{rate => infinity, interval => ?DEFAULT_INTERVAL}; 37 | verify_config(#{interarrival := Interarrival} = Config) 38 | when 1 =:= map_size(Config), ?POS_INT(Interarrival) -> 39 | #{rate => ?DEFAULT_INTERVAL div Interarrival, interval => ?DEFAULT_INTERVAL}; 40 | verify_config(#{rate := Rate, interval := Interval} = Config) 41 | when 2 =:= map_size(Config), ?TIMEOUT(Rate), ?NON_NEG_INT(Interval) -> 42 | Config; 43 | verify_config(#{rate := Rate} = Config) 44 | when 1 =:= map_size(Config), ?TIMEOUT(Rate) -> 45 | Config#{interval => ?DEFAULT_INTERVAL}; 46 | verify_config(_Config) -> 47 | {error, invalid_throttle}. 48 | 49 | -spec verify_gradual_config(amoc_throttle:gradual_plan()) -> gradual_plan() | {error, any()}. 50 | verify_gradual_config(Config) -> 51 | try do_verify_gradual_config(Config) of 52 | Change -> Change 53 | catch error:Reason:Stacktrace -> 54 | ?LOG_WARNING(#{what => bad_gradual_config, 55 | reason => Reason, stacktrace => Stacktrace}), 56 | {error, Reason} 57 | end. 58 | 59 | -spec pool_config(amoc_throttle:rate(), amoc_throttle:interval()) -> pool_config(). 60 | pool_config(infinity, _) -> 61 | Config = #{max_n => infinity, delay => 0, status => active, pid => undefined}, 62 | maps:from_keys(lists:seq(1, ?MODULE:no_of_processes()), Config); 63 | pool_config(0, _) -> 64 | Config = #{max_n => 0, delay => infinity, status => active, pid => undefined}, 65 | maps:from_keys(lists:seq(1, ?MODULE:no_of_processes()), Config); 66 | pool_config(Rate, 0) -> 67 | Config = #{max_n => Rate, delay => 0, status => inactive, pid => undefined}, 68 | PoolConfig = #{1 := First} = maps:from_keys(lists:seq(1, ?MODULE:no_of_processes()), Config), 69 | PoolConfig#{1 := First#{status => active}}; 70 | pool_config(Rate, Interval) when ?POS_INT(Rate), ?POS_INT(Interval) -> 71 | NoOfProcesses = ?MODULE:no_of_processes(), 72 | RatesPerProcess = calculate_rate_per_process(NoOfProcesses, Rate, Interval, +0.0, []), 73 | #{} = lists:foldl(fun assign_process/2, #{}, RatesPerProcess). 74 | 75 | -define(THRESHOLD, 10). 76 | calculate_rate_per_process(1, Rate, Interval, RoundingError, Acc) -> 77 | case delay(RoundingError, Rate, Interval) of 78 | {Delay, Remaining} when Delay =:= infinity; Remaining < 0.5 -> 79 | [{1, Rate, Delay} | Acc]; 80 | {Delay, _} -> 81 | [{1, Rate, Delay + 1} | Acc] 82 | end; 83 | calculate_rate_per_process(N, Rate, Interval, RoundingError, Acc) when is_integer(N), N > 1 -> 84 | ProcessRate = Rate div N, 85 | case ProcessRate of 86 | _ when ProcessRate =< ?THRESHOLD, Rate =< ?THRESHOLD -> 87 | {Delay, RoundingError1} = delay(RoundingError, Rate, Interval), 88 | Acc1 = [{N, Rate, Delay} | Acc], 89 | calculate_rate_per_process(N - 1, 0, Interval, RoundingError1, Acc1); 90 | _ when ProcessRate =< ?THRESHOLD -> 91 | {Delay, RoundingError1} = delay(RoundingError, ?THRESHOLD, Interval), 92 | Acc1 = [{N, ?THRESHOLD, Delay} | Acc], 93 | calculate_rate_per_process(N - 1, Rate - ?THRESHOLD, Interval, RoundingError1, Acc1); 94 | _ -> 95 | {Delay, RoundingError1} = delay(RoundingError, ProcessRate, Interval), 96 | Acc1 = [{N, ProcessRate, Delay} | Acc], 97 | calculate_rate_per_process(N - 1, Rate - ProcessRate, Interval, RoundingError1, Acc1) 98 | end. 99 | 100 | delay(RemainingError, 0, _Interval) -> 101 | {infinity, RemainingError}; 102 | delay(RemainingError, Rate, Interval) -> 103 | Remaining = Interval rem Rate, 104 | RemainingError1 = RemainingError + (Remaining / Rate), 105 | case {Interval div Rate, RemainingError1} of 106 | {DelayBetweenExecutions, _} when RemainingError1 >= 1.0 -> 107 | {DelayBetweenExecutions + 1, RemainingError1 - 1}; 108 | {DelayBetweenExecutions, _} -> 109 | {DelayBetweenExecutions, RemainingError1} 110 | end. 111 | 112 | assign_process({N, RatePerProcess, infinity}, Config) -> 113 | Config#{N => #{max_n => RatePerProcess, 114 | delay => infinity, 115 | status => inactive, 116 | pid => undefined}}; 117 | assign_process({N, RatePerProcess, Delay}, Config) -> 118 | Config#{N => #{max_n => RatePerProcess, 119 | delay => Delay, 120 | status => active, 121 | pid => undefined}}. 122 | 123 | -spec process_pool_config(pid(), pool_config()) -> pool_config(). 124 | process_pool_config(PoolSup, PoolConfig) -> 125 | Workers = amoc_throttle_pool:get_workers(PoolSup), 126 | Fun1 = fun(N, Config) -> Config#{pid => maps:get(N, Workers)} end, 127 | maps:map(Fun1, PoolConfig). 128 | 129 | -spec no_of_processes() -> non_neg_integer(). 130 | no_of_processes() -> 131 | min(30, 2 * erlang:system_info(schedulers_online)). 132 | 133 | -spec do_verify_gradual_config(amoc_throttle:gradual_plan()) -> gradual_plan(). 134 | do_verify_gradual_config( 135 | #{throttle := #{from_rate := FromRate, to_rate := ToRate, interval := Interval} = Throttle, 136 | plan := #{step_interval := StepInterval, step_count := StepCount} = Plan}) 137 | when 3 =:= map_size(Throttle), 2 =:= map_size(Plan), 138 | ?NON_NEG_INT(FromRate), ?NON_NEG_INT(ToRate), ?NON_NEG_INT(Interval), 139 | ?POS_INT(StepInterval), ?POS_INT(StepCount) -> 140 | StepRate = (ToRate - FromRate) / StepCount, 141 | StepPlan = [ calculate_step(Step, StepCount, StepRate, FromRate, ToRate) 142 | || Step <- lists:seq(0, StepCount) ], 143 | #{rates => StepPlan, interval => Interval, step_interval => StepInterval}; 144 | 145 | do_verify_gradual_config( 146 | #{throttle := #{from_rate := _, to_rate := _} = Throttle} = Config0) 147 | when 2 =:= map_size(Throttle) -> 148 | Config1 = Config0#{throttle := Throttle#{interval => ?DEFAULT_INTERVAL}}, 149 | do_verify_gradual_config(Config1); 150 | 151 | do_verify_gradual_config( 152 | #{throttle := #{from_interarrival := FromInterarrival, 153 | to_interarrival := ToInterarrival} = Throttle} = Config0) 154 | when ?NON_NEG_INT(FromInterarrival), ?NON_NEG_INT(ToInterarrival), 2 =:= map_size(Throttle) -> 155 | FromRate = ?DEFAULT_INTERVAL div FromInterarrival, 156 | ToRate = ?DEFAULT_INTERVAL div ToInterarrival, 157 | Config1 = Config0#{throttle := #{from_rate => FromRate, to_rate => ToRate}}, 158 | do_verify_gradual_config(Config1); 159 | 160 | do_verify_gradual_config( 161 | #{throttle := #{from_rate := FromRate, to_rate := ToRate, interval := Interval} = Throttle, 162 | plan := #{duration := Duration} = Plan} = Config0) 163 | when 3 =:= map_size(Throttle), 1 =:= map_size(Plan), 164 | ?NON_NEG_INT(FromRate), ?NON_NEG_INT(ToRate), ?NON_NEG_INT(Interval), ?POS_INT(Duration) -> 165 | StepCount = abs(Duration div ?DEFAULT_STEP_INTERVAL), 166 | Config1 = Config0#{plan := #{step_interval => ?DEFAULT_STEP_INTERVAL, step_count => StepCount}}, 167 | do_verify_gradual_config(Config1). 168 | 169 | -spec calculate_step( 170 | Step :: non_neg_integer(), 171 | StepCount :: non_neg_integer(), 172 | StepRate :: float(), 173 | FromRate :: non_neg_integer(), 174 | ToRate :: non_neg_integer()) -> 175 | non_neg_integer(). 176 | calculate_step(N, N, _, _, To) -> To; 177 | calculate_step(0, _, _, From, _) -> From; 178 | calculate_step(N, _, StepRate, From, _) -> 179 | From + round(StepRate * N). 180 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_pool.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | -module(amoc_throttle_pool). 5 | 6 | -behaviour(supervisor). 7 | 8 | -export([get_workers/1]). 9 | -export([start_link/2, init/1]). 10 | 11 | -spec get_workers(pid()) -> #{non_neg_integer() := pid()}. 12 | get_workers(PoolSup) -> 13 | Processes = supervisor:which_children(PoolSup), 14 | Workers = [ {N, Pid} || {{amoc_throttle_process, N}, Pid, _, _} <- Processes, is_pid(Pid) ], 15 | maps:from_list(Workers). 16 | 17 | -spec start_link(amoc_throttle:name(), amoc_throttle_config:pool_config()) -> 18 | supervisor:startlink_ret(). 19 | start_link(Name, PoolConfig) -> 20 | supervisor:start_link(?MODULE, {Name, PoolConfig}). 21 | 22 | -spec init({amoc_throttle:name(), amoc_throttle_config:pool_config()}) -> 23 | {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 24 | init({Name, ConfigPerProcess}) -> 25 | Children = [ 26 | #{id => {amoc_throttle_process, N}, 27 | start => {amoc_throttle_process, start_link, [Name, MaxN, Delay]}, 28 | type => worker, 29 | shutdown => timer:seconds(60), 30 | restart => transient, 31 | modules => [amoc_throttle_process] 32 | } 33 | || {N, #{max_n := MaxN, delay := Delay}} <- maps:to_list(ConfigPerProcess) 34 | ], 35 | SupFlags = #{strategy => one_for_one, intensity => 0}, 36 | {ok, {SupFlags, Children}}. 37 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_pooler.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | -module(amoc_throttle_pooler). 5 | 6 | -behaviour(supervisor). 7 | 8 | -export([start_pool/2, stop_pool/1]). 9 | -export([start_link/0, init/1]). 10 | 11 | -spec start_pool(amoc_throttle:name(), amoc_throttle_config:pool_config()) -> 12 | supervisor:startchild_ret(). 13 | start_pool(Name, PoolConfig) -> 14 | supervisor:start_child(?MODULE, [Name, PoolConfig]). 15 | 16 | -spec stop_pool(pid()) -> ok. 17 | stop_pool(Pool) -> 18 | ok = supervisor:terminate_child(?MODULE, Pool). 19 | 20 | -spec start_link() -> supervisor:startlink_ret(). 21 | start_link() -> 22 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 23 | 24 | -spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 25 | init([]) -> 26 | ChildSpec = #{id => amoc_throttle_pool, 27 | start => {amoc_throttle_pool, start_link, []}, 28 | type => supervisor, 29 | shutdown => infinity, 30 | restart => transient, 31 | modules => [amoc_throttle_pool] }, 32 | SupFlags = #{strategy => simple_one_for_one, intensity => 0}, 33 | {ok, {SupFlags, [ChildSpec]}}. 34 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_process.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %% @doc This process's only responsibility is to notify runners that 5 | %% they can run exactly when allowed by the throttling mechanism. 6 | -module(amoc_throttle_process). 7 | -behaviour(gen_server). 8 | 9 | %% API 10 | -export([run/2, update/3]). 11 | 12 | %% gen_server behaviour 13 | -export([start_link/3, 14 | init/1, 15 | handle_call/3, 16 | handle_info/2, 17 | handle_cast/2, 18 | handle_continue/2, 19 | terminate/2, 20 | format_status/1]). 21 | 22 | -define(DEFAULT_MSG_TIMEOUT, 60000). %% one minute 23 | 24 | -record(state, {name :: atom(), 25 | delay_between_executions :: timeout(), %% ms 26 | max_n :: infinity | non_neg_integer(), 27 | n = 0 :: non_neg_integer(), 28 | can_run_fn = true :: boolean(), 29 | tref :: timer:tref() | undefined, 30 | schedule = [] :: [AmocThrottleRunnerProcess :: pid()], 31 | schedule_reversed = [] :: [AmocThrottleRunnerProcess :: pid()] 32 | }). 33 | 34 | -type state() :: #state{}. 35 | %%------------------------------------------------------------------------------ 36 | %% Exported functions 37 | %%------------------------------------------------------------------------------ 38 | 39 | -spec start_link(atom(), amoc_throttle:rate(), timeout()) -> gen_server:start_ret(). 40 | start_link(Name, MaxN, Delay) -> 41 | gen_server:start_link(?MODULE, {Name, MaxN, Delay}, []). 42 | 43 | -spec run(pid(), pid()) -> ok. 44 | run(Pid, RunnerPid) -> 45 | gen_server:cast(Pid, {schedule, RunnerPid}). 46 | 47 | %% @doc See `initial_state/1'. 48 | %% 49 | %% Setting the delay to infinity results in the effective pausing of the process. 50 | -spec update(pid(), amoc_throttle:rate(), timeout()) -> ok. 51 | update(Pid, MaxN, Delay) -> 52 | gen_server:cast(Pid, {update, MaxN, Delay}). 53 | 54 | %%------------------------------------------------------------------------------ 55 | %% gen_server behaviour 56 | %%------------------------------------------------------------------------------ 57 | 58 | %% We're assuming that this throttle process is getting configured sensible values 59 | %% for Delay and MaxN, and it is the responsibility of the controller 60 | %% to give all workers values that aggregate to the desired throttling. 61 | -spec init({amoc_throttle:name(), amoc_throttle:rate(), timeout()}) -> 62 | {ok, state(), timeout()}. 63 | init({Name, MaxN, Delay}) -> 64 | InitialState = initial_state(Name, MaxN, Delay), 65 | StateWithTimer = maybe_start_timer(InitialState), 66 | {ok, StateWithTimer, timeout(InitialState)}. 67 | 68 | -spec handle_info(Req, state()) -> 69 | {noreply, state(), {continue, maybe_run_fn}} 70 | when Req :: {'DOWN', reference(), process, pid(), term()} 71 | | delay_between_executions 72 | | timeout. 73 | handle_info({'DOWN', _, process, _, _}, State) -> 74 | {noreply, dec_n(State), {continue, maybe_run_fn}}; 75 | handle_info(delay_between_executions, State) -> 76 | {noreply, State#state{can_run_fn = true}, {continue, maybe_run_fn}}; 77 | handle_info(timeout, State) -> 78 | internal_event(<<"is inactive">>, State), 79 | {noreply, State, {continue, maybe_run_fn}}. 80 | 81 | -spec handle_cast(Req, state()) -> 82 | {noreply, state(), {continue, maybe_run_fn}} 83 | when Req :: {update, amoc_throttle:rate(), timeout()} 84 | | {schedule, pid()}. 85 | handle_cast({schedule, RunnerPid}, #state{schedule_reversed = SchRev, name = Name} = State) -> 86 | amoc_throttle_controller:telemetry_event(Name, request), 87 | {noreply, State#state{schedule_reversed = [RunnerPid | SchRev]}, {continue, maybe_run_fn}}; 88 | handle_cast({update, MaxN, Delay}, #state{name = Name} = State) -> 89 | NewState = merge_state(initial_state(Name, MaxN, Delay), State), 90 | internal_event(<<"state update">>, NewState), 91 | {noreply, NewState, {continue, maybe_run_fn}}. 92 | 93 | -spec handle_call(any(), gen_server:from(), state()) -> 94 | {reply, {error, not_implemented}, state(), {continue, maybe_run_fn}}. 95 | handle_call(_, _, State) -> 96 | {reply, {error, not_implemented}, State, {continue, maybe_run_fn}}. 97 | 98 | -spec handle_continue(maybe_run_fn, state()) -> {noreply, state(), timeout()}. 99 | handle_continue(maybe_run_fn, State) -> 100 | NewState = maybe_run_fn(State), 101 | {noreply, NewState, timeout(NewState)}. 102 | 103 | -spec terminate(term(), state()) -> ok. 104 | terminate(_, State) -> %% Flush all pending actions 105 | maybe_run_fn(State#state{can_run_fn = true, max_n = infinity}), 106 | ok. 107 | 108 | -spec format_status(gen_server:format_status()) -> gen_server:format_status(). 109 | format_status(#{state := State} = FormatStatus) -> 110 | FormatStatus#{state := printable_state(State)}. 111 | 112 | %%------------------------------------------------------------------------------ 113 | %% internal functions 114 | %%------------------------------------------------------------------------------ 115 | 116 | %% - If `Delay' is infinity, we mean to pause the process, see how at `maybe_start_timer/1' 117 | %% a delay of infinity will set `can_run_fn = false'. 118 | %% 119 | %% - If `MaxN' is infinity and `Delay' is a number, we mean no limits to throttling, 120 | %% see how `maybe_start_timer/1' will not actually start any timer 121 | %% and `maybe_run_fn/1' with `max_n = infinity' will loop without pause. 122 | %% 123 | %% - If both `MaxN' and `Delay' are numbers, this will be the actual rate/interval. 124 | %% Note however that if delay is zero, we effectively limit parallelism to `MaxN'. 125 | -spec initial_state(Name :: atom(), MaxN :: amoc_throttle:rate(), Delay :: timeout()) -> state(). 126 | initial_state(Name, MaxN, Delay) -> 127 | #state{name = Name, max_n = MaxN, delay_between_executions = Delay}. 128 | 129 | merge_state(#state{delay_between_executions = D, max_n = MaxN}, #state{} = OldState) -> 130 | maybe_stop_timer(OldState), 131 | NewState = OldState#state{delay_between_executions = D, max_n = MaxN, tref = undefined}, 132 | maybe_start_timer(NewState). 133 | 134 | maybe_start_timer(#state{delay_between_executions = infinity, tref = undefined} = State) -> 135 | State#state{can_run_fn = false}; 136 | maybe_start_timer(#state{delay_between_executions = 0, tref = undefined} = State) -> 137 | State#state{can_run_fn = true}; 138 | maybe_start_timer(#state{delay_between_executions = D, tref = undefined} = State) -> 139 | {ok, TRef} = timer:send_interval(D, delay_between_executions), 140 | State#state{can_run_fn = false, tref = TRef}. 141 | 142 | maybe_stop_timer(#state{tref = undefined}) -> 143 | ok; 144 | maybe_stop_timer(#state{tref = TRef}) -> 145 | {ok, cancel} = timer:cancel(TRef), 146 | consume_all_timer_ticks(delay_between_executions). 147 | 148 | timeout(#state{delay_between_executions = infinity}) -> 149 | infinity; 150 | timeout(#state{delay_between_executions = Delay}) -> 151 | Delay + ?DEFAULT_MSG_TIMEOUT. 152 | 153 | consume_all_timer_ticks(Msg) -> 154 | receive 155 | Msg -> consume_all_timer_ticks(Msg) 156 | after 0 -> ok 157 | end. 158 | 159 | maybe_run_fn(#state{schedule = [], schedule_reversed = []} = State) -> 160 | State; 161 | maybe_run_fn(#state{schedule = [], schedule_reversed = SchRev} = State) -> 162 | NewSchedule = lists:reverse(SchRev), 163 | NewState = State#state{schedule = NewSchedule, schedule_reversed = []}, 164 | maybe_run_fn(NewState); 165 | maybe_run_fn(#state{can_run_fn = true, max_n = infinity} = State) -> 166 | NewState = run_fn(State), 167 | maybe_run_fn(NewState); 168 | maybe_run_fn(#state{can_run_fn = true, n = N, max_n = MaxN} = State) when N < MaxN -> 169 | NewState = run_fn(State), 170 | NewState#state{can_run_fn = false}; 171 | maybe_run_fn(State) -> 172 | State. 173 | 174 | run_fn(#state{schedule = [RunnerPid | T], name = Name, n = N} = State) -> 175 | erlang:monitor(process, RunnerPid), 176 | amoc_throttle_runner:run(RunnerPid), 177 | amoc_throttle_controller:telemetry_event(Name, execute), 178 | State#state{schedule = T, n = N + 1}. 179 | 180 | dec_n(#state{name = Name, n = 0} = State) -> 181 | PrintableState = printable_state(State), 182 | Msg = <<"throttle proccess has invalid N">>, 183 | Metadata = #{name => Name, n => 0, state => PrintableState}, 184 | amoc_telemetry:execute_log(error, [throttle, process], Metadata, Msg), 185 | State; 186 | dec_n(#state{n = N} = State) -> 187 | State#state{n = N - 1}. 188 | 189 | -spec internal_event(binary(), state()) -> any(). 190 | internal_event(Msg, #state{name = Name} = State) -> 191 | PrintableState = printable_state(State), 192 | amoc_telemetry:execute_log( 193 | debug, [throttle, process], #{self => self(), name => Name, state => PrintableState}, Msg). 194 | 195 | printable_state(#state{} = State) -> 196 | Fields = record_info(fields, state), 197 | [_ | Values] = tuple_to_list(State#state{schedule = [], schedule_reversed = []}), 198 | StateMap = maps:from_list(lists:zip(Fields, Values)), 199 | StateMap#{ 200 | schedule := length(State#state.schedule), 201 | schedule_reversed := length(State#state.schedule_reversed)}. 202 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_runner.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %% @doc Asynchronous runner that always runs together with the requesting process. 5 | %% 6 | %% Knows how to distinguist between the different operations the caller needs. 7 | -module(amoc_throttle_runner). 8 | 9 | -export([throttle/2, run/1]). 10 | -export([async_runner/4]). 11 | 12 | -type action() :: wait | {pid(), term()} | fun(() -> any()). 13 | 14 | -spec run(pid()) -> term(). 15 | run(RunnerPid) -> 16 | RunnerPid ! '$scheduled'. 17 | 18 | -spec throttle(amoc_throttle:name(), action()) -> ok | {error, any()}. 19 | throttle(Name, Action) -> 20 | case amoc_throttle_controller:get_throttle_process(Name) of 21 | {ok, ThrottlerPid} -> 22 | Args = [Name, self(), ThrottlerPid, Action], 23 | RunnerPid = erlang:spawn_link(?MODULE, async_runner, Args), 24 | amoc_throttle_process:run(ThrottlerPid, RunnerPid), 25 | maybe_wait(Action, RunnerPid); 26 | Error -> 27 | Error 28 | end. 29 | 30 | -spec maybe_wait(action(), pid()) -> ok. 31 | maybe_wait(wait, RunnerPid) -> 32 | receive 33 | {'EXIT', RunnerPid, Reason} -> 34 | exit({throttle_wait_died, RunnerPid, Reason}); 35 | '$scheduled' -> 36 | ok 37 | end; 38 | maybe_wait(_, _) -> 39 | ok. 40 | 41 | -spec async_runner(amoc_throttle:name(), pid(), pid(), action()) -> true. 42 | async_runner(Name, Caller, ThrottlerPid, Action) -> 43 | ThrottlerMonitor = erlang:monitor(process, ThrottlerPid), 44 | amoc_throttle_controller:raise_event_on_slave_node(Name, request), 45 | receive 46 | {'DOWN', ThrottlerMonitor, process, ThrottlerPid, Reason} -> 47 | exit({throttler_worker_died, ThrottlerPid, Reason}); 48 | '$scheduled' -> 49 | execute(Caller, Action), 50 | amoc_throttle_controller:raise_event_on_slave_node(Name, execute), 51 | %% If Action failed, unlink won't be called and the caller will receive an exit signal 52 | erlang:unlink(Caller) 53 | end. 54 | 55 | -spec execute(pid(), action()) -> term(). 56 | execute(Caller, wait) -> 57 | Caller ! '$scheduled'; 58 | execute(_Caller, Fun) when is_function(Fun, 0) -> 59 | Fun(); 60 | execute(_Caller, {Pid, Msg}) -> 61 | Pid ! Msg. 62 | -------------------------------------------------------------------------------- /src/throttle/amoc_throttle_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @see amoc_throttle 3 | %% @copyright 2024 Erlang Solutions Ltd. 4 | %% @doc Top supervisor for the controller, the pooler, and the process group 5 | %% 6 | %% The supervision tree is as follows 7 | %% 8 | %% amoc_sup 9 | %% | 10 | %% amoc_throttle_sup 11 | %% / | \ 12 | %% / | \ 13 | %% / | \ 14 | %% pooler controller pg 15 | %% || 16 | %% (dynamically) 17 | %% || 18 | %% pool 19 | %% / | \ 20 | %% [processes()] 21 | %% 22 | %% Where the pool, on creation, subscribes all its children to the named process group 23 | %% @end 24 | -module(amoc_throttle_sup). 25 | 26 | -behaviour(supervisor). 27 | 28 | -export([start_link/0, init/1]). 29 | 30 | -spec start_link() -> supervisor:startlink_ret(). 31 | start_link() -> 32 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 33 | 34 | -spec init(term()) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 35 | init([]) -> 36 | SupFlags = #{strategy => one_for_one}, 37 | Pg = #{id => pg, 38 | start => {pg, start_link, [amoc_throttle_controller:pg_scope()]}, 39 | type => worker, 40 | shutdown => timer:seconds(5), 41 | restart => permanent, 42 | modules => [pg]}, 43 | Controller = #{id => amoc_throttle_controller, 44 | start => {amoc_throttle_controller, start_link, []}, 45 | type => worker, 46 | shutdown => timer:seconds(5), 47 | restart => permanent, 48 | modules => [amoc_throttle_controller]}, 49 | Pooler = #{id => amoc_throttle_pooler, 50 | start => {amoc_throttle_pooler, start_link, []}, 51 | type => supervisor, 52 | shutdown => infinity, 53 | restart => permanent, 54 | modules => [amoc_throttle_pooler]}, 55 | {ok, {SupFlags, [Pg, Controller, Pooler]}}. 56 | -------------------------------------------------------------------------------- /src/users/amoc_user.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2024 Erlang Solutions Ltd. 3 | -module(amoc_user). 4 | 5 | %% API 6 | -export([start_link/3]). 7 | -export([stop/0, stop/2]). 8 | -export([init/4]). 9 | 10 | -type state() :: term(). 11 | 12 | -spec start_link(amoc:scenario(), amoc_scenario:user_id(), state()) -> 13 | {ok, pid()} | {error, term()}. 14 | start_link(Scenario, Id, State) -> 15 | proc_lib:start_link(?MODULE, init, [self(), Scenario, Id, State]). 16 | 17 | -spec stop() -> ok. 18 | stop() -> 19 | stop(self(), false). 20 | 21 | -spec stop(pid(), boolean()) -> ok. 22 | stop(Pid, Force) when is_pid(Pid) -> 23 | amoc_users_sup:stop_child(Pid, Force). 24 | 25 | -spec init(pid(), amoc:scenario(), amoc_scenario:user_id(), state()) -> term(). 26 | init(Parent, Scenario, Id, State) -> 27 | proc_lib:init_ack(Parent, {ok, self()}), 28 | process_flag(trap_exit, true), 29 | amoc_scenario:start(Scenario, Id, State). 30 | -------------------------------------------------------------------------------- /src/users/amoc_users_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2024 Erlang Solutions Ltd. 3 | %% @doc Top supervisor of the pooled users supervisor. 4 | %% 5 | %% It spawns a pool of workers as big as online schedulers. When starting a new user, as the user is 6 | %% identified by ID, a worker will be chosen for this user based on its ID 7 | %% (see get_sup_for_user_id/1). 8 | %% 9 | %% The currently running number of users is stored in an atomic that all workers update and the 10 | %% controller can read. 11 | -module(amoc_users_sup). 12 | 13 | -behaviour(supervisor). 14 | 15 | %% Supervisor 16 | -export([start_link/0, init/1]). 17 | 18 | %% API 19 | -export([incr_no_of_users/1, decr_no_of_users/1, count_no_of_users/0, 20 | start_child/3, stop_child/2, start_children/3, stop_children/2, terminate_all_children/0]). 21 | 22 | -export([distribute/2, get_all_children/0]). 23 | 24 | -type count() :: non_neg_integer(). 25 | -type assignment() :: [{pid(), count()}]. 26 | 27 | -record(storage, { 28 | %% an array of atomics whose index works as follows: 29 | %% * index=1 - overall number of Users 30 | %% * index>1 - number of users supervised by worker 31 | user_count :: atomics:atomics_ref(), 32 | sups :: tuple(), 33 | sups_indexed :: [{pid(), pos_integer()}], 34 | sups_count :: pos_integer() 35 | }). 36 | 37 | %% Supervisor 38 | 39 | %% @private 40 | -spec start_link() -> supervisor:startlink_ret(). 41 | start_link() -> 42 | Ret = supervisor:start_link({local, ?MODULE}, ?MODULE, no_args), 43 | UserSups = supervisor:which_children(?MODULE), 44 | IndexedSupsUnsorted = [ {Pid, N} || {{amoc_users_worker_sup, N}, Pid, _, _} <- UserSups], 45 | IndexedSups = lists:keysort(2, IndexedSupsUnsorted), 46 | UserSupPidsTuple = list_to_tuple([ Pid || {Pid, _} <- IndexedSups ]), 47 | SupCount = tuple_size(UserSupPidsTuple), 48 | Atomics = atomics:new(1 + SupCount, [{signed, false}]), 49 | Storage = #storage{user_count = Atomics, sups = UserSupPidsTuple, 50 | sups_indexed = IndexedSups, sups_count = SupCount}, 51 | persistent_term:put(?MODULE, Storage), 52 | Ret. 53 | 54 | %% @private 55 | -spec init(no_args) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 56 | init(no_args) -> 57 | Specs = [ 58 | #{ 59 | id => {amoc_users_worker_sup, N}, 60 | start => {amoc_users_worker_sup, start_link, [N]}, 61 | restart => permanent, 62 | shutdown => infinity, 63 | type => worker, 64 | modules => [amoc_users_worker_sup] 65 | } 66 | || N <- indexes() ], 67 | Strategy = #{strategy => one_for_one, intensity => 0}, 68 | {ok, {Strategy, Specs}}. 69 | 70 | indexes() -> 71 | lists:seq(2, erlang:system_info(schedulers_online) + 1). 72 | 73 | %% API 74 | -spec count_no_of_users() -> count(). 75 | count_no_of_users() -> 76 | #storage{user_count = Atomics} = persistent_term:get(?MODULE), 77 | atomics:get(Atomics, 1). 78 | 79 | -spec incr_no_of_users(non_neg_integer()) -> any(). 80 | incr_no_of_users(SupNum) when SupNum > 1 -> 81 | #storage{user_count = Atomics} = persistent_term:get(?MODULE), 82 | atomics:add(Atomics, SupNum, 1), 83 | atomics:add(Atomics, 1, 1). 84 | 85 | -spec decr_no_of_users(non_neg_integer()) -> ok. 86 | decr_no_of_users(SupNum) when SupNum > 1 -> 87 | #storage{user_count = Atomics} = persistent_term:get(?MODULE), 88 | atomics:sub(Atomics, SupNum, 1), 89 | case atomics:sub_get(Atomics, 1, 1) of 90 | 0 -> 91 | amoc_controller:zero_users_running(); 92 | _ -> 93 | ok 94 | end. 95 | 96 | -spec start_child(amoc:scenario(), amoc_scenario:user_id(), any()) -> ok. 97 | start_child(Scenario, Id, ScenarioState) -> 98 | Sup = get_sup_for_user_id(Id), 99 | amoc_users_worker_sup:start_child(Sup, Scenario, Id, ScenarioState). 100 | 101 | -spec stop_child(pid(), boolean()) -> ok. 102 | stop_child(Pid, Force) -> 103 | amoc_users_worker_sup:stop_child(Pid, Force). 104 | 105 | %% Group all children based on ID to their respective worker supervisor and cast a request with each 106 | %% group at once. This way we reduce the number of casts to each worker to always one, instead of 107 | %% depending on the number of users. 108 | -spec start_children(amoc:scenario(), [amoc_scenario:user_id()], any()) -> ok. 109 | start_children(Scenario, UserIds, ScenarioState) -> 110 | State = persistent_term:get(?MODULE), 111 | #storage{sups = Supervisors, sups_indexed = IndexedSups, sups_count = SupCount} = State, 112 | Acc = maps:from_list([ {Sup, []} || {Sup, _} <- IndexedSups ]), 113 | Assignments = assign_users_to_sups(SupCount, Supervisors, UserIds, Acc), 114 | CastFun = fun(Sup, Users) -> 115 | amoc_users_worker_sup:start_children(Sup, Scenario, Users, ScenarioState) 116 | end, 117 | maps:foreach(CastFun, Assignments). 118 | 119 | %% Assign a count of children each worker needs to remove 120 | %% in order to load-balance the request among all workers. 121 | -spec stop_children(non_neg_integer(), boolean()) -> non_neg_integer(). 122 | stop_children(Count, Force) -> 123 | {CountRemove, Assignments} = assign_counts(Count), 124 | [ amoc_users_worker_sup:stop_children(Sup, Int, Force) || {Sup, Int} <- Assignments ], 125 | CountRemove. 126 | 127 | -spec get_all_children() -> [{pid(), amoc_scenario:user_id()}]. 128 | get_all_children() -> 129 | #storage{sups_indexed = IndexedSups} = persistent_term:get(?MODULE), 130 | All = [ amoc_users_worker_sup:get_all_children(Sup) || {Sup, _} <- IndexedSups ], 131 | lists:flatten(All). 132 | 133 | -spec terminate_all_children() -> any(). 134 | terminate_all_children() -> 135 | #storage{sups_indexed = IndexedSups} = persistent_term:get(?MODULE), 136 | [ amoc_users_worker_sup:terminate_all_children(Sup) || {Sup, _} <- IndexedSups ]. 137 | 138 | %% Helpers 139 | -spec get_sup_for_user_id(amoc_scenario:user_id()) -> pid(). 140 | get_sup_for_user_id(Id) -> 141 | #storage{sups = Supervisors, sups_count = SupCount} = persistent_term:get(?MODULE), 142 | Index = erlang:phash2(Id, SupCount) + 1, 143 | element(Index, Supervisors). 144 | 145 | %% assign which users each worker will be requested to add 146 | -spec assign_users_to_sups(pos_integer(), tuple(), [amoc_scenario:user_id()], Acc) -> 147 | Acc when Acc :: #{pid() := [amoc_scenario:user_id()]}. 148 | assign_users_to_sups(SupCount, Supervisors, [Id | Ids], Acc) -> 149 | Index = erlang:phash2(Id, SupCount) + 1, 150 | ChosenSup = element(Index, Supervisors), 151 | Vs = maps:get(ChosenSup, Acc), 152 | NewAcc = Acc#{ChosenSup := [Id | Vs]}, 153 | assign_users_to_sups(SupCount, Supervisors, Ids, NewAcc); 154 | assign_users_to_sups(_, _, [], Acc) -> 155 | Acc. 156 | 157 | %% assign how many users each worker will be requested to remove, 158 | %% taking care of the fact that worker might not have enough users. 159 | -spec assign_counts(count()) -> {count(), assignment()}. 160 | assign_counts(Total) -> 161 | #storage{user_count = Atomics, sups_indexed = Indexed} = persistent_term:get(?MODULE), 162 | SupervisorsWithCounts = [ {Sup, atomics:get(Atomics, SupPos)} || {Sup, SupPos} <- Indexed ], 163 | distribute(Total, SupervisorsWithCounts). 164 | 165 | -spec distribute(count(), assignment()) -> {count(), assignment()}. 166 | distribute(Total, SupervisorsWithCounts) -> 167 | SupervisorWithPositiveCounts = [ T || T = {_, Count} <- SupervisorsWithCounts, Count =/= 0], 168 | Data = maps:from_list(SupervisorWithPositiveCounts), 169 | N = remove_n(Total, Data), 170 | distribute(#{}, Data, SupervisorWithPositiveCounts, Total, N). 171 | 172 | -spec remove_n(count(), map()) -> non_neg_integer(). 173 | remove_n(Total, Data) when map_size(Data) > 0 -> 174 | case Total div map_size(Data) of 175 | 0 -> 1; 176 | N -> N 177 | end; 178 | remove_n(_Total, _Data) -> 0. 179 | 180 | -spec distribute(#{pid() := count()}, #{pid() := count()}, assignment(), count(), count()) -> 181 | {count(), assignment()}. 182 | %% Already assigned all, or not enough active users, we're done 183 | distribute(Acc, Data, _, Left, _N) when 0 =:= Left; 0 =:= map_size(Data) -> 184 | {lists:sum(maps:values(Acc)), maps:to_list(Acc)}; 185 | %% Already assigned one round and still have counts left and running users available, loop again 186 | distribute(Acc, Data, [], Left, _N) -> 187 | NewData = maps:filter(fun(_K, V) -> V > 0 end, Data), 188 | NewN = remove_n(Left, NewData), 189 | distribute(Acc, NewData, maps:to_list(NewData), Left, NewN); 190 | distribute(Acc, Data, [{Sup, UsersInSup} | Rest], Left, N) -> 191 | case UsersInSup =< N of 192 | true -> 193 | %% Assigning the last possible users to this supervisor 194 | NewAcc = increment(Sup, UsersInSup, Acc), 195 | NewData = maps:put(Sup, 0, Data), 196 | distribute(NewAcc, NewData, Rest, Left - UsersInSup, N); 197 | false -> 198 | %% Assign N more to this supervisor and continue assigning 199 | NewAcc = increment(Sup, N, Acc), 200 | NewData = maps:put(Sup, UsersInSup - N, Data), 201 | distribute(NewAcc, NewData, Rest, Left - N, N) 202 | end. 203 | 204 | increment(Key, Increment, Acc) -> 205 | maps:update_with(Key, fun(V) -> V + Increment end, Increment, Acc). 206 | -------------------------------------------------------------------------------- /src/users/amoc_users_worker_sup.erl: -------------------------------------------------------------------------------- 1 | %% @private 2 | %% @copyright 2024 Erlang Solutions Ltd. 3 | %% @doc Supervisor-like gen_server with some tracking responsibilities over the users 4 | %% 5 | %% We want to keep consistent track of all users running globally by running special code upon a 6 | %% children's death. Standard solutions don't cut it because: 7 | %% - A supervisor doesn't expose callbacks on user termination 8 | %% - Implementing code on the user process before it dies risks inconsistencies if it is killed 9 | %% More improvements that could be made would be to distribute the supervision tree like ranch did, 10 | %% see https://stressgrid.com/blog/100k_cps_with_elixir/ 11 | %% @end 12 | -module(amoc_users_worker_sup). 13 | 14 | -behaviour(gen_server). 15 | 16 | %% gen_server callbacks 17 | -export([start_link/1]). 18 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). 19 | 20 | -export([start_child/4, stop_child/2, start_children/4, stop_children/3, terminate_all_children/1]). 21 | 22 | -export([get_all_children/1]). 23 | 24 | -record(state, { 25 | index :: non_neg_integer(), 26 | table :: ets:table(), 27 | tasks = #{} :: #{reference() := pid()} 28 | }). 29 | -type state() :: #state{}. 30 | 31 | -define(SHUTDOWN_TIMEOUT, 2000). %% 2 seconds 32 | 33 | %% @private 34 | -spec start_link(non_neg_integer()) -> gen_server:start_ret(). 35 | start_link(N) -> 36 | gen_server:start_link(?MODULE, N, []). 37 | 38 | -spec start_child(pid(), amoc:scenario(), amoc_scenario:user_id(), any()) -> ok. 39 | start_child(Sup, Scenario, Id, ScenarioState) -> 40 | gen_server:cast(Sup, {start_child, Scenario, Id, ScenarioState}). 41 | 42 | -spec start_children(pid(), amoc:scenario(), [amoc_scenario:user_id()], any()) -> ok. 43 | start_children(Sup, Scenario, UserIds, ScenarioState) -> 44 | gen_server:cast(Sup, {start_children, Scenario, UserIds, ScenarioState}). 45 | 46 | -spec stop_children(pid(), non_neg_integer(), boolean()) -> ok. 47 | stop_children(Sup, Count, Force) -> 48 | gen_server:cast(Sup, {stop_children, Count, Force}). 49 | 50 | -spec terminate_all_children(pid()) -> any(). 51 | terminate_all_children(Sup) -> 52 | gen_server:cast(Sup, terminate_all_children). 53 | 54 | -spec stop_child(pid(), boolean()) -> ok. 55 | stop_child(Pid, false) -> 56 | exit(Pid, shutdown), 57 | ok; 58 | stop_child(Pid, true) -> 59 | spawn(shutdown_and_kill_after_timeout_fun(Pid)), 60 | ok. 61 | 62 | -spec get_all_children(pid()) -> [{pid(), amoc_scenario:user_id()}]. 63 | get_all_children(Sup) -> 64 | gen_server:call(Sup, get_all_children, infinity). 65 | 66 | %% @private 67 | -spec init(non_neg_integer()) -> {ok, state()}. 68 | init(N) -> 69 | process_flag(trap_exit, true), 70 | Name = list_to_atom(atom_to_list(?MODULE) ++ "_" ++ integer_to_list(N)), 71 | Table = ets:new(Name, [ordered_set, protected, named_table]), 72 | {ok, #state{index = N, table = Table}}. 73 | 74 | %% @private 75 | -spec handle_call(any(), any(), state()) -> {reply, term(), state()}. 76 | handle_call(get_all_children, _From, #state{table = Table} = State) -> 77 | Children = ets:tab2list(Table), 78 | {reply, Children, State}. 79 | 80 | %% @private 81 | -spec handle_cast(Request, state()) -> {noreply, state()} when 82 | Request :: {start_child, amoc:scenario(), amoc_scenario:user_id(), amoc_scenario:state()} | 83 | {start_children, amoc:scenario(), [amoc_scenario:user_id()], amoc_scenario:state()} | 84 | {stop_children, non_neg_integer(), boolean()} | 85 | terminate_all_children. 86 | handle_cast({start_child, Scenario, Id, ScenarioState}, State) -> 87 | do_start_child(Scenario, Id, ScenarioState, State), 88 | {noreply, State}; 89 | handle_cast({start_children, Scenario, Ids, ScenarioState}, State) -> 90 | [ do_start_child(Scenario, Id, ScenarioState, State) || Id <- Ids], 91 | {noreply, State}; 92 | handle_cast({stop_children, 0, _}, State) -> 93 | {noreply, State}; 94 | handle_cast({stop_children, Int, ForceRemove}, #state{table = Table} = State) -> 95 | Pids = case ets:match_object(Table, '$1', Int) of 96 | '$end_of_table' -> 97 | []; 98 | {Objects, _} -> 99 | [Pid || {Pid, _Id} <- Objects] 100 | end, 101 | NewState = maybe_track_task_to_stop_my_children(State, Pids, ForceRemove), 102 | {noreply, NewState}; 103 | handle_cast(terminate_all_children, State) -> 104 | NewState = do_terminate_all_my_children(State), 105 | {noreply, NewState}; 106 | handle_cast(_Msg, State) -> 107 | {noreply, State}. 108 | 109 | %% @private 110 | -spec handle_info(Request, state()) -> {noreply, state()} when 111 | Request :: {child_up, pid(), amoc_scenario:user_id()} | 112 | {'DOWN', reference(), process, pid(), term()} | 113 | {'EXIT', pid(), term()}. 114 | handle_info({'DOWN', Ref, process, _Pid, _Reason}, #state{tasks = Tasks} = State) -> 115 | {noreply, State#state{tasks = maps:remove(Ref, Tasks)}}; 116 | handle_info({'EXIT', Pid, _Reason}, #state{index = N, table = Table} = State) -> 117 | handle_down_user(Table, Pid, N), 118 | {noreply, State}; 119 | handle_info(_Info, State) -> 120 | {noreply, State}. 121 | 122 | %% @private 123 | -spec terminate(term(), state()) -> state(). 124 | terminate(_Reason, State) -> 125 | do_terminate_all_my_children(State). 126 | 127 | %% Helpers 128 | 129 | -spec do_start_child(module(), amoc_scenario:user_id(), term(), state()) -> any(). 130 | do_start_child(Scenario, Id, ScenarioState, #state{index = N, table = Table}) -> 131 | case amoc_user:start_link(Scenario, Id, ScenarioState) of 132 | {ok, Pid} -> 133 | handle_up_user(Table, Pid, Id, N); 134 | _ -> 135 | ok 136 | end. 137 | 138 | -spec handle_up_user(ets:table(), pid(), amoc_scenario:user_id(), non_neg_integer()) -> any(). 139 | handle_up_user(Table, Pid, Id, SupNum) -> 140 | ets:insert(Table, {Pid, Id}), 141 | amoc_users_sup:incr_no_of_users(SupNum). 142 | 143 | -spec handle_down_user(ets:table(), pid(), non_neg_integer()) -> ok. 144 | handle_down_user(Table, Pid, SupNum) -> 145 | ets:delete(Table, Pid), 146 | amoc_users_sup:decr_no_of_users(SupNum). 147 | 148 | %% @doc Stop a list of users in parallel. 149 | %% We don't want to ever block the supervisor on `timer:sleep/1' so we spawn that async. 150 | %% However we don't want free processes roaming around, we want monitoring that can be traced. 151 | -spec maybe_track_task_to_stop_my_children(state(), [pid()], boolean()) -> state(). 152 | maybe_track_task_to_stop_my_children(State, [], _) -> 153 | State; 154 | maybe_track_task_to_stop_my_children(State, Pids, false) -> 155 | [ exit(Pid, shutdown) || Pid <- Pids ], 156 | State; 157 | maybe_track_task_to_stop_my_children(#state{tasks = Tasks} = State, Pids, true) -> 158 | {Pid, Ref} = spawn_monitor(shutdown_and_kill_after_timeout_fun(Pids)), 159 | State#state{tasks = Tasks#{Pid => Ref}}. 160 | 161 | -spec shutdown_and_kill_after_timeout_fun(pid() | [pid()]) -> fun(() -> term()). 162 | shutdown_and_kill_after_timeout_fun([_ | _] = Pids) -> 163 | fun() -> 164 | [ exit(Pid, shutdown) || Pid <- Pids ], 165 | timer:sleep(?SHUTDOWN_TIMEOUT), 166 | [ exit(Pid, kill) || Pid <- Pids ] 167 | end; 168 | shutdown_and_kill_after_timeout_fun(Pid) when is_pid(Pid) -> 169 | fun() -> 170 | exit(Pid, shutdown), 171 | timer:sleep(?SHUTDOWN_TIMEOUT), 172 | exit(Pid, kill) 173 | end. 174 | 175 | -spec do_terminate_all_my_children(state()) -> state(). 176 | do_terminate_all_my_children(#state{table = Table} = State) -> 177 | Match = ets:match_object(Table, '$1', 200), 178 | do_terminate_all_my_children(State, Match). 179 | 180 | %% ets:continuation/0 type is unfortunately not exported from the ets module. 181 | -spec do_terminate_all_my_children(state(), {[tuple()], term()} | '$end_of_table') -> state(). 182 | do_terminate_all_my_children(State, {Objects, Continuation}) -> 183 | Pids = [Pid || {Pid, _Id} <- Objects], 184 | NewState = maybe_track_task_to_stop_my_children(State, Pids, true), 185 | Match = ets:match_object(Continuation), 186 | do_terminate_all_my_children(NewState, Match); 187 | do_terminate_all_my_children(State, '$end_of_table') -> 188 | State. 189 | -------------------------------------------------------------------------------- /test/amoc_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(amoc_SUITE). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -compile([export_all, nowarn_export_all]). 6 | 7 | all() -> 8 | [ 9 | bad_config_fails_to_start, 10 | start_with_no_users, 11 | start_with_some_users, 12 | start_and_add_some_users, 13 | start_and_then_force_remove_some_users, 14 | start_and_then_soft_remove_some_users, 15 | start_and_then_force_remove_more_users_than_running, 16 | force_remove_more_users_with_no_running, 17 | start_and_then_soft_remove_users_that_ignore_the_error, 18 | start_and_then_stop_cannot_rerun, 19 | after_reset_can_run_again 20 | ]. 21 | 22 | init_per_suite(Config) -> 23 | Config. 24 | 25 | end_per_suite(_) -> 26 | ok. 27 | 28 | init_per_testcase(_, Config) -> 29 | application:ensure_all_started(amoc), 30 | Config. 31 | 32 | end_per_testcase(_, _Config) -> 33 | application:stop(amoc), 34 | ok. 35 | 36 | %%----------------------------------------------------------------------------------- 37 | %% test cases 38 | %%----------------------------------------------------------------------------------- 39 | 40 | bad_config_fails_to_start(_) -> 41 | Ret = amoc_do(testing_scenario, 0, []), 42 | ?assertMatch({{error, _}, 0}, Ret). 43 | 44 | start_with_no_users(_) -> 45 | Ret = amoc_do(testing_scenario, 0), 46 | ?assertEqual(ok, Ret), 47 | test_helpers:wait_until_scenario_has_users(testing_scenario, 0, 0). 48 | 49 | start_with_some_users(_) -> 50 | Ret = amoc_do(testing_scenario, 1), 51 | ?assertEqual(ok, Ret), 52 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 1). 53 | 54 | start_and_add_some_users(_) -> 55 | Ret = amoc_do(testing_scenario, 0), 56 | ?assertEqual(ok, Ret), 57 | test_helpers:wait_until_scenario_has_users(testing_scenario, 0, 0), 58 | amoc:add(1), 59 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 1). 60 | 61 | start_and_then_force_remove_some_users(_) -> 62 | Ret = amoc_do(testing_scenario, 2), 63 | ?assertEqual(ok, Ret), 64 | test_helpers:wait_until_scenario_has_users(testing_scenario, 2, 2), 65 | Removed = amoc:remove(1, true), 66 | ?assertEqual({ok, 1}, Removed), 67 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 2). 68 | 69 | start_and_then_soft_remove_some_users(_) -> 70 | Ret = amoc_do(testing_scenario, 2), 71 | ?assertEqual(ok, Ret), 72 | test_helpers:wait_until_scenario_has_users(testing_scenario, 2, 2), 73 | Removed = amoc:remove(1, false), 74 | ?assertEqual({ok, 1}, Removed), 75 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 2). 76 | 77 | start_and_then_force_remove_more_users_than_running(_) -> 78 | Ret = amoc_do(testing_scenario, 2), 79 | ?assertEqual(ok, Ret), 80 | test_helpers:wait_until_scenario_has_users(testing_scenario, 2, 2), 81 | Removed = amoc:remove(10, true), 82 | ?assertEqual({ok, 2}, Removed), 83 | test_helpers:wait_until_scenario_has_users(testing_scenario, 0, 2). 84 | 85 | force_remove_more_users_with_no_running(_) -> 86 | Ret = amoc_do(testing_scenario, 0), 87 | ?assertEqual(ok, Ret), 88 | test_helpers:wait_until_scenario_has_users(testing_scenario, 0, 0), 89 | Removed = amoc:remove(10, true), 90 | ?assertEqual({ok, 0}, Removed), 91 | test_helpers:wait_until_scenario_has_users(testing_scenario, 0, 0). 92 | 93 | start_and_then_soft_remove_users_that_ignore_the_error(_) -> 94 | Ret = amoc_do(testing_scenario_with_state, 2, test_helpers:all_vars_with_state()), 95 | ?assertEqual(ok, Ret), 96 | test_helpers:wait_until_scenario_has_users(testing_scenario_with_state, 2, 2), 97 | Removed = amoc:remove(10, false), 98 | ?assertEqual({ok, 2}, Removed), 99 | timer:sleep(100), 100 | Status = amoc_controller:get_status(), 101 | ?assertMatch({running, #{scenario := testing_scenario_with_state, 102 | currently_running_users := 2, 103 | highest_user_id := 2}}, Status). 104 | 105 | start_and_then_stop_cannot_rerun(_) -> 106 | Ret = amoc_do(testing_scenario, 1), 107 | ?assertEqual(ok, Ret), 108 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 1), 109 | Status = amoc:stop(), 110 | ?assertMatch(ok, Status), 111 | Retry = amoc_do(testing_scenario, 1), 112 | ?assertMatch({{error, {invalid_status, _}}, 1}, Retry). 113 | 114 | after_reset_can_run_again(_) -> 115 | Ret = amoc_do(testing_scenario, 1), 116 | ?assertEqual(ok, Ret), 117 | test_helpers:wait_until_scenario_has_users(testing_scenario, 1, 1), 118 | Status = amoc:reset(), 119 | ?assertMatch(ok, Status), 120 | Retry = amoc_do(testing_scenario, 1), 121 | ?assertMatch(ok, Retry). 122 | 123 | %% Helpers 124 | amoc_do(Scenario) -> 125 | amoc_do(Scenario, 0, test_helpers:all_vars()). 126 | 127 | amoc_do(Scenario, Count) -> 128 | amoc_do(Scenario, Count, test_helpers:all_vars()). 129 | 130 | amoc_do(Scenario, Count, Config) -> 131 | amoc:do(Scenario, Count, Config). 132 | -------------------------------------------------------------------------------- /test/amoc_code_server_SUITE_data/module.mustache: -------------------------------------------------------------------------------- 1 | 2 | %% this narrowed-down template allows generation of the module 3 | %% with one and the same MD5 but a different binary returned 4 | %% by the code:get_object_code/1 interface. 5 | 6 | -module({{module_name}}). 7 | 8 | {{tag}} 9 | 10 | -export([some_function/0]). 11 | 12 | some_function() -> ok. 13 | 14 | another_function() -> ok. 15 | -------------------------------------------------------------------------------- /test/amoc_config_attributes_SUITE.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(amoc_config_attributes_SUITE). 3 | 4 | -include_lib("eunit/include/eunit.hrl"). 5 | -include("../src/config/amoc_config.hrl"). 6 | 7 | -export([all/0]). 8 | -export([get_module_attributes/1, 9 | get_module_configuration/1, 10 | errors_reporting/1, 11 | one_of_function/1]). 12 | 13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 14 | %% these attributes and functions are required for the testing purposes %% 15 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 16 | -required_variable(#{name => config_attrs_var0, 17 | description => "config_attrs_var0"}). 18 | -required_variable(#{name => config_attrs_var1, 19 | description => "config_attrs_var1", 20 | default_value => def1}). 21 | -required_variable([ 22 | #{name => config_attrs_var2, description => "config_attrs_var2", 23 | default_value => def2, verification => none}, 24 | #{name => config_attrs_var3, description => "config_attrs_var3", 25 | default_value => def3, verification => [def3, another_atom]}, 26 | #{name => config_attrs_var4, description => "config_attrs_var4", 27 | default_value => def4, verification => {?MODULE, verify_value, 1}}, 28 | #{name => config_attrs_var4b, description => "config_attrs_var4b", 29 | default_value => def4b, verification => fun ?MODULE:verify_value/1} 30 | ]). 31 | -required_variable(#{name => config_attrs_var5, description => "config_attrs_var5", 32 | default_value => def5, update => read_only}). 33 | -required_variable(#{name => config_attrs_var6, description => "config_attrs_var6", 34 | default_value => def6, verification => none, update => none}). 35 | -required_variable([ 36 | #{name => config_attrs_var7, description => "config_attrs_var7", 37 | default_value => def7, verification => none, update => {?MODULE, update_value, 2}} 38 | ]). 39 | -required_variable(#{name => config_attrs_var7b, description => "config_attrs_var7b", 40 | default_value => def7, verification => none, 41 | update => fun ?MODULE:update_value/2}). 42 | 43 | %% verification functions 44 | -export([verify_value/1]). 45 | %% update functions 46 | -export([update_value/2]). 47 | 48 | verify_value(_) -> true. 49 | update_value(_, _) -> ok. 50 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 51 | 52 | all() -> 53 | [get_module_attributes, 54 | get_module_configuration, 55 | errors_reporting, 56 | one_of_function]. 57 | 58 | get_module_attributes(_) -> 59 | Result = amoc_config_attributes:get_module_attributes(required_variable, ?MODULE), 60 | ExpectedResult = [ 61 | #{name => config_attrs_var0, description => "config_attrs_var0"}, 62 | #{name => config_attrs_var1, description => "config_attrs_var1", default_value => def1}, 63 | #{name => config_attrs_var2, description => "config_attrs_var2", default_value => def2, 64 | verification => none}, 65 | #{name => config_attrs_var3, description => "config_attrs_var3", default_value => def3, 66 | verification => [def3, another_atom]}, 67 | #{name => config_attrs_var4, description => "config_attrs_var4", default_value => def4, 68 | verification => {?MODULE, verify_value, 1}}, 69 | #{name => config_attrs_var4b, description => "config_attrs_var4b", default_value => def4b, 70 | verification => fun ?MODULE:verify_value/1}, 71 | #{name => config_attrs_var5, description => "config_attrs_var5", default_value => def5, 72 | update => read_only}, 73 | #{name => config_attrs_var6, description => "config_attrs_var6", default_value => def6, 74 | verification => none, update => none}, 75 | #{name => config_attrs_var7, description => "config_attrs_var7", default_value => def7, 76 | verification => none, update => {?MODULE, update_value, 2}}, 77 | #{name => config_attrs_var7b, description => "config_attrs_var7b", default_value => def7, 78 | verification => none, update => fun ?MODULE:update_value/2}], 79 | ?assertEqual(ExpectedResult, Result). 80 | 81 | get_module_configuration(_) -> 82 | {ok, Config} = 83 | amoc_config_attributes:get_module_configuration(required_variable, ?MODULE), 84 | VerifyValueFN = fun ?MODULE:verify_value/1, 85 | UpdateValueFN = fun ?MODULE:update_value/2, 86 | UpdateNone = fun amoc_config_attributes:none/2, 87 | VerificationNone = fun amoc_config_attributes:none/1, 88 | ?assertMatch( 89 | [#module_parameter{name = config_attrs_var0, mod = ?MODULE, value = undefined, 90 | verification_fn = VerificationNone, update_fn = read_only}, 91 | #module_parameter{name = config_attrs_var1, mod = ?MODULE, value = def1, 92 | verification_fn = VerificationNone, update_fn = read_only}, 93 | #module_parameter{name = config_attrs_var2, mod = ?MODULE, value = def2, 94 | verification_fn = VerificationNone, update_fn = read_only}, 95 | #module_parameter{name = config_attrs_var3, mod = ?MODULE, value = def3, 96 | update_fn = read_only}, %% verification_fn is checked in 97 | %% 'one_of_function' test case. 98 | #module_parameter{name = config_attrs_var4, mod = ?MODULE, value = def4, 99 | verification_fn = VerifyValueFN, update_fn = read_only}, 100 | #module_parameter{name = config_attrs_var4b, mod = ?MODULE, value = def4b, 101 | verification_fn = VerifyValueFN, update_fn = read_only}, 102 | #module_parameter{name = config_attrs_var5, mod = ?MODULE, value = def5, 103 | verification_fn = VerificationNone, update_fn = read_only}, 104 | #module_parameter{name = config_attrs_var6, mod = ?MODULE, value = def6, 105 | verification_fn = VerificationNone, update_fn = UpdateNone}, 106 | #module_parameter{name = config_attrs_var7, mod = ?MODULE, value = def7, 107 | verification_fn = VerificationNone, update_fn = UpdateValueFN}, 108 | #module_parameter{name = config_attrs_var7b, mod = ?MODULE, value = def7, 109 | verification_fn = VerificationNone, update_fn = UpdateValueFN}], 110 | Config). 111 | 112 | errors_reporting(_) -> 113 | InvalidParam0 = #{name => "invalid_var0", description => "config_attrs_var0"}, 114 | InvalidParam1 = #{name => invalid_var1, description => [$a, -2]}, 115 | InvalidParam2 = #{name => invalid_var2, description => "config_attrs_var2", 116 | verification => <<"invalid_verification_method">>}, 117 | ValidParam3 = #{name => valid_var3, description => "config_attrs_var3", default_value => def3, 118 | verification => [def3, another_atom]}, 119 | InvalidParam4 = #{name => invalid_var4, description => "config_attrs_var4", 120 | verification => {erlang, not_exported_function, 1}}, 121 | InvalidParam4b = #{name => invalid_var4b, description => "config_attrs_var4b", 122 | verification => fun erlang:not_exported_function/1}, 123 | InvalidParam5 = #{name => invalid_var5, description => "config_attrs_var5", 124 | update => invalid_update_method}, 125 | InvalidParam6 = #{name => invalid_var6, description => "config_attrs_var6", 126 | update => fun invalid_module:not_exported_function/2}, 127 | InvalidParam7 = #{name => invalid_var7, description => "config_attrs_var7", 128 | update => fun update_value/2}, %% local function 129 | InvalidParam7b = #{name => invalid_var7, description => "config_attrs_var7b", 130 | update => fun update_value/2}, %% local function 131 | Attributes = [InvalidParam0, InvalidParam1, InvalidParam2, ValidParam3, InvalidParam4, 132 | InvalidParam4b, InvalidParam5, InvalidParam6, InvalidParam7, InvalidParam7b], 133 | {error, invalid_attribute_format, Reason} = 134 | amoc_config_attributes:process_module_attributes(?MODULE, Attributes), 135 | ?assertEqual( 136 | [{InvalidParam0, invalid_attribute}, 137 | {InvalidParam1, invalid_attribute}, 138 | {InvalidParam2, invalid_verification_method}, 139 | {InvalidParam4, verification_method_not_exported}, 140 | {InvalidParam4b, verification_method_not_exported}, 141 | {InvalidParam5, invalid_update_method}, 142 | {InvalidParam6, update_method_not_exported}, 143 | {InvalidParam7, update_method_not_exported}, 144 | {InvalidParam7b, update_method_not_exported}], 145 | Reason). 146 | 147 | one_of_function(_) -> 148 | OneOf = [def3, another_atom], 149 | Param = #{name => config_attrs_var0, description => "config_attrs_var0", default_value => def0, 150 | verification => OneOf}, 151 | {ok, [#module_parameter{verification_fn = OneOfFN}]} = 152 | amoc_config_attributes:process_module_attributes(?MODULE, [Param]), 153 | assert_one_of_fn(OneOfFN, OneOf). 154 | 155 | assert_one_of_fn(OneOfFN, OneOfValue) -> 156 | ?assertEqual({arity, 1}, erlang:fun_info(OneOfFN, arity)), 157 | ?assertEqual({module, amoc_config_attributes}, erlang:fun_info(OneOfFN, module)), 158 | ?assertEqual({type, local}, erlang:fun_info(OneOfFN, type)), 159 | ?assertEqual({env, [OneOfValue]}, erlang:fun_info(OneOfFN, env)). 160 | -------------------------------------------------------------------------------- /test/amoc_config_env_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(amoc_config_env_SUITE). 2 | 3 | -include_lib("proper/include/proper.hrl"). 4 | -include_lib("eunit/include/eunit.hrl"). 5 | 6 | -import(amoc_config_helper, [format_value/1, 7 | format_value/2, 8 | get_env/1, 9 | get_env/2, 10 | set_os_env/2, 11 | unset_os_env/1, 12 | set_empty_os_env/1]). 13 | 14 | -compile(export_all). 15 | 16 | -define(MOCK_MOD, custom_parser). 17 | 18 | all() -> 19 | [parse_value_prop_test, 20 | default_parser_test, 21 | valid_custom_parser_test, 22 | invalid_custom_parser_test]. 23 | 24 | init_per_suite(Config) -> 25 | %% set some predefined variables 26 | set_os_env(set_var, some_value), 27 | unset_os_env(unset_var), 28 | set_empty_os_env(empty_var), 29 | ct:pal("OS env:~n ~p", [os:getenv()]), 30 | Config. 31 | 32 | end_per_suite(_) -> 33 | %% unset predefined variables 34 | unset_os_env(set_var), 35 | unset_os_env(empty_var). 36 | 37 | init_per_testcase(valid_custom_parser_test, Config) -> 38 | meck:new(?MOCK_MOD, [non_strict, no_link]), 39 | set_config_parser_module(?MOCK_MOD), 40 | ct:pal("AMOC_* OS env. variables:~n ~p", 41 | [[AmocEnv || "AMOC_" ++ _ = AmocEnv <- os:getenv()]]), 42 | Config; 43 | init_per_testcase(invalid_custom_parser_test, Config) -> 44 | %% setting non-existing config parser module 45 | set_config_parser_module(invalid_parser_module), 46 | Config; 47 | init_per_testcase(_, Config) -> 48 | Config. 49 | 50 | end_per_testcase(valid_custom_parser_test, _Config) -> 51 | unset_config_parser_module(), 52 | meck:unload(); 53 | end_per_testcase(invalid_custom_parser_test, _Config) -> 54 | unset_config_parser_module(); 55 | end_per_testcase(_, _Config) -> 56 | ok. 57 | %%----------------------------------------------------------------------------------- 58 | %% test cases 59 | %%----------------------------------------------------------------------------------- 60 | 61 | parse_value_prop_test(_) -> 62 | RealAnyType = weighted_union([{1, map(any(), any())}, 63 | {10, any()}]), 64 | ProperTestStrings = 65 | ?FORALL(Value, RealAnyType, 66 | amoc_config_parser:parse_value(format_value(Value)) =:= {ok, Value}), 67 | ?assertEqual(true, proper:quickcheck(ProperTestStrings, [quiet])), 68 | 69 | ProperTestBinaries = 70 | ?FORALL(Value, RealAnyType, 71 | amoc_config_parser:parse_value(format_value(Value, binary)) =:= {ok, Value}), 72 | ?assertEqual(true, proper:quickcheck(ProperTestBinaries, [quiet])). 73 | 74 | default_parser_test(_) -> 75 | %% testing default config parser module (amoc_config_parser) 76 | ?assertEqual(some_value, get_env(set_var)), 77 | ?assertEqual(some_value, get_env(set_var, default_value)), 78 | ?assertEqual(undefined, get_env(unset_var)), 79 | ?assertEqual(default_value, get_env(unset_var, default_value)), 80 | ?assertEqual(undefined, get_env(empty_var)), 81 | ?assertEqual(default_value, get_env(empty_var, default_value)). 82 | 83 | valid_custom_parser_test(_) -> 84 | %% testing invalid custom config parser module 85 | Self = self(), 86 | 87 | %% successful parsing 88 | meck:expect(?MOCK_MOD, parse_value, ['_'], {ok, another_value}), 89 | ?assertEqual(another_value, get_env(set_var)), 90 | ?assertEqual(another_value, get_env(set_var, default_value)), 91 | OkCall = {Self, {?MOCK_MOD, parse_value, ["some_value"]}, {ok, another_value}}, 92 | ?assertEqual([OkCall, OkCall], meck:history(?MOCK_MOD)), 93 | meck:reset(?MOCK_MOD), 94 | 95 | %% gracefully failing parsing 96 | meck:expect(?MOCK_MOD, parse_value, ['_'], {error, some_error}), 97 | ?assertEqual(undefined, get_env(set_var)), 98 | ?assertEqual(default_value, get_env(set_var, default_value)), 99 | ErrCall = {Self, {?MOCK_MOD, parse_value, ["some_value"]}, {error, some_error}}, 100 | ?assertEqual([ErrCall, ErrCall], meck:history(?MOCK_MOD)), 101 | meck:reset(?MOCK_MOD), 102 | 103 | %% invalid parsing return value 104 | meck:expect(?MOCK_MOD, parse_value, ['_'], invalid_ret_value), 105 | ?assertEqual(undefined, get_env(set_var)), 106 | ?assertEqual(default_value, get_env(set_var, default_value)), 107 | InvRetValCall = {Self, {?MOCK_MOD, parse_value, ["some_value"]}, invalid_ret_value}, 108 | ?assertEqual([InvRetValCall, InvRetValCall], meck:history(?MOCK_MOD)), 109 | meck:reset(?MOCK_MOD), 110 | 111 | %% crash during parsing 112 | meck:expect(?MOCK_MOD, parse_value, fun(_) -> error(bang) end), 113 | ?assertEqual(undefined, get_env(set_var)), 114 | ?assertEqual(default_value, get_env(set_var, default_value)), 115 | ?assertMatch([{Self, {?MOCK_MOD, parse_value, ["some_value"]}, error, bang, _}, 116 | {Self, {?MOCK_MOD, parse_value, ["some_value"]}, error, bang, _}], 117 | meck:history(?MOCK_MOD)), 118 | meck:reset(?MOCK_MOD), 119 | 120 | %% unset and empty env variables (custom parsing module is not triggered) 121 | ?assertEqual(undefined, get_env(unset_var)), 122 | ?assertEqual(default_value, get_env(unset_var, default_value)), 123 | ?assertEqual(undefined, get_env(empty_var)), 124 | ?assertEqual(default_value, get_env(empty_var, default_value)), 125 | ?assertEqual([], meck:history(?MOCK_MOD)). 126 | 127 | invalid_custom_parser_test(_) -> 128 | %% testing invalid custom config parser module 129 | ?assertEqual(undefined, get_env(set_var)), 130 | ?assertEqual(default_value, get_env(set_var, default_value)), 131 | ?assertEqual(undefined, get_env(unset_var)), 132 | ?assertEqual(default_value, get_env(unset_var, default_value)), 133 | ?assertEqual(undefined, get_env(empty_var)), 134 | ?assertEqual(default_value, get_env(empty_var, default_value)). 135 | 136 | %%----------------------------------------------------------------------------------- 137 | %% helper functions 138 | %%----------------------------------------------------------------------------------- 139 | 140 | set_config_parser_module(Mod) -> 141 | application:set_env(amoc, config_parser_mod, Mod). 142 | 143 | unset_config_parser_module() -> 144 | application:unset_env(amoc, config_parser_mod). 145 | -------------------------------------------------------------------------------- /test/amoc_config_helper.erl: -------------------------------------------------------------------------------- 1 | -module(amoc_config_helper). 2 | 3 | -compile([export_all, nowarn_export_all]). 4 | 5 | set_os_env(Name, Value) -> 6 | os:putenv(env_name(Name), format_value(Value)). 7 | 8 | set_empty_os_env(Name) -> 9 | os:putenv(env_name(Name), ""). 10 | 11 | unset_os_env(Name) -> 12 | os:unsetenv(env_name(Name)). 13 | 14 | env_name(Name) -> 15 | "AMOC_" ++ string:uppercase(erlang:atom_to_list(Name)). 16 | 17 | get_env(Name) -> 18 | get_env(Name, undefined). 19 | 20 | get_env(Name, Default) -> 21 | amoc_config_env:get(Name, Default). 22 | 23 | format_value(Value) -> 24 | amoc_config_parser:format(Value, string). 25 | 26 | format_value(Value, Format) -> 27 | amoc_config_parser:format(Value, Format). 28 | -------------------------------------------------------------------------------- /test/amoc_config_verification_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(amoc_config_verification_SUITE). 2 | -include_lib("eunit/include/eunit.hrl"). 3 | -include("../src/config/amoc_config.hrl"). 4 | 5 | -compile(export_all). 6 | 7 | -import(amoc_config_helper, [set_os_env/2, 8 | unset_os_env/1, 9 | set_app_env/3, 10 | unset_app_env/2]). 11 | 12 | -define(MOD, ct). 13 | -define(APP, common_test). 14 | 15 | all() -> 16 | [process_scenario_config_uses_default_values, 17 | process_scenario_config_shadows_default_values, 18 | process_scenario_config_returns_error_for_invalid_values, 19 | process_scenario_config_returns_preprocessed_value]. 20 | 21 | process_scenario_config_uses_default_values(_) -> 22 | ScenarioConfig = correct_scenario_config(), 23 | given_scenario_parameters_not_set(ScenarioConfig), 24 | Result = amoc_config_verification:process_scenario_config(ScenarioConfig, []), 25 | ?assertEqual({ok, ScenarioConfig}, Result). 26 | 27 | process_scenario_config_shadows_default_values(_) -> 28 | ScenarioConfig = correct_scenario_config(), 29 | AnotherScenarioConfig = another_correct_scenario_config(), 30 | YetAnotherScenarioConfig = yet_another_correct_scenario_config(), 31 | %% set also os parameters 32 | given_scenario_parameters_set(AnotherScenarioConfig), 33 | Result2 = amoc_config_verification:process_scenario_config(ScenarioConfig, []), 34 | ?assertEqual({ok, AnotherScenarioConfig}, Result2), 35 | %% provide settings 36 | Settings = settings_from_scenario_config(YetAnotherScenarioConfig), 37 | Result3 = amoc_config_verification:process_scenario_config(ScenarioConfig, Settings), 38 | ?assertEqual({ok, YetAnotherScenarioConfig}, Result3), 39 | %% unset os parameters 40 | given_scenario_parameters_not_set(ScenarioConfig), 41 | Result4 = amoc_config_verification:process_scenario_config(ScenarioConfig, []), 42 | ?assertEqual({ok, ScenarioConfig}, Result4). 43 | 44 | process_scenario_config_returns_error_for_invalid_values(_) -> 45 | VerificationFn = fun(_) -> {false, some_reason} end, 46 | IncorrectScenarioConfig = 47 | [#module_parameter{name = wrong_param, mod = ?MOD, value = any_value, 48 | verification_fn = VerificationFn} 49 | | incorrect_scenario_config()], 50 | given_scenario_parameters_not_set(IncorrectScenarioConfig), 51 | Result = amoc_config_verification:process_scenario_config(IncorrectScenarioConfig, []), 52 | ?assertEqual({error, parameters_verification_failed, 53 | [{wrong_param, any_value, {verification_failed, some_reason}}, 54 | {some_int, wrong_int, verification_failed}, 55 | {some_atom, <<"wrong_atom">>, verification_failed}, 56 | {some_binary, "wrong_binary", verification_failed}]}, 57 | Result). 58 | 59 | process_scenario_config_returns_preprocessed_value(_) -> 60 | VerificationFn = fun(_) -> {true, another_atom} end, 61 | PreprocessedParam = #module_parameter{name = preprocessed_param, 62 | mod = ?MOD, value = an_atom, 63 | verification_fn = VerificationFn}, 64 | Result = amoc_config_verification:process_scenario_config([PreprocessedParam], []), 65 | ?assertEqual({ok, [PreprocessedParam#module_parameter{value = another_atom}]}, Result). 66 | 67 | correct_scenario_config() -> 68 | scenario_configuration(1, some_atom, <<"some_binary">>). 69 | 70 | another_correct_scenario_config() -> 71 | scenario_configuration(2, another_atom, <<"another_binary">>). 72 | 73 | yet_another_correct_scenario_config() -> 74 | scenario_configuration(3, yet_another_atom, <<"yet_another_binary">>). 75 | 76 | incorrect_scenario_config() -> 77 | scenario_configuration(wrong_int, <<"wrong_atom">>, "wrong_binary"). 78 | 79 | scenario_configuration(Int, Atom, Binary) -> 80 | [ 81 | #module_parameter{name = some_int, mod = ?MOD, value = Int, 82 | verification_fn = fun erlang:is_integer/1}, 83 | #module_parameter{name = some_atom, mod = ?MOD, value = Atom, 84 | verification_fn = fun erlang:is_atom/1}, 85 | #module_parameter{name = some_binary, mod = ?MOD, value = Binary, 86 | verification_fn = fun erlang:is_binary/1} 87 | ]. 88 | 89 | settings_from_scenario_config(ScenarioConfig) -> 90 | [{Name, Value} || #module_parameter{name = Name, value = Value} <- ScenarioConfig]. 91 | 92 | given_scenario_parameters_not_set(ScenarioConfig) -> 93 | [unset_os_env(Name) || #module_parameter{name = Name} <- ScenarioConfig]. 94 | 95 | given_scenario_parameters_set(ScenarioConfig) -> 96 | [set_os_env(Name, Value) 97 | || #module_parameter{name = Name, value = Value} <- ScenarioConfig]. 98 | -------------------------------------------------------------------------------- /test/telemetry_helpers.erl: -------------------------------------------------------------------------------- 1 | -module(telemetry_helpers). 2 | 3 | -define(HANDLER, telemetry_handler). 4 | -define(CONFIG, #{dummy_config => true}). 5 | -define(CONFIG_MATCH, #{dummy_config := true}). 6 | 7 | -export([start/1, reset/0, stop/0, get_calls/1]). 8 | 9 | start(TelemetryEvents) -> 10 | meck:new(?HANDLER, [non_strict, no_link]), 11 | meck:expect(?HANDLER, handler, ['_', '_', '_', '_'], ok), 12 | application:start(telemetry), 13 | TelemetryHandler = fun ?HANDLER:handler/4, 14 | telemetry:attach_many(?HANDLER, TelemetryEvents, TelemetryHandler, ?CONFIG). 15 | 16 | reset() -> 17 | meck:reset(?HANDLER). 18 | 19 | stop() -> 20 | meck:unload(?HANDLER). 21 | 22 | get_calls(Prefix) -> 23 | History = meck:history(?HANDLER), 24 | Filter = fun({_Pid, Call, _Ret}) -> 25 | {?HANDLER, handler, 26 | [EventName, Measurements, Metadata, ?CONFIG_MATCH]} = Call, 27 | lists:prefix(Prefix, EventName) 28 | andalso {true, {EventName, Measurements, Metadata}} 29 | end, 30 | lists:filtermap(Filter, History). 31 | -------------------------------------------------------------------------------- /test/test_helpers.erl: -------------------------------------------------------------------------------- 1 | -module(test_helpers). 2 | 3 | -compile([export_all, nowarn_export_all]). 4 | 5 | wait_until_scenario_has_users(Scenario, Current, HighestId) -> 6 | wait_until_scenario_has_users(Scenario, Current, HighestId, #{}). 7 | 8 | wait_until_scenario_has_users(Scenario, Current, HighestId, ExtraConfig) -> 9 | WaitUntilFun = fun amoc_controller:get_status/0, 10 | WaitUntilValue = {running, #{scenario => Scenario, 11 | currently_running_users => Current, 12 | highest_user_id => HighestId}}, 13 | wait_helper:wait_until(WaitUntilFun, WaitUntilValue, ExtraConfig). 14 | 15 | all_vars() -> 16 | [{interarrival, 1}, {testing_var1, def1}, 17 | {config_scenario_var1, unused_value}]. 18 | 19 | regular_vars() -> 20 | [{interarrival, 1}, {testing_var1, def1}]. 21 | 22 | all_vars_with_state() -> 23 | [{interarrival, 1}, {testing_state_var1, def1}, 24 | {config_scenario_var1, unused_value}]. 25 | 26 | regular_vars_with_state() -> 27 | [{interarrival, 1}, {testing_state_var1, def1}]. 28 | 29 | other_vars_to_keep_quiet() -> 30 | [{config_scenario_var1, unused_value}]. 31 | -------------------------------------------------------------------------------- /test/testing_scenario.erl: -------------------------------------------------------------------------------- 1 | -module(testing_scenario). 2 | 3 | -behaviour(amoc_scenario). 4 | 5 | -required_variable(#{name => testing_var1, description => "description1", 6 | verification => [def1, another_value]}). 7 | 8 | %% amoc_scenario behaviour 9 | -export([init/0, start/1, terminate/0]). 10 | 11 | -spec init() -> ok. 12 | init() -> 13 | ok. 14 | 15 | -spec start(amoc_scenario:user_id()) -> any(). 16 | start(_Id) -> 17 | %% Wait for any message to be send 18 | receive 19 | Msg -> 20 | ct:pal("Msg ~p~n", [Msg]), 21 | timer:sleep(100), 22 | Msg 23 | end. 24 | 25 | -spec terminate() -> term(). 26 | terminate() -> 27 | ok. 28 | -------------------------------------------------------------------------------- /test/testing_scenario_with_error_in_init.erl: -------------------------------------------------------------------------------- 1 | -module(testing_scenario_with_error_in_init). 2 | 3 | -behaviour(amoc_scenario). 4 | 5 | -required_variable(#{name => testing_var1, description => "description1", 6 | verification => [def1, another_value]}). 7 | 8 | %% amoc_scenario behaviour 9 | -export([init/0, start/1, terminate/0]). 10 | 11 | -spec init() -> term(). 12 | init() -> 13 | {error, error}. 14 | 15 | -spec start(amoc_scenario:user_id()) -> any(). 16 | start(_Id) -> 17 | %% Wait for any message to be send 18 | receive 19 | Msg -> 20 | ct:pal("Msg ~p~n", [Msg]), 21 | timer:sleep(100), 22 | Msg 23 | end. 24 | 25 | -spec terminate() -> term(). 26 | terminate() -> 27 | ok. 28 | -------------------------------------------------------------------------------- /test/testing_scenario_with_state.erl: -------------------------------------------------------------------------------- 1 | -module(testing_scenario_with_state). 2 | 3 | -behaviour(amoc_scenario). 4 | 5 | -required_variable(#{name => testing_state_var1, description => "description1", 6 | verification => [def1, another_value]}). 7 | 8 | %% amoc_scenario behaviour 9 | -export([init/0, start/2, terminate/1]). 10 | 11 | -type state() :: #{_ := _}. 12 | -export_type([state/0]). 13 | 14 | -spec init() -> {ok, state()}. 15 | init() -> 16 | {ok, #{some_state => this_has_state}}. 17 | 18 | -spec start(amoc_scenario:user_id(), state()) -> any(). 19 | start(Id, #{some_state := this_has_state} = State) -> 20 | %% Wait for anymessage to be send 21 | receive 22 | {'EXIT', _, _} -> 23 | start(Id, State); 24 | Msg -> 25 | ct:pal("Msg ~p~n", [Msg]), 26 | timer:sleep(100), 27 | Msg 28 | end. 29 | 30 | -spec terminate(state()) -> any(). 31 | terminate(#{some_state := GiveState}) -> 32 | throw(GiveState). 33 | -------------------------------------------------------------------------------- /test/testing_scenario_with_throttle.erl: -------------------------------------------------------------------------------- 1 | -module(testing_scenario_with_throttle). 2 | 3 | -behaviour(amoc_scenario). 4 | 5 | -required_variable(#{name => testing_var1, description => "description1", 6 | verification => [def1, another_value]}). 7 | 8 | %% amoc_scenario behaviour 9 | -export([init/0, start/1, terminate/0]). 10 | 11 | -spec init() -> ok. 12 | init() -> 13 | ok. 14 | 15 | -spec start(amoc_scenario:user_id()) -> any(). 16 | start(_Id) -> 17 | %% Wait for any message to be send 18 | receive 19 | Msg -> 20 | ct:pal("Msg ~p~n", [Msg]), 21 | timer:sleep(100), 22 | Msg 23 | end. 24 | 25 | -spec terminate() -> term(). 26 | terminate() -> 27 | ok. 28 | -------------------------------------------------------------------------------- /test/testing_scenario_without_callbacks.erl: -------------------------------------------------------------------------------- 1 | -module(testing_scenario_without_callbacks). 2 | 3 | -behaviour(amoc_scenario). 4 | 5 | -required_variable(#{name => testing_var1, description => "description1", 6 | verification => [def1, another_value]}). 7 | 8 | %% amoc_scenario behaviour 9 | -export([init/0, terminate/0]). 10 | 11 | -spec init() -> ok. 12 | init() -> 13 | ok. 14 | 15 | -spec terminate() -> term(). 16 | terminate() -> 17 | ok. 18 | --------------------------------------------------------------------------------