├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── elvis.config ├── rebar.config ├── rebar.lock ├── src ├── gen_buffer.app.src ├── gen_buffer.erl ├── gen_buffer_app.erl ├── gen_buffer_dist.erl ├── gen_buffer_lib.erl ├── gen_buffer_sup.erl └── gen_buffer_worker.erl └── test ├── gen_buffer_SUITE.erl ├── gen_buffer_dist_SUITE.erl ├── gen_buffer_meta_SUITE.erl ├── gen_buffer_partitioned_SUITE.erl ├── shared └── gen_buffer_test_cases.erl └── support ├── gen_buffer_ct.erl └── handlers ├── test_message_handler.erl ├── test_message_handler2.erl ├── test_message_handler3.erl ├── test_message_handler4.erl ├── test_message_handler5.erl └── test_message_handler6.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .rebar3 2 | _* 3 | .eunit 4 | *.o 5 | *.beam 6 | *.plt 7 | *.swp 8 | *.swo 9 | .erlang.cookie 10 | ebin 11 | log 12 | erl_crash.dump 13 | .rebar 14 | _rel 15 | _deps 16 | _plugins 17 | _tdeps 18 | logs 19 | _build 20 | .DS_Store 21 | ._* 22 | _* 23 | *~ 24 | .idea 25 | *.iml 26 | log 27 | logs 28 | data 29 | *.sh 30 | edoc 31 | priv/*.so 32 | *.crashdump 33 | *_plt 34 | doc 35 | docs 36 | rebar3 37 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | otp_release: 3 | - 21.3 4 | - 20.3 5 | before_script: 6 | - epmd -daemon 7 | script: 8 | - make ci 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Carlos Bolanos. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## Taken from https://github.com/cabol/shards 2 | 3 | REBAR = $(shell which rebar3) 4 | 5 | EPMD_PROC_NUM = $(shell ps -ef | grep epmd | grep -v "grep") 6 | 7 | LOCAL_SUITES = "test/gen_buffer_SUITE,test/gen_buffer_partitioned_SUITE" 8 | 9 | .PHONY: all check_rebar compile clean distclean dialyzer test shell doc 10 | 11 | all: check_rebar compile 12 | 13 | compile: check_rebar 14 | $(REBAR) compile 15 | 16 | clean: check_rebar 17 | rm -rf ebin/* test/*.beam logs log 18 | $(REBAR) clean 19 | 20 | distclean: clean 21 | $(REBAR) clean --all 22 | rm -rf _build logs log doc *.dump *_plt *.crashdump priv 23 | 24 | dialyzer: check_rebar 25 | $(REBAR) dialyzer 26 | 27 | test: check_rebar check_epmd check_plt 28 | $(REBAR) do ct, cover 29 | 30 | local_test: check_rebar check_epmd 31 | $(REBAR) do ct --suite=$(LOCAL_SUITES), cover 32 | 33 | dist_test: check_rebar check_epmd 34 | $(REBAR) do ct --suite=test/gen_buffer_dist_SUITE, cover 35 | 36 | ## The option '--readable=false' is added due to the compatibility issue of rebar3 with OTP 21 37 | ci: check_epmd check_plt 38 | $(call get_rebar) 39 | $(REBAR) as test,ci do ct --readable=false, cover 40 | rm -rf rebar3 41 | 42 | shell: check_rebar 43 | $(REBAR) shell 44 | 45 | doc: check_rebar 46 | $(REBAR) edoc 47 | 48 | check_rebar: 49 | ifeq ($(REBAR),) 50 | ifeq ($(wildcard rebar3),) 51 | $(call get_rebar) 52 | else 53 | $(eval REBAR=./rebar3) 54 | endif 55 | endif 56 | 57 | check_plt: 58 | ifeq (,$(wildcard ./*_plt)) 59 | @echo " ---> Running dialyzer ..." 60 | $(REBAR) dialyzer 61 | endif 62 | 63 | check_epmd: 64 | ifeq ($(EPMD_PROC_NUM),) 65 | epmd -daemon 66 | @echo " ---> Started epmd!" 67 | endif 68 | 69 | define get_rebar 70 | curl -O https://s3.amazonaws.com/rebar3/rebar3 71 | chmod a+x rebar3 72 | ./rebar3 update 73 | $(eval REBAR=./rebar3) 74 | endef 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gen_buffer 2 | > ### High scalable message buffer for Erlang/Elixir 3 | > A generic message buffer behaviour with support for pooling, back-pressure, 4 | sharding, and distribution. 5 | 6 | [![Build Status](https://travis-ci.org/cabol/gen_buffer.svg?branch=master)](https://travis-ci.org/cabol/gen_buffer) 7 | 8 | ## Overview 9 | 10 | The `gen_buffer` can be illustrated as: 11 | 12 | ``` 13 | |--[gen_buffer_worker] 14 | | 15 | +--[gen_buffer_sup]--+ |--[gen_buffer_worker] 16 | | | | 17 | | |--|--[gen_buffer_worker] -> 18 | | | | 19 | +--------------------+ |--[gen_buffer_worker] 20 | ^ | 21 | | |--[gen_buffer_worker] 22 | (enqueue message) | ^ 23 | | | (sends message to worker) 24 | [gen_buffer]--------------------------+ 25 | ^ (worker available?) 26 | | 27 | messages 28 | ``` 29 | 30 | Some implementation notes: 31 | 32 | * A buffer is represented by its own supervision tree. The main supervisor 33 | creates the buffer itself (using an ETS table) and a pool of workers for it; 34 | each worker is a `gen_server`. 35 | 36 | * At the moment a message is sent to the buffer, it tries to find an available 37 | worker (`gen_server`) and then dispatch the message directly to it. If there 38 | is not any available worker, the message is stored into a buffer data struct 39 | created and handled using [ets_buffer][ets_buffer]. Once a worker becomes 40 | available, those buffered messages are processed. 41 | 42 | * In order to get better and/or higher scalability, the buffer can be 43 | partitioned, it supports sharding under-the-hood. Therefore, in a partitioned 44 | buffer, the incoming messages are spread across the configured partitions 45 | (for load balancing); you can configure the desired number of partitions with 46 | the option `n_partitions` when you start the buffer for the first time. 47 | 48 | * It is also possible to run the buffer in a distributed fashion on multiple 49 | nodes using the `gen_buffer_dist` module instead. 50 | 51 | [ets_buffer]: https://github.com/duomark/epocxy/blob/master/src/ets_buffer.erl 52 | 53 | ## Installation 54 | 55 | ### Erlang 56 | 57 | In your `rebar.config`: 58 | 59 | ```erlang 60 | {deps, [ 61 | {gen_buffer, {git, "https://github.com/cabol/gen_buffer.git", {branch, "master"}}} 62 | ]}. 63 | ``` 64 | 65 | ### Elixir 66 | 67 | In your `mix.exs`: 68 | 69 | ```elixir 70 | def deps do 71 | [{:gen_buffer, github: "cabol/gen_buffer", branch: "master"}] 72 | end 73 | ``` 74 | 75 | ## Usage 76 | 77 | ### Creating message buffers 78 | 79 | First of all, we have to create a message handler module: 80 | 81 | ```erlang 82 | -module(my_message_handler). 83 | 84 | -behaviour(gen_buffer). 85 | 86 | %% gen_buffer callbacks 87 | -export([ 88 | init/1, 89 | handle_message/3 90 | ]). 91 | 92 | %%%=================================================================== 93 | %%% gen_buffer callbacks 94 | %%%=================================================================== 95 | 96 | %% @hidden 97 | init(_Args) -> 98 | % initialize your handler state 99 | {ok, #{}}. 100 | 101 | %% @hidden 102 | handle_message(Buffer, Message, State) -> 103 | % your logic to process incoming messages goes here 104 | Response = {Buffer, Message}, 105 | {ok, Response, State}. 106 | 107 | %% Optionally you can implement `handle_info/3` and `terminate/3` 108 | ``` 109 | 110 | Now we can create our buffer calling `gen_buffer:start_link(BufferName, Opts)` 111 | 112 | ```erlang 113 | gen_buffer:start_link(my_buffer, #{message_handler => my_message_handler}). 114 | ``` 115 | 116 | It is also possible to start the buffer as part of your supervision tree in your 117 | app. In your supervisor, within the `init/1` callback, you can add the buffer 118 | spec to the supervisor's children list: 119 | 120 | ```erlang 121 | %% @hidden 122 | init(_) -> 123 | Children = [ 124 | gen_buffer:child_spec(#{buffer => my_buffer, message_handler => my_message_handler}) 125 | ], 126 | 127 | {ok, {{one_for_one, 0, 1}, Children}}. 128 | ``` 129 | 130 | > You can run `observer:start()` to see how the buffer looks like. 131 | 132 | ### Options for buffer creation 133 | 134 | The following are the options for `gen_buffer:start_link/2`: 135 | 136 | Option | Description | Required | Default 137 | :----- | :---------- | :------- | ------- 138 | `message_handler` | Message handler module that implements the `gen_buffer` behaviour | YES | NA 139 | `init_args` | Optional arguments passed to `init/1` callback when a worker starts | NO | `undefined` 140 | `workers` | Number of workers | NO | `erlang:system_info(schedulers_online)` 141 | `send_replies` | Determines whether or not to reply with the result of the `handle_message` to the given process when the function `send/2,3` is called | NO | `false` 142 | `buffer_type` | Buffer type according to [ets_buffer][ets_buffer]. Possible values: `ring`, `fifo`, `lifo` | NO | `fifo` 143 | `buffer` | Buffer name. This option is only required for `gen_buffer:child_spec/1` function | NO | NA 144 | `n_partitions` | The number of partitions for the buffer. The load will be balanced across the defined partitions and each partition has its own pool of workers. | NO | `1` 145 | 146 | ### Sending messages 147 | 148 | Messaging is asynchronous by nature, as well as `gen_buffer`, then when you send 149 | a message to a buffer, it is dispatched to another process to be processed 150 | asynchronously via your message handler. If you want to receive the result of 151 | your message handler, you have to start the buffer with `send_replies` option 152 | to `true`. For sending messages we use the function `gen_buffer:send/2,3` as 153 | follows: 154 | 155 | ```erlang 156 | % if option send_replies has been set to true, the buffer sends the reply to 157 | % the caller process 158 | Ref1 = gen_buffer:send(my_buffer, "hello"). 159 | 160 | % or you can specify explicitly to what process the buffer should reply to; 161 | % but send_replies has to be set to true 162 | Ref2 = gen_buffer:send(my_buffer, "hello", ReplyToThisPID). 163 | ``` 164 | 165 | ### Receiving replies 166 | 167 | When we send a message calling `gen_buffer:send/2,3`, a reference is returned 168 | and it can be used to receive the reply, like so: 169 | 170 | ```erlang 171 | % this is a blocking call that waits for your reply for 5000 milliseconds 172 | % by default. If none reply is received during that time, {error, timeout} 173 | % is returned 174 | gen_buffer:recv(my_buffer, Ref). 175 | 176 | % or you can pass the timeout explicitly 177 | gen_buffer:recv(my_buffer, Ref, 1000). 178 | ``` 179 | 180 | ### Sending messages and receiving replies in the same call 181 | 182 | There is a function `gen_buffer:send_recv/2,3` which combines the previous 183 | two functions in one, meaning that, when you send a message using this function, 184 | it gets blocked until the reply arrives or until the timeout expires. 185 | 186 | ```erlang 187 | % by default, the timeout is 5000 milliseconds 188 | gen_buffer:send_recv(my_buffer, "hello"). 189 | 190 | % or you can pass it explicitly 191 | gen_buffer:send_recv(my_buffer, "hello", 1000). 192 | ``` 193 | 194 | ### Increase/decrease number of workers dynamically (for throttling purposes) 195 | 196 | It is also possible to increase or decrease the number of workers in runtime for 197 | traffic throttling. Controlling the number of workers we can control the 198 | throughput and/or processing rate too. 199 | 200 | ```erlang 201 | % adding one worker inheriting the initial options at startup time 202 | gen_buffer:incr_worker(my_buffer). 203 | 204 | % passing/changing the options 205 | gen_buffer:incr_worker(my_buffer, OtherOpts). 206 | 207 | % removing one worker (a random worker) 208 | gen_buffer:decr_worker(my_buffer). 209 | 210 | % adding 5 workers at once inheriting the initial options at startup time 211 | gen_buffer:set_workers(my_buffer, 5). 212 | 213 | % passing/changing the options 214 | gen_buffer:set_workers(my_buffer, 5, OtherOpts). 215 | ``` 216 | 217 | ### Getting buffer info 218 | 219 | There are several functions to get info about the buffer, such as: 220 | 221 | ```erlang 222 | % getting the buffer size (number of buffered messages) 223 | gen_buffer:size(my_buffer). 224 | 225 | % getting all available info about the buffer 226 | % check gen_buffer:buffer_info() typespec 227 | gen_buffer:info(my_buffer) 228 | 229 | % getting all available info about all created buffers 230 | % check gen_buffer:buffers_info() typespec 231 | gen_buffer:info() 232 | 233 | % getting one random worker PID 234 | gen_buffer:get_worker(my_buffer) 235 | 236 | % getting all buffer workers (list of PIDs) 237 | gen_buffer:get_worker(my_buffer) 238 | ``` 239 | 240 | ## Evaluating message handler logic directly 241 | 242 | This is not recommended since it breaks the essence of messaging; remember we 243 | mentioned before the messaging is asynchronous by nature. Nevertheless, there is 244 | a way to execute the `handle_message` logic for a message bypassing the normal 245 | path, it just finds a worker and executing the logic directly with it. 246 | 247 | ```erlang 248 | % if the evaluation fails, it retries one more time by default 249 | gen_buffer:eval(my_buffer, "hello") 250 | 251 | % or you can pass the desired number of retries 252 | gen_buffer:eval(my_buffer, "hello", 5) 253 | ``` 254 | 255 | > The main use cases are for testing, debugging, etc. 256 | 257 | ## Testing 258 | 259 | ``` 260 | $ make test 261 | ``` 262 | 263 | You can find tests results in `_build/test/logs`, and coverage in 264 | `_build/test/cover`. 265 | 266 | > **NOTE:** `gen_buffer` comes with a helper `Makefile`, but it is just a simple 267 | wrapper on top of `rebar3`, therefore, you can run the tests using `rebar3` 268 | directly, like so: `rebar3 do ct, cover`. 269 | 270 | ## Building Edoc 271 | 272 | ``` 273 | $ make doc 274 | ``` 275 | 276 | > **NOTE:** Once you run the previous command, a new folder `doc` is created, 277 | and you'll have a pretty nice HTML documentation. 278 | 279 | ## Copyright and License 280 | 281 | Copyright (c) 2019 Carlos Bolanos. 282 | 283 | **gen_buffer** source code is licensed under the [MIT License](LICENSE.md). 284 | -------------------------------------------------------------------------------- /elvis.config: -------------------------------------------------------------------------------- 1 | [ 2 | {elvis, [ 3 | {config, [ 4 | #{ 5 | dirs => ["src/*"], 6 | filter => "*.erl", 7 | rules => [ 8 | {elvis_style, invalid_dynamic_call, disable}, 9 | {elvis_style, dont_repeat_yourself, disable}, 10 | {elvis_style, macro_names, disable}, 11 | {elvis_style, line_length, #{limit => 100}}, 12 | {elvis_style, god_modules, #{limit => 30}} 13 | ], 14 | ruleset => erl_files 15 | }, 16 | #{ 17 | dirs => ["."], 18 | filter => "rebar.config", 19 | ruleset => rebar_config 20 | }, 21 | #{ 22 | dirs => ["."], 23 | filter => "elvis.config", 24 | ruleset => elvis_config 25 | } 26 | ]} 27 | ]} 28 | ]. 29 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% == Erlang Compiler == 2 | 3 | {minimum_otp_vsn, "20.0"}. 4 | 5 | {erl_opts, [ 6 | debug_info, 7 | warnings_as_errors, 8 | warn_unused_vars, 9 | ewarn_export_all, 10 | warn_shadow_vars, 11 | warn_unused_import, 12 | warn_unused_function, 13 | warn_bif_clash, 14 | warn_unused_record, 15 | warn_deprecated_function, 16 | warn_obsolete_guard, 17 | strict_validation, 18 | warn_export_vars, 19 | warn_exported_vars, 20 | warn_untyped_record 21 | ]}. 22 | 23 | %% == Dependencies == 24 | 25 | {deps, [ 26 | {epocxy, "1.1.0"} 27 | ]}. 28 | 29 | %% == Profiles == 30 | 31 | {profiles, [ 32 | {test, [ 33 | {deps, [ 34 | {mixer, "0.1.5", {pkg, inaka_mixer}}, 35 | {katana, "0.4.0"}, 36 | {katana_test, "1.0.1"} 37 | ]}, 38 | {cover_excl_mods, [ 39 | gen_buffer_sup, 40 | gen_buffer_app, 41 | gen_buffer_test_cases, 42 | test_message_handler, 43 | test_message_handler2, 44 | test_message_handler3, 45 | test_message_handler4, 46 | test_message_handler5, 47 | test_message_handler6, 48 | gen_buffer_ct, 49 | gen_buffer_SUITE, 50 | gen_buffer_partitioned_SUITE, 51 | gen_buffer_dist_SUITE 52 | ]} 53 | ]}, 54 | {ci, [ 55 | {erl_opts, [{d, 'CI'}]} 56 | ]} 57 | ]}. 58 | 59 | %% == Cover == 60 | 61 | {cover_enabled, true}. 62 | 63 | {cover_opts, [verbose]}. 64 | 65 | %% == Common Test == 66 | 67 | {ct_compile_opts, [ 68 | debug_info, 69 | warnings_as_errors, 70 | warn_unused_vars, 71 | ewarn_export_all, 72 | warn_shadow_vars, 73 | warn_unused_import, 74 | warn_unused_function, 75 | warn_bif_clash, 76 | warn_unused_record, 77 | warn_deprecated_function, 78 | warn_obsolete_guard, 79 | strict_validation, 80 | warn_export_vars, 81 | warn_exported_vars, 82 | warn_untyped_record 83 | ]}. 84 | 85 | %% == EDoc == 86 | 87 | {edoc_opts, []}. 88 | 89 | %% == Dialyzer == 90 | 91 | {dialyzer, [ 92 | {warnings, [ 93 | no_return, 94 | unmatched_returns, 95 | error_handling, 96 | unknown 97 | ]}, 98 | {plt_apps, top_level_deps}, 99 | {plt_extra_apps, [epocxy]}, 100 | {plt_location, local}, 101 | {plt_prefix, "gen_buffer"}, 102 | {base_plt_location, "."}, 103 | {base_plt_prefix, "gen_buffer"} 104 | ]}. 105 | 106 | %% == Shell == 107 | 108 | {shell, [{apps, [gen_buffer]}]}. 109 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.1.0", 2 | [{<<"epocxy">>,{pkg,<<"epocxy">>,<<"1.1.0">>},0}]}. 3 | [ 4 | {pkg_hash,[ 5 | {<<"epocxy">>, <<"125E8CDD2346A807530F0510EA0A5022E63874B3FCDCE4DD80E16F38717A044E">>}]} 6 | ]. 7 | -------------------------------------------------------------------------------- /src/gen_buffer.app.src: -------------------------------------------------------------------------------- 1 | {application, gen_buffer, [ 2 | {description, "Generic message buffer behaviour with back-pressure and pooling for Erlang/Elixir."}, 3 | {vsn, "0.1.0"}, 4 | {registered, []}, 5 | {mod, {gen_buffer_app, []}}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | epocxy 10 | ]}, 11 | {env,[]}, 12 | {modules, []}, 13 | {maintainers, [ 14 | "Carlos A. Bolanos" 15 | ]}, 16 | {links, []} 17 | ]}. 18 | -------------------------------------------------------------------------------- /src/gen_buffer.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @doc 3 | %%% gen_buffer. 4 | %%% @end 5 | %%%------------------------------------------------------------------- 6 | -module(gen_buffer). 7 | 8 | %% API 9 | -export([ 10 | start_link/2, 11 | stop/1, 12 | child_spec/1, 13 | eval/2, 14 | eval/3, 15 | send/2, 16 | send/3, 17 | poll/1, 18 | recv/2, 19 | recv/3, 20 | send_recv/2, 21 | send_recv/3, 22 | get_worker/1, 23 | get_workers/1, 24 | incr_worker/1, 25 | incr_worker/2, 26 | decr_worker/1, 27 | set_workers/2, 28 | set_workers/3, 29 | size/1, 30 | info/1, 31 | info/0 32 | ]). 33 | 34 | %% Global Utilities 35 | -export([ 36 | pg2_namespace/1 37 | ]). 38 | 39 | %%%=================================================================== 40 | %%% Types 41 | %%%=================================================================== 42 | 43 | -type t() :: atom(). 44 | 45 | -type message() :: any(). 46 | 47 | -type opt() :: {buffer, atom()} 48 | | {message_handler, module()} 49 | | {workers, non_neg_integer()} 50 | | {buffer_type, ring | fifo | lifo} 51 | | {buffer_size, non_neg_integer()} 52 | | {send_replies, boolean()} 53 | | {init_args, any()}. 54 | 55 | -type opts_map() :: #{ 56 | buffer => atom(), 57 | message_handler => module(), 58 | workers => non_neg_integer(), 59 | buffer_type => ring | fifo | lifo, 60 | buffer_size => non_neg_integer(), 61 | send_replies => boolean(), 62 | init_args => any() 63 | }. 64 | 65 | -type opts() :: opts_map() | [opt()]. 66 | 67 | -type buffer_info() :: #{ 68 | workers := non_neg_integer(), 69 | size := non_neg_integer() 70 | }. 71 | 72 | -type buffers_info() :: #{ 73 | t() => buffer_info() 74 | }. 75 | 76 | -type init_res() :: {ok, State :: any()} 77 | | {ok, State :: any(), Timeout :: timeout()} 78 | | {ok, State :: any(), hibernate} 79 | | {stop, Reason :: any()} 80 | | ignore. 81 | 82 | -type handle_info_res() :: {noreply, NewState :: any()} 83 | | {noreply, NewState :: any(), Timeout :: timeout()} 84 | | {noreply, NewState :: any(), hibernate} 85 | | {stop, Reason :: normal | any(), NewState :: any()}. 86 | 87 | -type terminate_reason() :: normal | shutdown | {shutdown, any()} | any(). 88 | 89 | -export_type([ 90 | t/0, 91 | message/0, 92 | opts/0, 93 | opts_map/0, 94 | buffer_info/0, 95 | buffers_info/0 96 | ]). 97 | 98 | %%%=================================================================== 99 | %%% Callbacks 100 | %%%=================================================================== 101 | 102 | -optional_callbacks([ 103 | init/1, 104 | handle_info/3, 105 | terminate/3 106 | ]). 107 | 108 | -callback init(Args :: any()) -> init_res(). 109 | 110 | -callback handle_message( 111 | Buffer :: gen_buffer:t(), 112 | Message :: gen_buffer:message(), 113 | State :: any() 114 | ) -> {ok, Reply :: any(), NewState :: any()}. 115 | 116 | -callback handle_info( 117 | Buffer :: gen_buffer:t(), 118 | Info :: any(), 119 | State :: any() 120 | ) -> handle_info_res(). 121 | 122 | -callback terminate( 123 | Buffer :: gen_buffer:t(), 124 | Reason :: terminate_reason(), 125 | State :: any() 126 | ) -> any(). 127 | 128 | %%%=================================================================== 129 | %%% API 130 | %%%=================================================================== 131 | 132 | -spec start_link( 133 | Buffer :: t(), 134 | Opts :: opts() 135 | ) -> {ok, pid()} | {error, term()}. 136 | start_link(Buffer, Opts) -> 137 | gen_buffer_sup:start_link(Buffer, Opts). 138 | 139 | -spec stop(Buffer :: t()) -> ok. 140 | stop(Buffer) -> 141 | case whereis(Buffer) of 142 | undefined -> ok; 143 | Pid -> gen:stop(Pid, normal, infinity) 144 | end. 145 | 146 | -spec child_spec(Opts :: opts()) -> supervisor:child_spec(). 147 | child_spec(Opts) when is_list(Opts) -> 148 | child_spec(maps:from_list(Opts)); 149 | 150 | child_spec(Opts) when is_map(Opts) -> 151 | Buffer = maps:get(buffer, Opts), 152 | 153 | #{ 154 | id => Buffer, 155 | start => {?MODULE, start_link, [Buffer, Opts]}, 156 | type => supervisor 157 | }. 158 | 159 | %% @equiv eval(Buffer, Message, 1) 160 | eval(Buffer, Message) -> 161 | eval(Buffer, Message, 1). 162 | 163 | -spec eval( 164 | Buffer :: t(), 165 | Message :: any(), 166 | Retries :: non_neg_integer() 167 | ) -> any(). 168 | eval(Buffer, Message, Retries) when is_integer(Retries); Retries >= 0 -> 169 | case gen_buffer_lib:get_available_worker(Buffer) of 170 | no_available_workers -> 171 | maybe_retry(Buffer, Message, Retries - 1); 172 | 173 | Worker -> 174 | gen_buffer_worker:eval(Worker, Message) 175 | end. 176 | 177 | %% @equiv send(Buffer, Message, self()) 178 | send(Buffer, Message) -> 179 | send(Buffer, Message, self()). 180 | 181 | -spec send( 182 | Buffer :: t(), 183 | Message :: any(), 184 | ReplyTo :: pid() | atom() | {atom(), node()} 185 | ) -> reference(). 186 | send(Buffer, Message, ReplyTo) -> 187 | Ref = make_ref(), 188 | WorkerMessage = {ReplyTo, Ref, Message}, 189 | 190 | case gen_buffer_lib:get_available_worker(Buffer) of 191 | no_available_workers -> 192 | BufferPartition = gen_buffer_lib:get_partition(Buffer), 193 | _ = ets_buffer:write_dedicated(BufferPartition, WorkerMessage), 194 | Ref; 195 | 196 | Worker -> 197 | ok = gen_buffer_worker:send(Worker, WorkerMessage), 198 | Ref 199 | end. 200 | 201 | -spec poll(Buffer :: t()) -> ok | {error, term()}. 202 | poll(Buffer) when is_atom(Buffer) -> 203 | case get_worker(Buffer) of 204 | {ok, Worker} -> 205 | gen_buffer_worker:poll(Worker); 206 | 207 | Error -> 208 | Error 209 | end. 210 | 211 | %% @equiv recv(Buffer, Ref, infinity) 212 | recv(Buffer, Ref) -> 213 | recv(Buffer, Ref, infinity). 214 | 215 | -spec recv( 216 | Buffer :: t(), 217 | Ref :: reference(), 218 | Timeout :: timeout() 219 | ) -> {ok, HandlerResponse :: any()} | {error, Reason :: any()}. 220 | recv(Buffer, Ref, Timeout) -> 221 | receive 222 | {reply, Ref, Buffer, Payload} -> 223 | {ok, Payload}; 224 | 225 | {error, Ref, Buffer, Error} -> 226 | {error, Error} 227 | after 228 | Timeout -> {error, timeout} 229 | end. 230 | 231 | %% @equiv send_recv(Buffer, Message, infinity) 232 | send_recv(Buffer, Message) -> 233 | send_recv(Buffer, Message, infinity). 234 | 235 | -spec send_recv( 236 | Buffer :: t(), 237 | Message :: any(), 238 | Timeout :: timeout() 239 | ) -> {ok, HandlerResponse :: any()} | {error, Reason :: any()}. 240 | send_recv(Buffer, Message, Timeout) -> 241 | Ref = send(Buffer, Message), 242 | recv(Buffer, Ref, Timeout). 243 | 244 | -spec get_worker(Buffer :: t()) -> {ok, pid()} | {error, term()}. 245 | get_worker(Buffer) -> 246 | case get_children(Buffer) of 247 | {ok, Children} -> 248 | Nth = erlang:phash2(os:timestamp(), length(Children)) + 1, 249 | {_, Pid, _, _} = lists:nth(Nth, Children), 250 | {ok, Pid}; 251 | 252 | Error -> 253 | Error 254 | end. 255 | 256 | -spec get_workers(Buffer :: t()) -> {ok, [pid()]} | {error, term()}. 257 | get_workers(Buffer) -> 258 | case get_children(Buffer) of 259 | {ok, Children} -> 260 | {ok, [Pid || {_, Pid, _, _} <- Children]}; 261 | 262 | Error -> 263 | Error 264 | end. 265 | 266 | %% @equiv incr_worker(Buffer, #{}) 267 | incr_worker(Buffer) -> 268 | incr_worker(Buffer, #{}). 269 | 270 | -spec incr_worker( 271 | Buffer :: t(), 272 | Opts :: opts() 273 | ) -> supervisor:startchild_ret(). 274 | incr_worker(Buffer, Opts) -> 275 | gen_buffer_sup:start_child(Buffer, Opts). 276 | 277 | -spec decr_worker(Buffer :: t()) -> ok. 278 | decr_worker(Buffer) -> 279 | gen_buffer_sup:terminate_child(Buffer). 280 | 281 | %% @equiv set_workers(Buffer, N, #{}) 282 | set_workers(Buffer, N) -> 283 | set_workers(Buffer, N, #{}). 284 | 285 | -spec set_workers( 286 | Buffer :: t(), 287 | N :: non_neg_integer(), 288 | Opts :: opts() 289 | ) -> {ok, [pid()]} | {error, term()}. 290 | set_workers(Buffer, N, Opts) when N > 0 -> 291 | try 292 | NumWorkers = 293 | case get_workers(Buffer) of 294 | {ok, Workers} -> 295 | length(Workers); 296 | 297 | {error, no_available_workers} -> 298 | 0; 299 | 300 | Error -> 301 | throw(Error) 302 | end, 303 | 304 | {Fun, Diff} = 305 | case N >= NumWorkers of 306 | true -> {fun incr_worker/2, N - NumWorkers}; 307 | false -> {fun decr_worker/1, NumWorkers - N} 308 | end, 309 | 310 | ok = 311 | lists:foreach(fun 312 | (_) when is_function(Fun, 2) -> Fun(Buffer, Opts); 313 | (_) when is_function(Fun, 1) -> Fun(Buffer) 314 | end, lists:seq(1, Diff)), 315 | 316 | get_workers(Buffer) 317 | catch 318 | throw:Ex -> Ex 319 | end. 320 | 321 | -spec size(Buffer :: t()) -> integer() | undefined. 322 | size(Buffer) -> 323 | case whereis(Buffer) of 324 | undefined -> 325 | undefined; 326 | 327 | _ -> 328 | % we have to subtract 1 because ets_buffer uses one entry for metadata 329 | Partitions = gen_buffer_lib:get_one_metadata_value(Buffer, partitions, []), 330 | 331 | lists:foldl(fun(Partition, Acc) -> 332 | Acc + (ets:info(Partition, size) - 1) 333 | end, 0, Partitions) 334 | end. 335 | 336 | -spec info(Buffer :: t()) -> buffer_info() | undefined. 337 | info(Buffer) -> 338 | buffer_info(Buffer). 339 | 340 | -spec info() -> buffers_info(). 341 | info() -> 342 | lists:foldl(fun 343 | ({gen_buffer, Buffer}, Acc) -> 344 | case buffer_info(Buffer) of 345 | undefined -> Acc; 346 | InfoChann -> Acc#{Buffer => InfoChann} 347 | end; 348 | 349 | (_, Acc) -> 350 | Acc 351 | end, #{}, pg2:which_groups()). 352 | 353 | %%%=================================================================== 354 | %%% Global Utilities 355 | %%%=================================================================== 356 | 357 | -spec pg2_namespace(gen_buffer:t()) -> any(). 358 | pg2_namespace(Buffer) -> 359 | {gen_buffer, Buffer}. 360 | 361 | %%%=================================================================== 362 | %%% Internal functions 363 | %%%=================================================================== 364 | 365 | %% @private 366 | get_children(undefined) -> 367 | {error, no_available_buffer}; 368 | 369 | get_children(Buffer) when is_atom(Buffer) -> 370 | get_children(whereis(Buffer)); 371 | 372 | get_children(Buffer) when is_pid(Buffer) -> 373 | case supervisor:which_children(Buffer) of 374 | [] -> {error, no_available_workers}; 375 | Children -> {ok, Children} 376 | end. 377 | 378 | %% @private 379 | maybe_retry(_Buffer, _Message, 0) -> 380 | no_available_workers; 381 | 382 | maybe_retry(Buffer, Message, Retries) when Retries < 0 -> 383 | eval(Buffer, Message, 0); 384 | 385 | maybe_retry(Buffer, Message, Retries) -> 386 | eval(Buffer, Message, Retries - 1). 387 | 388 | %% @private 389 | buffer_info(Buffer) -> 390 | case get_workers(Buffer) of 391 | {ok, WorkerList} -> 392 | info_map(length(WorkerList), ?MODULE:size(Buffer)); 393 | 394 | {error, no_available_workers} -> 395 | info_map(0, ?MODULE:size(Buffer)); 396 | 397 | {error, no_available_buffer} -> 398 | undefined 399 | end. 400 | 401 | %% @private 402 | info_map(Workers, QueueSize) -> 403 | #{ 404 | workers => Workers, 405 | size => QueueSize 406 | }. 407 | -------------------------------------------------------------------------------- /src/gen_buffer_app.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @doc 3 | %%% gen_buffer app 4 | %%% @end 5 | %%%------------------------------------------------------------------- 6 | -module(gen_buffer_app). 7 | 8 | -behaviour(application). 9 | -behaviour(supervisor). 10 | 11 | %% Application callbacks 12 | -export([ 13 | start/2, 14 | stop/1 15 | ]). 16 | 17 | %% Supervisor callbacks 18 | -export([init/1]). 19 | 20 | -define(SUPERVISOR, gen_buffer_sup). 21 | 22 | %%%=================================================================== 23 | %%% API 24 | %%%=================================================================== 25 | 26 | %% @hidden 27 | start(_StartType, _StartArgs) -> 28 | supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []). 29 | 30 | %% @hidden 31 | stop(_State) -> 32 | ok. 33 | 34 | %%%=================================================================== 35 | %%% Supervisor callbacks 36 | %%%=================================================================== 37 | 38 | %% @hidden 39 | init(_) -> 40 | {ok, {{one_for_all, 0, 1}, []}}. 41 | -------------------------------------------------------------------------------- /src/gen_buffer_dist.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @doc 3 | %%% gen_buffer Distributed Interface. 4 | %%% @end 5 | %%%------------------------------------------------------------------- 6 | -module(gen_buffer_dist). 7 | 8 | %% API 9 | -export([ 10 | start_link/2, 11 | stop/1, 12 | eval/2, 13 | eval/3, 14 | send/2, 15 | poll/1, 16 | recv/2, 17 | recv/3, 18 | send_recv/2, 19 | send_recv/3, 20 | get_worker/1, 21 | get_workers/1, 22 | set_workers/2, 23 | set_workers/3, 24 | set_workers/4, 25 | size/1, 26 | size/2, 27 | info/0, 28 | info/1, 29 | info/2 30 | ]). 31 | 32 | %% Global Utilities 33 | -export([ 34 | pick_node/1, 35 | pick_node/2, 36 | get_nodes/1 37 | ]). 38 | 39 | -define(LOCAL, gen_buffer). 40 | 41 | %%%=================================================================== 42 | %%% Types 43 | %%%=================================================================== 44 | 45 | -type dist_res(R) :: [{Node ::node(), R}]. 46 | -type rpc_error() :: {badrpc, Reason :: any()} | no_return(). 47 | 48 | %%%=================================================================== 49 | %%% API 50 | %%%=================================================================== 51 | 52 | %% @equiv gen_buffer:start_link(Buffer, Opts) 53 | start_link(Buffer, Opts) -> 54 | gen_buffer:start_link(Buffer, Opts). 55 | 56 | %% @equiv gen_buffer:stop(Buffer) 57 | stop(Buffer) -> 58 | gen_buffer:stop(Buffer). 59 | 60 | %% @equiv eval(Buffer, Message, 1) 61 | eval(Buffer, Message) -> 62 | eval(Buffer, Message, 1). 63 | 64 | -spec eval( 65 | Buffer :: gen_buffer:t(), 66 | Message :: any(), 67 | Retries :: integer() 68 | ) -> any() | rpc_error(). 69 | eval(Buffer, Message, Retries)-> 70 | rpc_call(Buffer, eval, [Buffer, Message, Retries]). 71 | 72 | -spec send( 73 | Buffer :: gen_buffer:t(), 74 | Message :: any() 75 | ) -> reference() | rpc_error(). 76 | send(Buffer, Message) -> 77 | rpc_call(Buffer, send, [Buffer, Message, self()]). 78 | 79 | -spec poll( 80 | Buffer :: gen_buffer:t() 81 | ) -> ok | {error, Reason :: any()} | rpc_error(). 82 | poll(Buffer) -> 83 | rpc_call(Buffer, poll, [Buffer]). 84 | 85 | %% @equiv recv(Buffer, Ref, infinity) 86 | recv(Buffer, Ref) -> 87 | recv(Buffer, Ref, infinity). 88 | 89 | -spec recv( 90 | Buffer :: gen_buffer:t(), 91 | Ref :: reference(), 92 | Timeout :: timeout() 93 | ) -> {ok, any()} | {error, any()} | {badrpc, any()}. 94 | recv(Buffer, Ref, Timeout) -> 95 | gen_buffer:recv(Buffer, Ref, Timeout). 96 | 97 | %% @equiv send_recv(Buffer, Message, infinity) 98 | send_recv(Buffer, Message) -> 99 | send_recv(Buffer, Message, infinity). 100 | 101 | -spec send_recv( 102 | Buffer :: gen_buffer:t(), 103 | Message :: any(), 104 | Timeout :: timeout() 105 | ) -> {ok, any()} | {error, any()} | rpc_error(). 106 | send_recv(Buffer, Message, Timeout) -> 107 | rpc_call(Buffer, send_recv, [Buffer, Message, Timeout]). 108 | 109 | -spec get_worker(Buffer :: gen_buffer:t()) -> {ok, pid()} | {error, any()}. 110 | get_worker(Buffer) -> 111 | gen_buffer:get_worker(Buffer). 112 | 113 | -spec get_workers(Buffer :: gen_buffer:t()) -> [{node(), [pid()]}]. 114 | get_workers(Buffer) -> 115 | Nodes = get_nodes(Buffer), 116 | {ResL, _} = rpc:multicall(Nodes, ?LOCAL, get_workers, [Buffer]), 117 | WorkersL = [Workers || {OkOrErr, Workers} <- ResL, OkOrErr =:= ok], 118 | lists:zip(Nodes, WorkersL). 119 | 120 | %% @equiv set_workers(Buffer, N, #{}) 121 | set_workers(Buffer, N) -> 122 | set_workers(Buffer, N, #{}). 123 | 124 | %% @equiv set_workers(get_nodes(Buffer), Buffer, N, Opts) 125 | set_workers(Buffer, N, Opts) -> 126 | set_workers(get_nodes(Buffer), Buffer, N, Opts). 127 | 128 | -spec set_workers( 129 | Nodes :: [node()], 130 | Buffer :: gen_buffer:t(), 131 | N :: non_neg_integer(), 132 | Opts :: gen_buffer:opts() 133 | ) -> dist_res([pid()]). 134 | set_workers(Nodes, Buffer, N, Opts) when N > 0 -> 135 | {ResL, _} = rpc:multicall(Nodes, ?LOCAL, set_workers, [Buffer, N, Opts]), 136 | WorkersL = [Workers || {OkOrErr, Workers} <- ResL, OkOrErr =:= ok], 137 | lists:zip(Nodes, WorkersL). 138 | 139 | %% @equiv size(get_nodes(Buffer), Buffer) 140 | size(Buffer) -> 141 | size(get_nodes(Buffer), Buffer). 142 | 143 | -spec size( 144 | Nodes :: [node()], 145 | Buffer :: gen_buffer:t() 146 | ) -> dist_res(non_neg_integer()). 147 | size(Nodes, Buffer) -> 148 | {ResL, _} = rpc:multicall(Nodes, ?LOCAL, size, [Buffer]), 149 | SizeL = [Size || Size <- ResL, Size =/= undefined], 150 | lists:zip(Nodes, SizeL). 151 | 152 | %% equiv info(Nodes) 153 | info() -> 154 | Nodes = 155 | lists:foldl(fun 156 | ({gen_buffer, _} = Group, Acc) -> 157 | GroupNodes = [node(Member) || Member <- pg2:get_members(Group)], 158 | GroupNodes ++ Acc; 159 | 160 | (_, Acc) -> 161 | Acc 162 | end, [], pg2:which_groups()), 163 | 164 | info(Nodes). 165 | 166 | -spec info( 167 | BufferOrNodes :: gen_buffer:t() | [node()] 168 | ) -> dist_res(gen_buffer:buffers_info()). 169 | info(Buffer) when is_atom(Buffer) -> 170 | info(get_nodes(Buffer), Buffer); 171 | 172 | info(Nodes) when is_list(Nodes) -> 173 | {ResL, _} = rpc:multicall(Nodes, ?LOCAL, info, []), 174 | InfoL = [Info || Info <- ResL], 175 | lists:zip(Nodes, InfoL). 176 | 177 | -spec info( 178 | Nodes :: [node()], 179 | Buffer :: gen_buffer:t() 180 | ) -> dist_res(gen_buffer:buffer_info()). 181 | info(Nodes, Buffer) -> 182 | {ResL, _} = rpc:multicall(Nodes, ?LOCAL, info, [Buffer]), 183 | InfoL = [Info || Info <- ResL, Info =/= undefined], 184 | lists:zip(Nodes, InfoL). 185 | 186 | %%%=================================================================== 187 | %%% Global Utilities 188 | %%%=================================================================== 189 | 190 | %% @equiv pick_node(Buffer, os:timestamp()) 191 | pick_node(Buffer) -> 192 | pick_node(Buffer, os:timestamp()). 193 | 194 | -spec pick_node(Buffer :: gen_buffer:t(), Key :: any()) -> node(). 195 | pick_node(Buffer, Key) -> 196 | Nodes = get_nodes(Buffer), 197 | Nth = erlang:phash2(Key, length(Nodes)) + 1, 198 | lists:nth(Nth, Nodes). 199 | 200 | -spec get_nodes(Buffer :: gen_buffer:t()) -> [node()]. 201 | get_nodes(Buffer) -> 202 | Group = gen_buffer:pg2_namespace(Buffer), 203 | 204 | case pg2:get_members(Group) of 205 | {error, {no_such_group, Group}} -> 206 | ok = pg2:create(Group), 207 | get_nodes(Buffer); 208 | 209 | [] -> 210 | error(no_available_nodes); 211 | 212 | Pids -> 213 | [node(Pid) || Pid <- Pids] 214 | end. 215 | 216 | %%%=================================================================== 217 | %%% Internal functions 218 | %%%=================================================================== 219 | 220 | %% @private 221 | rpc_call(Buffer, Fun, Args) -> 222 | rpc_call(Buffer, ?LOCAL, Fun, Args). 223 | 224 | %% @private 225 | rpc_call(Buffer, Mod, Fun, Args) -> 226 | LocalNode = node(), 227 | case pick_node(Buffer) of 228 | LocalNode -> apply(Mod, Fun, Args); 229 | RemoteNode -> rpc:call(RemoteNode, Mod, Fun, Args) 230 | end. 231 | -------------------------------------------------------------------------------- /src/gen_buffer_lib.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_lib). 2 | 3 | %% API 4 | -export([ 5 | get_metadata_value/2, 6 | get_metadata_value/3, 7 | get_one_metadata_value/3, 8 | set_metadata_value/3, 9 | del_metadata_value/3, 10 | get_available_workers/1, 11 | get_available_worker/1, 12 | get_partition/1, 13 | get_partition/2, 14 | partition_name/2 15 | ]). 16 | 17 | %%%=================================================================== 18 | %%% API 19 | %%%=================================================================== 20 | 21 | %% @equiv get_metadata_value(Buffer, Key, undefined) 22 | get_metadata_value(Buffer, Key) -> 23 | get_metadata_value(Buffer, Key, undefined). 24 | 25 | -spec get_metadata_value(Buffer :: gen_buffer:t(), Key :: any(), Default :: any()) -> any(). 26 | get_metadata_value(Buffer, Key, Default) -> 27 | try 28 | ets:lookup_element(Buffer, Key, 2) 29 | catch 30 | _:_ -> Default 31 | end. 32 | 33 | -spec get_one_metadata_value(Buffer :: gen_buffer:t(), Key :: any(), Default :: any()) -> any(). 34 | get_one_metadata_value(Buffer, Key, Default) -> 35 | case get_metadata_value(Buffer, Key, Default) of 36 | Value when is_list(Value) -> 37 | hd(Value); 38 | 39 | Value -> 40 | Value 41 | end. 42 | 43 | -spec set_metadata_value(Buffer :: gen_buffer:t(), Key :: any(), Value :: any()) -> ok. 44 | set_metadata_value(Buffer, Key, Value) -> 45 | _ = ets:insert(Buffer, {Key, Value}), 46 | ok. 47 | 48 | -spec del_metadata_value(Buffer :: gen_buffer:t(), Key :: any(), Value :: any()) -> ok. 49 | del_metadata_value(Buffer, Key, Value) -> 50 | _ = ets:delete_object(Buffer, {Key, Value}), 51 | ok. 52 | 53 | -spec get_available_workers(Buffer :: gen_buffer:t()) -> [pid()]. 54 | get_available_workers(Buffer) -> 55 | get_metadata_value(Buffer, Buffer, []). 56 | 57 | -spec get_available_worker(Buffer :: gen_buffer:t()) -> pid() | no_available_workers. 58 | get_available_worker(Buffer) -> 59 | case get_available_workers(Buffer) of 60 | [] -> 61 | no_available_workers; 62 | 63 | Workers -> 64 | pick_worker(Workers) 65 | end. 66 | 67 | -spec get_partition(Buffer :: gen_buffer:t()) -> gen_buffer:t(). 68 | get_partition(Buffer) -> 69 | NumPartitions = get_one_metadata_value(Buffer, n_partitions, 1), 70 | get_partition(Buffer, NumPartitions). 71 | 72 | -spec get_partition(Buffer :: gen_buffer:t(), NumPartitions :: pos_integer()) -> atom(). 73 | get_partition(Buffer, NumPartitions) -> 74 | Partition = erlang:phash2(os:timestamp(), NumPartitions), 75 | partition_name(Buffer, Partition). 76 | 77 | -spec partition_name(Buffer :: gen_buffer:t(), Partition :: non_neg_integer()) -> atom(). 78 | partition_name(Buffer, Partition) -> 79 | Bin = <<(atom_to_binary(Buffer, utf8))/binary, ".", (integer_to_binary(Partition))/binary>>, 80 | binary_to_atom(Bin, utf8). 81 | 82 | %%%=================================================================== 83 | %%% Internal functions 84 | %%%=================================================================== 85 | 86 | %% @private 87 | pick_worker([]) -> 88 | no_available_workers; 89 | 90 | pick_worker([Worker | Rest]) -> 91 | case process_info(Worker, message_queue_len) of 92 | {message_queue_len, Len} when Len > 0 -> 93 | pick_worker(Rest); 94 | 95 | _ -> 96 | Worker 97 | end. 98 | -------------------------------------------------------------------------------- /src/gen_buffer_sup.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @doc 3 | %%% gen_buffer supervisor. 4 | %%% @end 5 | %%%------------------------------------------------------------------- 6 | -module(gen_buffer_sup). 7 | 8 | -behaviour(supervisor). 9 | 10 | %% API 11 | -export([ 12 | start_link/2, 13 | start_child/2, 14 | terminate_child/1 15 | ]). 16 | 17 | %% Supervisor callbacks 18 | -export([init/1]). 19 | 20 | %% Buffer worker module 21 | -define(WORKER, gen_buffer_worker). 22 | 23 | %% Default number of workers 24 | -define(DEFAULT_WORKERS, erlang:system_info(schedulers_online)). 25 | 26 | %%%=================================================================== 27 | %%% API functions 28 | %%%=================================================================== 29 | 30 | -spec start_link(gen_buffer:t(), gen_buffer:opts()) -> {ok, pid()} | ignore | {error, term()}. 31 | start_link(Buffer, Opts) when is_list(Opts) -> 32 | start_link(Buffer, maps:from_list(Opts)); 33 | 34 | start_link(Buffer, Opts) when is_map(Opts) -> 35 | case supervisor:start_link({local, Buffer}, ?MODULE, {Buffer, Opts}) of 36 | {ok, Pid} = Ok -> 37 | ok = init_pg2(Buffer, Pid), 38 | ok = start_children(Buffer, Opts), 39 | Ok; 40 | 41 | Error -> 42 | Error 43 | end. 44 | 45 | -spec start_child(gen_buffer:t(), gen_buffer:opts()) -> supervisor:startchild_ret(). 46 | start_child(Buffer, Opts) when is_list(Opts) -> 47 | start_child(Buffer, maps:from_list(Opts)); 48 | 49 | start_child(Buffer, Opts) when is_map(Opts) -> 50 | supervisor:start_child(Buffer, [Opts#{buffer => Buffer}]). 51 | 52 | -spec terminate_child(gen_buffer:t()) -> ok | {error, not_found | simple_one_for_one}. 53 | terminate_child(Buffer) -> 54 | case whereis(Buffer) of 55 | undefined -> 56 | ok; 57 | _ -> 58 | case gen_buffer:get_worker(Buffer) of 59 | {ok, Worker} -> 60 | supervisor:terminate_child(Buffer, Worker); 61 | 62 | Error -> 63 | Error 64 | end 65 | end. 66 | 67 | %%%=================================================================== 68 | %%% Supervisor callbacks 69 | %%%=================================================================== 70 | 71 | %% @hidden 72 | init({Buffer, Opts}) -> 73 | _ = create_buffer(Buffer, Opts), 74 | Children = [child(?WORKER, [Opts#{buffer => Buffer}])], 75 | supervise(Children, #{strategy => simple_one_for_one}). 76 | 77 | %%%=================================================================== 78 | %%% Internal functions 79 | %%%=================================================================== 80 | 81 | %% @private 82 | child(Module, Args) -> 83 | #{ 84 | id => Module, 85 | start => {Module, start_link, Args} 86 | }. 87 | 88 | %% @private 89 | supervise(Children, SupFlagsMap) -> 90 | Strategy = maps:get(strategy, SupFlagsMap, one_for_one), 91 | Intensity = maps:get(intensity, SupFlagsMap, 10), 92 | Period = maps:get(period, SupFlagsMap, 10), 93 | {ok, {{Strategy, Intensity, Period}, Children}}. 94 | 95 | create_buffer(Buffer, Opts) -> 96 | NumPartitions = maps:get(n_partitions, Opts, 1), 97 | create_buffer(Buffer, Opts, NumPartitions). 98 | 99 | %% @private 100 | create_buffer(Buffer, #{buffer_type := ring} = Opts, NumPartitions) -> 101 | Size = maps:get(buffer_size, Opts, 100), 102 | create_partitioned_buffer(Buffer, [ring, Size], NumPartitions); 103 | 104 | create_buffer(Buffer, #{buffer_type := lifo}, NumPartitions) -> 105 | create_partitioned_buffer(Buffer, [lifo], NumPartitions); 106 | 107 | create_buffer(Buffer, _, NumPartitions) -> 108 | create_partitioned_buffer(Buffer, [fifo], NumPartitions). 109 | 110 | %% @private 111 | create_partitioned_buffer(Buffer, Args, NumPartitions) -> 112 | % create metadata table 113 | Buffer = create_metadata(Buffer), 114 | 115 | % store number of partitions 116 | true = ets:insert(Buffer, {n_partitions, NumPartitions}), 117 | 118 | % create partitions (one buffer per partition) 119 | Partitions = [begin 120 | PartitionName = gen_buffer_lib:partition_name(Buffer, Partition), 121 | _ = apply(ets_buffer, create_dedicated, [PartitionName | Args]), 122 | PartitionName 123 | end || Partition <- lists:seq(0, NumPartitions - 1)], 124 | 125 | % store partition names 126 | true = ets:insert(Buffer, {partitions, Partitions}), 127 | ok. 128 | 129 | %% @private 130 | create_metadata(Buffer) -> 131 | ets:new(Buffer, [ 132 | named_table, 133 | public, 134 | duplicate_bag, 135 | {read_concurrency, true}, 136 | {write_concurrency, true} 137 | ]). 138 | 139 | %% @private 140 | start_children(Buffer, Opts) -> 141 | lists:foreach(fun(_) -> 142 | case start_child(Buffer, Opts) of 143 | {ok, Child} when is_pid(Child); Child =:= undefined -> 144 | ok; 145 | 146 | {error, Error} -> 147 | exit(whereis(Buffer), Error) 148 | end 149 | end, lists:seq(1, maps:get(workers, Opts, ?DEFAULT_WORKERS))). 150 | 151 | %% @private 152 | init_pg2(Buffer, SupPid) -> 153 | Group = gen_buffer:pg2_namespace(Buffer), 154 | ok = pg2:create(Group), 155 | ok = pg2:join(Group, SupPid), 156 | ok. 157 | -------------------------------------------------------------------------------- /src/gen_buffer_worker.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @doc 3 | %%% gen_buffer worker. 4 | %%% @end 5 | %%%------------------------------------------------------------------- 6 | -module(gen_buffer_worker). 7 | 8 | -behaviour(gen_server). 9 | 10 | %% API 11 | -export([ 12 | start_link/1, 13 | start_link/2, 14 | eval/2, 15 | send/2, 16 | poll/1 17 | ]). 18 | 19 | %% gen_server callbacks 20 | -export([ 21 | init/1, 22 | handle_call/3, 23 | handle_cast/2, 24 | handle_info/2, 25 | terminate/2 26 | ]). 27 | 28 | %%%=================================================================== 29 | %%% API 30 | %%%=================================================================== 31 | 32 | %% @equiv start_link(Opts, #{}) 33 | start_link(Opts) when is_map(Opts) -> 34 | start_link(Opts, #{}). 35 | 36 | -spec start_link( 37 | Opts :: gen_buffer:opts_map(), 38 | ExtOpts :: gen_buffer:opts_map() 39 | ) -> {ok, pid()} | ignore | {error, term()}. 40 | start_link(Opts, ExtOpts) when is_map(Opts), is_map(ExtOpts) -> 41 | gen_server:start_link(?MODULE, maps:merge(Opts, ExtOpts), []). 42 | 43 | -spec eval(Worker :: pid(), Message :: any()) -> ok. 44 | eval(Worker, Message) -> 45 | gen_server:call(Worker, {eval, Message}). 46 | 47 | -spec send(Worker :: pid(), Message :: any()) -> ok. 48 | send(Worker, Message) -> 49 | gen_server:cast(Worker, {send, Message}). 50 | 51 | -spec poll(Worker :: pid()) -> ok. 52 | poll(Worker) -> 53 | gen_server:cast(Worker, poll). 54 | 55 | %%%=================================================================== 56 | %%% gen_server callbacks 57 | %%%=================================================================== 58 | 59 | %% @hidden 60 | init(#{buffer := Buffer} = Opts) -> 61 | _ = process_flag(trap_exit, true), 62 | ok = worker_available(Buffer, true), 63 | 64 | State = validate_opts(Opts), 65 | Handler = maps:get(message_handler, State), 66 | InitResult = 67 | case erlang:function_exported(Handler, init, 1) of 68 | true -> Handler:init(maps:get(init_args, State, undefined)); 69 | false -> {ok, undefined} 70 | end, 71 | 72 | handle_init_result(InitResult, State). 73 | 74 | %% @hidden 75 | handle_call({eval, Message}, _From, State) -> 76 | {ok, Res, NewHState} = do_tx(fun eval_callback/2, [Message, State], State), 77 | {reply, Res, State#{handler_state := NewHState}}; 78 | 79 | handle_call(_Request, _From, State) -> 80 | {reply, ok, State}. 81 | 82 | %% @hidden 83 | handle_cast({send, Message}, State) -> 84 | NewState = do_tx(fun do_poll/2, [Message, State], State), 85 | {noreply, NewState}; 86 | 87 | handle_cast(poll, State) -> 88 | {noreply, do_poll(undefined, State)}. 89 | 90 | %% @hidden 91 | handle_info( 92 | Info, 93 | #{ 94 | buffer := Buffer, 95 | message_handler := Handler, 96 | handler_state := HState 97 | } = State 98 | ) -> 99 | case erlang:function_exported(Handler, handle_info, 3) of 100 | true -> 101 | try 102 | ok = worker_available(Buffer, false), 103 | Result = Handler:handle_info(Buffer, Info, HState), 104 | handle_info_result(Result, State) 105 | catch 106 | throw:Error -> handle_info_result(Error, State) 107 | after 108 | ok = worker_available(Buffer, true) 109 | end; 110 | 111 | false -> 112 | {noreply, State} 113 | end. 114 | 115 | %% @hidden 116 | terminate(Reason, #{buffer := Buffer, message_handler := Handler, handler_state := HState}) -> 117 | ok = worker_available(Buffer, false), 118 | case erlang:function_exported(Handler, terminate, 3) of 119 | true -> 120 | try 121 | Handler:terminate(Buffer, Reason, HState) 122 | catch 123 | throw:Response -> Response 124 | end; 125 | 126 | false -> 127 | ok 128 | end. 129 | 130 | %%%=================================================================== 131 | %%% Internal functions 132 | %%%=================================================================== 133 | 134 | %% @private 135 | validate_opts(Opts) -> 136 | validate_handler(Opts). 137 | 138 | %% @private 139 | validate_handler(Opts) -> 140 | case maps:get(message_handler, Opts, nil) of 141 | nil -> 142 | exit({missing_option, message_handler}); 143 | 144 | Handler when is_atom(Handler) -> 145 | validate_message_handler_mod(Handler, Opts); 146 | 147 | Handler -> 148 | exit({invalid_message_handler, Handler}) 149 | end. 150 | 151 | %% @private 152 | validate_message_handler_mod(Handler, Opts) -> 153 | case code:ensure_loaded(Handler) of 154 | {module, Handler} -> 155 | validate_message_handler_funs(Handler, Opts); 156 | 157 | {error, _} -> 158 | exit({invalid_message_handler, Handler}) 159 | end. 160 | 161 | %% @private 162 | validate_message_handler_funs(Handler, Opts) -> 163 | case erlang:function_exported(Handler, handle_message, 3) of 164 | true -> 165 | Opts; 166 | 167 | false -> 168 | exit({missing_callback, handle_message}) 169 | end. 170 | 171 | %% @priate 172 | do_poll(Message, #{buffer := Buffer} = State) -> 173 | State1 = 174 | case Message of 175 | undefined -> 176 | State; 177 | 178 | _ -> 179 | {ok, _Res, NewHState} = eval_callback(Message, State), 180 | State#{handler_state := NewHState} 181 | end, 182 | 183 | lists:foldl(fun(Partition, Acc) -> 184 | PartitionName = gen_buffer_lib:partition_name(Buffer, Partition), 185 | do_partition_poll(PartitionName, Acc) 186 | end, State1, lists:seq(0, maps:get(n_partitions, State, 1) - 1)). 187 | 188 | %% @priate 189 | do_partition_poll(BufferPartition, State) -> 190 | case ets_buffer:read_dedicated(BufferPartition) of 191 | [] -> 192 | State; 193 | 194 | [Message] -> 195 | {ok, _Res, NewHState} = eval_callback(Message, State), 196 | do_partition_poll(BufferPartition, State#{handler_state := NewHState}); 197 | 198 | {missing_ets_data, BufferPartition, _ReadLoc} -> 199 | % this case is expected under heavy concurrency scenario; 200 | % see ets_buffer for more details 201 | do_partition_poll(BufferPartition, State) 202 | end. 203 | 204 | %% @priate 205 | do_tx(Fun, Args, #{buffer := Buffer}) -> 206 | try 207 | ok = worker_available(Buffer, false), 208 | apply(Fun, Args) 209 | after 210 | ok = worker_available(Buffer, true) 211 | end. 212 | 213 | -ifdef(OTP_RELEASE). 214 | %% OTP 21 or higher 215 | 216 | %% @private 217 | eval_callback( 218 | {From, Ref, Msg}, 219 | #{ 220 | buffer := Buffer, 221 | message_handler := Handler, 222 | handler_state := HState 223 | } = State 224 | ) -> 225 | try Handler:handle_message(Buffer, Msg, HState) of 226 | {ok, Reply, _} = Res -> 227 | _ = maybe_reply({reply, Ref, Buffer, Reply}, From, State), 228 | Res 229 | catch 230 | Class:Exception:Stacktrace -> 231 | _ = maybe_reply({error, Ref, Buffer, Exception}, From, State), 232 | erlang:raise(Class, Exception, Stacktrace) 233 | end; 234 | 235 | eval_callback(Msg, #{buffer := Buffer, message_handler := Handler, handler_state := HState}) -> 236 | Handler:handle_message(Buffer, Msg, HState). 237 | 238 | -else. 239 | %% OTP 20 or lower 240 | 241 | %% @private 242 | eval_callback( 243 | {From, Ref, Msg}, 244 | #{ 245 | buffer := Buffer, 246 | message_handler := Handler, 247 | handler_state := HState 248 | } = State 249 | ) -> 250 | try Handler:handle_message(Buffer, Msg, HState) of 251 | {ok, Reply, _} = Res -> 252 | _ = maybe_reply({reply, Ref, Buffer, Reply}, From, State), 253 | Res 254 | catch 255 | Class:Exception -> 256 | _ = maybe_reply({error, Ref, Buffer, Exception}, From, State), 257 | erlang:raise(Class, Exception, erlang:get_stacktrace()) 258 | end; 259 | 260 | eval_callback(Msg, #{buffer := Buffer, message_handler := Handler, handler_state := HState}) -> 261 | Handler:handle_message(Buffer, Msg, HState). 262 | 263 | -endif. 264 | 265 | %% @private 266 | maybe_reply(Reply, From, #{send_replies := true}) -> From ! Reply; 267 | maybe_reply(_, _, _) -> ok. 268 | 269 | %% @private 270 | handle_init_result({ok, HState}, State) -> 271 | {ok, State#{handler_state => HState}}; 272 | 273 | handle_init_result({ok, HState, hibernate}, State) -> 274 | {ok, State#{handler_state => HState}, hibernate}; 275 | 276 | handle_init_result({ok, HState, Timeout}, State) -> 277 | {ok, State#{handler_state => HState}, Timeout}; 278 | 279 | handle_init_result(ignore, _State) -> 280 | ignore; 281 | 282 | handle_init_result({stop, Reason}, #{buffer := Buffer} = State) -> 283 | ok = worker_available(Buffer, false), 284 | {stop, Reason, State}. 285 | 286 | %% @private 287 | handle_info_result({noreply, NewHState}, State) -> 288 | {noreply, State#{handler_state => NewHState}}; 289 | 290 | handle_info_result({noreply, NewHState, hibernate}, State) -> 291 | {noreply, State#{handler_state => NewHState}, hibernate}; 292 | 293 | handle_info_result({noreply, NewHState, Timeout}, State) -> 294 | {noreply, State#{handler_state => NewHState}, Timeout}; 295 | 296 | handle_info_result({stop, Reason, NewHState}, State) -> 297 | {stop, Reason, State#{handler_state => NewHState}}. 298 | 299 | %% @private 300 | worker_available(Buffer, true) -> 301 | gen_buffer_lib:set_metadata_value(Buffer, Buffer, self()); 302 | 303 | worker_available(Buffer, false) -> 304 | gen_buffer_lib:del_metadata_value(Buffer, Buffer, self()). 305 | -------------------------------------------------------------------------------- /test/gen_buffer_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% Common Test 6 | -export([ 7 | all/0, 8 | init_per_suite/1, 9 | end_per_suite/1, 10 | init_per_testcase/2, 11 | end_per_testcase/2 12 | ]). 13 | 14 | %% Common Test Cases 15 | -include_lib("mixer/include/mixer.hrl"). 16 | -mixin([ 17 | {gen_buffer_test_cases, [ 18 | t_eval/1, 19 | t_eval_error/1, 20 | t_send_and_recv/1, 21 | t_send_and_recv_errors/1, 22 | t_send_recv/1, 23 | t_fire_and_forget/1, 24 | t_add_del_workers/1, 25 | t_set_workers/1, 26 | t_size/1, 27 | t_info_buffer/1, 28 | t_info/1, 29 | t_worker_polling/1, 30 | t_worker_distribution/1 31 | ]} 32 | ]). 33 | 34 | %% Test Cases 35 | -export([ 36 | t_child_spec/1, 37 | t_create_errors/1, 38 | t_delete_buffer/1, 39 | t_buffer_types/1, 40 | t_missing_buffer_worker_funs/1, 41 | t_handle_message_state/1, 42 | t_callback_init/1, 43 | t_callback_handle_info/1, 44 | t_callback_terminate/1, 45 | t_restart_workers/1, 46 | t_gen_buffer_dist_locally/1, 47 | t_gen_buffer_worker_missing_ets_data/1, 48 | t_gen_buffer_lib_missing_funs/1 49 | ]). 50 | 51 | %% Helpers 52 | -export([ 53 | producer/2 54 | ]). 55 | 56 | -define(EXCLUDED_FUNS, [ 57 | module_info, 58 | all, 59 | init_per_suite, 60 | end_per_suite, 61 | init_per_testcase, 62 | end_per_testcase, 63 | producer 64 | ]). 65 | 66 | -define(BUFFER, gen_buffer_test). 67 | 68 | %%%=================================================================== 69 | %%% Common Test 70 | %%%=================================================================== 71 | 72 | all() -> 73 | Exports = ?MODULE:module_info(exports), 74 | [F || {F, _} <- Exports, not lists:member(F, ?EXCLUDED_FUNS)]. 75 | 76 | init_per_suite(Config) -> 77 | Opts = #{ 78 | message_handler => test_message_handler, 79 | send_replies => true, 80 | init_args => ok 81 | }, 82 | 83 | [{opts, Opts}, {module, gen_buffer} | Config]. 84 | 85 | end_per_suite(Config) -> 86 | Config. 87 | 88 | init_per_testcase(_, Config) -> 89 | Config. 90 | 91 | end_per_testcase(_, Config) -> 92 | Mod = ?config(module, Config), 93 | ok = cleanup_buffers(Mod), 94 | Config. 95 | 96 | %%%=================================================================== 97 | %%% Tests Cases 98 | %%%=================================================================== 99 | 100 | t_child_spec(_Config) -> 101 | #{ 102 | id := test_buffer, 103 | start := { 104 | gen_buffer, 105 | start_link, 106 | [ 107 | test_buffer, 108 | #{ 109 | buffer := test_buffer, 110 | message_handler := ?MODULE 111 | } 112 | ] 113 | }, 114 | type := supervisor 115 | } = gen_buffer:child_spec([{buffer, test_buffer}, {message_handler, ?MODULE}]). 116 | 117 | t_create_errors(Config) -> 118 | Mod = ?config(module, Config), 119 | Opts = ?config(opts, Config), 120 | 121 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 122 | 123 | {error, {already_started, _}} = 124 | gen_buffer_ct:create_buffer(?BUFFER, [{message_handler, handler}], Mod, Config), 125 | 126 | _ = process_flag(trap_exit, true), 127 | 128 | {{invalid_message_handler, handler}, _} = 129 | try 130 | gen_buffer_ct:create_buffer(test, [{message_handler, handler}], Mod, Config) 131 | catch 132 | _:E2 -> E2 133 | end, 134 | 135 | {{missing_option, message_handler}, _} = 136 | try 137 | gen_buffer_ct:create_buffer(test, [], Mod, Config) 138 | catch 139 | _:E3 -> E3 140 | end, 141 | 142 | {{missing_callback, handle_message}, _} = 143 | try 144 | gen_buffer_ct:create_buffer( 145 | test, 146 | Opts#{message_handler => test_message_handler3}, 147 | Mod, 148 | Config 149 | ) 150 | catch 151 | _:E4 -> E4 152 | end, 153 | 154 | {{invalid_message_handler,"wrong_handler"}, _} = 155 | try 156 | gen_buffer_ct:create_buffer(test, Opts#{message_handler => "wrong_handler"}, Mod, Config) 157 | catch 158 | _:E5 -> E5 159 | end, 160 | 161 | {ok, _} = 162 | gen_buffer_ct:create_buffer( 163 | test1, 164 | Opts#{message_handler => test_message_handler2}, 165 | Mod, 166 | Config 167 | ), 168 | 169 | {ok, _} = gen_buffer_ct:create_buffer(test, Opts, Mod, Config). 170 | 171 | t_delete_buffer(Config) -> 172 | Mod = ?config(module, Config), 173 | _ = gen_buffer_ct:create_buffer(?BUFFER, ?config(opts, Config), Mod, Config), 174 | 175 | true = is_pid(whereis(?BUFFER)), 176 | ok = Mod:stop(?BUFFER), 177 | ok = Mod:stop(wrong_buffer), 178 | undefined = whereis(?BUFFER). 179 | 180 | t_buffer_types(Config) -> 181 | Mod = ?config(module, Config), 182 | 183 | Opts = (?config(opts, Config))#{ 184 | send_replies => false, 185 | workers => 0 186 | }, 187 | 188 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 189 | _ = gen_buffer_ct:create_buffer(ring, Opts#{buffer_type => ring, buffer_size => 5}, Mod, Config), 190 | _ = gen_buffer_ct:create_buffer(lifo, Opts#{buffer_type => lifo}, Mod, Config), 191 | ChannName = gen_buffer_lib:partition_name(?BUFFER, 0), 192 | 193 | lists:foreach(fun(M) -> 194 | Mod:send(?BUFFER, M) 195 | end, lists:seq(1, 10)), 196 | 197 | [{_, _, 1}] = ets_buffer:read_dedicated(ChannName), 198 | 199 | lists:foreach(fun(M) -> 200 | Mod:send(lifo, M) 201 | end, lists:seq(1, 10)), 202 | 203 | [{_, _, 10}] = ets_buffer:read_dedicated('lifo.0'), 204 | 205 | lists:foreach(fun(M) -> 206 | Mod:send(ring, M) 207 | end, lists:seq(1, 10)), 208 | 209 | [{_, _, 6}] = ets_buffer:read_dedicated('ring.0'). 210 | 211 | t_missing_buffer_worker_funs(Config) -> 212 | Mod = ?config(module, Config), 213 | Opts = ?config(opts, Config), 214 | 215 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 216 | _ = gen_buffer_worker:start_link(Opts#{buffer => ?BUFFER}), 217 | 218 | {ok, Worker} = Mod:get_worker(?BUFFER), 219 | ok = gen_server:call(Worker, ping). 220 | 221 | t_handle_message_state(Config) -> 222 | Mod = ?config(module, Config), 223 | Opts = ?config(opts, Config), 224 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts#{workers => 1}, Mod, Config), 225 | 226 | ok = lists:foreach(fun(M) -> 227 | {ok, M} = Mod:send_recv(?BUFFER, M) 228 | end, lists:seq(1, 10)), 229 | 230 | 11 = Mod:eval(?BUFFER, 11), 231 | 232 | _ = timer:sleep(1000), 233 | {ok, MsgL} = Mod:send_recv(?BUFFER, messages), 234 | 11 = length(MsgL). 235 | 236 | t_callback_init(Config) -> 237 | Mod = ?config(module, Config), 238 | Opts = (?config(opts, Config))#{workers => 1}, 239 | 240 | _ = process_flag(trap_exit, true), 241 | Opts2 = Opts#{init_args => {test, {stop, kill}}}, 242 | 243 | try gen_buffer_ct:create_buffer(?BUFFER, Opts2, Mod, Config) 244 | catch 245 | exit:kill -> ok 246 | end, 247 | 248 | OptsL = [ 249 | Opts#{init_args => {test, {ok, #{}}}}, 250 | Opts#{init_args => {test, {ok, #{}, hibernate}}}, 251 | Opts#{init_args => {test, {ok, #{}, 5000}}}, 252 | Opts#{init_args => {test, ignore}} 253 | ], 254 | 255 | lists:foreach(fun(Args) -> 256 | _ = gen_buffer_ct:create_buffer(?BUFFER, Args, Mod, Config), 257 | _ = timer:sleep(1000), 258 | Mod:stop(?BUFFER) 259 | end, OptsL). 260 | 261 | t_callback_handle_info(Config) -> 262 | Mod = ?config(module, Config), 263 | Opts = ?config(opts, Config), 264 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 265 | 266 | Messages = [ 267 | {noreply, #{}}, 268 | {noreply, #{}, hibernate}, 269 | {noreply, #{}, 1}, 270 | {stop, kill, #{}}, 271 | throw, 272 | terminate 273 | ], 274 | 275 | ok = lists:foreach(fun(Info) -> 276 | {ok, Worker} = gen_buffer:get_worker(?BUFFER), 277 | Worker ! Info 278 | end, Messages), 279 | 280 | Opts1 = Opts#{message_handler => test_message_handler2}, 281 | _ = gen_buffer_ct:create_buffer(test, Opts1, Mod, Config), 282 | 283 | {ok, Worker} = gen_buffer:get_worker(test), 284 | Worker ! {stop, kill, #{}}, 285 | 286 | Opts2 = Opts#{message_handler => test_message_handler5}, 287 | _ = gen_buffer_ct:create_buffer(test2, Opts2, Mod, Config), 288 | 289 | {ok, Worker2} = gen_buffer:get_worker(test2), 290 | Worker2 ! {stop, kill, #{}}, 291 | 292 | Opts3 = Opts#{message_handler => test_message_handler4}, 293 | _ = gen_buffer_ct:create_buffer(test3, Opts3, Mod, Config), 294 | 295 | {ok, Worker3} = gen_buffer:get_worker(test3), 296 | Worker3 ! "hello", 297 | _ = timer:sleep(500), 298 | AvailableWorkers = gen_buffer_lib:get_available_workers(test3), 299 | false = lists:member(Worker3, AvailableWorkers). 300 | 301 | t_callback_terminate(Config) -> 302 | Mod = ?config(module, Config), 303 | Opts = ?config(opts, Config), 304 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 305 | 306 | {ok, Worker1} = gen_buffer:get_worker(?BUFFER), 307 | _ = exit(Worker1, shutdown), 308 | 309 | {ok, Worker2} = gen_buffer:get_worker(?BUFFER), 310 | _ = exit(Worker2, throw), 311 | 312 | Opts1 = Opts#{message_handler => test_message_handler2}, 313 | _ = gen_buffer_ct:create_buffer(test, Opts1, Mod, Config), 314 | 315 | {ok, Worker3} = gen_buffer:get_worker(test), 316 | _ = exit(Worker3, shutdown). 317 | 318 | t_restart_workers(Config) -> 319 | Mod = ?config(module, Config), 320 | Opts = ?config(opts, Config), 321 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 322 | 323 | {ok, WorkerL1} = gen_buffer:get_workers(?BUFFER), 324 | Len1 = length(WorkerL1), 325 | 326 | {ok, Worker1} = gen_buffer:get_worker(?BUFFER), 327 | _ = exit(Worker1, shutdown), 328 | 329 | {ok, Worker2} = gen_buffer:get_worker(?BUFFER), 330 | _ = exit(Worker2, shutdown), 331 | 332 | {ok, WorkerL2} = gen_buffer:get_workers(?BUFFER), 333 | Len1 = length(WorkerL2). 334 | 335 | t_gen_buffer_dist_locally(Config) -> 336 | Mod = ?config(module, Config), 337 | Opts = ?config(opts, Config), 338 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 339 | 340 | {ok, "hello"} = gen_buffer_dist:send_recv(?BUFFER, "hello"). 341 | 342 | t_gen_buffer_worker_missing_ets_data(Config) -> 343 | Mod = ?config(module, Config), 344 | Opts = ?config(opts, Config), 345 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 346 | 347 | _ = process_flag(trap_exit, true), 348 | Pids = [spawn_link(?MODULE, producer, [Mod, ?BUFFER]) || _ <- lists:seq(1, 20)], 349 | _ = timer:sleep(10000), 350 | ok = lists:foreach(fun(Pid) -> exit(Pid, normal) end, Pids), 351 | Mod:stop(?BUFFER). 352 | 353 | t_gen_buffer_lib_missing_funs(Config) -> 354 | Mod = ?config(module, Config), 355 | Opts = ?config(opts, Config), 356 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 357 | 358 | undefined = gen_buffer_lib:get_metadata_value(?BUFFER, hello), 359 | hello = gen_buffer_lib:get_one_metadata_value(?BUFFER, hello, hello). 360 | 361 | %%%=================================================================== 362 | %%% Helpers 363 | %%%=================================================================== 364 | 365 | producer(Mod, Buffer) -> 366 | _ = Mod:send(Buffer, "hello"), 367 | ok = Mod:poll(Buffer), 368 | producer(Mod, Buffer). 369 | 370 | %%%=================================================================== 371 | %%% Internal functions 372 | %%%=================================================================== 373 | 374 | cleanup_buffers(Mod) -> 375 | lists:foreach(fun(Ch) -> 376 | Mod:stop(Ch) 377 | end, [?BUFFER, test, test2, test3]). 378 | -------------------------------------------------------------------------------- /test/gen_buffer_dist_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_dist_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% Common Test 6 | -export([ 7 | all/0, 8 | init_per_suite/1, 9 | end_per_suite/1, 10 | init_per_testcase/2, 11 | end_per_testcase/2 12 | ]). 13 | 14 | %% Common Test Cases 15 | -include_lib("mixer/include/mixer.hrl"). 16 | -mixin([ 17 | {gen_buffer_test_cases, [ 18 | t_eval/1, 19 | t_eval_error/1, 20 | t_send_and_recv_errors/1, 21 | t_send_recv/1, 22 | t_fire_and_forget/1 23 | ]} 24 | ]). 25 | 26 | %% Test Cases 27 | -export([ 28 | t_send_and_recv/1, 29 | t_get_set_workers/1, 30 | t_size/1, 31 | t_info_buffer/1, 32 | t_info/1, 33 | t_no_available_nodes/1 34 | ]). 35 | 36 | -define(EXCLUDED_FUNS, [ 37 | module_info, 38 | all, 39 | init_per_suite, 40 | end_per_suite, 41 | init_per_testcase, 42 | end_per_testcase 43 | ]). 44 | 45 | -define(BUFFER, gen_buffer_test). 46 | -define(SLAVES, ['node1@127.0.0.1', 'node2@127.0.0.1']). 47 | 48 | %%%=================================================================== 49 | %%% Common Test 50 | %%%=================================================================== 51 | 52 | all() -> 53 | Exports = ?MODULE:module_info(exports), 54 | [F || {F, _} <- Exports, not lists:member(F, ?EXCLUDED_FUNS)]. 55 | 56 | init_per_suite(Config) -> 57 | ok = start_primary_node(), 58 | {ok, _} = application:ensure_all_started(gen_buffer), 59 | ok = allow_boot(), 60 | Nodes = start_slaves(?SLAVES), 61 | 62 | Opts = #{ 63 | message_handler => test_message_handler, 64 | send_replies => true, 65 | init_args => ok 66 | }, 67 | 68 | [{nodes, Nodes}, {opts, Opts}, {module, gen_buffer_dist} | Config]. 69 | 70 | end_per_suite(Config) -> 71 | stop_slaves(?SLAVES), 72 | Config. 73 | 74 | init_per_testcase(_, Config) -> 75 | Config. 76 | 77 | end_per_testcase(_, Config) -> 78 | ok = cleanup_remote_buffers(), 79 | Config. 80 | 81 | %%%=================================================================== 82 | %%% Tests Cases 83 | %%%=================================================================== 84 | 85 | t_send_and_recv(Config) -> 86 | Mod = ?config(module, Config), 87 | Opts = ?config(opts, Config), 88 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 89 | 90 | Ref1 = Mod:send(?BUFFER, "hello"), 91 | {reply, Ref1, ?BUFFER, "hello"} = gen_buffer_ct:wait_for_msg(), 92 | 93 | Ref2 = Mod:send(?BUFFER, "hello"), 94 | {ok, "hello"} = Mod:recv(?BUFFER, Ref2), 95 | 96 | ok = Mod:poll(?BUFFER), 97 | {error, timeout} = gen_buffer_ct:wait_for_msg(200), 98 | 99 | ok = Mod:stop(?BUFFER), 100 | ok = Mod:stop(test). 101 | 102 | t_get_set_workers(Config) -> 103 | Mod = ?config(module, Config), 104 | Opts = ?config(opts, Config), 105 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 106 | 107 | [ 108 | {'ct@127.0.0.1', Workers1_N1}, 109 | {'node1@127.0.0.1', Workers1_N2}, 110 | {'node2@127.0.0.1', Workers1_N3} 111 | ] = lists:usort(Mod:get_workers(?BUFFER)), 112 | 113 | Len1 = erlang:system_info(schedulers_online), 114 | ok = lists:foreach(fun(WL) -> 115 | Len1 = length(WL) 116 | end, [Workers1_N1, Workers1_N2, Workers1_N3]), 117 | 118 | [ 119 | {'ct@127.0.0.1', Workers2_N1}, 120 | {'node1@127.0.0.1', Workers2_N2}, 121 | {'node2@127.0.0.1', Workers2_N3} 122 | ] = lists:usort(Mod:set_workers(?BUFFER, 3)), 123 | 124 | ok = lists:foreach(fun(WL) -> 125 | 3 = length(WL) 126 | end, [Workers2_N1, Workers2_N2, Workers2_N3]), 127 | 128 | {ok, _} = Mod:get_worker(?BUFFER). 129 | 130 | t_size(Config) -> 131 | Mod = ?config(module, Config), 132 | Opts = ?config(opts, Config), 133 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 134 | 135 | [ 136 | {'ct@127.0.0.1', Size}, 137 | {'node1@127.0.0.1', _}, 138 | {'node2@127.0.0.1', _} 139 | ] = lists:usort(Mod:size(?BUFFER)), 140 | 141 | true = is_integer(Size). 142 | 143 | t_info_buffer(Config) -> 144 | Mod = ?config(module, Config), 145 | Opts = ?config(opts, Config), 146 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 147 | 148 | [ 149 | {'ct@127.0.0.1', Data}, 150 | {'node1@127.0.0.1', _}, 151 | {'node2@127.0.0.1', _} 152 | ] = lists:usort(Mod:info(?BUFFER)), 153 | 154 | #{workers := _, size := _} = Data. 155 | 156 | t_info(Config) -> 157 | Mod = ?config(module, Config), 158 | Opts = ?config(opts, Config), 159 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 160 | _ = gen_buffer_ct:create_buffer(test, Opts, Mod, Config), 161 | 162 | ok = pg2:create(yet_another_group), 163 | 164 | [ 165 | {'ct@127.0.0.1', Data}, 166 | {'node1@127.0.0.1', _}, 167 | {'node2@127.0.0.1', _} 168 | ] = lists:usort(Mod:info()), 169 | 170 | #{ 171 | ?BUFFER := #{workers := _, size := _}, 172 | test := #{workers := _, size := _} 173 | } = Data. 174 | 175 | t_no_available_nodes(Config) -> 176 | Mod = ?config(module, Config), 177 | Opts = ?config(opts, Config), 178 | 179 | ok = pg2:delete(gen_buffer:pg2_namespace(?BUFFER)), 180 | try 181 | Mod:send(?BUFFER, "hello") 182 | catch 183 | error:no_available_nodes -> ok 184 | end, 185 | 186 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 187 | _ = Mod:send(?BUFFER, "hello"). 188 | 189 | %%%=================================================================== 190 | %%% Internal functions 191 | %%%=================================================================== 192 | 193 | %% @private 194 | start_primary_node() -> 195 | {ok, _} = net_kernel:start(['ct@127.0.0.1']), 196 | true = erlang:set_cookie(node(), gen_buffer), 197 | ok. 198 | 199 | %% @private 200 | allow_boot() -> 201 | _ = erl_boot_server:start([]), 202 | {ok, IPv4} = inet:parse_ipv4_address("127.0.0.1"), 203 | erl_boot_server:add_slave(IPv4). 204 | 205 | %% @private 206 | start_slaves(Slaves) -> 207 | start_slaves(Slaves, []). 208 | 209 | %% @private 210 | start_slaves([], Acc) -> 211 | lists:usort(Acc); 212 | 213 | start_slaves([Node | T], Acc) -> 214 | start_slaves(T, [spawn_node(Node) | Acc]). 215 | 216 | %% @private 217 | spawn_node(Node) -> 218 | Cookie = atom_to_list(erlang:get_cookie()), 219 | InetLoaderArgs = "-loader inet -hosts 127.0.0.1 -setcookie " ++ Cookie, 220 | 221 | {ok, Node} = 222 | slave:start( 223 | "127.0.0.1", 224 | node_name(Node), 225 | InetLoaderArgs 226 | ), 227 | 228 | ok = rpc:block_call(Node, code, add_paths, [code:get_path()]), 229 | {ok, _} = rpc:block_call(Node, application, ensure_all_started, [gen_buffer]), 230 | ok = load_support_files(Node), 231 | Node. 232 | 233 | %% @private 234 | node_name(Node) -> 235 | [Name, _] = binary:split(atom_to_binary(Node, utf8), <<"@">>), 236 | binary_to_atom(Name, utf8). 237 | 238 | %% @private 239 | load_support_files(Node) -> 240 | {module, gen_buffer_test_cases} = 241 | rpc:block_call(Node, code, load_file, [gen_buffer_test_cases]), 242 | ok. 243 | 244 | %% @private 245 | stop_slaves(Slaves) -> 246 | stop_slaves(Slaves, []). 247 | 248 | %% @private 249 | stop_slaves([], Acc) -> 250 | lists:usort(Acc); 251 | stop_slaves([Node | T], Acc) -> 252 | ok = slave:stop(Node), 253 | pang = net_adm:ping(Node), 254 | stop_slaves(T, [Node | Acc]). 255 | 256 | cleanup_remote_buffers() -> 257 | _ = register(ct, self()), 258 | Buffers = [parent_gen_buffer_test, parent_test, parent_test2], 259 | [begin 260 | {Name, Node} ! exit, 261 | gen_buffer_ct:wait_for_msg(300) 262 | end || Name <- Buffers, Node <- ?SLAVES], 263 | ok. 264 | -------------------------------------------------------------------------------- /test/gen_buffer_meta_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_meta_SUITE). 2 | 3 | -include_lib("mixer/include/mixer.hrl"). 4 | -mixin([ktn_meta_SUITE]). 5 | 6 | -export([init_per_suite/1, end_per_suite/1]). 7 | 8 | -spec init_per_suite(xdb_ct:config()) -> xdb_ct:config(). 9 | init_per_suite(Config) -> 10 | Dialyzer = {dialyzer_warnings, [ 11 | no_return, 12 | unmatched_returns, 13 | error_handling, 14 | unknown 15 | ]}, 16 | 17 | [{application, gen_buffer}, {dirs, ["ebin"]}, Dialyzer | Config]. 18 | 19 | -spec end_per_suite(xdb_ct:config()) -> ok. 20 | end_per_suite(_) -> 21 | ok. 22 | -------------------------------------------------------------------------------- /test/gen_buffer_partitioned_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_partitioned_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% Common Test 6 | -export([ 7 | all/0, 8 | init_per_suite/1, 9 | end_per_suite/1, 10 | init_per_testcase/2, 11 | end_per_testcase/2 12 | ]). 13 | 14 | %% Common Test Cases 15 | -include_lib("mixer/include/mixer.hrl"). 16 | -mixin([ 17 | {gen_buffer_test_cases, [ 18 | t_eval/1, 19 | t_eval_error/1, 20 | t_send_and_recv/1, 21 | t_send_and_recv_errors/1, 22 | t_send_recv/1, 23 | t_fire_and_forget/1, 24 | t_add_del_workers/1, 25 | t_set_workers/1, 26 | t_size/1, 27 | t_info_buffer/1, 28 | t_info/1, 29 | t_worker_polling/1, 30 | t_worker_distribution/1 31 | ]} 32 | ]). 33 | 34 | -ifndef('CI'). 35 | 36 | %% Test Cases 37 | -export([ 38 | t_load_balancing/1 39 | ]). 40 | 41 | -endif. 42 | 43 | -define(EXCLUDED_FUNS, [ 44 | module_info, 45 | all, 46 | init_per_suite, 47 | end_per_suite, 48 | init_per_testcase, 49 | end_per_testcase, 50 | producer 51 | ]). 52 | 53 | -define(BUFFER, gen_buffer_test). 54 | 55 | %%%=================================================================== 56 | %%% Common Test 57 | %%%=================================================================== 58 | 59 | all() -> 60 | Exports = ?MODULE:module_info(exports), 61 | [F || {F, _} <- Exports, not lists:member(F, ?EXCLUDED_FUNS)]. 62 | 63 | init_per_suite(Config) -> 64 | Opts = #{ 65 | message_handler => test_message_handler, 66 | send_replies => true, 67 | init_args => ok, 68 | n_partitions => erlang:system_info(schedulers_online) 69 | }, 70 | 71 | [{opts, Opts}, {module, gen_buffer} | Config]. 72 | 73 | end_per_suite(Config) -> 74 | Config. 75 | 76 | init_per_testcase(_, Config) -> 77 | Config. 78 | 79 | end_per_testcase(_, Config) -> 80 | Config. 81 | 82 | %%%=================================================================== 83 | %%% Tests Cases 84 | %%%=================================================================== 85 | 86 | -ifndef('CI'). 87 | 88 | t_load_balancing(Config) -> 89 | Mod = ?config(module, Config), 90 | Opts = ?config(opts, Config), 91 | _ = gen_buffer_ct:create_buffer(?BUFFER, Opts#{workers => 0}, Mod, Config), 92 | 93 | ok = lists:foreach(fun(M) -> 94 | Mod:send(?BUFFER, {self(), M}) 95 | end, lists:seq(1, 100)), 96 | 97 | ok = lists:foreach(fun(P) -> 98 | true = ets:info(gen_buffer_lib:partition_name(?BUFFER, P), size) > 5 99 | end, lists:seq(0, 3)), 100 | 101 | {ok, [_, _, _, _]} = Mod:set_workers(?BUFFER, 4), 102 | ok = Mod:poll(?BUFFER), 103 | _ = timer:sleep(2000), 104 | 105 | lists:foreach(fun(P) -> 106 | 1 = ets:info(gen_buffer_lib:partition_name(?BUFFER, P), size) 107 | end, lists:seq(0, 3)). 108 | 109 | -endif. 110 | -------------------------------------------------------------------------------- /test/shared/gen_buffer_test_cases.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_test_cases). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% Test Cases 6 | -export([ 7 | t_eval/1, 8 | t_eval_error/1, 9 | t_send_and_recv/1, 10 | t_send_and_recv_errors/1, 11 | t_send_recv/1, 12 | t_fire_and_forget/1, 13 | t_add_del_workers/1, 14 | t_set_workers/1, 15 | t_size/1, 16 | t_info_buffer/1, 17 | t_info/1, 18 | t_worker_polling/1, 19 | t_worker_distribution/1 20 | ]). 21 | 22 | -define(BUFFER, gen_buffer_test). 23 | 24 | %%%=================================================================== 25 | %%% Tests Cases 26 | %%%=================================================================== 27 | 28 | t_eval(Config) -> 29 | Mod = ?config(module, Config), 30 | Opts = ?config(opts, Config), 31 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 32 | 33 | "hello" = Mod:eval(?BUFFER, "hello"), 34 | 1 = Mod:eval(?BUFFER, 1), 35 | {ok, "hello"} = Mod:eval(?BUFFER, {ok, "hello"}), 36 | 37 | try Mod:eval(?BUFFER, error) 38 | catch 39 | exit:{{handler_exception, _}, _} -> ok 40 | end. 41 | 42 | t_eval_error(Config) -> 43 | Mod = ?config(module, Config), 44 | Opts = ?config(opts, Config), 45 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts#{workers => 0}, Mod, Config), 46 | 47 | no_available_workers = Mod:eval(?BUFFER, "hello", 5), 48 | 49 | true = register(ct, self()), 50 | _ = process_flag(trap_exit, true), 51 | 52 | Pid = 53 | spawn_link(fun() -> 54 | Res = Mod:eval(?BUFFER, "hello", 0), 55 | ct ! Res 56 | end), 57 | 58 | {error, timeout} = gen_buffer_ct:wait_for_msg(1000), 59 | true = unregister(ct), 60 | true = exit(Pid, normal). 61 | 62 | t_send_and_recv(Config) -> 63 | Mod = ?config(module, Config), 64 | Opts = ?config(opts, Config), 65 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 66 | 67 | Ref1 = Mod:send(?BUFFER, "hello"), 68 | {reply, Ref1, ?BUFFER, "hello"} = gen_buffer_ct:wait_for_msg(), 69 | 70 | Ref2 = Mod:send(?BUFFER, "hello"), 71 | {ok, "hello"} = Mod:recv(?BUFFER, Ref2). 72 | 73 | t_send_and_recv_errors(Config) -> 74 | Mod = ?config(module, Config), 75 | Opts = ?config(opts, Config), 76 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 77 | 78 | _ = Mod:send(?BUFFER, error), 79 | {error, _, ?BUFFER, handler_exception} = gen_buffer_ct:wait_for_msg(1000), 80 | 81 | {ok, _} = gen_buffer_ct:create_buffer(test, Opts#{send_replies => false}, Mod, Config), 82 | 83 | Ref = Mod:send(test, error), 84 | {error, timeout} = Mod:recv(test, Ref, 1000). 85 | 86 | t_send_recv(Config) -> 87 | Mod = ?config(module, Config), 88 | Opts = ?config(opts, Config), 89 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 90 | 91 | {ok, "hello"} = Mod:send_recv(?BUFFER, "hello"), 92 | 93 | Opts1 = Opts#{message_handler => test_message_handler4}, 94 | {ok, _} = gen_buffer_ct:create_buffer(test, Opts1, Mod, Config), 95 | 96 | {error, handler_exception} = Mod:send_recv(test, "hello"), 97 | 98 | Opts2 = Opts#{message_handler => test_message_handler4, send_replies => false}, 99 | {ok, _} = gen_buffer_ct:create_buffer(test2, Opts2, Mod, Config), 100 | 101 | {error, timeout} = Mod:send_recv(test2, "hello", 1000). 102 | 103 | t_fire_and_forget(Config) -> 104 | Mod = ?config(module, Config), 105 | Opts = (?config(opts, Config))#{message_handler => test_message_handler4, send_replies => false}, 106 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 107 | 108 | Ref = Mod:send(?BUFFER, "hello"), 109 | {error, timeout} = Mod:recv(?BUFFER, Ref, 500), 110 | 111 | {ok, _} = 112 | gen_buffer_ct:create_buffer( 113 | test, Opts#{message_handler => test_message_handler4}, Mod, Config), 114 | 115 | Ref2 = Mod:send(test, {self(), "hello"}), 116 | {error, timeout} = Mod:recv(?BUFFER, Ref2, 500). 117 | 118 | t_add_del_workers(Config) -> 119 | Mod = ?config(module, Config), 120 | Opts = maps:to_list(?config(opts, Config)), 121 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 122 | Workers = num_workers(Mod, ?BUFFER), 123 | 124 | ok = lists:foreach(fun(_) -> 125 | Mod:incr_worker(?BUFFER) 126 | end, lists:seq(1, 2)), 127 | 128 | ok = lists:foreach(fun(_) -> 129 | Mod:incr_worker(?BUFFER, Opts) 130 | end, lists:seq(3, 5)), 131 | 132 | Workers1 = num_workers(Mod, ?BUFFER), 133 | Workers1 = Workers + 5, 134 | 135 | ok = Mod:decr_worker(?BUFFER), 136 | ok = Mod:decr_worker(wrong_buffer), 137 | 138 | Workers2 = num_workers(Mod, ?BUFFER), 139 | Workers2 = Workers + 4, 140 | 141 | ok = lists:foreach(fun(_) -> 142 | Mod:decr_worker(?BUFFER) 143 | end, lists:seq(1, Workers2)), 144 | 145 | {error, no_available_buffer} = Mod:get_worker(wrong_buffer), 146 | {error, no_available_buffer} = Mod:get_workers(wrong_buffer), 147 | {error, no_available_buffer} = Mod:set_workers(wrong_buffer, 10, Opts), 148 | {error, no_available_workers} = Mod:decr_worker(?BUFFER), 149 | {error, no_available_workers} = Mod:poll(?BUFFER). 150 | 151 | t_set_workers(Config) -> 152 | Mod = ?config(module, Config), 153 | Opts = maps:to_list(?config(opts, Config)), 154 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 155 | Workers = num_workers(Mod, ?BUFFER), 156 | 157 | {ok, WorkersL1} = Mod:set_workers(?BUFFER, Workers + 2), 158 | Workers1 = length(WorkersL1), 159 | Workers1 = Workers + 2, 160 | 161 | {ok, WorkersL2} = Mod:set_workers(?BUFFER, Workers1 - 3, Opts), 162 | Workers2 = length(WorkersL2), 163 | Workers2 = Workers - 1, 164 | 165 | try 166 | Mod:set_workers(?BUFFER, -1, Opts) 167 | catch 168 | error:function_clause -> ok 169 | end. 170 | 171 | t_size(Config) -> 172 | Mod = ?config(module, Config), 173 | Opts = ?config(opts, Config), 174 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts#{workers => 0}, Mod, Config), 175 | 176 | 0 = Mod:size(?BUFFER), 177 | 178 | ok = lists:foreach(fun(_) -> 179 | Mod:send(?BUFFER, "hello") 180 | end, lists:seq(1, 10)), 181 | 182 | 10 = Mod:size(?BUFFER), 183 | undefined = Mod:size(undefined). 184 | 185 | t_info_buffer(Config) -> 186 | Mod = ?config(module, Config), 187 | Opts = ?config(opts, Config), 188 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 189 | 190 | #{workers := W, size := QS} = Mod:info(?BUFFER), 191 | true = is_integer(W), 192 | true = is_integer(QS), 193 | undefined = Mod:info(undefined). 194 | 195 | t_info(Config) -> 196 | Mod = ?config(module, Config), 197 | Opts = ?config(opts, Config), 198 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts, Mod, Config), 199 | {ok, _} = gen_buffer_ct:create_buffer(test, Opts, Mod, Config), 200 | {ok, _} = gen_buffer_ct:create_buffer(test2, Opts#{workers => 0}, Mod, Config), 201 | ok = pg2:create(yet_another_group), 202 | 203 | InfoMap = Mod:info(), 204 | 3 = map_size(InfoMap), 205 | 206 | #{ 207 | ?BUFFER := #{workers := W, size := QS}, 208 | test := #{workers := _, size := _}, 209 | test2 := #{workers := 0, size := _} 210 | } = InfoMap, 211 | 212 | true = is_integer(W), 213 | true = is_integer(QS). 214 | 215 | t_worker_polling(Config) -> 216 | true = register(ct, self()), 217 | Mod = ?config(module, Config), 218 | 219 | Opts = ?config(opts, Config), 220 | 221 | Opts1 = Opts#{ 222 | workers => 0, 223 | send_replies => false, 224 | message_handler => test_message_handler6 225 | }, 226 | 227 | {ok, _} = gen_buffer_ct:create_buffer(?BUFFER, Opts1, Mod, Config), 228 | 229 | 0 = Mod:size(?BUFFER), 230 | 231 | ok = lists:foreach(fun(_) -> 232 | Mod:send(?BUFFER, "hello") 233 | end, lists:seq(1, 5)), 234 | 235 | 5 = Mod:size(?BUFFER), 236 | 237 | _ = Mod:set_workers(?BUFFER, 1), 238 | _ = timer:sleep(500), 239 | _ = Mod:poll(?BUFFER), 240 | 241 | ok = lists:foreach(fun(_) -> 242 | "hello" = gen_buffer_ct:wait_for_msg(1000) 243 | end, lists:seq(1, 5)), 244 | 245 | 0 = Mod:size(?BUFFER), 246 | true = unregister(ct). 247 | 248 | t_worker_distribution(Config) -> 249 | Mod = ?config(module, Config), 250 | Opts = ?config(opts, Config), 251 | 252 | {ok, _} = 253 | gen_buffer_ct:create_buffer( 254 | ?BUFFER, Opts#{message_handler => test_message_handler5}, Mod, Config), 255 | 256 | {ok, CurrentWorkers} = gen_buffer:get_workers(?BUFFER), 257 | NumMsgs = length(CurrentWorkers) * 100, 258 | Refs = [Mod:send(?BUFFER, M) || M <- lists:seq(1, NumMsgs)], 259 | no_available_workers = gen_buffer_lib:get_available_worker(?BUFFER), 260 | 261 | Replies = 262 | lists:foldl(fun(_, Acc) -> 263 | receive 264 | {reply, Ref, ?BUFFER, {_, Worker}} -> 265 | case lists:member(Ref, Refs) of 266 | true -> 267 | Count = maps:get(Worker, Acc, 0), 268 | Acc#{Worker => Count + 1}; 269 | 270 | false -> 271 | Acc 272 | end; 273 | 274 | Error -> 275 | error(Error) 276 | after 277 | 5000 -> error(timeout) 278 | end 279 | end, #{}, lists:seq(1, NumMsgs)), 280 | 281 | _ = timer:sleep(1500), 282 | {ok, Workers1} = gen_buffer:get_workers(?BUFFER), 283 | Workers2 = lists:usort(gen_buffer_lib:get_available_workers(?BUFFER)), 284 | Workers2 = lists:usort(Workers1), 285 | WorkersLen = maps:size(Replies), 286 | WorkersLen = length(Workers1), 287 | WorkersLen = length(Workers2), 288 | 289 | lists:foreach(fun(Worker) -> 290 | true = maps:get(Worker, Replies) > 1 291 | end, Workers2). 292 | 293 | %%%=================================================================== 294 | %%% Internal functions 295 | %%%=================================================================== 296 | 297 | num_workers(Mod, Buffer) -> 298 | {ok, Workers} = Mod:get_workers(Buffer), 299 | length(Workers). 300 | -------------------------------------------------------------------------------- /test/support/gen_buffer_ct.erl: -------------------------------------------------------------------------------- 1 | -module(gen_buffer_ct). 2 | 3 | -export([ 4 | wait_for_msg/0, 5 | wait_for_msg/1, 6 | messages/0, 7 | messages/1, 8 | create_buffer/4 9 | ]). 10 | 11 | %%%=================================================================== 12 | %%% API 13 | %%%=================================================================== 14 | 15 | wait_for_msg() -> 16 | wait_for_msg(infinity). 17 | 18 | wait_for_msg(Timeout) -> 19 | receive 20 | Msg -> Msg 21 | after 22 | Timeout -> {error, timeout} 23 | end. 24 | 25 | messages() -> 26 | messages(self()). 27 | 28 | messages(Pid) -> 29 | {messages, Messages} = erlang:process_info(Pid, messages), 30 | Messages. 31 | 32 | create_buffer(Buffer, Opts, gen_buffer_dist, Config) -> 33 | {ok, _} = gen_buffer_dist:start_link(Buffer, Opts), 34 | {nodes, Nodes} = lists:keyfind(nodes, 1, Config), 35 | Parent = {ct, 'ct@127.0.0.1'}, 36 | 37 | Fun = fun() -> 38 | Name = list_to_atom("parent_" ++ atom_to_list(Buffer)), 39 | _ = register(Name, self()), 40 | 41 | case gen_buffer:start_link(Buffer, Opts) of 42 | {ok, _} -> 43 | ok; 44 | 45 | {error, {already_started, _}} -> 46 | ok = gen_buffer:start_link(Buffer), 47 | {ok, _} = gen_buffer:start_link(Buffer, Opts) 48 | end, 49 | 50 | receive 51 | exit -> Parent ! {exit, Name} 52 | end 53 | end, 54 | 55 | {ResL, []} = rpc:multicall(Nodes, erlang, spawn_link, [Fun]), 56 | _ = timer:sleep(1000), 57 | {ok, ResL}; 58 | 59 | create_buffer(Buffer, Opts, Mod, _) -> 60 | Mod:start_link(Buffer, Opts). 61 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler). 2 | 3 | -behaviour(gen_buffer). 4 | 5 | -export([ 6 | init/1, 7 | handle_message/3, 8 | handle_info/3, 9 | terminate/3 10 | ]). 11 | 12 | init({test, Result}) -> 13 | Result; 14 | init(_Args) -> 15 | _ = process_flag(trap_exit, true), 16 | {ok, #{messages => []}}. 17 | 18 | handle_message(_Buffer, error, _State) -> 19 | error(handler_exception); 20 | handle_message(_Buffer, messages, #{messages := MsgL} = State) -> 21 | {ok, MsgL, State}; 22 | handle_message(_Buffer, Msg, #{messages := MsgL} = State) -> 23 | {ok, Msg, State#{messages := [Msg | MsgL]}}. 24 | 25 | handle_info(_Buffer, {'EXIT', _From, Reason}, State) -> 26 | {stop, Reason, State}; 27 | handle_info(_Buffer, throw, State) -> 28 | throw({stop, throw, State}); 29 | handle_info(_Buffer, Info, _State) -> 30 | Info. 31 | 32 | terminate(_Buffer, throw, _State) -> 33 | throw(ok); 34 | terminate(_Buffer, _Info, _State) -> 35 | ok. 36 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler2.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler2). 2 | 3 | -behaviour(gen_buffer). 4 | 5 | -export([ 6 | init/1, 7 | handle_message/3, 8 | handle_info/3 9 | ]). 10 | 11 | init(_Args) -> 12 | _ = process_flag(trap_exit, true), 13 | {ok, #{}}. 14 | 15 | handle_message(_BufferName, _Msg, _State) -> 16 | error(handler_exception). 17 | 18 | handle_info(_Buffer, _Info, _State) -> 19 | throw(handler_exception). 20 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler3.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler3). 2 | 3 | -export([ 4 | handle_message/2 5 | ]). 6 | 7 | handle_message(_BufferName, _Msg) -> 8 | error(handler_exception). 9 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler4.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler4). 2 | 3 | -behaviour(gen_buffer). 4 | 5 | -export([ 6 | handle_message/3, 7 | handle_info/3 8 | ]). 9 | 10 | handle_message(_BufferName, _Msg, _State) -> 11 | error(handler_exception). 12 | 13 | handle_info(_Buffer, Info, _State) -> 14 | _ = timer:sleep(5000), 15 | {noreply, Info}. 16 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler5.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler5). 2 | 3 | -behaviour(gen_buffer). 4 | 5 | -export([ 6 | handle_message/3 7 | ]). 8 | 9 | handle_message(_Buffer, Msg, State) -> 10 | _ = timer:sleep(500), 11 | {ok, {Msg, self()}, State}. 12 | -------------------------------------------------------------------------------- /test/support/handlers/test_message_handler6.erl: -------------------------------------------------------------------------------- 1 | -module(test_message_handler6). 2 | 3 | -behaviour(gen_buffer). 4 | 5 | -export([ 6 | handle_message/3 7 | ]). 8 | 9 | handle_message(_Buffer, Msg, State) -> 10 | ct ! Msg, 11 | {ok, Msg, State}. 12 | --------------------------------------------------------------------------------