├── .github ├── FUNDING.yml └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── TODO.org ├── doc ├── README.md ├── edoc-info ├── erlang.png ├── erlang07g-wiger.pdf ├── jobs.md ├── jobs_app.md ├── jobs_info.md ├── jobs_lib.md ├── jobs_prod_simple.md ├── jobs_queue.md ├── jobs_queue_list.md ├── jobs_sampler.md ├── jobs_sampler_cpu.md ├── jobs_sampler_history.md ├── jobs_sampler_mnesia.md ├── jobs_server.md ├── jobs_stateful_simple.md ├── overview.edoc └── stylesheet.css ├── examples ├── jobs_cpu.gnu ├── performance_logger.erl └── performer.erl ├── include └── jobs.hrl ├── rebar.config ├── rebar.lock ├── rebar3 ├── src ├── jobs.app.src ├── jobs.erl ├── jobs_app.erl ├── jobs_info.erl ├── jobs_lib.erl ├── jobs_prod_simple.erl ├── jobs_queue.erl ├── jobs_queue_list.erl ├── jobs_sampler.erl ├── jobs_sampler_cpu.erl ├── jobs_sampler_history.erl ├── jobs_sampler_mnesia.erl ├── jobs_server.erl └── jobs_stateful_simple.erl └── test ├── jobs_eqc_queue.erl ├── jobs_queue_model.erl ├── jobs_sampler_slave.erl ├── jobs_server_tests.erl ├── jobs_tests.erl └── t.erl /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [uwiger] 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: [ $default-branch ] 7 | release: 8 | types: 9 | - created 10 | 11 | jobs: 12 | test: 13 | name: "Erlang Test" 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | otp: [21, 22, 23, 24] 18 | fail-fast: false 19 | container: 20 | image: erlang:${{ matrix.otp }} 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Test 24 | run: make ci 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /deps/ 2 | /deps/ 3 | *.beam 4 | ebin/jobs.app 5 | 6 | /.jobs.plt 7 | .eunit 8 | .rebar 9 | _build 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REBAR3=$(shell which rebar3 || echo ./rebar3) 2 | 3 | .PHONY: all test clean doc dialyzer 4 | 5 | all: compile 6 | 7 | compile: 8 | $(REBAR3) compile 9 | 10 | test: all 11 | $(REBAR3) eunit 12 | 13 | ci: test xref dialyzer 14 | 15 | clean: 16 | $(REBAR3) clean 17 | 18 | doc: 19 | $(REBAR3) doc 20 | 21 | xref: 22 | $(REBAR3) xref 23 | 24 | dialyzer: 25 | $(REBAR3) dialyzer 26 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Jobs, copyright 2014 Ulf Wiger 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # jobs - a Job scheduler for load regulation # 4 | 5 | Copyright (c) 2014-2018 Ulf Wiger 6 | 7 | __Version:__ 0.9.0 8 | 9 | JOBS 10 | ==== 11 | 12 | Jobs is a job scheduler for load regulation of Erlang applications. 13 | It provides a queueing framework where each queue can be configured 14 | for throughput rate, credit pool and feedback compensation. 15 | Queues can be added and modified at runtime, and customizable 16 | "samplers" propagate load status across all nodes in the system. 17 | 18 | Specifically, jobs provides three features: 19 | 20 | * Job scheduling: A job is scheduled according to certain constraints. 21 | For instance, you may want to define that no more than 9 jobs of a 22 | certain type can execute simultaneously and the maximal rate at 23 | which you can start such jobs are 300 per second. 24 | * Job queueing: When load is higher than the scheduling limits 25 | additional jobs are *queued* by the system to be run later when load 26 | clears. Certain rules govern queues: are they dequeued in FIFO or 27 | LIFO order? How many jobs can the queue take before it is full? Is 28 | there a deadline after which jobs should be rejected. When we hit 29 | the queue limits we reject the job. This provides a feedback 30 | mechanism on the client of the queue so you can take action. 31 | * Sampling and dampening: Periodic samples of the Erlang VM can 32 | provide information about the health of the system in general. If we 33 | have high CPU load or high memory usage, we apply dampening to the 34 | scheduling rules: we may lower the concurrency count or the rate at 35 | which we execute jobs. When the health problem clears, we remove the 36 | dampener and run at full speed again. 37 | 38 | Error recovery 39 | -------------- 40 | 41 | The Jobs server is designed to not crash. However, in the unlikely event 42 | that it should occur (and it has!) Jobs does not automatically restore changes 43 | that have been effected through the API. This can be enabled, setting the 44 | Jobs environment variable `auto_restore` to `true`, or calling the function 45 | `jobs_server:auto_restore(true)`. This will tell the jobs_server to remember 46 | every configuration change and replay them, in order, after a process restart. 47 | 48 | Examples 49 | -------- 50 | 51 | The following examples are fetched from the EUC 2013 presentation on Jobs. 52 | 53 | 54 | #### Regulate incoming HTTP requests (e.g. JSON-RPC) #### 55 | 56 | ```erlang 57 | 58 | %% @doc Handle a JSON-RPC request. 59 | handler_session(Arg) -> 60 | jobs:run( 61 | rpc_from_web, 62 | fun() -> 63 | try 64 | yaws_rpc:handler_session( 65 | maybe_multipart(Arg),{?MODULE, web_rpc}) 66 | catch 67 | error:E -> 68 | ... 69 | end 70 | end). 71 | 72 | ``` 73 | 74 | 75 | #### From Riak prototype, using explicit ask/done #### 76 | 77 | ```erlang 78 | 79 | case jobs:ask(riak_kv_fsm) of 80 | {ok, JobId} -> 81 | try 82 | {ok, Pid} = riak_kv_get_fsm_sup:start_get_fsm(...), 83 | Timeout = recv_timeout(Options), 84 | wait_for_reqid(ReqId, Timeout) 85 | after 86 | jobs:done(JobId) %% Only needed if process stays alive 87 | end; 88 | {error, rejected} -> %% Overload! 89 | {error, timeout} 90 | end 91 | 92 | ``` 93 | 94 | 95 | #### Shell demo - simple rate-limited queue #### 96 | 97 | ```erlang 98 | 99 | 2> jobs:add_queue(q, [{standard_rate,1}]). 100 | ok 101 | 3> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 102 | job: {14,37,7} 103 | ok 104 | 4> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 105 | job: {14,37,8} 106 | ok 107 | ... 108 | 5> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 109 | job: {14,37,10} 110 | ok 111 | 6> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 112 | job: {14,37,11} 113 | ok 114 | 115 | ``` 116 | 117 | 118 | #### Shell demo - "stateful" queues #### 119 | 120 | ```erlang 121 | 122 | Eshell V5.9.2 (abort with ^G) 123 | 1> application:start(jobs). 124 | ok 125 | 2> jobs:add_queue(q, 126 | [{standard_rate,1}, 127 | {stateful,fun(init,_) -> {0,5}; 128 | ({call,{size,Sz},_,_},_) -> {reply,ok,{0,Sz}}; 129 | ({N,Sz},_) -> {N, {(N+1) rem Sz,Sz}} 130 | end}]). 131 | ok 132 | 3> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 133 | 0 134 | 4> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 135 | 1 136 | 5> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 137 | 2 138 | 6> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 139 | 3 140 | 7> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 141 | 4 142 | 8> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 143 | 0 144 | 9> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 145 | 1 146 | %% Resize the 'pool' 147 | 10> jobs:ask_queue(q, {size,3}). 148 | ok 149 | 11> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 150 | 0 151 | 12> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 152 | 1 153 | 13> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 154 | 2 155 | 14> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 156 | 0 157 | ... 158 | 159 | ``` 160 | 161 | 162 | #### Demo - producers #### 163 | 164 | ```erlang 165 | 166 | Eshell V5.9.2 (abort with ^G) 167 | 1> application:start(jobs). 168 | ok 169 | 2> jobs:add_queue(p, 170 | [{producer, fun() -> io:fwrite("job: ~p~n",[time()]) end}, 171 | {standard_rate,1}]). 172 | job: {14,33,51} 173 | ok 174 | 3> job: {14,33,52} 175 | job: {14,33,53} 176 | job: {14,33,54} 177 | job: {14,33,55} 178 | ... 179 | 180 | ``` 181 | 182 | 183 | #### Demo - passive queues #### 184 | 185 | ```erlang 186 | 187 | 2> jobs:add_queue(q,[passive]). 188 | ok 189 | 3> Fun = fun() -> io:fwrite("~p starting...~n",[self()]), 190 | 3> Res = jobs:dequeue(q, 3), 191 | 3> io:fwrite("Res = ~p~n", [Res]) 192 | 3> end. 193 | #Fun 194 | 4> jobs:add_queue(p, [{standard_counter,3},{producer,Fun}]). 195 | <0.47.0> starting... 196 | <0.48.0> starting... 197 | <0.49.0> starting... 198 | ok 199 | 5> jobs:enqueue(q, job1). 200 | Res = [{113214444910647,job1}] 201 | ok 202 | <0.54.0> starting... 203 | 204 | ``` 205 | 206 | #### Demo - linked queues #### 207 | 208 | ```erlang 209 | 3> Pid = spawn(fun() -> receive stop -> ok end end). 210 | <0.131.0> 211 | 4> jobs:add_queue(q, [{standard_rate,1}, {link, Pid}]). 212 | ok 213 | 5> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 214 | job: {19,33,37} 215 | ok 216 | 6> exit(Pid, kill). 217 | 218 | =INFO REPORT==== 29-May-2020::19:33:45 === 219 | jobs: removing_queue 220 | name: q 221 | reason: linked 222 | true 223 | 7> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 224 | ** exception error: bad argument 225 | in function jobs_server:call/3 (/home/uwiger/uw/jobs/src/jobs_server.erl, line 236) 226 | in call from jobs_server:run/2 (/home/uwiger/uw/jobs/src/jobs_server.erl, line 117) 227 | 8> jobs:queue_info(q). 228 | undefined 229 | ``` 230 | 231 | #### Demo - queue status #### 232 | 233 | ```erlang 234 | 235 | (a@uwair)1> jobs:queue_info(q). 236 | {queue,[{name,q}, 237 | {mod,jobs_queue}, 238 | {type,fifo}, 239 | {group,undefined}, 240 | {regulators,[{rr,[{name,{rate,q,1}}, 241 | {rate,{rate,[{limit,1}, 242 | {preset_limit,1}, 243 | {interval,1.0e3}, 244 | {modifiers, 245 | [{cpu,10},{memory,10}]}, 246 | {active_modifiers,[]} 247 | ]}}]}]}, 248 | {max_time,undefined}, 249 | {max_size,undefined}, 250 | {latest_dispatch,113216378663298}, 251 | {approved,4}, 252 | {queued,0}, 253 | ..., 254 | {stateful,undefined}, 255 | {st,{st,45079}}]} 256 | 257 | ``` 258 | --------- 259 | 260 | ##Scenarios and Corresponding Configuration Examples 261 | 262 | ####EXAMPLE 1: 263 | * Add counter regulated queue called ___heavy_crunches___ to limit your cpu intensive code executions to no more than 7 at a time 264 | 265 | Configuration: 266 | 267 | ```erlang 268 | 269 | { jobs, [ 270 | { queues, [ 271 | { heavy_crunches, [ { regulators, [{ counter, [{ limit, 7 }] } ] }] } 272 | ] 273 | } 274 | ] 275 | } 276 | 277 | ``` 278 | 279 | Anywhere in your code wrap cpu-intensive work in a call to jobs server and-- __voilà!__ --it is counter-regulated: 280 | 281 | ```erlang 282 | 283 | jobs:run( heavy_crunches,fun()->my_cpu_intensive_calculation() end) 284 | 285 | ``` 286 | 287 | ####EXAMPLE 2: 288 | * Add rate regulated queue called ___http_requests___ to ensure that your http server gets no more than 1000 requests per second. 289 | * Additionally, set the queue size to 10,000 (i.e. to control queue memory consumption) 290 | 291 | Configuration: 292 | 293 | ```erlang 294 | 295 | { jobs, [ 296 | { queues, [ 297 | { http_requests, [ { max_size, 10000}, {regulators, [{ rate, [{limit, 1000}]}]}]} 298 | ] 299 | } 300 | ] 301 | } 302 | 303 | ``` 304 | 305 | Wrap your request entry point in a call to jobs server and it will end up being rate-regulated. 306 | 307 | ```erlang 308 | 309 | jobs:run(http_requests,fun()->handle_http_request() end) 310 | 311 | ``` 312 | 313 | NOTE: with the config above, once 10,000 requests accumulates in the queue any incoming requests are dropped on the floor. 314 | 315 | ####EXAMPLE 3: 316 | * HTTP requests will always have a reasonable execution time. No point in keeping them in the queue past the timeout. 317 | 318 | * Let's create ___patient_user_requests___ queue that will keep requests in the queue for up to 10 seconds 319 | 320 | ```erlang 321 | 322 | { patient_user_requests, [ 323 | { max_time, 10000}, 324 | { regulators, [{rate, [ { limit, 1000 } ] } 325 | ] 326 | } 327 | 328 | ``` 329 | 330 | * Let's create ___impatient_user_requests___ queue that will keep requests in the queue for up to 200 milliseconds. 331 | Additionally, we'll make it a LIFO queue. Unfair, but if we assume that happy/unhappy is a boolean 332 | we're likely to maximize the happy users! 333 | 334 | ```erlang 335 | 336 | { impatient_user_requests, [ 337 | { max_time, 200}, 338 | { type, lifo}, 339 | { regulators, [{rate, [ { limit, 1000 } ] } 340 | ] 341 | } 342 | 343 | ``` 344 | 345 | NOTE: In order to pace requests from both queues at 1000 per second, use __group_rate__ regulation (EXAMPLE 4) 346 | 347 | ####EXAMPLE 4: 348 | * Rate regulate http requests from multiple queues 349 | 350 | Create __group_rates__ regulator called ___http_request_rate___ and assign it to both _impatient_user_requests_ and _patient_user_requests_ 351 | 352 | ```erlang 353 | 354 | { jobs, [ 355 | { group_rates,[{ http_request_rate, [{limit,1000}] }] }, 356 | { queues, [ 357 | { impatient_user_requests, 358 | [ {max_time, 200}, 359 | {type, lifo}, 360 | {regulators,[{ group_rate, http_request_rate}]} 361 | ] 362 | }, 363 | { patient_user_requests, 364 | [ {max_time, 10000}, 365 | {regulators,[{ group_rate, http_request_rate} 366 | ] 367 | } 368 | ] 369 | } 370 | ] 371 | } 372 | 373 | ``` 374 | 375 | ####EXAMPLE 5: 376 | * Can't afford to drop http requests on the floor once max_size is reached? 377 | * Implement and use your own queue to persist those unfortunate http requests and serve them eventually 378 | 379 | ```erlang 380 | 381 | -module(my_persistent_queue). 382 | -behaviour(jobs_queue). 383 | -export([ new/2, 384 | delete/1, 385 | in/3, 386 | out/2, 387 | peek/1, 388 | info/2, 389 | all/1]). 390 | 391 | ## implementation 392 | ... 393 | 394 | ``` 395 | 396 | Configuration: 397 | 398 | ```erlang 399 | 400 | { jobs, [ 401 | { queues, [ 402 | { http_requests, [ 403 | { mod, my_persistent_queue}, 404 | { max_size, 10000 }, 405 | { regulators, [ { rate, [ { limit, 1000 } ] } ] } 406 | ] 407 | } 408 | ] 409 | } 410 | ] 411 | } 412 | 413 | ``` 414 | 415 | ###The use of sampler framework 416 | 1. Get a sampler running and sending feedback to the jobs server. 417 | 2. Apply its feedback to a regulator limit. 418 | 419 | ####EXAMPLE 6: 420 | * Adjust rate regulator limit on the fly based on the feedback from __jobs_sampler_cpu__ named ___cpu_feedback___ 421 | 422 | ```erlang 423 | 424 | { jobs, [ 425 | { samplers, [{ cpu_feedback, jobs_sampler_cpu, [] } ] }, 426 | { queues, [ 427 | { http_requests, [ 428 | { regulators, [ { rate, [ { limit,1000 } ] }, 429 | { modifiers, [ { cpu_feedback, 10} ] } %% 10 = % increment by which to modify the limit 430 | ] 431 | } 432 | ] 433 | } 434 | ] 435 | } 436 | 437 | ``` 438 | 439 | Prerequisites 440 | ------------- 441 | This application requires 'exprecs'. 442 | The 'exprecs' module is part of http://github.com/uwiger/parse_trans 443 | 444 | Contribute 445 | ---------- 446 | For issues, comments or feedback please [create an issue!] [1] 447 | 448 | [1]: http://github.com/uwiger/jobs/issues "jobs issues" 449 | 450 | 451 | ## Modules ## 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 |
jobs
jobs_app
jobs_info
jobs_lib
jobs_prod_simple
jobs_queue
jobs_queue_list
jobs_sampler
jobs_sampler_cpu
jobs_sampler_history
jobs_sampler_mnesia
jobs_stateful_simple
467 | 468 | -------------------------------------------------------------------------------- /TODO.org: -------------------------------------------------------------------------------- 1 | * DONE Change jobs_eqc_model so it understands LIFO and FIFO. 2 | * DONE Change test so we always generate fifo with jobs queue 3 | * DONE Change test so we always generate life with jobs queue list 4 | * TODO Add tests for the "timedout" parameter. 5 | ** DONE Implement timedout in the model 6 | ** TODO Introduce meck 7 | ** TODO Meck up jobs_lib:timestamp() 8 | 9 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # jobs - a Job scheduler for load regulation # 4 | 5 | Copyright (c) 2014-2018 Ulf Wiger 6 | 7 | __Version:__ 0.9.0 8 | 9 | JOBS 10 | ==== 11 | 12 | Jobs is a job scheduler for load regulation of Erlang applications. 13 | It provides a queueing framework where each queue can be configured 14 | for throughput rate, credit pool and feedback compensation. 15 | Queues can be added and modified at runtime, and customizable 16 | "samplers" propagate load status across all nodes in the system. 17 | 18 | Specifically, jobs provides three features: 19 | 20 | * Job scheduling: A job is scheduled according to certain constraints. 21 | For instance, you may want to define that no more than 9 jobs of a 22 | certain type can execute simultaneously and the maximal rate at 23 | which you can start such jobs are 300 per second. 24 | * Job queueing: When load is higher than the scheduling limits 25 | additional jobs are *queued* by the system to be run later when load 26 | clears. Certain rules govern queues: are they dequeued in FIFO or 27 | LIFO order? How many jobs can the queue take before it is full? Is 28 | there a deadline after which jobs should be rejected. When we hit 29 | the queue limits we reject the job. This provides a feedback 30 | mechanism on the client of the queue so you can take action. 31 | * Sampling and dampening: Periodic samples of the Erlang VM can 32 | provide information about the health of the system in general. If we 33 | have high CPU load or high memory usage, we apply dampening to the 34 | scheduling rules: we may lower the concurrency count or the rate at 35 | which we execute jobs. When the health problem clears, we remove the 36 | dampener and run at full speed again. 37 | 38 | Error recovery 39 | -------------- 40 | 41 | The Jobs server is designed to not crash. However, in the unlikely event 42 | that it should occur (and it has!) Jobs does not automatically restore changes 43 | that have been effected through the API. This can be enabled, setting the 44 | Jobs environment variable `auto_restore` to `true`, or calling the function 45 | `jobs_server:auto_restore(true)`. This will tell the jobs_server to remember 46 | every configuration change and replay them, in order, after a process restart. 47 | 48 | Examples 49 | -------- 50 | 51 | The following examples are fetched from the EUC 2013 presentation on Jobs. 52 | 53 | 54 | #### Regulate incoming HTTP requests (e.g. JSON-RPC) #### 55 | 56 | ```erlang 57 | 58 | %% @doc Handle a JSON-RPC request. 59 | handler_session(Arg) -> 60 | jobs:run( 61 | rpc_from_web, 62 | fun() -> 63 | try 64 | yaws_rpc:handler_session( 65 | maybe_multipart(Arg),{?MODULE, web_rpc}) 66 | catch 67 | error:E -> 68 | ... 69 | end 70 | end). 71 | 72 | ``` 73 | 74 | 75 | #### From Riak prototype, using explicit ask/done #### 76 | 77 | ```erlang 78 | 79 | case jobs:ask(riak_kv_fsm) of 80 | {ok, JobId} -> 81 | try 82 | {ok, Pid} = riak_kv_get_fsm_sup:start_get_fsm(...), 83 | Timeout = recv_timeout(Options), 84 | wait_for_reqid(ReqId, Timeout) 85 | after 86 | jobs:done(JobId) %% Only needed if process stays alive 87 | end; 88 | {error, rejected} -> %% Overload! 89 | {error, timeout} 90 | end 91 | 92 | ``` 93 | 94 | 95 | #### Shell demo - simple rate-limited queue #### 96 | 97 | ```erlang 98 | 99 | 2> jobs:add_queue(q, [{standard_rate,1}]). 100 | ok 101 | 3> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 102 | job: {14,37,7} 103 | ok 104 | 4> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 105 | job: {14,37,8} 106 | ok 107 | ... 108 | 5> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 109 | job: {14,37,10} 110 | ok 111 | 6> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end). 112 | job: {14,37,11} 113 | ok 114 | 115 | ``` 116 | 117 | 118 | #### Shell demo - "stateful" queues #### 119 | 120 | ```erlang 121 | 122 | Eshell V5.9.2 (abort with ^G) 123 | 1> application:start(jobs). 124 | ok 125 | 2> jobs:add_queue(q, 126 | [{standard_rate,1}, 127 | {stateful,fun(init,_) -> {0,5}; 128 | ({call,{size,Sz},_,_},_) -> {reply,ok,{0,Sz}}; 129 | ({N,Sz},_) -> {N, {(N+1) rem Sz,Sz}} 130 | end}]). 131 | ok 132 | 3> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 133 | 0 134 | 4> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 135 | 1 136 | 5> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 137 | 2 138 | 6> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 139 | 3 140 | 7> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 141 | 4 142 | 8> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 143 | 0 144 | 9> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 145 | 1 146 | %% Resize the 'pool' 147 | 10> jobs:ask_queue(q, {size,3}). 148 | ok 149 | 11> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 150 | 0 151 | 12> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 152 | 1 153 | 13> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 154 | 2 155 | 14> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end). 156 | 0 157 | ... 158 | 159 | ``` 160 | 161 | 162 | #### Demo - producers #### 163 | 164 | ```erlang 165 | 166 | Eshell V5.9.2 (abort with ^G) 167 | 1> application:start(jobs). 168 | ok 169 | 2> jobs:add_queue(p, 170 | [{producer, fun() -> io:fwrite("job: ~p~n",[time()]) end}, 171 | {standard_rate,1}]). 172 | job: {14,33,51} 173 | ok 174 | 3> job: {14,33,52} 175 | job: {14,33,53} 176 | job: {14,33,54} 177 | job: {14,33,55} 178 | ... 179 | 180 | ``` 181 | 182 | 183 | #### Demo - passive queues #### 184 | 185 | ```erlang 186 | 187 | 2> jobs:add_queue(q,[passive]). 188 | ok 189 | 3> Fun = fun() -> io:fwrite("~p starting...~n",[self()]), 190 | 3> Res = jobs:dequeue(q, 3), 191 | 3> io:fwrite("Res = ~p~n", [Res]) 192 | 3> end. 193 | #Fun 194 | 4> jobs:add_queue(p, [{standard_counter,3},{producer,Fun}]). 195 | <0.47.0> starting... 196 | <0.48.0> starting... 197 | <0.49.0> starting... 198 | ok 199 | 5> jobs:enqueue(q, job1). 200 | Res = [{113214444910647,job1}] 201 | ok 202 | <0.54.0> starting... 203 | 204 | ``` 205 | 206 | 207 | #### Demo - queue status #### 208 | 209 | ```erlang 210 | 211 | (a@uwair)1> jobs:queue_info(q). 212 | {queue,[{name,q}, 213 | {mod,jobs_queue}, 214 | {type,fifo}, 215 | {group,undefined}, 216 | {regulators,[{rr,[{name,{rate,q,1}}, 217 | {rate,{rate,[{limit,1}, 218 | {preset_limit,1}, 219 | {interval,1.0e3}, 220 | {modifiers, 221 | [{cpu,10},{memory,10}]}, 222 | {active_modifiers,[]} 223 | ]}}]}]}, 224 | {max_time,undefined}, 225 | {max_size,undefined}, 226 | {latest_dispatch,113216378663298}, 227 | {approved,4}, 228 | {queued,0}, 229 | ..., 230 | {stateful,undefined}, 231 | {st,{st,45079}}]} 232 | 233 | ``` 234 | --------- 235 | 236 | ##Scenarios and Corresponding Configuration Examples 237 | 238 | ####EXAMPLE 1: 239 | * Add counter regulated queue called ___heavy_crunches___ to limit your cpu intensive code executions to no more than 7 at a time 240 | 241 | Configuration: 242 | 243 | ```erlang 244 | 245 | { jobs, [ 246 | { queues, [ 247 | { heavy_crunches, [ { regulators, [{ counter, [{ limit, 7 }] } ] }] } 248 | ] 249 | } 250 | ] 251 | } 252 | 253 | ``` 254 | 255 | Anywhere in your code wrap cpu-intensive work in a call to jobs server and-- __voilà!__ --it is counter-regulated: 256 | 257 | ```erlang 258 | 259 | jobs:run( heavy_crunches,fun()->my_cpu_intensive_calculation() end) 260 | 261 | ``` 262 | 263 | ####EXAMPLE 2: 264 | * Add rate regulated queue called ___http_requests___ to ensure that your http server gets no more than 1000 requests per second. 265 | * Additionally, set the queue size to 10,000 (i.e. to control queue memory consumption) 266 | 267 | Configuration: 268 | 269 | ```erlang 270 | 271 | { jobs, [ 272 | { queues, [ 273 | { http_requests, [ { max_size, 10000}, {regulators, [{ rate, [{limit, 1000}]}]}]} 274 | ] 275 | } 276 | ] 277 | } 278 | 279 | ``` 280 | 281 | Wrap your request entry point in a call to jobs server and it will end up being rate-regulated. 282 | 283 | ```erlang 284 | 285 | jobs:run(http_requests,fun()->handle_http_request() end) 286 | 287 | ``` 288 | 289 | NOTE: with the config above, once 10,000 requests accumulates in the queue any incoming requests are dropped on the floor. 290 | 291 | ####EXAMPLE 3: 292 | * HTTP requests will always have a reasonable execution time. No point in keeping them in the queue past the timeout. 293 | 294 | * Let's create ___patient_user_requests___ queue that will keep requests in the queue for up to 10 seconds 295 | 296 | ```erlang 297 | 298 | { patient_user_requests, [ 299 | { max_time, 10000}, 300 | { regulators, [{rate, [ { limit, 1000 } ] } 301 | ] 302 | } 303 | 304 | ``` 305 | 306 | * Let's create ___impatient_user_requests___ queue that will keep requests in the queue for up to 200 milliseconds. 307 | Additionally, we'll make it a LIFO queue. Unfair, but if we assume that happy/unhappy is a boolean 308 | we're likely to maximize the happy users! 309 | 310 | ```erlang 311 | 312 | { impatient_user_requests, [ 313 | { max_time, 200}, 314 | { type, lifo}, 315 | { regulators, [{rate, [ { limit, 1000 } ] } 316 | ] 317 | } 318 | 319 | ``` 320 | 321 | NOTE: In order to pace requests from both queues at 1000 per second, use __group_rate__ regulation (EXAMPLE 4) 322 | 323 | ####EXAMPLE 4: 324 | * Rate regulate http requests from multiple queues 325 | 326 | Create __group_rates__ regulator called ___http_request_rate___ and assign it to both _impatient_user_requests_ and _patient_user_requests_ 327 | 328 | ```erlang 329 | 330 | { jobs, [ 331 | { group_rates,[{ http_request_rate, [{limit,1000}] }] }, 332 | { queues, [ 333 | { impatient_user_requests, 334 | [ {max_time, 200}, 335 | {type, lifo}, 336 | {regulators,[{ group_rate, http_request_rate}]} 337 | ] 338 | }, 339 | { patient_user_requests, 340 | [ {max_time, 10000}, 341 | {regulators,[{ group_rate, http_request_rate} 342 | ] 343 | } 344 | ] 345 | } 346 | ] 347 | } 348 | 349 | ``` 350 | 351 | ####EXAMPLE 5: 352 | * Can't afford to drop http requests on the floor once max_size is reached? 353 | * Implement and use your own queue to persist those unfortunate http requests and serve them eventually 354 | 355 | ```erlang 356 | 357 | -module(my_persistent_queue). 358 | -behaviour(jobs_queue). 359 | -export([ new/2, 360 | delete/1, 361 | in/3, 362 | out/2, 363 | peek/1, 364 | info/2, 365 | all/1]). 366 | 367 | ## implementation 368 | ... 369 | 370 | ``` 371 | 372 | Configuration: 373 | 374 | ```erlang 375 | 376 | { jobs, [ 377 | { queues, [ 378 | { http_requests, [ 379 | { mod, my_persistent_queue}, 380 | { max_size, 10000 }, 381 | { regulators, [ { rate, [ { limit, 1000 } ] } ] } 382 | ] 383 | } 384 | ] 385 | } 386 | ] 387 | } 388 | 389 | ``` 390 | 391 | ###The use of sampler framework 392 | 1. Get a sampler running and sending feedback to the jobs server. 393 | 2. Apply its feedback to a regulator limit. 394 | 395 | ####EXAMPLE 6: 396 | * Adjust rate regulator limit on the fly based on the feedback from __jobs_sampler_cpu__ named ___cpu_feedback___ 397 | 398 | ```erlang 399 | 400 | { jobs, [ 401 | { samplers, [{ cpu_feedback, jobs_sampler_cpu, [] } ] }, 402 | { queues, [ 403 | { http_requests, [ 404 | { regulators, [ { rate, [ { limit,1000 } ] }, 405 | { modifiers, [ { cpu_feedback, 10} ] } %% 10 = % increment by which to modify the limit 406 | ] 407 | } 408 | ] 409 | } 410 | ] 411 | } 412 | 413 | ``` 414 | 415 | Prerequisites 416 | ------------- 417 | This application requires 'exprecs'. 418 | The 'exprecs' module is part of http://github.com/uwiger/parse_trans 419 | 420 | Contribute 421 | ---------- 422 | For issues, comments or feedback please [create an issue!] [1] 423 | 424 | [1]: http://github.com/uwiger/jobs/issues "jobs issues" 425 | 426 | 427 | ## Modules ## 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 |
jobs
jobs_app
jobs_info
jobs_lib
jobs_prod_simple
jobs_queue
jobs_queue_list
jobs_sampler
jobs_sampler_cpu
jobs_sampler_history
jobs_sampler_mnesia
jobs_server
jobs_stateful_simple
444 | 445 | -------------------------------------------------------------------------------- /doc/edoc-info: -------------------------------------------------------------------------------- 1 | %% encoding: UTF-8 2 | {application,jobs}. 3 | {modules,[jobs,jobs_app,jobs_info,jobs_lib,jobs_prod_simple,jobs_queue, 4 | jobs_queue_list,jobs_sampler,jobs_sampler_cpu,jobs_sampler_history, 5 | jobs_sampler_mnesia,jobs_server,jobs_stateful_simple]}. 6 | -------------------------------------------------------------------------------- /doc/erlang.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uwiger/jobs/867ac4d33f50ce2628f68625568ebc25c5e38d97/doc/erlang.png -------------------------------------------------------------------------------- /doc/erlang07g-wiger.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uwiger/jobs/867ac4d33f50ce2628f68625568ebc25c5e38d97/doc/erlang07g-wiger.pdf -------------------------------------------------------------------------------- /doc/jobs.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | This is the public API of the JOBS framework. 9 | 10 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 11 | 12 | 13 | 14 | ## Description ## 15 | 16 | 17 | ## Function Index ## 18 | 19 | 20 |
add_counter/2Adds a named counter to the load regulator on the current node.
add_group_rate/2Adds a group rate regulator to the load regulator on the current node.
add_queue/2Installs a new queue in the load regulator on the current node.
ask/1Asks permission to run a job of Type.
ask_queue/2Sends a synchronous request to a specific queue.
delete_counter/1Deletes a named counter from the load regulator on the current node.
delete_group_rate/1
delete_queue/1Deletes the named queue from the load regulator on the current node.
dequeue/2Extracts up to N items from a passive queue.
done/1Signals completion of an executed task.
enqueue/2Inserts Item` into a passive queue. 21 | 22 | Note that this function only works on passive queues. An exception will be 23 | raised if the queue doesnt exist, or isn't passive.
info/1
job_info/1Retrieves job-specific information from the Opaque data object.
modify_counter/2
modify_group_rate/2
modify_queue/2Modifies queue parameters of existing queue.
modify_regulator/4
queue_info/1
queue_info/2
run/2Executes Function() when permission has been granted by job regulator.
24 | 25 | 26 | 27 | 28 | ## Function Details ## 29 | 30 | 31 | 32 | ### add_counter/2 ### 33 | 34 | 

 35 | add_counter(Name, Options) -> ok
 36 |
37 |
38 | 39 | Adds a named counter to the load regulator on the current node. 40 | Fails if there already is a counter the name `Name`. 41 | 42 | 43 | 44 | ### add_group_rate/2 ### 45 | 46 | 

 47 | add_group_rate(Name, Options) -> ok
 48 |
49 |
50 | 51 | Adds a group rate regulator to the load regulator on the current node. 52 | Fails if there is already a group rate regulator of the same name. 53 | 54 | 55 | 56 | ### add_queue/2 ### 57 | 58 | 

 59 | add_queue(Name::any(), Options::[{Key, Value}]) -> ok
 60 |
61 |
62 | 63 | Installs a new queue in the load regulator on the current node. 64 | 65 | Valid options are: 66 | 67 | * `{regulators, Rs}`, where `Rs` is a list of rate- or counter-based 68 | regulators. Valid regulators listed below. Default: []. 69 | 70 | * `{type, Type}` - type of queue. Valid types listed below. Default: `fifo`. 71 | 72 | * `{action, Action}` - automatic action to perform for each request. 73 | Valid actions described below. Default: `undefined`. 74 | 75 | * `{check_interval, I}` - If specified (in ms), this overrides the interval 76 | derived from any existing rate regulator. Note that regardless of how often 77 | the queue is checked, enough jobs will be dispatched at each interval to 78 | maintain the highest allowed rate possible, but the check interval may 79 | thus affect how many jobs are dispatched at the same time. Normally, this 80 | should not have to be specified. 81 | 82 | * `{max_time, T}`, specifies how long (in ms) a job is allowed to wait 83 | in the queue before it is automatically rejected. 84 | If `undefined`, no limit is imposed. 85 | 86 | * `{max_size, S}`, indicates how many items can be queued before requests 87 | are automatically rejected. Strictly speaking, size is whatever the queue 88 | behavior reports as the size; in the default queue behavior, it is the 89 | number of elements in the queue. 90 | If `undefined`, no limit is imposed. 91 | 92 | * `{mod, M}`, indicates which queue behavior to use. Default is `jobs_queue`. 93 | 94 | In addition, some 'abbreviated' options are supported: 95 | 96 | * `{standard_rate, R}` - equivalent to 97 | `[{regulators,[{rate,[{limit,R}, {modifiers,[{cpu,10},{memory,10}]}]}]}]` 98 | 99 | * `{standard_counter, C}` - equivalent to 100 | `[{regulators,[{counter,[{limit,C}, {modifiers,[{cpu,10},{memory,10}]}]}]}]` 101 | 102 | * `{producer, F}` - equivalent to `{type, {producer, F}}` 103 | 104 | * `passive` - equivalent to `{type, {passive, fifo}}` 105 | 106 | * `approve | reject` - equivalent to `{action, approve | reject}` 107 | 108 | __Regulators__ 109 | 110 | * `{rate, Opts}` - rate regulator. Valid options are 111 | 112 | 1. `{limit, Limit}` where `Limit` is the maximum rate (requests/sec) 113 | 114 | 1. `{modifiers, Mods}`, control feedback-based regulation. See below. 115 | 116 | 1. `{name, Name}`, optional. The default name for the regulator is 117 | `{rate, QueueName, N}`, where `N` is an index indicating which rate regulator 118 | in the list is referred. Currently, at most one rate regulator is allowed, 119 | so `N` will always be `1`. 120 | 121 | 122 | * `{counter, Opts}` - counter regulator. Valid options are 123 | 124 | 1. `{limit, Limit}`, where `Limit` is the number of concurrent jobs 125 | allowed. 126 | 127 | 1. `{increment, Incr}`, increment per job. Default is `1`. 128 | 129 | 1. `{modifiers, Mods}`, control feedback-based regulation. See below. 130 | 131 | 132 | * `{named_counter, Name, Incr}`, use an existing counter, incrementing it 133 | with `Incr` for each job. `Name` can either refer to a named top-level 134 | counter (see [`add_counter/2`](#add_counter-2)), or a queue-specific counter 135 | (these are named `{counter,Qname,N}`, where `N` is an index specifying 136 | their relative position in the regulators list - e.g. first or second 137 | counter). 138 | 139 | * `{group_rate, R}`, refers to a top-level group rate `R`. 140 | See [`add_group_rate/2`](#add_group_rate-2). 141 | 142 | __Types__ 143 | 144 | * `fifo | lifo` - these are the types supported by the default queue 145 | behavior. While lifo may sound like an odd choice, it may have benefits 146 | for stochastic traffic with time constraints: there is no point to 147 | 'fairness', since requests cannot control their place in the queue, and 148 | choosing the 'freshest' job may increase overall goodness critera. 149 | 150 | * `{producer, F}`, the queue is not for incoming requests, but rather 151 | generates jobs. Valid options for `F` are 152 | (for details, see [`jobs_prod_simpe`](jobs_prod_simpe.md)): 153 | 154 | 1. A fun of arity 0, indicating a stateless producer 155 | 156 | 1. A fun of arity 2, indicating a stateful producer 157 | 158 | 1. `{M, F, A}`, indicating a stateless producer 159 | 160 | 1. `{Mod, Args}` indicating a stateful producer 161 | 162 | 163 | * `{action, approve | reject}`, specifies an automatic response to every 164 | request. This can be used to either block a queue (`reject`) or set it as 165 | a pass-through ('approve'). 166 | 167 | __Modifiers__ 168 | 169 | Jobs supports feedback-based modification of regulators. 170 | 171 | The sampler framework sends feedback messages of type 172 | `[{Modifier, Local, Remote::[{node(), Level}]}]`. 173 | 174 | Each regulator can specify a list of modifier instructions: 175 | 176 | * `{Modifier, Local, Remote}` - `Modifier` can be any label used by the 177 | samplers (see [`jobs_sampler`](jobs_sampler.md)). `Local` and `Remote` indicate 178 | increments in percent by which to reduce the limit of the given regulator. 179 | The `Local` increment is used for feedback info pertaining to the local 180 | node, and the `Remote` increment is used for remote indicators. `Local` 181 | is given as a percentage value (e.g. `10` for `10 %`). The `Remote` 182 | increment is either `{avg, Percent}` or `{max, Percent}`, indicating whether 183 | to respond to the average load of other nodes or to the most loaded node. 184 | The correction from `Local` and the correction from `Remote` are summed 185 | before applying to the regulator limit. 186 | 187 | * `{Modifier, Local}` - same as above, but responding only to local 188 | indications, ignoring the load on remote nodes. 189 | 190 | * `{Modifier, F::function((Local, Remote) -> integer())}` - the function 191 | `F(Local, Remote)` is applied and expected to return a correction value, 192 | in percentage units. 193 | 194 | * `{Modifier, {Module, Function}}` - `Module:Function(Local Remote)` 195 | is applied an expected to return a correction value in percentage units. 196 | 197 | For example, if a rate regulator has a limit of `100` and has a modifier, 198 | `{cpu, 10}`, then a feedback message of `{cpu, 2, _Remote}` will reduce 199 | the rate limit by `2*10` percent, i.e. down to `80`. 200 | 201 | Note that modifiers are always applied to the _preset_ limit, 202 | not the current limit. Thus, the next round of feedback messages in our 203 | example will be applied to the preset limit of `100`, not the `80` that 204 | resulted from the previous feedback messages. A correction value of `0` 205 | will reset the limit to the preset value. 206 | 207 | If there are more than one modifier with the same name, the last one in the 208 | list will be the one used. 209 | 210 | 211 | 212 | ### ask/1 ### 213 | 214 | 

215 | ask(Type) -> {ok, Opaque} | {error, Reason}
216 |
217 |
218 | 219 | Asks permission to run a job of Type. Returns when permission granted. 220 | 221 | The simplest way to have jobs regulated is to spawn a request per job. 222 | The process should immediately call this function, and when granted 223 | permission, execute the job, and then terminate. 224 | If for some reason the process needs to remain, to execute more jobs, 225 | it should explicitly call `jobs:done(Opaque)`. 226 | This is not strictly needed when regulation is rate-based, but as the 227 | regulation strategy may change over time, it is the prudent thing to do. 228 | 229 | 230 | 231 | ### ask_queue/2 ### 232 | 233 | 

234 | ask_queue(QueueName, Request) -> Reply
235 |
236 |
237 | 238 | Sends a synchronous request to a specific queue. 239 | 240 | This function is mainly intended to be used for back-end processes that act 241 | as custom extensions to the load regulator itself. It should not be used by 242 | regular clients. Sophisticated queue behaviours could export gen_server-like 243 | logic allowing them to respond to synchronous calls, either for special 244 | inspection, or for influencing the queue state. 245 | 246 | 247 | 248 | ### delete_counter/1 ### 249 | 250 | 

251 | delete_counter(Name) -> boolean()
252 |
253 |
254 | 255 | Deletes a named counter from the load regulator on the current node. 256 | Returns `true` if there was in fact such a counter; `false` otherwise. 257 | 258 | 259 | 260 | ### delete_group_rate/1 ### 261 | 262 | `delete_group_rate(Name) -> any()` 263 | 264 | 265 | 266 | ### delete_queue/1 ### 267 | 268 | 

269 | delete_queue(Name) -> boolean()
270 |
271 |
272 | 273 | Deletes the named queue from the load regulator on the current node. 274 | Returns `true` if there was in fact such a queue; `false` otherwise. 275 | 276 | 277 | 278 | ### dequeue/2 ### 279 | 280 | 

281 | dequeue(Queue, N) -> [{JobID, Item}]
282 |
283 |
284 | 285 | Extracts up to `N` items from a passive queue 286 | 287 | Note that this function only works on passive queues. An exception will be 288 | raised if the queue doesn't exist, or if it isn't passive. 289 | 290 | This function will block until at least one item can be extracted from the 291 | queue (see [`enqueue/2`](#enqueue-2)). No more than `N` items will be extracted. 292 | 293 | The items returned are on the form `{JobID, Item}`, where `JobID` is in 294 | the form of a microsecond timestamp 295 | (see [`jobs_lib:timestamp_to_datetime/1`](jobs_lib.md#timestamp_to_datetime-1)), and `Item` is whatever was 296 | provided in [`enqueue/2`](#enqueue-2). 297 | 298 | 299 | 300 | ### done/1 ### 301 | 302 | 

303 | done(Opaque) -> ok
304 |
305 |
306 | 307 | Signals completion of an executed task. 308 | 309 | This is used when the current process wants to submit more jobs to load 310 | regulation. It is mandatory when performing counter-based regulation 311 | (unless the process terminates after completing the task). It has no 312 | effect if the job type is purely rate-regulated. 313 | 314 | 315 | 316 | ### enqueue/2 ### 317 | 318 | 

319 | enqueue(Queue, Item) -> ok | {error, Reason}
320 |
321 |
322 | 323 | Inserts `Item` into a passive queue. 324 | 325 | Note that this function only works on passive queues. An exception will be 326 | raised if the queue doesn`t exist, or isn't passive. 327 | 328 | Returns `ok` if `Item` was successfully entered into the queue, 329 | `{error, Reason}` otherwise (e.g. if the queue is full). 330 | 331 | 332 | 333 | ### info/1 ### 334 | 335 | `info(Item) -> any()` 336 | 337 | 338 | 339 | ### job_info/1 ### 340 | 341 | 

342 | job_info(X1::Opaque) -> undefined | Info
343 |
344 |
345 | 346 | Retrieves job-specific information from the `Opaque` data object. 347 | 348 | The queue could choose to return specific information that is passed to a 349 | granted job request. This could be used e.g. for load-balancing strategies. 350 | 351 | 352 | 353 | ### modify_counter/2 ### 354 | 355 | `modify_counter(CName, Opts) -> any()` 356 | 357 | 358 | 359 | ### modify_group_rate/2 ### 360 | 361 | `modify_group_rate(GRName, Opts) -> any()` 362 | 363 | 364 | 365 | ### modify_queue/2 ### 366 | 367 | 

368 | modify_queue(Name::any(), Options::[{Key, Value}]) -> ok | {error, Reason}
369 |
370 |
371 | 372 | Modifies queue parameters of existing queue. 373 | 374 | The queue parameters that can be modified are `max_size` and `max_time`. 375 | 376 | 377 | 378 | ### modify_regulator/4 ### 379 | 380 | `modify_regulator(Type, QName, RegName, Opts) -> any()` 381 | 382 | 383 | 384 | ### queue_info/1 ### 385 | 386 | `queue_info(Name) -> any()` 387 | 388 | 389 | 390 | ### queue_info/2 ### 391 | 392 | `queue_info(Name, Item) -> any()` 393 | 394 | 395 | 396 | ### run/2 ### 397 | 398 | 

399 | run(Queue::Type, Function::function()) -> Result
400 |
401 |
402 | 403 | Executes Function() when permission has been granted by job regulator. 404 | 405 | This is equivalent to performing the following sequence: 406 | 407 | ``` 408 | 409 | case jobs:ask(Type) of 410 | {ok, Opaque} -> 411 | try Function() 412 | after 413 | jobs:done(Opaque) 414 | end; 415 | {error, Reason} -> 416 | erlang:error(Reason) 417 | end. 418 | ``` 419 | 420 | -------------------------------------------------------------------------------- /doc/jobs_app.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_app # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | Application module for JOBS. 9 | 10 | 11 | 12 | ## Description ## 13 | 14 | Normally, JOBS is configured at startup, using a static configuration. 15 | There is a reconfiguration API [`jobs`](jobs.md), which is mainly for evolution 16 | of the system. 17 | 18 | 19 | ### Configuring JOBS ### 20 | 21 | A static configuration can be provided via application environment 22 | variables for the `jobs` application. The following is a list of 23 | recognised configuration parameters. 24 | 25 | 26 | #### {config, Filename} #### 27 | 28 | Evaluate a file using [file:script/1](https://www.erlang.org/doc/man/file.html#script-1), treating the data 29 | returned from the script as a list of configuration options. 30 | 31 | 32 | #### {queues, QueueOptions} #### 33 | 34 | Configure a list of queues according to the provided QueueOptions. 35 | If no queues are specified, a queue named `default` will be created 36 | with default characteristics. 37 | 38 | Below are the different queue configuration options: 39 | 40 |
{Name, Options}
41 | 42 | This is the generic queue configuration pattern. 43 | `Name :: any()` is used to identify the queue. 44 | 45 | Options: 46 | 47 | `{mod, Module::atom()}` provides the name of the queueing module. 48 | The default module is `jobs_queue`. 49 | 50 | `{type, fifo | lifo | approve | reject | {producer, F}}` 51 | specifies the semantics of the queue. Note that the specified queue module 52 | may be limited to only one type (e.g. the `jobs_queue_list` module only 53 | supports `lifo` semantics). 54 | 55 | If the type is `{producer, F}`, it doesn't matter which queue module is 56 | used, as it is not possible to submit job requests to a producer queue. 57 | The producer queue will initiate jobs using `spawn_monitor(F)` at the 58 | rate given by the regulators for the queue. 59 | 60 | If the type is `approve` or `reject`, respectively, all other options will 61 | be irrelevant. Any request to the queue will either be immediately approved 62 | or immediately rejected. 63 | 64 | `{max_time, integer() | undefined}` specifies the longest time that a job 65 | request may spend in the queue. If `undefined`, no limit is imposed. 66 | 67 | `{max_size, integer() | undefined}` specifies the maximum length (number 68 | of job requests) of the queue. If the queue has reached the maximum length, 69 | subsequent job requests will be rejected unless it is possible to remove 70 | enough requests that have exceeded the maximum allowed time in the queue. 71 | If `undefined`, no limit is imposed. 72 | 73 | `{regulators, [{regulator_type(), Opts]}` specifies the regulation 74 | characteristics of the queue. 75 | 76 | The following types of regulator are supported: 77 | 78 | `regulator_type() :: rate | counter | group_rate` 79 | 80 | It is possible to combine different types of regulator on the same queue, 81 | e.g. a queue may have both rate- and counter regulation. It is not possible 82 | to have two different rate regulators for the same queue. 83 | 84 | Common regulator options: 85 | 86 | `{name, term()}` names the regulator; by default, a name will be generated. 87 | 88 | `{limit, integer()}` defines the limit for the regulator. If it is a rate 89 | regulator, the value represents the maximum number of jobs/second; if it 90 | is a counter regulator, it represents the total number of "credits" 91 | available. 92 | 93 | `{modifiers, [modifier()]}` 94 | 95 | ``` 96 | 97 | modifier() :: {IndicatorName :: any(), unit()} 98 | | {Indicator, local_unit(), remote_unit()} 99 | | {Indicator, Fun} 100 | local_unit() :: unit() :: integer() 101 | remote_unit() :: {avg, unit()} | {max, unit()} 102 | ``` 103 | 104 | Feedback indicators are sent from the sampler framework. Each indicator 105 | has the format `{IndicatorName, LocalLoadFactor, Remote}`. 106 | 107 | `Remote :: [{Node, LoadFactor}]` 108 | 109 | `IndicatorName` defines the type of indicator. It could be e.g. `cpu`, 110 | `memory`, `mnesia`, or any other name defined by one of the sampler plugins. 111 | 112 | The effect of a modifier is calculated as the sum of the effects from local 113 | and remote load. As the remote load is represented as a list of 114 | `{Node,Factor}` it is possible to multiply either the average or the max 115 | load on the remote nodes with the given factor: `{avg,Unit} | {max, Unit}`. 116 | 117 | For custom interpretation of the feedback indicator, it is possible to 118 | specify a function `F(LocalFactor, Remote) -> Effect`, where Effect is a 119 | positive integer. 120 | 121 | The resulting effect value is used to reduce the predefined regulator limit 122 | with the given number of percentage points, e.g. if a rate regulator has 123 | a predefined limit of 100 jobs/sec, and `Effect = 20`, the current rate 124 | limit will become 80 jobs/sec. 125 | 126 | `{rate, Opts}` - rate regulation 127 | 128 | Currently, no special options exist for rate regulators. 129 | 130 | `{counter, Opts}` - counter regulation 131 | 132 | The option `{increment, I}` can be used to specify how much of the credit 133 | pool should be assigned to each job. The default increment is 1. 134 | 135 | `{named_counter, Name, Increment}` reuses an existing counter regulator. 136 | This can be used to link multiple queues to a shared credit pool. Note that 137 | this does not use the existing counter regulator as a template, but actually 138 | shares the credits with any other queues using the same named counter. 139 | 140 | __NOTE__ Currently, if there is no counter corresponding to the alias, 141 | the entry will simply be ignored during regulation. It is likely that this 142 | behaviour will change in the future. 143 | 144 |
{Name, standard_rate, R}
145 | 146 | A simple rate-regulated queue with throughput rate `R`, and basic cpu- and 147 | memory-related feedback compensation. 148 | 149 |
{Name, standard_counter, N}
150 | 151 | A simple counter-regulated queue, giving each job a weight of 1, and thus 152 | allowing at most `N` jobs to execute concurrently. Basic cpu- and memory- 153 | related feedback compensation. 154 | 155 |
{Name, producer, F, Options}
156 | 157 | A producer queue is not open for incoming jobs, but will rather initiate 158 | jobs at the given rate. 159 | 160 | ## Function Index ## 161 | 162 | 163 |
init/1
start/2
stop/1
164 | 165 | 166 | 167 | 168 | ## Function Details ## 169 | 170 | 171 | 172 | ### init/1 ### 173 | 174 | `init(X1) -> any()` 175 | 176 | 177 | 178 | ### start/2 ### 179 | 180 | `start(X1, X2) -> any()` 181 | 182 | 183 | 184 | ### stop/1 ### 185 | 186 | `stop(X1) -> any()` 187 | 188 | -------------------------------------------------------------------------------- /doc/jobs_info.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_info # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | ## Function Index ## 10 | 11 | 12 |
pp/1
13 | 14 | 15 | 16 | 17 | ## Function Details ## 18 | 19 | 20 | 21 | ### pp/1 ### 22 | 23 | `pp(L) -> any()` 24 | 25 | -------------------------------------------------------------------------------- /doc/jobs_lib.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_lib # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | ## Function Index ## 10 | 11 | 12 |
time_compat/0
timestamp/0
timestamp_to_datetime/1
13 | 14 | 15 | 16 | 17 | ## Function Details ## 18 | 19 | 20 | 21 | ### time_compat/0 ### 22 | 23 | `time_compat() -> any()` 24 | 25 | 26 | 27 | ### timestamp/0 ### 28 | 29 | `timestamp() -> any()` 30 | 31 | 32 | 33 | ### timestamp_to_datetime/1 ### 34 | 35 | `timestamp_to_datetime(TS) -> any()` 36 | 37 | -------------------------------------------------------------------------------- /doc/jobs_prod_simple.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_prod_simple # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | ## Function Index ## 10 | 11 | 12 |
init/2
next/3
13 | 14 | 15 | 16 | 17 | ## Function Details ## 18 | 19 | 20 | 21 | ### init/2 ### 22 | 23 | `init(F, Info) -> any()` 24 | 25 | 26 | 27 | ### next/3 ### 28 | 29 | `next(Opaque, Stateful, Info) -> any()` 30 | 31 | -------------------------------------------------------------------------------- /doc/jobs_queue.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_queue # 4 | * [Description](#description) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | Default queue behaviour for JOBS (using ordered_set ets). 9 | 10 | __This module defines the `jobs_queue` behaviour.__
Required callback functions: `new/2`, `delete/1`, `in/3`, `peek/1`, `out/2`, `all/1`, `info/2`. 11 | 12 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 13 | 14 | 15 | 16 | ## Description ## 17 | This module implements the default queue behaviour for JOBS, and also 18 | specifies the behaviour itself. 19 | 20 | ## Function Index ## 21 | 22 | 23 |
all/1Return all the job entries in the queue, not removing them from the queue.
behaviour_info/1
delete/1Queue is being deleted; remove any external data structures.
empty/1
in/3Enqueue a job reference; return the updated queue.
info/2Return information about the queue.
is_empty/1
new/2Instantiate a new queue.
out/2Dequeue a batch of N jobs; return the modified queue.
peek/1Looks at the first item in the queue, without removing it.
representation/1A representation of a queue which can be inspected.
timedout/1Return all entries that have been in the queue longer than MaxTime.
timedout/2
24 | 25 | 26 | 27 | 28 | ## Function Details ## 29 | 30 | 31 | 32 | ### all/1 ### 33 | 34 | 

 35 | all(Queue::#queue{}) -> [JobEntry]
 36 |
37 |
38 | 39 | Return all the job entries in the queue, not removing them from the queue. 40 | 41 | 42 | 43 | ### behaviour_info/1 ### 44 | 45 | `behaviour_info(X1) -> any()` 46 | 47 | 48 | 49 | ### delete/1 ### 50 | 51 | 

 52 | delete(Queue::#queue{}) -> any()
 53 |
54 |
55 | 56 | Queue is being deleted; remove any external data structures. 57 | 58 | If the queue behaviour has created an ETS table or similar, this is the place 59 | to get rid of it. 60 | 61 | 62 | 63 | ### empty/1 ### 64 | 65 | `empty(Queue) -> any()` 66 | 67 | 68 | 69 | ### in/3 ### 70 | 71 | 

 72 | in(TS::Timestamp, Job, Queue::#queue{}) -> #queue{}
 73 |
74 |
75 | 76 | Enqueue a job reference; return the updated queue. 77 | 78 | This puts a job into the queue. The callback function is responsible for 79 | updating the #queue.oldest_job attribute, if needed. The #queue.oldest_job 80 | attribute shall either contain the Timestamp of the oldest job in the queue, 81 | or `undefined` if the queue is empty. It may be noted that, especially in the 82 | fairly trivial case of the `in/3` function, the oldest job would be 83 | `erlang:min(Timestamp, PreviousOldest)`, even if `PreviousOldest == undefined`. 84 | 85 | 86 | 87 | ### info/2 ### 88 | 89 | 

 90 | info(X1::Item, Queue::#queue{}) -> Info
 91 |
92 | 93 | 94 | 95 | Return information about the queue. 96 | 97 | 98 | 99 | ### is_empty/1 ### 100 | 101 | 

102 | is_empty(Queue::#queue{}) -> boolean()
103 |
104 |
105 | 106 | 107 | 108 | ### new/2 ### 109 | 110 | 

111 | new(Options, Q::#queue{}) -> #queue{}
112 |
113 |
114 | 115 | Instantiate a new queue. 116 | 117 | Options is the list of options provided when defining the queue. 118 | Q is an initial #queue{} record. It can be used directly by including 119 | `jobs/include/jobs.hrl`, or by using exprecs-style record accessors in the 120 | module `jobs_info`. 121 | See [parse_trans](http://github.com/uwiger/parse_trans) for more info 122 | on exprecs. In the `new/2` function, the #queue.st attribute will normally be 123 | used to keep track of the queue data structure. 124 | 125 | 126 | 127 | ### out/2 ### 128 | 129 | 

130 | out(N::integer(), Queue::#queue{}) -> {[Entry], #queue{}}
131 |
132 |
133 | 134 | Dequeue a batch of N jobs; return the modified queue. 135 | 136 | Note that this function may need to update the #queue.oldest_job attribute, 137 | especially if the queue becomes empty. 138 | 139 | 140 | 141 | ### peek/1 ### 142 | 143 | 

144 | peek(Queue::#queue{}) -> JobEntry | undefined
145 |
146 |
147 | 148 | Looks at the first item in the queue, without removing it. 149 | 150 | 151 | 152 | ### representation/1 ### 153 | 154 | `representation(Queue) -> any()` 155 | 156 | A representation of a queue which can be inspected 157 | 158 | 159 | 160 | ### timedout/1 ### 161 | 162 | 

163 | timedout(Queue::#queue{}) -> {[Entry], #queue{}}
164 |
165 |
166 | 167 | Return all entries that have been in the queue longer than MaxTime. 168 | 169 | NOTE: This is an inspection function; it doesn't remove the job entries. 170 | 171 | 172 | 173 | ### timedout/2 ### 174 | 175 | `timedout(TO, Queue) -> any()` 176 | 177 | -------------------------------------------------------------------------------- /doc/jobs_queue_list.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_queue_list # 4 | * [Data Types](#types) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 9 | 10 | 11 | 12 | ## Data Types ## 13 | 14 | 15 | 16 | 17 | ### entry() ### 18 | 19 | 20 | 

 21 | entry() = {timestamp(), job()}
 22 |
23 | 24 | 25 | 26 | 27 | ### info_item() ### 28 | 29 | 30 | 

 31 | info_item() = max_time | oldest_job | length
 32 |
33 | 34 | 35 | 36 | 37 | ### job() ### 38 | 39 | 40 | 

 41 | job() = {pid(), reference()}
 42 |
43 | 44 | 45 | 46 | ## Function Index ## 47 | 48 | 49 |
all/1
delete/1
empty/1
in/3
info/2
is_empty/1
new/2
out/2
peek/1
representation/1
timedout/1
timedout/2
50 | 51 | 52 | 53 | 54 | ## Function Details ## 55 | 56 | 57 | 58 | ### all/1 ### 59 | 60 | 

 61 | all(Queue::#queue{}) -> [entry()]
 62 |
63 |
64 | 65 | 66 | 67 | ### delete/1 ### 68 | 69 | `delete(Queue) -> any()` 70 | 71 | 72 | 73 | ### empty/1 ### 74 | 75 | `empty(Queue) -> any()` 76 | 77 | 78 | 79 | ### in/3 ### 80 | 81 | 

 82 | in(TS::timestamp(), Job::job(), Queue::#queue{}) -> #queue{}
 83 |
84 |
85 | 86 | 87 | 88 | ### info/2 ### 89 | 90 | 

 91 | info(X1::info_item(), Queue::#queue{}) -> any()
 92 |
93 |
94 | 95 | 96 | 97 | ### is_empty/1 ### 98 | 99 | 

100 | is_empty(Queue::#queue{}) -> boolean()
101 |
102 |
103 | 104 | 105 | 106 | ### new/2 ### 107 | 108 | `new(Options, Q) -> any()` 109 | 110 | 111 | 112 | ### out/2 ### 113 | 114 | 

115 | out(N::integer(), Queue::#queue{}) -> {[entry()], #queue{}}
116 |
117 |
118 | 119 | 120 | 121 | ### peek/1 ### 122 | 123 | `peek(Queue) -> any()` 124 | 125 | 126 | 127 | ### representation/1 ### 128 | 129 | `representation(Queue) -> any()` 130 | 131 | 132 | 133 | ### timedout/1 ### 134 | 135 | 

136 | timedout(Queue::#queue{}) -> {[entry()], #queue{}}
137 |
138 |
139 | 140 | 141 | 142 | ### timedout/2 ### 143 | 144 | `timedout(TO, Queue) -> any()` 145 | 146 | -------------------------------------------------------------------------------- /doc/jobs_sampler.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_sampler # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | __This module defines the `jobs_sampler` behaviour.__
Required callback functions: `init/2`, `sample/2`, `handle_msg/3`, `calc/2`. 8 | 9 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 10 | 11 | 12 | 13 | ## Function Index ## 14 | 15 | 16 |
calc/3
code_change/3
end_subscription/0
handle_call/3
handle_cast/2
handle_info/2
init/1
start_link/0
start_link/1
subscribe/0Subscribes to feedback indicator information.
tell_sampler/2
terminate/2
trigger_sample/0
17 | 18 | 19 | 20 | 21 | ## Function Details ## 22 | 23 | 24 | 25 | ### calc/3 ### 26 | 27 | `calc(Type, Template, History) -> any()` 28 | 29 | 30 | 31 | ### code_change/3 ### 32 | 33 | `code_change(FromVsn, State, Extra) -> any()` 34 | 35 | 36 | 37 | ### end_subscription/0 ### 38 | 39 | `end_subscription() -> any()` 40 | 41 | 42 | 43 | ### handle_call/3 ### 44 | 45 | `handle_call(X1, From, State) -> any()` 46 | 47 | 48 | 49 | ### handle_cast/2 ### 50 | 51 | `handle_cast(X1, S) -> any()` 52 | 53 | 54 | 55 | ### handle_info/2 ### 56 | 57 | `handle_info(Msg, State) -> any()` 58 | 59 | 60 | 61 | ### init/1 ### 62 | 63 | `init(Opts) -> any()` 64 | 65 | 66 | 67 | ### start_link/0 ### 68 | 69 | `start_link() -> any()` 70 | 71 | 72 | 73 | ### start_link/1 ### 74 | 75 | `start_link(Opts) -> any()` 76 | 77 | 78 | 79 | ### subscribe/0 ### 80 | 81 | 

 82 | subscribe() -> ok
 83 |
84 |
85 | 86 | Subscribes to feedback indicator information 87 | 88 | This function allows a process to receive the same information as the 89 | jobs_server any time the information changes. 90 | 91 | The notifications are delivered on the format `{jobs_indicators, Info}`, 92 | where 93 | 94 | ``` 95 | 96 | Info :: [{IndicatorName, LocalValue, Remote}] 97 | Remote :: [{NodeName, Value}] 98 | ``` 99 | 100 | This information could be used e.g. to aggregate the information and generate 101 | new sampler information (which could be passed to a sampler plugin using 102 | [`tell_sampler/2`](#tell_sampler-2), or to a specific queue using [`jobs:ask_queue/2`](jobs.md#ask_queue-2). 103 | 104 | 105 | 106 | ### tell_sampler/2 ### 107 | 108 | `tell_sampler(P, Msg) -> any()` 109 | 110 | 111 | 112 | ### terminate/2 ### 113 | 114 | `terminate(X1, S) -> any()` 115 | 116 | 117 | 118 | ### trigger_sample/0 ### 119 | 120 | `trigger_sample() -> any()` 121 | 122 | -------------------------------------------------------------------------------- /doc/jobs_sampler_cpu.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_sampler_cpu # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | __Behaviours:__ [`jobs_sampler`](jobs_sampler.md). 8 | 9 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 10 | 11 | 12 | 13 | ## Function Index ## 14 | 15 | 16 |
calc/2
handle_msg/3
init/2
sample/2
17 | 18 | 19 | 20 | 21 | ## Function Details ## 22 | 23 | 24 | 25 | ### calc/2 ### 26 | 27 | `calc(History, St) -> any()` 28 | 29 | 30 | 31 | ### handle_msg/3 ### 32 | 33 | `handle_msg(Msg, Timestamp, ModS) -> any()` 34 | 35 | 36 | 37 | ### init/2 ### 38 | 39 | `init(Name, Opts) -> any()` 40 | 41 | 42 | 43 | ### sample/2 ### 44 | 45 | `sample(Timestamp, St) -> any()` 46 | 47 | -------------------------------------------------------------------------------- /doc/jobs_sampler_history.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_sampler_history # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | ## Function Index ## 10 | 11 | 12 |
add/2
from_list/2
new/1
take_last/2
to_list/1
13 | 14 | 15 | 16 | 17 | ## Function Details ## 18 | 19 | 20 | 21 | ### add/2 ### 22 | 23 | `add(Entry, Jsh) -> any()` 24 | 25 | 26 | 27 | ### from_list/2 ### 28 | 29 | `from_list(MaxL, L0) -> any()` 30 | 31 | 32 | 33 | ### new/1 ### 34 | 35 | `new(Length) -> any()` 36 | 37 | 38 | 39 | ### take_last/2 ### 40 | 41 | `take_last(F, Jsh) -> any()` 42 | 43 | 44 | 45 | ### to_list/1 ### 46 | 47 | `to_list(Jsh) -> any()` 48 | 49 | -------------------------------------------------------------------------------- /doc/jobs_sampler_mnesia.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_sampler_mnesia # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | __Behaviours:__ [`jobs_sampler`](jobs_sampler.md). 8 | 9 | 10 | 11 | ## Function Index ## 12 | 13 | 14 |
calc/2
handle_msg/3
init/2
sample/2
15 | 16 | 17 | 18 | 19 | ## Function Details ## 20 | 21 | 22 | 23 | ### calc/2 ### 24 | 25 | `calc(History, St) -> any()` 26 | 27 | 28 | 29 | ### handle_msg/3 ### 30 | 31 | `handle_msg(X1, T, S) -> any()` 32 | 33 | 34 | 35 | ### init/2 ### 36 | 37 | `init(Name, Opts) -> any()` 38 | 39 | 40 | 41 | ### sample/2 ### 42 | 43 | `sample(T, S) -> any()` 44 | 45 | -------------------------------------------------------------------------------- /doc/jobs_server.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_server # 4 | * [Data Types](#types) 5 | * [Function Index](#index) 6 | * [Function Details](#functions) 7 | 8 | __Behaviours:__ [`gen_server`](gen_server.md). 9 | 10 | __Authors:__ : Ulf Wiger ([`ulf@wiger.net`](mailto:ulf@wiger.net)). 11 | 12 | 13 | 14 | ## Data Types ## 15 | 16 | 17 | 18 | 19 | ### ctx_options() ### 20 | 21 | 22 | 

 23 | ctx_options() = [{_Ctxt::env | opts, _App::user | atom(), [option()]}]
 24 |
25 | 26 | 27 | 28 | 29 | ### info_category() ### 30 | 31 | 32 | 

 33 | info_category() = queues | group_rates | counters
 34 |
35 | 36 | 37 | 38 | 39 | ### queue_name() ### 40 | 41 | 42 | 

 43 | queue_name() = any()
 44 |
45 | 46 | 47 | 48 | ## Function Index ## 49 | 50 | 51 |
add_counter/2
add_group_rate/2
add_queue/2
ask/0
ask/1
ask_queue/2Invoke the Q:handle_call/3 function (if it exists).
auto_restore/1
code_change/3
delete_counter/1
delete_group_rate/1
delete_queue/1
dequeue/2
done/1
enqueue/2
handle_call/3
handle_cast/2
handle_info/2
info/1
init/1
modify_counter/2
modify_group_rate/2
modify_queue/2
modify_regulator/4
queue_info/1
queue_info/2
run/1
run/2
set_modifiers/1
start_link/0
start_link/1
terminate/2
timestamp/0
timestamp_to_datetime/1
52 | 53 | 54 | 55 | 56 | ## Function Details ## 57 | 58 | 59 | 60 | ### add_counter/2 ### 61 | 62 | `add_counter(Name, Options) -> any()` 63 | 64 | 65 | 66 | ### add_group_rate/2 ### 67 | 68 | `add_group_rate(Name, Options) -> any()` 69 | 70 | 71 | 72 | ### add_queue/2 ### 73 | 74 | 

 75 | add_queue(Name::queue_name(), Options::q_opts()) -> ok
 76 |
77 |
78 | 79 | 80 | 81 | ### ask/0 ### 82 | 83 | 

 84 | ask() -> {ok, any()} | {error, rejected | timeout}
 85 |
86 |
87 | 88 | 89 | 90 | ### ask/1 ### 91 | 92 | 

 93 | ask(Type::job_class()) -> {ok, reg_obj()} | {error, rejected | timeout}
 94 |
95 |
96 | 97 | 98 | 99 | ### ask_queue/2 ### 100 | 101 | 

102 | ask_queue(QName, Request) -> Reply
103 |
104 |
105 | 106 | Invoke the Q:handle_call/3 function (if it exists). 107 | 108 | Send a request to a specific queue in the JOBS server. 109 | Each queue has its own local state, allowing it to collect special statistics. 110 | This function allows a client to send a request that is handled by a specific 111 | queue instance, either to pull information from the queue, or to influence its 112 | state. 113 | 114 | 115 | 116 | ### auto_restore/1 ### 117 | 118 | `auto_restore(Flag) -> any()` 119 | 120 | 121 | 122 | ### code_change/3 ### 123 | 124 | `code_change(FromVsn, St, Extra) -> any()` 125 | 126 | 127 | 128 | ### delete_counter/1 ### 129 | 130 | `delete_counter(Name) -> any()` 131 | 132 | 133 | 134 | ### delete_group_rate/1 ### 135 | 136 | `delete_group_rate(Name) -> any()` 137 | 138 | 139 | 140 | ### delete_queue/1 ### 141 | 142 | 

143 | delete_queue(Name::queue_name()) -> ok
144 |
145 |
146 | 147 | 148 | 149 | ### dequeue/2 ### 150 | 151 | 

152 | dequeue(Type::job_class(), N::integer() | infinity) -> [{timestamp(), any()}]
153 |
154 |
155 | 156 | 157 | 158 | ### done/1 ### 159 | 160 | 

161 | done(Opaque::reg_obj()) -> ok
162 |
163 |
164 | 165 | 166 | 167 | ### enqueue/2 ### 168 | 169 | 

170 | enqueue(Type::job_class(), Item::any()) -> ok
171 |
172 |
173 | 174 | 175 | 176 | ### handle_call/3 ### 177 | 178 | `handle_call(Req, From, S) -> any()` 179 | 180 | 181 | 182 | ### handle_cast/2 ### 183 | 184 | `handle_cast(Msg, St) -> any()` 185 | 186 | 187 | 188 | ### handle_info/2 ### 189 | 190 | `handle_info(Msg, St) -> any()` 191 | 192 | 193 | 194 | ### info/1 ### 195 | 196 | 

197 | info(Item::info_category()) -> [any()]
198 |
199 |
200 | 201 | 202 | 203 | ### init/1 ### 204 | 205 | 

206 | init(Opts::[option()]) -> {ok, #st{queues = [#queue{}] | [tuple()], group_rates = [#grp{}], counters = [#cr{}], monitors = any(), q_select = atom(), q_select_st = any(), default_queue = any(), info_f = any()}}
207 |
208 |
209 | 210 | 211 | 212 | ### modify_counter/2 ### 213 | 214 | `modify_counter(Name, Opts) -> any()` 215 | 216 | 217 | 218 | ### modify_group_rate/2 ### 219 | 220 | `modify_group_rate(Name, Opts) -> any()` 221 | 222 | 223 | 224 | ### modify_queue/2 ### 225 | 226 | 

227 | modify_queue(Name::queue_name(), Options::q_opts()) -> ok
228 |
229 |
230 | 231 | 232 | 233 | ### modify_regulator/4 ### 234 | 235 | `modify_regulator(Type, QName, RegName, Opts) -> any()` 236 | 237 | 238 | 239 | ### queue_info/1 ### 240 | 241 | `queue_info(Name) -> any()` 242 | 243 | 244 | 245 | ### queue_info/2 ### 246 | 247 | `queue_info(Name, Item) -> any()` 248 | 249 | 250 | 251 | ### run/1 ### 252 | 253 | 

254 | run(Fun::fun(() -> X)) -> X
255 |
256 |
257 | 258 | 259 | 260 | ### run/2 ### 261 | 262 | 

263 | run(Type::job_class(), Fun::fun(() -> X)) -> X
264 |
265 |
266 | 267 | 268 | 269 | ### set_modifiers/1 ### 270 | 271 | `set_modifiers(Modifiers) -> any()` 272 | 273 | 274 | 275 | ### start_link/0 ### 276 | 277 | 

278 | start_link() -> {ok, pid()}
279 |
280 |
281 | 282 | 283 | 284 | ### start_link/1 ### 285 | 286 | 

287 | start_link(Opts0::ctx_options()) -> {ok, pid()}
288 |
289 |
290 | 291 | 292 | 293 | ### terminate/2 ### 294 | 295 | `terminate(X1, X2) -> any()` 296 | 297 | 298 | 299 | ### timestamp/0 ### 300 | 301 | `timestamp() -> any()` 302 | 303 | 304 | 305 | ### timestamp_to_datetime/1 ### 306 | 307 | `timestamp_to_datetime(TS) -> any()` 308 | 309 | -------------------------------------------------------------------------------- /doc/jobs_stateful_simple.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Module jobs_stateful_simple # 4 | * [Function Index](#index) 5 | * [Function Details](#functions) 6 | 7 | 8 | 9 | ## Function Index ## 10 | 11 | 12 |
handle_call/4
init/2
next/3
13 | 14 | 15 | 16 | 17 | ## Function Details ## 18 | 19 | 20 | 21 | ### handle_call/4 ### 22 | 23 | `handle_call(Req, From, Stateful, Info) -> any()` 24 | 25 | 26 | 27 | ### init/2 ### 28 | 29 | `init(F, Info) -> any()` 30 | 31 | 32 | 33 | ### next/3 ### 34 | 35 | `next(Opaque, Stateful, Info) -> any()` 36 | 37 | -------------------------------------------------------------------------------- /doc/overview.edoc: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | @copyright 2014-2018 Ulf Wiger 18 | @version 0.9.0 19 | @title jobs - a Job scheduler for load regulation 20 | 21 | @doc 22 | 23 | JOBS 24 | ==== 25 | 26 | Jobs is a job scheduler for load regulation of Erlang applications. 27 | It provides a queueing framework where each queue can be configured 28 | for throughput rate, credit pool and feedback compensation. 29 | Queues can be added and modified at runtime, and customizable 30 | "samplers" propagate load status across all nodes in the system. 31 | 32 | Specifically, jobs provides three features: 33 | 34 | * Job scheduling: A job is scheduled according to certain constraints. 35 | For instance, you may want to define that no more than 9 jobs of a 36 | certain type can execute simultaneously and the maximal rate at 37 | which you can start such jobs are 300 per second. 38 | * Job queueing: When load is higher than the scheduling limits 39 | additional jobs are *queued* by the system to be run later when load 40 | clears. Certain rules govern queues: are they dequeued in FIFO or 41 | LIFO order? How many jobs can the queue take before it is full? Is 42 | there a deadline after which jobs should be rejected. When we hit 43 | the queue limits we reject the job. This provides a feedback 44 | mechanism on the client of the queue so you can take action. 45 | * Sampling and dampening: Periodic samples of the Erlang VM can 46 | provide information about the health of the system in general. If we 47 | have high CPU load or high memory usage, we apply dampening to the 48 | scheduling rules: we may lower the concurrency count or the rate at 49 | which we execute jobs. When the health problem clears, we remove the 50 | dampener and run at full speed again. 51 | 52 | Error recovery 53 | -------------- 54 | 55 | The Jobs server is designed to not crash. However, in the unlikely event 56 | that it should occur (and it has!) Jobs does not automatically restore changes 57 | that have been effected through the API. This can be enabled, setting the 58 | Jobs environment variable `auto_restore' to `true', or calling the function 59 | `jobs_server:auto_restore(true)'. This will tell the jobs_server to remember 60 | every configuration change and replay them, in order, after a process restart. 61 | 62 | Examples 63 | -------- 64 | 65 | The following examples are fetched from the EUC 2013 presentation on Jobs. 66 | 67 |

Regulate incoming HTTP requests (e.g. JSON-RPC)

68 | 69 | 
 70 | %% @doc Handle a JSON-RPC request.
 71 | handler_session(Arg) ->
 72 |     jobs:run(
 73 |         rpc_from_web,
 74 |         fun() ->
 75 |                try
 76 |                   yaws_rpc:handler_session(
 77 |                     maybe_multipart(Arg),{?MODULE, web_rpc})
 78 |                catch
 79 |                    error:E ->
 80 |                        ...
 81 |                end
 82 |     end).
 83 |
84 | 85 |

From Riak prototype, using explicit ask/done

86 | 87 | 
 88 | case jobs:ask(riak_kv_fsm) of
 89 |   {ok, JobId} ->
 90 |     try
 91 |       {ok, Pid} = riak_kv_get_fsm_sup:start_get_fsm(...),
 92 |       Timeout = recv_timeout(Options),
 93 |       wait_for_reqid(ReqId, Timeout)
 94 |     after
 95 |       jobs:done(JobId)  %% Only needed if process stays alive
 96 |     end;
 97 |   {error, rejected} ->  %% Overload!
 98 |     {error, timeout}
 99 | end
100 |
101 | 102 |

Shell demo - simple rate-limited queue

103 | 104 | 
105 | 2> jobs:add_queue(q, [{standard_rate,1}]).
106 | ok
107 | 3> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end).
108 | job: {14,37,7}
109 | ok
110 | 4> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end).
111 | job: {14,37,8}
112 | ok
113 | ...
114 | 5> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end).
115 | job: {14,37,10}
116 | ok
117 | 6> jobs:run(q, fun() -> io:fwrite("job: ~p~n", [time()]) end).
118 | job: {14,37,11}
119 | ok
120 |
121 | 122 |

Shell demo - "stateful" queues

123 | 124 | 
125 | Eshell V5.9.2 (abort with ^G)
126 | 1> application:start(jobs).
127 | ok
128 | 2> jobs:add_queue(q,
129 |      [{standard_rate,1},
130 |       {stateful,fun(init,_) -> {0,5};
131 |                    ({call,{size,Sz},_,_},_) -> {reply,ok,{0,Sz}};
132 |                    ({N,Sz},_) -> {N, {(N+1) rem Sz,Sz}}
133 |                 end}]).
134 | ok
135 | 3> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
136 | 0
137 | 4> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
138 | 1
139 | 5> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
140 | 2
141 | 6> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
142 | 3
143 | 7> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
144 | 4
145 | 8> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
146 | 0
147 | 9> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
148 | 1
149 | %% Resize the 'pool'
150 | 10> jobs:ask_queue(q, {size,3}).
151 | ok
152 | 11> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
153 | 0
154 | 12> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
155 | 1
156 | 13> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
157 | 2
158 | 14> jobs:run(q,fun(Opaque) -> jobs:job_info(Opaque) end).
159 | 0
160 | ...
161 |
162 | 163 |

Demo - producers

164 | 165 | 
166 | Eshell V5.9.2 (abort with ^G)
167 | 1> application:start(jobs).
168 | ok
169 | 2> jobs:add_queue(p,
170 |   [{producer, fun() -> io:fwrite("job: ~p~n",[time()]) end},
171 |    {standard_rate,1}]).
172 | job: {14,33,51}
173 | ok
174 | 3> job: {14,33,52}
175 | job: {14,33,53}
176 | job: {14,33,54}
177 | job: {14,33,55}
178 | ...
179 |
180 | 181 |

Demo - passive queues

182 | 183 | 
184 | 2> jobs:add_queue(q,[passive]).
185 | ok
186 | 3> Fun = fun() -> io:fwrite("~p starting...~n",[self()]),
187 | 3>                Res = jobs:dequeue(q, 3),
188 | 3>                io:fwrite("Res = ~p~n", [Res])
189 | 3>       end.
190 | #Fun<erl_eval.20.82930912>
191 | 4> jobs:add_queue(p, [{standard_counter,3},{producer,Fun}]).
192 | <0.47.0> starting...
193 | <0.48.0> starting...
194 | <0.49.0> starting...
195 | ok
196 | 5> jobs:enqueue(q, job1).
197 | Res = [{113214444910647,job1}]
198 | ok
199 | <0.54.0> starting...
200 |
201 | 202 |

Demo - queue status

203 | 204 | 
205 | (a@uwair)1> jobs:queue_info(q).
206 | {queue,[{name,q},
207 |         {mod,jobs_queue},
208 |         {type,fifo},
209 |         {group,undefined},
210 |         {regulators,[{rr,[{name,{rate,q,1}},
211 |                           {rate,{rate,[{limit,1},
212 |                           {preset_limit,1},
213 |                           {interval,1.0e3},
214 |                           {modifiers,
215 |                            [{cpu,10},{memory,10}]},
216 |                           {active_modifiers,[]}
217 |                          ]}}]}]},
218 |         {max_time,undefined},
219 |         {max_size,undefined},
220 |         {latest_dispatch,113216378663298},
221 |         {approved,4},
222 |         {queued,0},
223 |         ...,
224 |         {stateful,undefined},
225 |         {st,{st,45079}}]}
226 |
227 | 228 |
229 | 230 | ##Scenarios and Corresponding Configuration Examples 231 | 232 | ####EXAMPLE 1: 233 | * Add counter regulated queue called heavy_crunches to limit your cpu intensive code executions to no more than 7 at a time 234 | 235 | Configuration: 236 | 
237 | { jobs, [
238 |     { queues, [
239 |         { heavy_crunches, [ { regulators, [{ counter, [{ limit, 7 }] } ] }] }
240 |       ]
241 |     }
242 |   ]
243 | }
244 |
245 | 246 | Anywhere in your code wrap cpu-intensive work in a call to jobs server and-- voilà! --it is counter-regulated: 247 | 248 | 
249 | jobs:run( heavy_crunches,fun()->my_cpu_intensive_calculation() end)
250 |
251 | 252 | ####EXAMPLE 2: 253 | * Add rate regulated queue called http_requests to ensure that your http server gets no more than 1000 requests per second. 254 | * Additionally, set the queue size to 10,000 (i.e. to control queue memory consumption) 255 | 256 | Configuration: 257 | 258 | 
259 | { jobs, [
260 |       { queues, [
261 |             { http_requests, [ { max_size, 10000},  {regulators, [{ rate, [{limit, 1000}]}]}]}
262 |         ]
263 |       }
264 |   ]
265 | }
266 |
267 | 268 | Wrap your request entry point in a call to jobs server and it will end up being rate-regulated. 269 | 270 | 
271 | jobs:run(http_requests,fun()->handle_http_request() end)
272 |
273 | 274 | NOTE: with the config above, once 10,000 requests accumulates in the queue any incoming requests are dropped on the floor. 275 | 276 | ####EXAMPLE 3: 277 | * HTTP requests will always have a reasonable execution time. No point in keeping them in the queue past the timeout. 278 | 279 | * Let's create patient_user_requests queue that will keep requests in the queue for up to 10 seconds 280 | 281 | 
282 | { patient_user_requests, [
283 |     { max_time, 10000},
284 |     { regulators, [{rate, [ { limit, 1000 } ] }
285 |   ]
286 | }
287 |
288 | * Let's create impatient_user_requests queue that will keep requests in the queue for up to 200 milliseconds. 289 | Additionally, we'll make it a LIFO queue. Unfair, but if we assume that happy/unhappy is a boolean 290 | we're likely to maximize the happy users! 291 | 292 | 293 | 294 | 
295 | { impatient_user_requests, [
296 |     { max_time, 200},
297 |     { type, lifo},
298 |     { regulators, [{rate, [ { limit, 1000 } ] }
299 |   ]
300 | }
301 |
302 | 303 | NOTE: In order to pace requests from both queues at 1000 per second, use group_rate regulation (EXAMPLE 4) 304 | 305 | 306 | ####EXAMPLE 4: 307 | * Rate regulate http requests from multiple queues 308 | 309 | Create group_rates regulator called http_request_rate and assign it to both impatient_user_requests and patient_user_requests 310 | 311 | 
312 | { jobs, [
313 |     { group_rates,[{ http_request_rate, [{limit,1000}] }] },
314 |     { queues, [
315 |         { impatient_user_requests,
316 |             [ {max_time, 200},
317 |               {type, lifo},
318 |               {regulators,[{ group_rate, http_request_rate}]}
319 |             ]
320 |         },
321 |         { patient_user_requests,
322 |             [ {max_time, 10000},
323 |               {regulators,[{ group_rate, http_request_rate}
324 |             ]
325 |         }
326 |       ]
327 |     }
328 |   ]
329 | }
330 |
331 | 332 | ####EXAMPLE 5: 333 | * Can't afford to drop http requests on the floor once max_size is reached? 334 | * Implement and use your own queue to persist those unfortunate http requests and serve them eventually 335 | 336 | 337 | 
338 |  -module(my_persistent_queue).
339 |  -behaviour(jobs_queue).
340 |  -export([  new/2,
341 |             delete/1,
342 |             in/3,
343 |             out/2,
344 |             peek/1,
345 |             info/2,
346 |             all/1]).
347 | 
348 |  ## implementation
349 |  ...
350 |
351 | 352 | Configuration: 353 | 354 | 
355 | { jobs, [
356 |     { queues, [
357 |         { http_requests, [
358 |             { mod, my_persistent_queue},
359 |             { max_size, 10000 },
360 |             { regulators, [ { rate, [ { limit, 1000 } ] } ] }
361 |           ]
362 |         }
363 |       ]
364 |     }
365 |   ]
366 | }
367 |
368 | 369 | 370 | ###The use of sampler framework 371 | 1. Get a sampler running and sending feedback to the jobs server. 372 | 2. Apply its feedback to a regulator limit. 373 | 374 | ####EXAMPLE 6: 375 | * Adjust rate regulator limit on the fly based on the feedback from jobs_sampler_cpu named cpu_feedback 376 | 377 | 
378 | { jobs, [
379 |     { samplers, [{ cpu_feedback, jobs_sampler_cpu, [] } ] },
380 |     { queues, [
381 |         { http_requests, [
382 |             { regulators,   [ { rate, [ { limit,1000 } ]  },
383 |             { modifiers,    [ { cpu_feedback,  10} ] } %% 10 = % increment by which to modify the limit
384 |           ]
385 |         }
386 |       ]
387 |     }
388 |   ]
389 | }
390 |
391 | 392 | 393 | Prerequisites 394 | ------------- 395 | This application requires 'exprecs'. 396 | The 'exprecs' module is part of http://github.com/uwiger/parse_trans 397 | 398 | Contribute 399 | ---------- 400 | For issues, comments or feedback please [create an issue!] [1] 401 | 402 | [1]: http://github.com/uwiger/jobs/issues "jobs issues" 403 | 404 | @end -------------------------------------------------------------------------------- /doc/stylesheet.css: -------------------------------------------------------------------------------- 1 | /* standard EDoc style sheet */ 2 | body { 3 | font-family: Verdana, Arial, Helvetica, sans-serif; 4 | margin-left: .25in; 5 | margin-right: .2in; 6 | margin-top: 0.2in; 7 | margin-bottom: 0.2in; 8 | color: #000000; 9 | background-color: #ffffff; 10 | } 11 | h1,h2 { 12 | margin-left: -0.2in; 13 | } 14 | div.navbar { 15 | background-color: #add8e6; 16 | padding: 0.2em; 17 | } 18 | h2.indextitle { 19 | padding: 0.4em; 20 | background-color: #add8e6; 21 | } 22 | h3.function,h3.typedecl { 23 | background-color: #add8e6; 24 | padding-left: 1em; 25 | } 26 | div.spec { 27 | margin-left: 2em; 28 | background-color: #eeeeee; 29 | } 30 | a.module { 31 | text-decoration:none 32 | } 33 | a.module:hover { 34 | background-color: #eeeeee; 35 | } 36 | ul.definitions { 37 | list-style-type: none; 38 | } 39 | ul.index { 40 | list-style-type: none; 41 | background-color: #eeeeee; 42 | } 43 | 44 | /* 45 | * Minor style tweaks 46 | */ 47 | ul { 48 | list-style-type: square; 49 | } 50 | table { 51 | border-collapse: collapse; 52 | } 53 | td { 54 | padding: 3 55 | } 56 | -------------------------------------------------------------------------------- /examples/jobs_cpu.gnu: -------------------------------------------------------------------------------- 1 | set terminal png transparent nocrop enhanced font arial 8 size 800,600 2 | 3 | set autoscale y 4 | set autoscale y2 5 | 6 | set key autotitle columnhead 7 | plot "jobs_cpu.dat" using 2 with lines axes x1y1, "jobs_cpu.dat" using 3 with lines axes x2y2, "jobs_cpu.dat" using 4 with impulses axes x1y1, "jobs_cpu.dat" using 5 with impulses axes x1y1, "jobs_cpu.dat" using 6 with lines axes x1y1 8 | 9 | -------------------------------------------------------------------------------- /examples/performance_logger.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | -module(performance_logger). 18 | -behaviour(gen_server). 19 | 20 | %% Interface 21 | -export([start_link/0, 22 | increment_counter/2, 23 | decrement_counter/2, 24 | set_counter/2, 25 | start_recording/1, 26 | end_recording/0, 27 | save_recorded_data_to_file/1]). 28 | 29 | %% gen_server-specific 30 | -export([init/1, 31 | handle_cast/2]). 32 | 33 | %% internal 34 | -export([tick/0]). 35 | 36 | %% testing 37 | -export([test_recording/0]). 38 | 39 | -export([spec_to_gnuplot_script/1]). 40 | 41 | -define(SAMPLING_TIME, 125). %% Duration between samples, in msec. 42 | -define(PFL_ETS_NAME, pfl_stats). 43 | 44 | -define(TIME_COMPUTATION_SCALE, 1). 45 | 46 | %% :) 47 | -define(WITH_OPEN_FILE(Anaphora,FileName,Modes,Code), case file:open(FileName,Modes) of 48 | {ok, Anaphora} -> Code, file:close(Anaphora); 49 | SthElse -> SthElse 50 | end). 51 | 52 | %% Data spec description 53 | -type counter_type() :: fun(() -> any()) | atom(). %% For custom counters one can supply a 0-arity fun instead of an usual name. 54 | -type counter_name() :: string(). %% Name used on chart as a label. 55 | -type sampling_type() :: diff | identity | accumulate. 56 | -type data_spec() :: [{counter_type(), counter_name(), sampling_type()}]. 57 | 58 | 59 | %% GENSERVERIFY BEGIN 60 | %% Interface 61 | start_link() -> 62 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 63 | 64 | %%% Counter operations 65 | increment_counter(CounterName, Increment) -> 66 | gen_server:cast(?MODULE, {incf, CounterName, Increment}). 67 | 68 | decrement_counter(CounterName, Decrement) -> 69 | gen_server:cast(?MODULE, {decf, CounterName, Decrement}). 70 | 71 | set_counter(CounterName, NewValue) -> 72 | gen_server:cast(?MODULE, {setf, CounterName, NewValue}). 73 | 74 | %%% Data recording 75 | %% Note the DataSpec. 76 | -spec start_recording(DataSpec :: data_spec()) -> any(). 77 | start_recording(DataSpec) -> 78 | gen_server:cast(?MODULE, {start_recording, DataSpec}). 79 | 80 | end_recording() -> 81 | gen_server:cast(?MODULE, end_recording). 82 | 83 | save_recorded_data_to_file(FileName) -> 84 | gen_server:cast(?MODULE, {save_data, FileName}). 85 | 86 | %% A call to tick causes the Logger to take a snapshot of current counter values and save that as a sample. 87 | %% NOTE if we get flooded with calls to 88 | %% increment/decrement/set_counter and gen_server starts falling 89 | %% behind, delays between samples may not exactly match the specified rate. 90 | tick() -> 91 | gen_server:cast(?MODULE, ping). 92 | 93 | %% Implementation 94 | -record(state, {data_spec, %% data spec, see description at the top of the file 95 | start_time, %% the time of the start of start_recording,it is used to calculate the offset from this start time, the offset is used to indicate the descending order of the ets table "pfl_stats" key 96 | timer}). %% reference to timer that makes us take samples every now and then 97 | 98 | init(_State) -> 99 | ets:new(?PFL_ETS_NAME, [ordered_set, named_table, public]), 100 | {ok, #state{}}. 101 | 102 | handle_cast({incf, CounterName, Increment}, _State) -> 103 | setf(CounterName, getf(CounterName) + Increment), 104 | {noreply, _State}; 105 | 106 | handle_cast({decf, CounterName, Decrement}, _State) -> 107 | setf(CounterName, getf(CounterName) - Decrement), 108 | {noreply, _State}; 109 | 110 | handle_cast({setf, CounterName, NewValue}, _State) -> 111 | setf(CounterName, NewValue), 112 | {noreply, _State}; 113 | 114 | %%% Data collection 115 | handle_cast(ping, State = #state{data_spec = DataSpec}) -> 116 | NewSample = calculate_sample(DataSpec), 117 | store_sample({compute_time_offset(State), NewSample}), 118 | {noreply, State}; 119 | 120 | %%% Data recording 121 | %% Note the DataSpec. 122 | handle_cast({start_recording, DataSpec}, State) -> 123 | ets:delete_all_objects(?PFL_ETS_NAME), 124 | {ok, TRef} = timer:apply_interval(?SAMPLING_TIME, ?MODULE, tick, []), 125 | {noreply, State#state{start_time = erlang:now(), 126 | %% we add two default counters, that measure CPU load and memory usage. 127 | data_spec = [{fun cpu_load/0, "CPULoad", identity}, 128 | {fun memory_use/0, "MemoryUse", identity} | DataSpec], 129 | timer = TRef}}; 130 | 131 | handle_cast(end_recording, State) -> 132 | timer:cancel(State#state.timer), 133 | {noreply, State}; 134 | 135 | 136 | handle_cast({save_data, FileName}, State = #state{data_spec = DataSpec}) -> 137 | FirstElement = first_element(), 138 | ?WITH_OPEN_FILE(File, FileName, [write], 139 | begin 140 | save_header(DataSpec, File), 141 | save_data(FirstElement, 142 | next_element(FirstElement), 143 | lists:duplicate(length(DataSpec), 0), %% we need an auxiliary list for each counter, initialized to all zeros 144 | DataSpec, 145 | File) 146 | end), 147 | {noreply, State}. 148 | 149 | sample_to_string({Timestamp, Entry}) -> 150 | io_lib:format("~p~s~n", [Timestamp, lists:foldl(fun(W, Acc) -> Acc ++ "\t" ++ io_lib:format("~p",[W]) end, [], Entry)]). 151 | 152 | %% Measure an approximate current CPU load value. 153 | cpu_load() -> 154 | cpu_sup:util(). 155 | memory_use() -> 156 | erlang:memory(total). 157 | 158 | store_sample(Sample) -> 159 | ets:insert(?PFL_ETS_NAME, Sample). 160 | 161 | %%%% Tools for victory. No refunds. 162 | %%%% (also, not really tested) 163 | setf(Counter, Value) -> 164 | put(Counter, Value). 165 | 166 | getf(Counter) when is_function(Counter) -> 167 | Counter(); 168 | 169 | %% NOTE that getf will convert 'undefined' atom to 0, so that 170 | %% we may assume we're working with numerical counters. 171 | getf(Counter) -> 172 | case get(Counter) of 173 | undefined -> 174 | 0; 175 | Value -> 176 | Value 177 | end. 178 | 179 | 180 | calculate_sample(Counters) -> 181 | lists:map(fun({Counter, _Name, _SamplingType}) -> 182 | getf(Counter) end, 183 | Counters). 184 | 185 | compute_time_offset(#state{start_time = StartTime}) -> 186 | timer:now_diff(erlang:now(), StartTime) * ?TIME_COMPUTATION_SCALE. 187 | 188 | %% 189 | save_header(DataSpec, File) -> 190 | file:write(File, "Timestamp" ++ lists:foldl(fun({_, Name, _}, Acc) -> Acc ++ "\t" ++ Name end, [], DataSpec) ++ "\n"). 191 | 192 | %% Save measured data to file. 193 | %% save_data(Stream of data, 194 | %% Stream of data, shifted by one element left, 195 | %% Additional buffer for integration, 196 | %% DataSpec, 197 | %% File to save to). 198 | %% 199 | %% All data in the input stream represent the measured values of counters. However, the API allows us to specify that 200 | %% some counters should have their data integrated, and others should record differences between recent values. 201 | %% Feeding in the data stream both as normal and shifted by one element allows us to compute differences, while 202 | %% additional buffer allows us to integrate specific counters. 203 | save_data(_, end_of_data, _, _, _) -> 204 | ok; 205 | save_data(Fn_1, Fn = {Timestamp, _}, Result, Data, File) -> 206 | Out = compute_sample_value(Fn_1, Fn, Result, Data), 207 | file:write(File, sample_to_string({Timestamp, Out})), 208 | save_data(Fn, next_element(Fn), Out, Data, File). 209 | 210 | compute_sample_value({_, Fn_1}, {_, Fn}, Result, Data) -> 211 | zipwith4(fun(Cntr_prev, Cntr, Cntr_local, {_, _, CntrType}) -> 212 | case CntrType of 213 | identity -> Cntr; 214 | diff -> Cntr - Cntr_prev; 215 | accumulate -> Cntr + Cntr_local 216 | end 217 | end, 218 | Fn_1, 219 | Fn, 220 | Result, 221 | Data). 222 | 223 | %% Helper functions for working with stream that comes from a specific ETS table. 224 | first_element() -> 225 | [WhatINeed] = ets:lookup(?PFL_ETS_NAME, ets:first(?PFL_ETS_NAME)), 226 | WhatINeed. 227 | 228 | next_element({Key, _Value}) -> 229 | case ets:next(?PFL_ETS_NAME, Key) of 230 | '$end_of_table' -> end_of_data; 231 | NextKey -> [WhatINeed] = ets:lookup(?PFL_ETS_NAME, NextKey), 232 | WhatINeed 233 | end. 234 | 235 | %% zipwith4 - lists:zipwith for 4 lists. 236 | zipwith4(Combine, List1, List2, List3, List4) -> 237 | zipwith4(Combine, List1, List2, List3, List4, []). 238 | 239 | zipwith4(_, [], [], [], [], Accu) -> 240 | lists:reverse(Accu); 241 | zipwith4(Combine, [H1 | T1], [H2 | T2], [H3 | T3], [H4 | T4], Accu) -> 242 | zipwith4(Combine, T1, T2, T3, T4, [Combine(H1, H2, H3, H4) | Accu]). 243 | 244 | %% TODO needs real implementation 245 | spec_to_gnuplot_script(DataSpec) -> 246 | io_lib:format("set terminal png transparent nocrop enhanced font arial 8 size 800,600 247 | 248 | set autoscale y 249 | set autoscale y2 250 | 251 | set key autotitle columnhead 252 | 253 | plot \"foobar\" ", []). 254 | 255 | 256 | %% Test if the whole recording business works. 257 | %% NOTE, to verify the test, one needs to inspect the proper .txt file saved by it. 258 | test_recording() -> 259 | increment_counter(foobar, 42), 260 | increment_counter(firebirds, 15), 261 | increment_counter(jane, 24), 262 | start_recording([{foobar, "foobziubar", identity}, 263 | {jane, "JaneDwim", diff}, 264 | {firebirds, "24zachody", accumulate}]), 265 | timer:sleep(500), 266 | decrement_counter(foobar, 42), 267 | decrement_counter(jane, 14), 268 | decrement_counter(firebirds, 14), 269 | timer:sleep(1500), 270 | decrement_counter(foobar, 42), 271 | decrement_counter(jane, 14), 272 | decrement_counter(firebirds, 14), 273 | timer:sleep(1500), 274 | decrement_counter(foobar, 42), 275 | decrement_counter(firebirds, 14), 276 | timer:sleep(1500), 277 | increment_counter(foobar, 45), 278 | increment_counter(jane, 55), 279 | increment_counter(firebirds, 55), 280 | timer:sleep(1500), 281 | increment_counter(jane, 55), 282 | timer:sleep(1500), 283 | end_recording(), 284 | save_recorded_data_to_file("itworks.txt"). 285 | -------------------------------------------------------------------------------- /examples/performer.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | -module(performer). 18 | 19 | -export([start/0, stop/0]). 20 | -export([a_test_scenario/0]). 21 | -define(DELAY_BETWEEN_TASKS, 15). 22 | start() -> 23 | register(?MODULE, spawn(fun run_cpu_stresstests/0)). 24 | 25 | stop() -> 26 | ?MODULE ! stop. 27 | 28 | run_cpu_stresstests() -> 29 | jobs:add_queue(ramirez, 30 | [{regulators, [{rate, [{limit, 20}, 31 | {modifiers, 32 | [{cpu, 15, {max, 0}}]}]}]}]), 33 | spawn_cpu_intensive_jobs(). 34 | spawn_cpu_intensive_jobs() -> 35 | receive 36 | stop -> 37 | ok 38 | after ?DELAY_BETWEEN_TASKS -> 39 | spawn(fun() -> jobs:run(ramirez, fun cpu_intensive_job/0) end), 40 | performance_logger:increment_counter(jobs_enqueued, 1), 41 | spawn_cpu_intensive_jobs() 42 | end. 43 | 44 | cpu_intensive_job() -> 45 | %% TODO insert real CPU-intensive task here. 46 | %% NOTE that somehow, this seems like enough to crank up the CPU usage. 47 | timer:sleep(500), 48 | performance_logger:increment_counter(jobs_done, 1). 49 | 50 | queue_frequency() -> 51 | jobs:queue_info(ramirez, rate_limit). 52 | 53 | %% A simple test scenario that should show you the basic feedback reaction to CPU usage. 54 | %% NOTE that you need to start Jobs with following environmental variable setting: 55 | %% samplers <= [{foobar, jobs_sampler_cpu, []}] 56 | %% where 'foobar' can be - as far as I can tell - any atom. 57 | %% NOTE that you need to set 'samplers', not 'sampler' - setting the latter will result in Jobs not working. 58 | a_test_scenario() -> 59 | performance_logger:set_counter(jobs_enqueued, 0), 60 | performance_logger:set_counter(jobs_done, 0), 61 | start(), 62 | timer:sleep(500), 63 | performance_logger:start_recording([{jobs_enqueued, "\"Jobs enqueued\"", diff}, 64 | {jobs_done, "\"Jobs done\"", diff}, 65 | {fun queue_frequency/0, "\"Queue frequency\"", identity}]), 66 | timer:sleep(12000), 67 | stop(), 68 | timer:sleep(16000), 69 | performance_logger:end_recording(), 70 | performance_logger:save_recorded_data_to_file("jobs_cpu.dat"), 71 | ok. 72 | -------------------------------------------------------------------------------- /include/jobs.hrl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %%------------------------------------------------------------------- 18 | %% File : jobs.hrl 19 | %% @author : Ulf Wiger 20 | %% @end 21 | %% Description : 22 | %% 23 | %% Created : 15 Jan 2010 by Ulf Wiger 24 | %%------------------------------------------------------------------- 25 | 26 | -export_type([counter/0, reg_obj/0]). 27 | 28 | -type job_class() :: any(). 29 | 30 | -type mod_args() :: {atom(), list()}. 31 | -type mod_fun() :: {atom(), atom()}. 32 | 33 | -type option() :: {queues, [q_spec()]} 34 | | {config, file:name()} 35 | | {group_rates, [{q_name(), [option()]}]} 36 | | {counters, [{q_name(), [option()]}]} 37 | | {interval, integer()}. 38 | -type timestamp() :: integer(). % microseconds with a special epoch 39 | 40 | 41 | -type q_name() :: any(). 42 | -type q_std_type() :: standard_rate | standard_counter. 43 | -type q_check_interval() :: integer() | infinity | mfa(). 44 | -type q_producer() :: fun() | mfa() | mod_args(). 45 | 46 | -type q_reg_rate() :: {limit, integer()} 47 | | {modifiers, q_modifiers()} 48 | | {name, any()}. 49 | -type q_reg_counter() :: {limit, integer()} 50 | | {increment, integer()} 51 | | {modifiers, q_modifiers()} 52 | | {name, any()}. 53 | -type q_reg_opt() :: {rate, q_reg_rate()} 54 | | {counter, q_reg_counter()} 55 | | {named_counter, any(), integer()} 56 | | {group_rate, q_reg_rate()}. 57 | -type q_opt_action() :: approve | reject. 58 | -type q_opt_type() :: fifo | lifo | {producer, q_producer()} 59 | | {action, q_opt_action()}. 60 | 61 | -type q_opt() :: {regulators, [q_reg_opt()]} 62 | | {type, q_opt_type()} 63 | | {producer, q_producer()} 64 | | passive 65 | | {passive, fifo} 66 | | {action, q_opt_action()} 67 | | q_opt_action() 68 | | {check_interval, q_check_interval()} 69 | | {max_time, integer()} 70 | | {max_size, integer()} 71 | | {mod, atom()} 72 | | {standard_rate, integer()} 73 | | {standard_counter, integer()}. 74 | 75 | 76 | -type q_opts() :: [q_opt()]. 77 | -type q_spec() :: {q_name(), q_std_type(), q_opts()} 78 | | {q_name(), q_opts()}. 79 | 80 | -type q_modifier_name() :: cpu % predefined 81 | | memory % predefined 82 | | any(). % user-defined 83 | 84 | -type q_modifier_remote() :: {avg | max, integer()}. 85 | -type q_modifier() :: {q_modifier_name(), integer()} 86 | | {q_modifier_name(), integer(), q_modifier_remote()} 87 | | {q_modifier_name(), fun( 88 | (integer(), q_modifier_remote()) -> integer() 89 | )} 90 | | {q_modifier_name(), mod_fun()}. 91 | -type q_modifiers() :: [q_modifier()]. 92 | 93 | 94 | -record(rate, {limit = 0, 95 | preset_limit = 0, 96 | interval, 97 | modifiers = [], 98 | active_modifiers = []}). 99 | 100 | -record(counter, {name, increment = undefined}). 101 | -record(group_rate, {name}). 102 | 103 | -record(rr, 104 | %% Rate-based regulation 105 | {name, 106 | rate = #rate{}}). 107 | % limit = 0 :: float(), 108 | % interval = 0 :: undefined | float(), 109 | % modifiers = [] :: [{atom(),integer()}], 110 | % active_modifiers = [] :: [{atom(),integer()}], 111 | % preset_limit = 0}). 112 | 113 | -record(cr, 114 | %% Counter-based regulation 115 | {name, 116 | increment = 1, 117 | value = 0, 118 | rate = #rate{}, 119 | owner, 120 | queues = [], 121 | % limit = 5, 122 | % interval = 50, 123 | % modifiers = [] :: [{atom(),integer()}], 124 | % active_modifiers = [] :: [{atom(),integer()}], 125 | % preset_limit = 5, 126 | shared = false}). 127 | 128 | -opaque counter() :: {#cr{}, non_neg_integer()}. 129 | -opaque reg_obj() :: {reference(), [{info, any()} | {counters, [counter()]}]}. 130 | 131 | -record(grp, {name, 132 | rate = #rate{}, 133 | latest_dispatch=0 :: integer()}). 134 | % modifiers = [] :: [{atom(),integer()}], 135 | % active_modifiers = [] :: [{atom(),integer()}], 136 | % limit = 0 :: float(), 137 | % preset_limit = 0 :: float(), 138 | % interval :: float()}). 139 | 140 | -type regulator() :: #rr{} | #cr{} | regulator_ref(). 141 | -type regulator_ref() :: #group_rate{} | #counter{}. 142 | 143 | 144 | %% -record(producer, {f={erlang,error,[undefined_producer]} 145 | %% :: mfa() | fun(), 146 | %% mode = spawn :: spawn | {stateful, }). 147 | -record(producer, {mod = jobs_prod_simple, 148 | state}). 149 | 150 | %% -record(producer, {f={erlang,error,[undefined_producer]} 151 | %% :: mfa() | fun()}). 152 | -record(passive , {type = fifo :: fifo}). 153 | -record(action , {a = approve :: q_opt_action()}). 154 | 155 | -record(queue, {name :: any(), 156 | mod :: atom(), 157 | type = fifo :: fifo | lifo | #producer{} | #passive{} 158 | | #action{} | q_opt_type(), 159 | group :: atom(), 160 | regulators = [] :: [regulator() | regulator_ref()], 161 | max_time :: integer() | undefined, 162 | max_size :: integer() | undefined, 163 | latest_dispatch = 0 :: integer(), 164 | approved = 0, 165 | queued = 0, 166 | check_interval :: q_check_interval() | undefined, 167 | oldest_job :: integer() | undefined, 168 | timer, 169 | link_ref = undefined :: undefined | reference(), 170 | check_counter = 0 :: integer(), 171 | empty = false :: boolean(), 172 | depleted = false :: boolean(), 173 | waiters = [] :: [{pid(), reference()}], 174 | stateful, 175 | st 176 | }). 177 | 178 | -record(sampler, {name, 179 | mod, 180 | mod_state, 181 | type, % binary | meter 182 | step, % {seconds, [{Secs,Step}]}|{levels,[{Level,Step}]} 183 | hist_length = 10, 184 | history = queue:new()}). 185 | 186 | -record(stateless, {f}). 187 | -record(stateful, {f, st}). 188 | 189 | %% Gproc counter objects for counter-based regulation 190 | %% Each worker process gets a counter object. The aggregated counter, 191 | %% owned by the jobs_server, maintains a running tally of the concurrently 192 | %% existing counter objects of the given name. 193 | %% 194 | -define(COUNTER(Name), {c,l,{?MODULE,Name}}). 195 | -define( AGGR(Name), {a,l,{?MODULE,Name}}). 196 | 197 | -define(COUNTER_SAMPLE_INTERVAL, infinity). 198 | 199 | %% The jobs_server may, under certain circumstances, generate error reports 200 | %% This value, in microseconds, defines the highest frequency with which 201 | %% it can issue error reports. Any reports that would cause this limit to 202 | %% be exceeded are simply discarded. 203 | % 204 | -define(MAX_ERROR_RPT_INTERVAL_US, 1000000). 205 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | {erl_opts, [debug_info]}. 3 | {xref_checks, [undefined_function_calls]}. 4 | 5 | {cover_enabled, true}. 6 | {eunit_opts, [verbose]}. 7 | 8 | {clean_files, ["*~","*/*~","*/*.xfm","test/*.beam"]}. 9 | 10 | {deps, [ {parse_trans, "3.4.0"} 11 | , {setup, "2.1.0"} 12 | ]}. 13 | 14 | {profiles, 15 | [ 16 | {docs, 17 | [ 18 | {deps, 19 | [ 20 | {edown, "0.8.1"} 21 | ]}, 22 | {edoc_opts, [{doclet, edown_doclet}, 23 | {top_level_readme, 24 | {"./README.md", 25 | "http://github.com/uwiger/jobs"}}, 26 | {app_default, "https://www.erlang.org/doc/man"}]} 27 | ]}, 28 | {test, 29 | [ 30 | {deps, 31 | [ 32 | {meck, "0.8.11"} 33 | ]} 34 | ]} 35 | ]}. 36 | 37 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.2.0", 2 | [{<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.0">>},0}, 3 | {<<"setup">>,{pkg,<<"setup">>,<<"2.1.0">>},0}]}. 4 | [ 5 | {pkg_hash,[ 6 | {<<"parse_trans">>, <<"BB87AC362A03CA674EBB7D9D498F45C03256ADED7214C9101F7035EF44B798C7">>}, 7 | {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]}, 8 | {pkg_hash_ext,[ 9 | {<<"parse_trans">>, <<"F99E368830BEA44552224E37E04943A54874F08B8590485DE8D13832B63A2DC3">>}, 10 | {<<"setup">>, <<"EFD072578F0CF85BEA96CAAFFC7ADB0992398272522660A136E10567377071C5">>}]} 11 | ]. 12 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uwiger/jobs/867ac4d33f50ce2628f68625568ebc25c5e38d97/rebar3 -------------------------------------------------------------------------------- /src/jobs.app.src: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | %%============================================================================== 3 | %% Copyright 2014 Ulf Wiger 4 | %% 5 | %% Licensed under the Apache License, Version 2.0 (the "License"); 6 | %% you may not use this file except in compliance with the License. 7 | %% You may obtain a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, software 12 | %% distributed under the License is distributed on an "AS IS" BASIS, 13 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | %% See the License for the specific language governing permissions and 15 | %% limitations under the License. 16 | %%============================================================================== 17 | 18 | {application, jobs, 19 | [ 20 | {vsn, git}, 21 | {description, "Job scheduler"}, 22 | {applications, [kernel, stdlib]}, 23 | {registered, []}, 24 | {mod, {jobs_app, []}}, 25 | {env, []}, 26 | 27 | {maintainers, ["Ulf Wiger"]}, 28 | {licenses, ["Apache 2.0"]}, 29 | {links, [{"Github", "https://github.com/uwiger/jobs"}]} 30 | ]}. 31 | -------------------------------------------------------------------------------- /src/jobs.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %%------------------------------------------------------------------- 18 | %% File : jobs.erl 19 | %% @author : Ulf Wiger 20 | %% @doc 21 | %% This is the public API of the JOBS framework. 22 | %% 23 | %% @end 24 | %% Created : 15 Jan 2010 by Ulf Wiger 25 | %%------------------------------------------------------------------- 26 | -module(jobs). 27 | 28 | -export([ask/1, 29 | done/1, 30 | job_info/1, 31 | run/2, 32 | enqueue/2, 33 | dequeue/2]). 34 | 35 | -export([ask_queue/2]). 36 | 37 | 38 | %% Configuration API 39 | -export([add_queue/2, 40 | modify_queue/2, 41 | delete_queue/1, 42 | info/1, 43 | queue_info/1, 44 | queue_info/2, 45 | modify_regulator/4, 46 | add_counter/2, 47 | modify_counter/2, 48 | delete_counter/1, 49 | add_group_rate/2, 50 | modify_group_rate/2, 51 | delete_group_rate/1]). 52 | 53 | 54 | %% @spec ask(Type) -> {ok, Opaque} | {error, Reason} 55 | %% @doc Asks permission to run a job of Type. Returns when permission granted. 56 | %% 57 | %% The simplest way to have jobs regulated is to spawn a request per job. 58 | %% The process should immediately call this function, and when granted 59 | %% permission, execute the job, and then terminate. 60 | %% If for some reason the process needs to remain, to execute more jobs, 61 | %% it should explicitly call `jobs:done(Opaque)'. 62 | %% This is not strictly needed when regulation is rate-based, but as the 63 | %% regulation strategy may change over time, it is the prudent thing to do. 64 | %% @end 65 | %% 66 | ask(Type) -> 67 | jobs_server:ask(Type). 68 | 69 | %% @spec done(Opaque) -> ok 70 | %% @doc Signals completion of an executed task. 71 | %% 72 | %% This is used when the current process wants to submit more jobs to load 73 | %% regulation. It is mandatory when performing counter-based regulation 74 | %% (unless the process terminates after completing the task). It has no 75 | %% effect if the job type is purely rate-regulated. 76 | %% @end 77 | %% 78 | done(Opaque) -> 79 | jobs_server:done(Opaque). 80 | 81 | %% @spec run(Type, Function::function()) -> Result 82 | %% @doc Executes Function() when permission has been granted by job regulator. 83 | %% 84 | %% This is equivalent to performing the following sequence: 85 | %% 
 86 | %% case jobs:ask(Type) of
 87 | %%    {ok, Opaque} ->
 88 | %%       try Function()
 89 | %%         after
 90 | %%           jobs:done(Opaque)
 91 | %%       end;
 92 | %%    {error, Reason} ->
 93 | %%       erlang:error(Reason)
 94 | %% end.
 95 | %%
96 | %% @end 97 | %% 98 | run(Queue, F) when is_function(F, 0); is_function(F, 1) -> 99 | jobs_server:run(Queue, F). 100 | 101 | %% @spec enqueue(Queue, Item) -> ok | {error, Reason} 102 | %% @doc Inserts `Item` into a passive queue. 103 | %% 104 | %% Note that this function only works on passive queues. An exception will be 105 | %% raised if the queue doesn't exist, or isn't passive. 106 | %% 107 | %% Returns `ok' if `Item' was successfully entered into the queue, 108 | %% `{error, Reason}' otherwise (e.g. if the queue is full). 109 | %% @end 110 | enqueue(Queue, Item) -> 111 | jobs_server:enqueue(Queue, Item). 112 | 113 | %% @spec dequeue(Queue, N) -> [{JobID, Item}] 114 | %% @doc Extracts up to `N' items from a passive queue 115 | %% 116 | %% Note that this function only works on passive queues. An exception will be 117 | %% raised if the queue doesn't exist, or if it isn't passive. 118 | %% 119 | %% This function will block until at least one item can be extracted from the 120 | %% queue (see {@link enqueue/2}). No more than `N' items will be extracted. 121 | %% 122 | %% The items returned are on the form `{JobID, Item}', where `JobID' is in 123 | %% the form of a microsecond timestamp 124 | %% (see {@link jobs_lib:timestamp_to_datetime/1}), and `Item' is whatever was 125 | %% provided in {@link enqueue/2}. 126 | %% @end 127 | dequeue(Queue, N) when N =:= infinity; is_integer(N), N > 0 -> 128 | jobs_server:dequeue(Queue, N). 129 | 130 | %% @spec job_info(Opaque) -> undefined | Info 131 | %% @doc Retrieves job-specific information from the `Opaque' data object. 132 | %% 133 | %% The queue could choose to return specific information that is passed to a 134 | %% granted job request. This could be used e.g. for load-balancing strategies. 135 | %% @end 136 | %% 137 | job_info({_, Opaque}) -> 138 | proplists:get_value(info, Opaque). 139 | 140 | %% @spec add_queue(Name::any(), Options::[{Key,Value}]) -> ok 141 | %% @doc Installs a new queue in the load regulator on the current node. 142 | %% 143 | %% Valid options are: 144 | %% 145 | %% * `{regulators, Rs}', where `Rs' is a list of rate- or counter-based 146 | %% regulators. Valid regulators listed below. Default: []. 147 | %% 148 | %% * `{type, Type}' - type of queue. Valid types listed below. Default: `fifo'. 149 | %% 150 | %% * `{action, Action}' - automatic action to perform for each request. 151 | %% Valid actions described below. Default: `undefined'. 152 | %% 153 | %% * `{check_interval, I}' - If specified (in ms), this overrides the interval 154 | %% derived from any existing rate regulator. Note that regardless of how often 155 | %% the queue is checked, enough jobs will be dispatched at each interval to 156 | %% maintain the highest allowed rate possible, but the check interval may 157 | %% thus affect how many jobs are dispatched at the same time. Normally, this 158 | %% should not have to be specified. 159 | %% 160 | %% * `{max_time, T}', specifies how long (in ms) a job is allowed to wait 161 | %% in the queue before it is automatically rejected. 162 | %% If `undefined', no limit is imposed. 163 | %% 164 | %% * `{max_size, S}', indicates how many items can be queued before requests 165 | %% are automatically rejected. Strictly speaking, size is whatever the queue 166 | %% behavior reports as the size; in the default queue behavior, it is the 167 | %% number of elements in the queue. 168 | %% If `undefined', no limit is imposed. 169 | %% 170 | %% * `{link, undefined | pid()}', links the queue to a process. The queue 171 | %% will automatically be removed if the process dies. Default is `undefined', 172 | %% which means that the queue will not be linked. An exception will be raised 173 | %% if the option value is anything other than `undefined' or a valid pid. 174 | %% Note that if the referenced pid is not alive at queue creation time, the 175 | %% queue will be removed directly afterwards. An info report will be issued 176 | %% when the queue is removed. 177 | %% 178 | %% * `{mod, M}', indicates which queue behavior to use. Default is `jobs_queue'. 179 | %% 180 | %% In addition, some 'abbreviated' options are supported: 181 | %% 182 | %% * `{standard_rate, R}' - equivalent to 183 | %% `[{regulators,[{rate,[{limit,R}, {modifiers,[{cpu,10},{memory,10}]}]}]}]' 184 | %% 185 | %% * `{standard_counter, C}' - equivalent to 186 | %% `[{regulators,[{counter,[{limit,C}, {modifiers,[{cpu,10},{memory,10}]}]}]}]' 187 | %% 188 | %% * `{producer, F}' - equivalent to `{type, {producer, F}}' 189 | %% 190 | %% * `passive' - equivalent to `{type, {passive, fifo}}' 191 | %% 192 | %% * `approve | reject' - equivalent to `{action, approve | reject}' 193 | %% 194 | %% Regulators 195 | %% 196 | %% * `{rate, Opts}' - rate regulator. Valid options are 197 | %%
    198 | %%
  1. `{limit, Limit}' where `Limit' is the maximum rate (requests/sec)
    2. 199 | %%
  2. `{modifiers, Mods}', control feedback-based regulation. See below.
    3. 200 | %%
  3. `{name, Name}', optional. The default name for the regulator is 201 | %% `{rate, QueueName, N}', where `N' is an index indicating which rate regulator 202 | %% in the list is referred. Currently, at most one rate regulator is allowed, 203 | %% so `N' will always be `1'.
    4. 204 | %%
205 | %% 206 | %% * `{counter, Opts}' - counter regulator. Valid options are 207 | %%
    208 | %%
  1. `{limit, Limit}', where `Limit' is the number of concurrent jobs 209 | %% allowed.
    2. 210 | %%
  2. `{increment, Incr}', increment per job. Default is `1'.
    3. 211 | %%
  3. `{modifiers, Mods}', control feedback-based regulation. See below.
    4. 212 | %%
213 | %% 214 | %% * `{named_counter, Name, Incr}', use an existing counter, incrementing it 215 | %% with `Incr' for each job. `Name' can either refer to a named top-level 216 | %% counter (see {@link add_counter/2}), or a queue-specific counter 217 | %% (these are named `{counter,Qname,N}', where `N' is an index specifying 218 | %% their relative position in the regulators list - e.g. first or second 219 | %% counter). 220 | %% 221 | %% * `{group_rate, R}', refers to a top-level group rate `R'. 222 | %% See {@link add_group_rate/2}. 223 | %% 224 | %% Types 225 | %% 226 | %% * `fifo | lifo' - these are the types supported by the default queue 227 | %% behavior. While lifo may sound like an odd choice, it may have benefits 228 | %% for stochastic traffic with time constraints: there is no point to 229 | %% 'fairness', since requests cannot control their place in the queue, and 230 | %% choosing the 'freshest' job may increase overall goodness critera. 231 | %% 232 | %% * `{producer, F}', the queue is not for incoming requests, but rather 233 | %% generates jobs. Valid options for `F' are 234 | %% (for details, see {@link jobs_prod_simpe}): 235 | %%
    236 | %%
  1. A fun of arity 0, indicating a stateless producer
    2. 237 | %%
  2. A fun of arity 2, indicating a stateful producer
    3. 238 | %%
  3. `{M, F, A}', indicating a stateless producer
    4. 239 | %%
  4. `{Mod, Args}' indicating a stateful producer
    5. 240 | %%
241 | %% 242 | %% * `{action, approve | reject}', specifies an automatic response to every 243 | %% request. This can be used to either block a queue (`reject') or set it as 244 | %% a pass-through ('approve'). 245 | %% 246 | %% Modifiers 247 | %% 248 | %% Jobs supports feedback-based modification of regulators. 249 | %% 250 | %% The sampler framework sends feedback messages of type 251 | %% `[{Modifier, Local, Remote::[{node(), Level}]}]'. 252 | %% 253 | %% Each regulator can specify a list of modifier instructions: 254 | %% 255 | %% * `{Modifier, Local, Remote}' - `Modifier' can be any label used by the 256 | %% samplers (see {@link jobs_sampler}). `Local' and `Remote' indicate 257 | %% increments in percent by which to reduce the limit of the given regulator. 258 | %% The `Local' increment is used for feedback info pertaining to the local 259 | %% node, and the `Remote' increment is used for remote indicators. `Local' 260 | %% is given as a percentage value (e.g. `10' for `10 %'). The `Remote' 261 | %% increment is either `{avg, Percent}' or `{max, Percent}', indicating whether 262 | %% to respond to the average load of other nodes or to the most loaded node. 263 | %% The correction from `Local' and the correction from `Remote' are summed 264 | %% before applying to the regulator limit. 265 | %% 266 | %% * `{Modifier, Local}' - same as above, but responding only to local 267 | %% indications, ignoring the load on remote nodes. 268 | %% 269 | %% * `{Modifier, F::function((Local, Remote) -> integer())}' - the function 270 | %% `F(Local, Remote)' is applied and expected to return a correction value, 271 | %% in percentage units. 272 | %% 273 | %% * `{Modifier, {Module, Function}}' - `Module:Function(Local Remote)' 274 | %% is applied an expected to return a correction value in percentage units. 275 | %% 276 | %% For example, if a rate regulator has a limit of `100' and has a modifier, 277 | %% `{cpu, 10}', then a feedback message of `{cpu, 2, _Remote}' will reduce 278 | %% the rate limit by `2*10' percent, i.e. down to `80'. 279 | %% 280 | %% Note that modifiers are always applied to the preset limit, 281 | %% not the current limit. Thus, the next round of feedback messages in our 282 | %% example will be applied to the preset limit of `100', not the `80' that 283 | %% resulted from the previous feedback messages. A correction value of `0' 284 | %% will reset the limit to the preset value. 285 | %% 286 | %% If there are more than one modifier with the same name, the last one in the 287 | %% list will be the one used. 288 | %% 289 | %% @end 290 | %% 291 | add_queue(Name, Options) -> 292 | jobs_server:add_queue(Name, Options). 293 | 294 | %% @spec modify_queue(Name::any(), Options::[{Key,Value}]) -> 295 | %% ok | {error, Reason} 296 | %% @doc Modifies queue parameters of existing queue. 297 | %% 298 | %% The queue parameters that can be modified are `max_size' and `max_time'. 299 | %% @end 300 | modify_queue(Name, Options) -> 301 | jobs_server:modify_queue(Name, Options). 302 | 303 | %% @spec delete_queue(Name) -> boolean() 304 | %% @doc Deletes the named queue from the load regulator on the current node. 305 | %% Returns `true' if there was in fact such a queue; `false' otherwise. 306 | %% @end 307 | %% 308 | delete_queue(Name) -> 309 | jobs_server:delete_queue(Name). 310 | 311 | %% @spec ask_queue(QueueName, Request) -> Reply 312 | %% @doc Sends a synchronous request to a specific queue. 313 | %% 314 | %% This function is mainly intended to be used for back-end processes that act 315 | %% as custom extensions to the load regulator itself. It should not be used by 316 | %% regular clients. Sophisticated queue behaviours could export gen_server-like 317 | %% logic allowing them to respond to synchronous calls, either for special 318 | %% inspection, or for influencing the queue state. 319 | %% @end 320 | %% 321 | ask_queue(QueueName, Request) -> 322 | jobs_server:ask_queue(QueueName, Request). 323 | 324 | %% @spec add_counter(Name, Options) -> ok 325 | %% @doc Adds a named counter to the load regulator on the current node. 326 | %% Fails if there already is a counter the name `Name'. 327 | %% @end 328 | %% 329 | add_counter(Name, Options) -> 330 | jobs_server:add_counter(Name, Options). 331 | 332 | %% @spec delete_counter(Name) -> boolean() 333 | %% @doc Deletes a named counter from the load regulator on the current node. 334 | %% Returns `true' if there was in fact such a counter; `false' otherwise. 335 | %% @end 336 | %% 337 | delete_counter(Name) -> 338 | jobs_server:delete_counter(Name). 339 | 340 | %% @spec add_group_rate(Name, Options) -> ok 341 | %% @doc Adds a group rate regulator to the load regulator on the current node. 342 | %% Fails if there is already a group rate regulator of the same name. 343 | %% @end 344 | %% 345 | add_group_rate(Name, Options) -> 346 | jobs_server:add_group_rate(Name, Options). 347 | 348 | delete_group_rate(Name) -> 349 | jobs_server:delete_group_rate(Name). 350 | 351 | info(Item) -> 352 | jobs_server:info(Item). 353 | 354 | queue_info(Name) -> 355 | jobs_server:queue_info(Name). 356 | 357 | queue_info(Name, Item) -> 358 | jobs_server:queue_info(Name, Item). 359 | 360 | modify_regulator(Type, QName, RegName, Opts) when Type==counter;Type==rate -> 361 | jobs_server:modify_regulator(Type, QName, RegName, Opts). 362 | 363 | modify_counter(CName, Opts) -> 364 | jobs_server:modify_counter(CName, Opts). 365 | 366 | modify_group_rate(GRName, Opts) -> 367 | jobs_server:modify_group_rate(GRName, Opts). 368 | -------------------------------------------------------------------------------- /src/jobs_app.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %% @doc Application module for JOBS. 18 | %% Normally, JOBS is configured at startup, using a static configuration. 19 | %% There is a reconfiguration API {@link jobs}, which is mainly for evolution 20 | %% of the system. 21 | %% 22 | %% == Configuring JOBS == 23 | %% A static configuration can be provided via application environment 24 | %% variables for the `jobs' application. The following is a list of 25 | %% recognised configuration parameters. 26 | %% 27 | %% === {config, Filename} === 28 | %% Evaluate a file using {@link //kernel/file:script/1. file:script/1}, treating the data 29 | %% returned from the script as a list of configuration options. 30 | %% 31 | %% === {queues, QueueOptions} === 32 | %% Configure a list of queues according to the provided QueueOptions. 33 | %% If no queues are specified, a queue named `default' will be created 34 | %% with default characteristics. 35 | %% 36 | %% Below are the different queue configuration options: 37 | %% 38 | %% ==== {Name, Options} ==== 39 | %% This is the generic queue configuration pattern. 40 | %% `Name :: any()' is used to identify the queue. 41 | %% 42 | %% Options: 43 | %% 44 | %% `{mod, Module::atom()}' provides the name of the queueing module. 45 | %% The default module is `jobs_queue'. 46 | %% 47 | %% `{type, fifo | lifo | approve | reject | {producer, F}}' 48 | %% specifies the semantics of the queue. Note that the specified queue module 49 | %% may be limited to only one type (e.g. the `jobs_queue_list' module only 50 | %% supports `lifo' semantics). 51 | %% 52 | %% If the type is `{producer, F}', it doesn't matter which queue module is 53 | %% used, as it is not possible to submit job requests to a producer queue. 54 | %% The producer queue will initiate jobs using `spawn_monitor(F)' at the 55 | %% rate given by the regulators for the queue. 56 | %% 57 | %% If the type is `approve' or `reject', respectively, all other options will 58 | %% be irrelevant. Any request to the queue will either be immediately approved 59 | %% or immediately rejected. 60 | %% 61 | %% `{max_time, integer() | undefined}' specifies the longest time that a job 62 | %% request may spend in the queue. If `undefined', no limit is imposed. 63 | %% 64 | %% `{max_size, integer() | undefined}' specifies the maximum length (number 65 | %% of job requests) of the queue. If the queue has reached the maximum length, 66 | %% subsequent job requests will be rejected unless it is possible to remove 67 | %% enough requests that have exceeded the maximum allowed time in the queue. 68 | %% If `undefined', no limit is imposed. 69 | %% 70 | %% `{regulators, [{regulator_type(), Opts]}' specifies the regulation 71 | %% characteristics of the queue. 72 | %% 73 | %% The following types of regulator are supported: 74 | %% 75 | %% `regulator_type() :: rate | counter | group_rate' 76 | %% 77 | %% It is possible to combine different types of regulator on the same queue, 78 | %% e.g. a queue may have both rate- and counter regulation. It is not possible 79 | %% to have two different rate regulators for the same queue. 80 | %% 81 | %% Common regulator options: 82 | %% 83 | %% `{name, term()}' names the regulator; by default, a name will be generated. 84 | %% 85 | %% `{limit, integer()}' defines the limit for the regulator. If it is a rate 86 | %% regulator, the value represents the maximum number of jobs/second; if it 87 | %% is a counter regulator, it represents the total number of "credits" 88 | %% available. 89 | %% 90 | %% `{modifiers, [modifier()]}' 91 | %% 92 | %% 
 93 | %% modifier() :: {IndicatorName :: any(), unit()}
 94 | %%               | {Indicator, local_unit(), remote_unit()}
 95 | %%               | {Indicator, Fun}
 96 | %%
 97 | %% local_unit() :: unit() :: integer()
 98 | %% remote_unit() :: {avg, unit()} | {max, unit()}
 99 | %%
100 | %% 101 | %% Feedback indicators are sent from the sampler framework. Each indicator 102 | %% has the format `{IndicatorName, LocalLoadFactor, Remote}'. 103 | %% 104 | %% `Remote :: [{Node, LoadFactor}]' 105 | %% 106 | %% `IndicatorName' defines the type of indicator. It could be e.g. `cpu', 107 | %% `memory', `mnesia', or any other name defined by one of the sampler plugins. 108 | %% 109 | %% The effect of a modifier is calculated as the sum of the effects from local 110 | %% and remote load. As the remote load is represented as a list of 111 | %% `{Node,Factor}' it is possible to multiply either the average or the max 112 | %% load on the remote nodes with the given factor: `{avg,Unit} | {max, Unit}'. 113 | %% 114 | %% For custom interpretation of the feedback indicator, it is possible to 115 | %% specify a function `F(LocalFactor, Remote) -> Effect', where Effect is a 116 | %% positive integer. 117 | %% 118 | %% The resulting effect value is used to reduce the predefined regulator limit 119 | %% with the given number of percentage points, e.g. if a rate regulator has 120 | %% a predefined limit of 100 jobs/sec, and `Effect = 20', the current rate 121 | %% limit will become 80 jobs/sec. 122 | %% 123 | %% `{rate, Opts}' - rate regulation 124 | %% 125 | %% Currently, no special options exist for rate regulators. 126 | %% 127 | %% `{counter, Opts}' - counter regulation 128 | %% 129 | %% The option `{increment, I}' can be used to specify how much of the credit 130 | %% pool should be assigned to each job. The default increment is 1. 131 | %% 132 | %% `{named_counter, Name, Increment}' reuses an existing counter regulator. 133 | %% This can be used to link multiple queues to a shared credit pool. Note that 134 | %% this does not use the existing counter regulator as a template, but actually 135 | %% shares the credits with any other queues using the same named counter. 136 | %% 137 | %% __NOTE__ Currently, if there is no counter corresponding to the alias, 138 | %% the entry will simply be ignored during regulation. It is likely that this 139 | %% behaviour will change in the future. 140 | %% 141 | %% ==== {Name, standard_rate, R} ==== 142 | %% A simple rate-regulated queue with throughput rate `R', and basic cpu- and 143 | %% memory-related feedback compensation. 144 | %% 145 | %% ==== {Name, standard_counter, N} ==== 146 | %% A simple counter-regulated queue, giving each job a weight of 1, and thus 147 | %% allowing at most `N' jobs to execute concurrently. Basic cpu- and memory- 148 | %% related feedback compensation. 149 | %% 150 | %% ==== {Name, producer, F, Options} ==== 151 | %% A producer queue is not open for incoming jobs, but will rather initiate 152 | %% jobs at the given rate. 153 | %% @end 154 | %% 155 | -module(jobs_app). 156 | 157 | -export([start/2, stop/1, 158 | init/1]). 159 | 160 | 161 | start(_, _) -> 162 | supervisor:start_link({local,?MODULE},?MODULE,[]). 163 | 164 | stop(_) -> 165 | ok. 166 | 167 | 168 | init([]) -> 169 | {ok, {{rest_for_one,3,10}, 170 | [{jobs_server, {jobs_server,start_link,[]}, 171 | permanent, 3000, worker, [jobs_server]}| 172 | sampler_spec()]}}. 173 | 174 | 175 | sampler_spec() -> 176 | Mod = case application:get_env(sampler) of 177 | {ok,M} when M =/= undefined -> M; 178 | _ -> jobs_sampler 179 | end, 180 | [{jobs_sampler, {Mod,start_link,[]}, permanent, 3000, worker, [Mod]}]. 181 | -------------------------------------------------------------------------------- /src/jobs_info.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | -module(jobs_info). 17 | 18 | -export([pp/1]). 19 | 20 | -include("jobs.hrl"). 21 | -include_lib("parse_trans/include/exprecs.hrl"). 22 | 23 | -export_records([rr, cr, grp, rate, queue, sampler]). 24 | 25 | 26 | pp(L) when is_list(L) -> 27 | [pp(X) || X <- L]; 28 | pp(X) -> 29 | case '#is_record-'(X) of 30 | true -> 31 | RecName = element(1,X), 32 | {RecName, lists:zip( 33 | '#info-'(RecName,fields), 34 | pp(tl(tuple_to_list(X))))}; 35 | false -> 36 | if is_tuple(X) -> 37 | list_to_tuple(pp(tuple_to_list(X))); 38 | true -> 39 | X 40 | end 41 | end. 42 | -------------------------------------------------------------------------------- /src/jobs_lib.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | -module(jobs_lib). 18 | %% We don't want warnings about the use of erlang:now/0 in 19 | %% this module. 20 | -compile(nowarn_deprecated_function). 21 | 22 | -export([timestamp/0, 23 | timestamp_to_datetime/1, 24 | time_compat/0]). 25 | 26 | 27 | timestamp() -> 28 | %% Invented epoc is {1258,0,0}, or 2009-11-12, 4:26:40 29 | {MS,S,US} = time_compat(), 30 | (MS-1258)*1000000000 + S*1000 + US div 1000. 31 | 32 | timestamp_to_datetime(TS) -> 33 | %% Our internal timestamps are relative to Now = {1258,0,0} 34 | %% It doesn't really matter much how we construct a now()-like tuple, 35 | %% as long as the weighted sum of the three numbers is correct. 36 | S = TS div 1000, 37 | MS = TS rem 1000, 38 | %% return {Datetime, Milliseconds} 39 | {calendar:now_to_datetime({1258,S,0}), MS}. 40 | 41 | %% create the jobs_time_compat module for efficiency 42 | time_compat() -> 43 | case erlang:is_builtin(erlang,timestamp,0) of 44 | true -> erlang:timestamp(); 45 | false -> erlang:now() 46 | end. -------------------------------------------------------------------------------- /src/jobs_prod_simple.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | -module(jobs_prod_simple). 17 | 18 | -export([init/2, 19 | next/3]). 20 | 21 | -include("jobs.hrl"). 22 | 23 | init(F, _Info) when is_function(F, 0) -> 24 | #stateless{f = F}; 25 | init(F, Info) when is_function(F, 2) -> 26 | #stateful{f = F, st = F(init, Info)}; 27 | init({_, F, A} = MFA, _Info) when is_atom(F), is_list(A) -> 28 | #stateless{f = MFA}. 29 | 30 | next(_Opaque, #stateful{f = F, st = St} = P, Info) -> 31 | case F(St, Info) of 32 | {F1, St1} when is_function(F1, 0) -> 33 | {F1, P#stateful{st = St1}}; 34 | Other -> 35 | erlang:error({bad_producer_next, Other}) 36 | end; 37 | next(_Opaque, #stateless{f = F} = P, _Info) -> 38 | case F of 39 | {M,Fn,A} -> 40 | {fun() -> apply(M, Fn, A) end, P}; 41 | F when is_function(F, 0) -> 42 | {F, P} 43 | end. 44 | -------------------------------------------------------------------------------- /src/jobs_queue.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %%------------------------------------------------------------------- 18 | %% File : jobs_queue.erl 19 | %% @author : Ulf Wiger 20 | %% @end 21 | %% Description : 22 | %% 23 | %% Created : 15 Jan 2010 by Ulf Wiger 24 | %%------------------------------------------------------------------- 25 | %% @doc Default queue behaviour for JOBS (using ordered_set ets). 26 | %% 27 | %% This module implements the default queue behaviour for JOBS, and also 28 | %% specifies the behaviour itself. 29 | %% @end 30 | 31 | -module(jobs_queue). 32 | -author('ulf@wiger.net'). 33 | -copyright('Ulf Wiger'). 34 | 35 | -export([new/2, 36 | delete/1]). 37 | -export([in/3, 38 | out/2, 39 | peek/1, 40 | info/2, 41 | all/1, 42 | empty/1, 43 | is_empty/1, 44 | representation/1, 45 | timedout/1, timedout/2]). 46 | 47 | -export([behaviour_info/1]). 48 | 49 | -include("jobs.hrl"). 50 | -import(jobs_server, [timestamp/0]). 51 | 52 | -record(st, {table}). 53 | 54 | %-type timestamp() :: integer(). 55 | -type job() :: {pid(), reference()}. 56 | -type entry() :: {timestamp(), job()}. 57 | 58 | behaviour_info(callbacks) -> 59 | [{new , 2}, 60 | {delete , 1}, 61 | {in , 3}, 62 | {peek , 1}, 63 | {out , 2}, 64 | {all , 1}, 65 | {info , 2}]; 66 | behaviour_info(_) -> 67 | undefined. 68 | 69 | 70 | %% @spec new(Options, #queue{}) -> #queue{} 71 | %% @doc Instantiate a new queue. 72 | %% 73 | %% Options is the list of options provided when defining the queue. 74 | %% Q is an initial #queue{} record. It can be used directly by including 75 | %% `jobs/include/jobs.hrl', or by using exprecs-style record accessors in the 76 | %% module `jobs_info'. 77 | %% See parse_trans for more info 78 | %% on exprecs. In the `new/2' function, the #queue.st attribute will normally be 79 | %% used to keep track of the queue data structure. 80 | %% @end 81 | %% 82 | new(Options, Q) -> 83 | case proplists:get_value(type, Options, fifo) of 84 | fifo -> 85 | Tab = ets:new(?MODULE, [ordered_set]), 86 | Q#queue{st = #st{table = Tab}} 87 | end. 88 | 89 | %% @doc A representation of a queue which can be inspected 90 | %% @end 91 | representation( 92 | #queue { oldest_job = OJ, 93 | st = #st { table = Tab}}) -> 94 | Contents = ets:match_object(Tab, '$1'), 95 | [{oldest_job, OJ}, 96 | {contents, [X || {X} <- Contents]}]. 97 | 98 | %% @spec delete(#queue{}) -> any() 99 | %% @doc Queue is being deleted; remove any external data structures. 100 | %% 101 | %% If the queue behaviour has created an ETS table or similar, this is the place 102 | %% to get rid of it. 103 | %% @end 104 | %% 105 | delete(#queue{st = undefined}) -> 106 | ok; 107 | delete(#queue{st = #st{table = T}}) -> 108 | ets:delete(T). 109 | 110 | 111 | 112 | -spec in(timestamp(), job(), #queue{}) -> #queue{}. 113 | %% @spec in(Timestamp, Job, #queue{}) -> #queue{} 114 | %% @doc Enqueue a job reference; return the updated queue. 115 | %% 116 | %% This puts a job into the queue. The callback function is responsible for 117 | %% updating the #queue.oldest_job attribute, if needed. The #queue.oldest_job 118 | %% attribute shall either contain the Timestamp of the oldest job in the queue, 119 | %% or `undefined' if the queue is empty. It may be noted that, especially in the 120 | %% fairly trivial case of the `in/3' function, the oldest job would be 121 | %% `erlang:min(Timestamp, PreviousOldest)', even if `PreviousOldest == undefined'. 122 | %% @end 123 | %% 124 | in(TS, Job, #queue{st = #st{table = Tab}, oldest_job = OJ} = Q) -> 125 | OJ1 = erlang:min(TS, OJ), % Works even if OJ==undefined 126 | ets:insert(Tab, {{TS, Job}}), 127 | Q#queue{oldest_job = OJ1}. 128 | 129 | 130 | -spec peek(#queue{}) -> entry(). 131 | %% @spec peek(#queue{}) -> JobEntry | undefined 132 | %% @doc Looks at the first item in the queue, without removing it. 133 | %% 134 | peek(#queue{st = #st{table = T}}) -> 135 | case ets:first(T) of 136 | '$end_of_table' -> 137 | undefined; 138 | Key -> 139 | Key 140 | end. 141 | 142 | -spec out(N :: integer(), #queue{}) -> {[entry()], #queue{}}. 143 | %% @spec out(N :: integer(), #queue{}) -> {[Entry], #queue{}} 144 | %% @doc Dequeue a batch of N jobs; return the modified queue. 145 | %% 146 | %% Note that this function may need to update the #queue.oldest_job attribute, 147 | %% especially if the queue becomes empty. 148 | %% @end 149 | %% 150 | out(N,#queue{st = #st{table = T}}=Q) when N >= 0 -> 151 | {out1(N, T), set_oldest_job(Q)}. 152 | 153 | 154 | -spec all(#queue{}) -> [entry()]. 155 | %% @spec all(#queue{}) -> [JobEntry] 156 | %% @doc Return all the job entries in the queue, not removing them from the queue. 157 | %% 158 | all(#queue{st = #st{table = T}}) -> 159 | ets:select(T, [{{'$1'},[],['$1']}]). 160 | 161 | 162 | -type info_item() :: max_time | oldest_job | length. 163 | 164 | -spec info(info_item(), #queue{}) -> any(). 165 | %% @spec info(Item, #queue{}) -> Info 166 | %% Item = max_time | oldest_job | length 167 | %% @doc Return information about the queue. 168 | %% 169 | info(max_time , #queue{max_time = T} ) -> T; 170 | info(oldest_job, #queue{oldest_job = OJ}) -> OJ; 171 | info(length , #queue{st = #st{table = Tab}}) -> 172 | ets:info(Tab, size). 173 | 174 | -spec timedout(#queue{}) -> {[entry()], #queue{}}. 175 | %% @spec timedout(#queue{}) -> {[Entry], #queue{}} 176 | %% @doc Return all entries that have been in the queue longer than MaxTime. 177 | %% 178 | %% NOTE: This is an inspection function; it doesn't remove the job entries. 179 | %% @end 180 | %% 181 | timedout(#queue{max_time = undefined} = Q) -> {[], Q}; 182 | timedout(#queue{max_time = TO} = Q) -> 183 | timedout(TO, Q). 184 | 185 | timedout(_ , #queue{oldest_job = undefined} = Q) -> {[], Q}; 186 | timedout(TO, #queue{st = #st{table = Tab}} = Q) -> 187 | Now = timestamp(), 188 | Objs = find_expired(Tab, Now, TO), 189 | OJ = case ets:first(Tab) of 190 | '$end_of_table' -> undefined; 191 | {TS, _} -> TS 192 | end, 193 | {Objs, Q#queue{oldest_job = OJ}}. 194 | 195 | 196 | 197 | -spec is_empty(#queue{}) -> boolean(). 198 | %% 199 | %% Check whether the queue is empty. 200 | %% 201 | is_empty(#queue{type = {producer, _}}) -> false; 202 | is_empty(#queue{oldest_job = undefined}) -> true; 203 | is_empty(#queue{}) -> 204 | false. 205 | 206 | 207 | out1(0, _Tab) -> []; 208 | out1(1, Tab) -> 209 | case ets:first(Tab) of 210 | '$end_of_table' -> 211 | []; 212 | {_TS,_Client} = Key -> 213 | ets:delete(Tab, Key), 214 | [Key] 215 | end; 216 | out1(N, Tab) when N > 0 -> 217 | %% We impose an arbitrary limit of 100 jobs fetched in one chunk. 218 | %% The main reason for capping the limit is that ets:select/3 will 219 | %% crash if N is a bignum; we probably don't want to chunk that many 220 | %% objects anyway, so we set the limit much lower. 221 | Limit = erlang:min(N, 100), 222 | case ets:select(Tab, [{{'$1'},[],['$1']}], Limit) of 223 | '$end_of_table' -> 224 | []; 225 | {Keys, _} -> 226 | [ets:delete(Tab, K) || K <- Keys], 227 | Keys 228 | end. 229 | 230 | set_oldest_job(#queue{st = #st{table = Tab}} = Q) -> 231 | OJ = case ets:first(Tab) of 232 | '$end_of_table' -> 233 | undefined; 234 | {TS,_} -> 235 | TS 236 | end, 237 | Q#queue{oldest_job = OJ}. 238 | 239 | 240 | find_expired(Tab, Now, TO) -> 241 | find_expired(ets:first(Tab), Tab, Now, TO * 1000, []). 242 | 243 | %% we return the reversed list, but I don't think that matters here. 244 | find_expired('$end_of_table', _, _, _, Acc) -> 245 | Acc; 246 | find_expired({TS, _} = Key, Tab, Now, TO, Acc) -> 247 | case is_expired(TS, Now, TO) of 248 | true -> 249 | ets:delete(Tab, Key), 250 | find_expired(ets:first(Tab), Tab, Now, TO, [Key|Acc]); 251 | false -> 252 | Acc 253 | end. 254 | 255 | empty(#queue{st = #st{table = T}} = Q) -> 256 | ets:delete_all_objects(T), 257 | Q#queue{oldest_job = undefined}. 258 | 259 | 260 | is_expired(TS, Now, TO) -> 261 | MS = Now - TS, 262 | MS > TO. 263 | -------------------------------------------------------------------------------- /src/jobs_queue_list.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %%------------------------------------------------------------------- 18 | %% File : jobs_queue.erl 19 | %% @author : Ulf Wiger 20 | %% @end 21 | %% Description : 22 | %% 23 | %% Created : 15 Jan 2010 by Ulf Wiger 24 | %%------------------------------------------------------------------- 25 | 26 | -module(jobs_queue_list). 27 | -author('ulf@wiger.net'). 28 | -copyright('Ulf Wiger'). 29 | 30 | -export([new/2, 31 | delete/1]). 32 | -export([in/3, 33 | out/2, 34 | info/2, 35 | peek/1, 36 | all/1, 37 | empty/1, 38 | is_empty/1, 39 | representation/1, 40 | timedout/1, timedout/2]). 41 | 42 | -include("jobs.hrl"). 43 | 44 | %-type timestamp() :: integer(). 45 | -type job() :: {pid(), reference()}. 46 | -type entry() :: {timestamp(), job()}. 47 | 48 | 49 | new(Options, Q) -> 50 | case proplists:get_value(type, Options, lifo) of 51 | lifo -> Q#queue{st = []} 52 | end. 53 | 54 | delete(#queue{}) -> true. 55 | 56 | -spec in(timestamp(), job(), #queue{}) -> #queue{}. 57 | %% 58 | %% Enqueue a job reference; return the updated queue 59 | %% 60 | in(TS, Job, #queue{st = []} = Q) -> 61 | Q#queue{st = [{TS, Job}], oldest_job = TS}; 62 | in(TS, Job, #queue{st = L} = Q) -> 63 | Q#queue{st = [{TS, Job} | L]}. 64 | 65 | -spec out(N :: integer(), #queue{}) -> {[entry()], #queue{}}. 66 | %% 67 | %% Dequeue a batch of N jobs; return the modified queue. 68 | %% 69 | out(N, #queue{st = L, oldest_job = OJ} = Q) when N >= 0 -> 70 | {Out, Rest} = split(N, L), 71 | OJ1 = case Rest of 72 | [] -> undefined; 73 | _ -> OJ 74 | end, 75 | {Out, Q#queue{st = Rest, oldest_job = OJ1}}. 76 | 77 | representation(#queue { st = L, oldest_job = OJ}) -> 78 | [{oldest_job, OJ}, 79 | {contents, L}]. 80 | 81 | split(N, L) -> 82 | split(N, L, []). 83 | 84 | split(_, [], Acc) -> 85 | {lists:reverse(Acc), []}; 86 | split(N, [H|T], Acc) when N > 0 -> 87 | split(N-1, T, [H|Acc]); 88 | split(0, T, Acc) -> 89 | {lists:reverse(Acc), T}. 90 | 91 | 92 | peek(#queue{st = []}) -> undefined; 93 | peek(#queue { st = [H | _]}) -> H. 94 | 95 | 96 | -spec all(#queue{}) -> [entry()]. 97 | %% 98 | %% Return all the job entries in the queue 99 | %% 100 | all(#queue{st = L}) -> 101 | L. 102 | 103 | 104 | -type info_item() :: max_time | oldest_job | length. 105 | 106 | -spec info(info_item(), #queue{}) -> any(). 107 | %% 108 | %% Return information about the queue. 109 | %% 110 | info(max_time , #queue{max_time = T} ) -> T; 111 | info(oldest_job, #queue{oldest_job = OJ}) -> OJ; 112 | info(length , #queue{st = L}) -> 113 | length(L). 114 | 115 | -spec timedout(#queue{}) -> {[entry()], #queue{}}. 116 | %% 117 | %% Return all entries that have been in the queue longer than MaxTime. 118 | %% 119 | timedout(#queue{max_time = undefined} = Q) -> {[],Q}; 120 | timedout(#queue{max_time = TO} = Q) -> 121 | timedout(TO, Q). 122 | 123 | timedout(_ , #queue{oldest_job = undefined} = Q) -> {[],Q}; 124 | timedout(TO, #queue{st = L} = Q) -> 125 | Now = jobs_server:timestamp(), 126 | {Left, Timedout} = lists:splitwith(fun({TS,_}) -> 127 | not(is_expired(TS,Now,TO)) 128 | end, L), 129 | OJ = get_oldest_job(Left), 130 | {Timedout, Q#queue{oldest_job = OJ, st = Left}}. 131 | 132 | get_oldest_job([]) -> undefined; 133 | get_oldest_job(L) -> 134 | element(1, hd(lists:reverse(L))). 135 | 136 | 137 | -spec is_empty(#queue{}) -> boolean(). 138 | %% 139 | %% Check whether the queue is empty. 140 | %% 141 | is_empty(#queue{st = []}) -> true; 142 | is_empty(_) -> 143 | false. 144 | 145 | empty(#queue{} = Q) -> 146 | Q#queue{oldest_job = undefined, st = []}. 147 | 148 | is_expired(TS, Now, TO) -> 149 | MS = Now - TS, 150 | MS > TO. 151 | -------------------------------------------------------------------------------- /src/jobs_sampler.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | %%------------------------------------------------------------------- 18 | %% File : jobs_sampler.erl 19 | %% @author : Ulf Wiger 20 | %% @end 21 | %% Description : 22 | %% 23 | %% Created : 15 Jan 2010 by Ulf Wiger 24 | %%------------------------------------------------------------------- 25 | -module(jobs_sampler). 26 | 27 | -export([start_link/0, start_link/1, 28 | trigger_sample/0, 29 | tell_sampler/2, 30 | subscribe/0, 31 | end_subscription/0, 32 | calc/3]). 33 | 34 | -export([init/1, 35 | handle_call/3, 36 | handle_cast/2, 37 | handle_info/2, 38 | terminate/2, 39 | code_change/3]). 40 | 41 | -include("jobs.hrl"). 42 | 43 | %% -record(indicators, {mnesia_dumper = 0, 44 | %% mnesia_tm = 0, 45 | %% mnesia_remote = []}). 46 | 47 | 48 | -define(SAMPLE_INTERVAL, 10000). 49 | 50 | 51 | -record(state, {modified = false, 52 | update_delay = 0, 53 | sample_interval = ?SAMPLE_INTERVAL, 54 | %% indicators = [], 55 | %% remote_indicators = [], 56 | samplers = [] :: [#sampler{}], 57 | subscribers = [], 58 | modifiers = orddict:new(), 59 | remote_modifiers = []}). 60 | 61 | 62 | -callback init(Name :: any(), Opts :: proplists:proplist()) -> {ok, State :: any()}. 63 | -callback sample(Timestamp :: integer(), State :: any()) -> {Result :: any(), State :: any()}. 64 | -callback handle_msg(Msg :: any(), Timestamp :: integer(), State :: any()) 65 | -> {ignore, State :: any()} | {log, Sample :: any(), State :: any()}. 66 | -callback calc(History :: any(), State :: any()) -> {Modifiers :: q_modifiers(), State :: any()}. 67 | 68 | trigger_sample() -> 69 | gen_server:cast(?MODULE, sample). 70 | 71 | start_link() -> 72 | Opts = application:get_all_env(jobs), 73 | start_link(Opts). 74 | 75 | start_link(Opts) -> 76 | gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). 77 | 78 | 79 | tell_sampler(P, Msg) -> 80 | gen_server:call(?MODULE, {tell_sampler, P, timestamp(), Msg}). 81 | 82 | %% @spec subscribe() -> ok 83 | %% @doc Subscribes to feedback indicator information 84 | %% 85 | %% This function allows a process to receive the same information as the 86 | %% jobs_server any time the information changes. 87 | %% 88 | %% The notifications are delivered on the format `{jobs_indicators, Info}', 89 | %% where 90 | %% 
 91 | %% Info :: [{IndicatorName, LocalValue, Remote}]
 92 | %%  Remote :: [{NodeName, Value}]
 93 | %%
94 | %% 95 | %% This information could be used e.g. to aggregate the information and generate 96 | %% new sampler information (which could be passed to a sampler plugin using 97 | %% {@link tell_sampler/2}, or to a specific queue using {@link jobs:ask_queue/2}. 98 | %% 99 | subscribe() -> 100 | gen_server:call(?MODULE, subscribe). 101 | 102 | %% 103 | end_subscription() -> 104 | gen_server:call(?MODULE, end_subscription). 105 | 106 | %% ========================================================== 107 | %% Gen_server callbacks 108 | 109 | init(Opts) -> 110 | Samplers = init_samplers(Opts), 111 | S0 = #state{samplers = Samplers}, 112 | UpdateDelay = proplists:get_value( 113 | sample_update_delay, Opts, S0#state.update_delay), 114 | SampleInterval = proplists:get_value( 115 | sample_interval, Opts, S0#state.sample_interval), 116 | timer:apply_interval(SampleInterval, ?MODULE, trigger_sample, []), 117 | gen_server:abcast(nodes(), ?MODULE, {get_status, node()}), 118 | {ok, #state{samplers = Samplers, 119 | update_delay = UpdateDelay, 120 | sample_interval = SampleInterval}}. 121 | 122 | 123 | handle_call({tell_sampler, Name, TS, Msg}, _From, #state{samplers = Samplers0} = St) -> 124 | case lists:keyfind(Name, #sampler.name, Samplers0) of 125 | false -> 126 | {reply, {error, not_found}, St}; 127 | #sampler{} = Sampler -> 128 | Sampler1 = one_handle_info(Msg, TS, Sampler), 129 | Samplers1 = lists:keyreplace(Name, #sampler.name, Samplers0, Sampler1), 130 | {reply, ok, St#state{samplers = Samplers1}} 131 | end; 132 | handle_call(subscribe, {Pid,_}, #state{subscribers = Subs} = St) -> 133 | MRef = erlang:monitor(process, Pid), 134 | case lists:keymember(Pid,1,Subs) of 135 | true -> 136 | {reply, ok, St}; 137 | false -> 138 | Subs1 = [{Pid, MRef} | Subs], 139 | {reply, ok, St#state{subscribers = Subs1}} 140 | end; 141 | handle_call(end_subscription, {Pid,_}, #state{subscribers = Subs} = St) -> 142 | case lists:keyfind(Pid, 1, Subs) of 143 | false -> 144 | {reply, ok, St}; 145 | {_, MRef} = Found -> 146 | erlang:demonitor(MRef), 147 | {reply, ok, St#state{subscribers = Subs -- [Found]}} 148 | end. 149 | 150 | 151 | handle_info({?MODULE, update}, #state{modified = IsModified} = S) -> 152 | case IsModified of 153 | true -> 154 | {noreply, report_global( 155 | report_local(S#state{modified = false}))}; 156 | false -> 157 | {noreply, S} 158 | end; 159 | handle_info({'DOWN', MRef, _, _, _}, #state{subscribers = Subs} = St) -> 160 | {noreply, St#state{subscribers = lists:keydelete(MRef,2,Subs)}}; 161 | handle_info(Msg, #state{samplers = Samplers0} = S) -> 162 | Samplers = map_handle_info(Msg, Samplers0), 163 | {noreply, calc_modifiers(S#state{samplers = Samplers})}. 164 | 165 | 166 | handle_cast({get_status, Node}, S) -> 167 | tell_node(Node, S), 168 | {noreply, S}; 169 | handle_cast(sample, #state{samplers = Samplers0} = S) -> 170 | Samplers = collect_samples(Samplers0), 171 | {noreply, calc_modifiers(S#state{samplers = Samplers})}; 172 | handle_cast({remote, Node, Modifiers}, #state{remote_modifiers = Ds} = S) -> 173 | NewDs = 174 | lists:keysort(1, ([{{K,Node},V} || {K,V} <- Modifiers] 175 | ++ [D || {{_,N},_} = D <- Ds, N =/= Node])), 176 | {noreply, report_local(S#state{remote_modifiers = NewDs})}. 177 | 178 | 179 | terminate(_, _S) -> 180 | ok. 181 | 182 | 183 | code_change(_FromVsn, State, _Extra) -> 184 | {ok, State}. 185 | 186 | %% end Gen_server callbacks 187 | %% ========================================================== 188 | 189 | 190 | init_samplers(Opts) -> 191 | Samplers = proplists:get_value(samplers, Opts, []), 192 | lists:map( 193 | fun({Name, Mod, Args}) -> 194 | {ok, ModSt} = Mod:init(Name, Args), 195 | #sampler{name = Name, 196 | mod = Mod, 197 | mod_state = ModSt} 198 | end, Samplers). 199 | 200 | 201 | 202 | group_modifiers(Local, Remote) -> 203 | RemoteRegrouped = lists:foldl( 204 | fun({{K,N},V}, D) -> 205 | orddict:append(K,{N,V},D) 206 | end, orddict:new(), Remote), 207 | [{K, V, remote_modifiers(K,RemoteRegrouped)} 208 | || {K,V} <- Local] 209 | ++ 210 | [{K, 0, Vs} || {K,Vs} <- RemoteRegrouped, 211 | not lists:keymember(K, 1, Local)]. 212 | 213 | remote_modifiers(K, Remote) -> 214 | case orddict:find(K, Remote) of 215 | {ok, Vs} -> 216 | Vs; 217 | error -> 218 | [] 219 | end. 220 | 221 | 222 | collect_samples(Samplers) -> 223 | [one_sample(S) || S <- Samplers]. 224 | 225 | 226 | one_sample(#sampler{mod = M, 227 | mod_state = ModS} = Sampler) -> 228 | Timestamp = timestamp(), 229 | try M:sample(Timestamp, ModS) of 230 | {Res, NewModS} -> 231 | add_to_history(Res, Timestamp, 232 | Sampler#sampler{mod_state = NewModS}); 233 | ignore -> 234 | Sampler 235 | catch 236 | error:Err -> 237 | sampler_error(Err, Sampler) 238 | end. 239 | 240 | 241 | map_handle_info(Msg, Samplers) -> 242 | Timestamp = timestamp(), 243 | [one_handle_info(Msg, Timestamp, S) || S <- Samplers]. 244 | 245 | one_handle_info(Msg, TS, #sampler{mod = M, mod_state = ModS} = Sampler) -> 246 | try M:handle_msg(Msg, TS, ModS) of 247 | {ignore, ModS1} -> 248 | Sampler#sampler{mod_state = ModS1}; 249 | {log, Sample, ModS1} -> 250 | add_to_history(Sample, TS, Sampler#sampler{mod_state = ModS1}) 251 | catch 252 | error:Err -> 253 | sampler_error(Err, Sampler) 254 | end. 255 | 256 | 257 | add_to_history(Result, Timestamp, #sampler{hist_length = Len, 258 | history = History} = S) -> 259 | Item = {Timestamp, Result}, 260 | NewHistory = 261 | case queue:len(History) of 262 | HL when HL >= Len -> 263 | queue:in(Item, queue:drop(History)); 264 | _ -> 265 | queue:in(Item, History) 266 | end, 267 | S#sampler{history = NewHistory}. 268 | 269 | 270 | sampler_error(Err, Sampler) -> 271 | error_logger:error_report([{?MODULE, sampler_error}, 272 | {error, Err}, 273 | {sampler, Sampler}]), 274 | % For now, don't modify the sampler (disable it...?) 275 | Sampler. 276 | 277 | 278 | report_local(#state{modifiers = Local, remote_modifiers = Remote, 279 | subscribers = Subs} = S) -> 280 | Grouped = group_modifiers(Local, Remote), 281 | jobs_server:set_modifiers(Grouped), 282 | [Pid ! {jobs_indicators, Grouped} || {Pid,_} <- Subs], 283 | S. 284 | 285 | report_global(#state{modifiers = Local} = S) -> 286 | gen_server:abcast(nodes(), ?MODULE, {remote, node(), Local}), 287 | S. 288 | 289 | 290 | 291 | calc_modifiers(#state{samplers = Samplers} = S) -> 292 | S1 = calc_modifiers(Samplers, S), 293 | case S1#state.modified of 294 | true -> 295 | erlang:send_after(S#state.update_delay, self(), {?MODULE,update}); 296 | false -> 297 | skip 298 | end, 299 | S1. 300 | 301 | 302 | calc_modifiers(Samplers, #state{modifiers = Modifiers0} = S) -> 303 | {Samplers1, {Modifiers1, IsModified}} = 304 | lists:mapfoldl( 305 | fun(#sampler{mod = M, 306 | mod_state = ModS, 307 | history = History} = Sx, {Acc,Flg}) -> 308 | try M:calc(History, ModS) of 309 | {NewModifiers, NewModSt} -> 310 | {Sx#sampler{mod_state = NewModSt}, 311 | {merge_modifiers(orddict:from_list(NewModifiers), Acc), true}}; 312 | false -> 313 | {Sx, {Acc, Flg}} 314 | catch 315 | error:Err -> 316 | sampler_error(Err, Sx), 317 | {Sx, {Acc,Flg}} 318 | end 319 | end, {orddict:new(), false}, Samplers), 320 | FinalMods = orddict:merge(fun(_,V,_) -> V end,Modifiers1,Modifiers0), 321 | S#state{samplers = Samplers1, modifiers = FinalMods, modified = IsModified}. 322 | 323 | 324 | merge_modifiers(New, Modifiers) -> 325 | orddict:merge( 326 | fun(_, V1, V2) -> erlang:max(V1, V2) end, New, Modifiers). 327 | 328 | 329 | tell_node(Node, S) -> 330 | gen_server:cast({?MODULE, Node}, {remote, node(), S#state.modifiers}). 331 | 332 | 333 | %% example: type = time , step = {seconds, [{0,1},{30,2},{45,3},{50,4}]} 334 | %% type = value, step = [{80,1},{85,2},{90,3},{95,4},{100,5}] 335 | 336 | calc(Type, Template, History) -> 337 | case queue:is_empty(History) of 338 | true -> 0; 339 | false -> calc1(Type, Template, History) 340 | end. 341 | 342 | calc1(time, Template, History) -> 343 | Now = timestamp(), 344 | {Unit, Steps} = case Template of 345 | T when is_list(T) -> 346 | {msec, T}; 347 | {U, T} -> 348 | U1 = if 349 | U==sec; U==seconds -> sec; 350 | U==ms; msec -> msec 351 | end, 352 | {U1, T} 353 | end, 354 | case true_since(History) of 355 | 0 -> 0; 356 | Since -> 357 | %% timestamps are in milliseconds 358 | Time = case Unit of 359 | sec -> (Now - Since) div 1000; 360 | msec -> Now - Since 361 | end, 362 | pick_step(Time, Steps) 363 | end; 364 | calc1(value, Template, History) -> 365 | {value, {_, Level}} = queue:peek_r(History), 366 | pick_step(Level, Template). 367 | 368 | true_since(Q) -> 369 | true_since(queue:out_r(Q), 0). 370 | 371 | true_since({{value,{_,false}},_}, Since) -> 372 | Since; 373 | true_since({empty, _}, Since) -> 374 | Since; 375 | true_since({{value,{T,true}},Q1}, _) -> 376 | true_since(queue:out_r(Q1), T). 377 | 378 | 379 | pick_step(Level, Ls) -> 380 | take_last(fun({L,_}) -> 381 | Level >= L 382 | end, Ls, 0). 383 | 384 | take_last(F, [{_,V} = H|T], Last) -> 385 | case F(H) of 386 | true -> take_last(F, T, V); 387 | false -> Last 388 | end; 389 | take_last(_, [], Last) -> 390 | Last. 391 | 392 | 393 | %% millisecond timestamp, never wraps 394 | timestamp() -> 395 | jobs_server:timestamp() div 1000. 396 | -------------------------------------------------------------------------------- /src/jobs_sampler_cpu.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4; indent-tabs-mode: nil -*- 2 | %%============================================================================== 3 | %% Copyright 2014 Ulf Wiger 4 | %% 5 | %% Licensed under the Apache License, Version 2.0 (the "License"); 6 | %% you may not use this file except in compliance with the License. 7 | %% You may obtain a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, software 12 | %% distributed under the License is distributed on an "AS IS" BASIS, 13 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | %% See the License for the specific language governing permissions and 15 | %% limitations under the License. 16 | %%============================================================================== 17 | 18 | %%------------------------------------------------------------------- 19 | %% File : jobs_sampler_cpu.erl 20 | %% @author : Ulf Wiger 21 | %% @end 22 | %% Description : 23 | %% 24 | %% Created : 15 Jan 2010 by Ulf Wiger 25 | %%------------------------------------------------------------------- 26 | -module(jobs_sampler_cpu). 27 | -behaviour(jobs_sampler). 28 | 29 | -export([init/2, 30 | sample/2, 31 | handle_msg/3, 32 | calc/2]). 33 | 34 | -record(st, {levels = []}). 35 | 36 | default_levels() -> [{80,1},{90,2},{100,3}]. 37 | 38 | 39 | init(_Name, Opts) -> 40 | cpu_sup:util([per_cpu]), % first return value is rubbish, per the docs 41 | Levels = proplists:get_value(levels, Opts, default_levels()), 42 | {ok, #st{levels = Levels}}. 43 | 44 | handle_msg(_Msg, _Timestamp, ModS) -> 45 | {ignore, ModS}. 46 | 47 | sample(_Timestamp, #st{} = S) -> 48 | Result = 49 | case cpu_sup:util([per_cpu]) of 50 | Info when is_list(Info) -> 51 | Utils = [U || {_,U,_,_} <- Info], 52 | case Utils of 53 | [U] -> 54 | %% only one cpu 55 | U; 56 | [_,_|_] -> 57 | %% This is a form of ad-hoc averaging, which tries to 58 | %% account for the possibility that the application 59 | %% loads the cores unevenly. 60 | calc_avg_util(Utils) 61 | end; 62 | _ -> 63 | undefined 64 | end, 65 | {Result, S}. 66 | 67 | calc_avg_util(Utils) -> 68 | case minmax(Utils) of 69 | {A,B} when B-A > 50 -> 70 | %% very uneven load 71 | High = [U || U <- Utils, 72 | B-U > 20], 73 | lists:sum(High)/length(High); 74 | {Low,High} -> 75 | (High+Low)/2 76 | end. 77 | 78 | 79 | minmax([H|T]) -> 80 | lists:foldl( 81 | fun(X, {Min,Max}) -> 82 | {erlang:min(X,Min), erlang:max(X,Max)} 83 | end, {H,H}, T). 84 | 85 | 86 | calc(History, #st{levels = Levels} = St) -> 87 | L = jobs_sampler:calc(value, Levels, History), 88 | {[{cpu,L}], St}. 89 | -------------------------------------------------------------------------------- /src/jobs_sampler_history.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | -module(jobs_sampler_history). 18 | -export([new/1, 19 | add/2, 20 | to_list/1, 21 | from_list/2, 22 | take_last/2]). 23 | 24 | -record(jsh, {max_length, 25 | length = 0, 26 | history = queue:new()}). 27 | 28 | new(Length) -> 29 | #jsh{max_length = Length}. 30 | 31 | 32 | add(Entry, #jsh{length = L, 33 | max_length = L} = R) -> 34 | In = {timestamp(), Entry}, 35 | do_add(In, drop(R)); 36 | add(Entry, #jsh{} = R) -> 37 | do_add({timestamp(), Entry}, R). 38 | 39 | 40 | do_add(Item, #jsh{length = L, history = H} = R) -> 41 | R#jsh{length = L+1, history = queue:in(Item, H)}. 42 | 43 | drop(#jsh{length = 0} = R) -> 44 | R; 45 | drop(#jsh{length = L, history = H} = R) -> 46 | R#jsh{length = L-1, history = queue:drop(H)}. 47 | 48 | 49 | from_list(MaxL, L0) -> 50 | {Length, L} = case length(L0) of 51 | Len when Len > MaxL -> 52 | {MaxL, lists:sublist(L0, MaxL)}; 53 | Len -> 54 | {Len, L0} 55 | end, 56 | #jsh{max_length = MaxL, 57 | length = Length, 58 | history = queue:from_list(L)}. 59 | 60 | to_list(#jsh{history = Q}) -> 61 | queue:to_list(Q). 62 | 63 | 64 | take_last(F, #jsh{history = Q}) -> 65 | take_last(F, queue:to_list(Q), []). 66 | 67 | take_last(F, [H|T], Last) -> 68 | case F(H) of 69 | true -> take_last(F, T, H); 70 | false -> Last 71 | end; 72 | take_last(_, [], Last) -> 73 | Last. 74 | 75 | 76 | 77 | %% Millisecond timestamp, never wraps 78 | timestamp() -> 79 | jobs_server:timestamp() div 1000. 80 | -------------------------------------------------------------------------------- /src/jobs_sampler_mnesia.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | 17 | -module(jobs_sampler_mnesia). 18 | 19 | -behaviour(jobs_sampler). 20 | 21 | -export([init/2, 22 | sample/2, 23 | handle_msg/3, 24 | calc/2]). 25 | 26 | -record(st, {levels = [], 27 | subscriber}). 28 | 29 | 30 | 31 | init(Name, Opts) -> 32 | Pid = spawn_link(fun() -> 33 | mnesia:subscribe(system), 34 | subscriber_loop(Name) 35 | end), 36 | Levels = proplists:get_value(levels, Opts, default_levels()), 37 | {ok, #st{levels = Levels, subscriber = Pid}}. 38 | 39 | default_levels() -> 40 | {seconds, [{0,1}, {30,2}, {45,3}, {60,4}]}. 41 | 42 | handle_msg({mnesia_system_event, {mnesia,{dump_log,_}}}, _T, S) -> 43 | {log, true, S}; 44 | handle_msg({mnesia_system_event, {mnesia_tm, message_queue_len, _}}, _T, S) -> 45 | {log, true, S}; 46 | handle_msg(_, _T, S) -> 47 | {ignore, S}. 48 | 49 | sample(_T, S) -> 50 | {is_overload(), S}. 51 | 52 | calc(History, #st{levels = Levels} = S) -> 53 | {[{mnesia,jobs_sampler:calc(time, Levels, History)}], S}. 54 | 55 | 56 | subscriber_loop(Name) -> 57 | receive 58 | Msg -> 59 | case jobs_sampler:tell_sampler(Name, Msg) of 60 | ok -> 61 | subscriber_loop(Name); 62 | {error, _} -> 63 | %% sampler likely removed 64 | exit(normal) 65 | end 66 | end. 67 | 68 | 69 | is_overload() -> 70 | %% e.g: [{mnesia_tm,true},{mnesia_dump_log,false}] 71 | lists:keymember(true, 2, mnesia_overload_read()). 72 | 73 | mnesia_overload_read() -> 74 | %% This function is not present in mnesia versions older than R14A 75 | case erlang:function_exported(mnesia_lib,overload_read,0) of 76 | false -> 77 | []; 78 | true -> 79 | mnesia_lib:overload_read() 80 | end. 81 | -------------------------------------------------------------------------------- /src/jobs_stateful_simple.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% Copyright 2014 Ulf Wiger 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%============================================================================== 16 | -module(jobs_stateful_simple). 17 | 18 | -export([init/2, 19 | next/3, 20 | handle_call/4]). 21 | 22 | -include("jobs.hrl"). 23 | 24 | init(F, Info) when is_function(F, 2) -> 25 | #stateful{f = F, st = F(init, Info)}. 26 | 27 | next(_Opaque, #stateful{f = F, st = St} = P, Info) -> 28 | case F(St, Info) of 29 | {V, St1} -> 30 | {V, P#stateful{st = St1}}; 31 | Other -> 32 | erlang:error({bad_stateful_next, Other}) 33 | end. 34 | 35 | handle_call(Req, From, #stateful{f = F, st = St} = P, Info) -> 36 | case F({call, Req, From, St}, Info) of 37 | {reply, Reply, St1} -> 38 | {reply, Reply, P#stateful{st = St1}}; 39 | {noreply, St1} -> 40 | {noreply, P#stateful{st = St1}} 41 | end. 42 | -------------------------------------------------------------------------------- /test/jobs_eqc_queue.erl: -------------------------------------------------------------------------------- 1 | -module(jobs_eqc_queue). 2 | 3 | -ifdef(EQC). 4 | -include_lib("eqc/include/eqc.hrl"). 5 | -include("jobs.hrl"). 6 | 7 | -compile([export_all, nowarn_export_all]). 8 | 9 | -record(model, 10 | { time = jobs_lib:timestamp(), 11 | st = undefined }). 12 | 13 | test() -> 14 | test(300). 15 | 16 | test(N) -> 17 | meck:new(jobs_lib, [passthrough]), 18 | eqc:module({numtests, N}, ?MODULE), 19 | meck:unload(jobs_lib). 20 | 21 | g_job() -> 22 | {make_ref(), make_ref()}. 23 | 24 | g_scheduling_order() -> 25 | elements([fifo, lifo]). 26 | 27 | g_options(jobs_queue) -> return([{type, fifo}]); 28 | g_options(jobs_queue_list) -> return([{type, lifo}]). 29 | 30 | g_queue_record() -> 31 | ?LET(N, nat(), 32 | #queue { max_time = N }). 33 | 34 | g_start_time() -> 35 | ?LET({T, N}, {jobs_lib:timestamp(), nat()}, 36 | T + N). 37 | 38 | g_time_advance() -> 39 | ?LET(N, nat(), N+1). 40 | 41 | g_model_type() -> 42 | oneof([jobs_queue_list, jobs_queue]). 43 | 44 | g_model(Ty) -> 45 | ?SIZED(Size, g_model(Size, Ty)). 46 | 47 | g_model(0, Ty) -> 48 | oneof([{call, ?MODULE, new, [Ty, 49 | g_options(Ty), 50 | g_queue_record(), 51 | g_start_time()]}]); 52 | g_model(N, Ty) -> 53 | frequency([{1, g_model(0, Ty)}, 54 | {N, 55 | ?LET(M, g_model(max(0, N-2), Ty), 56 | frequency( 57 | [ 58 | {200, {call, ?MODULE, advance_time, 59 | [M, g_time_advance()]}}, 60 | {200, {call, ?MODULE, in, [Ty, g_job(), M]}}, 61 | {100, {call, ?MODULE, out, [Ty, choose(0,100), M]}}, 62 | {20, {call, ?MODULE, timedout, [Ty, M]}}, 63 | {1, {call, ?MODULE, empty, [Ty, M]}} 64 | ]))} 65 | ]). 66 | 67 | new(Mod, Opts, Q, T) -> 68 | advance_time( 69 | #model { st = Mod:new(Opts, Q), 70 | time = T}, 1). 71 | 72 | advance_time(#model { time = T} = M, N) -> 73 | M#model { time = T + N}. 74 | 75 | timedout(Mod, #model { st = Q} = M) -> 76 | set_time(M), 77 | NQ = case Mod:timedout(Q) of 78 | [] -> Q; 79 | {_, Q1} -> Q1 80 | end, 81 | advance_time(M#model { st = NQ }, 1). 82 | 83 | timedout_obs(Mod, #model { st = Q} = M) -> 84 | set_time(M), 85 | case Mod:timedout(Q) of 86 | [] -> []; 87 | {TO, _} -> lists:sort(TO) 88 | end. 89 | 90 | in(Mod, Job, #model { time = T, st = Q} = M) -> 91 | set_time(M), 92 | advance_time( 93 | M#model { st = Mod:in(T, Job, Q)}, 94 | 1). 95 | 96 | out(Mod, N, #model { st = Q} = M) -> 97 | set_time(M), 98 | NQ = element(2, Mod:out(N, Q)), 99 | advance_time( 100 | M#model { st = NQ }, 101 | 1). 102 | 103 | empty(Mod, #model { st = Q} = M) -> 104 | set_time(M), 105 | advance_time(M#model { st = Mod:empty(Q)}, 1). 106 | 107 | is_empty(M, #model { st = Q}) -> 108 | M:is_empty(Q). 109 | 110 | peek(M, #model { st = Q}) -> 111 | M:peek(Q). 112 | 113 | all(M, #model { st = Q}) -> 114 | M:all(Q). 115 | 116 | info(M, I, #model { st = Q}) -> 117 | M:info(I, Q). 118 | 119 | g_info() -> 120 | oneof([oldest_job, length, max_time]). 121 | 122 | obs() -> 123 | ?LET(Ty, g_model_type(), 124 | begin 125 | M = g_model(Ty), 126 | oneof([{call, ?MODULE, all, [Ty, M]}, 127 | {call, ?MODULE, peek, [Ty, M]}, 128 | {call, ?MODULE, info, [Ty, g_info(), M]}, 129 | {call, ?MODULE, timedout_obs, [Ty, M]}, 130 | {call, ?MODULE, is_empty, [Ty, M]}]) 131 | end). 132 | 133 | model({call, ?MODULE, F, [W | Args]}) when W == jobs_queue; 134 | W == jobs_queue_list -> 135 | apply(?MODULE, F, model([jobs_queue_model | Args])); 136 | model({call, ?MODULE, F, Args}) -> 137 | apply(?MODULE, F, model(Args)); 138 | model([H|T]) -> 139 | [model(H) | model(T)]; 140 | model(X) -> 141 | X. 142 | 143 | prop_oldest_job_match() -> 144 | ?LET(Ty, g_model_type(), 145 | ?FORALL(M, g_model(Ty), 146 | begin 147 | R = eval(M), 148 | Repr = Ty:representation(R#model.st), 149 | OJ = proplists:get_value(oldest_job, Repr), 150 | Cts = proplists:get_value(contents, Repr), 151 | case OJ of 152 | undefined -> 153 | Cts == []; 154 | V -> 155 | lists:min([TS || {TS, _} <- Cts]) == V 156 | end 157 | end)). 158 | 159 | prop_queue() -> 160 | ?LET(Ty, g_model_type(), 161 | ?FORALL(M, g_model(Ty), 162 | equals( 163 | catching(fun() -> 164 | R = eval(M), 165 | Ty:representation(R#model.st) 166 | end), 167 | catching(fun () -> 168 | R = model(M), 169 | jobs_queue_model:representation(R#model.st) 170 | end)))). 171 | 172 | prop_observe() -> 173 | ?FORALL(Obs, obs(), 174 | equals( 175 | catch eval(Obs), 176 | catch model(Obs))). 177 | 178 | catching(F) -> 179 | try F() 180 | catch C:E -> 181 | {exception, C, E} 182 | end. 183 | 184 | set_time(#model { time = T}) -> 185 | meck:expect(jobs_lib, timestamp, fun() -> T end). 186 | 187 | -endif. 188 | -------------------------------------------------------------------------------- /test/jobs_queue_model.erl: -------------------------------------------------------------------------------- 1 | -module(jobs_queue_model). 2 | 3 | -include("jobs.hrl"). 4 | 5 | -compile([export_all, nowarn_export_all]). 6 | 7 | new(Options, Q) -> 8 | case proplists:get_value(type, Options) of 9 | fifo -> 10 | Q#queue { type = fifo, 11 | st = queue:new() }; 12 | lifo -> 13 | Q#queue { type = lifo, 14 | st = queue:new() } 15 | end. 16 | 17 | is_empty(#queue { st = Q}) -> 18 | queue:is_empty(Q). 19 | 20 | info(oldest_job, #queue { oldest_job = OJ}) -> 21 | OJ; 22 | info(max_time, #queue { max_time = MT}) -> MT; 23 | info(length, #queue { st = Q}) -> 24 | queue:len(Q). 25 | 26 | timedout(#queue { max_time = undefined}) -> 27 | %% This return value is highly illogical, but it is what the code returns! 28 | []; 29 | timedout(#queue { type = Ty, 30 | max_time = TO, st = Queue} = Q) -> 31 | Now = jobs_lib:timestamp(), 32 | QL = queue:to_list(Queue), 33 | {Left, Timedout} = lists:partition( 34 | fun({TS, _}) -> 35 | not(is_expired(TS, Now, TO)) 36 | end, QL), 37 | OJ = get_oldest_job(Left), 38 | {case Ty of 39 | fifo -> Timedout; 40 | lifo -> lists:reverse(Timedout) 41 | end, Q#queue { oldest_job = OJ, 42 | st = queue:from_list(Left)}}. 43 | 44 | is_expired(TS, Now, TO) -> 45 | MS = Now - TS, 46 | MS > TO. 47 | 48 | get_oldest_job([]) -> undefined; 49 | get_oldest_job(L) -> 50 | lists:min([TS || {TS, _} <- L]). 51 | 52 | peek(#queue { type = fifo, st = Q }) -> 53 | case queue:peek(Q) of 54 | empty -> undefined; 55 | {value, K} -> K 56 | end; 57 | peek(#queue { type = lifo, st = Q }) -> 58 | case queue:peek_r(Q) of 59 | empty -> undefined; 60 | {value, K} -> K 61 | end. 62 | 63 | all(#queue { type = fifo, st = Q}) -> 64 | queue:to_list(Q); 65 | all(#queue { type = lifo, st = Q}) -> 66 | queue:to_list(queue:reverse(Q)). 67 | 68 | in(TS, E, #queue { st = Q, 69 | oldest_job = OJ } = S) -> 70 | S#queue { st = queue:in({TS, E}, Q), 71 | oldest_job = case OJ of undefined -> TS; 72 | _ -> OJ 73 | end}. 74 | 75 | out(N, #queue { type = Ty, st = Q} = S) -> 76 | {Elems, NQ} = out(Ty, N, Q, []), 77 | {Elems, S#queue { st = NQ, 78 | oldest_job = set_oldest_job(Ty, NQ) }}. 79 | 80 | set_oldest_job(fifo, Q) -> 81 | case queue:out(Q) of 82 | {{value, {TS, _}}, _} -> 83 | TS; 84 | {empty, _} -> 85 | undefined 86 | end; 87 | set_oldest_job(lifo, Q) -> 88 | case queue:out(Q) of 89 | {{value, {TS, _}}, _} -> 90 | TS; 91 | {empty, _} -> 92 | undefined 93 | end. 94 | 95 | out(fifo, 0, Q, Taken) -> 96 | {lists:reverse(Taken), Q}; 97 | out(lifo, 0, Q, Taken) -> 98 | {Taken, Q}; 99 | out(fifo, K, Q, Acc) when K > 0 -> 100 | case queue:out(Q) of 101 | {{value, E}, NQ} -> 102 | out(fifo, K-1, NQ, [E | Acc]); 103 | {empty, NQ} -> 104 | out(fifo, 0, NQ, Acc) 105 | end; 106 | out(lifo, K, Q, Acc) -> 107 | case queue:out_r(Q) of 108 | {{value, E}, NQ} -> 109 | out(lifo, K-1, NQ, [E | Acc]); 110 | {empty, NQ} -> 111 | out(lifo, 0, NQ, Acc) 112 | end. 113 | 114 | 115 | empty(#queue {} = Q) -> 116 | Q#queue { st = queue:new(), 117 | oldest_job = undefined }. 118 | 119 | representation(#queue { type = fifo, st = Q, oldest_job = OJ} ) -> 120 | Cts = queue:to_list(Q), 121 | [{oldest_job, OJ}, 122 | {contents, Cts}]; 123 | representation(#queue { type = lifo, st = Q, oldest_job = OJ} ) -> 124 | Cts = queue:to_list(queue:reverse(Q)), 125 | [{oldest_job, OJ}, 126 | {contents, Cts}]; 127 | representation(O) -> 128 | io:format("Otherwise: ~p", [O]), 129 | exit(fail). 130 | 131 | 132 | -------------------------------------------------------------------------------- /test/jobs_sampler_slave.erl: -------------------------------------------------------------------------------- 1 | -module(jobs_sampler_slave). 2 | 3 | -behaviour(jobs_sampler). 4 | 5 | -export([init/2, 6 | sample/2, 7 | handle_msg/3, 8 | calc/2]). 9 | 10 | -define(NOTEST, 1). 11 | 12 | init(_Name, {Type, Levels}) -> 13 | {ok, {Type, Levels}}. 14 | 15 | 16 | handle_msg({test, log, V}, _T, S) -> 17 | {log, V, S}; 18 | handle_msg(_Msg, _T, S) -> 19 | {ignore, S}. 20 | 21 | 22 | sample(_, _S) -> 23 | ignore. 24 | 25 | calc(History, {Type, Levels} = S) -> 26 | {[{test,jobs_sampler:calc(Type, Levels, History)}], S}. 27 | 28 | -------------------------------------------------------------------------------- /test/jobs_server_tests.erl: -------------------------------------------------------------------------------- 1 | -module(jobs_server_tests). 2 | -export([start_test_server/1]). 3 | 4 | -include_lib("eunit/include/eunit.hrl"). 5 | 6 | rate_test_() -> 7 | {foreachx, 8 | fun(Type) -> start_test_server(Type) end, 9 | fun(_, _) -> stop_server() end, 10 | [{{rate,1}, fun(_,_) -> [fun() -> serial(1,2,2) end] end} 11 | , {{rate, 5}, fun(_,_) -> [fun() -> serial(5,5,1) end] end} 12 | , {{rate, 50}, fun(_,_) -> [fun() -> serial(50,50,1) end] end} 13 | , {{rate, 100}, fun(_,_) -> [fun() -> serial(100,100,1) end] end} 14 | , {{rate, 300}, fun(_,_) -> [fun() -> serial(300,300,1) end] end} 15 | , {{rate, 500}, fun(_,_) -> [fun() -> serial(500,500,1) end] end} 16 | , {{rate, 1000}, fun(_,_) -> [fun() -> serial(1000,1000,1) end] end} 17 | %% , {{rate, 100}, fun(O,_) -> [fun() -> rate_test(O,1) end] end} 18 | , {{rate, 400}, fun(_,_) -> [fun() -> par_run(400,400,1) end] end} 19 | , {{rate, 600}, fun(_,_) -> [fun() -> par_run(600,600,1) end] end} 20 | , {{rate,1000}, fun(_,_) -> [fun() -> par_run(1000,1000,1) end] end} 21 | , {{rate,2000}, fun(_,_) -> [fun() -> par_run(2000,2000,1) end] end} 22 | %% , {[{rate,100}, 23 | %% {group,50}], fun(O,_) -> [fun() -> max_rate_test(O,1) end] end} 24 | , {{count,3}, fun(_,_) -> [fun() -> counter_run(30,1) end] end} 25 | , {[{rate,5},{count,3}], 26 | fun(_,_) -> [fun() -> par_run(5,5,1) end] end} 27 | , {{timeout,500}, fun(_,_) -> [fun() -> 28 | ?debugVal(timeout_test(500)) 29 | end] end} 30 | ]}. 31 | 32 | link_test_() -> 33 | Name = {q, ?LINE}, 34 | Pid = start_link_pid(), 35 | {setup, 36 | fun() -> 37 | {Pid, start_test_server(false, {Name, [{standard_counter, 1}, 38 | {link, Pid}]})} 39 | end, 40 | fun(_) -> 41 | stop_server() 42 | end, 43 | [ fun() -> 44 | queue_works(Name) 45 | end, 46 | fun() -> 47 | queue_removed(Pid, Name) 48 | end 49 | ]}. 50 | 51 | start_link_pid() -> 52 | Parent = self(), 53 | spawn(fun() -> 54 | MRef = monitor(process, Parent), 55 | receive 56 | {'DOWN', MRef, _, _, _} -> 57 | done 58 | end 59 | end). 60 | 61 | queue_works(Name) -> 62 | ok = jobs:run(Name, fun() -> 63 | ok 64 | end). 65 | 66 | queue_removed(Pid, Name) -> 67 | {queue, _} = jobs:queue_info(Name), 68 | MRef = monitor(process, Pid), 69 | exit(Pid, kill), 70 | receive 71 | {'DOWN', MRef, process, Pid, _} -> 72 | timer:sleep(500), 73 | undefined = jobs:queue_info(Name) 74 | after 1000 -> 75 | error(timeout) 76 | end. 77 | 78 | config_test_() -> 79 | {foreachx, 80 | fun(Conf) -> start_server_w_config(Conf) end, 81 | fun(_, _) -> stop_server() end, 82 | [ {#{ restore => {true, preset} 83 | , preset => [{queue,q}] 84 | , dynamic => [{queue,q1}]}, fun config_test_f/2} 85 | , {#{ restore => {false, preset} 86 | , preset => [{queue,q}] 87 | , dynamic => [{queue,q1}] }, fun config_test_f/2} 88 | , {#{ restore => {true, preset} 89 | , preset => [] 90 | , dynamic => [{queue,q1,[{standard_rate,100}]}] }, fun rate_recovers/2} 91 | ]}. 92 | 93 | config_test_f(Conf, Pre) -> 94 | fun() -> 95 | restart_jobs_server(), 96 | compare_configs(Pre, prune_info(jobs:info(all)), Conf) 97 | end. 98 | 99 | rate_recovers(#{dynamic := [{queue,Q,[{standard_rate,R}]}]} = _Conf, _Pre) -> 100 | fun() -> 101 | true = test_rate(Q, R), 102 | restart_jobs_server(), 103 | true = test_rate(Q, R) 104 | end. 105 | 106 | restart_jobs_server() -> 107 | exit(whereis(jobs_server), kill), 108 | await_jobs_server(1000). 109 | 110 | test_rate(Q, R) -> 111 | T0 = jobs_lib:timestamp(), 112 | Times = [jobs:run(Q, fun() -> 113 | jobs_lib:timestamp() 114 | end) || _ <- lists:seq(1,10)], 115 | TimeLimit = 1000 * 10 * (1/R), % in milliseconds 116 | Actual = (lists:last(Times) - T0), 117 | %% allow for 10% deviation 118 | {true, _} = {Actual =< (TimeLimit * 1.1), 119 | {TimeLimit, Actual, T0, Times}}, 120 | true. 121 | 122 | 123 | start_server_w_config(Conf) -> 124 | ensure_jobs_cleared(), 125 | application:load(jobs), 126 | set_env_w_config(Conf), 127 | application:ensure_started(jobs), 128 | post_start_config(Conf), 129 | apply_dynamic(maps:get(dynamic, Conf, [])), 130 | prune_info(jobs:info(all)). 131 | 132 | ensure_jobs_cleared() -> 133 | application:stop(jobs), 134 | application:unload(jobs). 135 | 136 | set_env_w_config(#{preset := Preset, restore := {Restore, Ctxt}}) -> 137 | Queues = [{Q, [{standard_rate,2}]} || {queue, Q} <- Preset], 138 | application:set_env(jobs, queues, Queues), 139 | [application:set_env(jobs, auto_restore, Restore) || Ctxt == preset], 140 | ok. 141 | 142 | post_start_config(#{restore := {Restore, dynamic}}) -> 143 | jobs_server:auto_restore(Restore); 144 | post_start_config(_) -> 145 | ok. 146 | 147 | apply_dynamic(Dyn) -> 148 | lists:foreach( 149 | fun({queue, Q}) -> 150 | ok = jobs:add_queue(Q, [{standard_rate, 2}]); 151 | ({queue, Q, Opts}) -> 152 | ok = jobs:add_queue(Q, Opts) 153 | end, Dyn). 154 | 155 | await_jobs_server(Timeout) -> 156 | TRef = erlang:start_timer(Timeout, self(), await_jobs_server), 157 | await_jobs_server_(TRef). 158 | 159 | await_jobs_server_(TRef) -> 160 | case whereis(jobs_server) of 161 | undefined -> 162 | receive 163 | {timeout, TRef, await_jobs_server} -> 164 | erlang:error(timeout) 165 | after 10 -> 166 | await_jobs_server_(TRef) 167 | end; 168 | Pid when is_pid(Pid) -> 169 | case is_process_alive(Pid) of 170 | true -> 171 | erlang:cancel_timer(TRef), 172 | Pid; 173 | false -> 174 | await_jobs_server_(TRef) 175 | end 176 | end. 177 | 178 | prune_info(Info) -> 179 | lists:map(fun prune_info_/1, Info). 180 | 181 | prune_info_({queues, Qs}) -> 182 | {queues, [{Q, prune_q_info(Iq)} || {Q, Iq} <- Qs]}; 183 | prune_info_(I) -> 184 | I. 185 | 186 | prune_q_info(I) -> 187 | [X || {K,_} = X <- I, 188 | not lists:member(K, [check_interval, 189 | latest_dispatch, approved, queued, 190 | oldest_job, timer, empty, depleted, 191 | waiters, stateful, st])]. 192 | 193 | compare_configs(_, _, #{restore := {false,_}}) -> true; 194 | compare_configs(I1, I2, Conf) -> 195 | {I1, I2, _} = {I2, I1, Conf}. 196 | 197 | serial(R, N, TargetRatio) -> 198 | Expected = (N div R) * 1000000 * TargetRatio, 199 | ?debugVal({R,N,Expected}), 200 | {T,Ts} = tc(fun() -> run_jobs(q,N) end), 201 | time_eval(R, N, T, Ts, Expected). 202 | 203 | par_run(R, N, TargetRatio) -> 204 | Expected = (N div R) * 1000000 * TargetRatio, 205 | ?debugVal({R,N,Expected}), 206 | {T,Ts} = tc(fun() -> pmap(fun() -> 207 | run_job(q, one_job(time)) 208 | end, N) 209 | end), 210 | time_eval(R, N, T, Ts, Expected). 211 | 212 | counter_run(N, Target) -> 213 | ?debugVal({N, Target}), 214 | {T, Ts} = tc(fun() -> 215 | pmap(fun() -> run_job(q, one_job(count)) end, N) 216 | end), 217 | ?debugVal({T,Ts}). 218 | 219 | timeout_test(T) -> 220 | case timer:tc(jobs, ask, [q]) of 221 | {US, {error, timeout}} -> 222 | case (US div 1000) - T of 223 | Diff when Diff < 5 -> 224 | ok; 225 | Other -> 226 | error({timeout_too_late, Other}) 227 | end; 228 | Other -> 229 | error({timeout_expected, Other}) 230 | end. 231 | 232 | time_eval(_R, _N, T, Ts, Expected) -> 233 | [{Hd,_}|Tl] = lists:sort(Ts), 234 | Diffs = [X-Hd || {X,_} <- Tl], 235 | Ratio = T/Expected, 236 | Max = lists:max(Diffs), 237 | {Mean, Variance} = time_variance(Diffs), 238 | io:fwrite(user, 239 | "Time: ~p, Ratio = ~.1f, Max = ~p, " 240 | "Mean = ~.1f, Variance = ~.1f~n", 241 | [T, Ratio, Max, Mean, Variance]). 242 | 243 | 244 | time_variance(L) -> 245 | N = length(L), 246 | Mean = lists:sum(L) / N, 247 | SQ = fun(X) -> X*X end, 248 | {Mean, math:sqrt(lists:sum([SQ(X-Mean) || X <- L]) / N)}. 249 | 250 | 251 | 252 | %% counter_test(Count) -> 253 | %% start_test_server({count,Count}), 254 | %% Res = tc(fun() -> 255 | %% pmap(fun() -> jobs:run(q, one_job(count)) end, Count * 2) 256 | %% end), 257 | %% io:fwrite(user, "~p~n", [Res]), 258 | %% stop_server(). 259 | 260 | 261 | pmap(F, N) -> 262 | Pids = [spawn_monitor(fun() -> exit(F()) end) || _ <- lists:seq(1,N)], 263 | collect(Pids). 264 | 265 | collect([{_P,Ref}|Ps]) -> 266 | receive 267 | {'DOWN', Ref, _, _, Res} -> 268 | [Res|collect(Ps)] 269 | end; 270 | collect([]) -> 271 | []. 272 | 273 | start_test_server(Conf) -> 274 | start_test_server(true, Conf). 275 | 276 | start_test_server(Silent, {rate,Rate}) -> 277 | start_with_conf(Silent, [{queues, [{q, [{regulators, 278 | [{rate,[ 279 | {limit, Rate}] 280 | }]} 281 | %% , {mod, jobs_queue_list} 282 | ]} 283 | ]} 284 | ]), 285 | Rate; 286 | start_test_server(Silent, [{rate,Rate},{group,Grp}]) -> 287 | start_with_conf(Silent, 288 | [{group_rates, [{gr, [{limit, Grp}]}]}, 289 | {queues, [{q, [{regulators, 290 | [{rate,[{limit, Rate}]}, 291 | {group_rate, gr}]} 292 | ]} 293 | ]} 294 | ]), 295 | Grp; 296 | start_test_server(Silent, {count, Count}) -> 297 | start_with_conf(Silent, 298 | [{queues, [{q, [{regulators, 299 | [reg({count, Count})] 300 | }] 301 | }] 302 | }]); 303 | start_test_server(Silent, {timeout, T}) -> 304 | start_with_conf(Silent, 305 | [{queues, [{q, [{regulators, 306 | [{counter,[ 307 | {limit, 0} 308 | ]} 309 | ]}, 310 | {max_time, T} 311 | ]} 312 | ]} 313 | ]); 314 | start_test_server(Silent, {_Name, [_|_]} = Q) -> 315 | start_with_conf(Silent, [{queues, [Q]}]); 316 | start_test_server(Silent, [_|_] = Rs) -> 317 | start_with_conf(Silent, 318 | [{queues, [{q, 319 | [{regulators, [reg(R) || R <- Rs]}] 320 | }] 321 | }]). 322 | 323 | 324 | reg({rate, R}) -> 325 | {rate, [{limit, R}]}; 326 | reg({count, C}) -> 327 | {counter, [{limit, C}]}. 328 | 329 | 330 | 331 | start_with_conf(Silent, Conf) -> 332 | application:unload(jobs), 333 | application:load(jobs), 334 | [application:set_env(jobs, K, V) || {K,V} <- Conf], 335 | if Silent == true -> 336 | error_logger:delete_report_handler(error_logger_tty_h); 337 | true -> 338 | ok 339 | end, 340 | application:start(jobs). 341 | 342 | 343 | stop_server() -> 344 | application:stop(jobs). 345 | 346 | tc(F) -> 347 | T1 = jobs_lib:time_compat(), 348 | R = (catch F()), 349 | T2 = jobs_lib:time_compat(), 350 | {timer:now_diff(T2,T1), R}. 351 | 352 | run_jobs(Q,N) -> 353 | [run_job(Q, one_job(time)) || _ <- lists:seq(1,N)]. 354 | 355 | run_job(Q,F) -> 356 | timer:tc(jobs,run,[Q,F]). 357 | 358 | one_job(time) -> 359 | fun timestamp/0; 360 | one_job(count) -> 361 | fun() -> 362 | 1 363 | end. 364 | 365 | 366 | timestamp() -> 367 | jobs_server:timestamp(). 368 | -------------------------------------------------------------------------------- /test/jobs_tests.erl: -------------------------------------------------------------------------------- 1 | -module(jobs_tests). 2 | 3 | -compile([export_all, nowarn_export_all]). 4 | -export([with_msg_sampler/1]). 5 | 6 | -include_lib("eunit/include/eunit.hrl"). 7 | 8 | 9 | 10 | msg_test_() -> 11 | Rate = 100, 12 | {foreach, 13 | fun() -> with_msg_sampler(Rate) end, 14 | fun(_) -> stop_jobs() end, 15 | [ 16 | {with, [fun apply_feedback/1]} 17 | ]}. 18 | 19 | dist_test_() -> 20 | Rate = 100, 21 | Name = jobs_eunit_slave, 22 | {foreach, 23 | fun() -> 24 | ?assertEqual(Rate, with_msg_sampler(Rate)), 25 | Remote = start_slave(Name), 26 | RpcRes = rpc:call(Remote, ?MODULE, with_msg_sampler, [Rate]), 27 | ?assertEqual(Rate, RpcRes), 28 | {Remote, Rate} 29 | end, 30 | fun({Remote, _}) -> 31 | Res = rpc:call(Remote, erlang, halt, []), 32 | io:fwrite(user, "Halting remote: ~p~n", [Res]), 33 | stop_jobs() 34 | end, 35 | [ 36 | {with, [fun apply_feedback/1]} 37 | ]}. 38 | 39 | 40 | 41 | with_msg_sampler(Rate) -> 42 | application:unload(jobs), 43 | ok = application:load(jobs), 44 | [application:set_env(jobs, K, V) || 45 | {K,V} <- [{queues, [{q, [{regulators, 46 | [{rate, [ 47 | {limit, Rate}, 48 | {modifiers, 49 | [{test,10, {max,5}}]}]}]} 50 | ]} 51 | ]}, 52 | {samplers, [{test, jobs_sampler_slave, 53 | {value, [{1,1},{2,2},{3,3}]}} 54 | ]} 55 | ] 56 | ], 57 | ok = application:start(jobs), 58 | Rate. 59 | 60 | start_slave(Name) -> 61 | case node() of 62 | nonode@nohost -> 63 | os:cmd("epmd -daemon"), 64 | {ok, _} = net_kernel:start([jobs_eunit_master, shortnames]); 65 | _ -> 66 | ok 67 | end, 68 | D1 = filename:absname(code:lib_dir(jobs, test)), 69 | D2 = filename:absname(code:lib_dir(jobs, ebin)), 70 | {ok, Node} = slave:start(host(), Name, "-pa " ++ D1 ++ " -pz " ++ D2), 71 | io:fwrite(user, "Slave node: ~p~n", [Node]), 72 | Node. 73 | 74 | host() -> 75 | [_Name, Host] = re:split(atom_to_list(node()), "@", [{return, list}]), 76 | list_to_atom(Host). 77 | 78 | 79 | stop_jobs() -> 80 | dbg:stop(), 81 | application:stop(jobs). 82 | 83 | apply_feedback(Rate) when is_integer(Rate) -> 84 | R0 = get_rate(), 85 | ?assertEqual(R0, Rate), 86 | io:fwrite(user, "R0 = ~p~n", [R0]), 87 | kick_sampler(1), 88 | io:fwrite(user, "get_rate() -> ~p~n", [get_rate()]), 89 | ?assertEqual(get_rate(), Rate - 10), 90 | kick_sampler(2), 91 | io:fwrite(user, "get_rate() -> ~p~n", [get_rate()]), 92 | ?assertEqual(get_rate(), Rate - 20), 93 | kick_sampler(3), 94 | io:fwrite(user, "get_rate() -> ~p~n", [get_rate()]), 95 | ?assertEqual(get_rate(), Rate - 30); 96 | apply_feedback({Remote, Rate}) -> 97 | R0=get_rate(), 98 | ?assertEqual(R0, Rate), 99 | io:fwrite(user, "R0 = ~p~n", [R0]), 100 | ?assertEqual(rpc:call(Remote,?MODULE,get_rate,[]), Rate), 101 | kick_sampler(Remote, 1), 102 | io:fwrite(user, "[Remote] get_rate() -> ~p~n", [get_rate()]), 103 | ?assertEqual(get_rate(), Rate - 5), 104 | kick_sampler(Remote, 2), 105 | io:fwrite(user, "[Remote] get_rate() -> ~p~n", [get_rate()]), 106 | ?assertEqual(get_rate(), Rate - 10), 107 | kick_sampler(Remote, 3), 108 | io:fwrite(user, "[Remote] get_rate() -> ~p~n", [get_rate()]), 109 | ?assertEqual(get_rate(), Rate - 15). 110 | 111 | 112 | get_rate() -> 113 | jobs:queue_info(q, rate_limit). 114 | 115 | kick_sampler(N) -> 116 | jobs_sampler ! {test, log, N}, 117 | timer:sleep(1000). 118 | 119 | 120 | kick_sampler(Remote, N) -> 121 | io:fwrite("Kicking sampler (N=~p) at ~p~n", [N, Remote]), 122 | {jobs_sampler, Remote} ! {test, log, N}, 123 | timer:sleep(1000). 124 | 125 | -------------------------------------------------------------------------------- /test/t.erl: -------------------------------------------------------------------------------- 1 | -module(t). 2 | 3 | -compile([export_all, nowarn_export_all]). 4 | 5 | t() -> 6 | t(300). 7 | 8 | t(N) -> 9 | jobs_eqc_queue:test(N). 10 | --------------------------------------------------------------------------------