├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.org ├── include ├── ct_boilerplate.hrl └── snabbkaffe.hrl ├── rebar.config ├── rebar.lock ├── src ├── asciiart.erl ├── snabbkaffe.app.src ├── snabbkaffe.erl ├── snabbkaffe_collector.erl ├── snabbkaffe_internal.hrl ├── snabbkaffe_nemesis.erl └── snabbkaffe_sup.erl └── test ├── causality_tests.erl ├── collector_SUITE.erl ├── complete_tests.erl ├── concuerror_tests.erl ├── is_subset_tests.erl ├── misc_tests.erl ├── nemesis_SUITE.erl ├── overlap_depth_tests.erl ├── split_tests.erl └── unique_tests.erl /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # Unix-style, newlines, indent style of 2 spaces, with a newline ending every file. 4 | [*.{e,h}rl] 5 | end_of_line = lf 6 | insert_final_newline = true 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | indent_style = space 10 | indent_size = 2 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | deps 3 | *.o 4 | *.beam 5 | *.plt 6 | erl_crash.dump 7 | ebin/*.beam 8 | rel/example_project 9 | .concrete/DEV_MODE 10 | .rebar 11 | _build 12 | concuerror_report.txt 13 | TAGS 14 | rebar3.crashdump 15 | snabbkaffe -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | otp_release: 3 | - 23.0 4 | - 19.3 5 | 6 | script: | 7 | make 8 | if [ $TRAVIS_OTP_RELEASE = '23.0' ]; then 9 | make concuerror_test 10 | fi 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019-2020 Klarna Bank AB 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BUILD_DIR := $(CURDIR)/_build 2 | CONCUERROR := $(BUILD_DIR)/Concuerror/bin/concuerror 3 | CONCUERROR_RUN := $(CONCUERROR) \ 4 | --treat_as_normal shutdown --treat_as_normal normal \ 5 | -x code -x code_server -x error_handler \ 6 | -pa $(BUILD_DIR)/concuerror+test/lib/snabbkaffe/ebin 7 | 8 | .PHONY: compile 9 | compile: 10 | rebar3 do dialyzer,eunit,ct 11 | 12 | concuerror = $(CONCUERROR_RUN) -f $(BUILD_DIR)/concuerror+test/lib/snabbkaffe/test/concuerror_tests.beam -t $(1) || \ 13 | { cat concuerror_report.txt; exit 1; } 14 | 15 | .PHONY: concuerror_test 16 | concuerror_test: $(CONCUERROR) 17 | rebar3 as concuerror eunit 18 | $(call concuerror,race_test) 19 | $(call concuerror,causality_test) 20 | $(call concuerror,fail_test) 21 | 22 | $(CONCUERROR): 23 | mkdir -p _build/ 24 | cd _build && git clone https://github.com/parapluu/Concuerror.git 25 | $(MAKE) -C _build/Concuerror/ 26 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Snabbkaffe 2 | 3 | [[https://travis-ci.org/klarna/snabbkaffe.svg?branch=master]] 4 | 5 | * Introduction 6 | 7 | This library provides functions for simple trace-based testing. 8 | 9 | It works like this: 10 | 11 | 1) Programmer manually instruments the code with trace points 12 | 2) Testcases are split in two parts: 13 | - *Run stage* where the program runs and emits event trace 14 | - *Check stage* where trace is collected and validated against the 15 | spec(s) 16 | 3) Trace points become ordinary log messages in the release build 17 | 18 | This approach can be used in a component test involving an ensemble of 19 | interacting processes. It has a few nice properties: 20 | 21 | + Checks can be separated from the program execution 22 | + Checks are independent from each other and fully composable 23 | + Trace contains complete history of the process execution, thus 24 | making certain types of concurrency bugs, like livelocks, easier to 25 | detect 26 | 27 | * Usage 28 | 29 | ** Instrumenting the code 30 | 31 | Code instrumentation is done manually by inserting =tp= macros at the 32 | points of interest: 33 | 34 | #+BEGIN_SRC erlang 35 | ?tp(my_server_state_change, #{old_state => foo, next_state => bar}) 36 | #+END_SRC 37 | 38 | The above line of code, when compiled in test mode, emits an event of 39 | /kind/ =my_server_state_change=, with some additional data specified 40 | in the second argument. Any event has a /kind/, which is an atom 41 | identifying type of the event. The second argument must be a map, and 42 | it can contain any data. 43 | 44 | In the release build this macro will become a [[https://github.com/tolbrino/hut/][hut]] log message with 45 | =debug= level. One can as well tune trace point log level: 46 | 47 | #+BEGIN_SRC erlang 48 | ?tp(notice, my_server_state_change, #{old_state => foo, next_state => bar}) 49 | #+END_SRC 50 | 51 | ** Collecting trace 52 | 53 | Trace collector process must be started before running the test. Full 54 | workflow looks like this: 55 | 56 | #+BEGIN_SRC erlang 57 | ok = snabbkaffe:start_trace(), 58 | Return = RunYourCode(), 59 | Trace = snabbkaffe:collect_trace(Timeout), 60 | snabbkaffe:stop(), 61 | RunCheck1(Return, Trace), 62 | RunCheck2(Return, Trace), 63 | ... 64 | RunCheckN(Return, Trace). 65 | #+END_SRC 66 | 67 | Note that =collect_trace= function is destructive: it cleans event 68 | buffer of the collector process. Its argument =Timeout= specifies how 69 | long the call will wait after the last event is received. Setting this 70 | parameter to a non-zero value is useful when /run stage/ starts some 71 | asynchronous tasks. By default =Timeout= is 0. 72 | 73 | ** Checking traces 74 | 75 | Trace is just a list of maps. Any standard function can work with 76 | it. Nonetheless, /Snabbkaffe/ comes with a few useful macros for trace 77 | analysis. 78 | 79 | Select events of certain kind: 80 | 81 | #+BEGIN_SRC erlang 82 | ?of_kind(foo, Trace) 83 | #+END_SRC 84 | 85 | Extract values of a certain event field(s) to a list: 86 | 87 | #+BEGIN_SRC erlang 88 | [1, 2] = ?projection(foo, [#{foo => 1, quux => 1}, #{foo => 2}]), 89 | 90 | [{1, 1}, {2, 2}] = ?projection([foo, bar], [#{foo => 1, bar => 1}, #{foo => 2, bar => 2}]) 91 | #+END_SRC 92 | 93 | Check that events occur in certain order, and throw an exception 94 | otherwise: 95 | 96 | #+BEGIN_SRC erlang 97 | ?causality( #{?snk_kind := foo, foo := _A} 98 | , #{?snk_kind := bar, foo := _A} 99 | , Trace 100 | ) 101 | #+END_SRC 102 | 103 | First argument of =?causality= macro matches an event that is called 104 | the /cause/, and the second one matches so called /effect/. The above 105 | example checks events of kind =bar= occur only after corresponding 106 | events of kind =foo=. 107 | 108 | This macro can be extended with a guard as well. Here guard checks 109 | that two events actually make up a pair: 110 | 111 | #+BEGIN_SRC erlang 112 | ?causality( #{?snk_kind := foo, foo := _A} 113 | , #{?snk_kind := bar, foo := _B} 114 | , _A + 1 =:= _B 115 | , Trace 116 | ) 117 | #+END_SRC 118 | 119 | There is a version of the above macro that checks that all cause have 120 | an effect: 121 | 122 | #+BEGIN_SRC erlang 123 | ?strict_causality( #{?snk_kind := foo, foo := _A} 124 | , #{?snk_kind := bar, foo := _A} 125 | , Trace 126 | ) 127 | #+END_SRC 128 | 129 | Otherwise it works just like =?causality=. 130 | 131 | Both =?causality= and =?strict_causality= are actually based on a more 132 | powerful =?find_pairs= macro that is invoked like this: 133 | 134 | #+BEGIN_SRC erlang 135 | ?find_pairs( Strict 136 | , MatchCause 137 | , MatchEffect 138 | [, Guard] 139 | , Trace 140 | ) 141 | #+END_SRC 142 | 143 | where =Strict= is a boolean that determines whether events that matched 144 | as =Effect= may precede their cause. 145 | 146 | It returns a list of tuples of type =snabbkaffe:maybe_pair/0= that is 147 | defined like that: 148 | 149 | #+BEGIN_SRC erlang 150 | -type maybe_pair() :: {pair, event(), event()} 151 | | {singleton, event()}. 152 | #+END_SRC 153 | 154 | ** Gathering it all together 155 | 156 | =?check_trace= is a convenience wrapper that starts the trace 157 | collector process, executes /run stage/, collects traces and then 158 | executes /check stage/: 159 | 160 | #+BEGIN_SRC erlang 161 | ?check_trace(begin 162 | RunStage 163 | end, 164 | fun(ReturnValue, Trace) -> 165 | CheckStage 166 | end). 167 | #+END_SRC 168 | 169 | There is an extended version of this macro that takes additional 170 | configuration as the first argument: 171 | 172 | #+BEGIN_SRC erlang 173 | ?check_trace(#{timeout => Timeout, bucket => Bucket}, 174 | begin 175 | RunStage 176 | end, 177 | fun(ReturnValue, Trace) -> 178 | CheckStage 179 | end). 180 | #+END_SRC 181 | 182 | or: 183 | 184 | #+BEGIN_SRC erlang 185 | ?check_trace(Bucket, 186 | begin 187 | RunStage 188 | end, 189 | fun(ReturnValue, Trace) -> 190 | CheckStage 191 | end). 192 | #+END_SRC 193 | 194 | =Bucket= is a parameter used for benchmarking, more on that later. 195 | 196 | ** Blocking execution of testcase until certain event is emitted 197 | 198 | Even though philosophy of this library lies in separation of run and 199 | verify stages, sometimes the former needs to be somewhat aware of the 200 | events. For example, the testcase may need to wait for asynchronous 201 | initialization of some resource. 202 | 203 | In this case =?block_until= macro should be used. It allows the 204 | testcase to peek into the trace. Example usage: 205 | 206 | #+BEGIN_SRC erlang 207 | ?block_until(#{?snk_kind := Kind}, Timeout, BackInTime) 208 | #+END_SRC 209 | 210 | Note: it's tempting to use this macro to check the result of some 211 | asynchronous action, like this: 212 | 213 | #+BEGIN_SRC erlang 214 | {ok, Pid} = foo:async_init(), 215 | {ok, Event} = ?block_until(#{?snk_kind := foo_init, pid := Pid}), 216 | do_stuff(Pid) 217 | #+END_SRC 218 | 219 | However it's not a good idea, because the event can be emitted before 220 | =?block_until= has a chance to run. Use the following macro to avoid 221 | this race condition: 222 | 223 | #+BEGIN_SRC 224 | {{ok, Pid}, {ok, Event}} = ?wait_async_action( foo:async_init() 225 | , #{?snk_kind := foo_init, pid := Pid} 226 | ), 227 | do_stuff(Pid) 228 | #+END_SRC 229 | 230 | ** Declarative fault injection 231 | 232 | Any trace point can also be used to inject crashes into the 233 | system. This is extremely useful for testing fault-tolerance 234 | properties of the system and tuning the supervision trees. This is 235 | done using =?inject_crash= macro, like in the below example: 236 | 237 | #+BEGIN_SRC erlang 238 | FaultId = ?inject_crash( #{?snk_kind := some_kind, value := 42} % Pattern for matching trace points 239 | , snabbkaffe_nemesis:always_crash() % Fault scenario 240 | , notmyday % Error reason 241 | ) 242 | #+END_SRC 243 | 244 | Running this command in the run stage of the testcase will ensure that 245 | every time the system tries to emit a trace event matching the 246 | pattern, the system will crash with a reason =notmyday=, and emit a 247 | trace event of kind =snabbkaffe_crash=. 248 | 249 | First argument of the macro is a pattern that is used for matching 250 | trace events. Second argument is a "fault scenario", that determines 251 | how often the system should fail. The following scenarios are 252 | implemented: 253 | 254 | + =snabbkaffe_nemesis:always_crash()= -- always crash, emulates 255 | unrecoverable errors 256 | + =snabbkaffe_nemesis:recover_after(N)= -- crash =N= times, and then 257 | proceed normally, emulates recoverable errors 258 | + =snabbkaffe_nemesis:random_crash(P)= -- crash in a pseudo-random 259 | pattern with probability =P=, emulates an unreliable resource 260 | + =snabbkaffe_nemesis:periodic_crash(Period, DutyCycle, Phase)= -- 261 | crash periodically, like this: 262 | =[ok, ok, ok, crash, crash, ok, ok, ok, crash, crash|...]= 263 | - =Period= is an integer that specifies period of the crash-recover 264 | cycle 265 | - =DutyCycle= is a float in =[0..1]= range, that specifies relative 266 | amount of time when the trace point is /not/ crushing. (For 267 | example, 1 means the system doesn't crash, and 0 means it always 268 | crashes) 269 | - =Phase= is a float in =[0..2*math:pi()]= range that allows to 270 | shift the phase of the periodic scenario 271 | 272 | Finally, the third argument is a crash reason. It is optional, and 273 | defaults to the atom =notmyday=. 274 | 275 | Please note that fault scenarios work independently for each /trace 276 | point/. E.g. if there are two trace point that both match the same 277 | fault injection pattern with =recover_after= scenario, they will 278 | recover at different times. 279 | 280 | Later =snabbkaffe_nemesis:fix_crash(FaultId)= call can be used to 281 | delete the injected crash. 282 | 283 | ** PropER integration 284 | 285 | There are two useful macros for running /snabbkaffe/ together with [[https://proper-testing.github.io/][propER]]: 286 | 287 | #+BEGIN_SRC erlang 288 | Config = [{proper, #{ numtests => 100 289 | , timeout => 5000 290 | , max_size => 100 291 | }}, ...], 292 | ?run_prop(Config, PROP) 293 | #+END_SRC 294 | 295 | =Config= parameter should be a proplist or a map, that (optionally) 296 | contains =proper= key. It can be used to pass different parameters to 297 | proper. Snabbkaffe will fall back to the default values (shown above) 298 | when parameter is absent. 299 | 300 | =PROP= is a proper spec that looks something like this: 301 | 302 | #+BEGIN_SRC erlang 303 | ?FORALL({Ret, L}, {term(), list()}, 304 | ?check_trace( 305 | begin 306 | RunStage 307 | end, 308 | fun(Return, Trace) -> 309 | CheckStage 310 | end)) 311 | #+END_SRC 312 | 313 | There is another macro for the most common type of proper checks where 314 | property is a simple =?FORALL= clause (like in the above example). 315 | 316 | #+BEGIN_SRC erlang 317 | ?forall_trace({Ret, L}, {term(), list()}, 318 | begin 319 | RunStage 320 | end, 321 | fun(Return, Trace) -> 322 | CheckStage 323 | end) 324 | #+END_SRC 325 | 326 | It combines =?FORALL= and =?run_prop=. 327 | 328 | ** Concuerror support 329 | 330 | Snabbkaffe has (highly) experimental support for [[https://concuerror.com][Concuerror]]. It 331 | requires recompiling this library with special options, so creating a 332 | special build profile is recommended. This can be done by adding the 333 | following code to the =rebar.config=: 334 | 335 | #+BEGIN_SRC erlang 336 | {profiles, 337 | [ {concuerror, 338 | [ {overrides, 339 | [{add, [{erl_opts, 340 | [ {d, 'CONCUERROR'} 341 | , {d, 'HUT_IOFORMAT'} 342 | ]}]}]} 343 | ]} 344 | ]}. 345 | #+END_SRC 346 | 347 | Run concuerror with the following flags: 348 | 349 | #+BEGIN_SRC bash 350 | $(CONCUERROR) --treat_as_normal shutdown --treat_as_normal normal \ 351 | -x code -x code_server -x error_handler \ 352 | --pa $(BUILD_DIR)/concuerror+test/lib/snabbkaffe/ebin 353 | #+END_SRC 354 | 355 | P.S. Again, this feature is experimental, use at your own risk. 356 | 357 | * Benchmarking 358 | 359 | /Snabbkaffe/ automatically adds timestamps to the events, which makes 360 | it a very unscientific benchmarking library. 361 | 362 | There is a family of functions for reporting metric data. 363 | 364 | Report a scalar metric called =my_metric1=: 365 | 366 | #+BEGIN_SRC erlang 367 | snabbkaffe:push_stat(my_metric1, 42), 368 | snabbkaffe:push_stats(my_metric1, [42, 43, 42]), 369 | %% Or even: 370 | snabbkaffe:push_stats(my_metric1, [{pair, Event1, Event2}, {pair, Event3, Event4}, ...]), 371 | #+END_SRC 372 | 373 | Sometimes it is entertaining to see how metric value depends on the 374 | size of the input data: 375 | 376 | #+BEGIN_SRC erlang 377 | snabbkaffe:push_stat(my_metric1, SizeOfData, 42), 378 | snabbkaffe:push_stats(my_metric1, SizeOfData, [42, 43, 42]) 379 | #+END_SRC 380 | 381 | Metrics can be reported by calling =snabbkaffe:analyze_statistics/0= 382 | function that prints statistics for each reported metric, like in the 383 | above example: 384 | 385 | #+BEGIN_EXAMPLE 386 | ------------------------------- 387 | foo_bar statistics: 388 | [{min,9.999999999999999e-6}, 389 | {max,9.999999999999999e-6}, 390 | {arithmetic_mean,1.000000000000002e-5}, 391 | {geometric_mean,1.0000000000000123e-5}, 392 | {harmonic_mean,9.999999999999997e-6}, 393 | {median,9.999999999999999e-6}, 394 | {variance,4.174340734454146e-40}, 395 | {standard_deviation,2.0431203426264804e-20}, 396 | {skewness,-0.9850375627355535}, 397 | {kurtosis,-2.0199000000000003}, 398 | {percentile,[{50,9.999999999999999e-6}, 399 | {75,9.999999999999999e-6}, 400 | {90,9.999999999999999e-6}, 401 | {95,9.999999999999999e-6}, 402 | {99,9.999999999999999e-6}, 403 | {999,9.999999999999999e-6}]}, 404 | {histogram,[{9.999999999999999e-6,100}]}, 405 | {n,100}] 406 | 407 | Statisitics of test 408 | 100.479087 ^ * 409 | | * 410 | | * 411 | | * 412 | | 413 | | * 414 | | * 415 | | 416 | | * 417 | | * 418 | | * 419 | | 420 | | * 421 | | * 422 | 0 +---------------------------------------------------------------------> 423 | 0 1100 424 | 425 | N min max avg 426 | 110 1.23984e+0 1.09774e+1 5.97581e+0 427 | 209 1.10121e+1 2.08884e+1 1.60011e+1 428 | 308 2.13004e+1 3.09071e+1 2.60224e+1 429 | 407 3.10212e+1 4.09074e+1 3.59904e+1 430 | 506 4.10095e+1 5.09904e+1 4.60456e+1 431 | 605 5.11370e+1 6.08557e+1 5.60354e+1 432 | 704 6.10493e+1 7.09071e+1 6.59642e+1 433 | 803 7.11237e+1 8.07733e+1 7.59588e+1 434 | 902 8.10944e+1 9.09766e+1 8.60179e+1 435 | 1001 9.10459e+1 9.99404e+1 9.54548e+1 436 | 1100 1.00004e+2 1.00939e+2 1.00479e+2 437 | #+END_EXAMPLE 438 | 439 | Note: =?run_prop= does this automatically. 440 | -------------------------------------------------------------------------------- /include/ct_boilerplate.hrl: -------------------------------------------------------------------------------- 1 | -export([init_per_testcase/2, end_per_testcase/2, all/0]). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | -include_lib("stdlib/include/assert.hrl"). 5 | -include_lib("proper/include/proper.hrl"). 6 | -include_lib("snabbkaffe/include/snabbkaffe.hrl"). 7 | 8 | init_per_testcase(TestCase, Config) -> 9 | case os:getenv("KEEP_CT_LOGGING") of 10 | false -> 11 | ?log(notice, asciiart:visible($%, "Running ~p", [TestCase])); 12 | _ -> 13 | ok 14 | end, 15 | Config1 = try apply(?MODULE, common_init_per_testcase, [TestCase, Config]) 16 | catch 17 | error:undef -> Config 18 | end, 19 | Config2 = try apply(?MODULE, TestCase, [{init, Config1}]) 20 | catch 21 | error:function_clause -> Config1 22 | end, 23 | ok = snabbkaffe:start_trace(), 24 | Config2. 25 | 26 | end_per_testcase(TestCase, Config) -> 27 | try apply(?MODULE, TestCase, [{'end', Config}]) 28 | catch 29 | error:function_clause -> ok 30 | end, 31 | try apply(?MODULE, common_end_per_testcase, [TestCase, Config]) 32 | catch 33 | error:undef -> ok 34 | end, 35 | snabbkaffe:analyze_statistics(), 36 | snabbkaffe:stop(), 37 | case os:getenv("KEEP_CT_LOGGING") of 38 | false -> 39 | ?log(notice, asciiart:visible($%, "End of ~p", [TestCase])); 40 | _ -> 41 | ok 42 | end, 43 | ok. 44 | 45 | all() -> 46 | application:ensure_all_started(snabbkaffe), 47 | snabbkaffe:mk_all(?MODULE). 48 | -------------------------------------------------------------------------------- /include/snabbkaffe.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(SNABBKAFFE_HRL). 2 | -define(SNABBKAFFE_HRL, true). 3 | 4 | -include_lib("hut/include/hut.hrl"). 5 | 6 | -ifdef(TEST). 7 | -ifndef(SNK_COLLECTOR). 8 | -define(SNK_COLLECTOR, true). 9 | -endif. %% SNK_COLLECTOR 10 | -endif. %% TEST 11 | 12 | -define(snk_kind, '$kind'). %% "$" will make kind field go first when maps are printed. 13 | 14 | -ifdef(SNK_COLLECTOR). 15 | 16 | -define(panic(KIND, ARGS), 17 | error({panic, (ARGS) #{?snk_kind => (KIND)}})). 18 | 19 | %% Dirty hack: we use reference to a local function as a key that can 20 | %% be used to refer error injection points. This works, because all 21 | %% invokations of this macro create a new fun object with unique id. 22 | -define(__snkStaticUniqueToken, fun() -> ok end). 23 | 24 | -define(maybe_crash(KIND, DATA), 25 | snabbkaffe_nemesis:maybe_crash(KIND, DATA#{?snk_kind => KIND})). 26 | 27 | -define(maybe_crash(DATA), 28 | snabbkaffe_nemesis:maybe_crash(?__snkStaticUniqueToken, DATA)). 29 | 30 | -define(tp(LEVEL, KIND, EVT), 31 | (fun() -> 32 | ?maybe_crash(EVT #{?snk_kind => KIND}), 33 | snabbkaffe_collector:tp(KIND, EVT) 34 | end)()). 35 | 36 | -define(tp(KIND, EVT), ?tp(debug, KIND, EVT)). 37 | 38 | -define(of_kind(KIND, TRACE), 39 | snabbkaffe:events_of_kind(KIND, TRACE)). 40 | 41 | -define(projection(FIELDS, TRACE), 42 | snabbkaffe:projection(FIELDS, TRACE)). 43 | 44 | -define(snk_int_match_arg(ARG), 45 | fun(__SnkArg) -> 46 | case __SnkArg of 47 | ARG -> true; 48 | _ -> false 49 | end 50 | end). 51 | 52 | -define(snk_int_match_arg2(M1, M2, GUARD), 53 | fun(__SnkArg1, __SnkArg2) -> 54 | case __SnkArg1 of 55 | M1 -> 56 | case __SnkArg2 of 57 | M2 -> (GUARD); 58 | _ -> false 59 | end; 60 | _ -> false 61 | end 62 | end). 63 | 64 | -define(find_pairs(STRICT, M1, M2, GUARD, TRACE), 65 | snabbkaffe:find_pairs( STRICT 66 | , ?snk_int_match_arg(M1) 67 | , ?snk_int_match_arg(M2) 68 | , ?snk_int_match_arg2(M1, M2, GUARD) 69 | , (TRACE) 70 | )). 71 | 72 | -define(find_pairs(STRICT, M1, M2, TRACE), 73 | ?find_pairs(STRICT, M1, M2, true, TRACE)). 74 | 75 | -define(causality(M1, M2, GUARD, TRACE), 76 | snabbkaffe:causality( false 77 | , ?snk_int_match_arg(M1) 78 | , ?snk_int_match_arg(M2) 79 | , ?snk_int_match_arg2(M1, M2, GUARD) 80 | , (TRACE) 81 | )). 82 | 83 | -define(causality(M1, M2, Trace), 84 | ?causality(M1, M2, true, Trace)). 85 | 86 | -define(strict_causality(M1, M2, GUARD, TRACE), 87 | snabbkaffe:causality( true 88 | , ?snk_int_match_arg(M1) 89 | , ?snk_int_match_arg(M2) 90 | , ?snk_int_match_arg2(M1, M2, GUARD) 91 | , (TRACE) 92 | )). 93 | 94 | -define(strict_causality(M1, M2, TRACE), 95 | ?strict_causality(M1, M2, true, TRACE)). 96 | 97 | -define(pair_max_depth(PAIRS), snabbkaffe:pair_max_depth(PAIRS)). 98 | 99 | -define(projection_complete(FIELD, TRACE, L), 100 | snabbkaffe:projection_complete(FIELD, TRACE, L)). 101 | 102 | -define(projection_is_subset(FIELD, TRACE, L), 103 | snabbkaffe:projection_is_subset(FIELD, TRACE, L)). 104 | 105 | -define(check_trace(BUCKET, RUN, CHECK), 106 | (case snabbkaffe:run( (fun() -> BUCKET end)() 107 | , fun() -> RUN end 108 | , begin CHECK end 109 | ) of 110 | true -> true; 111 | ok -> true; 112 | {error, {panic, CrashKind, Args}} -> ?panic(CrashKind, Args); 113 | A -> ?panic("Unexpected result", #{result => A}) 114 | end)). 115 | 116 | -define(check_trace(RUN, CHECK), 117 | ?check_trace(#{}, RUN, CHECK)). 118 | 119 | -define(run_prop(CONFIG, PROPERTY), 120 | (fun() -> 121 | __SnkTimeout = snabbkaffe:get_cfg([proper, timeout], CONFIG, 5000), 122 | __SnkNumtests = snabbkaffe:get_cfg([proper, numtests], CONFIG, 100), 123 | __SnkMaxSize = snabbkaffe:get_cfg([proper, max_size], CONFIG, 100), 124 | __SnkColors = case os:getenv("TERM") of 125 | "dumb" -> [nocolors]; 126 | _ -> [] 127 | end, 128 | __SnkRet = proper:quickcheck( 129 | ?TIMEOUT( __SnkTimeout 130 | , begin 131 | ?log(info, asciiart:visible($', "Runnung ~s", [??PROPERTY])), 132 | PROPERTY 133 | end) 134 | , [ {numtests, __SnkNumtests} 135 | , {max_size, __SnkMaxSize} 136 | , {on_output, fun snabbkaffe:proper_printout/2} 137 | ] ++ __SnkColors 138 | ), 139 | case __SnkRet of 140 | true -> 141 | ok; 142 | Error -> 143 | ?log(critical, asciiart:visible($!, "Proper test failed: ~p", [Error])), 144 | exit(fail) 145 | end 146 | end)()). 147 | 148 | -define(forall_trace(XS, XG, BUCKET, RUN, CHECK), 149 | ?FORALL(XS, XG, ?check_trace(BUCKET, RUN, CHECK))). 150 | 151 | -define(forall_trace(XS, XG, RUN, CHECK), 152 | ?forall_trace(XS, XG, #{}, RUN, CHECK)). 153 | 154 | -define(give_or_take(EXPECTED, DEVIATION, VALUE), 155 | (fun() -> 156 | __SnkValue = (VALUE), 157 | __SnkExpected = (EXPECTED), 158 | __SnkDeviation = (DEVIATION), 159 | case catch erlang:abs(__SnkValue - __SnkExpected) of 160 | __SnkDelta when is_integer(__SnkDelta), 161 | __SnkDelta =< __SnkDeviation -> 162 | true; 163 | _ -> 164 | erlang:error({assertGiveOrTake, 165 | [ {module, ?MODULE} 166 | , {line, ?LINE} 167 | , {expected, __SnkExpected} 168 | , {value, __SnkValue} 169 | , {expression, (??VALUE)} 170 | , {max_deviation, __SnkDeviation} 171 | ]}) 172 | end 173 | end)()). 174 | 175 | -define(retry(TIMEOUT, N, FUN), snabbkaffe:retry(TIMEOUT, N, fun() -> FUN end)). 176 | 177 | -define(block_until(PATTERN, TIMEOUT, BACK_IN_TIME), 178 | snabbkaffe:block_until(?snk_int_match_arg(PATTERN), (TIMEOUT), (BACK_IN_TIME))). 179 | 180 | -define(wait_async_action(ACTION, PATTERN, TIMEOUT), 181 | snabbkaffe:wait_async_action( fun() -> ACTION end 182 | , ?snk_int_match_arg(PATTERN) 183 | , (TIMEOUT) 184 | )). 185 | 186 | -define(wait_async_action(ACTION, PATTERN), 187 | ?wait_async_action(ACTION, PATTERN, infinity)). 188 | 189 | -define(block_until(PATTERN, TIMEOUT), 190 | ?block_until(PATTERN, (TIMEOUT), infinity)). 191 | 192 | -define(block_until(MATCH), 193 | ?block_until(MATCH, infinity)). 194 | 195 | -define(split_trace_at(PATTERN, TRACE), 196 | lists:splitwith(?snk_int_match_arg(PATTERN), (TRACE))). 197 | 198 | -define(splitl_trace(PATTERN, TRACE), 199 | snabbkaffe:splitl(?snk_int_match_arg(PATTERN), (TRACE))). 200 | 201 | -define(splitr_trace(PATTERN, TRACE), 202 | snabbkaffe:splitr(?snk_int_match_arg(PATTERN), (TRACE))). 203 | 204 | -define(inject_crash(PATTERN, STRATEGY, REASON), 205 | snabbkaffe_nemesis:inject_crash( ?snk_int_match_arg(PATTERN) 206 | , (STRATEGY) 207 | , (REASON) 208 | )). 209 | 210 | -define(inject_crash(PATTERN, STRATEGY), 211 | ?inject_crash(PATTERN, STRATEGY, notmyday)). 212 | 213 | -else. %% SNK_COLLECTOR 214 | 215 | -define(tp(LEVEL, KIND, EVT), ?slog(LEVEL, EVT #{?snk_kind => KIND})). 216 | 217 | -define(tp(KIND, EVT), ?tp(debug, KIND, EVT)). 218 | 219 | -define(maybe_crash(KIND, DATA), ok). 220 | 221 | -define(maybe_crash(DATA), ok). 222 | 223 | -endif. %% SNK_COLLECTOR 224 | -endif. %% SNABBKAFFE_HRL 225 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | {erl_opts, 3 | [debug_info]}. 4 | 5 | {ct_opts, 6 | [{enable_builtin_hooks, false}]}. 7 | %{ct_readable, false}. 8 | 9 | {minimum_otp_vsn, "19.0"}. 10 | 11 | {eunit_opts, 12 | [verbose]}. 13 | 14 | {deps, 15 | [ {hut, "1.3.0"} 16 | ]}. 17 | 18 | {profiles, 19 | [ {test, 20 | [{deps, [ {proper, "1.3.0"} 21 | , {bear, "0.8.7"} 22 | ]}]} 23 | , {dev, 24 | [{plugins, [rebar3_hex]}]} 25 | , {concuerror, 26 | [{erl_opts, [ {d, 'CONCUERROR'} 27 | , {d, 'HUT_IOFORMAT'} 28 | ]}]} 29 | ]}. 30 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.2.0", 2 | [{<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},0}, 3 | {<<"hut">>,{pkg,<<"hut">>,<<"1.3.0">>},0}, 4 | {<<"proper">>,{pkg,<<"proper">>,<<"1.3.0">>},0}]}. 5 | [ 6 | {pkg_hash,[ 7 | {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, 8 | {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, 9 | {<<"proper">>, <<"C1ACD51C51DA17A2FE91D7A6FC6A0C25A6A9849D8DC77093533109D1218D8457">>}]}, 10 | {pkg_hash_ext,[ 11 | {<<"bear">>, <<"534217DCE6A719D59E54FB0EB7A367900DBFC5F85757E8C1F94269DF383F6D9B">>}, 12 | {<<"hut">>, <<"7E15D28555D8A1F2B5A3A931EC120AF0753E4853A4C66053DB354F35BF9AB563">>}, 13 | {<<"proper">>, <<"4AA192FCCDDD03FDBE50FEF620BE9D4D2F92635B54F55FB83AEC185994403CBC">>}]} 14 | ]. 15 | -------------------------------------------------------------------------------- /src/asciiart.erl: -------------------------------------------------------------------------------- 1 | %% Copyright 2019-2020 Klarna Bank AB 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | -module(asciiart). 15 | 16 | -export([ init/0 17 | , dimensions/1 18 | , render/1 19 | , char/3 20 | , char/2 21 | , line/4 22 | , line/3 23 | , string/4 24 | , string/3 25 | , plot/1 26 | , plot/2 27 | , draw/2 28 | , draw/1 29 | , visible/3 30 | ]). 31 | 32 | %%==================================================================== 33 | %% Types 34 | %%==================================================================== 35 | 36 | -type vector() :: {integer(), integer()}. 37 | 38 | -type cont() :: fun((canvas()) -> canvas()). 39 | 40 | -opaque canvas() :: #{vector() => char()}. 41 | 42 | -type plot_data() :: [{char(), [{float(), float()}]}]. 43 | 44 | -export_type([vector/0, canvas/0]). 45 | 46 | -define(epsilon, 1.0e-6). 47 | 48 | %%==================================================================== 49 | %% API functions 50 | %%==================================================================== 51 | 52 | -spec init() -> canvas(). 53 | init() -> 54 | #{}. 55 | 56 | -spec render(canvas()) -> iolist(). 57 | render(Cnv) -> 58 | {{Xm, Ym}, {XM, YM}} = dimensions(Cnv), 59 | [[[maps:get({X, Y}, Cnv, $ ) || X <- lists:seq(Xm, XM)], $\n] 60 | || Y <- lists:reverse(lists:seq(Ym, YM))]. 61 | 62 | -spec draw([cont()], canvas()) -> canvas(). 63 | draw(Ops, Cnv) -> 64 | lists:foldl(fun(F, Acc) -> F(Acc) end, Cnv, Ops). 65 | 66 | -spec draw([cont()]) -> canvas(). 67 | draw(Ops) -> 68 | draw(Ops, init()). 69 | 70 | -spec dimensions(canvas()) -> {vector(), vector()}. 71 | dimensions(Cnv) -> 72 | Fun = fun({X, Y}, _, {{Xm, Ym}, {XM, YM}}) -> 73 | { {min(X, Xm), min(Y, Ym)} 74 | , {max(X, XM), max(Y, YM)} 75 | } 76 | end, 77 | maps:fold(Fun, {{1, 1}, {1, 1}}, Cnv). 78 | 79 | -spec char(canvas(), vector(), char()) -> canvas(). 80 | char(Cnv, Pos, Char) -> 81 | Cnv #{Pos => Char}. 82 | 83 | -spec char(vector(), char()) -> cont(). 84 | char(Pos, Char) -> 85 | fun(Cnv) -> char(Cnv, Pos, Char) end. 86 | 87 | -spec line(canvas(), vector(), vector(), char()) -> canvas(). 88 | line(Cnv, {X1, Y1}, {X2, Y2}, Char) -> 89 | X = X2 - X1, 90 | Y = Y2 - Y1, 91 | N = max(1, max(abs(X), abs(Y))), 92 | lists:foldl( fun(Pos, Cnv) -> char(Cnv, Pos, Char) end 93 | , Cnv 94 | , [{ X1 + round(X * I / N) 95 | , Y1 + round(Y * I / N) 96 | } || I <- lists:seq(0, N)] 97 | ). 98 | 99 | -spec line(vector(), vector(), char()) -> cont(). 100 | line(F, T, C) -> 101 | fun(Cnv) -> line(Cnv, F, T, C) end. 102 | 103 | -spec string(canvas(), vector(), string(), left | right) -> canvas(). 104 | string(Cnv, _, [], _) -> 105 | Cnv; 106 | string(Cnv, {X, Y}, String, Direction) -> 107 | XL = case Direction of 108 | right -> 109 | lists:seq(X, X + length(String) - 1); 110 | left -> 111 | lists:seq(X - length(String) + 1, X) 112 | end, 113 | L = lists:zip(XL, String), 114 | lists:foldl( fun({X, Char}, Cnv) -> 115 | char(Cnv, {X, Y}, Char) 116 | end 117 | , Cnv 118 | , L 119 | ). 120 | 121 | -spec string(vector(), string(), left | right) -> cont(). 122 | string(Pos, Str, Dir) -> 123 | fun(Cnv) -> string(Cnv, Pos, Str, Dir) end. 124 | 125 | -spec plot(plot_data()) -> canvas(). 126 | plot(Datapoints) -> 127 | plot(Datapoints, #{}). 128 | 129 | -spec plot(plot_data(), map()) -> canvas(). 130 | plot(Datapoints, Config) -> 131 | AllDatapoints = lists:append([L || {_, L} <- Datapoints]), 132 | {XX, YY} = lists:unzip(AllDatapoints), 133 | Xm = bound(min, Config, XX), 134 | XM = bound(max, Config, XX), 135 | Ym = bound(min, Config, YY), 136 | YM = bound(max, Config, YY), 137 | DX = max(?epsilon, XM - Xm), 138 | DY = max(?epsilon, YM - Ym), 139 | %% Dimensions of the plot: 140 | AspectRatio = maps:get(aspect_ratio, Config, 0.2), 141 | Width = max(length(Datapoints) * 2, 70), 142 | Height = round(Width * AspectRatio), 143 | Frame = {{Xm, Ym}, {Width / DX, Height / DY}}, 144 | %% Draw axis 145 | Cnv0 = draw( [ %% Vertical: 146 | line({0, 0}, {0, Height - 1}, $|) 147 | , char({0, Height}, $^) 148 | %% Labels: 149 | , string({-2, 0}, print_num(Ym), left) 150 | , string({-2, Height}, print_num(YM), left) 151 | %% Horizontal: 152 | , line({0, 0}, {Width - 1, 0}, $-) 153 | , char({Width, 0}, $>) 154 | , char({0, 0}, $+) 155 | %% Labels 156 | , string({0, -1}, print_num(Xm), right) 157 | , string({Width, -1}, print_num(XM), left) 158 | ] 159 | , init() 160 | ), 161 | lists:foldl( fun({Char, Data}, Acc) -> 162 | draw_datapoints(Frame, Char, Data, Acc) 163 | end 164 | , Cnv0 165 | , Datapoints 166 | ). 167 | 168 | draw_datapoints(Frame, Char, Data, Acc) -> 169 | lists:foldl( fun(Coords, Acc) -> 170 | char(Acc, plot_coord(Frame, Coords), Char) 171 | end 172 | , Acc 173 | , Data 174 | ). 175 | 176 | print_num(Num) when is_integer(Num) -> 177 | integer_to_list(Num); 178 | print_num(Num) -> 179 | lists:flatten(io_lib:format("~.6..f", [Num])). 180 | 181 | plot_coord({{Xm, Ym}, {SX, SY}}, {X, Y}) -> 182 | {round((X - Xm) * SX), round((Y - Ym) * SY)}. 183 | 184 | bound(Fun, Cfg, L) -> 185 | N = case L of 186 | [] -> 0; 187 | _ -> lists:Fun(L) 188 | end, 189 | case maps:get(include_zero, Cfg, true) of 190 | true -> 191 | erlang:Fun(0, N); 192 | false -> 193 | N 194 | end. 195 | 196 | -spec visible(char(), string(), [term()]) -> iolist(). 197 | visible(Char, Fmt, Args) -> 198 | Str = lines(lists:flatten(io_lib:format(Fmt, Args))), 199 | Width = max(79, lists:max([length(I) || I <- Str])) + 1, 200 | N = length(Str), 201 | Text = [string({4, Y}, S, right) 202 | || {Y, S} <- lists:zip( lists:seq(1, N) 203 | , lists:reverse(Str) 204 | )], 205 | Cnv = draw([ asciiart:line({1, -1}, {Width, -1}, Char) 206 | , asciiart:line({1, N + 2}, {Width, N + 2}, Char) 207 | , asciiart:line({1, 0}, {1, N + 1}, Char) 208 | , asciiart:line({2, 0}, {2, N + 1}, Char) 209 | , asciiart:line({Width - 1, 0}, {Width - 1, N + 1}, Char) 210 | , asciiart:line({Width, 0}, {Width, N + 1}, Char) 211 | ] ++ Text), 212 | [$\n, render(Cnv), $\n]. 213 | 214 | -spec lines(string()) -> [string()]. 215 | lines(Str) -> 216 | re:split(Str, "\n", [{return, list}]). 217 | -------------------------------------------------------------------------------- /src/snabbkaffe.app.src: -------------------------------------------------------------------------------- 1 | {application,snabbkaffe, 2 | [{description,"Simple trace-based testing framework"}, 3 | {vsn,"git"}, 4 | {registered,[]}, 5 | {applications,[kernel,stdlib,hut]}, 6 | {env,[]}, 7 | {modules,[]}, 8 | {licenses,["Apache 2.0"]}, 9 | {links,[{"Github","https://github.com/k32/snabbkaffe"}]}]}. 10 | -------------------------------------------------------------------------------- /src/snabbkaffe.erl: -------------------------------------------------------------------------------- 1 | %% Copyright 2019-2020 Klarna Bank AB 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | -module(snabbkaffe). 15 | 16 | -ifndef(SNK_COLLECTOR). 17 | -define(SNK_COLLECTOR, true). 18 | -endif. 19 | 20 | -include("snabbkaffe_internal.hrl"). 21 | 22 | %% API exports 23 | -export([ start_trace/0 24 | , stop/0 25 | , collect_trace/0 26 | , collect_trace/1 27 | , block_until/2 28 | , block_until/3 29 | , wait_async_action/3 30 | , push_stat/2 31 | , push_stat/3 32 | , push_stats/2 33 | , push_stats/3 34 | , analyze_statistics/0 35 | , get_stats/0 36 | , run/3 37 | , get_cfg/3 38 | , fix_ct_logging/0 39 | , splitl/2 40 | , splitr/2 41 | , proper_printout/2 42 | ]). 43 | 44 | -export([ events_of_kind/2 45 | , projection/2 46 | , erase_timestamps/1 47 | , find_pairs/5 48 | , causality/5 49 | , unique/1 50 | , projection_complete/3 51 | , projection_is_subset/3 52 | , pair_max_depth/1 53 | , inc_counters/2 54 | , dec_counters/2 55 | , strictly_increasing/1 56 | ]). 57 | 58 | -export([ mk_all/1 59 | , retry/3 60 | ]). 61 | 62 | %%==================================================================== 63 | %% Types 64 | %%==================================================================== 65 | 66 | -type kind() :: atom(). 67 | 68 | -type metric() :: atom(). 69 | 70 | -type timestamp() :: integer(). 71 | 72 | -type event() :: 73 | #{ ?snk_kind := kind() 74 | , _ => _ 75 | }. 76 | 77 | -type timed_event() :: 78 | #{ ?snk_kind := kind() 79 | , ts := timestamp() 80 | , _ => _ 81 | }. 82 | 83 | -type trace() :: [event()]. 84 | 85 | -type maybe_pair() :: {pair, timed_event(), timed_event()} 86 | | {singleton, timed_event()}. 87 | 88 | -type maybe(A) :: {just, A} | nothing. 89 | 90 | -type run_config() :: 91 | #{ bucket => integer() 92 | , timeout => integer() 93 | }. 94 | 95 | -type predicate() :: fun((event()) -> boolean()). 96 | 97 | -export_type([ kind/0, timestamp/0, event/0, timed_event/0, trace/0 98 | , maybe_pair/0, maybe/1, metric/0, run_config/0, predicate/0 99 | ]). 100 | 101 | %%==================================================================== 102 | %% Macros 103 | %%==================================================================== 104 | 105 | %%==================================================================== 106 | %% API functions 107 | %%==================================================================== 108 | 109 | -spec collect_trace() -> trace(). 110 | collect_trace() -> 111 | collect_trace(0). 112 | 113 | -spec collect_trace(integer()) -> trace(). 114 | collect_trace(Timeout) -> 115 | snabbkaffe_collector:get_trace(Timeout). 116 | 117 | %% @equiv block_until(Predicate, Timeout, 100) 118 | -spec block_until(predicate(), timeout()) -> {ok, event()} | timeout. 119 | block_until(Predicate, Timeout) -> 120 | block_until(Predicate, Timeout, 100). 121 | 122 | -spec wait_async_action(fun(() -> Return), predicate(), timeout()) -> 123 | {Return, {ok, event()} | timeout}. 124 | wait_async_action(Action, Predicate, Timeout) -> 125 | Ref = make_ref(), 126 | Self = self(), 127 | Callback = fun(Result) -> 128 | Self ! {Ref, Result} 129 | end, 130 | snabbkaffe_collector:notify_on_event(Predicate, Timeout, Callback), 131 | Return = Action(), 132 | receive 133 | {Ref, Result} -> 134 | {Return, Result} 135 | end. 136 | 137 | %% @doc Block execution of the run stage of a testcase until an event 138 | %% matching `Predicate' is received or until `Timeout'. 139 | %% 140 | %% Note: since the most common use case for this function is 141 | %% the following: 142 | %% 143 | %% ```trigger_produce_event_async(), 144 | %% snabbkaffe:block_until(MatchEvent, 1000) 145 | %% ``` 146 | %% 147 | %% there is a possible situation when the event is emitted before 148 | %% `block_until' function has a chance to run. In this case the latter 149 | %% will time out for no good reason. In order to work around this, 150 | %% `block_until' function actually searches for events matching 151 | %% `Predicate' in the past. `BackInTime' parameter determines how far 152 | %% back into past this function peeks. 153 | %% 154 | %% Note: In the current implementation `Predicate' runs for 155 | %% every received event. It means this function should be lightweight 156 | -spec block_until(predicate(), timeout(), timeout()) -> 157 | event() | timeout. 158 | block_until(Predicate, Timeout, BackInTime) -> 159 | snabbkaffe_collector:block_until(Predicate, Timeout, BackInTime). 160 | 161 | -spec start_trace() -> ok. 162 | start_trace() -> 163 | case snabbkaffe_sup:start_link() of 164 | {ok, _} -> 165 | ok; 166 | {error, {already_started, _}} -> 167 | ok 168 | end. 169 | 170 | -spec stop() -> ok. 171 | stop() -> 172 | snabbkaffe_sup:stop(), 173 | ok. 174 | 175 | %% @doc Extract events of certain kind(s) from the trace 176 | -spec events_of_kind(kind() | [kind()], trace()) -> trace(). 177 | events_of_kind(Kind, Events) when is_atom(Kind) -> 178 | events_of_kind([Kind], Events); 179 | events_of_kind(Kinds, Events) -> 180 | [E || E = #{?snk_kind := Kind} <- Events, lists:member(Kind, Kinds)]. 181 | 182 | -spec projection([atom()] | atom(), trace()) -> list(). 183 | projection(Field, Trace) when is_atom(Field) -> 184 | [maps:get(Field, I) || I <- Trace]; 185 | projection(Fields, Trace) -> 186 | [list_to_tuple([maps:get(F, I) || F <- Fields]) || I <- Trace]. 187 | 188 | -spec erase_timestamps(trace()) -> trace(). 189 | erase_timestamps(Trace) -> 190 | [maps:without([ts], I) || I <- Trace]. 191 | 192 | %% @doc Find pairs of complimentary events 193 | -spec find_pairs( boolean() 194 | , fun((event()) -> boolean()) 195 | , fun((event()) -> boolean()) 196 | , fun((event(), event()) -> boolean()) 197 | , trace() 198 | ) -> [maybe_pair()]. 199 | find_pairs(Strict, CauseP, EffectP, Guard, L) -> 200 | Fun = fun(A) -> 201 | C = fun_matches1(CauseP, A), 202 | E = fun_matches1(EffectP, A), 203 | if C orelse E -> 204 | {true, {A, C, E}}; 205 | true -> 206 | false 207 | end 208 | end, 209 | L1 = lists:filtermap(Fun, L), 210 | do_find_pairs(Strict, Guard, L1). 211 | 212 | -spec run( run_config() | integer() 213 | , fun() 214 | , fun() 215 | ) -> boolean() | {error, _}. 216 | run(Bucket, Run, Check) when is_integer(Bucket) -> 217 | run(#{bucket => Bucket}, Run, Check); 218 | run(Config, Run, Check) -> 219 | Timeout = maps:get(timeout, Config, 0), 220 | Bucket = maps:get(bucket, Config, undefined), 221 | start_trace(), 222 | %% Wipe the trace buffer clean: 223 | _ = collect_trace(0), 224 | snabbkaffe_collector:tp('$trace_begin', #{}), 225 | try 226 | Return = Run(), 227 | Trace = collect_trace(Timeout), 228 | RunTime = ?find_pairs( false 229 | , #{?snk_kind := '$trace_begin'} 230 | , #{?snk_kind := '$trace_end'} 231 | , Trace 232 | ), 233 | ?SNK_CONCUERROR orelse push_stats(run_time, Bucket, RunTime), 234 | try Check(Return, Trace) 235 | catch EC1:Error1 ?BIND_STACKTRACE(Stack1) -> 236 | ?GET_STACKTRACE(Stack1), 237 | Filename1 = dump_trace(Trace), 238 | ?log(critical, "Check stage failed: ~p~n~p~nStacktrace: ~p~n" 239 | "Trace dump: ~p~n" 240 | , [EC1, Error1, Stack1, Filename1] 241 | ), 242 | {error, {check_mode_failed, EC1, Error1, Stack1}} 243 | end 244 | catch EC:Error ?BIND_STACKTRACE(Stack) -> 245 | ?GET_STACKTRACE(Stack), 246 | Filename = dump_trace(collect_trace(0)), 247 | ?log(critical, "Run stage failed: ~p:~p~nStacktrace: ~p~n" 248 | "Trace dump: ~p~n" 249 | , [EC, Error, Stack, Filename] 250 | ), 251 | {error, {run_stage_failed, EC, Error, Stack}} 252 | end. 253 | 254 | -spec proper_printout(string(), list()) -> _. 255 | proper_printout(Char, []) when Char =:= "."; 256 | Char =:= "x"; 257 | Char =:= "!" -> 258 | io:put_chars(standard_error, Char); 259 | proper_printout(Fmt, Args) -> 260 | ?log(notice, Fmt, Args). 261 | 262 | %%==================================================================== 263 | %% List manipulation functions 264 | %%==================================================================== 265 | 266 | %% @doc Split list by predicate like this: 267 | %% ```[true, true, false, true, true, false] -> 268 | %% [[true, true], [false, true, true], [false]] 269 | %% ''' 270 | -spec splitr(fun((A) -> boolean()), [A]) -> [[A]]. 271 | splitr(_, []) -> 272 | []; 273 | splitr(Pred, L) -> 274 | case lists:splitwith(Pred, L) of 275 | {[], [X|Rest]} -> 276 | {A, B} = lists:splitwith(Pred, Rest), 277 | [[X|A]|splitr(Pred, B)]; 278 | {A, B} -> 279 | [A|splitr(Pred, B)] 280 | end. 281 | 282 | %% @doc Split list by predicate like this: 283 | %% ```[true, true, false, true, true, false] -> 284 | %% [[true, true, false], [true, true, false]] 285 | %% ''' 286 | -spec splitl(fun((A) -> boolean()), [A]) -> [[A]]. 287 | splitl(_, []) -> 288 | []; 289 | splitl(Pred, L) -> 290 | {A, B} = splitwith_(Pred, L, []), 291 | [A|splitl(Pred, B)]. 292 | 293 | %%==================================================================== 294 | %% CT overhauls 295 | %%==================================================================== 296 | 297 | %% @doc Implement `all/0' callback for Common Test 298 | -spec mk_all(module()) -> [atom() | {group, atom()}]. 299 | mk_all(Module) -> 300 | io:format(user, "Module: ~p", [Module]), 301 | Groups = try Module:groups() 302 | catch 303 | error:undef -> [] 304 | end, 305 | [{group, element(1, I)} || I <- Groups] ++ 306 | [F || {F, _A} <- Module:module_info(exports), 307 | case atom_to_list(F) of 308 | "t_" ++ _ -> true; 309 | _ -> false 310 | end]. 311 | 312 | -spec retry(integer(), non_neg_integer(), fun(() -> Ret)) -> Ret. 313 | retry(_, 0, Fun) -> 314 | Fun(); 315 | retry(Timeout, N, Fun) -> 316 | try Fun() 317 | catch 318 | EC:Err ?BIND_STACKTRACE(Stack) -> 319 | ?GET_STACKTRACE(Stack), 320 | timer:sleep(Timeout), 321 | ?slog(debug, #{ what => retry_fun 322 | , ec => EC 323 | , error => Err 324 | , stacktrace => Stack 325 | }), 326 | retry(Timeout, N - 1, Fun) 327 | end. 328 | 329 | -spec get_cfg([atom()], map() | proplists:proplist(), A) -> A. 330 | get_cfg([Key|T], Cfg, Default) when is_list(Cfg) -> 331 | case lists:keyfind(Key, 1, Cfg) of 332 | false -> 333 | Default; 334 | {_, Val} -> 335 | case T of 336 | [] -> Val; 337 | _ -> get_cfg(T, Val, Default) 338 | end 339 | end; 340 | get_cfg(Key, Cfg, Default) when is_map(Cfg) -> 341 | get_cfg(Key, maps:to_list(Cfg), Default). 342 | 343 | -spec fix_ct_logging() -> ok. 344 | -ifdef(OTP_RELEASE). 345 | %% OTP21+, we have logger: 346 | fix_ct_logging() -> 347 | %% Fix CT logging by overriding it 348 | LogLevel = case os:getenv("LOGLEVEL") of 349 | S when S =:= "debug"; 350 | S =:= "info"; 351 | S =:= "error"; 352 | S =:= "critical"; 353 | S =:= "alert"; 354 | S =:= "emergency" -> 355 | list_to_atom(S); 356 | _ -> 357 | notice 358 | end, 359 | case os:getenv("KEEP_CT_LOGGING") of 360 | false -> 361 | logger:set_primary_config(level, LogLevel), 362 | logger:remove_handler(default), 363 | logger:add_handler( default 364 | , logger_std_h 365 | , #{ formatter => {logger_formatter, 366 | #{ depth => 100 367 | , single_line => false 368 | %% , template => [msg] 369 | }} 370 | } 371 | ); 372 | _ -> 373 | ok 374 | end. 375 | -else. 376 | fix_ct_logging() -> 377 | ok. 378 | -endif. 379 | 380 | %%==================================================================== 381 | %% Statistical functions 382 | %%==================================================================== 383 | 384 | -spec push_stat(metric(), number()) -> ok. 385 | push_stat(Metric, Num) -> 386 | snabbkaffe_collector:push_stat(Metric, undefined, Num). 387 | 388 | -spec push_stat(metric(), number() | undefined, number()) -> ok. 389 | push_stat(Metric, X, Y) -> 390 | snabbkaffe_collector:push_stat(Metric, X, Y). 391 | 392 | -spec push_stats(metric(), number(), [maybe_pair()] | number()) -> ok. 393 | push_stats(Metric, Bucket, Pairs) -> 394 | lists:foreach( fun(Val) -> push_stat(Metric, Bucket, Val) end 395 | , transform_stats(Pairs) 396 | ). 397 | 398 | -spec push_stats(metric(), [maybe_pair()] | number()) -> ok. 399 | push_stats(Metric, Pairs) -> 400 | lists:foreach( fun(Val) -> push_stat(Metric, Val) end 401 | , transform_stats(Pairs) 402 | ). 403 | 404 | get_stats() -> 405 | {ok, Stats} = gen_server:call(snabbkaffe_collector, get_stats, infinity), 406 | Stats. 407 | 408 | analyze_statistics() -> 409 | Stats = get_stats(), 410 | maps:map(fun analyze_metric/2, Stats), 411 | ok. 412 | 413 | %%==================================================================== 414 | %% Checks 415 | %%==================================================================== 416 | 417 | -spec causality( boolean() 418 | , fun((event()) -> ok) 419 | , fun((event()) -> ok) 420 | , fun((event(), event()) -> boolean()) 421 | , trace() 422 | ) -> ok. 423 | causality(Strict, CauseP, EffectP, Guard, Trace) -> 424 | Pairs = find_pairs(true, CauseP, EffectP, Guard, Trace), 425 | if Strict -> 426 | [?panic("Cause without effect", #{cause => I}) 427 | || {singleton, I} <- Pairs]; 428 | true -> 429 | ok 430 | end, 431 | ok. 432 | 433 | -spec unique(trace()) -> true. 434 | unique(Trace) -> 435 | Trace1 = erase_timestamps(Trace), 436 | Fun = fun(A, Acc) -> inc_counters([A], Acc) end, 437 | Counters = lists:foldl(Fun, #{}, Trace1), 438 | Dupes = [E || E = {_, Val} <- maps:to_list(Counters), Val > 1], 439 | case Dupes of 440 | [] -> 441 | true; 442 | _ -> 443 | ?panic("Duplicate elements found", #{dupes => Dupes}) 444 | end. 445 | 446 | -spec projection_complete(atom() | [atom()], trace(), [term()]) -> true. 447 | projection_complete(Fields, Trace, Expected) -> 448 | Got = ordsets:from_list(projection(Fields, Trace)), 449 | Expected1 = ordsets:from_list(Expected), 450 | case ordsets:subtract(Expected1, Got) of 451 | [] -> 452 | true; 453 | Missing -> 454 | ?panic("Trace is missing elements", #{missing => Missing}) 455 | end. 456 | 457 | -spec projection_is_subset(atom() | [atom()], trace(), [term()]) -> true. 458 | projection_is_subset(Fields, Trace, Expected) -> 459 | Got = ordsets:from_list(projection(Fields, Trace)), 460 | Expected1 = ordsets:from_list(Expected), 461 | case ordsets:subtract(Got, Expected1) of 462 | [] -> 463 | true; 464 | Unexpected -> 465 | ?panic("Trace contains unexpected elements", #{unexpected => Unexpected}) 466 | end. 467 | 468 | -spec pair_max_depth([maybe_pair()]) -> non_neg_integer(). 469 | pair_max_depth(Pairs) -> 470 | TagPair = 471 | fun({pair, #{ts := T1}, #{ts := T2}}) -> 472 | [{T1, 1}, {T2, -1}]; 473 | ({singleton, #{ts := T}}) -> 474 | [{T, 1}] 475 | end, 476 | L0 = lists:flatmap(TagPair, Pairs), 477 | L = lists:keysort(1, L0), 478 | CalcDepth = 479 | fun({_T, A}, {N0, Max}) -> 480 | N = N0 + A, 481 | {N, max(N, Max)} 482 | end, 483 | {_, Max} = lists:foldl(CalcDepth, {0, 0}, L), 484 | Max. 485 | 486 | -spec strictly_increasing(list()) -> true. 487 | strictly_increasing(L) -> 488 | case L of 489 | [Init|Rest] -> 490 | Fun = fun(A, B) -> 491 | A > B orelse 492 | ?panic("Elements of list are not strictly increasing", 493 | #{ '1st_element' => A 494 | , '2nd_element' => B 495 | , list => L 496 | }), 497 | A 498 | end, 499 | lists:foldl(Fun, Init, Rest), 500 | true; 501 | [] -> 502 | true 503 | end. 504 | 505 | %%==================================================================== 506 | %% Internal functions 507 | %%==================================================================== 508 | 509 | -spec do_find_pairs( boolean() 510 | , fun((event(), event()) -> boolean()) 511 | , [{event(), boolean(), boolean()}] 512 | ) -> [maybe_pair()]. 513 | do_find_pairs(_Strict, _Guard, []) -> 514 | []; 515 | do_find_pairs(Strict, Guard, [{A, C, E}|T]) -> 516 | FindEffect = fun({B, _, true}) -> 517 | fun_matches2(Guard, A, B); 518 | (_) -> 519 | false 520 | end, 521 | case {C, E} of 522 | {true, _} -> 523 | case take(FindEffect, T) of 524 | {{B, _, _}, T1} -> 525 | [{pair, A, B}|do_find_pairs(Strict, Guard, T1)]; 526 | T1 -> 527 | [{singleton, A}|do_find_pairs(Strict, Guard, T1)] 528 | end; 529 | {false, true} when Strict -> 530 | ?panic("Effect occurs before cause", #{effect => A}); 531 | _ -> 532 | do_find_pairs(Strict, Guard, T) 533 | end. 534 | 535 | -spec dump_trace(trace()) -> file:filename(). 536 | -ifndef(CONCUERROR). 537 | dump_trace(Trace) -> 538 | {ok, CWD} = file:get_cwd(), 539 | Filename = integer_to_list(os:system_time()) ++ ".log", 540 | FullPath = filename:join([CWD, "snabbkaffe", Filename]), 541 | filelib:ensure_dir(FullPath), 542 | {ok, Handle} = file:open(FullPath, [write]), 543 | try 544 | lists:foreach(fun(I) -> io:format(Handle, "~99999p.~n", [I]) end, Trace) 545 | after 546 | file:close(Handle) 547 | end, 548 | FullPath. 549 | -else. 550 | dump_trace(Trace) -> 551 | lists:foreach(fun(I) -> io:format("~99999p.~n", [I]) end, Trace). 552 | -endif. %% CONCUERROR 553 | 554 | -spec inc_counters([Key], Map) -> Map 555 | when Map :: #{Key => integer()}. 556 | inc_counters(Keys, Map) -> 557 | Inc = fun(V) -> V + 1 end, 558 | lists:foldl( fun(Key, Acc) -> 559 | maps:update_with(Key, Inc, 1, Acc) 560 | end 561 | , Map 562 | , Keys 563 | ). 564 | 565 | -spec dec_counters([Key], Map) -> Map 566 | when Map :: #{Key => integer()}. 567 | dec_counters(Keys, Map) -> 568 | Dec = fun(V) -> V - 1 end, 569 | lists:foldl( fun(Key, Acc) -> 570 | maps:update_with(Key, Dec, -1, Acc) 571 | end 572 | , Map 573 | , Keys 574 | ). 575 | 576 | -spec fun_matches1(fun((A) -> boolean()), A) -> boolean(). 577 | fun_matches1(Fun, A) -> 578 | try Fun(A) 579 | catch 580 | error:function_clause -> false 581 | end. 582 | 583 | -spec fun_matches2(fun((A, B) -> boolean()), A, B) -> boolean(). 584 | fun_matches2(Fun, A, B) -> 585 | try Fun(A, B) 586 | catch 587 | error:function_clause -> false 588 | end. 589 | 590 | -spec take(fun((A) -> boolean()), [A]) -> {A, [A]} | [A]. 591 | take(Pred, L) -> 592 | take(Pred, L, []). 593 | 594 | take(_Pred, [], Acc) -> 595 | lists:reverse(Acc); 596 | take(Pred, [A|T], Acc) -> 597 | case Pred(A) of 598 | true -> 599 | {A, lists:reverse(Acc) ++ T}; 600 | false -> 601 | take(Pred, T, [A|Acc]) 602 | end. 603 | 604 | analyze_metric(MetricName, DataPoints = [N|_]) when is_number(N) -> 605 | %% This is a simple metric: 606 | Stats = bear:get_statistics(DataPoints), 607 | ?log(notice, "-------------------------------~n" 608 | "~p statistics:~n~p~n" 609 | , [MetricName, Stats]); 610 | analyze_metric(MetricName, Datapoints = [{_, _}|_]) -> 611 | %% This "clustering" is not scientific at all 612 | {XX, _} = lists:unzip(Datapoints), 613 | Min = lists:min(XX), 614 | Max = lists:max(XX), 615 | NumBuckets = 10, 616 | BucketSize = max(1, (Max - Min) div NumBuckets), 617 | PushBucket = 618 | fun({X, Y}, Acc) -> 619 | B0 = (X - Min) div BucketSize, 620 | B = Min + B0 * BucketSize, 621 | maps:update_with( B 622 | , fun(L) -> [Y|L] end 623 | , [Y] 624 | , Acc 625 | ) 626 | end, 627 | Buckets0 = lists:foldl(PushBucket, #{}, Datapoints), 628 | BucketStats = 629 | fun({Key, Vals}) when length(Vals) > 5 -> 630 | Stats = bear:get_statistics(Vals), 631 | {true, {Key, Stats}}; 632 | (_) -> 633 | false 634 | end, 635 | Buckets = lists:filtermap( BucketStats 636 | , lists:keysort(1, maps:to_list(Buckets0)) 637 | ), 638 | %% Print per-bucket stats: 639 | PlotPoints = [{Bucket, proplists:get_value(arithmetic_mean, Stats)} 640 | ||{Bucket, Stats} <- Buckets], 641 | Plot = asciiart:plot([{$*, PlotPoints}]), 642 | BucketStatsToString = 643 | fun({Key, Stats}) -> 644 | io_lib:format( "~10b ~e ~e ~e~n" 645 | , [ Key 646 | , proplists:get_value(min, Stats) * 1.0 647 | , proplists:get_value(max, Stats) * 1.0 648 | , proplists:get_value(arithmetic_mean, Stats) * 1.0 649 | ]) 650 | end, 651 | StatsStr = [ "Statisitics of ", atom_to_list(MetricName), $\n 652 | , asciiart:render(Plot) 653 | , "\n N min max avg\n" 654 | , [BucketStatsToString(I) || I <- Buckets] 655 | ], 656 | ?log(notice, "~s~n", [StatsStr]), 657 | %% Print more elaborate info for the last bucket 658 | case length(Buckets) of 659 | 0 -> 660 | ok; 661 | _ -> 662 | {_, Last} = lists:last(Buckets), 663 | ?log(info, "Stats:~n~p~n", [Last]) 664 | end. 665 | 666 | transform_stats(Data) -> 667 | Fun = fun({pair, #{ts := T1}, #{ts := T2}}) -> 668 | Dt = erlang:convert_time_unit( T2 - T1 669 | , native 670 | , millisecond 671 | ), 672 | {true, Dt * 1.0e-6}; 673 | (Num) when is_number(Num) -> 674 | {true, Num}; 675 | (_) -> 676 | false 677 | end, 678 | lists:filtermap(Fun, Data). 679 | 680 | %% @private Version of `lists:splitwith/2' that appends element that 681 | %% doesn't match predicate to the tail of the first tuple element 682 | splitwith_(Pred, [Hd|Tail], Taken) -> 683 | case Pred(Hd) of 684 | true -> splitwith_(Pred, Tail, [Hd|Taken]); 685 | false -> {lists:reverse([Hd|Taken]), Tail} 686 | end; 687 | splitwith_(Pred, [], Taken) -> 688 | {lists:reverse(Taken), []}. 689 | -------------------------------------------------------------------------------- /src/snabbkaffe_collector.erl: -------------------------------------------------------------------------------- 1 | %% Copyright 2019-2020 Klarna Bank AB 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | -module(snabbkaffe_collector). 15 | 16 | -include("snabbkaffe_internal.hrl"). 17 | 18 | -behaviour(gen_server). 19 | 20 | %% API 21 | -export([ start_link/0 22 | , get_trace/1 23 | , get_stats/0 24 | , block_until/3 25 | , notify_on_event/3 26 | , tp/2 27 | , push_stat/3 28 | ]). 29 | 30 | %% gen_server callbacks 31 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 32 | terminate/2, code_change/3]). 33 | 34 | -export_type([async_action/0]). 35 | 36 | -define(SERVER, ?MODULE). 37 | 38 | -type datapoints() :: [{number(), number()}] 39 | | number(). 40 | 41 | -type async_action() :: fun(({ok, snabbkaffe:event()} | timeout) -> _). 42 | 43 | -record(callback, 44 | { async_action :: async_action() 45 | , predicate :: snabbkaffe:predicate() 46 | , tref :: reference() | undefined 47 | , ref :: reference() 48 | }). 49 | 50 | -record(s, 51 | { trace :: [snabbkaffe:timed_event()] 52 | , stats = #{} :: #{snabbkaffe:metric() => datapoints()} 53 | , last_event_ts = 0 :: integer() 54 | , callbacks = [] :: [#callback{}] 55 | }). 56 | 57 | %%%=================================================================== 58 | %%% API 59 | %%%=================================================================== 60 | 61 | -spec tp(atom(), map()) -> ok. 62 | tp(Kind, Event) -> 63 | Event1 = Event #{ ts => timestamp() 64 | , ?snk_kind => Kind 65 | }, 66 | ?slog(debug, Event1), 67 | %% Call or cast? This is a tricky question, since we need to 68 | %% preserve causality of trace events. Per documentation, Erlang 69 | %% doesn't guarantee order of messages from different processes. So 70 | %% call looks like a safer option. However, when testing under 71 | %% concuerror, calls to snabbkaffe generate a lot (really!) of 72 | %% undesirable interleavings. In the current BEAM implementation, 73 | %% however, sender process gets blocked while the message is being 74 | %% copied to the local receiver's mailbox. That leads to 75 | %% preservation of causality. Concuerror uses this fact, as it runs 76 | %% with `--instant_delivery true` by default. 77 | %% 78 | %% Above reasoning is only valid for local processes. 79 | gen_server:cast(?SERVER, {trace, Event1}). 80 | 81 | -spec push_stat(snabbkaffe:metric(), number() | undefined, number()) -> ok. 82 | push_stat(Metric, X, Y) -> 83 | Val = case X of 84 | undefined -> 85 | Y; 86 | _ -> 87 | {X, Y} 88 | end, 89 | gen_server:call(?SERVER, {push_stat, Metric, Val}, infinity). 90 | 91 | start_link() -> 92 | gen_server:start({local, ?SERVER}, ?MODULE, [], []). 93 | 94 | -spec get_stats() -> datapoints(). 95 | get_stats() -> 96 | gen_server:call(?SERVER, get_stats, infinity). 97 | 98 | %% NOTE: Concuerror only supports `Timeout=0' 99 | -spec get_trace(integer()) -> snabbkaffe:timed_trace(). 100 | get_trace(Timeout) -> 101 | {ok, Trace} = gen_server:call(?SERVER, {get_trace, Timeout}, infinity), 102 | Trace. 103 | 104 | %% NOTE: concuerror supports only `Timeout = infinity' and `BackInType = infinity' 105 | %% or `BackInTime = 0' 106 | -spec block_until(snabbkaffe:predicate(), timeout(), timeout()) -> 107 | {ok, snabbkaffe:event()} | timeout. 108 | block_until(Predicate, Timeout, BackInTime) -> 109 | Infimum = infimum(BackInTime), 110 | gen_server:call( ?SERVER 111 | , {block_until, Predicate, Timeout, Infimum} 112 | , infinity 113 | ). 114 | 115 | -spec notify_on_event(snabbkaffe:predicate(), timeout(), async_action()) -> 116 | ok. 117 | notify_on_event(Predicate, Timeout, Callback) -> 118 | gen_server:call( ?SERVER 119 | , {notify_on_event, Callback, Predicate, Timeout} 120 | , infinity 121 | ). 122 | 123 | %%%=================================================================== 124 | %%% gen_server callbacks 125 | %%%=================================================================== 126 | 127 | init([]) -> 128 | TS = timestamp(), 129 | BeginTrace = #{ ts => TS 130 | , ?snk_kind => '$trace_begin' 131 | }, 132 | {ok, #s{ trace = [BeginTrace] 133 | , last_event_ts = TS 134 | }}. 135 | 136 | handle_cast({trace, Evt}, State0 = #s{trace = T0, callbacks = CB0}) -> 137 | CB = maybe_unblock_someone(Evt, CB0), 138 | State = State0#s{ trace = [Evt|T0] 139 | , last_event_ts = timestamp() 140 | , callbacks = CB 141 | }, 142 | {noreply, State}; 143 | handle_cast(Evt, State) -> 144 | {noreply, State}. 145 | 146 | handle_call({trace, Evt}, _From, State0 = #s{trace = T0, callbacks = CB0}) -> 147 | CB = maybe_unblock_someone(Evt, CB0), 148 | State = State0#s{ trace = [Evt|T0] 149 | , last_event_ts = timestamp() 150 | , callbacks = CB 151 | }, 152 | {reply, ok, State}; 153 | handle_call({push_stat, Metric, Stat}, _From, State0) -> 154 | Stats = maps:update_with( Metric 155 | , fun(L) -> [Stat|L] end 156 | , [Stat] 157 | , State0#s.stats 158 | ), 159 | {reply, ok, State0#s{stats = Stats}}; 160 | handle_call(get_stats, _From, State) -> 161 | {reply, {ok, State#s.stats}, State}; 162 | handle_call({get_trace, Timeout}, From, State) -> 163 | send_after(Timeout, self(), {flush, From, Timeout}), 164 | {noreply, State}; 165 | handle_call({block_until, Predicate, Timeout, Infimum}, From, State0) -> 166 | Callback = fun(Result) -> 167 | gen_server:reply(From, Result) 168 | end, 169 | State = maybe_subscribe(Predicate, Timeout, Infimum, Callback, State0), 170 | {noreply, State}; 171 | handle_call({notify_on_event, Callback, Predicate, Timeout}, _From, State0) -> 172 | Now = erlang:monotonic_time(), 173 | State = maybe_subscribe(Predicate, Timeout, Now, Callback, State0), 174 | {reply, ok, State}; 175 | handle_call(_Request, _From, State) -> 176 | Reply = unknown_call, 177 | {reply, Reply, State}. 178 | 179 | handle_info({timeout, Ref}, State) -> 180 | #s{callbacks = CB0} = State, 181 | Fun = fun(#callback{ref = Ref1, async_action = AsyncAction}) 182 | when Ref1 =:= Ref -> 183 | AsyncAction(timeout), 184 | false; 185 | (C) -> 186 | {true, C} 187 | end, 188 | CB = lists:filtermap(Fun, CB0), 189 | {noreply, State#s{callbacks = CB}}; 190 | handle_info(Event = {flush, To, Timeout}, State) -> 191 | #s{ trace = Trace 192 | , last_event_ts = LastEventTs 193 | } = State, 194 | Finished = 195 | if Timeout > 0 -> 196 | Dt = erlang:convert_time_unit( timestamp() - LastEventTs 197 | , native 198 | , millisecond 199 | ), 200 | Dt >= Timeout; 201 | true -> 202 | %% Logically, this branch is redundand, but it's here as a 203 | %% workaround for concuerror 204 | true 205 | end, 206 | if Finished -> 207 | TraceEnd = #{ ?snk_kind => '$trace_end' 208 | , ts => LastEventTs 209 | }, 210 | Result = lists:reverse([TraceEnd|Trace]), 211 | gen_server:reply(To, {ok, Result}), 212 | {noreply, State #s{trace = []}}; 213 | true -> 214 | send_after(Timeout, self(), Event), 215 | {noreply, State} 216 | end; 217 | handle_info(_, State) -> 218 | {noreply, State}. 219 | 220 | terminate(_Reason, _State) -> 221 | ok. 222 | 223 | code_change(_OldVsn, State, _Extra) -> 224 | {ok, State}. 225 | 226 | %%%=================================================================== 227 | %%% Internal functions 228 | %%%=================================================================== 229 | 230 | -spec maybe_unblock_someone( snabbkaffe:event() 231 | , [#callback{}] 232 | ) -> [#callback{}]. 233 | maybe_unblock_someone(Evt, Callbacks) -> 234 | Fun = fun(Callback) -> 235 | #callback{ predicate = Predicate 236 | , async_action = AsyncAction 237 | , tref = TRef 238 | } = Callback, 239 | case Predicate(Evt) of 240 | false -> 241 | {true, Callback}; 242 | true -> 243 | cancel_timer(TRef), 244 | AsyncAction({ok, Evt}), 245 | false 246 | end 247 | end, 248 | lists:filtermap(Fun, Callbacks). 249 | 250 | -spec maybe_subscribe( snabbkaffe:predicate() 251 | , timeout() 252 | , integer() 253 | , async_action() 254 | , #s{} 255 | ) -> #s{}. 256 | maybe_subscribe(Predicate, Timeout, Infimum, AsyncAction, State0) -> 257 | #s{ trace = Trace 258 | , callbacks = CB0 259 | } = State0, 260 | try 261 | %% 1. Search in the past events 262 | [case Evt of 263 | #{ts := Ts} when Ts > Infimum -> 264 | case Predicate(Evt) of 265 | true -> 266 | throw({found, Evt}); 267 | false -> 268 | ok 269 | end; 270 | _ -> 271 | throw(not_found) 272 | end 273 | || Evt <- Trace], 274 | throw(not_found) 275 | catch 276 | {found, Event} -> 277 | AsyncAction({ok, Event}), 278 | State0; 279 | not_found -> 280 | %% 2. Postpone reply 281 | Ref = make_ref(), 282 | TRef = send_after(Timeout, self(), {timeout, Ref}), 283 | Callback = #callback{ async_action = AsyncAction 284 | , predicate = Predicate 285 | , tref = TRef 286 | , ref = Ref 287 | }, 288 | State0#s{ callbacks = [Callback|CB0] 289 | } 290 | end. 291 | 292 | -spec send_after(timeout(), pid(), _Msg) -> 293 | reference() | undefined. 294 | send_after(infinity, _, _) -> 295 | undefined; 296 | send_after(Timeout, Pid, Msg) -> 297 | erlang:send_after(Timeout, Pid, Msg). 298 | 299 | -spec infimum(integer()) -> integer(). 300 | -ifndef(CONCUERROR). 301 | infimum(infinity) -> 302 | beginning_of_times(); 303 | infimum(BackInTime0) -> 304 | BackInTime = erlang:convert_time_unit( BackInTime0 305 | , millisecond 306 | , native 307 | ), 308 | erlang:monotonic_time() - BackInTime. 309 | -else. 310 | infimum(infinity) -> 311 | beginning_of_times(); 312 | infimum(_) -> 313 | %% With concuerror, all events have `timestamp=-1', so 314 | %% starting from 0 should not match any events: 315 | 0. 316 | -endif. 317 | 318 | -spec cancel_timer(reference() | undefined) -> _. 319 | cancel_timer(undefined) -> 320 | ok; 321 | cancel_timer(TRef) -> 322 | erlang:cancel_timer(TRef). 323 | 324 | -spec timestamp() -> integer(). 325 | -ifndef(CONCUERROR). 326 | timestamp() -> 327 | erlang:monotonic_time(). 328 | -else. 329 | timestamp() -> 330 | -1. 331 | -endif. 332 | 333 | -spec beginning_of_times() -> integer(). 334 | -ifndef(CONCUERROR). 335 | beginning_of_times() -> 336 | erlang:system_info(start_time). 337 | -else. 338 | beginning_of_times() -> 339 | -2. 340 | -endif. 341 | -------------------------------------------------------------------------------- /src/snabbkaffe_internal.hrl: -------------------------------------------------------------------------------- 1 | -include("snabbkaffe.hrl"). 2 | 3 | -ifdef(CONCUERROR). 4 | -define(SNK_CONCUERROR, true). 5 | -else. 6 | -define(SNK_CONCUERROR, false). 7 | -endif. 8 | 9 | -ifdef(OTP_RELEASE). 10 | -define(BIND_STACKTRACE(V), : V). 11 | -define(GET_STACKTRACE(V), ok). 12 | -else. 13 | -define(BIND_STACKTRACE(V),). 14 | -define(GET_STACKTRACE(V), V = erlang:get_stacktrace()). 15 | -endif. 16 | -------------------------------------------------------------------------------- /src/snabbkaffe_nemesis.erl: -------------------------------------------------------------------------------- 1 | %% Copyright 2019-2020 Klarna Bank AB 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | %% @doc This module implements "nemesis" process that injects faults 16 | %% into system under test in order to test its fault-tolerance. 17 | %% 18 | %% == Usage == 19 | %% 20 | %% === Somewhere in the tested code === 21 | %% 22 | %% ``` 23 | %% ?maybe_crash(kind1, #{data1 => Foo, field2 => Bar}) 24 | %% ''' 25 | %% 26 | %% === Somewhere in the run stage === 27 | %% 28 | %% ``` 29 | %% ?inject_crash( #{?snk_kind := kind1, data1 := 42} 30 | %% , snabbkaffe_nemesis:always_crash() 31 | %% ) 32 | %% ''' 33 | %% @end 34 | -module(snabbkaffe_nemesis). 35 | 36 | -include("snabbkaffe_internal.hrl"). 37 | 38 | -behaviour(gen_server). 39 | 40 | %% API 41 | -export([ start_link/0 42 | , inject_crash/2 43 | , inject_crash/3 44 | , fix_crash/1 45 | , maybe_crash/2 46 | %% Failure scenarios 47 | , always_crash/0 48 | , recover_after/1 49 | , random_crash/1 50 | , periodic_crash/3 51 | ]). 52 | 53 | -export_type([fault_scenario/0]). 54 | 55 | %% gen_server callbacks 56 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 57 | terminate/2]). 58 | 59 | -define(SERVER, ?MODULE). 60 | 61 | -define(ERROR_TAB, snabbkaffe_injected_errors). 62 | -define(STATE_TAB, snabbkaffe_fault_states). 63 | -define(SINGLETON_KEY, 0). 64 | 65 | %%%=================================================================== 66 | %%% Types 67 | %%%=================================================================== 68 | 69 | %% @doc 70 | %% Type of fault patterns, such as "always fail", "fail randomly" or 71 | %% "recover after N attempts" 72 | %% @end 73 | %% 74 | %% This type is pretty magical. For performance reasons, state of the 75 | %% failure scenario is encoded as an integer counter, that is 76 | %% incremented every time the scenario is run. (BEAM VM can do this 77 | %% atomically and fast). Therefore "failure scenario" should map 78 | %% integer to boolean. 79 | -opaque fault_scenario() :: fun((integer()) -> boolean()). 80 | 81 | -type fault_key() :: term(). 82 | 83 | %% State of failure point (it's a simple counter, see above comment): 84 | -type fault_state() :: {fault_key(), integer()}. 85 | 86 | %% Injected error: 87 | -record(fault, 88 | { reference :: reference() 89 | , predicate :: snabbkaffe:prediacate() 90 | , scenario :: snabbkaffe:fault_scenario() 91 | , reason :: term() 92 | }). 93 | 94 | %% Currently this gen_server just holds the ets tables and 95 | %% synchronizes writes to the fault table, but in the future it may be 96 | %% used to mess up the system in more interesting ways 97 | -record(s, 98 | { injected_errors :: ets:tid() 99 | , fault_states :: ets:tid() 100 | }). 101 | 102 | %%%=================================================================== 103 | %%% API 104 | %%%=================================================================== 105 | 106 | %% @doc Start the server 107 | start_link() -> 108 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 109 | 110 | %% @equiv inject_crash(Predicate, Scenario, notmyday) 111 | -spec inject_crash(snabbkaffe:predicate(), fault_scenario()) -> reference(). 112 | inject_crash(Predicate, Scenario) -> 113 | inject_crash(Predicate, Scenario, notmayday). 114 | 115 | %% @doc Inject crash into the system 116 | -spec inject_crash(snabbkaffe:predicate(), fault_scenario(), term()) -> reference(). 117 | inject_crash(Predicate, Scenario, Reason) -> 118 | Ref = make_ref(), 119 | Crash = #fault{ reference = Ref 120 | , predicate = Predicate 121 | , scenario = Scenario 122 | , reason = Reason 123 | }, 124 | ok = gen_server:call(?SERVER, {inject_crash, Crash}, infinity), 125 | Ref. 126 | 127 | %% @doc Remove injected fault 128 | -spec fix_crash(reference()) -> ok. 129 | fix_crash(Ref) -> 130 | gen_server:call(?SERVER, {fix_crash, Ref}, infinity). 131 | 132 | %% @doc Check if there are any injected crashes that match this data, 133 | %% and respond with the crash reason if so. 134 | -spec maybe_crash(fault_key(), map()) -> ok. 135 | maybe_crash(Key, Data) -> 136 | [{_, Faults}] = ets:lookup(?ERROR_TAB, ?SINGLETON_KEY), 137 | %% Check if any of the injected errors have predicates matching my 138 | %% data: 139 | Fun = fun(#fault{predicate = P}) -> P(Data) end, 140 | case lists:filter(Fun, Faults) of 141 | [] -> 142 | %% None of the injected faults match my data: 143 | ok; 144 | [#fault{scenario = S, reason = R}|_] -> 145 | NewVal = ets:update_counter(?STATE_TAB, Key, {2, 1}, {Key, 0}), 146 | %% Run fault_scenario function to see if we need to crash this 147 | %% time: 148 | case S(NewVal) of 149 | true -> 150 | snabbkaffe_collector:tp(snabbkaffe_crash, Data#{ crash_kind => Key 151 | }), 152 | error(R); 153 | false -> 154 | ok 155 | end 156 | end. 157 | 158 | %%%=================================================================== 159 | %%% Fault scenarios 160 | %%%=================================================================== 161 | 162 | -spec always_crash() -> fault_scenario(). 163 | always_crash() -> 164 | fun(_) -> 165 | true 166 | end. 167 | 168 | -spec recover_after(non_neg_integer()) -> fault_scenario(). 169 | recover_after(Times) -> 170 | fun(X) -> 171 | X =< Times 172 | end. 173 | 174 | -spec random_crash(float()) -> fault_scenario(). 175 | random_crash(CrashProbability) -> 176 | fun(X) -> 177 | Range = 2 bsl 16, 178 | %% Turn a sequential number into a sufficiently plausible 179 | %% pseudorandom one: 180 | Val = erlang:phash2(X, Range), 181 | Val < CrashProbability * Range 182 | end. 183 | 184 | %% @doc A type of fault that occurs and fixes periodically. 185 | -spec periodic_crash(integer(), float(), float()) -> fault_scenario(). 186 | periodic_crash(Period, DutyCycle, Phase) -> 187 | DC = DutyCycle * Period, 188 | P = round(Phase/(math:pi()*2)*Period), 189 | fun(X) -> 190 | (X + P - 1) rem Period >= DC 191 | end. 192 | 193 | %%%=================================================================== 194 | %%% gen_server callbacks 195 | %%%=================================================================== 196 | 197 | %% @private 198 | init([]) -> 199 | ST = ets:new(?STATE_TAB, [ named_table 200 | , {write_concurrency, true} 201 | , {read_concurrency, true} 202 | , public 203 | ]), 204 | FT = ets:new(?ERROR_TAB, [ named_table 205 | , {write_concurrency, false} 206 | , {read_concurrency, true} 207 | , protected 208 | ]), 209 | ets:insert(?ERROR_TAB, {?SINGLETON_KEY, []}), 210 | {ok, #s{ injected_errors = FT 211 | , fault_states = ST 212 | }}. 213 | 214 | %% @private 215 | handle_call({inject_crash, Crash}, _From, State) -> 216 | [{_, Faults}] = ets:lookup(?ERROR_TAB, ?SINGLETON_KEY), 217 | ets:insert(?ERROR_TAB, {?SINGLETON_KEY, [Crash|Faults]}), 218 | {reply, ok, State}; 219 | handle_call({fix_crash, Ref}, _From, State) -> 220 | [{_, Faults0}] = ets:lookup(?ERROR_TAB, ?SINGLETON_KEY), 221 | Faults = lists:keydelete(Ref, #fault.reference, Faults0), 222 | ets:insert(?ERROR_TAB, {?SINGLETON_KEY, Faults}), 223 | {reply, ok, State}; 224 | handle_call(_Request, _From, State) -> 225 | {reply, ok, State}. 226 | 227 | %% @private 228 | handle_cast(_Request, State) -> 229 | {noreply, State}. 230 | 231 | %% @private 232 | handle_info(_Info, State) -> 233 | {noreply, State}. 234 | 235 | %% @private 236 | terminate(_Reason, _State) -> 237 | ok. 238 | 239 | %%%=================================================================== 240 | %%% Internal functions 241 | %%%=================================================================== 242 | -------------------------------------------------------------------------------- /src/snabbkaffe_sup.erl: -------------------------------------------------------------------------------- 1 | %% Copyright 2019-2020 Klarna Bank AB 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | -module(snabbkaffe_sup). 15 | 16 | -behaviour(supervisor). 17 | 18 | %% API 19 | -export([start_link/0, stop/0]). 20 | 21 | %% Supervisor callbacks 22 | -export([init/1]). 23 | 24 | -define(SERVER, ?MODULE). 25 | 26 | %%%=================================================================== 27 | %%% API functions 28 | %%%=================================================================== 29 | 30 | start_link() -> 31 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 32 | 33 | stop() -> 34 | case whereis(?MODULE) of 35 | undefined -> 36 | ok; 37 | Pid -> 38 | monitor(process, Pid), 39 | unlink(Pid), 40 | exit(Pid, shutdown), 41 | receive 42 | {'DOWN', _MRef, process, Pid, _} -> ok 43 | end 44 | end. 45 | 46 | %%%=================================================================== 47 | %%% Supervisor callbacks 48 | %%%=================================================================== 49 | 50 | init([]) -> 51 | SupFlags = #{ strategy => one_for_all 52 | , intensity => 0 53 | }, 54 | Collector = #{ id => snabbkaffe_collector 55 | , start => {snabbkaffe_collector, start_link, []} 56 | }, 57 | Nemesis = #{ id => snabbkaffe_nemesis 58 | , start => {snabbkaffe_nemesis, start_link, []} 59 | }, 60 | {ok, {SupFlags, [Collector, Nemesis]}}. 61 | 62 | %%%=================================================================== 63 | %%% Internal functions 64 | %%%=================================================================== 65 | -------------------------------------------------------------------------------- /test/causality_tests.erl: -------------------------------------------------------------------------------- 1 | -module(causality_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | %% Strict Pairs No Guards 7 | -define(SPNG(Trace), 8 | ?find_pairs( true 9 | , #{foo := _A}, #{bar := _A} 10 | , Trace 11 | )). 12 | 13 | %% Fuzzy Pairs No Guards 14 | -define(FPNG(Trace), 15 | ?find_pairs( false 16 | , #{foo := _A}, #{bar := _A} 17 | , Trace 18 | )). 19 | 20 | %% Strict Pairs With Guard 21 | -define(SPWG(Trace), 22 | ?find_pairs( true 23 | , #{foo := _A}, #{bar := _B}, _A + 1 =:= _B 24 | , Trace 25 | )). 26 | 27 | %% Strict Cuasality No Guard 28 | -define(SCNG(Trace), 29 | ?strict_causality( #{foo := _A} when true, #{bar := _A} when true 30 | , Trace 31 | )). 32 | 33 | %% Weak Cuasality No Guard 34 | -define(WCNG(Trace), 35 | ?causality( #{foo := _A}, #{bar := _A} 36 | , Trace 37 | )). 38 | 39 | -define(foo(A), #{foo => A}). 40 | -define(bar(A), #{bar => A}). 41 | 42 | -define(pair(A), {pair, #{foo := A}, #{bar := A}}). 43 | -define(singleton(A), {singleton, #{foo := A}}). 44 | 45 | -define(error_msg, "Effect occurs before cause"). 46 | 47 | -define(error_msg_cause, "Cause without effect"). 48 | 49 | spng_success_test() -> 50 | ?assertMatch( [] 51 | , ?SPNG([]) 52 | ), 53 | ?assertMatch( [] 54 | , ?SPNG([#{quux => 1}, #{quux => 2}]) 55 | ), 56 | ?assertMatch( [?pair(1), ?pair(owo)] 57 | , ?SPNG([?foo(1), ?foo(owo), foo, ?bar(owo), ?bar(1), bar]) 58 | ), 59 | ?assertMatch( [?singleton(1), ?pair(2)] 60 | , ?SPNG([?foo(1), ?foo(2), foo, ?bar(2), bar]) 61 | ), 62 | ?assertMatch( [?singleton(1), ?pair(2), ?pair(owo)] 63 | , ?SPNG([?foo(1), ?foo(2), ?bar(2), ?foo(owo), ?bar(owo)]) 64 | ). 65 | 66 | spng_fail_test() -> 67 | ?assertError( {panic, #{?snk_kind := ?error_msg}} 68 | , ?SPNG([?foo(1), ?bar(owo), foo, ?foo(owo), ?bar(1), bar]) 69 | ). 70 | 71 | fpng_success_test() -> 72 | ?assertMatch( [] 73 | , ?FPNG([]) 74 | ), 75 | ?assertMatch( [] 76 | , ?FPNG([#{quux => 1}, #{quux => 2}]) 77 | ), 78 | ?assertMatch( [?pair(1), ?pair(owo)] 79 | , ?FPNG([?foo(1), ?foo(owo), foo, ?bar(owo), ?bar(1), bar]) 80 | ), 81 | ?assertMatch( [?singleton(1), ?pair(2)] 82 | , ?FPNG([?foo(1), ?foo(2), foo, ?bar(2), bar]) 83 | ), 84 | ?assertMatch( [?singleton(1), ?pair(2), ?pair(owo)] 85 | , ?FPNG([?foo(1), ?foo(2), ?bar(2), ?foo(owo), ?bar(owo)]) 86 | ), 87 | ?assertMatch( [?singleton(1), ?singleton(owo), ?pair(2)] 88 | , ?FPNG([ ?foo(1), ?bar(owo), ?foo(owo), ?foo(2) 89 | , foo, ?bar(2), ?bar(44) 90 | ]) 91 | ). 92 | 93 | fpng_scoped_test() -> 94 | %% Test that match specs in ?find_pairs capture variables from the 95 | %% context: 96 | _A = owo, 97 | ?assertMatch( [?pair(owo)] 98 | , ?FPNG([?foo(1), ?foo(owo), foo, ?bar(owo), ?bar(1), bar]) 99 | ). 100 | 101 | -define(pair_inc(A), {pair, #{foo := A}, #{bar := A + 1}}). 102 | 103 | spwg_success_test() -> 104 | ?assertMatch( [] 105 | , ?SPWG([]) 106 | ), 107 | ?assertMatch( [] 108 | , ?SPWG([#{quux => 1}, #{quux => 2}]) 109 | ), 110 | ?assertMatch( [?pair_inc(1), ?pair_inc(2)] 111 | , ?SPWG([?foo(1), foo, ?foo(2), ?bar(2), ?bar(3)]) 112 | ), 113 | ?assertMatch( [?singleton(1), ?pair_inc(2)] 114 | , ?SPWG([?foo(1), ?foo(2), foo, ?bar(3), bar]) 115 | ). 116 | 117 | scng_succ_test() -> 118 | ?assertMatch( ok 119 | , ?SCNG([]) 120 | ), 121 | ?assertMatch( ok 122 | , ?SCNG([#{quux => 1}, #{quux => 2}]) 123 | ), 124 | ?assertMatch( ok 125 | , ?SCNG([?foo(1), ?foo(2), foo, ?bar(2), ?bar(1)]) 126 | ). 127 | 128 | scng_fail_test() -> 129 | ?assertError( {panic, #{?snk_kind := ?error_msg_cause}} 130 | , ?SCNG([?foo(1), foo]) 131 | ). 132 | 133 | wcng_succ_test() -> 134 | ?assertMatch( ok 135 | , ?WCNG([]) 136 | ), 137 | ?assertMatch( ok 138 | , ?WCNG([#{quux => 1}, #{quux => 2}]) 139 | ), 140 | ?assertMatch( ok 141 | , ?WCNG([?foo(1), ?foo(2), foo, ?bar(2)]) 142 | ). 143 | -------------------------------------------------------------------------------- /test/collector_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(collector_SUITE). 2 | 3 | -compile(export_all). 4 | 5 | -include_lib("snabbkaffe/include/ct_boilerplate.hrl"). 6 | 7 | %%==================================================================== 8 | %% CT callbacks 9 | %%==================================================================== 10 | 11 | suite() -> 12 | [{timetrap, {seconds, 30}}]. 13 | 14 | init_per_suite(Config) -> 15 | snabbkaffe:fix_ct_logging(), 16 | Config. 17 | 18 | end_per_suite(_Config) -> 19 | ok. 20 | 21 | init_per_group(_GroupName, Config) -> 22 | Config. 23 | 24 | end_per_group(_GroupName, _Config) -> 25 | ok. 26 | 27 | groups() -> 28 | []. 29 | 30 | %%==================================================================== 31 | %% Testcases 32 | %%==================================================================== 33 | 34 | t_all_collected(_Config) when is_list(_Config) -> 35 | [?tp(foo, #{foo => I}) || I <- lists:seq(1, 1000)], 36 | Trace = snabbkaffe:collect_trace(), 37 | ?assertMatch(1000, length(?of_kind(foo, Trace))), 38 | ok. 39 | 40 | t_check_trace(_Config) when is_list(_Config) -> 41 | ?check_trace( 42 | 42, 43 | fun(Ret, Trace) -> 44 | ?assertMatch(42, Ret), 45 | ?assertMatch( [ #{?snk_kind := '$trace_begin'} 46 | , #{?snk_kind := '$trace_end'} 47 | ] 48 | , Trace) 49 | end). 50 | 51 | prop_async_collect() -> 52 | ?FORALL( 53 | {MaxWaitTime, Events}, 54 | ?LET(MaxWaitTime, range(1, 100), 55 | {MaxWaitTime, [range(0, MaxWaitTime)]}), 56 | ?check_trace( 57 | #{timeout => MaxWaitTime + 10}, 58 | %% Emit events with some sleep in between: 59 | [begin 60 | Id = make_ref(), 61 | spawn(fun() -> 62 | timer:sleep(Sleep), 63 | ?tp(async, #{id => Id}) 64 | end), 65 | Id 66 | end || Sleep <- Events], 67 | %% Check that all events have been collected: 68 | fun(Ids, Trace) -> 69 | ?projection_complete( id 70 | , ?of_kind(async, Trace) 71 | , Ids 72 | ) 73 | end)). 74 | 75 | t_async_collect(Config) when is_list(Config) -> 76 | %% Verify that trace collection is delayed until last event (within 77 | %% timeout) is received: 78 | ?run_prop(Config, prop_async_collect()). 79 | 80 | t_bar({init, Config}) -> 81 | Config; 82 | t_bar({'end', _Config}) -> 83 | ok; 84 | t_bar(Config) when is_list(Config) -> 85 | ok. 86 | 87 | t_simple_metric(_Config) when is_list(_Config) -> 88 | [snabbkaffe:push_stat(test, rand:uniform()) 89 | || I <- lists:seq(1, 100)], 90 | ok. 91 | 92 | t_bucket_metric(_Config) when is_list(_Config) -> 93 | [snabbkaffe:push_stat(test, 100 + I*10, I + rand:uniform()) 94 | || I <- lists:seq(1, 100) 95 | , _ <- lists:seq(1, 10)], 96 | ok. 97 | 98 | t_pair_metric(_Config) when is_list(_Config) -> 99 | [?tp(foo, #{i => I}) || I <- lists:seq(1, 100)], 100 | timer:sleep(10), 101 | [?tp(bar, #{i => I}) || I <- lists:seq(1, 100)], 102 | Trace = snabbkaffe:collect_trace(), 103 | Pairs = ?find_pairs( true 104 | , #{?snk_kind := foo, i := I}, #{?snk_kind := bar, i := I} 105 | , Trace 106 | ), 107 | snabbkaffe:push_stats(foo_bar, Pairs). 108 | 109 | t_pair_metric_buckets(_Config) when is_list(_Config) -> 110 | [?tp(foo, #{i => I}) || I <- lists:seq(1, 100)], 111 | timer:sleep(10), 112 | [?tp(bar, #{i => I}) || I <- lists:seq(1, 100)], 113 | Trace = snabbkaffe:collect_trace(), 114 | Pairs = ?find_pairs( true 115 | , #{?snk_kind := foo, i := I}, #{?snk_kind := bar, i := I} 116 | , Trace 117 | ), 118 | snabbkaffe:push_stats(foo_bar, 10, Pairs). 119 | 120 | t_run_1(_Config) when is_list(_Config) -> 121 | [?check_trace( I 122 | , begin 123 | [?tp(foo, #{}) || J <- lists:seq(1, I)], 124 | true 125 | end 126 | , fun(Ret, Trace) -> 127 | ?assertMatch(true, Ret), 128 | ?assertMatch(I, length(?of_kind(foo, Trace))) 129 | end 130 | ) 131 | || I <- lists:seq(1, 1000)]. 132 | 133 | prop1() -> 134 | ?FORALL( 135 | {Ret, L}, {term(), list()}, 136 | ?check_trace( 137 | length(L), 138 | begin 139 | [?tp(foo, #{i => I}) || I <- L], 140 | Ret 141 | end, 142 | fun(Ret1, Trace) -> 143 | ?assertMatch(Ret, Ret1), 144 | Foos = ?of_kind(foo, Trace), 145 | ?assertMatch(L, ?projection(i, Foos)), 146 | true 147 | end)). 148 | 149 | t_proper(Config) when is_list(Config) -> 150 | ?run_prop(Config, prop1()). 151 | 152 | t_forall_trace(Config0) when is_list(Config0) -> 153 | Config = [{proper, #{ max_size => 100 154 | , numtests => 1000 155 | , timeout => 30000 156 | }} | Config0], 157 | Prop = 158 | ?forall_trace( 159 | {Ret, L}, {term(), list()}, 160 | length(L), %% Bucket 161 | begin 162 | [?tp(foo, #{i => I}) || I <- L], 163 | Ret 164 | end, 165 | fun(Ret1, Trace) -> 166 | ?assertMatch(Ret, Ret1), 167 | ?assertMatch(L, ?projection(i, ?of_kind(foo, Trace))), 168 | true 169 | end), 170 | ?run_prop(Config, Prop). 171 | 172 | t_prop_fail_false(Config) when is_list(Config) -> 173 | Prop = ?forall_trace( 174 | X, list(), 175 | 42, 176 | fun(_, _) -> 177 | false 178 | end 179 | ), 180 | ?assertExit( fail 181 | , ?run_prop(Config, Prop) 182 | ). 183 | 184 | t_prop_run_exception(Config) when is_list(Config) -> 185 | Prop = ?forall_trace( 186 | X, list(), 187 | 42, %% Bucket 188 | begin 189 | 1 = 2 190 | end, 191 | fun(_, _) -> 192 | false 193 | end 194 | ), 195 | ?assertExit( fail 196 | , ?run_prop(Config, Prop) 197 | ). 198 | 199 | t_prop_check_exception(Config) when is_list(Config) -> 200 | ?log(notice, "Don't mind the below crashes, they are intentional!", []), 201 | Prop = ?forall_trace( 202 | X, list(), 203 | 42, %% Bucket 204 | ok, 205 | fun(_, _) -> 206 | 1 = 2 207 | end 208 | ), 209 | ?assertExit( fail 210 | , ?run_prop(Config, Prop) 211 | ). 212 | 213 | t_block_until(Config) when is_list(Config) -> 214 | Kind = foo, 215 | ?check_trace( 216 | begin 217 | spawn(fun() -> 218 | timer:sleep(100), 219 | %% This event should not be matched (kind =/= Kind): 220 | ?tp(bar, #{data => 44}), 221 | %% This event should not be matched (data is too small): 222 | ?tp(foo, #{data => 1}), 223 | %% This one should be matched: 224 | ?tp(foo, #{data => 43}), 225 | %% This one matches the pattern but is ignored, the 226 | %% previous one should already unlock the caller: 227 | ?tp(foo, #{data => 44}) 228 | end), 229 | %% Note that here `Kind' variable is captured from the context 230 | %% (and used to match events) and `Data' is bound in the guard: 231 | ?block_until(#{?snk_kind := Kind, data := Data} when Data > 42) 232 | end, 233 | fun(Ret, _Trace) -> 234 | ?assertMatch( {ok, #{?snk_kind := foo, data := 43}} 235 | , Ret 236 | ) 237 | end). 238 | 239 | t_block_until_from_past(Config) when is_list(Config) -> 240 | Kind = foo, 241 | ?check_trace( 242 | begin 243 | %% This one matches the pattern but is ignored, the 244 | %% next one should unlock the caller: 245 | ?tp(foo, #{data => 43}), 246 | %% This one should be matched: 247 | ?tp(foo, #{data => 44}), 248 | %% This event should not be matched (kind =/= Kind): 249 | ?tp(bar, #{data => 1}), 250 | %% This event should not be matched (data is too small): 251 | ?tp(foo, #{data => 1}), 252 | ?block_until(#{?snk_kind := Kind, data := Data} when Data > 42) 253 | end, 254 | fun(Ret, _Trace) -> 255 | ?assertMatch( {ok, #{?snk_kind := foo, data := 44}} 256 | , Ret 257 | ) 258 | end). 259 | 260 | t_block_until_timeout(Config) when is_list(Config) -> 261 | ?check_trace( 262 | begin 263 | ?block_until(#{?snk_kind := foo}, 100) 264 | end, 265 | fun(Ret, _Trace) -> 266 | ?assertMatch(timeout, Ret) 267 | end). 268 | 269 | t_block_until_past_limit(Config) when is_list(Config) -> 270 | ?check_trace( 271 | begin 272 | %% This event should be ignored, it's too far back in time: 273 | ?tp(foo, #{}), 274 | timer:sleep(200), 275 | ?block_until(#{?snk_kind := foo}, 100, 100) 276 | end, 277 | fun(Ret, _Trace) -> 278 | ?assertMatch(timeout, Ret) 279 | end). 280 | 281 | wait_async_action_prop() -> 282 | MinDiff = 10, 283 | ?FORALL( 284 | {Delay, Timeout}, {range(0, 100), range(0, 100)}, 285 | ?IMPLIES( 286 | abs(Delay - Timeout) > MinDiff, 287 | ?check_trace( 288 | ?wait_async_action( 289 | begin 290 | timer:sleep(Delay), 291 | ?tp(bar, #{}), 292 | foo 293 | end, 294 | #{?snk_kind := bar}, 295 | Timeout), 296 | fun({Result, Event}, _Trace) -> 297 | ?assertMatch(foo, Result), 298 | if Delay < Timeout -> 299 | ?assertMatch( {ok, #{?snk_kind := bar}} 300 | , Event 301 | ); 302 | true -> 303 | ?assertMatch( timeout 304 | , Event 305 | ) 306 | end, 307 | true 308 | end))). 309 | 310 | t_wait_async_action(Config) when is_list(Config) -> 311 | ?run_prop(Config, wait_async_action_prop()). 312 | -------------------------------------------------------------------------------- /test/complete_tests.erl: -------------------------------------------------------------------------------- 1 | -module(complete_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | -define(foo(A), #{foo => A, bar => bar}). 7 | 8 | -define(valid(T, L), 9 | ?assertMatch( true 10 | , ?projection_complete(foo, [?foo(I) || I <- T], L) 11 | )). 12 | 13 | -define(invalid(T, L), 14 | ?assertError( {panic, #{?snk_kind := "Trace is missing elements"}} 15 | , ?projection_complete(foo, [?foo(I) || I <- T], L) 16 | )). 17 | 18 | complete_succ_test() -> 19 | ?valid([], []), 20 | ?valid([1, 3, 4], [1, 3, 4]), 21 | ?valid([1, 2, 2, 3], [1, 2, 3]). 22 | 23 | complete_fail_test() -> 24 | ?invalid([1, 3, 4], [1, 3, 4, 5]), 25 | ?invalid([1, 2, 2, 3], [1, 2, 3, 4]). 26 | 27 | multiple_fields_test() -> 28 | Fields = [foo, bar], 29 | Trace = [#{foo => 1, bar => 1}, #{foo => 2, bar => 2, baz => 2}], 30 | Pattern = [{1, 1}, {2, 2}], 31 | ?projection_complete(Fields, Trace, Pattern). 32 | -------------------------------------------------------------------------------- /test/concuerror_tests.erl: -------------------------------------------------------------------------------- 1 | -module(concuerror_tests). 2 | 3 | -include("snabbkaffe.hrl"). 4 | -include_lib("stdlib/include/assert.hrl"). 5 | 6 | -export([race_test/0, causality_test/0, fail_test/0]). 7 | 8 | race_test() -> 9 | ?check_trace( 10 | begin 11 | Pid = spawn_link(fun() -> 12 | receive 13 | {ping, N} -> 14 | ?tp(pong, #{winner => N}) 15 | end 16 | end), 17 | %% Spawn two processes competing to send ping message to the 18 | %% first one: 19 | spawn_link(fun() -> 20 | catch ?tp(ping, #{id => 1}), 21 | Pid ! {ping, 1}, 22 | ok 23 | end), 24 | spawn_link(fun() -> 25 | catch ?tp(ping, #{id => 2}), 26 | Pid ! {ping, 2}, 27 | ok 28 | end), 29 | %% Wait for the termination of the receiving process: 30 | ?block_until(#{?snk_kind := pong}) 31 | end, 32 | fun(_Ret, Trace) -> 33 | %% Validate that there's always a pair of events 34 | ?assertMatch( [{pair, _, _} | _] 35 | , ?find_pairs( true 36 | , #{?snk_kind := ping} 37 | , #{?snk_kind := pong} 38 | , Trace 39 | ) 40 | ), 41 | %% TODO: I validated manually that value of `winner' field is 42 | %% indeed nondeterministic, and therefore snabbkaffe doesn't 43 | %% interfere with concuerror interleavings; however, it would 44 | %% be nice to check this property automatically as well, but 45 | %% it requires "testing outside the box": 46 | %% 47 | %% Both asserts are true: 48 | %% ?assertMatch([#{winner := 2}], ?of_kind(pong, Trace)), 49 | %% ?assertMatch([#{winner := 1}], ?of_kind(pong, Trace)), 50 | true 51 | end). 52 | 53 | causality_test() -> 54 | ?check_trace( 55 | begin 56 | C = spawn(fun() -> 57 | receive ping -> 58 | ?tp(pong, #{id => c}) 59 | end 60 | end), 61 | B = spawn(fun() -> 62 | receive ping -> 63 | ?tp(pong, #{id => b}), 64 | C ! ping 65 | end 66 | end), 67 | A = spawn(fun() -> 68 | ?tp(pong, #{id => a}), 69 | B ! ping 70 | end), 71 | ?block_until(#{?snk_kind := pong, id := c}) 72 | end, 73 | fun(_, Trace) -> 74 | ?assertEqual([a,b,c], ?projection(id, ?of_kind(pong, Trace))) 75 | end). 76 | 77 | %% Check that testcases fail gracefully and don't try to do anything 78 | %% that concuerror doesn't understand, like opening files: 79 | fail_test() -> 80 | try 81 | ?check_trace( 82 | begin 83 | ?tp(foo, #{}) 84 | end, 85 | fun(_, _) -> 86 | error(deliberate) 87 | end) 88 | catch 89 | _:_ -> ok 90 | end. 91 | -------------------------------------------------------------------------------- /test/is_subset_tests.erl: -------------------------------------------------------------------------------- 1 | -module(is_subset_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | -define(foo(A), #{foo => A, bar => bar}). 7 | 8 | -define(valid(T, L), 9 | ?assertMatch( true 10 | , ?projection_is_subset(foo, [?foo(I) || I <- T], L) 11 | )). 12 | 13 | -define(invalid(T, L), 14 | ?assertError( {panic, #{?snk_kind := "Trace contains unexpected elements"}} 15 | , ?projection_is_subset(foo, [?foo(I) || I <- T], L) 16 | )). 17 | 18 | complete_succ_test() -> 19 | ?valid([], []), 20 | ?valid([1, 3, 3, 4], [1, 3, 4, 4]), 21 | ?valid([1, 2, 3], [1, 2, 3, 7]). 22 | 23 | complete_fail_test() -> 24 | ?invalid([1, 2], [1]), 25 | ?invalid([1, 2, 2, 3], [1, 2, 4]). 26 | 27 | 28 | multiple_fields_test() -> 29 | Fields = [foo, bar], 30 | Trace = [#{foo => 2, bar => 2, baz => 2}], 31 | Pattern = [{1, 1}, {2, 2}], 32 | ?projection_is_subset(Fields, Trace, Pattern). 33 | -------------------------------------------------------------------------------- /test/misc_tests.erl: -------------------------------------------------------------------------------- 1 | -module(misc_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | -define(foo(A), #{foo => A, bar => bar, baz => baz}). 7 | 8 | projection_1_test() -> 9 | ?assertMatch( [1, 2, 3] 10 | , snabbkaffe:projection( foo 11 | , [?foo(1), ?foo(2), ?foo(3)] 12 | )). 13 | 14 | projection_2_test() -> 15 | ?assertMatch( [{1, bar}, {2, bar}, {3, bar}] 16 | , snabbkaffe:projection( [foo, bar] 17 | , [?foo(1), ?foo(2), ?foo(3)] 18 | )). 19 | 20 | strictly_increasing_test() -> 21 | ?assertMatch( true 22 | , snabbkaffe:strictly_increasing([]) 23 | ), 24 | ?assertMatch( true 25 | , snabbkaffe:strictly_increasing([1]) 26 | ), 27 | ?assertMatch( true 28 | , snabbkaffe:strictly_increasing([1, 2, 5, 6]) 29 | ), 30 | ?assertError( _ 31 | , snabbkaffe:strictly_increasing([1, 2, 5, 3]) 32 | ), 33 | ?assertError( _ 34 | , snabbkaffe:strictly_increasing([1, 2, 2, 3]) 35 | ). 36 | 37 | get_cfg_test() -> 38 | Cfg1 = [{proper, [ {numtests, 1000} 39 | , {timeout, 42} 40 | ]} 41 | ], 42 | Cfg2 = #{ proper => #{ numtests => 1000 43 | , timeout => 42 44 | } 45 | }, 46 | Cfg3 = [{proper, #{ numtests => 1000 47 | , timeout => 42 48 | } 49 | }], 50 | ?assertMatch(42, snabbkaffe:get_cfg([proper, timeout], Cfg1, 10)), 51 | ?assertMatch(42, snabbkaffe:get_cfg([proper, foo], Cfg1, 42)), 52 | ?assertMatch(42, snabbkaffe:get_cfg([proper, timeout], Cfg2, 10)), 53 | ?assertMatch(42, snabbkaffe:get_cfg([proper, foo], Cfg2, 42)), 54 | ?assertMatch(42, snabbkaffe:get_cfg([proper, timeout], Cfg3, 10)), 55 | ?assertMatch(42, snabbkaffe:get_cfg([proper, foo], Cfg3, 42)). 56 | -------------------------------------------------------------------------------- /test/nemesis_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(nemesis_SUITE). 2 | 3 | -compile(export_all). 4 | 5 | -include_lib("snabbkaffe/include/ct_boilerplate.hrl"). 6 | 7 | %%==================================================================== 8 | %% CT callbacks 9 | %%==================================================================== 10 | 11 | suite() -> 12 | [{timetrap, {seconds, 30}}]. 13 | 14 | init_per_suite(Config) -> 15 | snabbkaffe:fix_ct_logging(), 16 | Config. 17 | 18 | end_per_suite(_Config) -> 19 | ok. 20 | 21 | %%==================================================================== 22 | %% Testcases 23 | %%==================================================================== 24 | 25 | t_always_crash(Config) when is_list(Config) -> 26 | Run = fun(N) -> 27 | ?maybe_crash(foo, #{data => N}) 28 | end, 29 | ?check_trace( 30 | begin 31 | ?assertMatch(ok, Run(1)), 32 | Ref = ?inject_crash( #{?snk_kind := foo, data := 1} 33 | , snabbkaffe_nemesis:always_crash() 34 | ), 35 | ?assertMatch(ok, Run(2)), 36 | ?assertError(notmyday, Run(1)), 37 | snabbkaffe_nemesis:fix_crash(Ref), 38 | ?assertMatch(ok, Run(1)) 39 | end, 40 | fun(_Result, Trace) -> 41 | ?assertMatch( [#{crash_kind := foo, data := 1}] 42 | , ?of_kind(snabbkaffe_crash, Trace) 43 | ) 44 | end). 45 | 46 | t_recover(Config) when is_list(Config) -> 47 | N = 4, 48 | ?check_trace( 49 | begin 50 | ?inject_crash( #{?snk_kind := foo} 51 | , snabbkaffe_nemesis:recover_after(N) 52 | ), 53 | [catch ?maybe_crash(#{?snk_kind => foo}) || _ <- lists:seq(1, 2*N)] 54 | end, 55 | fun(_Result, Trace) -> 56 | ?assertEqual( N 57 | , length(?of_kind(snabbkaffe_crash, Trace)) 58 | ) 59 | end). 60 | 61 | t_periodic(Config) when is_list(Config) -> 62 | F1 = snabbkaffe_nemesis:periodic_crash(5, 0.6, 0), 63 | ?assertEqual( [false, false, false, true, true, false, false, false, true, true] 64 | , [F1(I) || I <- lists:seq(1, 10)] 65 | ), 66 | F2 = snabbkaffe_nemesis:periodic_crash(5, 0.6, math:pi()), 67 | ?assertEqual( [true, true, false, false, false, true, true, false, false, false] 68 | , [F2(I) || I <- lists:seq(1, 10)] 69 | ). 70 | 71 | %% Check that error can be injected at random trace point 72 | t_break_trace_point(Config) when is_list(Config) -> 73 | N = 4, 74 | ?check_trace( 75 | begin 76 | ?inject_crash( #{?snk_kind := foo} 77 | , snabbkaffe_nemesis:recover_after(N) 78 | ), 79 | [catch ?tp(foo, #{}) || _ <- lists:seq(1, 2*N)] 80 | end, 81 | fun(_Result, Trace) -> 82 | ?assertEqual( N 83 | , length(?of_kind(snabbkaffe_crash, Trace)) 84 | ) 85 | end). 86 | 87 | %% Check that static unique tokens are indeed unique 88 | t_static_unique_points(Config) when is_list(Config) -> 89 | A = ?__snkStaticUniqueToken, B = ?__snkStaticUniqueToken, 90 | ?assertEqual(false, A =:= B). 91 | -------------------------------------------------------------------------------- /test/overlap_depth_tests.erl: -------------------------------------------------------------------------------- 1 | -module(overlap_depth_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | -define(p(A, B), {pair, #{ts => A}, #{ts => B}}). 7 | -define(s(A), {singleton, #{ts => A}}). 8 | 9 | -define(matchDepth(N, L), 10 | ?assertMatch(N, ?pair_max_depth(L))). 11 | 12 | disjoint_test() -> 13 | ?matchDepth(0, []), 14 | ?matchDepth(1, [?p(1, 1)]), 15 | ?matchDepth(1, [?p(0, 1)]), 16 | ?matchDepth(1, [?p(0, 1), ?p(1, 2)]), 17 | ?matchDepth(1, [?s(4), ?p(0, 1), ?p(2, 3)]). 18 | 19 | nested_test() -> 20 | ?matchDepth(2, [?s(0), ?p(0, 1), ?p(2, 3)]), 21 | ?matchDepth(2, [?p(0, 3), ?p(1, 2), ?p(2, 3)]), 22 | ?matchDepth(3, [?p(0, 3), ?p(1, 3), ?p(2, 2)]), 23 | ?matchDepth(5, [?p(0, 3), ?s(1), ?p(1, 3), ?s(2), ?p(2, 2)]). 24 | 25 | overlap_test() -> 26 | ?matchDepth(2, [?p(0, 2), ?p(1, 3)]), 27 | ?matchDepth(3, [?p(0, 2), ?p(1, 3), ?p(1.5, 2.5)]). 28 | -------------------------------------------------------------------------------- /test/split_tests.erl: -------------------------------------------------------------------------------- 1 | -module(split_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | splitl_test() -> 6 | Pred = fun is_atom/1, 7 | ?assertMatch( [] 8 | , snabbkaffe:splitl(Pred, []) 9 | ), 10 | L1 = [[a]], 11 | ?assertMatch( L1 12 | , snabbkaffe:splitl(Pred, lists:append(L1)) 13 | ), 14 | L2 = [[a, b, 1], [c, d, 2], [d]], 15 | ?assertMatch( L2 16 | , snabbkaffe:splitl(Pred, lists:append(L2)) 17 | ), 18 | L3 = [[1], [2], [3]], 19 | ?assertMatch( L3 20 | , snabbkaffe:splitl(Pred, lists:append(L3)) 21 | ). 22 | 23 | splitr_test() -> 24 | Pred = fun is_atom/1, 25 | ?assertMatch( [] 26 | , snabbkaffe:splitr(Pred, []) 27 | ), 28 | L1 = [[a]], 29 | ?assertMatch( L1 30 | , snabbkaffe:splitr(Pred, lists:append(L1)) 31 | ), 32 | L2 = [[a, b], [1, c, d], [2, d]], 33 | ?assertMatch( L2 34 | , snabbkaffe:splitr(Pred, lists:append(L2)) 35 | ), 36 | L3 = [[1], [2], [3]], 37 | ?assertMatch( L3 38 | , snabbkaffe:splitr(Pred, lists:append(L3)) 39 | ). 40 | -------------------------------------------------------------------------------- /test/unique_tests.erl: -------------------------------------------------------------------------------- 1 | -module(unique_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | -include_lib("snabbkaffe.hrl"). 5 | 6 | -define(valid(L), 7 | ?assertMatch(true, snabbkaffe:unique(L))). 8 | 9 | -define(invalid(L), 10 | ?assertError( {panic, #{?snk_kind := "Duplicate elements found"}} 11 | , snabbkaffe:unique(L) 12 | )). 13 | 14 | unique_succ_test() -> 15 | ?valid([]), 16 | ?valid([ #{foo => 1, ts => 1} 17 | , #{bar => 1, ts => 2} 18 | , #{bar => 2, ts => 2} 19 | ]). 20 | 21 | unique_fail_test() -> 22 | ?invalid([ #{foo => 1, ts => 1} 23 | , #{bar => 1, ts => 2} 24 | , #{foo => 1, ts => 2} 25 | ]). 26 | --------------------------------------------------------------------------------