├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── include └── tr.hrl ├── rebar.config ├── rebar.lock ├── src ├── erlang_doctor.app.src ├── erlang_doctor_app.erl ├── erlang_doctor_sup.erl └── tr.erl └── test ├── tr_SUITE.erl └── tr_helper.erl /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | common-test: 6 | name: Run Common Test 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | otp: [21, 22, 23, 24, 25, 26, 27] 11 | container: 12 | image: erlang:${{ matrix.otp }}-alpine 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | - name: Run Common Test 17 | run: | 18 | rebar3 xref 19 | rebar3 dialyzer 20 | rebar3 ct 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .rebar3 2 | _* 3 | .eunit 4 | *.o 5 | *.beam 6 | *.plt 7 | *.swp 8 | *.swo 9 | .erlang.cookie 10 | ebin 11 | log 12 | erl_crash.dump 13 | .rebar 14 | logs 15 | _build 16 | .idea 17 | rebar3.crashdump 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2017, Paweł Chrząszcz . 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Erlang Doctor 2 | 3 | [![Hex.pm Version](https://img.shields.io/hexpm/v/erlang_doctor)](https://hex.pm/packages/erlang_doctor) 4 | [![Hex Docs](https://img.shields.io/badge/hex-docs-yellow.svg)](https://hexdocs.pm/erlang_doctor/) 5 | [![GitHub Actions](https://github.com/chrzaszcz/erlang_doctor/actions/workflows/test.yml/badge.svg)](https://github.com/chrzaszcz/erlang_doctor/actions) 6 | 7 | Lightweight tracing, debugging and profiling tool, which collects traces in an ETS table, putting minimal impact on your system. 8 | After collecting the traces, you can query and analyse them. 9 | By separating data collection from analysis, this tool helps you limit unnecessary repetition and guesswork. 10 | There is [ExDoctor](https://hex.pm/packages/ex_doctor) for Elixir as well. 11 | 12 | ## Quick start 13 | 14 | To quickly try it out right now, copy & paste the following to your Erlang shell: 15 | 16 | ```erlang 17 | P = "/tmp/tr.erl", ssl:start(), inets:start(), {ok, {{_, 200, _}, _, Src}} = httpc:request("https://git.io/fj024"), file:write_file(P, Src), {ok, tr, B} = compile:file(P, binary), code:load_binary(tr, P, B), rr(P), tr:start(). 18 | ``` 19 | 20 | This snippet downloads, compiles and starts the `tr` module from the `master` branch. 21 | Your Erlang Doctor is now ready to use! 22 | 23 | The easiest way to use it is the following: 24 | 25 | ```erlang 26 | tr:trace([your_module]). 27 | your_module:some_function(). 28 | tr:select(). 29 | ``` 30 | 31 | You should see the collected traces for the call and return of `your_module:some_function/0`. 32 | 33 | This compact tool is capable of much more - see below. 34 | 35 | ### Include it as a dependency 36 | 37 | To avoid copy-pasting the snippet shown above, you can include `erlang_doctor` in your dependencies in `rebar.config`. 38 | There is a [Hex package](https://hex.pm/packages/erlang_doctor) as well. 39 | 40 | ### Use it during development 41 | 42 | You can make Erlang Doctor available in the Erlang/Rebar3 shell during development by cloning it to `ERLANG_DOCTOR_PATH`, 43 | calling `rebar3 compile`, and loading it in your `~/.erlang` file: 44 | 45 | ```erlang 46 | code:add_path("ERLANG_DOCTOR_PATH/erlang_doctor/_build/default/lib/erlang_doctor/ebin"). 47 | code:load_file(tr). 48 | ``` 49 | 50 | ## Tracing: data collection 51 | 52 | The test suite helpers from `tr_SUITE.erl` are used here as examples. 53 | You can follow these examples on your own - just call `rebar3 as test shell` in `ERLANG_DOCTOR_PATH`. 54 | 55 | ### Setting up: `start`, `start_link` 56 | 57 | The first thing to do is to start the tracer with `tr:start/0`. 58 | 59 | There is also `tr:start/1`, which accepts a [map of options](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:init_options/0), including: 60 | 61 | - `tab`: collected traces are stored in an ETS table with this name (default: `trace`), 62 | - `limit`: maximum number of traces in the table - when it is reached, tracing is stopped (default: no limit). 63 | 64 | There are `tr:start_link/0` and `tr:start_link/1` as well, and they are intended for use with the whole `erlang_doctor` application. 65 | 66 | For this tutorial we start the `tr` module in the simplest way: 67 | 68 | ```erlang 69 | 1> tr:start(). 70 | {ok, <0.218.0>} 71 | ``` 72 | 73 | ### Tracing with `trace` 74 | 75 | To trace function calls for given modules, use `tr:trace/1`, providing a list of traced modules: 76 | 77 | ```erlang 78 | 2> tr:trace([tr_SUITE]). 79 | ok 80 | ``` 81 | 82 | You can provide `{Module, Function, Arity}` tuples in the list as well. 83 | The function `tr:trace_app/1` traces an application, and `tr:trace_apps/1` traces multiple ones. 84 | If you need to trace an application and some additional modules, use `tr:app_modules/1` to get the list of modules for an application: 85 | 86 | ```erlang 87 | tr:trace([Module1, Module2 | tr:app_modules(YourApp)]). 88 | ``` 89 | 90 | If you want to trace selected processes instead of all of them, you can use `tr:trace/2`: 91 | 92 | ```erlang 93 | tr:trace([Module1, Module2], [Pid1, Pid2]). 94 | ``` 95 | 96 | The `tr:trace/1` function accepts a [map of options](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:trace_options/0), which include: 97 | 98 | - `modules`: a list of module names or `{Module, Function, Arity}` tuples. The list is empty by default. 99 | - `pids`: a list of Pids of processes to trace, or the atom `all` (default) to trace all processes. 100 | - `msg`: `none` (default), `all`, `send` or `recv`. Specifies which message events will be traced. By default no messages are traced. 101 | - `msg_trigger`: `after_traced_call` (default) or `always`. By default, traced messages in each process are stored after the first traced function call in that process. The goal is to limit the number of traced messages, which can be huge in the entire Erlang system. If you want all messages, set it to `always`. 102 | 103 | ### Calling the traced function 104 | 105 | Now we can call some functions - let's trace the following function call. 106 | It calculates the factorial recursively and sleeps 1 ms between each step. 107 | 108 | ```erlang 109 | 3> tr_SUITE:sleepy_factorial(3). 110 | 6 111 | ``` 112 | 113 | ### Stopping tracing 114 | 115 | You can stop tracing with the following function: 116 | 117 | ```erlang 118 | 4> tr:stop_tracing(). 119 | ok 120 | ``` 121 | 122 | It's good to stop it as soon as possible to avoid accumulating too many traces in the ETS table. 123 | Usage of `tr` on production systems is risky, but if you have to do it, start and stop the tracer in the same command, 124 | e.g. for one second with: 125 | 126 | ```erlang 127 | tr:trace(Modules), timer:sleep(1000), tr:stop_tracing(). 128 | ``` 129 | 130 | ## Debugging: data analysis 131 | 132 | The collected traces are stored in an ETS table (default name: `trace`). 133 | They are stored as [`tr`](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:tr/0) records with the following fields: 134 | 135 | - `index`: trace identifier, auto-incremented for each received trace. 136 | - `pid`: process identifier associated with the trace. 137 | - `event`: `call`, `return` or `exception` for function traces; `send` or `recv` for messages. 138 | - `mfa`: `{Module, Function, Arity}` for function traces; `no_mfa` for messages. 139 | - `data`: argument list (for calls), returned value (for returns) or class and value (for exceptions). 140 | - `timestamp` in microseconds. 141 | - `info`: For function traces and `recv` events it is `no_info`. For `send` events it is a `{To, Exists}` tuple, where `To` is the recipient pid, and `Exists` is a boolean indicating if the recipient process existed. 142 | 143 | It's useful to read the record definitions before trace analysis: 144 | 145 | ```erlang 146 | 5> rr(tr). 147 | [node,tr] 148 | ``` 149 | 150 | The snippet shown at the top of this page includes this already. 151 | 152 | ### Trace selection: `select` 153 | 154 | Use `tr:select/0` to select all collected traces. 155 | 156 | ```erlang 157 | 6> tr:select(). 158 | [#tr{index = 1,pid = <0.395.0>,event = call, 159 | mfa = {tr_SUITE,sleepy_factorial,1}, 160 | data = [3], 161 | ts = 1705475521743239,info = no_info}, 162 | #tr{index = 2,pid = <0.395.0>,event = call, 163 | mfa = {tr_SUITE,sleepy_factorial,1}, 164 | data = [2], 165 | ts = 1705475521744690,info = no_info}, 166 | #tr{index = 3,pid = <0.395.0>,event = call, 167 | mfa = {tr_SUITE,sleepy_factorial,1}, 168 | data = [1], 169 | ts = 1705475521746470,info = no_info}, 170 | #tr{index = 4,pid = <0.395.0>,event = call, 171 | mfa = {tr_SUITE,sleepy_factorial,1}, 172 | data = [0], 173 | ts = 1705475521748499,info = no_info}, 174 | #tr{index = 5,pid = <0.395.0>,event = return, 175 | mfa = {tr_SUITE,sleepy_factorial,1}, 176 | data = 1,ts = 1705475521750451,info = no_info}, 177 | #tr{index = 6,pid = <0.395.0>,event = return, 178 | mfa = {tr_SUITE,sleepy_factorial,1}, 179 | data = 1,ts = 1705475521750453,info = no_info}, 180 | #tr{index = 7,pid = <0.395.0>,event = return, 181 | mfa = {tr_SUITE,sleepy_factorial,1}, 182 | data = 2,ts = 1705475521750454,info = no_info}, 183 | #tr{index = 8,pid = <0.395.0>,event = return, 184 | mfa = {tr_SUITE,sleepy_factorial,1}, 185 | data = 6,ts = 1705475521750455,info = no_info}] 186 | ``` 187 | 188 | The `tr:select/1` function accepts a fun that is passed to `ets:fun2ms/1`. 189 | This way you can limit the selection to specific items and select only some fields from the [`tr`](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:tr/0) record: 190 | 191 | ```erlang 192 | 7> tr:select(fun(#tr{event = call, data = [N]}) -> N end). 193 | [3, 2, 1, 0] 194 | ``` 195 | 196 | Use `tr:select/2` to further filter the results by searching for a term in `#tr.data` (recursively searching in lists, tuples and maps). 197 | 198 | ```erlang 199 | 8> tr:select(fun(T) -> T end, 2). 200 | [#tr{index = 2,pid = <0.395.0>,event = call, 201 | mfa = {tr_SUITE,sleepy_factorial,1}, 202 | data = [2], 203 | ts = 1705475521744690,info = no_info}, 204 | #tr{index = 7,pid = <0.395.0>,event = return, 205 | mfa = {tr_SUITE,sleepy_factorial,1}, 206 | data = 2,ts = 1705475521750454,info = no_info}] 207 | ``` 208 | 209 | ### Trace filtering: `filter` 210 | 211 | Sometimes it might be easier to use `tr:filter/1`, because it can accept any function as the argument. 212 | You can use `tr:contains_data/2` to search for the same term as in the example above. 213 | 214 | ```erlang 215 | 9> Traces = tr:filter(fun(T) -> tr:contains_data(2, T) end). 216 | [#tr{index = 2,pid = <0.395.0>,event = call, 217 | mfa = {tr_SUITE,sleepy_factorial,1}, 218 | data = [2], 219 | ts = 1705475521744690,info = no_info}, 220 | #tr{index = 7,pid = <0.395.0>,event = return, 221 | mfa = {tr_SUITE,sleepy_factorial,1}, 222 | data = 2,ts = 1705475521750454,info = no_info}] 223 | ``` 224 | 225 | The provided function is a predicate, which has to return `true` for the matching traces. 226 | For other traces it can return another value, or even raise an exception: 227 | 228 | ```erlang 229 | 10> tr:filter(fun(#tr{data = [2]}) -> true end). 230 | [#tr{index = 2,pid = <0.395.0>,event = call, 231 | mfa = {tr_SUITE,sleepy_factorial,1}, 232 | data = [2], 233 | ts = 1705475521744690,info = no_info}] 234 | ``` 235 | 236 | There is also `tr:filter/2`, which can be used to search in a different table than the current one - or in a list: 237 | 238 | ```erlang 239 | 11> tr:filter(fun(#tr{event = call}) -> true end, Traces). 240 | [#tr{index = 2,pid = <0.395.0>,event = call, 241 | mfa = {tr_SUITE,sleepy_factorial,1}, 242 | data = [2], 243 | ts = 1705475521744690,info = no_info}] 244 | ``` 245 | 246 | ### Tracebacks for filtered traces: `tracebacks` 247 | 248 | To find the tracebacks (stack traces) for matching traces, use `tr:tracebacks/1`: 249 | 250 | ```erlang 251 | 12> tr:tracebacks(fun(#tr{data = 1}) -> true end). 252 | [[#tr{index = 3,pid = <0.395.0>,event = call, 253 | mfa = {tr_SUITE,sleepy_factorial,1}, 254 | data = [1], 255 | ts = 1705475521746470,info = no_info}, 256 | #tr{index = 2,pid = <0.395.0>,event = call, 257 | mfa = {tr_SUITE,sleepy_factorial,1}, 258 | data = [2], 259 | ts = 1705475521744690,info = no_info}, 260 | #tr{index = 1,pid = <0.395.0>,event = call, 261 | mfa = {tr_SUITE,sleepy_factorial,1}, 262 | data = [3], 263 | ts = 1705475521743239,info = no_info}]] 264 | ``` 265 | 266 | Note, that by specifying `data = 1` we are only matching return traces, as call traces always have a list in `data`. 267 | Only one traceback is returned. It starts with a call that returned `1`. What follows is the stack trace for this call. 268 | 269 | One can notice that the call for 0 also returned 1, but the call tree got pruned - whenever two tracebacks overlap, only the shorter one is left. 270 | You can change this by returning tracebacks for all matching traces even if they overlap, setting the `output` option to `all`. All options are specified in the second argument, which is a map: 271 | 272 | ```erlang 273 | 13> tr:tracebacks(fun(#tr{data = 1}) -> true end, #{output => all}). 274 | [[#tr{index = 4,pid = <0.395.0>,event = call, 275 | mfa = {tr_SUITE,sleepy_factorial,1}, 276 | data = [0], 277 | ts = 1705475521748499,info = no_info}, 278 | #tr{index = 3,pid = <0.395.0>,event = call, 279 | mfa = {tr_SUITE,sleepy_factorial,1}, 280 | data = [1], 281 | ts = 1705475521746470,info = no_info}, 282 | #tr{index = 2,pid = <0.395.0>,event = call, 283 | mfa = {tr_SUITE,sleepy_factorial,1}, 284 | data = [2], 285 | ts = 1705475521744690,info = no_info}, 286 | #tr{index = 1,pid = <0.395.0>,event = call, 287 | mfa = {tr_SUITE,sleepy_factorial,1}, 288 | data = [3], 289 | ts = 1705475521743239,info = no_info}], 290 | [#tr{index = 3,pid = <0.395.0>,event = call, 291 | mfa = {tr_SUITE,sleepy_factorial,1}, 292 | data = [1], 293 | ts = 1705475521746470,info = no_info}, 294 | #tr{index = 2,pid = <0.395.0>,event = call, 295 | mfa = {tr_SUITE,sleepy_factorial,1}, 296 | data = [2], 297 | ts = 1705475521744690,info = no_info}, 298 | #tr{index = 1,pid = <0.395.0>,event = call, 299 | mfa = {tr_SUITE,sleepy_factorial,1}, 300 | data = [3], 301 | ts = 1705475521743239,info = no_info}]] 302 | ``` 303 | 304 | The third possibility is `output => longest` which does the opposite of pruning, leaving only the longest tracabecks when they overlap: 305 | 306 | ```erlang 307 | 14> tr:tracebacks(fun(#tr{data = 1}) -> true end, #{output => longest}). 308 | [[#tr{index = 4,pid = <0.395.0>,event = call, 309 | mfa = {tr_SUITE,sleepy_factorial,1}, 310 | data = [0], 311 | ts = 1705475521748499,info = no_info}, 312 | #tr{index = 3,pid = <0.395.0>,event = call, 313 | mfa = {tr_SUITE,sleepy_factorial,1}, 314 | data = [1], 315 | ts = 1705475521746470,info = no_info}, 316 | #tr{index = 2,pid = <0.395.0>,event = call, 317 | mfa = {tr_SUITE,sleepy_factorial,1}, 318 | data = [2], 319 | ts = 1705475521744690,info = no_info}, 320 | #tr{index = 1,pid = <0.395.0>,event = call, 321 | mfa = {tr_SUITE,sleepy_factorial,1}, 322 | data = [3], 323 | ts = 1705475521743239,info = no_info}]] 324 | ``` 325 | 326 | Possible [options](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:tb_options/0) for `tr:tracebacks/2` include: 327 | 328 | - `tab` is the table or list which is like the second argument of `tr:filter/2`, 329 | - `output` - `shortest` (default), `all`, `longest` - see above. 330 | - `format` - `list` (default), `tree` - returns a list of (possibly merged) call trees instead of tracebacks, `root` - returns a list of root calls. Trees and roots don't distinguish between `all` and `longest` output formats. Using `root` is equivalent to using `tree`, and then calling `tr:roots/1` on the results. There is also `tr:root/1` for a single tree. 331 | - `order` - `top_down` (default), `bottom_up` - call order in each tracaback; only for the `list` format. 332 | - `limit` - positive integer or `infinity` (default) - limits the number of matched traces. The actual number of tracebacks returned can be smaller unless `output => all` 333 | 334 | There are also functions `tr:traceback/1` and `tr:traceback/2`. They set `limit` to one and return only one trace if it exists. The options for `tr:traceback/2` are the same as for `tr:traceback/2` except `limit` and `format` (which are not supported). Additionally, it is possible to pass a [`tr`](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:tr/0) record (or an index) as the first argument to `tr:traceback/1` or `tr:traceback/2` to obtain the traceback for the provided trace event. 335 | 336 | ### Trace ranges for filtered traces: `ranges` 337 | 338 | To get a list of traces between each matching call and the corresponding return, use `tr:ranges/1`: 339 | 340 | ```erlang 341 | 15> tr:ranges(fun(#tr{data = [1]}) -> true end). 342 | [[#tr{index = 3,pid = <0.395.0>,event = call, 343 | mfa = {tr_SUITE,sleepy_factorial,1}, 344 | data = [1], 345 | ts = 1705475521746470,info = no_info}, 346 | #tr{index = 4,pid = <0.395.0>,event = call, 347 | mfa = {tr_SUITE,sleepy_factorial,1}, 348 | data = [0], 349 | ts = 1705475521748499,info = no_info}, 350 | #tr{index = 5,pid = <0.395.0>,event = return, 351 | mfa = {tr_SUITE,sleepy_factorial,1}, 352 | data = 1,ts = 1705475521750451,info = no_info}, 353 | #tr{index = 6,pid = <0.395.0>,event = return, 354 | mfa = {tr_SUITE,sleepy_factorial,1}, 355 | data = 1,ts = 1705475521750453,info = no_info}]] 356 | ``` 357 | 358 | There is also `tr:ranges/2` - it accepts a [map of options](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:range_options/0), including: 359 | 360 | - `tab` is the table or list which is like the second argument of `tr:filter/2`, 361 | - `max_depth` is the maximum depth of nested calls. A message event also adds 1 to the depth. 362 | You can use `#{max_depth => 1}` to see only the top-level call and the corresponding return. 363 | - `output` - `all` (default), `complete` or `incomplete` - decides whether the output should contain 364 | complete and/or incomplete ranges. A range is complete if the root call has a return. 365 | For example, you can use `#{output => incomplete}` to see only the traces with missing returns. 366 | 367 | When you combine the options into `#{output => incomplete, max_depth => 1}`, 368 | you get all the calls which didn't return (they were still executing when tracing was stopped). 369 | 370 | There are two additional functions: `tr:range/1` and `tr:range/2`, which return only one range if it exists. It is possible to pass a [`tr`](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:tr/0) record or an index as the first argument to `tr:range/1` or `tr:range/2` as well. 371 | 372 | ### Calling a function from a trace: `do` 373 | 374 | It is easy to replay a particular function call with `tr:do/1`: 375 | 376 | ```erlang 377 | 16> [T] = tr:filter(fun(#tr{data = [3]}) -> true end). 378 | [#tr{index = 1,pid = <0.395.0>,event = call, 379 | mfa = {tr_SUITE,sleepy_factorial,1}, 380 | data = [3], 381 | ts = 1705475521743239,info = no_info}] 382 | 17> tr:do(T). 383 | 6 384 | ``` 385 | 386 | This is useful e.g. for checking if a bug has been fixed without running the whole test suite. 387 | This function can be called with an index as the argument. 388 | 389 | ### Getting a single trace for the index: `lookup` 390 | 391 | Use `tr:lookup/1` to obtain the trace for an index. 392 | 393 | ## Profiling 394 | 395 | You can quickly get a hint about possible bottlenecks and redundancies in your system with function call statistics. 396 | 397 | ### Call statistics: `call_stat` 398 | 399 | The argument of `tr:call_stat/1` is a function that returns a key by which the traces are grouped. 400 | The simplest way to use this function is to look at the total number of calls and their time. 401 | To do this, we group all calls under one key, e.g. `total`: 402 | 403 | ```erlang 404 | 18> tr:call_stat(fun(_) -> total end). 405 | #{total => {4,7216,7216}} 406 | ``` 407 | 408 | Values of the returned map have the following format (time is in microseconds): 409 | 410 | ```{call_count(), acc_time(), own_time()}``` 411 | 412 | In the example there are four calls, which took 7216 microseconds in total. 413 | For nested calls we only take into account the outermost call, so this means that the whole calculation took 7.216 ms. 414 | Let's see how this looks like for individual steps - we can group the stats by the function argument: 415 | 416 | ```erlang 417 | 19> tr:call_stat(fun(#tr{data = [N]}) -> N end). 418 | #{0 => {1,1952,1952}, 419 | 1 => {1,3983,2031}, 420 | 2 => {1,5764,1781}, 421 | 3 => {1,7216,1452}} 422 | ``` 423 | 424 | You can use the provided function to do filtering as well: 425 | 426 | ```erlang 427 | 20> tr:call_stat(fun(#tr{data = [N]}) when N < 3 -> N end). 428 | #{0 => {1,1952,1952},1 => {1,3983,2031},2 => {1,5764,1781}} 429 | ``` 430 | 431 | ### Sorted call statistics: `sorted_call_stat` 432 | 433 | You can sort the call stat by accumulated time (descending) with `tr:sorted_call_stat/1`: 434 | 435 | ```erlang 436 | 21> tr:sorted_call_stat(fun(#tr{data = [N]}) -> N end). 437 | [{3,1,7216,1452}, 438 | {2,1,5764,1781}, 439 | {1,1,3983,2031}, 440 | {0,1,1952,1952}] 441 | ``` 442 | 443 | The first element of each tuple is the key, the rest is the same as above. 444 | To pretty-print it, use `tr:print_sorted_call_stat/2`. 445 | The second argument limits the table row number, e.g. we can only print the top 3 items: 446 | 447 | ```erlang 448 | 22> tr:print_sorted_call_stat(fun(#tr{data = [N]}) -> N end, 3). 449 | 3 1 7216 1452 450 | 2 1 5764 1781 451 | 1 1 3983 2031 452 | ok 453 | ``` 454 | 455 | ### Call tree statistics: `top_call_trees` 456 | 457 | The function `tr:top_call_trees/0` makes it possible to detect complete call trees that repeat several times, 458 | where corresponding function calls and returns have the same arguments and return values, respectively. 459 | When such functions take a lot of time and do not have useful side effects, they can be often optimized. 460 | 461 | As an example, let's trace the call to a function which calculates the 4th element of the Fibonacci Sequence 462 | in a recursive way. The `trace` table should be empty, so let's clean it up first: 463 | 464 | ```erlang 465 | 23> tr:clean(). 466 | ok 467 | 24> tr:trace([tr_SUITE]). 468 | ok 469 | 25> tr_SUITE:fib(4). 470 | 3 471 | 26> tr:stop_tracing(). 472 | ok 473 | ``` 474 | 475 | Now it is possible to print the most time consuming call trees that repeat at least twice: 476 | 477 | ```erlang 478 | 27> tr:top_call_trees(). 479 | [{13,2, 480 | #node{module = tr_SUITE,function = fib, 481 | args = [2], 482 | children = [#node{module = tr_SUITE,function = fib, 483 | args = [1], 484 | children = [], 485 | result = {return,1}}, 486 | #node{module = tr_SUITE,function = fib, 487 | args = [0], 488 | children = [], 489 | result = {return,0}}], 490 | result = {return,1}}}, 491 | {5,3, 492 | #node{module = tr_SUITE,function = fib, 493 | args = [1], 494 | children = [], 495 | result = {return,1}}}] 496 | ``` 497 | 498 | The resulting list contains tuples `{Time, Count, Tree}` where `Time` is the accumulated time (in microseconds) spent in the tree, 499 | and `Count` is the number of times the tree repeated. The list is sorted by `Time`, descending. 500 | In the example above `fib(2)` was called twice and `fib(1)` was called 3 times, 501 | what already shows that the recursive implementation is suboptimal. 502 | 503 | There is also `tr:top_call_trees/1`, which takes a [map of options](https://hexdocs.pm/erlang_doctor/0.2.8/tr.html#t:top_call_trees_options/0), including: 504 | - `output` is `reduced` by default, but it can be set to `complete` where subtrees of already listed trees are also listed. 505 | - `min_count` is the minimum number of times a tree has to occur to be listed, the default is 2. 506 | - `min_time` is the minimum accumulated time for a tree, by default there is no minimum. 507 | - `max_size` is the maximum number of trees presented, the default is 10. 508 | 509 | As an exercise, try calling `tr:top_call_trees(#{min_count => 1000})` for `fib(20)`. 510 | 511 | ## Exporting and importing traces 512 | 513 | To get the current table name, use `tr:tab/0`: 514 | 515 | ```erlang 516 | 28> tr:tab(). 517 | trace 518 | ``` 519 | 520 | To switch to a new table, use `tr:set_tab/1`. The table need not exist. 521 | 522 | ```erlang 523 | 29> tr:set_tab(tmp). 524 | ok 525 | ``` 526 | 527 | Now you can collect traces to the new table without changing the original one. 528 | 529 | ```erlang 530 | 30> tr:trace([lists]), lists:seq(1, 10), tr:stop_tracing(). 531 | ok 532 | 31> tr:select(). 533 | [#tr{index = 1, pid = <0.175.0>, event = call, 534 | mfa = {lists, ukeysort, 2}, 535 | data = [1, 536 | [{'Traces', [#tr{index = 2, pid = <0.175.0>, event = call, 537 | mfa = {tr_SUITE, sleepy_factorial, 1}, 538 | data = [2], 539 | (...) 540 | ``` 541 | 542 | You can dump a table to file with `tr:dump/1` - let's dump the `tmp` table: 543 | 544 | ```erlang 545 | 32> tr:dump("tmp.ets"). 546 | ok 547 | ``` 548 | 549 | In a new Erlang session we can load the data with `tr:load/1`. This will set the current table name to `tmp`. 550 | 551 | ```erlang 552 | 1> tr:start(). 553 | {ok, <0.181.0>} 554 | 2> tr:load("tmp.ets"). 555 | {ok, tmp} 556 | 3> tr:select(). 557 | (...) 558 | 4> tr:tab(). 559 | tmp 560 | ``` 561 | 562 | Finally, you can remove all traces from the ETS table with `tr:clean/0`. 563 | 564 | ```erlang 565 | 5> tr:clean(). 566 | ok 567 | ``` 568 | 569 | To stop `tr`, just call `tr:stop/0`. 570 | 571 | # Example use cases 572 | 573 | ## Debugging a vague error 574 | 575 | While reworking the LDAP connection layer in [MongooseIM](https://github.com/esl/MongooseIM), the following error occured in the logs: 576 | 577 | ``` 578 | 14:46:35.002 [warning] lager_error_logger_h dropped 79 messages in the last second that exceeded the limit of 50 messages/sec 579 | 14:46:35.002 [error] gen_server 'wpool_pool-mongoose_wpool$ldap$global$bind-1' terminated with reason: no case clause matching {badkey,handle} in wpool_process:handle_info/2 line 123 580 | 14:46:35.003 [error] CRASH REPORT Process 'wpool_pool-mongoose_wpool$ldap$global$bind-1' with 1 neighbours crashed with reason: no case clause matching {badkey,handle} in wpool_process:handle_info/2 line 123 581 | 14:46:35.003 [error] Supervisor 'wpool_pool-mongoose_wpool$ldap$global$bind-process-sup' had child 'wpool_pool-mongoose_wpool$ldap$global$bind-1' started with wpool_process:start_link('wpool_pool-mongoose_wpool$ldap$global$bind-1', mongoose_ldap_worker, [{port,3636},{encrypt,tls},{tls_options,[{verify,verify_peer},{cacertfile,"priv/ssl/cacert.pem"},...]}], [{queue_manager,'wpool_pool-mongoose_wpool$ldap$global$bind-queue-manager'},{time_checker,'wpool_pool-mongoose_wpool$ldap$global$bind-time-checker'},...]) at <0.28894.0> exit with reason no case clause matching {badkey,handle} in wpool_process:handle_info/2 line 123 in context child_terminated 582 | 14:46:35.009 [info] Connected to LDAP server 583 | 14:46:35.009 [error] gen_server 'wpool_pool-mongoose_wpool$ldap$global$default-1' terminated with reason: no case clause matching {badkey,handle} in wpool_process:handle_info/2 line 123 584 | 14:46:35.009 [error] CRASH REPORT Process 'wpool_pool-mongoose_wpool$ldap$global$default-1' with 1 neighbours crashed with reason: no case clause matching {badkey,handle} in wpool_process:handle_info/2 line 123 585 | ``` 586 | 587 | As this messages appear every 10 seconds (on each attempt to reconnect to LDAP), we can start tracing. 588 | The most lkely culprit is the `mongoose_ldap_worker` module, so let's trace it: 589 | 590 | ```erlang 591 | (mongooseim@localhost)16> tr:trace([mongoose_ldap_worker]). 592 | ok 593 | ``` 594 | 595 | A few seconds (and error messages) later we can check the traces for the `badkey` value we saw in the logs: 596 | 597 | ```erlang 598 | (mongooseim@localhost)17> tr:filter(fun(T) -> tr:contains_data(badkey, T) end). 599 | [#tr{index = 255, pid = <0.8118.1>, event = exception, 600 | mfa = {mongoose_ldap_worker, connect, 1}, 601 | data = {error, {badkey, handle}}, 602 | ts = 1557838064073778}, 603 | (...) 604 | ``` 605 | 606 | This means that the key `handle` was missing from a map. 607 | Let's see the traceback to find the exact place in the code: 608 | 609 | ```erlang 610 | (mongooseim@localhost)18> tr:traceback(fun(T) -> tr:contains_data(badkey, T) end). 611 | [#tr{index = 254, pid = <0.8118.1>, event = call, 612 | mfa = {mongoose_ldap_worker, connect, 1}, 613 | data = [#{connect_interval => 10000, encrypt => tls, password => <<>>, 614 | port => 3636, root_dn => <<>>, 615 | servers => ["localhost"], 616 | tls_options => 617 | [{verify, verify_peer}, 618 | {cacertfile, "priv/ssl/cacert.pem"}, 619 | {certfile, "priv/ssl/fake_cert.pem"}, 620 | {keyfile, "priv/ssl/fake_key.pem"}]}], 621 | ts = 1557838064052121}, ...] 622 | ``` 623 | 624 | We can see that the `handle` key is missing from the map passed to `mongoose_ldap_worker:connect/1`. 625 | After looking at the source code of this function and searching for `handle` we can see only one matching line: 626 | 627 | ```erlang 628 | State#{handle := Handle}; 629 | ``` 630 | 631 | The `:=` operator assumes that the key is already present in the map. 632 | The solution would be to either change it to `=>` or ensure that the map already contains that key. 633 | 634 | ## Loading traces to the `trace` table after tracing to file 635 | 636 | It's possible to use `tr` with a file generated by `dbg:trace_port/2` tracing. 637 | The file may be generated on another system. 638 | 639 | ```erlang 640 | 1> {ok, St} = tr:init({}). 641 | {ok, #{index => 0, traced_modules => []}} 642 | 2> dbg:trace_client(file, "/Users/erszcz/work/myproject/long-pong.dbg.trace", {fun tr:handle_trace/2, St}). 643 | <0.178.0> 644 | 3> tr:select(). 645 | [#tr{index = 1, pid = <14318.7477.2537>, event = call, 646 | mfa = {mod_ping, user_ping_response_metric, 3}, 647 | data = [{jid, <<"user1">>, <<"myproject.com">>, <<"res1">>, 648 | <<"user1">>, <<"myproject.com">>, <<"res1">>}, 649 | {iq, <<"EDC1944CF88F67C6">>, result, <<>>, <<"en">>, []}, 650 | 5406109], 651 | ts = 1553517330696515}, 652 | ... 653 | ``` 654 | -------------------------------------------------------------------------------- /include/tr.hrl: -------------------------------------------------------------------------------- 1 | -record(tr, {index :: tr:index(), 2 | pid :: pid(), 3 | event :: call | return | exception | send | recv, 4 | mfa = no_mfa :: mfa() | no_mfa, 5 | data :: term(), 6 | ts :: integer(), 7 | info = no_info :: tr:recipient() | no_info}). 8 | 9 | -record(node, {module :: module(), 10 | function :: atom(), 11 | args :: list(), 12 | children = [] :: [#node{}], 13 | result :: {return | exception, any()}}). 14 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info]}. 2 | 3 | {deps, []}. 4 | 5 | {xref_checks, [undefined_function_calls]}. 6 | 7 | {project_plugins, [rebar3_ex_doc]}. 8 | 9 | {hex, [{doc, ex_doc}]}. 10 | 11 | {ex_doc, [{extras, ["README.md"]}, 12 | {main, "README.md"}, 13 | {homepage_url, "https://github.com/chrzaszcz/erlang_doctor"}, 14 | {source_url, "https://github.com/chrzaszcz/erlang_doctor"}, 15 | {api_reference, false} 16 | ]}. 17 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/erlang_doctor.app.src: -------------------------------------------------------------------------------- 1 | {application, erlang_doctor, 2 | [{description, "Lightweight tracing, debugging and profiling utility for Erlang"}, 3 | {vsn, "0.2.8"}, 4 | {registered, []}, 5 | {mod, { erlang_doctor_app, []}}, 6 | {applications, 7 | [kernel, 8 | stdlib 9 | ]}, 10 | {env, [{limit, infinity}]}, 11 | {modules, []}, 12 | {maintainers, []}, 13 | {licenses, ["Apache 2.0"]}, 14 | {links, [{"GitHub", "https://github.com/chrzaszcz/erlang_doctor"}]} 15 | ]}. 16 | -------------------------------------------------------------------------------- /src/erlang_doctor_app.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %% @doc erlang_doctor public API 3 | %% @end 4 | %%%------------------------------------------------------------------- 5 | 6 | -module(erlang_doctor_app). 7 | 8 | -behaviour(application). 9 | 10 | %% Application callbacks 11 | -export([start/2, stop/1]). 12 | 13 | %%==================================================================== 14 | %% API 15 | %%==================================================================== 16 | 17 | start(_StartType, _StartArgs) -> 18 | erlang_doctor_sup:start_link(). 19 | 20 | %%-------------------------------------------------------------------- 21 | stop(_State) -> 22 | ok. 23 | 24 | %%==================================================================== 25 | %% Internal functions 26 | %%==================================================================== 27 | -------------------------------------------------------------------------------- /src/erlang_doctor_sup.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %% @doc erlang_doctor top level supervisor. 3 | %% @end 4 | %%%------------------------------------------------------------------- 5 | 6 | -module(erlang_doctor_sup). 7 | 8 | -behaviour(supervisor). 9 | 10 | %% API 11 | -export([start_link/0]). 12 | 13 | %% Supervisor callbacks 14 | -export([init/1]). 15 | 16 | -define(SERVER, ?MODULE). 17 | 18 | %% API functions 19 | 20 | start_link() -> 21 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 22 | 23 | %% Supervisor callbacks 24 | 25 | -spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. 26 | init([]) -> 27 | {ok, {#{}, [tr_spec()]}}. 28 | 29 | %% Internal functions 30 | 31 | -spec tr_spec() -> supervisor:child_spec(). 32 | tr_spec() -> 33 | #{id => tr, 34 | start => {tr, start_link, []}, 35 | shutdown => brutal_kill}. 36 | -------------------------------------------------------------------------------- /src/tr.erl: -------------------------------------------------------------------------------- 1 | %% @doc Erlang Doctor API module. 2 | -module(tr). 3 | 4 | -behaviour(gen_server). 5 | 6 | %% API - capturing, data manipulation 7 | -export([start_link/0, start_link/1, 8 | start/0, start/1, 9 | trace_app/1, 10 | trace_apps/1, 11 | trace/1, trace/2, 12 | stop_tracing/0, 13 | stop/0, 14 | tab/0, 15 | set_tab/1, 16 | load/1, 17 | dump/1, 18 | clean/0]). 19 | 20 | %% API - analysis 21 | -export([select/0, select/1, select/2, 22 | filter/1, filter/2, 23 | traceback/1, traceback/2, 24 | tracebacks/1, tracebacks/2, 25 | roots/1, root/1, 26 | range/1, range/2, 27 | ranges/1, ranges/2, 28 | call_tree_stat/0, call_tree_stat/1, 29 | reduce_call_trees/1, 30 | top_call_trees/0, top_call_trees/1, top_call_trees/2, 31 | print_sorted_call_stat/2, 32 | sorted_call_stat/1, 33 | call_stat/1, call_stat/2]). 34 | 35 | %% API - utilities 36 | -export([contains_data/2, 37 | do/1, 38 | lookup/1, 39 | app_modules/1, 40 | mfarity/1, 41 | mfargs/2, 42 | ts/1]). 43 | 44 | %% gen_server callbacks 45 | -export([init/1, 46 | handle_call/3, 47 | handle_cast/2, 48 | handle_info/2, 49 | terminate/2, 50 | code_change/3]). 51 | 52 | -type recipient() :: {pid(), boolean()}. % Recipient pid with a boolean indicating if it exists. 53 | 54 | %% copied, not included from tr.hrl to make it self-contained 55 | -record(tr, {index :: index(), 56 | pid :: pid(), 57 | event :: call | return | exception | send | recv, 58 | mfa = no_mfa :: mfa() | no_mfa, 59 | data :: term(), 60 | ts :: integer(), 61 | info = no_info :: recipient() | no_info}). 62 | 63 | -define(is_return(Event), (Event =:= return orelse Event =:= exception)). 64 | -define(is_msg(Event), (Event =:= send orelse Event =:= recv)). 65 | 66 | -type tr() :: #tr{}. 67 | %% Trace record, storing one collected trace event. 68 | %% 69 | %% Record fields: 70 | %% 79 | 80 | -type pred() :: fun((tr()) -> boolean()). 81 | %% Predicate returning `true' for matching traces. 82 | %% 83 | %% For other traces it can return a different value or fail. 84 | 85 | -type selector(Data) :: fun((tr()) -> Data). 86 | %% Trace selector function. 87 | %% 88 | %% For selected traces, it returns `Data'. For other traces, it should fail. 89 | 90 | -type call_count() :: non_neg_integer(). % Total number of aggregated calls. 91 | -type acc_time() :: non_neg_integer(). % Total accumulated time. 92 | -type own_time() :: non_neg_integer(). % Total own time (without other called functions). 93 | -type pids() :: [pid()] | all. % A list of processes to trace. Default: `all'. 94 | -type limit() :: pos_integer() | infinity. % Maximum number of items. 95 | -type index() :: pos_integer(). % Unique, auto-incremented identifier of a `t:tr()' record. 96 | -type table() :: atom(). % ETS table name. 97 | -type mfargs() :: {module(), atom(), list()}. % Module, function and arguments. 98 | 99 | -type init_options() :: #{tab => table(), index => index(), limit => limit()}. 100 | %% Initialization options. 101 | %% 102 | %% `tab' is the ETS table used for storing traces (default: `trace'). 103 | %% `index' is the index value of the first inserted trace (default: 1). 104 | %% When size of `tab' reaches the optional `limit', tracing is stopped. 105 | 106 | -type state() :: #{tab := table(), 107 | index := index(), 108 | limit := limit(), 109 | trace := none | trace_spec(), 110 | tracer_pid := none | pid()}. 111 | 112 | -type trace_spec() :: #{modules := module_spec(), 113 | pids := pids(), 114 | msg := message_event_types(), 115 | msg_trigger := msg_trigger()}. 116 | -type trace_options() :: #{modules => module_spec(), 117 | pids => pids(), 118 | msg => message_event_types(), 119 | msg_trigger => msg_trigger()}. 120 | %% Options for tracing. 121 | 122 | -type message_event_types() :: send | recv | all | none. 123 | %% Message event types to trace. Default: `none'. 124 | 125 | -type msg_trigger() :: after_traced_call | always. 126 | %% Condition checked before collecting message traces for a process. 127 | %% 128 | %% `after_traced_call' (default) means that a process needs to call at least one traced 129 | %% function before its message events start being collected. 130 | %% `always' means that messages for all traced processes are collected. 131 | 132 | -type module_spec() :: [module() | mfa()]. 133 | %% Specifies traced modules and/or individual functions. Default: `[]'. 134 | 135 | -type erlang_trace_flags() :: [call | timestamp | send | 'receive']. 136 | -type traced_pids_tab() :: none | ets:table(). 137 | 138 | -type tr_source() :: table() | [tr()]. 139 | %% Source of traces: an ETS table or a list of traces. Default: `tab()'. 140 | 141 | -type range_options() :: #{tab => tr_source(), 142 | max_depth => limit(), 143 | output => range_output()}. 144 | %% Options for trace ranges. 145 | %% 146 | %% Optional `limit' is the maximum depth of calls in the returned ranges. 147 | %% All traces (including messages) exceeding that depth are skipped. 148 | 149 | -type range_output() :: complete | incomplete | all. 150 | %% Which ranges to return. Incomplete ranges are missing at least one return. 151 | %% By default, all ranges are returned. 152 | 153 | -type tb_options() :: #{tab => tr_source(), 154 | output => tb_output(), 155 | format => tb_format(), 156 | order => tb_order(), 157 | limit => limit()}. 158 | %% Traceback options. 159 | %% 160 | %% Optional `limit' is the maximum number of tracebacks to collect 161 | %% before filtering them according to `output'. 162 | 163 | -type tb_output() :: shortest | longest | all. 164 | %% Which tracebacks to return if they overlap. Default: `shortest'. 165 | 166 | -type tb_format() :: list | tree | root. 167 | %% Format in which tracebacks are returned. 168 | %% 169 | %% `list' (default) returns a list of tracebacks. 170 | %% `tree' merges them into a list of trees. 171 | %% `root' returns only the root of each tree. 172 | 173 | -type tb_order() :: top_down | bottom_up. 174 | %% Order of calls in each returned traceback. Default: `top_down'. 175 | 176 | -type tb_tree() :: tr() | {tr(), [tb_tree()]}. 177 | %% Multiple tracebacks with a common root merged into a tree structure. 178 | 179 | -type tb_acc_tree() :: [{tr(), tb_acc_tree()}]. 180 | -type tb_acc_list() :: [[tr()]]. 181 | -type tb_acc() :: tb_acc_tree() | tb_acc_list(). 182 | 183 | -type call() :: {call, {module(), atom(), list()}}. 184 | -type result() :: {return | exception, any()}. % Result of a function call. 185 | -type simple_tr() :: call() | result(). 186 | 187 | -type call_tree_stat_options() :: #{tab => table()}. 188 | -type call_tree_stat_state() :: #{pid_states := map(), tab := ets:tid()}. 189 | 190 | -type pid_call_state() :: [tree() | call()]. 191 | 192 | -type top_call_trees_output() :: reduced | complete. 193 | %% Specifies the behaviour for overlapping call trees. 194 | %% 195 | %% `reduced' (default) hides subtrees, while `complete' keeps them. 196 | 197 | -type top_call_trees_options() :: #{max_size => pos_integer(), 198 | min_count => call_tree_count(), 199 | min_time => acc_time(), 200 | output => top_call_trees_output()}. 201 | %% Options for repeated call tree statistics. 202 | %% 203 | %% `min_time' is an optional minimum accumulated time of a tree. 204 | %% `min_count' (default: 2) specifies minimum number of repetitions of a tree. 205 | %% `max_size' (default: 10) specifies maximum number of listed call trees. 206 | 207 | -type call_tree_count() :: pos_integer(). % Number of occurrences of a given call tree. 208 | 209 | -record(node, {module :: module(), 210 | function :: atom(), 211 | args = [] :: list(), 212 | children = [] :: [#node{}], 213 | result :: result()}). 214 | 215 | -type tree() :: #node{}. 216 | %% Function call tree node. 217 | %% 218 | %% Record fields: 219 | %% 226 | 227 | -type tree_item() :: {acc_time(), call_tree_count(), tree()}. 228 | %% Function call tree with its accumulated time and number of repetitions. 229 | 230 | -export_type([tr/0, index/0, recipient/0]). 231 | 232 | %% API - capturing, data manipulation 233 | 234 | %% @doc Starts `tr' as part of a supervision tree. 235 | %% @see start/1 236 | -spec start_link() -> {ok, pid()}. 237 | start_link() -> 238 | start_link(#{}). 239 | 240 | %% @doc Start `tr' as part of a supervision tree. 241 | %% @see start/1 242 | -spec start_link(init_options()) -> {ok, pid()}. 243 | start_link(Opts) -> 244 | gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). 245 | 246 | %% @doc Starts `tr' as a stand-alone `gen_server'. Intended for interactive use. 247 | %% @see start/1 248 | -spec start() -> {ok, pid()}. 249 | start() -> 250 | start(#{}). 251 | 252 | %% @doc Starts `tr' as a stand-alone `gen_server'. Intended for interactive use. 253 | %% 254 | %% You can override the selected `Opts'. 255 | -spec start(init_options()) -> {ok, pid()}. 256 | start(Opts) -> 257 | gen_server:start({local, ?MODULE}, ?MODULE, Opts, []). 258 | 259 | %% @doc Starts tracing of all modules in an application. 260 | -spec trace_app(atom()) -> ok | {error, already_tracing}. 261 | trace_app(App) -> 262 | trace_apps([App]). 263 | 264 | %% @doc Starts tracing of all modules in all provided applications. 265 | -spec trace_apps([atom()]) -> ok | {error, already_tracing}. 266 | trace_apps(Apps) -> 267 | trace(lists:flatmap(fun app_modules/1, Apps)). 268 | 269 | %% @doc Starts tracing of the specified functions/modules and/or message events. 270 | %% 271 | %% You can either provide a list of modules/functions or a more generic map of options. 272 | -spec trace(module_spec() | trace_options()) -> ok | {error, already_tracing}. 273 | trace(Modules) when is_list(Modules) -> 274 | trace(#{modules => Modules}); 275 | trace(Opts) -> 276 | DefaultOpts = #{modules => [], pids => all, 277 | msg => none, msg_trigger => after_traced_call}, 278 | gen_server:call(?MODULE, {start_trace, call, maps:merge(DefaultOpts, Opts)}). 279 | 280 | %% @doc Starts tracing of the specified functions/modules in specific processes. 281 | -spec trace(module_spec(), pids()) -> ok | {error, already_tracing}. 282 | trace(Modules, Pids) -> 283 | trace(#{modules => Modules, pids => Pids}). 284 | 285 | %% @doc Stops tracing, disabling all trace specs. 286 | %% 287 | %% Any future messages from the Erlang tracer will be ignored. 288 | -spec stop_tracing() -> ok | {error, not_tracing}. 289 | stop_tracing() -> 290 | gen_server:call(?MODULE, {stop_trace, call}). 291 | 292 | %% @doc Stops the whole `tr' server process. 293 | -spec stop() -> ok. 294 | stop() -> 295 | gen_server:stop(?MODULE). 296 | 297 | %% @doc Returns the name of the current ETS trace table in use. 298 | -spec tab() -> table(). 299 | tab() -> 300 | gen_server:call(?MODULE, get_tab). 301 | 302 | %% @doc Sets a new ETS table for collecting traces, creating it if it doesn't exist. 303 | -spec set_tab(table()) -> ok. 304 | set_tab(Tab) when is_atom(Tab) -> 305 | gen_server:call(?MODULE, {set_tab, Tab}). 306 | 307 | %% @doc Loads an ETS trace table from a file, and makes it the current table. 308 | %% 309 | %% Fails if the table already exists. 310 | -spec load(file:name_all()) -> {ok, table()} | {error, any()}. 311 | load(File) when is_binary(File) -> 312 | load(binary_to_list(File)); 313 | load(File) when is_list(File) -> 314 | gen_server:call(?MODULE, {load, File}, timer:minutes(2)). 315 | 316 | %% @doc Dumps the `tab()' table to a file. 317 | -spec dump(file:name_all()) -> ok | {error, any()}. 318 | dump(File) when is_binary(File) -> 319 | dump(binary_to_list(File)); 320 | dump(File) when is_list(File) -> 321 | gen_server:call(?MODULE, {dump, File}, timer:minutes(2)). 322 | 323 | %% @doc Removes all traces from the current ETS table. 324 | -spec clean() -> ok. 325 | clean() -> 326 | gen_server:call(?MODULE, clean). 327 | 328 | %% API - analysis 329 | 330 | %% @doc Returns a list of all collected traces from `tab()'. 331 | -spec select() -> [tr()]. 332 | select() -> 333 | ets:tab2list(tab()). 334 | 335 | %% @doc Selects data from matching traces from `tab()' with `ets:fun2ms(F)'. 336 | -spec select(selector(Data)) -> [Data]. 337 | select(F) -> 338 | ets:select(tab(), ets:fun2ms(F)). 339 | 340 | %% @doc Selects data from matching traces from `tab()' with `ets:fun2ms(F)'. 341 | %% 342 | %% Additionally, the selected traces have to contain `DataVal' in `#tr.data'. 343 | %% `DataVal' can occur in (possibly nested) tuples, maps or lists. 344 | -spec select(selector(Data), term()) -> [Data]. 345 | select(F, DataVal) -> 346 | MS = ets:fun2ms(F), 347 | SelectRes = ets:select(tab(), MS, 1000), 348 | select(MS, DataVal, [], SelectRes). 349 | 350 | %% @doc Returns matching traces from `tab()'. 351 | -spec filter(pred()) -> [tr()]. 352 | filter(F) -> 353 | filter(F, tab()). 354 | 355 | %% @doc Returns matching traces from `t:tr_source()'. 356 | -spec filter(pred(), tr_source()) -> [tr()]. 357 | filter(F, Tab) -> 358 | Traces = foldl(fun(Tr, State) -> filter_trace(F, Tr, State) end, [], Tab), 359 | lists:reverse(Traces). 360 | 361 | %% @doc Returns traceback of the first matching trace from `t:tr_source()'. 362 | %% 363 | %% Matching can be done with a predicate function, an index value or a `tr' record. 364 | %% Fails if no trace is matched. 365 | %% 366 | %% @see traceback/2 367 | -spec traceback(pred() | index() | tr()) -> [tr()]. 368 | traceback(Pred) -> 369 | traceback(Pred, #{}). 370 | 371 | %% @doc Returns traceback of the first matching trace from `t:tr_source()'. 372 | %% 373 | %% Fails if no trace is matched. 374 | %% The options `limit' and `format' do not apply. 375 | -spec traceback(pred() | index() | tr(), tb_options()) -> [tr()]. 376 | traceback(Index, Options) when is_integer(Index) -> 377 | traceback(fun(#tr{index = I}) -> Index =:= I end, Options); 378 | traceback(T = #tr{}, Options) -> 379 | traceback(fun(Tr) -> Tr =:= T end, Options); 380 | traceback(PredF, Options) when is_function(PredF, 1) -> 381 | [TB] = tracebacks(PredF, Options#{limit => 1, format => list}), 382 | TB. 383 | 384 | %% @doc Returns tracebacks of all matching traces from `tab()'. 385 | %% 386 | %% @see tracebacks/2 387 | -spec tracebacks(pred()) -> [[tr()]] | [tb_tree()]. 388 | tracebacks(PredF) -> 389 | tracebacks(PredF, #{}). 390 | 391 | %% @doc Returns tracebacks of all matching traces from `t:tr_source()'. 392 | -spec tracebacks(pred(), tb_options()) -> [[tr()]] | [tb_tree()]. 393 | tracebacks(PredF, Options) when is_map(Options) -> 394 | Tab = maps:get(tab, Options, tab()), 395 | Output = maps:get(output, Options, shortest), 396 | Format = maps:get(format, Options, list), 397 | Limit = maps:get(limit, Options, infinity), 398 | InitialState = #{tbs => [], call_stacks => #{}, 399 | output => Output, format => Format, 400 | count => 0, limit => Limit}, 401 | #{tbs := TBs} = 402 | foldl(fun(T, State) -> tb_step(PredF, T, State) end, InitialState, Tab), 403 | finalize_tracebacks(TBs, Output, Format, Options). 404 | 405 | %% @doc Returns the root call of each `t:tb_tree()` from the provided list. 406 | -spec roots([tb_tree()]) -> [tr()]. 407 | roots(Trees) -> 408 | lists:map(fun root/1, Trees). 409 | 410 | %% @doc Returns the root call of the provided `t:tb_tree()`. 411 | -spec root(tb_tree()) -> tr(). 412 | root(#tr{} = T) -> T; 413 | root({#tr{} = T, _}) -> T. 414 | 415 | %% @doc Returns a list of traces from `tab()' between the first matched call and the corresponding return. 416 | %% 417 | %% Matching can be done with a predicate function, an index value or a `t:tr()' record. 418 | %% Fails if no trace is matched. 419 | %% 420 | %% @see range/2 421 | -spec range(pred() | index() | tr()) -> [tr()]. 422 | range(PredF) -> 423 | range(PredF, #{}). 424 | 425 | %% @doc Returns a list of traces from `t:tr_source()' between the first matched call and the corresponding return. 426 | %% 427 | %% Fails if no call is matched. 428 | -spec range(pred() | index() | tr(), range_options()) -> [tr()]. 429 | range(Index, Options) when is_integer(Index) -> 430 | range(fun(#tr{index = I}) -> Index =:= I end, Options); 431 | range(T = #tr{}, Options) -> 432 | range(fun(Tr) -> Tr =:= T end, Options); 433 | range(PredF, Options) when is_function(PredF, 1) -> 434 | hd(ranges(PredF, Options)). 435 | 436 | %% @doc Returns lists of traces from `tab()' between matched calls and corresponding returns. 437 | %% 438 | %% @see ranges/2 439 | -spec ranges(pred()) -> [[tr()]]. 440 | ranges(PredF) -> 441 | ranges(PredF, #{}). 442 | 443 | %% @doc Returns lists of traces from `t:tr_source()' between matched calls and corresponding returns. 444 | -spec ranges(pred(), range_options()) -> [[tr()]]. 445 | ranges(PredF, Options) when is_map(Options) -> 446 | Tab = maps:get(tab, Options, tab()), 447 | Output = maps:get(output, Options, all), 448 | InitialState = maps:merge(#{traces => [], pid_states => #{}, max_depth => infinity}, Options), 449 | FinalState = foldl(fun(T, S) -> range_step(PredF, T, S) end, InitialState, Tab), 450 | complete_ranges(FinalState, Output) ++ incomplete_ranges(FinalState, Output). 451 | 452 | complete_ranges(#{}, incomplete) -> 453 | []; 454 | complete_ranges(#{traces := Traces}, Output) when Output =:= all; Output =:= complete -> 455 | lists:reverse(Traces). 456 | 457 | incomplete_ranges(#{}, complete) -> 458 | []; 459 | incomplete_ranges(#{pid_states := States}, Output) when Output =:= all; Output =:= incomplete -> 460 | lists:sort([Range || #{trace := Range} <- maps:values(States)]). 461 | 462 | %% @doc Prints sorted call time statistics for the selected traces from `tab()'. 463 | %% 464 | %% The statistics are sorted according to `t:acc_time()', descending. 465 | %% Only top `Limit' rows are printed. 466 | %% @see sorted_call_stat/1 467 | -spec print_sorted_call_stat(selector(_), limit()) -> ok. 468 | print_sorted_call_stat(KeyF, Limit) -> 469 | pretty_print_tuple_list(sorted_call_stat(KeyF), Limit). 470 | 471 | %% @doc Returns sorted call time statistics for the selected traces from `tab()'. 472 | %% 473 | %% The statistics are sorted according to `t:acc_time()', descending. 474 | %% @see call_stat/1 475 | -spec sorted_call_stat(selector(Key)) -> [{Key, call_count(), acc_time(), own_time()}]. 476 | sorted_call_stat(KeyF) -> 477 | lists:reverse(sort_by_time(call_stat(KeyF))). 478 | 479 | %% @doc Returns call time statistics for traces selected from `tab()'. 480 | %% 481 | %% @see call_stat/2 482 | -spec call_stat(selector(Key)) -> #{Key => {call_count(), acc_time(), own_time()}}. 483 | call_stat(KeyF) -> 484 | call_stat(KeyF, tab()). 485 | 486 | %% @doc Returns call time statistics for traces selected from `t:tr_source()'. 487 | %% 488 | %% Calls are aggregated by `Key' returned by `KeyF'. 489 | -spec call_stat(selector(Key), tr_source()) -> #{Key => {call_count(), acc_time(), own_time()}}. 490 | call_stat(KeyF, Tab) -> 491 | {#{}, State} = foldl(fun(Tr, State) -> call_stat_step(KeyF, Tr, State) end, {#{}, #{}}, Tab), 492 | State. 493 | 494 | %% API - utilities 495 | 496 | %% @doc Looks for `DataVal' in `#tr.data'. 497 | %% 498 | %% `DataVal' can occur in (possibly nested) tuples, maps or lists. 499 | -spec contains_data(term(), tr()) -> boolean(). 500 | contains_data(DataVal, #tr{data = Data}) -> 501 | contains_val(DataVal, Data). 502 | 503 | %% @doc Executes the function call for the provided `t:tr()' record or index. 504 | -spec do(tr()) -> term(). 505 | do(Index) when is_integer(Index) -> 506 | do(lookup(Index)); 507 | do(#tr{event = call, mfa = {M, F, Arity}, data = Args}) when length(Args) =:= Arity -> 508 | apply(M, F, Args). 509 | 510 | %% @doc Returns the `t:tr()' record from `tab()' for an index. 511 | -spec lookup(index()) -> tr(). 512 | lookup(Index) when is_integer(Index) -> 513 | [T] = ets:lookup(tab(), Index), 514 | T. 515 | 516 | %% @doc Returns all module names for an application. 517 | -spec app_modules(atom()) -> [module()]. 518 | app_modules(AppName) -> 519 | Path = filename:join(code:lib_dir(AppName), ebin), 520 | {ok, FileNames} = file:list_dir(Path), 521 | BeamFileNames = lists:filter(fun(Name) -> filename:extension(Name) =:= ".beam" end, FileNames), 522 | [list_to_atom(filename:rootname(Name)) || Name <- BeamFileNames]. 523 | 524 | %% @doc Replaces arguments with arity in an MFA tuple. 525 | -spec mfarity(mfa() | mfargs()) -> mfa(). 526 | mfarity({M, F, A}) -> {M, F, maybe_length(A)}. 527 | 528 | %% @doc Replaces arity with `Args' in an MFA tuple. 529 | -spec mfargs(mfa(), list()) -> mfargs(). 530 | mfargs({M, F, Arity}, Args) when length(Args) =:= Arity -> {M, F, Args}. 531 | 532 | %% @doc Returns human-readable timestamp according to RFC 3339. 533 | -spec ts(tr()) -> string(). 534 | ts(#tr{ts = TS}) -> calendar:system_time_to_rfc3339(TS, [{unit, microsecond}]). 535 | 536 | %% gen_server callbacks 537 | 538 | %% @private 539 | -spec init(init_options()) -> {ok, state()}. 540 | init(Opts) -> 541 | process_flag(trap_exit, true), 542 | Defaults = #{tab => default_tab(), 543 | index => initial_index(), 544 | limit => application:get_env(erlang_doctor, limit, infinity)}, 545 | FinalOpts = #{tab := Tab} = maps:merge(Defaults, Opts), 546 | State = maps:merge(FinalOpts, #{trace => none, tracer_pid => none}), 547 | create_tab(Tab), 548 | {ok, State}. 549 | 550 | %% @private 551 | -spec handle_call(any(), {pid(), any()}, state()) -> {reply, ok | {error, atom()}, state()}. 552 | handle_call({start_trace, call, Spec}, _From, State = #{trace := none, tracer_pid := none}) -> 553 | {reply, ok, start_trace(State, Spec)}; 554 | handle_call({start_trace, call, _Spec}, _From, State = #{trace := none}) -> 555 | {reply, {error, tracer_not_terminated}, State}; 556 | handle_call({start_trace, call, _Spec}, _From, State = #{}) -> 557 | {reply, {error, already_tracing}, State}; 558 | handle_call({stop_trace, call}, _From, State = #{trace := none}) -> 559 | {reply, {error, not_tracing}, State}; 560 | handle_call({stop_trace, call}, _From, State = #{trace := #{}}) -> 561 | {reply, ok, stop_trace(State)}; 562 | handle_call(clean, _From, State = #{tab := Tab}) -> 563 | ets:delete_all_objects(Tab), 564 | {reply, ok, State#{index := index(Tab)}}; 565 | handle_call({dump, File}, _From, State = #{tab := Tab}) -> 566 | Reply = ets:tab2file(Tab, File), 567 | {reply, Reply, State}; 568 | handle_call({load, File}, _From, State) -> 569 | Reply = ets:file2tab(File), 570 | NewState = case Reply of 571 | {ok, Tab} -> State#{tab := Tab, index := index(Tab)}; 572 | _ -> State 573 | end, 574 | {reply, Reply, NewState}; 575 | handle_call(get_tab, _From, State = #{tab := Tab}) -> 576 | {reply, Tab, State}; 577 | handle_call({set_tab, NewTab}, _From, State) -> 578 | case ets:info(NewTab) of 579 | undefined -> create_tab(NewTab); 580 | [_|_] -> ok 581 | end, 582 | {reply, ok, State#{tab := NewTab, index := index(NewTab)}}; 583 | handle_call(Req, From, State) -> 584 | logger:error("Unexpected call ~p from ~p.", [Req, From]), 585 | {reply, ok, State}. 586 | 587 | %% @private 588 | -spec handle_cast(any(), state()) -> {noreply, state()}. 589 | handle_cast(Msg, State) -> 590 | logger:error("Unexpected message ~p.", [Msg]), 591 | {noreply, State}. 592 | 593 | %% @private 594 | -spec handle_info(any(), state()) -> {noreply, state()}. 595 | handle_info({'EXIT', TracerPid, Reason}, State = #{tracer_pid := TracerPid}) -> 596 | logger:warning("Tracer process ~p exited with reason ~p.", [TracerPid, Reason]), 597 | {noreply, disable_trace_patterns(State#{tracer_pid := none})}; 598 | handle_info(Msg, State) -> 599 | logger:error("Unexpected message ~p.", [Msg]), 600 | {noreply, State}. 601 | 602 | %% @private 603 | -spec terminate(any(), state()) -> ok. 604 | terminate(_Reason, #{trace := none}) -> 605 | ok; 606 | terminate(_, State) -> 607 | stop_trace(State), 608 | ok. 609 | 610 | %% @private 611 | -spec code_change(any(), state(), any()) -> {ok, state()}. 612 | code_change(_OldVsn, State, _Extra) -> 613 | {ok, State}. 614 | 615 | %% Internal functions 616 | %%------------------- 617 | 618 | create_tab(Tab) -> 619 | ets:new(Tab, [named_table, public, ordered_set, {keypos, 2}]). 620 | 621 | select(_MS, _DataVal, DataAcc, '$end_of_table') -> 622 | lists:append(lists:reverse(DataAcc)); 623 | select(MS, DataVal, DataAcc, {Matched, Cont}) -> 624 | Filtered = lists:filter(fun(#tr{data = Data}) -> contains_val(DataVal, Data); 625 | (T) -> contains_val(DataVal, T) 626 | end, Matched), 627 | SelectRes = ets:select(Cont), 628 | select(MS, DataVal, [Filtered | DataAcc], SelectRes). 629 | 630 | filter_trace(F, T, State) -> 631 | case catch F(T) of 632 | true -> [T | State]; 633 | _ -> State 634 | end. 635 | 636 | contains_val(DataVal, DataVal) -> true; 637 | contains_val(DataVal, L) when is_list(L) -> lists:any(fun(El) -> contains_val(DataVal, El) end, L); 638 | contains_val(DataVal, T) when is_tuple(T) -> contains_val(DataVal, tuple_to_list(T)); 639 | contains_val(DataVal, M) when is_map(M) -> contains_val(DataVal, maps:to_list(M)); 640 | contains_val(_, _) -> false. 641 | 642 | index(Tab) -> 643 | case ets:last(Tab) of 644 | I when is_integer(I) -> I + 1; 645 | _ -> initial_index() 646 | end. 647 | 648 | -spec start_trace(state(), trace_spec()) -> state(). 649 | start_trace(State, Spec = #{modules := ModSpecs, pids := Pids}) -> 650 | TracerPid = spawn_link(fun() -> start_tracer_loop(State, Spec) end), 651 | [enable_trace_pattern(ModSpec) || ModSpec <- ModSpecs], 652 | set_tracing(Pids, true, trace_flags(TracerPid, Spec)), 653 | State#{trace := Spec, tracer_pid := TracerPid}. 654 | 655 | -spec stop_trace(state()) -> state(). 656 | stop_trace(State) -> 657 | disable_trace_patterns(shut_down_tracer(State)). 658 | 659 | -spec shut_down_tracer(state()) -> state(). 660 | shut_down_tracer(State = #{tracer_pid := none}) -> 661 | State; 662 | shut_down_tracer(State = #{tracer_pid := TracerPid}) -> 663 | exit(TracerPid, shutdown), 664 | receive 665 | {'EXIT', TracerPid, Reason} -> 666 | handle_tracer_exit(TracerPid, Reason), 667 | State#{tracer_pid := none} 668 | after 1000 -> 669 | logger:warning("Timeout when waiting for tracer process ~p to terminate.", [TracerPid]), 670 | State 671 | end. 672 | 673 | -spec disable_trace_patterns(state()) -> state(). 674 | disable_trace_patterns(State = #{trace := none}) -> 675 | State; 676 | disable_trace_patterns(State = #{trace := #{modules := ModSpecs}}) -> 677 | [disable_trace_pattern(ModSpec) || ModSpec <- ModSpecs], 678 | State#{trace := none}. 679 | 680 | handle_tracer_exit(_TracerPid, shutdown) -> 681 | ok; 682 | handle_tracer_exit(TracerPid, Reason) -> 683 | logger:error("Tracer process ~p exited with reason ~p.", [TracerPid, Reason]). 684 | 685 | -spec trace_flags(pid(), trace_spec()) -> erlang_trace_flags(). 686 | trace_flags(TracerPid, Spec) -> 687 | [{tracer, TracerPid} | basic_trace_flags()] ++ msg_trace_flags(Spec). 688 | 689 | -spec msg_trace_flags(trace_spec()) -> erlang_trace_flags(). 690 | msg_trace_flags(#{msg := all}) -> [send, 'receive']; 691 | msg_trace_flags(#{msg := send}) -> [send]; 692 | msg_trace_flags(#{msg := recv}) -> ['receive']; 693 | msg_trace_flags(#{msg := none}) -> []. 694 | 695 | basic_trace_flags() -> [call, timestamp]. 696 | 697 | -spec set_tracing(pids(), boolean(), erlang_trace_flags()) -> ok. 698 | set_tracing(all, How, FlagList) -> 699 | erlang:trace(all, How, FlagList), 700 | ok; 701 | set_tracing(Pids, How, FlagList) when is_list(Pids) -> 702 | [trace_pid(Pid, How, FlagList) || Pid <- Pids], 703 | ok. 704 | 705 | trace_pid(Pid, How, FlagList) when is_pid(Pid) -> 706 | try 707 | erlang:trace(Pid, How, FlagList) 708 | catch Class:Reason -> 709 | logger:warning("Could not switch tracing to ~p for pid ~p, ~p:~p", 710 | [How, Pid, Class, Reason]) 711 | end. 712 | 713 | -spec setup_msg_tracing(trace_spec()) -> traced_pids_tab(). 714 | setup_msg_tracing(#{msg := none}) -> 715 | none; 716 | setup_msg_tracing(#{msg := _, msg_trigger := Trigger}) -> 717 | case Trigger of 718 | after_traced_call -> ets:new(traced_pids, []); 719 | always -> none 720 | end. 721 | 722 | enable_trace_pattern(ModSpec) -> 723 | {MFA = {M, _, _}, Opts} = trace_pattern_and_opts(ModSpec), 724 | {module, _} = code:ensure_loaded(M), 725 | erlang:trace_pattern(MFA, [{'_', [], [{exception_trace}]}], Opts). 726 | 727 | disable_trace_pattern(Mod) -> 728 | {MFA, Opts} = trace_pattern_and_opts(Mod), 729 | erlang:trace_pattern(MFA, false, Opts). 730 | 731 | trace_pattern_and_opts(Mod) when is_atom(Mod) -> trace_pattern_and_opts({Mod, '_', '_'}); 732 | trace_pattern_and_opts({_, _, _} = MFA) -> {MFA, [local, call_time]}; 733 | trace_pattern_and_opts({Mod, Opts}) when is_atom(Mod) -> {{Mod, '_', '_'}, Opts}; 734 | trace_pattern_and_opts({{_M, _F, _A} = MFA, Opts}) -> {MFA, Opts}. 735 | 736 | -spec start_tracer_loop(state(), trace_spec()) -> no_return(). 737 | start_tracer_loop(#{tab := Tab, index := Index, limit := Limit}, Spec) -> 738 | PidsTab = setup_msg_tracing(Spec), 739 | loop(Tab, Index, Limit, PidsTab). 740 | 741 | -spec loop(table(), index(), limit(), traced_pids_tab()) -> no_return(). 742 | loop(_Tab, Index, Limit, _PidsTab) when Index > Limit -> 743 | logger:warning("Reached trace limit ~p, stopping the tracer.", [Limit]), 744 | exit(shutdown); 745 | loop(Tab, Index, Limit, PidsTab) -> 746 | receive 747 | Msg -> 748 | case handle_trace(Msg, Index, PidsTab) of 749 | skip -> 750 | loop(Tab, Index, Limit, PidsTab); 751 | Tr -> 752 | ets:insert(Tab, Tr), 753 | loop(Tab, Index + 1, Limit, PidsTab) 754 | end 755 | end. 756 | 757 | -spec handle_trace(term(), index(), traced_pids_tab()) -> tr() | skip. 758 | handle_trace({trace_ts, Pid, call, MFA = {_, _, Args}, TS}, Index, PidsTab) -> 759 | case are_messages_skipped(Pid, PidsTab) of 760 | true -> stop_skipping_messages(Pid, PidsTab); 761 | false -> ok 762 | end, 763 | #tr{index = Index, pid = Pid, event = call, mfa = mfarity(MFA), data = Args, 764 | ts = usec_from_now(TS)}; 765 | handle_trace({trace_ts, Pid, return_from, MFArity, Res, TS}, Index, _PidsTab) -> 766 | #tr{index = Index, pid = Pid, event = return, mfa = MFArity, data = Res, 767 | ts = usec_from_now(TS)}; 768 | handle_trace({trace_ts, Pid, exception_from, MFArity, {Class, Value}, TS}, Index, _PidsTab) -> 769 | #tr{index = Index, pid = Pid, event = exception, mfa = MFArity, data = {Class, Value}, 770 | ts = usec_from_now(TS)}; 771 | handle_trace({trace_ts, Pid, Event, Msg, To, TS}, Index, PidsTab) 772 | when Event =:= send orelse Event =:= send_to_non_existing_process -> 773 | case are_messages_skipped(Pid, PidsTab) of 774 | true -> skip; 775 | false -> #tr{index = Index, pid = Pid, event = send, data = Msg, ts = usec_from_now(TS), 776 | info = {To, Event =:= send}} 777 | end; 778 | handle_trace({trace_ts, Pid, 'receive', Msg, TS}, Index, PidsTab) -> 779 | case are_messages_skipped(Pid, PidsTab) of 780 | true -> skip; 781 | false -> #tr{index = Index, pid = Pid, event = recv, data = Msg, ts = usec_from_now(TS)} 782 | end; 783 | handle_trace(Trace, _Index, _State) -> 784 | logger:error("Tracer process received unexpected message ~p.", [Trace]), 785 | skip. 786 | 787 | are_messages_skipped(_Pid, none) -> 788 | false; 789 | are_messages_skipped(Pid, PidsTab) -> 790 | ets:lookup(PidsTab, Pid) =:= []. 791 | 792 | stop_skipping_messages(Pid, PidsTab) -> 793 | ets:insert(PidsTab, {Pid}). 794 | 795 | %% Filter tracebacks 796 | 797 | -spec tb_step(pred(), tr(), map()) -> map(). 798 | tb_step(PredF, T = #tr{pid = Pid, event = Event}, 799 | State = #{tbs := TBs, call_stacks := CallStacks, output := Output, format := Format, 800 | count := Count, limit := Limit}) -> 801 | CallStack = maps:get(Pid, CallStacks, []), 802 | NewStack = update_call_stack(T, CallStack), 803 | NewState = State#{call_stacks := CallStacks#{Pid => NewStack}}, 804 | case catch PredF(T) of 805 | true when Count < Limit -> 806 | TB = if Event =:= call -> NewStack; 807 | ?is_return(Event) orelse ?is_msg(Event) -> CallStack 808 | end, 809 | NewState#{tbs := add_tb(lists:reverse(TB), TBs, Output, Format), 810 | count := Count + 1}; 811 | _ -> 812 | NewState 813 | end. 814 | 815 | -spec update_call_stack(tr(), [tr()]) -> [tr()]. 816 | update_call_stack(T = #tr{event = call}, Stack) -> [T|Stack]; 817 | update_call_stack(#tr{event = Event, mfa = MFArity}, [#tr{mfa = MFArity} | Stack]) 818 | when ?is_return(Event) -> 819 | Stack; 820 | update_call_stack(#tr{event = Event, mfa = {M, F, Arity}}, Stack) when ?is_return(Event) -> 821 | logger:warning("Found a return trace from ~p:~p/~p without a call trace", [M, F, Arity]), 822 | Stack; 823 | update_call_stack(#tr{event = Event}, Stack) when ?is_msg(Event) -> 824 | Stack. 825 | 826 | -spec add_tb([tr()], tb_acc(), tb_output(), tb_format()) -> tb_acc(). 827 | add_tb(TB, TBs, all, list) -> [TB | TBs]; %% The only case which uses a list of TBs 828 | add_tb([], _Tree, shortest, _) -> []; %% Other cases use a tree 829 | add_tb([], Tree, Output, _Format) when Output =:= longest; 830 | Output =:= all -> Tree; 831 | add_tb([Call | Rest], Tree, Output, Format) -> 832 | case lists:keyfind(Call, 1, Tree) of 833 | {Call, SubTree} -> 834 | lists:keyreplace(Call, 1, Tree, {Call, add_tb(Rest, SubTree, Output, Format)}); 835 | false -> 836 | [{Call, add_tb(Rest, [], Output, Format)} | Tree] 837 | end. 838 | 839 | -spec finalize_tracebacks(tb_acc(), tb_output(), tb_format(), tb_options()) -> 840 | tb_tree() | [[tr()]]. 841 | finalize_tracebacks(TBs, all, list, Options) -> 842 | reorder_tb(lists:reverse(TBs), maps:get(order, Options, top_down)); 843 | finalize_tracebacks(TBs, _, list, Options) -> 844 | reorder_tb(tree_to_list(TBs), maps:get(order, Options, top_down)); 845 | finalize_tracebacks(TBs, _, root, _Options) -> 846 | lists:map(fun({T, _}) -> T end, lists:reverse(TBs)); 847 | finalize_tracebacks(TBs, _, tree, _Options) -> 848 | finalize_tree(TBs). 849 | 850 | -spec reorder_tb([[tr()]], tb_order()) -> [[tr()]]. 851 | reorder_tb(TBs, top_down) -> [lists:reverse(TB) || TB <- TBs]; 852 | reorder_tb(TBs, bottom_up) -> TBs. 853 | 854 | -spec tree_to_list(tb_acc_tree()) -> [[tr()]]. 855 | tree_to_list(Tree) -> 856 | lists:foldl(fun({K, []}, Res) -> [[K] | Res]; 857 | ({K, V}, Res) -> [[K | Rest] || Rest <- tree_to_list(V)] ++ Res 858 | end, [], Tree). 859 | 860 | %% Reverse order and simplify leaf nodes 861 | -spec finalize_tree(tb_acc_tree()) -> [tb_tree()]. 862 | finalize_tree(Tree) -> 863 | lists:foldl(fun({K, []}, Res) -> [K | Res]; 864 | ({K, V}, Res) -> [{K, finalize_tree(V)} | Res] 865 | end, [], Tree). 866 | 867 | %% Filter ranges 868 | 869 | range_step(PredF, T = #tr{pid = Pid}, State = #{traces := Traces, pid_states := States, max_depth := MaxDepth}) -> 870 | PidState = maps:get(Pid, States, no_state), 871 | case filter_range(PredF, T, PidState, MaxDepth) of 872 | {complete, Trace} -> State#{traces := [Trace | Traces], pid_states := maps:remove(Pid, States)}; 873 | {incomplete, NewPidState} -> State#{pid_states := States#{Pid => NewPidState}}; 874 | none -> State 875 | end. 876 | 877 | filter_range(_PredF, T = #tr{event = call}, State = #{depth := Depth, trace := Trace}, MaxDepth) 878 | when Depth < MaxDepth -> 879 | {incomplete, State#{depth => Depth + 1, trace => [T|Trace]}}; 880 | filter_range(_PredF, #tr{event = call}, State = #{depth := Depth, trace := Trace}, _) -> 881 | {incomplete, State#{depth => Depth + 1, trace => Trace}}; 882 | filter_range(PredF, T = #tr{event = call}, no_state, _) -> 883 | case catch PredF(T) of 884 | true -> {incomplete, #{depth => 1, trace => [T]}}; 885 | _ -> none 886 | end; 887 | filter_range(_PredF, T = #tr{event = Event}, #{depth := 1, trace := Trace}, _) when ?is_return(Event) -> 888 | {complete, lists:reverse([T|Trace])}; 889 | filter_range(_PredF, T = #tr{event = Event}, State = #{depth := Depth, trace := Trace}, MaxDepth) 890 | when ?is_return(Event), Depth =< MaxDepth -> 891 | {incomplete, State#{depth => Depth - 1, trace => [T|Trace]}}; 892 | filter_range(_PredF, #tr{event = Event}, State = #{depth := Depth, trace := Trace}, _) when ?is_return(Event) -> 893 | {incomplete, State#{depth => Depth - 1, trace => Trace}}; 894 | filter_range(_PredF, #tr{event = Event}, no_state, _) when ?is_return(Event) -> 895 | none; 896 | filter_range(_PrefF, T = #tr{event = Event}, State = #{depth := Depth, trace := Trace}, MaxDepth) 897 | when Depth < MaxDepth, ?is_msg(Event) -> 898 | {incomplete, State#{depth => Depth, trace => [T|Trace]}}; 899 | filter_range(_PrefF, #tr{event = Event}, _State, _MaxDepth) when ?is_msg(Event) -> 900 | none. 901 | 902 | %% Call stat 903 | 904 | sort_by_time(MapStat) -> 905 | lists:keysort(3, [{Key, Count, AccTime, OwnTime} || {Key, {Count, AccTime, OwnTime}} <- maps:to_list(MapStat)]). 906 | 907 | call_stat_step(_KeyF, #tr{event = Event}, State) when ?is_msg(Event) -> 908 | State; 909 | call_stat_step(KeyF, Tr = #tr{pid = Pid}, {ProcessStates, Stat}) -> 910 | {LastTr, Stack, Depths} = maps:get(Pid, ProcessStates, {no_tr, [], #{}}), 911 | {NewStack, Key} = get_key_and_update_stack(KeyF, Stack, Tr), 912 | TSDepth = maps:get(Key, Depths, none), 913 | NewDepths = set_ts_and_depth(Depths, Tr, Key, TSDepth), 914 | ProcessStates1 = ProcessStates#{Pid => {Tr, NewStack, NewDepths}}, 915 | Stat1 = update_stat(Stat, LastTr, Tr, Key, TSDepth, Stack), 916 | {ProcessStates1, Stat1}. 917 | 918 | get_key_and_update_stack(KeyF, Stack, T = #tr{event = call}) -> 919 | Key = try KeyF(T) 920 | catch _:_ -> no_key 921 | end, 922 | {[Key | Stack], Key}; 923 | get_key_and_update_stack(_KeyF, [Key | Rest], #tr{event = Event}) when ?is_return(Event) -> 924 | {Rest, Key}; 925 | get_key_and_update_stack(_, [], #tr{event = Event, mfa = MFA}) when ?is_return(Event) -> 926 | {M, F, A} = mfarity(MFA), 927 | logger:warning("Found a return trace from ~p:~p/~p without a call trace", [M, F, A]), 928 | {[], no_key}. 929 | 930 | set_ts_and_depth(Depths, _, no_key, none) -> 931 | Depths; 932 | set_ts_and_depth(Depths, #tr{event = call, ts = TS}, Key, none) -> 933 | Depths#{Key => {TS, 0}}; 934 | set_ts_and_depth(Depths, #tr{event = call}, Key, {OrigTS, Depth}) -> 935 | Depths#{Key => {OrigTS, Depth + 1}}; 936 | set_ts_and_depth(Depths, #tr{event = Event}, Key, {OrigTS, Depth}) when ?is_return(Event), Depth > 0 -> 937 | Depths#{Key => {OrigTS, Depth - 1}}; 938 | set_ts_and_depth(Depths, #tr{event = Event}, Key, {_OrigTS, 0}) when ?is_return(Event) -> 939 | maps:remove(Key, Depths). 940 | 941 | update_stat(Stat, LastTr, Tr, Key, TSDepth, Stack) -> 942 | Stat1 = update_count(Tr, Key, Stat), 943 | Stat2 = update_acc_time(TSDepth, Tr, Key, Stat1), 944 | ParentKey = case Stack of 945 | [K | _] -> K; 946 | [] -> no_key 947 | end, 948 | update_own_time(LastTr, ParentKey, Tr, Key, Stat2). 949 | 950 | update_count(Tr, Key, Stat) -> 951 | case count_key(Tr, Key) of 952 | no_key -> 953 | Stat; 954 | KeyToUpdate -> 955 | {Count, AccTime, OwnTime} = maps:get(KeyToUpdate, Stat, {0, 0, 0}), 956 | Stat#{KeyToUpdate => {Count + 1, AccTime, OwnTime}} 957 | end. 958 | 959 | update_acc_time(TSDepth, Tr, Key, Stat) -> 960 | case acc_time_key(TSDepth, Tr, Key) of 961 | no_key -> 962 | Stat; 963 | KeyToUpdate -> 964 | {Count, AccTime, OwnTime} = maps:get(KeyToUpdate, Stat, {0, 0, 0}), 965 | {OrigTS, _Depth} = TSDepth, 966 | NewAccTime = AccTime + Tr#tr.ts - OrigTS, 967 | Stat#{KeyToUpdate => {Count, NewAccTime, OwnTime}} 968 | end. 969 | 970 | update_own_time(LastTr, ParentKey, Tr, Key, Stat) -> 971 | case own_time_key(LastTr, ParentKey, Tr, Key) of 972 | no_key -> 973 | Stat; 974 | KeyToUpdate -> 975 | {Count, AccTime, OwnTime} = maps:get(KeyToUpdate, Stat, {0, 0, 0}), 976 | NewOwnTime = OwnTime + Tr#tr.ts - LastTr#tr.ts, 977 | Stat#{KeyToUpdate => {Count, AccTime, NewOwnTime}} 978 | end. 979 | 980 | count_key(#tr{event = call}, Key) 981 | when Key =/= no_key -> Key; 982 | count_key(#tr{}, _) -> no_key. 983 | 984 | acc_time_key({_, 0}, #tr{event = Event}, Key) 985 | when ?is_return(Event), Key =/= no_key -> Key; 986 | acc_time_key(_, #tr{}, _) -> no_key. 987 | 988 | own_time_key(#tr{event = call}, PKey, #tr{}, _) 989 | when PKey =/= no_key -> PKey; 990 | own_time_key(#tr{event = LEvent}, PKey, #tr{event = call}, _) 991 | when ?is_return(LEvent), PKey =/= no_key -> PKey; 992 | own_time_key(_, _, #tr{event = Event}, Key) 993 | when ?is_return(Event), Key =/= no_key -> Key; 994 | own_time_key(_, _, #tr{}, _) -> no_key. 995 | 996 | %% Call tree statistics for redundancy check 997 | 998 | %% @private 999 | -spec call_tree_stat() -> ets:tid(). 1000 | call_tree_stat() -> 1001 | call_tree_stat(#{}). 1002 | 1003 | %% @private 1004 | -spec call_tree_stat(call_tree_stat_options()) -> ets:tid(). 1005 | call_tree_stat(Options) when is_map(Options) -> 1006 | TraceTab = maps:get(tab, Options, tab()), 1007 | CallTreeTab = ets:new(call_tree_stat, [public, {keypos, 3}]), 1008 | InitialState = maps:merge(#{pid_states => #{}, tab => CallTreeTab}, Options), 1009 | foldl(fun(T, S) -> call_tree_stat_step(T, S) end, InitialState, TraceTab), 1010 | CallTreeTab. 1011 | 1012 | -spec call_tree_stat_step(tr(), call_tree_stat_state()) -> call_tree_stat_state(). 1013 | call_tree_stat_step(#tr{event = Event}, State) when ?is_msg(Event) -> 1014 | State; 1015 | call_tree_stat_step(Tr = #tr{pid = Pid, ts = TS}, State = #{pid_states := PidStates, tab := TreeTab}) -> 1016 | PidState = maps:get(Pid, PidStates, []), 1017 | Item = simplify_trace_item(Tr), 1018 | {Status, NewPidState} = update_call_trees(Item, TS, PidState), 1019 | case Status of 1020 | {new_node, CallTS} -> 1021 | insert_call_tree(hd(NewPidState), TS - CallTS, TreeTab); 1022 | no_new_nodes -> 1023 | ok 1024 | end, 1025 | NewPidStates = PidStates#{Pid => NewPidState}, 1026 | State#{pid_states => NewPidStates}. 1027 | 1028 | -spec simplify_trace_item(tr()) -> simple_tr(). 1029 | simplify_trace_item(#tr{event = call, mfa = MFA, data = Args}) -> 1030 | {call, mfargs(MFA, Args)}; 1031 | simplify_trace_item(#tr{event = return, data = Value}) -> 1032 | {return, Value}; 1033 | simplify_trace_item(#tr{event = exception, data = Value}) -> 1034 | {exception, Value}. 1035 | 1036 | -spec update_call_trees(simple_tr(), integer(), pid_call_state()) -> 1037 | {no_new_nodes | {new_node, integer()}, pid_call_state()}. 1038 | update_call_trees(Item = {call, _}, TS, PidState) -> 1039 | {no_new_nodes, [{Item, TS} | PidState]}; 1040 | update_call_trees(Item, _TS, PidState) -> 1041 | {CallTS, NewPidState} = build_node(PidState, #node{result = Item}), 1042 | {{new_node, CallTS}, NewPidState}. 1043 | 1044 | -spec build_node(pid_call_state(), tree()) -> {integer(), pid_call_state()}. 1045 | build_node([Child = #node{} | State], Node = #node{children = Children}) -> 1046 | build_node(State, Node#node{children = [Child | Children]}); 1047 | build_node([{Call, CallTS} | State], Node) -> 1048 | {call, {M, F, Args}} = Call, 1049 | FinalNode = Node#node{module = M, function = F, args = Args}, 1050 | {CallTS, [FinalNode | State]}. 1051 | 1052 | -spec insert_call_tree(tree(), acc_time(), ets:tid()) -> true. 1053 | insert_call_tree(CallTree, Time, TreeTab) -> 1054 | TreeItem = case ets:lookup(TreeTab, CallTree) of 1055 | [] -> {Time, 1, CallTree}; 1056 | [{PrevTime, Count, _}] -> {PrevTime + Time, Count + 1, CallTree} 1057 | end, 1058 | ets:insert(TreeTab, TreeItem). 1059 | 1060 | -spec reduce_call_trees(ets:tid()) -> true. 1061 | reduce_call_trees(TreeTab) -> 1062 | ets:foldl(fun reduce_tree_item/2, TreeTab, TreeTab). 1063 | 1064 | -spec reduce_tree_item(tree_item(), ets:tid()) -> ok. 1065 | reduce_tree_item({_, Count, #node{children = Children}}, TreeTab) -> 1066 | [reduce_subtree(Child, Count, TreeTab) || Child <- Children], 1067 | TreeTab. 1068 | 1069 | -spec reduce_subtree(tree(), call_tree_count(), ets:tid()) -> any(). 1070 | reduce_subtree(Node, Count, TreeTab) -> 1071 | case ets:lookup(TreeTab, Node) of 1072 | [{_, Count, _} = Item] -> 1073 | reduce_tree_item(Item, TreeTab), 1074 | ets:delete(TreeTab, Node); 1075 | [{_, OtherCount, _}] when OtherCount > Count -> 1076 | has_more_callers; 1077 | [] -> 1078 | already_deleted 1079 | end. 1080 | 1081 | %% @doc Returns statistics of repeated function call trees that took most time. 1082 | %% 1083 | %% @see top_call_trees/1 1084 | -spec top_call_trees() -> [tree_item()]. 1085 | top_call_trees() -> 1086 | top_call_trees(#{}). 1087 | 1088 | %% @doc Returns statistics of repeated function call trees that took most time. 1089 | %% 1090 | %% Two call trees repeat if they contain the same function calls and returns 1091 | %% in the same order taking the same arguments and returning the same values, respectively. 1092 | %% The results are sorted according to accumulated time. 1093 | -spec top_call_trees(top_call_trees_options()) -> [tree_item()]. 1094 | top_call_trees(Options) when is_map(Options) -> 1095 | TreeTab = call_tree_stat(), 1096 | case maps:get(output, Options, reduced) of 1097 | reduced -> reduce_call_trees(TreeTab); 1098 | complete -> ok 1099 | end, 1100 | TopTrees = top_call_trees(TreeTab, Options), 1101 | ets:delete(TreeTab), 1102 | TopTrees. 1103 | 1104 | %% @private 1105 | -spec top_call_trees(ets:tid(), top_call_trees_options()) -> [tree_item()]. 1106 | top_call_trees(TreeTab, Options) -> 1107 | MaxSize = maps:get(max_size, Options, 10), 1108 | MinCount = maps:get(min_count, Options, 2), 1109 | MinTime = maps:get(min_time, Options, 0), 1110 | Set = ets:foldl(fun(TreeItem = {Time, Count, _}, T) when Count >= MinCount, Time >= MinTime -> 1111 | insert_top_call_trees_item(TreeItem, T, MaxSize); 1112 | (_, T) -> 1113 | T 1114 | end, gb_sets:empty(), TreeTab), 1115 | lists:reverse(lists:sort(gb_sets:to_list(Set))). 1116 | 1117 | -spec insert_top_call_trees_item(tree_item(), gb_sets:set(tree_item()), pos_integer()) -> 1118 | gb_sets:set(tree_item()). 1119 | insert_top_call_trees_item(TreeItem, Set, MaxSize) -> 1120 | NewSet = gb_sets:add(TreeItem, Set), 1121 | case gb_sets:size(NewSet) of 1122 | N when N =< MaxSize -> 1123 | NewSet; 1124 | N when N =:= MaxSize + 1 -> 1125 | {_, ReducedSet} = gb_sets:take_smallest(NewSet), 1126 | ReducedSet 1127 | end. 1128 | 1129 | %% Helpers 1130 | 1131 | -spec initial_index() -> index(). 1132 | initial_index() -> 1133 | 1. 1134 | 1135 | default_tab() -> 1136 | trace. 1137 | 1138 | foldl(F, InitialState, List) when is_list(List) -> lists:foldl(F, InitialState, List); 1139 | foldl(F, InitialState, Tab) -> ets:foldl(F, InitialState, Tab). 1140 | 1141 | maybe_length(L) when is_list(L) -> length(L); 1142 | maybe_length(I) when is_integer(I) -> I. 1143 | 1144 | pretty_print_tuple_list(TList, MaxRows) -> 1145 | Head = lists:sublist(TList, MaxRows), 1146 | MaxSize = lists:max([tuple_size(T) || T <- Head]), 1147 | L = [pad([lists:flatten(io_lib:format("~p", [Item])) || Item <- tuple_to_list(T)], MaxSize) || T <- Head], 1148 | Widths = lists:foldl(fun(T, CurWidths) -> 1149 | [max(W, length(Item)+2) || {W, Item} <- lists:zip(CurWidths, T)] 1150 | end, lists:duplicate(MaxSize, 2), L), 1151 | [begin 1152 | X = [[Item, lists:duplicate(W-length(Item), $ )] || {W, Item} <- lists:zip(Widths, T)], 1153 | io:format("~s~n", [X]) 1154 | end || T<- L], 1155 | ok. 1156 | 1157 | pad(L, Size) when length(L) < Size -> L ++ lists:duplicate(Size-length(L), ""); 1158 | pad(L, _) -> L. 1159 | 1160 | usec_from_now({MegaSecs, Secs, Usecs}) -> 1161 | (MegaSecs * 1000000 + Secs) * 1000000 + Usecs. 1162 | -------------------------------------------------------------------------------- /test/tr_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(tr_SUITE). 2 | -compile([export_all, nowarn_export_all]). 3 | 4 | -include_lib("eunit/include/eunit.hrl"). 5 | -include("tr.hrl"). 6 | 7 | -import(tr_helper, [wait_for_traces/1]). 8 | 9 | expect_log(Level, Regexp) -> 10 | receive 11 | {Level, Log} -> 12 | case re:run(Log, Regexp) of 13 | nomatch -> 14 | ct:pal("Skipping unexpected log message (~p): ~p", [Level, Log]), 15 | expect_log(Level, Regexp); 16 | {match, _} -> 17 | ok 18 | end 19 | after 5000 -> 20 | ct:fail("Expected log message not received") 21 | end. 22 | 23 | %% CT callbacks 24 | 25 | all() -> 26 | [{group, start_stop}, 27 | {group, trace}, 28 | {group, range}, 29 | {group, traceback}, 30 | {group, util}, 31 | {group, call_stat}, 32 | {group, call_tree_stat}]. 33 | 34 | suite() -> 35 | [{timetrap, {seconds, 30}}]. 36 | 37 | groups() -> 38 | [{start_stop, [limit, 39 | stop_while_tracing, 40 | duplicates]}, 41 | {trace, [single_pid, 42 | single_pid_with_msg, 43 | msg_after_traced_call]}, 44 | {range, [ranges, 45 | ranges_max_depth, 46 | range, 47 | range_max_depth, 48 | ranges_with_messages, 49 | incomplete_ranges]}, 50 | {traceback, [single_tb, 51 | single_tb_with_messages, 52 | tb, 53 | tb_bottom_up, 54 | tb_limit, 55 | tb_longest, 56 | tb_all, 57 | tb_all_limit, 58 | tb_tree, 59 | tb_tree_longest, 60 | tb_roots]}, 61 | {util, [do]}, 62 | {call_stat, [simple_total, 63 | tree_total, 64 | simple_total_with_messages, 65 | acc_and_own_for_recursion, 66 | acc_and_own_for_recursion_with_exception, 67 | acc_and_own_for_indirect_recursion, 68 | dump_and_load, 69 | interleave, 70 | call_without_return, 71 | return_without_call]}, 72 | {call_tree_stat, [top_call_trees, 73 | top_call_trees_with_messages]}]. 74 | 75 | init_per_group(start_stop, Config) -> 76 | Config; 77 | init_per_group(_Group, Config) -> 78 | ok = application:start(erlang_doctor), 79 | Config. 80 | 81 | end_per_group(start_stop, _Config) -> 82 | ok; 83 | end_per_group(_Group, _Config) -> 84 | ok = application:stop(erlang_doctor). 85 | 86 | init_per_testcase(_TC, Config) -> 87 | logger:add_handler(?MODULE, ?MODULE, #{config => self()}), 88 | Config. 89 | 90 | end_per_testcase(_TC, _Config) -> 91 | logger:remove_handler(?MODULE), 92 | case lists:keymember(erlang_doctor, 1, application:which_applications()) of 93 | false -> 94 | catch tr:stop(); % cleanup in case of failed tests 95 | true -> 96 | catch tr:stop_tracing(), % cleanup in case of failed tests 97 | tr:clean() 98 | end. 99 | 100 | %% Test cases 101 | 102 | limit(_Config) -> 103 | tr:start(#{limit => 2}), 104 | MFA = {?MODULE, factorial, 1}, 105 | tr:trace([MFA]), 106 | factorial(1), 107 | expect_log(warning, "Reached trace limit 2"), 108 | expect_log(warning, "Tracer process <.+> exited with reason shutdown"), 109 | wait_for_traces(2), 110 | 111 | %% expect only first 2 calls 112 | [#tr{index = 1, event = call, mfa = MFA, data = [1]}, 113 | #tr{index = 2, event = call, mfa = MFA, data = [0]}] = tr:select(), 114 | tr:stop(). 115 | 116 | stop_while_tracing(_Config) -> 117 | tr:start(), 118 | tr:trace([{?MODULE, fib, 1}]), 119 | fib(1), 120 | wait_for_traces(2), 121 | tr:stop(), %% Stop tr when tracing is still enabled 122 | tr:start(), 123 | MFA = {?MODULE, factorial, 1}, 124 | tr:trace([MFA]), 125 | fib(0), % Shouldn't be traced anymore 126 | factorial(0), 127 | 128 | %% Expect only factorial 129 | [#tr{index = 1, event = call, mfa = MFA, data = [0]}, 130 | #tr{index = 2, event = return, mfa = MFA, data = 1}] = tr:select(), 131 | tr:stop(). 132 | 133 | duplicates(_Config) -> 134 | {ok, Pid} = tr:start(), 135 | {error, {already_started, Pid}} = tr:start(), 136 | 137 | ok = tr:trace([?MODULE]), 138 | {error, already_tracing} = tr:trace([?MODULE]), 139 | 140 | ok = tr:stop_tracing(), 141 | {error, not_tracing} = tr:stop_tracing(), 142 | 143 | ok = tr:stop(), 144 | ?assertExit(noproc, tr:stop()). 145 | 146 | single_pid(_Config) -> 147 | Pid = spawn_link(fun ?MODULE:async_factorial/0), 148 | MFA = {?MODULE, factorial, 1}, 149 | tr:trace([MFA], [Pid]), 150 | factorial(0), 151 | Pid ! {do_factorial, 0, self()}, 152 | receive {ok, _} -> ok end, 153 | wait_for_traces(2), 154 | tr:stop_tracing(), 155 | Pid ! stop, 156 | 157 | %% expect only traces from Pid 158 | [#tr{index = 1, event = call, mfa = MFA, pid = Pid, data = [0]}, 159 | #tr{index = 2, event = return, mfa = MFA, pid = Pid, data = 1}] = tr:select(). 160 | 161 | single_pid_with_msg(_Config) -> 162 | Pid = spawn_link(fun ?MODULE:async_factorial/0), 163 | MFA = {?MODULE, factorial, 1}, 164 | tr:trace(#{modules => [MFA], pids => [Pid], msg => all, msg_trigger => always}), 165 | factorial(0), 166 | Self = self(), 167 | Pid ! {do_factorial, 0, Self}, 168 | receive {ok, _} -> ok end, 169 | wait_for_traces(4), 170 | tr:stop_tracing(), 171 | Pid ! stop, 172 | 173 | %% expect only traces from Pid 174 | [#tr{index = 1, event = recv, pid = Pid, data = {do_factorial, 0, Self}}, 175 | #tr{index = 2, event = call, mfa = MFA, pid = Pid, data = [0]}, 176 | #tr{index = 3, event = return, mfa = MFA, pid = Pid, data = 1}, 177 | #tr{index = 4, event = send, pid = Pid, data = {ok, 1}, info = {Self, true}}] = tr:select(). 178 | 179 | msg_after_traced_call(_Config) -> 180 | Pid = spawn_link(fun ?MODULE:async_factorial/0), 181 | MFA = {?MODULE, factorial, 1}, 182 | tr:trace(#{modules => [MFA], msg => all}), 183 | Self = self(), 184 | Pid ! {do_factorial, 0, Self}, 185 | receive {ok, _} -> ok end, 186 | Pid ! stop, 187 | wait_for_traces(4), 188 | tr:stop_tracing(), 189 | 190 | %% message traces for Pid are stored after the call to factorial/1 191 | [#tr{index = 1, event = call, mfa = MFA, pid = Pid, data = [0]}, 192 | #tr{index = 2, event = return, mfa = MFA, pid = Pid, data = 1}, 193 | #tr{index = 3, event = send, pid = Pid, data = {ok, 1}, info = {Self, true}}, 194 | #tr{index = 4, event = recv, pid = Pid, data = stop}] = tr:select(). 195 | 196 | ranges(_Config) -> 197 | Traces = trace_fib3(), 198 | 199 | %% all traces from fib(3) 200 | [Traces] = tr:ranges(fun(#tr{event = call}) -> true end), 201 | 202 | %% nothing because 'return' is not a call 203 | [] = tr:ranges(fun(#tr{event = return}) -> true end), 204 | 205 | %% all traces from both calls to fib(2) 206 | Range1a = lists:sublist(Traces, 3, 2), 207 | Range1b = lists:sublist(Traces, 8, 2), 208 | [Range1a, Range1b] = tr:ranges(fun(#tr{data = [1]}) -> true end). 209 | 210 | ranges_max_depth(_Config) -> 211 | Traces = trace_fib3(), 212 | 213 | %% fib(3) and its return 214 | Range1 = [hd(Traces), lists:last(Traces)], 215 | [Range1] = tr:ranges(fun(#tr{event = call}) -> true end, #{max_depth => 1}), 216 | 217 | %% fib(3) with fib(2) and fib(1) inside 218 | Range2 = lists:sublist(Traces, 2) ++ lists:sublist(Traces, 7, 4), 219 | [Range2] = tr:ranges(fun(#tr{event = call}) -> true end, #{max_depth => 2}). 220 | 221 | range(_Config) -> 222 | Traces = trace_fib3(), 223 | 224 | %% all traces from fib(3) 225 | Traces = tr:range(fun(#tr{event = call}) -> true end), 226 | Traces = tr:range(1), 227 | Traces = tr:range(hd(Traces)), 228 | 229 | %% return is not a call 230 | ?assertException(error, badarg, tr:range(fun(#tr{event = return}) -> true end)), 231 | 232 | %% all traces from the first call to fib(2) 233 | Range1 = lists:sublist(Traces, 3, 2), 234 | Range1 = tr:range(fun(#tr{data = [1]}) -> true end). 235 | 236 | range_max_depth(_Config) -> 237 | Traces = trace_fib3(), 238 | 239 | %% fib(3) and its return 240 | Range1 = [hd(Traces), lists:last(Traces)], 241 | Range1 = tr:range(fun(#tr{event = call}) -> true end, #{max_depth => 1}), 242 | 243 | %% fib(3) with fib(2) and fib(1) inside 244 | Range2 = lists:sublist(Traces, 2) ++ lists:sublist(Traces, 7, 4), 245 | Range2 = tr:range(fun(#tr{event = call}) -> true end, #{max_depth => 2}). 246 | 247 | ranges_with_messages(_Config) -> 248 | Traces = trace_wait_and_reply(), 249 | 250 | %% all traces from the root call with messages 251 | [Traces] = tr:ranges(fun(#tr{}) -> true end), 252 | 253 | %% nothing because 'send' is not a call 254 | [] = tr:ranges(fun(#tr{event = send}) -> true end), 255 | 256 | %% root call and its return without messages (they have depth of 2) 257 | Range1 = [hd(Traces), lists:last(Traces)], 258 | [Range1] = tr:ranges(fun(#tr{}) -> true end, #{max_depth => 1}). 259 | 260 | incomplete_ranges(_Config) -> 261 | Self = self(), 262 | tr:trace(#{modules => [MFA = {?MODULE, wait_and_reply, 1}]}), 263 | Pid1 = spawn_link(?MODULE, wait_and_reply, [self()]), 264 | Pid2 = spawn_link(?MODULE, wait_and_reply, [self()]), 265 | receive {started, Pid1} -> ok end, 266 | receive {started, Pid2} -> ok end, 267 | Pid2 ! reply, 268 | receive {finished, Pid2} -> ok end, 269 | 270 | %% Pid1 finished, but Pid1 didn't 271 | wait_for_traces(3), 272 | tr:stop_tracing(), 273 | Pid1 ! reply, 274 | receive {finished, Pid1} -> ok end, 275 | 276 | [#tr{index = 1, pid = Pid1, event = call, mfa = MFA, data = [Self]}, 277 | #tr{index = 2, pid = Pid2, event = call, mfa = MFA, data = [Self]}, 278 | #tr{index = 3, pid = Pid2, event = return, mfa = MFA, data = {finished, Pid2}}] = 279 | [T1, T2, T3] = tr:select(), 280 | 281 | %% Ranges with missing returns are included at the end 282 | [[T2, T3], [T1]] = tr:ranges(fun(#tr{}) -> true end), 283 | [[T2, T3], [T1]] = tr:ranges(fun(#tr{}) -> true end, #{output => all}), 284 | 285 | %% Skip ranges with missing returns 286 | [[T2, T3]] = tr:ranges(fun(#tr{}) -> true end, #{output => complete}), 287 | 288 | %% Skip ranges with missing returns 289 | [[T1]] = tr:ranges(fun(#tr{}) -> true end, #{output => incomplete}). 290 | 291 | do(_Config) -> 292 | tr:trace([{?MODULE, fib, 1}]), 293 | ?MODULE:fib(2), 294 | wait_for_traces(6), 295 | tr:stop_tracing(), 296 | MFA = {?MODULE, fib, 1}, 297 | [T1 = #tr{index = 1, event = call, mfa = MFA, data = [2]}, 298 | #tr{index = 2, event = call, mfa = MFA, data = [1]}, 299 | #tr{index = 3, event = return, mfa = MFA, data = 1}, 300 | T4 = #tr{index = 4, event = call, mfa = MFA, data = [0]}, 301 | #tr{index = 5, event = return, mfa = MFA, data = 0}, 302 | #tr{index = 6, event = return, mfa = MFA, data = 1}] = tr:select(), 303 | 1 = tr:do(T1), 304 | 0 = tr:do(T4), 305 | 1 = tr:do(1), 306 | 0 = tr:do(4). 307 | 308 | single_tb(_Config) -> 309 | tr:trace([{?MODULE, fib, 1}]), 310 | ?MODULE:fib(4), 311 | wait_for_traces(18), 312 | tr:stop_tracing(), 313 | 314 | PredF = fun(#tr{event = return, data = N}) when N < 2 -> true end, 315 | TB = tr:traceback(PredF), 316 | ct:pal("Top-down TB: ~p~n", [TB]), 317 | ?assertMatch([#tr{data = [1]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], TB), 318 | 319 | TB2 = tr:traceback(PredF, #{order => bottom_up}), 320 | ct:pal("Bottom-up TB: ~p~n", [TB2]), 321 | ?assertEqual(lists:reverse(TB), TB2). 322 | 323 | single_tb_with_messages(_Config) -> 324 | Traces = [Call, Send, Recv|_] = trace_wait_and_reply(), 325 | Return = lists:last(Traces), 326 | ?assertEqual([Call], tr:traceback(Send)), 327 | ?assertEqual([Call], tr:traceback(Recv)), 328 | ?assertEqual([Call], tr:traceback(Return)). 329 | 330 | tb(_Config) -> 331 | tr:trace([{?MODULE, fib, 1}]), 332 | ?MODULE:fib(4), 333 | wait_for_traces(18), 334 | tr:stop_tracing(), 335 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end), 336 | ct:pal("~p~n", [TBs]), 337 | ?assertMatch([[#tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 338 | [#tr{data = [1]}, #tr{data = [3]}, #tr{data = [4]}], 339 | [#tr{data = [2]}, #tr{data = [4]}]], TBs). 340 | 341 | tb_bottom_up(_Config) -> 342 | tr:trace([{?MODULE, fib, 1}]), 343 | ?MODULE:fib(4), 344 | wait_for_traces(18), 345 | tr:stop_tracing(), 346 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 347 | #{order => bottom_up}), 348 | ct:pal("~p~n", [TBs]), 349 | ?assertMatch([[#tr{data = [4]}, #tr{data = [3]}, #tr{data = [2]}], 350 | [#tr{data = [4]}, #tr{data = [3]}, #tr{data = [1]}], 351 | [#tr{data = [4]}, #tr{data = [2]}]], TBs). 352 | 353 | tb_limit(_Config) -> 354 | tr:trace([{?MODULE, fib, 1}]), 355 | ?MODULE:fib(4), 356 | wait_for_traces(18), 357 | tr:stop_tracing(), 358 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 359 | #{limit => 3}), 360 | ct:pal("~p~n", [TBs]), 361 | ?assertMatch([[#tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}]], TBs). 362 | 363 | tb_all(_Config) -> 364 | tr:trace([{?MODULE, fib, 1}]), 365 | ?MODULE:fib(4), 366 | wait_for_traces(18), 367 | tr:stop_tracing(), 368 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 369 | #{output => all}), 370 | ct:pal("~p~n", [TBs]), 371 | ?assertMatch([[#tr{data = [1]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 372 | [#tr{data = [0]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 373 | [#tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 374 | [#tr{data = [1]}, #tr{data = [3]}, #tr{data = [4]}], 375 | [#tr{data = [1]}, #tr{data = [2]}, #tr{data = [4]}], 376 | [#tr{data = [0]}, #tr{data = [2]}, #tr{data = [4]}], 377 | [#tr{data = [2]}, #tr{data = [4]}]], TBs). 378 | 379 | tb_longest(_Config) -> 380 | tr:trace([{?MODULE, fib, 1}]), 381 | ?MODULE:fib(4), 382 | wait_for_traces(18), 383 | tr:stop_tracing(), 384 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 385 | #{output => longest}), 386 | ct:pal("~p~n", [TBs]), 387 | ?assertMatch([[#tr{data = [1]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 388 | [#tr{data = [0]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 389 | [#tr{data = [1]}, #tr{data = [3]}, #tr{data = [4]}], 390 | [#tr{data = [1]}, #tr{data = [2]}, #tr{data = [4]}], 391 | [#tr{data = [0]}, #tr{data = [2]}, #tr{data = [4]}]], TBs). 392 | 393 | tb_all_limit(_Config) -> 394 | tr:trace([{?MODULE, fib, 1}]), 395 | ?MODULE:fib(4), 396 | wait_for_traces(18), 397 | tr:stop_tracing(), 398 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 399 | #{limit => 3, output => all}), 400 | ct:pal("~p~n", [TBs]), 401 | ?assertMatch([[#tr{data = [1]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 402 | [#tr{data = [0]}, #tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}], 403 | [#tr{data = [2]}, #tr{data = [3]}, #tr{data = [4]}]], TBs). 404 | 405 | tb_tree(_Config) -> 406 | tr:trace([{?MODULE, fib, 1}]), 407 | ?MODULE:fib(4), 408 | wait_for_traces(18), 409 | tr:stop_tracing(), 410 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 411 | #{format => tree}), 412 | ct:pal("~p~n", [TBs]), 413 | ?assertMatch([{#tr{data = [4]}, [{#tr{data = [3]}, [#tr{data = [2]}, 414 | #tr{data = [1]}]}, 415 | #tr{data = [2]}]}], TBs). 416 | 417 | tb_tree_longest(_Config) -> 418 | tr:trace([{?MODULE, fib, 1}]), 419 | ?MODULE:fib(4), 420 | wait_for_traces(18), 421 | tr:stop_tracing(), 422 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 423 | #{format => tree, output => longest}), 424 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 425 | #{format => tree, output => all}), %% same result for trees 426 | ct:pal("~p~n", [TBs]), 427 | ?assertMatch([{#tr{data = [4]}, [{#tr{data = [3]}, [{#tr{data = [2]}, [#tr{data = [1]}, 428 | #tr{data = [0]}]}, 429 | #tr{data = [1]}]}, 430 | {#tr{data = [2]}, [#tr{data = [1]}, 431 | #tr{data = [0]}]}]}], TBs). 432 | 433 | tb_roots(_Config) -> 434 | tr:trace([{?MODULE, fib, 1}]), 435 | ?MODULE:fib(4), 436 | wait_for_traces(18), 437 | tr:stop_tracing(), 438 | 439 | %% Option 1: call root/1 or roots/1 on a tree 440 | TBs = tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 441 | #{format => tree}), 442 | Roots = tr:roots(TBs), 443 | ct:pal("Roots: ~p~n", [Roots]), 444 | ?assertMatch([#tr{data = [4]}], Roots), 445 | ?assertEqual(hd(Roots), tr:root(hd(TBs))), 446 | 447 | %% Option 2: directly use the root format 448 | ?assertEqual(Roots, tr:tracebacks(fun(#tr{event = return, data = N}) when N < 2 -> true end, 449 | #{format => root})). 450 | 451 | simple_total(_Config) -> 452 | tr:trace([{?MODULE, factorial, 1}]), 453 | ?MODULE:factorial(2), 454 | wait_for_traces(6), 455 | [{total, 3, Acc1, Acc1}] = tr:sorted_call_stat(fun(_) -> total end), 456 | ?MODULE:factorial(1), 457 | wait_for_traces(10), 458 | [{total, 5, Acc2, Acc2}] = tr:sorted_call_stat(fun(_) -> total end), 459 | ?assertEqual(true, Acc1 < Acc2), 460 | tr:stop_tracing(), 461 | timer:sleep(10), 462 | 463 | %% Tracing disabled 464 | ?MODULE:factorial(1), 465 | timer:sleep(10), 466 | [{total, 5, Acc2, Acc2}] = tr:sorted_call_stat(fun(_) -> total end), 467 | 468 | %% Tracing enabled for a different function 469 | tr:trace([{?MODULE, factorial2, 1}]), 470 | ?MODULE:factorial(1), 471 | timer:sleep(10), 472 | [{total, 5, Acc2, Acc2}] = tr:sorted_call_stat(fun(_) -> total end), 473 | tr:stop_tracing(). 474 | 475 | tree_total(_Config) -> 476 | %% Check that acc and own times are equal for a branching call tree 477 | trace_fib3(), 478 | [{total, 5, Acc, Acc}] = tr:sorted_call_stat(fun(_) -> total end), 479 | ?assert(Acc > 0). 480 | 481 | simple_total_with_messages(_Config) -> 482 | _Traces = trace_wait_and_reply(), 483 | [{total, 1, Acc1, Acc1}] = tr:sorted_call_stat(fun(_) -> total end). 484 | 485 | acc_and_own_for_recursion(_Config) -> 486 | tr:trace([{?MODULE, sleepy_factorial, 1}]), 487 | ?MODULE:sleepy_factorial(2), 488 | wait_for_traces(6), 489 | tr:stop_tracing(), 490 | Stat = tr:sorted_call_stat(fun(#tr{data = [Arg]}) -> Arg end), 491 | ct:pal("Stat: ~p~n", [Stat]), 492 | [{2, 1, Acc2, Own2}, 493 | {1, 1, Acc1, Own1}, 494 | {0, 1, Acc0, Acc0}] = Stat, 495 | ?assertEqual(Acc2, Own2 + Own1 + Acc0), 496 | ?assertEqual(Acc1, Own1 + Acc0). 497 | 498 | acc_and_own_for_recursion_with_exception(_Config) -> 499 | tr:trace([{?MODULE, bad_factorial, 1}]), 500 | catch ?MODULE:bad_factorial(2), 501 | wait_for_traces(6), 502 | tr:stop_tracing(), 503 | Stat = tr:sorted_call_stat(fun(#tr{data = [Arg]}) -> Arg end), 504 | ct:pal("Stat: ~p~n", [Stat]), 505 | [{2, 1, Acc2, Own2}, 506 | {1, 1, Acc1, Own1}, 507 | {0, 1, Acc0, Acc0}] = Stat, 508 | ?assertEqual(Acc2, Own2 + Own1 + Acc0), 509 | ?assertEqual(Acc1, Own1 + Acc0). 510 | 511 | acc_and_own_for_indirect_recursion(_Config) -> 512 | tr:trace([?MODULE]), 513 | ?MODULE:factorial_with_helper(2), 514 | wait_for_traces(10), 515 | tr:stop_tracing(), 516 | Traces = ets:tab2list(trace), 517 | ct:pal("~p~n", [Traces]), 518 | [#tr{event = call, mfa = {_, factorial_with_helper, 1}, data = [2], ts = T1}, 519 | #tr{event = call, mfa = {_, factorial_helper, 1}, data = [2], ts = T2}, 520 | #tr{event = call, mfa = {_, factorial_with_helper, 1}, data = [1], ts = T3}, 521 | #tr{event = call, mfa = {_, factorial_helper, 1}, data = [1], ts = T4}, 522 | #tr{event = call, mfa = {_, factorial_with_helper, 1}, data = [0], ts = T5}, 523 | #tr{event = return, mfa = {_, factorial_with_helper, 1}, data = 1, ts = T6}, 524 | #tr{event = return, mfa = {_, factorial_helper, 1}, data = 1, ts = T7}, 525 | #tr{event = return, mfa = {_, factorial_with_helper, 1}, data = 1, ts = T8}, 526 | #tr{event = return, mfa = {_, factorial_helper, 1}, data = 2, ts = T9}, 527 | #tr{event = return, mfa = {_, factorial_with_helper, 1}, data = 2, ts = T10}] = Traces, 528 | Stat = tr:sorted_call_stat(fun(#tr{mfa = {_, factorial_with_helper, _}, data = Args}) -> Args end), 529 | ct:pal("Stat: ~p~n", [Stat]), 530 | [{[2], 1, Acc1, Own1}, 531 | {[1], 1, Acc2, Own2}, 532 | {[0], 1, Acc3, Acc3}] = Stat, 533 | ?assertEqual(Acc1, T10 - T1), 534 | ?assertEqual(Own1, (T2 - T1) + (T10 - T9)), 535 | ?assertEqual(Acc2, T8 - T3), 536 | ?assertEqual(Own2, (T4 - T3) + (T8 - T7)), 537 | ?assertEqual(Acc3, T6 - T5), 538 | FStat = tr:sorted_call_stat(fun(#tr{mfa = {_, F, _}}) -> F end), 539 | ct:pal("FStat: ~p~n", [FStat]), 540 | [{factorial_with_helper, 3, Acc1, OwnF1}, 541 | {factorial_helper, 2, AccF2, OwnF2}] = FStat, 542 | ?assertEqual(AccF2, T9 - T2), 543 | ?assertEqual(OwnF1, Own1 + Own2 + Acc3), 544 | ?assertEqual(OwnF2, (T3 - T2) + (T5 - T4) + (T7 - T6) + (T9 - T8)). 545 | 546 | interleave(_Config) -> 547 | tr:trace([?MODULE]), 548 | Self = self(), 549 | P1 = spawn_link(?MODULE, wait_and_reply, [Self]), 550 | receive {started, P1} -> ok end, 551 | P2 = spawn_link(?MODULE, wait_and_reply, [Self]), 552 | receive {started, P2} -> ok end, 553 | P1 ! reply, 554 | receive {finished, P1} -> ok end, 555 | P2 ! reply, 556 | receive {finished, P2} -> ok end, 557 | wait_for_traces(4), 558 | tr:stop_tracing(), 559 | [#tr{pid = P1, event = call, ts = T1}, 560 | #tr{pid = P2, event = call, ts = T2}, 561 | #tr{pid = P1, event = return, ts = T3}, 562 | #tr{pid = P2, event = return, ts = T4}] = ets:tab2list(trace), 563 | Stat = tr:sorted_call_stat(fun(#tr{mfa = {_, F, _}, data = Data}) -> {F, Data} end), 564 | ct:pal("Stat: ~p~n", [Stat]), 565 | [{{wait_and_reply, [Self]}, 2, DT, DT}] = Stat, 566 | ?assertEqual(DT, (T3 - T1) + (T4 - T2)). 567 | 568 | call_without_return(_Config) -> 569 | tr:trace([?MODULE]), 570 | Self = self(), 571 | P1 = spawn_link(?MODULE, wait_and_reply, [Self]), 572 | receive {started, P1} -> ok end, 573 | wait_for_traces(1), 574 | tr:stop_tracing(), 575 | P1 ! reply, 576 | receive {finished, P1} -> ok end, 577 | [#tr{pid = P1, event = call}] = tr:select(), 578 | Stat = tr:sorted_call_stat(fun(#tr{mfa = {_, F, _}, data = Data}) -> {F, Data} end), 579 | ct:pal("Stat: ~p~n", [Stat]), 580 | ?assertEqual([{{wait_and_reply, [Self]}, 1, 0, 0}], Stat). % time is zero 581 | 582 | return_without_call(_Config) -> 583 | P1 = spawn_link(?MODULE, wait_and_reply, [self()]), 584 | receive {started, P1} -> ok end, 585 | tr:trace([?MODULE]), 586 | P1 ! reply, 587 | receive {finished, P1} -> ok end, 588 | timer:sleep(10), 589 | tr:stop_tracing(), 590 | [] = tr:select(), % return is not registered if call was not traced 591 | Stat = tr:sorted_call_stat(fun(#tr{mfa = {_, F, _}, data = Data}) -> {F, Data} end), 592 | ct:pal("Stat: ~p~n", [Stat]), 593 | ?assertEqual([], Stat). 594 | 595 | dump_and_load(_Config) -> 596 | DumpFile = "dump", 597 | tr:trace([{?MODULE, factorial, 1}]), 598 | factorial(1), 599 | wait_for_traces(4), 600 | tr:stop_tracing(), 601 | BeforeDump = tr:sorted_call_stat(fun(_) -> total end), 602 | tr:dump(DumpFile), 603 | tr:load(DumpFile), 604 | AfterLoad = tr:sorted_call_stat(fun(_) -> total end), 605 | ?assertEqual(BeforeDump, AfterLoad), 606 | file:delete(DumpFile). 607 | 608 | top_call_trees(_Config) -> 609 | tr:trace([?MODULE]), 610 | ?MODULE:fib(3), 611 | ?MODULE:fib(4), 612 | wait_for_traces(28), 613 | tr:stop_tracing(), 614 | N = #node{module = ?MODULE, function = fib}, 615 | Fib0 = N#node{args = [0], result = {return, 0}}, 616 | Fib1 = N#node{args = [1], result = {return, 1}}, 617 | Fib2 = N#node{args = [2], children = [Fib1, Fib0], result = {return, 1}}, 618 | Fib3 = N#node{args = [3], children = [Fib2, Fib1], result = {return, 2}}, 619 | Complete = tr:top_call_trees(#{output => complete}), 620 | Reduced = tr:top_call_trees(), 621 | ct:pal("Top call trees (complete): ~p~n", [Complete]), 622 | ct:pal("Top call trees (reduced): ~p~n", [Reduced]), 623 | [{T0, 3, Fib0}] = Complete -- Reduced, 624 | [{T3, 2, Fib3}, {T2, 3, Fib2}, {T1, 5, Fib1}] = lists:keysort(2, Reduced), 625 | ?assert(T2 + T3 > T1), 626 | ?assert(T2 > T0). 627 | 628 | top_call_trees_with_messages(_Config) -> 629 | _Traces = trace_wait_and_reply(), 630 | [{_T, 1, #node{module = ?MODULE, function = wait_and_reply, 631 | args = [_], result = {return, {finished, _}}}}] = 632 | tr:top_call_trees(#{min_count => 1}). 633 | 634 | %% Helpers 635 | 636 | trace_fib3() -> 637 | tr:trace([MFA = {?MODULE, fib, 1}]), 638 | fib(3), 639 | wait_for_traces(10), 640 | tr:stop_tracing(), 641 | [#tr{index = 1, event = call, mfa = MFA, data = [3]}, 642 | #tr{index = 2, event = call, mfa = MFA, data = [2]}, 643 | #tr{index = 3, event = call, mfa = MFA, data = [1]}, 644 | #tr{index = 4, event = return, mfa = MFA, data = 1}, 645 | #tr{index = 5, event = call, mfa = MFA, data = [0]}, 646 | #tr{index = 6, event = return, mfa = MFA, data = 0}, 647 | #tr{index = 7, event = return, mfa = MFA, data = 1}, 648 | #tr{index = 8, event = call, mfa = MFA, data = [1]}, 649 | #tr{index = 9, event = return, mfa = MFA, data = 1}, 650 | #tr{index = 10, event = return, mfa = MFA, data = 2}] = tr:select(). 651 | 652 | trace_wait_and_reply() -> 653 | Self = self(), 654 | tr:trace(#{modules => [MFA = {?MODULE, wait_and_reply, 1}], msg => all}), 655 | Pid = spawn_link(?MODULE, wait_and_reply, [self()]), 656 | receive {started, Pid} -> ok end, 657 | Pid ! reply, 658 | receive {finished, Pid} -> ok end, 659 | wait_for_traces(5), 660 | tr:stop_tracing(), 661 | [#tr{index = 1, pid = Pid, event = call, mfa = MFA, data = [Self]}, 662 | #tr{index = 2, pid = Pid, event = send, data = {started, Pid}, info = {Self, true}}, 663 | #tr{index = 3, pid = Pid, event = recv, data = reply}, 664 | #tr{index = 4, pid = Pid, event = send, data = {finished, Pid}, info = {Self, true}}, 665 | #tr{index = 5, pid = Pid, event = return, mfa = MFA, data = {finished, Pid}} 666 | ] = tr:select(). 667 | 668 | factorial(N) when N > 0 -> N * factorial(N - 1); 669 | factorial(0) -> 1. 670 | 671 | async_factorial() -> 672 | receive 673 | {do_factorial, N, From} -> 674 | Res = ?MODULE:factorial(N), 675 | From ! {ok, Res}, 676 | async_factorial(); 677 | stop -> ok 678 | end. 679 | 680 | sleepy_factorial(N) when N > 0 -> 681 | timer:sleep(1), 682 | N * sleepy_factorial(N-1); 683 | sleepy_factorial(0) -> 684 | timer:sleep(1), 685 | 1. 686 | 687 | -spec bad_factorial(integer()) -> no_return(). 688 | bad_factorial(N) when N > 0 -> 689 | timer:sleep(1), 690 | N * bad_factorial(N - 1). 691 | 692 | factorial_with_helper(N) when N > 0 -> 693 | timer:sleep(1), 694 | factorial_helper(N); 695 | factorial_with_helper(0) -> 696 | timer:sleep(1), 697 | 1. 698 | 699 | factorial_helper(N) -> N * factorial_with_helper(N - 1). 700 | 701 | fib(N) when N > 1 -> fib(N - 1) + fib(N - 2); 702 | fib(1) -> 1; 703 | fib(0) -> 0. 704 | 705 | wait_and_reply(Sender) -> 706 | Sender ! {started, self()}, 707 | receive reply -> ok end, 708 | Sender ! {finished, self()}. 709 | 710 | log(#{msg := {Format, Args}, level := Level}, #{config := Pid}) -> 711 | Pid ! {Level, lists:flatten(io_lib:format(Format, Args))}; 712 | log(_Event, _Config) -> 713 | ok. 714 | -------------------------------------------------------------------------------- /test/tr_helper.erl: -------------------------------------------------------------------------------- 1 | -module(tr_helper). 2 | -compile([export_all, nowarn_export_all]). 3 | 4 | wait_for_traces(ExpectedSize) -> 5 | wait_for_traces(tr:tab(), 10, 500, ExpectedSize). 6 | 7 | wait_for_traces(Table, Interval, Retries, ExpectedSize) -> 8 | case ets:info(Table, size) of 9 | ExpectedSize -> 10 | ct:pal("~p~n", [ets:tab2list(trace)]), 11 | ok; 12 | TooMany when TooMany > ExpectedSize -> 13 | ct:pal("~p~n", [ets:tab2list(trace)]), 14 | ct:fail({"Too many traces", Table, TooMany, ExpectedSize}); 15 | _TooFew when Retries > 0 -> 16 | timer:sleep(Interval), 17 | wait_for_traces(Table, Interval, Retries - 1, ExpectedSize); 18 | TooFew -> 19 | ct:pal("~p~n", [ets:tab2list(trace)]), 20 | ct:fail({"Too few traces", Table, TooFew, ExpectedSize}) 21 | end. 22 | --------------------------------------------------------------------------------