├── .gitignore ├── LICENSE ├── README.md ├── doc └── screenshots │ ├── bin_vheap_delta_flame.png │ ├── bin_vheap_delta_heat.png │ ├── bin_vheap_flame.png │ ├── bin_vheap_heat.png │ ├── mbuf_size_delta_heat.png │ ├── reductions_delta_flame.png │ ├── reductions_delta_heat.png │ └── samples_heat.png ├── rebar.config └── src ├── flame_prof.app.src ├── flame_prof.erl └── seltor@flame_prof.erl /.gitignore: -------------------------------------------------------------------------------- 1 | ## 2 | ## Copyright 2020 Medical-Objects Pty Ltd. 3 | ## 4 | ## Licensed under the Apache License, Version 2.0 (the "License"); 5 | ## you may not use this file except in compliance with the License. 6 | ## You may obtain a copy of the License at 7 | ## 8 | ## http://www.apache.org/licenses/LICENSE-2.0 9 | ## 10 | ## Unless required by applicable law or agreed to in writing, software 11 | ## distributed under the License is distributed on an "AS IS" BASIS, 12 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | ## See the License for the specific language governing permissions and 14 | ## limitations under the License. 15 | ## 16 | 17 | ## Rebar 3 18 | .rebar3 19 | _* 20 | .eunit 21 | *.o 22 | *.beam 23 | *.plt 24 | *.swp 25 | *.swo 26 | .erlang.cookie 27 | ebin 28 | log 29 | erl_crash.dump 30 | .rebar 31 | logs 32 | _build 33 | .idea 34 | *.iml 35 | rebar3.crashdump 36 | .vscode 37 | ## Custom 38 | /tmp 39 | .vscode 40 | /samples 41 | /doc/* 42 | !/doc/screenshots 43 | rebar.lock -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 Medical-Objects Pty Ltd. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # **flame_prof** 2 | 3 |

NOTE: This project is under active development. Reference documentation, 4 | tutorials and a number features are in the works.

5 | 6 | 7 | ## Contents 8 | 9 | - [Overview](#Overview) 10 | - [Screenshots](#Screenshots) 11 | - [Example](#Example) 12 | - [Samples](#Samples) 13 | - [High scheduler load](#High-scheduler-load) 14 | - [Off-heap binaries](#Off-heap-binaries) 15 | - [There's much more](#Theres-much-more) 16 | - [Build](#Build) 17 | 18 | 19 | ## Overview 20 | 21 | **flame_prof** is a general-purpose Erlang profiler a little like OTP's 22 | **fprof**, except ... 23 | 24 | + It generates Linux [perf_events](https://en.wikipedia.org/wiki/Perf_(Linux)) 25 | script [output](https://linux.die.net/man/1/perf-script) (even on Win/macOS) 26 | intended to be consumed by, and analysed with, [a fork of Netflix's Flamescope](https://github.com/ebegumisa/flamescope). 27 | 28 | + It uses a call-stack sampling approach rather than attempting to measure each 29 | individual call. So it does not _need_ to use Erlang tracing (though it _may_ 30 | use tracing for certain optional features). 31 | 32 | + It retains calling process information including process status, memory usage, 33 | message queue lengths and garbage collection info. 34 | 35 | + Provides control over output file writing (e.g. sample flush frequency, output 36 | file rotation). 37 | 38 | + It provides means to automatically select processes to be profiled 39 | (e.g. top 100 by reductions). Automatically triggering profiling in controlled 40 | manner is coming soon. 41 | 42 | ## Screenshots 43 | 44 | ### Example 45 | 46 | Below are a few (of many) sub-second heatmaps and Erlang code flamegraphs 47 | generated from a node being profiled once for a little over a minute. The 48 | profiler was configured to, every 5 seconds, auto-select and profile the top 100 49 | processes. 50 | 51 | ### Samples 52 | 53 | Colour saturation in the heatmap below is scaled to the number of callstack 54 | samples taken by the profiler. Grey areas show where the node was so busy, the 55 | profiler was unable to take regular samples (long lines of grey are a special 56 | case -- these are due to the profiler flushing samples to disk every 15 57 | seconds.) 58 | 59 | Despite the load, there are still plenty of samples in the middle busy part to 60 | be able to gain insights from the other heatmaps and flamegraphs in the sections 61 | that follow. Moreover because the other heatmaps and flamegraphs integrate 62 | metrics mesurements like reductions differences, even large gaps between samples 63 | still gives us usable data. 64 | 65 | ![samples heatmap screenshot](doc/screenshots/samples_heat.png) 66 | 67 | ### High scheduler load 68 | 69 | The reductions heatmap and corresponding flamegraph below show what callstacks 70 | contributed most to schedular load in the example profile. Colour saturation in 71 | the heatmap and widths in the flamegraph are scaled to reductions. _Scaling 72 | flamegraph widths by metrics measured during sampling_ is the first of 73 | **flame_prof's two main innovations**. 74 | 75 | Crucially, in the flamegraph, the callstacks are categorised by the _process 76 | status_ of upto 100 processes running that particular code (the example profile 77 | auto-selected the top 100.) Such _callstack categorisation/grouping_ is 78 | **flame_prof's other main innovation**. It has proven extremely useful 79 | especially in identifying code causing concurrency bottlenecks or memory issues. 80 | 81 | ![reductions_delta heatmap screenshot](doc/screenshots/reductions_delta_heat.png) 82 | 83 | ![reductions_delta flamegraph screenshot](doc/screenshots/reductions_delta_flame.png) 84 | 85 | ### Off-heap binaries 86 | 87 | In addition to CPU load, **flame_prof** can also be used to identify code 88 | causing memory and/or garbage collection problems. 89 | 90 | Take binaries for instance. The bin_vheap_size heatmaps and corresponding 91 | flamegraphs below show size of off-heap binaries referenced by the callstacks. 92 | Colour saturation in the heatmap and widths in the flamegraph are scaled to 93 | words. 94 | 95 | Using some more callstack categorisation, the flamegraph is able to further 96 | group callstacks by the memory used by those processes when the calls were being 97 | made. 98 | 99 | ![bin_vheap heatmap screenshot](doc/screenshots/bin_vheap_heat.png) 100 | 101 | ![bin_vheap flamegraph screenshot](doc/screenshots/bin_vheap_flame.png) 102 | 103 | Below, we again see bin_vheap_size heatmaps and flamegraphs but this time by the 104 | delta. Green hue in the heatmap shows releases with saturation scaled to words. 105 | The flamegraph is further still categorised by positive (references) and 106 | negative (releases), the width of both scaled to words. 107 | 108 | ![bin_vheap_delta heatmap screenshot](doc/screenshots/bin_vheap_delta_heat.png) 109 | 110 | ![bin_vheap_delta flamegraph screenshot](doc/screenshots/bin_vheap_delta_flame.png) 111 | 112 | ## There's much more 113 | 114 | In addition to above, **flame_prof** generates heatmaps and flamegraphs for 115 | process message queue lengths, message buffer sizes, stack memory, heap memory 116 | and garbage collection (heap block sizes, etc) allowing the developer to easily 117 | and meaningfully cross reference these metrics with thousands of callstacks 118 | sampled from thousands of processes over a timeline. 119 | 120 | ## Build 121 | 122 | $ rebar3 compile -------------------------------------------------------------------------------- /doc/screenshots/bin_vheap_delta_flame.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/bin_vheap_delta_flame.png -------------------------------------------------------------------------------- /doc/screenshots/bin_vheap_delta_heat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/bin_vheap_delta_heat.png -------------------------------------------------------------------------------- /doc/screenshots/bin_vheap_flame.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/bin_vheap_flame.png -------------------------------------------------------------------------------- /doc/screenshots/bin_vheap_heat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/bin_vheap_heat.png -------------------------------------------------------------------------------- /doc/screenshots/mbuf_size_delta_heat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/mbuf_size_delta_heat.png -------------------------------------------------------------------------------- /doc/screenshots/reductions_delta_flame.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/reductions_delta_flame.png -------------------------------------------------------------------------------- /doc/screenshots/reductions_delta_heat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/reductions_delta_heat.png -------------------------------------------------------------------------------- /doc/screenshots/samples_heat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ebegumisa/flame_prof/8532cf07e1de1b29e8f6350da847cebab8eccbfc/doc/screenshots/samples_heat.png -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright 2020 Medical-Objects Pty Ltd. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %% 16 | 17 | {erl_opts, [debug_info,warn_export_vars,warn_shadow_vars,warn_obsolete_guard]}. -------------------------------------------------------------------------------- /src/flame_prof.app.src: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright 2020 Medical-Objects Pty Ltd. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %% 16 | 17 | {application, flame_prof, 18 | [{description, "Heatmap and flamegraph sampling profiler for Erlang"}, 19 | {vsn, "0.1.0"}, 20 | {applications, [kernel,stdlib]}, 21 | {modules, []}, 22 | {maintainers, ["Edmond Begumisa"]}, 23 | {licenses, ["Apache 2.0"]} 24 | ]}. 25 | -------------------------------------------------------------------------------- /src/flame_prof.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright 2020 Medical-Objects Pty Ltd. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %% 16 | 17 | %%% ============================================================================ 18 | %%% @doc Erlang profiler which generates Linux perf_events output (even on 19 | %%% Windows/MacOS). 20 | %%% 21 | %%% This is a general-purpose erlang profiler a little like OTP's `fprof', 22 | %%% except... 23 | %%% 24 | %%%
  • It generates Linux perf_events [1] script output [2] (even on 25 | %%% Win/macOS) intended to be consumed by, and analysed with, a fork of 26 | %%% Netflix's Flamescope [3].
    • 27 | %%%
  • It uses a call-stack sampling approach rather than attempting to 28 | %%% measure each individual call. So it does not need to use Erlang 29 | %%% tracing (though it may use tracing for certain optional features). 30 | %%%
    • 31 | %%%
  • It retains calling process information including process status, 32 | %%% memory usage, message queue lengths and garbage collection info. 33 | %%%
    • 34 | %%%
  • Provides control over output file writing (e.g. sample flush 35 | %%% frequency, output file rotation).
    • 36 | %%%
  • It provides means to automatically select processes to be profiled 37 | %%% (e.g. top 100 by reductions). Automatically triggering profiling in 38 | %%% in a controlled manner is coming soon.
    • 39 | %%% 40 | %%% [1] [https://en.wikipedia.org/wiki/Perf_(Linux)]
    41 | %%% [2] [https://linux.die.net/man/1/perf-script]
    42 | %%% [3] [https://github.com/ebegumisa/flamescope]
    43 | %%% @end 44 | %%% ============================================================================ 45 | -module(flame_prof). 46 | 47 | %%% -- Public Control -- 48 | -export([start/0, start/1, start/2, 49 | start_link/0, start_link/1, start_link/2, 50 | stop/0, stop/1, stop/2]). 51 | 52 | %%% -- Public Interface -- 53 | -export([apply/1, apply/2, 54 | profile_start/0, profile_start/1, 55 | profile_stop/0]). 56 | -type process_status_t() :: 57 | exiting | garbage_collecting | waiting | running | runnable | suspended. 58 | -type viral_mode_t() :: 59 | true | false | spawn | spawn_and_send | send | self_only. 60 | -type process_opts_t() :: 61 | #{viral => undefined | viral_mode_t(), 62 | status => undefined | all | process_status_t()}. 63 | -type start_opts_t() :: 64 | #{basename => undefined | file:filename(), 65 | flatten_recursive => undefined | boolean(), 66 | sample_interval => undefined | pos_integer(), 67 | flush_interval => undefined | pos_integer(), 68 | flush_close_max => undefined | infinity | pos_integer(), 69 | files_max => undefined | infinity | pos_integer(), 70 | auto_select => 71 | undefined 72 | | {top,ProcessCount :: pos_integer()} 73 | | {top,ProcessCount :: pos_integer(),Interval :: pos_integer()}, 74 | auto_process_opts => process_opts_t()}. 75 | -export_type([start_opts_t/0, 76 | process_status_t/0, viral_mode_t/0, 77 | process_opts_t/0]). 78 | 79 | %%% -- Private Interface -- 80 | -export([sampler/3]). 81 | -export([system_continue/3, system_terminate/4, system_get_state/1]). 82 | 83 | -type perf_counter_t() :: integer(). 84 | -type millisecs_t() :: non_neg_integer(). 85 | -type stack_initial_item_t() :: 86 | {Mod :: module(), Func :: atom(), Arity :: non_neg_integer()}. 87 | -type stack_item_t() :: 88 | {Mod :: module(), Func :: atom(), Arity :: non_neg_integer(), 89 | [{file,string()} | {line,non_neg_integer()}]}. 90 | -type stack_t() :: [stack_initial_item_t() | stack_item_t()]. 91 | -type stacks_t() :: #{pid() => stack_t()}. 92 | -type process_metrics_t() :: 93 | [{atom(),pos_integer()} | {atom(),[{atom(),pos_integer()}]}]. 94 | -type process_metrics_delta_t() :: 95 | [{atom(),pos_integer(),integer()} | 96 | {atom(),[{atom(),pos_integer(),integer()}]}]. 97 | -type processes_t() :: 98 | #{pid() => 99 | {StatusFilter :: all | process_status_t(), 100 | Metrics :: undefined | process_metrics_t()}}. 101 | -type process_grp_t() :: pos_integer(). 102 | -type sample_t() :: {TS :: perf_counter_t(), 103 | MetricsDelta :: process_metrics_delta_t(), 104 | Stacks :: stacks_t(), 105 | InitialCalls :: [stack_initial_item_t()]}. 106 | -type reg_names_t() :: #{pid() => 0 | atom()}. 107 | 108 | %%% -- Private Preprocess -- 109 | -define(DFLT_SAMPLE_INTERVAL, 9). % 99 hz (combined for all processes) 110 | -define(DFLT_FLUSH_INTERVAL, 15000). % 15 secs 111 | -define(DFLT_FLUSH_CLOSE_MAX, 8). % 2 mins worth 112 | -define(DFLT_FILES_MAX, 15). % 30 mins worth (30 / 2). 113 | -define(DFLT_TOP_INTERVAL, 10000). % 10 secs (etop's default). 114 | 115 | -define(DELAY_SIZE, 10*1024*1024). % 10 MiB 116 | -define(DELAY_INTERVAL, 5*60*1000). % 5 mins 117 | 118 | -define(BACKTRACE_DEPTH, 255). 119 | 120 | -record(st_r, 121 | {flush_tout :: perf_counter_t(), 122 | flush_close_max :: non_neg_integer(), 123 | flush_count=0 :: non_neg_integer(), 124 | sample_tout :: perf_counter_t(), 125 | sample_tout_ms :: millisecs_t(), 126 | prev_sample_ts :: undefined | perf_counter_t(), 127 | prev_flush_ts :: undefined | perf_counter_t(), 128 | ps_grps=#{} :: #{process_grp_t() => processes_t()}, 129 | expl_ps=#{} :: #{pid() => process_grp_t()}, 130 | auto_ps=#{} :: #{pid() => process_grp_t()}, 131 | auto_ts :: undefined | perf_counter_t(), 132 | reg_names=#{} :: reg_names_t(), 133 | samples=[] :: [sample_t()], 134 | file :: undefined | file:io_device(), 135 | paths :: undefined | {non_neg_integer(),queue:queue(file:filename())}, 136 | auto_selector :: undefined | pid(), 137 | auto_status :: process_status_t(), 138 | auto_traceopts :: [set_on_spawn|send]}). 139 | 140 | -include_lib("kernel/include/file.hrl"). 141 | 142 | %%% -- Control ----------------------------------------------------------------- 143 | 144 | -spec start() -> pid() | {error,{already_started,pid()}}. 145 | 146 | start() -> 147 | start(#{}). 148 | 149 | 150 | -spec start(start_opts_t() | non_neg_integer() | infinity) -> 151 | pid() | {error,{already_started,pid()}}. 152 | 153 | start(#{}=Opts) -> 154 | do_start(false, Opts, 5000); 155 | start(TOut) -> 156 | do_start(false, #{}, TOut). 157 | 158 | 159 | -spec start(start_opts_t(), non_neg_integer() | infinity) -> 160 | pid() | {error,{already_started,pid()}}. 161 | 162 | start(#{}=Opts, TOut) -> 163 | do_start(false, Opts, TOut). 164 | 165 | 166 | -spec start_link() -> pid() | {error,{already_started,pid()}}. 167 | 168 | start_link() -> 169 | start_link(#{}). 170 | 171 | 172 | -spec start_link(start_opts_t() | non_neg_integer() | infinity) -> 173 | pid() | {error,{already_started,pid()}}. 174 | 175 | start_link(#{}=Opts) -> 176 | do_start(true, Opts, 5000); 177 | start_link(TOut) -> 178 | do_start(true, #{}, TOut). 179 | 180 | 181 | -spec start_link(start_opts_t(), non_neg_integer() | infinity) -> 182 | pid() | {error,{already_started,pid()}}. 183 | 184 | start_link(#{}=Opts, TOut) -> 185 | do_start(true, Opts, TOut). 186 | 187 | 188 | do_start(ShouldLnk, #{}=Opts, TOut) 189 | when is_boolean(ShouldLnk), is_integer(TOut) orelse TOut=:=infinity 190 | -> 191 | A = [self(),chk_start_opts(Opts),TOut], 192 | case whereis(?MODULE) of 193 | undefined when ShouldLnk -> 194 | proc_lib:start_link(?MODULE, sampler, A, TOut, [{priority,max}]); 195 | undefined -> 196 | proc_lib:start(?MODULE, sampler, A, TOut, [{priority,max}]); 197 | P -> 198 | {error,{already_started,P}} 199 | end. 200 | 201 | 202 | stop() -> 203 | stop(?MODULE). 204 | 205 | stop(TOut) when is_integer(TOut); TOut=:=infinity -> 206 | stop(?MODULE, TOut); 207 | stop(P) -> 208 | stop(P, 5000). 209 | 210 | stop(P, TOut) when is_pid(P) orelse is_atom(P), 211 | is_integer(TOut) orelse TOut=:=infinity 212 | -> 213 | try proc_lib:stop(P, normal, TOut) 214 | catch exit:noproc -> ok 215 | end. 216 | 217 | 218 | %%% -- Interface --------------------------------------------------------------- 219 | 220 | -spec apply(fun(() -> term())) -> term(). 221 | %% 222 | %% @doc Same as calling {@link apply/2} with empty profiling process options 223 | %% (i.e. default process options). 224 | %% 225 | apply(Fn) -> 226 | ?MODULE:apply(Fn, #{}). 227 | 228 | 229 | -spec apply(fun(() -> term()), process_opts_t()) -> term(). 230 | %% 231 | %% @doc Profile the evaluation of a given function in the context of the calling 232 | %% process using the given profiling process options. 233 | %% 234 | apply(Fn, Opts) -> 235 | profile_start(Opts), 236 | try Fn() 237 | after profile_stop() 238 | end. 239 | 240 | 241 | -spec profile_start() -> non_neg_integer(). 242 | %% 243 | %% @doc Same as calling {@link profile_start/1} with emtpy profiling process 244 | %% options (i.e. default process options). 245 | %% 246 | profile_start() -> 247 | profile_start(#{}). 248 | 249 | 250 | -spec profile_start(process_opts_t()) -> non_neg_integer(). 251 | %% 252 | %% @doc Start profiling the calling process using the given profiling process 253 | %% options. 254 | %% 255 | profile_start(Opts) -> 256 | profile_start1(chk_p_opts(Opts, spawn)). 257 | 258 | profile_start1(#{viral:=Viral,status:=Status}) -> 259 | Sampler = whereis(?MODULE), 260 | true = is_pid(Sampler) orelse error({not_started,?MODULE}), 261 | ViralTraceOpts = viral_to_traceopts(Viral), 262 | % Notify sampler about this process by briefly tracing a single call to 263 | % explicit/1. 264 | erlang:trace_pattern({?MODULE,explicit,1}, true, [local]), 265 | erlang:trace(self(), true, [{tracer,Sampler},'call']), 266 | try Status = explicit(Status) 267 | after 268 | erlang:trace(self(), false, [all]), 269 | erlang:trace_pattern({?MODULE,process,1}, false, [local]) 270 | end, 271 | % Now that the sampler knows to sample us, profile without tracing any 272 | % calls. 273 | erlang:system_flag(backtrace_depth, ?BACKTRACE_DEPTH), 274 | maybe_trace_send_on(ViralTraceOpts), 275 | trace_p_on(self(), Sampler, ViralTraceOpts). 276 | 277 | 278 | -spec profile_stop() -> non_neg_integer(). 279 | 280 | profile_stop() -> 281 | trace_p_off(self()). 282 | 283 | %%% -- sampler proc_lib impl -- 284 | 285 | explicit(Status) -> Status. 286 | 287 | sampler(Parent, 288 | #{sample_interval:=SampleMs,flush_interval:=FlushMs, 289 | flush_close_max:=CloseMax,auto_select:=AutoSel, 290 | auto_process_opts:=#{status:=AutoStatus,viral:=AutoViral}}=Opts, 291 | TOut) when is_pid(Parent) 292 | -> 293 | process_flag(trap_exit, true), 294 | try register(?MODULE, self()) of 295 | true -> 296 | S = #st_r{sample_tout_ms=SampleMs, 297 | sample_tout=pc_from_millisecs(SampleMs), 298 | flush_tout=pc_from_millisecs(FlushMs), 299 | flush_close_max=CloseMax, 300 | paths=del_excess_files(Opts), 301 | auto_status=AutoStatus, 302 | auto_traceopts=viral_to_traceopts(AutoViral), 303 | auto_selector= 304 | case undefined=:=AutoSel 305 | orelse 306 | seltor@flame_prof:start_link( 307 | AutoSel, [{timeout,TOut}]) 308 | of 309 | {ok,P} -> P; 310 | true -> undefined 311 | end}, 312 | ok = proc_lib:init_ack(Parent, {ok,self()}), 313 | sampler_loop(Parent, nil, S, Opts) 314 | catch error:badarg -> 315 | {error,{already_started,erlang:whereis(?MODULE)}} 316 | end. 317 | 318 | sampler_loop(Parent, nil, #st_r{}=S, Opts) 319 | when map_size(S#st_r.expl_ps)=:=0, map_size(S#st_r.auto_ps)=:=0 320 | -> 321 | % Nothing to sample as yet, so wait here until there is. 322 | receive Msg -> sampler_loop(Parent, Msg, S, Opts) end; 323 | sampler_loop(Parent, nil, #st_r{}=S0, Opts) -> 324 | % Figure out if it's time to sample and/or flush the stacks. 325 | TS = os:perf_counter(), 326 | S1 = sampler_maybe_sample(TS, S0, Opts), 327 | % Wait for more profile events, otherwise check if we need to sample roughly 328 | % after the sample interval. 329 | receive Msg -> sampler_loop(Parent, Msg, S1, Opts) 330 | after S1#st_r.sample_tout_ms -> sampler_loop(Parent, nil, S1, Opts) 331 | end; 332 | sampler_loop(Parent, {auto_selected,[_|_]=Ps}, #st_r{}=S, Opts) -> 333 | % seltor@flame_prof has sent the set of processes to automatically 334 | % profile. Replace the ones we have. 335 | TS = os:perf_counter(), 336 | erlang:system_flag(backtrace_depth, ?BACKTRACE_DEPTH), 337 | maybe_trace_send_on(S#st_r.auto_traceopts), 338 | sampler_loop( 339 | Parent, nil, sampler_ps_add_auto(Ps, S#st_r{auto_ts=TS}), Opts); 340 | sampler_loop(Parent, {trace,P,call,{?MODULE,explicit,[Status]}}, 341 | #st_r{}=S, Opts) 342 | -> 343 | % Add calling process to map of processes to explicitly sample 344 | % (call is as result of an external call to apply/N or profile_start/N). 345 | sampler_loop(Parent, nil, sampler_p_add_expl(P, Status, S), Opts); 346 | sampler_loop(Parent, {trace,_,call,_}, #st_r{}=S, Opts) -> 347 | sampler_loop(Parent, nil, S, Opts); 348 | sampler_loop(Parent, {trace,_,send,_,_}, #st_r{}=S, Opts) 349 | when map_size(S#st_r.expl_ps)=:=0, map_size(S#st_r.auto_ps)=:=0 350 | -> 351 | trace_send_off(), 352 | sampler_loop(Parent, nil, S, Opts); 353 | sampler_loop(Parent, {trace,_,send,_,{NmeOrP,Nd}}=Ev, #st_r{}=S, Opts) -> 354 | case Nd=:=node() andalso (is_pid(NmeOrP) orelse whereis(NmeOrP)) of 355 | true -> 356 | sampler_loop(Parent, setelement(5, Ev, NmeOrP), S, Opts); 357 | false -> 358 | sampler_loop(Parent, nil, S, Opts); 359 | undefined -> 360 | sampler_loop(Parent, nil, S, Opts); 361 | P when is_pid(P) -> 362 | sampler_loop(Parent, setelement(5, Ev, P), S, Opts) 363 | end; 364 | sampler_loop(Parent, {trace,Frm,send,_,To}, #st_r{}=S0, Opts) 365 | when is_pid(Frm), is_pid(To) 366 | -> 367 | % Add local recipient process to map of processes to explicitly sample 368 | % if the sending process is being explicitly sampled (i.e. send is as result 369 | % of an exernal call to apply/N or profile_start/N) 370 | S1 = case node(To)=:=node() andalso S0#st_r.expl_ps of 371 | #{Frm:=ExplGrp} -> 372 | case S0#st_r.ps_grps of 373 | #{ExplGrp:=#{Frm:=ExplStatus}} -> 374 | sampler_p_add_expl(To, ExplStatus, S0); 375 | #{} -> 376 | local 377 | end; 378 | #{} -> 379 | local; 380 | false -> 381 | remote 382 | end, 383 | S2 = case S1=:=local andalso S1#st_r.auto_ps of 384 | #{Frm:=AutoGrp} -> 385 | % Add local recipient process to map of processes to auto sample 386 | % if sending process is being auto sampled. 387 | case S1#st_r.ps_grps of 388 | #{AutoGrp:=#{Frm:=AutoStatus}} -> 389 | sampler_p_add_auto(To, AutoStatus, S1); 390 | #{} -> 391 | S0 392 | end; 393 | #{} -> 394 | S1; 395 | false -> 396 | S0 397 | end, 398 | sampler_loop(Parent, nil, S2, Opts); 399 | sampler_loop(Parent, {trace,Frm,send,_,_}=Ev, #st_r{}=S, Opts) when is_atom(Frm) 400 | -> 401 | case whereis(Frm) of 402 | undefined -> 403 | sampler_loop(Parent, nil, S, Opts); 404 | P -> 405 | sampler_loop(Parent, setelement(2, Ev, P), S, Opts) 406 | end; 407 | sampler_loop(Parent, {trace,_,send,_,To}=Ev, #st_r{}=S, Opts) when is_atom(To) -> 408 | case whereis(To) of 409 | undefined -> 410 | sampler_loop(Parent, nil, S, Opts); 411 | P -> 412 | sampler_loop(Parent, setelement(5, Ev, P), S, Opts) 413 | end; 414 | sampler_loop(Parent, {trace,_,send,_,To}, #st_r{}=S, Opts) when is_port(To) -> 415 | sampler_loop(Parent, nil, S, Opts); 416 | sampler_loop(Parent, {trace,_,send_to_non_existing_process,_,_}, #st_r{}=S, 417 | Opts) 418 | -> 419 | map_size(S#st_r.expl_ps)=:=0 andalso map_size(S#st_r.auto_ps)=:=0 andalso 420 | trace_send_off(), 421 | sampler_loop(Parent, nil, S, Opts); 422 | sampler_loop(Parent, {trace,P,exit,_}, #st_r{}=S, Opts) -> 423 | sampler_loop(Parent, nil, sampler_p_remove(P, S), Opts); 424 | sampler_loop(Parent, {trace,P,register,Nme}, #st_r{reg_names=Nmes}=S, Opts) -> 425 | sampler_loop(Parent, nil, S#st_r{reg_names=Nmes#{P=>Nme}}, Opts); 426 | sampler_loop(Parent, {trace,P,unregister,_}, #st_r{reg_names=Nmes}=S, Opts) -> 427 | sampler_loop(Parent, nil, S#st_r{reg_names=Nmes#{P=>0}}, Opts); 428 | sampler_loop(Parent, {trace,_,spawn,_,_}, #st_r{}=S, Opts) -> 429 | sampler_loop(Parent, nil, S, Opts); 430 | sampler_loop(Parent, {trace,_,spawned,_,_}, #st_r{}=S, Opts) -> 431 | sampler_loop(Parent, nil, S, Opts); 432 | sampler_loop(Parent, {trace,_,link,_}, #st_r{}=S, Opts) -> 433 | sampler_loop(Parent, nil, S, Opts); 434 | sampler_loop(Parent, {trace,_,unlink,_}, #st_r{}=S, Opts) -> 435 | sampler_loop(Parent, nil, S, Opts); 436 | sampler_loop(Parent, {trace,_,getting_linked,_}, #st_r{}=S, Opts) -> 437 | sampler_loop(Parent, nil, S, Opts); 438 | sampler_loop(Parent, {trace,_,getting_unlinked,_}, #st_r{}=S, Opts) -> 439 | sampler_loop(Parent, nil, S, Opts); 440 | sampler_loop(Parent, Other, #st_r{auto_selector=AutoSeltor}=S, Opts) -> 441 | % Handle 'EXIT' messages, sys messages and unexpected trace events. 442 | case Other of 443 | {'EXIT',Parent,Reason} -> 444 | system_terminate(Reason, Parent, undefined, {S,Opts}); 445 | {'EXIT',AutoSeltor,Reason} -> 446 | case is_terminate_crash(Reason) of 447 | true -> system_terminate(Reason, Parent, undefined, {S,Opts}); 448 | false -> sampler_loop(Parent, nil, S, Opts) 449 | end; 450 | {system,Frm,Rq} -> 451 | sys:handle_system_msg( 452 | Rq, Frm, Parent, ?MODULE, undefined, {S,Opts}); 453 | Other when Other=/=nil -> 454 | system_terminate({unexpected,Other}, Parent, undefined, {S,Opts}) 455 | end. 456 | 457 | sampler_p_remove(P, #st_r{ps_grps=PsGrps}=S) -> 458 | case S#st_r.expl_ps of 459 | #{P:=Grp} -> 460 | #{Grp:=Ps} = PsGrps, 461 | false = maps:is_key(P, S#st_r.auto_ps), 462 | S#st_r{ps_grps=PsGrps#{Grp:=maps:remove(P, Ps)}, 463 | expl_ps=maps:remove(P, S#st_r.expl_ps), 464 | reg_names=maps:remove(P, S#st_r.reg_names)}; 465 | #{} -> 466 | case S#st_r.auto_ps of 467 | #{P:=Grp} -> 468 | #{Grp:=Ps} = PsGrps, 469 | false = maps:is_key(P, S#st_r.expl_ps), 470 | S#st_r{ps_grps=PsGrps#{Grp:=maps:remove(P, Ps)}, 471 | reg_names=maps:remove(P, S#st_r.reg_names)}; 472 | #{} -> 473 | false = maps:is_key(P, S#st_r.reg_names), 474 | S 475 | end 476 | end. 477 | 478 | sampler_p_add_grp(P, Status, PsGrps, Ps) -> 479 | Grp = case maps:is_key(P, Ps) of 480 | true -> maps:get(P, Ps); 481 | false -> rand:uniform(11) 482 | end, 483 | {case PsGrps of 484 | #{Grp:=#{P:=X}=PsGrp} -> 485 | {_,Metrics} = X, 486 | PsGrps#{Grp:=PsGrp#{P=>{Status,Metrics}}}; 487 | #{Grp:=PsGrp} -> 488 | PsGrps#{Grp:=PsGrp#{P=>{Status,undefined}}}; 489 | #{} -> 490 | PsGrps#{Grp=>#{P=>{Status,undefined}}} 491 | end, 492 | Ps#{P=>Grp}}. 493 | 494 | sampler_p_add_expl(P, Status, S) -> 495 | {PsGrps,ExplPs} = 496 | sampler_p_add_grp(P, Status, S#st_r.ps_grps, S#st_r.expl_ps), 497 | S#st_r{ps_grps=PsGrps, 498 | expl_ps=ExplPs, 499 | auto_ps=maps:remove(P, S#st_r.auto_ps)}. 500 | 501 | sampler_p_add_auto(P, Status, S) -> 502 | case maps:is_key(P, S#st_r.expl_ps) of 503 | false -> 504 | {PsGrps,AutoPs} = 505 | sampler_p_add_grp(P, Status, S#st_r.ps_grps, S#st_r.auto_ps), 506 | S#st_r{ps_grps=PsGrps,auto_ps=AutoPs}; 507 | true -> 508 | S 509 | end. 510 | 511 | sampler_ps_add_auto(Ps, S) -> 512 | sampler_ps_add_auto1(Ps, S#st_r.auto_ps, [], S). 513 | 514 | sampler_ps_add_auto1([H|T], OrigAutoPs, AccPs, S) -> 515 | case S of 516 | #st_r{expl_ps=#{H:=_}} -> 517 | sampler_ps_add_auto1(T, OrigAutoPs, AccPs, S); 518 | #st_r{auto_status=AutoStatus} -> 519 | % Ensure we're tracing proccess if it's to be virally profiled. 520 | S#st_r.auto_traceopts=/=[] andalso 521 | trace_p_on(H, self(), S#st_r.auto_traceopts), 522 | sampler_ps_add_auto1( 523 | T, OrigAutoPs, [H|AccPs], sampler_p_add_auto(H, AutoStatus, S)) 524 | end; 525 | sampler_ps_add_auto1([], #{}=OrigAutoPs, [_|_]=AccPs, S0) -> 526 | if 527 | map_size(OrigAutoPs)=:=0 -> 528 | #st_r{}=S0; 529 | true -> 530 | % Stop profiling processes that were, but are no longer, in auto 531 | % profiled set. 532 | maps:fold( 533 | fun(P, _, S1) -> 534 | S0#st_r.auto_traceopts=/=[] andalso 535 | trace_p_off(P), % Stop tracing virally profiled process 536 | sampler_p_remove(P, S1) 537 | end, S0, maps:without(AccPs, OrigAutoPs)) 538 | end; 539 | sampler_ps_add_auto1([], #{}, [], S) -> 540 | S. 541 | 542 | sampler_maybe_sample(TS, #st_r{prev_flush_ts=undefined}=S, _) -> 543 | % First loop 544 | S#st_r{prev_sample_ts=TS,prev_flush_ts=TS}; 545 | sampler_maybe_sample(TS, S, Opts) -> 546 | if 547 | TS - S#st_r.prev_flush_ts >= S#st_r.flush_tout -> 548 | % Sample once more, flush all samples then possibly close. 549 | FileS = 550 | if 551 | S#st_r.file=:=undefined -> 552 | TS - S#st_r.prev_flush_ts; 553 | S#st_r.flush_count >= S#st_r.flush_close_max -> 554 | close; 555 | true -> 556 | opened 557 | end, 558 | flush_samples( 559 | FileS, 560 | sampler_maybe_sample1( 561 | TS, S#st_r{prev_sample_ts=TS,prev_flush_ts=TS}), 562 | Opts); 563 | TS - S#st_r.prev_sample_ts >= S#st_r.sample_tout -> 564 | % Sample and accumulate 565 | sampler_maybe_sample1(TS, S#st_r{prev_sample_ts=TS}); 566 | true -> 567 | S 568 | end. 569 | 570 | sampler_maybe_sample1(TS, #st_r{ps_grps=PsGrps}=S) -> 571 | Grp = rand:uniform(11), 572 | case PsGrps of 573 | #{Grp:=Ps0} -> 574 | case maps:fold(fun sample_p/3, {undefined,Ps0}, Ps0) of 575 | {#{}=Stks,#{}=Ps1} -> 576 | Samples = [{TS,Stks}|S#st_r.samples], 577 | S#st_r{samples=Samples,ps_grps=PsGrps#{Grp:=Ps1}}; 578 | {undefined,_} -> 579 | S 580 | end; 581 | #{} -> 582 | S 583 | end. 584 | 585 | system_continue(Parent, _, {S,Opts}) -> 586 | sampler_loop(Parent, nil, S, Opts). 587 | 588 | system_get_state({S,Opts}) -> 589 | {ok,{{'$hidden',erts_debug:size(S)},Opts}}. 590 | 591 | system_terminate(Reason, _, _, {S,Opts}) -> 592 | try flush_samples(close, S, Opts) 593 | after trace_send_off() 594 | end, 595 | exit(Reason). 596 | 597 | %%% -- Helpers ----------------------------------------------------------------- 598 | 599 | chk_start_opts(#{}=Opts0) -> 600 | Opts1 = 601 | lists:foldl( 602 | fun(K, Acc) -> 603 | Acc#{K=>chk_start_opt(K, maps:get(K, Acc, undefined))} 604 | end, Opts0, [basename,sample_interval,flush_interval, 605 | flush_close_max,flatten_recursive,files_max, 606 | auto_select,auto_process_opts]), 607 | #{flush_interval:=FlushMs,sample_interval:=SampleMs} = Opts1, 608 | (FlushMs - 1000) < SampleMs andalso error({badarg,flush_interval}), 609 | Opts1. 610 | 611 | chk_start_opt(basename, V) when is_list(V); is_binary(V) -> 612 | V; 613 | chk_start_opt(flatten_recursive, V) when is_boolean(V) -> 614 | V; 615 | chk_start_opt(sample_interval, V) when is_integer(V), V>0 -> 616 | V; 617 | chk_start_opt(flush_interval, V) when is_integer(V), V>0 -> 618 | V; 619 | chk_start_opt(flush_close_max, V) when is_integer(V), V>=0 -> 620 | V; 621 | chk_start_opt(flush_close_max, infinity) -> 622 | 0; 623 | chk_start_opt(files_max, infinity) -> 624 | infinity; 625 | chk_start_opt(files_max, V) when is_integer(V), V>0 -> 626 | V; 627 | chk_start_opt(auto_select, {top,0}) -> 628 | undefined; 629 | chk_start_opt(auto_select, {top,0,Ival}) when is_integer(Ival), Ival>0 -> 630 | undefined; 631 | chk_start_opt(auto_select, {top,N}) -> 632 | chk_start_opt(auto_select, {top,N,?DFLT_TOP_INTERVAL}); 633 | chk_start_opt(auto_select, {top,N,Ival}=V) 634 | when is_integer(N), N>0, is_integer(Ival), Ival>0 635 | -> 636 | V; 637 | chk_start_opt(auto_process_opts, undefined) -> 638 | chk_start_opt(auto_process_opts, #{}); 639 | chk_start_opt(auto_process_opts, V) -> 640 | chk_p_opts(V, self_only); 641 | % ^^^ Default for auto sampled processes as opposed to 'spawn' 642 | % which is the default for explicitly sampled processes. 643 | 644 | chk_start_opt(K, undefined) -> 645 | dflt_start_opt(K); 646 | chk_start_opt(K, V) when is_boolean(V) -> 647 | error({badarg,{K,V}}). 648 | 649 | dflt_start_opt(basename) -> 650 | {home,Home} = lists:keyfind(home, 1, init:get_arguments()), 651 | filename:join(Home, ?MODULE); 652 | dflt_start_opt(flatten_recursive) -> 653 | true; 654 | dflt_start_opt(sample_interval) -> 655 | ?DFLT_SAMPLE_INTERVAL; 656 | dflt_start_opt(flush_interval) -> 657 | ?DFLT_FLUSH_INTERVAL; 658 | dflt_start_opt(flush_close_max) -> 659 | ?DFLT_FLUSH_CLOSE_MAX; 660 | dflt_start_opt(files_max) -> 661 | ?DFLT_FILES_MAX; 662 | dflt_start_opt(auto_select) -> 663 | undefined. 664 | 665 | chk_p_opts(#{}=Opts, DfltViral) -> 666 | lists:foldl( 667 | fun 668 | (viral, Acc) -> 669 | case chk_p_opt(viral, maps:get(viral, Acc, undefined)) of 670 | undefined -> 671 | Acc#{viral=>DfltViral}; 672 | V -> 673 | Acc#{viral=>V} 674 | end; 675 | (K, Acc) -> 676 | Acc#{K=>chk_p_opt(K, maps:get(K, Acc, undefined))} 677 | end, Opts, [viral,status]). 678 | 679 | chk_p_opt(viral, true) -> 680 | spawn_and_send; 681 | chk_p_opt(viral, false) -> 682 | self_only; 683 | chk_p_opt(viral, V) 684 | when V=:=spawn; V=:=spawn_and_send; V=:=send; V=:=self_only 685 | -> 686 | V; 687 | chk_p_opt(status, V) 688 | when V=:=all; V=:=exiting; V=:=garbage_collecting; V=:=waiting; 689 | V=:=running; V=:=runnable; V=:=suspended 690 | -> 691 | V; 692 | chk_p_opt(viral, undefined) -> 693 | undefined; % Intentional. See chk_p_opts/2. 694 | chk_p_opt(status, undefined) -> 695 | all; 696 | chk_p_opt(K, V) -> 697 | error({badarg,{K,V}}). 698 | 699 | viral_to_traceopts(self_only) -> 700 | []; 701 | viral_to_traceopts(spawn) -> 702 | [set_on_spawn]; 703 | viral_to_traceopts(spawn_and_send) -> 704 | [send,set_on_spawn]; % NB: We rely on 'send' always being at head. 705 | viral_to_traceopts(send) -> 706 | [send]. % Ditto NB above. 707 | 708 | sample_p(P, {Filter,PrevMetrics}, {Stks,Ps}) -> 709 | case erlang:process_info(P, [status]) of 710 | [{status,Status}] when Filter=:=all; Status=:=Filter -> 711 | case erlang:process_info( 712 | P, [current_function,current_stacktrace,initial_call, 713 | dictionary,reductions,message_queue_len, 714 | memory,total_heap_size,garbage_collection_info, 715 | garbage_collection]) 716 | of 717 | [{current_function,CurrMFA0},{current_stacktrace,Stk}, 718 | {initial_call,Init},{dictionary,Dict}|CurrMetrics] 719 | -> 720 | % TODO: Confirm that this may be different from head of 721 | % current_stacktrace. If not, remove it. 722 | CurrMFA1 = 723 | case CurrMFA0 of 724 | undefined -> native; 725 | _ -> CurrMFA0 726 | end, 727 | Inits = 728 | case lists:keyfind('$initial_call', 1, Dict) of 729 | {'$initial_call',{M,F,N}}=GenInit 730 | when is_atom(M), is_atom(F), is_integer(N) 731 | -> 732 | [Init,GenInit]; 733 | _ -> 734 | [Init] 735 | end, 736 | MetricsDelta = metrics_delta(PrevMetrics, CurrMetrics), 737 | Sample = {Status,MetricsDelta,[CurrMFA1|Stk],Inits}, 738 | if 739 | undefined=:=Stks -> 740 | {#{P=>Sample},Ps#{P:={Filter,CurrMetrics}}}; 741 | true -> 742 | {Stks#{P=>Sample},Ps#{P:={Filter,CurrMetrics}}} 743 | end; 744 | undefined -> 745 | {Stks,Ps} % TODO: Remove P for st_r.ps so we don't have to trace exits 746 | end; 747 | [{status,_}] -> 748 | {Stks,Ps}; 749 | undefined -> 750 | {Stks,Ps} % TODO: Ditto above. 751 | end. 752 | 753 | metrics_delta(undefined, [{_,[]}=PairB|TB]) -> 754 | [PairB|metrics_delta(undefined, TB)]; 755 | metrics_delta([{K,[]}|TA], [{K,[]}=PairB|TB]) -> 756 | [PairB|metrics_delta(TA, TB)]; 757 | metrics_delta(undefined, [{garbage_collection,LB}|TB]) -> 758 | [{garbage_collection,[{K,V,0} || {K,V} <- LB, K=:=minor_gcs]} 759 | |metrics_delta(undefined, TB)]; 760 | metrics_delta([{garbage_collection,LA}|TA], 761 | [{garbage_collection,LB}|TB]) 762 | -> 763 | {_,VA} = lists:keyfind(minor_gcs, 1, LA), 764 | {_,VB} = lists:keyfind(minor_gcs, 1, LB), 765 | VDelta = if VB>VA -> VB-VA; true -> VB end, 766 | % NB: VB-VA ^^^ doesn't make sense if there's been one or more full sweeps 767 | % between samples [1]. If VBVB, we need to somehow find detect if there's been a full 771 | % sweep and similarly take VB as the delta. This case we're currently 772 | % not handling. 773 | % [1] https://github.com/erlang/otp/blob/de00be8f6723dededcbcff6f635394167b609367/erts/emulator/beam/erl_bif_info.c#L1812 774 | % https://github.com/erlang/otp/blob/a67f636bcf14dd5eebea9b1bf1e7b89b38895b22/erts/emulator/beam/erl_gc.c#L1855 775 | [{garbage_collection,[{minor_gcs,VB,VDelta}]}|metrics_delta(TA, TB)]; 776 | metrics_delta(undefined, [{garbage_collection_info,LB}|TB]) -> 777 | [{garbage_collection_info,[{K,VB,0} || {K,VB} <- LB]} 778 | |metrics_delta(undefined, TB)]; 779 | metrics_delta([{garbage_collection_info,LA}|TA], 780 | [{garbage_collection_info,LB}|TB]) 781 | -> 782 | [{garbage_collection_info, 783 | fun 784 | Delta([{K,VA}|TAInner], [{K,VB}|TBInner]) -> 785 | [{K,VB,VB-VA}|Delta(TAInner, TBInner)]; 786 | Delta([], []) -> 787 | [] 788 | end(LA, LB)} | metrics_delta(TA, TB)]; 789 | metrics_delta(undefined, [{reductions,VB}|TB]) -> 790 | [{reductions,VB,1}|metrics_delta(undefined, TB)]; 791 | metrics_delta(undefined, [{K,VB}|TB]) -> 792 | [{K,VB,0}|metrics_delta(undefined, TB)]; 793 | metrics_delta([{reductions,VA}|TA], [{reductions,VB}|TB]) -> 794 | [{reductions,VB,case VB-VA of X when X>=0 -> X+1; X -> -X end} 795 | % XXX: Negative does happen ^ 796 | % (maybe due to a scheduler race?) 797 | |metrics_delta(TA, TB)]; 798 | metrics_delta([{K,VA}|TA], [{K,VB}|TB]) -> 799 | [{K,VB,VB-VA}|metrics_delta(TA, TB)]; 800 | metrics_delta(undefined, []) -> 801 | []; 802 | metrics_delta([], []) -> 803 | []. 804 | 805 | flush_samples(Dur, #st_r{file=undefined,samples=[]}=S, #{}) when is_integer(Dur) 806 | -> 807 | S; 808 | flush_samples(Dur, #st_r{file=undefined}=S, #{basename:=BseNme, 809 | files_max:=Max}=Opts) 810 | when is_integer(Dur) 811 | -> 812 | StartSysTm = erlang:system_time(perf_counter) - Dur, 813 | {{Yy,Mn,Dd},{Hh,Mm,Ss}} = 814 | calendar:system_time_to_local_time(StartSysTm, perf_counter), 815 | Leaf = io_lib:format( 816 | "~4..0w-~2..0w-~2..0w@~2..0w~2..0w.~2..0w.erl_perf.log", 817 | [Yy,Mn,Dd,Hh,Mm,Ss]), 818 | Path = filename:join(BseNme, Leaf), 819 | _ = filelib:ensure_dir(Path), 820 | {ok,Fd} = file:open(Path, [append,read, 821 | {delayed_write,?DELAY_SIZE,?DELAY_INTERVAL}]), 822 | flush_samples1(Fd, S#st_r.samples, S#st_r.reg_names, Opts), 823 | file_flush_delayed(Fd), 824 | Paths = 825 | case S#st_r.paths of 826 | undefined when Max=:=undefined -> 827 | undefined; 828 | {N,Q0} when N=:=Max -> 829 | {{value,FPath},Q1} = queue:out(Q0), 830 | _ = file:delete(FPath), 831 | {Max,queue:in(Path, Q1)}; 832 | {N,Q} when N>=0, N 833 | {N+1,queue:in(Path, Q)} 834 | end, 835 | S#st_r{samples=[],flush_count=0,paths=Paths,file=Fd}; 836 | flush_samples(close, #st_r{file=undefined,samples=[]}=S, #{}) -> 837 | S; 838 | flush_samples(close, #st_r{file=undefined,samples=[_|_]}=S, #{}=Opts) -> 839 | flush_samples(close, flush_samples(0, S, Opts), Opts); 840 | flush_samples(close, #st_r{}=S, Opts) -> 841 | Fd = flush_samples1(S#st_r.file, S#st_r.samples, S#st_r.reg_names, Opts), 842 | file_flush_delayed(Fd), 843 | _ = file:close(Fd), 844 | S#st_r{samples=[],flush_count=0,file=undefined}; 845 | flush_samples(opened, #st_r{samples=[]}=S, #{}) -> 846 | S; 847 | flush_samples(opened, #st_r{flush_count=FlushCount}=S, #{}=Opts) -> 848 | S#st_r{samples=[],flush_count=FlushCount+1, 849 | file=flush_samples1( 850 | S#st_r.file, S#st_r.samples, S#st_r.reg_names, Opts)}. 851 | 852 | flush_samples1(Fd, [], #{}, #{}) -> 853 | Fd; 854 | flush_samples1(Fd, [_|_]=Samples, Nmes, #{flatten_recursive:=Flatten}) -> 855 | flush_samples2(Fd, lists:reverse(Samples), Nmes, Flatten). 856 | % TODO: ^^^ Does the order really matter to FlameScope or 857 | % flamegraph.pl? If not, no need to bother with 858 | % reverse. Mabye a start op to retain order? 859 | 860 | flush_samples2(Fd, [], _, _) -> 861 | Fd; 862 | flush_samples2(Fd, [{TS,#{}=Ps}|TSamples], Nmes, Flatten) -> 863 | % For format, see... 864 | % https://github.com/Netflix/flamescope/blob/141238b4f69feee1d026c2156951eb78857e3953/app/perf/regexp.py#L24 865 | % https://github.com/Netflix/flamescope/blob/141238b4f69feee1d026c2156951eb78857e3953/app/perf/flame_graph.py#L185 866 | % https://github.com/Netflix/flamescope/blob/141238b4f69feee1d026c2156951eb78857e3953/app/common/fileutil.py#L45 867 | maps:fold( 868 | fun 869 | Fold(P, {Status,Metrics,Stk,[_|_]=Inits}, sample) -> 870 | PStr = pid_str(P), 871 | %Nme = case maps:get(P, Nmes, 0) of 872 | % 0 -> PStr; 873 | % X -> atom_to_list(X) 874 | % end, 875 | Us = pc_to_microsecs(TS), 876 | io:put_chars( 877 | Fd, [atom_to_list(Status),$\t,PStr," [000] ", 878 | microsecs_to_secs_str(Us),": cpu-clock:"]), 879 | flush_metrics(Fd, Metrics), 880 | io:put_chars(Fd, "\n"), 881 | Fold(P, Stk, Inits); 882 | Fold(P, [{M,F,N,_}]=Stk, [{M,F,N}|TNext]) -> 883 | Fold(P, Stk, TNext); 884 | Fold(P, [{M,F,N}]=Stk, [{M,F,N}|TNext]) -> 885 | Fold(P, Stk, TNext); 886 | Fold(P, [{M,F,N,_}]=Stk, [{'$initial_call',{M,F,N}}]) -> 887 | Fold(P, Stk, []); 888 | Fold(P, [{M,F,N}]=Stk, [{'$initial_call',{M,F,N}}]) -> 889 | Fold(P, Stk, []); 890 | Fold(P, [{M,F,N},{M,F,N,_}=Second|TStk], Next) -> 891 | Fold(P, [Second|TStk], Next); 892 | Fold(P, [{M,F,N}|TStk], Next) -> 893 | Fold(P, [{M,F,N,[]}|TStk], Next); 894 | Fold(P, [{M,F,N,_},{M,F,N,_}=Recursive|TStk], Next) 895 | when Flatten 896 | -> 897 | Fold(P, [Recursive|TStk], Next); 898 | Fold(P, [{M,F,N,[{file,File},{line,Ln}|_]}|TStk], Next) -> 899 | io:put_chars(Fd, ["\tffffffff ",atom_to_list(M),$:, 900 | atom_to_list(F),$/,integer_to_list(N),$;, 901 | integer_to_list(Ln)," (",File,")\n"]), 902 | Fold(P, TStk, Next); 903 | Fold(P, [{M,F,N,[]}|TStk], Next) -> 904 | MStr = atom_to_list(M), 905 | io:put_chars(Fd, ["\tffffffff ",MStr,$:,atom_to_list(F),$/, 906 | integer_to_list(N)," (",MStr,".erl)\n"]), 907 | Fold(P, TStk, Next); 908 | Fold(P, [{'$initial_call',{M,F,N}}|TStk], Next) -> 909 | MStr = atom_to_list(M), 910 | io:put_chars(Fd, ["\tffffffff (",MStr,$:,atom_to_list(F),$/, 911 | integer_to_list(N),") (",MStr,".erl)\n"]), 912 | Fold(P, TStk, Next); 913 | Fold(P, [native|TStk], Next) -> 914 | io:put_chars(Fd, ["\tffffffff native ()\n"]), 915 | Fold(P, TStk, Next); 916 | Fold(P, [], [_|_]=Inits) -> 917 | Fold(P, Inits, sample); 918 | Fold(P, [], []) -> 919 | Fold(P, [], sample); 920 | Fold(_, [], sample) -> 921 | io:put_chars(Fd, "\n"), 922 | sample 923 | end, sample, Ps), 924 | flush_samples2(Fd, TSamples, Nmes, Flatten). 925 | 926 | flush_metrics(Fd, [{_,[]}|T]) -> 927 | flush_metrics(Fd, T); 928 | flush_metrics(Fd, [{_,[_|_]=L}|T]) -> 929 | io:put_chars( 930 | Fd, [["\tmetric_erlang_",atom_to_list(K),": ",integer_to_list(V),", ", 931 | integer_to_list(Delta)] || {K,V,Delta} <- L]), 932 | flush_metrics(Fd, T); 933 | flush_metrics(Fd, [{K,V,Delta}|T]) -> 934 | io:put_chars( 935 | Fd, ["\tmetric_erlang_",atom_to_list(K),": ",integer_to_list(V),", ", 936 | integer_to_list(Delta)]), 937 | flush_metrics(Fd, T); 938 | flush_metrics(_, []) -> 939 | ok. 940 | 941 | %%% 942 | %% Delete all but the lexicographically last `files_max' *.erl_perf.log files in 943 | %% the basename dir, returning their count and paths. 944 | %% 945 | del_excess_files(#{files_max:=infinity}) -> 946 | undefined; 947 | del_excess_files(#{files_max:=Max,basename:=BseNme}) when Max>=0 -> 948 | Paths = filelib:wildcard(filename:join(BseNme,"*.erl_perf.log")), 949 | lists:foldr( 950 | fun 951 | (Path, {N,_}=Acc) when N=:=Max -> 952 | _ = file:delete(Path), 953 | Acc; 954 | (Path, {N,Q}=Acc) when N 955 | case file_del_if_zero(Path) of 956 | true -> 957 | _ = file:delete(Path), 958 | Acc; 959 | false -> 960 | {N+1,queue:in_r(Path, Q)} 961 | end 962 | end, {0,queue:new()}, Paths). 963 | 964 | file_del_if_zero(Path) -> 965 | case file:read_file_info(Path) of 966 | {ok,#file_info{size=0}} -> 967 | _ = file:delete(Path), 968 | true; 969 | {ok,#file_info{}} -> 970 | false; 971 | {error,_} -> 972 | false 973 | end. 974 | 975 | file_flush_delayed(Fd) -> 976 | case file:read(Fd, 1) of 977 | {ok,_} -> ok; 978 | eof -> ok; 979 | {error,enoent} -> ok; 980 | {error,enotdir} -> ok 981 | end. 982 | 983 | %%% 984 | %% Flamescope and flamegraph.pl don't like the "<" and ">" in the process name 985 | %% and PID columns of the output file, so strip them from the erlang process 986 | %% pid. 987 | %% 988 | pid_str(P) when is_pid(P) -> 989 | pid_str(pid_to_list(P)); 990 | pid_str([$>]) -> 991 | []; 992 | pid_str([$<|T]) -> 993 | pid_str(T); 994 | pid_str([C|T]) -> 995 | [C|pid_str(T)]. 996 | 997 | maybe_trace_send_on([send|_]) -> 998 | erlang:trace_pattern(send, true, []); 999 | maybe_trace_send_on([_|_]) -> 1000 | 0; 1001 | maybe_trace_send_on([]) -> 1002 | 0. 1003 | 1004 | trace_send_off() -> 1005 | erlang:trace_pattern(send, false, []). 1006 | 1007 | trace_p_on(P, Sampler, [_|_]=ViralTraceOpts) -> 1008 | TraceOpts = [{tracer,Sampler},procs|ViralTraceOpts], 1009 | if 1010 | P=:=self() -> 1011 | erlang:trace(P, true, TraceOpts); 1012 | true -> 1013 | try erlang:trace(P, true, TraceOpts) 1014 | catch error:badarg -> 1015 | case is_process_alive(P) of 1016 | false -> 0; 1017 | true -> error(badarg) 1018 | end 1019 | end 1020 | end. 1021 | 1022 | trace_p_off(P) when P=:=self() -> 1023 | erlang:trace(P, false, [all]); 1024 | trace_p_off(P) -> 1025 | try erlang:trace(P, false, [all]) 1026 | catch error:badarg -> 1027 | case is_process_alive(P) of 1028 | false -> 0; 1029 | true -> error(badarg) 1030 | end 1031 | end. 1032 | 1033 | pc_from_millisecs(Ms) -> 1034 | erlang:convert_time_unit(Ms, millisecond, perf_counter). 1035 | 1036 | pc_to_microsecs(Pc) -> 1037 | erlang:convert_time_unit(Pc, perf_counter, microsecond). 1038 | 1039 | microsecs_to_secs_str(Us) -> 1040 | SubSecs = Us rem 1000000, 1041 | [integer_to_list(Us div 1000000),$.| 1042 | if 1043 | SubSecs>99999 -> integer_to_list(SubSecs); 1044 | SubSecs>9999 -> [$0|integer_to_list(SubSecs)]; 1045 | SubSecs>999 -> [$0,$0|integer_to_list(SubSecs)]; 1046 | SubSecs>99 -> [$0,$0,$0|integer_to_list(SubSecs)]; 1047 | SubSecs>9 -> [$0,$0,$0,$0|integer_to_list(SubSecs)]; 1048 | SubSecs>0 -> [$0,$0,$0,$0,$0|integer_to_list(SubSecs)]; 1049 | SubSecs=:=0 -> "000000" 1050 | end]. 1051 | 1052 | is_terminate_crash(normal) -> false; 1053 | is_terminate_crash(shutdown) -> false; 1054 | is_terminate_crash({shutdown,_}) -> false; 1055 | is_terminate_crash(killed) -> false; 1056 | is_terminate_crash(_) -> true. 1057 | -------------------------------------------------------------------------------- /src/seltor@flame_prof.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% Copyright 2020 Medical-Objects Pty Ltd. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %% 16 | 17 | %%% ============================================================================ 18 | %%% @doc Process selector for `flame_prof' 19 | %%% 20 | %%% This process auto selects processes to be profiled by `flame_prof' on a 21 | %%% timer and sends them to `flame_prof'. 22 | %%% @end 23 | %%% ============================================================================ 24 | -module(seltor@flame_prof). 25 | 26 | %%% -- Public Control -- 27 | -export([start_link/2]). 28 | 29 | %%% -- Private Interface -- 30 | -behaviour(gen_server). 31 | -export([init/1, handle_continue/2, 32 | handle_call/3, handle_cast/2, handle_info/2, 33 | terminate/2, code_change/3, format_status/2]). 34 | 35 | -record(st_r, 36 | {sampler :: pid(), 37 | auto_select :: {top,ProcessCount :: pos_integer(), 38 | IntervalMillisecs :: pos_integer(), 39 | IntervalPc :: integer()}, 40 | snapshot=#{} :: #{pid() => {ReductionsDelta :: integer(), 41 | Reductions :: non_neg_integer()}}}). 42 | 43 | %%% -- Public Control ---------------------------------------------------------- 44 | 45 | start_link({top,_,_}=AutoSel, SysOpts) when is_list(SysOpts) -> 46 | gen_server:start_link({local,?MODULE}, ?MODULE, {AutoSel,self()}, SysOpts). 47 | 48 | %%% -- Public Interface -------------------------------------------------------- 49 | 50 | %%% -- 'gen_server' callbacks --- 51 | 52 | init({AutoSel,Sampler}) -> 53 | process_flag(trap_exit, true), 54 | S = #st_r{sampler=Sampler,auto_select=AutoSel}, 55 | {ok,S,{continue,init}}. 56 | 57 | handle_continue(init, #st_r{auto_select={top,PCount,Ival}}=S) -> 58 | {Ps,Snap} = auto_select_top(PCount, get_reductions_new()), 59 | []=/=Ps andalso (S#st_r.sampler ! {auto_selected,Ps}), 60 | {noreply,S#st_r{snapshot=Snap},Ival}. 61 | 62 | handle_call(never_match, _, S) -> 63 | {stop,never_match,S}. 64 | 65 | handle_cast(never_match, S) -> 66 | {stop,never_match,S}. 67 | 68 | handle_info(timeout, #st_r{auto_select={top,PCount,Ival}}=S) -> 69 | {Ps,Snap} = auto_select_top(PCount, get_reductions_delta(S#st_r.snapshot)), 70 | []=/=Ps andalso (S#st_r.sampler ! {auto_selected,Ps}), 71 | {noreply,S#st_r{snapshot=Snap},Ival}; 72 | handle_info({'EXIT',Sampler,Reason}, #st_r{sampler=Sampler}=S) -> 73 | {stop,Reason,S}; 74 | handle_info({'EXIT',SamplerParent,Reason}, #st_r{}=S) -> 75 | [_,SamplerParent] = get('$ancestors'), 76 | {stop,Reason,S}. 77 | 78 | terminate(_, _) -> 79 | ok. 80 | 81 | code_change(_, #st_r{auto_select={top,_,_},snapshot=#{}}=S, _) -> 82 | {ok,S}. 83 | 84 | format_status(_, [_,#st_r{snapshot=Snap}=S]) -> 85 | [{data, [{"State",S#st_r{snapshot={'$hidden',erts_debug:size(Snap)}}}]}]. 86 | 87 | %%% -- Helpers ----------------------------------------------------------------- 88 | 89 | %%% 90 | %% @doc Get the top N processes by reductions. 91 | %% 92 | auto_select_top(N, {RedPairs,Snap}) when is_integer(N), N>0 -> 93 | % See {@link observer_backend:etop_collect/1} [https://github.com/erlang/otp/blob/a67f636bcf14dd5eebea9b1bf1e7b89b38895b22/lib/runtime_tools/src/observer_backend.erl#L325] 94 | % See {@link observer_backend:etop_collect/2} [https://github.com/erlang/otp/blob/a67f636bcf14dd5eebea9b1bf1e7b89b38895b22/lib/runtime_tools/src/observer_backend.erl#L371] 95 | Decc = lists:sort( % <<< TODO: Use more efficient partial sort instead. 96 | fun({_,{_,_}=PairA}, {_,{_,_}=PairB}) -> PairA>PairB end, 97 | RedPairs), 98 | {auto_select_top1(N, Decc), Snap}. 99 | 100 | auto_select_top1(0, Decc) when is_list(Decc) -> 101 | []; 102 | auto_select_top1(N, [{P,{_,_}}|T]) -> 103 | [P|auto_select_top1(N-1, T)]; 104 | auto_select_top1(_, []) -> 105 | []. 106 | 107 | get_reductions_new() -> 108 | do_get_reductions(processes(), #{}, #{}, [], true). 109 | 110 | get_reductions_delta(PrevSnap) -> 111 | do_get_reductions(processes(), PrevSnap, #{}, [], true). 112 | 113 | do_get_reductions([H|T], PrevSnap, CurrSnap, Acc, WantDelta) -> 114 | case process_info(H, reductions) of 115 | {_,CurrReds} when WantDelta -> 116 | CurrPair = 117 | case PrevSnap of 118 | #{H:=PrevPair} -> 119 | {_,PrevReds} = PrevPair, 120 | {CurrReds-PrevReds,CurrReds}; 121 | #{} -> 122 | {0,CurrReds} 123 | end, 124 | do_get_reductions( 125 | T, maps:remove(H, PrevSnap), CurrSnap#{H=>CurrPair}, 126 | [{H,CurrPair}|Acc], true); 127 | {_,CurrReds} -> 128 | CurrPair = {0,CurrReds}, 129 | do_get_reductions( 130 | T, maps:remove(H, PrevSnap), CurrSnap#{H=>CurrPair}, 131 | [{H,CurrPair}|Acc], false); 132 | undefined -> 133 | do_get_reductions( 134 | T, maps:remove(H, PrevSnap), CurrSnap, Acc, WantDelta) 135 | end; 136 | do_get_reductions([], #{}, #{}=CurrSnap, Acc, _) -> 137 | {Acc,CurrSnap}. 138 | 139 | --------------------------------------------------------------------------------