├── .gitignore ├── LICENSE ├── README.md ├── rebar.config ├── rebar.lock ├── src ├── alexander.app.src └── alexander.erl └── test ├── alexander_SUITE.erl └── gen_server_test.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .rebar3 2 | _* 3 | .eunit 4 | *.o 5 | *.beam 6 | *.plt 7 | *.swp 8 | *.swo 9 | .erlang.cookie 10 | ebin 11 | log 12 | erl_crash.dump 13 | .rebar 14 | logs 15 | _build 16 | .idea 17 | *.iml 18 | rebar3.crashdump 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2018, Andrew Thompson . 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Alexander 2 | ===== 3 | 4 | An OTP library for cutting the Gordian knot of stuck OTP processes. 5 | 6 | One of the hardest problems to debug in Erlang/OTP is why a 7 | gen_server/gen_statem/etc is deadlocked. Often this is due to a blocking call 8 | cycle where A calls B who calls C who calls A. A is waiting for the return from 9 | B when the second call comes in, so it cannot respond to it, which means all the 10 | processes deadlock until the call timeout hits, or forever if the timeouts are 11 | infinite. 12 | 13 | When a process is stuck like this, you can't use sys:get_status or install a 14 | debug trace, those use messages, which cannot be handled. All you can do is look 15 | at erlang:process_info for the process, which often does not contain enough 16 | information to understand why it is deadlocked. 17 | 18 | Alexander, the Great, original cutter of the Gordian knot, is a parse transform 19 | that instruments blocking OTP calls by writing the blocking call's information 20 | to the process dictionary before the blocking call and then removing it 21 | afterwards. It can then use that information to detect call cycles because other 22 | processes can read the process dictionary via erlang:process_info. So if A calls 23 | B, A first records it is calling B (and is thus unavailable to handle a call 24 | until it returns) and then examines B's process dictionary to see if B is 25 | blocked making a call of its own. If B is blocked we follow the chain of 26 | processes until we run out of processes to follow, or we find a loop. If a loop 27 | is detected, an exception is thrown. 28 | 29 | How to use it 30 | =========== 31 | 32 | Simply enable the parse transform on the modules you want to check for call 33 | cycles: 34 | 35 | ```erlang 36 | -compile([{parse_transform, alexander}]). 37 | ``` 38 | 39 | Or use it in your Erlang compiler flags: 40 | 41 | ``` 42 | {parse_transform, alexander} 43 | ``` 44 | 45 | Alexander can only see call cycles between processes it has transformed at 46 | compile time, so you might want to use something like rebar3's overrides 47 | feature: 48 | 49 | ```erlang 50 | {overrides, [{add, [{erl_opts, [{parse_transform, alexander}]}]}]}. 51 | ``` 52 | 53 | For this to work you likely need a compiled copy of alexander in your OTP code 54 | path somewhere. If you are using other parse transforms, Alexander will try to 55 | run them for you before applying its own transform. This should work for most 56 | use cases but it is not well tested. 57 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info]}. 2 | {deps, []}. -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/alexander.app.src: -------------------------------------------------------------------------------- 1 | {application, alexander, 2 | [{description, "An OTP library"}, 3 | {vsn, "0.1.0"}, 4 | {registered, []}, 5 | {applications, 6 | [kernel, 7 | stdlib 8 | ]}, 9 | {env,[]}, 10 | {modules, []}, 11 | 12 | {maintainers, []}, 13 | {licenses, ["Apache 2.0"]}, 14 | {links, []} 15 | ]}. 16 | -------------------------------------------------------------------------------- /src/alexander.erl: -------------------------------------------------------------------------------- 1 | -module(alexander). 2 | 3 | %% API exports 4 | -export([parse_transform/2, check/0, enter/1, exit/0]). 5 | 6 | %%==================================================================== 7 | %% API functions 8 | %%==================================================================== 9 | 10 | parse_transform(AST, Options) -> 11 | put(targets, [{gen_server, call, 2}, 12 | {gen_server, call, 3}, 13 | {gen_fsm, sync_send_event, 2}, 14 | {gen_fsm, sync_send_event, 3}, 15 | {gen_fsm, sync_send_all_state_event, 2}, 16 | {gen_fsm, sync_send_all_state_event, 3}, 17 | {gen_statem, call, 2}, 18 | {gen_statem, call, 3}, 19 | {sys, get_state, 1}, 20 | {sys, get_state, 2}, 21 | {sys, get_status, 1}, 22 | {sys, get_status, 2}, 23 | {gen_event, call, 3}, 24 | {gen_event, call, 4}, 25 | {gen_event, sync_notify, 2}, 26 | {gen_event, which_handlers, 1} 27 | ]), 28 | %% run any other parse transforms first 29 | NewAST = lists:foldl(fun({parse_transform, alexander}, Acc) -> 30 | Acc; 31 | ({parse_transform, Transform}, Acc) -> 32 | Transform:parse_transform(Acc, Options -- [{parse_transform, alexander}]); 33 | (_, Acc) -> 34 | Acc 35 | end, AST, Options), 36 | walk_ast([], NewAST). 37 | 38 | check() -> 39 | true. 40 | 41 | enter({_TargetMod, _TargetFun, _Module, _Function, _Line, _Destination} = Signature) -> 42 | %io:format("~p entering blocking call to ~s:~s in function ~s:~s line ~b with destination ~p~n", [self(), TargetMod, TargetFun, Module, Function, Line, Destination]), 43 | %% search for someone in the chain that is calling us 44 | check_loop(self(), [Signature]), 45 | erlang:put('__alexander', Signature), 46 | ok. 47 | 48 | exit() -> 49 | %io:format("~p exiting blocking call~n", [self()]), 50 | erlang:erase('__alexander'), 51 | ok. 52 | 53 | %%==================================================================== 54 | %% Internal functions 55 | %%==================================================================== 56 | 57 | walk_ast(Acc, []) -> 58 | lists:reverse(Acc); 59 | walk_ast(Acc, [{attribute, _, module, Module}=H|T]) -> 60 | put(module, Module), 61 | walk_ast([H|Acc], T); 62 | walk_ast(Acc, [{function, Line, Name, Arity, Clauses}|T]) -> 63 | put(function, Name), 64 | walk_ast([{function, Line, Name, Arity, 65 | walk_clauses([], Clauses)}|Acc], T); 66 | walk_ast(Acc, [H|T]) -> 67 | walk_ast([H|Acc], T). 68 | 69 | walk_clauses(Acc, []) -> 70 | lists:reverse(Acc); 71 | walk_clauses(Acc, [{clause, Line, Arguments, Guards, Body}|T]) -> 72 | walk_clauses([{clause, Line, Arguments, Guards, walk_body([], Body)}|Acc], T). 73 | 74 | walk_body(Acc, []) -> 75 | lists:reverse(lists:flatten(Acc)); 76 | walk_body(Acc, [H|T]) -> 77 | walk_body([transform_statement(H, get(targets))|Acc], T). 78 | 79 | transform_statement({call, Line, {remote, _Line1, {atom, _Line2, Module}, 80 | {atom, _Line3, Function}}, Arguments0} = Stmt, 81 | Targets) -> 82 | Signature = {Module, Function, length(Arguments0)}, 83 | case lists:member(Signature, Targets) of 84 | true -> 85 | Destination = extract(Signature, Arguments0), 86 | %io:format("detected signature on line ~p ~p~n", [Line, Arguments0]), 87 | %% use a try/after to install the metadata, run the code, return the actual value we expect, and remove the metadata once the call has completed 88 | {'try',Line, 89 | [{call, Line, {remote, Line, {atom, Line, ?MODULE}, {atom, Line, enter}}, [{tuple, Line, [{atom, Line, Module}, {atom, Line, Function}, {atom, Line, get(module)}, {atom, Line, get(function)}, {integer, Line, Line}, Destination]}]}], 90 | [{clause,Line, 91 | [{var,Line,'_'}], 92 | [], 93 | [Stmt]}], 94 | [], 95 | [{call, Line, {remote, Line, {atom, Line, ?MODULE}, {atom, Line, exit}},[]}]}; 96 | false -> 97 | list_to_tuple(transform_statement(tuple_to_list(Stmt), Targets)) 98 | end; 99 | transform_statement(Stmt, Targets) when is_tuple(Stmt) -> 100 | list_to_tuple(transform_statement(tuple_to_list(Stmt), Targets)); 101 | transform_statement(Stmt, Targets) when is_list(Stmt) -> 102 | [transform_statement(S, Targets) || S <- Stmt]; 103 | transform_statement(Stmt, _Targets) -> 104 | Stmt. 105 | 106 | extract({gen_server, call, _}, [Destination|_Tail]) -> 107 | Destination; 108 | extract({gen_fsm, sync_send_event, _}, [Destination|_Tail]) -> 109 | Destination; 110 | extract({gen_fsm, sync_send_all_state_event, _}, [Destination|_]) -> 111 | Destination; 112 | extract({gen_statem, call, _}, [Destination|_]) -> 113 | Destination; 114 | extract({sys, get_state, _}, [Destination |_]) -> 115 | Destination; 116 | extract({sys, get_status, _}, [Destination |_]) -> 117 | Destination; 118 | extract({gen_event, call, _}, [Destination|_]) -> 119 | Destination; 120 | extract({gen_event, sync_notify, 2}, [Destination, _]) -> 121 | Destination; 122 | extract({gen_event, which_handlers, 1}, [Destination]) -> 123 | Destination. 124 | 125 | check_loop(Source, [{_TargetMod, _TargetFun, _Module, _Function, _Line, Destination}|_]=Stack) -> 126 | Pid = case Destination of 127 | D when is_pid(D) -> D; 128 | D when is_atom(D) -> whereis(D) 129 | end, 130 | case Pid == Source of 131 | true -> 132 | erlang:error({call_loop_detected, unwind_stack(Stack)}); 133 | false -> 134 | ok 135 | end, 136 | case erlang:process_info(Pid, [dictionary]) of 137 | [{dictionary, Dict}] -> 138 | case lists:keyfind('__alexander', 1, Dict) of 139 | false -> 140 | ok; 141 | {'__alexander', {_NewTargetMod, _NewTargetFun, _NewModule, _NewFunction, _NewLine, _NewDestination}=AlexanderTuple} -> 142 | check_loop(Source, [AlexanderTuple|Stack]) 143 | end; 144 | _ -> 145 | ok 146 | end. 147 | 148 | unwind_stack(Stack) -> 149 | unwind_stack(Stack, []). 150 | 151 | unwind_stack([], Acc) -> 152 | lists:flatten(Acc); 153 | unwind_stack([{TargetMod, TargetFun, Module, Function, Line, Destination}|Tail], Acc) -> 154 | E = io_lib:format("From ~s:~s(~p) in ~s:~s line ~b~n", [TargetMod, TargetFun, Destination, Module, Function, Line]), 155 | unwind_stack(Tail, [E|Acc]). 156 | -------------------------------------------------------------------------------- /test/alexander_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(alexander_SUITE). 2 | 3 | -export([all/0]). 4 | -export([non_loop/1, loop/1, recursive_call/1]). 5 | 6 | all() -> 7 | [ 8 | non_loop, 9 | loop, 10 | recursive_call 11 | ]. 12 | 13 | non_loop(_) -> 14 | {ok, Pid1} = gen_server_test:start_link(), 15 | {ok, Pid2} = gen_server_test:start_link(), 16 | {ok, Pid3} = gen_server_test:start_link(), 17 | {ok, Pid4} = gen_server_test:start_link(), 18 | no_loop = gen_server_test:blocking_call(Pid1, [{gen_server_test, Pid2}, 19 | {gen_server_test, Pid3}, 20 | {gen_server_test, Pid4}, 21 | no_loop]), 22 | [gen_server:stop(P) || P <- [Pid1, Pid2, Pid3, Pid4]]. 23 | 24 | loop(_) -> 25 | process_flag(trap_exit, true), 26 | {ok, Pid1} = gen_server_test:start_link(), 27 | {ok, Pid2} = gen_server_test:start_link(), 28 | {ok, Pid3} = gen_server_test:start_link(), 29 | {ok, Pid4} = gen_server_test:start_link(), 30 | try gen_server_test:blocking_call(Pid1, [{gen_server_test, Pid2}, 31 | {gen_server_test, Pid3}, 32 | {gen_server_test, Pid4}, 33 | {gen_server_test, Pid1}, 34 | loop]) 35 | of 36 | _ -> 37 | ct:fail(did_not_crash) 38 | catch 39 | What:Why -> 40 | ct:pal("~p ~p", [What, Why]) 41 | end, 42 | [gen_server:stop(P) || P <- [Pid1, Pid2, Pid3, Pid4], is_process_alive(P)]. 43 | 44 | recursive_call(_) -> 45 | process_flag(trap_exit, true), 46 | {ok, Pid} = gen_server_test:start_link(), 47 | try gen_server_test:recursive_call(Pid) of 48 | _ -> 49 | ct:fail(did_not_crash) 50 | catch 51 | What:Why -> 52 | ct:pal("~p ~p", [What, Why]) 53 | end. 54 | -------------------------------------------------------------------------------- /test/gen_server_test.erl: -------------------------------------------------------------------------------- 1 | -module(gen_server_test). 2 | -compile([{parse_transform, alexander}]). 3 | 4 | -behaviour(gen_server). 5 | 6 | -export([start_link/0, blocking_call/2, blocking_call/1, blocking_infinite_call/2, blocking_timeout_call/2, recursive_call/1]). 7 | 8 | %% gen_server callbacks 9 | -export([init/1, 10 | handle_call/3, 11 | handle_cast/2, 12 | handle_info/2, 13 | terminate/2, 14 | code_change/3]). 15 | 16 | 17 | -record(state, {}). 18 | 19 | start_link() -> 20 | gen_server:start_link(?MODULE, [], []). 21 | 22 | blocking_call(Pid, Cmd) -> 23 | gen_server:call(Pid, Cmd). 24 | 25 | blocking_call(Cmd) -> 26 | gen_server:call(?MODULE, Cmd). 27 | 28 | blocking_infinite_call(Pid, Cmd) -> 29 | gen_server:call(Pid, Cmd, infinity). 30 | 31 | blocking_timeout_call(Pid, Cmd) -> 32 | gen_server:call(Pid, Cmd, 500). 33 | 34 | recursive_call(Pid) -> 35 | gen_server:call(Pid, recurse). 36 | 37 | init([]) -> 38 | {ok, #state{}}. 39 | 40 | handle_call(recurse, _From, State) -> 41 | {reply, recursive_call(self()), State}; 42 | handle_call([Result], _From, State) -> 43 | {reply, Result, State}; 44 | handle_call([{Module, Pid}|Tail], _From, State) -> 45 | {reply, Module:blocking_call(Pid, Tail), State}; 46 | handle_call(_Request, _From, State) -> 47 | {reply, ignored, State}. 48 | 49 | handle_cast(_Msg, State) -> 50 | {noreply, State}. 51 | 52 | handle_info(_Info, State) -> 53 | {noreply, State}. 54 | 55 | terminate(_Reason, _State) -> 56 | ok. 57 | 58 | code_change(_OldVsn, State, _Extra) -> 59 | {ok, State}. 60 | --------------------------------------------------------------------------------