├── rebar ├── src ├── detest.hrl ├── detest.app.src ├── detest_pmd.erl ├── detest_shaper.erl ├── detest_net.erl └── detest.erl ├── Makefile ├── rebar.config ├── test ├── app.config └── test.erl ├── escriptize.escript └── README.md /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/biokoda/detest/HEAD/rebar -------------------------------------------------------------------------------- /src/detest.hrl: -------------------------------------------------------------------------------- 1 | -define(PATH,".detest"). 2 | -define(INF(F,Param), 3 | case butil:ds_val(quiet,etscfg) of 4 | true -> 5 | ok; 6 | _ -> 7 | io:format("~p ~p: ~s~n",[time(),?MODULE,io_lib:fwrite(F,Param)]) 8 | end). 9 | -define(INF(F),?INF(F,[])). 10 | 11 | -define(CFG(K),butil:ds_val(K,etscfg)). -------------------------------------------------------------------------------- /src/detest.app.src: -------------------------------------------------------------------------------- 1 | {application, detest, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | compiler 10 | ]}, 11 | {mod, { detest, []}}, 12 | {env, []} 13 | ]}. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | ./rebar get-deps 3 | ./rebar compile 4 | ./escriptize.escript 5 | 6 | clean: 7 | ./rebar clean 8 | 9 | 10 | 11 | # cp deps/procket/priv/procket . 12 | # cp deps/procket/priv/procket.so . 13 | # ./escriptize.escript procket procket.so 14 | # erl -pa ebin -noinput -eval "detest:ez(),init:stop()" 15 | # rm procket procket.so 16 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [{parse_transform, lager_transform}]}. 2 | 3 | {lib_dirs, ["deps"]}. 4 | 5 | {erl_opts, [{src_dirs,["src"]}]}. 6 | 7 | %{escript_incl_apps, [erlydtl,lager,merl,goldrush,eunit_formatters,damocles,bkdcore]}. 8 | %{escript_name,"detest"}. 9 | 10 | {deps, [ 11 | {erlydtl, ".*", {git, "https://github.com/brigadier/erlydtl", {branch, "master"}}}, 12 | {bkdcore,".*",{git,"http://github.com/biokoda/bkdcore.git",{branch,"master"}}} 13 | %{damocles, ".*", {git, "https://github.com/lostcolony/damocles.git", {branch, "master"}}} 14 | ]}. 15 | 16 | 17 | %{tunctl, ".*", {git, "https://github.com/biokoda/tunctl.git", {branch, "master"}}}, 18 | %{pkt, ".*", {git, "https://github.com/msantos/pkt.git", {branch, "master"}}} 19 | -------------------------------------------------------------------------------- /test/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | %% {{basepath}} is provided by detest. This is the base folder where you should write your data. 4 | %% {{name}} is taken from cfg/1 nodes parameter. 5 | {myapp,[ 6 | {i_want_my_files_here,"{{ basepath }}/{{ name }}"} 7 | ]}, 8 | 9 | 10 | 11 | {lager, [ 12 | {colored, true}, 13 | {handlers, [ 14 | {lager_console_backend,[debug,{lager_default_formatter, [time," ",pid," ",node," ",module," ",line, 15 | " [",severity,"] ", message, "\n"]}]}, 16 | {lager_file_backend, [{file, "{{ basepath }}/log/{{ name }}.error.log"}, {level, error}, {formatter, lager_default_formatter}, 17 | {formatter_config, [date, " ", time," [",severity,"] ",pid," ",module, " ",line, " ", message, "\n"]}]}, 18 | {lager_file_backend, [{file, "{{ basepath }}/log/{{ name }}.console.log"}, {level, info}]} 19 | ]} 20 | ]}, 21 | 22 | %% SASL config 23 | {sasl, [ 24 | {sasl_error_logger, {file, "{{ basepath }}/log/{{ name }}.sasl-error.log"}}, 25 | {errlog_type, error}, 26 | {error_logger_mf_dir, "{{ basepath }}/log/{{ name }}"}, % Log directory 27 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 28 | {error_logger_mf_maxfiles, 5} % 5 files max 29 | ]} 30 | ]. 31 | 32 | -------------------------------------------------------------------------------- /src/detest_pmd.erl: -------------------------------------------------------------------------------- 1 | -module(detest_pmd). 2 | 3 | -export([start_link/0, 4 | register_node/2, 5 | register_node/3, 6 | port_please/2, 7 | names/1]). 8 | 9 | %% The supervisor module erl_distribution tries to add us as a child 10 | %% process. We don't need a child process, so return 'ignore'. 11 | start_link() -> 12 | ignore. 13 | 14 | register_node(_Name, _Port) -> 15 | %% This is where we would connect to epmd and tell it which port 16 | %% we're listening on, but since we're epmd-less, we don't do that. 17 | 18 | %% Need to return a "creation" number between 1 and 3. 19 | Creation = rand:uniform(3), 20 | {ok, Creation}. 21 | 22 | %% As of Erlang/OTP 19.1, register_node/3 is used instead of 23 | %% register_node/2, passing along the address family, 'inet_tcp' or 24 | %% 'inet6_tcp'. This makes no difference for our purposes. 25 | register_node(Name, Port, _Family) -> 26 | register_node(Name, Port). 27 | 28 | port_please(Name, _IP) -> 29 | % Port = epmdless:dist_port(Name), 30 | % %% The distribution protocol version number has been 5 ever since 31 | % %% Erlang/OTP R6. 32 | Version = 5, 33 | {port, node_to_port(Name), Version}. 34 | 35 | names(_Hostname) -> 36 | %% Since we don't have epmd, we don't really know what other nodes 37 | %% there are. 38 | {error, address}. 39 | 40 | node_to_port(Node) -> 41 | Nodes = butil:ds_val(nodes,etscfg), 42 | butil:ds_val(butil:toatom(Node),nodes_split(Nodes)). 43 | 44 | nodes_split(Nodes) -> 45 | [begin 46 | {butil:toatom(butil:ds_val(name,Nd)),butil:ds_val(dist_port,Nd)} 47 | end || Nd <- Nodes]. 48 | 49 | -------------------------------------------------------------------------------- /escriptize.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | 3 | 4 | main(BinFiles1) -> 5 | BinFiles = ["deps/bkdcore/ebin/butil.beam","deps/bkdcore/ebin/bjson.beam","deps/bkdcore/ebin/bmochinum.beam"] ++ BinFiles1, 6 | Apps = [erlydtl,lager,merl,goldrush,eunit_formatters,damocles], 7 | 8 | %% Add ebin paths to our path 9 | true = code:add_path("ebin"), 10 | ok = code:add_paths(filelib:wildcard("deps/*/ebin")), 11 | 12 | %% Read the contents of the files in ebin(s) 13 | Files1 = [begin 14 | FileList = filelib:wildcard("deps/"++atom_to_list(Dir)++"/ebin/*.*") ++ filelib:wildcard("ebin/*.*"), 15 | [{filename:basename(Nm),element(2,file:read_file(Nm))} || Nm <- FileList] 16 | end || Dir <- Apps], 17 | 18 | Files = [{filename:basename(Fn),element(2,file:read_file(Fn))} || Fn <- BinFiles]++lists:flatten(Files1), 19 | 20 | case zip:create("mem", Files, [memory]) of 21 | {ok, {"mem", ZipBin}} -> 22 | %% Archive was successfully created. Prefix that with header and write to "edis" file 23 | Script = <<"#!/usr/bin/env escript\n%%! +Bc +K true -start_epmd false -epmd_module detest_pmd -smp enable\n", ZipBin/binary>>, 24 | case file:write_file("detest", Script) of 25 | ok -> ok; 26 | {error, WriteError} -> 27 | io:format("Failed to write detest: ~p\n", [WriteError]), 28 | halt(1) 29 | end; 30 | {error, ZipError} -> 31 | io:format("Failed to construct detest archive: ~p\n", [ZipError]), 32 | halt(1) 33 | end, 34 | 35 | %% Finally, update executable perms for our script 36 | case os:type() of 37 | {unix,_} -> 38 | [] = os:cmd("chmod u+x detest"), 39 | ok; 40 | _ -> 41 | ok 42 | end. 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Detest is a tool for running distributed erlang tests and it also works very well as a generic library/tool for running erlang VMs. 2 | 3 | It is designed to be simple and flexible. It sets up a distributed environment of multiple nodes, then it calls your code to do whatever you wish with that environment. 4 | 5 | Detest is an escript and should be executed from the root folder of your project (like rebar). 6 | 7 | For an example look at: test/test.erl and test/app.config 8 | 9 | You can run all or some nodes with valgrind/gdb. 10 | 11 | As a parameter it takes in path to your test script. This script needs to export at least: cfg/1, run/1, setup/1, cleanup/1. 12 | 13 | Detest is configured with your cfg/1 function within the script. It instructs detest how many nodes to run, where their configuration files are, how to execute your app, etc.Detest will then execute the nodes, connect to each (as a hidden node) wait for your app to start, then call run/1 function in your script. You are then free to do whatever you wish. RPC is possible to any of the nodes. You can also terminate the test at any time by entering q to terminal. 14 | 15 | Run it like so: 16 | 17 | ./detest test/test.erl 18 | 19 | You can print all console output from your nodes: 20 | 21 | ./detest -v test/test.erl 22 | 23 | You can also send arguments to your script: 24 | 25 | ./detest -v test/test.erl dotest1 26 | 27 | Detest also supports executing nodes over ssh. So if you want to test network conditions, use LXC and configure detest to SSH to containers and execute nodes in them. 28 | 29 | You can also simulate network partitions the easy way, using detest:isolate/2 and detest:isolate_end/1. This will simulate network splits by changing erlang cookies. 30 | 31 | detest will create a folder named .detest at location it is running from. This is where node logs and state should go. 32 | 33 | Detest supports: 34 | 35 | * embedded mode within an app 36 | * executing nodes over SSH 37 | * network split simulation without using containers/virtualization 38 | * executing some or all nodes with GDB/valgrind 39 | * manually executing some nodes (if you want to debug one of them) 40 | * django template files as configs -------------------------------------------------------------------------------- /test/test.erl: -------------------------------------------------------------------------------- 1 | -module(test). 2 | % mandatory detest functions 3 | -export([cfg/1,run/1,setup/1,cleanup/1]). 4 | % test functions 5 | -export([call_start/1,call_receive/1]). 6 | % assert macros 7 | -include_lib("eunit/include/eunit.hrl"). 8 | 9 | % Nodes will be started with dist name: name@127.0.0.1 10 | -define(ND1,[{name,node1}]). % node1@127.0.0.1 11 | -define(ND2,[{name,node2}]). % node2@127.0.0.1 12 | -define(ND3,[{name,node3}]). % node3@127.0.0.1 13 | 14 | %% Example for running nodes over ssh. Add option: {ssh,NodeHostName,SshPort,Cwd,SshOpts} 15 | %-define(ND1,[{name,node1},{ssh,"node1",22,"/opt/detest",[{user,"root"}]}]). 16 | %-define(ND2,[{name,node2},{ssh,"node2",22,"/opt/detest",[{user,"root"}]}]). 17 | %-define(ND3,[{name,node3},{ssh,"node3",22,"/opt/detest",[{user,"root"}]}]). 18 | 19 | 20 | cfg(_TestArgs) -> 21 | % KV = [{Param1,Val1}], 22 | [ 23 | %% global_cfg is a list of config files for your nodes. They are template files (erlydtl). 24 | %% When rendering, they get nodes value as a parameter + whatever you add here. 25 | % {global_cfg,[{"test/nodes.yaml",[{fixedvals,KV}]},"test/withoutparams.yaml"]}, 26 | 27 | %% Config files per node. For every node, its property list is added when rendering. 28 | %% if name contains app.config or vm.args it gets automatically added to run node command 29 | %% do not set cookie or name of node in vm.args this is set by detest 30 | {per_node_cfg,["test/app.config"]}, 31 | 32 | %% Detest node address is by default detest@127.0.0.1 33 | %% If running nodes over ssh you need to set a valid non loopback address 34 | %{detest_name,'detest@home'}, 35 | 36 | %% cmd is appended to erl execute command, it should execute your app. 37 | %% It can be set for every node individually. Add it to that list if you need it, it will override this value. 38 | %% You should just start your app with this command and set start flags. Leave other code for run call. 39 | %% We add +S 2 and +A 2 to keep test node small. 40 | {cmd,"-s lager +S 2 +A 2"}, 41 | 42 | %% which app to wait for to consider node started 43 | {wait_for_app,lager}, 44 | 45 | %% List of nodes that will be started initially. Additional nodes and this cfg can be change during test. 46 | %% It must be a property list. It must have {name,nameofnode} parameter. Name of node must not be a full 47 | %% distributed name. Detest will add @someip to the end. 48 | {nodes,[?ND1,?ND2]} 49 | 50 | %% What RPC to execute for stopping nodes. 51 | % {stop,{init,stop,[]}}, 52 | 53 | %% optional command to start erlang with (def. is "erl") 54 | % {erlcmd,"../otp/bin/cerl -valgrind"}, 55 | 56 | %% optional environment variables for erlang (def. is []) 57 | % {erlenv,[{"VALGRIND_MISC_FLAGS","-v --leak-check=full --tool=memcheck --track-origins=yes "++ 58 | % "--suppressions=../otp/erts/emulator/valgrind/suppress.standard --show-possibly-lost=no"}]}, 59 | 60 | %% in ms, how long to wait to connect to node. If running with valgrind it can take a while. (def. is 10000) 61 | % {connect_timeout,60000}, 62 | 63 | %% in ms, how long to wait for application start once node is started (def. is 30000) 64 | % {app_wait_timeout,60000*5} 65 | ]. 66 | 67 | 68 | setup(_Param) -> 69 | ok. 70 | 71 | cleanup(_Param) -> 72 | ok. 73 | 74 | 75 | run(Param) -> 76 | lager:info("Script params: ~p",[Param]), 77 | % Get full dist name for RPC. 78 | Node1 = proplists:get_value(node1,Param), 79 | Node2 = proplists:get_value(node2,Param), 80 | {ok,Node2} = rpc:call(Node1,?MODULE,call_start,[Node2],200), 81 | {ok,Node1} = rpc:call(Node2,?MODULE,call_start,[Node1],200), 82 | 83 | Node3 = detest:add_node(?ND3), 84 | {ok,Node1} = rpc:call(Node3,?MODULE,call_start,[Node1],200), 85 | 86 | % Place node2 in nd2_isolation group 87 | detest:isolate(Node2,nd2_isolation), 88 | % Calling from node2 to node1 or node3 fails 89 | {badrpc,_} = rpc:call(Node2,?MODULE,call_start,[Node1],200), 90 | {badrpc,_} = rpc:call(Node2,?MODULE,call_start,[Node3],200), 91 | % node3 and node1 are still connected 92 | {ok,Node1} = rpc:call(Node3,?MODULE,call_start,[Node1],200), 93 | % We are still connected to node2 94 | pong = net_adm:ping(Node2), 95 | 96 | % add node3 to node2 group. Now node1 is alone. 97 | detest:isolate(Node3,nd2_isolation), 98 | {badrpc,_} = rpc:call(Node1,?MODULE,call_start,[Node2],200), 99 | {badrpc,_} = rpc:call(Node1,?MODULE,call_start,[Node3],200), 100 | % node2 and node3 are still connected 101 | {ok,Node2} = rpc:call(Node3,?MODULE,call_start,[Node2],200), 102 | % We are still connected to node1 103 | pong = net_adm:ping(Node1), 104 | 105 | % Back to normal 106 | detest:isolate_end([Node3,Node2]), 107 | {ok,Node2} = rpc:call(Node1,?MODULE,call_start,[Node2],200), 108 | {ok,Node3} = rpc:call(Node1,?MODULE,call_start,[Node3],200), 109 | {ok,Node2} = rpc:call(Node3,?MODULE,call_start,[Node2],200), 110 | 111 | ok. 112 | 113 | 114 | % This module is loaded inside every executed node. So we can rpc to these functions on every node. 115 | call_start(Nd) -> 116 | lager:info("Calling from=~p to=~p, at=~p~n",[node(), Nd, time()]), 117 | rpc:call(Nd,?MODULE,call_receive,[node()],200). 118 | 119 | call_receive(From) -> 120 | lager:info("Received call on=~p from=~p, at=~p~n",[node(), From, time()]), 121 | {ok,node()}. 122 | -------------------------------------------------------------------------------- /src/detest_shaper.erl: -------------------------------------------------------------------------------- 1 | -module(detest_shaper). 2 | -include("detest.hrl"). 3 | -export([call/1,cast/1, run/0]). 4 | 5 | run() -> 6 | spawn(fun() -> shaper() end). 7 | cast(Msg) -> 8 | case whereis(?MODULE) of 9 | undefined = Pid -> 10 | exit(normal); 11 | Pid -> 12 | ok 13 | end, 14 | Pid ! {call,undefined, make_ref(), Msg}, 15 | ok. 16 | call(Msg) -> 17 | case whereis(?MODULE) of 18 | undefined = Pid -> 19 | exit(normal); 20 | Pid -> 21 | ok 22 | end, 23 | Ref = make_ref(), 24 | Pid ! {call,self(), Ref, Msg}, 25 | receive 26 | {Ref,Resp} -> 27 | Resp 28 | end. 29 | 30 | wait_runner() -> 31 | case whereis(detest_net) of 32 | undefined -> 33 | timer:sleep(20), 34 | wait_runner(); 35 | Pid -> 36 | Pid 37 | end. 38 | 39 | -record(sp,{runner, callers = [], nodes = {}, 40 | node_mods = #{}, 41 | isolated_nodes = [], 42 | isolation_groups = [], 43 | transition = false, paused_callers = [], calls = [], 44 | shaping = false, reshape_timer, shape_mod_fixed = #{}}). 45 | shaper() -> 46 | register(?MODULE,self()), 47 | Run = wait_runner(), 48 | link(Run), 49 | erlang:monitor(process,Run), 50 | Me = self(), 51 | spawn(fun() -> 52 | erlang:monitor(process,Me), 53 | receive 54 | {'DOWN',_Ref,_,_Pid,normal} -> 55 | ok; 56 | {'DOWN',_Ref,_,_Pid,Reason} -> 57 | ?INF("shaper died ~p",[Reason]), 58 | ok 59 | end 60 | end), 61 | shaper(#sp{runner = Run}). 62 | shaper(P) -> 63 | receive 64 | {'DOWN',_Ref,_,Pid,_} when Pid == P#sp.runner -> 65 | ok; 66 | {'DOWN',_Ref,_,Pid,_} -> 67 | shaper(shaper_call(P,{Pid,undefined},unreg_caller)); 68 | reshape -> 69 | ?INF("Reshape timer"), 70 | case P#sp.shaping of 71 | true -> 72 | shaper(transition(timer(P#sp{transition = true}))); 73 | false -> 74 | shaper(unwrap(reshape(P#sp{calls = [], reshape_timer = undefined}),P#sp.calls)) 75 | end; 76 | {call, From, Ref, Msg} -> 77 | shaper(shaper_call(P,{From,Ref},Msg)) 78 | end. 79 | 80 | timer(P) -> 81 | P#sp{reshape_timer = erlang:send_after(1000 + rand:uniform(20000), self(), reshape)}. 82 | 83 | shaper_call(P, {Pid,Ref}, shape_traffic_get) -> 84 | butil:safesend(Pid,{Ref,P#sp.node_mods}), 85 | P; 86 | shaper_call(P, {Pid,Ref}, {shape_traffic_set,Mods}) -> 87 | shaper_call(P#sp{shape_mod_fixed = Mods},{Pid,Ref}, shape_traffic_start); 88 | shaper_call(P, {Pid,Ref}, {isolation_group_remove,Id}) -> 89 | butil:safesend(Pid,{Ref,ok}), 90 | P#sp{isolation_groups = lists:delete(Id,P#sp.isolation_groups)}; 91 | shaper_call(P, {Pid,Ref}, {isolation_group_set,_Nodes,Id}) -> 92 | Grps = butil:lists_add(Id, P#sp.isolation_groups), 93 | case P#sp.isolation_groups of 94 | [] -> 95 | shaper_call(P#sp{isolation_groups = Grps},{Pid,Ref}, shape_traffic_stop); 96 | _ -> 97 | butil:safesend(Pid,{Ref,ok}), 98 | P#sp{isolation_groups = Grps} 99 | end; 100 | shaper_call(P,{Pid,Ref}, shape_traffic_stop) -> 101 | case P#sp.shaping of 102 | false when P#sp.reshape_timer == undefined -> 103 | butil:safesend(Pid,{Ref,ok}), 104 | P; 105 | false -> 106 | erlang:cancel_timer(P#sp.reshape_timer), 107 | self() ! reshape, 108 | P#sp{reshape_timer = undefined, calls = [{{Pid,Ref}, shape_traffic_stop}|P#sp.calls]}; 109 | true -> 110 | erlang:cancel_timer(P#sp.reshape_timer), 111 | self() ! reshape, 112 | P#sp{reshape_timer = undefined, shaping = false, calls = [{{Pid,Ref}, shape_traffic_stop}|P#sp.calls]} 113 | end; 114 | shaper_call(P,{Pid,Ref}, shape_traffic_rand) -> 115 | shaper_call(P#sp{shape_mod_fixed = #{}},{Pid,Ref}, shape_traffic_start); 116 | shaper_call(P,{Pid,Ref}, shape_traffic_start) -> 117 | butil:safesend(Pid,{Ref,ok}), 118 | case P#sp.shaping of 119 | true -> 120 | P; 121 | false -> 122 | ?INF("shape_traffic_start ~p ~p",[P#sp.callers, P#sp.paused_callers]), 123 | transition(timer(P#sp{shaping = true, transition = true})) 124 | end; 125 | shaper_call(P,{Pid,Ref}, {add_node,Nd}) -> 126 | butil:safesend(Pid,{Ref,ok}), 127 | P#sp{nodes = list_to_tuple(butil:lists_add(Nd,tuple_to_list(P#sp.nodes)))}; 128 | shaper_call(#sp{transition = true} = P,{Pid,Ref}, majority_node) -> 129 | case lists:keytake(Pid, 1, P#sp.callers) of 130 | {value,Caller,NewCallers} -> 131 | transition(P#sp{callers = NewCallers, paused_callers = [Caller|P#sp.paused_callers], calls = [{{Pid,Ref},majority_node}|P#sp.calls]}); 132 | _ -> 133 | butil:safesend(Pid,{Ref,undefined}), 134 | P 135 | end; 136 | shaper_call(P,{Pid,Ref}, majority_node) -> 137 | case lists:keyfind(Pid, 1, P#sp.callers) of 138 | false -> 139 | butil:safesend(Pid,{Ref,undefined}), 140 | P; 141 | _ -> 142 | butil:safesend(Pid,{Ref, randnd(P)}), 143 | P 144 | end; 145 | shaper_call(#sp{transition = true} = P,{Pid,Ref}, reg_caller) -> 146 | case lists:keymember(Pid,1,P#sp.paused_callers) of 147 | false -> 148 | butil:safesend(Pid,{Ref,ok}), 149 | MonRef = erlang:monitor(process,Pid), 150 | P#sp{paused_callers = [{Pid,MonRef}|P#sp.paused_callers]}; 151 | true -> 152 | P 153 | end; 154 | shaper_call(#sp{transition = false} = P,{Pid,Ref}, reg_caller) -> 155 | butil:safesend(Pid,{Ref,ok}), 156 | case lists:keymember(Pid,1,P#sp.callers) of 157 | false -> 158 | MonRef = erlang:monitor(process,Pid), 159 | P#sp{callers = [{Pid,MonRef}|P#sp.callers]}; 160 | true -> 161 | P 162 | end; 163 | shaper_call(P,{Pid,Ref}, unreg_caller) -> 164 | butil:safesend(Pid,{Ref,ok}), 165 | case lists:keyfind(Pid,1,P#sp.callers) of 166 | false -> 167 | case lists:keyfind(Pid,1,P#sp.paused_callers) of 168 | false -> 169 | P; 170 | {_,MonRef} -> 171 | erlang:demonitor(MonRef,[flush]), 172 | P#sp{paused_callers = lists:keydelete(Pid,1,P#sp.paused_callers)} 173 | end; 174 | {_,MonRef} -> 175 | ?INF("Unreg caller ~p",[Pid]), 176 | erlang:demonitor(MonRef,[flush]), 177 | transition(P#sp{callers = lists:keydelete(Pid,1,P#sp.callers)}) 178 | end. 179 | 180 | randnd(P) -> 181 | element(rand:uniform(tuple_size(P#sp.nodes)), P#sp.nodes). 182 | 183 | transition(#sp{transition = true, callers = [], calls = Calls} = P) -> 184 | unwrap(reshape(P#sp{transition = false, callers = P#sp.paused_callers, calls = []}), Calls); 185 | transition(P) -> 186 | P. 187 | 188 | reshape(#sp{shaping = true} = P) -> 189 | % Remove and set new modifications 190 | node_cleanup(maps:to_list(P#sp.node_mods)), 191 | case maps:size(P#sp.shape_mod_fixed) of 192 | 0 -> 193 | Nodes = tuple_to_list(P#sp.nodes), 194 | NewMods = set_offline(Nodes,set_latencies(Nodes,#{})); 195 | NewMods -> 196 | ok 197 | end, 198 | node_set(maps:to_list(NewMods)), 199 | {OnlineNodes,IsolatedNodes} = split_offline(tuple_to_list(P#sp.nodes), [], [], NewMods), 200 | ?INF("Network shape: ~p, non-majority:~p",[NewMods,IsolatedNodes]), 201 | P#sp{node_mods = NewMods, nodes = list_to_tuple(OnlineNodes), isolated_nodes = IsolatedNodes}; 202 | reshape(P) -> 203 | node_cleanup(maps:to_list(P#sp.node_mods)), 204 | P#sp{node_mods = #{}}. 205 | 206 | split_offline([Node|T],Online,Offline,Mods) -> 207 | case maps:get(Node,Mods,undefined) of 208 | undefined -> 209 | split_offline(T,[Node|Online],Offline,Mods); 210 | Obj -> 211 | case is_offline(Obj) of 212 | true -> 213 | split_offline(T,Online,[Node|Offline],Mods); 214 | false -> 215 | split_offline(T,[Node|Online],Offline,Mods) 216 | end 217 | end; 218 | split_offline([],Online,Offline,_) -> 219 | {Online,Offline}. 220 | 221 | set_latencies([Node|T],Obj) -> 222 | ObjMods = maps:get(Node,Obj,[]), 223 | Lat = rand:uniform(1000), 224 | % detest_net:call({node_latency, Node, Lat}), 225 | set_latencies(T,Obj#{Node => [{latency, Lat}|ObjMods]}); 226 | set_latencies([],Obj) -> 227 | Obj. 228 | 229 | set_offline(Nodes,Obj) -> 230 | set_offline(length(Nodes) div 2, Nodes, [], Obj). 231 | set_offline(Max,_, Offlines, Obj) when Max == length(Offlines) -> 232 | Obj; 233 | set_offline(Max,[Node|T], Offlines, Obj) -> 234 | case rand:uniform(2) of 235 | 1 -> 236 | set_offline(Max,T, Offlines, Obj); 237 | 2 -> 238 | ObjMods = maps:get(Node,Obj,[]), 239 | OfflineType = offline_type(Node), 240 | case OfflineType of 241 | {latency,X} -> 242 | set_offline(Max, T, [Node|Offlines], Obj#{Node => [{latency,X}|lists:keydelete(latency,1,ObjMods)]}); 243 | _ -> 244 | set_offline(Max, T, [Node|Offlines], Obj#{Node => [OfflineType|ObjMods]}) 245 | end 246 | end; 247 | set_offline(_Max,[],_,Obj) -> 248 | Obj. 249 | 250 | is_offline(Mods) -> 251 | lists:member(offline,Mods) orelse lists:member(stop,Mods) orelse lists:member({latency,20000},Mods). 252 | 253 | offline_type(_Node) -> 254 | case rand:uniform(3) of 255 | 1 -> 256 | % detest_node:call({node_online, Node, false}), 257 | offline; 258 | 2 -> 259 | % detest_net:call({node_latency, Node, 20000}), 260 | {latency,20000}; 261 | 3 -> 262 | % detest:stop_node(Node), 263 | stop 264 | end. 265 | 266 | node_set([{Node,Mods}|T]) -> 267 | [detest_net:call({node_latency, Node, Lat}) || {latency,Lat} <- Mods], 268 | [detest_net:call({node_online, Node, false}) || offline <- Mods], 269 | [detest:stop_node(Node) || stop <- Mods], 270 | node_set(T); 271 | node_set([]) -> 272 | ok. 273 | 274 | node_cleanup([{Node,Mods}|T]) -> 275 | [detest_net:call({node_latency, Node, 0}) || {latency,_} <- Mods], 276 | [detest_net:call({node_online, Node, true}) || offline <- Mods], 277 | [begin 278 | NodeInfo = butil:findobj(distname,Node,?CFG(nodes)), 279 | detest:add_node(NodeInfo) 280 | end || stop <- Mods], 281 | node_cleanup(T); 282 | node_cleanup([]) -> 283 | ok. 284 | 285 | unwrap(P,[{{Pid,Ref},Msg}|T]) -> 286 | unwrap(shaper_call(P,{Pid,Ref}, Msg), T); 287 | unwrap(P,[]) -> 288 | P. 289 | -------------------------------------------------------------------------------- /src/detest_net.erl: -------------------------------------------------------------------------------- 1 | -module(detest_net). 2 | -include("detest.hrl"). 3 | -export([start/0, add_node/1,node_bw/2, node_latency/2, 4 | node_online/2, isolation_group_remove/1, isolation_group_set/2]). 5 | -export([reg_caller/0, unreg_caller/0, majority_node/0, 6 | shape_traffic_rand/0,shape_traffic_stop/0, shape_traffic_set/1, shape_traffic_get/0]). 7 | -export([call/1]). 8 | -define(INTERVAL, 3). 9 | -define(SAMPLES_PER_SEC, (1000 / ?INTERVAL)). 10 | 11 | start() -> 12 | detest_shaper:run(), 13 | spawn(fun() -> run() end). 14 | 15 | node_bw(Node,Bw) -> 16 | call({node_bw, butil:toatom(Node), Bw}). 17 | node_latency(Node,Latency) -> 18 | call({node_latency, butil:toatom(Node), Latency}). 19 | 20 | % This disables any automatic shaping 21 | node_online(Node,Bool) -> 22 | case Bool of 23 | true -> 24 | detest_shaper:call({isolation_group_remove,Node}); 25 | false -> 26 | detest_shaper:call({isolation_group_set,Node}) 27 | end, 28 | call({node_online, butil:toatom(Node), Bool}). 29 | % This disables any automatic shaping 30 | isolation_group_set(Nodes1,Id) -> 31 | Nodes = [butil:toatom(Node) || Node <- Nodes1], 32 | detest_shaper:call({isolation_group_set,Nodes,Id}), 33 | call({group_set, Nodes, Id}). 34 | % This disables any automatic shaping 35 | isolation_group_remove(Id) -> 36 | detest_shaper:call({isolation_group_remove,Id}), 37 | call({group_remove, Id}). 38 | % Should not be called client side 39 | add_node(Node) -> 40 | detest_shaper:cast({add_node,Node}), 41 | call({add_node, butil:toatom(Node)}). 42 | 43 | % Registers self for executing requests 44 | reg_caller() -> 45 | detest_shaper:call(reg_caller). 46 | % Must unreg whenever there is a delay in executing requests or complete stop from caller 47 | unreg_caller() -> 48 | detest_shaper:call(unreg_caller). 49 | % Get a node that is safe to call and request must succeed. 50 | % If network in flux, will block. 51 | majority_node() -> 52 | detest_shaper:call(majority_node). 53 | % Change network conditions every 1-20s. 54 | % Always keeps a majority of nodes online. 55 | % Randomly sets latency between 0 - 1s for online nodes 56 | % Can take node offline by shutting down, set very high latency or isolates network. 57 | shape_traffic_rand() -> 58 | detest_shaper:call(shape_traffic_rand). 59 | shape_traffic_get() -> 60 | detest_shaper:call(shape_traffic_get). 61 | shape_traffic_set(Mods) -> 62 | detest_shaper:call({shape_traffic_set,Mods}). 63 | shape_traffic_stop() -> 64 | detest_shaper:call(shape_traffic_stop). 65 | 66 | 67 | 68 | tm() -> 69 | erlang:monotonic_time(millisecond). 70 | 71 | call(Msg) -> 72 | Ref = make_ref(), 73 | ?MODULE ! {call,self(), Ref, Msg}, 74 | receive 75 | {Ref,Resp} -> 76 | Resp 77 | end. 78 | 79 | -record(pkt,{bin = <<>>, tm = 0}). 80 | -record(nd,{rpc_port, dist_port, offset = 0, online = true, groups = [], initialized = false}). 81 | -record(sock,{src_node, dst_node, other, bw = 1024 * 1024, recv_int_bytes = 0, int_budget = 1024, latency = 0, queue}). 82 | -record(lsock,{src_node, dst_node, accept_ref, dst_port, port, blocked = false}). 83 | -record(dp,{sockets = #{}, nodes = #{}, bytes = 0}). 84 | 85 | run() -> 86 | register(?MODULE,self()), 87 | erlang:send_after(?INTERVAL, self(), timeout), 88 | Nodes = nodes_parse(), 89 | Nodes1 = [{NdNm, Nd#nd{initialized = true}} || {NdNm,Nd} <- maps:to_list(Nodes) ], 90 | run(#dp{nodes = maps:from_list(Nodes1), 91 | sockets = create_listeners(maps:to_list(Nodes),maps:to_list(Nodes),#{})}). 92 | run(P) -> 93 | receive 94 | {call, Src, Ref, Msg} -> 95 | Src ! {Ref, ok}, 96 | run(modify(P,Msg)); 97 | timeout -> 98 | % ?INF("Transfered ~p",[P#dp.bytes]), 99 | erlang:send_after(?INTERVAL, self(), timeout), 100 | run(P#dp{sockets = socket_activate(maps:to_list(P#dp.sockets), P#dp.sockets)}); 101 | {tcp,Socket,Bin} -> 102 | case maps:get(Socket, P#dp.sockets,undefined) of 103 | undefined -> 104 | run(P); 105 | #sock{queue = Q} = SI -> 106 | Now = tm(), 107 | NQ = queue:in(#pkt{bin = Bin, tm = Now}, Q), 108 | IntBytes = SI#sock.recv_int_bytes + byte_size(Bin), 109 | run(P#dp{bytes = P#dp.bytes + byte_size(Bin), 110 | sockets = (P#dp.sockets)#{Socket => SI#sock{queue = NQ, recv_int_bytes = IntBytes}}}) 111 | end; 112 | {tcp_closed,Socket} -> 113 | case maps:get(Socket, P#dp.sockets,undefined) of 114 | undefined -> 115 | run(P); 116 | #sock{other = Other} -> 117 | gen_tcp:close(Other), 118 | run(P#dp{sockets = maps:remove(Other,maps:remove(Socket,P#dp.sockets))}) 119 | end; 120 | {tcp_passive,Socket} -> 121 | case maps:get(Socket, P#dp.sockets,undefined) of 122 | undefined -> 123 | run(P); 124 | #sock{int_budget = Budget, recv_int_bytes = Bytes} -> 125 | case Budget > Bytes of 126 | true -> 127 | inet:setopts(Socket,[{active,once}]), 128 | run(P); 129 | false -> 130 | run(P) 131 | end 132 | end; 133 | {inet_async, LSock, _Ref, {ok, Sock}} -> 134 | % ?INF("inet_async ~p",[LSock]), 135 | run(accept(P,LSock, Sock)); 136 | {inet_async, _LSock, _Ref, _Error} -> 137 | ?INF("async accept error ignore ~p",[_Error]), 138 | run(P); 139 | Other -> 140 | ?INF("detest_net unknown_msg ~p",[Other]), 141 | run(P) 142 | end. 143 | 144 | modify(P,{group_set, Nodes, Id}) -> 145 | ModNodes = [{NdNm,NI#nd{groups = butil:lists_add(Id, NI#nd.groups)}} || 146 | {NdNm,NI} <- maps:to_list(P#dp.nodes), lists:member(NdNm,Nodes) andalso lists:member(Id,NI#nd.groups) == false], 147 | ?INF("group_set ~p ~p ~p",[ModNodes,Nodes,P#dp.nodes ]), 148 | case ModNodes of 149 | [] -> 150 | P; 151 | _ -> 152 | NP = P#dp{nodes = put_all(ModNodes,P#dp.nodes)}, 153 | Op = {group_set, Nodes}, 154 | NP#dp{sockets = mod_socks(NP#dp.nodes, maps:to_list(NP#dp.sockets), Op, NP#dp.sockets)} 155 | end; 156 | modify(P,{group_remove, Id}) -> 157 | ModNodes = [{NdNm,NI#nd{groups = lists:delete(Id, NI#nd.groups)}} || 158 | {NdNm,NI} <- maps:to_list(P#dp.nodes), lists:member(Id,NI#nd.groups)], 159 | case ModNodes of 160 | [] -> 161 | P; 162 | _ -> 163 | Op = {group_remove, Id}, 164 | NSocks = mod_socks(P#dp.nodes, maps:to_list(P#dp.sockets), Op, P#dp.sockets), 165 | UpdatedNodes = put_all(ModNodes,P#dp.nodes), 166 | P#dp{nodes = UpdatedNodes, 167 | sockets = create_listeners(maps:to_list(UpdatedNodes),maps:to_list(UpdatedNodes),NSocks)} 168 | end; 169 | modify(P,{add_node, Node}) -> 170 | case maps:get(Node,P#dp.nodes, undefined) of 171 | undefined -> 172 | #{Node := NI} = nodes_parse(), 173 | Nodes = maps:put(Node, NI, P#dp.nodes), 174 | P#dp{nodes = Nodes, 175 | sockets = create_listeners([{Node,NI}],maps:to_list(Nodes),P#dp.sockets)}; 176 | _ -> 177 | P 178 | end; 179 | modify(P,{_What, Node, _Bw} = Op) -> 180 | #{Node := #nd{online = NodeOnline} = NI} = P#dp.nodes, 181 | case Op of 182 | {node_online, Node, Bool} when Bool == NodeOnline -> 183 | P; 184 | _ -> 185 | case Op of 186 | {node_online, _, true} -> 187 | P#dp{nodes = (P#dp.nodes)#{Node => NI#nd{online = true}}, 188 | sockets = create_listeners([{Node,NI}],maps:to_list(P#dp.nodes),P#dp.sockets)}; 189 | {node_online, _, false} -> 190 | Out = P#dp{sockets = mod_socks(P#dp.nodes, maps:to_list(P#dp.sockets), Op, P#dp.sockets)}, 191 | Out#dp{nodes = (P#dp.nodes)#{Node => NI#nd{online = false}}}; 192 | _ -> 193 | P#dp{sockets = mod_socks(P#dp.nodes, maps:to_list(P#dp.sockets), Op, P#dp.sockets)} 194 | end 195 | end. 196 | 197 | src_node(#sock{src_node = Src}) -> 198 | Src; 199 | src_node(#lsock{src_node = Src}) -> 200 | Src. 201 | dst_node(#sock{dst_node = Dst}) -> 202 | Dst; 203 | dst_node(#lsock{dst_node = Dst}) -> 204 | Dst. 205 | 206 | mod_socks(Nodes,[{S,SI}|T], {group_remove, GrpId} = Op, M) -> 207 | SrcNd = src_node(SI), 208 | DstNd = dst_node(SI), 209 | SrcNI = maps:get(SrcNd, Nodes), 210 | DstNI = maps:get(DstNd, Nodes), 211 | MemberSrc = lists:member(GrpId, SrcNI#nd.groups), 212 | MemberDst = lists:member(GrpId, DstNI#nd.groups), 213 | case MemberSrc orelse MemberDst of 214 | false -> 215 | mod_socks(Nodes,T, Op, M); 216 | true -> 217 | MemberSrc1 = lists:delete(GrpId, SrcNI#nd.groups), 218 | MemberDst1 = lists:delete(GrpId, DstNI#nd.groups), 219 | NoCommon = (SrcNI#nd.groups -- DstNI#nd.groups) == SrcNI#nd.groups, 220 | case ok of 221 | _ when MemberSrc1 == [], MemberDst1 == [] -> 222 | mod_socks(Nodes,T, Op, M); 223 | _ when NoCommon -> 224 | mod_socks(Nodes,T, Op, sock_close(S, SI, M)); 225 | _ -> 226 | mod_socks(Nodes,T, Op, M) 227 | end 228 | end; 229 | mod_socks(Nodes,[{S,SI}|T], {group_set, GNodes} = Op, M) -> 230 | ?INF("group_set ~p",[GNodes]), 231 | SrcNd = src_node(SI), 232 | DstNd = dst_node(SI), 233 | MemberSrc = lists:member(SrcNd, GNodes), 234 | MemberDst = lists:member(DstNd, GNodes), 235 | case MemberSrc orelse MemberDst of 236 | true when MemberSrc andalso MemberDst -> 237 | mod_socks(Nodes,T, Op, M); 238 | true -> 239 | SrcNI = maps:get(SrcNd, Nodes), 240 | DstNI = maps:get(DstNd, Nodes), 241 | case (SrcNI#nd.groups -- DstNI#nd.groups) /= SrcNI#nd.groups of 242 | % they have a common group 243 | true -> 244 | mod_socks(Nodes,T, Op, M); 245 | false -> 246 | mod_socks(Nodes,T, Op, sock_close(S, SI, M)) 247 | end; 248 | false -> 249 | mod_socks(Nodes,T, Op, M) 250 | end; 251 | mod_socks(Nodes,[{S,SI}|T], {node_bw, Node, Bw} = Op, M) when SI#sock.src_node == Node; SI#sock.dst_node == Node -> 252 | mod_socks(Nodes,T, Op, M#{S => sock_bw(SI,Bw)}); 253 | mod_socks(Nodes,[{S,SI}|T], {node_latency, Node, Lat} = Op, M) when SI#sock.src_node == Node; SI#sock.dst_node == Node -> 254 | mod_socks(Nodes,T, Op, M#{S => SI#sock{latency = Lat}}); 255 | mod_socks(Nodes,[{S,SI}|T], {node_online, Node, Bool} = Op, M) when SI#sock.src_node == Node; SI#sock.dst_node == Node -> 256 | case Bool of 257 | false when SI#sock.src_node /= SI#sock.dst_node -> 258 | mod_socks(Nodes,T, Op, M); 259 | false -> 260 | mod_socks(Nodes,T, Op, sock_close(S, SI, M)); 261 | true -> 262 | mod_socks(Nodes,T, Op, M) 263 | end; 264 | mod_socks(Nodes,[{S,SI}|T], {node_online, Node, Bool} = Op, M) when SI#lsock.src_node == Node; SI#lsock.dst_node == Node -> 265 | case Bool of 266 | false when SI#lsock.src_node /= SI#lsock.dst_node -> 267 | mod_socks(Nodes,T, Op, M); 268 | false -> 269 | mod_socks(Nodes,T, Op, sock_close(S,SI,M)); 270 | true -> 271 | mod_socks(Nodes,T, Op, M) 272 | end; 273 | mod_socks(Nodes,[_|T], Op, M) -> 274 | mod_socks(Nodes,T, Op, M); 275 | mod_socks(_,[],_, M) -> 276 | M. 277 | 278 | sock_close(S, SI, M) when element(1,SI) == sock -> 279 | gen_tcp:close(S), 280 | gen_tcp:close(SI#sock.other), 281 | maps:remove(S,maps:remove(SI#sock.other,M)); 282 | sock_close(S, SI, M) when element(1,SI) == lsock -> 283 | ?INF("sock_close ~p",[SI]), 284 | gen_tcp:close(S), 285 | maps:remove(S,M). 286 | 287 | flush_q(Sock, #sock{other = Other} = SI) -> 288 | case queue:out(SI#sock.queue) of 289 | {empty,_} -> 290 | SI; 291 | {{value,Oldest},NQ} -> 292 | Now = tm(), 293 | case Oldest#pkt.tm + SI#sock.latency < Now of 294 | true -> 295 | gen_tcp:send(Other, Oldest#pkt.bin), 296 | flush_q(Sock, SI#sock{queue = NQ}); 297 | false -> 298 | SI 299 | end 300 | end. 301 | 302 | socket_activate([{S,#sock{recv_int_bytes = Recv, int_budget = Budget} = SI}|T], M) -> 303 | case ok of 304 | _ when Recv =< Budget -> 305 | NRecv = 0, 306 | inet:setopts(S,[{active,once}]); 307 | _ -> 308 | NRecv = Recv - Budget 309 | end, 310 | socket_activate(T, M#{S => flush_q(S,SI#sock{recv_int_bytes = NRecv})}); 311 | socket_activate([_|T], M) -> 312 | socket_activate(T, M); 313 | socket_activate([], M) -> 314 | M. 315 | 316 | create_listeners([{_NdName,#nd{online = false}}|T],Nodes,M) -> 317 | create_listeners(T,Nodes,M); 318 | create_listeners([{NdName,Nd}|T],Nodes,M) -> 319 | create_listeners(T,Nodes,create_listeners(NdName, Nd, Nodes, M)); 320 | create_listeners([],_,M) -> 321 | M. 322 | create_listeners(NdName,Nd,[{OtherNd,Other}|T], M) -> % when NdName /= OtherNd 323 | Groups = Nd#nd.groups, 324 | OtherGroups = Other#nd.groups, 325 | ?INF("create listeners ~p ~p ~p ~p",[NdName, OtherNd, Groups, OtherGroups]), 326 | case ok of 327 | _ when Groups == [], OtherGroups == [] -> 328 | Doit = true; 329 | _ -> 330 | % If this this node and other node have a common group 331 | Doit = (Groups -- OtherGroups) /= Groups 332 | end, 333 | case Doit of 334 | true when NdName == OtherNd, Nd#nd.initialized -> 335 | Changes = []; 336 | true -> 337 | L1 = lsock(NdName, Other#nd.rpc_port + Nd#nd.offset, Other#nd.rpc_port, OtherNd), 338 | L2 = lsock(NdName, Other#nd.dist_port + Nd#nd.offset, Other#nd.dist_port, OtherNd), 339 | Changes = L1++L2; 340 | false -> 341 | Changes = [] 342 | end, 343 | create_listeners(NdName, Nd, T, put_all(Changes,M)); 344 | % create_listeners(A,B,[_|T],M) -> 345 | % create_listeners(A,B,T,M); 346 | create_listeners(_,_,[],M) -> 347 | M. 348 | 349 | lsock(Nd,Port,DstPort, DstNode) -> 350 | Opts = [binary, {packet, 0}, {reuseaddr, true}, 351 | {keepalive, true}, {active, false}], 352 | case gen_tcp:listen(Port, Opts) of 353 | {ok,LSock} -> 354 | ?INF("lsock create from=~p to=~p, port=~p",[Nd,DstNode,Port]), 355 | {ok, Ref} = prim_inet:async_accept(LSock, -1), 356 | [{LSock,#lsock{accept_ref = Ref, src_node = Nd, dst_node = DstNode, dst_port = DstPort, port = Port}}]; 357 | _ -> 358 | [] 359 | end. 360 | 361 | accept(P, LSock, Sock) -> 362 | case is_port(Sock) of 363 | true -> 364 | case set_sockopt(LSock, Sock) of 365 | ok -> 366 | gen_tcp:controlling_process(Sock, self()); 367 | {error, _Reason} -> 368 | ?INF("async accept fail ~p",[_Reason]), 369 | ok 370 | end; 371 | false -> 372 | ok 373 | end, 374 | case prim_inet:async_accept(LSock, -1) of 375 | {ok, NewRef} -> 376 | ok; 377 | {error, NewRef} -> 378 | ?INF("async accept error"), 379 | ok 380 | end, 381 | case maps:get(LSock, P#dp.sockets, undefined) of 382 | undefined -> 383 | P; 384 | LInfo -> 385 | Opts = [{keepalive,true},binary,{active,false},{nodelay,true},{sndbuf,1024*4},{recbuf,1024*4}], 386 | case gen_tcp:connect({127,0,0,1},LInfo#lsock.dst_port,Opts) of 387 | {ok,Other} -> 388 | BW = ?CFG(internode_bw), 389 | DF = sock_bw(#sock{queue = queue:new()}, BW), 390 | ?INF("Accepted sock from=~p to=~p",[LInfo#lsock.src_node, LInfo#lsock.dst_node]), 391 | OI = DF#sock{other = Sock, dst_node = LInfo#lsock.src_node, src_node = LInfo#lsock.dst_node}, 392 | SI = DF#sock{other = Other,src_node = LInfo#lsock.src_node, dst_node = LInfo#lsock.dst_node}, 393 | Updates = [{LSock,LInfo#lsock{accept_ref = NewRef}}, 394 | {Other, OI}, 395 | {Sock, SI}]; 396 | _ -> 397 | gen_tcp:close(Sock), 398 | Updates = [{LSock,LInfo#lsock{accept_ref = NewRef}}] 399 | end, 400 | P#dp{sockets = put_all(Updates, P#dp.sockets)} 401 | end. 402 | 403 | sock_bw(SI,BW) -> 404 | SI#sock{bw = BW, int_budget = erlang:round(BW / ?SAMPLES_PER_SEC)}. 405 | 406 | 407 | put_all([{K,H}|T],M) -> 408 | put_all(T,maps:put(K,H,M)); 409 | put_all([],M) -> 410 | M. 411 | 412 | set_sockopt(LSocket, Socket) -> 413 | case inet_db:lookup_socket(LSocket) of 414 | {error,E} -> 415 | gen_tcp:close(Socket), 416 | {error,E}; 417 | {ok, Mod} -> 418 | true = inet_db:register_socket(Socket, Mod), 419 | case prim_inet:getopts(LSocket, [active, nodelay, keepalive, delay_send, priority, tos]) of 420 | {ok, Opts} -> 421 | case prim_inet:setopts(Socket, Opts) of 422 | ok -> ok; 423 | Error -> gen_tcp:close(Socket), Error 424 | end; 425 | Error -> 426 | gen_tcp:close(Socket), Error 427 | end 428 | end. 429 | 430 | nodes_parse() -> 431 | Nodes = ?CFG(nodes), 432 | % ?INF("Nodes ~p",[Nodes]), 433 | maps:from_list([begin 434 | Node = butil:toatom(butil:ds_val(distname,Nd)), 435 | detest_shaper:cast({add_node,Node}), 436 | {Node, readnd(Nd)} 437 | end || Nd <- Nodes]). 438 | 439 | readnd(Nd) -> 440 | #nd{rpc_port = butil:ds_val(rpcport,Nd,0), 441 | dist_port = butil:ds_val(dist_port,Nd), 442 | offset = butil:ds_val(connect_offset,Nd,0)}. 443 | 444 | -------------------------------------------------------------------------------- /src/detest.erl: -------------------------------------------------------------------------------- 1 | -module(detest). 2 | % run for running detest embedded. dist erlang must be running. 3 | % main for running detest as escript. 4 | -export([main/1,run/1,run/2,run/3,ez/0]). 5 | % API for test module 6 | -export([add_node/1,add_node/2, stop_node/1, ip/1, cmd/2, isolate/2,isolate_end/1]). 7 | -include("detest.hrl"). 8 | 9 | ip(Distname) -> 10 | [_,IP] = string:tokens(butil:tolist(Distname),"@"), 11 | IP. 12 | 13 | % Isolate node or nodes on Id 14 | % All nodes with this Id will see each other. 15 | isolate(Nodes,Id) when is_list(Nodes) -> 16 | [begin 17 | rpc:call(Nd,erlang,set_cookie,[Nd,butil:toatom(Id)]), 18 | erlang:set_cookie(Nd,butil:toatom(Id)), 19 | HisNodes = rpc:call(Nd,erlang,nodes,[]), 20 | [rpc:call(Nd,erlang,disconnect_node,[HN]) || HN <- HisNodes], 21 | pong = net_adm:ping(Nd), 22 | case rpc:call(Nd,erlang,whereis,[bkdcore_sup]) of 23 | undefined -> 24 | ok; 25 | _ -> 26 | rpc:call(Nd,bkdcore_rpc,isolate,[true]) 27 | end, 28 | NodeInfo = butil:findobj(distname,Nd,butil:ds_val(nodes,etscfg)), 29 | [rpc:call(butil:ds_val(distname,Nd1), bkdcore_rpc, isolate_from,[butil:tobin(butil:ds_val(name,NodeInfo)),true]) || Nd1 <- butil:ds_val(nodes,etscfg)] 30 | end || Nd <- Nodes]; 31 | isolate(Node,Id) -> 32 | isolate([Node],Id). 33 | 34 | % Back to default 35 | isolate_end(Nodes) when is_list(Nodes) -> 36 | [begin 37 | rpc:call(Nd,erlang,set_cookie,[Nd,erlang:get_cookie()]), 38 | erlang:set_cookie(Nd,erlang:get_cookie()), 39 | pong = net_adm:ping(Nd), 40 | case rpc:call(Nd,erlang,whereis,[bkdcore_sup]) of 41 | undefined -> 42 | ok; 43 | _ -> 44 | % Cons = rpc:call(Nd,ranch_server,get_connections_sup,[bkdcore_in]), 45 | % L = rpc:call(Nd,supervisor,which_children,[Cons]), 46 | % [rpc:call(Nd,bkdcore_rpc,isolate,[Pid,false]) || {bkdcore_rpc,Pid,worker,[bkdcore_rpc]} <- L] 47 | rpc:call(Nd,bkdcore_rpc,isolate,[false]) 48 | end, 49 | NodeInfo = butil:findobj(distname,Nd,butil:ds_val(nodes,etscfg)), 50 | [rpc:call(butil:ds_val(distname,Nd1), bkdcore_rpc, isolate_from,[butil:tobin(butil:ds_val(name,NodeInfo)),false]) || Nd1 <- butil:ds_val(nodes,etscfg)] 51 | end || Nd <- Nodes]; 52 | isolate_end(N) -> 53 | isolate_end([N]). 54 | 55 | % Execute command on host where Node is running at. 56 | % Useful for nodes connected over ssh, otherwise just os:cmd/1 57 | cmd(Node,Cmd) -> 58 | case lists:keyfind(ssh,1,Node) of 59 | false -> 60 | os:cmd(Cmd); 61 | {ssh,Host,SshPort,_Cwd,Opts} -> 62 | {ok,SshCon} = ssh:connect(Host,SshPort,[{silently_accept_hosts,true}|Opts]), 63 | {ok,SshChan} = ssh_connection:session_channel(SshCon,5000), 64 | ssh_connection:exec(SshCon, SshChan,Cmd, infinity), 65 | Rec = receive_ssh_resp([]), 66 | ssh:close(SshCon), 67 | Rec 68 | end. 69 | 70 | receive_ssh_resp(L) -> 71 | receive 72 | {ssh_cm,_,{data,_,_,Bin}} -> 73 | receive_ssh_resp([Bin|L]); 74 | {ssh_cm,_,{closed,_}} -> 75 | iolist_to_binary(lists:reverse(L)) 76 | end. 77 | 78 | add_node(P1) -> 79 | add_node(P1,[]). 80 | add_node(P1,NewCfg) -> 81 | DistName = distname(P1), 82 | % update configs if we received them 83 | GlobCfgs = butil:ds_val(global_cfg,NewCfg,butil:ds_val(global_cfg,etscfg)), 84 | NodeCfgs = butil:ds_val(per_node_cfg,NewCfg,butil:ds_val(per_node_cfg,etscfg)), 85 | butil:ds_add(global_cfg,GlobCfgs,etscfg), 86 | butil:ds_add(per_node_cfg,NodeCfgs,etscfg), 87 | % add node to nodes 88 | P = butil:lists_add({distname, DistName},P1), 89 | % node might already be in cfg 90 | case butil:findobj(distname,DistName,butil:ds_val(nodes,etscfg)) of 91 | false -> 92 | Nodes = [P|butil:ds_val(nodes,etscfg)], 93 | butil:ds_add(nodes,Nodes,etscfg), 94 | detest_net:add_node(butil:ds_val(name,P)); 95 | _ -> 96 | butil:ds_val(nodes,etscfg) 97 | end, 98 | 99 | write_global_cfgs(), 100 | write_per_node_cfgs(), 101 | 102 | Pid = start_node(P), 103 | RunPids = butil:ds_val(runpids,etscfg), 104 | butil:ds_add(runpids,[Pid|RunPids],etscfg), 105 | 106 | ok = connect([P]), 107 | ok = wait_app([P],butil:ds_val(wait_for_app,etscfg),butil:ds_val(scriptload,etscfg)), 108 | DistName. 109 | 110 | stop_node([_|_] = Nd) -> 111 | stop_node(distname(Nd)); 112 | stop_node(Nm) when is_atom(Nm) -> 113 | io:format("stopnode ~p~n",[Nm]), 114 | Nm ! stop, 115 | wait_pids([whereis(Nm)]). 116 | 117 | 118 | 119 | main([]) -> 120 | io:format("Command: ./detest yourscript.erl ~n"++ 121 | "Options:~n" 122 | "-h print help~n"++ 123 | "-v print stdout from nodes~n"++ 124 | "-q quiet~n"++ 125 | "-ez create ez package from beams in project (if your tests need external libraries)~n"); 126 | main(["-h"]) -> 127 | main([]); 128 | main(Param) -> 129 | case lists:member("-ez",Param) of 130 | true -> 131 | ez(filename:basename(filename:dirname(filename:absname(escript:script_name())))), 132 | halt(1); 133 | false -> 134 | ok 135 | end, 136 | ets:new(etscfg, [named_table,public,set,{read_concurrency,true}]), 137 | butil:ds_add(verbose,lists:member("-v",Param),etscfg), 138 | butil:ds_add(quiet,lists:member("-q",Param),etscfg), 139 | butil:ds_add(stopby,halt,etscfg), 140 | butil:ds_add(basepath,?PATH,etscfg), 141 | butil:ds_add(dostdin,true,etscfg), 142 | case script_param(Param) of 143 | {ScriptNm,ScriptArg} -> 144 | ok; 145 | _ -> 146 | ScriptNm = ScriptArg = undefined, 147 | io:format("Invalid params~n"), 148 | halt(1) 149 | end, 150 | [] = os:cmd(epmd_path() ++ " -daemon"), 151 | application:ensure_all_started(lager), 152 | 153 | case compile:file(ScriptNm,[binary,return_errors,{parse_transform, lager_transform}]) of 154 | {ok,Mod,Bin} -> 155 | ScriptLoad = {Mod,Bin,filename:basename(ScriptNm)}, 156 | code:load_binary(Mod, filename:basename(ScriptNm), Bin); 157 | Err -> 158 | ScriptLoad = Mod = undefined, 159 | ?INF("Unable to compile: ~p",[Err]), 160 | halt(1) 161 | end, 162 | run(Mod,ScriptArg,ScriptLoad). 163 | 164 | stop(Reason) -> 165 | case butil:ds_val(stopby,etscfg) of 166 | halt -> 167 | halt(1); 168 | _ -> 169 | throw(Reason) 170 | end. 171 | 172 | % Run will execute until Mod:run concludes and erlang nodes have stopped. 173 | % Mod - test module atom 174 | % ScriptArg - list of script arguments 175 | % Opts - [{basepath,".detest"},{verbose,true},{quiet,true},..] 176 | run(Mod) -> 177 | run(Mod,[]). 178 | run(Mod,ScriptArg) -> 179 | run(Mod,ScriptArg, []). 180 | run(Mod,ScriptArg,[{_,_}|_] = Opts) -> 181 | case ets:info(etscfg,size) of 182 | undefined -> 183 | ets:new(etscfg, [named_table,public,set,{read_concurrency,true}]); 184 | _ -> 185 | ets:delete_all_objects(etscfg) 186 | end, 187 | [butil:ds_add(K,V,etscfg) || {K,V} <- Opts], 188 | case butil:ds_val(basepath,etscfg) of 189 | undefined -> 190 | butil:ds_add(basepath,?PATH,etscfg); 191 | _ -> 192 | ok 193 | end, 194 | run(Mod,ScriptArg,code:get_object_code(Mod)); 195 | run(Mod,ScriptArg,{Mod,_ModBin,_ModFilename} = ScriptLoad) -> 196 | Cfg = apply(Mod,cfg,[ScriptArg]), 197 | GlobCfgs = butil:ds_val(global_cfg,Cfg,[]), 198 | NodeCfgs = butil:ds_val(per_node_cfg,Cfg,[]), 199 | Nodes1 = butil:ds_val(nodes,Cfg), 200 | Nodes = [[{distname,distname(Nd)}|Nd] || Nd <- Nodes1], 201 | 202 | case node() of 203 | 'nonode@nohost' -> 204 | DetestDist = butil:ds_val(detest_name,Cfg,'detest@127.0.0.1'), 205 | [_,DistHost] = string:tokens(butil:tolist(DetestDist),"@"), 206 | case string:tokens(DistHost,".") of 207 | [_] -> 208 | {ok, _} = net_kernel:start([DetestDist, shortnames]); 209 | _ -> 210 | {ok, _} = net_kernel:start([DetestDist, longnames]) 211 | end; 212 | _ -> 213 | ok 214 | end, 215 | 216 | case [lists:keyfind(ssh,1,Nd) || Nd <- Nodes, lists:keyfind(ssh,1,Nd) /= false] of 217 | [] -> 218 | % case os:type() of 219 | % {unix,linux} -> 220 | % % case os:cmd("sudo whoami") of 221 | % % "root"++_ -> 222 | % % application:ensure_all_started(damocles); 223 | % % _ -> 224 | % % ok 225 | % % end; 226 | % ok; 227 | % _ -> 228 | % ok 229 | % end; 230 | ok; 231 | [_|_] -> 232 | ssh:start() 233 | end, 234 | 235 | butil:ds_add(global_cfg,GlobCfgs,etscfg), 236 | butil:ds_add(per_node_cfg,NodeCfgs,etscfg), 237 | butil:ds_add(nodes,Nodes,etscfg), 238 | butil:ds_add(cmd,butil:ds_val(cmd,Cfg,""),etscfg), 239 | butil:ds_add(wait_for_app,butil:ds_val(wait_for_app,Cfg),etscfg), 240 | butil:ds_add(scriptload,ScriptLoad,etscfg), 241 | butil:ds_add(erlcmd,butil:ds_val(erlcmd,GlobCfgs,"erl"),etscfg), 242 | butil:ds_add(erlenv,butil:ds_val(erlenv,GlobCfgs,[]),etscfg), 243 | butil:ds_add(internode_bw,butil:ds_val(internode_bw,Cfg,1024*1024),etscfg), 244 | butil:ds_add(connect_timeout,butil:ds_val(connect_timeout,Cfg,10000),etscfg), 245 | butil:ds_add(app_wait_timeout,butil:ds_val(app_wait_timeout,Cfg,30000),etscfg), 246 | 247 | [begin 248 | case lists:keyfind(ssh,1,Nd) of 249 | false -> 250 | ok; 251 | {ssh,_,_,BasePth1,_} -> 252 | %?INF("Cleanup ~p",["rm -rf "++BasePth1++"/"++butil:ds_val(basepath,etscfg)]), 253 | cmd(Nd,"rm -rf "++BasePth1++"/"++butil:ds_val(basepath,etscfg)) 254 | end 255 | end || Nd <- Nodes], 256 | 257 | os:cmd("rm -rf "++butil:ds_val(basepath,etscfg)), 258 | 259 | % case detest_net:start([{butil:ip_to_tuple(ndaddr(Nd)), 260 | % proplists:get_value(delay,Nd,{0,0}), 261 | % ndnm(Nd)} || Nd <- Nodes]) of 262 | % {error,NetPids} -> 263 | % DetestName = undefined, 264 | % ?INF("Unable to setup interfaces. Run as sudo? Kill your vpn app?~nFailed:~p",[NetPids]), 265 | % halt(1); 266 | % {ok,NetPids,DetestName} -> 267 | % ok 268 | % end, 269 | % NetPids = [], 270 | 271 | compile_cfgs(GlobCfgs++NodeCfgs), 272 | 273 | % create files in etc 274 | write_global_cfgs(Nodes,GlobCfgs), 275 | write_per_node_cfgs(Nodes,NodeCfgs), 276 | 277 | DistNames = [{butil:ds_val(name,Nd),butil:ds_val(distname,Nd)} || Nd <- Nodes], 278 | ScriptParam = DistNames++[{path,butil:ds_val(basepath,etscfg)},{args,ScriptArg}], 279 | %{damocles,butil:is_app_running(damocles)} 280 | 281 | case catch apply(Mod,setup,[ScriptParam]) of 282 | {'EXIT',Err0} -> 283 | ?INF("Setup failed ~p",[Err0]), 284 | stop({error,setup,Err0}); 285 | _ -> 286 | ok 287 | end, 288 | 289 | os:cmd("chmod -R a+rw "++butil:ds_val(basepath,etscfg)), 290 | detest_net:start(), 291 | % spawn nodes 292 | RunPids = [start_node(Nd,Cfg) || Nd <- Nodes], 293 | butil:ds_add(runpids,RunPids,etscfg), 294 | 295 | case connect(Nodes) of 296 | {error,Node} -> 297 | Pid1 = Pid2 = undefined, 298 | ?INF("Unable to connect to ~p",[Node]); 299 | ok -> 300 | timer:sleep(500), 301 | case wait_app(Nodes,butil:ds_val(wait_for_app,Cfg),ScriptLoad) of 302 | {error,Node} -> 303 | Pid1 = Pid2 = undefined, 304 | ?INF("Timeout waiting for app to start on ~p",[Node]); 305 | ok -> 306 | {Pid1,_} = spawn_monitor(fun() -> runproc(Mod,ScriptParam) end), 307 | {Pid2,_} = spawn_monitor(fun() -> stdinproc() end) 308 | end 309 | end, 310 | wait_done(Pid1,Pid2), 311 | do_stop(Mod,Cfg, ScriptParam). 312 | 313 | wait_done(Pid1,Pid2) -> 314 | case is_pid(Pid1) andalso is_pid(Pid2) of 315 | true -> 316 | receive 317 | % no matter which process exits, kill both 318 | {'DOWN',_Ref,_,Pid,_} when Pid == Pid1; Pid == Pid2 -> 319 | exit(Pid1,stop), 320 | exit(Pid2,stop) 321 | end; 322 | false -> 323 | ok 324 | end. 325 | 326 | prompt_continue() -> 327 | case io:get_line("Enter c to continue: ") of 328 | "c"++_ -> 329 | ok; 330 | "C"++_ -> 331 | ok; 332 | _A -> 333 | prompt_continue() 334 | end. 335 | 336 | stdinproc() -> 337 | case butil:ds_val(dostdin,etscfg) of 338 | true -> 339 | case io:get_line("Enter q to quit test: ") of 340 | "q"++_ -> 341 | ok; 342 | "Q"++_ -> 343 | ok; 344 | _A -> 345 | stdinproc() 346 | end; 347 | _ -> 348 | timer:sleep(infinity) 349 | end. 350 | 351 | runproc(Mod,ScriptParam) -> 352 | register(runproc,self()), 353 | case catch apply(Mod,run,[ScriptParam]) of 354 | {'EXIT',Err1} -> 355 | ?INF("Test failed ~p",[Err1]); 356 | _ -> 357 | ?INF("Test finished.") 358 | end. 359 | 360 | script_param(["-"++_N|T]) -> 361 | script_param(T); 362 | script_param([N|T]) -> 363 | case filename:extension(N) of 364 | ".erl" -> 365 | {N,T}; 366 | _ -> 367 | script_param(T) 368 | end; 369 | script_param([]) -> 370 | false. 371 | 372 | do_stop(Mod,Cfg,ScriptParam) -> 373 | RunPids = ?CFG(runpids), 374 | {StopMod,StopFunc,StopArg} = butil:ds_val(stop,Cfg,{init,stop,[]}), 375 | % Nodelist may have changed, read it from ets 376 | Nodes = ?CFG(nodes), 377 | timer:sleep(800), 378 | Pids = [spawn(fun() -> rpc:call(distname(Nd),StopMod,StopFunc,StopArg,10000) end) || Nd <- Nodes], 379 | wait_pids(Pids), 380 | % [butil:safesend(NetPid,stop) || NetPid <- NetPids], 381 | application:stop(damocles), 382 | % If nodes still alive, kill them forcefully 383 | [RP ! stop || RP <- RunPids], 384 | timer:sleep(200), 385 | % wait_pids(NetPids), 386 | apply(Mod,cleanup,[ScriptParam]), 387 | ok. 388 | 389 | start_node(Nd) -> 390 | start_node(Nd,etscfg). 391 | start_node(Nd,GlobCfg) -> 392 | start_node(Nd,butil:ds_val(cmd,GlobCfg,""),butil:ds_val(erlcmd,GlobCfg,"erl"),butil:ds_val(erlenv,GlobCfg,[])). 393 | start_node(Nd,GlobCmd,ErlCmd1,ErlEnv1) -> 394 | AppPth = lists:flatten([?CFG(basepath),"/",ndnm(Nd),"/etc/app.config"]), 395 | RunCmd = butil:ds_val(cmd,Nd,GlobCmd), 396 | ErlCmd = butil:ds_val(erlcmd,Nd,ErlCmd1), 397 | ErlEnv = butil:ds_val(erlenv,Nd,ErlEnv1), 398 | case filelib:is_regular(AppPth) of 399 | true -> 400 | AppCmd = " -config "++AppPth; 401 | false -> 402 | AppCmd = "" 403 | end, 404 | VmArgs = lists:flatten([butil:ds_val(basepath,etscfg),"/",ndnm(Nd),"/etc/vm.args"]), 405 | case filelib:is_regular(VmArgs) of 406 | true -> 407 | VmCmd = " -args_file "++VmArgs; 408 | false -> 409 | VmCmd = "" 410 | end, 411 | 412 | %Ebins = [filename:absname(Nm) || Nm <- ["ebin"|filelib:wildcard("deps/*/ebin")]]," "), 413 | Ebins = string:join(["ebin"|filelib:wildcard("deps/*/ebin")]," "), 414 | Dist = atom_to_list(butil:ds_val(distname,Nd)), 415 | case string:tokens(Dist,".") of 416 | [_] -> 417 | NameStr = " -sname "; 418 | _ -> 419 | NameStr = " -name " 420 | end, 421 | Cmd = ErlCmd++" -noshell -noinput -setcookie "++butil:tolist(erlang:get_cookie())++NameStr++Dist++ 422 | " -pa "++Ebins++" "++AppCmd++VmCmd++" "++RunCmd, 423 | case butil:ds_val(extrun,Nd) of 424 | true -> 425 | ?INF("extrun set for node. Run it manually now. Command detest would use (in current folder): ~n~p",[Cmd]), 426 | prompt_continue(), 427 | runerl(Nd,butil:ds_val(distname,Nd),undefined,[]); 428 | _ -> 429 | timer:sleep(300), 430 | runerl(Nd,butil:ds_val(distname,Nd),Cmd,ErlEnv) 431 | end. 432 | 433 | write_global_cfgs() -> 434 | write_global_cfgs(butil:ds_val(nodes,etscfg),butil:ds_val(global_cfg,etscfg)). 435 | write_global_cfgs(Nodes,GlobCfgs) -> 436 | [begin 437 | filelib:ensure_dir([butil:ds_val(basepath,etscfg),"/",ndnm(Nd),"/etc"]), 438 | filelib:ensure_dir([butil:ds_val(basepath,etscfg),"/log"]), 439 | [begin 440 | FBin = render_cfg(G,[{basepath,butil:ds_val(basepath,etscfg)},{nodes,Nodes}]), 441 | Nm = [butil:ds_val(basepath,etscfg),"/",ndnm(Nd),"/etc/",filename:basename(dtlendnm(G))], 442 | filelib:ensure_dir(Nm), 443 | ok = file:write_file(Nm,FBin) 444 | end || G <- GlobCfgs] 445 | end || Nd <- Nodes]. 446 | 447 | write_per_node_cfgs() -> 448 | write_per_node_cfgs(butil:ds_val(nodes,etscfg),butil:ds_val(per_node_cfg,etscfg)). 449 | write_per_node_cfgs(Nodes,NodeCfgs) -> 450 | [begin 451 | [begin 452 | case lists:keyfind(ssh,1,Nd) of 453 | false -> 454 | BasePth = {basepath,butil:ds_val(basepath,etscfg)}; 455 | {ssh,_,_,BasePth1,_} -> 456 | BasePth = {basepath,BasePth1++"/"++butil:ds_val(basepath,etscfg)} 457 | end, 458 | FBin = render_cfg(NC,[BasePth|Nd]), 459 | Nm = [butil:ds_val(basepath,etscfg),"/",ndnm(Nd),"/etc/",filename:basename(dtlendnm(NC))], 460 | filelib:ensure_dir(Nm), 461 | ok = file:write_file(Nm,FBin) 462 | end || NC <- NodeCfgs] 463 | end || Nd <- Nodes]. 464 | 465 | wait_app(L,App,ScriptLoad) -> 466 | wait_app(L,App,ScriptLoad,os:timestamp(),butil:ds_val(app_wait_timeout,etscfg)). 467 | wait_app([H|T],App,ScriptLoad,Started,Timeout) -> 468 | Node = butil:ds_val(distname,H), 469 | case timer:now_diff(os:timestamp(),Started) > Timeout*1000 of 470 | true -> 471 | {error,H}; 472 | _ -> 473 | case rpc:call(Node,application,which_applications,[],1000) of 474 | [_|_] = L -> 475 | case App == undefined orelse lists:keymember(App,1,L) of 476 | true -> 477 | load_modules(Node,ScriptLoad), 478 | wait_app(T,App,ScriptLoad,Started,Timeout); 479 | false -> 480 | wait_app([H|T],App,ScriptLoad,Started,Timeout) 481 | end; 482 | _ -> 483 | wait_app([H|T],App,ScriptLoad,Started,Timeout) 484 | end 485 | end; 486 | wait_app([],_,_,_,_) -> 487 | ok. 488 | 489 | 490 | runerl(Nd,Name,Cmd,ErlEnv) when is_atom(Name) -> 491 | ?INF("Running ~s",[Cmd]), 492 | SshInfo = lists:keyfind(ssh,1,Nd), 493 | spawn(fun() -> 494 | register(Name,self()), 495 | case Cmd of 496 | [_|_] when SshInfo == false -> 497 | Port = open_port({spawn,Cmd},[exit_status,use_stdio,binary,stream,{env,ErlEnv}]), 498 | {os_pid,OsPid} = erlang:port_info(Port,os_pid); 499 | [_|_] -> 500 | {ssh,Host,SshPort,Cwd,Opts} = SshInfo, 501 | {ok,SshCon} = ssh:connect(Host,SshPort,[{silently_accept_hosts,true}|Opts]), 502 | {ok,SshChan} = ssh_connection:session_channel(SshCon,5000), 503 | ssh_connection:exec(SshCon, SshChan, "cd "++Cwd++" && "++Cmd, infinity), 504 | Port = SshCon, 505 | OsPid = undefined; 506 | _ -> 507 | Port = undefined, 508 | OsPid = undefined 509 | end, 510 | runerl(Port,OsPid) 511 | end). 512 | runerl(Port,OsPid) -> 513 | receive 514 | {Port,{exit_status,_Status}} -> 515 | ok; 516 | {Port,{data,Bin}} -> 517 | case butil:ds_val(verbose,etscfg) of 518 | true -> 519 | io:format("~s",[Bin]); 520 | _ -> 521 | ok 522 | end, 523 | runerl(Port,OsPid); 524 | {ssh_cm,Port,{data,_,_,Bin}} -> 525 | case butil:ds_val(verbose,etscfg) of 526 | true -> 527 | io:format("~s",[Bin]); 528 | _ -> 529 | ok 530 | end, 531 | runerl(Port,OsPid); 532 | stop when is_integer(OsPid) -> 533 | os:cmd("kill -KILL "++integer_to_list(OsPid)); 534 | stop -> 535 | ok 536 | end. 537 | 538 | wait_pids([undefined|T]) -> 539 | wait_pids(T); 540 | wait_pids([H|T]) -> 541 | case erlang:is_process_alive(H) of 542 | true -> 543 | timer:sleep(100), 544 | wait_pids([H|T]); 545 | false -> 546 | wait_pids(T) 547 | end; 548 | wait_pids([]) -> 549 | ok. 550 | 551 | connect(L) -> 552 | connect(L,os:timestamp(),butil:ds_val(connect_timeout,etscfg)). 553 | connect([H|T],Start,Timeout) -> 554 | case timer:now_diff(os:timestamp(),Start) > Timeout*1000 of 555 | true -> 556 | {error,H}; 557 | _ -> 558 | ?INF("Connecting to ~p",[ndnm(H)]), 559 | Node = butil:ds_val(distname,H), 560 | case net_kernel:hidden_connect_node(Node) of 561 | true -> 562 | case net_adm:ping(Node) of 563 | pang -> 564 | timer:sleep(100), 565 | connect([H|T],Start,Timeout); 566 | pong -> 567 | connect(T,os:timestamp(),Timeout) 568 | end; 569 | false -> 570 | timer:sleep(100), 571 | connect([H|T],Start,Timeout) 572 | end 573 | end; 574 | connect([],_,_) -> 575 | ok. 576 | 577 | % Every node also has test and this module loaded 578 | load_modules(Node,{Mod,Bin,Filename}) -> 579 | {module,Mod} = rpc:call(Node,code,load_binary,[Mod,Filename,Bin]), 580 | {_,ThisBin,ThisNm} = code:get_object_code(?MODULE), 581 | {module,?MODULE} = rpc:call(Node,code,load_binary,[?MODULE,ThisNm,ThisBin]). 582 | 583 | render_cfg(Cfg,P) -> 584 | case Cfg of 585 | {{FN,_},Param} -> 586 | ok; 587 | {FN,Param} -> 588 | ok; 589 | FN -> 590 | Param = [] 591 | end, 592 | case apply(modnm(FN),render,[P++Param]) of 593 | {ok,Bin} -> 594 | Bin; 595 | Err -> 596 | ?INF("Error rendering ~p~nParam:~p~nError:~p",[FN,P,Err]), 597 | stop({error,cfg_render,FN,Err}) 598 | end. 599 | 600 | modnm(FN) -> 601 | list_to_atom(filename:basename(FN)). 602 | 603 | ndnm([{_,_}|_] = N) -> 604 | ndnm(proplists:get_value(name,N,"")); 605 | ndnm(N) -> 606 | NS = atom_to_list(N), 607 | [NS1|_] = string:tokens(NS,"@"), 608 | NS1. 609 | 610 | distname([{_,_}|_] = Nd) -> 611 | Name = butil:ds_val(name,Nd), 612 | true = Name /= undefined, 613 | case lists:keyfind(ssh,1,Nd) of 614 | {ssh,Host,_Port,_Cwd,_Opts} -> 615 | butil:toatom(butil:tolist(Name)++"@"++butil:tolist(Host)); 616 | _ -> 617 | distname(butil:ds_val(name,Nd,"")) 618 | end; 619 | distname(N) -> 620 | case butil:ds_val(N,etscfg) of 621 | undefined -> 622 | HaveDamocles = butil:is_app_running(damocles), 623 | case os:type() of 624 | {unix,linux} when HaveDamocles -> 625 | IP = butil:to_ip(detest_net:find_free_ip()), 626 | damocles:add_interface(IP), 627 | Nm = list_to_atom(butil:tolist(N)++"@"++IP); 628 | _ -> 629 | Nm = list_to_atom(hd(string:tokens(butil:tolist(N),"@"))++"@127.0.0.1") 630 | end, 631 | butil:ds_add(N,Nm,etscfg), 632 | Nm; 633 | Nm -> 634 | Nm 635 | end. 636 | 637 | % ndaddr([{_,_}|_] = N) -> 638 | % ndaddr(proplists:get_value(name,N,"")); 639 | % ndaddr(N) -> 640 | % NS = atom_to_list(N), 641 | % [_,Addr] = string:tokens(NS,"@"), 642 | % Addr. 643 | 644 | dtlendnm(G) -> 645 | case G of 646 | {{_,GNm},_} -> 647 | GNm; 648 | {GNm,_} -> 649 | GNm; 650 | GNm -> 651 | GNm 652 | end. 653 | 654 | dtlnm(G) -> 655 | case G of 656 | {{GNm,_},_} -> 657 | GNm; 658 | {GNm,_} -> 659 | GNm; 660 | GNm -> 661 | GNm 662 | end. 663 | 664 | compile_cfgs(L) -> 665 | [begin 666 | case erlydtl:compile_file(dtlnm(Cfg), list_to_atom(filename:basename(dtlnm(Cfg))),[{out_dir,false},return,{auto_escape,false}]) of 667 | ok -> 668 | ok; 669 | {ok,_} -> 670 | ok; 671 | {ok,_,_} -> 672 | ok; 673 | Err -> 674 | ?INF("Error compiling ~p~n~p",[dtlnm(Cfg),Err]), 675 | stop({error,compile,Cfg,Err}) 676 | end 677 | end || Cfg <- L]. 678 | 679 | 680 | epmd_path() -> 681 | ErtsBinDir = filename:dirname(element(2,file:get_cwd())), 682 | Name = "epmd", 683 | case os:find_executable(Name, ErtsBinDir) of 684 | false -> 685 | case os:find_executable(Name) of 686 | false -> 687 | ?INF("Could not find epmd.~n"), 688 | stop({error,noepmd}); 689 | GlobalEpmd -> 690 | GlobalEpmd 691 | end; 692 | Epmd -> 693 | Epmd 694 | end. 695 | 696 | ez() -> 697 | ez("detest"). 698 | ez(Name) -> 699 | Ebin = filelib:wildcard("ebin/*"), 700 | Deps = filelib:wildcard("deps/*/ebin/*"), 701 | Files = 702 | [begin 703 | {ok,Bin} = file:read_file(Fn), 704 | {filename:basename(Fn),Bin} 705 | end || Fn <- Ebin++Deps], 706 | %["procket","procket.so"] 707 | {ok,{_,Bin}} = zip:create(Name++".ez",Files,[memory]), 708 | file:write_file(Name++".ez",Bin). 709 | --------------------------------------------------------------------------------