├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── rebar ├── rebar.config ├── rebar.config.script ├── src ├── examples │ ├── gen_gossip_aggregate.erl │ └── gen_gossip_epidemic.erl ├── gen_gossip.app.src ├── gen_gossip.erl └── gen_gossip.hrl └── test └── gen_gossip_test.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .eunit 3 | deps 4 | priv 5 | *.o 6 | *.beam 7 | *.plt 8 | ebin/*.app 9 | erl_crash.dump 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Richard Ramsden, Alex Arnell 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 15 | * The names of its contributors may not be used to endorse or promote 16 | products derived from this software without specific prior written 17 | permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REBAR := ./rebar 2 | 3 | all : compile 4 | 5 | compile : get-deps 6 | $(REBAR) compile 7 | 8 | get-deps : 9 | $(REBAR) get-deps 10 | 11 | clean : 12 | $(REBAR) clean 13 | 14 | test : REBAR := EGOSSIP_TEST=1 $(REBAR) 15 | test : clean compile 16 | $(REBAR) eunit skip_deps=true 17 | 18 | dist-clean : clean 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | gen_gossip 2 | ========== 3 | 4 | *gen_gossip* allows you to quickly implement gossip protocols in Erlang. 5 | 6 | Reason 7 | ====== 8 | 9 | This application was built to DRY up one of my existing projects. Having 10 | implemented several gossip protocols in the past I needed a way to re-use shared 11 | code common to many gossip based algorithms. 12 | 13 | Features 14 | ======== 15 | 16 | * Supports Aggregation-Based Protocols based on this [whitepaper](http://www.cs.unibo.it/bison/publications/aggregation-tocs.pdf) 17 | * Safe-guards for preventing your network from being flood with gossip messages 18 | * Ability to subscribe to node events: netsplits, joining, leaving, etc. 19 | 20 | Usage 21 | ===== 22 | 23 | See examples folder in src directory 24 | 25 | Status 26 | ====== 27 | 28 | This is still a young project if you see anything wrong please 29 | open an issue or send a pull request. 30 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rramsden/gen_gossip/f6bc695284d1ff9c53453560473873331aa62459/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {lib_dirs, ["deps"]}. 2 | {deps, []}. 3 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% vim: set ts=4 sw=4 ft=erlang et: 3 | 4 | ExtraDeps = [{meck, ".*", 5 | {git, "https://github.com/eproxus/meck.git", {tag, "cf476475b06"}}}], 6 | 7 | case os:getenv("EGOSSIP_TEST") of 8 | false -> 9 | CONFIG; 10 | _ -> 11 | case lists:keysearch(deps, 1, CONFIG) of 12 | {value, {deps, Deps}} -> 13 | NDeps = Deps ++ ExtraDeps, 14 | lists:keyreplace(deps, 1, CONFIG, {deps, NDeps}); 15 | false -> 16 | CONFIG ++ [{deps, ExtraDeps}] 17 | end 18 | end. 19 | -------------------------------------------------------------------------------- /src/examples/gen_gossip_aggregate.erl: -------------------------------------------------------------------------------- 1 | %% @doc 2 | %% Implements a simple aggregation-based protocol. This will 3 | %% calculate the sum for an entire cluster of nodes. 4 | %% round_length/2 defines how long it takes to converge on the answer. 5 | %% We calculate the sum of the cluster by taking the average of the value 6 | %% and multiplying it by the number of nodes in the conversation. 7 | %% 8 | %% Usage: 9 | %% 10 | %% (a@machine1)> gen_gossip_aggregate:start_link(25). 11 | %% (b@machine1)> gen_gossip_aggregate:start_link(25). 12 | %% (b@machine1)> net_adm:ping('a@machine1'). 13 | %% 14 | %% @end 15 | -module(gen_gossip_aggregate). 16 | -behaviour(gen_gossip). 17 | 18 | %% API 19 | -export([start_link/1]). 20 | 21 | %% gen_gossip callbacks 22 | -export([init/1, 23 | gossip_freq/1, 24 | round_finish/2, 25 | round_length/2, 26 | digest/1, 27 | join/2, 28 | expire/2, 29 | handle_gossip/4]). 30 | 31 | -record(state, { 32 | value = 0 33 | }). 34 | 35 | %%%=================================================================== 36 | %%% API 37 | %%%=================================================================== 38 | 39 | start_link(Number) -> 40 | gen_gossip:register_handler(?MODULE, [Number], aggregate). 41 | 42 | %%%=================================================================== 43 | %%% gen_gossip callbacks 44 | %%%=================================================================== 45 | 46 | init([Number]) -> 47 | {ok, #state{value=Number}}. 48 | 49 | % Defines how frequently we want to send a gossip message. 50 | % In milliseconds. 51 | gossip_freq(State) -> 52 | {reply, 1000, State}. 53 | 54 | % The total number of cycles needed to reach convergence. 55 | % Best to experiment and figure out how many cycles it takes 56 | % your algorithm to reach convergence then assign that number 57 | round_length(NodeCount, State) -> 58 | Length = ceil(math:log(NodeCount * NodeCount)) + 1, 59 | {reply, Length, State}. 60 | 61 | % Callback signifiying end of a round 62 | round_finish(NodeCount, State) -> 63 | io:format("=== end of round ===~n"), 64 | io:format(">>> SUM : ~p~n", [State#state.value * NodeCount]), 65 | {noreply, State}. 66 | 67 | % First message sent when talking to another node. 68 | digest(State) -> 69 | {reply, State#state.value, _HandleToken = push, State}. 70 | 71 | % Callback triggered when you join a cluster of nodes 72 | join(Nodelist, State) -> 73 | io:format("Joined cluster ~p~n", [Nodelist]), 74 | {noreply, State}. 75 | 76 | % Callback triggered when a node crashes 77 | expire(_Node, State) -> 78 | {noreply, State}. 79 | 80 | handle_gossip(push, Value, _From, State) -> 81 | io:format("got push~n"), 82 | NewValue = (Value + State#state.value) / 2, 83 | {reply, State#state.value, _HandleToken = pull, State#state{value=NewValue}}; 84 | 85 | handle_gossip(pull, Value, _From, State) -> 86 | io:format("got sym push~n"), 87 | NewValue = (Value + State#state.value) / 2, 88 | {noreply, State#state{value=NewValue}}. 89 | 90 | %%%=================================================================== 91 | %%% Internal Functions 92 | %%%=================================================================== 93 | 94 | ceil(X) -> 95 | T = erlang:trunc(X), 96 | case (X - T) of 97 | Neg when Neg < 0 -> T; 98 | Pos when Pos > 0 -> T + 1; 99 | _ -> T 100 | end. 101 | -------------------------------------------------------------------------------- /src/examples/gen_gossip_epidemic.erl: -------------------------------------------------------------------------------- 1 | %% Simple epidemic based protocol. Gossips an ever increasing epoch 2 | %% value around the cluster 3 | %% 4 | %% Usage: 5 | %% 6 | %% (a@machine1)> gen_gossip_epidemic:start_link(). 7 | %% (b@machine1)> gen_gossip_epidemic:start_link(). 8 | %% (b@machine1)> net_adm:ping('a@machine1'). 9 | %% 10 | -module(gen_gossip_epidemic). 11 | -behaviour(gen_gossip). 12 | 13 | %% api 14 | -export([start_link/0]). 15 | 16 | %% gen_gossip callbacks 17 | -export([init/1, 18 | gossip_freq/1, 19 | digest/1, 20 | join/2, 21 | expire/2, 22 | handle_gossip/4]). 23 | 24 | -record(state, { 25 | epoch = 0 26 | }). 27 | 28 | %%%=================================================================== 29 | %%% API 30 | %%%=================================================================== 31 | 32 | start_link() -> 33 | gen_gossip:register_handler(?MODULE, [], epidemic). 34 | 35 | %%%=================================================================== 36 | %%% gen_gossip callbacks 37 | %%%=================================================================== 38 | 39 | init([]) -> 40 | {ok, #state{}}. 41 | 42 | % how often do we want to send a message? in milliseconds. 43 | gossip_freq(State) -> 44 | {reply, 1000, State}. 45 | 46 | % defines what we're gossiping 47 | digest(#state{epoch=Epoch0} = State) -> 48 | Epoch1 = Epoch0 + 1, 49 | Value = State#state.epoch, 50 | HandleToken = push, 51 | io:format("Epoch = ~p~n", [Epoch1]), 52 | {reply, Value, HandleToken, State#state{epoch=Epoch1}}. 53 | 54 | % received a push 55 | handle_gossip(push, Epoch, _From, State) when Epoch >= State#state.epoch -> 56 | {noreply, State#state{epoch=Epoch}}; 57 | handle_gossip(push, _Epoch, _From, State) -> 58 | {reply, State#state.epoch, _HandleToken = pull, State}; 59 | 60 | % received a symmetric push 61 | handle_gossip(pull, Epoch, _From, State) -> 62 | {noreply, State#state{epoch=Epoch}}. 63 | 64 | % joined cluster 65 | join(_Nodelist, State) -> 66 | {noreply, State}. 67 | 68 | % node left 69 | expire(_Node, State) -> 70 | {noreply, State}. 71 | -------------------------------------------------------------------------------- /src/gen_gossip.app.src: -------------------------------------------------------------------------------- 1 | {application, gen_gossip, 2 | [ 3 | {description, ""}, 4 | {vsn, "1.0.0"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {env, []} 11 | ]}. 12 | -------------------------------------------------------------------------------- /src/gen_gossip.erl: -------------------------------------------------------------------------------- 1 | %% @doc 2 | %% Behaviour module for gen_gossip. gen_gossip must be implemented by 3 | %% the user. There's two gossiping modes: 4 | %% 5 | %% 1. Aggregation Protocols 6 | %% ------------------------ 7 | %% 8 | %% If you want to converge on a value over a period of time then you want to implement 9 | %% an aggregation protocol. These protocols will prevent nodes from joining a round in 10 | %% progress. They do this by keeping a version number of the current conversation. 11 | %% If two versions don't match then nodes will not gossip with eachother. Lower 12 | %% versioned nodes will wait for the next version to join in when the next round rolls over. 13 | %% 14 | %% 2. Epidemic Protocols 15 | %% --------------------- 16 | %% 17 | %% These don't have any kind of versioning. All nodes will always be able to 18 | %% gossip with eachother. 19 | %% 20 | %% Implementing a module 21 | %% --------------------- 22 | %% 23 | %% To use gen_gossip you will need a user module 24 | %% to implement it. You must define the following callbacks: 25 | %% 26 | %% init(Args) 27 | %% ==> {ok, State} 28 | %% 29 | %% gossip_freq(State) 30 | %% | Handles how frequently a gossip message is sent. Time is 31 | %% | in milliseconds 32 | %% ==> {reply, Tick :: Integer, State} 33 | %% 34 | %% join(Nodelist, State) 35 | %% | Notifies callback module when the CURRENT NODE joins another cluster 36 | %% ==> {noreply, State} 37 | %% 38 | %% expire(Node, State) 39 | %% | Notifies callback module when a node leaves the cluster 40 | %% ==> {noreply, State} 41 | %% 42 | %% digest(State) 43 | %% | Message you want to be gossiped around cluster 44 | %% ==> {reply, Reply, HandleToken State} 45 | %% 46 | %% handle_gossip(Token, Msg, From, State) 47 | %% | Called when we receive a gossip message from another node 48 | %% ==> {reply, Reply, HandleToken, State} | {noreply, State} 49 | %% 50 | %% You will need to implment the following if you're implementing an aggregation 51 | %% protocol: 52 | %% 53 | %% round_finish(NodeCount, State) 54 | %% | User module is notified when a round finishes, passing 55 | %% | the number of nodes that were in on the current conversation 56 | %% ==> {noreply, State} 57 | %% 58 | %% round_length(NodeCount, State) 59 | %% | This returns the number of cycles in each round needed 60 | %% | to trigger round_finish. 61 | %% ==> Integer 62 | %% 63 | %% Optionally, you can also implement gen_server callbacks: 64 | %% 65 | %% handle_info(Msg, State) 66 | %% handle_call(Msg, From, State) 67 | %% handle_cast(Msg, State) 68 | %% terminate(Reason, State) 69 | %% code_chnage(OldVsn, State, Extra) 70 | %% 71 | %% Gossip Communication 72 | %% -------------------- 73 | %% 74 | %% NODE A NODE B 75 | %% send digest -----------------> Module:handle_gossip/4 76 | %% Module:handle_gossip/4 <----------------- send pull 77 | %% send commit -----------------> Module:handle_commit/3 78 | %% 79 | %% @end 80 | -module(gen_gossip). 81 | -behaviour(gen_fsm). 82 | 83 | -include("gen_gossip.hrl"). 84 | 85 | %% API 86 | -export([register_handler/3, call/2, cast/2]). 87 | 88 | %% gen_server callbacks 89 | -export([init/1, 90 | gossiping/2, 91 | waiting/2, 92 | handle_info/3, 93 | handle_event/3, 94 | handle_sync_event/4, 95 | terminate/3, 96 | code_change/4]). 97 | 98 | -define(SERVER(Module), Module). 99 | -define(TRY(Code), (catch begin Code end)). 100 | 101 | -ifdef(TEST). 102 | -export([reconcile_nodes/4, send_gossip/4, node_name/0, nodelist/0]). 103 | -define(mockable(Fun), ?MODULE:Fun). 104 | -else. 105 | -define(mockable(Fun), Fun). 106 | -endif. 107 | 108 | %%%=================================================================== 109 | %%% API 110 | %%%=================================================================== 111 | 112 | -callback init(Args :: [any()]) -> 113 | {ok, module_state()}. 114 | -callback gossip_freq(State :: term()) -> 115 | {reply, Tick :: pos_integer(), NewState :: term()}. 116 | -callback digest(State :: term()) -> 117 | {reply, Reply :: term(), NewState :: term()}. 118 | -callback join(nodelist(), State :: term()) -> 119 | {noreply, NewState :: term()}. 120 | -callback expire(node(), NewState :: term()) -> 121 | {noreply, NewState :: term()}. 122 | -callback handle_gossip(HandleToken :: atom(), Msg :: term(), From :: node(), State :: term()) -> 123 | {reply, Reply :: term(), HandleToken :: atom(), NewState :: term()} | {noreply, NewState :: term()}. 124 | 125 | %% @doc 126 | %% Starts gen_gossip server with registered handler module 127 | %% @end 128 | -spec register_handler(module(), list(atom()), Mode :: atom()) -> {error, Reason :: atom()} | {ok, pid()}. 129 | 130 | register_handler(Module, Args, Mode) -> 131 | case lists:member(Mode, [aggregate, epidemic]) of 132 | true -> 133 | gen_fsm:start_link({local, ?SERVER(Module)}, ?MODULE, [Module, Args, Mode], []); 134 | false -> 135 | {error, invalid_mode} 136 | end. 137 | 138 | %% @doc 139 | %% Cals gen_fsm:sync_send_all_state_event/2 140 | %% @end 141 | -spec call(FsmRef :: pid(), Event :: term()) -> term(). 142 | 143 | call(FsmRef, Event) -> 144 | gen_fsm:sync_send_all_state_event(FsmRef, Event). 145 | 146 | %% @doc 147 | %% Calls gen_fsm:send_all_state_event/2 148 | %% @end 149 | -spec cast(FsmRef :: pid(), Event ::term()) -> term(). 150 | 151 | cast(FsmRef, Request) -> 152 | gen_fsm:send_all_state_event(FsmRef, Request). 153 | 154 | %%%=================================================================== 155 | %%% gen_server callbacks 156 | %%%=================================================================== 157 | 158 | init([Module, Args, Mode]) -> 159 | {ok, MState0} = Module:init(Args), 160 | 161 | State0 = #state{module=Module, mode=Mode, mstate=MState0}, 162 | {reply, Tick, MState1} = Module:gossip_freq(MState0), 163 | 164 | send_after(Tick, '$gen_gossip_tick'), 165 | 166 | {ok, gossiping, State0#state{mstate=MState1}}. 167 | 168 | %% @doc 169 | %% A node will transition into waiting state if there exists 170 | %% a higher epoch converstation happening. The node will then 171 | %% wait for (epoch + 1) to roll around to join in on the conversation 172 | %% @end 173 | 174 | -spec waiting({epoch(), message(), [node()]}, StateData :: term()) -> handle_event_ret(). 175 | 176 | waiting({R_Epoch, _, _} = Msg, #state{wait_for=R_Epoch} = State) -> 177 | gossiping(Msg, State#state{wait_for=undefined, max_wait=0, epoch=R_Epoch}); 178 | waiting({R_Epoch, _, _}, #state{wait_for=Epoch} = State0) 179 | when R_Epoch > Epoch -> 180 | % if there's a larger epoch around then wait for that one 181 | WaitFor = R_Epoch + 1, 182 | {next_state, waiting, State0#state{wait_for=WaitFor}}; 183 | waiting(_, State) -> 184 | {next_state, waiting, State}. 185 | 186 | %% @doc 187 | %% Nodes which have the same epoch and haven't been split into islands 188 | %% will be able to gossip in a conversation. Here's what happens 189 | %% in the other cases: 190 | %% 191 | %% 1. epochs and nodelists match 192 | %% - gossip 193 | %% 2. (epoch_remote > epoch_local) 194 | %% - use higher epoch, gossip 195 | %% 3. (epoch_remote > epoch_local) and nodelists don't match 196 | %% if intersection non-empty 197 | %% - merge lists, goto #2 198 | %% else 199 | %% - wait for (epoch_remote + 1) 200 | %% end 201 | %% 4. epochs match and nodelists mismatch 202 | %% - merge nodelists 203 | %% @end 204 | -spec gossiping({epoch(), message(), [node()]}, StateData :: term()) -> handle_event_ret(). 205 | 206 | % 1. 207 | gossiping({Epoch, {Token, Msg, From}, Nodelist}, 208 | #state{module=Module, epoch=Epoch, nodes=Nodelist} = State0) -> 209 | {ok, State1} = do_gossip(Module, Token, Msg, From, State0), 210 | {next_state, gossiping, State1}; 211 | 212 | % 2. 213 | gossiping({R_Epoch, {Token, Msg, From}, Nodelist}, 214 | #state{epoch=Epoch, module=Module, nodes=Nodelist} = State0) 215 | when R_Epoch > Epoch -> 216 | % This case handles when a node flips over the next epoch and contacts 217 | % another node that hasn't updated its epoch yet. This happens due to 218 | % clock-drift between nodes. You'll never have everything perfectly in-sync 219 | % unless your Google and have atomic GPS clocks... To keep up with the highest 220 | % epoch we simply set our epoch to the new value keeping things in-sync. 221 | {ok, State1} = set_round(R_Epoch, State0), 222 | {ok, State2} = do_gossip(Module, Token, Msg, From, State1), 223 | {next_state, gossiping, State2}; 224 | 225 | % 3. 226 | gossiping({R_Epoch, {Token, Msg, From}, R_Nodelist}, 227 | #state{epoch=Epoch, module=Module, mstate=MState0, nodes=Nodelist} = State0) 228 | when R_Epoch > Epoch, R_Nodelist =/= Nodelist -> 229 | % The intersection is taken to prevent nodes from waiting twice 230 | % to enter into the next epoch. This happens when islands 231 | % are trying to join. For example: 232 | % 233 | % Suppose we have two islands [a,b] and [c,d] 234 | % 'a' and 'b' are waiting, 'c' and 'd' both have a higher epoch 235 | % 'a' reconciles with [c,d] when epoch rolls around forming island [a,c,d] 236 | % 'a' then talks to 'b', since [a,b] =/= [a,c,d] 'b' its forced to wait again 237 | % 238 | case intersection(R_Nodelist, Nodelist) of 239 | [] -> 240 | % wait twice the amount of cycles for nodes to join each other. 241 | % we do this because if the node we're waiting on crashes 242 | % we could end up waiting forever. 243 | ClusterSize = length(union(Nodelist, R_Nodelist)), 244 | 245 | {reply, RoundLength, MState1} = Module:round_length(ClusterSize, MState0), 246 | {next_state, waiting, State0#state{max_wait = (RoundLength * 2), 247 | mstate=MState1, 248 | wait_for = (R_Epoch + 1)}}; 249 | _NonEmpty -> 250 | {ok, State1} = set_round(R_Epoch, State0), 251 | {MState1, NewNodes} = reconcile_nodes(Nodelist, R_Nodelist, From, State1), 252 | {ok, State2} = do_gossip(Module, Token, Msg, From, State1#state{mstate=MState1}), 253 | {next_state, gossiping, State2#state{nodes=NewNodes}} 254 | end; 255 | % 4. 256 | gossiping({Epoch, {Token, Msg, From}, R_Nodelist}, 257 | #state{module=Module, nodes=Nodelist, epoch=Epoch} = State0) 258 | when R_Nodelist =/= Nodelist -> 259 | {MState1, NewNodes} = reconcile_nodes(Nodelist, R_Nodelist, From, State0), 260 | {ok, State1} = do_gossip(Module, Token, Msg, From, State0#state{mstate=MState1}), 261 | {next_state, gossiping, State1#state{nodes=NewNodes}}; 262 | gossiping({_, _, _}, State) -> 263 | {next_state, gossiping, State}. 264 | 265 | handle_info('$gen_gossip_tick', StateName, #state{nodes=Nodelist0, max_wait=MaxWait, 266 | mstate=MState0, module=Module} = State0) -> 267 | % schedule another tick 268 | {reply, Tick, MState1} = Module:gossip_freq(MState0), 269 | send_after(Tick, '$gen_gossip_tick'), 270 | 271 | {Nodelist1, MState2} = expire_downed_nodes(Nodelist0, Module, MState1), 272 | State1 = State0#state{nodes=Nodelist1, mstate=MState2}, 273 | 274 | case StateName == gossiping of 275 | true -> 276 | {ok, State2} = gossip(State1), 277 | {ok, State3} = next_cycle(State2), 278 | {next_state, gossiping, State3}; 279 | false -> 280 | % prevent a node from waiting forever on a crashed node 281 | ExceededMaxWait = (MaxWait == 0), 282 | case ExceededMaxWait of 283 | true -> 284 | {next_state, gossiping, State1}; 285 | false -> 286 | {next_state, waiting, State1#state{max_wait=(MaxWait-1)}} 287 | end 288 | end; 289 | 290 | handle_info(Msg, StateName, #state{module=Module, mstate=MState0} = State) -> 291 | case erlang:function_exported(Module, handle_info, 2) of 292 | true -> 293 | Reply = Module:handle_info(Msg, MState0), 294 | handle_reply(Reply, StateName, State); 295 | false -> 296 | {next_state, StateName, State} 297 | end. 298 | 299 | handle_event(Event, StateName, #state{module=Module, mstate=MState0} = State) -> 300 | case erlang:function_exported(Module, handle_cast, 2) of 301 | true -> 302 | Reply = Module:handle_cast(Event, MState0), 303 | handle_reply(Reply, StateName, State); 304 | false -> 305 | {next_state, StateName, State} 306 | end. 307 | 308 | handle_sync_event(Event, From, StateName, #state{module=Module, mstate=MState0} = State) -> 309 | case erlang:function_exported(Module, handle_call, 3) of 310 | true -> 311 | Reply = Module:handle_call(Event, From, MState0), 312 | handle_reply(Reply, StateName, State); 313 | false -> 314 | {next_state, StateName, State} 315 | end. 316 | 317 | terminate(Reason, _StateName, #state{module=Module, mstate=MState0}) -> 318 | case erlang:function_exported(Module, terminate, 2) of 319 | true -> 320 | Module:terminate(Reason, MState0); 321 | false -> 322 | ok 323 | end. 324 | 325 | code_change(OldVsn, _StateName, #state{module=Module, mstate=MState} = State, Extra) -> 326 | case erlang:function_exported(Module, code_change, 3) of 327 | true -> 328 | case Module:code_change(OldVsn, MState, Extra) of 329 | {ok, NewState} -> 330 | {ok, State#state{mstate=NewState}}; 331 | Error -> Error 332 | end; 333 | false -> 334 | {ok, State} 335 | end. 336 | 337 | %%%=================================================================== 338 | %%% Internal functions 339 | %%%=================================================================== 340 | 341 | gossip(#state{mstate=MState0, module=Module} = State) -> 342 | case get_peer(visible) of 343 | none_available -> 344 | {ok, State}; 345 | {ok, Node} -> 346 | {reply, Digest, HandleToken, MState1} = Module:digest(MState0), 347 | ?mockable( send_gossip(Node, HandleToken, Digest, State#state{mstate=MState1}) ) 348 | end. 349 | 350 | expire_downed_nodes(Nodelist0, Module, MState0) -> 351 | Alive = ?mockable( nodelist() ), 352 | DownedNodes = subtract(Nodelist0, Alive), 353 | Nodelist1 = subtract(Nodelist0, DownedNodes), 354 | 355 | % expire nodes that have left the cluster 356 | MState1 = lists:foldl(fun(Node, Acc) -> 357 | {noreply, NewState} = Module:expire(Node, Acc), 358 | NewState 359 | end, MState0, DownedNodes), 360 | 361 | {Nodelist1, MState1}. 362 | 363 | handle_reply(Msg, StateName, State) -> 364 | case Msg of 365 | {reply, Reply, MState0} -> 366 | {reply, Reply, StateName, State#state{mstate=MState0}}; 367 | {reply, Reply, MState0, Extra} -> 368 | {reply, Reply, StateName, State#state{mstate=MState0}, Extra}; 369 | {noreply, MState0} -> 370 | {next_state, StateName, State#state{mstate=MState0}}; 371 | {noreply, MState0, Extra} -> 372 | {next_state, StateName, State#state{mstate=MState0}, Extra}; 373 | {stop, Reason, Reply, MState0} -> 374 | {stop, Reason, Reply, State#state{mstate=MState0}}; 375 | {stop, Reason, MState0} -> 376 | {stop, Reason, State#state{mstate=MState0}} 377 | end. 378 | 379 | do_gossip(Module, Token, Msg, From, #state{mstate=MState0} = State0) -> 380 | case Module:handle_gossip(Token, Msg, From, MState0) of 381 | {reply, Reply, HandleToken, MState1} -> 382 | ?mockable( send_gossip(From, HandleToken, Reply, State0#state{mstate=MState1}) ); 383 | {noreply, MState1} -> 384 | {ok, State0#state{mstate=MState1}} 385 | end. 386 | 387 | next_cycle(#state{mode=Mode} = State) when Mode =/= aggregate -> 388 | {ok, State}; 389 | next_cycle(#state{module=Module, mstate=MState0, cycle=Cycle, epoch=Epoch, nodes=Nodes} = State0) -> 390 | NextCycle = Cycle + 1, 391 | NodeCount = length(Nodes), 392 | 393 | {reply, RoundLength, MState1} = Module:round_length(NodeCount, MState0), 394 | 395 | case NextCycle > RoundLength of 396 | true -> 397 | set_round(Epoch + 1, State0#state{mstate=MState1}); 398 | false -> 399 | {ok, State0#state{cycle=NextCycle, mstate=MState1}} 400 | end. 401 | 402 | set_round(N, #state{module=Module, mstate=MState0} = State) -> 403 | NodeCount = length(State#state.nodes), 404 | {noreply, MState1} = Module:round_finish(NodeCount, MState0), 405 | {ok, State#state{epoch=N, cycle=0, mstate=MState1}}. 406 | 407 | %% @doc 408 | %% Figures out how we join islands together. This is important because 409 | %% this bit of code figures out which nodes should trigger Module:join 410 | %% callbacks. 411 | %% @end 412 | reconcile_nodes(Local, Remote, From, #state{mstate=MState0, module=Module}) -> 413 | NodeName = ?mockable( node_name() ), 414 | Alive = ?mockable( nodelist() ), 415 | 416 | A = Local, 417 | B = intersection(Alive, Remote), % prevent dead nodes from being re-added 418 | 419 | LenA = length(A), 420 | LenB = length(B), 421 | Intersection = intersection(A, B), 422 | 423 | if 424 | % if the intersection is one this means that the node in question has 425 | % Left our island to join another. If the intersection is greater than 426 | % or equal to 2 this means we are in the process of forming a larger island 427 | % so we can simply union the two islands together. 428 | length(Intersection) >= 2 -> 429 | {MState0, union(A, B)}; 430 | LenA == LenB -> 431 | TieBreaker = lists:sort(A) < lists:sort(B), 432 | case TieBreaker of 433 | true -> 434 | {MState0, union(A, [From])}; 435 | false -> 436 | {noreply, MState1} = Module:join(B, MState0), 437 | {MState1, union([NodeName], B)} 438 | end; 439 | LenA > LenB -> 440 | % if my island is bigger than the remotes i consume it 441 | {MState0, union(A, [From])}; 442 | LenA < LenB -> 443 | % my island is smaller, I have to leave it and join the remotes 444 | {noreply, MState1} = Module:join(B, MState0), 445 | {MState1, union([NodeName], B)} 446 | end. 447 | 448 | nodelist() -> 449 | [?mockable( node_name() ) | nodes()]. 450 | 451 | node_name() -> 452 | node(). 453 | 454 | get_peer(Opts) -> 455 | case nodes(Opts) of 456 | [] -> none_available; 457 | Nodes -> 458 | N = random:uniform(length(Nodes)), 459 | {ok, lists:nth(N, Nodes)} 460 | end. 461 | 462 | send_after(never, _Message) -> 463 | ok; 464 | send_after({Num,Sec}, Message) -> 465 | send_after(trunc(1000 / (Num/Sec)), Message); 466 | send_after(After, Message) -> 467 | erlang:send_after(After, self(), Message). 468 | 469 | send_gossip(ToNode, Token, Message, #state{module=Module, nodes=Nodelist} = State0) -> 470 | Payload = {State0#state.epoch, {Token, Message, node()}, Nodelist}, 471 | gen_fsm:send_event({?SERVER(Module), ToNode}, Payload), 472 | {ok, State0}. 473 | 474 | union(L1, L2) -> 475 | sets:to_list(sets:union(sets:from_list(L1), sets:from_list(L2))). 476 | 477 | intersection(A, B) -> 478 | sets:to_list(sets:intersection(sets:from_list(A), sets:from_list(B))). 479 | 480 | subtract(A, B) -> 481 | sets:to_list(sets:subtract(sets:from_list(A), sets:from_list(B))). 482 | -------------------------------------------------------------------------------- /src/gen_gossip.hrl: -------------------------------------------------------------------------------- 1 | -record(state, { 2 | nodes = [node()], % list of nodes and their respective epochs 3 | epoch = 0, 4 | wait_for :: integer(), % set to epoch we're waiting for 5 | max_wait = 0, 6 | cycle = 0, 7 | module :: module(), 8 | mstate :: term(), 9 | mode :: mode() 10 | }). 11 | 12 | -type mode() :: epidemic | aggregate. 13 | -type from() :: node(). 14 | -type token() :: atom(). 15 | -type epoch() :: integer(). 16 | -type message() :: {token(), term(), from()}. 17 | -type module_state() :: any(). 18 | -type nodelist() :: [node()]. 19 | 20 | -type handle_event_ret() :: {next_state, NextStateName :: atom(), NewStateData :: term()} | 21 | {next_state, NextStateName :: atom(), NewStateData :: term(), 22 | timeout() | hibernate} | 23 | {stop, Reason :: term(), NewStateData :: term()}. 24 | -------------------------------------------------------------------------------- /test/gen_gossip_test.erl: -------------------------------------------------------------------------------- 1 | -module(gen_gossip_test). 2 | -include_lib("eunit/include/eunit.hrl"). 3 | 4 | -include("src/gen_gossip.hrl"). 5 | 6 | 7 | app_test_() -> 8 | {foreach, 9 | fun setup/0, 10 | fun cleanup/1, 11 | [ 12 | fun reconcile_equal_win_tiebreaker_/1, 13 | fun reconcile_equal_lost_tiebreaker_/1, 14 | fun reconcile_smaller_with_bigger_/1, 15 | fun reconcile_larger_with_smaller_/1, 16 | fun reconcile_intersection_with_1_/1, 17 | fun reconcile_intersection_with_2_/1, 18 | fun reconcile_removes_downed_nodes_/1, 19 | fun prevent_forever_wait_/1, 20 | fun transition_wait_to_gossip_state_/1, 21 | fun transition_gossip_to_wait_state_/1, 22 | fun gossips_if_nodelist_and_epoch_match_/1, 23 | fun use_latest_epoch_if_nodelist_match_/1, 24 | fun remove_downed_node_/1, 25 | fun dont_increment_cycle_in_wait_state_/1, 26 | fun dont_increment_cycle_for_other_modes_/1, 27 | fun dont_gossip_in_wait_state_/1, 28 | fun dont_wait_forever_/1, 29 | fun proxies_out_of_band_messages_to_callback_module_/1 30 | ]}. 31 | 32 | setup() -> 33 | meck:new(gen_gossip, [passthrough]), 34 | meck:expect(gen_gossip, send_gossip, fun(_, _, _, State) -> {ok, State} end), 35 | meck:expect(gen_gossip, node_name, 0, a), 36 | 37 | Module = gossip_test, 38 | meck:new(Module), 39 | meck:expect(Module, init, 1, {ok, state}), 40 | meck:expect(Module, gossip_freq, 1, {reply, 1000, state}), 41 | meck:expect(Module, round_finish, 2, {noreply, state}), 42 | meck:expect(Module, round_length, 2, {reply, 10, state}), 43 | meck:expect(Module, digest, 1, {reply, digest, push, state}), 44 | meck:expect(Module, handle_gossip, 4, {reply, digest, pull, state}), 45 | meck:expect(Module, handle_info, 2, {noreply, state}), 46 | meck:expect(Module, handle_call, 3, {reply, ok, state}), 47 | meck:expect(Module, handle_cast, 2, {noreply, state}), 48 | meck:expect(Module, code_change, 3, {ok, state}), 49 | meck:expect(Module, terminate, 2, ok), 50 | meck:expect(Module, join, 2, {noreply, state}), 51 | meck:expect(Module, expire, 2, {noreply, state}), 52 | Module. 53 | 54 | cleanup(Module) -> 55 | meck:unload(gen_gossip), 56 | meck:unload(Module). 57 | 58 | called(Mod, Fun) -> 59 | History = meck:history(Mod), 60 | List = [match || {_, {M, F, _, _}} <- History, M == Mod, F == Fun], 61 | List =/= []. 62 | 63 | dont_gossip_in_wait_state_(Module) -> 64 | fun() -> 65 | State0 = #state{module=Module}, 66 | 67 | {next_state, gossiping, State1} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State0), 68 | ?assert( not meck:called(gen_gossip, send_gossip, [from, pull, digest, State1]) ) 69 | end. 70 | 71 | reconcile_equal_win_tiebreaker_(Module) -> 72 | fun() -> 73 | State = #state{module=Module, mstate=state}, 74 | MyNode = a, 75 | MyList = [a,b], 76 | RemoteNode = c, 77 | RemoteList = [c,d], 78 | 79 | Expected = [a, b, c], 80 | 81 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d]), 82 | meck:expect(gen_gossip, node_name, 0, MyNode), 83 | 84 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 85 | ?assertEqual(Expected, Nodelist), 86 | ?assert(not called( Module, join )) 87 | end. 88 | 89 | reconcile_equal_lost_tiebreaker_(Module) -> 90 | fun() -> 91 | State = #state{module=Module, mstate=state}, 92 | MyNode = c, 93 | MyList = [c,d], 94 | RemoteNode = c, 95 | RemoteList = [a,b], 96 | 97 | Expected = [a, b, c], 98 | 99 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d]), 100 | meck:expect(gen_gossip, node_name, 0, MyNode), 101 | 102 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 103 | ?assertEqual(Expected, Nodelist), 104 | ?assert( meck:called(Module, join, [ [a,b], state ]) ) 105 | end. 106 | 107 | reconcile_smaller_with_bigger_(Module) -> 108 | fun() -> 109 | State = #state{module=Module, mstate=state}, 110 | MyNode = a, 111 | MyList = [a,b,c], 112 | RemoteNode = c, 113 | RemoteList = [d,e,f,g], 114 | 115 | Expected = [a,d,e,f,g], 116 | 117 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d,e,f,g]), 118 | meck:expect(gen_gossip, node_name, 0, MyNode), 119 | 120 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 121 | ?assertEqual(Expected, Nodelist), 122 | ?assert( meck:called(Module, join, [ [d,e,f,g], state ]) ) 123 | end. 124 | 125 | reconcile_larger_with_smaller_(Module) -> 126 | fun() -> 127 | State = #state{module=Module, mstate=state}, 128 | MyNode = a, 129 | MyList = [a,b,c], 130 | RemoteNode = d, 131 | RemoteList = [d,e], 132 | 133 | Expected = [a,b,c,d], 134 | 135 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d,e]), 136 | meck:expect(gen_gossip, node_name, 0, MyNode), 137 | 138 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 139 | ?assertEqual(Expected, Nodelist), 140 | ?assert(not called( Module, join )) 141 | end. 142 | 143 | reconcile_intersection_with_1_(Module) -> 144 | fun() -> 145 | % will still trigger a join because intersection has one item in it 146 | State = #state{module=Module, mstate=state}, 147 | MyNode = b, 148 | MyList = [a,b,c], 149 | RemoteNode = d, 150 | RemoteList = [a,d,e,f], 151 | 152 | Expected = [a,b,d,e,f], 153 | 154 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d,e,f]), 155 | meck:expect(gen_gossip, node_name, 0, MyNode), 156 | 157 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 158 | ?assertEqual(Expected, Nodelist), 159 | ?assert( meck:called(Module, join, [ [a,d,e,f], state ]) ) 160 | end. 161 | 162 | reconcile_intersection_with_2_(Module) -> 163 | fun() -> 164 | % will still trigger a join because intersection has one item in it 165 | State = #state{module=Module, mstate=state}, 166 | MyNode = b, 167 | MyList = [a,b], 168 | RemoteNode = d, 169 | RemoteList = [a,b,c], 170 | 171 | Expected = [a,b,c], 172 | 173 | meck:expect(gen_gossip, nodelist, 0, [a,b,c,d]), 174 | meck:expect(gen_gossip, node_name, 0, MyNode), 175 | 176 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 177 | ?assertEqual(Expected, Nodelist), 178 | ?assert( not called(Module, join) ) 179 | end. 180 | 181 | reconcile_removes_downed_nodes_(Module) -> 182 | % Using the erlang nodes() call we can see which nodes are alive in the cluster. 183 | % Inside of reconcile_nodes/4 we strip out dead nodes from the remotes gnodelist 184 | % to ensure that they won't be re-added back into our local gnodelist 185 | fun() -> 186 | State = #state{module=Module, mstate=state}, 187 | MyNode = b, 188 | MyList = [a,b], 189 | RemoteNode = d, 190 | RemoteList = [c,d], 191 | 192 | Expected = [a,b,d], 193 | 194 | meck:expect(gen_gossip, nodelist, 0, [a,b,d]), % node 'c' is dead 195 | meck:expect(gen_gossip, node_name, 0, MyNode), 196 | 197 | {_, Nodelist} = gen_gossip:reconcile_nodes(MyList, RemoteList, RemoteNode, State), 198 | ?assertEqual(Expected, Nodelist) 199 | end. 200 | 201 | prevent_forever_wait_(Module) -> 202 | % by some freak chance if we were waiting for an epoch to roll around 203 | % that never occurred because a higher epoch appeared then we should 204 | % wait for the next highest to occur to prevent waiting forever 205 | fun() -> 206 | R_Epoch = 2, 207 | R_Nodelist = [b], 208 | WaitFor = 1, 209 | Nodelist = [a], 210 | 211 | meck:expect(gen_gossip, nodelist, 0, [a,b]), 212 | 213 | State0 = #state{module=Module, wait_for=WaitFor, nodes=Nodelist}, 214 | Send = {R_Epoch, {push, msg, from}, R_Nodelist}, 215 | 216 | {next_state, waiting, State1} = gen_gossip:waiting(Send, State0), 217 | 218 | ?assertEqual(State1#state.wait_for, R_Epoch + 1) 219 | end. 220 | 221 | transition_wait_to_gossip_state_(Module) -> 222 | % to transition from waiting -> gossiping the epoch 223 | % specified in #state.wait_for must equal the callers epoch 224 | fun() -> 225 | R_Epoch = 1, 226 | R_Nodelist = [b], 227 | Epoch = 1, 228 | Nodelist = [a], 229 | 230 | meck:expect(gen_gossip, nodelist, 0, [a,b]), 231 | 232 | State0 = #state{module=Module, wait_for=Epoch, nodes=Nodelist}, 233 | Msg = {R_Epoch, {push, msg, from}, R_Nodelist}, 234 | 235 | {next_state, gossiping, _} = gen_gossip:waiting(Msg, State0) 236 | end. 237 | 238 | transition_gossip_to_wait_state_(Module) -> 239 | fun() -> 240 | R_Epoch = 2, 241 | R_Nodelist = [b], 242 | Epoch = 1, 243 | Nodelist = [a], 244 | 245 | meck:expect(gen_gossip, nodelist, 0, [a,b]), 246 | 247 | State0 = #state{module=Module, nodes=Nodelist, epoch=Epoch}, 248 | Msg = {R_Epoch, {push, msg, from}, R_Nodelist}, 249 | 250 | {next_state, waiting, _} = gen_gossip:gossiping(Msg, State0) 251 | end. 252 | 253 | gossips_if_nodelist_and_epoch_match_(Module) -> 254 | fun() -> 255 | R_Epoch = 1, 256 | R_Nodelist = [a,b], 257 | Epoch = 1, 258 | Nodelist = [a,b], 259 | 260 | meck:expect(gen_gossip, nodelist, 0, [a,b]), 261 | 262 | State0 = #state{mstate=state, module=Module, nodes=Nodelist, epoch=Epoch}, 263 | Msg = {R_Epoch, {push, msg, from}, R_Nodelist}, 264 | 265 | {next_state, gossiping, _} = gen_gossip:gossiping(Msg, State0), 266 | 267 | % some data was pushed to the module, so it should reply with a pull 268 | ?assert( meck:called(Module, handle_gossip, [ push, msg, from, state ]) ), 269 | ?assert( meck:called(gen_gossip, send_gossip, [from, pull, digest, State0]) ) 270 | end. 271 | 272 | use_latest_epoch_if_nodelist_match_(Module) -> 273 | % since there is clock-drift, ticks will never truly be in sync. 274 | % this causes other nodes to switch to the next epoch before another. 275 | % to do our best at synchrnoization we always use the latest epoch. 276 | fun() -> 277 | R_Epoch = 10, 278 | R_Nodelist = [a,b], 279 | Epoch = 1, 280 | Nodelist = [a,b], 281 | 282 | meck:expect(gen_gossip, nodelist, 0, [a,b]), 283 | 284 | State0 = #state{module=Module, nodes=Nodelist, epoch=Epoch}, 285 | Send = {R_Epoch, {push, msg, from}, R_Nodelist}, 286 | 287 | {next_state, gossiping, State1} = gen_gossip:gossiping(Send, State0), 288 | 289 | % should also send a gossip message back 290 | ?assert( meck:called(gen_gossip, send_gossip, [from, pull, digest, State1]) ), 291 | ?assertEqual(State1#state.epoch, R_Epoch) 292 | end. 293 | 294 | remove_downed_node_(Module) -> 295 | fun() -> 296 | Nodelist = [a,b,c], 297 | Expected = [a,b], % node 'c' left cluster 298 | 299 | meck:expect(gen_gossip, nodelist, 0, Expected), 300 | 301 | State0 = #state{cycle=0, module=Module, nodes=Nodelist, epoch=0}, 302 | 303 | {next_state, _, #state{nodes=Result}} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State0), 304 | 305 | ?assertEqual(Expected, Result) 306 | end. 307 | 308 | dont_increment_cycle_in_wait_state_(Module) -> 309 | fun() -> 310 | Nodelist = [a,b,c], 311 | Epoch = 1, 312 | 313 | State0 = #state{mode=aggregate, cycle=0, max_wait=1, module=Module, nodes=Nodelist, epoch=Epoch}, 314 | 315 | meck:expect(gen_gossip, nodelist, 0, [a,b,c]), 316 | 317 | {next_state, waiting, State1} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State0), 318 | 319 | % just making sure cycle is being incremented in gossip state 320 | {next_state, gossiping, State2} = gen_gossip:handle_info('$gen_gossip_tick', gossiping, State0#state{max_wait=0}), 321 | 322 | ?assertEqual(0, State1#state.cycle), 323 | ?assertEqual(1, State2#state.cycle) 324 | end. 325 | 326 | dont_increment_cycle_for_other_modes_(Module) -> 327 | % should only increment the cycle and change rounds when 328 | % were in aggregate mode 329 | fun() -> 330 | Nodelist = [a,b,c], 331 | Epoch = 1, 332 | 333 | State0 = #state{mode=epidemic, cycle=0, max_wait=0, module=Module, nodes=Nodelist, epoch=Epoch}, 334 | 335 | meck:expect(gen_gossip, nodelist, 0, [a,b,c]), 336 | 337 | {next_state, gossiping, State1} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State0), 338 | 339 | % just making sure cycle is being incremented in gossip state 340 | {next_state, gossiping, State2} = gen_gossip:handle_info('$gen_gossip_tick', gossiping, State0), 341 | 342 | ?assertEqual(0, State1#state.cycle), 343 | ?assertEqual(0, State2#state.cycle) 344 | end. 345 | 346 | dont_wait_forever_(Module) -> 347 | fun() -> 348 | MaxWait = 2, 349 | Nodelist = [a,b,c], 350 | Epoch = 1, 351 | 352 | meck:expect(gen_gossip, nodelist, 0, [a,b,c]), 353 | 354 | State0 = #state{cycle=0, max_wait=MaxWait, module=Module, nodes=Nodelist, epoch=Epoch}, 355 | 356 | {next_state, waiting, State1} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State0), 357 | {next_state, waiting, State2} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State1), 358 | {next_state, gossiping, _} = gen_gossip:handle_info('$gen_gossip_tick', waiting, State2) 359 | end. 360 | 361 | proxies_out_of_band_messages_to_callback_module_(Module) -> 362 | fun() -> 363 | State0 = #state{module=Module, mstate=state}, 364 | 365 | {next_state, gossiping, _} = gen_gossip:handle_info(out_of_band, gossiping, State0), 366 | ?assert( meck:called( Module, handle_info, [out_of_band, state] ) ), 367 | 368 | {next_state, gossiping, _} = gen_gossip:handle_event(out_of_band, gossiping, State0), 369 | ?assert( meck:called( Module, handle_cast, [out_of_band, state] ) ), 370 | 371 | {reply, ok, gossiping, _} = gen_gossip:handle_sync_event(out_of_band, from, gossiping, State0), 372 | ?assert( meck:called( Module, handle_call, [out_of_band, from, state] ) ), 373 | 374 | ok = gen_gossip:terminate(shutdown, gossiping, State0), 375 | ?assert( meck:called( Module, terminate, [shutdown, state] ) ), 376 | 377 | {ok, State0} = gen_gossip:code_change(1, gossiping, State0, []), 378 | ?assert( meck:called( Module, code_change, [1, state, []]) ) 379 | end. 380 | --------------------------------------------------------------------------------