├── README.markdown ├── combined_version └── gen_leader.erl ├── hanssv+serge_version └── gen_leader.erl ├── opencsm_version ├── CHANGELOG ├── CHANGES ├── gen_leader.erl └── source-to-current.diff ├── original_version └── gen_leader.erl └── powerset_version ├── CHANGELOG ├── gen_leader.erl └── source-to-current.diff /README.markdown: -------------------------------------------------------------------------------- 1 | Mission 2 | ======= 3 | 4 | This is a project to revive and modernize the [gen_leader library](http://www.cs.chalmers.se/~hanssv/leader_election/doc/gen_leader.html). 5 | Numerous versions of this project exist, developed by disparate groups with different aims. By collecting and integrating these we hope 6 | to provide a new standard-library-quality module for the Erlang runtime that provides leader-election functionality without many of the 7 | difficulties traditionally associated with such. 8 | 9 | Which Version Should I Use? 10 | =========================== 11 | 12 | *You should not use this version!* I helped organize the glr over a year ago, but I am currently busy with other projects. A much better place to look for updated & integrated versions of this code are at [abecciu's repository](https://github.com/abecciu/gen_leader_revival). Go there. 13 | 14 | What Exactly Does It Do? 15 | ======================== 16 | 17 | Let us get back to you on that. 18 | 19 | Current Participants 20 | ==================== 21 | 22 | + Andrew Thompson (andrew@hijacked.us) 23 | + Dave Fayram (dfayram@gmail.com) 24 | + Hans Svensson (hanssv@chalmers.se) 25 | + Ulf Wiger (ulf@wiger.net) 26 | -------------------------------------------------------------------------------- /opencsm_version/CHANGES: -------------------------------------------------------------------------------- 1 | * Don't hang when only one candidate is specified 2 | * Implemented handle_DOWN to match docs 3 | * Added gen_leader_cast 4 | * Allowed multiple gen_leader instances per node by prefixing the module name to 5 | the incarnation filename 6 | * Add some documentation where the code was unclear 7 | * Runtime added worker support (uses code_change) 8 | * Fix obsolete guards (atom() -> is_atom()) 9 | * R13B fix 10 | -------------------------------------------------------------------------------- /opencsm_version/gen_leader.erl: -------------------------------------------------------------------------------- 1 | %% ``The contents of this file are subject to the Erlang Public License, 2 | %% Version 1.1, (the "License"); you may not use this file except in 3 | %% compliance with the License. You should have received a copy of the 4 | %% Erlang Public License along with this software. If not, it can be 5 | %% retrieved via the world wide web at http://www.erlang.org/. 6 | %% 7 | %% Software distributed under the License is distributed on an "AS IS" 8 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 | %% the License for the specific language governing rights and limitations 10 | %% under the License. 11 | %% 12 | %% The Initial Developer of the Original Code is Ericsson Utvecklings AB. 13 | %% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings 14 | %% AB. All Rights Reserved.'' 15 | %% 16 | %% 17 | %% $Id: gen_leader.erl,v 1.2 2005/07/04 06:55:56 hanssv Exp $ 18 | %% 19 | %% @author Hans Svensson 20 | %% @author Thomas Arts 21 | %% @author Ulf Wiger 22 | %% 23 | %% @doc Leader election behavior. 24 | %%

This application implements a leader election behavior modeled after 25 | %% gen_server. This behavior intends to make it reasonably 26 | %% straightforward to implement a fully distributed server with 27 | %% master-slave semantics.

28 | %%

The gen_leader behavior supports nearly everything that gen_server 29 | %% does (some functions, such as multicall() and the internal timeout, 30 | %% have been removed), and adds a few callbacks and API functions to 31 | %% support leader election etc.

32 | %%

Also included is an example program, a global dictionary, based 33 | %% on the modules gen_leader and dict. The callback implementing the 34 | %% global dictionary is called 'test_cb', for no particularly logical 35 | %% reason.

36 | %%

New version: The internal leader election algorithm was faulty 37 | %% and has been replaced with a new version based on a different leader 38 | %% election algorithm. As a consequence of this the query functions 39 | %% alive and down can no longer be provided. 40 | %% The new algorithm also make use of an incarnation parameter, by 41 | %% default written to disk in the function incarnation. This 42 | %% implies that only one gen_leader per node is permitted, if 43 | %% used in a diskless environment, incarnation must be adapted. 44 | %%

45 | %% @end 46 | %% 47 | %% @type election() = tuple(). Opaque state of the gen_leader behaviour. 48 | %% @type node() = atom(). A node name. 49 | %% @type name() = atom(). A locally registered name. 50 | %% @type serverRef() = Name | {name(),node()} | {global,Name} | pid(). 51 | %% See gen_server. 52 | %% @type callerRef() = {pid(), reference()}. See gen_server. 53 | %% 54 | 55 | %% This version debugged/modified by Andrew Thompson (andrew@hijacked.us) 56 | %% for SpiceCSM 57 | -module(gen_leader). 58 | 59 | % Time between rounds of query from the leader 60 | -define(TAU,250). 61 | 62 | % Exports for quickcheck 63 | %-export([safe_loop/4,loop/4]). 64 | 65 | -export([start/6, 66 | start_link/6, 67 | leader_call/2, leader_call/3, leader_cast/2, 68 | call/2, call/3, cast/2, 69 | reply/2]). 70 | 71 | %% Query functions 72 | -export([%% alive/1, 73 | %% down/1, 74 | candidates/1, 75 | workers/1]). 76 | 77 | -export([ 78 | system_continue/3, 79 | system_terminate/4, 80 | system_code_change/4, 81 | format_status/2 82 | ]). 83 | 84 | -export([behaviour_info/1]). 85 | 86 | %% Internal exports 87 | -export([init_it/6, print_event/3 88 | %%, safe_send/2 89 | ]). 90 | 91 | -import(error_logger , [format/2]). 92 | -import(lists, [foldl/3, 93 | foreach/2, 94 | member/2, 95 | keydelete/3, 96 | keysearch/3]). 97 | 98 | % Include for QuickCheck 99 | % -include("eqc.hrl"). 100 | 101 | -record(election,{leader = none, 102 | name, 103 | leadernode = none, 104 | candidate_nodes = [], 105 | worker_nodes = [], 106 | alive = [], 107 | down = [], 108 | monitored = [], 109 | buffered = [], 110 | status, 111 | elid, 112 | acks = [], 113 | work_down = [], 114 | pendack, 115 | incarn, 116 | nextel 117 | }). 118 | 119 | -record(server, {parent, 120 | mod, 121 | state, 122 | debug}). 123 | 124 | 125 | %%% --------------------------------------------------- 126 | %%% Interface functions. 127 | %%% --------------------------------------------------- 128 | 129 | %% @hidden 130 | behaviour_info(callbacks) -> 131 | [{init,1}, 132 | {elected,2}, 133 | {surrendered,3}, 134 | {handle_leader_call,4}, 135 | {handle_leader_cast,3}, 136 | {from_leader,3}, 137 | {handle_call,3}, 138 | {handle_cast,2}, 139 | {handle_DOWN,3}, 140 | {handle_info,2}, 141 | {terminate,2}, 142 | {code_change,4}]; 143 | behaviour_info(_Other) -> 144 | undefined. 145 | 146 | %% @spec start(Name::node(), CandidateNodes::[node()], 147 | %% Workers::[node()], Mod::atom(), Arg, Options::list()) -> 148 | %% {ok,pid()} 149 | %% 150 | %% @doc Starts a gen_leader process without linking to the parent. 151 | %% 152 | % ADT - don't allow an empty candidate list 153 | start(_Name, [], _Workers, _Mod, _Arg, _Options) -> 154 | {error, nocandidates}; 155 | start(Name, CandidateNodes, Workers, Mod, Arg, Options) when is_atom(Name) -> 156 | gen:start(?MODULE, nolink, {local,Name}, 157 | Mod, {CandidateNodes, Workers, Arg}, Options). 158 | 159 | %% @spec start_link(Name::atom(), CandidateNodes::[atom()], 160 | %% Workers::[atom()], Mod::atom(), Arg, Options::list()) -> 161 | %% {ok, pid()} 162 | %% 163 | %% @doc Starts a gen_leader process. 164 | %% 165 | %% 166 | %% 168 | %% 169 | %% 171 | %% 172 | %% 173 | %% 174 | %%
NameThe locally registered name of the process
CandidateNodesThe names of nodes capable of assuming 167 | %% a leadership role
WorkersThe names of nodes that will be part of the "cluster", 170 | %% but cannot ever assume a leadership role.
ModThe name of the callback module
ArgArgument passed on to Mod:init/1
OptionsSame as gen_server's Options
175 | %% 176 | %%

The list of candidates needs to be known from the start. Workers 177 | %% can be added at runtime.

178 | %% @end 179 | % ADT - don't allow an empty candidate list 180 | start_link(_Name, [], _Workers, _Mod, _Arg, _Options) -> 181 | {error, nocandidates}; 182 | start_link(Name, CandidateNodes, Workers, 183 | Mod, Arg, Options) when is_atom(Name) -> 184 | % Random delay for QuickCheck 185 | % timer:sleep(random:uniform(400)), 186 | gen:start(?MODULE, link, {local,Name}, Mod, 187 | {CandidateNodes, Workers, Arg}, Options). 188 | 189 | 190 | %% Query functions to be used from the callback module 191 | 192 | %% alive(#election{alive = Alive}) -> 193 | %% Alive. 194 | 195 | %% down(#election{down = Down}) -> 196 | %% Down. 197 | 198 | %% @spec candidates(E::election()) -> [node()] 199 | %% 200 | %% @doc Returns a list of known candidates. 201 | %% 202 | candidates(#election{candidate_nodes = Cands}) -> 203 | Cands. 204 | 205 | %% @spec workers(E::election()) -> [node()] 206 | %% 207 | %% @doc Returns a list of known workers. 208 | %% 209 | workers(#election{worker_nodes = Workers}) -> 210 | Workers. 211 | 212 | % 213 | % Make a call to a generic server. 214 | % If the server is located at another node, that node will 215 | % be monitored. 216 | % If the client is trapping exits and is linked server termination 217 | % is handled here (? Shall we do that here (or rely on timeouts) ?). 218 | % 219 | %% @spec call(Name::serverRef(), Request) -> term() 220 | %% 221 | %% @doc Equivalent to gen_server:call/2, but with a slightly 222 | %% different exit reason if something goes wrong. This function calls 223 | %% the gen_leader process exactly as if it were a gen_server 224 | %% (which, for practical purposes, it is.) 225 | %% @end 226 | call(Name, Request) -> 227 | case catch gen:call(Name, '$gen_call', Request) of 228 | {ok,Res} -> 229 | Res; 230 | {'EXIT',Reason} -> 231 | exit({Reason, {?MODULE, local_call, [Name, Request]}}) 232 | end. 233 | 234 | %% @spec call(Name::serverRef(), Request, Timeout::integer()) -> 235 | %% Reply 236 | %% 237 | %% Reply = term() 238 | %% 239 | %% @doc Equivalent to gen_server:call/3, but with a slightly 240 | %% different exit reason if something goes wrong. This function calls 241 | %% the gen_leader process exactly as if it were a gen_server 242 | %% (which, for practical purposes, it is.) 243 | %% @end 244 | call(Name, Request, Timeout) -> 245 | case catch gen:call(Name, '$gen_call', Request, Timeout) of 246 | {ok,Res} -> 247 | Res; 248 | {'EXIT',Reason} -> 249 | exit({Reason, {?MODULE, local_call, [Name, Request, Timeout]}}) 250 | end. 251 | 252 | %% @spec leader_call(Name::name(), Request::term()) 253 | %% -> Reply 254 | %% 255 | %% Reply = term() 256 | %% 257 | %% @doc Makes a call (similar to gen_server:call/2) to the 258 | %% leader. The call is forwarded via the local gen_leader instance, if 259 | %% that one isn't actually the leader. The client will exit if the 260 | %% leader dies while the request is outstanding. 261 | %%

This function uses gen:call/3, and is subject to the 262 | %% same default timeout as e.g. gen_server:call/2.

263 | %% @end 264 | %% 265 | leader_call(Name, Request) -> 266 | case catch gen:call(Name, '$leader_call', Request) of 267 | {ok,{leader,reply,Res}} -> 268 | Res; 269 | {ok,{error, leader_died}} -> 270 | exit({leader_died, {?MODULE, leader_call, [Name, Request]}}); 271 | {'EXIT',Reason} -> 272 | exit({Reason, {?MODULE, leader_call, [Name, Request]}}) 273 | end. 274 | 275 | %% @spec leader_call(Name::name(), Request::term(), Timeout::integer()) 276 | %% -> Reply 277 | %% 278 | %% Reply = term() 279 | %% 280 | %% @doc Makes a call (similar to gen_server:call/3) to the 281 | %% leader. The call is forwarded via the local gen_leader instance, if 282 | %% that one isn't actually the leader. The client will exit if the 283 | %% leader dies while the request is outstanding. 284 | %% @end 285 | %% 286 | leader_call(Name, Request, Timeout) -> 287 | case catch gen:call(Name, '$leader_call', Request, Timeout) of 288 | {ok,{leader,reply,Res}} -> 289 | Res; 290 | {'EXIT',Reason} -> 291 | exit({Reason, {?MODULE, leader_call, [Name, Request, Timeout]}}) 292 | end. 293 | 294 | 295 | %% @equiv gen_server:cast/2 296 | cast(Name, Request) -> 297 | catch do_cast('$gen_cast', Name, Request), 298 | ok. 299 | 300 | %% @spec leader_cast(Name::name(), Msg::term()) -> ok 301 | %% @doc Similar to gen_server:cast/2 but will be forwarded to 302 | %% the leader via the local gen_leader instance. 303 | leader_cast(Name, Request) -> 304 | catch do_cast('$leader_cast', Name, Request), 305 | ok. 306 | 307 | 308 | do_cast(Tag, Name, Request) when is_atom(Name) -> 309 | Name ! {Tag, Request}; 310 | do_cast(Tag, Pid, Request) when is_pid(Pid) -> 311 | Pid ! {Tag, Request}. 312 | 313 | 314 | %% @spec reply(From::callerRef(), Reply::term()) -> Void 315 | %% @equiv gen_server:reply/2 316 | reply({To, Tag}, Reply) -> 317 | catch To ! {Tag, Reply}. 318 | 319 | 320 | %%% --------------------------------------------------- 321 | %%% Initiate the new process. 322 | %%% Register the name using the Rfunc function 323 | %%% Calls the Mod:init/Args function. 324 | %%% Finally an acknowledge is sent to Parent and the main 325 | %%% loop is entered. 326 | %%% --------------------------------------------------- 327 | %%% @hidden 328 | % ADT - R13B passes {local, Name} or {global, Name} instead of just Name 329 | init_it(Starter, Parent, {local, Name}, Mod, {CandidateNodes, Workers, Arg}, Options) -> 330 | init_it(Starter, Parent, Name, Mod, 331 | {CandidateNodes, Workers, Arg}, Options); % ADT - R13B compatability 332 | init_it(Starter, self, Name, Mod, {CandidateNodes, Workers, Arg}, Options) -> 333 | init_it(Starter, self(), Name, Mod, 334 | {CandidateNodes, Workers, Arg}, Options); 335 | init_it(Starter,Parent,Name,Mod,{CandidateNodes,Workers,Arg},Options) -> 336 | 337 | %% The following row is needed in case of trace analysis, 338 | %% starting tracing is too slow otherwise! 339 | %receive after 100 -> ok end, 340 | 341 | Debug = debug_options(Name, Options), 342 | 343 | AmCandidate = member(node(), CandidateNodes), 344 | 345 | Election = #election{candidate_nodes = CandidateNodes, 346 | worker_nodes = Workers, 347 | name = Name, 348 | nextel = 0}, 349 | 350 | case {catch Mod:init(Arg), AmCandidate} of 351 | {{stop, Reason},_} -> 352 | proc_lib:init_ack(Starter, {error, Reason}), 353 | exit(Reason); 354 | {ignore,_} -> 355 | proc_lib:init_ack(Starter, ignore), 356 | exit(normal); 357 | {{'EXIT', Reason},_} -> 358 | proc_lib:init_ack(Starter, {error, Reason}), 359 | exit(Reason); 360 | {{ok, State}, true} -> 361 | NewE = startStage1(Election#election{incarn = incarnation(Name, node())}), 362 | 363 | proc_lib:init_ack(Starter, {ok, self()}), 364 | 365 | % ADT - handle the case where there's only one candidate worker and we can't 366 | % rely on DOWN messages to trigger the elected() call because we never get 367 | % a DOWN for ourselves 368 | case (length(CandidateNodes) == 1) and (CandidateNodes =:= [node()]) of 369 | true -> 370 | % there's only one candidate leader; us 371 | hasBecomeLeader(NewE,#server{parent = Parent,mod = Mod, 372 | state = State,debug = Debug},{init}); 373 | false -> 374 | % more than one candidate worker, continue as normal 375 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 376 | candidate, NewE,{init}) 377 | end; 378 | {{ok, State}, false} -> 379 | % ADT - support for dynamic workers 380 | io:format("new worker~n", []), 381 | % broadcast to all the candidate nodes we know about 382 | proc_lib:init_ack(Starter, {ok, self()}), 383 | lists:foreach( 384 | fun(Node) -> 385 | {Name,Node} ! {workerStart, self()} 386 | end,Election#election.candidate_nodes), 387 | 388 | timer:send_after(?TAU,{workerstart_timeout}), 389 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 390 | waiting_worker, Election,{init}); 391 | Else -> 392 | Error = {bad_return_value, Else}, 393 | proc_lib:init_ack(Starter, {error, Error}), 394 | exit(Error) 395 | end. 396 | 397 | 398 | 399 | 400 | %%% --------------------------------------------------- 401 | %%% The MAIN loops. 402 | %%% --------------------------------------------------- 403 | 404 | 405 | % safe_loop is for when new candidates/workers are starting or when 406 | % the leader is being re-elected 407 | 408 | safe_loop(#server{mod = Mod, state = State} = Server, Role, 409 | #election{name = Name} = E, _PrevMsg) -> 410 | % Event for QuickCheck 411 | % ?EVENT({Role,E}), 412 | receive 413 | {system, From, Req} -> 414 | #server{parent = Parent, debug = Debug} = Server, 415 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 416 | [safe, Server, Role, E]); 417 | {'EXIT', _, Reason} = Msg -> 418 | terminate(Reason, Msg, Server, Role, E); 419 | {halt,T,From} = Msg -> 420 | NewE = halting(E,T,From), 421 | From ! {ackLeader,T,self()}, 422 | safe_loop(Server,Role,NewE,Msg); 423 | {hasLeader,Ldr,T,_} = Msg -> 424 | NewE1 = mon_node(E,Ldr), 425 | case ( (E#election.status == elec2) and (E#election.acks /= []) ) of 426 | true -> 427 | lists:foreach( 428 | fun(Node) -> 429 | {Name,Node} ! {hasLeader,Ldr,T,self()} 430 | end,E#election.acks); 431 | false -> 432 | ok 433 | end, 434 | NewE = NewE1#election{elid = T, 435 | status = wait, 436 | leadernode = node(Ldr), 437 | down = E#election.down -- [node(Ldr)], 438 | acks = []}, 439 | Ldr ! {isLeader,T,self()}, 440 | safe_loop(Server,Role,NewE,Msg); 441 | {isLeader,T,From} = Msg -> 442 | From ! {notLeader,T,self()}, 443 | safe_loop(Server,Role,E,Msg); 444 | {notLeader,T,_} = Msg -> 445 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 446 | true -> 447 | NewE = startStage1(E); 448 | false -> 449 | NewE = E 450 | end, 451 | safe_loop(Server,Role,NewE,Msg); 452 | {ackLeader,T,From} = Msg -> 453 | case ( (E#election.status == elec2) and (E#election.elid == T) and 454 | (E#election.pendack == node(From)) ) of 455 | true -> 456 | NewE = continStage2(E#election{acks = [node(From)|E#election.acks]}); 457 | false -> 458 | NewE = E 459 | end, 460 | hasBecomeLeader(NewE,Server,Msg); 461 | {ldr,Synch,Candidates,Workers,T,From} = Msg -> 462 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 463 | true -> 464 | NewE1 = mon_node(E,From), 465 | NewE = NewE1#election{leader = From, 466 | leadernode = node(From), 467 | candidate_nodes = Candidates, 468 | worker_nodes = Workers, 469 | status = norm}, 470 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 471 | loop(Server#server{state = NewState},surrendered,NewE,Msg); 472 | false -> 473 | safe_loop(Server,Role,E,Msg) 474 | end; 475 | {normQ,T,From} = Msg -> 476 | case ( (E#election.status == elec1) or 477 | ( (E#election.status == wait) and (E#election.elid == T))) of 478 | true -> 479 | NewE = halting(E,T,From), 480 | From ! {notNorm,T,self()}; 481 | false -> 482 | NewE = E 483 | end, 484 | safe_loop(Server,Role,NewE,Msg); 485 | 486 | {notNorm,_,_} = Msg -> 487 | safe_loop(Server,Role,E,Msg); 488 | {workerAlive,T,From} = Msg -> 489 | case E#election.leadernode == none of 490 | true -> % We should initiate activation, monitor the possible leader! 491 | NewE = mon_node(E#election{leadernode = node(From), 492 | elid = T}, 493 | From), 494 | From ! {workerIsAlive,T,self()}; 495 | false -> 496 | % We should acutally ignore this, the present activation 497 | % will complete or abort first... 498 | NewE = E 499 | end, 500 | safe_loop(Server,Role,NewE,Msg); 501 | {workerIsAlive,_,_} = Msg -> 502 | % If this happens, the activation process should abort 503 | % This process is no longer the leader! 504 | % The sender will notice this via a DOWN message 505 | safe_loop(Server,Role,E,Msg); 506 | {activateWorker,T,Synch,From} = Msg -> 507 | case ((T == E#election.elid) and (node(From) == E#election.leadernode)) of 508 | true -> 509 | NewE = E#election{ leader = From, 510 | status = worker }, 511 | % XXX - only workers should get this message, and workers shouldn't 512 | % call Mod:surrendered! 513 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 514 | % actually enter the main loop 515 | loop(Server#server{state = NewState},worker,NewE,Msg); 516 | false -> 517 | % This should be a VERY special case... 518 | % But doing nothing is the right thing! 519 | % A DOWN message should arrive to solve this situation 520 | safe_loop(Server,Role,E,Msg) 521 | end; 522 | 523 | {tau_timeout} = Msg -> 524 | safe_loop(Server,Role,E,Msg); 525 | {workerstart_timeout} = Msg -> 526 | io:format("worker start timeout~n"), 527 | lists:foreach( 528 | fun(Node) -> 529 | {E#election.name,Node} ! {workerStart, self()} 530 | end,E#election.candidate_nodes), 531 | timer:send_after(?TAU,{workerstart_timeout}), 532 | safe_loop(Server, Role, E, Msg); 533 | {'DOWN',_Ref,process,From,_Reason} = Msg when Role == waiting_worker -> 534 | % We are only monitoring one proc, the leader! 535 | Node = case From of 536 | {Name,_Node} -> _Node; 537 | _ when is_pid(From) -> node(From) 538 | end, 539 | case Node == E#election.leadernode of 540 | true -> 541 | NewE = E#election{ leader = none, leadernode = none, 542 | status = waiting_worker, 543 | monitored = []}; 544 | false -> 545 | NewE = E 546 | end, 547 | safe_loop(Server, Role, NewE,Msg); 548 | {'DOWN',Ref,process,From,_Reason} = Msg -> 549 | Node = case From of 550 | {Name,_Node} -> _Node; 551 | _ when is_pid(From) -> node(From) 552 | end, 553 | NewMon = E#election.monitored -- [{Ref,Node}], 554 | case lists:member(Node,E#election.candidate_nodes) of 555 | true -> 556 | NewDown = [Node | E#election.down], 557 | E1 = E#election{down = NewDown, monitored = NewMon}, 558 | case ( pos(Node,E#election.candidate_nodes) < 559 | pos(node(),E#election.candidate_nodes) ) of 560 | true -> 561 | NewServer = Server, 562 | Lesser = lesser(node(),E#election.candidate_nodes), 563 | LesserIsSubset = (Lesser -- NewDown) == [], 564 | case ((E#election.status == wait) and (Node == E#election.leadernode)) of 565 | true -> 566 | NewE = startStage1(E1); 567 | false -> 568 | case ((E#election.status == elec1) and LesserIsSubset) of 569 | true -> 570 | NewE = startStage2(E1#election{down = Lesser}); 571 | false -> 572 | NewE = E1 573 | end 574 | end; 575 | false -> 576 | % ADT if we're the leader, call Mod:handle_DOWN as per the documentation 577 | case (E#election.leader == self()) of 578 | true -> 579 | {ok,NewState} = (Server#server.mod):handle_DOWN(Node, Server#server.state, E), 580 | NewServer = Server#server{state = NewState}; 581 | false -> 582 | NewServer = Server 583 | end, 584 | case ( (E#election.status == elec2) and (Node == E#election.pendack) ) of 585 | true -> 586 | NewE = continStage2(E1); 587 | false -> 588 | case ( (E#election.status == wait) and 589 | (Node == E#election.leadernode)) of 590 | true -> 591 | NewE = startStage1(E1); 592 | false -> 593 | NewE = E1 594 | end 595 | end 596 | end 597 | end, 598 | hasBecomeLeader(NewE,NewServer,Msg) 599 | end. 600 | 601 | 602 | loop(#server{parent = Parent, 603 | mod = Mod, 604 | state = State, 605 | debug = Debug} = Server, Role, #election{name = Name} = E, _PrevMsg) -> 606 | % Event for QuickCheck 607 | % ?EVENT({Role,E}), 608 | receive 609 | Msg -> 610 | 611 | case Msg of 612 | {system, From, Req} -> 613 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 614 | [normal, Server, Role, E]); 615 | {'EXIT', Parent, Reason} -> 616 | terminate(Reason, Msg, Server, Role, E); 617 | 618 | {halt,_,From} -> 619 | % we already have a leader, so send it back 620 | From ! {hasLeader, E#election.leader,E#election.elid,self()}, 621 | loop(Server,Role,E,Msg); 622 | {hasLeader,_,_,_} -> 623 | loop(Server,Role,E,Msg); 624 | {isLeader,T,From} -> 625 | case (self() == E#election.leader) of 626 | true -> 627 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 628 | From), 629 | {ok,Synch,NewState} = Mod:elected(State,NewE), 630 | % ADT - send the candidate and worker lists so they can be synchronized 631 | From ! {ldr,Synch,E#election.candidate_nodes, E#election.worker_nodes, 632 | E#election.elid,self()}, 633 | loop(Server#server{state = NewState},Role,NewE,Msg); 634 | false -> 635 | From ! {notLeader,T,self()}, 636 | loop(Server,Role,E,Msg) 637 | end; 638 | {ackLeader,_,_} -> 639 | loop(Server,Role,E,Msg); 640 | {notLeader,_,_} -> 641 | loop(Server,Role,E,Msg); 642 | {ack,_,_} -> 643 | loop(Server,Role,E,Msg); 644 | {ldr,_,_,_} -> 645 | loop(Server,Role,E,Msg); 646 | {normQ,_,_} -> 647 | loop(Server,Role,E,Msg); 648 | {notNorm,T,From} -> 649 | case ( (E#election.leader == self()) and (E#election.elid == T) ) of 650 | true -> 651 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 652 | From), 653 | {ok,Synch,NewState} = Mod:elected(State,NewE), 654 | % ADT - send the candidate and worker lists so they can be synchronized 655 | From ! {ldr,Synch,E#election.candidate_nodes, E#election.worker_nodes, 656 | E#election.elid,self()}, 657 | loop(Server#server{state = NewState},Role,NewE,Msg); 658 | false -> 659 | loop(Server,Role,E,Msg) 660 | end; 661 | % ADT - support for dynamic workers 662 | {workerStart,From} -> 663 | case self() == E#election.leader of 664 | true -> 665 | io:format("Added new worker to the pool ~p~n", [From]), 666 | NewE = E#election{work_down = lists:umerge(E#election.work_down, 667 | [node(From)])}, 668 | loop(Server,Role,NewE,Msg); 669 | false -> 670 | loop(Server,Role,E,Msg) 671 | end; 672 | {workerAlive,_,_} -> 673 | % Do nothing if we get this from a new leader 674 | % We will soon notice that the prev leader has died, and 675 | % get the same message again when we are back in safe_loop! 676 | loop(Server,Role,E,Msg); 677 | {activateWorker,_,_,_} -> 678 | % We ignore this, we are already active... 679 | % It must be an old message! 680 | loop(Server,Role,E,Msg); 681 | {workerIsAlive,T,From} -> 682 | case ((T == E#election.elid) and (self() == E#election.leader) 683 | % and iselem(node(From),E#election.monitored) 684 | ) of 685 | true -> 686 | Extra = {add_worker, node(From)}, 687 | Fun = fun() -> 688 | case whereis(E#election.name) of 689 | undefined -> 690 | io:format("~p is not running on ~p~n", [E#election.name, node()]), 691 | ok; 692 | Pid -> 693 | io:format("suspending ~p on ~p~n", [Pid, node()]), 694 | sys:suspend(Pid), 695 | io:format("changing code for ~p on ~p~n", [Pid, node()]), 696 | sys:change_code(Pid, ?MODULE, foo, Extra), 697 | io:format("resuming ~p on ~p~n", [Pid, node()]), 698 | sys:resume(Pid) 699 | end 700 | end, 701 | 702 | Nodes = lists:append(E#election.candidate_nodes -- [node()], 703 | [node()]), 704 | 705 | lists:foreach( 706 | fun(Node) -> 707 | spawn(Node, Fun) 708 | end, Nodes), 709 | io:format("done spawning~n"), 710 | 711 | NewE = mon_node( 712 | E#election{work_down = E#election.work_down -- [node(From)]}, 713 | From), 714 | % NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 715 | % XXX - this isn't a candidate, the docs say only candidates should 716 | % trigger an elected() call 717 | 718 | {ok,Synch,NewState} = Mod:elected(State,NewE), 719 | From ! {activateWorker,T,Synch,self()}, 720 | loop(Server#server{state = NewState},Role,NewE,Msg); 721 | false -> 722 | loop(Server,Role,E,Msg) 723 | end; 724 | {tau_timeout} -> 725 | case (E#election.leader == self()) of 726 | true -> 727 | lists:foreach( 728 | fun(Node) -> 729 | {Name,Node} ! {normQ,E#election.elid,self()} 730 | end,E#election.down -- [lists:nth(1,E#election.candidate_nodes)]), 731 | 732 | lists:foreach( 733 | fun(Node) -> 734 | {Name,Node} ! {workerAlive,E#election.elid,self()} 735 | end,E#election.work_down), 736 | 737 | timer:send_after(?TAU,{tau_timeout}); 738 | false -> 739 | ok 740 | end, 741 | loop(Server,Role,E,Msg); 742 | {workerstart_timeout} = Msg -> 743 | loop(Server, Role, E, Msg); 744 | {'DOWN',_Ref,process,From,_Reason} when Role == worker -> 745 | % We are only monitoring one proc, the leader! 746 | Node = case From of 747 | {Name,_Node} -> _Node; 748 | _ when is_pid(From) -> node(From) 749 | end, 750 | case Node == E#election.leadernode of 751 | true -> 752 | NewE = E#election{ leader = none, leadernode = none, 753 | status = waiting_worker, 754 | monitored = []}, 755 | safe_loop(Server, waiting_worker, NewE,Msg); 756 | false -> 757 | loop(Server, Role, E,Msg) 758 | end; 759 | {'DOWN',Ref,process,From,_Reason} -> 760 | Node = case From of 761 | {Name,_Node} -> _Node; 762 | _ when is_pid(From) -> node(From) 763 | end, 764 | NewMon = E#election.monitored -- [{Ref,Node}], 765 | case lists:member(Node,E#election.candidate_nodes) of 766 | true -> 767 | NewDown = [Node | E#election.down], 768 | E1 = E#election{down = NewDown, monitored = NewMon}, 769 | case (Node == E#election.leadernode) of 770 | true -> 771 | NewE = startStage1(E1), 772 | safe_loop(Server, candidate, NewE,Msg); 773 | 774 | false -> 775 | % ADT if we're the leader, call Mod:handle_DOWN as per the documentation 776 | case (E#election.leader == self()) of 777 | true -> 778 | {ok,NewState} = (Server#server.mod):handle_DOWN(Node, Server#server.state, E), 779 | NewServer = Server#server{state = NewState}; 780 | false -> 781 | NewServer = Server 782 | end, 783 | loop(NewServer, Role, E1,Msg) 784 | end; 785 | false -> 786 | % I am the leader, make sure the dead worker is in work_down. 787 | E1 = E#election{monitored = NewMon, 788 | work_down = [Node | (E#election.work_down -- [Node])] 789 | }, 790 | loop(Server, Role, E1,Msg) 791 | end; 792 | _Msg when Debug == [] -> 793 | handle_msg(Msg, Server, Role, E); 794 | _Msg -> 795 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 796 | E#election.name, {in, Msg}), 797 | handle_msg(Msg, Server#server{debug = Debug1}, Role, E) 798 | end 799 | end. 800 | 801 | %%----------------------------------------------------------------- 802 | %% Callback functions for system messages handling. 803 | %%----------------------------------------------------------------- 804 | %% @hidden 805 | system_continue(_Parent, _Debug, [safe, Server, Role, E]) -> 806 | safe_loop(Server, Role, E,{}); 807 | system_continue(_Parent, _Debug, [normal, Server, Role, E]) -> 808 | loop(Server, Role, E,{}). 809 | 810 | %% @hidden 811 | system_terminate(Reason, _Parent, _Debug, [_Mode, Server, Role, E]) -> 812 | terminate(Reason, [], Server, Role, E). 813 | 814 | %% @hidden 815 | system_code_change([Mode, Server, Role, E], _Module, _OldVsn, 816 | {add_worker, Worker}) -> 817 | % ADT - using code_change to add a dynamic worker to the shared list of workers 818 | NewE = E#election{worker_nodes = lists:umerge(E#election.worker_nodes, 819 | [Worker])}, 820 | io:format("Added worker ~p to election state for ~p~n", [Worker, 821 | E#election.name]), 822 | {ok, [Mode, Server, Role, NewE]}; 823 | system_code_change([Mode, Server, Role, E], _Module, OldVsn, Extra) -> 824 | #server{mod = Mod, state = State} = Server, 825 | case catch Mod:code_change(OldVsn, State, E, Extra) of 826 | {ok, NewState} -> 827 | NewServer = Server#server{state = NewState}, 828 | {ok, [Mode, NewServer, Role, E]}; 829 | {ok, NewState, NewE} -> 830 | NewServer = Server#server{state = NewState}, 831 | {ok, [Mode, NewServer, Role, NewE]}; 832 | Else -> Else 833 | end. 834 | 835 | %%----------------------------------------------------------------- 836 | %% Format debug messages. Print them as the call-back module sees 837 | %% them, not as the real erlang messages. Use trace for that. 838 | %%----------------------------------------------------------------- 839 | %% @hidden 840 | print_event(Dev, {in, Msg}, Name) -> 841 | case Msg of 842 | {'$gen_call', {From, _Tag}, Call} -> 843 | io:format(Dev, "*DBG* ~p got local call ~p from ~w~n", 844 | [Name, Call, From]); 845 | {'$leader_call', {From, _Tag}, Call} -> 846 | io:format(Dev, "*DBG* ~p got global call ~p from ~w~n", 847 | [Name, Call, From]); 848 | {'$gen_cast', Cast} -> 849 | io:format(Dev, "*DBG* ~p got local cast ~p~n", 850 | [Name, Cast]); 851 | {'$leader_cast', Cast} -> 852 | io:format(Dev, "*DBG* ~p got global cast ~p~n", 853 | [Name, Cast]); 854 | _ -> 855 | io:format(Dev, "*DBG* ~p got ~p~n", [Name, Msg]) 856 | end; 857 | print_event(Dev, {out, Msg, To, State}, Name) -> 858 | io:format(Dev, "*DBG* ~p sent ~p to ~w, new state ~w~n", 859 | [Name, Msg, To, State]); 860 | print_event(Dev, {noreply, State}, Name) -> 861 | io:format(Dev, "*DBG* ~p new state ~w~n", [Name, State]); 862 | print_event(Dev, Event, Name) -> 863 | io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). 864 | 865 | 866 | handle_msg({'$leader_call', From, Request} = Msg, 867 | #server{mod = Mod, state = State} = Server, elected = Role, E) -> 868 | case catch Mod:handle_leader_call(Request, From, State, E) of 869 | {reply, Reply, NState} -> 870 | NewServer = reply(From, {leader,reply,Reply}, 871 | Server#server{state = NState}, Role, E), 872 | loop(NewServer, Role, E,Msg); 873 | {reply, Reply, Broadcast, NState} -> 874 | NewE = broadcast({from_leader,Broadcast}, E), 875 | NewServer = reply(From, {leader,reply,Reply}, 876 | Server#server{state = NState}, Role, 877 | NewE), 878 | loop(NewServer, Role, NewE,Msg); 879 | {noreply, NState} = Reply -> 880 | NewServer = handle_debug(Server#server{state = NState}, 881 | Role, E, Reply), 882 | loop(NewServer, Role, E,Msg); 883 | {stop, Reason, Reply, NState} -> 884 | {'EXIT', R} = 885 | (catch terminate(Reason, Msg, 886 | Server#server{state = NState}, 887 | Role, E)), 888 | reply(From, Reply), 889 | exit(R); 890 | Other -> 891 | handle_common_reply(Other, Msg, Server, Role, E) 892 | end; 893 | handle_msg({from_leader, Cmd} = Msg, 894 | #server{mod = Mod, state = State} = Server, Role, E) -> 895 | handle_common_reply(catch Mod:from_leader(Cmd, State, E), 896 | Msg, Server, Role, E); 897 | handle_msg({'$leader_call', From, Request} = Msg, Server, Role, 898 | #election{buffered = Buffered, leader = Leader} = E) -> 899 | Ref = make_ref(), 900 | Leader ! {'$leader_call', {self(),Ref}, Request}, 901 | NewBuffered = [{Ref,From}|Buffered], 902 | loop(Server, Role, E#election{buffered = NewBuffered},Msg); 903 | handle_msg({Ref, {leader,reply,Reply}} = Msg, Server, Role, 904 | #election{buffered = Buffered} = E) -> 905 | {value, {_,From}} = keysearch(Ref,1,Buffered), 906 | NewServer = reply(From, {leader,reply,Reply}, Server, Role, 907 | E#election{buffered = keydelete(Ref,1,Buffered)}), 908 | loop(NewServer, Role, E, Msg); 909 | handle_msg({'$gen_call', From, Request} = Msg, 910 | #server{mod = Mod, state = State} = Server, Role, E) -> 911 | case catch Mod:handle_call(Request, From, State) of 912 | {reply, Reply, NState} -> 913 | NewServer = reply(From, Reply, 914 | Server#server{state = NState}, Role, E), 915 | loop(NewServer, Role, E, Msg); 916 | {noreply, NState} = Reply -> 917 | NewServer = handle_debug(Server#server{state = NState}, 918 | Role, E, Reply), 919 | loop(NewServer, Role, E, Msg); 920 | {stop, Reason, Reply, NState} -> 921 | {'EXIT', R} = 922 | (catch terminate(Reason, Msg, Server#server{state = NState}, 923 | Role, E)), 924 | reply(From, Reply), 925 | exit(R); 926 | Other -> 927 | handle_common_reply(Other, Msg, Server, Role, E) 928 | end; 929 | handle_msg({'$gen_cast',Msg} = Cast, 930 | #server{mod = Mod, state = State} = Server, Role, E) -> 931 | handle_common_reply(catch Mod:handle_cast(Msg, State), 932 | Cast, Server, Role, E); 933 | % ADT - implement gen_leader_cast 934 | handle_msg({'$leader_cast', Msg} = Cast, 935 | #server{mod = Mod, state = State} = Server, elected = Role, E) -> 936 | case catch Mod:handle_leader_cast(Msg, State, E) of 937 | {noreply, NState} -> 938 | NewServer = handle_debug(Server#server{state = NState}, 939 | Role, E, Cast), 940 | loop(NewServer, Role, E,Cast); 941 | Other -> 942 | handle_common_reply(Other, Msg, Server, Role, E) 943 | end; 944 | handle_msg({'$leader_cast', Msg} = Cast, Server, Role, 945 | #election{leader = Leader} = E) -> 946 | Leader ! {'$leader_cast', Msg}, 947 | loop(Server, Role, E, Cast); 948 | handle_msg(Msg, 949 | #server{mod = Mod, state = State} = Server, Role, E) -> 950 | handle_common_reply(catch Mod:handle_info(Msg, State), 951 | Msg, Server, Role, E). 952 | 953 | 954 | handle_common_reply(Reply, Msg, Server, Role, E) -> 955 | case Reply of 956 | % ADT - support for noreply replies from eg. gen_cast 957 | {noreply, NState} -> 958 | NewServer = handle_debug(Server#server{state = NState}, 959 | Role, E, Reply), 960 | loop(NewServer, Role, E, Msg); 961 | {ok, NState} -> 962 | NewServer = handle_debug(Server#server{state = NState}, 963 | Role, E, Reply), 964 | loop(NewServer, Role, E, Msg); 965 | {stop, Reason, NState} -> 966 | terminate(Reason, Msg, Server#server{state = NState}, Role, E); 967 | {'EXIT', Reason} -> 968 | terminate(Reason, Msg, Server, Role, E); 969 | _ -> 970 | terminate({bad2_return_value, Reply}, Msg, Server, Role, E) 971 | end. 972 | 973 | 974 | reply({To, Tag}, Reply, #server{state = State} = Server, Role, E) -> 975 | reply({To, Tag}, Reply), 976 | handle_debug(Server, Role, E, {out, Reply, To, State}). 977 | 978 | 979 | handle_debug(#server{debug = []} = Server, _Role, _E, _Event) -> 980 | Server; 981 | handle_debug(#server{debug = Debug} = Server, _Role, E, Event) -> 982 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 983 | E#election.name, Event), 984 | Server#server{debug = Debug1}. 985 | 986 | %%% --------------------------------------------------- 987 | %%% Terminate the server. 988 | %%% --------------------------------------------------- 989 | 990 | terminate(Reason, Msg, #server{mod = Mod, 991 | state = State, 992 | debug = Debug} = _Server, _Role, 993 | #election{name = Name} = _E) -> 994 | case catch Mod:terminate(Reason, State) of 995 | {'EXIT', R} -> 996 | error_info(R, Name, Msg, State, Debug), 997 | exit(R); 998 | _ -> 999 | case Reason of 1000 | normal -> 1001 | exit(normal); 1002 | shutdown -> 1003 | exit(shutdown); 1004 | _ -> 1005 | error_info(Reason, Name, Msg, State, Debug), 1006 | exit(Reason) 1007 | end 1008 | end. 1009 | 1010 | %% Maybe we shouldn't do this? We have the crash report... 1011 | error_info(Reason, Name, Msg, State, Debug) -> 1012 | format("** Generic leader ~p terminating \n" 1013 | "** Last message in was ~p~n" 1014 | "** When Server state == ~p~n" 1015 | "** Reason for termination == ~n** ~p~n", 1016 | [Name, Msg, State, Reason]), 1017 | sys:print_log(Debug), 1018 | ok. 1019 | 1020 | %%% --------------------------------------------------- 1021 | %%% Misc. functions. 1022 | %%% --------------------------------------------------- 1023 | 1024 | opt(Op, [{Op, Value}|_]) -> 1025 | {ok, Value}; 1026 | opt(Op, [_|Options]) -> 1027 | opt(Op, Options); 1028 | opt(_, []) -> 1029 | false. 1030 | 1031 | debug_options(Name, Opts) -> 1032 | case opt(debug, Opts) of 1033 | {ok, Options} -> dbg_options(Name, Options); 1034 | _ -> dbg_options(Name, []) 1035 | end. 1036 | 1037 | dbg_options(Name, []) -> 1038 | Opts = 1039 | case init:get_argument(generic_debug) of 1040 | error -> 1041 | []; 1042 | _ -> 1043 | [log, statistics] 1044 | end, 1045 | dbg_opts(Name, Opts); 1046 | dbg_options(Name, Opts) -> 1047 | dbg_opts(Name, Opts). 1048 | 1049 | dbg_opts(Name, Opts) -> 1050 | case catch sys:debug_options(Opts) of 1051 | {'EXIT',_} -> 1052 | format("~p: ignoring erroneous debug options - ~p~n", 1053 | [Name, Opts]), 1054 | []; 1055 | Dbg -> 1056 | Dbg 1057 | end. 1058 | 1059 | %%----------------------------------------------------------------- 1060 | %% Status information 1061 | %%----------------------------------------------------------------- 1062 | %% @hidden 1063 | format_status(Opt, StatusData) -> 1064 | [PDict, SysState, Parent, Debug, [_Mode, Server, _Role, E]] = StatusData, 1065 | Header = lists:concat(["Status for generic server ", E#election.name]), 1066 | Log = sys:get_debug(log, Debug, []), 1067 | #server{mod = Mod, state = State} = Server, 1068 | Specific = 1069 | case erlang:function_exported(Mod, format_status, 2) of 1070 | true -> 1071 | case catch apply(Mod, format_status, [Opt, [PDict, State]]) of 1072 | {'EXIT', _} -> [{data, [{"State", State}]}]; 1073 | Else -> Else 1074 | end; 1075 | _ -> 1076 | [{data, [{"State", State}]}] 1077 | end, 1078 | [{header, Header}, 1079 | {data, [{"Status", SysState}, 1080 | {"Parent", Parent}, 1081 | {"Logged events", Log}]} | 1082 | Specific]. 1083 | 1084 | 1085 | %%----------------------------------------------------------------- 1086 | %% Leader-election functions 1087 | %%----------------------------------------------------------------- 1088 | 1089 | %% Corresponds to startStage1 in Figure 1 in the Stoller-article 1090 | startStage1(E) -> 1091 | Elid = {pos(node(),E#election.candidate_nodes),E#election.incarn,E#election.nextel}, 1092 | NewE = E#election{ 1093 | elid = Elid, 1094 | nextel = E#election.nextel + 1, 1095 | down = [], 1096 | status = elec1}, 1097 | case ( pos(node(),E#election.candidate_nodes) == 1) of 1098 | true -> 1099 | startStage2(NewE); 1100 | false -> 1101 | mon_nodes(NewE,lesser(node(),E#election.candidate_nodes)) 1102 | end. 1103 | 1104 | %% Corresponds to startStage2 1105 | startStage2(E) -> 1106 | continStage2(E#election{ 1107 | status = elec2, 1108 | pendack = node(), 1109 | acks = []}). 1110 | 1111 | continStage2(E) -> 1112 | % is the current node at a lower position than the last element in the list 1113 | case pos(E#election.pendack, E#election.candidate_nodes) < length(E#election.candidate_nodes) of 1114 | true -> 1115 | % finds the candidate node in the list right after the one matching node() 1116 | Pendack = next(E#election.pendack,E#election.candidate_nodes), 1117 | NewE = mon_nodes(E,[Pendack]), % monitor it 1118 | % send a halt message, the pendack process should return ackLeader 1119 | % or hasLeader if it thinks it has a leader already 1120 | {E#election.name,Pendack} ! {halt,E#election.elid,self()}, 1121 | NewE#election{pendack = Pendack}; 1122 | false -> 1123 | % I am the leader 1124 | % io:format("I am the leader (Node ~w) ~n", [node()]), 1125 | E#election{leader = self(), 1126 | leadernode = node(), 1127 | status = norm} 1128 | end. 1129 | 1130 | %% corresponds to Halting 1131 | halting(E,T,From) -> 1132 | NewE = mon_node(E,From), 1133 | NewE#election{elid = T, 1134 | status = wait, 1135 | leadernode = node(From), 1136 | down = E#election.down -- [node(From)] 1137 | }. 1138 | 1139 | %% Start monitor a bunch of nodes 1140 | mon_nodes(E,Nodes) -> 1141 | foldl( 1142 | fun(Node,_E) -> 1143 | mon_node(_E,{_E#election.name,Node}) 1144 | end,E,Nodes). 1145 | 1146 | %% Star monitoring one Process 1147 | mon_node(E,Proc) -> 1148 | Node = case Proc of 1149 | {_Name,Node_} -> 1150 | Node_; 1151 | Pid when is_pid(Pid) -> 1152 | node(Pid) 1153 | end, 1154 | case iselem(Node,E#election.monitored) of 1155 | true -> 1156 | E; 1157 | false -> 1158 | Ref = erlang:monitor(process,Proc), 1159 | E#election{monitored = [{Ref,Node} | E#election.monitored]} 1160 | end. 1161 | 1162 | %% Stop monitoring of a bunch of nodes 1163 | %demon_nodes(E) -> 1164 | % foreach(fun({R,_}) -> 1165 | % erlang:demonitor(R) 1166 | % end,E#election.monitored), 1167 | % E#election{monitored = []}. 1168 | 1169 | %% checks if the proc has become the leader, if so switch to loop 1170 | hasBecomeLeader(E,Server,Msg) -> 1171 | case ((E#election.status == norm) and (E#election.leader == self())) of 1172 | true -> 1173 | {ok,Synch,NewState} = (Server#server.mod):elected(Server#server.state,E), 1174 | lists:foreach( 1175 | fun(Node) -> 1176 | {E#election.name,Node} ! 1177 | {ldr, Synch, E#election.candidate_nodes, E#election.worker_nodes, 1178 | E#election.elid, self()} 1179 | end,E#election.acks), 1180 | 1181 | % Make sure we will try to contact all workers! 1182 | NewE = E#election{work_down = E#election.worker_nodes}, 1183 | 1184 | % Set the internal timeout (corresponds to Periodically) 1185 | timer:send_after(?TAU,{tau_timeout}), %It's meaningful only when I am the leader! 1186 | loop(Server#server{state = NewState},elected,NewE,Msg); 1187 | false -> 1188 | safe_loop(Server,candidate,E,Msg) 1189 | end. 1190 | 1191 | 1192 | 1193 | 1194 | %%% 1195 | % 1196 | % incarnation should return an integer value for the next 1197 | % incarnation of this node. We create a file for each node, 1198 | % this file contains a counter. When starting the system for the 1199 | % first time, the files should be intialized with 0 incarnation 1200 | % counter for all nodes orelse be removed, since we create 1201 | % files if not present with counter 1. 1202 | % 1203 | % Atomicity: This approach is safe as long as there is only 1204 | % one gen_leader running per node. 1205 | % 1206 | % ADT - modified to use the locally registered 1207 | % name in addition to the nodename to allow multiple gen_leaders 1208 | % per node 1209 | % 1210 | incarnation(Name, Node) -> 1211 | FileName = lists:flatten(io_lib:format(".~p.~p-~p", [?MODULE, Name, Node])), 1212 | case file:read_file_info(FileName) of 1213 | {error,_Reason} -> 1214 | ok = file:write_file(FileName,term_to_binary(1)), 1215 | 0; 1216 | {ok,_} -> 1217 | {ok,Bin} = file:read_file(FileName), 1218 | Incarn = binary_to_term(Bin), 1219 | ok = file:write_file(FileName,term_to_binary(Incarn+1)), 1220 | Incarn 1221 | end. 1222 | 1223 | 1224 | broadcast(Msg, #election{monitored = Monitored} = E) -> 1225 | %% This function is used for broadcasts, 1226 | %% and we make sure only to broadcast to already known nodes. 1227 | ToNodes = [N || {_,N} <- Monitored], 1228 | broadcast(Msg, ToNodes, E). 1229 | 1230 | broadcast({from_leader, Msg}, ToNodes, E) -> 1231 | foreach( 1232 | fun(Node) -> 1233 | {E#election.name,Node} ! {from_leader, Msg} 1234 | end,ToNodes), 1235 | E. 1236 | 1237 | iselem(_,[]) -> 1238 | false; 1239 | iselem(P,[{_,P}|_]) -> 1240 | true; 1241 | iselem(P,[_ | Ns]) -> 1242 | iselem(P,Ns). 1243 | 1244 | lesser(_,[]) -> 1245 | []; 1246 | lesser(N,[N|_]) -> 1247 | []; 1248 | lesser(N,[M|Ms]) -> 1249 | [M|lesser(N,Ms)]. 1250 | 1251 | %% returns the next element after N in a list 1252 | next(_,[]) -> 1253 | no_val; 1254 | next(N,[N|Ms]) -> 1255 | lists:nth(1,Ms); 1256 | next(N,[_|Ms]) -> 1257 | next(N,Ms). 1258 | 1259 | %% find the posiion of N1 in a list 1260 | pos(N1,[N1|_]) -> 1261 | 1; 1262 | pos(N1,[_|Ns]) -> 1263 | 1+pos(N1,Ns). 1264 | -------------------------------------------------------------------------------- /opencsm_version/source-to-current.diff: -------------------------------------------------------------------------------- 1 | --- original_version/gen_leader.erl 2009-05-19 15:16:57.000000000 -0400 2 | +++ opencsm_version/gen_leader.erl 2009-05-19 15:32:25.000000000 -0400 3 | @@ -51,6 +51,9 @@ 4 | %% See gen_server. 5 | %% @type callerRef() = {pid(), reference()}. See gen_server. 6 | %% 7 | + 8 | +%% This version debugged/modified by Andrew Thompson (andrew@hijacked.us) 9 | +%% for SpiceCSM 10 | -module(gen_leader). 11 | 12 | % Time between rounds of query from the leader 13 | @@ -146,6 +149,9 @@ 14 | %% 15 | %% @doc Starts a gen_leader process without linking to the parent. 16 | %% 17 | +% ADT - don't allow an empty candidate list 18 | +start(_Name, [], _Workers, _Mod, _Arg, _Options) -> 19 | + {error, nocandidates}; 20 | start(Name, CandidateNodes, Workers, Mod, Arg, Options) when is_atom(Name) -> 21 | gen:start(?MODULE, nolink, {local,Name}, 22 | Mod, {CandidateNodes, Workers, Arg}, Options). 23 | @@ -170,6 +176,9 @@ 24 | %%

The list of candidates needs to be known from the start. Workers 25 | %% can be added at runtime.

26 | %% @end 27 | +% ADT - don't allow an empty candidate list 28 | +start_link(_Name, [], _Workers, _Mod, _Arg, _Options) -> 29 | + {error, nocandidates}; 30 | start_link(Name, CandidateNodes, Workers, 31 | Mod, Arg, Options) when is_atom(Name) -> 32 | % Random delay for QuickCheck 33 | @@ -296,9 +305,9 @@ 34 | ok. 35 | 36 | 37 | -do_cast(Tag, Name, Request) when atom(Name) -> 38 | +do_cast(Tag, Name, Request) when is_atom(Name) -> 39 | Name ! {Tag, Request}; 40 | -do_cast(Tag, Pid, Request) when pid(Pid) -> 41 | +do_cast(Tag, Pid, Request) when is_pid(Pid) -> 42 | Pid ! {Tag, Request}. 43 | 44 | 45 | @@ -316,6 +325,10 @@ 46 | %%% loop is entered. 47 | %%% --------------------------------------------------- 48 | %%% @hidden 49 | +% ADT - R13B passes {local, Name} or {global, Name} instead of just Name 50 | +init_it(Starter, Parent, {local, Name}, Mod, {CandidateNodes, Workers, Arg}, Options) -> 51 | + init_it(Starter, Parent, Name, Mod, 52 | + {CandidateNodes, Workers, Arg}, Options); % ADT - R13B compatability 53 | init_it(Starter, self, Name, Mod, {CandidateNodes, Workers, Arg}, Options) -> 54 | init_it(Starter, self(), Name, Mod, 55 | {CandidateNodes, Workers, Arg}, Options); 56 | @@ -345,13 +358,34 @@ 57 | proc_lib:init_ack(Starter, {error, Reason}), 58 | exit(Reason); 59 | {{ok, State}, true} -> 60 | - NewE = startStage1(Election#election{incarn = incarnation(node())}), 61 | + NewE = startStage1(Election#election{incarn = incarnation(Name, node())}), 62 | 63 | proc_lib:init_ack(Starter, {ok, self()}), 64 | + 65 | + % ADT - handle the case where there's only one candidate worker and we can't 66 | + % rely on DOWN messages to trigger the elected() call because we never get 67 | + % a DOWN for ourselves 68 | + case (length(CandidateNodes) == 1) and (CandidateNodes =:= [node()]) of 69 | + true -> 70 | + % there's only one candidate leader; us 71 | + hasBecomeLeader(NewE,#server{parent = Parent,mod = Mod, 72 | + state = State,debug = Debug},{init}); 73 | + false -> 74 | + % more than one candidate worker, continue as normal 75 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 76 | - candidate, NewE,{init}); 77 | + candidate, NewE,{init}) 78 | + end; 79 | {{ok, State}, false} -> 80 | + % ADT - support for dynamic workers 81 | + io:format("new worker~n", []), 82 | + % broadcast to all the candidate nodes we know about 83 | proc_lib:init_ack(Starter, {ok, self()}), 84 | + lists:foreach( 85 | + fun(Node) -> 86 | + {Name,Node} ! {workerStart, self()} 87 | + end,Election#election.candidate_nodes), 88 | + 89 | + timer:send_after(?TAU,{workerstart_timeout}), 90 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 91 | waiting_worker, Election,{init}); 92 | Else -> 93 | @@ -368,8 +402,11 @@ 94 | %%% --------------------------------------------------- 95 | 96 | 97 | +% safe_loop is for when new candidates/workers are starting or when 98 | +% the leader is being re-elected 99 | + 100 | safe_loop(#server{mod = Mod, state = State} = Server, Role, 101 | - #election{name = Name} = E, PrevMsg) -> 102 | + #election{name = Name} = E, _PrevMsg) -> 103 | % Event for QuickCheck 104 | % ?EVENT({Role,E}), 105 | receive 106 | @@ -421,12 +458,14 @@ 107 | NewE = E 108 | end, 109 | hasBecomeLeader(NewE,Server,Msg); 110 | - {ldr,Synch,T,From} = Msg -> 111 | + {ldr,Synch,Candidates,Workers,T,From} = Msg -> 112 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 113 | true -> 114 | NewE1 = mon_node(E,From), 115 | NewE = NewE1#election{leader = From, 116 | leadernode = node(From), 117 | + candidate_nodes = Candidates, 118 | + worker_nodes = Workers, 119 | status = norm}, 120 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 121 | loop(Server#server{state = NewState},surrendered,NewE,Msg); 122 | @@ -465,11 +504,14 @@ 123 | % The sender will notice this via a DOWN message 124 | safe_loop(Server,Role,E,Msg); 125 | {activateWorker,T,Synch,From} = Msg -> 126 | - case ( (T == E#election.elid) and (node(From) == E#election.leadernode)) of 127 | + case ((T == E#election.elid) and (node(From) == E#election.leadernode)) of 128 | true -> 129 | NewE = E#election{ leader = From, 130 | status = worker }, 131 | + % XXX - only workers should get this message, and workers shouldn't 132 | + % call Mod:surrendered! 133 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 134 | + % actually enter the main loop 135 | loop(Server#server{state = NewState},worker,NewE,Msg); 136 | false -> 137 | % This should be a VERY special case... 138 | @@ -480,11 +522,19 @@ 139 | 140 | {tau_timeout} = Msg -> 141 | safe_loop(Server,Role,E,Msg); 142 | - {'DOWN',Ref,process,From,Reason} = Msg when Role == waiting_worker -> 143 | + {workerstart_timeout} = Msg -> 144 | + io:format("worker start timeout~n"), 145 | + lists:foreach( 146 | + fun(Node) -> 147 | + {E#election.name,Node} ! {workerStart, self()} 148 | + end,E#election.candidate_nodes), 149 | + timer:send_after(?TAU,{workerstart_timeout}), 150 | + safe_loop(Server, Role, E, Msg); 151 | + {'DOWN',_Ref,process,From,_Reason} = Msg when Role == waiting_worker -> 152 | % We are only monitoring one proc, the leader! 153 | Node = case From of 154 | {Name,_Node} -> _Node; 155 | - _ when pid(From) -> node(From) 156 | + _ when is_pid(From) -> node(From) 157 | end, 158 | case Node == E#election.leadernode of 159 | true -> 160 | @@ -495,10 +545,10 @@ 161 | NewE = E 162 | end, 163 | safe_loop(Server, Role, NewE,Msg); 164 | - {'DOWN',Ref,process,From,Reason} = Msg -> 165 | + {'DOWN',Ref,process,From,_Reason} = Msg -> 166 | Node = case From of 167 | {Name,_Node} -> _Node; 168 | - _ when pid(From) -> node(From) 169 | + _ when is_pid(From) -> node(From) 170 | end, 171 | NewMon = E#election.monitored -- [{Ref,Node}], 172 | case lists:member(Node,E#election.candidate_nodes) of 173 | @@ -508,6 +558,7 @@ 174 | case ( pos(Node,E#election.candidate_nodes) < 175 | pos(node(),E#election.candidate_nodes) ) of 176 | true -> 177 | + NewServer = Server, 178 | Lesser = lesser(node(),E#election.candidate_nodes), 179 | LesserIsSubset = (Lesser -- NewDown) == [], 180 | case ((E#election.status == wait) and (Node == E#election.leadernode)) of 181 | @@ -522,6 +573,14 @@ 182 | end 183 | end; 184 | false -> 185 | + % ADT if we're the leader, call Mod:handle_DOWN as per the documentation 186 | + case (E#election.leader == self()) of 187 | + true -> 188 | + {ok,NewState} = (Server#server.mod):handle_DOWN(Node, Server#server.state, E), 189 | + NewServer = Server#server{state = NewState}; 190 | + false -> 191 | + NewServer = Server 192 | + end, 193 | case ( (E#election.status == elec2) and (Node == E#election.pendack) ) of 194 | true -> 195 | NewE = continStage2(E1); 196 | @@ -536,14 +595,14 @@ 197 | end 198 | end 199 | end, 200 | - hasBecomeLeader(NewE,Server,Msg) 201 | + hasBecomeLeader(NewE,NewServer,Msg) 202 | end. 203 | 204 | 205 | loop(#server{parent = Parent, 206 | mod = Mod, 207 | state = State, 208 | - debug = Debug} = Server, Role, #election{name = Name} = E, PrevMsg) -> 209 | + debug = Debug} = Server, Role, #election{name = Name} = E, _PrevMsg) -> 210 | % Event for QuickCheck 211 | % ?EVENT({Role,E}), 212 | receive 213 | @@ -557,7 +616,8 @@ 214 | terminate(Reason, Msg, Server, Role, E); 215 | 216 | {halt,_,From} -> 217 | - From ! {hasLeader,E#election.leader,E#election.elid,self()}, 218 | + % we already have a leader, so send it back 219 | + From ! {hasLeader, E#election.leader,E#election.elid,self()}, 220 | loop(Server,Role,E,Msg); 221 | {hasLeader,_,_,_} -> 222 | loop(Server,Role,E,Msg); 223 | @@ -567,7 +627,9 @@ 224 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 225 | From), 226 | {ok,Synch,NewState} = Mod:elected(State,NewE), 227 | - From ! {ldr,Synch,E#election.elid,self()}, 228 | + % ADT - send the candidate and worker lists so they can be synchronized 229 | + From ! {ldr,Synch,E#election.candidate_nodes, E#election.worker_nodes, 230 | + E#election.elid,self()}, 231 | loop(Server#server{state = NewState},Role,NewE,Msg); 232 | false -> 233 | From ! {notLeader,T,self()}, 234 | @@ -589,11 +651,24 @@ 235 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 236 | From), 237 | {ok,Synch,NewState} = Mod:elected(State,NewE), 238 | - From ! {ldr,Synch,E#election.elid,self()}, 239 | + % ADT - send the candidate and worker lists so they can be synchronized 240 | + From ! {ldr,Synch,E#election.candidate_nodes, E#election.worker_nodes, 241 | + E#election.elid,self()}, 242 | loop(Server#server{state = NewState},Role,NewE,Msg); 243 | false -> 244 | loop(Server,Role,E,Msg) 245 | end; 246 | + % ADT - support for dynamic workers 247 | + {workerStart,From} -> 248 | + case self() == E#election.leader of 249 | + true -> 250 | + io:format("Added new worker to the pool ~p~n", [From]), 251 | + NewE = E#election{work_down = lists:umerge(E#election.work_down, 252 | + [node(From)])}, 253 | + loop(Server,Role,NewE,Msg); 254 | + false -> 255 | + loop(Server,Role,E,Msg) 256 | + end; 257 | {workerAlive,_,_} -> 258 | % Do nothing if we get this from a new leader 259 | % We will soon notice that the prev leader has died, and 260 | @@ -608,10 +683,38 @@ 261 | % and iselem(node(From),E#election.monitored) 262 | ) of 263 | true -> 264 | + Extra = {add_worker, node(From)}, 265 | + Fun = fun() -> 266 | + case whereis(E#election.name) of 267 | + undefined -> 268 | + io:format("~p is not running on ~p~n", [E#election.name, node()]), 269 | + ok; 270 | + Pid -> 271 | + io:format("suspending ~p on ~p~n", [Pid, node()]), 272 | + sys:suspend(Pid), 273 | + io:format("changing code for ~p on ~p~n", [Pid, node()]), 274 | + sys:change_code(Pid, ?MODULE, foo, Extra), 275 | + io:format("resuming ~p on ~p~n", [Pid, node()]), 276 | + sys:resume(Pid) 277 | + end 278 | + end, 279 | + 280 | + Nodes = lists:append(E#election.candidate_nodes -- [node()], 281 | + [node()]), 282 | + 283 | + lists:foreach( 284 | + fun(Node) -> 285 | + spawn(Node, Fun) 286 | + end, Nodes), 287 | + io:format("done spawning~n"), 288 | + 289 | NewE = mon_node( 290 | E#election{work_down = E#election.work_down -- [node(From)]}, 291 | From), 292 | % NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 293 | + % XXX - this isn't a candidate, the docs say only candidates should 294 | + % trigger an elected() call 295 | + 296 | {ok,Synch,NewState} = Mod:elected(State,NewE), 297 | From ! {activateWorker,T,Synch,self()}, 298 | loop(Server#server{state = NewState},Role,NewE,Msg); 299 | @@ -636,11 +739,13 @@ 300 | ok 301 | end, 302 | loop(Server,Role,E,Msg); 303 | - {'DOWN',Ref,process,From,Reason} when Role == worker -> 304 | + {workerstart_timeout} = Msg -> 305 | + loop(Server, Role, E, Msg); 306 | + {'DOWN',_Ref,process,From,_Reason} when Role == worker -> 307 | % We are only monitoring one proc, the leader! 308 | Node = case From of 309 | {Name,_Node} -> _Node; 310 | - _ when pid(From) -> node(From) 311 | + _ when is_pid(From) -> node(From) 312 | end, 313 | case Node == E#election.leadernode of 314 | true -> 315 | @@ -651,10 +756,10 @@ 316 | false -> 317 | loop(Server, Role, E,Msg) 318 | end; 319 | - {'DOWN',Ref,process,From,Reason} -> 320 | + {'DOWN',Ref,process,From,_Reason} -> 321 | Node = case From of 322 | {Name,_Node} -> _Node; 323 | - _ when pid(From) -> node(From) 324 | + _ when is_pid(From) -> node(From) 325 | end, 326 | NewMon = E#election.monitored -- [{Ref,Node}], 327 | case lists:member(Node,E#election.candidate_nodes) of 328 | @@ -667,7 +772,15 @@ 329 | safe_loop(Server, candidate, NewE,Msg); 330 | 331 | false -> 332 | - loop(Server, Role, E1,Msg) 333 | + % ADT if we're the leader, call Mod:handle_DOWN as per the documentation 334 | + case (E#election.leader == self()) of 335 | + true -> 336 | + {ok,NewState} = (Server#server.mod):handle_DOWN(Node, Server#server.state, E), 337 | + NewServer = Server#server{state = NewState}; 338 | + false -> 339 | + NewServer = Server 340 | + end, 341 | + loop(NewServer, Role, E1,Msg) 342 | end; 343 | false -> 344 | % I am the leader, make sure the dead worker is in work_down. 345 | @@ -689,16 +802,24 @@ 346 | %% Callback functions for system messages handling. 347 | %%----------------------------------------------------------------- 348 | %% @hidden 349 | -system_continue(Parent, Debug, [safe, Server, Role, E]) -> 350 | +system_continue(_Parent, _Debug, [safe, Server, Role, E]) -> 351 | safe_loop(Server, Role, E,{}); 352 | -system_continue(Parent, Debug, [normal, Server, Role, E]) -> 353 | +system_continue(_Parent, _Debug, [normal, Server, Role, E]) -> 354 | loop(Server, Role, E,{}). 355 | 356 | %% @hidden 357 | -system_terminate(Reason, _Parent, Debug, [Mode, Server, Role, E]) -> 358 | +system_terminate(Reason, _Parent, _Debug, [_Mode, Server, Role, E]) -> 359 | terminate(Reason, [], Server, Role, E). 360 | 361 | %% @hidden 362 | +system_code_change([Mode, Server, Role, E], _Module, _OldVsn, 363 | + {add_worker, Worker}) -> 364 | + % ADT - using code_change to add a dynamic worker to the shared list of workers 365 | + NewE = E#election{worker_nodes = lists:umerge(E#election.worker_nodes, 366 | + [Worker])}, 367 | + io:format("Added worker ~p to election state for ~p~n", [Worker, 368 | + E#election.name]), 369 | + {ok, [Mode, Server, Role, NewE]}; 370 | system_code_change([Mode, Server, Role, E], _Module, OldVsn, Extra) -> 371 | #server{mod = Mod, state = State} = Server, 372 | case catch Mod:code_change(OldVsn, State, E, Extra) of 373 | @@ -809,6 +930,21 @@ 374 | #server{mod = Mod, state = State} = Server, Role, E) -> 375 | handle_common_reply(catch Mod:handle_cast(Msg, State), 376 | Cast, Server, Role, E); 377 | +% ADT - implement gen_leader_cast 378 | +handle_msg({'$leader_cast', Msg} = Cast, 379 | + #server{mod = Mod, state = State} = Server, elected = Role, E) -> 380 | + case catch Mod:handle_leader_cast(Msg, State, E) of 381 | + {noreply, NState} -> 382 | + NewServer = handle_debug(Server#server{state = NState}, 383 | + Role, E, Cast), 384 | + loop(NewServer, Role, E,Cast); 385 | + Other -> 386 | + handle_common_reply(Other, Msg, Server, Role, E) 387 | + end; 388 | +handle_msg({'$leader_cast', Msg} = Cast, Server, Role, 389 | + #election{leader = Leader} = E) -> 390 | + Leader ! {'$leader_cast', Msg}, 391 | + loop(Server, Role, E, Cast); 392 | handle_msg(Msg, 393 | #server{mod = Mod, state = State} = Server, Role, E) -> 394 | handle_common_reply(catch Mod:handle_info(Msg, State), 395 | @@ -817,6 +953,11 @@ 396 | 397 | handle_common_reply(Reply, Msg, Server, Role, E) -> 398 | case Reply of 399 | + % ADT - support for noreply replies from eg. gen_cast 400 | + {noreply, NState} -> 401 | + NewServer = handle_debug(Server#server{state = NState}, 402 | + Role, E, Reply), 403 | + loop(NewServer, Role, E, Msg); 404 | {ok, NState} -> 405 | NewServer = handle_debug(Server#server{state = NState}, 406 | Role, E, Reply), 407 | @@ -835,9 +976,9 @@ 408 | handle_debug(Server, Role, E, {out, Reply, To, State}). 409 | 410 | 411 | -handle_debug(#server{debug = []} = Server, _Role, _E, Event) -> 412 | +handle_debug(#server{debug = []} = Server, _Role, _E, _Event) -> 413 | Server; 414 | -handle_debug(#server{debug = Debug} = Server, Role, E, Event) -> 415 | +handle_debug(#server{debug = Debug} = Server, _Role, E, Event) -> 416 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 417 | E#election.name, Event), 418 | Server#server{debug = Debug1}. 419 | @@ -848,8 +989,8 @@ 420 | 421 | terminate(Reason, Msg, #server{mod = Mod, 422 | state = State, 423 | - debug = Debug} = Server, Role, 424 | - #election{name = Name} = E) -> 425 | + debug = Debug} = _Server, _Role, 426 | + #election{name = Name} = _E) -> 427 | case catch Mod:terminate(Reason, State) of 428 | {'EXIT', R} -> 429 | error_info(R, Name, Msg, State, Debug), 430 | @@ -920,7 +1061,7 @@ 431 | %%----------------------------------------------------------------- 432 | %% @hidden 433 | format_status(Opt, StatusData) -> 434 | - [PDict, SysState, Parent, Debug, [Mode, Server, Role, E]] = StatusData, 435 | + [PDict, SysState, Parent, Debug, [_Mode, Server, _Role, E]] = StatusData, 436 | Header = lists:concat(["Status for generic server ", E#election.name]), 437 | Log = sys:get_debug(log, Debug, []), 438 | #server{mod = Mod, state = State} = Server, 439 | @@ -968,10 +1109,14 @@ 440 | acks = []}). 441 | 442 | continStage2(E) -> 443 | - case pos(E#election.pendack,E#election.candidate_nodes) < length(E#election.candidate_nodes) of 444 | + % is the current node at a lower position than the last element in the list 445 | + case pos(E#election.pendack, E#election.candidate_nodes) < length(E#election.candidate_nodes) of 446 | true -> 447 | + % finds the candidate node in the list right after the one matching node() 448 | Pendack = next(E#election.pendack,E#election.candidate_nodes), 449 | - NewE = mon_nodes(E,[Pendack]), 450 | + NewE = mon_nodes(E,[Pendack]), % monitor it 451 | + % send a halt message, the pendack process should return ackLeader 452 | + % or hasLeader if it thinks it has a leader already 453 | {E#election.name,Pendack} ! {halt,E#election.elid,self()}, 454 | NewE#election{pendack = Pendack}; 455 | false -> 456 | @@ -1001,9 +1146,9 @@ 457 | %% Star monitoring one Process 458 | mon_node(E,Proc) -> 459 | Node = case Proc of 460 | - {Name,Node_} -> 461 | + {_Name,Node_} -> 462 | Node_; 463 | - Pid when pid(Pid) -> 464 | + Pid when is_pid(Pid) -> 465 | node(Pid) 466 | end, 467 | case iselem(Node,E#election.monitored) of 468 | @@ -1014,7 +1159,6 @@ 469 | E#election{monitored = [{Ref,Node} | E#election.monitored]} 470 | end. 471 | 472 | - 473 | %% Stop monitoring of a bunch of nodes 474 | %demon_nodes(E) -> 475 | % foreach(fun({R,_}) -> 476 | @@ -1030,7 +1174,8 @@ 477 | lists:foreach( 478 | fun(Node) -> 479 | {E#election.name,Node} ! 480 | - {ldr, Synch, E#election.elid, self()} 481 | + {ldr, Synch, E#election.candidate_nodes, E#election.worker_nodes, 482 | + E#election.elid, self()} 483 | end,E#election.acks), 484 | 485 | % Make sure we will try to contact all workers! 486 | @@ -1058,15 +1203,20 @@ 487 | % Atomicity: This approach is safe as long as there is only 488 | % one gen_leader running per node. 489 | % 490 | -incarnation(Node) -> 491 | - case file:read_file_info(Node) of 492 | - {error,Reason} -> 493 | - ok = file:write_file(Node,term_to_binary(1)), 494 | +% ADT - modified to use the locally registered 495 | +% name in addition to the nodename to allow multiple gen_leaders 496 | +% per node 497 | +% 498 | +incarnation(Name, Node) -> 499 | + FileName = lists:flatten(io_lib:format(".~p.~p-~p", [?MODULE, Name, Node])), 500 | + case file:read_file_info(FileName) of 501 | + {error,_Reason} -> 502 | + ok = file:write_file(FileName,term_to_binary(1)), 503 | 0; 504 | {ok,_} -> 505 | - {ok,Bin} = file:read_file(Node), 506 | + {ok,Bin} = file:read_file(FileName), 507 | Incarn = binary_to_term(Bin), 508 | - ok = file:write_file(Node,term_to_binary(Incarn+1)), 509 | + ok = file:write_file(FileName,term_to_binary(Incarn+1)), 510 | Incarn 511 | end. 512 | 513 | @@ -1098,6 +1248,7 @@ 514 | lesser(N,[M|Ms]) -> 515 | [M|lesser(N,Ms)]. 516 | 517 | +%% returns the next element after N in a list 518 | next(_,[]) -> 519 | no_val; 520 | next(N,[N|Ms]) -> 521 | @@ -1105,6 +1256,7 @@ 522 | next(N,[_|Ms]) -> 523 | next(N,Ms). 524 | 525 | +%% find the posiion of N1 in a list 526 | pos(N1,[N1|_]) -> 527 | 1; 528 | pos(N1,[_|Ns]) -> 529 | -------------------------------------------------------------------------------- /original_version/gen_leader.erl: -------------------------------------------------------------------------------- 1 | %% ``The contents of this file are subject to the Erlang Public License, 2 | %% Version 1.1, (the "License"); you may not use this file except in 3 | %% compliance with the License. You should have received a copy of the 4 | %% Erlang Public License along with this software. If not, it can be 5 | %% retrieved via the world wide web at http://www.erlang.org/. 6 | %% 7 | %% Software distributed under the License is distributed on an "AS IS" 8 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 | %% the License for the specific language governing rights and limitations 10 | %% under the License. 11 | %% 12 | %% The Initial Developer of the Original Code is Ericsson Utvecklings AB. 13 | %% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings 14 | %% AB. All Rights Reserved.'' 15 | %% 16 | %% 17 | %% $Id: gen_leader.erl,v 1.2 2005/07/04 06:55:56 hanssv Exp $ 18 | %% 19 | %% @author Hans Svensson 20 | %% @author Thomas Arts 21 | %% @author Ulf Wiger 22 | %% 23 | %% @doc Leader election behavior. 24 | %%

This application implements a leader election behavior modeled after 25 | %% gen_server. This behavior intends to make it reasonably 26 | %% straightforward to implement a fully distributed server with 27 | %% master-slave semantics.

28 | %%

The gen_leader behavior supports nearly everything that gen_server 29 | %% does (some functions, such as multicall() and the internal timeout, 30 | %% have been removed), and adds a few callbacks and API functions to 31 | %% support leader election etc.

32 | %%

Also included is an example program, a global dictionary, based 33 | %% on the modules gen_leader and dict. The callback implementing the 34 | %% global dictionary is called 'test_cb', for no particularly logical 35 | %% reason.

36 | %%

New version: The internal leader election algorithm was faulty 37 | %% and has been replaced with a new version based on a different leader 38 | %% election algorithm. As a consequence of this the query functions 39 | %% alive and down can no longer be provided. 40 | %% The new algorithm also make use of an incarnation parameter, by 41 | %% default written to disk in the function incarnation. This 42 | %% implies that only one gen_leader per node is permitted, if 43 | %% used in a diskless environment, incarnation must be adapted. 44 | %%

45 | %% @end 46 | %% 47 | %% @type election() = tuple(). Opaque state of the gen_leader behaviour. 48 | %% @type node() = atom(). A node name. 49 | %% @type name() = atom(). A locally registered name. 50 | %% @type serverRef() = Name | {name(),node()} | {global,Name} | pid(). 51 | %% See gen_server. 52 | %% @type callerRef() = {pid(), reference()}. See gen_server. 53 | %% 54 | -module(gen_leader). 55 | 56 | %% Time between rounds of query from the leader 57 | -define(TAU,250). 58 | 59 | %% Exports for quickcheck 60 | %%-export([safe_loop/4,loop/4]). 61 | 62 | -export([start/6, 63 | start_link/6, 64 | leader_call/2, leader_call/3, leader_cast/2, 65 | call/2, call/3, cast/2, 66 | reply/2]). 67 | 68 | %% Query functions 69 | -export([%% alive/1, 70 | %% down/1, 71 | candidates/1, 72 | workers/1]). 73 | 74 | -export([ 75 | system_continue/3, 76 | system_terminate/4, 77 | system_code_change/4, 78 | format_status/2 79 | ]). 80 | 81 | -export([behaviour_info/1]). 82 | 83 | %% Internal exports 84 | -export([init_it/6, print_event/3 85 | %%, safe_send/2 86 | ]). 87 | 88 | -import(error_logger , [format/2]). 89 | -import(lists, [foldl/3, 90 | foreach/2, 91 | member/2, 92 | keydelete/3, 93 | keysearch/3]). 94 | 95 | %% Include for QuickCheck 96 | %% -include("eqc.hrl"). 97 | 98 | -record(election,{leader = none, 99 | name, 100 | leadernode = none, 101 | candidate_nodes = [], 102 | worker_nodes = [], 103 | alive = [], 104 | down = [], 105 | monitored = [], 106 | buffered = [], 107 | status, 108 | elid, 109 | acks = [], 110 | work_down = [], 111 | pendack, 112 | incarn, 113 | nextel 114 | }). 115 | 116 | -record(server, {parent, 117 | mod, 118 | state, 119 | debug}). 120 | 121 | 122 | %%% --------------------------------------------------- 123 | %%% Interface functions. 124 | %%% --------------------------------------------------- 125 | 126 | %% @hidden 127 | behaviour_info(callbacks) -> 128 | [{init,1}, 129 | {elected,2}, 130 | {surrendered,3}, 131 | {handle_leader_call,4}, 132 | {handle_leader_cast,3}, 133 | {from_leader,3}, 134 | {handle_call,3}, 135 | {handle_cast,2}, 136 | {handle_DOWN,3}, 137 | {handle_info,2}, 138 | {terminate,2}, 139 | {code_change,4}]; 140 | behaviour_info(_Other) -> 141 | undefined. 142 | 143 | %% @spec start(Name::node(), CandidateNodes::[node()], 144 | %% Workers::[node()], Mod::atom(), Arg, Options::list()) -> 145 | %% {ok,pid()} 146 | %% 147 | %% @doc Starts a gen_leader process without linking to the parent. 148 | %% 149 | start(Name, CandidateNodes, Workers, Mod, Arg, Options) when is_atom(Name) -> 150 | gen:start(?MODULE, nolink, {local,Name}, 151 | Mod, {CandidateNodes, Workers, Arg}, Options). 152 | 153 | %% @spec start_link(Name::atom(), CandidateNodes::[atom()], 154 | %% Workers::[atom()], Mod::atom(), Arg, Options::list()) -> 155 | %% {ok, pid()} 156 | %% 157 | %% @doc Starts a gen_leader process. 158 | %% 159 | %% 160 | %% 162 | %% 163 | %% 165 | %% 166 | %% 167 | %% 168 | %%
NameThe locally registered name of the process
CandidateNodesThe names of nodes capable of assuming 161 | %% a leadership role
WorkersThe names of nodes that will be part of the "cluster", 164 | %% but cannot ever assume a leadership role.
ModThe name of the callback module
ArgArgument passed on to Mod:init/1
OptionsSame as gen_server's Options
169 | %% 170 | %%

The list of candidates needs to be known from the start. Workers 171 | %% can be added at runtime.

172 | %% @end 173 | start_link(Name, CandidateNodes, Workers, 174 | Mod, Arg, Options) when is_atom(Name) -> 175 | %% Random delay for QuickCheck 176 | %% timer:sleep(random:uniform(400)), 177 | gen:start(?MODULE, link, {local,Name}, Mod, 178 | {CandidateNodes, Workers, Arg}, Options). 179 | 180 | 181 | %% Query functions to be used from the callback module 182 | 183 | %% alive(#election{alive = Alive}) -> 184 | %% Alive. 185 | 186 | %% down(#election{down = Down}) -> 187 | %% Down. 188 | 189 | %% @spec candidates(E::election()) -> [node()] 190 | %% 191 | %% @doc Returns a list of known candidates. 192 | %% 193 | candidates(#election{candidate_nodes = Cands}) -> 194 | Cands. 195 | 196 | %% @spec workers(E::election()) -> [node()] 197 | %% 198 | %% @doc Returns a list of known workers. 199 | %% 200 | workers(#election{worker_nodes = Workers}) -> 201 | Workers. 202 | 203 | %% 204 | %% Make a call to a generic server. 205 | %% If the server is located at another node, that node will 206 | %% be monitored. 207 | %% If the client is trapping exits and is linked server termination 208 | %% is handled here (? Shall we do that here (or rely on timeouts) ?). 209 | %% 210 | %% @spec call(Name::serverRef(), Request) -> term() 211 | %% 212 | %% @doc Equivalent to gen_server:call/2, but with a slightly 213 | %% different exit reason if something goes wrong. This function calls 214 | %% the gen_leader process exactly as if it were a gen_server 215 | %% (which, for practical purposes, it is.) 216 | %% @end 217 | call(Name, Request) -> 218 | case catch gen:call(Name, '$gen_call', Request) of 219 | {ok,Res} -> 220 | Res; 221 | {'EXIT',Reason} -> 222 | exit({Reason, {?MODULE, local_call, [Name, Request]}}) 223 | end. 224 | 225 | %% @spec call(Name::serverRef(), Request, Timeout::integer()) -> 226 | %% Reply 227 | %% 228 | %% Reply = term() 229 | %% 230 | %% @doc Equivalent to gen_server:call/3, but with a slightly 231 | %% different exit reason if something goes wrong. This function calls 232 | %% the gen_leader process exactly as if it were a gen_server 233 | %% (which, for practical purposes, it is.) 234 | %% @end 235 | call(Name, Request, Timeout) -> 236 | case catch gen:call(Name, '$gen_call', Request, Timeout) of 237 | {ok,Res} -> 238 | Res; 239 | {'EXIT',Reason} -> 240 | exit({Reason, {?MODULE, local_call, [Name, Request, Timeout]}}) 241 | end. 242 | 243 | %% @spec leader_call(Name::name(), Request::term()) 244 | %% -> Reply 245 | %% 246 | %% Reply = term() 247 | %% 248 | %% @doc Makes a call (similar to gen_server:call/2) to the 249 | %% leader. The call is forwarded via the local gen_leader instance, if 250 | %% that one isn't actually the leader. The client will exit if the 251 | %% leader dies while the request is outstanding. 252 | %%

This function uses gen:call/3, and is subject to the 253 | %% same default timeout as e.g. gen_server:call/2.

254 | %% @end 255 | %% 256 | leader_call(Name, Request) -> 257 | case catch gen:call(Name, '$leader_call', Request) of 258 | {ok,{leader,reply,Res}} -> 259 | Res; 260 | {ok,{error, leader_died}} -> 261 | exit({leader_died, {?MODULE, leader_call, [Name, Request]}}); 262 | {'EXIT',Reason} -> 263 | exit({Reason, {?MODULE, leader_call, [Name, Request]}}) 264 | end. 265 | 266 | %% @spec leader_call(Name::name(), Request::term(), Timeout::integer()) 267 | %% -> Reply 268 | %% 269 | %% Reply = term() 270 | %% 271 | %% @doc Makes a call (similar to gen_server:call/3) to the 272 | %% leader. The call is forwarded via the local gen_leader instance, if 273 | %% that one isn't actually the leader. The client will exit if the 274 | %% leader dies while the request is outstanding. 275 | %% @end 276 | %% 277 | leader_call(Name, Request, Timeout) -> 278 | case catch gen:call(Name, '$leader_call', Request, Timeout) of 279 | {ok,{leader,reply,Res}} -> 280 | Res; 281 | {'EXIT',Reason} -> 282 | exit({Reason, {?MODULE, leader_call, [Name, Request, Timeout]}}) 283 | end. 284 | 285 | 286 | %% @equiv gen_server:cast/2 287 | cast(Name, Request) -> 288 | catch do_cast('$gen_cast', Name, Request), 289 | ok. 290 | 291 | %% @spec leader_cast(Name::name(), Msg::term()) -> ok 292 | %% @doc Similar to gen_server:cast/2 but will be forwarded to 293 | %% the leader via the local gen_leader instance. 294 | leader_cast(Name, Request) -> 295 | catch do_cast('$leader_cast', Name, Request), 296 | ok. 297 | 298 | 299 | do_cast(Tag, Name, Request) when atom(Name) -> 300 | Name ! {Tag, Request}; 301 | do_cast(Tag, Pid, Request) when pid(Pid) -> 302 | Pid ! {Tag, Request}. 303 | 304 | 305 | %% @spec reply(From::callerRef(), Reply::term()) -> Void 306 | %% @equiv gen_server:reply/2 307 | reply({To, Tag}, Reply) -> 308 | catch To ! {Tag, Reply}. 309 | 310 | 311 | %%% --------------------------------------------------- 312 | %%% Initiate the new process. 313 | %%% Register the name using the Rfunc function 314 | %%% Calls the Mod:init/Args function. 315 | %%% Finally an acknowledge is sent to Parent and the main 316 | %%% loop is entered. 317 | %%% --------------------------------------------------- 318 | %%% @hidden 319 | init_it(Starter, self, Name, Mod, {CandidateNodes, Workers, Arg}, Options) -> 320 | init_it(Starter, self(), Name, Mod, 321 | {CandidateNodes, Workers, Arg}, Options); 322 | init_it(Starter,Parent,Name,Mod,{CandidateNodes,Workers,Arg},Options) -> 323 | 324 | %% The following row is needed in case of trace analysis, 325 | %% starting tracing is too slow otherwise! 326 | %% receive after 100 -> ok end, 327 | 328 | Debug = debug_options(Name, Options), 329 | 330 | AmCandidate = member(node(), CandidateNodes), 331 | 332 | Election = #election{candidate_nodes = CandidateNodes, 333 | worker_nodes = Workers, 334 | name = Name, 335 | nextel = 0}, 336 | 337 | case {catch Mod:init(Arg), AmCandidate} of 338 | {{stop, Reason},_} -> 339 | proc_lib:init_ack(Starter, {error, Reason}), 340 | exit(Reason); 341 | {ignore,_} -> 342 | proc_lib:init_ack(Starter, ignore), 343 | exit(normal); 344 | {{'EXIT', Reason},_} -> 345 | proc_lib:init_ack(Starter, {error, Reason}), 346 | exit(Reason); 347 | {{ok, State}, true} -> 348 | NewE = startStage1(Election#election{incarn = incarnation(node())}), 349 | 350 | proc_lib:init_ack(Starter, {ok, self()}), 351 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 352 | candidate, NewE,{init}); 353 | {{ok, State}, false} -> 354 | proc_lib:init_ack(Starter, {ok, self()}), 355 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 356 | waiting_worker, Election,{init}); 357 | Else -> 358 | Error = {bad_return_value, Else}, 359 | proc_lib:init_ack(Starter, {error, Error}), 360 | exit(Error) 361 | end. 362 | 363 | 364 | 365 | 366 | %%% --------------------------------------------------- 367 | %%% The MAIN loops. 368 | %%% --------------------------------------------------- 369 | 370 | 371 | safe_loop(#server{mod = Mod, state = State} = Server, Role, 372 | #election{name = Name} = E, PrevMsg) -> 373 | %% Event for QuickCheck 374 | %% ?EVENT({Role,E}), 375 | receive 376 | {system, From, Req} -> 377 | #server{parent = Parent, debug = Debug} = Server, 378 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 379 | [safe, Server, Role, E]); 380 | {'EXIT', _, Reason} = Msg -> 381 | terminate(Reason, Msg, Server, Role, E); 382 | {halt,T,From} = Msg -> 383 | NewE = halting(E,T,From), 384 | From ! {ackLeader,T,self()}, 385 | safe_loop(Server,Role,NewE,Msg); 386 | {hasLeader,Ldr,T,_} = Msg -> 387 | NewE1 = mon_node(E,Ldr), 388 | case ( (E#election.status == elec2) and (E#election.acks /= []) ) of 389 | true -> 390 | lists:foreach( 391 | fun(Node) -> 392 | {Name,Node} ! {hasLeader,Ldr,T,self()} 393 | end,E#election.acks); 394 | false -> 395 | ok 396 | end, 397 | NewE = NewE1#election{elid = T, 398 | status = wait, 399 | leadernode = node(Ldr), 400 | down = E#election.down -- [node(Ldr)], 401 | acks = []}, 402 | Ldr ! {isLeader,T,self()}, 403 | safe_loop(Server,Role,NewE,Msg); 404 | {isLeader,T,From} = Msg -> 405 | From ! {notLeader,T,self()}, 406 | safe_loop(Server,Role,E,Msg); 407 | {notLeader,T,_} = Msg -> 408 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 409 | true -> 410 | NewE = startStage1(E); 411 | false -> 412 | NewE = E 413 | end, 414 | safe_loop(Server,Role,NewE,Msg); 415 | {ackLeader,T,From} = Msg -> 416 | case ( (E#election.status == elec2) and (E#election.elid == T) and 417 | (E#election.pendack == node(From)) ) of 418 | true -> 419 | NewE = continStage2(E#election{acks = [node(From)|E#election.acks]}); 420 | false -> 421 | NewE = E 422 | end, 423 | hasBecomeLeader(NewE,Server,Msg); 424 | {ldr,Synch,T,From} = Msg -> 425 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 426 | true -> 427 | NewE1 = mon_node(E,From), 428 | NewE = NewE1#election{leader = From, 429 | leadernode = node(From), 430 | status = norm}, 431 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 432 | loop(Server#server{state = NewState},surrendered,NewE,Msg); 433 | false -> 434 | safe_loop(Server,Role,E,Msg) 435 | end; 436 | {normQ,T,From} = Msg -> 437 | case ( (E#election.status == elec1) or 438 | ( (E#election.status == wait) and (E#election.elid == T))) of 439 | true -> 440 | NewE = halting(E,T,From), 441 | From ! {notNorm,T,self()}; 442 | false -> 443 | NewE = E 444 | end, 445 | safe_loop(Server,Role,NewE,Msg); 446 | 447 | {notNorm,_,_} = Msg -> 448 | safe_loop(Server,Role,E,Msg); 449 | {workerAlive,T,From} = Msg -> 450 | case E#election.leadernode == none of 451 | true -> %% We should initiate activation, monitor the possible leader! 452 | NewE = mon_node(E#election{leadernode = node(From), 453 | elid = T}, 454 | From), 455 | From ! {workerIsAlive,T,self()}; 456 | false -> 457 | %% We should acutally ignore this, the present activation 458 | %% will complete or abort first... 459 | NewE = E 460 | end, 461 | safe_loop(Server,Role,NewE,Msg); 462 | {workerIsAlive,_,_} = Msg -> 463 | %% If this happens, the activation process should abort 464 | %% This process is no longer the leader! 465 | %% The sender will notice this via a DOWN message 466 | safe_loop(Server,Role,E,Msg); 467 | {activateWorker,T,Synch,From} = Msg -> 468 | case ( (T == E#election.elid) and (node(From) == E#election.leadernode)) of 469 | true -> 470 | NewE = E#election{ leader = From, 471 | status = worker }, 472 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 473 | loop(Server#server{state = NewState},worker,NewE,Msg); 474 | false -> 475 | %% This should be a VERY special case... 476 | %% But doing nothing is the right thing! 477 | %% A DOWN message should arrive to solve this situation 478 | safe_loop(Server,Role,E,Msg) 479 | end; 480 | 481 | {tau_timeout} = Msg -> 482 | safe_loop(Server,Role,E,Msg); 483 | {'DOWN',Ref,process,From,Reason} = Msg when Role == waiting_worker -> 484 | %% We are only monitoring one proc, the leader! 485 | Node = case From of 486 | {Name,_Node} -> _Node; 487 | _ when pid(From) -> node(From) 488 | end, 489 | case Node == E#election.leadernode of 490 | true -> 491 | NewE = E#election{ leader = none, leadernode = none, 492 | status = waiting_worker, 493 | monitored = []}; 494 | false -> 495 | NewE = E 496 | end, 497 | safe_loop(Server, Role, NewE,Msg); 498 | {'DOWN',Ref,process,From,Reason} = Msg -> 499 | Node = case From of 500 | {Name,_Node} -> _Node; 501 | _ when pid(From) -> node(From) 502 | end, 503 | NewMon = E#election.monitored -- [{Ref,Node}], 504 | case lists:member(Node,E#election.candidate_nodes) of 505 | true -> 506 | NewDown = [Node | E#election.down], 507 | E1 = E#election{down = NewDown, monitored = NewMon}, 508 | case ( pos(Node,E#election.candidate_nodes) < 509 | pos(node(),E#election.candidate_nodes) ) of 510 | true -> 511 | Lesser = lesser(node(),E#election.candidate_nodes), 512 | LesserIsSubset = (Lesser -- NewDown) == [], 513 | case ((E#election.status == wait) and (Node == E#election.leadernode)) of 514 | true -> 515 | NewE = startStage1(E1); 516 | false -> 517 | case ((E#election.status == elec1) and LesserIsSubset) of 518 | true -> 519 | NewE = startStage2(E1#election{down = Lesser}); 520 | false -> 521 | NewE = E1 522 | end 523 | end; 524 | false -> 525 | case ( (E#election.status == elec2) and (Node == E#election.pendack) ) of 526 | true -> 527 | NewE = continStage2(E1); 528 | false -> 529 | case ( (E#election.status == wait) and 530 | (Node == E#election.leadernode)) of 531 | true -> 532 | NewE = startStage1(E1); 533 | false -> 534 | NewE = E1 535 | end 536 | end 537 | end 538 | end, 539 | hasBecomeLeader(NewE,Server,Msg) 540 | end. 541 | 542 | 543 | loop(#server{parent = Parent, 544 | mod = Mod, 545 | state = State, 546 | debug = Debug} = Server, Role, #election{name = Name} = E, PrevMsg) -> 547 | %% Event for QuickCheck 548 | %% ?EVENT({Role,E}), 549 | receive 550 | Msg -> 551 | case Msg of 552 | {system, From, Req} -> 553 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 554 | [normal, Server, Role, E]); 555 | {'EXIT', Parent, Reason} -> 556 | terminate(Reason, Msg, Server, Role, E); 557 | 558 | {halt,_,From} -> 559 | From ! {hasLeader,E#election.leader,E#election.elid,self()}, 560 | loop(Server,Role,E,Msg); 561 | {hasLeader,_,_,_} -> 562 | loop(Server,Role,E,Msg); 563 | {isLeader,T,From} -> 564 | case (self() == E#election.leader) of 565 | true -> 566 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 567 | From), 568 | {ok,Synch,NewState} = Mod:elected(State,NewE), 569 | From ! {ldr,Synch,E#election.elid,self()}, 570 | loop(Server#server{state = NewState},Role,NewE,Msg); 571 | false -> 572 | From ! {notLeader,T,self()}, 573 | loop(Server,Role,E,Msg) 574 | end; 575 | {ackLeader,_,_} -> 576 | loop(Server,Role,E,Msg); 577 | {notLeader,_,_} -> 578 | loop(Server,Role,E,Msg); 579 | {ack,_,_} -> 580 | loop(Server,Role,E,Msg); 581 | {ldr,_,_,_} -> 582 | loop(Server,Role,E,Msg); 583 | {normQ,_,_} -> 584 | loop(Server,Role,E,Msg); 585 | {notNorm,T,From} -> 586 | case ( (E#election.leader == self()) and (E#election.elid == T) ) of 587 | true -> 588 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 589 | From), 590 | {ok,Synch,NewState} = Mod:elected(State,NewE), 591 | From ! {ldr,Synch,E#election.elid,self()}, 592 | loop(Server#server{state = NewState},Role,NewE,Msg); 593 | false -> 594 | loop(Server,Role,E,Msg) 595 | end; 596 | {workerAlive,_,_} -> 597 | %% Do nothing if we get this from a new leader 598 | %% We will soon notice that the prev leader has died, and 599 | %% get the same message again when we are back in safe_loop! 600 | loop(Server,Role,E,Msg); 601 | {activateWorker,_,_,_} -> 602 | %% We ignore this, we are already active... 603 | %% It must be an old message! 604 | loop(Server,Role,E,Msg); 605 | {workerIsAlive,T,From} -> 606 | case ((T == E#election.elid) and (self() == E#election.leader) 607 | %% and iselem(node(From),E#election.monitored) 608 | ) of 609 | true -> 610 | NewE = mon_node( 611 | E#election{work_down = E#election.work_down -- [node(From)]}, 612 | From), 613 | %% NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 614 | {ok,Synch,NewState} = Mod:elected(State,NewE), 615 | From ! {activateWorker,T,Synch,self()}, 616 | loop(Server#server{state = NewState},Role,NewE,Msg); 617 | false -> 618 | loop(Server,Role,E,Msg) 619 | end; 620 | {tau_timeout} -> 621 | case (E#election.leader == self()) of 622 | true -> 623 | lists:foreach( 624 | fun(Node) -> 625 | {Name,Node} ! {normQ,E#election.elid,self()} 626 | end,E#election.down -- [lists:nth(1,E#election.candidate_nodes)]), 627 | 628 | lists:foreach( 629 | fun(Node) -> 630 | {Name,Node} ! {workerAlive,E#election.elid,self()} 631 | end,E#election.work_down), 632 | 633 | timer:send_after(?TAU,{tau_timeout}); 634 | false -> 635 | ok 636 | end, 637 | loop(Server,Role,E,Msg); 638 | {'DOWN',Ref,process,From,Reason} when Role == worker -> 639 | %% We are only monitoring one proc, the leader! 640 | Node = case From of 641 | {Name,_Node} -> _Node; 642 | _ when pid(From) -> node(From) 643 | end, 644 | case Node == E#election.leadernode of 645 | true -> 646 | NewE = E#election{ leader = none, leadernode = none, 647 | status = waiting_worker, 648 | monitored = []}, 649 | safe_loop(Server, waiting_worker, NewE,Msg); 650 | false -> 651 | loop(Server, Role, E,Msg) 652 | end; 653 | {'DOWN',Ref,process,From,Reason} -> 654 | Node = case From of 655 | {Name,_Node} -> _Node; 656 | _ when pid(From) -> node(From) 657 | end, 658 | NewMon = E#election.monitored -- [{Ref,Node}], 659 | case lists:member(Node,E#election.candidate_nodes) of 660 | true -> 661 | NewDown = [Node | E#election.down], 662 | E1 = E#election{down = NewDown, monitored = NewMon}, 663 | case (Node == E#election.leadernode) of 664 | true -> 665 | NewE = startStage1(E1), 666 | safe_loop(Server, candidate, NewE,Msg); 667 | 668 | false -> 669 | loop(Server, Role, E1,Msg) 670 | end; 671 | false -> 672 | %% I am the leader, make sure the dead worker is in work_down. 673 | E1 = E#election{monitored = NewMon, 674 | work_down = [Node | (E#election.work_down -- [Node])] 675 | }, 676 | loop(Server, Role, E1,Msg) 677 | end; 678 | _Msg when Debug == [] -> 679 | handle_msg(Msg, Server, Role, E); 680 | _Msg -> 681 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 682 | E#election.name, {in, Msg}), 683 | handle_msg(Msg, Server#server{debug = Debug1}, Role, E) 684 | end 685 | end. 686 | 687 | %%----------------------------------------------------------------- 688 | %% Callback functions for system messages handling. 689 | %%----------------------------------------------------------------- 690 | %% @hidden 691 | system_continue(Parent, Debug, [safe, Server, Role, E]) -> 692 | safe_loop(Server, Role, E,{}); 693 | system_continue(Parent, Debug, [normal, Server, Role, E]) -> 694 | loop(Server, Role, E,{}). 695 | 696 | %% @hidden 697 | system_terminate(Reason, _Parent, Debug, [Mode, Server, Role, E]) -> 698 | terminate(Reason, [], Server, Role, E). 699 | 700 | %% @hidden 701 | system_code_change([Mode, Server, Role, E], _Module, OldVsn, Extra) -> 702 | #server{mod = Mod, state = State} = Server, 703 | case catch Mod:code_change(OldVsn, State, E, Extra) of 704 | {ok, NewState} -> 705 | NewServer = Server#server{state = NewState}, 706 | {ok, [Mode, NewServer, Role, E]}; 707 | {ok, NewState, NewE} -> 708 | NewServer = Server#server{state = NewState}, 709 | {ok, [Mode, NewServer, Role, NewE]}; 710 | Else -> Else 711 | end. 712 | 713 | %%----------------------------------------------------------------- 714 | %% Format debug messages. Print them as the call-back module sees 715 | %% them, not as the real erlang messages. Use trace for that. 716 | %%----------------------------------------------------------------- 717 | %% @hidden 718 | print_event(Dev, {in, Msg}, Name) -> 719 | case Msg of 720 | {'$gen_call', {From, _Tag}, Call} -> 721 | io:format(Dev, "*DBG* ~p got local call ~p from ~w~n", 722 | [Name, Call, From]); 723 | {'$leader_call', {From, _Tag}, Call} -> 724 | io:format(Dev, "*DBG* ~p got global call ~p from ~w~n", 725 | [Name, Call, From]); 726 | {'$gen_cast', Cast} -> 727 | io:format(Dev, "*DBG* ~p got local cast ~p~n", 728 | [Name, Cast]); 729 | {'$leader_cast', Cast} -> 730 | io:format(Dev, "*DBG* ~p got global cast ~p~n", 731 | [Name, Cast]); 732 | _ -> 733 | io:format(Dev, "*DBG* ~p got ~p~n", [Name, Msg]) 734 | end; 735 | print_event(Dev, {out, Msg, To, State}, Name) -> 736 | io:format(Dev, "*DBG* ~p sent ~p to ~w, new state ~w~n", 737 | [Name, Msg, To, State]); 738 | print_event(Dev, {noreply, State}, Name) -> 739 | io:format(Dev, "*DBG* ~p new state ~w~n", [Name, State]); 740 | print_event(Dev, Event, Name) -> 741 | io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). 742 | 743 | 744 | handle_msg({'$leader_call', From, Request} = Msg, 745 | #server{mod = Mod, state = State} = Server, elected = Role, E) -> 746 | case catch Mod:handle_leader_call(Request, From, State, E) of 747 | {reply, Reply, NState} -> 748 | NewServer = reply(From, {leader,reply,Reply}, 749 | Server#server{state = NState}, Role, E), 750 | loop(NewServer, Role, E,Msg); 751 | {reply, Reply, Broadcast, NState} -> 752 | NewE = broadcast({from_leader,Broadcast}, E), 753 | NewServer = reply(From, {leader,reply,Reply}, 754 | Server#server{state = NState}, Role, 755 | NewE), 756 | loop(NewServer, Role, NewE,Msg); 757 | {noreply, NState} = Reply -> 758 | NewServer = handle_debug(Server#server{state = NState}, 759 | Role, E, Reply), 760 | loop(NewServer, Role, E,Msg); 761 | {stop, Reason, Reply, NState} -> 762 | {'EXIT', R} = 763 | (catch terminate(Reason, Msg, 764 | Server#server{state = NState}, 765 | Role, E)), 766 | reply(From, Reply), 767 | exit(R); 768 | Other -> 769 | handle_common_reply(Other, Msg, Server, Role, E) 770 | end; 771 | handle_msg({from_leader, Cmd} = Msg, 772 | #server{mod = Mod, state = State} = Server, Role, E) -> 773 | handle_common_reply(catch Mod:from_leader(Cmd, State, E), 774 | Msg, Server, Role, E); 775 | handle_msg({'$leader_call', From, Request} = Msg, Server, Role, 776 | #election{buffered = Buffered, leader = Leader} = E) -> 777 | Ref = make_ref(), 778 | Leader ! {'$leader_call', {self(),Ref}, Request}, 779 | NewBuffered = [{Ref,From}|Buffered], 780 | loop(Server, Role, E#election{buffered = NewBuffered},Msg); 781 | handle_msg({Ref, {leader,reply,Reply}} = Msg, Server, Role, 782 | #election{buffered = Buffered} = E) -> 783 | {value, {_,From}} = keysearch(Ref,1,Buffered), 784 | NewServer = reply(From, {leader,reply,Reply}, Server, Role, 785 | E#election{buffered = keydelete(Ref,1,Buffered)}), 786 | loop(NewServer, Role, E, Msg); 787 | handle_msg({'$gen_call', From, Request} = Msg, 788 | #server{mod = Mod, state = State} = Server, Role, E) -> 789 | case catch Mod:handle_call(Request, From, State) of 790 | {reply, Reply, NState} -> 791 | NewServer = reply(From, Reply, 792 | Server#server{state = NState}, Role, E), 793 | loop(NewServer, Role, E, Msg); 794 | {noreply, NState} = Reply -> 795 | NewServer = handle_debug(Server#server{state = NState}, 796 | Role, E, Reply), 797 | loop(NewServer, Role, E, Msg); 798 | {stop, Reason, Reply, NState} -> 799 | {'EXIT', R} = 800 | (catch terminate(Reason, Msg, Server#server{state = NState}, 801 | Role, E)), 802 | reply(From, Reply), 803 | exit(R); 804 | Other -> 805 | handle_common_reply(Other, Msg, Server, Role, E) 806 | end; 807 | handle_msg({'$gen_cast',Msg} = Cast, 808 | #server{mod = Mod, state = State} = Server, Role, E) -> 809 | handle_common_reply(catch Mod:handle_cast(Msg, State), 810 | Cast, Server, Role, E); 811 | handle_msg(Msg, 812 | #server{mod = Mod, state = State} = Server, Role, E) -> 813 | handle_common_reply(catch Mod:handle_info(Msg, State), 814 | Msg, Server, Role, E). 815 | 816 | 817 | handle_common_reply(Reply, Msg, Server, Role, E) -> 818 | case Reply of 819 | {ok, NState} -> 820 | NewServer = handle_debug(Server#server{state = NState}, 821 | Role, E, Reply), 822 | loop(NewServer, Role, E, Msg); 823 | {stop, Reason, NState} -> 824 | terminate(Reason, Msg, Server#server{state = NState}, Role, E); 825 | {'EXIT', Reason} -> 826 | terminate(Reason, Msg, Server, Role, E); 827 | _ -> 828 | terminate({bad2_return_value, Reply}, Msg, Server, Role, E) 829 | end. 830 | 831 | 832 | reply({To, Tag}, Reply, #server{state = State} = Server, Role, E) -> 833 | reply({To, Tag}, Reply), 834 | handle_debug(Server, Role, E, {out, Reply, To, State}). 835 | 836 | 837 | handle_debug(#server{debug = []} = Server, _Role, _E, Event) -> 838 | Server; 839 | handle_debug(#server{debug = Debug} = Server, Role, E, Event) -> 840 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 841 | E#election.name, Event), 842 | Server#server{debug = Debug1}. 843 | 844 | %%% --------------------------------------------------- 845 | %%% Terminate the server. 846 | %%% --------------------------------------------------- 847 | 848 | terminate(Reason, Msg, #server{mod = Mod, 849 | state = State, 850 | debug = Debug} = Server, Role, 851 | #election{name = Name} = E) -> 852 | case catch Mod:terminate(Reason, State) of 853 | {'EXIT', R} -> 854 | error_info(R, Name, Msg, State, Debug), 855 | exit(R); 856 | _ -> 857 | case Reason of 858 | normal -> 859 | exit(normal); 860 | shutdown -> 861 | exit(shutdown); 862 | _ -> 863 | error_info(Reason, Name, Msg, State, Debug), 864 | exit(Reason) 865 | end 866 | end. 867 | 868 | %% Maybe we shouldn't do this? We have the crash report... 869 | error_info(Reason, Name, Msg, State, Debug) -> 870 | format("** Generic leader ~p terminating \n" 871 | "** Last message in was ~p~n" 872 | "** When Server state == ~p~n" 873 | "** Reason for termination == ~n** ~p~n", 874 | [Name, Msg, State, Reason]), 875 | sys:print_log(Debug), 876 | ok. 877 | 878 | %%% --------------------------------------------------- 879 | %%% Misc. functions. 880 | %%% --------------------------------------------------- 881 | 882 | opt(Op, [{Op, Value}|_]) -> 883 | {ok, Value}; 884 | opt(Op, [_|Options]) -> 885 | opt(Op, Options); 886 | opt(_, []) -> 887 | false. 888 | 889 | debug_options(Name, Opts) -> 890 | case opt(debug, Opts) of 891 | {ok, Options} -> dbg_options(Name, Options); 892 | _ -> dbg_options(Name, []) 893 | end. 894 | 895 | dbg_options(Name, []) -> 896 | Opts = 897 | case init:get_argument(generic_debug) of 898 | error -> 899 | []; 900 | _ -> 901 | [log, statistics] 902 | end, 903 | dbg_opts(Name, Opts); 904 | dbg_options(Name, Opts) -> 905 | dbg_opts(Name, Opts). 906 | 907 | dbg_opts(Name, Opts) -> 908 | case catch sys:debug_options(Opts) of 909 | {'EXIT',_} -> 910 | format("~p: ignoring erroneous debug options - ~p~n", 911 | [Name, Opts]), 912 | []; 913 | Dbg -> 914 | Dbg 915 | end. 916 | 917 | %%----------------------------------------------------------------- 918 | %% Status information 919 | %%----------------------------------------------------------------- 920 | %% @hidden 921 | format_status(Opt, StatusData) -> 922 | [PDict, SysState, Parent, Debug, [Mode, Server, Role, E]] = StatusData, 923 | Header = lists:concat(["Status for generic server ", E#election.name]), 924 | Log = sys:get_debug(log, Debug, []), 925 | #server{mod = Mod, state = State} = Server, 926 | Specific = 927 | case erlang:function_exported(Mod, format_status, 2) of 928 | true -> 929 | case catch apply(Mod, format_status, [Opt, [PDict, State]]) of 930 | {'EXIT', _} -> [{data, [{"State", State}]}]; 931 | Else -> Else 932 | end; 933 | _ -> 934 | [{data, [{"State", State}]}] 935 | end, 936 | [{header, Header}, 937 | {data, [{"Status", SysState}, 938 | {"Parent", Parent}, 939 | {"Logged events", Log}]} | 940 | Specific]. 941 | 942 | 943 | %%----------------------------------------------------------------- 944 | %% Leader-election functions 945 | %%----------------------------------------------------------------- 946 | 947 | %% Corresponds to startStage1 in Figure 1 in the Stoller-article 948 | startStage1(E) -> 949 | Elid = {pos(node(),E#election.candidate_nodes),E#election.incarn,E#election.nextel}, 950 | NewE = E#election{ 951 | elid = Elid, 952 | nextel = E#election.nextel + 1, 953 | down = [], 954 | status = elec1}, 955 | case ( pos(node(),E#election.candidate_nodes) == 1) of 956 | true -> 957 | startStage2(NewE); 958 | false -> 959 | mon_nodes(NewE,lesser(node(),E#election.candidate_nodes)) 960 | end. 961 | 962 | %% Corresponds to startStage2 963 | startStage2(E) -> 964 | continStage2(E#election{ 965 | status = elec2, 966 | pendack = node(), 967 | acks = []}). 968 | 969 | continStage2(E) -> 970 | case pos(E#election.pendack,E#election.candidate_nodes) < length(E#election.candidate_nodes) of 971 | true -> 972 | Pendack = next(E#election.pendack,E#election.candidate_nodes), 973 | NewE = mon_nodes(E,[Pendack]), 974 | {E#election.name,Pendack} ! {halt,E#election.elid,self()}, 975 | NewE#election{pendack = Pendack}; 976 | false -> 977 | %% I am the leader 978 | %% io:format("I am the leader (Node ~w) ~n", [node()]), 979 | E#election{leader = self(), 980 | leadernode = node(), 981 | status = norm} 982 | end. 983 | 984 | %% corresponds to Halting 985 | halting(E,T,From) -> 986 | NewE = mon_node(E,From), 987 | NewE#election{elid = T, 988 | status = wait, 989 | leadernode = node(From), 990 | down = E#election.down -- [node(From)] 991 | }. 992 | 993 | %% Start monitor a bunch of nodes 994 | mon_nodes(E,Nodes) -> 995 | foldl( 996 | fun(Node,_E) -> 997 | mon_node(_E,{_E#election.name,Node}) 998 | end,E,Nodes). 999 | 1000 | %% Star monitoring one Process 1001 | mon_node(E,Proc) -> 1002 | Node = case Proc of 1003 | {Name,Node_} -> 1004 | Node_; 1005 | Pid when pid(Pid) -> 1006 | node(Pid) 1007 | end, 1008 | case iselem(Node,E#election.monitored) of 1009 | true -> 1010 | E; 1011 | false -> 1012 | Ref = erlang:monitor(process,Proc), 1013 | E#election{monitored = [{Ref,Node} | E#election.monitored]} 1014 | end. 1015 | 1016 | 1017 | %% Stop monitoring of a bunch of nodes 1018 | %% demon_nodes(E) -> 1019 | %% foreach(fun({R,_}) -> 1020 | %% erlang:demonitor(R) 1021 | %% end,E#election.monitored), 1022 | %% E#election{monitored = []}. 1023 | 1024 | %% checks if the proc has become the leader, if so switch to loop 1025 | hasBecomeLeader(E,Server,Msg) -> 1026 | case ((E#election.status == norm) and (E#election.leader == self())) of 1027 | true -> 1028 | {ok,Synch,NewState} = (Server#server.mod):elected(Server#server.state,E), 1029 | lists:foreach( 1030 | fun(Node) -> 1031 | {E#election.name,Node} ! 1032 | {ldr, Synch, E#election.elid, self()} 1033 | end,E#election.acks), 1034 | 1035 | %% Make sure we will try to contact all workers! 1036 | NewE = E#election{work_down = E#election.worker_nodes}, 1037 | 1038 | %% Set the internal timeout (corresponds to Periodically) 1039 | timer:send_after(?TAU,{tau_timeout}), %It's meaningful only when I am the leader! 1040 | loop(Server#server{state = NewState},elected,NewE,Msg); 1041 | false -> 1042 | safe_loop(Server,candidate,E,Msg) 1043 | end. 1044 | 1045 | 1046 | 1047 | 1048 | %%% 1049 | %% 1050 | %% incarnation should return an integer value for the next 1051 | %% incarnation of this node. We create a file for each node, 1052 | %% this file contains a counter. When starting the system for the 1053 | %% first time, the files should be intialized with 0 incarnation 1054 | %% counter for all nodes orelse be removed, since we create 1055 | %% files if not present with counter 1. 1056 | %% 1057 | %% Atomicity: This approach is safe as long as there is only 1058 | %% one gen_leader running per node. 1059 | %% 1060 | incarnation(Node) -> 1061 | case file:read_file_info(Node) of 1062 | {error,Reason} -> 1063 | ok = file:write_file(Node,term_to_binary(1)), 1064 | 0; 1065 | {ok,_} -> 1066 | {ok,Bin} = file:read_file(Node), 1067 | Incarn = binary_to_term(Bin), 1068 | ok = file:write_file(Node,term_to_binary(Incarn+1)), 1069 | Incarn 1070 | end. 1071 | 1072 | 1073 | broadcast(Msg, #election{monitored = Monitored} = E) -> 1074 | %% This function is used for broadcasts, 1075 | %% and we make sure only to broadcast to already known nodes. 1076 | ToNodes = [N || {_,N} <- Monitored], 1077 | broadcast(Msg, ToNodes, E). 1078 | 1079 | broadcast({from_leader, Msg}, ToNodes, E) -> 1080 | foreach( 1081 | fun(Node) -> 1082 | {E#election.name,Node} ! {from_leader, Msg} 1083 | end,ToNodes), 1084 | E. 1085 | 1086 | iselem(_,[]) -> 1087 | false; 1088 | iselem(P,[{_,P}|_]) -> 1089 | true; 1090 | iselem(P,[_ | Ns]) -> 1091 | iselem(P,Ns). 1092 | 1093 | lesser(_,[]) -> 1094 | []; 1095 | lesser(N,[N|_]) -> 1096 | []; 1097 | lesser(N,[M|Ms]) -> 1098 | [M|lesser(N,Ms)]. 1099 | 1100 | next(_,[]) -> 1101 | no_val; 1102 | next(N,[N|Ms]) -> 1103 | lists:nth(1,Ms); 1104 | next(N,[_|Ms]) -> 1105 | next(N,Ms). 1106 | 1107 | pos(N1,[N1|_]) -> 1108 | 1; 1109 | pos(N1,[_|Ns]) -> 1110 | 1+pos(N1,Ns). 1111 | -------------------------------------------------------------------------------- /powerset_version/gen_leader.erl: -------------------------------------------------------------------------------- 1 | %% ``The contents of this file are subject to the Erlang Public License, 2 | %% Version 1.1, (the "License"); you may not use this file except in 3 | %% compliance with the License. You should have received a copy of the 4 | %% Erlang Public License along with this software. If not, it can be 5 | %% retrieved via the world wide web at http://www.erlang.org/. 6 | %% 7 | %% Software distributed under the License is distributed on an "AS IS" 8 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 | %% the License for the specific language governing rights and limitations 10 | %% under the License. 11 | %% 12 | %% The Initial Developer of the Original Code is Ericsson Utvecklings AB. 13 | %% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings 14 | %% AB. All Rights Reserved.'' 15 | %% 16 | %% 17 | %% $Id: gen_leader.erl,v 1.2 2005/07/04 06:55:56 hanssv Exp $ 18 | %% 19 | %% @author Hans Svensson 20 | %% @author Thomas Arts 21 | %% @author Ulf Wiger 22 | %% @author Dave Fayram 23 | %% 24 | %% @doc Leader election behavior. 25 | %%

This application implements a leader election behavior modeled after 26 | %% gen_server. This behavior intends to make it reasonably 27 | %% straightforward to implement a fully distributed server with 28 | %% master-slave semantics.

29 | %%

The gen_leader behavior supports nearly everything that gen_server 30 | %% does (some functions, such as multicall() and the internal timeout, 31 | %% have been removed), and adds a few callbacks and API functions to 32 | %% support leader election etc.

33 | %%

Also included is an example program, a global dictionary, based 34 | %% on the modules gen_leader and dict. The callback implementing the 35 | %% global dictionary is called 'test_cb', for no particularly logical 36 | %% reason.

37 | %%

New version: The internal leader election algorithm was faulty 38 | %% and has been replaced with a new version based on a different leader 39 | %% election algorithm. As a consequence of this the query functions 40 | %% alive and down can no longer be provided. 41 | %% The new algorithm also make use of an incarnation parameter, by 42 | %% default written to disk in the function incarnation. This 43 | %% implies that only one gen_leader per node is permitted, if 44 | %% used in a diskless environment, incarnation must be adapted. 45 | %%

46 | %% @end 47 | %% 48 | %% @type election() = tuple(). Opaque state of the gen_leader behaviour. 49 | %% @type node() = atom(). A node name. 50 | %% @type name() = atom(). A locally registered name. 51 | %% @type serverRef() = Name | {name(),node()} | {global,Name} | pid(). 52 | %% See gen_server. 53 | %% @type callerRef() = {pid(), reference()}. See gen_server. 54 | %% 55 | 56 | %% This version has been modified and debugged by Dave Fayram for 57 | %% Powerset/Microsoft. 58 | -module(gen_leader). 59 | 60 | % Time between rounds of query from the leader 61 | -define(TAU,250). 62 | 63 | % Exports for quickcheck 64 | %-export([safe_loop/4,loop/4]). 65 | 66 | -export([start/6, 67 | start_link/6, 68 | leader_call/2, leader_call/3, leader_cast/2, worker_announce/2, 69 | call/2, call/3, cast/2, 70 | reply/2]). 71 | 72 | %% Query functions 73 | -export([%% alive/1, 74 | %% down/1, 75 | candidates/1, 76 | workers/1]). 77 | 78 | -export([ 79 | system_continue/3, 80 | system_terminate/4, 81 | system_code_change/4, 82 | format_status/2 83 | ]). 84 | 85 | -export([behaviour_info/1]). 86 | 87 | %% Internal exports 88 | -export([init_it/6, print_event/3 89 | %%, safe_send/2 90 | ]). 91 | 92 | -import(error_logger , [format/2]). 93 | -import(lists, [foldl/3, 94 | foreach/2, 95 | member/2, 96 | keydelete/3, 97 | keysearch/3]). 98 | 99 | % Include for QuickCheck 100 | % -include("eqc.hrl"). 101 | 102 | -record(election,{leader = none, 103 | name, 104 | leadernode = none, 105 | candidate_nodes = [], 106 | worker_nodes = [], 107 | alive = [], 108 | down = [], 109 | monitored = [], 110 | buffered = [], 111 | status, 112 | elid, 113 | acks = [], 114 | work_down = [], 115 | pendack, 116 | incarn, 117 | nextel 118 | }). 119 | 120 | -record(server, {parent, 121 | mod, 122 | state, 123 | debug}). 124 | 125 | 126 | %%% --------------------------------------------------- 127 | %%% Interface functions. 128 | %%% --------------------------------------------------- 129 | 130 | %% @hidden 131 | behaviour_info(callbacks) -> 132 | [{init,1}, 133 | {elected,2}, 134 | {surrendered,3}, 135 | {handle_leader_call,4}, 136 | {handle_leader_cast,3}, 137 | {from_leader,3}, 138 | {handle_call,3}, 139 | {handle_cast,2}, 140 | {handle_DOWN,3}, 141 | {handle_info,2}, 142 | {terminate,2}, 143 | {code_change,4}]; 144 | behaviour_info(_Other) -> 145 | undefined. 146 | 147 | %% @spec start(Name::node(), CandidateNodes::[node()], 148 | %% Workers::[node()], Mod::atom(), Arg, Options::list()) -> 149 | %% {ok,pid()} 150 | %% 151 | %% @doc Starts a gen_leader process without linking to the parent. 152 | %% 153 | start(Name, CandidateNodes, Workers, Mod, Arg, Options) when is_atom(Name) -> 154 | gen:start(?MODULE, nolink, {local,Name}, 155 | Mod, {CandidateNodes, Workers, Arg}, Options). 156 | 157 | %% @spec start_link(Name::atom(), CandidateNodes::[atom()], 158 | %% Workers::[atom()], Mod::atom(), Arg, Options::list()) -> 159 | %% {ok, pid()} 160 | %% 161 | %% @doc Starts a gen_leader process. 162 | %% 163 | %% 164 | %% 166 | %% 167 | %% 169 | %% 170 | %% 171 | %% 172 | %%
NameThe locally registered name of the process
CandidateNodesThe names of nodes capable of assuming 165 | %% a leadership role
WorkersThe names of nodes that will be part of the "cluster", 168 | %% but cannot ever assume a leadership role.
ModThe name of the callback module
ArgArgument passed on to Mod:init/1
OptionsSame as gen_server's Options
173 | %% 174 | %%

The list of candidates needs to be known from the start. Workers 175 | %% can be added at runtime.

176 | %% @end 177 | start_link(Name, CandidateNodes, Workers, 178 | Mod, Arg, Options) when is_atom(Name) -> 179 | % Random delay for QuickCheck 180 | % timer:sleep(random:uniform(400)), 181 | gen:start(?MODULE, link, {local,Name}, Mod, 182 | {CandidateNodes, Workers, Arg}, Options). 183 | 184 | 185 | %% Query functions to be used from the callback module 186 | 187 | %% alive(#election{alive = Alive}) -> 188 | %% Alive. 189 | 190 | %% down(#election{down = Down}) -> 191 | %% Down. 192 | 193 | %% @spec candidates(E::election()) -> [node()] 194 | %% 195 | %% @doc Returns a list of known candidates. 196 | %% 197 | candidates(#election{candidate_nodes = Cands}) -> 198 | Cands. 199 | 200 | %% @spec workers(E::election()) -> [node()] 201 | %% 202 | %% @doc Returns a list of known workers. 203 | %% 204 | workers(#election{worker_nodes = Workers}) -> 205 | Workers. 206 | 207 | %% Used by dynamically added workers. 208 | %% @hidden 209 | worker_announce(Name, Pid) -> 210 | Name ! {add_worker, Pid}. 211 | 212 | % 213 | % Make a call to a generic server. 214 | % If the server is located at another node, that node will 215 | % be monitored. 216 | % If the client is trapping exits and is linked server termination 217 | % is handled here (? Shall we do that here (or rely on timeouts) ?). 218 | % 219 | %% @spec call(Name::serverRef(), Request) -> term() 220 | %% 221 | %% @doc Equivalent to gen_server:call/2, but with a slightly 222 | %% different exit reason if something goes wrong. This function calls 223 | %% the gen_leader process exactly as if it were a gen_server 224 | %% (which, for practical purposes, it is.) 225 | %% @end 226 | call(Name, Request) -> 227 | case catch gen:call(Name, '$gen_call', Request) of 228 | {ok,Res} -> 229 | Res; 230 | {'EXIT',Reason} -> 231 | exit({Reason, {?MODULE, local_call, [Name, Request]}}) 232 | end. 233 | 234 | %% @spec call(Name::serverRef(), Request, Timeout::integer()) -> 235 | %% Reply 236 | %% 237 | %% Reply = term() 238 | %% 239 | %% @doc Equivalent to gen_server:call/3, but with a slightly 240 | %% different exit reason if something goes wrong. This function calls 241 | %% the gen_leader process exactly as if it were a gen_server 242 | %% (which, for practical purposes, it is.) 243 | %% @end 244 | call(Name, Request, Timeout) -> 245 | case catch gen:call(Name, '$gen_call', Request, Timeout) of 246 | {ok,Res} -> 247 | Res; 248 | {'EXIT',Reason} -> 249 | exit({Reason, {?MODULE, local_call, [Name, Request, Timeout]}}) 250 | end. 251 | 252 | %% @spec leader_call(Name::name(), Request::term()) 253 | %% -> Reply 254 | %% 255 | %% Reply = term() 256 | %% 257 | %% @doc Makes a call (similar to gen_server:call/2) to the 258 | %% leader. The call is forwarded via the local gen_leader instance, if 259 | %% that one isn't actually the leader. The client will exit if the 260 | %% leader dies while the request is outstanding. 261 | %%

This function uses gen:call/3, and is subject to the 262 | %% same default timeout as e.g. gen_server:call/2.

263 | %% @end 264 | %% 265 | leader_call(Name, Request) -> 266 | case catch gen:call(Name, '$leader_call', Request) of 267 | {ok,{leader,reply,Res}} -> 268 | Res; 269 | {ok,{error, leader_died}} -> 270 | exit({leader_died, {?MODULE, leader_call, [Name, Request]}}); 271 | {'EXIT',Reason} -> 272 | exit({Reason, {?MODULE, leader_call, [Name, Request]}}) 273 | end. 274 | 275 | %% @spec leader_call(Name::name(), Request::term(), Timeout::integer()) 276 | %% -> Reply 277 | %% 278 | %% Reply = term() 279 | %% 280 | %% @doc Makes a call (similar to gen_server:call/3) to the 281 | %% leader. The call is forwarded via the local gen_leader instance, if 282 | %% that one isn't actually the leader. The client will exit if the 283 | %% leader dies while the request is outstanding. 284 | %% @end 285 | %% 286 | leader_call(Name, Request, Timeout) -> 287 | case catch gen:call(Name, '$leader_call', Request, Timeout) of 288 | {ok,{leader,reply,Res}} -> 289 | Res; 290 | {'EXIT',Reason} -> 291 | exit({Reason, {?MODULE, leader_call, [Name, Request, Timeout]}}) 292 | end. 293 | 294 | 295 | %% @equiv gen_server:cast/2 296 | cast(Name, Request) -> 297 | catch do_cast('$gen_cast', Name, Request), 298 | ok. 299 | 300 | %% @spec leader_cast(Name::name(), Msg::term()) -> ok 301 | %% @doc Similar to gen_server:cast/2 but will be forwarded to 302 | %% the leader via the local gen_leader instance. 303 | leader_cast(Name, Request) -> 304 | catch do_cast('$leader_cast', Name, Request), 305 | ok. 306 | 307 | 308 | do_cast(Tag, Name, Request) when atom(Name) -> 309 | Name ! {Tag, Request}; 310 | do_cast(Tag, Pid, Request) when pid(Pid) -> 311 | Pid ! {Tag, Request}. 312 | 313 | 314 | %% @spec reply(From::callerRef(), Reply::term()) -> Void 315 | %% @equiv gen_server:reply/2 316 | reply({To, Tag}, Reply) -> 317 | catch To ! {Tag, Reply}. 318 | 319 | 320 | %%% --------------------------------------------------- 321 | %%% Initiate the new process. 322 | %%% Register the name using the Rfunc function 323 | %%% Calls the Mod:init/Args function. 324 | %%% Finally an acknowledge is sent to Parent and the main 325 | %%% loop is entered. 326 | %%% --------------------------------------------------- 327 | %%% @hidden 328 | init_it(Starter, self, Name, Mod, {CandidateNodes, Workers, Arg}, Options) -> 329 | init_it(Starter, self(), Name, Mod, 330 | {CandidateNodes, Workers, Arg}, Options); 331 | init_it(Starter,Parent,Name,Mod,{CandidateNodes,Workers,Arg},Options) -> 332 | %% The following row is needed in case of trace analysis, 333 | %% starting tracing is too slow otherwise! 334 | receive after 100 -> ok end, 335 | 336 | Debug = debug_options(Name, Options), 337 | 338 | AmCandidate = member(node(), CandidateNodes), 339 | 340 | Election = #election{candidate_nodes = CandidateNodes, 341 | worker_nodes = Workers, 342 | name = Name, 343 | nextel = 0}, 344 | 345 | case {catch Mod:init(Arg), AmCandidate} of 346 | {{stop, Reason},_} -> 347 | proc_lib:init_ack(Starter, {error, Reason}), 348 | exit(Reason); 349 | {ignore,_} -> 350 | proc_lib:init_ack(Starter, ignore), 351 | exit(normal); 352 | {{'EXIT', Reason},_} -> 353 | proc_lib:init_ack(Starter, {error, Reason}), 354 | exit(Reason); 355 | {{ok, State}, true} -> 356 | NewE = startStage1(Election#election{incarn = incarnation(node())}), 357 | 358 | proc_lib:init_ack(Starter, {ok, self()}), 359 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 360 | candidate, NewE,{init}); 361 | {{ok, State}, false} -> 362 | proc_lib:init_ack(Starter, {ok, self()}), 363 | case lists:member(self(), Workers) of 364 | false -> 365 | rpc:multicall(CandidateNodes, gen_leader, worker_announce, [Name, node(self())]); 366 | _ -> nop 367 | end, 368 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 369 | waiting_worker, Election,{init}); 370 | Else -> 371 | Error = {bad_return_value, Else}, 372 | proc_lib:init_ack(Starter, {error, Error}), 373 | exit(Error) 374 | end. 375 | 376 | 377 | 378 | 379 | %%% --------------------------------------------------- 380 | %%% The MAIN loops. 381 | %%% --------------------------------------------------- 382 | 383 | 384 | safe_loop(#server{mod = Mod, state = State} = Server, Role, 385 | #election{name = Name} = E, _PrevMsg) -> 386 | % Event for QuickCheck 387 | % ?EVENT({Role,E}), 388 | receive 389 | {system, From, Req} -> 390 | #server{parent = Parent, debug = Debug} = Server, 391 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 392 | [safe, Server, Role, E]); 393 | {'EXIT', _, Reason} = Msg -> 394 | terminate(Reason, Msg, Server, Role, E); 395 | {report_leader, From} = Msg -> 396 | From ! {election_leader, self(), E#election.leader}, 397 | safe_loop(Server, Role, E, Msg); 398 | {halt,T,From} = Msg -> 399 | NewE = halting(E,T,From), 400 | From ! {ackLeader,T,self()}, 401 | safe_loop(Server,Role,NewE,Msg); 402 | {hasLeader,Ldr,T,_} = Msg -> 403 | NewE1 = mon_node(E,Ldr), 404 | case ( (E#election.status == elec2) and (E#election.acks /= []) ) of 405 | true -> 406 | lists:foreach( 407 | fun(Node) -> 408 | {Name,Node} ! {hasLeader,Ldr,T,self()} 409 | end,E#election.acks); 410 | false -> 411 | ok 412 | end, 413 | NewE = NewE1#election{elid = T, 414 | status = wait, 415 | leadernode = node(Ldr), 416 | down = E#election.down -- [node(Ldr)], 417 | acks = []}, 418 | Ldr ! {isLeader,T,self()}, 419 | safe_loop(Server,Role,NewE,Msg); 420 | {isLeader,T,From} = Msg -> 421 | From ! {notLeader,T,self()}, 422 | safe_loop(Server,Role,E,Msg); 423 | {notLeader,T,_} = Msg -> 424 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 425 | true -> 426 | NewE = startStage1(E); 427 | false -> 428 | NewE = E 429 | end, 430 | safe_loop(Server,Role,NewE,Msg); 431 | {ackLeader,T,From} = Msg -> 432 | case ( (E#election.status == elec2) and (E#election.elid == T) and 433 | (E#election.pendack == node(From)) ) of 434 | true -> 435 | NewE = continStage2(E#election{acks = [node(From)|E#election.acks]}); 436 | false -> 437 | NewE = E 438 | end, 439 | hasBecomeLeader(NewE,Server,Msg); 440 | {ldr,Synch,T,From, LeaderE} = Msg -> 441 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 442 | true -> 443 | NewE1 = mon_node(E,From), 444 | NewE = NewE1#election{leader = From, 445 | leadernode = node(From), 446 | status = norm, 447 | worker_nodes = LeaderE#election.worker_nodes}, 448 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 449 | loop(Server#server{state = NewState},surrendered,NewE,Msg); 450 | false -> 451 | safe_loop(Server,Role,E,Msg) 452 | end; 453 | {normQ,T,From} = Msg -> 454 | case ( (E#election.status == elec1) or 455 | ( (E#election.status == wait) and (E#election.elid == T))) of 456 | true -> 457 | NewE = halting(E,T,From), 458 | From ! {notNorm,T,self()}; 459 | false -> 460 | NewE = E 461 | end, 462 | safe_loop(Server,Role,NewE,Msg); 463 | 464 | {notNorm,_,_} = Msg -> 465 | safe_loop(Server,Role,E,Msg); 466 | {workerAlive,T,From} = Msg -> 467 | case E#election.leadernode == none of 468 | true -> % We should initiate activation, monitor the possible leader! 469 | NewE = mon_node(E#election{leadernode = node(From), 470 | elid = T}, 471 | From), 472 | From ! {workerIsAlive,T,self()}; 473 | false -> 474 | % We should acutally ignore this, the present activation 475 | % will complete or abort first... 476 | NewE = E 477 | end, 478 | safe_loop(Server,Role,NewE,Msg); 479 | {workerIsAlive,_,_} = Msg -> 480 | % If this happens, the activation process should abort 481 | % This process is no longer the leader! 482 | % The sender will notice this via a DOWN message 483 | safe_loop(Server,Role,E,Msg); 484 | {activateWorker,T,Synch,From,OldE} = Msg -> 485 | case ( (T == E#election.elid) and (node(From) == E#election.leadernode)) of 486 | true -> 487 | NewE = E#election{ leader = From, 488 | status = worker, worker_nodes = OldE#election.worker_nodes }, 489 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 490 | loop(Server#server{state = NewState},worker,NewE,Msg); 491 | false -> 492 | % This should be a VERY special case... 493 | % But doing nothing is the right thing! 494 | % A DOWN message should arrive to solve this situation 495 | safe_loop(Server,Role,E,Msg) 496 | end; 497 | 498 | {tau_timeout} = Msg -> 499 | safe_loop(Server,Role,E,Msg); 500 | {'DOWN',_Ref,process,From,_Reason} = Msg when Role == waiting_worker -> 501 | % We are only monitoring one proc, the leader! 502 | Node = case From of 503 | {Name,_Node} -> _Node; 504 | _ when pid(From) -> node(From) 505 | end, 506 | case Node == E#election.leadernode of 507 | true -> 508 | NewE = E#election{ leader = none, leadernode = none, 509 | status = waiting_worker, 510 | monitored = []}; 511 | false -> 512 | NewE = E 513 | end, 514 | safe_loop(Server, Role, NewE,Msg); 515 | {'DOWN',Ref,process,From,_Reason} = Msg -> 516 | Node = case From of 517 | {Name,_Node} -> _Node; 518 | _ when pid(From) -> node(From) 519 | end, 520 | NewMon = E#election.monitored -- [{Ref,Node}], 521 | case lists:member(Node,E#election.candidate_nodes) of 522 | true -> 523 | NewDown = [Node | E#election.down], 524 | E1 = E#election{down = NewDown, monitored = NewMon}, 525 | case ( pos(Node,E#election.candidate_nodes) < 526 | pos(node(),E#election.candidate_nodes) ) of 527 | true -> 528 | Lesser = lesser(node(),E#election.candidate_nodes), 529 | LesserIsSubset = (Lesser -- NewDown) == [], 530 | case ((E#election.status == wait) and (Node == E#election.leadernode)) of 531 | true -> 532 | NewE = startStage1(E1); 533 | false -> 534 | case ((E#election.status == elec1) and LesserIsSubset) of 535 | true -> 536 | NewE = startStage2(E1#election{down = Lesser}); 537 | false -> 538 | NewE = E1 539 | end 540 | end; 541 | false -> 542 | case ( (E#election.status == elec2) and (Node == E#election.pendack) ) of 543 | true -> 544 | NewE = continStage2(E1); 545 | false -> 546 | case ( (E#election.status == wait) and 547 | (Node == E#election.leadernode)) of 548 | true -> 549 | NewE = startStage1(E1); 550 | false -> 551 | NewE = E1 552 | end 553 | end 554 | end 555 | end, 556 | hasBecomeLeader(NewE,Server,Msg) 557 | end. 558 | 559 | 560 | loop(#server{parent = Parent, 561 | mod = Mod, 562 | state = State, 563 | debug = Debug} = Server, Role, #election{name = Name} = E, _PrevMsg) -> 564 | % Event for QuickCheck 565 | % ?EVENT({Role,E}), 566 | receive 567 | Msg -> 568 | 569 | case Msg of 570 | {system, From, Req} -> 571 | sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, 572 | [normal, Server, Role, E]); 573 | {'EXIT', Parent, Reason} -> 574 | terminate(Reason, Msg, Server, Role, E); 575 | 576 | {halt,_,From} -> 577 | From ! {hasLeader,E#election.leader,E#election.elid,self()}, 578 | loop(Server,Role,E,Msg); 579 | {hasLeader,_,_,_} -> 580 | loop(Server,Role,E,Msg); 581 | {isLeader,T,From} -> 582 | case (self() == E#election.leader) of 583 | true -> 584 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 585 | From), 586 | {ok,Synch,NewState} = Mod:elected(State,NewE), 587 | From ! {ldr,Synch,E#election.elid,self(), NewE}, 588 | loop(Server#server{state = NewState},Role,NewE,Msg); 589 | false -> 590 | From ! {notLeader,T,self()}, 591 | loop(Server,Role,E,Msg) 592 | end; 593 | {ackLeader,_,_} -> 594 | loop(Server,Role,E,Msg); 595 | {notLeader,_,_} -> 596 | loop(Server,Role,E,Msg); 597 | {ack,_,_} -> 598 | loop(Server,Role,E,Msg); 599 | {ldr,_,_,_,_} -> 600 | loop(Server,Role,E,Msg); 601 | {normQ,_,_} -> 602 | loop(Server,Role,E,Msg); 603 | {notNorm,T,From} -> 604 | case ( (E#election.leader == self()) and (E#election.elid == T) ) of 605 | true -> 606 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 607 | From), 608 | {ok,Synch,NewState} = Mod:elected(State,NewE), 609 | From ! {ldr,Synch,E#election.elid,self(), NewE}, 610 | loop(Server#server{state = NewState},Role,NewE,Msg); 611 | false -> 612 | loop(Server,Role,E,Msg) 613 | end; 614 | {workerAlive,_,_} -> 615 | % Do nothing if we get this from a new leader 616 | % We will soon notice that the prev leader has died, and 617 | % get the same message again when we are back in safe_loop! 618 | loop(Server,Role,E,Msg); 619 | {activateWorker,_,_,_} -> 620 | % We ignore this, we are already active... 621 | % It must be an old message! 622 | loop(Server,Role,E,Msg); 623 | {workerIsAlive,T,From} -> 624 | case ((T == E#election.elid) and (self() == E#election.leader) 625 | % and iselem(node(From),E#election.monitored) 626 | ) of 627 | true -> 628 | NewE = mon_node( 629 | E#election{work_down = E#election.work_down -- [node(From)]}, 630 | From), 631 | % NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 632 | {ok,Synch,NewState} = Mod:elected(State,NewE), 633 | From ! {activateWorker,T,Synch,self(), NewE}, 634 | loop(Server#server{state = NewState},Role,NewE,Msg); 635 | false -> 636 | loop(Server,Role,E,Msg) 637 | end; 638 | {tau_timeout} -> 639 | case (E#election.leader == self()) of 640 | true -> 641 | lists:foreach( 642 | fun(Node) -> 643 | {Name,Node} ! {normQ,E#election.elid,self()} 644 | end,E#election.down -- [lists:nth(1,E#election.candidate_nodes)]), 645 | 646 | lists:foreach( 647 | fun(Node) -> 648 | {Name,Node} ! {workerAlive,E#election.elid,self()} 649 | end,E#election.work_down), 650 | 651 | timer:send_after(?TAU,{tau_timeout}); 652 | false -> 653 | ok 654 | end, 655 | loop(Server,Role,E,Msg); 656 | {'DOWN',_Ref,process,From,_Reason} when Role == worker -> 657 | % We are only monitoring one proc, the leader! 658 | Node = case From of 659 | {Name,_Node} -> _Node; 660 | _ when pid(From) -> node(From) 661 | end, 662 | case Node == E#election.leadernode of 663 | true -> 664 | NewE = E#election{ leader = none, leadernode = none, 665 | status = waiting_worker, 666 | monitored = []}, 667 | safe_loop(Server, waiting_worker, NewE,Msg); 668 | false -> 669 | loop(Server, Role, E,Msg) 670 | end; 671 | {'DOWN',Ref,process,From,_Reason} -> 672 | Node = case From of 673 | {Name,_Node} -> _Node; 674 | _ when pid(From) -> node(From) 675 | end, 676 | NewMon = E#election.monitored -- [{Ref,Node}], 677 | case lists:member(Node,E#election.candidate_nodes) of 678 | true -> 679 | NewDown = [Node | E#election.down], 680 | E1 = E#election{down = NewDown, monitored = NewMon}, 681 | case (Node == E#election.leadernode) of 682 | true -> 683 | NewE = startStage1(E1), 684 | safe_loop(Server, candidate, NewE,Msg); 685 | 686 | false -> 687 | loop(Server, Role, E1,Msg) 688 | end; 689 | false -> 690 | % I am the leader, make sure the dead worker is in work_down. 691 | E1 = E#election{monitored = NewMon, 692 | work_down = [Node | (E#election.work_down -- [Node])] 693 | }, 694 | loop(Server, Role, E1,Msg) 695 | end; 696 | {add_worker, WorkerNode} -> 697 | case {node(self()) =:= node(self()), 698 | lists:member(WorkerNode, E#election.worker_nodes)} of 699 | {false, _} -> 700 | loop(Server, Role, E, Msg); 701 | {true, false} -> 702 | {WNodes, DNodes} = {E#election.worker_nodes, E#election.work_down}, 703 | loop(Server, Role, E#election{worker_nodes=[WorkerNode|WNodes], 704 | work_down=[WorkerNode|DNodes]}, 705 | Msg); 706 | {true, true} -> % Redundancy, meet the mirror 707 | loop(Server, Role, E, Msg) 708 | end; 709 | _Msg when Debug == [] -> 710 | handle_msg(Msg, Server, Role, E); 711 | _Msg -> 712 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 713 | E#election.name, {in, Msg}), 714 | handle_msg(Msg, Server#server{debug = Debug1}, Role, E) 715 | end 716 | end. 717 | 718 | %%----------------------------------------------------------------- 719 | %% Callback functions for system messages handling. 720 | %%----------------------------------------------------------------- 721 | %% @hidden 722 | system_continue(_Parent, _Debug, [safe, Server, Role, E]) -> 723 | safe_loop(Server, Role, E,{}); 724 | system_continue(_Parent, _Debug, [normal, Server, Role, E]) -> 725 | loop(Server, Role, E,{}). 726 | 727 | %% @hidden 728 | system_terminate(Reason, _Parent, _Debug, [_Mode, Server, Role, E]) -> 729 | terminate(Reason, [], Server, Role, E). 730 | 731 | %% @hidden 732 | system_code_change([Mode, Server, Role, E], _Module, OldVsn, Extra) -> 733 | #server{mod = Mod, state = State} = Server, 734 | case catch Mod:code_change(OldVsn, State, E, Extra) of 735 | {ok, NewState} -> 736 | NewServer = Server#server{state = NewState}, 737 | {ok, [Mode, NewServer, Role, E]}; 738 | {ok, NewState, NewE} -> 739 | NewServer = Server#server{state = NewState}, 740 | {ok, [Mode, NewServer, Role, NewE]}; 741 | Else -> Else 742 | end. 743 | 744 | %%----------------------------------------------------------------- 745 | %% Format debug messages. Print them as the call-back module sees 746 | %% them, not as the real erlang messages. Use trace for that. 747 | %%----------------------------------------------------------------- 748 | %% @hidden 749 | print_event(Dev, {in, Msg}, Name) -> 750 | case Msg of 751 | {'$gen_call', {From, _Tag}, Call} -> 752 | io:format(Dev, "*DBG* ~p got local call ~p from ~w~n", 753 | [Name, Call, From]); 754 | {'$leader_call', {From, _Tag}, Call} -> 755 | io:format(Dev, "*DBG* ~p got global call ~p from ~w~n", 756 | [Name, Call, From]); 757 | {'$gen_cast', Cast} -> 758 | io:format(Dev, "*DBG* ~p got local cast ~p~n", 759 | [Name, Cast]); 760 | {'$leader_cast', Cast} -> 761 | io:format(Dev, "*DBG* ~p got global cast ~p~n", 762 | [Name, Cast]); 763 | _ -> 764 | io:format(Dev, "*DBG* ~p got ~p~n", [Name, Msg]) 765 | end; 766 | print_event(Dev, {out, Msg, To, State}, Name) -> 767 | io:format(Dev, "*DBG* ~p sent ~p to ~w, new state ~w~n", 768 | [Name, Msg, To, State]); 769 | print_event(Dev, {noreply, State}, Name) -> 770 | io:format(Dev, "*DBG* ~p new state ~w~n", [Name, State]); 771 | print_event(Dev, Event, Name) -> 772 | io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). 773 | 774 | handle_msg({'$leader_call', From, Request} = Msg, 775 | #server{mod = Mod, state = State} = Server, elected = Role, E) -> 776 | case catch Mod:handle_leader_call(Request, From, State, E) of 777 | {reply, Reply, NState} -> 778 | NewServer = reply(From, {leader,reply,Reply}, 779 | Server#server{state = NState}, Role, E), 780 | loop(NewServer, Role, E,Msg); 781 | {reply, Reply, Broadcast, NState} -> 782 | NewE = broadcast({from_leader,Broadcast}, E), 783 | NewServer = reply(From, {leader,reply,Reply}, 784 | Server#server{state = NState}, Role, 785 | NewE), 786 | loop(NewServer, Role, NewE,Msg); 787 | {noreply, NState} = Reply -> 788 | NewServer = handle_debug(Server#server{state = NState}, 789 | Role, E, Reply), 790 | loop(NewServer, Role, E,Msg); 791 | {stop, Reason, Reply, NState} -> 792 | {'EXIT', R} = 793 | (catch terminate(Reason, Msg, 794 | Server#server{state = NState}, 795 | Role, E)), 796 | reply(From, Reply), 797 | exit(R); 798 | Other -> 799 | handle_common_reply(Other, Msg, Server, Role, E) 800 | end; 801 | handle_msg({'$leader_call', From, Request} = Msg, 802 | Server, Role, 803 | #election{buffered = Buffered, leader = Leader} = E) -> 804 | Ref = make_ref(), 805 | Leader ! {'$leader_call', {self(),Ref}, Request}, 806 | NewBuffered = [{Ref,From}|Buffered], 807 | loop(Server, Role, E#election{buffered = NewBuffered},Msg); 808 | handle_msg({'$leader_cast', Request} = Msg, 809 | #server{mod = Mod, state = State} = Server, 810 | elected = Role, 811 | E) -> 812 | case catch Mod:handle_leader_cast(Request, State, E) of 813 | {noreply, NState} -> 814 | loop(Server#server{state = NState}, Role, E, Msg); 815 | {noreply, Broadcast, NState} -> 816 | NewE = broadcast({from_leader, Broadcast}, E), 817 | loop(Server#server{state = NState}, Role, NewE, Msg); 818 | {stop, Reason, NState} -> 819 | {'EXIT', R} = 820 | (catch terminate(Reason, Msg, Server#server{state = NState}, Role, E)), 821 | exit(R); 822 | Other -> 823 | handle_common_reply(Other, Msg, Server, Role, E) 824 | end; 825 | handle_msg({'$leader_cast', _} = Msg, 826 | Server, 827 | Role, 828 | #election{buffered=_Buffered, leader = Leader} = E) -> 829 | Leader ! Msg, 830 | loop(Server, Role, E, Msg); 831 | handle_msg({from_leader, Cmd} = Msg, 832 | #server{mod = Mod, state = State} = Server, Role, E) -> 833 | handle_common_reply(catch Mod:from_leader(Cmd, State, E), 834 | Msg, Server, Role, E); 835 | handle_msg({Ref, {leader,reply,Reply}} = Msg, Server, Role, 836 | #election{buffered = Buffered} = E) -> 837 | {value, {_,From}} = keysearch(Ref,1,Buffered), 838 | NewServer = reply(From, {leader,reply,Reply}, Server, Role, 839 | E#election{buffered = keydelete(Ref,1,Buffered)}), 840 | loop(NewServer, Role, E, Msg); 841 | handle_msg({'$gen_call', From, Request} = Msg, 842 | #server{mod = Mod, state = State} = Server, Role, E) -> 843 | case catch Mod:handle_call(Request, From, State) of 844 | {reply, Reply, NState} -> 845 | NewServer = reply(From, Reply, 846 | Server#server{state = NState}, Role, E), 847 | loop(NewServer, Role, E, Msg); 848 | {noreply, NState} = Reply -> 849 | NewServer = handle_debug(Server#server{state = NState}, 850 | Role, E, Reply), 851 | loop(NewServer, Role, E, Msg); 852 | {stop, Reason, Reply, NState} -> 853 | {'EXIT', R} = 854 | (catch terminate(Reason, Msg, Server#server{state = NState}, 855 | Role, E)), 856 | reply(From, Reply), 857 | exit(R); 858 | Other -> 859 | handle_common_reply(Other, Msg, Server, Role, E) 860 | end; 861 | handle_msg({'$gen_cast',Msg} = Cast, 862 | #server{mod = Mod, state = State} = Server, Role, E) -> 863 | handle_common_reply(catch Mod:handle_cast(Msg, State), 864 | Cast, Server, Role, E); 865 | handle_msg(Msg, 866 | #server{mod = Mod, state = State} = Server, Role, E) -> 867 | handle_common_reply(catch Mod:handle_info(Msg, State), 868 | Msg, Server, Role, E). 869 | 870 | 871 | handle_common_reply(Reply, Msg, Server, Role, E) -> 872 | case Reply of 873 | {noreply, NState} -> 874 | NewServer = handle_debug(Server#server{state = NState}, 875 | Role, E, Reply), 876 | loop(NewServer, Role, E, Msg); 877 | {ok, NState} -> 878 | NewServer = handle_debug(Server#server{state = NState}, 879 | Role, E, Reply), 880 | loop(NewServer, Role, E, Msg); 881 | {stop, Reason, NState} -> 882 | terminate(Reason, Msg, Server#server{state = NState}, Role, E); 883 | {'EXIT', Reason} -> 884 | terminate(Reason, Msg, Server, Role, E); 885 | _ -> 886 | terminate({bad2_return_value, Reply}, Msg, Server, Role, E) 887 | end. 888 | 889 | 890 | reply({To, Tag}, Reply, #server{state = State} = Server, Role, E) -> 891 | reply({To, Tag}, Reply), 892 | handle_debug(Server, Role, E, {out, Reply, To, State}). 893 | 894 | 895 | handle_debug(#server{debug = []} = Server, _Role, _E, _Event) -> 896 | Server; 897 | handle_debug(#server{debug = Debug} = Server, _Role, E, Event) -> 898 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 899 | E#election.name, Event), 900 | Server#server{debug = Debug1}. 901 | 902 | %%% --------------------------------------------------- 903 | %%% Terminate the server. 904 | %%% --------------------------------------------------- 905 | 906 | terminate(Reason, Msg, #server{mod = Mod, 907 | state = State, 908 | debug = Debug} = _Server, _Role, 909 | #election{name = Name} = _E) -> 910 | case catch Mod:terminate(Reason, State) of 911 | {'EXIT', R} -> 912 | error_info(R, Name, Msg, State, Debug), 913 | exit(R); 914 | _ -> 915 | case Reason of 916 | normal -> 917 | exit(normal); 918 | shutdown -> 919 | exit(shutdown); 920 | _ -> 921 | error_info(Reason, Name, Msg, State, Debug), 922 | exit(Reason) 923 | end 924 | end. 925 | 926 | %% Maybe we shouldn't do this? We have the crash report... 927 | error_info(Reason, Name, Msg, State, Debug) -> 928 | format("** Generic leader ~p terminating \n" 929 | "** Last message in was ~p~n" 930 | "** When Server state == ~p~n" 931 | "** Reason for termination == ~n** ~p~n", 932 | [Name, Msg, State, Reason]), 933 | sys:print_log(Debug), 934 | ok. 935 | 936 | %%% --------------------------------------------------- 937 | %%% Misc. functions. 938 | %%% --------------------------------------------------- 939 | 940 | opt(Op, [{Op, Value}|_]) -> 941 | {ok, Value}; 942 | opt(Op, [_|Options]) -> 943 | opt(Op, Options); 944 | opt(_, []) -> 945 | false. 946 | 947 | debug_options(Name, Opts) -> 948 | case opt(debug, Opts) of 949 | {ok, Options} -> dbg_options(Name, Options); 950 | _ -> dbg_options(Name, []) 951 | end. 952 | 953 | dbg_options(Name, []) -> 954 | Opts = 955 | case init:get_argument(generic_debug) of 956 | error -> 957 | []; 958 | _ -> 959 | [log, statistics] 960 | end, 961 | dbg_opts(Name, Opts); 962 | dbg_options(Name, Opts) -> 963 | dbg_opts(Name, Opts). 964 | 965 | dbg_opts(Name, Opts) -> 966 | case catch sys:debug_options(Opts) of 967 | {'EXIT',_} -> 968 | format("~p: ignoring erroneous debug options - ~p~n", 969 | [Name, Opts]), 970 | []; 971 | Dbg -> 972 | Dbg 973 | end. 974 | 975 | %%----------------------------------------------------------------- 976 | %% Status information 977 | %%----------------------------------------------------------------- 978 | %% @hidden 979 | format_status(Opt, StatusData) -> 980 | [PDict, SysState, Parent, Debug, [_Mode, Server, _Role, E]] = StatusData, 981 | Header = lists:concat(["Status for generic server ", E#election.name]), 982 | Log = sys:get_debug(log, Debug, []), 983 | #server{mod = Mod, state = State} = Server, 984 | Specific = 985 | case erlang:function_exported(Mod, format_status, 2) of 986 | true -> 987 | case catch apply(Mod, format_status, [Opt, [PDict, State]]) of 988 | {'EXIT', _} -> [{data, [{"State", State}]}]; 989 | Else -> Else 990 | end; 991 | _ -> 992 | [{data, [{"State", State}]}] 993 | end, 994 | [{header, Header}, 995 | {data, [{"Status", SysState}, 996 | {"Parent", Parent}, 997 | {"Logged events", Log}]} | 998 | Specific]. 999 | 1000 | 1001 | %%----------------------------------------------------------------- 1002 | %% Leader-election functions 1003 | %%----------------------------------------------------------------- 1004 | 1005 | %% Corresponds to startStage1 in Figure 1 in the Stoller-article 1006 | startStage1(E) -> 1007 | Elid = {pos(node(),E#election.candidate_nodes),E#election.incarn,E#election.nextel}, 1008 | NewE = E#election{ 1009 | elid = Elid, 1010 | nextel = E#election.nextel + 1, 1011 | down = [], 1012 | status = elec1}, 1013 | case ( pos(node(),E#election.candidate_nodes) == 1) of 1014 | true -> 1015 | startStage2(NewE); 1016 | false -> 1017 | mon_nodes(NewE,lesser(node(),E#election.candidate_nodes)) 1018 | end. 1019 | 1020 | %% Corresponds to startStage2 1021 | startStage2(E) -> 1022 | continStage2(E#election{ 1023 | status = elec2, 1024 | pendack = node(), 1025 | acks = []}). 1026 | 1027 | continStage2(E) -> 1028 | case pos(E#election.pendack,E#election.candidate_nodes) < length(E#election.candidate_nodes) of 1029 | true -> 1030 | Pendack = next(E#election.pendack,E#election.candidate_nodes), 1031 | NewE = mon_nodes(E,[Pendack]), 1032 | {E#election.name,Pendack} ! {halt,E#election.elid,self()}, 1033 | NewE#election{pendack = Pendack}; 1034 | false -> 1035 | % I am the leader 1036 | % io:format("I am the leader (Node ~w) ~n", [node()]), 1037 | E#election{leader = self(), 1038 | leadernode = node(), 1039 | status = norm} 1040 | end. 1041 | 1042 | %% corresponds to Halting 1043 | halting(E,T,From) -> 1044 | NewE = mon_node(E,From), 1045 | NewE#election{elid = T, 1046 | status = wait, 1047 | leadernode = node(From), 1048 | down = E#election.down -- [node(From)] 1049 | }. 1050 | 1051 | %% Start monitor a bunch of nodes 1052 | mon_nodes(E,Nodes) -> 1053 | foldl( 1054 | fun(Node,_E) -> 1055 | mon_node(_E,{_E#election.name,Node}) 1056 | end,E,Nodes). 1057 | 1058 | %% Star monitoring one Process 1059 | mon_node(E,Proc) -> 1060 | Node = case Proc of 1061 | {_Name,Node_} -> 1062 | Node_; 1063 | Pid when pid(Pid) -> 1064 | node(Pid) 1065 | end, 1066 | case iselem(Node,E#election.monitored) of 1067 | true -> 1068 | E; 1069 | false -> 1070 | Ref = erlang:monitor(process,Proc), 1071 | E#election{monitored = [{Ref,Node} | E#election.monitored]} 1072 | end. 1073 | 1074 | 1075 | %% Stop monitoring of a bunch of nodes 1076 | %demon_nodes(E) -> 1077 | % foreach(fun({R,_}) -> 1078 | % erlang:demonitor(R) 1079 | % end,E#election.monitored), 1080 | % E#election{monitored = []}. 1081 | 1082 | %% checks if the proc has become the leader, if so switch to loop 1083 | hasBecomeLeader(E,Server,Msg) -> 1084 | case ((E#election.status == norm) and (E#election.leader == self())) of 1085 | true -> 1086 | {ok,Synch,NewState} = (Server#server.mod):elected(Server#server.state,E), 1087 | lists:foreach( 1088 | fun(Node) -> 1089 | {E#election.name,Node} ! 1090 | {ldr, Synch, E#election.elid, self(), E} 1091 | end,E#election.acks), 1092 | 1093 | % Make sure we will try to contact all workers! 1094 | NewE = E#election{work_down = E#election.worker_nodes}, 1095 | 1096 | % Set the internal timeout (corresponds to Periodically) 1097 | timer:send_after(?TAU,{tau_timeout}), %It's meaningful only when I am the leader! 1098 | loop(Server#server{state = NewState},elected,NewE,Msg); 1099 | false -> 1100 | safe_loop(Server,candidate,E,Msg) 1101 | end. 1102 | 1103 | 1104 | 1105 | 1106 | %%% 1107 | % 1108 | % incarnation should return an integer value for the next 1109 | % incarnation of this node. We create a file for each node, 1110 | % this file contains a counter. When starting the system for the 1111 | % first time, the files should be intialized with 0 incarnation 1112 | % counter for all nodes orelse be removed, since we create 1113 | % files if not present with counter 1. 1114 | % 1115 | % Atomicity: This approach is safe as long as there is only 1116 | % one gen_leader running per node. 1117 | % 1118 | incarnation(Node) -> 1119 | case file:read_file_info(Node) of 1120 | {error,_Reason} -> 1121 | ok = file:write_file(Node,term_to_binary(1)), 1122 | 0; 1123 | {ok,_} -> 1124 | {ok,Bin} = file:read_file(Node), 1125 | Incarn = binary_to_term(Bin), 1126 | ok = file:write_file(Node,term_to_binary(Incarn+1)), 1127 | Incarn 1128 | end. 1129 | 1130 | 1131 | broadcast(Msg, #election{monitored = Monitored} = E) -> 1132 | %% This function is used for broadcasts, 1133 | %% and we make sure only to broadcast to already known nodes. 1134 | ToNodes = [N || {_,N} <- Monitored], 1135 | broadcast(Msg, ToNodes, E). 1136 | 1137 | broadcast({from_leader, Msg}, ToNodes, E) -> 1138 | foreach( 1139 | fun(Node) -> 1140 | {E#election.name,Node} ! {from_leader, Msg} 1141 | end,ToNodes), 1142 | E. 1143 | 1144 | iselem(_,[]) -> 1145 | false; 1146 | iselem(P,[{_,P}|_]) -> 1147 | true; 1148 | iselem(P,[_ | Ns]) -> 1149 | iselem(P,Ns). 1150 | 1151 | lesser(_,[]) -> 1152 | []; 1153 | lesser(N,[N|_]) -> 1154 | []; 1155 | lesser(N,[M|Ms]) -> 1156 | [M|lesser(N,Ms)]. 1157 | 1158 | next(_,[]) -> 1159 | no_val; 1160 | next(N,[N|Ms]) -> 1161 | lists:nth(1,Ms); 1162 | next(N,[_|Ms]) -> 1163 | next(N,Ms). 1164 | 1165 | pos(N1,[N1|_]) -> 1166 | 1; 1167 | pos(N1,[_|Ns]) -> 1168 | 1+pos(N1,Ns). 1169 | -------------------------------------------------------------------------------- /powerset_version/source-to-current.diff: -------------------------------------------------------------------------------- 1 | --- original_version/gen_leader.erl 2009-05-19 15:16:57.000000000 -0400 2 | +++ powerset_version/gen_leader.erl 2009-05-19 14:05:38.000000000 -0400 3 | @@ -19,6 +19,7 @@ 4 | %% @author Hans Svensson 5 | %% @author Thomas Arts 6 | %% @author Ulf Wiger 7 | +%% @author Dave Fayram 8 | %% 9 | %% @doc Leader election behavior. 10 | %%

This application implements a leader election behavior modeled after 11 | @@ -51,6 +52,9 @@ 12 | %% See gen_server. 13 | %% @type callerRef() = {pid(), reference()}. See gen_server. 14 | %% 15 | + 16 | +%% This version has been modified and debugged by Dave Fayram for 17 | +%% Powerset/Microsoft. 18 | -module(gen_leader). 19 | 20 | % Time between rounds of query from the leader 21 | @@ -61,7 +65,7 @@ 22 | 23 | -export([start/6, 24 | start_link/6, 25 | - leader_call/2, leader_call/3, leader_cast/2, 26 | + leader_call/2, leader_call/3, leader_cast/2, worker_announce/2, 27 | call/2, call/3, cast/2, 28 | reply/2]). 29 | 30 | @@ -200,6 +204,11 @@ 31 | workers(#election{worker_nodes = Workers}) -> 32 | Workers. 33 | 34 | +%% Used by dynamically added workers. 35 | +%% @hidden 36 | +worker_announce(Name, Pid) -> 37 | + Name ! {add_worker, Pid}. 38 | + 39 | % 40 | % Make a call to a generic server. 41 | % If the server is located at another node, that node will 42 | @@ -320,10 +329,9 @@ 43 | init_it(Starter, self(), Name, Mod, 44 | {CandidateNodes, Workers, Arg}, Options); 45 | init_it(Starter,Parent,Name,Mod,{CandidateNodes,Workers,Arg},Options) -> 46 | - 47 | %% The following row is needed in case of trace analysis, 48 | %% starting tracing is too slow otherwise! 49 | - %receive after 100 -> ok end, 50 | + receive after 100 -> ok end, 51 | 52 | Debug = debug_options(Name, Options), 53 | 54 | @@ -352,6 +360,11 @@ 55 | candidate, NewE,{init}); 56 | {{ok, State}, false} -> 57 | proc_lib:init_ack(Starter, {ok, self()}), 58 | + case lists:member(self(), Workers) of 59 | + false -> 60 | + rpc:multicall(CandidateNodes, gen_leader, worker_announce, [Name, node(self())]); 61 | + _ -> nop 62 | + end, 63 | safe_loop(#server{parent = Parent,mod = Mod,state = State,debug = Debug}, 64 | waiting_worker, Election,{init}); 65 | Else -> 66 | @@ -369,7 +382,7 @@ 67 | 68 | 69 | safe_loop(#server{mod = Mod, state = State} = Server, Role, 70 | - #election{name = Name} = E, PrevMsg) -> 71 | + #election{name = Name} = E, _PrevMsg) -> 72 | % Event for QuickCheck 73 | % ?EVENT({Role,E}), 74 | receive 75 | @@ -379,6 +392,9 @@ 76 | [safe, Server, Role, E]); 77 | {'EXIT', _, Reason} = Msg -> 78 | terminate(Reason, Msg, Server, Role, E); 79 | + {report_leader, From} = Msg -> 80 | + From ! {election_leader, self(), E#election.leader}, 81 | + safe_loop(Server, Role, E, Msg); 82 | {halt,T,From} = Msg -> 83 | NewE = halting(E,T,From), 84 | From ! {ackLeader,T,self()}, 85 | @@ -421,13 +437,14 @@ 86 | NewE = E 87 | end, 88 | hasBecomeLeader(NewE,Server,Msg); 89 | - {ldr,Synch,T,From} = Msg -> 90 | + {ldr,Synch,T,From, LeaderE} = Msg -> 91 | case ( (E#election.status == wait) and (E#election.elid == T) ) of 92 | true -> 93 | NewE1 = mon_node(E,From), 94 | NewE = NewE1#election{leader = From, 95 | leadernode = node(From), 96 | - status = norm}, 97 | + status = norm, 98 | + worker_nodes = LeaderE#election.worker_nodes}, 99 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 100 | loop(Server#server{state = NewState},surrendered,NewE,Msg); 101 | false -> 102 | @@ -464,11 +481,11 @@ 103 | % This process is no longer the leader! 104 | % The sender will notice this via a DOWN message 105 | safe_loop(Server,Role,E,Msg); 106 | - {activateWorker,T,Synch,From} = Msg -> 107 | + {activateWorker,T,Synch,From,OldE} = Msg -> 108 | case ( (T == E#election.elid) and (node(From) == E#election.leadernode)) of 109 | true -> 110 | NewE = E#election{ leader = From, 111 | - status = worker }, 112 | + status = worker, worker_nodes = OldE#election.worker_nodes }, 113 | {ok,NewState} = Mod:surrendered(State,Synch,NewE), 114 | loop(Server#server{state = NewState},worker,NewE,Msg); 115 | false -> 116 | @@ -480,7 +497,7 @@ 117 | 118 | {tau_timeout} = Msg -> 119 | safe_loop(Server,Role,E,Msg); 120 | - {'DOWN',Ref,process,From,Reason} = Msg when Role == waiting_worker -> 121 | + {'DOWN',_Ref,process,From,_Reason} = Msg when Role == waiting_worker -> 122 | % We are only monitoring one proc, the leader! 123 | Node = case From of 124 | {Name,_Node} -> _Node; 125 | @@ -495,7 +512,7 @@ 126 | NewE = E 127 | end, 128 | safe_loop(Server, Role, NewE,Msg); 129 | - {'DOWN',Ref,process,From,Reason} = Msg -> 130 | + {'DOWN',Ref,process,From,_Reason} = Msg -> 131 | Node = case From of 132 | {Name,_Node} -> _Node; 133 | _ when pid(From) -> node(From) 134 | @@ -543,7 +560,7 @@ 135 | loop(#server{parent = Parent, 136 | mod = Mod, 137 | state = State, 138 | - debug = Debug} = Server, Role, #election{name = Name} = E, PrevMsg) -> 139 | + debug = Debug} = Server, Role, #election{name = Name} = E, _PrevMsg) -> 140 | % Event for QuickCheck 141 | % ?EVENT({Role,E}), 142 | receive 143 | @@ -567,7 +584,7 @@ 144 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 145 | From), 146 | {ok,Synch,NewState} = Mod:elected(State,NewE), 147 | - From ! {ldr,Synch,E#election.elid,self()}, 148 | + From ! {ldr,Synch,E#election.elid,self(), NewE}, 149 | loop(Server#server{state = NewState},Role,NewE,Msg); 150 | false -> 151 | From ! {notLeader,T,self()}, 152 | @@ -579,7 +596,7 @@ 153 | loop(Server,Role,E,Msg); 154 | {ack,_,_} -> 155 | loop(Server,Role,E,Msg); 156 | - {ldr,_,_,_} -> 157 | + {ldr,_,_,_,_} -> 158 | loop(Server,Role,E,Msg); 159 | {normQ,_,_} -> 160 | loop(Server,Role,E,Msg); 161 | @@ -589,7 +606,7 @@ 162 | NewE = mon_node(E#election{down = E#election.down -- [node(From)]}, 163 | From), 164 | {ok,Synch,NewState} = Mod:elected(State,NewE), 165 | - From ! {ldr,Synch,E#election.elid,self()}, 166 | + From ! {ldr,Synch,E#election.elid,self(), NewE}, 167 | loop(Server#server{state = NewState},Role,NewE,Msg); 168 | false -> 169 | loop(Server,Role,E,Msg) 170 | @@ -611,9 +628,9 @@ 171 | NewE = mon_node( 172 | E#election{work_down = E#election.work_down -- [node(From)]}, 173 | From), 174 | -% NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 175 | + % NewE = E#election{work_down = E#election.work_down -- [node(From)]}, 176 | {ok,Synch,NewState} = Mod:elected(State,NewE), 177 | - From ! {activateWorker,T,Synch,self()}, 178 | + From ! {activateWorker,T,Synch,self(), NewE}, 179 | loop(Server#server{state = NewState},Role,NewE,Msg); 180 | false -> 181 | loop(Server,Role,E,Msg) 182 | @@ -636,7 +653,7 @@ 183 | ok 184 | end, 185 | loop(Server,Role,E,Msg); 186 | - {'DOWN',Ref,process,From,Reason} when Role == worker -> 187 | + {'DOWN',_Ref,process,From,_Reason} when Role == worker -> 188 | % We are only monitoring one proc, the leader! 189 | Node = case From of 190 | {Name,_Node} -> _Node; 191 | @@ -651,7 +668,7 @@ 192 | false -> 193 | loop(Server, Role, E,Msg) 194 | end; 195 | - {'DOWN',Ref,process,From,Reason} -> 196 | + {'DOWN',Ref,process,From,_Reason} -> 197 | Node = case From of 198 | {Name,_Node} -> _Node; 199 | _ when pid(From) -> node(From) 200 | @@ -676,6 +693,19 @@ 201 | }, 202 | loop(Server, Role, E1,Msg) 203 | end; 204 | + {add_worker, WorkerNode} -> 205 | + case {node(self()) =:= node(self()), 206 | + lists:member(WorkerNode, E#election.worker_nodes)} of 207 | + {false, _} -> 208 | + loop(Server, Role, E, Msg); 209 | + {true, false} -> 210 | + {WNodes, DNodes} = {E#election.worker_nodes, E#election.work_down}, 211 | + loop(Server, Role, E#election{worker_nodes=[WorkerNode|WNodes], 212 | + work_down=[WorkerNode|DNodes]}, 213 | + Msg); 214 | + {true, true} -> % Redundancy, meet the mirror 215 | + loop(Server, Role, E, Msg) 216 | + end; 217 | _Msg when Debug == [] -> 218 | handle_msg(Msg, Server, Role, E); 219 | _Msg -> 220 | @@ -689,13 +719,13 @@ 221 | %% Callback functions for system messages handling. 222 | %%----------------------------------------------------------------- 223 | %% @hidden 224 | -system_continue(Parent, Debug, [safe, Server, Role, E]) -> 225 | +system_continue(_Parent, _Debug, [safe, Server, Role, E]) -> 226 | safe_loop(Server, Role, E,{}); 227 | -system_continue(Parent, Debug, [normal, Server, Role, E]) -> 228 | +system_continue(_Parent, _Debug, [normal, Server, Role, E]) -> 229 | loop(Server, Role, E,{}). 230 | 231 | %% @hidden 232 | -system_terminate(Reason, _Parent, Debug, [Mode, Server, Role, E]) -> 233 | +system_terminate(Reason, _Parent, _Debug, [_Mode, Server, Role, E]) -> 234 | terminate(Reason, [], Server, Role, E). 235 | 236 | %% @hidden 237 | @@ -741,7 +771,6 @@ 238 | print_event(Dev, Event, Name) -> 239 | io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). 240 | 241 | - 242 | handle_msg({'$leader_call', From, Request} = Msg, 243 | #server{mod = Mod, state = State} = Server, elected = Role, E) -> 244 | case catch Mod:handle_leader_call(Request, From, State, E) of 245 | @@ -769,16 +798,40 @@ 246 | Other -> 247 | handle_common_reply(Other, Msg, Server, Role, E) 248 | end; 249 | -handle_msg({from_leader, Cmd} = Msg, 250 | - #server{mod = Mod, state = State} = Server, Role, E) -> 251 | - handle_common_reply(catch Mod:from_leader(Cmd, State, E), 252 | - Msg, Server, Role, E); 253 | -handle_msg({'$leader_call', From, Request} = Msg, Server, Role, 254 | +handle_msg({'$leader_call', From, Request} = Msg, 255 | + Server, Role, 256 | #election{buffered = Buffered, leader = Leader} = E) -> 257 | Ref = make_ref(), 258 | Leader ! {'$leader_call', {self(),Ref}, Request}, 259 | NewBuffered = [{Ref,From}|Buffered], 260 | loop(Server, Role, E#election{buffered = NewBuffered},Msg); 261 | +handle_msg({'$leader_cast', Request} = Msg, 262 | + #server{mod = Mod, state = State} = Server, 263 | + elected = Role, 264 | + E) -> 265 | + case catch Mod:handle_leader_cast(Request, State, E) of 266 | + {noreply, NState} -> 267 | + loop(Server#server{state = NState}, Role, E, Msg); 268 | + {noreply, Broadcast, NState} -> 269 | + NewE = broadcast({from_leader, Broadcast}, E), 270 | + loop(Server#server{state = NState}, Role, NewE, Msg); 271 | + {stop, Reason, NState} -> 272 | + {'EXIT', R} = 273 | + (catch terminate(Reason, Msg, Server#server{state = NState}, Role, E)), 274 | + exit(R); 275 | + Other -> 276 | + handle_common_reply(Other, Msg, Server, Role, E) 277 | + end; 278 | +handle_msg({'$leader_cast', _} = Msg, 279 | + Server, 280 | + Role, 281 | + #election{buffered=_Buffered, leader = Leader} = E) -> 282 | + Leader ! Msg, 283 | + loop(Server, Role, E, Msg); 284 | +handle_msg({from_leader, Cmd} = Msg, 285 | + #server{mod = Mod, state = State} = Server, Role, E) -> 286 | + handle_common_reply(catch Mod:from_leader(Cmd, State, E), 287 | + Msg, Server, Role, E); 288 | handle_msg({Ref, {leader,reply,Reply}} = Msg, Server, Role, 289 | #election{buffered = Buffered} = E) -> 290 | {value, {_,From}} = keysearch(Ref,1,Buffered), 291 | @@ -817,6 +870,10 @@ 292 | 293 | handle_common_reply(Reply, Msg, Server, Role, E) -> 294 | case Reply of 295 | + {noreply, NState} -> 296 | + NewServer = handle_debug(Server#server{state = NState}, 297 | + Role, E, Reply), 298 | + loop(NewServer, Role, E, Msg); 299 | {ok, NState} -> 300 | NewServer = handle_debug(Server#server{state = NState}, 301 | Role, E, Reply), 302 | @@ -835,9 +892,9 @@ 303 | handle_debug(Server, Role, E, {out, Reply, To, State}). 304 | 305 | 306 | -handle_debug(#server{debug = []} = Server, _Role, _E, Event) -> 307 | +handle_debug(#server{debug = []} = Server, _Role, _E, _Event) -> 308 | Server; 309 | -handle_debug(#server{debug = Debug} = Server, Role, E, Event) -> 310 | +handle_debug(#server{debug = Debug} = Server, _Role, E, Event) -> 311 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 312 | E#election.name, Event), 313 | Server#server{debug = Debug1}. 314 | @@ -848,8 +905,8 @@ 315 | 316 | terminate(Reason, Msg, #server{mod = Mod, 317 | state = State, 318 | - debug = Debug} = Server, Role, 319 | - #election{name = Name} = E) -> 320 | + debug = Debug} = _Server, _Role, 321 | + #election{name = Name} = _E) -> 322 | case catch Mod:terminate(Reason, State) of 323 | {'EXIT', R} -> 324 | error_info(R, Name, Msg, State, Debug), 325 | @@ -920,7 +977,7 @@ 326 | %%----------------------------------------------------------------- 327 | %% @hidden 328 | format_status(Opt, StatusData) -> 329 | - [PDict, SysState, Parent, Debug, [Mode, Server, Role, E]] = StatusData, 330 | + [PDict, SysState, Parent, Debug, [_Mode, Server, _Role, E]] = StatusData, 331 | Header = lists:concat(["Status for generic server ", E#election.name]), 332 | Log = sys:get_debug(log, Debug, []), 333 | #server{mod = Mod, state = State} = Server, 334 | @@ -1001,7 +1058,7 @@ 335 | %% Star monitoring one Process 336 | mon_node(E,Proc) -> 337 | Node = case Proc of 338 | - {Name,Node_} -> 339 | + {_Name,Node_} -> 340 | Node_; 341 | Pid when pid(Pid) -> 342 | node(Pid) 343 | @@ -1030,7 +1087,7 @@ 344 | lists:foreach( 345 | fun(Node) -> 346 | {E#election.name,Node} ! 347 | - {ldr, Synch, E#election.elid, self()} 348 | + {ldr, Synch, E#election.elid, self(), E} 349 | end,E#election.acks), 350 | 351 | % Make sure we will try to contact all workers! 352 | @@ -1060,7 +1117,7 @@ 353 | % 354 | incarnation(Node) -> 355 | case file:read_file_info(Node) of 356 | - {error,Reason} -> 357 | + {error,_Reason} -> 358 | ok = file:write_file(Node,term_to_binary(1)), 359 | 0; 360 | {ok,_} -> 361 | --------------------------------------------------------------------------------