├── rebar ├── README ├── src ├── zab_engine.app.src ├── zab_engine_util.erl ├── zab_engine_app.erl ├── zabe_proposal_backend.erl ├── zab_engine_sup.erl ├── eleveldb_util.erl ├── zabe_util.erl ├── zabe_fast_leader_test.erl ├── zabe_learn_leveldb.erl ├── zabe_proposal_leveldb_backend.erl ├── zabe_fast_elect.erl └── gen_zab_server.erl ├── rebar.config ├── test ├── t.erl ├── zabe_util_SUITE.erl ├── zabe_fast_elect_SUITE.erl ├── elect2_SUITE.erl ├── leader_down_SUITE.erl ├── follow_online_recover_SUITE.erl ├── recover1_SUITE.erl ├── follow_down_SUITE.erl ├── zabe_learn_leveldb_SUITE.erl └── zabe_proposal_leveldb_backend_SUITE.erl ├── LICENCE └── include └── zabe_main.hrl /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xinmingyao/zab_engine/HEAD/rebar -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | zab protocol in erlang and use like gen_server 2 | 3 | use leveldb as log 4 | 5 | 6 | use: 7 | 8 | implements gen_zab_server callbacks 9 | 10 | look zabe_learn_leveldb.erl 11 | 12 | 13 | todo: 14 | 15 | recover 16 | 17 | backup 18 | 19 | -------------------------------------------------------------------------------- /src/zab_engine.app.src: -------------------------------------------------------------------------------- 1 | {application, zab_engine, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {mod, { zab_engine_app, []}}, 11 | {env, []} 12 | ]}. 13 | -------------------------------------------------------------------------------- /src/zab_engine_util.erl: -------------------------------------------------------------------------------- 1 | -module(zab_engine_util). 2 | -export([get_env/2]). 3 | 4 | -spec get_env(atom(), term()) -> term(). 5 | get_env(Key, Default) -> 6 | case application:get_env(zab_engine, Key) of 7 | {ok, Value} -> {ok, Value}; 8 | undefined -> {ok, Default} 9 | end. 10 | 11 | -------------------------------------------------------------------------------- /src/zab_engine_app.erl: -------------------------------------------------------------------------------- 1 | -module(zab_engine_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | zab_engine_sup:start_link(). 14 | 15 | stop(_State) -> 16 | ok. 17 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps_dirs,["deps"]}. 2 | {ct_use_short_names,true}. 3 | {test_node_name,"8@localhost"}. 4 | {deps, 5 | [ 6 | {meck, ".*", {git, "http://github.com/eproxus/meck.git", "master"}}, 7 | {eleveldb, ".*", {git, "http://github.com/basho/eleveldb.git", "master"}} , 8 | {lager, ".*", {git, "git://github.com/basho/lager", {branch, "master"}}} 9 | ]}. 10 | 11 | {erl_opts, [{parse_transform, lager_transform}]}. 12 | %% Make sure to set -fPIC when compiling redis 13 | %{pre_hooks, [{compile, "c_src/build_deps.sh"}]}. 14 | 15 | %{post_hooks, [{clean, "c_src/build_deps.sh clean"}]}. 16 | -------------------------------------------------------------------------------- /test/t.erl: -------------------------------------------------------------------------------- 1 | -module(t). 2 | -export([start/0]). 3 | 4 | start()-> 5 | 6 | Dir="/tmp/p3.db", 7 | {ok,Db}=eleveldb:open(Dir, [{create_if_missing, true},{max_open_files,50}]), 8 | {ok,It}=eleveldb:iterator(Db,[]), 9 | error_logger:info_msg("dddd~p",[eleveldb:iterator_move(It,first)]). 10 | s2()-> 11 | Dir="/tmp/p1.db", 12 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 13 | zabe_proposal_leveldb_backend:get_last_proposal([]). 14 | s3()-> 15 | Dir="/tmp/p3.db", 16 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 17 | zabe_proposal_leveldb_backend:iterate_zxid_count(fun({_K,V})-> 18 | V end,{1,0},100). 19 | -------------------------------------------------------------------------------- /src/zabe_proposal_backend.erl: -------------------------------------------------------------------------------- 1 | -module(zabe_proposal_backend). 2 | -include("zabe_main.hrl"). 3 | 4 | -type opts()::[ 5 | {prefix,V::string()} 6 | ]. 7 | %%%========================================================================= 8 | %%% API 9 | %%%========================================================================= 10 | -callback start(Dir::string(),Opts::opts())-> 11 | ok. 12 | -callback get_last_proposal(Opts::opts())-> 13 | {ok,Zxid::zxid()}|not_found. 14 | -callback put_proposal(Zxid :: zxid(),Proposal::#proposal{},Opts::[]) -> 15 | ok. 16 | -callback get_proposal(Zxid :: zxid(),Opts::[]) -> 17 | {ok, Proposal::#proposal{}} | 18 | {error,not_found} . 19 | -callback get_epoch_last_zxid(Epoch::non_neg_integer(),Opts::[])-> 20 | {ok,Zxid::zxid()}. 21 | -callback fold(Fun::fun(),StartZxid::zxid(),Opts::[])-> 22 | ok. 23 | -callback get_range_proposal(StartZxid::zxid,Range::integer(),Opts::[])-> 24 | {ok,Proposals::[#proposal{}]}. 25 | 26 | -callback iterate_zxid_count(Fun::fun(),Start::zxid(),Count::integer)-> 27 | ok. 28 | 29 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | ;;;; Copyright (c) 2012 xinming yao .All rights reserverd. 2 | ;;;; 3 | ;;;; Permission is hereby granted, free of charge, to any person obtaining 4 | ;;;; a copy of this software and associated documentation files (the 5 | ;;;; "Software"), to deal in the Software without restriction, including 6 | ;;;; without limitation the rights to use, copy, modify, merge, publish, 7 | ;;;; distribute, sublicense, and/or sell copies of the Software, and to 8 | ;;;; permit persons to whom the Software is furnished to do so, subject to 9 | ;;;; the following conditions: 10 | ;;;; 11 | ;;;; The above copyright notice and this permission notice shall be included 12 | ;;;; in all copies or substantial portions of the Software. 13 | ;;;; 14 | ;;;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | ;;;; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | ;;;; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | ;;;; IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | ;;;; CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | ;;;; TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | ;;;; SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /src/zab_engine_sup.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(zab_engine_sup). 3 | 4 | -behaviour(supervisor). 5 | 6 | %% API 7 | -export([start_link/0]). 8 | 9 | %% Supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% Helper macro for declaring children of supervisor 13 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 14 | 15 | %% =================================================================== 16 | %% API functions 17 | %% =================================================================== 18 | 19 | start_link() -> 20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 21 | 22 | %% =================================================================== 23 | %% Supervisor callbacks 24 | %% =================================================================== 25 | 26 | 27 | init([]) -> 28 | 29 | {ok,ProposalDir}=zab_engine_util:get_env(proposal_log_dir,"/tmp/proposal"), 30 | 31 | ProposalBackEnd = {proposal_backend, 32 | {zabe_proposal_leveldb_backend, start_link, 33 | [ProposalDir,[]]}, 34 | permanent, 5000, worker, [redis_back_end]}, 35 | 36 | 37 | 38 | % Build the process list... 39 | Processes = lists:flatten([ 40 | ProposalBackEnd 41 | 42 | ]), 43 | % Run the proesses... 44 | {ok, {{one_for_one, 10, 10}, Processes}}. 45 | 46 | 47 | -------------------------------------------------------------------------------- /include/zabe_main.hrl: -------------------------------------------------------------------------------- 1 | -type zxid()::{Epoch::non_neg_integer(),TxnId::non_neg_integer()}. 2 | -type peer()::{Name::atom(),Node::node()}. 3 | -type client()::{Name::atom(),Node::node()}|{Pid::pid(),Node::node()}. 4 | 5 | 6 | -define(ELECT_CMD,1). 7 | -define(VOTE_CMD,2). 8 | -define(PROPOSE_CMD,3). 9 | -define(TICK_CMD,4). 10 | -define(LEARN_CMD,5). 11 | -define(ZAB_CMD,6). 12 | 13 | -define(LOOKING,1). 14 | -define(LEADING,2). 15 | -define(FOLLOWING,3). 16 | -define(LEADER_RECOVER,4). 17 | -define(FOLLOWER_RECOVER,5). 18 | -define(OBSERVING,6). 19 | 20 | 21 | 22 | 23 | -type cmd()::?ELECT_CMD|?VOTE_CMD. 24 | 25 | 26 | -record(vote,{from::node() 27 | ,leader::node() 28 | ,zxid::zxid() 29 | ,epoch 30 | ,state::?LOOKING|?LEADING|?OBSERVING 31 | ,last_commit_zxid::zxid() 32 | }). 33 | 34 | 35 | -record(election, { 36 | parent ::atom(), 37 | ensemble = [] :: [node()], 38 | last_zxid ::zxid(), 39 | quorum::non_neg_integer(), 40 | last_commit_zxid::zxid(), 41 | logical_clock::integer() 42 | }). 43 | 44 | -record(msg,{cmd::cmd(), 45 | value::any(), 46 | epoch::integer() 47 | }). 48 | 49 | -record(transaction,{value::any(),zxid::zxid()}). 50 | 51 | 52 | -record(proposal,{sender::peer(),client::client(),transaction::#transaction{}}). 53 | 54 | -record(zab_req,{msg}). 55 | -record(zab_ack,{msg}). 56 | -record(zab_commit,{msg}). 57 | 58 | -record(zab_server_info,{leader,zab_state}). 59 | 60 | -record(proposal_rec,{zxid::zxid(),proposal::#proposal{},acks::list(),commit::boolean()}). 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/eleveldb_util.erl: -------------------------------------------------------------------------------- 1 | %% yaoxinming@gmail.com 2 | %% 3 | %% iterate db count record 4 | %% 5 | -module(eleveldb_util). 6 | -export([itera_end/2,zxid_fold/5]). 7 | -include("zabe_main.hrl"). 8 | 9 | 10 | 11 | zxid_fold(Ref,Fun,Acc,Start,Prefix)-> 12 | 13 | Key=zabe_util:encode_key(zabe_util:encode_zxid(Start),Prefix), 14 | fold(Ref, 15 | Fun,Acc,[],Prefix,list_to_binary(Key),itera_end). 16 | 17 | 18 | 19 | fold(Ref, Fun, Acc0, Opts,Prefix,Key,EndFun) -> 20 | {ok, Itr} = eleveldb:iterator(Ref, Opts), 21 | do_fold(Itr, Fun, Acc0, Opts,Prefix,Key,EndFun). 22 | 23 | do_fold(Itr, Fun, Acc0, _Opts,Prefix,Key,EndFun) -> 24 | try 25 | %% Extract {first_key, binary()} and seek to that key as a starting 26 | %% point for the iteration. The folding function should use throw if it 27 | %% wishes to terminate before the end of the fold. 28 | 29 | fold_loop(eleveldb:iterator_move(Itr,Key), Itr, Fun, Acc0,Prefix,EndFun) 30 | after 31 | eleveldb:iterator_close(Itr) 32 | end. 33 | 34 | fold_loop({error, invalid_iterator}, _Itr, _Fun, Acc0,_Prefix,_EndFun) -> 35 | Acc0; 36 | fold_loop({ok, _K, _V}, _Itr,_Fun, Acc0={_,0},_Prefix,_EndFun) -> 37 | Acc0; 38 | fold_loop({ok, K, V}, Itr, Fun, Acc0,Prefix,EndFun) -> 39 | case apply(?MODULE,EndFun,[K,Prefix]) of 40 | false-> 41 | Acc0; 42 | true-> 43 | K1=zabe_util:decode_zxid(zabe_util:decode_key(binary_to_list(K),Prefix)), 44 | Acc = Fun({K1, binary_to_term(V)},Acc0), 45 | % io:format("~p ~p ~n",[K1,Acc]), 46 | fold_loop(eleveldb:iterator_move(Itr, next), Itr, Fun, Acc,Prefix,EndFun) 47 | end. 48 | 49 | itera_end(K,Prefix)-> 50 | K1=binary_to_list(K), 51 | P=zabe_util:prefix_match(K1,Prefix), 52 | Len=length(K1)-length(Prefix), 53 | (P andalso Len==30) %todo fix magic number 54 | . 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /test/zabe_util_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(zabe_util_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | init_per_suite(Config)-> 8 | 9 | 10 | Config 11 | . 12 | 13 | 14 | 15 | end_per_suite(Config)-> 16 | 17 | Config. 18 | 19 | init_per_testcase(_,Config)-> 20 | 21 | Config 22 | . 23 | 24 | end_per_testcase(Config)-> 25 | 26 | ok. 27 | 28 | 29 | 30 | 31 | all()-> 32 | [e_2_s,encode,encode_key,prefix_match,zxid_compare]. 33 | 34 | encode_key(C)-> 35 | "55123"=zabe_util:encode_key("123","55"), 36 | "123"=zabe_util:decode_key("55123","55"), 37 | "123"=zabe_util:decode_key("123",""), 38 | ok. 39 | 40 | prefix_match(_C)-> 41 | true=zabe_util:prefix_match("1230000","123"), 42 | false=zabe_util:prefix_match("1230000","1234"), 43 | false=zabe_util:prefix_match("","1234"). 44 | 45 | encode(_)-> 46 | Zxid=zabe_util:encode_zxid({1,2}), 47 | "000000000100000000000000000002"=Zxid, 48 | {1,2}=zabe_util:decode_zxid(Zxid). 49 | e_2_s(_C)-> 50 | I="1", 51 | "01"=zabe_util:e_2_s(I,2), 52 | try zabe_util:e_2_s(1,2) of 53 | _->ct:fail("should error") 54 | catch 55 | _:_->ok 56 | end, 57 | "0000000001"=zabe_util:epoch_to_string(1), 58 | "00000000000000000001"=zabe_util:txn_to_string(1). 59 | 60 | zxid_compare(Config)-> 61 | equal=zabe_util:zxid_compare({0,0},{0,0}), 62 | big=zabe_util:zxid_compare({1,2},{1,1}), 63 | epoch_small=zabe_util:zxid_compare({1,2},{2,1}), 64 | ok. 65 | 66 | select(C)-> 67 | Que=ets:new(?MODULE,[{keypos,2},ordered_set]), 68 | ets:insert(Que, #proposal_rec{zxid={1,1},commit=true}), 69 | ets:insert(Que, #proposal_rec{zxid={1,2},commit=true}), 70 | ets:insert(Que, #proposal_rec{zxid={1,3},commit=true}), 71 | ets:insert(Que, #proposal_rec{zxid={1,4}}), 72 | zabe_util:select_ets(Que,fun(A)-> 73 | A end,{1,2}) 74 | . 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /src/zabe_util.erl: -------------------------------------------------------------------------------- 1 | -module(zabe_util). 2 | -export([e_2_s/2,decode_zxid/1,encode_zxid/1]). 3 | -export([zxid_eq/2,zxid_big/2,zxid_plus/1,zxid_compare/2]). 4 | -export([epoch_to_string/1,txn_to_string/1]). 5 | 6 | -export([decode_key/2,encode_key/2,prefix_match/2]). 7 | 8 | -export([zxid_big_eq/2,change_leader_zxid/1,select_ets/3]). 9 | 10 | 11 | -include("zabe_main.hrl"). 12 | 13 | 14 | select_ets(Tab,Fun,{Epoch,Txn}=StartZxid)-> 15 | ets:select(Tab,[{{'_','_','_','_','_'},[],[]}]). 16 | % ets:select(Tab,[{{'_','$1','_','_','_'},[{'>','$1',1}]}]). 17 | change_leader_zxid({E,_Txn})-> 18 | {E+1,0}. 19 | % if 20 | % E=:=0 andalso Txn =:=0-> 21 | % {1,1}; 22 | % true -> 23 | % {E+1,1} 24 | % end 25 | 26 | zxid_big_eq(Z1,Z2)-> 27 | zxid_eq(Z1,Z2) orelse zxid_big(Z1,Z2). 28 | 29 | prefix_match(String,Match)-> 30 | 31 | case re:run(String,Match) of 32 | {match,[{0,Len}]}when Len==length(Match)-> 33 | true; 34 | _ ->false 35 | end. 36 | 37 | decode_key(Key,Prefix)-> 38 | Len=erlang:length(Prefix), 39 | string:substr(Key,Len+1,length(Key)-Len). 40 | 41 | encode_key(Key,Prefix)-> 42 | Prefix++Key. 43 | 44 | -spec zxid_compare(Z1::zxid(),Z2::zxid())-> 45 | equal|big|epoch_smsall. 46 | zxid_compare(Z1={E1,_T1},Z2={E2,_T2})-> 47 | case zxid_eq(Z1,Z2) of 48 | true-> 49 | equal; 50 | false -> 51 | if 52 | E1 epoch_small ; 53 | true -> big 54 | end 55 | end. 56 | 57 | 58 | zxid_plus({E,T})-> 59 | {E,T+1}. 60 | 61 | zxid_eq({E1,T1},{E2,T2})-> 62 | E1==E2 andalso T1==T2. 63 | 64 | zxid_big({E1,T1},{E2,T2})-> 65 | E1 >E2 orelse ((E1==E2) andalso T1>T2). 66 | encode_zxid({Epoch,Txn})-> 67 | epoch_to_string(Epoch)++txn_to_string(Txn). 68 | decode_zxid(Zxid)-> 69 | Epoch=string:substr(Zxid,1,10), 70 | Txn=string:substr(Zxid,11,20), 71 | {erlang:list_to_integer(Epoch),list_to_integer(Txn)}. 72 | 73 | 74 | epoch_to_string(Epoch) when is_integer(Epoch)-> 75 | e_2_s(erlang:integer_to_list(Epoch),10) 76 | . 77 | 78 | 79 | txn_to_string(Txn) when is_integer(Txn)-> 80 | e_2_s(erlang:integer_to_list(Txn),20). 81 | 82 | e_2_s(S,L) when length(S)=:=L-> 83 | S; 84 | e_2_s(S,L) when length(S) < L -> 85 | e_2_s([$0|S],L); 86 | e_2_s(_,_) -> 87 | exit(epoch_errror). 88 | 89 | 90 | -------------------------------------------------------------------------------- /test/zabe_fast_elect_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(zabe_fast_elect_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | 8 | init_per_suite(Config)-> 9 | application:start(consensus_engine), 10 | 11 | Config 12 | . 13 | 14 | 15 | 16 | setup() -> 17 | % error_logger:tty(false), 18 | application:load(lager), 19 | application:set_env(lager, handlers, [{lager_console_backend, error} 20 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 21 | ]), 22 | application:set_env(lager, error_logger_redirect, false), 23 | application:start(compiler), 24 | application:start(syntax_tools), 25 | application:start(lager) 26 | % gen_event:call(lager_event, ?MODULE, flush) 27 | . 28 | 29 | cleanup(_) -> 30 | application:stop(lager), 31 | error_logger:tty(true). 32 | 33 | end_per_suite(Config)-> 34 | 35 | Config. 36 | 37 | init_per_testcase(_,Config)-> 38 | 39 | Config 40 | . 41 | 42 | end_per_testcase(Config)-> 43 | catch erlang:exit(whereis(zabe_fast_leader_test),normal), 44 | catch slave:stop('z1@localhost'), 45 | catch slave:stop('z2@localhost'), 46 | ok. 47 | 48 | -define(HOST,'localhost'). 49 | 50 | 51 | all()-> 52 | [elect]. 53 | 54 | elect(Config)-> 55 | % setup(), 56 | Nodes=[node(),'z1@localhost','z2@localhost'], 57 | Q=2, 58 | LastZxid={1,1}, 59 | Ps=code:get_path(), 60 | Arg=lists:foldl(fun(A,Acc)-> 61 | " -pa "++A++ Acc end, " ",Ps), 62 | slave:start(?HOST,z1,Arg), 63 | slave:start(?HOST,z2,Arg), 64 | setup(), 65 | rpc:call('z1@localhost',zabe_fast_elect_SUITE,setup,[]), 66 | rpc:call('z2@localhost',zabe_fast_elect_SUITE,setup,[]), 67 | zabe_fast_leader_test:start_link(LastZxid,Nodes,Q), 68 | 69 | rpc:call('z1@localhost',zabe_fast_leader_test,start_link,[LastZxid,Nodes,Q]), 70 | rpc:call('z2@localhost',zabe_fast_leader_test,start_link,[LastZxid,Nodes,Q]), 71 | ct:sleep(300), 72 | ?FOLLOWING=zabe_fast_leader_test:get_state(), 73 | ?FOLLOWING=rpc:call('z1@localhost',zabe_fast_leader_test,get_state,[]), 74 | ?LEADING=rpc:call('z2@localhost',zabe_fast_leader_test,get_state,[]), 75 | ct:sleep(200). 76 | 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /test/elect2_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(elect2_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | %% test one node join cluster and become follower 8 | %% 9 | %% 10 | %% 11 | 12 | init_per_suite(Config)-> 13 | application:start(consensus_engine), 14 | 15 | Config 16 | . 17 | 18 | 19 | 20 | setup() -> 21 | % error_logger:tty(false), 22 | application:load(lager), 23 | application:set_env(lager, handlers, [{lager_console_backend,error} 24 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 25 | ]), 26 | application:set_env(lager, error_logger_redirect, false), 27 | application:start(compiler), 28 | application:start(syntax_tools), 29 | application:start(lager) 30 | % gen_event:call(lager_event, ?MODULE, flush) 31 | . 32 | 33 | cleanup(_) -> 34 | application:stop(lager), 35 | error_logger:tty(true). 36 | 37 | end_per_suite(Config)-> 38 | 39 | Config. 40 | 41 | init_per_testcase(_,Config)-> 42 | 43 | Config 44 | . 45 | 46 | end_per_testcase(_Config)-> 47 | erlang:exit(whereis(zabe_fast_leader_test),normal), 48 | slave:stop('z1@localhost'), 49 | slave:stop('z2@localhost'), 50 | ok. 51 | 52 | -define(HOST,'localhost'). 53 | 54 | 55 | all()-> 56 | [elect]. 57 | 58 | elect(Config)-> 59 | % setup(), 60 | Nodes=[node(),'z1@localhost','z2@localhost'], 61 | Q=2, 62 | LastZxid={1,1}, 63 | Ps=code:get_path(), 64 | Arg=lists:foldl(fun(A,Acc)-> 65 | " -pa "++A++ Acc end, " ",Ps), 66 | slave:start(?HOST,z1,Arg), 67 | slave:start(?HOST,z2,Arg), 68 | setup(), 69 | rpc:call('z1@localhost',zabe_fast_elect_SUITE,setup,[]), 70 | rpc:call('z2@localhost',zabe_fast_elect_SUITE,setup,[]), 71 | zabe_fast_leader_test:start_link(LastZxid,Nodes,Q), 72 | 73 | rpc:call('z1@localhost',zabe_fast_leader_test,start_link,[LastZxid,Nodes,Q]), 74 | rpc:call('z2@localhost',zabe_fast_leader_test,start_link,[LastZxid,Nodes,Q]), 75 | ct:sleep(200), 76 | ?FOLLOWING=zabe_fast_leader_test:get_state(), 77 | ?LEADING=rpc:call('z2@localhost',zabe_fast_leader_test,get_state,[]), 78 | ?FOLLOWING=rpc:call('z1@localhost',zabe_fast_leader_test,get_state,[]), 79 | ct:sleep(10). 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /test/leader_down_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(leader_down_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | init_per_suite(Config)-> 8 | 9 | 10 | Config 11 | . 12 | 13 | 14 | 15 | setup(Dir) -> 16 | % error_logger:tty(false), 17 | application:load(lager), 18 | 19 | application:set_env(lager, handlers, [{lager_console_backend, error} 20 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 21 | ]), 22 | application:set_env(lager, error_logger_redirect, false), 23 | application:start(compiler), 24 | application:start(syntax_tools), 25 | application:start(lager), 26 | os:cmd("rm -rf /tmp/*.db"), 27 | timer:sleep(2), 28 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 29 | ok 30 | . 31 | 32 | cleanup(_) -> 33 | application:stop(lager), 34 | error_logger:tty(true). 35 | 36 | end_per_suite(Config)-> 37 | 38 | Config. 39 | 40 | 41 | init_per_testcase(_TestCase, Config) -> 42 | % os:cmd("rm -rf /tmp/proposal.bb"), 43 | % timer:sleep(2), 44 | % {ok,Pid}=zabe_proposal_leveldb_backend:start_link("/tmp/proposal.bb",[]), 45 | % [{pid,Pid}|Config] 46 | Config 47 | 48 | . 49 | 50 | end_per_testcase(_TestCase, Config) -> 51 | % Pid=?config(pid,Config), 52 | % erlang:exit(Pid,normal), 53 | slave:stop('z1@localhost'), 54 | slave:stop('z2@localhost'), 55 | ok. 56 | 57 | -define(HOST,'localhost'). 58 | 59 | 60 | all()-> 61 | [elect]. 62 | 63 | elect(Config)-> 64 | D1="/tmp/1.db", 65 | D2="/tmp/2.db", 66 | D3="/tmp/3.db", 67 | Op1=[{proposal_dir,"/tmp/p1.db"}], 68 | Op2=[{proposal_dir,"/tmp/p2.db"}], 69 | Op3=[{proposal_dir,"/tmp/p3.db"}], 70 | Nodes=[node(),'z1@localhost','z2@localhost'], 71 | Ps=code:get_path(), 72 | Arg=lists:foldl(fun(A,Acc)-> 73 | " -pa "++A++ Acc end, " ",Ps), 74 | slave:start(?HOST,z1,Arg), 75 | slave:start(?HOST,z2,Arg), 76 | setup("/tmp/p1.db"), 77 | ok=rpc:call('z1@localhost',?MODULE,setup,["/tmp/p2.db"]), 78 | ok=rpc:call('z2@localhost',?MODULE,setup,["/tmp/p3.db"]), 79 | timer:sleep(500), 80 | {ok,_}=zabe_learn_leveldb:start_link(Nodes,Op1,D1), 81 | {ok,_}=rpc:call('z1@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 82 | {ok,_}=rpc:call('z2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op3,D3]), 83 | timer:sleep(2000), 84 | 85 | Key="test1", 86 | Value="value1", 87 | ok=zabe_learn_leveldb:put(Key,Value), 88 | ok=zabe_learn_leveldb:put("k1","v1"), 89 | % ok=zabe_learn_leveldb:put("k2","v2"), 90 | % ok=zabe_learn_leveldb:put("k3","v3"), 91 | %{ok,Value}=zabe_learn_leveldb:get(Key), 92 | 93 | % {ok,Value}=rpc:call('z2@localhost',zabe_learn_leveldb,get,[Key]), 94 | % {ok,Value}=rpc:call('z1@localhost',zabe_learn_leveldb,get,[Key]), 95 | % {ok,"v3"}=rpc:call('z1@localhost',zabe_learn_leveldb,get,["k3"]), 96 | slave:stop('z2@localhost'), 97 | timer:sleep(2000), 98 | ok=zabe_learn_leveldb:put("k5","v5"), 99 | {ok,"v5"}=zabe_learn_leveldb:get("k5"), 100 | 101 | ok 102 | . 103 | 104 | -------------------------------------------------------------------------------- /test/follow_online_recover_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(follow_online_recover_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | init_per_suite(Config)-> 8 | 9 | 10 | Config 11 | . 12 | 13 | 14 | setup(Dir) -> 15 | % error_logger:tty(false), 16 | application:load(lager), 17 | application:set_env(lager, handlers, [{lager_console_backend, error} 18 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 19 | ]), 20 | application:set_env(lager, error_logger_redirect, false), 21 | application:start(compiler), 22 | application:start(syntax_tools), 23 | application:start(lager), 24 | os:cmd("rm -rf /tmp/*.db"), 25 | timer:sleep(2), 26 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 27 | ok 28 | . 29 | 30 | cleanup(_) -> 31 | application:stop(lager), 32 | error_logger:tty(true). 33 | 34 | end_per_suite(Config)-> 35 | 36 | Config. 37 | 38 | 39 | init_per_testcase(_TestCase, Config) -> 40 | % os:cmd("rm -rf /tmp/proposal.bb"), 41 | % timer:sleep(2), 42 | % {ok,Pid}=zabe_proposal_leveldb_backend:start_link("/tmp/proposal.bb",[]), 43 | % [{pid,Pid}|Config] 44 | Config 45 | 46 | . 47 | 48 | end_per_testcase(_TestCase, Config) -> 49 | catch erlang:exit(whereis(zabe_learn_leveldb),normal), 50 | catch slave:stop('z1@localhost'), 51 | catch slave:stop('z2@localhost'), 52 | catch slave:stop('z3@localhost'), 53 | % Pid=?config(pid,Config), 54 | % erlang:exit(Pid,normal), 55 | ok. 56 | 57 | -define(HOST,'localhost'). 58 | 59 | 60 | all()-> 61 | [elect]. 62 | 63 | elect(Config)-> 64 | D1="/tmp/1.db", 65 | D2="/tmp/2.db", 66 | D3="/tmp/3.db", 67 | Op1=[{proposal_dir,"/tmp/p1.db"}], 68 | Op2=[{proposal_dir,"/tmp/p2.db"}], 69 | Op3=[{proposal_dir,"/tmp/p3.db"}], 70 | Nodes=['z1@localhost','z2@localhost','z3@localhost'], 71 | Ps=code:get_path(), 72 | Arg=lists:foldl(fun(A,Acc)-> 73 | " -pa "++A++ Acc end, " ",Ps), 74 | slave:start(?HOST,z1,Arg), 75 | slave:start(?HOST,z2,Arg), 76 | slave:start(?HOST,z3,Arg), 77 | 78 | ok=rpc:call('z1@localhost',?MODULE,setup,["/tmp/p1.db"]), 79 | ok=rpc:call('z2@localhost',?MODULE,setup,["/tmp/p2.db"]), 80 | ok=rpc:call('z3@localhost',?MODULE,setup,["/tmp/p3.db"]), 81 | timer:sleep(500), 82 | 83 | {ok,_}=rpc:call('z1@localhost',zabe_learn_leveldb,start_link,[Nodes,Op1,D1]), 84 | 85 | {ok,_}=rpc:call('z3@localhost',zabe_learn_leveldb,start_link,[Nodes,Op3,D3]), 86 | timer:sleep(800), 87 | 88 | Key="test1", 89 | Value="value1", 90 | rpc:call('z1@localhost',zabe_learn_leveldb,put,[Key,Value]), 91 | 92 | {ok,_}=rpc:call('z2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 93 | 94 | lists:foreach(fun(A)->K=erlang:integer_to_list(A), 95 | timer:sleep(2), 96 | rpc:call('z1@localhost',zabe_learn_leveldb,put,[K,K]) end,lists:seq(1,50)), 97 | 98 | {ok,Value}=rpc:call('z1@localhost',zabe_learn_leveldb,get,[Key]), 99 | 100 | {ok,Value}=rpc:call('z3@localhost',zabe_learn_leveldb,get,[Key]), 101 | 102 | 103 | ct:sleep(2000), 104 | {ok,Value}=rpc:call('z2@localhost',zabe_learn_leveldb,get,[Key]), 105 | {ok,"1"}=rpc:call('z2@localhost',zabe_learn_leveldb,get,["1"]), 106 | {ok,"50"}=rpc:call('z2@localhost',zabe_learn_leveldb,get,["50"]), 107 | ok 108 | . 109 | 110 | -------------------------------------------------------------------------------- /test/recover1_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module('recover1_SUITE'). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | 8 | init_per_suite(Config)-> 9 | 10 | 11 | Config 12 | . 13 | 14 | 15 | 16 | setup(Dir) -> 17 | % error_logger:tty(false), 18 | application:load(lager), 19 | application:set_env(lager, handlers, [{lager_console_backend, error} 20 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 21 | ]), 22 | application:set_env(lager, error_logger_redirect, false), 23 | application:start(compiler), 24 | application:start(syntax_tools), 25 | application:start(lager), 26 | % os:cmd("rm -rf "++Dir), 27 | timer:sleep(2), 28 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 29 | ok 30 | . 31 | 32 | cleanup(_) -> 33 | application:stop(lager), 34 | error_logger:tty(true). 35 | 36 | end_per_suite(Config)-> 37 | 38 | Config. 39 | 40 | 41 | init_per_testcase(_TestCase, Config) -> 42 | % os:cmd("rm -rf /tmp/proposal.bb"), 43 | % timer:sleep(2), 44 | % {ok,Pid}=zabe_proposal_leveldb_backend:start_link("/tmp/proposal.bb",[]), 45 | % [{pid,Pid}|Config] 46 | Config 47 | 48 | . 49 | 50 | end_per_testcase(_TestCase, Config) -> 51 | % Pid=?config(pid,Config), 52 | % erlang:exit(Pid,normal), 53 | % Pid=?config(pid,Config), 54 | % erlang:exit(Pid,normal), 55 | 56 | erlang:exit(whereis(zabe_learn_leveldb),normal), 57 | slave:stop('z1@localhost'), 58 | slave:stop('z2@localhost'), 59 | ok. 60 | 61 | -define(HOST,'localhost'). 62 | 63 | 64 | all()-> 65 | [elect]. 66 | 67 | elect(Config)-> 68 | D1="/tmp/1.db", 69 | D2="/tmp/2.db", 70 | D3="/tmp/3.db", 71 | Op1=[{proposal_dir,"/tmp/p1.db"}], 72 | Op2=[{proposal_dir,"/tmp/p2.db"}], 73 | Op3=[{proposal_dir,"/tmp/p3.db"}], 74 | Nodes=[node(),'z1@localhost','z2@localhost'], 75 | 76 | D1="/tmp/1.db", 77 | D2="/tmp/2.db", 78 | D3="/tmp/3.db", 79 | Op1=[{proposal_dir,"/tmp/p1.db"}], 80 | Op2=[{proposal_dir,"/tmp/p2.db"}], 81 | Op3=[{proposal_dir,"/tmp/p3.db"}], 82 | Nodes=[node(),'z1@localhost','z2@localhost'], 83 | Ps=code:get_path(), 84 | Arg=lists:foldl(fun(A,Acc)-> 85 | " -pa "++A++ Acc end, " ",Ps), 86 | slave:start(?HOST,z1,Arg), 87 | slave:start(?HOST,z2,Arg), 88 | setup("/tmp/p1.db"), 89 | ok=rpc:call('z1@localhost',?MODULE,setup,["/tmp/p2.db"]), 90 | ok=rpc:call('z2@localhost',?MODULE,setup,["/tmp/p3.db"]), 91 | timer:sleep(500), 92 | {ok,_}=zabe_learn_leveldb:start_link(Nodes,Op1,D1), 93 | 94 | {ok,_}=rpc:call('z2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op3,D3]), 95 | timer:sleep(4000), 96 | 97 | Key="test1", 98 | Value="value1", 99 | ok=zabe_learn_leveldb:put(Key,Value), 100 | ok=zabe_learn_leveldb:put("k1","v1"), 101 | ok=zabe_learn_leveldb:put("k2","v2"), 102 | ok=zabe_learn_leveldb:put("k3","v3"), 103 | {ok,Value}=zabe_learn_leveldb:get(Key), 104 | 105 | {ok,Value}=rpc:call('z2@localhost',zabe_learn_leveldb,get,[Key]), 106 | 107 | {ok,_}=rpc:call('z1@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 108 | timer:sleep(500), 109 | {ok,Value}=rpc:call('z1@localhost',zabe_learn_leveldb,get,[Key]), 110 | {ok,"v3"}=rpc:call('z1@localhost',zabe_learn_leveldb,get,["k3"]), 111 | ok 112 | . 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /test/follow_down_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(follow_down_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | init_per_suite(Config)-> 8 | Config 9 | . 10 | 11 | setup(Dir) -> 12 | % error_logger:tty(false), 13 | application:load(lager), 14 | application:set_env(lager, handlers, [{lager_console_backend,error} 15 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 16 | ]), 17 | application:set_env(lager, error_logger_redirect, false), 18 | application:start(compiler), 19 | application:start(syntax_tools), 20 | application:start(lager), 21 | os:cmd("rm -rf /tmp/*.db"), 22 | timer:sleep(2), 23 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 24 | ok 25 | . 26 | 27 | cleanup(_) -> 28 | application:stop(lager), 29 | error_logger:tty(true). 30 | 31 | end_per_suite(Config)-> 32 | 33 | Config. 34 | 35 | 36 | init_per_testcase(_TestCase, Config) -> 37 | 38 | Config 39 | 40 | . 41 | 42 | end_per_testcase(_TestCase, Config) -> 43 | % Pid=?config(pid,Config), 44 | % erlang:exit(Pid,normal), 45 | catch erlang:exit(whereis(zabe_learn_leveldb),normal), 46 | % slave:stop('z1@localhost'), 47 | slave:stop('z2@localhost'), 48 | slave:stop('z3@localhost'), 49 | ok. 50 | 51 | -define(HOST,'localhost'). 52 | 53 | 54 | all()-> 55 | [elect]. 56 | 57 | elect(Config)-> 58 | 59 | D1="/tmp/1.db", 60 | D2="/tmp/2.db", 61 | D3="/tmp/3.db", 62 | Op1=[{proposal_dir,"/tmp/p1.db"}], 63 | Op2=[{proposal_dir,"/tmp/p2.db"}], 64 | Op3=[{proposal_dir,"/tmp/p3.db"}], 65 | Nodes=['z1@localhost','z2@localhost','z3@localhost'], 66 | Ps=code:get_path(), 67 | Arg=lists:foldl(fun(A,Acc)-> 68 | " -pa "++A++ Acc end, " ",Ps), 69 | slave:start(?HOST,z1,Arg), 70 | slave:start(?HOST,z2,Arg), 71 | slave:start(?HOST,z3,Arg), 72 | 73 | ok=rpc:call('z1@localhost',?MODULE,setup,["/tmp/p1.db"]), 74 | ok=rpc:call('z2@localhost',?MODULE,setup,["/tmp/p2.db"]), 75 | ok=rpc:call('z3@localhost',?MODULE,setup,["/tmp/p3.db"]), 76 | timer:sleep(500), 77 | 78 | {ok,_}=rpc:call('z1@localhost',zabe_learn_leveldb,start_link,[Nodes,Op1,D1]), 79 | {ok,_}=rpc:call('z2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 80 | {ok,_}=rpc:call('z3@localhost',zabe_learn_leveldb,start_link,[Nodes,Op3,D3]), 81 | timer:sleep(800), 82 | 83 | Key="test1", 84 | Value="value1", 85 | rpc:call('z1@localhost',zabe_learn_leveldb,put,[Key,Value]), 86 | 87 | % ok=zabe_learn_leveldb:put("k2","v2"), 88 | % ok=zabe_learn_leveldb:put("k3","v3"), 89 | %{ok,Value}=zabe_learn_leveldb:get(Key), 90 | 91 | % {ok,Value}=rpc:call('z2@localhost',zabe_learn_leveldb,get,[Key]), 92 | % {ok,Value}=rpc:call('z1@localhost',zabe_learn_leveldb,get,[Key]), 93 | % {ok,"v3"}=rpc:call('z1@localhost',zabe_learn_leveldb,get,["k3"]), 94 | slave:stop('z2@localhost'), 95 | 96 | {ok,Value}=rpc:call('z1@localhost',zabe_learn_leveldb,get,[Key]), 97 | slave:stop('z1@localhost'), 98 | {ok,Value}=rpc:call('z3@localhost',zabe_learn_leveldb,get,[Key]), 99 | ct:sleep(1000), 100 | {error,not_ready}=rpc:call('z3@localhost',zabe_learn_leveldb,put,[Key,Value]), 101 | slave:start(?HOST,z2,Arg), 102 | rpc:call('z2@localhost',?MODULE,setup,["/tmp/p2.db"]), 103 | ct:sleep(1000), 104 | rpc:call('z2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 105 | ct:sleep(1000), 106 | ok=rpc:call('z3@localhost',zabe_learn_leveldb,put,[Key,Value]), 107 | ok 108 | . 109 | 110 | -------------------------------------------------------------------------------- /test/zabe_learn_leveldb_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(zabe_learn_leveldb_SUITE). 2 | -compile(export_all). 3 | -include_lib("common_test/include/ct.hrl"). 4 | -compile([{parse_transform, lager_transform}]). 5 | -include("zabe_main.hrl"). 6 | 7 | 8 | init_per_suite(Config)-> 9 | 10 | 11 | Config 12 | . 13 | 14 | 15 | 16 | setup(Dir) -> 17 | % error_logger:tty(false), 18 | application:load(lager), 19 | application:set_env(lager, handlers, [{lager_console_backend, error} 20 | ,{lager_file_backend,[{"/tmp/console.log",error,10485760,"$D0",5}]} 21 | ]), 22 | application:set_env(lager, error_logger_redirect, false), 23 | application:start(compiler), 24 | application:start(syntax_tools), 25 | application:start(lager), 26 | %os:cmd("rm -rf "++Dir), 27 | timer:sleep(2), 28 | zabe_proposal_leveldb_backend:start_link(Dir,[]), 29 | ok 30 | . 31 | 32 | cleanup(_) -> 33 | application:stop(lager), 34 | error_logger:tty(true). 35 | 36 | end_per_suite(Config)-> 37 | 38 | Config. 39 | 40 | 41 | init_per_testcase(_TestCase, Config) -> 42 | % os:cmd("rm -rf /tmp/proposal.bb"), 43 | % timer:sleep(2), 44 | % {ok,Pid}=zabe_proposal_leveldb_backend:start_link("/tmp/proposal.bb",[]), 45 | % [{pid,Pid}|Config] 46 | Config 47 | 48 | . 49 | 50 | end_per_testcase(_TestCase, Config) -> 51 | 52 | erlang:exit(whereis(zabe_learn_leveldb),normal), 53 | slave:stop('n1@localhost'), 54 | slave:stop('n2@localhost'), 55 | ok. 56 | 57 | -define(HOST,'localhost'). 58 | 59 | 60 | all()-> 61 | [elect]. 62 | 63 | elect(Config)-> D1="/tmp/1.db", 64 | D2="/tmp/2.db", 65 | D3="/tmp/3.db", 66 | Op1=[{proposal_dir,"/tmp/p1.db"}], 67 | Op2=[{proposal_dir,"/tmp/p2.db"}], 68 | Op3=[{proposal_dir,"/tmp/p3.db"}], 69 | Nodes=[node(),'n1@localhost','n2@localhost'], 70 | 71 | D1="/tmp/1.db", 72 | D2="/tmp/2.db", 73 | D3="/tmp/3.db", 74 | Op1=[{proposal_dir,"/tmp/p1.db"}], 75 | Op2=[{proposal_dir,"/tmp/p2.db"}], 76 | Op3=[{proposal_dir,"/tmp/p3.db"}], 77 | Nodes=[node(),'n1@localhost','n2@localhost'], 78 | Ps=code:get_path(), 79 | Arg=lists:foldl(fun(A,Acc)-> 80 | " -pa "++A++ Acc end, " ",Ps), 81 | slave:start(?HOST,n1,Arg), 82 | slave:start(?HOST,n2,Arg), 83 | setup("/tmp/p1.db"), 84 | ok=rpc:call('n1@localhost',zabe_learn_leveldb_SUITE,setup,["/tmp/p2.db"]), 85 | ok=rpc:call('n2@localhost',zabe_learn_leveldb_SUITE,setup,["/tmp/p3.db"]), 86 | timer:sleep(800), 87 | {ok,_}=zabe_learn_leveldb:start_link(Nodes,Op1,D1), 88 | 89 | {ok,_}=rpc:call('n1@localhost',zabe_learn_leveldb,start_link,[Nodes,Op2,D2]), 90 | 91 | {ok,_}=rpc:call('n2@localhost',zabe_learn_leveldb,start_link,[Nodes,Op3,D3]), 92 | timer:sleep(4000), 93 | 94 | Key="test1", 95 | Value="value1", 96 | ok=zabe_learn_leveldb:put(Key,Value), 97 | {ok,Value}=zabe_learn_leveldb:get(Key), 98 | 99 | {ok,Value}=rpc:call('n2@localhost',zabe_learn_leveldb,get,[Key]), 100 | {ok,Value}=rpc:call('n1@localhost',zabe_learn_leveldb,get,[Key]), 101 | ok 102 | . 103 | 104 | 105 | elect1(Config)-> 106 | D1="/tmp/1.db", 107 | D2="/tmp/2.db", 108 | D3="/tmp/3.db", 109 | Op1=[{proposal_dir,"/tmp/p1.db"}], 110 | Op2=[{proposal_dir,"/tmp/p2.db"}], 111 | Op3=[{proposal_dir,"/tmp/p3.db"}], 112 | Nodes=[node(),'n1@localhost','n2@localhost'], 113 | 114 | D1="/tmp/1.db", 115 | D2="/tmp/2.db", 116 | D3="/tmp/3.db", 117 | Op1=[{proposal_dir,"/tmp/p1.db"}], 118 | Op2=[{proposal_dir,"/tmp/p2.db"}], 119 | Op3=[{proposal_dir,"/tmp/p3.db"}], 120 | Nodes=[node(),'n1@localhost','n2@localhost'], 121 | Ps=code:get_path(), 122 | 123 | {ok,P}=zabe_learn_leveldb:start_link(Nodes,Op1,D1), 124 | erlang:send(P,#msg{cmd=?ZAB_CMD,value=ttttt}), 125 | timer:sleep(50). 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /test/zabe_proposal_leveldb_backend_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author <> 3 | %%% @copyright (C) 2012, 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 19 Jun 2012 by <> 8 | %%%------------------------------------------------------------------- 9 | -module(zabe_proposal_leveldb_backend_SUITE). 10 | 11 | %% Note: This directive should only be used in test suites. 12 | -compile(export_all). 13 | 14 | -include_lib("common_test/include/ct.hrl"). 15 | -include("zabe_main.hrl"). 16 | 17 | 18 | suite() -> 19 | [{timetrap,{minutes,10}}]. 20 | 21 | init_per_suite(Config) -> 22 | Config 23 | . 24 | 25 | end_per_suite(Config) -> 26 | 27 | ok. 28 | 29 | init_per_group(_GroupName, Config) -> 30 | Config. 31 | 32 | 33 | end_per_group(_GroupName, _Config) -> 34 | ok. 35 | 36 | init_per_testcase(_TestCase, Config) -> 37 | os:cmd("rm -rf /tmp/proposal.bb"), 38 | timer:sleep(2), 39 | {ok,Pid}=zabe_proposal_leveldb_backend:start_link("/tmp/proposal.bb",[{prefix,"100"}]), 40 | [{pid,Pid}|Config] 41 | 42 | 43 | . 44 | 45 | end_per_testcase(_TestCase, Config) -> 46 | Pid=?config(pid,Config), 47 | erlang:exit(Pid,normal), 48 | ok. 49 | 50 | groups() -> 51 | []. 52 | 53 | all() -> 54 | [put_get,get_last,zxid_fold,get_epoch_last_zxid]. 55 | 56 | put_get(Config) -> 57 | Zxid={1,1}, 58 | Txn=#transaction{zxid=Zxid,value=test}, 59 | Proposal=#proposal{transaction=Txn}, 60 | ok=zabe_proposal_leveldb_backend:put_proposal(Zxid,Proposal,[]), 61 | {ok,Proposal}=zabe_proposal_leveldb_backend:get_proposal(Zxid,[]), 62 | ok. 63 | 64 | get_last(Config)-> 65 | Zxid={10,10}, 66 | Txn=#transaction{zxid=Zxid,value=test}, 67 | Proposal=#proposal{transaction=Txn}, 68 | Z2={20,20}, 69 | zabe_proposal_leveldb_backend:put_proposal(Zxid,Proposal,[]), 70 | zabe_proposal_leveldb_backend:put_proposal(Z2,Proposal,[]), 71 | {ok,Last}=zabe_proposal_leveldb_backend:get_last_proposal([]), 72 | Z2=Last, 73 | ok. 74 | get_epoch_last_zxid(Config)-> 75 | Zxid={10,10}, 76 | Txn=#transaction{zxid=Zxid,value=test}, 77 | Proposal=#proposal{transaction=Txn}, 78 | Z2={10,11}, 79 | Txn2=#transaction{zxid=Z2,value=test}, 80 | Proposal2=#proposal{transaction=Txn2}, 81 | Z3={20,20}, 82 | Txn3=#transaction{zxid=Z3,value=test}, 83 | Proposal3=#proposal{transaction=Txn3}, 84 | zabe_proposal_leveldb_backend:put_proposal(Zxid,Proposal,[]), 85 | zabe_proposal_leveldb_backend:put_proposal(Z2,Proposal2,[]), 86 | zabe_proposal_leveldb_backend:put_proposal(Z3,Proposal3,[]), 87 | {ok,Z2}=zabe_proposal_leveldb_backend:get_epoch_last_zxid(10,[]), 88 | ok. 89 | 90 | zxid_fold(Config)-> 91 | Zxid={10,10}, 92 | Txn=#transaction{zxid=Zxid,value=test}, 93 | Proposal=#proposal{transaction=Txn}, 94 | Z2={10,11}, 95 | Txn2=#transaction{zxid=Z2,value=test}, 96 | Proposal2=#proposal{transaction=Txn2}, 97 | Z3={20,20}, 98 | Txn3=#transaction{zxid=Z3,value=test}, 99 | Proposal3=#proposal{transaction=Txn3}, 100 | zabe_proposal_leveldb_backend:put_proposal(Zxid,Proposal,[]), 101 | zabe_proposal_leveldb_backend:put_proposal(Z2,Proposal2,[]), 102 | zabe_proposal_leveldb_backend:put_proposal(Z3,Proposal3,[]), 103 | {ok,{L2,_}}=zabe_proposal_leveldb_backend:fold(fun({Key,_Value},{Acc,Count})-> 104 | {[Key|Acc],Count} end,{[],infinite},{20,20},[]), 105 | 1=length(L2), 106 | {ok,{L3,_}}=zabe_proposal_leveldb_backend:fold(fun({Key,_Value},{Acc,Count})-> 107 | {[Key|Acc],Count} end,{[],infinite},{10,11},[]), 108 | 2=length(L3), 109 | ok. 110 | 111 | zxid_fold_count(Config)-> 112 | Zxid={10,10}, 113 | Txn=#transaction{zxid=Zxid,value=test}, 114 | Proposal=#proposal{transaction=Txn}, 115 | Z2={10,11}, 116 | Txn2=#transaction{zxid=Z2,value=test}, 117 | Proposal2=#proposal{transaction=Txn2}, 118 | Z3={20,20}, 119 | Txn3=#transaction{zxid=Z3,value=test}, 120 | Proposal3=#proposal{transaction=Txn3}, 121 | 122 | Z4={20,21}, 123 | Txn4=#transaction{zxid=Z4,value=test}, 124 | Proposal4=#proposal{transaction=Txn4}, 125 | 126 | zabe_proposal_leveldb_backend:put_proposal(Zxid,Proposal,[]), 127 | zabe_proposal_leveldb_backend:put_proposal(Z2,Proposal2,[]), 128 | zabe_proposal_leveldb_backend:put_proposal(Z3,Proposal3,[]), 129 | zabe_proposal_leveldb_backend:put_proposal(Z4,Proposal4,[]), 130 | % {ok,L1}=zabe_proposal_leveldb_backend:iterate_zxid_count(fun({_K,V})-> 131 | % V end,Zxid,2), 132 | 133 | {ok,{L3,_}}=zabe_proposal_leveldb_backend:fold(fun({Key,_Value},{Acc,Count})-> 134 | {[Key|Acc],Count-1} end,{[],2},Zxid,[]), 135 | 136 | 2=length(L3), 137 | 138 | % [Proposal,Proposal2]=L1, 139 | {ok,{[Proposal4,Proposal3],_}}= 140 | zabe_proposal_leveldb_backend:fold(fun({_Key,Value},{Acc,Count})-> 141 | {[Value|Acc],Count-1} end,{[],2},zabe_util:zxid_plus(Z2),[]), 142 | 143 | 144 | ok. 145 | -------------------------------------------------------------------------------- /src/zabe_fast_leader_test.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author <> 3 | %%% @copyright (C) 2012, 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 15 Jun 2012 by <> 8 | %%%------------------------------------------------------------------- 9 | -module(zabe_fast_leader_test). 10 | 11 | -behaviour(gen_server). 12 | 13 | %% API 14 | -export([start_link/3]). 15 | 16 | %% gen_server callbacks 17 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 18 | terminate/2, code_change/3]). 19 | 20 | -export([get_state/0]). 21 | 22 | -compile([{parse_transform, lager_transform}]). 23 | -define(SERVER, ?MODULE). 24 | 25 | -record(state, {vote,state,elect_pid}). 26 | -include("zabe_main.hrl"). 27 | 28 | %%%=================================================================== 29 | %%% API 30 | %%%=================================================================== 31 | get_state()-> 32 | gen_server:call(?SERVER,get). 33 | %%-------------------------------------------------------------------- 34 | %% @doc 35 | %% Starts the server 36 | %% 37 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 38 | %% @end 39 | %%-------------------------------------------------------------------- 40 | start_link(LastZxid,Ensemble,Quorum) -> 41 | 42 | gen_server:start({local, ?SERVER}, ?MODULE, [LastZxid,Ensemble,Quorum], []). 43 | 44 | %%%=================================================================== 45 | %%% gen_server callbacks 46 | %%%=================================================================== 47 | 48 | %%-------------------------------------------------------------------- 49 | %% @private 50 | %% @doc 51 | %% Initializes the server 52 | %% 53 | %% @spec init(Args) -> {ok, State} | 54 | %% {ok, State, Timeout} | 55 | %% ignore | 56 | %% {stop, Reason} 57 | %% @end 58 | %%-------------------------------------------------------------------- 59 | init([LastZxid,Ensemble,Quorum]) -> 60 | Election=#election{logical_clock=1,parent=?SERVER,last_zxid=LastZxid,ensemble=Ensemble,quorum=Quorum}, 61 | {ok,Pid}=zabe_fast_elect:start_link(Election), 62 | {ok, #state{elect_pid=Pid}}. 63 | 64 | %%-------------------------------------------------------------------- 65 | %% @private 66 | %% @doc 67 | %% Handling call messages 68 | %% 69 | %% @spec handle_call(Request, From, State) -> 70 | %% {reply, Reply, State} | 71 | %% {reply, Reply, State, Timeout} | 72 | %% {noreply, State} | 73 | %% {noreply, State, Timeout} | 74 | %% {stop, Reason, Reply, State} | 75 | %% {stop, Reason, State} 76 | %% @end 77 | %%-------------------------------------------------------------------- 78 | handle_call(get, _From, State) -> 79 | Reply = State#state.state, 80 | {reply, Reply, State}. 81 | 82 | %%-------------------------------------------------------------------- 83 | %% @private 84 | %% @doc 85 | %% Handling cast messages 86 | %% 87 | %% @spec handle_cast(Msg, State) -> {noreply, State} | 88 | %% {noreply, State, Timeout} | 89 | %% {stop, Reason, State} 90 | %% @end 91 | %%-------------------------------------------------------------------- 92 | handle_cast(_Msg, State) -> 93 | {noreply, State}. 94 | 95 | %%-------------------------------------------------------------------- 96 | %% @private 97 | %% @doc 98 | %% Handling all non call/cast messages 99 | %% 100 | %% @spec handle_info(Info, State) -> {noreply, State} | 101 | %% {noreply, State, Timeout} | 102 | %% {stop, Reason, State} 103 | %% @end 104 | %%-------------------------------------------------------------------- 105 | handle_info(#msg{value={elect_reply,{ok,V,_}}}, State) -> 106 | if 107 | V#vote.leader =:=node()-> 108 | {noreply,State#state{state=?LEADING}}; 109 | true-> 110 | {noreply, State#state{state=?FOLLOWING}} 111 | end; 112 | handle_info(Msg=#msg{cmd=?VOTE_CMD,value=V},State=#state{elect_pid=Pid}) -> 113 | lager:info("get msg ~p",[Msg]), 114 | gen_fsm:send_event(Pid,V), 115 | {noreply,State}. 116 | 117 | %%-------------------------------------------------------------------- 118 | %% @private 119 | %% @doc 120 | %% This function is called by a gen_server when it is about to 121 | %% terminate. It should be the opposite of Module:init/1 and do any 122 | %% necessary cleaning up. When it returns, the gen_server terminates 123 | %% with Reason. The return value is ignored. 124 | %% 125 | %% @spec terminate(Reason, State) -> void() 126 | %% @end 127 | %%-------------------------------------------------------------------- 128 | terminate(_Reason, _State) -> 129 | ok. 130 | 131 | %%-------------------------------------------------------------------- 132 | %% @private 133 | %% @doc 134 | %% Convert process state when code is changed 135 | %% 136 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 137 | %% @end 138 | %%-------------------------------------------------------------------- 139 | code_change(_OldVsn, State, _Extra) -> 140 | {ok, State}. 141 | 142 | %%%=================================================================== 143 | %%% Internal functions 144 | %%%=================================================================== 145 | -------------------------------------------------------------------------------- /src/zabe_learn_leveldb.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author <> 3 | %%% @copyright (C) 2012, 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 19 Jun 2012 by <> 8 | %%%------------------------------------------------------------------- 9 | -module(zabe_learn_leveldb). 10 | -behaviour(gen_zab_server). 11 | %% API 12 | -export([start_link/3]). 13 | -compile([{parse_transform, lager_transform}]). 14 | %% gen_server callbacks 15 | -export([init/1, handle_call/4, handle_cast/3, handle_info/3, 16 | terminate/3, code_change/4]). 17 | 18 | -define(SERVER, ?MODULE). 19 | -include("zabe_main.hrl"). 20 | 21 | -record(state, {leveldb}). 22 | 23 | -export([put/2,get/1]). 24 | 25 | -export([handle_commit/4]). 26 | 27 | -define(LAST_ZXID_KEY,<<"cen_last_zxid_key">>). 28 | 29 | %%%=================================================================== 30 | %%% API 31 | %%%=================================================================== 32 | 33 | put(Key,Value)-> 34 | gen_zab_server:proposal_call(?SERVER,{put,Key,Value}) 35 | . 36 | get(Key)-> 37 | gen_zab_server:call(?SERVER,{get,Key}). 38 | 39 | %%-------------------------------------------------------------------- 40 | %% @doc 41 | %% Starts the server 42 | %% 43 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 44 | %% @end 45 | %%-------------------------------------------------------------------- 46 | start_link(Nodes,Opts,DbDir) -> 47 | gen_zab_server:start_link(?SERVER,Nodes,Opts, ?MODULE, [DbDir], []). 48 | 49 | %%%=================================================================== 50 | %%% gen_server callbacks 51 | %%%=================================================================== 52 | 53 | %%-------------------------------------------------------------------- 54 | %% @private 55 | %% @doc 56 | %% Initializes the server 57 | %% 58 | %% @spec init(Args) -> {ok, State} | 59 | %% {ok, State, Timeout} | 60 | %% ignore | 61 | %% {stop, Reason} 62 | %% @end 63 | %%-------------------------------------------------------------------- 64 | init([WorkDir]) -> 65 | %%WorkDir="/home/erlang/tmp/proposal.bb", 66 | 67 | case eleveldb:open(WorkDir, [{create_if_missing, true},{max_open_files,50}]) of 68 | {ok, Ref} -> 69 | lager:info("open db on:~p ok",[WorkDir]), 70 | case eleveldb:get(Ref,?LAST_ZXID_KEY,[]) of 71 | {ok,Value}-> 72 | {ok,#state{leveldb=Ref},binary_to_term(Value)}; 73 | not_found-> 74 | {ok,#state{leveldb=Ref},{0,0}}; 75 | {error,Reason}-> 76 | lager:info("get last zxid error ~p",[Reason]), 77 | {error,Reason} 78 | end 79 | ; 80 | {error, Reason} -> 81 | lager:info("open db error ~p",[Reason]), 82 | {error, Reason} 83 | end. 84 | 85 | 86 | handle_commit({put,Key,Value},Zxid, State=#state{leveldb=Db},_ZabServerInfo) -> 87 | 88 | eleveldb:put(Db,?LAST_ZXID_KEY,term_to_binary(Zxid),[]), 89 | eleveldb:put(Db,list_to_binary(Key),erlang:term_to_binary(Value),[]), 90 | {ok,ok,State} 91 | . 92 | %%-------------------------------------------------------------------- 93 | %% @private 94 | %% @doc 95 | %% Handling call messages 96 | %% 97 | %% @spec handle_call(Request, From, State) -> 98 | %% {reply, Reply, State} | 99 | %% {reply, Reply, State, Timeout} | 100 | %% {noreply, State} | 101 | %% {noreply, State, Timeout} | 102 | %% {stop, Reason, Reply, State} | 103 | %% {stop, Reason, State} 104 | %% @end 105 | %%-------------------------------------------------------------------- 106 | 107 | handle_call({get,Key}, _From, State=#state{leveldb=Db},_) -> 108 | case eleveldb:get(Db,list_to_binary(Key),[]) of 109 | {ok,Value}-> 110 | {reply,{ok,erlang:binary_to_term(Value)}, State}; 111 | not_found-> 112 | {reply,not_found, State}; 113 | {error,Reason} -> 114 | {reply, {error,Reason}, State} 115 | end; 116 | handle_call(_, _From, State,_) -> 117 | {reply,ok,State}. 118 | 119 | %%-------------------------------------------------------------------- 120 | %% @private 121 | %% @doc 122 | %% Handling cast messages 123 | %% 124 | %% @spec handle_cast(Msg, State) -> {noreply, State} | 125 | %% {noreply, State, Timeout} | 126 | %% {stop, Reason, State} 127 | %% @end 128 | %%-------------------------------------------------------------------- 129 | handle_cast(_Msg, State,_) -> 130 | {noreply, State}. 131 | 132 | %%-------------------------------------------------------------------- 133 | %% @private 134 | %% @doc 135 | %% Handling all non call/cast messages 136 | %% 137 | %% @spec handle_info(Info, State) -> {noreply, State} | 138 | %% {noreply, State, Timeout} | 139 | %% {stop, Reason, State} 140 | %% @end 141 | %%-------------------------------------------------------------------- 142 | handle_info(_Info, State,_ZabServerInfo) -> 143 | {noreply, State}. 144 | 145 | %%-------------------------------------------------------------------- 146 | %% @private 147 | %% @doc 148 | %% This function is called by a gen_server when it is about to 149 | %% terminate. It should be the opposite of Module:init/1 and do any 150 | %% necessary cleaning up. When it returns, the gen_server terminates 151 | %% with Reason. The return value is ignored. 152 | %% 153 | %% @spec terminate(Reason, State) -> void() 154 | %% @end 155 | %%-------------------------------------------------------------------- 156 | terminate(_Reason, _State,_ZabServerInfo) -> 157 | ok. 158 | 159 | %%-------------------------------------------------------------------- 160 | %% @private 161 | %% @doc 162 | %% Convert process state when code is changed 163 | %% 164 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 165 | %% @end 166 | %%-------------------------------------------------------------------- 167 | code_change(_OldVsn, State, _Extra,_ZabServerInfo) -> 168 | {ok, State}. 169 | 170 | %%%=================================================================== 171 | %%% Internal functions 172 | %%%=================================================================== 173 | -------------------------------------------------------------------------------- /src/zabe_proposal_leveldb_backend.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author 3 | %%% @copyright (C) 2012, 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 19 Jun 2012 by <> 8 | %%%------------------------------------------------------------------- 9 | -module(zabe_proposal_leveldb_backend). 10 | -behaviour(gen_server). 11 | -behaviour(zabe_proposal_backend). 12 | %% API 13 | -export([start_link/2,start/1]). 14 | 15 | %% gen_server callbacks 16 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 17 | terminate/2, code_change/3]). 18 | 19 | -define(SERVER, ?MODULE). 20 | -include("zabe_main.hrl"). 21 | -compile([{parse_transform, lager_transform}]). 22 | 23 | -record(state, {leveldb}). 24 | -define(MAX_PROPOSAL,"999999999999999999999999999999"). 25 | 26 | 27 | -export([put_proposal/3,get_proposal/2,get_last_proposal/1,fold/4,get_epoch_last_zxid/2]). 28 | -export([delete_proposal/2]). 29 | 30 | 31 | 32 | %%%=================================================================== 33 | %%% API 34 | %%%=================================================================== 35 | get_last_proposal(Opts)-> 36 | gen_server:call(?SERVER,{get_last_proposal,Opts}) . 37 | put_proposal(Key,Proposal,Opts)-> 38 | gen_server:call(?SERVER,{put_proposal,Key,Proposal,Opts}) 39 | . 40 | get_proposal(Key,Opts)-> 41 | gen_server:call(?SERVER,{get_proposal,Key,Opts}) 42 | . 43 | delete_proposal(Key,Opts)-> 44 | gen_server:call(?SERVER,{delete_proposal,Key,Opts}) . 45 | 46 | fold(Fun,Acc,Start,Opts)-> 47 | gen_server:call(?SERVER,{fold,Fun,Acc,Start,Opts},infinity). 48 | get_epoch_last_zxid(Epoch,Opts)-> 49 | gen_server:call(?SERVER,{get_epoch_last_zxid,Epoch,Opts}). 50 | 51 | 52 | 53 | %%-------------------------------------------------------------------- 54 | %% @doc 55 | %% Starts the server 56 | %% 57 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 58 | %% @end 59 | %%-------------------------------------------------------------------- 60 | start(Dir)-> 61 | gen_server:start_link({local, ?SERVER}, ?MODULE, [Dir], []). 62 | start_link(Dir,Opts) -> 63 | gen_server:start_link({local, ?SERVER}, ?MODULE, [Dir,Opts], []). 64 | 65 | %%%=================================================================== 66 | %%% gen_server callbacks 67 | %%%=================================================================== 68 | 69 | %%-------------------------------------------------------------------- 70 | %% @private 71 | %% @doc 72 | %% Initializes the server 73 | %% 74 | %% @spec init(Args) -> {ok, State} | 75 | %% {ok, State, Timeout} | 76 | %% ignore | 77 | %% {stop, Reason} 78 | %% @end 79 | %%-------------------------------------------------------------------- 80 | -define(INIT_MAX,<<"zzzzzzzzzzzzzzzzzzz">>). 81 | 82 | 83 | init([Dir,_Opts]) -> 84 | % WorkDir="/home/erlang/tmp/proposal.bb", 85 | % Prefix = proplists:get_value(prefix,Opts,""), 86 | % eleveldb:repair(Dir,[]), 87 | case eleveldb:open(Dir, [{create_if_missing, true},{max_open_files,50}]) of 88 | {ok, Ref} -> 89 | lager:info("zab engine start db on log dir ~p ok",[Dir]), 90 | 91 | case eleveldb:get(Ref,?INIT_MAX,[]) of 92 | not_found-> 93 | eleveldb:put(Ref,?INIT_MAX,?INIT_MAX,[]); 94 | {error,Reason}-> 95 | lager:info("erorr ~p",[Reason]); 96 | _-> 97 | ok 98 | end , 99 | {ok, #state { leveldb = Ref}}; 100 | {error, Reason} -> 101 | lager:info("zab engine start db on log dir ~p error:",[Reason]), 102 | {error, Reason} 103 | end. 104 | 105 | 106 | 107 | %%-------------------------------------------------------------------- 108 | %% @private 109 | %% @doc 110 | %% Handling call messages 111 | %% 112 | %% @spec handle_call(Request, From, State) -> 113 | %% {reply, Reply, State} | 114 | %% {reply, Reply, State, Timeout} | 115 | %% {noreply, State} | 116 | %% {noreply, State, Timeout} | 117 | %% {stop, Reason, Reply, State} | 118 | %% {stop, Reason, State} 119 | %% @end 120 | %%-------------------------------------------------------------------- 121 | handle_call({put_proposal,Zxid,Proposal,Opts}, _From, State=#state{leveldb=Db}) -> 122 | % {_Epoch,TxnId}=Zxid, 123 | Prefix = proplists:get_value(prefix,Opts,""), 124 | Key=zabe_util:encode_key(zabe_util:encode_zxid(Zxid),Prefix), 125 | case eleveldb:put(Db,list_to_binary(Key),erlang:term_to_binary(Proposal),[]) of 126 | ok-> 127 | {reply, ok, State}; 128 | {error,Reason} -> 129 | {reply, {error,Reason}, State} 130 | end; 131 | handle_call({get_proposal,Zxid,Opts}, _From, State=#state{leveldb=Db}) -> 132 | % {_Epoch,TxnId}=Zxid, 133 | % Key=integer_to_list(TxnId), 134 | Prefix = proplists:get_value(prefix,Opts,""), 135 | Key=zabe_util:encode_key(zabe_util:encode_zxid(Zxid),Prefix), 136 | case eleveldb:get(Db,list_to_binary(Key),[]) of 137 | {ok,Value}-> 138 | {reply,{ok,erlang:binary_to_term(Value)}, State}; 139 | not_found-> 140 | {reply,not_found, State}; 141 | {error,Reason} -> 142 | {reply, {error,Reason}, State} 143 | end; 144 | 145 | handle_call({delete_proposal,Zxid,Opts}, _From, State=#state{leveldb=Db}) -> 146 | % {_Epoch,TxnId}=Zxid, 147 | % Key=integer_to_list(TxnId), 148 | Prefix = proplists:get_value(prefix,Opts,""), 149 | Key=zabe_util:encode_key(zabe_util:encode_zxid(Zxid),Prefix), 150 | case eleveldb:delete(Db,list_to_binary(Key),[]) of 151 | ok-> 152 | {reply,ok, State}; 153 | {error,Reason} -> 154 | {reply, {error,Reason}, State} 155 | end; 156 | 157 | handle_call({get_last_proposal,Opts}, _From, State=#state{leveldb=Db}) -> 158 | Prefix = proplists:get_value(prefix,Opts,""), 159 | {ok,Itr}=eleveldb:iterator(Db,[]), 160 | % Max= zabe_util:encode_key(?MAX_PROPOSAL,Prefix), 161 | Max=zabe_util:encode_zxid({999999999,1}), 162 | K1=list_to_binary(zabe_util:encode_key(Max,Prefix)), 163 | 164 | Zxid= case eleveldb:iterator_move(Itr,K1) of 165 | {error,invalid_iterator}-> 166 | not_found; 167 | {error,iterator_closed}-> 168 | not_found; 169 | {ok,_Key,_}-> 170 | case eleveldb:iterator_move(Itr,prev) of 171 | {error,invalid_iterator}-> 172 | not_found; 173 | {error,iterator_closed}-> 174 | not_found; 175 | {ok,Key1,_}-> 176 | case zabe_util:prefix_match(Key1,Prefix) of 177 | true-> 178 | zabe_util:decode_zxid(zabe_util:decode_key(binary_to_list(Key1),Prefix)); 179 | false -> 180 | not_found 181 | end 182 | end 183 | end 184 | 185 | , 186 | 187 | {reply,{ok,Zxid},State}; 188 | handle_call({get_epoch_last_zxid,Epoch,Opts}, _From, State=#state{leveldb=Db}) -> 189 | Prefix = proplists:get_value(prefix,Opts,""), 190 | {ok,Itr}=eleveldb:iterator(Db,[]), 191 | Start=zabe_util:encode_zxid({Epoch+1,0}), 192 | Zxid=case eleveldb:iterator_move(Itr,list_to_binary(zabe_util:encode_key(Start,Prefix))) of 193 | {error,invalid_iterator}-> 194 | not_found; 195 | {error,iterator_closed}-> 196 | not_found; 197 | {ok,_,_}-> 198 | case eleveldb:iterator_move(Itr,prev) of 199 | {error,invalid_iterator}-> 200 | not_found; 201 | {error,iterator_closed}-> 202 | not_found; 203 | {ok,Key,_}-> 204 | case zabe_util:prefix_match(Key,Prefix) of 205 | true-> 206 | zabe_util:decode_zxid(zabe_util:decode_key(binary_to_list(Key),Prefix)); 207 | false -> 208 | not_found 209 | end 210 | end 211 | end, 212 | {reply,{ok,Zxid},State}; 213 | 214 | handle_call({fold,Fun,Acc,Start,Opts}, _From, State=#state{leveldb=Db}) -> 215 | Prefix = proplists:get_value(prefix,Opts,""), 216 | % L=eleveldb:fold(Db, 217 | % Fun, 218 | % [], [{first_key, list_to_binary(zabe_util:encode_key(zabe_util:encode_zxid(Start),Prefix))}]), 219 | L=eleveldb_util:zxid_fold(Db,Fun,Acc,Start,Prefix), 220 | {reply,{ok,L},State}. 221 | 222 | 223 | 224 | %%-------------------------------------------------------------------- 225 | %% @private 226 | %% @doc 227 | %% Handling cast messages 228 | %% 229 | %% @spec handle_cast(Msg, State) -> {noreply, State} | 230 | %% {noreply, State, Timeout} | 231 | %% {stop, Reason, State} 232 | %% @end 233 | %%-------------------------------------------------------------------- 234 | handle_cast(_Msg, State) -> 235 | {noreply, State}. 236 | 237 | %%-------------------------------------------------------------------- 238 | %% @private 239 | %% @doc 240 | %% Handling all non call/cast messages 241 | %% 242 | %% @spec handle_info(Info, State) -> {noreply, State} | 243 | %% {noreply, State, Timeout} | 244 | %% {stop, Reason, State} 245 | %% @end 246 | %%-------------------------------------------------------------------- 247 | handle_info(_Info, State) -> 248 | {noreply, State}. 249 | 250 | %%-------------------------------------------------------------------- 251 | %% @private 252 | %% @doc 253 | %% This function is called by a gen_server when it is about to 254 | %% terminate. It should be the opposite of Module:init/1 and do any 255 | %% necessary cleaning up. When it returns, the gen_server terminates 256 | %% with Reason. The return value is ignored. 257 | %% 258 | %% @spec terminate(Reason, State) -> void() 259 | %% @end 260 | %%-------------------------------------------------------------------- 261 | terminate(_Reason, #state{leveldb=_Db}) -> 262 | % eleveldb:destroy(Db,[]), 263 | ok. 264 | 265 | %%-------------------------------------------------------------------- 266 | %% @private 267 | %% @doc 268 | %% Convert process state when code is changed 269 | %% 270 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 271 | %% @end 272 | %%-------------------------------------------------------------------- 273 | code_change(_OldVsn, State, _Extra) -> 274 | {ok, State}. 275 | 276 | %%%=================================================================== 277 | %%% Internal functions 278 | %%%=================================================================== 279 | -------------------------------------------------------------------------------- /src/zabe_fast_elect.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author <> 3 | %%% @copyright (C) 2012, 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 14 Jun 2012 by <> 8 | %%%------------------------------------------------------------------- 9 | -module(zabe_fast_elect). 10 | 11 | -behaviour(gen_fsm). 12 | %% API 13 | -export([start_link/1]). 14 | 15 | %% gen_fsm callbacks 16 | -export([init/1, looking/2, state_name/3, handle_event/3, 17 | handle_sync_event/4, handle_info/3, terminate/3, code_change/4]). 18 | 19 | 20 | -export([wait_outof_election/2,send_notifications/3,leading/2,following/2]). 21 | -compile([{parse_transform, lager_transform}]). 22 | 23 | -include("zabe_main.hrl"). 24 | -define(SERVER, ?MODULE). 25 | 26 | 27 | -record(state, {cur_vote::#vote{}, 28 | manager_name::atom(), 29 | ensemble, 30 | quorum, 31 | ntimeout, 32 | recv_votes::ets, 33 | outof_election::ets, 34 | wait_outof_timer::reference(), 35 | 36 | logical_clock::integer() 37 | }). 38 | 39 | 40 | -define(WAIT_TIMEOUT,500). 41 | -define(PROPOSED,proposed). 42 | -define(VOTE_TIMER,2800). 43 | 44 | 45 | 46 | 47 | %%%=================================================================== 48 | %%% API 49 | %%%=================================================================== 50 | 51 | %%-------------------------------------------------------------------- 52 | %% @doc 53 | %% fast leader select for zab,algorithem from zookeeper,FasterLeaderElection.java 54 | %% Creates a gen_fsm process which calls Module:init/1 to 55 | %% initialize. To ensure a synchronized start-up procedure, this 56 | %% function does not return until Module:init/1 has returned. 57 | %% 58 | %% @spec start_link() -> {ok, Pid} | ignore | {error, Error} 59 | %% @end 60 | %%-------------------------------------------------------------------- 61 | -spec start_link(Election::#election{})-> 62 | {ok,Pid::pid()}|{error,Reason::any()}. 63 | start_link(Election) -> 64 | gen_fsm:start(?MODULE, [Election], []). 65 | 66 | %%%=================================================================== 67 | %%% gen_fsm callbacks 68 | %%%=================================================================== 69 | 70 | %%-------------------------------------------------------------------- 71 | %% @private 72 | %% @doc 73 | %% Whenever a gen_fsm is started using gen_fsm:start/[3,4] or 74 | %% gen_fsm:start_link/[3,4], this function is called by the new 75 | %% process to initialize. 76 | %% 77 | %% @spec init(Args) -> {ok, StateName, State} | 78 | %% {ok, StateName, State, Timeout} | 79 | %% ignore | 80 | %% {stop, StopReason} 81 | %% @end 82 | %%-------------------------------------------------------------------- 83 | init([#election{logical_clock=LogicalClock,parent=ManagerName,last_zxid=LastZxid,ensemble=Ensemble,quorum=Quorum,last_commit_zxid=LastCommitZxid}]) -> 84 | lager:info("elect start"), 85 | % {Epoch,_TxnId}=LastZxid, 86 | LogicalClock=1, 87 | V=#vote{from=node(),leader=node(),zxid=LastZxid, 88 | last_commit_zxid=LastCommitZxid, 89 | epoch=LogicalClock,state=?LOOKING}, 90 | 91 | send_notifications(V,Ensemble,ManagerName), 92 | gen_fsm:send_event_after(?VOTE_TIMER,send_notifications), 93 | Recv=ets:new(list_to_atom(atom_to_list(ManagerName)++"_1"),[{keypos,2}]), 94 | OutOf=ets:new(list_to_atom(atom_to_list(ManagerName)++"_2"),[{keypos,2}]), 95 | put(?PROPOSED,V), 96 | {ok, looking, #state{ 97 | manager_name=ManagerName, 98 | ensemble=Ensemble, 99 | quorum=Quorum, 100 | recv_votes=Recv, 101 | outof_election=OutOf, 102 | logical_clock=LogicalClock 103 | }}. 104 | 105 | 106 | %%-------------------------------------------------------------------- 107 | %% @private 108 | %% @doc 109 | %% 110 | %% state name. Whenever a gen_fsm receives an event sent using 111 | %% gen_fsm:send_event/2, the instance of this function with the same 112 | %% name as the current state name StateName is called to handle 113 | %% the event. It is also called if a timeout occurs. 114 | %% 115 | %% @spec state_name(Event, State) -> 116 | %% {next_state, NextStateName, NextState} | 117 | %% {next_state, NextStateName, NextState, Timeout} | 118 | %% {stop, Reason, NewState} 119 | %% @end 120 | %%-------------------------------------------------------------------- 121 | looking(V=#vote{},State)-> 122 | lager:debug("receive vote:~p ~n",[V]), 123 | try looking1(V,State) of 124 | _-> 125 | {next_state, looking, State} 126 | catch 127 | throw:break -> 128 | {next_state, looking, State}; 129 | throw:{wait_outof,Ns}-> 130 | {next_state, wait_outof_election, Ns}; 131 | throw:finish-> 132 | V2=get(?PROPOSED), 133 | case V2#vote.leader of 134 | L when L=:=node()-> 135 | {next_state,leading,State}; 136 | _-> 137 | {next_state,following,State} 138 | end; 139 | _:_ -> 140 | {stop,"error"} 141 | end 142 | ; 143 | 144 | looking(send_notifications,State=#state{ensemble=Ensemble,manager_name=ManagerName})-> 145 | V=get(?PROPOSED), 146 | send_notifications(V,Ensemble,ManagerName), 147 | gen_fsm:send_event_after(?VOTE_TIMER,send_notifications), 148 | {next_state, looking, State}; 149 | looking(_Event, State) -> 150 | {next_state, looking, State}. 151 | 152 | 153 | looking1(Vote=#vote{from=_From,leader=Leader,state=PeerState,epoch=PeerEpoch}, State=#state{manager_name=ManagerName,ensemble=Ensemble}) -> 154 | 155 | P1=get(?PROPOSED), 156 | 157 | Epoch=P1#vote.epoch, 158 | case 159 | PeerState of 160 | ?LOOKING-> 161 | if 162 | PeerEpoch < Epoch-> 163 | send_notifications(P1,Ensemble,ManagerName), 164 | throw(break); 165 | PeerEpoch >Epoch-> 166 | case 167 | total_order_predicate(Vote#vote.leader,Vote#vote.zxid,P1#vote.leader,P1#vote.zxid) of 168 | true-> 169 | V1=P1#vote{leader=Leader,zxid=Vote#vote.zxid,epoch=PeerEpoch}, 170 | send_notifications(V1,Ensemble,ManagerName), 171 | ets:delete_all_objects(State#state.recv_votes), 172 | put(?PROPOSED,V1); 173 | false-> 174 | V1=P1#vote{epoch=PeerEpoch}, 175 | send_notifications(V1,Ensemble,ManagerName), 176 | ets:delete_all_objects(State#state.recv_votes), 177 | put(?PROPOSED,V1) 178 | end; 179 | true -> 180 | case 181 | total_order_predicate(Vote#vote.leader,Vote#vote.zxid,P1#vote.leader,P1#vote.zxid) of 182 | true-> 183 | 184 | V1=P1#vote{leader=Vote#vote.leader,zxid=Vote#vote.zxid,epoch=PeerEpoch}, 185 | 186 | send_notifications(V1,Ensemble,ManagerName), 187 | 188 | put(?PROPOSED,V1); 189 | false-> 190 | 191 | ok 192 | 193 | end 194 | 195 | end, 196 | RecvVote=State#state.recv_votes, 197 | ets:insert(RecvVote,Vote), 198 | 199 | ReceiveAll=ets:info(RecvVote,size) == ordsets:size(Ensemble), 200 | HaveQuorm=is_have_quorm(State#state.quorum,Vote,RecvVote), 201 | if ReceiveAll -> 202 | 203 | notify_manager(ManagerName,ets:tab2list(RecvVote)), 204 | throw(finish); 205 | HaveQuorm-> 206 | %wait after select 207 | TimeRef=gen_fsm:start_timer(?WAIT_TIMEOUT,wait_timeout), 208 | 209 | throw({wait_outof,State#state{wait_outof_timer=TimeRef}}) 210 | ; 211 | true -> 212 | ok 213 | end; 214 | ?OBSERVING-> 215 | ok; 216 | _-> 217 | if PeerEpoch =:=Epoch-> 218 | R2=State#state.recv_votes, 219 | ets:insert(R2,Vote), 220 | 221 | HaveQuorm=is_have_quorm(State#state.quorum,Vote,R2), 222 | CheckLeader=check_leader(R2,Vote#vote.leader,node()), 223 | if Vote#vote.state ==?LEADING orelse (CheckLeader andalso HaveQuorm)-> 224 | put(?PROPOSED,Vote#vote{from=node()}), 225 | 226 | notify_manager(ManagerName,ets:tab2list(State#state.recv_votes)), 227 | throw(finish); 228 | true -> 229 | ok 230 | end; 231 | true -> 232 | 233 | OutOf=State#state.outof_election, 234 | ets:insert(OutOf,Vote), 235 | HaveQuorm=is_have_quorm(State#state.quorum,Vote,OutOf), 236 | CheckLeader=check_leader(OutOf,Vote#vote.leader,node()), 237 | if HaveQuorm andalso CheckLeader-> 238 | % V1=P1#vote{epoch=PeerEpoch}, 239 | % put(?PROPOSED,V1), 240 | put(?PROPOSED,Vote#vote{from=node()}), 241 | 242 | notify_manager(ManagerName,ets:tab2list(State#state.recv_votes)), 243 | ets:delete_all_objects(State#state.recv_votes),%?? 244 | throw(finish) 245 | ; 246 | 247 | true -> 248 | ok 249 | end 250 | end 251 | end 252 | . 253 | 254 | wait_outof_election({timeout,_,wait_timeout},State=#state{manager_name=M})-> 255 | 256 | notify_manager(M,ets:tab2list(State#state.recv_votes)), 257 | V=get(?PROPOSED), 258 | case V#vote.leader of 259 | L when L=:=node()-> 260 | {next_state,leading,State}; 261 | _-> 262 | {next_state,following,State} 263 | end; 264 | 265 | wait_outof_election(Vote=#vote{leader=_Leader},State) -> 266 | % P1=get(?PROPOSED), 267 | % case total_order_predicate(Leader,Vote#vote.zxid,node(),P1#vote.zxid) of 268 | % true-> 269 | %send to myself 270 | gen_fsm:cancel_timer(State#state.wait_outof_timer), 271 | gen_fsm:send_event_after(1,Vote), 272 | {next_state, looking, State} 273 | % _-> 274 | %TimeRef=gen_fsm:start_timer(?WAIT_TIMEOUT,wait_timeout), 275 | % {next_state,wait_outof_election, State} 276 | % end. 277 | ; 278 | wait_outof_election(send_notifications,State=#state{ensemble=Ensemble,manager_name=ManagerName})-> 279 | V=get(?PROPOSED), 280 | send_notifications(V,Ensemble,ManagerName), 281 | gen_fsm:send_event_after(?VOTE_TIMER,send_notifications), 282 | {next_state, wait_outof_election, State}. 283 | 284 | leading(#vote{from=From,leader=_Leader,state=?LOOKING},State)-> 285 | V=get(?PROPOSED), 286 | Msg=#msg{cmd=?VOTE_CMD,value=V#vote{state=?LEADING}}, 287 | catch erlang:send({State#state.manager_name,From},Msg), 288 | {next_state,leading,State}; 289 | leading({re_elect,NewZxid,NewLastCommitZxid},State=#state{recv_votes=Recv,outof_election=Outof,logical_clock=LogicalClock 290 | ,ensemble=Ensemble,manager_name=ManagerName 291 | })-> 292 | L2=LogicalClock+1, 293 | V=#vote{from=node(),leader=node(),zxid=NewZxid, 294 | last_commit_zxid=NewLastCommitZxid, 295 | epoch=L2,state=?LOOKING}, 296 | 297 | send_notifications(V,Ensemble,ManagerName), 298 | gen_fsm:send_event_after(?VOTE_TIMER,send_notifications), 299 | 300 | ets:delete_all_objects(Recv), 301 | ets:delete_all_objects(Outof), 302 | put(?PROPOSED,V), 303 | {next_state, looking, State#state{ 304 | logical_clock=LogicalClock 305 | } 306 | }; 307 | 308 | leading(_,State) -> 309 | %flush msg 310 | {next_state,leading,State}. 311 | 312 | following(#vote{from=From,leader=_Leader,state=?LOOKING},State)-> 313 | V=get(?PROPOSED), 314 | Msg=#msg{cmd=?VOTE_CMD,value=V#vote{state=?FOLLOWING}}, 315 | catch erlang:send({State#state.manager_name,From},Msg), 316 | {next_state,following,State}; 317 | following({re_elect,NewZxid,NewLastCommitZxid},State=#state{recv_votes=Recv,outof_election=Outof,logical_clock=LogicalClock 318 | ,ensemble=Ensemble,manager_name=ManagerName 319 | })-> 320 | L2=LogicalClock+1, 321 | V=#vote{from=node(),leader=node(),zxid=NewZxid, 322 | last_commit_zxid=NewLastCommitZxid, 323 | epoch=L2,state=?LOOKING}, 324 | 325 | send_notifications(V,Ensemble,ManagerName), 326 | gen_fsm:send_event_after(?VOTE_TIMER,send_notifications), 327 | ets:delete_all_objects(Recv), 328 | ets:delete_all_objects(Outof), 329 | put(?PROPOSED,V), 330 | {next_state, looking, State#state{ 331 | logical_clock=LogicalClock 332 | } 333 | }; 334 | following(_,State) -> 335 | %flush msg 336 | {next_state,following,State}. 337 | 338 | is_have_quorm(Q,Proposed,RecvVote)-> 339 | C1=ets:foldl( 340 | fun(Vote,Count)-> 341 | Eq=is_eq(Vote,Proposed), 342 | if 343 | Eq-> Count+1; 344 | true -> Count 345 | end end ,0,RecvVote), 346 | if C1 >=Q-> 347 | true; 348 | true -> 349 | false 350 | end. 351 | is_eq(V1,V2)-> 352 | V1#vote.leader=:=V2#vote.leader andalso V1#vote.zxid=:=V2#vote.zxid andalso V1#vote.epoch=:=V2#vote.epoch. 353 | 354 | 355 | %%-------------------------------------------------------------------- 356 | %% @private 357 | %% @doc 358 | %% There should be one instance of this function for each possible 359 | %% state name. Whenever a gen_fsm receives an event sent using 360 | %% gen_fsm:sync_send_event/[2,3], the instance of this function with 361 | %% the same name as the current state name StateName is called to 362 | %% handle the event. 363 | %% 364 | %% @spec state_name(Event, From, State) -> 365 | %% {next_state, NextStateName, NextState} | 366 | %% {next_state, NextStateName, NextState, Timeout} | 367 | %% {reply, Reply, NextStateName, NextState} | 368 | %% {reply, Reply, NextStateName, NextState, Timeout} | 369 | %% {stop, Reason, NewState} | 370 | %% {stop, Reason, Reply, NewState} 371 | %% @end 372 | %%-------------------------------------------------------------------- 373 | state_name(_Event, _From, State) -> 374 | Reply = ok, 375 | {reply, Reply, state_name, State}. 376 | 377 | %%-------------------------------------------------------------------- 378 | %% @private 379 | %% @doc 380 | %% Whenever a gen_fsm receives an event sent using 381 | %% gen_fsm:send_all_state_event/2, this function is called to handle 382 | %% the event. 383 | %% 384 | %% @spec handle_event(Event, StateName, State) -> 385 | %% {next_state, NextStateName, NextState} | 386 | %% {next_state, NextStateName, NextState, Timeout} | 387 | %% {stop, Reason, NewState} 388 | %% @end 389 | %%-------------------------------------------------------------------- 390 | handle_event(_Event, StateName, State) -> 391 | {next_state, StateName, State}. 392 | 393 | %%-------------------------------------------------------------------- 394 | %% @private 395 | %% @doc 396 | %% Whenever a gen_fsm receives an event sent using 397 | %% gen_fsm:sync_send_all_state_event/[2,3], this function is called 398 | %% to handle the event. 399 | %% 400 | %% @spec handle_sync_event(Event, From, StateName, State) -> 401 | %% {next_state, NextStateName, NextState} | 402 | %% {next_state, NextStateName, NextState, Timeout} | 403 | %% {reply, Reply, NextStateName, NextState} | 404 | %% {reply, Reply, NextStateName, NextState, Timeout} | 405 | %% {stop, Reason, NewState} | 406 | %% {stop, Reason, Reply, NewState} 407 | %% @end 408 | %%-------------------------------------------------------------------- 409 | handle_sync_event(_Event, _From, StateName, State) -> 410 | Reply = ok, 411 | {reply, Reply, StateName, State}. 412 | 413 | %%-------------------------------------------------------------------- 414 | %% @private 415 | %% @doc 416 | %% This function is called by a gen_fsm when it receives any 417 | %% message other than a synchronous or asynchronous event 418 | %% (or a system message). 419 | %% 420 | %% @spec handle_info(Info,StateName,State)-> 421 | %% {next_state, NextStateName, NextState} | 422 | %% {next_state, NextStateName, NextState, Timeout} | 423 | %% {stop, Reason, NewState} 424 | %% @end 425 | %%-------------------------------------------------------------------- 426 | handle_info(_Info, StateName, State) -> 427 | {next_state, StateName, State}. 428 | 429 | %%-------------------------------------------------------------------- 430 | %% @private 431 | %% @doc 432 | %% This function is called by a gen_fsm when it is about to 433 | %% terminate. It should be the opposite of Module:init/1 and do any 434 | %% necessary cleaning up. When it returns, the gen_fsm terminates with 435 | %% Reason. The return value is ignored. 436 | %% 437 | %% @spec terminate(Reason, StateName, State) -> void() 438 | %% @end 439 | %%-------------------------------------------------------------------- 440 | terminate(_Reason, _StateName, _State) -> 441 | ok. 442 | 443 | %%-------------------------------------------------------------------- 444 | %% @private 445 | %% @doc 446 | %% Convert process state when code is changed 447 | %% 448 | %% @spec code_change(OldVsn, StateName, State, Extra) -> 449 | %% {ok, StateName, NewState} 450 | %% @end 451 | %%-------------------------------------------------------------------- 452 | code_change(_OldVsn, StateName, State, _Extra) -> 453 | {ok, StateName, State}. 454 | 455 | %%%=================================================================== 456 | %%% Internal functions 457 | %%%=================================================================== 458 | 459 | send_notifications(V=#vote{},Ensemble,ManagerName)-> 460 | lager:debug("broadcast send ~p ~p ~p ",[V,ManagerName, self()]), 461 | Msg=#msg{cmd=?VOTE_CMD,value=V}, 462 | lists:map(fun(N)-> 463 | catch erlang:send({ManagerName,N},Msg) end ,Ensemble). 464 | 465 | % R=[erlang:send({ManagerName,Node},Msg)||Node<-Ensemble], 466 | % lager:debug("send ~p",[R]) 467 | 468 | 469 | total_order_predicate(New,NewZxid,Cur,CurZxid)-> 470 | (NewZxid>CurZxid) orelse ((NewZxid==CurZxid) andalso (New>Cur)). 471 | notify_manager(ManagerName,RecvVotes) -> 472 | V=get(?PROPOSED), 473 | lager:debug("elect1 ~p~n",[V]), 474 | catch erlang:send(ManagerName,#msg{cmd=?ZAB_CMD,value={elect_reply,{ok,V,RecvVotes}}}). 475 | 476 | check_leader(_Votes,Leader,Leader) -> 477 | true; 478 | check_leader(Votes,Leader,_) -> 479 | case ets:lookup(Votes,Leader) of 480 | []-> 481 | false; 482 | _ -> 483 | true 484 | end. 485 | 486 | -------------------------------------------------------------------------------- /src/gen_zab_server.erl: -------------------------------------------------------------------------------- 1 | 2 | %% author 3 | %% implements zab protocol in erlang,detail see yahoo zab protocol paper 4 | %% state: init->looking->leader_recover->leading 5 | %% follow_recover->following ->end 6 | 7 | -module(gen_zab_server). 8 | 9 | %% Time between rounds of query from the leader 10 | -define(TAU,5000). 11 | 12 | -export([start/6, 13 | start_link/6, 14 | proposal_call/2, proposal_call/3, proposal_cast/2, 15 | call/2, call/3, cast/2, 16 | reply/2]). 17 | -export([system_continue/3, 18 | system_terminate/4, 19 | system_code_change/4, 20 | format_status/2 21 | 22 | ]). 23 | 24 | 25 | %% Internal exports 26 | -export([init_it/6, 27 | print_event/3 28 | ]). 29 | 30 | -export([loop/3,looking/4,leader_recover/4,follow_recover/4,following/4,leading/4]). 31 | -export([send_zab_msg/3]). 32 | 33 | 34 | 35 | %-define(LEADER_STATE_LOOKING,1). 36 | %-define(LEADER_STATE_RECORVER,2). 37 | %-define(LEADER_STATE_LEADING,3). 38 | %-define(LEADER_STATE_FOLLOWING,4). 39 | %-define(LEADER_STATE_OBSERVEING,5). 40 | 41 | 42 | -compile([{parse_transform, lager_transform}]). 43 | 44 | -include("zabe_main.hrl"). 45 | -type option() :: {'workers', Workers::[node()]} 46 | | {'proposal_dir', Dir::string()} 47 | | {'heartbeat', Seconds::integer()}. 48 | 49 | -type options() :: [option()]. 50 | %% A locally registered name 51 | -type name() :: atom(). 52 | -type server_ref() :: name() | {name(),node()} | {global,name()} | pid(). 53 | %% See gen_server. 54 | -type caller_ref() :: {pid(), reference()}. 55 | %% Opaque state of the gen_leader behaviour. 56 | 57 | -type opts()::[{prefix,V::any()}]. 58 | -record(server, { 59 | parent, 60 | mod, 61 | state, 62 | debug, 63 | role::?LEADING|?FOLLOWING, 64 | proposal_que::ets, 65 | last_zxid::zxid(), 66 | current_zxid::zxid(), 67 | ensemble::[node()|_], 68 | quorum::non_neg_integer(), 69 | last_commit_zxid, 70 | proposal_log_mod, 71 | leader::node(), 72 | observers::[], 73 | elect_pid::pid(), 74 | live_nodes::dict:new(), 75 | recover_acks::dict:new(), 76 | mon_leader_ref::reference(), 77 | mon_follow_refs::dict:new(), 78 | back_end_opts::[opts()], 79 | logical_clock::integer() 80 | }). 81 | 82 | %%% --------------------------------------------------- 83 | %%% Interface functions. 84 | %%% --------------------------------------------------- 85 | 86 | 87 | 88 | -callback init(Opts::[])-> 89 | ok. 90 | -callback handle_commit(Request::any(),Zxid::zxid(),State::any(),ZabServerInfo::#zab_server_info{})-> 91 | {ok,Result::any(),NewState::any()}. 92 | -callback handle_call(Msg::any(),From::any(),State::any(),ZabServerInfo::#zab_server_info{})-> 93 | {ok,Result::any(),NewState::any()}. 94 | 95 | -callback handle_cast(Msg::any(),State::any(),ZabServerInfo::#zab_server_info{})-> 96 | {ok,NewState::any()}. 97 | 98 | -callback handle_info(Msg::any(),State::any(),ZabServerInfo::#zab_server_info{})-> 99 | {ok,NewState::any()}. 100 | 101 | -callback terminate(Reason::any(),State::any(),ZabServerInfo::#zab_server_info{})-> 102 | {ok,NewState::any()}. 103 | -callback code_change(Msg::any(),From::any(),State::any(),ZabServerInfo::#zab_server_info{})-> 104 | {ok,NewState::any()}. 105 | 106 | 107 | 108 | -type start_ret() :: {'ok', pid()} | {'error', term()}. 109 | 110 | %% @doc Starts a gen_leader process without linking to the parent. 111 | %% @see start_link/6 112 | -spec start(Name::atom(), CandidateNodes::[node()], OptArgs::options(), 113 | Mod::module(), Arg::term(), Options::list()) -> start_ret(). 114 | start(Name, CandidateNodes, OptArgs, Mod, Arg, Options) 115 | when is_atom(Name), is_list(CandidateNodes), is_list(OptArgs) -> 116 | gen:start(?MODULE, nolink, {local,Name}, 117 | Mod, {CandidateNodes, OptArgs, Arg}, Options). 118 | 119 | 120 | -spec start_link(Name::atom(), CandidateNodes::[node()], OptArgs::options(), 121 | Mod::module(), Arg::term(), Options::list()) -> start_ret(). 122 | start_link(Name, CandidateNodes, OptArgs, Mod, Arg, Options) 123 | when is_atom(Name), is_list(CandidateNodes), is_list(OptArgs) -> 124 | gen:start(?MODULE, link, {local,Name}, 125 | Mod, {CandidateNodes, OptArgs, Arg}, Options). 126 | 127 | %% 128 | %% Make a call to a generic server. 129 | %% If the server is located at another node, that node will 130 | %% be monitored. 131 | %% If the client is trapping exits and is linked server termination 132 | %% is handled here (? Shall we do that here (or rely on timeouts) ?). 133 | %% 134 | %% @doc Equivalent to gen_server:call/2, but with a slightly 135 | %% different exit reason if something goes wrong. This function calls 136 | %% the gen_leader process exactly as if it were a gen_server 137 | %% (which, for practical purposes, it is.) 138 | %% @end 139 | -spec call(server_ref(), term()) -> term(). 140 | call(Name, Request) -> 141 | case catch gen:call(Name, '$gen_call', Request) of 142 | {ok,Res} -> 143 | Res; 144 | {'EXIT',Reason} -> 145 | exit({Reason, {?MODULE, local_call, [Name, Request]}}) 146 | end. 147 | 148 | %% @doc Equivalent to gen_server:call/3, but with a slightly 149 | %% different exit reason if something goes wrong. This function calls 150 | %% the gen_leader process exactly as if it were a gen_server 151 | %% (which, for practical purposes, it is.) 152 | %% @end 153 | -spec call(server_ref(), term(), integer()) -> term(). 154 | call(Name, Request, Timeout) -> 155 | case catch gen:call(Name, '$gen_call', Request, Timeout) of 156 | {ok,Res} -> 157 | Res; 158 | {'EXIT',Reason} -> 159 | exit({Reason, {?MODULE, local_call, [Name, Request, Timeout]}}) 160 | end. 161 | 162 | %% @doc Makes a call (similar to gen_server:call/2) to the 163 | %% leader. The call is forwarded via the local gen_leader instance, if 164 | %% that one isn't actually the leader. The client will exit if the 165 | %% leader dies while the request is outstanding. 166 | %%

This function uses gen:call/3, and is subject to the 167 | %% same default timeout as e.g. gen_server:call/2.

168 | %% @end 169 | %% 170 | -spec proposal_call(Name::server_ref(), Request::term()) -> term(). 171 | proposal_call(Name, Request) -> 172 | case catch gen:call(Name, '$proposal_call', Request) of 173 | 174 | {ok,Res={error,not_ready}} -> 175 | Res; 176 | {ok,Res} -> 177 | Res; 178 | {'EXIT',Reason} -> 179 | exit({Reason, {?MODULE, proposal_call, [Name, Request]}}) 180 | end. 181 | 182 | %% @doc Makes a call (similar to gen_server:call/3) to the 183 | %% leader. The call is forwarded via the local gen_leader instance, if 184 | %% that one isn't actually the leader. The client will exit if the 185 | %% leader dies while the request is outstanding. 186 | %% @end 187 | %% 188 | 189 | -spec proposal_call(Name::server_ref(), Request::term(), 190 | Timeout::integer()) -> term(). 191 | proposal_call(Name, Request, Timeout) -> 192 | case catch gen:call(Name, '$proposal_call', Request,Timeout) of 193 | 194 | {ok,Res={error,not_ready}} -> 195 | Res; 196 | {ok,Res} -> 197 | Res; 198 | {'EXIT',Reason} -> 199 | exit({Reason, {?MODULE, proposal_call, [Name, Request,Timeout]}}) 200 | end. 201 | 202 | 203 | 204 | %% @equiv gen_server:cast/2 205 | -spec cast(Name::name()|pid(), Request::term()) -> 'ok'. 206 | cast(Name, Request) -> 207 | catch do_cast('$gen_cast', Name, Request), 208 | ok. 209 | 210 | %% @doc Similar to gen_server:cast/2 but will be forwarded to 211 | %% the leader via the local gen_leader instance. 212 | -spec proposal_cast(Name::name()|pid(), Request::term()) -> 'ok'. 213 | proposal_cast(Name, Request) -> 214 | catch do_cast('$proposal_cast', Name, Request), 215 | ok. 216 | 217 | 218 | do_cast(Tag, Name, Request) when is_atom(Name) -> 219 | Name ! {Tag, Request}; 220 | do_cast(Tag, Pid, Request) when is_pid(Pid) -> 221 | Pid ! {Tag, Request}. 222 | 223 | 224 | %% @equiv gen_server:reply/2 225 | -spec reply(From::caller_ref(), Reply::term()) -> term(). 226 | reply({To, Tag}, Reply) -> 227 | catch To ! {Tag, Reply}. 228 | 229 | 230 | %%% --------------------------------------------------- 231 | %%% Initiate the new process. 232 | %%% Register the name using the Rfunc function 233 | %%% Calls the Mod:init/Args function. 234 | %%% Finally an acknowledge is sent to Parent and the main 235 | %%% loop is entered. 236 | %%% --------------------------------------------------- 237 | %%% @hidden 238 | init_it(Starter, Parent, {local, Name}, Mod, {CandidateNodes, Workers, Arg}, Options) -> 239 | %% R13B passes {local, Name} instead of just Name 240 | init_it(Starter, Parent, Name, Mod, 241 | {CandidateNodes, Workers, Arg}, Options); 242 | init_it(Starter, self, Name, Mod, {CandidateNodes, OptArgs, Arg}, Options) -> 243 | init_it(Starter, self(), Name, Mod, 244 | {CandidateNodes, OptArgs, Arg}, Options); 245 | init_it(Starter,Parent,Name,Mod,{CandidateNodes,OptArgs,Arg},Options) -> 246 | % Interval = proplists:get_value(heartbeat, OptArgs, ?TAU div 1000) * 1000, 247 | lager:info("gen_zab_server start"), 248 | ElectMod = proplists:get_value(elect_mod, OptArgs,zabe_fast_elect), 249 | ProposalLogMod = proplists:get_value(proposal_log_mod, OptArgs,zabe_proposal_leveldb_backend), 250 | Debug = debug_options(Name, Options), 251 | Prefix = proplists:get_value(prefix,OptArgs,""), 252 | BackEndOpts=[{prefix,Prefix}], 253 | % ProposalDir =proplists:get_value(proposal_dir,OptArgs,"/tmp/p1.ldb"), 254 | case Mod:init(Arg) of 255 | {stop, Reason} -> 256 | proc_lib:init_ack(Starter, {error, Reason}), 257 | exit(Reason); 258 | ignore -> 259 | proc_lib:init_ack(Starter, ignore), 260 | exit(normal); 261 | {'EXIT', Reason} -> 262 | proc_lib:init_ack(Starter, {error, Reason}), 263 | exit(Reason); 264 | {ok, State,LastCommitZxid} -> 265 | proc_lib:init_ack(Starter, {ok, self()}), 266 | Ensemble=CandidateNodes, 267 | Quorum=ordsets:size(Ensemble) div 2 +1, 268 | LastZxid= case ProposalLogMod:get_last_proposal(BackEndOpts) of 269 | {ok,not_found}->{0,0}; 270 | {ok,Z}->Z 271 | end, 272 | LogicalClock=1, 273 | Election=#election{logical_clock=LogicalClock,parent=Mod,last_zxid=LastZxid,ensemble=Ensemble,quorum=Quorum,last_commit_zxid=LastCommitZxid}, 274 | 275 | {ok,Pid}=ElectMod:start_link(Election), 276 | Que=ets:new(list_to_atom(atom_to_list(Mod)++"_que"),[{keypos,2},ordered_set]), 277 | loop(#server{parent = Parent,mod = Mod,elect_pid=Pid, 278 | ensemble=Ensemble, 279 | last_commit_zxid=LastCommitZxid, 280 | recover_acks=dict:new(), 281 | mon_follow_refs=dict:new(), 282 | back_end_opts=BackEndOpts, 283 | state = State,last_zxid=LastZxid,current_zxid=LastZxid,proposal_log_mod=ProposalLogMod, 284 | logical_clock=LogicalClock, 285 | debug = Debug,quorum=Quorum,proposal_que=Que},looking,#zab_server_info{} 286 | ) 287 | ; 288 | Else -> 289 | Error = {init_bad_return_value, Else}, 290 | proc_lib:init_ack(Starter, {error, Error}), 291 | exit(Error) 292 | end. 293 | 294 | %%% --------------------------------------------------- 295 | %%% The MAIN loops. 296 | %%% --------------------------------------------------- 297 | 298 | loop(Server=#server{debug=Debug,elect_pid=EPid,last_zxid=LastZxid, 299 | last_commit_zxid=LastCommitZxid,proposal_que=Que, 300 | mon_follow_refs=MonFollowRefs,quorum=Quorum, 301 | leader=Leader, 302 | mod=Mod,ensemble=Ensemble, 303 | mon_leader_ref=_ModLeaderRef},ZabState,ZabServerInfo)-> 304 | receive 305 | Msg1-> 306 | lager:debug("zab state:~p receive msg:~p",[ZabState,Msg1]), 307 | case Msg1 of 308 | {'DOWN',_ModLeaderRef,_,{_,Leader},_}-> 309 | ets:delete_all_objects(Que), 310 | 311 | gen_fsm:send_event(EPid,{re_elect,LastZxid,LastCommitZxid}), 312 | lager:debug("zxid:~p,commit zxid:~p,change state~p to looking",[LastZxid,LastCommitZxid,ZabState]), 313 | loop(Server#server{ 314 | recover_acks=dict:new(), 315 | mon_follow_refs=dict:new(), 316 | logical_clock=Server#server.logical_clock+1 317 | },looking,#zab_server_info{} 318 | ) 319 | ; 320 | {'DOWN',_MRef,process,F={_Mod,_Node},_} when ZabState=:=leading-> 321 | N1=dict:erase(F,MonFollowRefs), 322 | 323 | S1=dict:size(N1)+1 , 324 | if S1>=Quorum 325 | ->loop(Server#server{mon_follow_refs=N1},ZabState,ZabServerInfo); 326 | true-> 327 | ets:delete_all_objects(Que), 328 | lager:debug("zxid:~p,commit zxid:~p,change state~p to looking",[LastZxid,LastCommitZxid,ZabState]), 329 | gen_fsm:send_event(EPid,{re_elect,LastZxid,LastCommitZxid}), 330 | M1={partition,less_quorum}, 331 | abcast(Mod,lists:delete(node(),Ensemble),M1,Server#server.logical_clock), 332 | loop(Server#server{ 333 | recover_acks=dict:new(), 334 | mon_follow_refs=dict:new(), 335 | logical_clock=Server#server.logical_clock+1 336 | },looking,#zab_server_info{} 337 | ) 338 | end 339 | 340 | ; 341 | 342 | #msg{cmd=?VOTE_CMD,value=V} -> 343 | gen_fsm:send_event(EPid,V), 344 | loop(Server,ZabState,ZabServerInfo); 345 | #msg{epoch=Epoch} when Epoch < Server#server.logical_clock -> 346 | %fush invalidate msg 347 | %lager:error("~p ~p",[Msg1 ,Server#server.logical_clock]), 348 | loop(Server,ZabState,ZabServerInfo); 349 | %#msg{cmd=?VOTE_CMD,value=V}-> 350 | % V=erlang:get(vote), 351 | % V1=V#vote{from=node(),leader=V#vote.leader,zxid=LastZxid}, 352 | % send_zab_msg(V#vote.from,V1), 353 | % 354 | % loop(Server,ZabState,ZabServerInfo); 355 | 356 | %leader partiton 357 | #msg{value={partition,less_quorum}}-> 358 | ets:delete_all_objects(Que), 359 | 360 | gen_fsm:send_event(EPid,{re_elect,LastZxid,LastCommitZxid}), 361 | lager:debug("zxid:~p,commit zxid:~p,change state~p to looking",[LastZxid,LastCommitZxid,ZabState]), 362 | loop(Server#server{ 363 | recover_acks=dict:new(), 364 | mon_follow_refs=dict:new(), 365 | logical_clock=Server#server.logical_clock+1 366 | },looking,#zab_server_info{} 367 | ); 368 | 369 | #msg{cmd=?ZAB_CMD,value=Value}-> 370 | ?MODULE:ZabState(Server,Value,ZabState,ZabServerInfo); 371 | {'$proposal_call',From,_Msg} when (ZabState =/=leading andalso ZabState =/=following) -> 372 | reply(From,{error,not_ready}), 373 | loop(Server,ZabState,ZabServerInfo); 374 | V={'$proposal_call',_From,_Msg}-> 375 | ?MODULE:ZabState(Server,V,ZabState,ZabServerInfo); 376 | {'$proposal_cast',_From,_Msg} when (ZabState =/=?LEADING andalso ZabState =/=?FOLLOWING) -> 377 | flush,do_nothing, 378 | loop(Server,ZabState,ZabServerInfo); 379 | V={'$proposal_cast',_From,_Msg}-> 380 | ?MODULE:ZabState(Server,V,ZabState); 381 | _ when Debug == [] -> 382 | handle_msg(Msg1, Server,ZabState,ZabServerInfo); 383 | _ -> 384 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 385 | "nouse", {in, Msg1}), 386 | handle_msg(Msg1, Server#server{debug = Debug1},ZabState, ZabServerInfo) 387 | end 388 | end. 389 | 390 | 391 | looking(#server{mod = Mod, state = State,debug=_Debug,quorum=Quorum,elect_pid=EPid, 392 | last_zxid=LastZxid,last_commit_zxid=LastCommitZxid, 393 | mon_follow_refs=MonFollowRefs, 394 | proposal_que=Que, 395 | ensemble=Ensemble, 396 | proposal_log_mod=ProposalLogMod,back_end_opts=BackEndOpts} = Server 397 | ,Msg1,ZabState,ZabServerInfo)-> 398 | case Msg1 of 399 | {elect_reply,{ok,V=#vote{leader=Node},RecvVotes}} when Node=:=node() -> 400 | erlang:put(vote,V), 401 | Last=case zabe_util:zxid_eq(LastZxid,LastCommitZxid) of 402 | true -> LastCommitZxid; 403 | false->{ok,{T,_}}=ProposalLogMod:fold(fun({_Key,P1},{_LastCommit,Count})-> 404 | 405 | Mod:handle_commit(P1#proposal.transaction#transaction.value, 406 | P1#proposal.transaction#transaction.zxid,State,ZabServerInfo) , 407 | {P1#proposal.transaction#transaction.zxid,Count} 408 | end, 409 | {LastCommitZxid,infinite},LastCommitZxid,BackEndOpts), 410 | T 411 | end, 412 | 413 | %%monitor_follows(RecvVotes,Quorum), 414 | %% 415 | %lager:debug("monitors ~p~p",[MonFollowRefs,RecvVotes]), 416 | 417 | N5=lists:foldl(fun(#vote{from=From},Acc)-> 418 | 419 | if From=:=node() -> 420 | Acc; 421 | true-> 422 | case catch erlang:monitor(process,{Mod,From}) of 423 | {'EXIT',_}-> 424 | Acc; 425 | Ref-> 426 | dict:store({Mod,From},Ref,Acc) end end end,MonFollowRefs,RecvVotes), 427 | T6=dict:size(N5)+1, 428 | if T6 >=Quorum 429 | -> 430 | 431 | %% 432 | loop(Server#server{role=?LEADING,leader=Node,last_zxid=Last,mon_follow_refs=N5, 433 | last_commit_zxid=Last},leader_recover,ZabServerInfo) ; 434 | true-> 435 | ets:delete_all_objects(Que), 436 | gen_fsm:send_event(EPid,{re_elect,LastZxid,LastCommitZxid}), 437 | M1={partition,less_quorum}, 438 | abcast(Mod,lists:delete(node(),Ensemble),M1,Server#server.logical_clock), 439 | loop(Server#server{ 440 | recover_acks=dict:new(), 441 | mon_follow_refs=dict:new(), 442 | logical_clock=Server#server.logical_clock+1 443 | },looking,#zab_server_info{} 444 | ) 445 | end; 446 | {elect_reply,{ok,V=#vote{leader=Node,zxid=_LeaderZxid},_}} -> 447 | erlang:put(vote,V), 448 | %monitor_leader(Mod,Node), 449 | case catch erlang:monitor(process,{Mod,Node}) of 450 | {'EXIT',_}-> 451 | ets:delete_all_objects(Que), 452 | gen_fsm:send_event(EPid,{re_elect,LastZxid,LastCommitZxid}), 453 | lager:debug("zxid:~p,commit zxid:~p,change state~p to looking",[LastZxid,LastCommitZxid,ZabState]), 454 | loop(Server#server{ 455 | recover_acks=dict:new(), 456 | mon_follow_refs=dict:new(), 457 | logical_clock=Server#server.logical_clock+1 458 | },looking,#zab_server_info{} 459 | ); 460 | Ref-> 461 | % case zabe_util:zxid_compare(LeaderZxid,LastZxid) of 462 | % epoch_small1-> 463 | timer:sleep(50),%%todo fix this by leader handle this 464 | M1={truncate_req,{Mod,node()},LastZxid}, 465 | 466 | send_zab_msg({Mod,Node},M1,V#vote.epoch), 467 | loop(Server#server{leader=Node,logical_clock=V#vote.epoch,role=?FOLLOWING,mon_leader_ref=Ref},follow_recover,ZabServerInfo) 468 | end; 469 | 470 | 471 | 472 | % equal-> 473 | % timer:sleep(50), %%leader maybe slow,and in looking state todo fix this,use timer send to leader 474 | % M1={recover_ok,{Mod,node()}}, 475 | % send_zab_msg({Mod,Node},M1), 476 | % loop(Server#server{leader=Node,role=?FOLLOWING},following,ZabServerInfo) ; 477 | % _-> 478 | % timer:sleep(50), 479 | % M1={recover_req,{Mod,node()},LastZxid}, 480 | % send_zab_msg({Mod,Node},M1), 481 | % loop(Server#server{leader=Node,role=?FOLLOWING},follow_recover,ZabServerInfo) 482 | % end; 483 | 484 | _ -> %flush msg 485 | loop(Server,ZabState,ZabServerInfo) 486 | end 487 | . 488 | 489 | 490 | leader_recover(#server{mod = _Mod, state = _State,debug=_Debug,quorum=Quorum,elect_pid=_EPid, 491 | last_zxid=LastZxid,last_commit_zxid=_LastCommitZxid,recover_acks=RecoverAcks, 492 | proposal_log_mod=ProposalLogMod, 493 | back_end_opts=BackEndOpts, 494 | current_zxid=CurZxid,mon_follow_refs=MRefs 495 | } = Server 496 | ,Msg1,ZabState,ZabServerInfo)-> 497 | case Msg1 of 498 | {recover_ok,From} -> 499 | D1=dict:store(From,"",RecoverAcks), 500 | NRefs=case dict:is_key(From,MRefs) of 501 | true->MRefs; 502 | false-> 503 | case catch erlang:monitor(process,From) of 504 | {'EXIT',_}-> 505 | MRefs; 506 | Ref-> 507 | dict:store(From,Ref,MRefs) 508 | end 509 | end, 510 | Z=dict:size(D1)+1, 511 | if Z>=Quorum -> 512 | lager:info("leader recover change state to leading"), 513 | %%change epoch for handle old leader restart 514 | Z1=zabe_util:change_leader_zxid(LastZxid), 515 | loop(Server#server{recover_acks=D1,mon_follow_refs=NRefs,last_zxid=Z1,last_commit_zxid=Z1},leading,ZabServerInfo); 516 | true-> 517 | loop(Server#server{recover_acks=D1,mon_follow_refs=NRefs},ZabState,ZabServerInfo) 518 | end; 519 | {truncate_req,From,{Epoch1,_}} -> 520 | {LeaderEpoch,_}=LastZxid, 521 | EpochLastZxid=if LeaderEpoch > Epoch1 522 | ->{ok,EL}=ProposalLogMod:get_epoch_last_zxid(Epoch1,BackEndOpts),EL; 523 | true -> 524 | not_need 525 | end, 526 | M1={truncate_ack,EpochLastZxid}, 527 | send_zab_msg(From,M1,Server#server.logical_clock), 528 | loop(Server,ZabState,ZabServerInfo); 529 | 530 | {recover_req,From,StartZxid} -> 531 | % {ok,Res}=ProposalLogMod:iterate_zxid_count(fun({_K,V})-> 532 | % V end,StartZxid,100), 533 | 534 | {ok,{Res,_}}= 535 | ProposalLogMod:fold(fun({_Key,Value},{Acc,Count})-> 536 | {[Value|Acc],Count-1} end,{[],100},StartZxid,BackEndOpts), 537 | 538 | M1={recover_ack,lists:reverse(Res),CurZxid}, 539 | 540 | send_zab_msg(From,M1,Server#server.logical_clock), 541 | loop(Server,ZabState,ZabServerInfo); 542 | _ -> %flush msg 543 | loop(Server,ZabState,ZabServerInfo) 544 | end 545 | . 546 | 547 | follow_recover(#server{mod =Mod, state = State,quorum=_Quorum,leader=Leader, 548 | last_zxid=LastZxid,last_commit_zxid=LastCommitZxid,recover_acks=_RecoverAcks, 549 | proposal_que=Que, 550 | 551 | back_end_opts=BackEndOpts, 552 | proposal_log_mod=ProposalLogMod} = Server 553 | ,Msg1,ZabState,ZabServerInfo)-> 554 | case Msg1 of 555 | {recover_ack,Proposals,_LeaderCurrentZxid} -> 556 | Last=lists:foldl(fun(Proposal,_)-> 557 | ProposalLogMod:put_proposal(Proposal#proposal.transaction#transaction.zxid,Proposal,BackEndOpts), 558 | Mod:handle_commit(Proposal#proposal.transaction#transaction.value, 559 | Proposal#proposal.transaction#transaction.zxid,State,ZabServerInfo), 560 | Proposal#proposal.transaction#transaction.zxid 561 | end ,LastZxid,Proposals), 562 | QueSize=ets:info(Que,size), 563 | 564 | case Proposals of 565 | [] when QueSize=:=0-> 566 | M1={recover_ok,{Mod,node()}}, 567 | send_zab_msg({Mod,Leader},M1,Server#server.logical_clock), 568 | lager:info("follow recover ok,state to followiing"), 569 | loop(Server,following,ZabServerInfo); 570 | [] when QueSize>0-> %recover local 571 | 572 | F=fun(P1)-> 573 | Z1=P1#proposal_rec.proposal#proposal.transaction#transaction.zxid, 574 | ProposalLogMod:put_proposal(Z1,P1#proposal_rec.proposal,BackEndOpts), 575 | Mod:handle_commit(P1#proposal_rec.proposal,Z1,State,ZabServerInfo), 576 | Z1 577 | end, 578 | % NewZxid=fold_all(Que,F,Last), 579 | NewZxid=fold_all(Que,F,Last,ets:first(Que),LastZxid), 580 | lager:info("follow recover local ok,state to followiing"), 581 | loop(Server#server{last_zxid=NewZxid,last_commit_zxid=NewZxid},following,ZabServerInfo) 582 | ; 583 | 584 | _ when QueSize=:=0-> 585 | M1={recover_req,{Mod,node()},zabe_util:zxid_plus(Last)}, 586 | send_zab_msg({Mod,Leader},M1,Server#server.logical_clock), 587 | loop(Server,ZabState,ZabServerInfo); 588 | _ when QueSize>0 -> 589 | Min=ets:first(Que), 590 | % [M2|_]=ets:lookup(Que,Min), 591 | % Pro=M2#proposal_rec.proposal, 592 | % MinZxid = Pro#proposal.transaction#transaction.zxid, 593 | case zabe_util:zxid_big_eq(Last,Min) of 594 | true-> 595 | % lager:debug("recover local~p",[ets:tab2list(Que)]), 596 | % lager:debug(" recover local ~p ~p",[Last,MinZxid]), 597 | F=fun(P1)-> 598 | Z1=P1#proposal_rec.proposal#proposal.transaction#transaction.zxid, 599 | ProposalLogMod:put_proposal(Z1,P1#proposal_rec.proposal,BackEndOpts), 600 | Mod:handle_commit(P1#proposal_rec.proposal,Z1,State,ZabServerInfo), 601 | Z1 602 | end, 603 | % NewZxid=fold_all(Que,F,Last), 604 | NewZxid=fold_all(Que,F,Last,Min,LastZxid), 605 | %ets:delete_all_objects(Que), 606 | lager:info("follow recover ok,state to followiing"), 607 | loop(Server#server{last_zxid=NewZxid,last_commit_zxid=NewZxid},following,ZabServerInfo) 608 | ; 609 | false -> 610 | 611 | M1={recover_req,{Mod,node()},zabe_util:zxid_plus(Last)}, 612 | send_zab_msg({Mod,Leader},M1,Server#server.logical_clock), 613 | loop(Server,ZabState,ZabServerInfo) 614 | end 615 | end 616 | ; 617 | {truncate_ack,LeaderEpochLastZxid} -> 618 | % {E1,_}=LastZxid, 619 | MZxid=case LeaderEpochLastZxid of 620 | not_need->LastZxid; 621 | not_found->LastZxid; 622 | _-> 623 | 624 | {ok,{L2,_}}= 625 | ProposalLogMod:fold(fun({Key,_Value},{Acc,Count})-> 626 | {[Key|Acc],Count} end,{[],infinite}, 627 | zabe_util:zxid_plus(LeaderEpochLastZxid),BackEndOpts), 628 | % {ok,L1}=ProposalLogMod:trunc(fun({K,_V})-> 629 | % K end ,zabe_util:zxid_plus(LeaderEpochLastZxid),[]), 630 | lists:map(fun(Key)-> 631 | ProposalLogMod:delete_proposal(Key,BackEndOpts) end,lists:reverse(L2)), 632 | LeaderEpochLastZxid 633 | end, 634 | %%% recover local,redo local log 635 | 636 | {ok,{_T,_}}=ProposalLogMod:fold(fun({_Key,P1},{_LastCommit,Count})-> 637 | 638 | Mod:handle_commit(P1#proposal.transaction#transaction.value, 639 | P1#proposal.transaction#transaction.zxid,State,ZabServerInfo) , 640 | {P1#proposal.transaction#transaction.zxid,Count} 641 | end, 642 | {LastCommitZxid,infinite},LastCommitZxid,BackEndOpts), 643 | 644 | 645 | %%% 646 | 647 | M1={recover_req,{Mod,node()},zabe_util:zxid_plus(MZxid)}, 648 | send_zab_msg({Mod,Leader},M1,Server#server.logical_clock), 649 | loop(Server#server{last_zxid=MZxid,last_commit_zxid=MZxid},ZabState,ZabServerInfo); 650 | #zab_req{msg=Msg}-> 651 | Zxid1=Msg#proposal.transaction#transaction.zxid, 652 | ets:insert(Que,#proposal_rec{zxid=Zxid1,proposal=Msg,commit=false}), 653 | loop(Server,ZabState,ZabServerInfo); 654 | #zab_commit{msg=Zxid1}-> 655 | [P1|_]=ets:lookup(Que,Zxid1), 656 | ets:insert(Que,P1#proposal_rec{commit=true}), 657 | loop(Server,ZabState,ZabServerInfo); 658 | _ -> %flush msg %todo save proposal msg into que 659 | loop(Server,ZabState,ZabServerInfo) 660 | end 661 | . 662 | 663 | fold_all(Que,F,Last,Key,Acc)-> 664 | case ets:lookup(Que,Key) of 665 | [P1|_]-> 666 | lager:debug("dfdfd~p",[zabe_util:zxid_big_eq(Last,Key)]), 667 | case zabe_util:zxid_big_eq(Last,Key) of 668 | true-> 669 | ets:delete(Que,Key), 670 | fold_all(Que,F,Last,zabe_util:zxid_plus(Key),Acc); 671 | _-> 672 | case P1#proposal_rec.commit of 673 | true-> %ets:delete(Que,Key#proposal_rec.zxid), 674 | Acc1=F(P1), 675 | ets:delete(Que,Key), 676 | fold_all(Que,F,Last,zabe_util:zxid_plus(Key),Acc1); 677 | false -> 678 | Acc 679 | end 680 | end; 681 | _->Acc 682 | end 683 | . 684 | 685 | 686 | 687 | following(#server{mod = Mod, state = State, 688 | ensemble=_Ensemble, 689 | back_end_opts=BackEndOpts, 690 | proposal_log_mod=ProposalLogMod,elect_pid=_EPid, 691 | quorum=_Quorum,debug=_Debug,last_zxid=_Zxid,proposal_que=_Que,leader=Leader} = Server,Msg1,ZabState,ZabServerInfo)-> 692 | 693 | case Msg1 of 694 | {'$proposal_call',From,Msg} -> 695 | send_zab_msg({Mod,Leader},{zab_proposal,From,Msg,{Mod,node()}},Server#server.logical_clock), 696 | loop(Server,ZabState,ZabServerInfo) 697 | ; 698 | #zab_req{msg=Msg}-> 699 | Zxid1=Msg#proposal.transaction#transaction.zxid, 700 | ok=ProposalLogMod:put_proposal(Zxid1,Msg,BackEndOpts), 701 | Ack={Zxid1,{Mod,node()}}, 702 | 703 | send_zab_msg({Mod,Leader},#zab_ack{msg=Ack},Server#server.logical_clock), 704 | loop(Server#server{last_zxid=Zxid1},ZabState,ZabServerInfo); 705 | #zab_commit{msg=Zxid1}-> 706 | {ok,Proposal}=ProposalLogMod:get_proposal(Zxid1,BackEndOpts), 707 | Txn=Proposal#proposal.transaction, 708 | {ok,_,Ns}=Mod:handle_commit(Txn#transaction.value,Zxid1,State,ZabServerInfo), 709 | 710 | loop(Server#server{state=Ns,last_commit_zxid=Zxid1},ZabState,ZabServerInfo); 711 | _ -> 712 | loop(Server,ZabState,ZabServerInfo) 713 | 714 | end. 715 | 716 | 717 | leading(#server{mod = Mod, state = State, 718 | ensemble=Ensemble,mon_follow_refs=MRefs, 719 | proposal_log_mod=ProposalLogMod,elect_pid=_EPid, 720 | last_zxid=Zxid, 721 | back_end_opts=BackEndOpts, 722 | quorum=Quorum,debug=_Debug,current_zxid=CurZxid,proposal_que=Que,leader=_Leader} = Server 723 | ,Msg1,ZabState,ZabServerInfo)-> 724 | case Msg1 of 725 | 726 | {'$proposal_call',From,Msg} -> 727 | send_zab_msg(self(),{zab_proposal,From,Msg,{Mod,node()}},Server#server.logical_clock), 728 | loop(Server,ZabState,ZabServerInfo); 729 | {zab_proposal,From,Msg,Sender} -> 730 | {Epoch,Z}=Zxid, 731 | Z1 =Z+1, 732 | NewZxid={Epoch,Z1}, 733 | Tran=#transaction{zxid=NewZxid,value=Msg}, 734 | 735 | Proposal=#proposal{sender=Sender,client=From,transaction=Tran}, 736 | ZabReq=#zab_req{msg=Proposal}, 737 | Acks=dict:new(), 738 | 739 | A2=dict:store({Mod,node()},"",Acks), 740 | ets:insert(Que,#proposal_rec{zxid=NewZxid,proposal=Proposal,acks=A2}), 741 | %% leader learn first 742 | ProposalLogMod:put_proposal(NewZxid,Proposal,BackEndOpts), 743 | abcast(Mod,lists:delete(node(),Ensemble),ZabReq,Server#server.logical_clock), 744 | loop(Server#server{current_zxid=NewZxid,last_zxid=NewZxid},ZabState,ZabServerInfo) 745 | ; 746 | #zab_ack{msg={Zxid1,From}}-> 747 | case ets:lookup(Que,Zxid1) of 748 | []-> 749 | 750 | %%maybe delay msg lager:debug("1"), 751 | do_nothing,log, 752 | loop(Server,ZabState,ZabServerInfo); 753 | [Pro|_]-> 754 | Acks=Pro#proposal_rec.acks, 755 | A2=dict:store(From,"",Acks), 756 | Size=dict:size(A2), 757 | if 758 | Size >=Quorum -> 759 | %Result=Mod:handle_commit(Zxid,), 760 | P1=Pro#proposal_rec.proposal, 761 | T1=P1#proposal.transaction, 762 | V1=T1#transaction.value, 763 | {ok,Result,Ns}=Mod:handle_commit(V1,Zxid1,State,ZabServerInfo), 764 | ZabCommit=#zab_commit{msg=Zxid1}, 765 | ets:delete(Que,Zxid1), 766 | abcast(Mod,lists:delete(node(),Ensemble),ZabCommit,Server#server.logical_clock), 767 | reply(P1#proposal.client,Result), 768 | loop(Server#server{state=Ns,last_commit_zxid=Zxid1},ZabState,ZabServerInfo) 769 | ; 770 | true -> 771 | ets:insert(Que,Pro#proposal_rec{acks=A2}), 772 | loop(Server,ZabState,ZabServerInfo) 773 | end 774 | end; 775 | {truncate_req,From,{Epoch1,_}} -> 776 | {LeaderEpoch,_}=Zxid, 777 | EpochLastZxid=if LeaderEpoch > Epoch1 778 | ->{ok,EL}=ProposalLogMod:get_epoch_last_zxid(Epoch1,BackEndOpts),EL; 779 | true -> 780 | not_need 781 | end, 782 | % {ok,EpochLastZxid}=ProposalLogMod:get_epoch_last_zxid(Epoch1,[]), 783 | M1={truncate_ack,EpochLastZxid}, 784 | send_zab_msg(From,M1,Server#server.logical_clock), 785 | loop(Server,ZabState,ZabServerInfo); 786 | 787 | {recover_req,From,StartZxid} -> 788 | {ok,{Res,_}}= 789 | ProposalLogMod:fold(fun({_Key,Value},{Acc,Count})-> 790 | {[Value|Acc],Count-1} end,{[],100},StartZxid,BackEndOpts), 791 | 792 | % {ok,Res}=ProposalLogMod:iterate_zxid_count(fun({_K,V})-> 793 | % V end,StartZxid,100), 794 | 795 | M1={recover_ack,lists:reverse(Res),CurZxid}, 796 | 797 | send_zab_msg(From,M1,Server#server.logical_clock), 798 | loop(Server,ZabState,ZabServerInfo); 799 | {recover_ok,From} -> 800 | 801 | NRefs=case dict:is_key(From,MRefs) of 802 | true->MRefs; 803 | false-> 804 | case catch erlang:monitor(process,From) of 805 | {'EXIT',_}-> 806 | MRefs; 807 | Ref-> 808 | dict:store(From,Ref,MRefs) 809 | end 810 | end, 811 | loop(Server#server{mon_follow_refs=NRefs},ZabState,ZabServerInfo); 812 | _ -> 813 | loop(Server,ZabState,ZabServerInfo) 814 | end. 815 | abcast(Mod,Assemble,ZabReq,LC)-> 816 | [send_zab_msg({Mod,Node},ZabReq,LC)||Node<-Assemble], 817 | ok. 818 | 819 | 820 | 821 | %%----------------------------------------------------------------- 822 | %% Callback functions for system messages handling. 823 | %%----------------------------------------------------------------- 824 | %% @hidden 825 | system_continue(_Parent, _Debug, [ZabState,ZabServerInfo, Server]) -> 826 | loop(Server,ZabState,ZabServerInfo). 827 | 828 | 829 | %% @hidden 830 | system_terminate(Reason, _Parent, _Debug, [_Mode, Server, Role, E]) -> 831 | terminate(Reason, [], Server, Role, E). 832 | 833 | %% @hidden 834 | system_code_change([Mode, Server, Role, E], _Module, OldVsn, Extra) -> 835 | #server{mod = Mod, state = State} = Server, 836 | case catch Mod:code_change(OldVsn, State, E, Extra) of 837 | {ok, NewState} -> 838 | NewServer = Server#server{state = NewState}, 839 | {ok, [Mode, NewServer, Role, E]}; 840 | {ok, NewState, NewE} -> 841 | NewServer = Server#server{state = NewState}, 842 | {ok, [Mode, NewServer, Role, NewE]}; 843 | Else -> Else 844 | end. 845 | 846 | %%----------------------------------------------------------------- 847 | %% Format debug messages. Print them as the call-back module sees 848 | %% them, not as the real erlang messages. Use trace for that. 849 | %%----------------------------------------------------------------- 850 | %% @hidden 851 | print_event(Dev, {in, Msg}, Name) -> 852 | case Msg of 853 | {'$gen_call', {From, _Tag}, Call} -> 854 | io:format(Dev, "*DBG* ~p got local call ~p from ~w~n", 855 | [Name, Call, From]); 856 | {'$leader_call', {From, _Tag}, Call} -> 857 | io:format(Dev, "*DBG* ~p got global call ~p from ~w~n", 858 | [Name, Call, From]); 859 | {'$gen_cast', Cast} -> 860 | io:format(Dev, "*DBG* ~p got local cast ~p~n", 861 | [Name, Cast]); 862 | {'$leader_cast', Cast} -> 863 | io:format(Dev, "*DBG* ~p got global cast ~p~n", 864 | [Name, Cast]); 865 | _ -> 866 | io:format(Dev, "*DBG* ~p got ~p~n", [Name, Msg]) 867 | end; 868 | print_event(Dev, {out, Msg, To, State}, Name) -> 869 | io:format(Dev, "*DBG* ~p sent ~p to ~w, new state ~w~n", 870 | [Name, Msg, To, State]); 871 | print_event(Dev, {noreply, State}, Name) -> 872 | io:format(Dev, "*DBG* ~p new state ~w~n", [Name, State]); 873 | print_event(Dev, Event, Name) -> 874 | io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). 875 | 876 | 877 | handle_msg({'$gen_call', From, Request} = Msg, 878 | #server{mod = Mod, state = State} = Server, ZabState,E) -> 879 | case catch Mod:handle_call(Request, From, State,E) of 880 | {reply, Reply, NState} -> 881 | NewServer = reply(From, Reply, 882 | Server#server{state = NState}, ZabState, E), 883 | do_loop(NewServer,ZabState,E); 884 | {noreply, NState} = Reply -> 885 | NewServer = handle_debug(Server#server{state = NState}, 886 | ZabState, E, Reply), 887 | do_loop(NewServer,ZabState,E); 888 | {stop, Reason, Reply, NState} -> 889 | {'EXIT', R} = 890 | (catch terminate(Reason, Msg, Server#server{state = NState}, 891 | ZabState, E)), 892 | reply(From, Reply), 893 | exit(R); 894 | Other -> 895 | handle_common_reply(Other, Msg, Server,ZabState, E) 896 | end; 897 | handle_msg({'$gen_cast',Msg} = Cast, 898 | #server{mod = Mod, state = State} = Server, Role, E) -> 899 | handle_common_reply(catch Mod:handle_cast(Msg, State, E), 900 | Cast, Server, Role, E); 901 | 902 | 903 | handle_msg(Msg, #server{mod = Mod, state = State} = Server, Role, E) -> 904 | handle_common_reply(catch Mod:handle_info(Msg, State,E), 905 | Msg, Server, Role, E). 906 | 907 | 908 | handle_common_reply(Reply, Msg, Server, Role, E) -> 909 | case Reply of 910 | {noreply, NState} -> 911 | NewServer = handle_debug(Server#server{state = NState}, 912 | Role, E, Reply), 913 | do_loop(NewServer, Role,E); 914 | {ok, NState} -> 915 | NewServer = handle_debug(Server#server{state = NState}, 916 | Role, E, Reply), 917 | do_loop(NewServer, Role,E); 918 | {stop, Reason, NState} -> 919 | terminate(Reason, Msg, Server#server{state = NState}, Role, E); 920 | {'EXIT', Reason} -> 921 | terminate(Reason, Msg, Server, Role, E); 922 | _ -> 923 | terminate({bad2_return_value, Reply}, Msg, Server, Role, E) 924 | end. 925 | 926 | 927 | reply({To, Tag}, Reply, #server{state = State} = Server, Role, E) -> 928 | reply({To, Tag}, Reply), 929 | handle_debug(Server, Role, E, {out, Reply, To, State}). 930 | 931 | 932 | handle_debug(#server{debug = []} = Server, _Role, _E, _Event) -> 933 | Server; 934 | handle_debug(#server{debug = Debug} = Server, _Role, _E, Event) -> 935 | Debug1 = sys:handle_debug(Debug, {?MODULE, print_event}, 936 | "nouse", Event), 937 | Server#server{debug = Debug1}. 938 | 939 | 940 | do_loop(Server,ZabState,ZabServerInfo) -> 941 | loop(Server,ZabState,ZabServerInfo). 942 | 943 | %%% --------------------------------------------------- 944 | %%% Terminate the server. 945 | %%% --------------------------------------------------- 946 | 947 | terminate(Reason, Msg, #server{mod = Mod, 948 | state = State, 949 | debug = Debug} = _Server, _Role, 950 | E) -> 951 | 952 | case catch Mod:terminate(Reason, State,E) of 953 | {'EXIT', R} -> 954 | error_info(R,atom_to_list(?MODULE), Msg, State, Debug), 955 | exit(R); 956 | _ -> 957 | case Reason of 958 | normal -> 959 | exit(normal); 960 | shutdown -> 961 | exit(shutdown); 962 | _ -> 963 | error_info(Reason,atom_to_list(?MODULE), Msg, State, Debug), 964 | exit(Reason) 965 | end 966 | end. 967 | 968 | %% Maybe we shouldn't do this? We have the crash report... 969 | error_info(Reason, Name, Msg, State, Debug) -> 970 | error_logger:format("** Generic leader ~p terminating \n" 971 | "** Last message in was ~p~n" 972 | "** When Server state == ~p~n" 973 | "** Reason for termination == ~n** ~p~n", 974 | [Name, Msg, State, Reason]), 975 | sys:print_log(Debug), 976 | ok. 977 | 978 | %%% --------------------------------------------------- 979 | %%% Misc. functions. 980 | %%% --------------------------------------------------- 981 | 982 | opt(Op, [{Op, Value}|_]) -> 983 | {ok, Value}; 984 | opt(Op, [_|Options]) -> 985 | opt(Op, Options); 986 | opt(_, []) -> 987 | false. 988 | 989 | debug_options(Name, Opts) -> 990 | case opt(debug, Opts) of 991 | {ok, Options} -> dbg_options(Name, Options); 992 | _ -> dbg_options(Name, []) 993 | end. 994 | 995 | dbg_options(Name, []) -> 996 | Opts = 997 | case init:get_argument(generic_debug) of 998 | error -> 999 | []; 1000 | _ -> 1001 | [log, statistics] 1002 | end, 1003 | dbg_opts(Name, Opts); 1004 | dbg_options(Name, Opts) -> 1005 | dbg_opts(Name, Opts). 1006 | 1007 | dbg_opts(Name, Opts) -> 1008 | case catch sys:debug_options(Opts) of 1009 | {'EXIT',_} -> 1010 | error_logger:format("~p: ignoring erroneous debug options - ~p~n", 1011 | [Name, Opts]), 1012 | []; 1013 | Dbg -> 1014 | Dbg 1015 | end. 1016 | 1017 | %%----------------------------------------------------------------- 1018 | %% Status information 1019 | %%----------------------------------------------------------------- 1020 | %% @hidden 1021 | format_status(Opt, StatusData) -> 1022 | [PDict, SysState, Parent, Debug, [_Mode, Server, _Role, _E]] = StatusData, 1023 | Log = sys:get_debug(log, Debug, []), 1024 | #server{mod = Mod, state = State} = Server, 1025 | Specific = 1026 | case erlang:function_exported(Mod, format_status, 2) of 1027 | true -> 1028 | case catch apply(Mod, format_status, [Opt, [PDict, State]]) of 1029 | {'EXIT', _} -> [{data, [{"State", State}]}]; 1030 | Else -> Else 1031 | end; 1032 | _ -> 1033 | [{data, [{"State", State}]}] 1034 | end, 1035 | [{header, "Header"}, 1036 | {data, [{"Status", SysState}, 1037 | {"Parent", Parent}, 1038 | {"Logged events", Log}]} | 1039 | Specific]. 1040 | 1041 | 1042 | %%----------------------------------------------------------------- 1043 | %% Leader-election functions 1044 | %%----------------------------------------------------------------- 1045 | 1046 | send_zab_msg(To,Msg,LC)-> 1047 | catch erlang:send(To,#msg{cmd=?ZAB_CMD,value=Msg,epoch=LC}). 1048 | 1049 | 1050 | 1051 | --------------------------------------------------------------------------------