| Overview |  |
Generated by EDoc
15 | 16 | 17 | -------------------------------------------------------------------------------- /docs/stylesheet.css: -------------------------------------------------------------------------------- 1 | /* standard EDoc style sheet */ 2 | body { 3 | font-family: Verdana, Arial, Helvetica, sans-serif; 4 | margin-left: .25in; 5 | margin-right: .2in; 6 | margin-top: 0.2in; 7 | margin-bottom: 0.2in; 8 | color: #000000; 9 | background-color: #ffffff; 10 | } 11 | h1,h2 { 12 | margin-left: -0.2in; 13 | } 14 | div.navbar { 15 | background-color: #add8e6; 16 | padding: 0.2em; 17 | } 18 | h2.indextitle { 19 | padding: 0.4em; 20 | background-color: #add8e6; 21 | } 22 | h3.function,h3.typedecl { 23 | background-color: #add8e6; 24 | padding-left: 1em; 25 | } 26 | div.spec { 27 | margin-left: 2em; 28 | background-color: #eeeeee; 29 | } 30 | a.module { 31 | text-decoration:none 32 | } 33 | a.module:hover { 34 | background-color: #eeeeee; 35 | } 36 | ul.definitions { 37 | list-style-type: none; 38 | } 39 | ul.index { 40 | list-style-type: none; 41 | background-color: #eeeeee; 42 | } 43 | 44 | /* 45 | * Minor style tweaks 46 | */ 47 | ul { 48 | list-style-type: square; 49 | } 50 | table { 51 | border-collapse: collapse; 52 | } 53 | td { 54 | padding: 3 55 | } 56 | -------------------------------------------------------------------------------- /src/ra_file.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_file). 9 | 10 | -include("ra.hrl"). 11 | 12 | -define(RETRY_ON_ERROR(Op), 13 | case Op of 14 | {error, E} when E =:= eagain orelse E =:= eacces -> 15 | ?DEBUG("Error `~p` during file operation, retrying once in 20ms...", [E]), 16 | timer:sleep(20), 17 | case Op of 18 | {error, eagain} = Err -> 19 | ?DEBUG("Error `~p` again during file operation", [E]), 20 | Err; 21 | Res -> 22 | Res 23 | end; 24 | Res -> 25 | Res 26 | end). 27 | 28 | -export([ 29 | sync/1, 30 | rename/2 31 | ]). 32 | 33 | sync(Fd) -> 34 | ?RETRY_ON_ERROR(file:sync(Fd)). 35 | 36 | rename(Src, Dst) -> 37 | ?RETRY_ON_ERROR(prim_file:rename(Src, Dst)). 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: 🐛 Bug report 2 | description: Create a report to help us improve 3 | labels: bug 4 | 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! 10 | 11 | - type: textarea 12 | id: describe-bug 13 | attributes: 14 | label: Describe the bug 15 | description: A clear and concise description of what the bug is. 16 | validations: 17 | required: true 18 | 19 | - type: textarea 20 | id: reproduction-steps 21 | attributes: 22 | label: Reproduction steps 23 | description: Steps to reproduce the behavior 24 | value: | 25 | 1. 26 | 2. 27 | 3. 28 | ... 29 | validations: 30 | required: true 31 | 32 | - type: textarea 33 | id: expected-behavior 34 | attributes: 35 | label: Expected behavior 36 | description: A clear and concise description of what you expected to happen. 37 | validations: 38 | required: true 39 | 40 | - type: textarea 41 | id: additional-context 42 | attributes: 43 | label: Additional context 44 | description: Add any other context about the problem here. 45 | -------------------------------------------------------------------------------- /src/ra_log_read_plan.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_log_read_plan). 8 | 9 | 10 | -export([execute/2, 11 | execute/3, 12 | info/1]). 13 | 14 | -spec execute(ra_log:read_plan(), undefined | ra_flru:state()) -> 15 | {#{ra:index() => Command :: term()}, ra_flru:state()}. 16 | execute(Plan, Flru) -> 17 | execute(Plan, Flru, #{access_pattern => random, 18 | file_advise => normal}). 19 | 20 | -spec execute(ra_log:read_plan(), undefined | ra_flru:state(), 21 | ra_log_reader:read_plan_options()) -> 22 | {#{ra:index() => Command :: term()}, ra_flru:state()}. 23 | execute(Plan, Flru, Options) -> 24 | ra_log:execute_read_plan(Plan, Flru, 25 | fun ra_server:transform_for_partial_read/3, 26 | Options). 27 | 28 | -spec info(ra_log:read_plan()) -> map(). 29 | info(Plan) -> 30 | ra_log:read_plan_info(Plan). 31 | -------------------------------------------------------------------------------- /src/ra_env.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_env). 8 | 9 | -export([ 10 | data_dir/0, 11 | server_data_dir/2, 12 | configure_logger/1 13 | ]). 14 | 15 | -export_type([ 16 | ]). 17 | 18 | data_dir() -> 19 | DataDir = case application:get_env(ra, data_dir) of 20 | {ok, Dir} -> 21 | Dir; 22 | undefined -> 23 | {ok, Cwd} = file:get_cwd(), 24 | Cwd 25 | end, 26 | Node = ra_lib:to_list(node()), 27 | filename:join(DataDir, Node). 28 | 29 | server_data_dir(System, UId) when is_atom(System) -> 30 | #{data_dir := Dir} = ra_system:fetch(System), 31 | Me = ra_lib:to_list(UId), 32 | filename:join(Dir, Me). 33 | 34 | %% use this when interacting with Ra from a node without Ra running on it 35 | configure_logger(Module) -> 36 | persistent_term:put('$ra_logger', Module). 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | name: 🚀 Feature request 2 | description: Suggest an idea for this project 3 | labels: enhancement 4 | 5 | body: 6 | - type: textarea 7 | id: describe-problem 8 | attributes: 9 | label: Is your feature request related to a problem? Please describe. 10 | description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | validations: 12 | required: true 13 | 14 | - type: textarea 15 | id: describe-solution 16 | attributes: 17 | label: Describe the solution you'd like 18 | description: A clear and concise description of what you want to happen. 19 | validations: 20 | required: true 21 | 22 | - type: textarea 23 | id: describe-alternatives 24 | attributes: 25 | label: Describe alternatives you've considered 26 | description: A clear and concise description of any alternative solutions or features you've considered. 27 | validations: 28 | required: false 29 | 30 | - type: textarea 31 | id: additional-context 32 | attributes: 33 | label: Additional context 34 | description: Add any other context or screenshots about the feature request here. 35 | validations: 36 | required: false 37 | -------------------------------------------------------------------------------- /docs/modules-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 || ra |
| ra_aux |
| ra_counters |
| ra_dbg |
| ra_directory |
| ra_env |
| ra_leaderboard |
| ra_log_reader |
| ra_machine |
| ra_snapshot |
| ra_system |
| data_dir/0 | |
| server_data_dir/2 | |
| configure_logger/1 |
data_dir() -> any()
28 |
server_data_dir(System, UId) -> any()
34 |
configure_logger(Module) -> any()
40 |
Generated by EDoc
45 | 46 | 47 | -------------------------------------------------------------------------------- /test/tcp_inet_proxy_helpers.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(tcp_inet_proxy_helpers). 8 | 9 | -export([configure_dist_proxy/1, 10 | block_traffic_between/2, 11 | allow_traffic_between/2]). 12 | -export([wait_for_blocked/3]). 13 | configure_dist_proxy(Config) -> 14 | [{erlang_dist_module, inet_tcp_proxy_dist} | Config]. 15 | 16 | block_traffic_between(NodeA, NodeB) -> 17 | ok = retry_rpc(20, fun () -> rpc:call(NodeA, inet_tcp_proxy_dist, block, [NodeB]) end), 18 | ok = retry_rpc(20, fun () -> rpc:call(NodeB, inet_tcp_proxy_dist, block, [NodeA]) end), 19 | wait_for_blocked(NodeA, NodeB, 10). 20 | 21 | allow_traffic_between(NodeA, NodeB) -> 22 | ok = retry_rpc(20, fun () -> rpc:call(NodeA, inet_tcp_proxy_dist, allow, [NodeB]) end), 23 | ok = retry_rpc(20, fun () -> rpc:call(NodeB, inet_tcp_proxy_dist, allow, [NodeA]) end), 24 | wait_for_unblocked(NodeA, NodeB, 10). 25 | 26 | retry_rpc(1, Fun) -> 27 | Fun(); 28 | retry_rpc(N, Fun) -> 29 | try Fun() of 30 | {badrpc, _} -> 31 | timer:sleep(1000), 32 | retry_rpc(N-1, Fun); 33 | Result -> Result 34 | catch _:_ -> 35 | timer:sleep(1000), 36 | retry_rpc(N-1, Fun) 37 | end. 38 | 39 | wait_for_blocked(NodeA, NodeB, 0) -> 40 | error({failed_to_block, NodeA, NodeB, no_more_attempts}); 41 | wait_for_blocked(NodeA, NodeB, Attempts) -> 42 | case rpc:call(NodeA, rpc, call, [NodeB, erlang, node, []], 1000) of 43 | {badrpc, _} -> ok; 44 | _ -> 45 | timer:sleep(50), 46 | wait_for_blocked(NodeA, NodeB, Attempts - 1) 47 | end. 48 | 49 | wait_for_unblocked(NodeA, NodeB, 0) -> 50 | error({failed_to_unblock, NodeA, NodeB, no_more_attempts}); 51 | wait_for_unblocked(NodeA, NodeB, Attempts) -> 52 | case rpc:call(NodeA, rpc, call, [NodeB, erlang, node, []], 1000) of 53 | NodeB -> ok; 54 | {badrpc, _} -> 55 | timer:sleep(50), 56 | wait_for_unblocked(NodeA, NodeB, Attempts - 1) 57 | end. 58 | -------------------------------------------------------------------------------- /test/erlang_node_helpers.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(erlang_node_helpers). 8 | 9 | -export([start_erlang_nodes/2, start_erlang_node/2, stop_erlang_nodes/1, stop_erlang_node/1]). 10 | -include_lib("common_test/include/ct.hrl"). 11 | 12 | start_erlang_nodes(Nodes, Config) -> 13 | [start_erlang_node(Node, Config) || Node <- Nodes]. 14 | 15 | start_erlang_node(Node, Config) -> 16 | DistMod = ?config(erlang_dist_module, Config), 17 | StartArgs = case DistMod of 18 | undefined -> 19 | []; 20 | _ -> 21 | DistModS = atom_to_list(DistMod), 22 | DistModPath = filename:absname( 23 | filename:dirname( 24 | code:where_is_file(DistModS ++ ".beam"))), 25 | DistArg = re:replace(DistModS, "_dist$", "", 26 | [{return, list}]), 27 | ["-pa", DistModPath, "-proto_dist", DistArg, 28 | "-kernel", "prevent_overlapping_partitions", "false"] 29 | end, 30 | ct:log("Starting node ~p, with ~p", [Node, StartArgs]), 31 | {ok, Peer, _} = peer:start(#{name => Node, args => StartArgs, connection => standard_io}), 32 | wait_for_distribution(Node, 50), 33 | add_lib_dir(Node), 34 | Peer. 35 | 36 | add_lib_dir(Node) -> 37 | ct_rpc:call(Node, code, add_paths, [code:get_path()]). 38 | 39 | wait_for_distribution(Node, 0) -> 40 | error({distribution_failed_for, Node, no_more_attempts}); 41 | wait_for_distribution(Node, Attempts) -> 42 | ct:pal("Waiting for node ~p", [Node]), 43 | case ct_rpc:call(Node, net_kernel, set_net_ticktime, [15]) of 44 | {badrpc, nodedown} -> 45 | timer:sleep(100), 46 | wait_for_distribution(Node, Attempts - 1); 47 | _ -> ok 48 | end. 49 | 50 | stop_erlang_nodes(Nodes) -> 51 | [stop_erlang_node(Node) || Node <- Nodes]. 52 | 53 | stop_erlang_node(Peer) -> 54 | peer:stop(Peer). 55 | -------------------------------------------------------------------------------- /src/ra_metrics_ets.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_metrics_ets). 9 | -behaviour(gen_server). 10 | 11 | -export([start_link/0]). 12 | 13 | -export([init/1, 14 | handle_call/3, 15 | handle_cast/2, 16 | handle_info/2, 17 | terminate/2, 18 | code_change/3]). 19 | 20 | -record(state, {}). 21 | 22 | %%% here to own metrics ETS tables 23 | 24 | %%%=================================================================== 25 | %%% API functions 26 | %%%=================================================================== 27 | 28 | start_link() -> 29 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 30 | 31 | %%%=================================================================== 32 | %%% gen_server callbacks 33 | %%%=================================================================== 34 | 35 | init([]) -> 36 | TableFlags = [named_table, 37 | {read_concurrency, true}, 38 | {write_concurrency, true}, 39 | public], 40 | _ = ets:new(ra_log_metrics, [set | TableFlags]), 41 | ok = ra_counters:init(), 42 | ok = ra_leaderboard:init(), 43 | 44 | %% Table for ra processes to record their current snapshot index so that 45 | %% other processes such as the segment writer can use this value to skip 46 | %% stale records and avoid flushing unnecessary data to disk. 47 | %% This is written from the ra process so will need write_concurrency. 48 | %% {RaUId, ra_index()} 49 | _ = ets:new(ra_log_snapshot_state, [set | TableFlags]), 50 | {ok, #state{}}. 51 | 52 | handle_call(_, _From, State) -> 53 | {reply, ok, State}. 54 | 55 | handle_cast(_Msg, State) -> 56 | {noreply, State}. 57 | 58 | handle_info(_Info, State) -> 59 | {noreply, State}. 60 | 61 | terminate(_Reason, _State) -> 62 | ok. 63 | 64 | code_change(_OldVsn, State, _Extra) -> 65 | {ok, State}. 66 | 67 | %%%=================================================================== 68 | %%% Internal functions 69 | %%%=================================================================== 70 | -------------------------------------------------------------------------------- /src/ra_systems_sup.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_systems_sup). 9 | 10 | -behaviour(supervisor). 11 | 12 | -include("ra.hrl"). 13 | 14 | %% API functions 15 | -export([start_link/0, 16 | start_system/1, 17 | stop_system/1]). 18 | 19 | %% Supervisor callbacks 20 | -export([init/1]). 21 | 22 | -spec start_link() -> 23 | {ok, pid()} | ignore | {error, term()}. 24 | start_link() -> 25 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 26 | 27 | -spec start_system(ra_system:config()) -> supervisor:startchild_ret(). 28 | start_system(#{name := Name, 29 | names := _Names, 30 | data_dir := Dir} = Config) when is_atom(Name) -> 31 | ?INFO("starting Ra system: ~ts in directory: ~ts", [Name, Dir]), 32 | %% TODO: validate configuration 33 | ok = ra_system:store(Config), 34 | RaSystemsSup = #{id => Name, 35 | type => supervisor, 36 | start => {ra_system_sup, start_link, [Config]}}, 37 | supervisor:start_child(?MODULE, RaSystemsSup). 38 | 39 | -spec stop_system(ra_system:config() | atom()) -> ok | {error, any()}. 40 | stop_system(#{name := Name}) when is_atom(Name) -> 41 | stop_system(Name); 42 | stop_system(Name) when is_atom(Name) -> 43 | try 44 | case supervisor:terminate_child(?MODULE, Name) of 45 | ok -> 46 | cleanup(Name); 47 | {error, not_found} -> 48 | cleanup(Name); 49 | {error, _} = Error -> 50 | Error 51 | end 52 | catch 53 | exit:{noproc, _} -> 54 | cleanup(Name) 55 | end. 56 | 57 | cleanup(Name) when is_atom(Name) -> 58 | _ = (catch supervisor:delete_child(?MODULE, Name)), 59 | _ = persistent_term:erase({'$ra_system', Name}), 60 | ok. 61 | 62 | init([]) -> 63 | %% This is not something we want to expose. It helps test suites 64 | %% that crash Ra systems on purpose and may end up crashing 65 | %% the systems faster than we normally allow. 66 | {Intensity, Period} = application:get_env(ra, ra_systems_sup_intensity, {1, 5}), 67 | SupFlags = #{strategy => one_for_one, intensity => Intensity, period => Period}, 68 | {ok, {SupFlags, []}}. 69 | -------------------------------------------------------------------------------- /src/ra_ets_queue.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% @hidden 7 | -module(ra_ets_queue). 8 | 9 | -export([ 10 | new/0, 11 | in/2, 12 | reset/1, 13 | take/2, 14 | len/1 15 | ]). 16 | 17 | 18 | -record(?MODULE, {tbl :: ets:tid(), 19 | next_key = 0 :: non_neg_integer(), 20 | len = 0 :: non_neg_integer(), 21 | queue = queue:new() :: queue:queue()}). 22 | 23 | 24 | -opaque state() :: #?MODULE{}. 25 | -type item() :: term(). 26 | 27 | -export_type([ 28 | state/0 29 | ]). 30 | 31 | -spec new() -> state(). 32 | new() -> 33 | Tid = ets:new(?MODULE, [ordered_set, private]), 34 | #?MODULE{tbl = Tid}. 35 | 36 | -spec in(item(), state()) -> state(). 37 | in(Item, #?MODULE{tbl = Tbl, 38 | queue = Q, 39 | len = Len, 40 | next_key = NextKey} = State) -> 41 | true = ets:insert(Tbl, {NextKey, Item}), 42 | State#?MODULE{next_key = NextKey + 1, 43 | len = Len + 1, 44 | queue = queue:in(NextKey, Q)}. 45 | 46 | -spec len(state()) -> non_neg_integer(). 47 | len(#?MODULE{len = Len}) -> 48 | Len. 49 | 50 | -spec reset(state()) -> state(). 51 | reset(#?MODULE{tbl = Tbl}) -> 52 | true = ets:delete_all_objects(Tbl), 53 | #?MODULE{tbl = Tbl, 54 | len = 0, 55 | queue = queue:new()}. 56 | 57 | -spec take(non_neg_integer(), state()) -> 58 | {[item()], state()}. 59 | take(Num, #?MODULE{tbl = Tbl, 60 | len = Len, 61 | queue = Q0} = State) -> 62 | {Q, Taken, Out} = queue_take(Num, Q0, Tbl), 63 | {Out, State#?MODULE{len = Len - Taken, 64 | queue = Q}}. 65 | 66 | %% Internal 67 | 68 | queue_take(N, Q, Tbl) -> 69 | queue_take(N, Q, Tbl, 0, []). 70 | 71 | queue_take(0, Q, _Tbl, Taken, Acc) -> 72 | {Q, Taken, lists:reverse(Acc)}; 73 | queue_take(N, Q0, Tbl, Taken, Acc) -> 74 | case queue:out(Q0) of 75 | {{value, Key}, Q} -> 76 | [{Key, I}] = ets:take(Tbl, Key), 77 | queue_take(N - 1, Q, Tbl, Taken + 1, 78 | [I | Acc]); 79 | {empty, _} -> 80 | {Q0, Taken, lists:reverse(Acc)} 81 | end. 82 | -------------------------------------------------------------------------------- /docs/internals/ra-concepts.ascii: -------------------------------------------------------------------------------- 1 | +-----------------------------------------------------------------------------------------------------------------------------------------+ 2 | | +------------------------------------------+ +------------------------------------------+ +------------------------------------------+ | 3 | | | | | | | | | 4 | | | +----------------+ +----------------+ | | +----------------+ +----------------+ | | +----------------+ +----------------+ | | 5 | | | | Ra Cluster 1 | | Ra Cluster 2 | | | | Ra Cluster 1 | | Ra Cluster 2 | | | | Ra Cluster 1 | | Ra Cluster 2 | | | 6 | | | +----------------+ +----------------+ | | +----------------+ +----------------+ | | +----------------+ +----------------+ | | 7 | | | Ra clusters | | Ra clusters | | Ra clusters | | 8 | | | | | | | | | 9 | | | | | | | | | 10 | | | +-----------------+ +----------------+ | | +-----------------+ +----------------+ | | +-----------------+ +----------------+ | | 11 | | | | Write-ahead log | | Segment Writer | | | | Write-ahead log | | Segment Writer | | | | Write-ahead log | | Segment Writer | | | 12 | | | +-----------------+ +----------------+ | | +-----------------+ +----------------+ | | +-----------------+ +----------------+ | | 13 | | | Ra infrastructure | | Ra infrastructure | | Ra infrastructure | | 14 | | +------------------------------------------+ +------------------------------------------+ +------------------------------------------+ | 15 | | | 16 | | Erlang node Erlang node Erlang node | 17 | | | 18 | +-----------------------------------------------------------------------------------------------------------------------------------------+ 19 | 20 | Erlang cluster -------------------------------------------------------------------------------- /src/ra_aux.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | -module(ra_aux). 7 | 8 | -export([ 9 | machine_state/1, 10 | effective_machine_version/1, 11 | leader_id/1, 12 | last_applied/1, 13 | current_term/1, 14 | members_info/1, 15 | overview/1, 16 | log_last_index_term/1, 17 | log_fetch/2, 18 | log_stats/1 19 | ]). 20 | 21 | -include("ra.hrl"). 22 | -include("ra_server.hrl"). 23 | 24 | -opaque internal_state() :: ra_server:state(). 25 | 26 | -export_type([internal_state/0]). 27 | 28 | -spec machine_state(ra_aux:internal_state()) -> term(). 29 | machine_state(State) -> 30 | maps:get(?FUNCTION_NAME, State). 31 | 32 | -spec effective_machine_version(ra_aux:internal_state()) -> 33 | ra_machine:version(). 34 | effective_machine_version(#{cfg := Cfg}) -> 35 | Cfg#cfg.effective_machine_version. 36 | 37 | -spec leader_id(ra_aux:internal_state()) -> undefined | ra_server_id(). 38 | leader_id(State) -> 39 | maps:get(?FUNCTION_NAME, State). 40 | 41 | -spec last_applied(ra_aux:internal_state()) -> ra_index(). 42 | last_applied(State) -> 43 | maps:get(?FUNCTION_NAME, State). 44 | 45 | -spec current_term(ra_aux:internal_state()) -> ra_term(). 46 | current_term(State) -> 47 | maps:get(?FUNCTION_NAME, State). 48 | 49 | -spec members_info(ra_aux:internal_state()) -> ra_cluster(). 50 | members_info(State) -> 51 | ra_server:state_query(?FUNCTION_NAME, State). 52 | 53 | -spec overview(ra_aux:internal_state()) -> map(). 54 | overview(State) -> 55 | ra_server:state_query(?FUNCTION_NAME, State). 56 | 57 | -spec log_last_index_term(ra_aux:internal_state()) -> ra_idxterm(). 58 | log_last_index_term(#{log := Log}) -> 59 | ra_log:last_index_term(Log). 60 | 61 | -spec log_fetch(ra_index(), ra_aux:internal_state()) -> 62 | {undefined | 63 | {ra_term(), 64 | CmdMetadata :: ra_server:command_meta(), 65 | Command :: term()}, ra_aux:internal_state()}. 66 | log_fetch(Idx, #{log := Log0} = State) 67 | when is_integer(Idx) -> 68 | case ra_log:fetch(Idx, Log0) of 69 | {{Idx, Term, {'$usr', Meta, Cmd, _ReplyMode}}, Log} -> 70 | {{Term, Meta, Cmd}, State#{log => Log}}; 71 | {_, Log} -> 72 | %% we only allow user commands to be read 73 | {undefined, State#{log => Log}} 74 | end. 75 | 76 | -spec log_stats(ra_aux:internal_state()) -> ra_log:overview(). 77 | log_stats(#{log := Log}) -> 78 | ra_log:overview(Log). 79 | 80 | -------------------------------------------------------------------------------- /src/ra_dbg.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @doc utilities to debug ra 8 | 9 | -module(ra_dbg). 10 | 11 | %% API 12 | -export([replay_log/3, replay_log/4]). 13 | 14 | %% exported for tests 15 | -export([filter_duplicate_entries/1]). 16 | 17 | %% @doc Replays log file for a given state machine module with an initial state. 18 | %% 19 | %% @param The location of the WAL file. 20 | %% @param The Erlang module of the state machine. 21 | %% @param The initial state. 22 | %% 23 | %% @returns The state once the log has been replayed. 24 | -spec replay_log(string(), module(), term()) -> 25 | term(). 26 | replay_log(WalFile, Module, InitialState) -> 27 | replay_log(WalFile, Module, InitialState, fun(_, _) -> noop end). 28 | 29 | %% @doc Replays log file for a given state machine module with an initial state. 30 | %% 31 | %% @param The location of the WAL file. 32 | %% @param The Erlang module of the state machine. 33 | %% @param The initial state. 34 | %% @param A function to react to state and effects after each applied command 35 | %% 36 | %% @returns The state once the log has been replayed. 37 | -spec replay_log(string(), module(), term(), Fun) -> term() when 38 | Fun :: fun((State :: term(), Effects :: term()) -> term()). 39 | replay_log(WalFile, Module, InitialState, Func) -> 40 | Wal = filter_duplicate_entries(ra_log_wal:wal2list(WalFile)), 41 | WalFunc = fun({Index, Term, {'$usr', Metadata, Command, _}}, Acc) -> 42 | Metadata1 = Metadata#{index => Index, 43 | term => Term}, 44 | case Module:apply(Metadata1, Command, Acc) of 45 | {NewAcc, _Reply} -> 46 | Func(NewAcc, undefined), 47 | NewAcc; 48 | {NewAcc, _Reply, Effects} -> 49 | Func(NewAcc, Effects), 50 | NewAcc 51 | end; 52 | (_, Acc) -> 53 | Acc 54 | end, 55 | lists:foldl(WalFunc, InitialState, Wal). 56 | 57 | filter_duplicate_entries(WalInReverseOrder) -> 58 | {_IndexRegistry, OrderedAndFilteredWal} = 59 | lists:foldl(fun({Index, _Term, _Command} = Entry, {IndexRegistry, WalAcc}) -> 60 | case maps:is_key(Index, IndexRegistry) of 61 | true -> 62 | {IndexRegistry, WalAcc}; 63 | false -> 64 | {maps:put(Index, true, IndexRegistry), 65 | lists:append([Entry], WalAcc)} 66 | end 67 | end, {#{}, []}, WalInReverseOrder), 68 | OrderedAndFilteredWal. 69 | 70 | -------------------------------------------------------------------------------- /test/ra_log_meta_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_log_meta_SUITE). 8 | 9 | -compile(nowarn_export_all). 10 | -compile(export_all). 11 | 12 | -include_lib("common_test/include/ct.hrl"). 13 | -include_lib("eunit/include/eunit.hrl"). 14 | -define(SYS, default). 15 | 16 | %% common ra_log tests to ensure behaviour is equivalent across 17 | %% ra_log backends 18 | 19 | all() -> 20 | [ 21 | {group, tests} 22 | ]. 23 | 24 | all_tests() -> 25 | [ 26 | roundtrip, 27 | delete 28 | ]. 29 | 30 | groups() -> 31 | [ 32 | {tests, [], all_tests()} 33 | ]. 34 | 35 | init_per_group(_, Config) -> 36 | PrivDir = ?config(priv_dir, Config), 37 | {ok, _} = ra:start_in(PrivDir), 38 | Config. 39 | 40 | end_per_group(_, Config) -> 41 | Config. 42 | 43 | init_per_testcase(TestCase, Config) -> 44 | [{key, TestCase} | Config]. 45 | 46 | end_per_testcase(_, Config) -> 47 | Config. 48 | 49 | roundtrip(Config) -> 50 | Id = ?config(key, Config), 51 | ok = ra_log_meta:store_sync(ra_log_meta, Id, last_applied, 199), 52 | 199 = ra_log_meta:fetch(ra_log_meta, Id, last_applied), 53 | ok = ra_log_meta:store_sync(ra_log_meta, Id, current_term, 5), 54 | 5 = ra_log_meta:fetch(ra_log_meta, Id, current_term), 55 | ok = ra_log_meta:store(ra_log_meta, Id, voted_for, 'cream'), 56 | ok = ra_log_meta:store_sync(ra_log_meta, Id, voted_for, 'cøstard'), 57 | 'cøstard' = ra_log_meta:fetch(ra_log_meta, Id, voted_for), 58 | ok = ra_log_meta:store_sync(ra_log_meta, Id, voted_for, undefined), 59 | undefined = ra_log_meta:fetch(ra_log_meta, Id, voted_for), 60 | ok = ra_log_meta:store_sync(ra_log_meta, Id, voted_for, {custard, cream}), 61 | {custard, cream} = ra_log_meta:fetch(ra_log_meta, Id, voted_for), 62 | %% lose and re-open 63 | proc_lib:stop(whereis(ra_log_meta), killed, infinity), 64 | timer:sleep(200), 65 | % give it some time to restart 66 | 199 = ra_log_meta:fetch(ra_log_meta, Id, last_applied), 67 | 5 = ra_log_meta:fetch(ra_log_meta, Id, current_term), 68 | {custard, cream} = ra_log_meta:fetch(ra_log_meta, Id, voted_for), 69 | ok. 70 | 71 | delete(Config) -> 72 | Id = ?config(key, Config), 73 | ok = ra_log_meta:store_sync(ra_log_meta, Id, last_applied, 199), 74 | Oth = <<"some_other_id">>, 75 | ok = ra_log_meta:store_sync(ra_log_meta, Oth, last_applied, 1), 76 | ok = ra_log_meta:delete(ra_log_meta, Oth), %% async 77 | ok = ra_log_meta:delete_sync(ra_log_meta, Id), %% async 78 | %% store some other id just to make sure the delete is processed 79 | undefined = ra_log_meta:fetch(ra_log_meta, Oth, last_applied), 80 | undefined = ra_log_meta:fetch(ra_log_meta, Id, last_applied), 81 | ok. 82 | -------------------------------------------------------------------------------- /docs/ra_dbg.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 || replay_log/3 | Replays log file for a given state machine module with an initial state. |
| replay_log/4 | Replays log file for a given state machine module with an initial state. |
| filter_duplicate_entries/1 |
replay_log(WalFile :: string(), 28 | Module :: module(), 29 | InitialState :: term()) -> 30 | term()31 |
32 |
returns: The state once the log has been replayed.
33 |Replays log file for a given state machine module with an initial state. 34 |
35 | 36 |replay_log(WalFile :: string(), 39 | Module :: module(), 40 | InitialState :: term(), 41 | Fun) -> 42 | term()43 |
Fun = fun((State :: term(), Effects :: term()) -> term())
45 |
returns: The state once the log has been replayed.
46 |Replays log file for a given state machine module with an initial state. 47 |
48 | 49 |filter_duplicate_entries(WalInReverseOrder) -> any()
53 |
Generated by EDoc
58 | 59 | 60 | -------------------------------------------------------------------------------- /src/ra_system_recover.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% @hidden 7 | -module(ra_system_recover). 8 | 9 | -behaviour(gen_server). 10 | 11 | -include("ra.hrl"). 12 | %% API functions 13 | -export([start_link/1]). 14 | 15 | %% gen_server callbacks 16 | -export([init/1, 17 | handle_call/3, 18 | handle_cast/2, 19 | handle_info/2, 20 | terminate/2, 21 | code_change/3]). 22 | 23 | -record(state, {}). 24 | 25 | %%%=================================================================== 26 | %%% API functions 27 | %%%=================================================================== 28 | 29 | start_link(System) when is_atom(System) -> 30 | gen_server:start_link(?MODULE, [System], []). 31 | 32 | %%%=================================================================== 33 | %%% gen_server callbacks 34 | %%%=================================================================== 35 | 36 | init([System]) -> 37 | Conf = ra_system:fetch(System), 38 | case Conf of 39 | #{server_recovery_strategy := registered = Strat} -> 40 | Regd = ra_directory:list_registered(System), 41 | ?INFO("~s: ra system '~ts' server recovery strategy ~w, 42 | num servers ~b", 43 | [?MODULE, System, Strat, length(Regd)]), 44 | [begin 45 | case ra:restart_server(System, {N, node()}) of 46 | ok -> 47 | ok; 48 | Err -> 49 | ?WARN("~s: ra:restart_server/2 failed with ~p", 50 | [?MODULE, Err]), 51 | ok 52 | end 53 | end || {N, _Uid} <- Regd], 54 | ok; 55 | #{server_recovery_strategy := {Mod, Fun, Args}} -> 56 | ?INFO("~s: ra system '~ts' server recovery strategy ~s:~s", 57 | [?MODULE, System, Mod, Fun]), 58 | try apply(Mod, Fun, [System | Args]) of 59 | ok -> 60 | ok 61 | catch C:E:S -> 62 | ?ERROR("~s: ~s encountered during server recovery ~p. " 63 | "stack ~p", 64 | [?MODULE, C, E, S]), 65 | ok 66 | end; 67 | _ -> 68 | ?DEBUG("~s: no server recovery configured", [?MODULE]), 69 | ok 70 | end, 71 | {ok, #state{} , hibernate}. 72 | 73 | handle_call(_Request, _From, State) -> 74 | Reply = ok, 75 | {reply, Reply, State}. 76 | 77 | handle_cast(_Msg, State) -> 78 | {noreply, State}. 79 | 80 | handle_info(_Info, State) -> 81 | {noreply, State}. 82 | 83 | terminate(_Reason, _State) -> 84 | ok. 85 | 86 | code_change(_OldVsn, State, _Extra) -> 87 | {ok, State}. 88 | 89 | %%%=================================================================== 90 | %%% Internal functions 91 | %%%=================================================================== 92 | -------------------------------------------------------------------------------- /test/ra_props_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_props_SUITE). 8 | -compile(nowarn_export_all). 9 | -compile(export_all). 10 | 11 | -include("src/ra.hrl"). 12 | -include_lib("proper/include/proper.hrl"). 13 | -include_lib("common_test/include/ct.hrl"). 14 | -include_lib("eunit/include/eunit.hrl"). 15 | 16 | % just here to silence unused warning 17 | -export_type([op/0]). 18 | 19 | all() -> 20 | [ 21 | non_assoc 22 | ]. 23 | 24 | groups() -> 25 | [ {tests, [], all()} ]. 26 | 27 | -type op() :: {add, 1..100} | {subtract, 1..100} | {divide, 2..10} | {mult, 1..10}. 28 | 29 | init_per_suite(Config) -> 30 | {ok, _} = ra:start_in(?config(priv_dir, Config)), 31 | Config. 32 | 33 | init_per_testcase(non_assoc, Config) -> 34 | {ok, Cluster, _} = ra:start_cluster(default, non_assoc, 35 | {simple, fun non_assoc_apply/2, 0}, 36 | [{n1, node()}, 37 | {n2, node()}, 38 | {n3, node()}] 39 | ), 40 | [{ra_cluster, Cluster} | Config]. 41 | 42 | end_per_testcase(_, Config) -> 43 | terminate_cluster(?config(ra_cluster, Config)), 44 | Config. 45 | 46 | end_per_suite(Config) -> 47 | application:stop(ra), 48 | Config. 49 | 50 | %% this test mixes associative with non-associative operations to tests that 51 | %% all nodes apply operations in the same order 52 | non_assoc_prop([A, B, C], {Ops, Initial}) -> 53 | ct:pal("non_assoc_prop Ops: ~p Initial: ~p", [Ops, Initial]), 54 | Expected = lists:foldl(fun non_assoc_apply/2, Initial, Ops), 55 | % set cluster to 56 | {ok, _, Leader} = ra:process_command(A, {set, Initial}), 57 | [ra:process_command(Leader, Op) || Op <- Ops], 58 | {ok, _, Leader} = ra:consistent_query(A, fun(_) -> ok end), 59 | timer:sleep(100), 60 | {ok, {_, ARes}, _} = ra:local_query(A, fun id/1), 61 | {ok, {_, BRes}, _} = ra:local_query(B, fun id/1), 62 | {ok, {_, CRes}, _} = ra:local_query(C, fun id/1), 63 | % ct:pal("Result ~p ~p ~p Expected ~p Initial ~p", 64 | % [ARes, BRes, CRes, Expected, Initial]), 65 | % assert all nodes have the same final state 66 | Expected == ARes andalso Expected == BRes andalso Expected == CRes. 67 | 68 | non_assoc(Config) -> 69 | run_proper( 70 | fun () -> 71 | Cluster = ?config(ra_cluster, Config), 72 | ?FORALL(N, {list(op()), integer(1, 1000)}, 73 | non_assoc_prop(Cluster, N)) 74 | end, [], 25). 75 | 76 | id(X) -> X. 77 | 78 | non_assoc_apply({add, N}, State) -> State + N; 79 | non_assoc_apply({subtract, N}, State) -> State - N; 80 | non_assoc_apply({divide, N}, State) -> State / N; 81 | non_assoc_apply({mult, N}, State) -> State * N; 82 | non_assoc_apply({set, N}, _) -> N. 83 | 84 | terminate_cluster(Nodes) -> 85 | [gen_statem:stop(P, normal, 2000) || {P, _} <- Nodes]. 86 | 87 | run_proper(Fun, Args, NumTests) -> 88 | ?assertEqual( 89 | true, 90 | proper:counterexample(erlang:apply(Fun, Args), 91 | [{numtests, NumTests}, 92 | {on_output, fun(".", _) -> ok; % don't print the '.'s on new lines 93 | (F, A) -> ct:pal(?LOW_IMPORTANCE, F, A) end}])). 94 | -------------------------------------------------------------------------------- /test/ra_dbg_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_dbg_SUITE). 8 | 9 | -compile(nowarn_export_all). 10 | -compile(export_all). 11 | 12 | -include_lib("eunit/include/eunit.hrl"). 13 | 14 | all() -> 15 | [ 16 | {group, tests} 17 | ]. 18 | 19 | groups() -> 20 | [ 21 | {tests, [], all_tests()} 22 | ]. 23 | 24 | all_tests() -> 25 | [ 26 | replay, 27 | filter_entry_duplicate 28 | ]. 29 | 30 | init_per_suite(Config) -> 31 | Config. 32 | 33 | end_per_suite(_Config) -> 34 | ok. 35 | 36 | init_per_testcase(TestCase, Config) -> 37 | application:load(ra), 38 | Group = proplists:get_value(name, proplists:get_value(tc_group_properties, Config)), 39 | WorkDirectory = proplists:get_value(priv_dir, Config), 40 | ok = application:set_env(ra, data_dir, filename:join(WorkDirectory, atom_to_list(Group) ++ "-" ++ atom_to_list(TestCase))), 41 | Config. 42 | 43 | end_per_testcase(_TestCase, _Config) -> 44 | application:stop(ra), 45 | ok. 46 | 47 | replay(_Config) -> 48 | {Config, FinalState} = execute_state_machine(), 49 | WalFile = wal_file(), 50 | 51 | InitialState = ra_fifo:init(Config), 52 | Pid = spawn(?MODULE, report, [self(), 0]), 53 | %% check final state and replayed state are the same 54 | FinalState = ra_dbg:replay_log(WalFile, ra_fifo, InitialState, 55 | fun (_State, _Effects) -> 56 | Pid ! command_applied 57 | end), 58 | %% make sure the callback function has been called correctly 59 | Count = receive 60 | X -> X 61 | after 10000 -> 62 | timeout 63 | end, 64 | 5 = Count, 65 | ok. 66 | 67 | filter_entry_duplicate(_Config) -> 68 | execute_state_machine(), 69 | WalFile = wal_file(), 70 | 71 | WalInReverseOrder = ra_log_wal:wal2list(WalFile), 72 | Wal = lists:reverse(WalInReverseOrder), 73 | Wal = ra_dbg:filter_duplicate_entries(lists:append(WalInReverseOrder, WalInReverseOrder)), 74 | ok. 75 | 76 | execute_state_machine() -> 77 | %% creating a new WAL file with ra_fifo 78 | [Srv] = Nodes = [{ra_dbg, node()}], 79 | ClusterId = ra_dbg, 80 | Config = #{name => ClusterId, 81 | machine_version => 0}, 82 | Machine = {module, ra_fifo, Config}, 83 | ra:start(), 84 | {ok, _, _} = ra:start_cluster(default, ClusterId, Machine, Nodes), 85 | 86 | {ok, _, _} = ra:process_command(Srv, {enqueue, self(), 1, <<"1">>}), 87 | {ok, _, _} = ra:process_command(Srv, {enqueue, self(), 2, <<"2">>}), 88 | {ok, _, _} = ra:process_command(Srv, {enqueue, self(), 3, <<"3">>}), 89 | 90 | ConsumerId = {<<"ctag1">>, self()}, 91 | {ok, {dequeue, {MsgId, _}}, _} = ra:process_command(Srv, {checkout, {dequeue, unsettled}, ConsumerId}), 92 | 93 | {ok, _, _} = ra:process_command(Srv, {settle, [MsgId], ConsumerId}), 94 | {ok, FinalState, _} = ra:consistent_query(Srv, fun(State) -> State end), 95 | {Config, FinalState}. 96 | 97 | wal_file() -> 98 | {ok, RaDataDir} = application:get_env(ra, data_dir), 99 | filename:join([RaDataDir, node(), "0000000000000001.wal"]). 100 | 101 | report(Pid, Count) -> 102 | receive 103 | _ -> 104 | report(Pid, Count + 1) 105 | after 1000 -> 106 | Pid ! Count 107 | end. 108 | -------------------------------------------------------------------------------- /docs/internals/SNAPSHOTS.md: -------------------------------------------------------------------------------- 1 | # Snapshots 2 | 3 | Ra supports pluggable snapshot implementations by virtue of the `ra_snapshot` 4 | behaviour. The default implementation uses `term_to_binary/2` to write the 5 | snapshot to disk. 6 | 7 | Snapshot transfer between the leader and followers use distributed 8 | erlang and therefore it implements a "chunked transfer" approach where the 9 | snapshot is divided up into fixed size blocks that are transferred one by one 10 | so as to not block the distribution port when snapshot become very large. 11 | 12 | ## The `ra_snapshot` behaviour 13 | 14 | The `ra_snapshot` behaviour has 9 (!) callbacks: 15 | 16 | - `prepare(ra_index(), State :: term()) -> Ref :: term()`: 17 | 18 | This is called when the state machine has emitted a `release_cursor` effect 19 | and Ra has decided it is time to take a snapshot. This is called inside the 20 | Ra process and thus should not block unnecessarily. It can be used to trigger 21 | checkpoints or similar in disk-based state machines. 22 | 23 | 24 | - `write(Location :: file:filename(), meta(), Ref :: term()) -> ok | {error, term()}.`: 25 | 26 | This is called in a separate process and should write the snapshot into the 27 | directory specified by the Location argument. 28 | 29 | - `begin_read(ChunkSizeBytes :: non_neg_integer(), Location :: file:filename()) -> 30 | {ok, Crc :: non_neg_integer(), Meta :: meta(), ReadState :: term()} 31 | | {error, term()}.` 32 | 33 | This is called in a separate process when the leader needs to send a snapshot 34 | to a follower. `begin_read` returns the meta data (index, term and cluster configuration) 35 | as well as a continuation state that will be used to read chunks to be transferred. 36 | This function also returns a checksum to validate data transfer. 37 | 38 | - `read_chunk(ReadState, ChunkSizeBytes :: non_neg_integer(), Location :: file:filename()) -> 39 | {ok, Chunk :: term(), {next, ReadState} | last} | {error, term()}` 40 | 41 | This function reads a chunk of data to be sent. The data is read using ReadState 42 | initially received from `begin_read`. As long as it returns `{next, ReadState}` 43 | it will be called again for the next chunk. When reading the last chunk the 44 | function should return `last` instead. 45 | 46 | - `begin_accept/accept_chunk/complete_accept` 47 | 48 | These callbacks are used to implement the corresponding end of `read/2`. 49 | `begin_accept` will be called by the follower when it first receives an 50 | InstallSnapshotRpc message and should persist the meta data and return the 51 | initial accept state. After this `accept_chunk/2` will be called for each received 52 | chunk except the last which will call `complete_accept/2`. `complete_accept/2` should 53 | validate that the integrity of the snapshot is good before returning. 54 | 55 | - `recover/1`: is called at two different times. Immediately after a follower 56 | has completed a transfer and on init to recover the state of a stored snapshot. 57 | 58 | 59 | - `validate/1` should validate that an on-disk snapshot has no integrity faults. 60 | This is called when a Ra server is recovering after a restart. If this fails, 61 | the server will try to load the next available older snapshot, if available. 62 | 63 | - `read_meta/1` should return the meta data for a snapshot. This includes the 64 | Raft index and term as well as a list of member servers. 65 | 66 | 67 | ## On disk layout 68 | 69 | Snapshots are stored inside a `snapshots` directory inside the Ra server 70 | data directory. Each snapshot is a directory of the format: `Term_Index` in 71 | 64 bit hex encoded and zero padded format. E.g.: 72 | 73 | ``` 74 | <| init/0 | |
| record/3 | |
| clear/1 | |
| lookup_leader/1 | |
| lookup_members/1 | |
| overview/0 |
init() -> ok30 |
31 |
record(ClusterName :: ra:cluster_name(), 36 | Leader :: ra:server_id(), 37 | Members :: [ra:server_id()]) -> 38 | ok39 |
40 |
clear(ClusterName :: ra:cluster_name()) -> ok45 |
46 |
lookup_leader(ClusterName :: ra:cluster_name()) -> 51 | ra:server_id() | undefined52 |
53 |
lookup_members(ClusterName :: ra:cluster_name()) -> 58 | [ra:server_id()] | undefined59 |
60 |
overview() -> list()65 |
66 |
Generated by EDoc
71 | 72 | 73 | -------------------------------------------------------------------------------- /src/ra_flru.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_flru). 9 | 10 | %% small fixed size simple lru cache 11 | %% inefficient on larger sizes 12 | 13 | -export([ 14 | new/2, 15 | fetch/2, 16 | insert/3, 17 | evict/2, 18 | evict_all/1, 19 | size/1 20 | ]). 21 | 22 | -define(MAX_SIZE, 5). 23 | 24 | -type kv_item() :: {Key :: term(), Value :: term()}. 25 | 26 | -type handler_fun() :: fun((kv_item()) -> ok). 27 | 28 | -record(?MODULE, {max_size = ?MAX_SIZE :: non_neg_integer(), 29 | items = [] :: [term()], 30 | handler = fun (_) -> ok end :: handler_fun()}). 31 | 32 | -opaque state() :: #?MODULE{}. 33 | 34 | -export_type([ 35 | state/0 36 | ]). 37 | 38 | -spec new(non_neg_integer(), handler_fun()) -> 39 | state(). 40 | new(MaxSize, Handler) -> 41 | #?MODULE{handler = Handler, 42 | max_size = MaxSize}. 43 | 44 | -spec fetch(term(), state()) -> 45 | {ok, term(), state()} | error. 46 | fetch(Key, #?MODULE{items = [{Key, Value} | _]} = State) -> 47 | %% head optimisation 48 | {ok, Value, State}; 49 | fetch(Key, #?MODULE{items = Items0} = State0) -> 50 | case lists:keytake(Key, 1, Items0) of 51 | {value, {_, Value} = T, Items} -> 52 | {ok, Value, State0#?MODULE{items = [T | Items]}}; 53 | false -> 54 | error 55 | end. 56 | 57 | -spec insert(term(), term(), state()) -> state(). 58 | insert(Key, Value, #?MODULE{items = Items, 59 | max_size = M, 60 | handler = Handler} = State) 61 | when length(Items) =:= M -> 62 | %% cache is full, discard last item 63 | [Old | Rem] = lists:reverse(Items), 64 | %% call the handler 65 | ok = Handler(Old), 66 | State#?MODULE{items = [{Key, Value} | lists:reverse(Rem)]}; 67 | insert(Key, Value, #?MODULE{items = Items} = State) -> 68 | %% else just append it 69 | State#?MODULE{items = [{Key, Value} | Items]}. 70 | 71 | -spec evict(Key :: term(), state()) -> 72 | {Evicted :: kv_item(), state()} | error. 73 | evict(Key, #?MODULE{items = Items0, 74 | handler = Handler} = State) -> 75 | case lists:keytake(Key, 1, Items0) of 76 | {value, T, Items} -> 77 | ok = Handler(T), 78 | {T, State#?MODULE{items = Items}}; 79 | false -> 80 | error 81 | end. 82 | 83 | 84 | -spec evict_all(state()) -> state(). 85 | evict_all(#?MODULE{items = Items, 86 | handler = Handler}) -> 87 | [Handler(T) || T <- Items], 88 | #?MODULE{items = []}. 89 | 90 | -spec size(state()) -> non_neg_integer(). 91 | size(#?MODULE{items = Items}) -> 92 | length(Items). 93 | 94 | -ifdef(TEST). 95 | -include_lib("eunit/include/eunit.hrl"). 96 | 97 | evit_test() -> 98 | C0 = new(3, fun(I) -> ?debugFmt("~w evicted", [I]) end), 99 | C1 = insert(k1, v1, C0), 100 | C2 = insert(k2, v2, C1), 101 | {{k1, v1}, C3} = evict(k1, C2), 102 | error = evict(k3, C3), 103 | ok. 104 | 105 | 106 | basics_test() -> 107 | C0 = new(3, fun(I) -> ?debugFmt("~w evicted", [I]) end), 108 | C1 = insert(k1, v1, C0), 109 | C2 = insert(k2, v2, C1), 110 | C3 = insert(k3, v3, C2), 111 | C4 = insert(k4, v4, C3), 112 | ?assertEqual(error, fetch(k1, C4)), 113 | {ok, v2, C5} = fetch(k2, C4), 114 | C6 = insert(k5, v5, C5), 115 | %% k2 should still be readable here 116 | {ok, v2, C7} = fetch(k2, C6), 117 | %% k3 should have been evicted 118 | error = fetch(k3, C7), 119 | 120 | 121 | ok. 122 | 123 | 124 | -endif. 125 | -------------------------------------------------------------------------------- /test/ra_directory_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_directory_SUITE). 8 | 9 | -compile(nowarn_export_all). 10 | -compile(export_all). 11 | 12 | -export([ 13 | ]). 14 | 15 | -include_lib("common_test/include/ct.hrl"). 16 | -include_lib("eunit/include/eunit.hrl"). 17 | 18 | %%%=================================================================== 19 | %%% Common Test callbacks 20 | %%%=================================================================== 21 | 22 | -define(SYS, default). 23 | 24 | all() -> 25 | [ 26 | {group, tests} 27 | ]. 28 | 29 | 30 | all_tests() -> 31 | [ 32 | basics, 33 | persistence 34 | ]. 35 | 36 | groups() -> 37 | [ 38 | {tests, [], all_tests()} 39 | ]. 40 | 41 | init_per_suite(Config) -> 42 | Dir = ?config(priv_dir, Config), 43 | Cfg = ra_system:default_config(), 44 | ra_system:store(Cfg#{data_dir => Dir}), 45 | Config. 46 | 47 | end_per_suite(_Config) -> 48 | ok. 49 | 50 | init_per_group(_Group, Config) -> 51 | Config. 52 | 53 | end_per_group(_Group, _Config) -> 54 | ok. 55 | 56 | init_per_testcase(_TestCase, Config) -> 57 | Config. 58 | 59 | end_per_testcase(_TestCase, _Config) -> 60 | ok. 61 | 62 | %%%=================================================================== 63 | %%% Test cases 64 | %%%=================================================================== 65 | 66 | basics(_Config) -> 67 | ok = ra_directory:init(?SYS), 68 | UId = atom_to_binary(?FUNCTION_NAME), 69 | Self = self(), 70 | ok = ra_directory:register_name(?SYS, UId, Self, undefined, 71 | test1, <<"test_cluster_name">>), 72 | % registrations should always succeed - no negative test 73 | Self = ra_directory:where_is(?SYS, UId), 74 | UId = ra_directory:uid_of(?SYS, test1), 75 | ?assert(ra_directory:is_registered_uid(?SYS, UId)), 76 | % ensure it can be read from another process 77 | _ = spawn_link( 78 | fun () -> 79 | UId = ra_directory:uid_of(?SYS, test1), 80 | Self ! done 81 | end), 82 | receive done -> ok after 500 -> exit(timeout) end, 83 | test1 = ra_directory:name_of(?SYS, UId), 84 | <<"test_cluster_name">> = ra_directory:cluster_name_of(?SYS, UId), 85 | UId = ra_directory:unregister_name(?SYS, UId), 86 | undefined = ra_directory:where_is(?SYS, UId), 87 | undefined = ra_directory:name_of(?SYS, UId), 88 | undefined = ra_directory:cluster_name_of(?SYS, UId), 89 | undefined = ra_directory:uid_of(?SYS, test1), 90 | ?assertNot(ra_directory:is_registered_uid(?SYS, UId)), 91 | ok. 92 | 93 | persistence(_Config) -> 94 | ok = ra_directory:init(?SYS), 95 | UId = atom_to_binary(?FUNCTION_NAME), 96 | UId2 = <name() = term()20 | 21 | 22 |
| init/0 | |
| new/2 | |
| fetch/1 | |
| delete/1 | |
| overview/0 | |
| overview/1 | |
| counters/2 |
init() -> ok37 |
38 |
new(Name :: name(), FieldsSpec :: seshat:fields_spec()) -> 43 | counters:counters_ref()44 |
45 |
fetch(Name :: name()) -> undefined | counters:counters_ref()50 |
51 |
delete(Name :: term()) -> ok56 |
57 |
overview() -> #{name() => #{atom() => non_neg_integer()}}
62 | 63 |
overview(Name :: name()) -> #{atom() => non_neg_integer()}68 |
69 |
counters(Name :: name(), Fields :: [atom()]) -> 74 | #{atom() => non_neg_integer()} | undefined75 |
76 |
Generated by EDoc
81 | 82 | 83 | -------------------------------------------------------------------------------- /src/ra_log_pre_init.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% @hidden 7 | -module(ra_log_pre_init). 8 | 9 | -behaviour(gen_server). 10 | 11 | -include("ra.hrl"). 12 | %% API functions 13 | -export([start_link/1]). 14 | 15 | %% gen_server callbacks 16 | -export([init/1, 17 | handle_call/3, 18 | handle_cast/2, 19 | handle_info/2, 20 | terminate/2, 21 | code_change/3]). 22 | 23 | -record(state, {}). 24 | 25 | -define(ETSTBL, ra_log_snapshot_state). 26 | %%%=================================================================== 27 | %%% API functions 28 | %%%=================================================================== 29 | 30 | start_link(System) when is_atom(System) -> 31 | gen_server:start_link(?MODULE, [System], []). 32 | 33 | %%%=================================================================== 34 | %%% gen_server callbacks 35 | %%%=================================================================== 36 | 37 | init([System]) -> 38 | %% ra_log:pre_init ensures that the ra_log_snapshot_state table is 39 | %% populated before WAL recovery begins to avoid writing unnecessary 40 | %% indexes to segment files. 41 | Regd = ra_directory:list_registered(System), 42 | ?INFO("ra system '~ts' running pre init for ~b registered servers", 43 | [System, length(Regd)]), 44 | _ = [begin 45 | try pre_init(System, UId) of 46 | ok -> ok 47 | catch _:Err -> 48 | ?ERROR("pre_init failed in system ~s for UId ~ts with name ~ts" 49 | " This error may need manual intervention, Error ~p", 50 | [System, UId, Name, Err]), 51 | ok 52 | end 53 | end|| {Name, UId} <- Regd], 54 | {ok, #state{} , hibernate}. 55 | 56 | handle_call(_Request, _From, State) -> 57 | Reply = ok, 58 | {reply, Reply, State}. 59 | 60 | handle_cast(_Msg, State) -> 61 | {noreply, State}. 62 | 63 | handle_info(_Info, State) -> 64 | {noreply, State}. 65 | 66 | terminate(_Reason, _State) -> 67 | ok. 68 | 69 | code_change(_OldVsn, State, _Extra) -> 70 | {ok, State}. 71 | 72 | %%%=================================================================== 73 | %%% Internal functions 74 | %%%=================================================================== 75 | 76 | pre_init(System, UId) -> 77 | case ets:lookup(?ETSTBL, UId) of 78 | [{_, _}] -> 79 | %% already initialised 80 | ok; 81 | [] -> 82 | case ra_system:fetch(System) of 83 | undefined -> 84 | {error, system_not_started}; 85 | SysCfg -> 86 | %% check if the server dir exists, if not 87 | %% then just log and return instead of failing. 88 | Dir = ra_env:server_data_dir(System, UId), 89 | case ra_lib:is_dir(Dir) of 90 | true -> 91 | case ra_log:read_config(Dir) of 92 | {ok, #{log_init_args := Log}} -> 93 | ok = ra_log:pre_init(Log#{system_config => SysCfg}), 94 | ok; 95 | {error, Err} -> 96 | ?ERROR("pre_init failed to read config file for UId '~ts', Err ~p", 97 | [UId, Err]), 98 | ok 99 | end; 100 | false -> 101 | ?INFO("pre_init UId '~ts' is registered but no data 102 | directory was found, removing from ra directory", 103 | [UId]), 104 | _ = catch ra_directory:unregister_name(System, UId), 105 | ok 106 | end 107 | end 108 | end. 109 | 110 | -------------------------------------------------------------------------------- /test/enqueuer.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(enqueuer). 8 | 9 | -behaviour(gen_server). 10 | 11 | %% API functions 12 | -export([start_link/1, 13 | start_link/2, 14 | wait/2]). 15 | 16 | %% gen_server callbacks 17 | -export([init/1, 18 | handle_call/3, 19 | handle_cast/2, 20 | handle_info/2, 21 | terminate/2, 22 | code_change/3]). 23 | 24 | -include("src/ra.hrl"). 25 | 26 | -type config() :: #{cluster_name := ra_cluster_name(), 27 | servers := [ra_server_id()], 28 | num_messages := non_neg_integer(), 29 | spec := {Interval :: non_neg_integer(), Tag :: atom()} 30 | }. 31 | 32 | -record(state, {state :: ra_fifo_client:state(), 33 | next = 1 :: pos_integer(), 34 | max = 10 :: non_neg_integer(), 35 | tag :: atom(), 36 | applied = [] :: [non_neg_integer()], 37 | interval :: non_neg_integer(), 38 | waiting :: term()}). 39 | 40 | 41 | %%%=================================================================== 42 | %%% API functions 43 | %%%=================================================================== 44 | 45 | -spec start_link(config()) -> {ok, pid()} | ignore | {error, term()}. 46 | start_link(Config) -> 47 | gen_server:start_link({local, ?MODULE}, ?MODULE, [Config], []). 48 | 49 | start_link(Name, Config) -> 50 | gen_server:start_link({local, Name}, ?MODULE, [Config], []). 51 | 52 | wait(Pid, Timeout) -> 53 | gen_server:call(Pid, wait, Timeout). 54 | 55 | %%%=================================================================== 56 | %%% gen_server callbacks 57 | %%%=================================================================== 58 | 59 | init([#{spec := {Interval, Tag}, 60 | num_messages := Max, 61 | cluster_name := ClusterName, 62 | servers := Servers}]) -> 63 | erlang:send_after(Interval, self(), enqueue), 64 | F = ra_fifo_client:init(ClusterName, Servers), 65 | {ok, #state{state = F, 66 | max = Max, 67 | interval = Interval, 68 | tag = Tag}}. 69 | 70 | handle_call(wait, _From, #state{next = N, max = N, 71 | applied = Appd, 72 | state = F} = State) -> 73 | {reply, {applied, Appd, F}, State}; 74 | handle_call(wait, From, State) -> 75 | {noreply, State#state{waiting = From}}. 76 | 77 | handle_cast(Msg, State) -> 78 | ct:pal("enqeuer unhandled cast ~w", [Msg]), 79 | {noreply, State}. 80 | 81 | handle_info(enqueue, #state{tag = Tag, next = Next0, 82 | max = Max, 83 | interval = Interval, 84 | state = F0} = State0) -> 85 | Msg = {Tag, Next0}, 86 | Next = Next0 + 1, 87 | case ra_fifo_client:enqueue(Next0, Msg, F0) of 88 | {T, F} when T =/= error -> 89 | case Max of 90 | Next0 -> 91 | State = State0#state{state = F}, 92 | erlang:send_after(Interval, self(), reply), 93 | {noreply, State}; 94 | _ -> 95 | State = State0#state{next = Next, state = F}, 96 | erlang:send_after(Interval, self(), enqueue), 97 | {noreply, State} 98 | end; 99 | Err -> 100 | ?WARN("Enqueuer: error enqueue ~W", [Err, 5]), 101 | erlang:send_after(10, self(), enqueue), 102 | {noreply, State0} 103 | end; 104 | handle_info({ra_event, From, Evt}, 105 | #state{state = F0, applied = Appd} = State0) -> 106 | {internal, Applied, F} = ra_fifo_client:handle_ra_event(From, Evt, F0), 107 | {noreply, State0#state{state = F, applied = Appd ++ Applied}}; 108 | handle_info(reply, #state{waiting = undefined} = State0) -> 109 | {noreply, State0}; 110 | handle_info(reply, #state{state = F, waiting = From, 111 | applied = Applied} = State0) -> 112 | gen_server:reply(From, {applied, Applied, F}), 113 | {noreply, State0}. 114 | 115 | 116 | 117 | 118 | terminate(_Reason, _State) -> 119 | ok. 120 | 121 | code_change(_OldVsn, State, _Extra) -> 122 | {ok, State}. 123 | 124 | %%%=================================================================== 125 | %%% Internal functions 126 | %%%=================================================================== 127 | -------------------------------------------------------------------------------- /test/nemesis.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(nemesis). 8 | 9 | -behaviour(gen_server). 10 | 11 | %% API functions 12 | -export([start_link/1, 13 | wait_on_scenario/2]). 14 | 15 | %% gen_server callbacks 16 | -export([init/1, 17 | handle_call/3, 18 | handle_cast/2, 19 | handle_info/2, 20 | terminate/2, 21 | code_change/3]). 22 | 23 | -export([partition/1]). 24 | 25 | -include("src/ra.hrl"). 26 | 27 | -define(SYS, default). 28 | 29 | -type scenario() :: [{wait, non_neg_integer()} | 30 | {part, [node()], non_neg_integer()} | 31 | {app_restart, [ra_server_id()]} | 32 | heal]. 33 | 34 | -type config() :: #{nodes := [node()], 35 | scenario := scenario()}. 36 | 37 | -record(state, {config :: config(), 38 | nodes :: [node()], 39 | steps :: scenario(), 40 | waiting :: term()}). 41 | 42 | 43 | %%%=================================================================== 44 | %%% API functions 45 | %%%=================================================================== 46 | 47 | -spec start_link(config()) -> {ok, pid()} | ignore | {error, term()}. 48 | start_link(Config) -> 49 | gen_server:start_link({local, ?MODULE}, ?MODULE, [Config], []). 50 | 51 | wait_on_scenario(Pid, Timeout) -> 52 | gen_server:call(Pid, wait_on_scenario, Timeout). 53 | 54 | %%%=================================================================== 55 | %%% gen_server callbacks 56 | %%%=================================================================== 57 | 58 | init([#{scenario := Steps, 59 | nodes := Nodes} = Config]) -> 60 | State = handle_step(#state{config = Config, 61 | nodes = Nodes, 62 | steps = Steps}), 63 | {ok, State}. 64 | 65 | handle_call(wait_on_scenario, _From, #state{steps = []} = State) -> 66 | {reply, ok, State}; 67 | handle_call(wait_on_scenario, From, State) -> 68 | {noreply, State#state{waiting = From}}. 69 | 70 | handle_cast(_Msg, State) -> 71 | {noreply, State}. 72 | 73 | handle_info(next_step, State0) -> 74 | case handle_step(State0) of 75 | done -> 76 | case State0#state.waiting of 77 | undefined -> 78 | {noreply, State0}; 79 | From -> 80 | gen_server:reply(From, ok), 81 | {noreply, State0} 82 | end; 83 | State -> 84 | {noreply, State} 85 | end. 86 | 87 | 88 | terminate(_Reason, _State) -> 89 | ok. 90 | 91 | code_change(_OldVsn, State, _Extra) -> 92 | {ok, State}. 93 | 94 | partition(Partitions) -> 95 | partition(Partitions, fun tcp_inet_proxy_helpers:block_traffic_between/2). 96 | 97 | %%%=================================================================== 98 | %%% Internal functions 99 | %%%=================================================================== 100 | 101 | handle_step(#state{steps = [{wait, Time} | Rem]} = State) -> 102 | erlang:send_after(Time, self(), next_step), 103 | State#state{steps = Rem}; 104 | handle_step(#state{steps = [{part, Partition0, Time} | Rem], 105 | nodes = Nodes} = State) -> 106 | %% first we need to always heal 107 | heal(State#state.nodes), 108 | Partitions = {lists:usort(Partition0), Nodes -- Partition0}, 109 | _ = erlang:send_after(Time, self(), next_step), 110 | ok = partition(Partitions), 111 | % always heal after part 112 | State#state{steps = [heal | Rem]}; 113 | handle_step(#state{steps = [heal | Rem]} = State) -> 114 | heal(State#state.nodes), 115 | handle_step(State#state{steps = Rem}); 116 | handle_step(#state{steps = [{app_restart, Servers} | Rem]} = State) -> 117 | ct:pal("doing app restart of ~w", [Servers]), 118 | [begin 119 | rpc:call(N, application, stop, [ra]), 120 | rpc:call(N, ra, start, []), 121 | rpc:call(N, ra, restart_server, [?SYS, Id]) 122 | end || {_, N} = Id <- Servers], 123 | handle_step(State#state{steps = Rem}); 124 | handle_step(#state{steps = []}) -> 125 | done. 126 | 127 | 128 | 129 | partition({Partition1, Partition2}, PartitionFun) -> 130 | lists:foreach( 131 | fun(Node) -> 132 | [PartitionFun(Node, OtherNode) || OtherNode <- Partition2] 133 | end, 134 | Partition1), 135 | ok. 136 | 137 | heal(Nodes) -> 138 | ct:pal("Rejoining all nodes"), 139 | [tcp_inet_proxy_helpers:allow_traffic_between(Node, OtherNode) 140 | || OtherNode <- Nodes, 141 | Node <- Nodes, 142 | OtherNode =/= Node], 143 | [net_kernel:connect_node(N) 144 | || N <- Nodes]. 145 | -------------------------------------------------------------------------------- /test/ra_fifo_index.erl: -------------------------------------------------------------------------------- 1 | -module(ra_fifo_index). 2 | 3 | -export([ 4 | empty/0, 5 | fetch/2, 6 | append/3, 7 | return/3, 8 | delete/2, 9 | size/1, 10 | smallest/1, 11 | next_key_after/2, 12 | map/2 13 | ]). 14 | 15 | -include("src/ra.hrl"). 16 | 17 | -compile({no_auto_import, [size/1]}). 18 | 19 | -record(state, {data = #{} :: #{integer() => term()}, 20 | smallest :: undefined | non_neg_integer(), 21 | largest :: undefined | non_neg_integer() 22 | }). 23 | 24 | -opaque state() :: #state{}. 25 | 26 | -export_type([state/0]). 27 | 28 | -spec empty() -> state(). 29 | empty() -> 30 | #state{}. 31 | 32 | -spec fetch(integer(), state()) -> undefined | term(). 33 | fetch(Key, #state{data = Data}) -> 34 | maps:get(Key, Data, undefined). 35 | 36 | % only integer keys are supported 37 | -spec append(integer(), term(), state()) -> state(). 38 | append(Key, Value, 39 | #state{data = Data, 40 | smallest = Smallest, 41 | largest = Largest} = State) 42 | when Key > Largest orelse Largest =:= undefined -> 43 | State#state{data = maps:put(Key, Value, Data), 44 | smallest = ra_lib:default(Smallest, Key), 45 | largest = Key}. 46 | 47 | -spec return(integer(), term(), state()) -> state(). 48 | return(Key, Value, #state{data = Data, smallest = Smallest} = State) 49 | when is_integer(Key) andalso Key < Smallest -> 50 | % TODO: this could potentially result in very large gaps which would 51 | % result in poor performance of smallest/1 52 | % We could try to persist a linked list of "smallests" to make it quicker 53 | % to skip from one to the other - needs measurement 54 | State#state{data = maps:put(Key, Value, Data), 55 | smallest = Key}; 56 | return(Key, Value, #state{data = Data} = State) 57 | when is_integer(Key) -> 58 | State#state{data = maps:put(Key, Value, Data)}. 59 | 60 | -spec delete(integer(), state()) -> state(). 61 | delete(Smallest, #state{data = Data0, 62 | largest = Largest, 63 | smallest = Smallest} = State) -> 64 | Data = maps:remove(Smallest, Data0), 65 | case find_next(Smallest + 1, Largest, Data) of 66 | undefined -> 67 | State#state{data = Data, 68 | smallest = undefined, 69 | largest = undefined}; 70 | Next -> 71 | State#state{data = Data, smallest = Next} 72 | end; 73 | delete(Key, #state{data = Data} = State) -> 74 | State#state{data = maps:remove(Key, Data)}. 75 | 76 | -spec size(state()) -> non_neg_integer(). 77 | size(#state{data = Data}) -> 78 | maps:size(Data). 79 | 80 | -spec smallest(state()) -> undefined | {integer(), term()}. 81 | smallest(#state{smallest = undefined}) -> 82 | undefined; 83 | smallest(#state{smallest = Smallest, data = Data}) -> 84 | {Smallest, maps:get(Smallest, Data)}. 85 | 86 | 87 | -spec next_key_after(non_neg_integer(), state()) -> undefined | integer(). 88 | next_key_after(_Idx, #state{smallest = undefined}) -> 89 | % map must be empty 90 | undefined; 91 | next_key_after(Idx, #state{smallest = Smallest, 92 | largest = Largest}) 93 | when Idx+1 < Smallest orelse Idx+1 > Largest -> 94 | undefined; 95 | next_key_after(Idx, #state{data = Data} = State) -> 96 | Next = Idx+1, 97 | case maps:is_key(Next, Data) of 98 | true -> 99 | Next; 100 | false -> 101 | next_key_after(Next, State) 102 | end. 103 | 104 | -spec map(fun(), state()) -> state(). 105 | map(F, #state{data = Data} = State) -> 106 | State#state{data = maps:map(F, Data)}. 107 | 108 | 109 | %% internal 110 | 111 | find_next(Next, Last, _Map) when Next > Last -> 112 | undefined; 113 | find_next(Next, Last, Map) -> 114 | case Map of 115 | #{Next := _} -> 116 | Next; 117 | _ -> 118 | % in degenerate cases the range here could be very large 119 | % and hence this could be very slow 120 | % the typical case should idealy be better 121 | % assuming fifo-ish deletion of entries 122 | find_next(Next+1, Last, Map) 123 | end. 124 | 125 | -ifdef(TEST). 126 | -include_lib("eunit/include/eunit.hrl"). 127 | 128 | append_test() -> 129 | S0 = empty(), 130 | undefined = fetch(99, S0), 131 | undefined = smallest(S0), 132 | 0 = size(S0), 133 | S1 = append(1, one, S0), 134 | undefined = fetch(99, S1), 135 | one = fetch(1, S1), 136 | 1 = size(S1), 137 | {1, one} = smallest(S1), 138 | S2 = append(2, two, S1), 139 | two = fetch(2, S2), 140 | 2 = size(S2), 141 | {1, one} = smallest(S2), 142 | S3 = delete(1, S2), 143 | {2, two} = smallest(S3), 144 | 1 = size(S3), 145 | S4 = return(1, one, S3), 146 | one = fetch(1, S4), 147 | 2 = size(S4), 148 | {1, one} = smallest(S4), 149 | S5 = delete(2, delete(1, S4)), 150 | undefined = smallest(S5), 151 | 0 = size(S0), 152 | ok. 153 | 154 | next_after_test() -> 155 | S = append(3, three, 156 | append(2, two, 157 | append(1, one, 158 | empty()))), 159 | 1 = next_key_after(0, S), 160 | 2 = next_key_after(1, S), 161 | 3 = next_key_after(2, S), 162 | undefined = next_key_after(3, S), 163 | undefined = next_key_after(4, S), 164 | ok. 165 | 166 | -endif. 167 | -------------------------------------------------------------------------------- /docs/internals/LOG_V2.md: -------------------------------------------------------------------------------- 1 | # Ra log v2 internals 2 | 3 | ```mermaid 4 | sequenceDiagram 5 | participant ra-server-n 6 | participant wal 7 | participant segment-writer 8 | 9 | loop until wal full 10 | ra-server-n-)+wal: write(Index=1..N, Term=T) 11 | wal->>wal: write-batch([1,2,3]) 12 | wal-)-ra-server-n: written event: Term=T, Range=(1, 3) 13 | end 14 | wal-)+segment-writer: flush-wal-ranges 15 | segment-writer-->segment-writer: flush to segment files 16 | segment-writer-)ra-server-n: notify flushed segments 17 | ra-server-n-->ra-server-n: update mem-table-ranges 18 | ra-server-n-)ets-server: delete range from mem-table 19 | ``` 20 | 21 | In the Ra log v2 implementation some work previously done by the `ra_log_wal` 22 | process has now been either factored out or moved elsewhere. 23 | 24 | In Ra log v1 the WAL process would be responsible for both writing to disk and 25 | to memtables (ETS). Each writer (identified by a locally scoped binary "UId") would 26 | have a unique ETS table to cover the lifetime of each WAL file. Once the WAL breaches 27 | its' configured `max_wal_size_bytes` limit it closes the file and hands it over to 28 | the segment writer to flush any still live entries to per-server segments. 29 | The segment writer reads each entry from the memtables, not the WAL file. 30 | When all still live entries in the WAL have been flushed to segments the segment 31 | writer deletes the WAL file and notifies all relevant ra servers of the new 32 | segments. Once each ra server receives this notifications and updates their 33 | "seg-refs" they delete the whole memtable. 34 | 35 | In the v2 implementation the WAL no longer writes to memtables during normal 36 | operation (exception being the recovery phase). Instead the memtables are 37 | written to by the Ra servers before the write request is sent to the WAL. 38 | The removes the need for a separate ETS table per Ra server "cache" which was 39 | required in the v1 implementation. 40 | 41 | In v2 memtables aren't deleted after segment flush. Instead they are kept until 42 | a Ra server needs to overwrite some entries. This cannot be allowed due to the 43 | async nature of the log implementation. E.g. the segment writer could be reading 44 | from the memtables and if an index is overwritten it may generate an inconsistent 45 | end result. Overwrites are typically only needed when a leader has been replaced 46 | and have some written but uncommitted entries that another leader in a higher 47 | term has overwritten. 48 | 49 | 50 | ## In-Memory Tables (memtables) 51 | 52 | Module: `ra_mt` 53 | 54 | Mem tables are owned and created by the `ra_log_ets` process. Ra servers call 55 | into the process to create new memtables and a registry of current tables is 56 | kept in the `ra_log_open_memtables` table. From v2 the `ra_log_closed_memtables` 57 | ETS table is no longer used or created. 58 | 59 | Invariant: Entries can be written or deleted but never overwritten. 60 | 61 | During normal operation each Ra server only writes to a single ETS memtable. 62 | Entries that are no longer required to be kept in the memtable due to snapshotting 63 | or having been written to disk segments are deleted. The actual delete operation 64 | is performed by `ra_log_ets` on request by Ra servers. 65 | 66 | Memtables are no longer linked to the lifetime of a given WAL file as before. 67 | Apart from recovery after a system restart only the Ra servers write to 68 | memtables which reduces the workload of the WAL process. 69 | 70 | New memtables are only created when a server needs to overwrite indexes in its 71 | log. This typically only happens when a leader has been replaced and steps down 72 | to follower with uncommitted entries in its log. Due to the async nature of the 73 | Ra log implementation it is not safe to ever overwrite an entry in a memtable 74 | (as concurrent reads may be done by the segment writer process). Therefore a new 75 | memtable needs to be created when this situation occurs. 76 | 77 | When a new memtable is created the old ones will not be written to any further 78 | and will be deleted as soon as they are emptied. 79 | 80 | There is one additional condition at which a new memtable will be created and 81 | that is the case where the size of memtable exceed 1M entries. This is done 82 | to avoid very large tables which can still use substantial amounts of memory even 83 | after all entries have been individually deleted. 84 | 85 | ## WAL 86 | 87 | Module: `ra_log_wal` 88 | 89 | The `ra_log_wal` process now has the following responsibilities: 90 | 91 | * Write entries to disk and notify the writer processes when their entries 92 | have been synced to the underlying storage. 93 | * Track the ranges written by each writer (ra server) for which ETS table and 94 | notifies the segment writer when a WAL file has filled up. 95 | * Recover memtables from WAL files after a system restart. 96 | 97 | ## Segment Writer 98 | 99 | Module: `ra_log_segment_writer` 100 | 101 | The segment writer's responsibilities remain much as before. 102 | When a WAL file reaches it's max size limit the WAL will send the segment writer 103 | a map of `#{ra_uid() => [{ets:tid(), ra_range()}]}` describing the "tid ranges" 104 | that need to be written to disk for each `ra_uid()` (i.e. a Ra server). 105 | 106 | The range that is actually written can be dynamically truncated if the Ra server 107 | writes a snapshot before or during the segment flush. E.g. if the segment writer 108 | is asked to flush the range `{1000, 2000}` and the Ra server writes a snapshot 109 | at index 1500 the segment writer will update the range to `{1501, 2000}` to avoid flushing 110 | redundant entries to disk. 111 | 112 | The latest snapshot index for each Ra server is kept in the `ra_log_snapshot_state` 113 | ETS table. 114 | 115 | -------------------------------------------------------------------------------- /docs/ra_aux.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |abstract datatype: internal_state()
| machine_state/1 | |
| leader_id/1 | |
| last_applied/1 | |
| members_info/1 | |
| overview/1 | |
| log_last_index_term/1 | |
| log_fetch/2 | |
| log_stats/1 |
machine_state(State :: ra_aux:internal_state()) -> term()38 |
39 |
leader_id(State :: ra_aux:internal_state()) -> 44 | undefined | ra_server_id()45 |
46 |
last_applied(State :: ra_aux:internal_state()) -> ra_index()51 |
52 |
members_info(State :: ra_aux:internal_state()) -> ra_cluster()57 |
58 |
overview(State :: ra_aux:internal_state()) -> map()63 |
64 |
log_last_index_term(X1 :: ra_aux:internal_state()) -> ra_idxterm()69 |
70 |
log_fetch(Idx :: ra_index(), State :: ra_aux:internal_state()) -> 75 | {undefined | 76 | {ra_term(), 77 | CmdMetadata :: ra_server:command_meta(), 78 | Command :: term()}, 79 | ra_aux:internal_state()}80 |
81 |
log_stats(X1 :: ra_aux:internal_state()) -> ra_log:overview()86 |
87 |
Generated by EDoc
92 | 93 | 94 | -------------------------------------------------------------------------------- /src/ra_monitors.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% @hidden 7 | -module(ra_monitors). 8 | 9 | -include("ra.hrl"). 10 | -export([ 11 | init/0, 12 | add/3, 13 | remove/3, 14 | remove_all/2, 15 | handle_down/2, 16 | components/2 17 | ]). 18 | 19 | %% holds static or rarely changing fields 20 | 21 | -type component() :: machine | aux | snapshot_sender. 22 | 23 | -opaque state() :: #{pid() => {reference(), #{component() => ok}}, 24 | node() => #{component() => ok}}. 25 | 26 | -export_type([ 27 | state/0 28 | ]). 29 | 30 | -spec init() -> state(). 31 | init() -> 32 | #{}. 33 | 34 | -spec add(pid() | node(), component(), state()) -> state(). 35 | add(Pid, Component, Monitors) when is_pid(Pid) -> 36 | case Monitors of 37 | #{Pid := {MRef, Components}} -> 38 | Monitors#{Pid => {MRef, Components#{Component => ok}}}; 39 | _ -> 40 | MRef = erlang:monitor(process, Pid), 41 | Monitors#{Pid => {MRef, #{Component => ok}}} 42 | end; 43 | add(Node, Component, Monitors) when is_atom(Node) -> 44 | case Monitors of 45 | #{Node := Components} -> 46 | case maps:is_key(Component, Components) of 47 | false -> 48 | emit_current_node_state(Node); 49 | true -> 50 | ok 51 | end, 52 | Monitors#{Node => Components#{Component => ok}}; 53 | _ -> 54 | emit_current_node_state(Node), 55 | Monitors#{Node => #{Component => ok}} 56 | end. 57 | 58 | emit_current_node_state(Node) -> 59 | Nodes = [node() | nodes()], 60 | %% fake event for newly registered component 61 | %% so that it discovers the current node state 62 | case lists:member(Node, Nodes) of 63 | true -> 64 | self() ! {nodeup, Node, []}, 65 | ok; 66 | false -> 67 | self() ! {nodedown, Node, []}, 68 | ok 69 | end. 70 | 71 | 72 | -spec remove(pid() | node(), component(), state()) -> state(). 73 | remove(Target, Component, Monitors) -> 74 | case Monitors of 75 | #{Target := {MRef, #{Component := _} = Components0}} -> 76 | case maps:remove(Component, Components0) of 77 | Components when map_size(Components) == 0 -> 78 | %% if pid, demonitor using ref 79 | true = erlang:demonitor(MRef), 80 | maps:remove(Target, Monitors); 81 | Components -> 82 | Monitors#{Target => Components} 83 | end; 84 | #{Target := #{Component := _} = Components0} -> 85 | case maps:remove(Component, Components0) of 86 | Components when map_size(Components) == 0 -> 87 | maps:remove(Target, Monitors); 88 | Components -> 89 | Monitors#{Target => Components} 90 | end; 91 | _ -> 92 | Monitors 93 | end. 94 | 95 | -spec remove_all(component(), state()) -> state(). 96 | remove_all(Component, Monitors) -> 97 | lists:foldl(fun(T, Acc) -> 98 | remove(T, Component, Acc) 99 | end, Monitors, maps:keys(Monitors)). 100 | 101 | -spec handle_down(pid() | node(), state()) -> 102 | {[component()], state()}. 103 | handle_down(Target, Monitors0) 104 | when is_pid(Target) orelse is_atom(Target) -> 105 | case maps:take(Target, Monitors0) of 106 | {{_MRef, CompsMap}, Monitors} -> 107 | {maps:keys(CompsMap), Monitors}; 108 | {CompsMap, _Monitors} when is_map(CompsMap) -> 109 | %% nodes aren't removed when down 110 | {maps:keys(CompsMap), Monitors0}; 111 | error -> 112 | {[], Monitors0} 113 | end; 114 | handle_down(Target, Monitors0) -> 115 | ?DEBUG("ra_monitors: target ~w not recognised", [Target]), 116 | {[], Monitors0}. 117 | 118 | 119 | -spec components(pid() | node(), state()) -> [component()]. 120 | components(Target, Monitors) -> 121 | case maps:get(Target, Monitors, error) of 122 | {_MRef, CompsMap} -> 123 | maps:keys(CompsMap); 124 | CompsMap when is_map(CompsMap) -> 125 | maps:keys(CompsMap); 126 | error -> 127 | [] 128 | end. 129 | 130 | -ifdef(TEST). 131 | -include_lib("eunit/include/eunit.hrl"). 132 | 133 | basics_test() -> 134 | M0 = init(), 135 | M1 = add(self(), machine, M0), 136 | [machine] = components(self(), M1), 137 | M2 = add(self(), aux, M1), 138 | [aux, machine] = lists:sort(components(self(), M2)), 139 | M3 = remove(self(), machine, M2), 140 | [aux] = components(self(), M3), 141 | {Components, M5} = handle_down(self(), M2), 142 | [aux, machine] = lists:sort(Components), 143 | [] = components(self(), M5), 144 | ok. 145 | 146 | nodes_test() -> 147 | M0 = init(), 148 | M1 = add(fake@blah, machine, M0), 149 | receive 150 | {nodedown, fake@blah, _} -> ok 151 | after 100 -> exit(nodedown_timeout) 152 | end, 153 | M2 = add(fake2@blah, machine, M1), 154 | receive 155 | {nodedown, fake2@blah, _} -> ok 156 | after 100 -> exit(nodedown_timeout_2) 157 | end, 158 | M3 = add(fake2@blah, aux, M2), 159 | receive 160 | {nodedown, fake2@blah, _} -> ok 161 | after 100 -> exit(nodedown_timeout_3) 162 | end, 163 | M4 = remove_all(machine, M3), 164 | ?assertEqual([fake2@blah], maps:keys(M4)), 165 | ?assertEqual([#{aux => ok}], maps:values(M4)), 166 | ok. 167 | 168 | 169 | -endif. 170 | -------------------------------------------------------------------------------- /src/ra_lol.erl: -------------------------------------------------------------------------------- 1 | -module(ra_lol). 2 | %% sorted list of list 3 | 4 | -export([ 5 | new/0, 6 | append/2, 7 | search/2, 8 | takewhile/2, 9 | from_list/1, 10 | from_list/2, 11 | to_list/1, 12 | len/1 13 | ]). 14 | 15 | -define(MAX_ROW_LEN, 64). 16 | 17 | -type row() :: [term()]. 18 | -type gt_fun() :: fun((Item, Item) -> boolean()). 19 | 20 | -record(?MODULE, {len = 0 :: non_neg_integer(), 21 | append_row_len = 0 :: non_neg_integer(), 22 | gt_fun :: gt_fun(), 23 | rows = [] :: [row()]}). 24 | 25 | -opaque state() :: #?MODULE{}. 26 | 27 | %% a search continuation 28 | -opaque cont() :: [row()]. 29 | 30 | 31 | -export_type([state/0, 32 | cont/0]). 33 | 34 | -spec new() -> state(). 35 | new() -> 36 | #?MODULE{gt_fun = fun erlang:'>'/2}. 37 | 38 | -spec new(gt_fun()) -> state(). 39 | new(GtFun) -> 40 | #?MODULE{gt_fun = GtFun}. 41 | 42 | %% @doc append an item that is greater than the last appended item 43 | -spec append(Item, state()) -> 44 | state() | out_of_order 45 | when Item :: term(). 46 | append(Item, #?MODULE{rows = []} = State) -> 47 | State#?MODULE{rows = [[Item]], 48 | len = 1, 49 | append_row_len = 0}; 50 | append(Item, 51 | #?MODULE{len = Len, 52 | gt_fun = GtFun, 53 | append_row_len = RowLen, 54 | rows = [[LastItem | _] = Row | Rows]} = State) -> 55 | case GtFun(Item, LastItem) of 56 | true -> 57 | case RowLen of 58 | ?MAX_ROW_LEN -> 59 | %% time for a new row 60 | State#?MODULE{rows = [[Item], Row | Rows], 61 | len = Len + 1, 62 | append_row_len = 1}; 63 | _ -> 64 | State#?MODULE{rows = [[Item | Row] | Rows], 65 | len = Len + 1, 66 | append_row_len = RowLen + 1} 67 | end; 68 | false -> 69 | out_of_order 70 | end. 71 | 72 | 73 | -spec search(fun((term()) -> higher | lower | equal), 74 | state() | cont()) -> 75 | {term(), cont()} | undefined. 76 | search(SearchFun, #?MODULE{rows = Rows}) -> 77 | search(SearchFun, Rows); 78 | search(SearchFun, Rows) when is_list(Rows) -> 79 | case find_row(SearchFun, Rows) of 80 | [] -> 81 | undefined; 82 | [SearchRow | RemRows] -> 83 | case search_row(SearchFun, SearchRow) of 84 | undefined -> 85 | undefined; 86 | {Item, Rem} -> 87 | {Item, [Rem | RemRows]} 88 | end 89 | end. 90 | 91 | -spec takewhile(fun((Item) -> boolean()), state()) -> 92 | {[Item], state()} 93 | when Item :: term(). 94 | takewhile(Fun, #?MODULE{gt_fun = GtFun} = State) -> 95 | %% not the most efficient but rarely used 96 | {Taken, Left} = lists:splitwith(Fun, to_list(State)), 97 | {Taken, from_list(GtFun, lists:reverse(Left))}. 98 | 99 | 100 | %% @doc initialise from a list sorted in ascending order 101 | -spec from_list(list()) -> state(). 102 | from_list(List) -> 103 | from_list(fun erlang:'>'/2, List). 104 | 105 | -spec from_list(gt_fun(), list()) -> state(). 106 | from_list(GtFun, List) 107 | when is_list(List) -> 108 | lists:foldl(fun append/2, new(GtFun), List). 109 | 110 | -spec to_list(state()) -> list(). 111 | to_list(#?MODULE{rows = Rows}) -> 112 | lists:append(Rows). 113 | 114 | -spec len(state()) -> non_neg_integer(). 115 | len(#?MODULE{len = Len}) -> 116 | Len. 117 | 118 | 119 | %% Internals 120 | 121 | search_row(_SearchFun, []) -> 122 | undefined; 123 | search_row(SearchFun, [Item | Rem]) -> 124 | case SearchFun(Item) of 125 | equal -> 126 | {Item, Rem}; 127 | lower -> 128 | search_row(SearchFun, Rem); 129 | higher -> 130 | undefined 131 | end. 132 | 133 | 134 | find_row(SearchFun, [_Row, Row | Rem] = Rows) -> 135 | %% if last item of the second rows is higher than searching for 136 | %% then return all rows 137 | case SearchFun(hd(Row)) of 138 | higher -> 139 | Rows; 140 | _ -> 141 | %% else keep searching 142 | find_row(SearchFun, [Row | Rem]) 143 | end; 144 | find_row(_SearchFun, Rows) -> 145 | Rows. 146 | 147 | %%% =================== 148 | %%% Internal unit tests 149 | %%% =================== 150 | 151 | -ifdef(TEST). 152 | -include_lib("eunit/include/eunit.hrl"). 153 | 154 | basic_test() -> 155 | Items = lists:seq(1, 100), 156 | L0 = ra_lol:from_list(Items), 157 | ?assertEqual(100, ra_lol:len(L0)), 158 | ?assertEqual(Items, lists:reverse(ra_lol:to_list(L0))), 159 | ?assertMatch(out_of_order, ra_lol:append(1, L0)), 160 | L1 = ra_lol:append(101, L0), 161 | ?assertEqual(101, ra_lol:len(L1)), 162 | SearchFun = fun (T) -> 163 | fun (Item) -> 164 | if T == Item -> equal; 165 | T > Item -> higher; 166 | true -> lower 167 | end 168 | end 169 | end, 170 | [begin 171 | {T, _} = ra_lol:search(SearchFun(T), L1) 172 | end || T <- Items ++ [101]], 173 | 174 | %% test searching with a continuation 175 | _ = lists:foldl(fun (T, Acc) -> 176 | {T, Cont} = ra_lol:search(SearchFun(T), Acc), 177 | Cont 178 | end, L1, lists:reverse(Items ++ [101])), 179 | 180 | TakeFun = fun(Item) -> Item > 50 end, 181 | 182 | {Taken, L2} = takewhile(TakeFun, L1), 183 | ?assertEqual(50, ra_lol:len(L2)), 184 | ?assertEqual(51, length(Taken)), 185 | ?assertMatch(out_of_order, ra_lol:append(50, L2)), 186 | L3 = ra_lol:append(51, L2), 187 | ?assertEqual(51, ra_lol:len(L3)), 188 | 189 | ok. 190 | 191 | 192 | -endif. 193 | -------------------------------------------------------------------------------- /src/ra_range.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_range). 9 | 10 | -export([ 11 | new/1, 12 | new/2, 13 | add/2, 14 | in/2, 15 | extend/2, 16 | limit/2, 17 | truncate/2, 18 | size/1, 19 | overlap/2, 20 | subtract/2 21 | ]). 22 | 23 | 24 | -type range() :: undefined | {ra:index(), ra:index()}. 25 | 26 | -export_type([range/0]). 27 | 28 | -spec new(ra:index()) -> range(). 29 | new(Start) when is_integer(Start) -> 30 | {Start, Start}. 31 | 32 | -spec new(ra:index(), ra:index()) -> range(). 33 | new(Start, End) 34 | when is_integer(Start) andalso 35 | is_integer(End) andalso 36 | Start =< End -> 37 | {Start, End}; 38 | new(_Start, _End) -> 39 | undefined. 40 | 41 | -spec add(AddRange :: range(), CurRange :: range()) -> range(). 42 | add(undefined, Range) -> 43 | Range; 44 | add({AddStart, AddEnd}, {Start, End}) 45 | when Start =< AddEnd + 1 andalso 46 | End + 1 >= AddStart -> 47 | {min(AddStart, Start), max(AddEnd, End)}; 48 | add(AddRange, _Range) -> 49 | %% no overlap, return add range 50 | AddRange. 51 | 52 | -spec in(ra:index(), range()) -> boolean(). 53 | in(_Idx, undefined) -> 54 | false; 55 | in(Idx, {Start, End}) -> 56 | Idx >= Start andalso Idx =< End. 57 | 58 | -spec limit(ra:index(), range()) -> range(). 59 | limit(CeilExcl, {Start, _End}) 60 | when is_integer(CeilExcl) andalso 61 | CeilExcl =< Start -> 62 | undefined; 63 | limit(CeilExcl, {Start, End}) 64 | when is_integer(CeilExcl) andalso 65 | CeilExcl =< End -> 66 | {Start, CeilExcl - 1}; 67 | limit(CeilExcl, Range) 68 | when is_integer(CeilExcl) -> 69 | Range. 70 | 71 | -spec truncate(ra:index(), range()) -> range(). 72 | truncate(UpToIncl, {_Start, End}) 73 | when is_integer(UpToIncl) andalso 74 | UpToIncl >= End -> 75 | undefined; 76 | truncate(UpToIncl, {Start, End}) 77 | when is_integer(UpToIncl) andalso 78 | UpToIncl >= Start -> 79 | {UpToIncl + 1, End}; 80 | truncate(UpToIncl, Range) 81 | when is_integer(UpToIncl) -> 82 | Range. 83 | 84 | size(undefined) -> 85 | 0; 86 | size({Start, End}) -> 87 | End - Start + 1. 88 | 89 | -spec extend(ra:index(), range()) -> range(). 90 | extend(Idx, {Start, End}) 91 | when Idx == End + 1 -> 92 | {Start, Idx}; 93 | extend(Idx, undefined) when is_integer(Idx) -> 94 | ra_range:new(Idx); 95 | extend(Idx, Range) -> 96 | error({cannot_extend, Idx, Range}). 97 | 98 | -spec overlap(range(), range()) -> range(). 99 | overlap({ReqStart, ReqEnd}, {Start, End}) -> 100 | new(max(ReqStart, Start), min(ReqEnd, End)); 101 | overlap(_Range1, _Range2) -> 102 | undefined. 103 | 104 | %% @doc subtracts the range in the first argument 105 | %% from that of the range in the second arg. 106 | %% Returns a list of remaining ranges 107 | %% @end 108 | -spec subtract(range(), range()) -> [range()]. 109 | subtract(_Range1, undefined) -> 110 | []; 111 | subtract(undefined, Range) -> 112 | [Range]; 113 | subtract({_SubStart, _SubEnd} = SubRange, {Start, End} = Range) -> 114 | case overlap(SubRange, Range) of 115 | undefined -> 116 | [Range]; 117 | {OStart, OEnd} -> 118 | [R || {_, _} = R <- [new(Start, OStart -1), 119 | new(OEnd + 1, End)]] 120 | end. 121 | 122 | 123 | -ifdef(TEST). 124 | -include_lib("eunit/include/eunit.hrl"). 125 | 126 | subtract_test() -> 127 | ?assertEqual([], subtract({1, 10}, undefined)), 128 | ?assertEqual([{1, 10}], subtract(undefined, {1, 10})), 129 | ?assertEqual([], subtract({1, 10}, {1, 10})), 130 | ?assertEqual([], subtract({1, 10}, {4, 6})), 131 | ?assertEqual([{4, 6}], subtract({8, 10}, {4, 6})), 132 | 133 | ?assertEqual([{11, 20}], subtract({1, 10}, {1, 20})), 134 | ?assertEqual([{1, 1}, {11, 20}], subtract({2, 10}, {1, 20})), 135 | ?assertEqual([{1, 9}], subtract({10, 20}, {1, 20})), 136 | 137 | ok. 138 | 139 | overlap_test() -> 140 | ?assertEqual(undefined, overlap({1, 10}, undefined)), 141 | ?assertEqual(undefined, overlap(undefined, {1, 10})), 142 | ?assertEqual(undefined, overlap({1, 10}, {11, 15})), 143 | ?assertEqual(undefined, overlap({16, 20}, {11, 15})), 144 | ?assertEqual({11, 11}, overlap({1, 11}, {11, 15})), 145 | ?assertEqual({14, 15}, overlap({14, 20}, {11, 15})), 146 | ?assertEqual({12, 14}, overlap({12, 14}, {11, 15})), 147 | ?assertEqual({11, 15}, overlap({1, 20}, {11, 15})), 148 | ok. 149 | 150 | add_test() -> 151 | ?assertEqual(undefined, add(undefined, undefined)), 152 | ?assertEqual({1, 10}, add(undefined, {1, 10})), 153 | ?assertEqual({1, 10}, add({1, 10}, undefined)), 154 | 155 | ?assertEqual({1, 20}, add({1, 10}, {11, 20})), 156 | ?assertEqual({1, 20}, add({1, 10}, {5, 20})), 157 | 158 | ?assertEqual({1, 10}, add({1, 10}, {5, 9})), 159 | ?assertEqual({5, 10}, add({6, 10}, {5, 9})), 160 | ?assertEqual({1, 10}, add({6, 10}, {1, 5})), 161 | 162 | %% when the add range is smaller than the prior range 163 | %% return the additional range 164 | ?assertEqual({1, 3}, add({1, 3}, {6, 10})), 165 | ?assertEqual({1, 10}, add({6, 10}, {1, 7})), 166 | ok. 167 | 168 | truncate_test() -> 169 | ?assertEqual(undefined, truncate(9, undefined)), 170 | ?assertEqual({6, 10}, truncate(5, {1, 10})), 171 | ?assertEqual(undefined, truncate(11, {1, 10})), 172 | ?assertEqual({10, 20}, truncate(9, {10, 20})), 173 | ok. 174 | 175 | limit_test() -> 176 | ?assertEqual(undefined, limit(9, undefined)), 177 | ?assertEqual({1, 4}, limit(5, {1, 10})), 178 | ?assertEqual({1, 10}, limit(11, {1, 10})), 179 | ?assertEqual(undefined, limit(10, {10, 20})), 180 | ?assertEqual(undefined, limit(1, {10, 20})), 181 | ok. 182 | 183 | in_test() -> 184 | ?assertEqual(false, in(9, undefined)), 185 | ?assertEqual(true, in(5, {1, 10})), 186 | ?assertEqual(true, in(10, {1, 10})), 187 | ?assertEqual(true, in(1, {1, 10})), 188 | ?assertEqual(false, in(11, {1, 10})), 189 | ?assertEqual(false, in(0, {1, 10})), 190 | ok. 191 | 192 | extend_test() -> 193 | ?assertEqual({9, 9}, extend(9, undefined)), 194 | ?assertEqual({1, 11}, extend(11, {1, 10})), 195 | ?assertError({cannot_extend, 1, {5, 10}}, extend(1, {5, 10})), 196 | ?assertError({cannot_extend, 12, {1, 10}}, extend(12, {1, 10})), 197 | ok. 198 | 199 | -endif. 200 | -------------------------------------------------------------------------------- /src/ra_log_meta.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_log_meta). 9 | -behaviour(gen_batch_server). 10 | 11 | -export([start_link/1, 12 | init/1, 13 | handle_batch/2, 14 | terminate/2, 15 | format_status/1, 16 | store/4, 17 | store_sync/4, 18 | delete/2, 19 | delete_sync/2, 20 | fetch/3, 21 | fetch/4 22 | ]). 23 | 24 | -include("ra.hrl"). 25 | 26 | %% centralised meta data storage server for ra servers. 27 | 28 | -type key() :: current_term | voted_for | last_applied. 29 | -type value() :: non_neg_integer() | atom() | {atom(), atom()}. 30 | 31 | -define(TIMEOUT, 30000). 32 | -define(SYNC_INTERVAL, 5000). 33 | 34 | -record(?MODULE, {ref :: reference(), 35 | table_name :: atom()}). 36 | 37 | -opaque state() :: #?MODULE{}. 38 | 39 | -export_type([state/0]). 40 | 41 | -spec start_link(ra_system:config()) -> 42 | {ok, pid()} | {error, {already_started, pid()}}. 43 | start_link(#{names := #{log_meta := Name}} = Cfg) -> 44 | gen_batch_server:start_link({local, Name}, ?MODULE, Cfg, []). 45 | 46 | -spec init(ra_system:config()) -> {ok, state()}. 47 | init(#{name := System, 48 | data_dir := Dir, 49 | names := #{log_meta := TblName}}) -> 50 | process_flag(trap_exit, true), 51 | ok = ra_lib:make_dir(Dir), 52 | MetaFile = filename:join(Dir, "meta.dets"), 53 | {ok, Ref} = dets:open_file(TblName, [{file, MetaFile}, 54 | {auto_save, ?SYNC_INTERVAL}]), 55 | _ = ets:new(TblName, [named_table, public, {read_concurrency, true}]), 56 | TblName = dets:to_ets(TblName, TblName), 57 | ?INFO("ra: meta data store initialised for system ~ts. ~b record(s) recovered", 58 | [System, ets:info(TblName, size)]), 59 | {ok, #?MODULE{ref = Ref, 60 | table_name = TblName}}. 61 | 62 | handle_batch(Commands, #?MODULE{ref = Ref, 63 | table_name = TblName} = State) -> 64 | DoInsert = 65 | fun (Id, Key, Value, Inserts0) -> 66 | case Inserts0 of 67 | #{Id := Data} -> 68 | Inserts0#{Id => update_key(Key, Value, Data)}; 69 | _ -> 70 | case ets:lookup(TblName, Id) of 71 | [Data] -> 72 | Inserts0#{Id => update_key(Key, Value, Data)}; 73 | [] -> 74 | Data = {Id, undefined, undefined, undefined}, 75 | Inserts0#{Id => update_key(Key, Value, Data)} 76 | end 77 | end 78 | end, 79 | {Inserts, Replies, ShouldSync} = 80 | lists:foldl( 81 | fun ({cast, {store, Id, Key, Value}}, 82 | {Inserts0, Replies, DoSync}) -> 83 | {DoInsert(Id, Key, Value, Inserts0), Replies, DoSync}; 84 | ({call, From, {store, Id, Key, Value}}, 85 | {Inserts0, Replies, _DoSync}) -> 86 | {DoInsert(Id, Key, Value, Inserts0), 87 | [{reply, From, ok} | Replies], true}; 88 | ({cast, {delete, Id}}, 89 | {Inserts0, Replies, DoSync}) -> 90 | {handle_delete(TblName, Id, Ref, Inserts0), Replies, DoSync}; 91 | ({call, From, {delete, Id}}, 92 | {Inserts0, Replies, _DoSync}) -> 93 | {handle_delete(TblName, Id, Ref, Inserts0), 94 | [{reply, From, ok} | Replies], true} 95 | end, {#{}, [], false}, Commands), 96 | Objects = maps:values(Inserts), 97 | true = ets:insert(TblName, Objects), 98 | ok = dets:insert(TblName, Objects), 99 | case ShouldSync of 100 | true -> 101 | ok = dets:sync(TblName); 102 | false -> 103 | ok 104 | end, 105 | {ok, Replies, State}. 106 | 107 | terminate(_, #?MODULE{ref = Ref, 108 | table_name = TblName}) -> 109 | ?DEBUG("ra: meta data store is terminating", []), 110 | ok = dets:sync(TblName), 111 | _ = dets:close(Ref), 112 | ok. 113 | 114 | format_status(State) -> 115 | State. 116 | 117 | %% send a message to the meta data store using cast 118 | -spec store(atom(), ra_uid(), key(), value()) -> ok. 119 | store(Name, UId, Key, Value) when is_atom(Name) -> 120 | gen_batch_server:cast(Name, {store, UId, Key, Value}). 121 | 122 | %% waits until batch has been processed and synced. 123 | %% when it returns the store request has been safely flushed to disk 124 | -spec store_sync(atom(), ra_uid(), key(), value()) -> ok. 125 | store_sync(Name, UId, Key, Value) -> 126 | gen_batch_server:call(Name, {store, UId, Key, Value}, ?TIMEOUT). 127 | 128 | -spec delete(atom(), ra_uid()) -> ok. 129 | delete(Name, UId) -> 130 | gen_batch_server:cast(Name, {delete, UId}). 131 | 132 | -spec delete_sync(atom(), ra_uid()) -> ok. 133 | delete_sync(Name, UId) -> 134 | gen_batch_server:call(Name, {delete, UId}, ?TIMEOUT). 135 | 136 | %% READER API 137 | 138 | -spec fetch(atom(), ra_uid(), key()) -> value() | undefined. 139 | fetch(MetaName, Id, current_term) -> 140 | maybe_fetch(MetaName, Id, 2); 141 | fetch(MetaName, Id, voted_for) -> 142 | maybe_fetch(MetaName, Id, 3); 143 | fetch(MetaName, Id, last_applied) -> 144 | maybe_fetch(MetaName, Id, 4). 145 | 146 | -spec fetch(atom(), ra_uid(), key(), term()) -> value(). 147 | fetch(MetaName, Id, Key, Default) -> 148 | case fetch(MetaName, Id, Key) of 149 | undefined -> Default; 150 | Value -> Value 151 | end. 152 | 153 | %%% internal 154 | 155 | maybe_fetch(MetaName, Id, Pos) -> 156 | try ets:lookup_element(MetaName, Id, Pos) 157 | catch 158 | _:badarg -> 159 | undefined 160 | end. 161 | 162 | handle_delete(TblName, Id, Ref, Inserts) -> 163 | _ = dets:delete(Ref, Id), 164 | _ = ets:delete(TblName, Id), 165 | maps:remove(Id, Inserts). 166 | 167 | update_key(current_term, Value, Data) -> 168 | case element(2, Data) of 169 | %% current term matches the new value, nothing to do 170 | Value -> Data; 171 | %% current term has changed. Clear voted_for field as part of the update. 172 | %% See rabbitmq/ra#111. 173 | _ -> 174 | Data1 = setelement(3, Data, undefined), 175 | setelement(2, Data1, Value) 176 | end; 177 | update_key(voted_for, Value, Data) -> 178 | setelement(3, Data, Value); 179 | update_key(last_applied, Value, Data) -> 180 | setelement(4, Data, Value). 181 | -------------------------------------------------------------------------------- /test/ra_log_snapshot_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(ra_log_snapshot_SUITE). 8 | 9 | -compile(nowarn_export_all). 10 | -compile(export_all). 11 | 12 | -export([ 13 | ]). 14 | 15 | -include_lib("common_test/include/ct.hrl"). 16 | -include_lib("eunit/include/eunit.hrl"). 17 | 18 | %%%=================================================================== 19 | %%% Common Test callbacks 20 | %%%=================================================================== 21 | 22 | all() -> 23 | [ 24 | {group, tests} 25 | ]. 26 | 27 | 28 | all_tests() -> 29 | [ 30 | roundtrip, 31 | roundtrip_compat, 32 | accept, 33 | read_missing, 34 | read_other_file, 35 | read_invalid_version, 36 | recover_invalid_checksum, 37 | read_meta_data, 38 | recover_same_as_read 39 | ]. 40 | 41 | groups() -> 42 | [ 43 | {tests, [], all_tests()} 44 | ]. 45 | 46 | init_per_suite(Config) -> 47 | Config. 48 | 49 | end_per_suite(_Config) -> 50 | ok. 51 | 52 | init_per_group(_Group, Config) -> 53 | Config. 54 | 55 | end_per_group(_Group, _Config) -> 56 | ok. 57 | 58 | init_per_testcase(TestCase, Config) -> 59 | Dir = filename:join([?config(priv_dir, Config), TestCase, write]), 60 | ok = ra_lib:make_dir(Dir), 61 | AcceptDir = filename:join([?config(priv_dir, Config), TestCase, accept]), 62 | ok = ra_lib:make_dir(AcceptDir), 63 | [{accept_dir, AcceptDir}, {dir, Dir} | Config]. 64 | 65 | end_per_testcase(_TestCase, _Config) -> 66 | ok. 67 | 68 | %%%=================================================================== 69 | %%% Test cases 70 | %%%=================================================================== 71 | 72 | roundtrip(Config) -> 73 | Dir = ?config(dir, Config), 74 | SnapshotMeta = meta(33, 94, [{banana, node@jungle}, {banana, node@savanna}]), 75 | SnapshotRef = my_state, 76 | {ok, _} = ra_log_snapshot:write(Dir, SnapshotMeta, SnapshotRef, true), 77 | Context = #{can_accept_full_file => true}, 78 | ?assertEqual({SnapshotMeta, SnapshotRef}, read(Dir, Context)), 79 | ok. 80 | 81 | roundtrip_compat(Config) -> 82 | Dir = ?config(dir, Config), 83 | SnapshotMeta = meta(33, 94, [{banana, node@jungle}, {banana, node@savanna}]), 84 | SnapshotRef = my_state, 85 | {ok, _} = ra_log_snapshot:write(Dir, SnapshotMeta, SnapshotRef, true), 86 | ?assertEqual({SnapshotMeta, SnapshotRef}, read(Dir)), 87 | ok. 88 | 89 | dir(Base, Dir0) -> 90 | Dir = filename:join(Base, Dir0), 91 | ok = ra_lib:make_dir(Dir), 92 | Dir. 93 | 94 | 95 | accept(Config) -> 96 | test_accept(Config, one, 1024, true, 128), 97 | test_accept(Config, two, 1024, false, 128), 98 | test_accept(Config, three, 8, true, 64), 99 | test_accept(Config, four, 8, false, 64), 100 | ok. 101 | 102 | 103 | 104 | test_accept(Config, Name, DataSize, FullFile, ChunkSize) -> 105 | Dir = dir(?config(dir, Config), Name), 106 | AcceptDir = dir(?config(accept_dir, Config), Name), 107 | ct:pal("test_accept ~w ~b ~w ~b", [Name, DataSize, FullFile, ChunkSize]), 108 | SnapshotMeta = meta(33, 94, [{banana, node@jungle}, {banana, node@savanna}]), 109 | SnapshotRef = crypto:strong_rand_bytes(DataSize), 110 | {ok, _} = ra_log_snapshot:write(Dir, SnapshotMeta, SnapshotRef, true), 111 | Context = #{can_accept_full_file => FullFile}, 112 | {ok, Meta, St} = ra_log_snapshot:begin_read(Dir, Context), 113 | %% how to ensure 114 | [LastChunk | Chunks] = lists:reverse(read_all_chunks(St, Dir, ChunkSize, [])), 115 | {ok, A0} = ra_log_snapshot:begin_accept(AcceptDir, Meta), 116 | A1 = lists:foldl(fun (Ch, Acc0) -> 117 | {ok, Acc} = ra_log_snapshot:accept_chunk(Ch, Acc0), 118 | Acc 119 | end, A0, lists:reverse(Chunks)), 120 | ok = ra_log_snapshot:complete_accept(LastChunk, A1), 121 | ok. 122 | 123 | read(Dir) -> 124 | read(Dir, #{}). 125 | 126 | read(Dir, Context) -> 127 | case ra_log_snapshot:begin_read(Dir, Context) of 128 | {ok, _Meta, St} -> 129 | case iolist_to_binary( 130 | read_all_chunks(St, Dir, 128, [])) of 131 | <<"RASN", 1:8, _Crc:32/integer, 132 | MetaSz:32/integer, Meta:MetaSz/binary, 133 | Snap/binary>> -> 134 | {binary_to_term(Meta), binary_to_term(Snap)}; 135 | <<_Crc:32/integer, Snap/binary>> -> 136 | {_Meta, binary_to_term(Snap)} 137 | end; 138 | Err -> Err 139 | end. 140 | 141 | read_all_chunks(St, Dir, Size, Acc) -> 142 | case ra_log_snapshot:read_chunk(St, Size, Dir) of 143 | {ok, Data, {next, St1}} -> 144 | read_all_chunks(St1, Dir, Size, [Data | Acc]); 145 | {ok, Data, last} -> 146 | lists:reverse([Data | Acc]); 147 | Err -> Err 148 | end. 149 | 150 | read_missing(Config) -> 151 | File = ?config(file, Config), 152 | {error, enoent} = read(File), 153 | ok. 154 | 155 | read_other_file(Config) -> 156 | Dir = ?config(dir, Config), 157 | File = filename:join(Dir, "snapshot.dat"), 158 | file:write_file(File, <<"NSAR", 1:8/unsigned>>), 159 | {error, invalid_format} = read(Dir), 160 | ok. 161 | 162 | read_invalid_version(Config) -> 163 | Dir = ?config(dir, Config), 164 | File = filename:join(Dir, "snapshot.dat"), 165 | Data = term_to_binary(snapshot), 166 | Crc = erlang:crc32(Data), 167 | file:write_file(File, [<<"RASN", 99:8/unsigned, Crc:32/integer>>, Data]), 168 | {error, {invalid_version, 99}} = read(Dir), 169 | ok. 170 | 171 | recover_invalid_checksum(Config) -> 172 | Dir = ?config(dir, Config), 173 | File = filename:join(Dir, "snapshot.dat"), 174 | file:write_file(File, [<<"RASN">>, <<1:8/unsigned, 0:32/unsigned>>, 175 | term_to_binary(<<"hi">>)]), 176 | {error, checksum_error} = ra_log_snapshot:recover(Dir), 177 | ok. 178 | 179 | read_meta_data(Config) -> 180 | Dir = ?config(dir, Config), 181 | SnapshotMeta = meta(33, 94, [{banana, node@jungle}, {banana, node@savanna}]), 182 | SnapshotRef = my_state, 183 | {ok, _} = ra_log_snapshot:write(Dir, SnapshotMeta, SnapshotRef, true), 184 | {ok, SnapshotMeta} = ra_log_snapshot:read_meta(Dir), 185 | ok. 186 | 187 | recover_same_as_read(Config) -> 188 | Dir = ?config(dir, Config), 189 | SnapshotMeta = meta(33, 94, [{banana, node@jungle}, {banana, node@savanna}]), 190 | SnapshotData = my_state, 191 | {ok, _} = ra_log_snapshot:write(Dir, SnapshotMeta, SnapshotData, true), 192 | {ok, SnapshotMeta, SnapshotData} = ra_log_snapshot:recover(Dir), 193 | ok. 194 | 195 | %% Utility 196 | 197 | meta(Idx, Term, Cluster) -> 198 | #{index => Idx, 199 | term => Term, 200 | cluster => Cluster, 201 | machine_version => 1}. 202 | -------------------------------------------------------------------------------- /src/ra_log_ets.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_log_ets). 9 | -behaviour(gen_server). 10 | 11 | -export([start_link/1]). 12 | 13 | -export([ 14 | mem_table_please/2, 15 | mem_table_please/3, 16 | new_mem_table_please/3, 17 | delete_mem_tables/2, 18 | execute_delete/3]). 19 | 20 | -export([init/1, 21 | handle_call/3, 22 | handle_cast/2, 23 | handle_info/2, 24 | terminate/2, 25 | code_change/3]). 26 | 27 | -include("ra.hrl"). 28 | 29 | -record(state, {names :: ra_system:names(), 30 | table_opts :: list()}). 31 | 32 | -define(TABLE_OPTS, [set, 33 | {write_concurrency, auto}, 34 | public 35 | ]). 36 | 37 | %%% ra_log_ets - owns and creates mem_table ETS tables 38 | 39 | %%%=================================================================== 40 | %%% API functions 41 | %%%=================================================================== 42 | 43 | start_link(#{names := #{log_ets := Name}} = Cfg) -> 44 | gen_server:start_link({local, Name}, ?MODULE, [Cfg], []). 45 | 46 | -spec mem_table_please(ra_system:names(), ra:uid()) -> 47 | {ok, ra_mt:state()} | {error, term()}. 48 | mem_table_please(Names, UId) -> 49 | mem_table_please(Names, UId, read_write). 50 | 51 | -spec mem_table_please(ra_system:names(), ra:uid(), read | read_write) -> 52 | {ok, ra_mt:state()} | {error, term()}. 53 | mem_table_please(#{log_ets := Name, 54 | open_mem_tbls := OpnMemTbls}, UId, Mode) -> 55 | case ets:lookup(OpnMemTbls, UId) of 56 | [] -> 57 | case gen_server:call(Name, {mem_table_please, UId, #{}}, infinity) of 58 | {ok, [Tid | Rem]} -> 59 | Mt = lists:foldl( 60 | fun (T, Acc) -> 61 | ra_mt:init_successor(T, Mode, Acc) 62 | end, ra_mt:init(Tid, Mode), Rem), 63 | {ok, Mt}; 64 | Err -> 65 | Err 66 | end; 67 | [{_, Tid} | Rem] -> 68 | Mt = lists:foldl( 69 | fun ({_, T}, Acc) -> 70 | ra_mt:init_successor(T, Mode, Acc) 71 | end, ra_mt:init(Tid, Mode), Rem), 72 | {ok, Mt} 73 | end. 74 | 75 | -spec new_mem_table_please(ra_system:names(), ra:uid(), ra_mt:state()) -> 76 | {ok, ra_mt:state()} | {error, term()}. 77 | new_mem_table_please(#{log_ets := Name}, UId, Prev) -> 78 | case gen_server:call(Name, {new_mem_table_please, UId, #{}}, infinity) of 79 | {ok, Tid} -> 80 | {ok, ra_mt:init_successor(Tid, read_write, Prev)}; 81 | Err -> 82 | Err 83 | end. 84 | 85 | delete_mem_tables(#{log_ets := Name}, UId) -> 86 | gen_server:cast(Name, {delete_mem_tables, UId}). 87 | 88 | -spec execute_delete(ra_system:names(), 89 | ra:uid(), 90 | ra_mt:delete_spec()) -> 91 | ok. 92 | execute_delete(#{}, _UId, undefined) -> 93 | ok; 94 | execute_delete(#{log_ets := Name}, UId, Spec) -> 95 | gen_server:cast(Name, {exec_delete, UId, Spec}). 96 | 97 | 98 | %%%=================================================================== 99 | %%% gen_server callbacks 100 | %%%=================================================================== 101 | 102 | init([#{data_dir := DataDir, 103 | name := System, 104 | names := #{log_ets := LogEts, 105 | open_mem_tbls := OpenMemTbls} = Names} = Config]) -> 106 | process_flag(trap_exit, true), 107 | CompressMemTbls = maps:get(compress_mem_tables, Config, false), 108 | TblOpts = ?TABLE_OPTS ++ [{compressed, CompressMemTbls}], 109 | ?INFO("~s: in system ~s initialising. Mem table opts: ~w", 110 | [LogEts, System, TblOpts]), 111 | _ = ets:new(OpenMemTbls, [bag, protected, named_table]), 112 | ok = ra_directory:init(DataDir, Names), 113 | {ok, #state{names = Names, 114 | table_opts = TblOpts}}. 115 | 116 | handle_call({mem_table_please, UId, Opts}, _From, 117 | #state{names = #{open_mem_tbls := OpnMemTbls}} = State) -> 118 | case ets:lookup(OpnMemTbls, UId) of 119 | [] -> 120 | Name = maps:get(table_name, Opts, ra_mem_table), 121 | Tid = ets:new(Name, ?TABLE_OPTS), 122 | true = ets:insert(OpnMemTbls, {UId, Tid}), 123 | {reply, {ok, [Tid]}, State}; 124 | Tids -> 125 | {reply, {ok, Tids}, State#state{}} 126 | end; 127 | handle_call({new_mem_table_please, UId, Opts}, _From, 128 | #state{names = #{open_mem_tbls := OpnMemTbls}} = State) -> 129 | Name = maps:get(table_name, Opts, ra_mem_table), 130 | Tid = ets:new(Name, ?TABLE_OPTS), 131 | true = ets:insert(OpnMemTbls, {UId, Tid}), 132 | {reply, {ok, Tid}, State}; 133 | handle_call(Request, _From, State) -> 134 | ?INFO("ra_log_ets:handle_call/3 unhandled request ~w", [Request]), 135 | Reply = ok, 136 | {reply, Reply, State}. 137 | 138 | handle_cast({exec_delete, UId, Spec}, 139 | #state{names = #{open_mem_tbls := MemTables}} = State) -> 140 | 141 | %% delete from open mem tables if {delete, tid()} 142 | case Spec of 143 | {delete, Tid} -> 144 | ets:delete_object(MemTables, {UId, Tid}); 145 | _ -> 146 | ok 147 | end, 148 | 149 | try timer:tc(fun () -> ra_mt:delete(Spec) end) of 150 | {Time, Num} -> 151 | ?DEBUG_IF(Time > 25_000, 152 | "ra_log_ets: ra_mt:delete/1 took ~bms to delete ~w ~b entries", 153 | [Time div 1000, Spec, Num]), 154 | ok 155 | catch 156 | _:Err -> 157 | ?WARN("ra_log_ets: failed to delete ~w ~w ", 158 | [Spec, Err]), 159 | ok 160 | end, 161 | {noreply, State}; 162 | handle_cast({delete_mem_tables, UId}, 163 | #state{names = #{open_mem_tbls := MemTables}} = State) -> 164 | %% delete ets tables, 165 | %% we need to be defensive here. 166 | %% it is better to leak a table than to crash them all 167 | Objects = ets:take(MemTables, UId), 168 | [begin 169 | try timer:tc(fun () -> ets_delete(Tid) end) of 170 | {Time, true} -> 171 | ?DEBUG_IF(Time > 25_000, 172 | "ra_log_ets: ets:delete/1 took ~bms to delete ~w", 173 | [Time div 1000, Tid]), 174 | ok 175 | catch 176 | _:Err -> 177 | ?WARN("ra_log_ets: failed to delete ets table ~w with ~w " 178 | "This table may need to be cleaned up manually", 179 | [Tid, Err]), 180 | ok 181 | end 182 | end || {_, Tid} <- Objects], 183 | {noreply, State}; 184 | 185 | handle_cast(_Msg, State) -> 186 | {noreply, State}. 187 | 188 | handle_info(_Info, State) -> 189 | {noreply, State}. 190 | 191 | ets_delete(Tid) -> 192 | _ = ets:delete(Tid), 193 | true. 194 | 195 | terminate(Reason, #state{names = Names}) -> 196 | ?DEBUG("ra_log_ets: terminating with ~p", [Reason]), 197 | ok = ra_directory:deinit(Names), 198 | ok. 199 | 200 | code_change(_OldVsn, State, _Extra) -> 201 | {ok, State}. 202 | -------------------------------------------------------------------------------- /docs/ra_system.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |config() =
20 | #{name := atom(),
21 | names := names(),
22 | data_dir := file:filename_all(),
23 | wal_data_dir => file:filename_all(),
24 | wal_max_size_bytes => non_neg_integer(),
25 | wal_compute_checksums => boolean(),
26 | wal_max_batch_size => non_neg_integer(),
27 | wal_max_entries => undefined | non_neg_integer(),
28 | wal_write_strategy => default | o_sync | sync_after_notify,
29 | wal_sync_method => datasync | sync | none,
30 | wal_hibernate_after => non_neg_integer(),
31 | wal_garbage_collect => boolean(),
32 | wal_pre_allocate => boolean(),
33 | segment_max_entries => non_neg_integer(),
34 | segment_max_pending => non_neg_integer(),
35 | segment_compute_checksums => boolean(),
36 | snapshot_chunk_size => non_neg_integer(),
37 | receive_snapshot_timeout => non_neg_integer(),
38 | default_max_pipeline_count => non_neg_integer(),
39 | default_max_append_entries_rpc_batch_size =>
40 | non_neg_integer(),
41 | message_queue_data => on_heap | off_heap,
42 | wal_min_heap_size => non_neg_integer(),
43 | wal_min_bin_vheap_size => non_neg_integer(),
44 | server_min_bin_vheap_size => non_neg_integer(),
45 | server_min_heap_size => non_neg_integer(),
46 | compress_mem_tables => boolean(),
47 | low_priority_commands_flush_size => non_neg_integer(),
48 | low_priority_commands_in_memory_size => non_neg_integer(),
49 | server_recovery_strategy =>
50 | undefined | registered | {module(), atom(), list()}}
51 |
52 |
53 | names() =
55 | #{wal := atom(),
56 | wal_sup := atom(),
57 | log_sup := atom(),
58 | log_ets := atom(),
59 | log_meta := atom(),
60 | open_mem_tbls := atom(),
61 | segment_writer := atom(),
62 | server_sup := atom(),
63 | directory := atom(),
64 | directory_rev := atom()}
65 |
66 |
67 | | start/1 | |
| start_default/0 | |
| default_config/0 | |
| derive_names/1 | |
| store/1 | |
| fetch/1 | |
| fetch/2 | |
| lookup_name/2 | |
| stop/1 | |
| stop_default/0 |
start(Config :: ra_system:config()) -> supervisor:startchild_ret()85 |
86 |
start_default() -> supervisor:startchild_ret()91 |
92 |
default_config() -> ra_system:config()97 |
98 |
derive_names(SysName) -> any()
104 |
store(Config :: config()) -> ok109 |
110 |
fetch(Name :: atom()) -> config() | undefined115 |
116 |
fetch(Name :: atom(), Node :: atom()) -> 121 | config() | undefined | {badrpc, term()}122 |
123 |
lookup_name(System :: atom(), Key :: atom()) ->
128 | {ok, atom()} | {error, system_not_started}
129 | 130 |
stop(System :: ra_system:config() | atom()) -> ok | {error, any()}135 |
136 |
stop_default() -> ok | {error, any()}
141 | 142 |
Generated by EDoc
147 | 148 | 149 | -------------------------------------------------------------------------------- /src/ra_bench.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2017-2025 Broadcom. All Rights Reserved. The term Broadcom refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | %% @hidden 8 | -module(ra_bench). 9 | 10 | -behaviour(ra_machine). 11 | 12 | -compile(inline_list_funcs). 13 | -compile(inline). 14 | -compile({no_auto_import, [apply/3]}). 15 | 16 | -include_lib("eunit/include/eunit.hrl"). 17 | 18 | -define(PIPE_SIZE, 500). 19 | -define(DATA_SIZE, 256). 20 | 21 | -export([ 22 | init/1, 23 | apply/3, 24 | 25 | % profile/0, 26 | % stop_profile/0 27 | 28 | prepare/0, 29 | run/3, 30 | run/2, 31 | run/1, 32 | run/0, 33 | print_metrics/1 34 | 35 | ]). 36 | 37 | -define(TARGET_OPS_SEC, 100000). 38 | -define(TARGET_SEC, 60). 39 | -define(DEGREE, 5). 40 | 41 | 42 | init(#{}) -> 43 | undefined. 44 | 45 | % msg_ids are scoped per customer 46 | % ra_indexes holds all raft indexes for enqueues currently on queue 47 | apply(#{index := I}, {noop, _}, State) -> 48 | case I rem 100000 of 49 | 0 -> 50 | {State, ok, {release_cursor, I, State}}; 51 | _ -> 52 | {State, ok} 53 | end. 54 | 55 | run(Name, Nodes) -> 56 | run(Name, Nodes, ?PIPE_SIZE). 57 | 58 | run(Name, Nodes, Pipe) 59 | when is_atom(Name) 60 | andalso is_list(Nodes) -> 61 | run(#{name => Name, 62 | seconds => ?TARGET_SEC, 63 | target => ?TARGET_OPS_SEC, 64 | degree => ?DEGREE, 65 | pipe => Pipe, 66 | nodes => Nodes}). 67 | 68 | run() -> 69 | run(#{name => noop, 70 | seconds => ?TARGET_SEC, 71 | target => ?TARGET_OPS_SEC, 72 | degree => ?DEGREE, 73 | nodes => [node() | nodes()]}). 74 | 75 | print_counter(Last, Counter) -> 76 | V = counters:get(Counter, 1), 77 | io:format("ops/sec: ~b~n", [ V - Last]), 78 | timer:sleep(1000), 79 | print_counter(V, Counter). 80 | 81 | 82 | run(Nodes) when is_list(Nodes) -> 83 | run(#{name => noop, 84 | seconds => ?TARGET_SEC, 85 | target => ?TARGET_OPS_SEC, 86 | degree => ?DEGREE, 87 | nodes => Nodes}); 88 | run(#{name := Name, 89 | seconds := Secs, 90 | target := Target, 91 | nodes := Nodes, 92 | degree := Degree} = Conf) -> 93 | 94 | Pipe = maps:get(pipe, Conf, ?PIPE_SIZE), 95 | io:format("running ra benchmark config: ~p~n", [Conf]), 96 | io:format("starting servers on ~w~n", [Nodes]), 97 | {ok, ServerIds, []} = start(Name, Nodes), 98 | {ok, _, Leader} = ra:members(hd(ServerIds)), 99 | TotalOps = Secs * Target, 100 | Each = max(Pipe, TotalOps div Degree), 101 | DataSize = maps:get(data_size, Conf, ?DATA_SIZE), 102 | Counter = counters:new(1, [write_concurrency]), 103 | Pids = [spawn_client(self(), Leader, Each, DataSize, Pipe, Counter) 104 | || _ <- lists:seq(1, Degree)], 105 | io:format("running bench mark...~n", []), 106 | Start = erlang:system_time(millisecond), 107 | [P ! go || P <- Pids], 108 | CounterPrinter = spawn(fun () -> 109 | print_counter(counters:get(Counter, 1), Counter) 110 | end), 111 | %% wait for each pid 112 | Wait = ((Secs * 10000) * 4), 113 | [begin 114 | receive 115 | {done, P} -> 116 | io:format("~w is done ", [P]), 117 | ok 118 | after Wait -> 119 | exit({timeout, P}) 120 | end 121 | end || P <- Pids], 122 | End = erlang:system_time(millisecond), 123 | Taken = End - Start, 124 | exit(CounterPrinter, kill), 125 | io:format("benchmark completed: ~b ops in ~bms rate ~.2f ops/sec~n", 126 | [TotalOps, Taken, TotalOps / (Taken / 1000)]), 127 | 128 | BName = atom_to_binary(Name, utf8), 129 | _ = [rpc:call(N, ?MODULE, print_metrics, [BName]) 130 | || N <- Nodes], 131 | % _ = ra:delete_cluster(ServerIds), 132 | %% 133 | ok. 134 | 135 | start(Name, Nodes) when is_atom(Name) -> 136 | ServerIds = [{Name, N} || N <- Nodes], 137 | Configs = [begin 138 | rpc:call(N, ?MODULE, prepare, []), 139 | Id = {Name, N}, 140 | UId = ra:new_uid(ra_lib:to_binary(Name)), 141 | #{id => Id, 142 | uid => UId, 143 | cluster_name => Name, 144 | metrics_key => atom_to_binary(Name, utf8), 145 | log_init_args => #{uid => UId}, 146 | initial_members => ServerIds, 147 | machine => {module, ?MODULE, #{}}} 148 | end || N <- Nodes], 149 | ra:start_cluster(default, Configs). 150 | 151 | prepare() -> 152 | _ = application:ensure_all_started(ra), 153 | _ = ra_system:start_default(), 154 | ra_env:configure_logger(logger), 155 | LogFile = filename:join(ra_env:data_dir(), "ra-" ++ atom_to_list(node()) ++ ".log"), 156 | logger:set_primary_config(level, debug), 157 | Config = #{config => #{file => LogFile}}, 158 | logger:add_handler(ra_handler, logger_std_h, Config), 159 | % application:load(sasl), 160 | % application:set_env(sasl, sasl_error_logger, {file, SaslFile}), 161 | ok. 162 | 163 | send_n(_, _Data, 0, _Counter) -> ok; 164 | send_n(Leader, Data, N, Counter) -> 165 | ra:pipeline_command(Leader, {noop, Data}, make_ref(), normal), 166 | counters:add(Counter, 1, 1), 167 | send_n(Leader, Data, N-1, Counter). 168 | 169 | 170 | client_loop(0, 0, _Leader, _Data, _Counter) -> 171 | ok; 172 | client_loop(Num, Sent, _Leader, Data, Counter) -> 173 | receive 174 | {ra_event, Leader, {applied, Applied}} -> 175 | N = length(Applied), 176 | ToSend = min(Sent, N), 177 | send_n(Leader, Data, ToSend, Counter), 178 | client_loop(Num - N, Sent - ToSend, Leader, Data, Counter); 179 | {ra_event, _, {rejected, {not_leader, NewLeader, _}}} -> 180 | io:format("new leader ~w~n", [NewLeader]), 181 | send_n(NewLeader, Data, 1, Counter), 182 | client_loop(Num, Sent, NewLeader, Data, Counter); 183 | {ra_event, Leader, Evt} -> 184 | io:format("unexpected ra_event ~w~n", [Evt]), 185 | client_loop(Num, Sent, Leader, Data, Counter) 186 | end. 187 | 188 | spawn_client(Parent, Leader, Num, DataSize, Pipe, Counter) -> 189 | Data = crypto:strong_rand_bytes(DataSize), 190 | spawn_link( 191 | fun () -> 192 | %% first send one 1000 noop commands 193 | %% then top up as they are applied 194 | receive 195 | go -> 196 | send_n(Leader, Data, Pipe, Counter), 197 | ok = client_loop(Num, Num - Pipe, Leader, Data, Counter), 198 | Parent ! {done, self()} 199 | end 200 | end). 201 | 202 | print_metrics(_Name) -> 203 | io:format("Node: ~w~n", [node()]), 204 | io:format("counters ~p~n", [ra_counters:overview()]). 205 | 206 | 207 | 208 | % profile() -> 209 | % GzFile = atom_to_list(node()) ++ ".gz", 210 | % lg:trace([noop, ra_server, ra_server_proc, ra_snapshot, ra_machine, 211 | % ra_log, ra_flru, ra_machine, ra_log_meta, ra_log_segment], 212 | % lg_file_tracer, 213 | % GzFile, #{running => false, mode => profile}), 214 | % ok. 215 | 216 | % stop_profile() -> 217 | % lg:stop(), 218 | % Base = atom_to_list(node()), 219 | % GzFile = Base ++ ".gz.*", 220 | % lg_callgrind:profile_many(GzFile, Base ++ ".out",#{}), 221 | % ok. 222 | -------------------------------------------------------------------------------- /docs/ra_directory.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 || init/1 | |
| init/2 | |
| deinit/1 | |
| register_name/6 | |
| unregister_name/2 | |
| where_is/2 | |
| where_is_parent/2 | |
| name_of/2 | |
| cluster_name_of/2 | |
| pid_of/2 | |
| uid_of/2 | |
| overview/1 | |
| list_registered/1 | |
| is_registered_uid/2 |
init(System :: atom()) -> ok | {error, system_not_started}
38 | 39 |
init(Dir :: file:filename(), X2 :: ra_system:names()) -> ok44 |
45 |
deinit(System :: atom() | ra_system:names()) -> ok50 |
51 |
register_name(System :: ra_system:names() | atom(), 56 | UId :: ra_uid(), 57 | Pid :: pid(), 58 | ParentPid :: option(pid()), 59 | ServerName :: atom(), 60 | ClusterName :: ra_cluster_name()) -> 61 | ok62 |
63 |
unregister_name(System :: atom() | ra_system:names(), 68 | UId :: ra_uid()) -> 69 | ra_uid()70 |
71 |
where_is(System :: atom() | ra_system:names(), 76 | ServerName :: ra_uid() | atom()) -> 77 | pid() | undefined78 |
79 |
where_is_parent(System :: atom() | ra_system:names(), 84 | ServerName :: ra_uid() | atom()) -> 85 | pid() | undefined86 |
87 |
name_of(SystemOrNames :: atom() | ra_system:names(), 92 | UId :: ra_uid()) -> 93 | option(atom())94 |
95 |
cluster_name_of(SystemOrNames :: ra_system:names() | atom(), 100 | UId :: ra_uid()) -> 101 | option(ra_cluster_name())102 |
103 |
pid_of(SystemOrNames :: atom() | ra_system:names(), 108 | UId :: ra_uid()) -> 109 | option(pid())110 |
111 |
uid_of(System, ServerName) -> any()
117 |
overview(System) -> any()
123 |
list_registered(System :: atom()) -> [{atom(), ra_uid()}]
128 | 129 |
is_registered_uid(SystemOrNames :: atom() | ra_system:names(), 134 | UId :: ra_uid()) -> 135 | boolean()136 |
137 |
Generated by EDoc
142 | 143 | 144 | --------------------------------------------------------------------------------