├── .github ├── ISSUE_TEMPLATE │ ├── blank.md │ ├── bug_report.md │ ├── doc-request.md │ └── feature_request.md ├── actions-rs │ └── grcov.yml ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── README.md │ ├── chaos.yml │ ├── ci.yaml │ ├── commit-message-check.yml │ ├── coverage.yml │ ├── devskim-analysis.yml │ ├── gtp-translate.yml │ ├── issue-cmds.yml │ └── issue-welcome.yml ├── .gitignore ├── .mergify.yml ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile ├── README.md ├── book.toml ├── change-log.md ├── change-log ├── v0.6.1.md ├── v0.6.2-alpha.1.md ├── v0.6.2-alpha.10.md ├── v0.6.2-alpha.11.md ├── v0.6.2-alpha.12.md ├── v0.6.2-alpha.13.md ├── v0.6.2-alpha.14.md ├── v0.6.2-alpha.15.md ├── v0.6.2-alpha.16.md ├── v0.6.2-alpha.2.md ├── v0.6.2-alpha.3.md ├── v0.6.2-alpha.4.md ├── v0.6.2-alpha.5.md ├── v0.6.2-alpha.6.md ├── v0.6.2-alpha.7.md ├── v0.6.2-alpha.8.md ├── v0.6.2-alpha.9.md ├── v0.6.2-alpha.md ├── v0.6.2.md ├── v0.6.3.md ├── v0.6.4.md ├── v0.7.0-alpha.1.md ├── v0.7.0-alpha.2.md ├── v0.7.0-alpha.3.md ├── v0.7.0-alpha.4.md ├── v0.7.0-alpha.5.md ├── v0.7.0.md ├── v0.7.1.md ├── v0.7.2.md ├── v0.7.3.md ├── v0.7.4.md ├── v0.8.0.md ├── v0.8.1.md ├── v0.8.2.md ├── v0.8.3.md ├── v0.8.4.md └── v0.9.0.md ├── clippy.toml ├── cluster_benchmark ├── Cargo.toml ├── README.md └── tests │ ├── README.md │ └── benchmark │ ├── bench_cluster.rs │ ├── main.rs │ ├── network.rs │ ├── store.rs │ └── store │ └── test.rs ├── derived-from-async-raft.md ├── examples ├── README.md ├── client-http │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── lib.rs ├── mem-log │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── lib.rs │ │ └── log_store.rs │ └── test-cluster.sh ├── memstore ├── network-v1-http │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── lib.rs ├── raft-kv-memstore-grpc │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── build.rs │ ├── proto │ │ ├── app.proto │ │ ├── app_types.proto │ │ └── raft.proto │ ├── src │ │ ├── app.rs │ │ ├── bin │ │ │ └── main.rs │ │ ├── grpc │ │ │ ├── app_service.rs │ │ │ ├── mod.rs │ │ │ └── raft_service.rs │ │ ├── lib.rs │ │ ├── network │ │ │ └── mod.rs │ │ ├── pb_impl │ │ │ ├── impl_append_entries_request.rs │ │ │ ├── impl_append_entries_response.rs │ │ │ ├── impl_client_write_response.rs │ │ │ ├── impl_entry.rs │ │ │ ├── impl_leader_id.rs │ │ │ ├── impl_log_id.rs │ │ │ ├── impl_membership.rs │ │ │ ├── impl_snapshot_request.rs │ │ │ ├── impl_vote.rs │ │ │ ├── impl_vote_request.rs │ │ │ ├── impl_vote_response.rs │ │ │ └── mod.rs │ │ ├── store │ │ │ └── mod.rs │ │ └── test_store.rs │ ├── test-cluster.sh │ └── tests │ │ └── test_cluster.rs ├── raft-kv-memstore-network-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── api.rs │ │ ├── app.rs │ │ ├── lib.rs │ │ ├── network.rs │ │ ├── router.rs │ │ └── store.rs │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ ├── main.rs │ │ └── test_cluster.rs ├── raft-kv-memstore-opendal-snapshot-data │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── api.rs │ │ ├── app.rs │ │ ├── lib.rs │ │ ├── network.rs │ │ ├── router.rs │ │ └── store.rs │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ ├── main.rs │ │ └── test_cluster.rs ├── raft-kv-memstore-singlethreaded │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── api.rs │ │ ├── app.rs │ │ ├── lib.rs │ │ ├── network.rs │ │ ├── router.rs │ │ └── store.rs │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ ├── main.rs │ │ └── test_cluster.rs ├── raft-kv-memstore │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── app.rs │ │ ├── bin │ │ │ └── main.rs │ │ ├── lib.rs │ │ ├── network │ │ │ ├── api.rs │ │ │ ├── management.rs │ │ │ ├── mod.rs │ │ │ └── raft.rs │ │ ├── store │ │ │ └── mod.rs │ │ └── test.rs │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ ├── main.rs │ │ └── test_cluster.rs ├── raft-kv-rocksdb │ ├── .gitignore │ ├── Cargo.toml │ ├── src │ │ ├── app.rs │ │ ├── bin │ │ │ └── main.rs │ │ ├── lib.rs │ │ ├── network │ │ │ ├── api.rs │ │ │ ├── management.rs │ │ │ ├── mod.rs │ │ │ └── raft.rs │ │ └── store.rs │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ ├── main.rs │ │ └── test_cluster.rs ├── rocksstore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── lib.rs │ │ ├── log_store.rs │ │ └── test.rs │ └── test-cluster.sh └── utils │ ├── README.md │ └── declare_types.rs ├── guide ├── README.md └── src │ ├── custom.css │ └── images │ └── raft-overview.png ├── macros ├── Cargo.toml ├── README.md ├── src │ ├── expand.rs │ ├── lib.rs │ ├── lib_readme.md │ ├── since.rs │ └── utils.rs └── tests │ ├── expand │ ├── expand │ │ ├── keyed_let_0.expanded.rs │ │ ├── keyed_let_0.rs │ │ ├── keyed_let_1.expanded.rs │ │ ├── keyed_let_1.rs │ │ ├── keyed_let_dup_a.expanded.rs │ │ ├── keyed_let_dup_a.rs │ │ ├── not_keyed_let_dup_a.expanded.rs │ │ ├── not_keyed_let_dup_a.rs │ │ ├── variable_empty.expanded.rs │ │ ├── variable_empty.rs │ │ ├── variable_type_attributes.expanded.rs │ │ └── variable_type_attributes.rs │ └── fail │ │ ├── arg0_invalid_keyed.rs │ │ ├── arg0_invalid_keyed.stderr │ │ ├── variable_absent.rs │ │ └── variable_absent.stderr │ ├── since │ ├── expand │ │ ├── multi_since.expanded.rs │ │ ├── multi_since.rs │ │ ├── valid_semver.expanded.rs │ │ ├── valid_semver.rs │ │ ├── with_change.expanded.rs │ │ ├── with_change.rs │ │ ├── with_date.expanded.rs │ │ ├── with_date.rs │ │ ├── with_date_change.expanded.rs │ │ ├── with_date_change.rs │ │ ├── with_date_many_change.expanded.rs │ │ ├── with_date_many_change.rs │ │ ├── with_doc.expanded.rs │ │ └── with_doc.rs │ └── fail │ │ ├── invalid_date.rs │ │ ├── invalid_date.stderr │ │ ├── invalid_sem_ver.rs │ │ └── invalid_sem_ver.stderr │ ├── test_expand.rs │ └── test_since.rs ├── openraft ├── .gitignore ├── Cargo.toml ├── README.md └── src │ ├── base │ ├── finalized.rs │ └── mod.rs │ ├── change_members.rs │ ├── compat │ ├── mod.rs │ └── upgrade.rs │ ├── config │ ├── config.rs │ ├── config_test.rs │ ├── error.rs │ └── mod.rs │ ├── core │ ├── balancer.rs │ ├── heartbeat │ │ ├── event.rs │ │ ├── handle.rs │ │ ├── mod.rs │ │ └── worker.rs │ ├── mod.rs │ ├── notification.rs │ ├── raft_core.rs │ ├── raft_msg │ │ ├── external_command.rs │ │ └── mod.rs │ ├── replication_state.rs │ ├── server_state.rs │ ├── sm │ │ ├── command.rs │ │ ├── handle.rs │ │ ├── mod.rs │ │ ├── response.rs │ │ └── worker.rs │ └── tick.rs │ ├── display_ext.rs │ ├── display_ext │ ├── display_btreemap_opt_value.rs │ ├── display_instant.rs │ ├── display_option.rs │ ├── display_result.rs │ └── display_slice.rs │ ├── docs │ ├── cluster_control │ │ ├── cluster-control.md │ │ ├── cluster-formation.md │ │ ├── dynamic-membership.md │ │ ├── mod.rs │ │ └── node-lifecycle.md │ ├── components │ │ ├── mod.rs │ │ └── state-machine.md │ ├── data │ │ ├── effective-membership.md │ │ ├── extended-membership.md │ │ ├── leader-lease.md │ │ ├── leader_id.md │ │ ├── log_pointers.md │ │ ├── mod.rs │ │ ├── replication-session.md │ │ └── vote.md │ ├── docs.md │ ├── faq │ │ ├── Makefile │ │ ├── faq-toc.md │ │ ├── faq.md │ │ └── mod.rs │ ├── feature_flags │ │ ├── Makefile │ │ ├── feature-flags-toc.md │ │ ├── feature-flags.md │ │ └── mod.rs │ ├── getting_started │ │ ├── getting-started.md │ │ └── mod.rs │ ├── internal │ │ ├── architecture.md │ │ ├── internal.md │ │ ├── mod.rs │ │ └── threading.md │ ├── mod.rs │ ├── obsolete │ │ ├── blank-log-heartbeat.md │ │ ├── fast_commit.md │ │ └── mod.rs │ ├── protocol │ │ ├── leader_lease.md │ │ ├── log_replication.md │ │ ├── log_stream.md │ │ ├── mod.rs │ │ ├── read.md │ │ ├── replication.md │ │ └── snapshot_replication.md │ └── upgrade_guide │ │ ├── mod.rs │ │ ├── upgrade-v08-v09.md │ │ └── upgrade.md │ ├── engine │ ├── command.rs │ ├── command_kind.rs │ ├── engine_config.rs │ ├── engine_impl.rs │ ├── engine_output.rs │ ├── handler │ │ ├── establish_handler │ │ │ └── mod.rs │ │ ├── following_handler │ │ │ ├── append_entries_test.rs │ │ │ ├── commit_entries_test.rs │ │ │ ├── do_append_entries_test.rs │ │ │ ├── install_snapshot_test.rs │ │ │ ├── mod.rs │ │ │ ├── truncate_logs_test.rs │ │ │ └── update_committed_membership_test.rs │ │ ├── leader_handler │ │ │ ├── append_entries_test.rs │ │ │ ├── get_read_log_id_test.rs │ │ │ ├── mod.rs │ │ │ ├── send_heartbeat_test.rs │ │ │ └── transfer_leader_test.rs │ │ ├── log_handler │ │ │ ├── calc_purge_upto_test.rs │ │ │ ├── mod.rs │ │ │ └── purge_log_test.rs │ │ ├── mod.rs │ │ ├── replication_handler │ │ │ ├── append_membership_test.rs │ │ │ ├── mod.rs │ │ │ └── update_matching_test.rs │ │ ├── server_state_handler │ │ │ ├── mod.rs │ │ │ └── update_server_state_test.rs │ │ ├── snapshot_handler │ │ │ ├── mod.rs │ │ │ ├── trigger_snapshot_test.rs │ │ │ └── update_snapshot_test.rs │ │ └── vote_handler │ │ │ ├── accept_vote_test.rs │ │ │ ├── become_leader_test.rs │ │ │ ├── handle_message_vote_test.rs │ │ │ └── mod.rs │ ├── leader_log_ids.rs │ ├── log_id_list.rs │ ├── mod.rs │ ├── replication_progress.rs │ ├── testing.rs │ ├── tests │ │ ├── append_entries_test.rs │ │ ├── elect_test.rs │ │ ├── handle_vote_req_test.rs │ │ ├── handle_vote_resp_test.rs │ │ ├── initialize_test.rs │ │ ├── install_full_snapshot_test.rs │ │ ├── log_id_list_test.rs │ │ ├── startup_test.rs │ │ └── trigger_purge_log_test.rs │ └── time_state.rs │ ├── entry │ ├── mod.rs │ ├── payload.rs │ ├── raft_entry_ext.rs │ └── traits.rs │ ├── error.rs │ ├── error │ ├── allow_next_revert_error.rs │ ├── decompose.rs │ ├── into_ok.rs │ ├── into_raft_result.rs │ ├── invalid_sm.rs │ ├── membership_error.rs │ ├── node_not_found.rs │ ├── operation.rs │ ├── replication_closed.rs │ └── streaming_error.rs │ ├── feature_serde_test.rs │ ├── impls │ └── mod.rs │ ├── instant.rs │ ├── lib.rs │ ├── lib_readme.md │ ├── log_id │ ├── log_id_option_ext.rs │ ├── log_index_option_ext.rs │ ├── mod.rs │ ├── option_raft_log_id_ext.rs │ ├── option_ref_log_id_ext.rs │ ├── raft_log_id.rs │ ├── raft_log_id_ext.rs │ └── ref_log_id.rs │ ├── log_id_range.rs │ ├── membership │ ├── bench │ │ ├── is_quorum.rs │ │ └── mod.rs │ ├── effective_membership.rs │ ├── effective_membership_test.rs │ ├── into_nodes.rs │ ├── membership.rs │ ├── membership_test.rs │ ├── mod.rs │ └── stored_membership.rs │ ├── metrics │ ├── metric.rs │ ├── metric_display.rs │ ├── mod.rs │ ├── raft_metrics.rs │ ├── serde_instant.rs │ ├── wait.rs │ ├── wait_condition.rs │ └── wait_test.rs │ ├── network │ ├── backoff.rs │ ├── mod.rs │ ├── rpc_option.rs │ ├── rpc_type.rs │ ├── snapshot_transport.rs │ ├── v1 │ │ ├── factory.rs │ │ ├── mod.rs │ │ └── network.rs │ └── v2 │ │ ├── adapt_v1.rs │ │ ├── mod.rs │ │ └── network.rs │ ├── node.rs │ ├── progress │ ├── bench │ │ ├── mod.rs │ │ └── vec_progress_update.rs │ ├── entry │ │ ├── mod.rs │ │ ├── tests.rs │ │ └── update.rs │ ├── inflight │ │ ├── mod.rs │ │ └── tests.rs │ └── mod.rs │ ├── proposer │ ├── candidate.rs │ ├── leader.rs │ ├── leader_state.rs │ └── mod.rs │ ├── quorum │ ├── bench │ │ ├── is_quorum.rs │ │ └── mod.rs │ ├── coherent.rs │ ├── coherent_impl.rs │ ├── coherent_test.rs │ ├── joint.rs │ ├── joint_impl.rs │ ├── mod.rs │ ├── quorum_set.rs │ ├── quorum_set_impl.rs │ └── quorum_set_test.rs │ ├── raft │ ├── api │ │ ├── app.rs │ │ ├── management.rs │ │ ├── mod.rs │ │ └── protocol.rs │ ├── core_state.rs │ ├── declare_raft_types_test.rs │ ├── impl_raft_blocking_write.rs │ ├── linearizable_read │ │ ├── linearize_state.rs │ │ ├── linearizer.rs │ │ └── mod.rs │ ├── message │ │ ├── append_entries.rs │ │ ├── client_write.rs │ │ ├── install_snapshot.rs │ │ ├── mod.rs │ │ ├── transfer_leader.rs │ │ └── vote.rs │ ├── mod.rs │ ├── raft_inner.rs │ ├── responder │ │ ├── either.rs │ │ ├── impls.rs │ │ └── mod.rs │ ├── runtime_config_handle.rs │ └── trigger.rs │ ├── raft_state │ ├── io_state.rs │ ├── io_state │ │ ├── io_id.rs │ │ ├── io_progress.rs │ │ └── log_io_id.rs │ ├── log_state_reader.rs │ ├── membership_state │ │ ├── change_handler.rs │ │ ├── change_handler_test.rs │ │ ├── membership_state_test.rs │ │ └── mod.rs │ ├── mod.rs │ ├── tests │ │ ├── forward_to_leader_test.rs │ │ ├── is_initialized_test.rs │ │ ├── log_state_reader_test.rs │ │ └── validate_test.rs │ └── vote_state_reader.rs │ ├── raft_types.rs │ ├── replication │ ├── callbacks.rs │ ├── hint.rs │ ├── mod.rs │ ├── replication_session_id.rs │ ├── request.rs │ └── response.rs │ ├── runtime │ └── mod.rs │ ├── storage │ ├── callback.rs │ ├── helper.rs │ ├── log_reader_ext.rs │ ├── log_state.rs │ ├── mod.rs │ ├── snapshot.rs │ ├── snapshot_meta.rs │ ├── snapshot_signature.rs │ └── v2 │ │ ├── mod.rs │ │ ├── raft_log_reader.rs │ │ ├── raft_log_storage.rs │ │ ├── raft_log_storage_ext.rs │ │ ├── raft_snapshot_builder.rs │ │ └── raft_state_machine.rs │ ├── storage_error.rs │ ├── summary.rs │ ├── testing │ ├── common.rs │ ├── log │ │ ├── mod.rs │ │ ├── store_builder.rs │ │ └── suite.rs │ ├── mod.rs │ └── runtime │ │ └── mod.rs │ ├── try_as_ref.rs │ ├── type_config.rs │ ├── type_config │ ├── async_runtime │ │ ├── mod.rs │ │ ├── mpsc │ │ │ └── mod.rs │ │ ├── mpsc_unbounded │ │ │ ├── mod.rs │ │ │ ├── send_error.rs │ │ │ └── try_recv_error.rs │ │ ├── mutex.rs │ │ ├── oneshot.rs │ │ ├── tokio_impls │ │ │ └── tokio_runtime.rs │ │ └── watch │ │ │ ├── mod.rs │ │ │ └── watch_error.rs │ └── util.rs │ ├── utime.rs │ └── vote │ ├── committed.rs │ ├── leader_id │ ├── leader_id_adv.rs │ ├── leader_id_cmp.rs │ ├── leader_id_std.rs │ ├── mod.rs │ ├── raft_committed_leader_id.rs │ └── raft_leader_id.rs │ ├── mod.rs │ ├── non_committed.rs │ ├── raft_term │ ├── mod.rs │ └── raft_term_impls.rs │ ├── raft_vote.rs │ ├── ref_vote.rs │ ├── vote.rs │ └── vote_status.rs ├── raft-essentials.md ├── rt-compio ├── Cargo.toml ├── README.md └── src │ ├── lib.rs │ ├── mpsc.rs │ ├── mpsc_unbounded.rs │ ├── mutex.rs │ ├── oneshot.rs │ └── watch.rs ├── rt-monoio ├── Cargo.toml ├── README.md └── src │ └── lib.rs ├── rust-toolchain ├── rustfmt.toml ├── scripts ├── README.md ├── build_change_log.py ├── change-types.yaml ├── check.kdl ├── mprocs-check.yaml ├── requirements.txt └── watch-doc.sh ├── stores ├── README.md └── memstore │ ├── Cargo.toml │ ├── README.md │ └── src │ ├── lib.rs │ └── test.rs └── tests ├── Cargo.toml ├── src └── main.rs └── tests ├── README.md ├── append_entries ├── main.rs ├── t10_conflict_with_empty_entries.rs ├── t10_see_higher_vote.rs ├── t11_append_conflicts.rs ├── t11_append_entries_with_bigger_term.rs ├── t11_append_inconsistent_log.rs ├── t11_append_updates_membership.rs ├── t30_replication_1_voter_to_isolated_learner.rs ├── t60_enable_heartbeat.rs ├── t61_heartbeat_reject_vote.rs ├── t61_large_heartbeat.rs └── t90_issue_216_stale_last_log_id.rs ├── client_api ├── main.rs ├── t10_client_writes.rs ├── t11_client_reads.rs ├── t12_trigger_purge_log.rs ├── t13_begin_receiving_snapshot.rs ├── t13_get_snapshot.rs ├── t13_install_full_snapshot.rs ├── t13_trigger_snapshot.rs ├── t14_transfer_leader.rs ├── t16_with_raft_state.rs ├── t16_with_state_machine.rs ├── t50_lagging_network_write.rs └── t51_write_when_leader_quit.rs ├── elect ├── main.rs ├── t10_elect_compare_last_log.rs └── t11_elect_seize_leadership.rs ├── fixtures ├── logging.rs └── mod.rs ├── life_cycle ├── main.rs ├── t10_initialization.rs ├── t11_shutdown.rs ├── t50_follower_restart_does_not_interrupt.rs ├── t50_single_follower_restart.rs ├── t50_single_leader_restart_re_apply_logs.rs ├── t90_issue_607_single_restart.rs └── t90_issue_920_non_voter_leader_restart.rs ├── log_store ├── main.rs └── t10_save_committed.rs ├── management ├── main.rs └── t10_raft_config.rs ├── membership ├── main.rs ├── t10_learner_restart.rs ├── t10_single_node.rs ├── t11_add_learner.rs ├── t12_concurrent_write_and_add_learner.rs ├── t20_change_membership.rs ├── t21_change_membership_cases.rs ├── t30_commit_joint_config.rs ├── t30_elect_with_new_config.rs ├── t31_add_remove_follower.rs ├── t31_remove_leader.rs ├── t31_removed_follower.rs ├── t51_remove_unreachable_follower.rs ├── t52_change_membership_on_uninitialized_node.rs ├── t99_issue_471_adding_learner_uses_uninit_leader_id.rs ├── t99_issue_584_replication_state_reverted.rs └── t99_new_leader_auto_commit_uniform_config.rs ├── metrics ├── main.rs ├── t10_current_leader.rs ├── t10_leader_last_ack.rs ├── t10_purged.rs ├── t10_server_metrics_and_data_metrics.rs ├── t20_metrics_state_machine_consistency.rs ├── t30_leader_metrics.rs └── t40_metrics_wait.rs ├── replication ├── main.rs ├── t10_append_entries_partial_success.rs ├── t50_append_entries_backoff.rs ├── t50_append_entries_backoff_rejoin.rs ├── t51_append_entries_too_large.rs ├── t60_feature_loosen_follower_log_revert.rs └── t61_allow_follower_log_revert.rs ├── snapshot_building ├── main.rs ├── t10_build_snapshot.rs ├── t35_building_snapshot_does_not_block_append.rs ├── t35_building_snapshot_does_not_block_apply.rs └── t60_snapshot_policy_never.rs ├── snapshot_streaming ├── main.rs ├── t10_api_install_snapshot.rs ├── t10_api_install_snapshot_with_lower_vote.rs ├── t20_startup_snapshot.rs ├── t30_purge_in_snapshot_logs.rs ├── t31_snapshot_overrides_membership.rs ├── t32_snapshot_uses_prev_snap_membership.rs ├── t33_snapshot_delete_conflict_logs.rs ├── t34_replication_does_not_block_purge.rs ├── t50_snapshot_line_rate_to_snapshot.rs ├── t50_snapshot_when_lacking_log.rs ├── t51_after_snapshot_add_learner_and_request_a_log.rs ├── t60_snapshot_chunk_size.rs └── t90_issue_808_snapshot_to_unreachable_node_should_not_block.rs └── state_machine ├── main.rs ├── t10_total_order_apply.rs └── t20_state_machine_apply_membership.rs /.github/ISSUE_TEMPLATE/blank.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Blank 3 | about: Unclassified issue 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. 'cargo ...' 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Actual behavior** 22 | applicable, add screenshots to help explain your problem. 23 | 24 | **Env (please complete the following information):** 25 | - Openraft version [e.g., a tag such as `v0.7.2` or a commit hash] 26 | - Does the bug still there in the latest version?(`main` for the latest release branch, or `release-0.7` for 0.7 branch)? 27 | - Rust-toolchain: [e.g. `cargo 1.65.0-nightly (ce40690a5 2022-08-09)`; Find toolchain with `cargo version` in a crate dir] 28 | - OS: [e.g. mac os 12.0.1, centos 7.0] 29 | - CPU: [e.g. mac m1, x86_64] 30 | 31 | **Additional files:** 32 | - Logs: Attach logs generate by unittest(e.g., `./openraft/_log/*`), or logs output by examples(`examples/raft-kv-*/test-cluster.sh` will print logs in `n1.log`, `n2.log`, `n3.log`) 33 | - Cargo.lock: [e.g. `./openraft/Cargo.lock`, `./examples/raft-kv-memstore/Cargo.lock`] 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request for documentation 3 | about: Suggest more doc to explain something. 4 | title: '' 5 | labels: 'documentation' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What kind of doc do you ask for?** 11 | - [ ] Code comments. 12 | - [ ] Architecture design. 13 | - [ ] Algorithm explanation. 14 | - [ ] Guide. 15 | - [ ] Examples. 16 | 17 | **Describe the information you want** 18 | A clear and concise description of what you want. 19 | 20 | **Describe the status of the current doc** 21 | - Is there any doc yet?: 22 | - doc-1: [e.g. url] 23 | - doc-2: [e.g. url] 24 | - What other information should be added? 25 | - [e.g. ] 26 | 27 | - [ ] I'll open a pull request for it:) 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/actions-rs/grcov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "memstore*" 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "17:00" 8 | open-pull-requests-limit: 2 9 | 10 | - package-ecosystem: cargo 11 | directory: "/examples/raft-kv-memstore" 12 | schedule: 13 | interval: daily 14 | time: "17:00" 15 | open-pull-requests-limit: 2 16 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 15 | 16 | **Checklist** 17 | 18 | - [ ] Updated guide with pertinent info (may not always apply). 19 | - [ ] Squash down commits to one or two logical commits which clearly describe the work you've done. 20 | - [ ] Unittest is a friend:) 21 | -------------------------------------------------------------------------------- /.github/workflows/chaos.yml: -------------------------------------------------------------------------------- 1 | name: chaos-test 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | 7 | chaos_test: 8 | name: unittest 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | matrix: 13 | include: 14 | - toolchain: "nightly" 15 | store_defensive: "on" 16 | send_delay: "30" 17 | - toolchain: "nightly" 18 | store_defensive: "on" 19 | send_delay: "0" 20 | 21 | steps: 22 | - name: Setup | Checkout 23 | uses: actions/checkout@v4 24 | 25 | 26 | - name: Setup | Toolchain 27 | uses: actions-rs/toolchain@v1.0.6 28 | with: 29 | toolchain: "${{ matrix.toolchain }}" 30 | override: true 31 | components: rustfmt, clippy 32 | 33 | 34 | # TODO: unit test should be replaced 35 | - name: Unit Tests, with and without defensive store 36 | uses: actions-rs/cargo@v1 37 | with: 38 | command: test 39 | env: 40 | # Parallel tests block each other and result in timeout. 41 | RUST_TEST_THREADS: 2 42 | RUST_LOG: debug 43 | RUST_BACKTRACE: full 44 | OPENRAFT_STORE_DEFENSIVE: ${{ matrix.store_defensive }} 45 | OPENRAFT_NETWORK_SEND_DELAY: ${{ matrix.send_delay }} 46 | 47 | 48 | - name: Upload artifact 49 | uses: actions/upload-artifact@v4 50 | if: failure() 51 | with: 52 | name: ut 53 | path: | 54 | openraft/_log/ 55 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Unit test coverage 2 | on: 3 | push: 4 | branches: 5 | - "release-*" 6 | - "main" 7 | 8 | jobs: 9 | coverage: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - uses: actions-rs/toolchain@v1 15 | with: 16 | toolchain: nightly 17 | override: true 18 | components: llvm-tools-preview 19 | 20 | 21 | - name: Install cargo-llvm-cov 22 | uses: taiki-e/install-action@cargo-llvm-cov 23 | 24 | 25 | - name: Install cargo-expand 26 | run: cargo install cargo-expand 27 | 28 | 29 | - name: Generate coverage report 30 | # To test with all features: 31 | # run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info 32 | run: cargo llvm-cov --workspace --lcov --output-path lcov.info 33 | env: 34 | RUST_TEST_THREADS: 2 35 | 36 | 37 | - name: Upload to Coveralls 38 | uses: coverallsapp/github-action@v2 39 | with: 40 | github-token: ${{ secrets.GITHUB_TOKEN }} 41 | path-to-lcov: lcov.info 42 | -------------------------------------------------------------------------------- /.github/workflows/devskim-analysis.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | name: DevSkim 7 | 8 | on: 9 | push: 10 | branches: [ master ] 11 | pull_request: 12 | branches: [ master ] 13 | schedule: 14 | - cron: '35 13 * * 5' 15 | 16 | jobs: 17 | lint: 18 | name: DevSkim 19 | runs-on: ubuntu-20.04 20 | permissions: 21 | actions: read 22 | contents: read 23 | security-events: write 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v4 27 | 28 | - name: Run DevSkim scanner 29 | uses: microsoft/DevSkim-Action@v1 30 | 31 | - name: Upload DevSkim scan results to GitHub Security tab 32 | uses: github/codeql-action/upload-sarif@v1 33 | with: 34 | sarif_file: devskim-results.sarif 35 | -------------------------------------------------------------------------------- /.github/workflows/gtp-translate.yml: -------------------------------------------------------------------------------- 1 | name: GPT refine markdown 2 | 3 | on: 4 | issue_comment: 5 | types: [ created ] 6 | 7 | jobs: 8 | gpt_translate: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Run GPT Translate 15 | if: | 16 | contains(github.event.comment.body, '/gpt-translate') || 17 | contains(github.event.comment.body, '/gt') 18 | 19 | uses: drmingdrmer/gpt-refine-md@master 20 | with: 21 | apikey: ${{ secrets.OPENAI_API_KEY }} 22 | -------------------------------------------------------------------------------- /.github/workflows/issue-cmds.yml: -------------------------------------------------------------------------------- 1 | on: 2 | issue_comment: 3 | types: 4 | - created 5 | 6 | jobs: 7 | assignme: 8 | name: /assignme 9 | runs-on: ubuntu-latest 10 | if: startsWith(github.event.comment.body, '/assignme') 11 | 12 | steps: 13 | - uses: xt0rted/slash-command-action@v1 14 | with: 15 | repo-token: ${{ secrets.GITHUB_TOKEN }} 16 | command: assignme 17 | reaction: "true" 18 | reaction-type: "rocket" 19 | permission-level: read 20 | 21 | - uses: actions-ecosystem/action-add-assignees@v1 22 | with: 23 | github_token: ${{ secrets.github_token }} 24 | assignees: ${{ github.actor }} 25 | 26 | 27 | help: 28 | name: /help 29 | runs-on: ubuntu-latest 30 | if: startsWith(github.event.comment.body, '/help') 31 | 32 | steps: 33 | - uses: xt0rted/slash-command-action@v1 34 | with: 35 | repo-token: ${{ secrets.GITHUB_TOKEN }} 36 | command: help 37 | reaction: "true" 38 | reaction-type: "rocket" 39 | 40 | - uses: peter-evans/create-or-update-comment@v1 41 | with: 42 | issue-number: ${{ github.event.issue.number }} 43 | body: | 44 | Get help or engage by: 45 | 46 | - `/help` : to print help messages. 47 | - `/assignme` : to assign this issue to you. 48 | -------------------------------------------------------------------------------- /.github/workflows/issue-welcome.yml: -------------------------------------------------------------------------------- 1 | 2 | on: 3 | issues: 4 | types: 5 | - opened 6 | 7 | # https://github.com/marketplace/actions/create-or-update-comment 8 | 9 | jobs: 10 | welcome: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: peter-evans/create-or-update-comment@v1 14 | with: 15 | issue-number: ${{ github.event.issue.number }} 16 | body: | 17 | 👋 Thanks for opening this issue! 18 | 19 | Get help or engage by: 20 | 21 | - `/help` : to print help messages. 22 | - `/assignme` : to assign this issue to you. 23 | 24 | 25 | # Another way: actions/github-script 26 | # # https://github.com/actions/github-script#comment-on-an-issue 27 | 28 | # jobs: 29 | # welcome: 30 | # runs-on: ubuntu-latest 31 | # steps: 32 | # - uses: actions/github-script@v5 33 | # with: 34 | # script: | 35 | # github.rest.issues.createComment({ 36 | # issue_number: context.issue.number, 37 | # owner: context.repo.owner, 38 | # repo: context.repo.repo, 39 | # body: [ 40 | # '👋 Thanks for opening this issue!', 41 | # '', 42 | # 'Get help or engage by:', 43 | # '', 44 | # '- `/help` : to print help messages.', 45 | # '- `/assignme` : to assign this issue to you.', 46 | # ].join('\n') 47 | # }) 48 | 49 | 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Directory Ignores ########################################################## 3 | guide/book 4 | target 5 | vendor 6 | .idea 7 | tests/_log 8 | 9 | # File Ignores ############################################################### 10 | **/*.rs.bk 11 | Cargo.lock 12 | .DS_Store 13 | .vscode 14 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | queue_rules: 2 | - name: feature_queue 3 | conditions: 4 | # - '#check-pending=0' 5 | - '#check-success>=2' 6 | - check-success=check-subject 7 | - check-success=openraft-test (stable, 0) 8 | - check-success~=openraft-test 9 | 10 | pull_request_rules: 11 | 12 | - name: put into queue if approved 13 | conditions: 14 | - "#approved-reviews-by>=1" 15 | - "#changes-requested-reviews-by=0" 16 | - check-success=check-subject 17 | - check-success=openraft-test (stable, 0) 18 | actions: 19 | queue: 20 | name: feature_queue 21 | 22 | - name: Delete head branch after merge 23 | conditions: 24 | - merged 25 | actions: 26 | delete_head_branch: 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # CONTRIBUTING 2 | 3 | This is a Rust project, so [rustup](https://rustup.rs/) is the best place to start. 4 | 5 | ## The guide 6 | 7 | The guide for this project is built using [mdBook](https://rust-lang-nursery.github.io/mdBook/index.html). 8 | Review their guide for more details on how to work with mdBook. Here are a few of the pertinents: 9 | 10 | ``` 11 | # Install the CLI. 12 | cargo install mdbook 13 | 14 | # Build the guide. 15 | mdbook build 16 | 17 | # Watch the FS for changes & rebuild. 18 | mdbook watch 19 | ``` 20 | 21 | ## Working with git 22 | 23 | - **Write the commit message like writing an email to your friends**. There is a great [git commit format guide](https://cbea.ms/git-commit/). 24 | 25 | - Do **rebase** and **squash** the branch onto the latest `main` branch before publishing a PR. 26 | 27 | - Do **NOT** **rebase** after publishing PR. Only merge. 28 | 29 | 30 | ## Release checklist 31 | 32 | - `make`: pass the unit test, format and clippy check. 33 | 34 | - Any documentation updates should also be reflected in the guide. 35 | 36 | - Ensure the Cargo.toml version for openraft or memstore has been updated, depending on which is being released. 37 | 38 | - Once the release CI has been finished, navigate to the release page, update the release info and publish the release. 39 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | authors = ["databendlabs"] 3 | language = "en" 4 | multilingual = false 5 | src = "guide/src" 6 | title = "openraft" 7 | description = "The openraft user guide." 8 | 9 | [build] 10 | build-dir = "guide/book" 11 | create-missing = false 12 | 13 | [preprocessor.svgbob] 14 | text_width = 8.0 15 | text_height = 16.0 16 | class = "bob" 17 | font_family = "arial" 18 | font_size = 14.0 19 | stroke_width = 2.0 20 | # there's using css-variables from theme: 21 | stroke_color = "var(--fg)" # see default theme / variables.css 22 | background_color = "transparent" # also useful `var(--bg)` 23 | # all properties are optional. 24 | 25 | 26 | [output.html] 27 | additional-css = ["guide/src/custom.css"] 28 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [1ad17e8e](https://github.com/databendlabs/openraft/commit/1ad17e8edf18d98eeb687f00a65ebf528ab3aeb7) move wait_for_xxx util into metrics.; by drdr xp; 2021-06-16 4 | 5 | Introduce struct `Wait` as a wrapper of the metrics channel to impl 6 | wait-for utils: 7 | - `log()`: wait for log to apply. 8 | - `current_leader()`: wait for known leader. 9 | - `state()`: wait for the role. 10 | - `members()`: wait for membership_config.members. 11 | - `next_members()`: wait for membership_config.members_after_consensus. 12 | 13 | E.g.: 14 | 15 | ```rust 16 | // wait for ever for raft node's current leader to become 3: 17 | r.wait(None).current_leader(2).await?; 18 | ``` 19 | 20 | The timeout is now an option arg to all wait_for_xxx functions in 21 | fixtures. wait_for_xxx_timeout are all removed. 22 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.11.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.6.2-alpha.11.md -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.12.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.6.2-alpha.12.md -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.13.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [2eccb9e1](https://github.com/databendlabs/openraft/commit/2eccb9e1f82f1bf71f6a2cf9ef6da7bf6232fa84) install snapshot req with offset GE 0 should not start a new session.; by drdr xp; 2021-08-22 4 | 5 | A install-snapshot always ends with a req with data len to be 0 and 6 | offset GE 0. 7 | If such a req is re-sent, e.g., when timeout, the receiver will try to 8 | install a snapshot with empty data, if it just finished the previous 9 | install snapshot req(`snapshot_state` is None) and do not reject a 10 | install snapshot req with offset GE 0. 11 | Which results in a `fatal storage error`, since the storage tries to 12 | decode an empty snapshot data. 13 | 14 | - feature: add config `install_snapshot_timeout`. 15 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.14.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [eee8e534](https://github.com/databendlabs/openraft/commit/eee8e534e0b0b9abdb37dd94aeb64dc1affd3ef7) snapshot replication does not need to send a last 0 size chunk; by drdr xp; 2021-08-22 4 | 5 | - Fixed: [8cd24ba0](https://github.com/databendlabs/openraft/commit/8cd24ba0f0212e94e61f21a7be0ce0806fcc66d5) RaftCore.entries_cache is inconsistent with storage. removed it.; by drdr xp; 2021-08-23 6 | 7 | - When leader changes, `entries_cache` is cleared. 8 | Thus there may be cached entries wont be applied to state machine. 9 | 10 | - When applying finished, the applied entries are not removed from the 11 | cache. 12 | Thus there could be entries being applied more than once. 13 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.2.md: -------------------------------------------------------------------------------- 1 | ### Dependency: 2 | 3 | - Dependency: [70e1773e](https://github.com/databendlabs/openraft/commit/70e1773edcf5e2bc7369c7afe47bb1348bc2a274) adapt to changes of rand-0.8: gen_range() accepts a range instead of two args; by drdr xp; 2021-06-21 4 | 5 | ### Added: 6 | 7 | - Added: [32a67e22](https://github.com/databendlabs/openraft/commit/32a67e228cf26f9207593805a0386cf6aa4fe294) add metrics about leader; by drdr xp; 2021-06-29 8 | 9 | In LeaderState it also report metrics about the replication to other node when report metrics. 10 | 11 | When switched to other state, LeaderState will be destroyed as long as 12 | the cached replication metrics. 13 | 14 | Other state report an `None` to raft core to override the previous 15 | metrics data. 16 | 17 | At some point the raft core, without knonwning the state, just report 18 | metrics with an `Update::Ignore`, to indicate that leave replication 19 | metrics intact. 20 | 21 | ### Fixed: 22 | 23 | - Fixed: [d60f1e85](https://github.com/databendlabs/openraft/commit/d60f1e852d3e5b9455589593067599d261f695b2) client_read has using wrong quorum=majority-1; by drdr xp; 2021-07-02 24 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.3.md: -------------------------------------------------------------------------------- 1 | ### Dependency: 2 | 3 | - Dependency: [b351c87f](https://github.com/databendlabs/openraft/commit/b351c87f0adfd0a6f1105f55cf1223c6045ecf41) upgrade tokio from 1.7 to 1.8; by drdr xp; 2021-07-08 4 | 5 | ### Fixed: 6 | 7 | - Fixed: [cf4badd0](https://github.com/databendlabs/openraft/commit/cf4badd0d762757519e2db5ed2f2fc65c2f49d02) leader should re-create and send snapshot when `threshold/2 < last_log_index - snapshot < threshold`; by drdr xp; 2021-07-08 8 | 9 | The problem: 10 | 11 | If `last_log_index` advances `snapshot.applied_index` too many, i.e.: 12 | `threshold/2 < last_log_index - snapshot < threshold` 13 | (e.g., `10/2 < 16-10 < 20` in the test that reproduce this bug), the leader 14 | tries to re-create a new snapshot. But when 15 | `last_log_index < threshold`, it won't create, which result in a dead 16 | loop. 17 | 18 | Solution: 19 | 20 | In such case, force to create a snapshot without considering the 21 | threshold. 22 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.6.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [82a3f2f9](https://github.com/databendlabs/openraft/commit/82a3f2f9c7ac37a0f24c6e0e8993c8d3bcee5666) use LogId to track last applied instead of using just an index.; by drdr xp; 2021-07-19 4 | 5 | It provides more info by Using LogId to track last applied log. 6 | E.g. when creating a snapshot, it need to walk through logs to find the 7 | term of the last applied log, just like it did in memstore impl. 8 | 9 | Using LogId{term, index} is a more natural way in every aspect. 10 | 11 | changes: RaftCore: change type of `last_applied` from u64 to LogId. 12 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.7.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.6.2-alpha.7.md -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.8.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [adc24f55](https://github.com/databendlabs/openraft/commit/adc24f55d75d9c7c01fcd0f4f9e35dd5aae679aa) pass all logs to apply_entry_to_state_machine(), not just Normal logs.; by drdr xp; 2021-08-16 4 | 5 | Pass `Entry` to `apply_entry_to_state_machine()`, not just the only 6 | `EntryPayload::Normal(normal_log)`. 7 | 8 | Thus the state machine is able to save the membership changes if it 9 | prefers to. 10 | 11 | Why: 12 | 13 | In practice, a snapshot contains info about all applied logs, including 14 | the membership config log. 15 | Before this change, the state machine does not receive any membership 16 | log thus when making a snapshot, one needs to walk through all applied 17 | logs to get the last membership that is included in state machine. 18 | 19 | By letting the state machine remember the membership log applied, 20 | the snapshto creation becomes more convenient and intuitive: it does not 21 | need to scan the applied logs any more. 22 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.9.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [8b59966d](https://github.com/databendlabs/openraft/commit/8b59966dd0a6bf804eb0ba978b5375010bfbc3f3) MembershipConfig.member type is changed form HashSet BTreeSet; by drdr xp; 2021-08-17 4 | -------------------------------------------------------------------------------- /change-log/v0.6.3.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.6.3.md -------------------------------------------------------------------------------- /change-log/v0.6.4.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.6.4.md -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.2.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [30058c03](https://github.com/databendlabs/openraft/commit/30058c036de06e9d0d66dd290dc75cf06831e12e) #424 wrong range when searching for membership entries: `[end-step, end)`.; by 张炎泼; 2022-07-03 4 | 5 | The iterating range searching for membership log entries should be 6 | `[end-step, end)`, not `[start, end)`. 7 | With this bug it will return duplicated membership entries. 8 | 9 | - Bug: #424 10 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.3.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [f99ade30](https://github.com/databendlabs/openraft/commit/f99ade30a7f806f18ed19ace12e226cd62fd43ec) API: move default impl methods in RaftStorage to StorageHelper; by 张炎泼; 2022-07-04 4 | 5 | ### Fixed: 6 | 7 | - Fixed: [44381b0c](https://github.com/databendlabs/openraft/commit/44381b0c776cfbb7dfc7789de27346110776b7f6) when handling append-entries, if prev_log_id is purged, it should not delete any logs.; by 张炎泼; 2022-08-14 8 | 9 | When handling append-entries, if the local log at `prev_log_id.index` is 10 | purged, a follower should not believe it is a **conflict** and should 11 | not delete all logs. It will get committed log lost. 12 | 13 | To fix this issue, use `last_applied` instead of `committed`: 14 | `last_applied` is always the committed log id, while `committed` is not 15 | persisted and may be smaller than the actually applied, when a follower 16 | is restarted. 17 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.4.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.7.0-alpha.4.md -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.5.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.7.0-alpha.5.md -------------------------------------------------------------------------------- /change-log/v0.7.0.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/change-log/v0.7.0.md -------------------------------------------------------------------------------- /change-log/v0.7.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [ea696474](https://github.com/databendlabs/openraft/commit/ea696474191b82069fae465bb064a2e599537ede) add feature-flag: `bt` enables backtrace; by 张炎泼; 2022-03-12 4 | 5 | `--features bt` enables backtrace when generating errors. 6 | By default errors does not contain backtrace info. 7 | 8 | Thus openraft can be built on stable rust by default. 9 | 10 | To use on stable rust with backtrace, set `RUSTC_BOOTSTRAP=1`, e.g.: 11 | ``` 12 | RUSTUP_TOOLCHAIN=stable RUSTC_BOOTSTRAP=1 make test 13 | ``` 14 | -------------------------------------------------------------------------------- /change-log/v0.7.2.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [568ca470](https://github.com/databendlabs/openraft/commit/568ca470524f77bc721edb104206e1774e1555cc) add Raft::remove_learner(); by 张炎泼; 2022-09-02 4 | -------------------------------------------------------------------------------- /change-log/v0.7.3.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [25e94c36](https://github.com/databendlabs/openraft/commit/25e94c36e5c8ae640044196070f9a067d5f105a3) InstallSnapshotResponse: replies the last applied log id; Do not install a smaller snapshot; by 张炎泼; 2022-09-22 4 | 5 | A snapshot may not be installed by a follower if it already has a higher 6 | `last_applied` log id locally. 7 | In such a case, it just ignores the snapshot and respond with its local 8 | `last_applied` log id. 9 | 10 | This way the applied state(i.e., `last_applied`) will never revert back. 11 | 12 | ### Fixed: 13 | 14 | - Fixed: [21684bbd](https://github.com/databendlabs/openraft/commit/21684bbdfdc54b18daa68f623afc2b0be6718c72) potential inconsistency when installing snapshot; by 张炎泼; 2022-09-22 15 | 16 | The conflicting logs that are before `snapshot_meta.last_log_id` should 17 | be deleted before installing a snapshot. 18 | 19 | Otherwise there is chance the snapshot is installed but conflicting logs 20 | are left in the store, when a node crashes. 21 | -------------------------------------------------------------------------------- /change-log/v0.7.4.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [1bd22edc](https://github.com/databendlabs/openraft/commit/1bd22edc0a8dc7a9c314341370b3dfeb357411b5) remove AddLearnerError::Exists, which is not actually used; by 张炎泼; 2022-09-30 4 | 5 | - Changed: [c6fe29d4](https://github.com/databendlabs/openraft/commit/c6fe29d4a53b47f6c43d83a24e1610788a4c0166) change-membership does not return error when replication lags; by 张炎泼; 2022-10-22 6 | 7 | If `blocking` is `true`, `Raft::change_membership(..., blocking)` will 8 | block until repliication to new nodes become upto date. 9 | But it won't return an error when proposing change-membership log. 10 | 11 | - Change: remove two errors: `LearnerIsLagging` and `LearnerNotFound`. 12 | 13 | - Fix: #581 14 | 15 | ### Fixed: 16 | 17 | - Fixed: [2896b98e](https://github.com/databendlabs/openraft/commit/2896b98e34825a8623ec4650da405c79827ecbee) changing membership should not remove replication to all learners; by 张炎泼; 2022-09-30 18 | 19 | When changing membership, replications to the learners(non-voters) that 20 | are not added as voter should be kept. 21 | 22 | E.g.: with a cluster of voters `{0}` and learners `{1, 2, 3}`, changing 23 | membership to `{0, 1, 2}` should not remove replication to node `3`. 24 | 25 | Only replications to removed members should be removed. 26 | 27 | ### Added: 28 | 29 | - Added: [9a22bb03](https://github.com/databendlabs/openraft/commit/9a22bb035b1d456ab2949c8b3abdbaa630622c63) add rocks-store as a `RaftStorage` implementation based on rocks-db; by 张炎泼; 2023-02-22 30 | -------------------------------------------------------------------------------- /change-log/v0.8.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [b3c2ff7e](https://github.com/databendlabs/openraft/commit/b3c2ff7e37ce5996f572f43bc574cb50b2d0cdc2) add Membership methods: voter_ids(), learner_ids(), get_node(); by 张炎泼; 2023-02-28 4 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | too-many-arguments-threshold = 10 2 | cognitive-complexity-threshold = 25 3 | -------------------------------------------------------------------------------- /cluster_benchmark/Cargo.toml: -------------------------------------------------------------------------------- 1 | # These tests depend on `memstore`, while `memstore` enables `serde` feature of `openraft`. 2 | # This make it impossible to test openraft with `serde` feature off. 3 | 4 | [package] 5 | name = "cluster-benchmark" 6 | description = "openraft cluster benchmark" 7 | 8 | version = "0.1.0" 9 | edition = "2021" 10 | authors = [ 11 | "drdr xp ", 12 | ] 13 | categories = ["algorithms", "asynchronous", "data-structures"] 14 | homepage = "https://github.com/databendlabs/openraft" 15 | keywords = ["raft", "consensus"] 16 | license = "MIT OR Apache-2.0" 17 | repository = "https://github.com/databendlabs/openraft" 18 | 19 | [dependencies] 20 | 21 | [dev-dependencies] 22 | openraft = { path="../openraft", version = "0.10.0", features = ["serde", "type-alias"] } 23 | 24 | anyhow = "1.0.63" 25 | maplit = "1.0.2" 26 | serde = { version="1.0.114", features=["derive", "rc"]} 27 | serde_json = "1.0.57" 28 | tokio = { version="1.8", default-features=false, features=["fs", "io-util", "macros", "rt", "rt-multi-thread", "sync", "time"] } 29 | tracing = "0.1.29" 30 | 31 | [features] 32 | 33 | bt = ["openraft/bt"] 34 | single-term-leader = [] 35 | 36 | [profile.release] 37 | debug = 2 38 | lto = "thin" 39 | overflow-checks = false 40 | codegen-units = 1 41 | -------------------------------------------------------------------------------- /cluster_benchmark/README.md: -------------------------------------------------------------------------------- 1 | Benchmark openraft without any actual business. 2 | 3 | This benchmark is designed to measure the performance overhead caused by 4 | Openraft itself. 5 | 6 | The benchmark comes with an in-memory log store, a state machine with no data, 7 | and an in-process network that uses function calls to simulate RPC. 8 | 9 | 10 | ## Benchmark result 11 | 12 | | clients | put/s | ns/op | 13 | | --: | --: | --: | 14 | | 256 | **1,014,000** | 985 | 15 | | 64 | **730,000** | 1,369 | 16 | | 1 | 70,000 | **14,273** | 17 | 18 | The benchmark is carried out with varying numbers of clients because: 19 | - The `1 client` benchmark shows the average **latency** to commit each log. 20 | - The `64 client` benchmark shows the maximum **throughput**. 21 | 22 | The benchmark is conducted with the following settings: 23 | - No network. 24 | - In-memory store. 25 | - A cluster of 3 nodes in a single process on a Mac M1-Max laptop. 26 | - Request: empty 27 | - Response: empty 28 | 29 | 30 | ## Run it 31 | 32 | `make bench_cluster_of_3` in repo root folder, or in this folder: 33 | 34 | ```sh 35 | cargo test --test benchmark --release bench_cluster_of_3 -- --ignored --nocapture 36 | ``` 37 | -------------------------------------------------------------------------------- /cluster_benchmark/tests/README.md: -------------------------------------------------------------------------------- 1 | # Benchmark openraft cluster 2 | 3 | It builds a cluster locally with a **minimized** store and network layer, 4 | and is meant to benchmark and find performance bottleneck of the openraft framework. -------------------------------------------------------------------------------- /cluster_benchmark/tests/benchmark/main.rs: -------------------------------------------------------------------------------- 1 | #![deny(unused_crate_dependencies)] 2 | #![deny(unused_qualifications)] 3 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 4 | 5 | pub(crate) mod network; 6 | pub(crate) mod store; 7 | 8 | mod bench_cluster; 9 | -------------------------------------------------------------------------------- /cluster_benchmark/tests/benchmark/store/test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | use openraft::StorageError; 6 | 7 | use crate::store::LogStore; 8 | use crate::store::StateMachineStore; 9 | use crate::store::TypeConfig; 10 | 11 | struct Builder {} 12 | 13 | impl StoreBuilder, Arc> for Builder { 14 | async fn build(&self) -> Result<((), Arc, Arc), StorageError> { 15 | let log_store = LogStore::new_async().await; 16 | let sm = Arc::new(StateMachineStore::new()); 17 | Ok(((), log_store, sm)) 18 | } 19 | } 20 | 21 | #[tokio::test] 22 | pub async fn test_store() -> Result<(), StorageError> { 23 | Suite::test_all(Builder {}).await?; 24 | Ok(()) 25 | } 26 | -------------------------------------------------------------------------------- /examples/client-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "client-http" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example network implementation v1 built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus", "network"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 19 | 20 | reqwest = { version = "0.12.5", features = ["json"] } 21 | serde = { version = "1.0.114", features = ["derive"] } 22 | serde_json = "1.0.57" 23 | tokio = { version = "1.35.1", features = ["full"] } 24 | tracing = "0.1.40" 25 | 26 | [features] 27 | default = [] 28 | 29 | [package.metadata.docs.rs] 30 | all-features = true -------------------------------------------------------------------------------- /examples/mem-log/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mem-log" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example in-memory storage for `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", features = ["type-alias"] } 19 | 20 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 21 | 22 | [features] 23 | 24 | serde = ["openraft/serde"] 25 | 26 | [package.metadata.docs.rs] 27 | all-features = true 28 | -------------------------------------------------------------------------------- /examples/mem-log/README.md: -------------------------------------------------------------------------------- 1 | # memstore 2 | 3 | This is a simple in-memory implementation of `RaftLogStore`. 4 | Other example Raft based applications use this crate as a component to store raft logs. 5 | 6 | TODO: 7 | 8 | - [ ] Add `RaftStateMachine` -------------------------------------------------------------------------------- /examples/mem-log/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Provide storage layer implementation for examples. 2 | 3 | mod log_store; 4 | 5 | pub use log_store::LogStore; 6 | -------------------------------------------------------------------------------- /examples/mem-log/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for memstore" -------------------------------------------------------------------------------- /examples/memstore: -------------------------------------------------------------------------------- 1 | mem-log -------------------------------------------------------------------------------- /examples/network-v1-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "network-v1-http" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example network implementation v1 built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus", "network"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 19 | 20 | reqwest = { version = "0.12.5", features = ["json"] } 21 | serde = { version = "1.0.114", features = ["derive"] } 22 | serde_json = "1.0.57" 23 | tokio = { version = "1.35.1", features = ["full"] } 24 | tracing = "0.1.40" 25 | 26 | [features] 27 | default = [] 28 | 29 | [package.metadata.docs.rs] 30 | all-features = true -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-grpc" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "Sainath Singineedi ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example distributed key-value store built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [[bin]] 18 | name = "raft-key-value" 19 | path = "src/bin/main.rs" 20 | 21 | [dependencies] 22 | mem-log = { path = "../mem-log", features = [] } 23 | openraft = { path = "../../openraft", features = ["type-alias"] } 24 | 25 | clap = { version = "4.1.11", features = ["derive", "env"] } 26 | serde = { version = "1.0.114", features = ["derive"] } 27 | serde_json = "1.0.57" 28 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 29 | tracing = "0.1.29" 30 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 31 | tonic = "0.12.3" 32 | tonic-build = "0.12.3" 33 | dashmap = "6.1.0" 34 | prost = "0.13.4" 35 | futures = "0.3.31" 36 | 37 | [dev-dependencies] 38 | anyhow = "1.0.63" 39 | maplit = "1.0.2" 40 | 41 | [features] 42 | 43 | [build-dependencies] 44 | prost-build = "0.13.4" 45 | tonic-build = "0.12.3" 46 | 47 | [package.metadata.docs.rs] 48 | all-features = true 49 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> Result<(), Box> { 2 | println!("cargo:rerun-if-changed=src/*"); 3 | let mut config = prost_build::Config::new(); 4 | config.protoc_arg("--experimental_allow_proto3_optional"); 5 | let proto_files = ["proto/raft.proto", "proto/app_types.proto", "proto/app.proto"]; 6 | 7 | // TODO: remove serde 8 | 9 | tonic_build::configure() 10 | .btree_map(["."]) 11 | .type_attribute("openraftpb.Node", "#[derive(Eq)]") 12 | .type_attribute("openraftpb.SetRequest", "#[derive(Eq)]") 13 | .type_attribute("openraftpb.Response", "#[derive(Eq)]") 14 | .type_attribute("openraftpb.LeaderId", "#[derive(Eq)]") 15 | .type_attribute("openraftpb.Vote", "#[derive(Eq)]") 16 | .type_attribute("openraftpb.NodeIdSet", "#[derive(Eq)]") 17 | .type_attribute("openraftpb.Membership", "#[derive(Eq)]") 18 | .type_attribute("openraftpb.Entry", "#[derive(Eq)]") 19 | .compile_protos_with_config(config, &proto_files, &["proto"])?; 20 | Ok(()) 21 | } 22 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/proto/app_types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package openraftpb; 4 | 5 | // SetRequest represents a key-value pair to be stored 6 | message SetRequest { 7 | string key = 1; // Key to store 8 | string value = 2; // Value to associate with the key 9 | } 10 | 11 | // GetRequest represents a key lookup request 12 | message GetRequest { 13 | string key = 1; // Key to look up 14 | } 15 | 16 | // GetResponse contains the value associated with the requested key 17 | message Response { 18 | optional string value = 1; // Retrieved value 19 | } 20 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/grpc/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod app_service; 2 | pub mod raft_service; 3 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | pub mod app; 4 | pub mod grpc; 5 | pub mod network; 6 | pub mod store; 7 | 8 | pub mod protobuf { 9 | tonic::include_proto!("openraftpb"); 10 | } 11 | 12 | #[path = "../../utils/declare_types.rs"] 13 | pub mod typ; 14 | 15 | mod pb_impl; 16 | 17 | #[cfg(test)] 18 | mod test_store; 19 | 20 | use crate::protobuf as pb; 21 | 22 | openraft::declare_raft_types!( 23 | /// Declare the type configuration for example K/V store. 24 | pub TypeConfig: 25 | D = pb::SetRequest, 26 | R = pb::Response, 27 | LeaderId = pb::LeaderId, 28 | Vote = pb::Vote, 29 | Entry = pb::Entry, 30 | Node = pb::Node, 31 | SnapshotData = Vec, 32 | ); 33 | 34 | pub type NodeId = u64; 35 | pub type LogStore = store::LogStore; 36 | pub type StateMachineStore = store::StateMachineStore; 37 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_append_entries_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::AppendEntriesRequest; 3 | 4 | impl From for AppendEntriesRequest { 5 | fn from(proto_req: pb::AppendEntriesRequest) -> Self { 6 | AppendEntriesRequest { 7 | vote: proto_req.vote.unwrap(), 8 | prev_log_id: proto_req.prev_log_id.map(|log_id| log_id.into()), 9 | entries: proto_req.entries, 10 | leader_commit: proto_req.leader_commit.map(|log_id| log_id.into()), 11 | } 12 | } 13 | } 14 | 15 | impl From for pb::AppendEntriesRequest { 16 | fn from(value: AppendEntriesRequest) -> Self { 17 | pb::AppendEntriesRequest { 18 | vote: Some(value.vote), 19 | prev_log_id: value.prev_log_id.map(|log_id| log_id.into()), 20 | entries: value.entries, 21 | leader_commit: value.leader_commit.map(|log_id| log_id.into()), 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_client_write_response.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::*; 3 | 4 | impl From for ClientWriteResponse { 5 | fn from(r: pb::ClientWriteResponse) -> Self { 6 | ClientWriteResponse { 7 | log_id: r.log_id.unwrap().into(), 8 | data: r.data.unwrap(), 9 | membership: r.membership.map(|mem| mem.into()), 10 | } 11 | } 12 | } 13 | 14 | impl From for pb::ClientWriteResponse { 15 | fn from(r: ClientWriteResponse) -> Self { 16 | pb::ClientWriteResponse { 17 | log_id: Some(r.log_id.into()), 18 | data: Some(r.data), 19 | membership: r.membership.map(|mem| mem.into()), 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_entry.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use openraft::alias::LogIdOf; 4 | use openraft::entry::RaftEntry; 5 | use openraft::entry::RaftPayload; 6 | use openraft::EntryPayload; 7 | use openraft::Membership; 8 | 9 | use crate::protobuf as pb; 10 | use crate::TypeConfig; 11 | 12 | impl fmt::Display for pb::Entry { 13 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 14 | write!(f, "Entry{{term={},index={}}}", self.term, self.index) 15 | } 16 | } 17 | 18 | impl RaftPayload for pb::Entry { 19 | fn get_membership(&self) -> Option> { 20 | self.membership.clone().map(Into::into) 21 | } 22 | } 23 | 24 | impl RaftEntry for pb::Entry { 25 | fn new(log_id: LogIdOf, payload: EntryPayload) -> Self { 26 | let mut app_data = None; 27 | let mut membership = None; 28 | match payload { 29 | EntryPayload::Blank => {} 30 | EntryPayload::Normal(data) => app_data = Some(data), 31 | EntryPayload::Membership(m) => membership = Some(m.into()), 32 | } 33 | 34 | Self { 35 | term: log_id.leader_id, 36 | index: log_id.index, 37 | app_data, 38 | membership, 39 | } 40 | } 41 | 42 | fn log_id_parts(&self) -> (&u64, u64) { 43 | (&self.term, self.index) 44 | } 45 | 46 | fn set_log_id(&mut self, new: LogIdOf) { 47 | self.term = new.leader_id; 48 | self.index = new.index; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_log_id.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::LogId; 3 | 4 | impl From for pb::LogId { 5 | fn from(log_id: LogId) -> Self { 6 | pb::LogId { 7 | term: *log_id.committed_leader_id(), 8 | index: log_id.index(), 9 | } 10 | } 11 | } 12 | 13 | impl From for LogId { 14 | fn from(proto_log_id: pb::LogId) -> Self { 15 | LogId::new(proto_log_id.term, proto_log_id.index) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_membership.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::collections::BTreeSet; 3 | 4 | use openraft::Membership; 5 | 6 | use crate::pb; 7 | use crate::TypeConfig; 8 | 9 | impl From for Membership { 10 | fn from(value: pb::Membership) -> Self { 11 | let mut configs = vec![]; 12 | for c in value.configs { 13 | let config: BTreeSet = c.node_ids.keys().copied().collect(); 14 | configs.push(config); 15 | } 16 | let nodes = value.nodes; 17 | // TODO: do not unwrap() 18 | Membership::new(configs, nodes).unwrap() 19 | } 20 | } 21 | 22 | impl From> for pb::Membership { 23 | fn from(value: Membership) -> Self { 24 | let mut configs = vec![]; 25 | for c in value.get_joint_config() { 26 | let mut node_ids = BTreeMap::new(); 27 | for nid in c.iter() { 28 | node_ids.insert(*nid, ()); 29 | } 30 | configs.push(pb::NodeIdSet { node_ids }); 31 | } 32 | let nodes = value.nodes().map(|(nid, n)| (*nid, n.clone())).collect(); 33 | pb::Membership { configs, nodes } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_snapshot_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb::snapshot_request::Payload; 2 | use crate::protobuf as pb; 3 | impl pb::SnapshotRequest { 4 | pub fn into_meta(self) -> Option { 5 | let p = self.payload?; 6 | match p { 7 | Payload::Meta(meta) => Some(meta), 8 | Payload::Chunk(_) => None, 9 | } 10 | } 11 | 12 | pub fn into_data_chunk(self) -> Option> { 13 | let p = self.payload?; 14 | match p { 15 | Payload::Meta(_) => None, 16 | Payload::Chunk(chunk) => Some(chunk), 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use openraft::vote::RaftVote; 4 | 5 | use crate::pb; 6 | use crate::typ::LeaderId; 7 | use crate::TypeConfig; 8 | 9 | impl RaftVote for pb::Vote { 10 | fn from_leader_id(leader_id: LeaderId, committed: bool) -> Self { 11 | pb::Vote { 12 | leader_id: Some(leader_id), 13 | committed, 14 | } 15 | } 16 | 17 | fn leader_id(&self) -> Option<&LeaderId> { 18 | self.leader_id.as_ref() 19 | } 20 | 21 | fn is_committed(&self) -> bool { 22 | self.committed 23 | } 24 | } 25 | 26 | impl fmt::Display for pb::Vote { 27 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 28 | write!( 29 | f, 30 | "<{}:{}>", 31 | self.leader_id.as_ref().unwrap_or(&Default::default()), 32 | if self.is_committed() { "Q" } else { "-" } 33 | ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::VoteRequest; 3 | 4 | impl From for pb::VoteRequest { 5 | fn from(vote_req: VoteRequest) -> Self { 6 | pb::VoteRequest { 7 | vote: Some(vote_req.vote), 8 | last_log_id: vote_req.last_log_id.map(|log_id| log_id.into()), 9 | } 10 | } 11 | } 12 | 13 | impl From for VoteRequest { 14 | fn from(proto_vote_req: pb::VoteRequest) -> Self { 15 | let vote = proto_vote_req.vote.unwrap(); 16 | let last_log_id = proto_vote_req.last_log_id.map(|log_id| log_id.into()); 17 | VoteRequest::new(vote, last_log_id) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote_response.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::VoteResponse; 3 | 4 | impl From for pb::VoteResponse { 5 | fn from(vote_resp: VoteResponse) -> Self { 6 | pb::VoteResponse { 7 | vote: Some(vote_resp.vote), 8 | vote_granted: vote_resp.vote_granted, 9 | last_log_id: vote_resp.last_log_id.map(|log_id| log_id.into()), 10 | } 11 | } 12 | } 13 | 14 | impl From for VoteResponse { 15 | fn from(proto_vote_resp: pb::VoteResponse) -> Self { 16 | let vote = proto_vote_resp.vote.unwrap(); 17 | let last_log_id = proto_vote_resp.last_log_id.map(|log_id| log_id.into()); 18 | VoteResponse::new(vote, last_log_id, proto_vote_resp.vote_granted) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/mod.rs: -------------------------------------------------------------------------------- 1 | //! Implements traits for protobuf types 2 | 3 | mod impl_append_entries_request; 4 | mod impl_append_entries_response; 5 | mod impl_client_write_response; 6 | mod impl_entry; 7 | mod impl_leader_id; 8 | mod impl_log_id; 9 | mod impl_membership; 10 | mod impl_snapshot_request; 11 | mod impl_vote; 12 | mod impl_vote_request; 13 | mod impl_vote_response; 14 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/test_store.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | 6 | use crate::store::LogStore; 7 | use crate::store::StateMachineStore; 8 | use crate::typ::*; 9 | use crate::TypeConfig; 10 | 11 | struct MemKVStoreBuilder {} 12 | 13 | impl StoreBuilder, ()> for MemKVStoreBuilder { 14 | async fn build(&self) -> Result<((), LogStore, Arc), StorageError> { 15 | Ok(((), LogStore::default(), Arc::default())) 16 | } 17 | } 18 | 19 | #[tokio::test] 20 | pub async fn test_mem_store() -> Result<(), StorageError> { 21 | Suite::test_all(MemKVStoreBuilder {}).await?; 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-network-v2" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | description = "An example distributed key-value store built upon `openraft`." 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | [dependencies] 19 | mem-log = { path = "../mem-log", features = [] } 20 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 21 | 22 | serde = { version = "1.0.114", features = ["derive"] } 23 | serde_json = "1.0.57" 24 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 25 | tracing = "0.1.29" 26 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 27 | 28 | [dev-dependencies] 29 | maplit = "1.0.2" 30 | 31 | [features] 32 | 33 | [package.metadata.docs.rs] 34 | all-features = true 35 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/README.md: -------------------------------------------------------------------------------- 1 | # Example Openraft kv-store using `RaftNetworkV2` 2 | 3 | With `RaftNetworkV2`, Openraft allows application to use any data type for snapshot data, 4 | instead of a single-file like data format with `AsyncSeek + AsyncRead + AsyncWrite + Unpin` bounds. 5 | 6 | This example is similar to the basic raft-kv-memstore example 7 | but focuses on how to handle snapshot with `RaftNetworkV2::full_snapshot()`. 8 | Other aspects are minimized. 9 | 10 | To send a complete snapshot, Refer to implementation of `RaftNetworkV2::full_snapshot()` in this example. 11 | 12 | To receive a complete snapshot, Refer to implementation of `api::snapshot()` in this example. 13 | 14 | 15 | ## Run it 16 | 17 | Run it with `cargo test -- --nocapture`. -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/src/router.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::Arc; 3 | use std::sync::Mutex; 4 | 5 | use openraft::error::Unreachable; 6 | use tokio::sync::oneshot; 7 | 8 | use crate::app::RequestTx; 9 | use crate::decode; 10 | use crate::encode; 11 | use crate::typ::RaftError; 12 | use crate::NodeId; 13 | 14 | /// Simulate a network router. 15 | #[derive(Debug, Clone)] 16 | #[derive(Default)] 17 | pub struct Router { 18 | pub targets: Arc>>, 19 | } 20 | 21 | impl Router { 22 | /// Send request `Req` to target node `to`, and wait for response `Result>`. 23 | pub async fn send(&self, to: NodeId, path: &str, req: Req) -> Result 24 | where 25 | Req: serde::Serialize, 26 | Result: serde::de::DeserializeOwned, 27 | { 28 | let (resp_tx, resp_rx) = oneshot::channel(); 29 | 30 | let encoded_req = encode(req); 31 | tracing::debug!("send to: {}, {}, {}", to, path, encoded_req); 32 | 33 | { 34 | let mut targets = self.targets.lock().unwrap(); 35 | let tx = targets.get_mut(&to).unwrap(); 36 | 37 | tx.send((path.to_string(), encoded_req, resp_tx)).unwrap(); 38 | } 39 | 40 | let resp_str = resp_rx.await.unwrap(); 41 | tracing::debug!("resp from: {}, {}, {}", to, path, resp_str); 42 | 43 | let res = decode::>(&resp_str); 44 | res.map_err(|e| Unreachable::new(&e)) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-opendal-snapshot-data" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | "Xuanwo ", 11 | ] 12 | categories = ["algorithms", "asynchronous", "data-structures"] 13 | description = "An example distributed key-value store built upon `openraft`." 14 | homepage = "https://github.com/databendlabs/openraft" 15 | keywords = ["raft", "consensus"] 16 | license = "MIT OR Apache-2.0" 17 | repository = "https://github.com/databendlabs/openraft" 18 | 19 | [dependencies] 20 | mem-log = { path = "../mem-log", features = [] } 21 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 22 | 23 | bytes = "1.0" 24 | serde = { version = "1.0.114", features = ["derive"] } 25 | serde_json = "1.0.57" 26 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 27 | tracing = "0.1.29" 28 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 29 | opendal = "0.48.0" 30 | 31 | [dev-dependencies] 32 | maplit = "1.0.2" 33 | 34 | [features] 35 | 36 | [package.metadata.docs.rs] 37 | all-features = true 38 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/README.md: -------------------------------------------------------------------------------- 1 | # Example Openraft kv-store with snapshot stored in remote storage 2 | 3 | This example shows how to save and retrieve snapshot data from remote storage, 4 | allowing users to follow a similar pattern for implementing business logic such as snapshot backups. 5 | 6 | This example is similar to the basic raft-kv-memstore example 7 | but focuses on how to store and fetch snapshot data from remote storage. 8 | Other aspects are minimized. 9 | 10 | To send a complete snapshot, Refer to implementation of `RaftNetworkV2::full_snapshot()` in this example. 11 | 12 | To receive a complete snapshot, Refer to implementation of `api::snapshot()` in this example. 13 | 14 | 15 | ## Run it 16 | 17 | Run it with `cargo test -- --nocapture`. -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/src/router.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::Arc; 3 | use std::sync::Mutex; 4 | 5 | use openraft::error::Unreachable; 6 | use tokio::sync::oneshot; 7 | 8 | use crate::app::RequestTx; 9 | use crate::decode; 10 | use crate::encode; 11 | use crate::typ::RaftError; 12 | use crate::NodeId; 13 | 14 | /// Simulate a network router. 15 | #[derive(Debug, Clone)] 16 | #[derive(Default)] 17 | pub struct Router { 18 | pub targets: Arc>>, 19 | } 20 | 21 | impl Router { 22 | /// Send request `Req` to target node `to`, and wait for response `Result>`. 23 | pub async fn send(&self, to: NodeId, path: &str, req: Req) -> Result 24 | where 25 | Req: serde::Serialize, 26 | Result: serde::de::DeserializeOwned, 27 | { 28 | let (resp_tx, resp_rx) = oneshot::channel(); 29 | 30 | let encoded_req = encode(req); 31 | tracing::debug!("send to: {}, {}, {}", to, path, encoded_req); 32 | 33 | { 34 | let mut targets = self.targets.lock().unwrap(); 35 | let tx = targets.get_mut(&to).unwrap(); 36 | 37 | tx.send((path.to_string(), encoded_req, resp_tx)).unwrap(); 38 | } 39 | 40 | let resp_str = resp_rx.await.unwrap(); 41 | tracing::debug!("resp from: {}, {}, {}", to, path, resp_str); 42 | 43 | let res = decode::>(&resp_str); 44 | res.map_err(|e| Unreachable::new(&e)) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-singlethreaded" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | description = "An example distributed key-value store built upon `openraft`." 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | [dependencies] 19 | openraft = { path = "../../openraft", features = [ 20 | "serde", 21 | "singlethreaded", 22 | "type-alias", 23 | ] } 24 | 25 | serde = { version = "1.0.114", features = ["derive"] } 26 | serde_json = "1.0.57" 27 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 28 | tracing = "0.1.29" 29 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 30 | 31 | [dev-dependencies] 32 | maplit = "1.0.2" 33 | 34 | [features] 35 | 36 | [package.metadata.docs.rs] 37 | all-features = true 38 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/README.md: -------------------------------------------------------------------------------- 1 | # Example single threaded key-value store 2 | 3 | Example key-value store with `openraft`, single threaded, i.e., Openraft does not require `Send` for data types, 4 | by enabling feature flag `singlethreaded` 5 | 6 | In this example, `NodeId` and application request `Request` are not `Send`, by enabling feature flag `singlethreaded`: 7 | `openraft = { path = "../../openraft", features = ["singlethreaded"] }`, 8 | Openraft works happily with non-`Send` data types: 9 | 10 | ```rust 11 | pub struct NodeId { 12 | pub id: u64, 13 | // Make it !Send 14 | _p: PhantomData<*const ()>, 15 | } 16 | pub enum Request { 17 | Set { 18 | key: String, 19 | value: String, 20 | // Make it !Send 21 | _p: PhantomData<*const ()>, 22 | } 23 | } 24 | ``` 25 | 26 | ## Run it 27 | 28 | Run it with `cargo test -- --nocapture`. -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | description = "An example distributed key-value store built upon `openraft`." 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | [[bin]] 19 | name = "raft-key-value" 20 | path = "src/bin/main.rs" 21 | 22 | [dependencies] 23 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 24 | mem-log = { path = "../mem-log", features = [] } 25 | network-v1-http = { path = "../network-v1-http" } 26 | client-http = { path = "../client-http" } 27 | 28 | 29 | actix-web = "4.0.0-rc.2" 30 | clap = { version = "4.1.11", features = ["derive", "env"] } 31 | reqwest = { version = "0.12.5", features = ["json"] } 32 | serde = { version = "1.0.114", features = ["derive"] } 33 | serde_json = "1.0.57" 34 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 35 | tracing = "0.1.29" 36 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 37 | 38 | [dev-dependencies] 39 | anyhow = "1.0.63" 40 | maplit = "1.0.2" 41 | 42 | [features] 43 | 44 | [package.metadata.docs.rs] 45 | all-features = true 46 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/app.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use crate::NodeId; 4 | use crate::Raft; 5 | use crate::StateMachineStore; 6 | 7 | // Representation of an application state. This struct can be shared around to share 8 | // instances of raft, store and more. 9 | pub struct App { 10 | pub id: NodeId, 11 | pub addr: String, 12 | pub raft: Raft, 13 | pub state_machine_store: Arc, 14 | } 15 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use raft_kv_memstore::start_example_raft_node; 3 | use tracing_subscriber::EnvFilter; 4 | 5 | #[derive(Parser, Clone, Debug)] 6 | #[clap(author, version, about, long_about = None)] 7 | pub struct Opt { 8 | #[clap(long)] 9 | pub id: u64, 10 | 11 | #[clap(long)] 12 | pub http_addr: String, 13 | } 14 | 15 | #[actix_web::main] 16 | async fn main() -> std::io::Result<()> { 17 | // Setup the logger 18 | tracing_subscriber::fmt() 19 | .with_target(true) 20 | .with_thread_ids(true) 21 | .with_level(true) 22 | .with_ansi(false) 23 | .with_env_filter(EnvFilter::from_default_env()) 24 | .init(); 25 | 26 | // Parse the parameters passed by arguments. 27 | let options = Opt::parse(); 28 | 29 | start_example_raft_node(options.id, options.http_addr).await 30 | } 31 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod management; 3 | pub mod raft; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/network/raft.rs: -------------------------------------------------------------------------------- 1 | use actix_web::post; 2 | use actix_web::web::Data; 3 | use actix_web::web::Json; 4 | use actix_web::Responder; 5 | use openraft::raft::AppendEntriesRequest; 6 | use openraft::raft::InstallSnapshotRequest; 7 | use openraft::raft::VoteRequest; 8 | 9 | use crate::app::App; 10 | use crate::TypeConfig; 11 | 12 | // --- Raft communication 13 | 14 | #[post("/vote")] 15 | pub async fn vote(app: Data, req: Json>) -> actix_web::Result { 16 | let res = app.raft.vote(req.0).await; 17 | Ok(Json(res)) 18 | } 19 | 20 | #[post("/append")] 21 | pub async fn append(app: Data, req: Json>) -> actix_web::Result { 22 | let res = app.raft.append_entries(req.0).await; 23 | Ok(Json(res)) 24 | } 25 | 26 | #[post("/snapshot")] 27 | pub async fn snapshot( 28 | app: Data, 29 | req: Json>, 30 | ) -> actix_web::Result { 31 | let res = app.raft.install_snapshot(req.0).await; 32 | Ok(Json(res)) 33 | } 34 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | use openraft::StorageError; 6 | 7 | use crate::store::LogStore; 8 | use crate::store::StateMachineStore; 9 | use crate::TypeConfig; 10 | 11 | struct MemKVStoreBuilder {} 12 | 13 | impl StoreBuilder, ()> for MemKVStoreBuilder { 14 | async fn build(&self) -> Result<((), LogStore, Arc), StorageError> { 15 | Ok(((), LogStore::default(), Arc::default())) 16 | } 17 | } 18 | 19 | #[tokio::test] 20 | pub async fn test_mem_store() -> Result<(), StorageError> { 21 | Suite::test_all(MemKVStoreBuilder {}).await?; 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | *.db 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-rocksdb" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | "The Tremor Team", 11 | ] 12 | categories = ["algorithms", "asynchronous", "data-structures"] 13 | description = "An example distributed key-value store built upon `openraft`." 14 | homepage = "https://github.com/databendlabs/openraft" 15 | keywords = ["raft", "consensus"] 16 | license = "MIT OR Apache-2.0" 17 | repository = "https://github.com/databendlabs/openraft" 18 | 19 | [[bin]] 20 | name = "raft-key-value-rocks" 21 | path = "src/bin/main.rs" 22 | 23 | [dependencies] 24 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 25 | openraft-rocksstore = { path = "../rocksstore" } 26 | network-v1-http = { path = "../network-v1-http" } 27 | client-http = { path = "../client-http" } 28 | 29 | actix-web = "4.0.0-rc.2" 30 | tokio = { version = "1.35.1", features = ["full"] } 31 | byteorder = "1.4.3" 32 | clap = { version = "4.1.11", features = ["derive", "env"] } 33 | reqwest = { version = "0.12.5", features = ["json"] } 34 | rocksdb = "0.22.0" 35 | serde = { version = "1.0.114", features = ["derive"] } 36 | serde_json = "1.0.57" 37 | tracing = "0.1.40" 38 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 39 | 40 | [dev-dependencies] 41 | maplit = "1.0.2" 42 | tempfile = { version = "3.4.0" } 43 | 44 | 45 | [features] 46 | 47 | [package.metadata.docs.rs] 48 | all-features = true 49 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/app.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::Arc; 3 | 4 | use openraft::Config; 5 | use tokio::sync::RwLock; 6 | 7 | use crate::typ::Raft; 8 | use crate::NodeId; 9 | 10 | // Representation of an application state. This struct can be shared around to share 11 | // instances of raft, store and more. 12 | pub struct App { 13 | pub id: NodeId, 14 | pub addr: String, 15 | pub raft: Raft, 16 | pub key_values: Arc>>, 17 | pub config: Arc, 18 | } 19 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use raft_kv_rocksdb::start_example_raft_node; 3 | use tracing_subscriber::EnvFilter; 4 | 5 | #[derive(Parser, Clone, Debug)] 6 | #[clap(author, version, about, long_about = None)] 7 | pub struct Opt { 8 | #[clap(long)] 9 | pub id: u64, 10 | 11 | #[clap(long)] 12 | pub addr: String, 13 | } 14 | 15 | #[actix_web::main] 16 | async fn main() -> std::io::Result<()> { 17 | // Setup the logger 18 | tracing_subscriber::fmt() 19 | .with_target(true) 20 | .with_thread_ids(true) 21 | .with_level(true) 22 | .with_ansi(false) 23 | .with_env_filter(EnvFilter::from_default_env()) 24 | .init(); 25 | 26 | // Parse the parameters passed by arguments. 27 | let options = Opt::parse(); 28 | 29 | start_example_raft_node(options.id, format!("{}.db", options.addr), options.addr).await 30 | } 31 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod management; 3 | pub mod raft; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/network/raft.rs: -------------------------------------------------------------------------------- 1 | use actix_web::post; 2 | use actix_web::web::Data; 3 | use actix_web::web::Json; 4 | use actix_web::Responder; 5 | use openraft::error::decompose::DecomposeResult; 6 | 7 | use crate::app::App; 8 | use crate::typ::*; 9 | 10 | // --- Raft communication 11 | 12 | #[post("/vote")] 13 | pub async fn vote(app: Data, req: Json) -> actix_web::Result { 14 | let res = app.raft.vote(req.0).await.decompose().unwrap(); 15 | Ok(Json(res)) 16 | } 17 | 18 | #[post("/append")] 19 | pub async fn append(app: Data, req: Json) -> actix_web::Result { 20 | let res = app.raft.append_entries(req.0).await.decompose().unwrap(); 21 | Ok(Json(res)) 22 | } 23 | 24 | #[post("/snapshot")] 25 | pub async fn snapshot(app: Data, req: Json) -> actix_web::Result { 26 | let res = app.raft.install_snapshot(req.0).await.decompose().unwrap(); 27 | Ok(Json(res)) 28 | } 29 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/rocksstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-rocksstore" 3 | description = "A rocksdb based implementation of the `openraft::RaftLogStorage` and `openraft::RaftStateMachine` trait." 4 | documentation = "https://docs.rs/openraft-rocksstore" 5 | readme = "README.md" 6 | 7 | version = "0.1.0" 8 | edition = "2021" 9 | authors = [ 10 | "drdr xp ", 11 | ] 12 | categories = ["algorithms", "asynchronous", "data-structures"] 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 19 | 20 | [dependencies] 21 | openraft = { path= "../../openraft", version = "0.10.0", features=["serde", "type-alias"] } 22 | 23 | rocksdb = "0.22.0" 24 | rand = "0.9" 25 | byteorder = "1.4.3" 26 | 27 | serde = { version = "1.0.114", features = ["derive"] } 28 | serde_json = "1.0.57" 29 | tokio = { version = "1.22", default-features = false, features = [ 30 | "io-util", 31 | "macros", 32 | "rt", 33 | "rt-multi-thread", 34 | "sync", 35 | "time", 36 | ] } 37 | tracing = { version = "0.1.40" } 38 | 39 | [dev-dependencies] 40 | tempfile = { version = "3.4.0" } 41 | 42 | [features] 43 | bt = ["openraft/bt"] 44 | 45 | [package.metadata.docs.rs] 46 | all-features = true 47 | -------------------------------------------------------------------------------- /examples/rocksstore/README.md: -------------------------------------------------------------------------------- 1 | # openraft-rocksstore 2 | 3 | This is an example of v2 storage [`RaftLogStorage`] and [`RaftStateMachine`] implementation 4 | with [`rocksdb`](https://docs.rs/rocksdb/latest/rocksdb/) based on [openraft-0.8](https://github.com/databendlabs/openraft/tree/release-0.8). 5 | 6 | This crate is built mainly for testing or demonstrating purpose.:) 7 | -------------------------------------------------------------------------------- /examples/rocksstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use openraft::testing::log::StoreBuilder; 2 | use openraft::testing::log::Suite; 3 | use openraft::StorageError; 4 | use tempfile::TempDir; 5 | 6 | use crate::log_store::RocksLogStore; 7 | use crate::RocksStateMachine; 8 | use crate::TypeConfig; 9 | 10 | struct RocksBuilder {} 11 | 12 | impl StoreBuilder, RocksStateMachine, TempDir> for RocksBuilder { 13 | async fn build(&self) -> Result<(TempDir, RocksLogStore, RocksStateMachine), StorageError> { 14 | let td = TempDir::new().expect("couldn't create temp dir"); 15 | let (log_store, sm) = crate::new(td.path()).await; 16 | Ok((td, log_store, sm)) 17 | } 18 | } 19 | 20 | #[tokio::test] 21 | pub async fn test_rocks_store() -> Result<(), StorageError> { 22 | Suite::test_all(RocksBuilder {}).await?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /examples/rocksstore/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for rocksstore" -------------------------------------------------------------------------------- /examples/utils/README.md: -------------------------------------------------------------------------------- 1 | # Utils for examples 2 | 3 | `declare_types.rs` declares types for a specific `RaftTypeConfig` implementation. -------------------------------------------------------------------------------- /guide/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Backup of mdbook building 4 | 5 | `mdbook` based document is disabled and is moved to https://docs.rs/openraft/0.8.3/openraft/docs ; 6 | 7 | To build mdbook in github workflow, copy the following snippet: 8 | 9 | 10 | ``` 11 | on: 12 | push: 13 | branches: 14 | - main 15 | paths: 16 | - 'guide/**' 17 | - 'book.toml' 18 | - 'README.md' 19 | 20 | jobs: 21 | deploy-guide: 22 | runs-on: ubuntu-18.04 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | 27 | - name: Install mdbook 28 | uses: drmingdrmer/mdbook-full@main 29 | 30 | - name: Build mdbook 31 | run: mdbook build 32 | 33 | - name: Deploy to github page 34 | uses: peaceiris/actions-gh-pages@v3 35 | with: 36 | github_token: ${{ secrets.GITHUB_TOKEN }} 37 | publish_dir: ./guide/book 38 | ``` 39 | 40 | Github action `drmingdrmer/mdbook-full` is maintained in https://github.com/drmingdrmer/mdbook-full 41 | with several plugins enabled: 42 | 43 | Preprocessors: 44 | 45 | - [mdbook-svgbob](https://github.com/drmingdrmer/mdbook-svgbob): SvgBob mdbook preprocessor which swaps code-blocks with neat SVG. 46 | - [mdbook-katex](https://github.com/drmingdrmer/mdbook-katex): A preprocessor for mdBook, rendering LaTex equations to HTML at build time. 47 | 48 | Backends: 49 | 50 | - [mdbook-linkcheck](https://github.com/drmingdrmer/mdbook-linkcheck): A backend for `mdbook` which will check your links for you. 51 | 52 | 53 | -------------------------------------------------------------------------------- /guide/src/custom.css: -------------------------------------------------------------------------------- 1 | svg.bob line { 2 | stroke: #888; 3 | } 4 | 5 | svg.bob path { 6 | stroke: #888; 7 | } 8 | 9 | svg.bob line.dashed { 10 | stroke: #ccc; 11 | } 12 | 13 | svg.bob circle { 14 | stroke: #bbb; 15 | } 16 | -------------------------------------------------------------------------------- /guide/src/images/raft-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/8f64ebbe95c86962a51dbf747dc1863f4370314b/guide/src/images/raft-overview.png -------------------------------------------------------------------------------- /macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-macros" 3 | 4 | version = { workspace = true } 5 | edition = { workspace = true } 6 | authors = { workspace = true } 7 | categories = { workspace = true } 8 | description = { workspace = true } 9 | documentation = { workspace = true } 10 | homepage = { workspace = true } 11 | keywords = { workspace = true } 12 | license = { workspace = true } 13 | repository = { workspace = true } 14 | 15 | [lib] 16 | proc-macro = true 17 | 18 | [dependencies] 19 | chrono = { workspace = true } 20 | proc-macro2 = { workspace = true } 21 | quote = { workspace = true } 22 | semver = { workspace = true } 23 | syn = { workspace = true, features = ["full", "extra-traits"] } 24 | 25 | 26 | [dev-dependencies] 27 | macrotest = "1" 28 | trybuild = "1.0" 29 | 30 | 31 | [features] 32 | 33 | # Do not add `Send` bounds. 34 | singlethreaded = [] 35 | -------------------------------------------------------------------------------- /macros/README.md: -------------------------------------------------------------------------------- 1 | # openraft_macros 2 | 3 | This crate provides a set of macros to support Openraft. 4 | 5 | # Test 6 | 7 | - Macro expansions are tested with [`macrotest`] crate. 8 | - Macro compile errors are tested with [`trybuild`] crate. 9 | 10 | 11 | [`macrotest`]: https://crates.io/crates/macrotest 12 | [`trybuild`]: https://crates.io/crates/trybuild -------------------------------------------------------------------------------- /macros/src/lib_readme.md: -------------------------------------------------------------------------------- 1 | Supporting utils for [Openraft](https://crates.io/crates/openraft). 2 | 3 | # `add_async_trait` 4 | 5 | [`#[add_async_trait]`](`macro@crate::add_async_trait`) adds `Send` bounds to an async trait. 6 | 7 | ## Example 8 | 9 | ``` 10 | #[openraft_macros::add_async_trait] 11 | trait MyTrait { 12 | async fn my_method(&self) -> Result<(), String>; 13 | } 14 | ``` 15 | 16 | The above code will be transformed into: 17 | 18 | ```ignore 19 | trait MyTrait { 20 | fn my_method(&self) -> impl Future> + Send; 21 | } 22 | ``` 23 | 24 | 25 | # `since` 26 | 27 | [`#[since(version = "1.0.0")]`](`macro@crate::since`) adds a doc line `/// Since: 1.0.0`. 28 | 29 | ## Example 30 | 31 | ```rust,ignore 32 | /// Foo function 33 | #[since(version = "1.0.0")] 34 | fn foo() {} 35 | ``` 36 | 37 | The above code will be transformed into: 38 | 39 | ```rust,ignore 40 | /// Foo function 41 | /// 42 | /// Since: 1.0.0 43 | fn foo() {} 44 | ``` 45 | 46 | 47 | # `expand` a template 48 | 49 | [`expand!()`](`crate::expand!`) renders a template with arguments multiple times. 50 | 51 | # Example: 52 | 53 | ```rust 54 | # use openraft_macros::expand; 55 | # fn foo () { 56 | expand!(KEYED, // ignore duplicate by `K` 57 | (K, T, V) => {let K: T = V;}, 58 | (a, u64, 1), 59 | (a, u32, 2), // duplicate `a` will be ignored 60 | (c, Vec, vec![1,2]) 61 | ); 62 | # } 63 | ``` 64 | 65 | The above code will be transformed into: 66 | 67 | ```rust 68 | # fn foo () { 69 | let a: u64 = 1; 70 | let c: Vec = vec![1, 2]; 71 | # } 72 | ``` 73 | -------------------------------------------------------------------------------- /macros/src/utils.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | use proc_macro2::TokenTree; 3 | 4 | /// Check if the next two token is a doc attribute, such as `#[doc = "foo"]`. 5 | /// 6 | /// An doc attribute is composed of a `#` token and a `Group` token with a `Bracket` delimiter: 7 | /// ```ignore 8 | /// Punct { ch: '#', }, 9 | /// Group { 10 | /// delimiter: Bracket, 11 | /// stream: TokenStream [ 12 | /// Ident { ident: "doc", }, 13 | /// Punct { ch: '=', }, 14 | /// Literal { kind: Str, symbol: " Doc", }, 15 | /// ], 16 | /// }, 17 | /// ``` 18 | pub(crate) fn is_doc(curr: &TokenTree, next: &TokenTree) -> bool { 19 | let TokenTree::Punct(p) = curr else { 20 | return false; 21 | }; 22 | 23 | if p.as_char() != '#' { 24 | return false; 25 | } 26 | 27 | let TokenTree::Group(g) = &next else { 28 | return false; 29 | }; 30 | 31 | if g.delimiter() != proc_macro2::Delimiter::Bracket { 32 | return false; 33 | } 34 | let first = g.stream().into_iter().next(); 35 | let Some(first) = first else { 36 | return false; 37 | }; 38 | 39 | let TokenTree::Ident(i) = first else { 40 | return false; 41 | }; 42 | 43 | i == "doc" 44 | } 45 | 46 | pub(crate) fn token_stream_with_error(mut item: TokenStream, e: syn::Error) -> TokenStream { 47 | item.extend(TokenStream::from(e.into_compile_error())); 48 | item 49 | } 50 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_0.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() {} 2 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_0.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | ); 6 | } 7 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_1.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | } 4 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_1.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | ); 7 | } 8 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_dup_a.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | let b: String = "foo".to_string(); 4 | } 5 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_dup_a.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | (b, String, "foo".to_string()), 7 | (a, u32, 2), // duplicate `a` will be ignored 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/not_keyed_let_dup_a.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | let b: String = "foo".to_string(); 4 | let a: u32 = 2; 5 | } 6 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/not_keyed_let_dup_a.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | (b, String, "foo".to_string()), 7 | (a, u32, 2), 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_empty.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | c; 3 | c; 4 | u8; 5 | } 6 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_empty.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {K; T; V;}, 5 | (c, , ,), 6 | (c, , u8 ), 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_type_attributes.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let c: u8; 3 | #[allow(dead_code)] 4 | let c: u16; 5 | #[allow(dead_code)] 6 | #[allow(dead_code)] 7 | let c: u16; 8 | } 9 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_type_attributes.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, M, T) => {M let K: T;}, 5 | (c, , u8, ), 6 | (c, #[allow(dead_code)] , u16), 7 | (c, #[allow(dead_code)] #[allow(dead_code)] , u16), 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/arg0_invalid_keyed.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !FOO, 4 | (K, T, V) => {K; T; V;}, 5 | ); 6 | 7 | openraft_macros::expand!( 8 | FOO, 9 | (K, T, V) => {K; T; V;}, 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/arg0_invalid_keyed.stderr: -------------------------------------------------------------------------------- 1 | error: Expected KEYED 2 | --> tests/expand/fail/arg0_invalid_keyed.rs:3:10 3 | | 4 | 3 | !FOO, 5 | | ^^^ 6 | 7 | error: Expected KEYED 8 | --> tests/expand/fail/arg0_invalid_keyed.rs:8:9 9 | | 10 | 8 | FOO, 11 | | ^^^ 12 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/variable_absent.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {K; T; V;}, 5 | (c, , ), 6 | (c, , u8, ), 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/variable_absent.stderr: -------------------------------------------------------------------------------- 1 | error: Expected the same number of arguments(`c, ,`) as template variables(`["K", "T", "V"]`) 2 | --> tests/expand/fail/variable_absent.rs:5:10 3 | | 4 | 5 | (c, , ), 5 | | ^ 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/multi_since.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.1.0, Date(2021-02-01): add Foo; add Bar; add Baz 2 | /// 3 | /// Since: 1.0.0, Date(2021-01-01): Added 4 | const A: i32 = 0; 5 | -------------------------------------------------------------------------------- /macros/tests/since/expand/multi_since.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since( 2 | version = "1.1.0", 3 | date = "2021-02-01", 4 | change = "add Foo", 5 | change = "add Bar", 6 | change = "add Baz" 7 | )] 8 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01", change = "Added")] 9 | const A: i32 = 0; 10 | -------------------------------------------------------------------------------- /macros/tests/since/expand/valid_semver.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/valid_semver.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0: add Foo 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", change = "add Foo")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01) 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01): add Foo 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01", change = "add Foo")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_many_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01): add Foo; add Bar; add Baz 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_many_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since( 2 | version = "1.0.0", 3 | date = "2021-01-01", 4 | change = "add Foo", 5 | change = "add Bar", 6 | change = "add Baz" 7 | )] 8 | const A: i32 = 0; 9 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_doc.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Doc 2 | /// 3 | /// By default, `Send` bounds will be added to the trait and to the return bounds of any async 4 | /// functions defined within the trait. 5 | /// 6 | /// If the `singlethreaded` feature is enabled, the trait definition remains the same without any 7 | /// added `Send` bounds. 8 | /// 9 | /// # Example 10 | /// 11 | /// - list 12 | /// 13 | /// Since: 1.0.0, Date(2021-01-01) 14 | const A: i32 = 0; 15 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_doc.rs: -------------------------------------------------------------------------------- 1 | /// Doc 2 | /// 3 | /// By default, `Send` bounds will be added to the trait and to the return bounds of any async 4 | /// functions defined within the trait. 5 | /// 6 | /// If the `singlethreaded` feature is enabled, the trait definition remains the same without any 7 | /// added `Send` bounds. 8 | /// 9 | /// # Example 10 | /// 11 | /// - list 12 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01")] 13 | const A: i32 = 0; 14 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_date.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01--")] 2 | fn main() {} 3 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_date.stderr: -------------------------------------------------------------------------------- 1 | error: `date`(`2021-01--`) is not valid date string. Expected format: yyyy-mm-dd 2 | --> tests/since/fail/invalid_date.rs:1:52 3 | | 4 | 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01--")] 5 | | ^^^^^^^^^^^ 6 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_sem_ver.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0..0")] 2 | fn main() {} 3 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_sem_ver.stderr: -------------------------------------------------------------------------------- 1 | error: `version`(`1.0.0..0`) is not valid semver. 2 | --> tests/since/fail/invalid_sem_ver.rs:1:36 3 | | 4 | 1 | #[openraft_macros::since(version = "1.0.0..0")] 5 | | ^^^^^^^^^^ 6 | -------------------------------------------------------------------------------- /macros/tests/test_expand.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn fail() { 3 | let t = trybuild::TestCases::new(); 4 | t.compile_fail("tests/expand/fail/*.rs"); 5 | } 6 | 7 | #[test] 8 | fn pass() { 9 | macrotest::expand("tests/expand/expand/*.rs"); 10 | } 11 | -------------------------------------------------------------------------------- /macros/tests/test_since.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn fail() { 3 | let t = trybuild::TestCases::new(); 4 | t.compile_fail("tests/since/fail/*.rs"); 5 | } 6 | 7 | #[test] 8 | fn pass() { 9 | macrotest::expand("tests/since/expand/*.rs"); 10 | } 11 | -------------------------------------------------------------------------------- /openraft/.gitignore: -------------------------------------------------------------------------------- 1 | /_log/ 2 | -------------------------------------------------------------------------------- /openraft/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /openraft/src/base/finalized.rs: -------------------------------------------------------------------------------- 1 | //! Provides a marker trait to prevent external implementation of trait methods. 2 | 3 | /// A marker trait used to prevent specific already auto-implemented trait methods from being 4 | /// re-implemented outside their defining crate. 5 | /// 6 | /// This is achieved by adding this non-referencable marker trait to a trait method signature. 7 | /// 8 | /// # Example 9 | /// 10 | /// The following code demonstrates how `Final` prevents external implementation: 11 | /// 12 | /// ```ignore 13 | /// pub trait Trait { 14 | /// // This method cannot be implemented by users because it requires 15 | /// // the private `Final` trait in its bounds 16 | /// fn unimplementable(&self) where Self: Final { 17 | /// self.user_impl_this(); 18 | /// } 19 | /// 20 | /// // This method can be implemented by users 21 | /// fn user_impl_this(&self); 22 | /// } 23 | /// 24 | /// pub struct MyType; 25 | /// 26 | /// impl Trait for MyType { 27 | /// // Attempting to implement this method will fail to compile 28 | /// // because `Final` is not accessible from outside the crate 29 | /// fn unimplementable(&self) where Self: Final {} 30 | /// 31 | /// fn user_impl_this(&self) { 32 | /// println!("This implementation is allowed"); 33 | /// } 34 | /// } 35 | /// ``` 36 | pub trait Final {} 37 | 38 | impl Final for T {} 39 | -------------------------------------------------------------------------------- /openraft/src/compat/mod.rs: -------------------------------------------------------------------------------- 1 | //! This mod is a upgrade helper that provides functionalities for a newer openraft application to 2 | //! read data written by an older application. 3 | 4 | mod upgrade; 5 | 6 | pub use upgrade::Compat; 7 | pub use upgrade::Upgrade; 8 | -------------------------------------------------------------------------------- /openraft/src/compat/upgrade.rs: -------------------------------------------------------------------------------- 1 | /// Upgrade Self to type `To`. 2 | /// 3 | /// This trait is used to define types that can be directly upgraded from older versions of openraft 4 | /// to newer versions. For example, `LogId` can be upgrade: in openraft 0.7, `LogId` is `(term, 5 | /// index)`, which is upgraded to `(CommittedLeaderId, index)` in openraft 0.8. 6 | pub trait Upgrade { 7 | /// Upgrades the current instance to type To. 8 | fn upgrade(self) -> To; 9 | 10 | fn try_upgrade(self) -> Result 11 | where Self: Sized { 12 | Ok(self.upgrade()) 13 | } 14 | } 15 | 16 | /// `Compat` is a serialization compatible type that can be deserialized from both an older type 17 | /// and a newer type. It serves as an intermediate type container for newer programs to read old 18 | /// data. 19 | #[derive(Debug)] 20 | #[derive(serde::Deserialize, serde::Serialize)] 21 | #[serde(untagged)] 22 | pub enum Compat 23 | where From: Upgrade 24 | { 25 | /// Represents the older version of the data. 26 | Old(From), 27 | /// Represents the newer version of the data. 28 | New(To), 29 | } 30 | 31 | /// A compatible type can be upgraded to `To` if `From` can be upgraded to `To`. 32 | impl Upgrade for Compat 33 | where From: Upgrade 34 | { 35 | fn upgrade(self) -> To { 36 | match self { 37 | Self::Old(o) => o.upgrade(), 38 | Self::New(x) => x, 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /openraft/src/config/error.rs: -------------------------------------------------------------------------------- 1 | use anyerror::AnyError; 2 | 3 | /// Error variants related to configuration. 4 | #[derive(Debug, thiserror::Error)] 5 | #[derive(PartialEq, Eq)] 6 | pub enum ConfigError { 7 | #[error("ParseError: {source} while parsing ({args:?})")] 8 | ParseError { source: AnyError, args: Vec }, 9 | 10 | /// The min election timeout is not smaller than the max election timeout. 11 | #[error("election timeout: min({min}) must be < max({max})")] 12 | ElectionTimeout { min: u64, max: u64 }, 13 | 14 | #[error("max_payload_entries must be > 0")] 15 | MaxPayloadIs0, 16 | 17 | #[error("election_timeout_min({election_timeout_min}) must be > heartbeat_interval({heartbeat_interval})")] 18 | ElectionTimeoutLTHeartBeat { 19 | election_timeout_min: u64, 20 | heartbeat_interval: u64, 21 | }, 22 | 23 | #[error("snapshot policy string is invalid: '{invalid:?}' expect: '{syntax}'")] 24 | InvalidSnapshotPolicy { invalid: String, syntax: String }, 25 | 26 | #[error("{reason} when parsing {invalid:?}")] 27 | InvalidNumber { invalid: String, reason: String }, 28 | } 29 | -------------------------------------------------------------------------------- /openraft/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::module_inception)] 2 | mod config; 3 | mod error; 4 | 5 | #[cfg(test)] 6 | mod config_test; 7 | 8 | pub use config::Config; 9 | pub(crate) use config::RuntimeConfig; 10 | pub use config::SnapshotPolicy; 11 | pub use error::ConfigError; 12 | -------------------------------------------------------------------------------- /openraft/src/core/balancer.rs: -------------------------------------------------------------------------------- 1 | //! This mod defines and manipulates the ratio of multiple messages to handle. 2 | //! 3 | //! The ratio should be adjust so that every channel won't be starved. 4 | 5 | /// Balance the ratio of different kind of message to handle. 6 | pub(crate) struct Balancer { 7 | total: u64, 8 | 9 | /// The number of RaftMsg to handle in each round. 10 | raft_msg: u64, 11 | } 12 | 13 | impl Balancer { 14 | pub(crate) fn new(total: u64) -> Self { 15 | Self { 16 | total, 17 | // RaftMsg is the input entry. 18 | // We should consume as many as internal messages as possible. 19 | raft_msg: total / 10, 20 | } 21 | } 22 | 23 | pub(crate) fn raft_msg(&self) -> u64 { 24 | self.raft_msg 25 | } 26 | 27 | pub(crate) fn notification(&self) -> u64 { 28 | self.total - self.raft_msg 29 | } 30 | 31 | pub(crate) fn increase_notification(&mut self) { 32 | self.raft_msg = self.raft_msg * 15 / 16; 33 | if self.raft_msg == 0 { 34 | self.raft_msg = 1; 35 | } 36 | } 37 | 38 | pub(crate) fn increase_raft_msg(&mut self) { 39 | self.raft_msg = self.raft_msg * 17 / 16; 40 | 41 | // Always leave some budget for other channels 42 | if self.raft_msg > self.total / 2 { 43 | self.raft_msg = self.total / 2; 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /openraft/src/core/heartbeat/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod event; 2 | pub(crate) mod handle; 3 | pub(crate) mod worker; 4 | -------------------------------------------------------------------------------- /openraft/src/core/mod.rs: -------------------------------------------------------------------------------- 1 | //! The `RaftCore` is a `Runtime` supporting the raft algorithm implementation `Engine`. 2 | //! 3 | //! It passes events from an application or timer or network to `Engine` to drive it to run. 4 | //! Also it receives and execute `Command` emitted by `Engine` to apply raft state to underlying 5 | //! storage or forward messages to other raft nodes. 6 | 7 | pub(crate) mod balancer; 8 | pub(crate) mod heartbeat; 9 | pub(crate) mod notification; 10 | mod raft_core; 11 | pub(crate) mod raft_msg; 12 | mod replication_state; 13 | mod server_state; 14 | pub(crate) mod sm; 15 | mod tick; 16 | 17 | pub(crate) use raft_core::ApplyResult; 18 | pub use raft_core::RaftCore; 19 | pub(crate) use replication_state::replication_lag; 20 | pub use server_state::ServerState; 21 | pub(crate) use tick::Tick; 22 | pub(crate) use tick::TickHandle; 23 | -------------------------------------------------------------------------------- /openraft/src/core/replication_state.rs: -------------------------------------------------------------------------------- 1 | use crate::log_id::LogIndexOptionExt; 2 | 3 | /// Calculate the distance between the matched log index on a replication target and local last log 4 | /// index 5 | pub(crate) fn replication_lag(matched_log_index: &Option, last_log_index: &Option) -> u64 { 6 | last_log_index.next_index().saturating_sub(matched_log_index.next_index()) 7 | } 8 | 9 | #[cfg(test)] 10 | mod test { 11 | use crate::core::replication_state::replication_lag; 12 | 13 | #[test] 14 | fn test_replication_lag() -> anyhow::Result<()> { 15 | assert_eq!(0, replication_lag(&None, &None)); 16 | assert_eq!(4, replication_lag(&None, &Some(3))); 17 | assert_eq!(1, replication_lag(&Some(2), &Some(3))); 18 | assert_eq!(0, replication_lag(&Some(3), &Some(3))); 19 | assert_eq!(0, replication_lag(&Some(4), &Some(3))); 20 | Ok(()) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /openraft/src/core/server_state.rs: -------------------------------------------------------------------------------- 1 | /// All possible states of a Raft node. 2 | #[derive(Debug, Clone, Copy, Default)] 3 | #[derive(PartialEq, Eq)] 4 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 5 | pub enum ServerState { 6 | /// The node is completely passive; replicating entries, but neither voting nor timing out. 7 | #[default] 8 | Learner, 9 | /// The node is replicating logs from the leader. 10 | Follower, 11 | /// The node is campaigning to become the cluster leader. 12 | Candidate, 13 | /// The node is the Raft cluster leader. 14 | Leader, 15 | /// The Raft node is shutting down. 16 | Shutdown, 17 | } 18 | 19 | impl ServerState { 20 | /// Check if currently in learner state. 21 | pub fn is_learner(&self) -> bool { 22 | matches!(self, Self::Learner) 23 | } 24 | 25 | /// Check if currently in follower state. 26 | pub fn is_follower(&self) -> bool { 27 | matches!(self, Self::Follower) 28 | } 29 | 30 | /// Check if currently in candidate state. 31 | pub fn is_candidate(&self) -> bool { 32 | matches!(self, Self::Candidate) 33 | } 34 | 35 | /// Check if currently in leader state. 36 | pub fn is_leader(&self) -> bool { 37 | matches!(self, Self::Leader) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /openraft/src/core/sm/mod.rs: -------------------------------------------------------------------------------- 1 | //! State machine worker and its supporting types. 2 | //! 3 | //! This worker runs in a separate task and is the only one that can mutate the state machine. 4 | //! It is responsible for applying log entries, building/receiving snapshot and sending responses 5 | //! to the RaftCore. 6 | 7 | pub(crate) mod command; 8 | pub(crate) mod handle; 9 | pub(crate) mod response; 10 | pub(crate) mod worker; 11 | 12 | pub(crate) use command::Command; 13 | pub(crate) use response::CommandResult; 14 | pub(crate) use response::Response; 15 | -------------------------------------------------------------------------------- /openraft/src/display_ext.rs: -------------------------------------------------------------------------------- 1 | //! Implement [`std::fmt::Display`] for types such as `Option` and slice `&[T]`. 2 | 3 | pub(crate) mod display_btreemap_opt_value; 4 | pub(crate) mod display_instant; 5 | pub(crate) mod display_option; 6 | pub(crate) mod display_result; 7 | pub(crate) mod display_slice; 8 | 9 | pub(crate) use display_btreemap_opt_value::DisplayBTreeMapOptValue; 10 | #[allow(unused_imports)] 11 | pub(crate) use display_btreemap_opt_value::DisplayBtreeMapOptValueExt; 12 | #[allow(unused_imports)] 13 | pub(crate) use display_instant::DisplayInstant; 14 | pub(crate) use display_instant::DisplayInstantExt; 15 | pub(crate) use display_option::DisplayOption; 16 | pub(crate) use display_option::DisplayOptionExt; 17 | #[allow(unused_imports)] 18 | pub(crate) use display_result::DisplayResult; 19 | pub(crate) use display_result::DisplayResultExt; 20 | pub(crate) use display_slice::DisplaySlice; 21 | pub(crate) use display_slice::DisplaySliceExt; 22 | -------------------------------------------------------------------------------- /openraft/src/display_ext/display_option.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Implement `Display` for `Option` if T is `Display`. 4 | /// 5 | /// It outputs a literal string `"None"` if it is None. Otherwise it invokes the Display 6 | /// implementation for T. 7 | pub(crate) struct DisplayOption<'a, T: fmt::Display>(pub &'a Option); 8 | 9 | impl fmt::Display for DisplayOption<'_, T> { 10 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 11 | match &self.0 { 12 | None => { 13 | write!(f, "None") 14 | } 15 | Some(x) => x.fmt(f), 16 | } 17 | } 18 | } 19 | 20 | pub(crate) trait DisplayOptionExt<'a, T: fmt::Display> { 21 | fn display(&'a self) -> DisplayOption<'a, T>; 22 | } 23 | 24 | impl DisplayOptionExt<'_, T> for Option 25 | where T: fmt::Display 26 | { 27 | fn display(&self) -> DisplayOption { 28 | DisplayOption(self) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/display_ext/display_result.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Implement `Display` for `Result` if T and E are `Display`. 4 | /// 5 | /// It outputs `"Ok(...)"` or `"Err(...)"`. 6 | pub(crate) struct DisplayResult<'a, T: fmt::Display, E: fmt::Display>(pub &'a Result); 7 | 8 | impl fmt::Display for DisplayResult<'_, T, E> 9 | where 10 | T: fmt::Display, 11 | E: fmt::Display, 12 | { 13 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 14 | match &self.0 { 15 | Ok(ok) => { 16 | write!(f, "Ok({})", ok) 17 | } 18 | Err(err) => { 19 | write!(f, "Err({})", err) 20 | } 21 | } 22 | } 23 | } 24 | 25 | pub(crate) trait DisplayResultExt<'a, T: fmt::Display, E: fmt::Display> { 26 | fn display(&'a self) -> DisplayResult<'a, T, E>; 27 | } 28 | 29 | impl DisplayResultExt<'_, T, E> for Result 30 | where 31 | T: fmt::Display, 32 | E: fmt::Display, 33 | { 34 | fn display(&self) -> DisplayResult { 35 | DisplayResult(self) 36 | } 37 | } 38 | 39 | #[cfg(test)] 40 | mod tests { 41 | use super::*; 42 | 43 | #[test] 44 | fn test_display_result() { 45 | let result: Result = Ok(42); 46 | let display_result = DisplayResult(&result); 47 | assert_eq!(format!("{}", display_result), "Ok(42)"); 48 | 49 | let result: Result = Err("error"); 50 | let display_result = DisplayResult(&result); 51 | assert_eq!(format!("{}", display_result), "Err(error)"); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /openraft/src/docs/cluster_control/cluster-control.md: -------------------------------------------------------------------------------- 1 | # Managing Clusters: Adding/Removing Nodes 2 | 3 | The `Raft` type offers various API methods to control a raft cluster. 4 | Several concepts are associated with cluster management: 5 | 6 | - `Voter`: A raft node responsible for voting, electing itself as a `Candidate`, and becoming a `Leader` or `Follower`. 7 | 8 | - `Candidate`: A node attempting to elect itself as the Leader. 9 | 10 | - `Leader`: The sole node in a cluster that handles application requests. 11 | 12 | - `Follower`: A node that acknowledges the presence of a valid leader and only receives replicated logs. 13 | 14 | - `Learner`: A node that cannot vote but only receives logs. 15 | 16 | 17 | `Voter` state transition: 18 | 19 | ```text 20 | vote granted by a quorum 21 | .--> Candidate ----------------------> Leader 22 | heartbeat | | | 23 | timeout | | seen a higher Leader | seen a higher Leader 24 | | v | 25 | '----Follower <--------------------------' 26 | ``` 27 | -------------------------------------------------------------------------------- /openraft/src/docs/cluster_control/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("cluster-control.md")] 2 | 3 | pub mod cluster_formation { 4 | #![doc = include_str!("cluster-formation.md")] 5 | } 6 | 7 | pub mod dynamic_membership { 8 | #![doc = include_str!("dynamic-membership.md")] 9 | } 10 | 11 | pub mod node_lifecycle { 12 | #![doc = include_str!("node-lifecycle.md")] 13 | } 14 | -------------------------------------------------------------------------------- /openraft/src/docs/components/mod.rs: -------------------------------------------------------------------------------- 1 | //! Components and sub systems of the openraft project. 2 | //! 3 | //! - [`StateMachine`](state_machine) 4 | 5 | pub mod state_machine { 6 | #![doc = include_str!("state-machine.md")] 7 | } 8 | -------------------------------------------------------------------------------- /openraft/src/docs/data/mod.rs: -------------------------------------------------------------------------------- 1 | //! Data structures used by the Openraft protocol, such log, vote, snapshot, membership etc. 2 | 3 | pub mod leader_id { 4 | #![doc = include_str!("leader_id.md")] 5 | } 6 | 7 | pub mod vote { 8 | #![doc = include_str!("vote.md")] 9 | } 10 | 11 | pub mod log_pointers { 12 | #![doc = include_str!("log_pointers.md")] 13 | } 14 | 15 | pub mod leader_lease { 16 | #![doc = include_str!("leader-lease.md")] 17 | } 18 | 19 | pub mod extended_membership { 20 | #![doc = include_str!("extended-membership.md")] 21 | } 22 | 23 | pub mod effective_membership { 24 | #![doc = include_str!("effective-membership.md")] 25 | } 26 | 27 | pub mod replication_session { 28 | #![doc = include_str!("replication-session.md")] 29 | } 30 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | # dependency: 3 | # https://github.com/jonschlinkert/markdown-toc#cli 4 | # brew install markdown-toc 5 | markdown-toc faq.md > faq-toc.md 6 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/mod.rs: -------------------------------------------------------------------------------- 1 | //! # FAQ 2 | #![doc = include_str!("faq-toc.md")] 3 | #![doc = include_str!("faq.md")] 4 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | # dependency: 3 | # https://github.com/jonschlinkert/markdown-toc#cli 4 | # brew install markdown-toc 5 | markdown-toc feature-flags.md > feature-flags-toc.md 6 | echo "" >> feature-flags-toc.md 7 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/feature-flags-toc.md: -------------------------------------------------------------------------------- 1 | - [feature-flag `adapt-network-v1`](#feature-flag-adapt-network-v1) 2 | - [feature-flag `bench`](#feature-flag-bench) 3 | - [feature-flag `bt`](#feature-flag-bt) 4 | - [feature-flag `compat`](#feature-flag-compat) 5 | - [feature-flag `serde`](#feature-flag-serde) 6 | - [feature-flag `single-term-leader`](#feature-flag-single-term-leader) 7 | - [feature-flag `singlethreaded`](#feature-flag-singlethreaded) 8 | - [feature-flag `tokio-rt`](#feature-flag-tokio-rt) 9 | - [feature-flag `tracing-log`](#feature-flag-tracing-log) 10 | - [feature-flag `type-alias`](#feature-flag-type-alias) 11 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/mod.rs: -------------------------------------------------------------------------------- 1 | //! # Feature flags 2 | 3 | #![doc = include_str!("feature-flags-toc.md")] 4 | // 5 | #![doc = include_str!("feature-flags.md")] 6 | -------------------------------------------------------------------------------- /openraft/src/docs/getting_started/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("getting-started.md")] 2 | -------------------------------------------------------------------------------- /openraft/src/docs/internal/internal.md: -------------------------------------------------------------------------------- 1 | # Openraft internal design and implementation 2 | 3 | - [Architecture](`crate::docs::internal::architecture`) : The overall architecture of Openraft. 4 | - [Threading](`crate::docs::internal::threading`) : The threading model of Openraft. 5 | -------------------------------------------------------------------------------- /openraft/src/docs/internal/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("internal.md")] 2 | 3 | pub mod architecture { 4 | #![doc = include_str!("architecture.md")] 5 | } 6 | 7 | pub mod threading { 8 | #![doc = include_str!("threading.md")] 9 | } 10 | -------------------------------------------------------------------------------- /openraft/src/docs/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(rustdoc::redundant_explicit_links)] 2 | #![doc = include_str!("docs.md")] 3 | 4 | pub mod faq; 5 | 6 | pub mod getting_started; 7 | 8 | pub mod cluster_control; 9 | 10 | pub mod feature_flags; 11 | 12 | pub mod data; 13 | 14 | pub mod components; 15 | 16 | pub mod protocol; 17 | 18 | pub mod internal; 19 | 20 | pub mod upgrade_guide; 21 | 22 | pub mod obsolete; 23 | -------------------------------------------------------------------------------- /openraft/src/docs/obsolete/fast_commit.md: -------------------------------------------------------------------------------- 1 | ### Fast Commit(obsolete) 2 | 3 | **This algorithm is no longer used**, because it would be more flexible treating 4 | local-IO the same as replication to a remote node. As a result, applying a log does not 5 | need to wait for it to be flushed on local disk. 6 | 7 | #### Fast Commit Algorithm 8 | 9 | The algorithm assumes that an entry will be committed as soon as it is appended 10 | if the cluster has only one voter. 11 | 12 | However, if there is a membership log in the middle of the input entries, the 13 | condition to commit will change, and entries before and after the membership 14 | entry must be dealt with differently. 15 | 16 | When a membership entry is seen, update progress for all former entries. 17 | Then upgrade the quorum set for the Progress. 18 | 19 | E.g., if the input entries are `2..6`, entry 4 changes membership from `a` to `abc`. 20 | Then it will output a LeaderCommit command to commit entries `2,3`. 21 | 22 | ```text 23 | 1 2 3 4 5 6 24 | a x x a y y 25 | b 26 | c 27 | ``` 28 | 29 | If the input entries are `2..6`, entry 4 changes membership from `abc` to `a`. 30 | Then it will output a LeaderCommit command to commit entries `2,3,4,5,6`. 31 | 32 | ```text 33 | 1 2 3 4 5 6 34 | a x x a y y 35 | b 36 | c 37 | ``` 38 | -------------------------------------------------------------------------------- /openraft/src/docs/obsolete/mod.rs: -------------------------------------------------------------------------------- 1 | //! Obsolete Designs 2 | //! 3 | //! Several designs in Openraft have been discarded due to the problems they caused for 4 | //! applications. These designs were attempts at optimization or simplification, but ultimately 5 | //! proved to be inappropriate. They are included in this chapter as an archive to explain why they 6 | //! were discarded. 7 | 8 | pub mod heartbeat { 9 | #![doc = include_str!("blank-log-heartbeat.md")] 10 | } 11 | 12 | pub mod fast_commit { 13 | #![doc = include_str!("fast_commit.md")] 14 | } 15 | -------------------------------------------------------------------------------- /openraft/src/docs/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | //! The protocol used by Openraft to replicate data. 2 | 3 | pub mod read { 4 | #![doc = include_str!("read.md")] 5 | } 6 | 7 | pub mod replication { 8 | #![doc = include_str!("replication.md")] 9 | 10 | pub mod leader_lease { 11 | #![doc = include_str!("leader_lease.md")] 12 | } 13 | 14 | pub mod log_replication { 15 | #![doc = include_str!("log_replication.md")] 16 | } 17 | 18 | pub mod log_stream { 19 | #![doc = include_str!("log_stream.md")] 20 | } 21 | 22 | pub mod snapshot_replication { 23 | #![doc = include_str!("snapshot_replication.md")] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /openraft/src/docs/protocol/replication.md: -------------------------------------------------------------------------------- 1 | # Replication 2 | 3 | Appending entries is indeed the primary RPC for replicating logs from the leader to followers or learners in the Raft consensus algorithm. 4 | Installing a snapshot can be considered a special form of **appending logs** since it serves a similar purpose: 5 | ensuring that all nodes have a consistent state, particularly when log entries become too numerous or when a node has fallen far behind. 6 | 7 | - [Replication by Append-Entries](`crate::docs::protocol::replication::log_replication`). 8 | - [Replication by Snapshot](`crate::docs::protocol::replication::snapshot_replication`). 9 | 10 | -------------------------------------------------------------------------------- /openraft/src/docs/upgrade_guide/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("upgrade.md")] 2 | 3 | pub mod upgrade_06_07 { 4 | //! See: [Guide for upgrading v0.6 to v0.7](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_06_07) 5 | } 6 | pub mod upgrade_07_08 { 7 | //! See: [Guide for upgrading v0.7 to v0.8](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_07_08) 8 | } 9 | pub mod upgrade_083_084 { 10 | //! See: [Guide for upgrading v0.8.3 to v0.8.4](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_083_084) 11 | } 12 | pub mod upgrade_08_09 { 13 | #![doc = include_str!("upgrade-v08-v09.md")] 14 | } 15 | -------------------------------------------------------------------------------- /openraft/src/engine/command_kind.rs: -------------------------------------------------------------------------------- 1 | /// Command kind is used to categorize commands. 2 | /// 3 | /// Commands of the different kinds can be paralleled. 4 | #[allow(dead_code)] 5 | #[derive(Debug, Clone, Copy)] 6 | #[derive(PartialEq, Eq)] 7 | pub(crate) enum CommandKind { 8 | /// Log IO command 9 | Log, 10 | /// Network IO command 11 | Network, 12 | /// State machine IO command 13 | StateMachine, 14 | /// RaftCore main thread command 15 | Main, 16 | } 17 | -------------------------------------------------------------------------------- /openraft/src/engine/handler/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod establish_handler; 2 | pub(crate) mod following_handler; 3 | pub(crate) mod leader_handler; 4 | pub(crate) mod log_handler; 5 | pub(crate) mod replication_handler; 6 | pub(crate) mod server_state_handler; 7 | pub(crate) mod snapshot_handler; 8 | pub(crate) mod vote_handler; 9 | -------------------------------------------------------------------------------- /openraft/src/engine/handler/snapshot_handler/trigger_snapshot_test.rs: -------------------------------------------------------------------------------- 1 | use pretty_assertions::assert_eq; 2 | 3 | use crate::core::sm; 4 | use crate::engine::testing::UTConfig; 5 | use crate::engine::Command; 6 | use crate::engine::Engine; 7 | 8 | fn eng() -> Engine { 9 | let mut eng = Engine::testing_default(0); 10 | eng.state.enable_validation(false); // Disable validation for incomplete state 11 | 12 | eng 13 | } 14 | 15 | #[test] 16 | fn test_trigger_snapshot() -> anyhow::Result<()> { 17 | let mut eng = eng(); 18 | 19 | assert_eq!( 20 | false, 21 | eng.state.io_state_mut().building_snapshot(), 22 | "initially, snapshot is not triggered" 23 | ); 24 | 25 | // Trigger snapshot. 26 | 27 | let got = eng.snapshot_handler().trigger_snapshot(); 28 | 29 | assert_eq!(true, got); 30 | assert_eq!(true, eng.state.io_state_mut().building_snapshot()); 31 | assert_eq!( 32 | vec![ 33 | // 34 | Command::from(sm::Command::build_snapshot()), 35 | ], 36 | eng.output.take_commands() 37 | ); 38 | 39 | // Trigger twice will not trigger again. 40 | 41 | let got = eng.snapshot_handler().trigger_snapshot(); 42 | assert_eq!(false, got, "snapshot is already triggered"); 43 | assert_eq!(0, eng.output.take_commands().len()); 44 | 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /openraft/src/engine/replication_progress.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::progress::entry::ProgressEntry; 4 | use crate::RaftTypeConfig; 5 | 6 | #[derive(Debug)] 7 | #[derive(PartialEq, Eq)] 8 | pub(crate) struct ReplicationProgress(pub C::NodeId, pub ProgressEntry); 9 | 10 | impl fmt::Display for ReplicationProgress 11 | where C: RaftTypeConfig 12 | { 13 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 14 | write!(f, "ReplicationProgress({}={})", self.0, self.1) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /openraft/src/entry/raft_entry_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::entry::RaftEntry; 2 | use crate::log_id::ref_log_id::RefLogId; 3 | use crate::RaftTypeConfig; 4 | 5 | pub(crate) trait RaftEntryExt: RaftEntry 6 | where C: RaftTypeConfig 7 | { 8 | /// Returns a lightweight [`RefLogId`] that contains the log id information. 9 | fn ref_log_id(&self) -> RefLogId<'_, C> { 10 | let (leader_id, index) = self.log_id_parts(); 11 | RefLogId::new(leader_id, index) 12 | } 13 | } 14 | 15 | impl RaftEntryExt for T 16 | where 17 | C: RaftTypeConfig, 18 | T: RaftEntry, 19 | { 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/error/allow_next_revert_error.rs: -------------------------------------------------------------------------------- 1 | use crate::error::ForwardToLeader; 2 | use crate::error::NodeNotFound; 3 | use crate::RaftTypeConfig; 4 | 5 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 6 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 7 | pub enum AllowNextRevertError { 8 | #[error("Can not set allow_next_revert; error: {0}")] 9 | NodeNotFound(#[from] NodeNotFound), 10 | #[error("Can not set allow_next_revert; error: {0}")] 11 | ForwardToLeader(#[from] ForwardToLeader), 12 | } 13 | -------------------------------------------------------------------------------- /openraft/src/error/into_ok.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Infallible; 2 | 3 | /// Trait to convert `Result` to `T`, if `E` is a `never` type. 4 | pub(crate) trait UnwrapInfallible { 5 | fn into_ok(self) -> T; 6 | } 7 | 8 | impl UnwrapInfallible for Result 9 | where E: Into 10 | { 11 | fn into_ok(self) -> T { 12 | match self { 13 | Ok(t) => t, 14 | Err(_) => unreachable!(), 15 | } 16 | } 17 | } 18 | 19 | /// Convert `Result` to `T`, if `E` is a `never` type. 20 | pub(crate) fn into_ok(result: Result) -> T 21 | where E: Into { 22 | UnwrapInfallible::into_ok(result) 23 | } 24 | -------------------------------------------------------------------------------- /openraft/src/error/invalid_sm.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 2 | #[error( 3 | "User-defined function on the state machine failed to run; \ 4 | It may have used a different type \ 5 | of state machine from the one in RaftCore (`{actual_type}`)" 6 | )] 7 | pub struct InvalidStateMachineType { 8 | pub actual_type: &'static str, 9 | } 10 | 11 | impl InvalidStateMachineType { 12 | pub(crate) fn new() -> Self { 13 | Self { 14 | actual_type: std::any::type_name::(), 15 | } 16 | } 17 | } 18 | 19 | #[cfg(test)] 20 | mod tests { 21 | #[test] 22 | fn test_invalid_state_machine_type_to_string() { 23 | let err = super::InvalidStateMachineType::new::(); 24 | assert_eq!( 25 | err.to_string(), 26 | "User-defined function on the state machine failed to run; It may have used a different type of state machine from the one in RaftCore (`u32`)" 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /openraft/src/error/membership_error.rs: -------------------------------------------------------------------------------- 1 | use crate::error::ChangeMembershipError; 2 | use crate::error::EmptyMembership; 3 | use crate::error::LearnerNotFound; 4 | use crate::error::NodeNotFound; 5 | use crate::RaftTypeConfig; 6 | 7 | /// Errors occur when building a [`Membership`]. 8 | /// 9 | /// [`Membership`]: crate::membership::Membership 10 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 11 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 12 | pub enum MembershipError { 13 | #[error(transparent)] 14 | EmptyMembership(#[from] EmptyMembership), 15 | 16 | #[error(transparent)] 17 | NodeNotFound(#[from] NodeNotFound), 18 | } 19 | 20 | impl From> for ChangeMembershipError 21 | where C: RaftTypeConfig 22 | { 23 | fn from(me: MembershipError) -> Self { 24 | match me { 25 | MembershipError::EmptyMembership(e) => ChangeMembershipError::EmptyMembership(e), 26 | MembershipError::NodeNotFound(e) => { 27 | ChangeMembershipError::LearnerNotFound(LearnerNotFound { node_id: e.node_id }) 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /openraft/src/error/node_not_found.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Operation; 2 | use crate::RaftTypeConfig; 3 | 4 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 5 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 6 | #[error("Node {node_id} not found when: ({operation})")] 7 | pub struct NodeNotFound { 8 | pub node_id: C::NodeId, 9 | pub operation: Operation, 10 | } 11 | 12 | impl NodeNotFound { 13 | pub fn new(node_id: C::NodeId, operation: Operation) -> Self { 14 | Self { node_id, operation } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /openraft/src/error/replication_closed.rs: -------------------------------------------------------------------------------- 1 | /// Replication is closed intentionally. 2 | /// 3 | /// No further replication action should be taken. 4 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 5 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 6 | #[error("Replication is closed: {reason}")] 7 | pub struct ReplicationClosed { 8 | reason: String, 9 | } 10 | 11 | impl ReplicationClosed { 12 | pub fn new(reason: impl ToString) -> Self { 13 | Self { 14 | reason: reason.to_string(), 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /openraft/src/impls/mod.rs: -------------------------------------------------------------------------------- 1 | //! Collection of implementations of usually used traits defined by Openraft 2 | 3 | pub use crate::entry::Entry; 4 | pub use crate::node::BasicNode; 5 | pub use crate::node::EmptyNode; 6 | pub use crate::raft::responder::impls::OneshotResponder; 7 | #[cfg(feature = "tokio-rt")] 8 | pub use crate::type_config::async_runtime::tokio_impls::TokioRuntime; 9 | 10 | /// LeaderId implementation for advanced mode, allowing multiple leaders per term. 11 | pub mod leader_id_adv { 12 | pub use crate::vote::leader_id::leader_id_adv::LeaderId; 13 | } 14 | 15 | /// LeaderId implementation for standard Raft mode, enforcing single leader per term. 16 | pub mod leader_id_std { 17 | pub use crate::vote::leader_id::leader_id_std::LeaderId; 18 | } 19 | 20 | /// Default implementation of a raft log identity. 21 | pub use crate::log_id::LogId; 22 | /// Default [`RaftVote`] implementation for both standard Raft mode and multi-leader-per-term mode. 23 | /// 24 | /// The difference between the two modes is the implementation of [`RaftLeaderId`]. 25 | /// 26 | /// [`RaftVote`]: crate::vote::raft_vote::RaftVote 27 | /// [`RaftLeaderId`]: crate::vote::RaftLeaderId 28 | pub use crate::vote::Vote; 29 | -------------------------------------------------------------------------------- /openraft/src/lib_readme.md: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 |

Openraft

5 |
6 | 7 | ## API status 8 | 9 | **Openraft API is currently unstable**. 10 | Incompatibilities may arise in upgrades prior to `1.0.0`. 11 | Refer to our [Change-Log](https://github.com/databendlabs/openraft/blob/main/change-log.md) for details. 12 | [Upgrade Guides](crate::docs::upgrade_guide) explains how to upgrade. 13 | 14 | Each commit message begins with a keyword indicating the type of change: 15 | 16 | - `DataChange:` Changes to on-disk data types, possibly requiring manual upgrade. 17 | - `Change:` Introduces incompatible API changes. 18 | - `Feature:` Adds compatible, non-breaking new features. 19 | - `Fix:` Addresses bug fixes. 20 | 21 | -------------------------------------------------------------------------------- /openraft/src/log_id/log_id_option_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::log_id::raft_log_id::RaftLogId; 2 | use crate::RaftTypeConfig; 3 | 4 | /// This helper trait extracts information from an `Option`. 5 | pub trait LogIdOptionExt 6 | where C: RaftTypeConfig 7 | { 8 | /// Returns the log index if it is not a `None`. 9 | fn index(&self) -> Option; 10 | 11 | /// Returns the next log index. 12 | /// 13 | /// If self is `None`, it returns 0. 14 | fn next_index(&self) -> u64; 15 | } 16 | 17 | impl LogIdOptionExt for Option 18 | where 19 | C: RaftTypeConfig, 20 | T: RaftLogId, 21 | { 22 | fn index(&self) -> Option { 23 | self.as_ref().map(|x| x.index()) 24 | } 25 | 26 | fn next_index(&self) -> u64 { 27 | match self { 28 | None => 0, 29 | Some(log_id) => log_id.index() + 1, 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /openraft/src/log_id/log_index_option_ext.rs: -------------------------------------------------------------------------------- 1 | /// This helper trait extracts information from an `Option`. 2 | /// 3 | /// In openraft, `LogIndex` is a `u64`. 4 | pub trait LogIndexOptionExt { 5 | /// Return the next log index. 6 | /// 7 | /// If self is `None`, it returns 0. 8 | fn next_index(&self) -> u64; 9 | 10 | /// Return the previous log index. 11 | /// 12 | /// If self is `None`, it panics. 13 | fn prev_index(&self) -> Self; 14 | 15 | // TODO: unused, remove it 16 | /// Performs an "add" operation. 17 | fn add(&self, v: u64) -> Self; 18 | } 19 | 20 | impl LogIndexOptionExt for Option { 21 | fn next_index(&self) -> u64 { 22 | match self { 23 | None => 0, 24 | Some(v) => v + 1, 25 | } 26 | } 27 | 28 | fn prev_index(&self) -> Self { 29 | match self { 30 | None => { 31 | panic!("None has no previous value"); 32 | } 33 | Some(v) => { 34 | if *v == 0 { 35 | None 36 | } else { 37 | Some(*v - 1) 38 | } 39 | } 40 | } 41 | } 42 | 43 | fn add(&self, v: u64) -> Self { 44 | Some(self.next_index() + v).prev_index() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /openraft/src/log_id/option_ref_log_id_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::log_id::ref_log_id::RefLogId; 2 | use crate::type_config::alias::LogIdOf; 3 | use crate::RaftTypeConfig; 4 | 5 | pub(crate) trait OptionRefLogIdExt 6 | where C: RaftTypeConfig 7 | { 8 | /// Creates a new owned [`LogId`] from the reference log ID. 9 | /// 10 | /// [`LogId`]: crate::log_id::LogId 11 | fn to_log_id(&self) -> Option>; 12 | } 13 | 14 | impl OptionRefLogIdExt for Option> 15 | where C: RaftTypeConfig 16 | { 17 | fn to_log_id(&self) -> Option> { 18 | self.as_ref().map(|r| r.into_log_id()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/log_id/raft_log_id_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::log_id::raft_log_id::RaftLogId; 2 | use crate::log_id::ref_log_id::RefLogId; 3 | use crate::type_config::alias::LogIdOf; 4 | use crate::RaftTypeConfig; 5 | 6 | pub(crate) trait RaftLogIdExt 7 | where 8 | C: RaftTypeConfig, 9 | Self: RaftLogId, 10 | { 11 | /// Creates a new owned [`LogId`] from this log ID implementation. 12 | /// 13 | /// [`LogId`]: crate::log_id::LogId 14 | fn to_log_id(&self) -> LogIdOf { 15 | self.to_ref().into_log_id() 16 | } 17 | 18 | /// Creates a reference view of this log ID implementation via a [`RefLogId`]. 19 | fn to_ref(&self) -> RefLogId<'_, C> { 20 | RefLogId { 21 | leader_id: self.committed_leader_id(), 22 | index: self.index(), 23 | } 24 | } 25 | } 26 | 27 | impl RaftLogIdExt for T 28 | where 29 | C: RaftTypeConfig, 30 | T: RaftLogId, 31 | { 32 | } 33 | -------------------------------------------------------------------------------- /openraft/src/membership/bench/is_quorum.rs: -------------------------------------------------------------------------------- 1 | extern crate test; 2 | 3 | use maplit::btreeset; 4 | use test::black_box; 5 | use test::Bencher; 6 | 7 | use crate::engine::testing::UTConfig; 8 | use crate::quorum::QuorumSet; 9 | use crate::EffectiveMembership; 10 | use crate::Membership; 11 | 12 | #[bench] 13 | fn m12345_ids_slice(b: &mut Bencher) { 14 | let m = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5}], None); 15 | let m = EffectiveMembership::new(None, m); 16 | let x = [1, 2, 3, 6, 7]; 17 | 18 | b.iter(|| m.is_quorum(black_box(x.iter()))) 19 | } 20 | 21 | #[bench] 22 | fn m12345_ids_btreeset(b: &mut Bencher) { 23 | let m = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5}], None); 24 | let m = EffectiveMembership::new(None, m); 25 | let x = btreeset! {1, 2, 3, 6, 7}; 26 | 27 | b.iter(|| m.is_quorum(black_box(x.iter()))) 28 | } 29 | 30 | #[bench] 31 | fn m12345_678_ids_slice(b: &mut Bencher) { 32 | let m = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5}], None); 33 | let m = EffectiveMembership::new(None, m); 34 | let x = [1, 2, 3, 6, 7]; 35 | 36 | b.iter(|| m.is_quorum(black_box(x.iter()))) 37 | } 38 | 39 | #[bench] 40 | fn m12345_678_ids_btreeset(b: &mut Bencher) { 41 | let m = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5}], None); 42 | let m = EffectiveMembership::new(None, m); 43 | let x = btreeset! {1, 2, 3, 6, 7}; 44 | 45 | b.iter(|| m.is_quorum(black_box(x.iter()))) 46 | } 47 | -------------------------------------------------------------------------------- /openraft/src/membership/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod is_quorum; 2 | -------------------------------------------------------------------------------- /openraft/src/membership/effective_membership_test.rs: -------------------------------------------------------------------------------- 1 | use maplit::btreeset; 2 | 3 | use crate::engine::testing::UTConfig; 4 | use crate::quorum::QuorumSet; 5 | use crate::EffectiveMembership; 6 | use crate::Membership; 7 | 8 | #[test] 9 | fn test_effective_membership_majority() -> anyhow::Result<()> { 10 | { 11 | let m12345 = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5 }], []); 12 | let m = EffectiveMembership::::new(None, m12345); 13 | 14 | assert!(!m.is_quorum([0].iter())); 15 | assert!(!m.is_quorum([0, 1, 2].iter())); 16 | assert!(!m.is_quorum([6, 7, 8].iter())); 17 | assert!(m.is_quorum([1, 2, 3].iter())); 18 | assert!(m.is_quorum([3, 4, 5].iter())); 19 | assert!(m.is_quorum([1, 3, 4, 5].iter())); 20 | } 21 | 22 | { 23 | let m12345_678 = Membership::::new_with_defaults(vec![btreeset! {1,2,3,4,5 }, btreeset! {6,7,8}], []); 24 | let m = EffectiveMembership::::new(None, m12345_678); 25 | 26 | assert!(!m.is_quorum([0].iter())); 27 | assert!(!m.is_quorum([0, 1, 2].iter())); 28 | assert!(!m.is_quorum([6, 7, 8].iter())); 29 | assert!(!m.is_quorum([1, 2, 3].iter())); 30 | assert!(m.is_quorum([1, 2, 3, 6, 7].iter())); 31 | assert!(m.is_quorum([1, 2, 3, 4, 7, 8].iter())); 32 | } 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /openraft/src/membership/mod.rs: -------------------------------------------------------------------------------- 1 | mod effective_membership; 2 | mod into_nodes; 3 | #[allow(clippy::module_inception)] 4 | mod membership; 5 | mod stored_membership; 6 | 7 | #[cfg(feature = "bench")] 8 | #[cfg(test)] 9 | mod bench; 10 | 11 | #[cfg(test)] 12 | mod effective_membership_test; 13 | #[cfg(test)] 14 | mod membership_test; 15 | 16 | pub use effective_membership::EffectiveMembership; 17 | pub use into_nodes::IntoNodes; 18 | pub use membership::Membership; 19 | pub use stored_membership::StoredMembership; 20 | -------------------------------------------------------------------------------- /openraft/src/metrics/metric_display.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Formatter; 3 | 4 | use crate::display_ext::DisplayOption; 5 | use crate::metrics::Metric; 6 | use crate::RaftTypeConfig; 7 | 8 | /// Display the value of a metric. 9 | pub(crate) struct MetricDisplay<'a, C> 10 | where C: RaftTypeConfig 11 | { 12 | pub(crate) metric: &'a Metric, 13 | } 14 | 15 | impl fmt::Display for MetricDisplay<'_, C> 16 | where C: RaftTypeConfig 17 | { 18 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 19 | match self.metric { 20 | Metric::Term(v) => write!(f, "{}", v), 21 | Metric::Vote(v) => write!(f, "{}", v), 22 | Metric::LastLogIndex(v) => write!(f, "{}", DisplayOption(v)), 23 | Metric::Applied(v) => write!(f, "{}", DisplayOption(v)), 24 | Metric::AppliedIndex(v) => write!(f, "{}", DisplayOption(v)), 25 | Metric::Snapshot(v) => write!(f, "{}", DisplayOption(v)), 26 | Metric::Purged(v) => write!(f, "{}", DisplayOption(v)), 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /openraft/src/metrics/wait_condition.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::metrics::metric_display::MetricDisplay; 4 | use crate::metrics::Metric; 5 | use crate::RaftTypeConfig; 6 | 7 | /// A condition that the application wait for. 8 | #[derive(Debug)] 9 | pub(crate) enum Condition 10 | where C: RaftTypeConfig 11 | { 12 | GE(Metric), 13 | EQ(Metric), 14 | } 15 | 16 | impl Condition 17 | where C: RaftTypeConfig 18 | { 19 | /// Build a new condition which the application will await to meet or exceed. 20 | pub(crate) fn ge(v: Metric) -> Self { 21 | Self::GE(v) 22 | } 23 | 24 | /// Build a new condition which the application will await to meet. 25 | pub(crate) fn eq(v: Metric) -> Self { 26 | Self::EQ(v) 27 | } 28 | 29 | pub(crate) fn name(&self) -> &'static str { 30 | match self { 31 | Condition::GE(v) => v.name(), 32 | Condition::EQ(v) => v.name(), 33 | } 34 | } 35 | 36 | pub(crate) fn op(&self) -> &'static str { 37 | match self { 38 | Condition::GE(_) => ">=", 39 | Condition::EQ(_) => "==", 40 | } 41 | } 42 | 43 | pub(crate) fn value(&self) -> MetricDisplay<'_, C> { 44 | match self { 45 | Condition::GE(v) => v.value(), 46 | Condition::EQ(v) => v.value(), 47 | } 48 | } 49 | } 50 | 51 | impl fmt::Display for Condition 52 | where C: RaftTypeConfig 53 | { 54 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 55 | write!(f, "{}{}{}", self.name(), self.op(), self.value()) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /openraft/src/network/backoff.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::OptionalSend; 4 | 5 | /// A backoff instance that is an infinite iterator of durations to sleep before next retry, when a 6 | /// [`Unreachable`](`crate::error::Unreachable`) occurs. 7 | pub struct Backoff { 8 | #[cfg(not(feature = "singlethreaded"))] 9 | inner: Box + Send + 'static>, 10 | #[cfg(feature = "singlethreaded")] 11 | inner: Box + 'static>, 12 | } 13 | 14 | impl Backoff { 15 | pub fn new(iter: impl Iterator + OptionalSend + 'static) -> Self { 16 | Self { inner: Box::new(iter) } 17 | } 18 | } 19 | 20 | impl Iterator for Backoff { 21 | type Item = Duration; 22 | 23 | fn next(&mut self) -> Option { 24 | self.inner.next() 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /openraft/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | //! The Raft network interface. 2 | 3 | mod backoff; 4 | mod rpc_option; 5 | mod rpc_type; 6 | 7 | pub mod v1; 8 | pub mod v2; 9 | 10 | pub mod snapshot_transport; 11 | 12 | pub use backoff::Backoff; 13 | pub use rpc_option::RPCOption; 14 | pub use rpc_type::RPCTypes; 15 | pub use v1::RaftNetwork; 16 | pub use v1::RaftNetworkFactory; 17 | -------------------------------------------------------------------------------- /openraft/src/network/rpc_type.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | #[derive(Debug, Clone, Copy)] 4 | #[derive(PartialEq, Eq)] 5 | #[derive(Hash)] 6 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 7 | pub enum RPCTypes { 8 | Vote, 9 | AppendEntries, 10 | InstallSnapshot, 11 | TransferLeader, 12 | } 13 | 14 | impl fmt::Display for RPCTypes { 15 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 16 | write!(f, "{:?}", self) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/network/v1/mod.rs: -------------------------------------------------------------------------------- 1 | mod factory; 2 | mod network; 3 | 4 | pub use factory::RaftNetworkFactory; 5 | pub use network::RaftNetwork; 6 | -------------------------------------------------------------------------------- /openraft/src/network/v2/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(all(feature = "tokio-rt", feature = "adapt-network-v1"))] 2 | mod adapt_v1; 3 | mod network; 4 | 5 | pub use network::RaftNetworkV2; 6 | -------------------------------------------------------------------------------- /openraft/src/progress/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod vec_progress_update; 2 | -------------------------------------------------------------------------------- /openraft/src/progress/bench/vec_progress_update.rs: -------------------------------------------------------------------------------- 1 | extern crate test; 2 | 3 | use test::black_box; 4 | use test::Bencher; 5 | 6 | use crate::progress::Progress; 7 | use crate::progress::VecProgress; 8 | use crate::quorum::Joint; 9 | 10 | #[bench] 11 | fn progress_update_01234_567(b: &mut Bencher) { 12 | let membership: Vec> = vec![vec![0, 1, 2, 3, 4], vec![5, 6, 7]]; 13 | let quorum_set = Joint::from(membership); 14 | let mut progress = VecProgress::::new(quorum_set, 0..=7, || 0); 15 | 16 | let mut id = 0u64; 17 | let mut values = [0, 1, 2, 3, 4, 5, 6, 7]; 18 | b.iter(|| { 19 | id = (id + 1) & 7; 20 | values[id as usize] += 1; 21 | let v = values[id as usize]; 22 | 23 | let _ = progress.update(&black_box(id), black_box(v)); 24 | }); 25 | 26 | // It shows that is_quorum() is called at a rate of about 1/4 of update() 27 | // `Stat { update_count: 42997501, move_count: 10749381, is_quorum_count: 10749399 }` 28 | // println!("progress stat: {:?}", progress.stat()); 29 | } 30 | -------------------------------------------------------------------------------- /openraft/src/proposer/leader_state.rs: -------------------------------------------------------------------------------- 1 | use crate::proposer::Candidate; 2 | use crate::proposer::Leader; 3 | use crate::quorum::Joint; 4 | use crate::type_config::alias::NodeIdOf; 5 | 6 | /// The quorum set type used by `Leader`. 7 | pub(crate) type LeaderQuorumSet = Joint, Vec>, Vec>>>; 8 | 9 | pub(crate) type LeaderState = Option>>>; 10 | pub(crate) type CandidateState = Option>>; 11 | -------------------------------------------------------------------------------- /openraft/src/proposer/mod.rs: -------------------------------------------------------------------------------- 1 | //! A proposer includes the Candidate(phase-1) state and Leader(phase-2) state. 2 | 3 | pub(crate) mod candidate; 4 | pub(crate) mod leader; 5 | pub(crate) mod leader_state; 6 | 7 | pub(crate) use candidate::Candidate; 8 | pub(crate) use leader::Leader; 9 | pub(crate) use leader_state::CandidateState; 10 | pub(crate) use leader_state::LeaderQuorumSet; 11 | pub(crate) use leader_state::LeaderState; 12 | -------------------------------------------------------------------------------- /openraft/src/quorum/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod is_quorum; 2 | -------------------------------------------------------------------------------- /openraft/src/quorum/coherent.rs: -------------------------------------------------------------------------------- 1 | use crate::quorum::QuorumSet; 2 | 3 | /// **Coherent** quorum set A and B is defined as: `∀ qᵢ ∈ A, ∀ qⱼ ∈ B: qᵢ ∩ qⱼ != ø`, i.e., `A ~ 4 | /// B`. 5 | /// A distributed consensus protocol such as openraft is only allowed to switch membership 6 | /// between two **coherent** quorum sets. Being coherent is one of the two restrictions. The other 7 | /// restriction is to disable other smaller candidate to elect. 8 | pub(crate) trait Coherent 9 | where 10 | ID: PartialOrd + Ord + 'static, 11 | Self: QuorumSet, 12 | Other: QuorumSet, 13 | { 14 | /// Returns `true` if this QuorumSet is coherent with the other quorum set. 15 | fn is_coherent_with(&self, other: &Other) -> bool; 16 | } 17 | 18 | pub(crate) trait FindCoherent 19 | where 20 | ID: PartialOrd + Ord + 'static, 21 | Self: QuorumSet, 22 | Other: QuorumSet, 23 | { 24 | /// Build a QuorumSet `X` so that `self` is coherent with `X` and `X` is coherent with `other`, 25 | /// i.e., `self ~ X ~ other`. 26 | /// Then `X` is the intermediate QuorumSet when changing membership from `self` to `other`. 27 | /// 28 | /// E.g.(`cᵢcⱼ` is a joint of `cᵢ` and `cⱼ`): 29 | /// - `c₁.find_coherent(c₁)` returns `c₁` 30 | /// - `c₁.find_coherent(c₂)` returns `c₁c₂` 31 | /// - `c₁c₂.find_coherent(c₂)` returns `c₂` 32 | /// - `c₁c₂.find_coherent(c₁)` returns `c₁` 33 | /// - `c₁c2.find_coherent(c₃)` returns `c₂c₃` 34 | fn find_coherent(&self, other: Other) -> Self; 35 | } 36 | -------------------------------------------------------------------------------- /openraft/src/quorum/joint_impl.rs: -------------------------------------------------------------------------------- 1 | use crate::quorum::AsJoint; 2 | use crate::quorum::Joint; 3 | use crate::quorum::QuorumSet; 4 | 5 | /// Use a vec of some impl of `QuorumSet` as a joint quorum set. 6 | impl<'d, ID, QS> AsJoint<'d, ID, QS, &'d [QS]> for Vec 7 | where 8 | ID: 'static, 9 | QS: QuorumSet, 10 | { 11 | fn as_joint(&'d self) -> Joint 12 | where &'d [QS]: 'd { 13 | Joint::new(self) 14 | } 15 | } 16 | 17 | impl From> for Joint> 18 | where 19 | ID: 'static, 20 | QS: QuorumSet, 21 | { 22 | fn from(v: Vec) -> Self { 23 | Joint::new(v) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /openraft/src/quorum/mod.rs: -------------------------------------------------------------------------------- 1 | //! A quorum is a set of nodes a vote request or append-entries request has to contact to. 2 | //! The most common quorum is **majority**. 3 | //! A quorum set is a collection of quorums, e.g. the quorum set of majority of `{a,b,c}` is `{a,b}, 4 | //! {b,c}, {a,c}`. 5 | 6 | mod coherent; 7 | mod coherent_impl; 8 | mod joint; 9 | mod joint_impl; 10 | mod quorum_set; 11 | mod quorum_set_impl; 12 | 13 | #[cfg(feature = "bench")] 14 | #[cfg(test)] 15 | mod bench; 16 | 17 | #[cfg(test)] 18 | mod coherent_test; 19 | #[cfg(test)] 20 | mod quorum_set_test; 21 | 22 | pub(crate) use coherent::Coherent; 23 | pub(crate) use coherent::FindCoherent; 24 | pub(crate) use joint::AsJoint; 25 | pub(crate) use joint::Joint; 26 | pub(crate) use quorum_set::QuorumSet; 27 | -------------------------------------------------------------------------------- /openraft/src/quorum/quorum_set.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | /// A set of quorums is a collection of quorum. 4 | /// 5 | /// A quorum is a collection of nodes that a read or write operation in distributed system has to 6 | /// contact to. See: 7 | pub(crate) trait QuorumSet { 8 | type Iter: Iterator; 9 | 10 | /// Check if a series of ID constitute a quorum that is defined by this quorum set. 11 | fn is_quorum<'a, I: Iterator + Clone>(&self, ids: I) -> bool; 12 | 13 | /// Returns all ids in this QuorumSet 14 | fn ids(&self) -> Self::Iter; 15 | } 16 | 17 | impl> QuorumSet for Arc { 18 | type Iter = T::Iter; 19 | 20 | fn is_quorum<'a, I: Iterator + Clone>(&self, ids: I) -> bool { 21 | self.as_ref().is_quorum(ids) 22 | } 23 | 24 | fn ids(&self) -> Self::Iter { 25 | self.as_ref().ids() 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /openraft/src/raft/api/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod app; 2 | pub(crate) mod management; 3 | pub(crate) mod protocol; 4 | -------------------------------------------------------------------------------- /openraft/src/raft/core_state.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Fatal; 2 | use crate::error::Infallible; 3 | use crate::type_config::alias::JoinHandleOf; 4 | use crate::type_config::alias::WatchReceiverOf; 5 | use crate::RaftTypeConfig; 6 | 7 | /// The running state of RaftCore 8 | pub(in crate::raft) enum CoreState 9 | where C: RaftTypeConfig 10 | { 11 | /// The RaftCore task is still running. 12 | Running(JoinHandleOf>>), 13 | 14 | /// The RaftCore task is waiting for a signal to finish joining. 15 | Joining(WatchReceiverOf), 16 | 17 | /// The RaftCore task has finished. The return value of the task is stored. 18 | Done(Result>), 19 | } 20 | 21 | impl CoreState 22 | where C: RaftTypeConfig 23 | { 24 | /// Returns `true` if the RaftCore task is still running. 25 | pub(in crate::raft) fn is_running(&self) -> bool { 26 | matches!(self, CoreState::Running(_)) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /openraft/src/raft/declare_raft_types_test.rs: -------------------------------------------------------------------------------- 1 | //! Test the `declare_raft_types` macro with default values 2 | 3 | #![allow(dead_code)] 4 | 5 | use std::io::Cursor; 6 | 7 | use crate::declare_raft_types; 8 | use crate::impls::TokioRuntime; 9 | 10 | declare_raft_types!( 11 | All: 12 | NodeId = u64, 13 | Node = (), 14 | 15 | /// This is AppData 16 | D = (), 17 | #[allow(dead_code)] 18 | #[allow(dead_code)] 19 | R = (), 20 | Term = u64, 21 | LeaderId = crate::impls::leader_id_std::LeaderId, 22 | Entry = crate::Entry, 23 | Vote = crate::impls::Vote, 24 | SnapshotData = Cursor>, 25 | AsyncRuntime = TokioRuntime, 26 | Responder = crate::impls::OneshotResponder, 27 | ); 28 | 29 | declare_raft_types!( 30 | WithoutD: 31 | R = (), 32 | NodeId = u64, 33 | Node = (), 34 | Entry = crate::Entry, 35 | SnapshotData = Cursor>, 36 | AsyncRuntime = TokioRuntime, 37 | ); 38 | 39 | declare_raft_types!( 40 | WithoutR: 41 | D = (), 42 | NodeId = u64, 43 | Node = (), 44 | Entry = crate::Entry, 45 | SnapshotData = Cursor>, 46 | AsyncRuntime = TokioRuntime, 47 | ); 48 | 49 | declare_raft_types!(EmptyWithColon:); 50 | 51 | declare_raft_types!(Empty); 52 | -------------------------------------------------------------------------------- /openraft/src/raft/linearizable_read/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines the linearizable read protocol. 2 | 3 | mod linearize_state; 4 | mod linearizer; 5 | 6 | pub use linearize_state::LinearizeState; 7 | pub use linearizer::Linearizer; 8 | -------------------------------------------------------------------------------- /openraft/src/raft/message/mod.rs: -------------------------------------------------------------------------------- 1 | //! Raft protocol messages and types. 2 | //! 3 | //! Request and response types for an application to talk to the Raft, 4 | //! and are also used by network layer to talk to other Raft nodes. 5 | 6 | mod append_entries; 7 | mod install_snapshot; 8 | mod transfer_leader; 9 | mod vote; 10 | 11 | mod client_write; 12 | 13 | pub use append_entries::AppendEntriesRequest; 14 | pub use append_entries::AppendEntriesResponse; 15 | pub use client_write::ClientWriteResponse; 16 | pub use client_write::ClientWriteResult; 17 | pub use install_snapshot::InstallSnapshotRequest; 18 | pub use install_snapshot::InstallSnapshotResponse; 19 | pub use install_snapshot::SnapshotResponse; 20 | pub use transfer_leader::TransferLeaderRequest; 21 | pub use vote::VoteRequest; 22 | pub use vote::VoteResponse; 23 | -------------------------------------------------------------------------------- /openraft/src/raft/responder/either.rs: -------------------------------------------------------------------------------- 1 | use crate::impls::OneshotResponder; 2 | use crate::raft::responder::Responder; 3 | use crate::raft::ClientWriteResult; 4 | use crate::RaftTypeConfig; 5 | 6 | /// Either an oneshot responder or a user-defined responder. 7 | /// 8 | /// It is used in RaftCore to enqueue responder to client. 9 | pub(crate) enum OneshotOrUserDefined 10 | where C: RaftTypeConfig 11 | { 12 | Oneshot(OneshotResponder), 13 | UserDefined(C::Responder), 14 | } 15 | 16 | impl Responder for OneshotOrUserDefined 17 | where C: RaftTypeConfig 18 | { 19 | fn send(self, res: ClientWriteResult) { 20 | match self { 21 | Self::Oneshot(responder) => responder.send(res), 22 | Self::UserDefined(responder) => responder.send(res), 23 | } 24 | } 25 | 26 | type Receiver = (); 27 | 28 | fn from_app_data(_app_data: ::D) -> (::D, Self, Self::Receiver) 29 | where Self: Sized { 30 | unimplemented!("OneshotOrUserDefined is just a wrapper and does not support building from app_data") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /openraft/src/raft_state/tests/is_initialized_test.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::engine::testing::log_id; 4 | use crate::engine::testing::UTConfig; 5 | use crate::engine::LogIdList; 6 | use crate::type_config::TypeConfigExt; 7 | use crate::utime::Leased; 8 | use crate::RaftState; 9 | use crate::Vote; 10 | 11 | #[test] 12 | fn test_is_initialized() { 13 | // empty 14 | { 15 | let rs = RaftState:: { ..Default::default() }; 16 | 17 | assert_eq!(false, rs.is_initialized()); 18 | } 19 | 20 | // Vote is set but is default 21 | { 22 | let rs = RaftState:: { 23 | vote: Leased::new(UTConfig::<()>::now(), Duration::from_millis(500), Vote::default()), 24 | ..Default::default() 25 | }; 26 | 27 | assert_eq!(false, rs.is_initialized()); 28 | } 29 | 30 | // Vote is non-default value 31 | { 32 | let rs = RaftState:: { 33 | vote: Leased::new(UTConfig::<()>::now(), Duration::from_millis(500), Vote::new(1, 2)), 34 | ..Default::default() 35 | }; 36 | 37 | assert_eq!(true, rs.is_initialized()); 38 | } 39 | 40 | // Logs are non-empty 41 | { 42 | let rs = RaftState:: { 43 | log_ids: LogIdList::new([log_id(0, 0, 0)]), 44 | ..Default::default() 45 | }; 46 | 47 | assert_eq!(true, rs.is_initialized()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /openraft/src/raft_state/tests/validate_test.rs: -------------------------------------------------------------------------------- 1 | use validit::Validate; 2 | 3 | use crate::engine::testing::UTConfig; 4 | use crate::engine::LogIdList; 5 | use crate::storage::SnapshotMeta; 6 | use crate::type_config::alias::LogIdOf; 7 | use crate::RaftState; 8 | 9 | fn log_id(term: u64, index: u64) -> LogIdOf { 10 | crate::engine::testing::log_id(term, 0, index) 11 | } 12 | 13 | #[test] 14 | fn test_raft_state_validate_snapshot_is_none() -> anyhow::Result<()> { 15 | // Some app does not persist snapshot, when restarted, purged is not None but snapshot_last_log_id 16 | // is None. This is a valid state and should not emit error. 17 | 18 | let mut rs = RaftState:: { 19 | log_ids: LogIdList::new(vec![log_id(1, 1), log_id(3, 4)]), 20 | purged_next: 2, 21 | purge_upto: Some(log_id(1, 1)), 22 | snapshot_meta: SnapshotMeta::default(), 23 | ..Default::default() 24 | }; 25 | 26 | rs.apply_progress_mut().accept(log_id(1, 1)); 27 | 28 | assert!(rs.validate().is_ok()); 29 | 30 | Ok(()) 31 | } 32 | -------------------------------------------------------------------------------- /openraft/src/raft_state/vote_state_reader.rs: -------------------------------------------------------------------------------- 1 | use crate::type_config::alias::VoteOf; 2 | use crate::RaftTypeConfig; 3 | 4 | // TODO: remove it? 5 | /// APIs to get vote. 6 | #[allow(dead_code)] 7 | pub(crate) trait VoteStateReader 8 | where C: RaftTypeConfig 9 | { 10 | /// Get a reference to the current vote. 11 | fn vote_ref(&self) -> &VoteOf; 12 | } 13 | -------------------------------------------------------------------------------- /openraft/src/raft_types.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | use std::fmt::Formatter; 3 | 4 | /// Id of a snapshot stream. 5 | /// 6 | /// Everytime a snapshot is created, it is assigned with a globally unique id. 7 | /// Such an id is used by followers to distinguish different install-snapshot streams. 8 | pub type SnapshotId = String; 9 | 10 | /// The identity of a segment of a snapshot. 11 | #[derive(Debug, Default, Clone, PartialOrd, PartialEq, Eq)] 12 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 13 | pub struct SnapshotSegmentId { 14 | /// The unique identifier of the snapshot stream. 15 | pub id: SnapshotId, 16 | 17 | /// The offset of this segment in the entire snapshot data. 18 | pub offset: u64, 19 | } 20 | 21 | impl From<(D, u64)> for SnapshotSegmentId { 22 | fn from(v: (D, u64)) -> Self { 23 | SnapshotSegmentId { 24 | id: v.0.to_string(), 25 | offset: v.1, 26 | } 27 | } 28 | } 29 | 30 | impl Display for SnapshotSegmentId { 31 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 32 | write!(f, "{}+{}", self.id, self.offset) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /openraft/src/replication/hint.rs: -------------------------------------------------------------------------------- 1 | //! Defines config hint for replication RPC 2 | 3 | /// Temporary config hint for replication 4 | #[derive(Clone, Debug, Default)] 5 | pub(crate) struct ReplicationHint { 6 | n: u64, 7 | 8 | /// How many times this hint can be used. 9 | ttl: u64, 10 | } 11 | 12 | impl ReplicationHint { 13 | /// Create a new `ReplicationHint` 14 | pub(crate) fn new(n: u64, ttl: u64) -> Self { 15 | Self { n, ttl } 16 | } 17 | 18 | pub(crate) fn get(&mut self) -> Option { 19 | if self.ttl > 0 { 20 | self.ttl -= 1; 21 | Some(self.n) 22 | } else { 23 | None 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /openraft/src/storage/log_reader_ext.rs: -------------------------------------------------------------------------------- 1 | use anyerror::AnyError; 2 | use openraft_macros::add_async_trait; 3 | 4 | use crate::entry::RaftEntry; 5 | use crate::type_config::alias::LogIdOf; 6 | use crate::RaftLogReader; 7 | use crate::RaftTypeConfig; 8 | use crate::StorageError; 9 | 10 | #[add_async_trait] 11 | pub trait RaftLogReaderExt: RaftLogReader 12 | where C: RaftTypeConfig 13 | { 14 | /// Try to get an log entry. 15 | /// 16 | /// It does not return an error if the log entry at `log_index` is not found. 17 | async fn try_get_log_entry(&mut self, log_index: u64) -> Result, StorageError> { 18 | let mut res = self.try_get_log_entries(log_index..(log_index + 1)).await?; 19 | Ok(res.pop()) 20 | } 21 | 22 | /// Get the log id of the entry at `index`. 23 | async fn get_log_id(&mut self, log_index: u64) -> Result, StorageError> { 24 | let entries = self.try_get_log_entries(log_index..=log_index).await?; 25 | 26 | if entries.is_empty() { 27 | return Err(StorageError::read_log_at_index( 28 | log_index, 29 | AnyError::error("log entry not found"), 30 | )); 31 | } 32 | 33 | Ok(entries[0].log_id()) 34 | } 35 | } 36 | 37 | impl RaftLogReaderExt for LR 38 | where 39 | C: RaftTypeConfig, 40 | LR: RaftLogReader + ?Sized, 41 | { 42 | } 43 | -------------------------------------------------------------------------------- /openraft/src/storage/log_state.rs: -------------------------------------------------------------------------------- 1 | use crate::type_config::alias::LogIdOf; 2 | use crate::RaftTypeConfig; 3 | 4 | /// The state about logs. 5 | /// 6 | /// Invariance: last_purged_log_id <= last_applied <= last_log_id 7 | #[derive(Clone, Debug, Default, PartialEq, Eq)] 8 | pub struct LogState { 9 | /// The greatest log id that has been purged after being applied to state machine. 10 | pub last_purged_log_id: Option>, 11 | 12 | /// The log id of the last present entry if there are any entries. 13 | /// Otherwise the same value as `last_purged_log_id`. 14 | pub last_log_id: Option>, 15 | } 16 | -------------------------------------------------------------------------------- /openraft/src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | //! The Raft storage interface and data types. 2 | 3 | mod callback; 4 | mod helper; 5 | mod log_reader_ext; 6 | mod log_state; 7 | mod snapshot; 8 | mod snapshot_meta; 9 | mod snapshot_signature; 10 | mod v2; 11 | 12 | pub use self::callback::IOFlushed; 13 | pub use self::callback::LogApplied; 14 | #[allow(deprecated)] 15 | pub use self::callback::LogFlushed; 16 | pub use self::helper::StorageHelper; 17 | pub use self::log_reader_ext::RaftLogReaderExt; 18 | pub use self::log_state::LogState; 19 | pub use self::snapshot::Snapshot; 20 | pub use self::snapshot_meta::SnapshotMeta; 21 | pub use self::snapshot_signature::SnapshotSignature; 22 | pub use self::v2::RaftLogReader; 23 | pub use self::v2::RaftLogStorage; 24 | pub use self::v2::RaftLogStorageExt; 25 | pub use self::v2::RaftSnapshotBuilder; 26 | pub use self::v2::RaftStateMachine; 27 | -------------------------------------------------------------------------------- /openraft/src/storage/snapshot.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use openraft_macros::since; 4 | 5 | use crate::storage::SnapshotMeta; 6 | use crate::RaftTypeConfig; 7 | 8 | /// The data associated with the current snapshot. 9 | #[since(version = "0.10.0", change = "SnapshotData without Box")] 10 | #[derive(Debug, Clone)] 11 | pub struct Snapshot 12 | where C: RaftTypeConfig 13 | { 14 | /// metadata of a snapshot 15 | pub meta: SnapshotMeta, 16 | 17 | /// A read handle to the associated snapshot. 18 | pub snapshot: C::SnapshotData, 19 | } 20 | 21 | impl Snapshot 22 | where C: RaftTypeConfig 23 | { 24 | #[allow(dead_code)] 25 | pub(crate) fn new(meta: SnapshotMeta, snapshot: C::SnapshotData) -> Self { 26 | Self { meta, snapshot } 27 | } 28 | } 29 | 30 | impl fmt::Display for Snapshot 31 | where C: RaftTypeConfig 32 | { 33 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 34 | write!(f, "Snapshot{{meta: {}}}", self.meta) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /openraft/src/storage/v2/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines [`RaftLogStorage`] and [`RaftStateMachine`] trait. 2 | //! 3 | //! [`RaftLogStorage`] is responsible for storing logs, 4 | //! and [`RaftStateMachine`] is responsible for storing state machine and snapshot. 5 | 6 | mod raft_log_reader; 7 | mod raft_log_storage; 8 | mod raft_log_storage_ext; 9 | mod raft_snapshot_builder; 10 | mod raft_state_machine; 11 | 12 | pub use self::raft_log_reader::RaftLogReader; 13 | pub use self::raft_log_storage::RaftLogStorage; 14 | pub use self::raft_log_storage_ext::RaftLogStorageExt; 15 | pub use self::raft_snapshot_builder::RaftSnapshotBuilder; 16 | pub use self::raft_state_machine::RaftStateMachine; 17 | -------------------------------------------------------------------------------- /openraft/src/testing/common.rs: -------------------------------------------------------------------------------- 1 | //! Testing utilities used by all kinds of tests. 2 | 3 | use std::collections::BTreeSet; 4 | 5 | use crate::entry::RaftEntry; 6 | use crate::type_config::alias::LogIdOf; 7 | use crate::vote::RaftLeaderIdExt; 8 | use crate::RaftTypeConfig; 9 | 10 | /// Builds a log id, for testing purposes. 11 | pub fn log_id(term: u64, node_id: C::NodeId, index: u64) -> LogIdOf 12 | where 13 | C: RaftTypeConfig, 14 | C::Term: From, 15 | { 16 | LogIdOf::::new(C::LeaderId::new_committed(term.into(), node_id), index) 17 | } 18 | 19 | /// Create a blank log entry for test. 20 | pub fn blank_ent(term: u64, node_id: C::NodeId, index: u64) -> crate::Entry 21 | where 22 | C: RaftTypeConfig, 23 | C::Term: From, 24 | { 25 | crate::Entry::::new_blank(log_id::(term, node_id, index)) 26 | } 27 | 28 | /// Create a membership log entry without learner config for test. 29 | pub fn membership_ent(term: u64, node_id: C::NodeId, index: u64, config: Vec>) -> crate::Entry 30 | where 31 | C: RaftTypeConfig, 32 | C::Term: From, 33 | C::Node: Default, 34 | { 35 | crate::Entry::new_membership( 36 | log_id::(term, node_id, index), 37 | crate::Membership::new_with_defaults(config, []), 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /openraft/src/testing/log/mod.rs: -------------------------------------------------------------------------------- 1 | //! Suite for testing implementations of [`RaftLogStorage`] and [`RaftStateMachine`]. 2 | //! 3 | //! [`RaftLogStorage`]: crate::storage::RaftLogStorage 4 | //! [`RaftStateMachine`]: crate::storage::RaftStateMachine 5 | 6 | mod store_builder; 7 | mod suite; 8 | 9 | pub use store_builder::StoreBuilder; 10 | pub use suite::Suite; 11 | -------------------------------------------------------------------------------- /openraft/src/testing/log/store_builder.rs: -------------------------------------------------------------------------------- 1 | use openraft_macros::add_async_trait; 2 | 3 | use crate::storage::RaftLogStorage; 4 | use crate::storage::RaftStateMachine; 5 | use crate::RaftTypeConfig; 6 | use crate::StorageError; 7 | 8 | /// The trait to build a [`RaftLogStorage`] and [`RaftStateMachine`] implementation. 9 | /// 10 | /// The generic parameter `C` is type config for a `RaftLogStorage` and `RaftStateMachine` 11 | /// implementation, 12 | /// `LS` is the type that implements `RaftLogStorage`, 13 | /// `SM` is the type that implements `RaftStateMachine`, 14 | /// and `G` is a guard type that cleanup resource when being dropped. 15 | /// 16 | /// By default `G` is a trivial guard `()`. To test a store that is backed by a folder on disk, `G` 17 | /// could be the dropper of the temp-dir that stores data. 18 | #[add_async_trait] 19 | pub trait StoreBuilder: Send + Sync 20 | where 21 | C: RaftTypeConfig, 22 | LS: RaftLogStorage, 23 | SM: RaftStateMachine, 24 | { 25 | /// Build a [`RaftLogStorage`] and [`RaftStateMachine`] implementation 26 | async fn build(&self) -> Result<(G, LS, SM), StorageError>; 27 | } 28 | -------------------------------------------------------------------------------- /openraft/src/testing/mod.rs: -------------------------------------------------------------------------------- 1 | //! Testing utilities for OpenRaft. 2 | 3 | pub mod common; 4 | pub mod log; 5 | pub mod runtime; 6 | 7 | pub use common::*; 8 | -------------------------------------------------------------------------------- /openraft/src/try_as_ref.rs: -------------------------------------------------------------------------------- 1 | /// Similar to [`AsRef`], it does a cheap reference to reference conversion, but with the ability 2 | /// to return `None` if it is unable to perform the conversion to `&T`. 3 | pub trait TryAsRef { 4 | /// Try to convert this type into a shared reference of `T`. 5 | fn try_as_ref(&self) -> Option<&T>; 6 | } 7 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mpsc_unbounded/send_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by the `Sender`. 4 | #[derive(PartialEq, Eq, Clone, Copy)] 5 | pub struct SendError(pub T); 6 | 7 | impl fmt::Debug for SendError { 8 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 9 | f.debug_struct("SendError").finish_non_exhaustive() 10 | } 11 | } 12 | 13 | impl fmt::Display for SendError { 14 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 15 | write!(fmt, "channel closed") 16 | } 17 | } 18 | 19 | impl std::error::Error for SendError {} 20 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mpsc_unbounded/try_recv_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by `try_recv`. 4 | #[derive(PartialEq, Eq, Clone, Copy, Debug)] 5 | pub enum TryRecvError { 6 | /// This **channel** is currently empty, but the **Sender**(s) have not yet 7 | /// disconnected, so data may yet become available. 8 | Empty, 9 | /// The **channel**'s sending half has become disconnected, and there will 10 | /// never be any more data received on it. 11 | Disconnected, 12 | } 13 | 14 | impl fmt::Display for TryRecvError { 15 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 16 | match *self { 17 | TryRecvError::Empty => "receiving on an empty channel".fmt(fmt), 18 | TryRecvError::Disconnected => "receiving on a closed channel".fmt(fmt), 19 | } 20 | } 21 | } 22 | 23 | impl std::error::Error for TryRecvError {} 24 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mutex.rs: -------------------------------------------------------------------------------- 1 | use std::future::Future; 2 | use std::ops::DerefMut; 3 | 4 | use crate::OptionalSend; 5 | use crate::OptionalSync; 6 | 7 | /// Represents an implementation of an asynchronous Mutex. 8 | pub trait Mutex: OptionalSend + OptionalSync { 9 | /// Handle to an acquired lock, should release it when dropped. 10 | type Guard<'a>: DerefMut + OptionalSend 11 | where Self: 'a; 12 | 13 | /// Creates a new lock. 14 | fn new(value: T) -> Self; 15 | 16 | /// Locks this Mutex. 17 | fn lock(&self) -> impl Future> + OptionalSend; 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/watch/watch_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by the `WatchSender`. 4 | #[derive(PartialEq, Eq, Clone, Copy)] 5 | pub struct SendError(pub T); 6 | 7 | impl fmt::Debug for SendError { 8 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 9 | f.debug_struct("SendError").finish_non_exhaustive() 10 | } 11 | } 12 | 13 | impl fmt::Display for SendError { 14 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 15 | write!(fmt, "watch channel closed") 16 | } 17 | } 18 | 19 | impl std::error::Error for SendError {} 20 | 21 | /// Error returned by the `WatchReceiver`. 22 | #[derive(PartialEq, Eq, Clone, Copy)] 23 | pub struct RecvError(pub ()); 24 | 25 | impl fmt::Debug for RecvError { 26 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 27 | f.debug_struct("RecvError").finish_non_exhaustive() 28 | } 29 | } 30 | 31 | impl fmt::Display for RecvError { 32 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 33 | write!(fmt, "watch channel closed") 34 | } 35 | } 36 | 37 | impl std::error::Error for RecvError {} 38 | -------------------------------------------------------------------------------- /openraft/src/vote/leader_id/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod leader_id_adv; 2 | pub mod leader_id_std; 3 | 4 | pub(crate) mod leader_id_cmp; 5 | pub(crate) mod raft_committed_leader_id; 6 | pub(crate) mod raft_leader_id; 7 | 8 | #[cfg(feature = "single-term-leader")] 9 | compile_error!( 10 | r#"`single-term-leader` is removed. 11 | To enable standard Raft mode: 12 | - either add `LeaderId = openraft::impls::leader_id_std::LeaderId` to `declare_raft_types!(YourTypeConfig)` statement, 13 | - or add `type LeaderId: opernaft::impls::leader_id_std::LeaderId` to the `RaftTypeConfig` implementation."# 14 | ); 15 | -------------------------------------------------------------------------------- /openraft/src/vote/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines election related types. 2 | 3 | pub(crate) mod committed; 4 | pub(crate) mod leader_id; 5 | pub(crate) mod non_committed; 6 | pub(crate) mod raft_term; 7 | pub(crate) mod raft_vote; 8 | pub(crate) mod ref_vote; 9 | #[allow(clippy::module_inception)] 10 | mod vote; 11 | pub(crate) mod vote_status; 12 | 13 | pub use leader_id::raft_committed_leader_id::RaftCommittedLeaderId; 14 | pub use leader_id::raft_leader_id::RaftLeaderId; 15 | pub use leader_id::raft_leader_id::RaftLeaderIdExt; 16 | pub use raft_term::RaftTerm; 17 | pub use raft_vote::RaftVote; 18 | 19 | pub use self::leader_id::leader_id_adv; 20 | pub use self::leader_id::leader_id_cmp::LeaderIdCompare; 21 | pub use self::leader_id::leader_id_std; 22 | pub use self::vote::Vote; 23 | -------------------------------------------------------------------------------- /openraft/src/vote/raft_term/mod.rs: -------------------------------------------------------------------------------- 1 | mod raft_term_impls; 2 | 3 | use std::fmt::Debug; 4 | use std::fmt::Display; 5 | 6 | use openraft_macros::since; 7 | 8 | use crate::base::OptionalFeatures; 9 | 10 | /// Type representing a Raft term number. 11 | /// 12 | /// A term is a logical clock in Raft that is used to detect obsolete information, 13 | /// such as old leaders. It must be totally ordered and monotonically increasing. 14 | /// 15 | /// Common implementations are provided for standard integer types like `u64`, `i64` etc. 16 | #[since(version = "0.10.0")] 17 | pub trait RaftTerm 18 | where Self: OptionalFeatures + Ord + Debug + Display + Copy + Default + 'static 19 | { 20 | /// Returns the next term. 21 | /// 22 | /// Must satisfy: `self < self.next()` 23 | fn next(&self) -> Self; 24 | } 25 | -------------------------------------------------------------------------------- /openraft/src/vote/vote_status.rs: -------------------------------------------------------------------------------- 1 | use crate::vote::committed::CommittedVote; 2 | use crate::vote::non_committed::NonCommittedVote; 3 | use crate::RaftTypeConfig; 4 | 5 | #[derive(Debug, Clone, PartialEq, Eq)] 6 | pub(crate) enum VoteStatus { 7 | Committed(CommittedVote), 8 | Pending(NonCommittedVote), 9 | } 10 | -------------------------------------------------------------------------------- /rt-compio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-rt-compio" 3 | description = "compio AsyncRuntime support for Openraft" 4 | documentation = "https://docs.rs/openraft-rt-compio" 5 | readme = "README.md" 6 | version = "0.10.0" 7 | edition = "2021" 8 | authors = ["Databend Authors "] 9 | categories = ["algorithms", "asynchronous", "data-structures"] 10 | homepage = "https://github.com/databendlabs/openraft" 11 | keywords = ["consensus", "raft"] 12 | license = "MIT OR Apache-2.0" 13 | repository = "https://github.com/databendlabs/openraft" 14 | 15 | [dependencies] 16 | openraft = { path = "../openraft", version = "0.10.0", default-features = false, features = ["singlethreaded"] } 17 | compio = { version = "0.14.0", features = ["runtime", "time"] } 18 | tokio = { version = "1.22", features = ["sync"], default-features = false } 19 | rand = "0.9" 20 | futures = "0.3" 21 | pin-project-lite = "0.2.16" 22 | 23 | [dev-dependencies] 24 | compio = { version = "0.14.0", features = ["macros"] } 25 | -------------------------------------------------------------------------------- /rt-compio/README.md: -------------------------------------------------------------------------------- 1 | # openraft-rt-compio 2 | 3 | compio [`AsyncRuntime`][rt_link] support for Openraft. 4 | 5 | [rt_link]: https://docs.rs/openraft/latest/openraft/async_runtime/trait.AsyncRuntime.html 6 | -------------------------------------------------------------------------------- /rt-compio/src/mutex.rs: -------------------------------------------------------------------------------- 1 | use std::future::Future; 2 | 3 | use openraft::type_config::async_runtime::mutex; 4 | use openraft::OptionalSend; 5 | 6 | pub struct TokioMutex(tokio::sync::Mutex); 7 | 8 | impl mutex::Mutex for TokioMutex 9 | where T: OptionalSend + 'static 10 | { 11 | type Guard<'a> = tokio::sync::MutexGuard<'a, T>; 12 | 13 | #[inline] 14 | fn new(value: T) -> Self { 15 | TokioMutex(tokio::sync::Mutex::new(value)) 16 | } 17 | 18 | #[inline] 19 | fn lock(&self) -> impl Future> + OptionalSend { 20 | self.0.lock() 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /rt-compio/src/oneshot.rs: -------------------------------------------------------------------------------- 1 | //! Oneshot channel wrapper types and their trait impl. 2 | 3 | use openraft::type_config::async_runtime::oneshot; 4 | use openraft::OptionalSend; 5 | 6 | pub struct FuturesOneshot; 7 | 8 | pub struct FuturesOneshotSender(futures::channel::oneshot::Sender); 9 | 10 | impl oneshot::Oneshot for FuturesOneshot { 11 | type Sender = FuturesOneshotSender; 12 | type Receiver = futures::channel::oneshot::Receiver; 13 | type ReceiverError = futures::channel::oneshot::Canceled; 14 | 15 | #[inline] 16 | fn channel() -> (Self::Sender, Self::Receiver) 17 | where T: OptionalSend { 18 | let (tx, rx) = futures::channel::oneshot::channel(); 19 | let tx_wrapper = FuturesOneshotSender(tx); 20 | 21 | (tx_wrapper, rx) 22 | } 23 | } 24 | 25 | impl oneshot::OneshotSender for FuturesOneshotSender 26 | where T: OptionalSend 27 | { 28 | #[inline] 29 | fn send(self, t: T) -> Result<(), T> { 30 | self.0.send(t) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rt-monoio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-rt-monoio" 3 | description = "monoio AsyncRuntime support for Openraft" 4 | documentation = "https://docs.rs/openraft-rt-monoio" 5 | readme = "README.md" 6 | version = "0.10.0" 7 | edition = "2021" 8 | authors = [ 9 | "Databend Authors ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../openraft", version = "0.10.0", default-features = false, features = ["singlethreaded"] } 19 | 20 | rand = "0.9" 21 | 22 | futures = { version = "0.3" } 23 | local-sync = "0.1.1" 24 | 25 | monoio = "0.2.3" 26 | tokio = { version = "1.22", features = ["sync"] } 27 | -------------------------------------------------------------------------------- /rt-monoio/README.md: -------------------------------------------------------------------------------- 1 | # openraft-rt-monoio 2 | 3 | monoio [`AsyncRuntime`][rt_link] support for Openraft. 4 | 5 | [rt_link]: https://docs.rs/openraft/latest/openraft/async_runtime/trait.AsyncRuntime.html -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | nightly-2025-05-01 2 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # unstable_features = true 2 | # edition = "2018" 3 | 4 | # fn_args_layout = "Compressed" 5 | # use_small_heuristics = "Default" 6 | # use_try_shorthand = true 7 | 8 | # # pre-unstable 9 | 10 | # single_line_if_else_max_width = 75 11 | # space_around_attr_eq = false 12 | # struct_lit_width = 50 13 | 14 | # # unstable 15 | 16 | # condense_wildcard_suffixes = true 17 | # format_code_in_doc_comments = true 18 | # format_strings = true 19 | # match_block_trailing_comma = false 20 | # normalize_comments = true 21 | # normalize_doc_attributes = true 22 | # reorder_impl_items = true 23 | # struct_lit_single_line = true 24 | # use_field_init_shorthand = true 25 | 26 | 27 | reorder_imports = true 28 | imports_granularity = "Item" 29 | group_imports = "StdExternalCrate" 30 | where_single_line = true 31 | trailing_comma = "Vertical" 32 | overflow_delimited_expr = true 33 | wrap_comments = true 34 | comment_width = 100 35 | max_width = 120 36 | inline_attribute_width = 0 37 | 38 | merge_derives = false 39 | 40 | # pre-unstable 41 | 42 | chain_width = 100 43 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | ### `build_change_log.py` 2 | 3 | Build change log since a previous git-tag 4 | 5 | - Usage: `./scripts/build_change_log.py`: build change log since last tag. 6 | - Usage: `./scripts/build_change_log.py `: build since specified commit. 7 | - Install: `pip install -r requirements.txt`. 8 | 9 | This script builds change log from git commit messages, outputs a versioned 10 | change log to `./change-log/.md` and concatenates all versioned change log 11 | into `./change-log.md`. 12 | 13 | If `./change-log/.md` exists, it skips re-building it from git-log. 14 | Thus you can generate semi-automatic change-logs by editing `./change-log/.md` and running the script again, which will just only the `./change-log.md`. 15 | 16 | 17 | ### `check.kdl` 18 | 19 | Run daily tests in parallel: 20 | 21 | - Usage: `zellij --layout ./scripts/check.kdl`. 22 | - Install: `cargo install zellij`. 23 | 24 | It opens 3 panes to run: 25 | - `cargo test --libs`: run only lib tests 26 | - `cargo test --test *`: run only integration tests. 27 | - `cargo clippy`: lint 28 | -------------------------------------------------------------------------------- /scripts/change-types.yaml: -------------------------------------------------------------------------------- 1 | - data-change 2 | - api-change 3 | - new-feature 4 | - improve 5 | - internal 6 | - dep 7 | - doc 8 | - test 9 | - ci 10 | - refactor 11 | - fixbug 12 | - fixdoc 13 | - other 14 | -------------------------------------------------------------------------------- /scripts/check.kdl: -------------------------------------------------------------------------------- 1 | // Use zellij to run test in parallel. 2 | // 3 | // Install: 4 | // cargo install zellij 5 | // 6 | // Usage: 7 | // zellij action new-tab --layout check.kdl 8 | // zellij --layout check.kdl 9 | 10 | simplified_ui true 11 | 12 | layout { 13 | 14 | tab name="3m1q" { 15 | // tab-bar 16 | pane size=1 borderless=true { 17 | plugin location="zellij:tab-bar" 18 | } 19 | 20 | pane split_direction="vertical" { 21 | pane { 22 | command "cargo" 23 | args "test" "--lib" 24 | } 25 | pane { 26 | command "cargo" 27 | args "test" "--test" "*" 28 | } 29 | pane { 30 | command "cargo" 31 | args "clippy" "--no-deps" "--all-targets" "--" "-D" "warnings" 32 | } 33 | } 34 | // status-bar 35 | pane size=2 borderless=true { 36 | plugin location="zellij:status-bar" 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /scripts/mprocs-check.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env mprocs --config 2 | 3 | 4 | # run local check in parallel with mprocs 5 | # 6 | # Usage: 7 | # mprocs --config ./scripts/check.yaml 8 | # 9 | # Install: 10 | # cargo install mprocs 11 | # 12 | # See: https://github.com/pvolok/mprocs 13 | 14 | 15 | procs: 16 | test-lib: 17 | cmd: ["cargo", "test", "--lib"] 18 | it: 19 | cmd: ["cargo", "test", "--test", "*"] 20 | clippy: 21 | cmd: ["cargo", "clippy", "--no-deps", "--all-targets", "--", "-D", "warnings"] 22 | 23 | # # keeps examples: 24 | # xx: 25 | # shell: "nodemon server.js" 26 | # webpack: "webpack serve" 27 | # tests: 28 | # shell: "jest -w" 29 | # env: 30 | # NODE_ENV: test 31 | -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | semantic_version 2 | toml 3 | jinja2 4 | PyYAML 5 | -------------------------------------------------------------------------------- /scripts/watch-doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Appearence order is disabled 4 | # RUSTDOCFLAGS='-Z unstable-options --sort-modules-by-appearance' cargo watch -x 'doc --document-private-items --all --no-deps' 5 | 6 | if [ ".$1" = ".-p" ]; then 7 | cargo watch -x 'doc --document-private-items --all --no-deps' 8 | else 9 | cargo watch -x 'doc --all --no-deps' 10 | fi 11 | -------------------------------------------------------------------------------- /stores/README.md: -------------------------------------------------------------------------------- 1 | Example Storage implementations. 2 | 3 | - `memstore` is in-memory storage and is used by the test cases `./tests`. 4 | 5 | If a crate has different feature flags enabled, it must not be members of the workspace. 6 | A feature flag will be enabled for the entire workspace if a member crate enables it. 7 | -------------------------------------------------------------------------------- /stores/memstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-memstore" 3 | description = "A in-memory implementation of the `openraft::RaftLogStorage` and `openraft::RaftStateMachine` trait." 4 | documentation = "https://docs.rs/openraft-memstore" 5 | readme = "README.md" 6 | 7 | version = { workspace = true } 8 | edition = { workspace = true } 9 | authors = { workspace = true } 10 | categories = { workspace = true } 11 | homepage = { workspace = true } 12 | keywords = { workspace = true } 13 | license = { workspace = true } 14 | repository = { workspace = true } 15 | 16 | [dependencies] 17 | openraft = { path= "../../openraft", version = "0.10.0", features=["serde", "type-alias"] } 18 | 19 | serde = { workspace = true } 20 | serde_json = { workspace = true } 21 | tokio = { workspace = true } 22 | tracing = { workspace = true } 23 | 24 | [dev-dependencies] 25 | 26 | [features] 27 | bt = ["openraft/bt"] 28 | single-term-leader = [] 29 | 30 | [package.metadata.docs.rs] 31 | all-features = true 32 | -------------------------------------------------------------------------------- /stores/memstore/README.md: -------------------------------------------------------------------------------- 1 | # openraft-memstore 2 | 3 | This is an in-memory example `RaftLogStorage` and `RaftStateMachine` implementation based on [openraft](https://github.com/databendlabs/openraft/). 4 | 5 | This crate is built mainly for testing or demonstrating purpose.:) 6 | -------------------------------------------------------------------------------- /stores/memstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | use openraft::StorageError; 6 | 7 | use crate::MemLogStore; 8 | use crate::MemStateMachine; 9 | use crate::TypeConfig; 10 | 11 | struct MemStoreBuilder {} 12 | 13 | impl StoreBuilder, Arc, ()> for MemStoreBuilder { 14 | async fn build(&self) -> Result<((), Arc, Arc), StorageError> { 15 | let (log_store, sm) = crate::new_mem_store(); 16 | Ok(((), log_store, sm)) 17 | } 18 | } 19 | 20 | #[tokio::test] 21 | pub async fn test_mem_store() -> Result<(), StorageError> { 22 | Suite::test_all(MemStoreBuilder {}).await?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /tests/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /tests/tests/README.md: -------------------------------------------------------------------------------- 1 | # Openraft integration tests 2 | 3 | The integration tests for Openraft are stored in this directory and rely on 4 | `memstore` with `serde` enabled. 5 | Certain tests in Openraft require the `serde` feature to be disabled. 6 | To avoid enabling `serde` for all tests in Openraft, we must relocate the 7 | integration tests to a separate crate. 8 | 9 | 10 | ## Case naming convention 11 | 12 | A test file name starts with `t[\d\d]_`, where `\d\d` is the test case number indicating priority. 13 | 14 | - `t00`: not used. 15 | - `t10`: basic behaviors. 16 | - `t20`: life cycle test cases. 17 | - `t30`: special cases for an API. 18 | - `t40`: not used. 19 | - `t50`: environment depended behaviors. 20 | - `t60`: config related behaviors. 21 | - `t70`: not used. 22 | - `t80`: not used. 23 | - `t90`: issue fixes. 24 | -------------------------------------------------------------------------------- /tests/tests/append_entries/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_conflict_with_empty_entries; 11 | mod t10_see_higher_vote; 12 | mod t11_append_conflicts; 13 | mod t11_append_entries_with_bigger_term; 14 | mod t11_append_inconsistent_log; 15 | mod t11_append_updates_membership; 16 | mod t30_replication_1_voter_to_isolated_learner; 17 | mod t60_enable_heartbeat; 18 | mod t61_heartbeat_reject_vote; 19 | mod t61_large_heartbeat; 20 | mod t90_issue_216_stale_last_log_id; 21 | -------------------------------------------------------------------------------- /tests/tests/append_entries/t61_large_heartbeat.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use anyhow::Result; 5 | use maplit::btreeset; 6 | use openraft::Config; 7 | 8 | use crate::fixtures::ut_harness; 9 | use crate::fixtures::RaftRouter; 10 | 11 | /// Large heartbeat should not block replication. 12 | /// I.e., replication should not be driven by heartbeat. 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn large_heartbeat() -> Result<()> { 16 | // Setup test dependencies. 17 | let config = Arc::new( 18 | Config { 19 | heartbeat_interval: 10_000, 20 | election_timeout_min: 20_000, 21 | election_timeout_max: 30_000, 22 | max_payload_entries: 2, 23 | ..Default::default() 24 | } 25 | .validate()?, 26 | ); 27 | let mut router = RaftRouter::new(config.clone()); 28 | 29 | let mut log_index = router.new_cluster(btreeset! {0}, btreeset! {1}).await?; 30 | 31 | router.client_request_many(0, "foo", 10).await?; 32 | log_index += 10; 33 | 34 | router.wait(&1, Some(Duration::from_millis(3_000))).applied_index(Some(log_index), "").await?; 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /tests/tests/client_api/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // See ./README.md 9 | 10 | mod t10_client_writes; 11 | mod t11_client_reads; 12 | mod t12_trigger_purge_log; 13 | mod t13_begin_receiving_snapshot; 14 | mod t13_get_snapshot; 15 | mod t13_install_full_snapshot; 16 | mod t13_trigger_snapshot; 17 | mod t14_transfer_leader; 18 | mod t16_with_raft_state; 19 | mod t16_with_state_machine; 20 | mod t50_lagging_network_write; 21 | mod t51_write_when_leader_quit; 22 | -------------------------------------------------------------------------------- /tests/tests/client_api/t13_begin_receiving_snapshot.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::ut_harness; 8 | use crate::fixtures::RaftRouter; 9 | 10 | #[tracing::instrument] 11 | #[test_harness::test(harness = ut_harness)] 12 | async fn begin_receiving_snapshot() -> anyhow::Result<()> { 13 | let config = Arc::new( 14 | Config { 15 | enable_heartbeat: false, 16 | enable_elect: false, 17 | ..Default::default() 18 | } 19 | .validate()?, 20 | ); 21 | 22 | let mut router = RaftRouter::new(config.clone()); 23 | 24 | tracing::info!("--- initializing cluster"); 25 | let mut log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 26 | 27 | tracing::info!(log_index, "--- isolate node 2 so that it can receive snapshot"); 28 | router.set_unreachable(2, true); 29 | 30 | tracing::info!(log_index, "--- write to make node-0,1 have more logs"); 31 | { 32 | log_index += router.client_request_many(0, "foo", 3).await?; 33 | router.wait(&0, timeout()).applied_index(Some(log_index), "write more log").await?; 34 | router.wait(&1, timeout()).applied_index(Some(log_index), "write more log").await?; 35 | } 36 | 37 | tracing::info!(log_index, "--- got a snapshot data"); 38 | { 39 | let n1 = router.get_raft_handle(&1)?; 40 | let _resp = n1.begin_receiving_snapshot().await?; 41 | } 42 | 43 | Ok(()) 44 | } 45 | 46 | fn timeout() -> Option { 47 | Some(Duration::from_millis(1_000)) 48 | } 49 | -------------------------------------------------------------------------------- /tests/tests/client_api/t16_with_raft_state.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::error::Fatal; 6 | use openraft::Config; 7 | 8 | use crate::fixtures::log_id; 9 | use crate::fixtures::ut_harness; 10 | use crate::fixtures::RaftRouter; 11 | 12 | /// Access Raft state via [`Raft::with_raft_state()`](openraft::Raft::with_raft_state) 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn with_raft_state() -> Result<()> { 16 | let config = Arc::new( 17 | Config { 18 | enable_heartbeat: false, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | 26 | tracing::info!("--- initializing cluster"); 27 | let log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 28 | 29 | let n0 = router.get_raft_handle(&0)?; 30 | 31 | let committed = n0.with_raft_state(|st| st.committed().cloned()).await?; 32 | assert_eq!(committed, Some(log_id(1, 0, log_index))); 33 | 34 | tracing::info!("--- shutting down node 0"); 35 | n0.shutdown().await?; 36 | 37 | let res = n0.with_raft_state(|st| st.committed().cloned()).await; 38 | assert_eq!(Err(Fatal::Stopped), res); 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /tests/tests/elect/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_elect_compare_last_log; 11 | mod t11_elect_seize_leadership; 12 | -------------------------------------------------------------------------------- /tests/tests/elect/t11_elect_seize_leadership.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use anyhow::Result; 5 | use maplit::btreeset; 6 | use openraft::Config; 7 | use openraft::ServerState; 8 | 9 | use crate::fixtures::ut_harness; 10 | use crate::fixtures::RaftRouter; 11 | 12 | /// A node with higher term takes leadership from the current leader. 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn elect_seize_leadership() -> Result<()> { 16 | let config = Arc::new( 17 | Config { 18 | enable_heartbeat: false, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | 26 | tracing::info!("--- create cluster of 0,1,2"); 27 | let log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 28 | 29 | let n0 = router.get_raft_handle(&0)?; 30 | n0.wait(timeout()).state(ServerState::Leader, "node 0 becomes leader").await?; 31 | 32 | tracing::info!(log_index, "--- trigger election on node 1"); 33 | { 34 | let n1 = router.get_raft_handle(&1)?; 35 | n1.trigger().elect().await?; 36 | 37 | n1.wait(timeout()).state(ServerState::Leader, "node 1 becomes leader").await?; 38 | } 39 | 40 | Ok(()) 41 | } 42 | 43 | fn timeout() -> Option { 44 | Some(Duration::from_millis(2000)) 45 | } 46 | -------------------------------------------------------------------------------- /tests/tests/life_cycle/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_initialization; 11 | mod t11_shutdown; 12 | mod t50_follower_restart_does_not_interrupt; 13 | mod t50_single_follower_restart; 14 | mod t50_single_leader_restart_re_apply_logs; 15 | mod t90_issue_607_single_restart; 16 | mod t90_issue_920_non_voter_leader_restart; 17 | -------------------------------------------------------------------------------- /tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use openraft::storage::RaftLogStorage; 5 | use openraft::Config; 6 | use openraft::ServerState; 7 | use openraft::Vote; 8 | 9 | use crate::fixtures::ut_harness; 10 | use crate::fixtures::RaftRouter; 11 | 12 | /// Special case: A leader that is not a member(neither a voter or non-voter) should be started too, 13 | /// as a learner. 14 | #[tracing::instrument] 15 | #[test_harness::test(harness = ut_harness)] 16 | async fn issue_920_non_member_leader_restart() -> anyhow::Result<()> { 17 | let config = Arc::new( 18 | Config { 19 | enable_heartbeat: false, 20 | ..Default::default() 21 | } 22 | .validate()?, 23 | ); 24 | 25 | let mut router = RaftRouter::new(config.clone()); 26 | 27 | let (mut log_store, sm) = router.new_store(); 28 | // Set committed vote that believes node 0 is the leader. 29 | log_store.save_vote(&Vote::new_committed(1, 0)).await?; 30 | router.new_raft_node_with_sto(0, log_store, sm).await; 31 | 32 | router 33 | .wait(&0, timeout()) 34 | .state(ServerState::Learner, "node 0 becomes learner when startup") 35 | .await?; 36 | 37 | Ok(()) 38 | } 39 | 40 | fn timeout() -> Option { 41 | Some(Duration::from_millis(1000)) 42 | } 43 | -------------------------------------------------------------------------------- /tests/tests/log_store/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_save_committed; 11 | -------------------------------------------------------------------------------- /tests/tests/log_store/t10_save_committed.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use anyhow::Result; 5 | use maplit::btreeset; 6 | use openraft::storage::RaftLogStorage; 7 | use openraft::Config; 8 | 9 | use crate::fixtures::log_id; 10 | use crate::fixtures::ut_harness; 11 | use crate::fixtures::RaftRouter; 12 | 13 | /// Before applying log, write `committed` log id to log store. 14 | #[tracing::instrument] 15 | #[test_harness::test(harness = ut_harness)] 16 | async fn write_committed_log_id_to_log_store() -> Result<()> { 17 | let config = Arc::new( 18 | Config { 19 | enable_tick: false, 20 | ..Default::default() 21 | } 22 | .validate()?, 23 | ); 24 | 25 | let mut router = RaftRouter::new(config.clone()); 26 | 27 | tracing::info!("--- initializing cluster"); 28 | let mut log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 29 | 30 | log_index += router.client_request_many(0, "0", 10).await?; 31 | 32 | for i in [0, 1, 2] { 33 | router.wait(&i, timeout()).applied_index(Some(log_index), "write logs").await?; 34 | } 35 | 36 | for id in [0, 1, 2] { 37 | let (_, mut ls, _) = router.remove_node(id).unwrap(); 38 | let committed = ls.read_committed().await?; 39 | assert_eq!(Some(log_id(1, 0, log_index)), committed, "node-{} committed", id); 40 | } 41 | 42 | Ok(()) 43 | } 44 | 45 | fn timeout() -> Option { 46 | Some(Duration::from_millis(1000)) 47 | } 48 | -------------------------------------------------------------------------------- /tests/tests/management/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_raft_config; 11 | -------------------------------------------------------------------------------- /tests/tests/management/t10_raft_config.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::ut_harness; 8 | use crate::fixtures::RaftRouter; 9 | 10 | /// Get config via [`Raft::config`](openraft::Raft::config) 11 | #[tracing::instrument] 12 | #[test_harness::test(harness = ut_harness)] 13 | async fn raft_config() -> Result<()> { 14 | let config = Arc::new( 15 | Config { 16 | enable_tick: false, 17 | election_timeout_min: 123, 18 | election_timeout_max: 124, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | 26 | tracing::info!("--- initializing cluster"); 27 | let log_index = router.new_cluster(btreeset! {0}, btreeset! {}).await?; 28 | 29 | tracing::info!(log_index, "--- get config"); 30 | { 31 | let n0 = router.get_raft_handle(&0)?; 32 | let c = n0.config(); 33 | 34 | #[allow(clippy::bool_assert_comparison)] 35 | { 36 | assert_eq!(c.enable_tick, false); 37 | } 38 | assert_eq!(c.election_timeout_min, 123); 39 | assert_eq!(c.election_timeout_max, 124); 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /tests/tests/membership/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | 4 | #[macro_use] 5 | #[path = "../fixtures/mod.rs"] 6 | mod fixtures; 7 | 8 | // The number indicate the preferred running order for these case. 9 | // The later tests may depend on the earlier ones. 10 | 11 | mod t10_learner_restart; 12 | mod t10_single_node; 13 | mod t11_add_learner; 14 | mod t12_concurrent_write_and_add_learner; 15 | mod t20_change_membership; 16 | mod t21_change_membership_cases; 17 | mod t30_commit_joint_config; 18 | mod t30_elect_with_new_config; 19 | mod t31_add_remove_follower; 20 | mod t31_remove_leader; 21 | mod t31_removed_follower; 22 | mod t51_remove_unreachable_follower; 23 | mod t52_change_membership_on_uninitialized_node; 24 | mod t99_issue_471_adding_learner_uses_uninit_leader_id; 25 | mod t99_issue_584_replication_state_reverted; 26 | mod t99_new_leader_auto_commit_uniform_config; 27 | -------------------------------------------------------------------------------- /tests/tests/membership/t52_change_membership_on_uninitialized_node.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreemap; 5 | use openraft::ChangeMembers; 6 | use openraft::Config; 7 | 8 | use crate::fixtures::ut_harness; 9 | use crate::fixtures::RaftRouter; 10 | 11 | /// Call `Raft::change_membership()` on an uninitialized node should not panic due to empty 12 | /// membership. 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn change_membership_on_uninitialized_node() -> Result<()> { 16 | let config = Arc::new( 17 | Config { 18 | enable_heartbeat: false, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | router.new_raft_node(0).await; 26 | 27 | let n0 = router.get_raft_handle(&0)?; 28 | let res = n0.change_membership(ChangeMembers::AddVoters(btreemap! {0=>()}), false).await; 29 | tracing::info!("{:?}", res); 30 | 31 | let err = res.unwrap_err(); 32 | tracing::info!("{}", err); 33 | 34 | assert!(err.to_string().contains("forward request to")); 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /tests/tests/membership/t99_issue_471_adding_learner_uses_uninit_leader_id.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::ut_harness; 8 | use crate::fixtures::RaftRouter; 9 | 10 | /// When adding learner and waiting for the learner to become up to date, 11 | /// it should not try to use `matched.leader_id` which may be uninitialized, i.e., `(0,0)`. 12 | /// https://github.com/databendlabs/openraft/issues/471 13 | /// 14 | /// - Brings up 1 leader. 15 | /// - Add learner at once. 16 | /// - It should not panic. 17 | #[tracing::instrument] 18 | #[test_harness::test(harness = ut_harness)] 19 | async fn adding_learner_do_not_use_matched_leader_id() -> Result<()> { 20 | let config = Arc::new( 21 | Config { 22 | // Replicate log one by one, to trigger a state report with matched=(0,0,0), which is 23 | // the first log id. 24 | max_payload_entries: 1, 25 | ..Default::default() 26 | } 27 | .validate()?, 28 | ); 29 | let mut router = RaftRouter::new(config.clone()); 30 | 31 | router.new_cluster(btreeset! {0}, btreeset! {}).await?; 32 | 33 | tracing::info!("--- feed 2 log to make replication busy"); 34 | { 35 | router.client_request_many(0, "foo", 2).await?; 36 | } 37 | 38 | // Delay replication. 39 | router.network_send_delay(100); 40 | 41 | tracing::info!("--- add learner: node-1"); 42 | { 43 | router.new_raft_node(1).await; 44 | router.add_learner(0, 1).await?; 45 | } 46 | 47 | Ok(()) 48 | } 49 | -------------------------------------------------------------------------------- /tests/tests/metrics/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_current_leader; 11 | mod t10_leader_last_ack; 12 | mod t10_purged; 13 | mod t10_server_metrics_and_data_metrics; 14 | mod t20_metrics_state_machine_consistency; 15 | mod t30_leader_metrics; 16 | mod t40_metrics_wait; 17 | -------------------------------------------------------------------------------- /tests/tests/metrics/t10_current_leader.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::ut_harness; 8 | use crate::fixtures::RaftRouter; 9 | 10 | /// Current leader tests. 11 | /// 12 | /// What does this test do? 13 | /// 14 | /// - create a stable 3-node cluster. 15 | /// - call the current_leader interface on the all nodes, and assert success. 16 | #[tracing::instrument] 17 | #[test_harness::test(harness = ut_harness)] 18 | async fn current_leader() -> Result<()> { 19 | let config = Arc::new( 20 | Config { 21 | enable_heartbeat: false, 22 | ..Default::default() 23 | } 24 | .validate()?, 25 | ); 26 | 27 | let mut router = RaftRouter::new(config.clone()); 28 | 29 | let _log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 30 | 31 | // Get the ID of the leader, and assert that current_leader succeeds. 32 | let leader = router.leader().expect("leader not found"); 33 | assert_eq!(leader, 0, "expected leader to be node 0, got {}", leader); 34 | 35 | for i in 0..3 { 36 | let leader = router.current_leader(i).await; 37 | assert_eq!(leader, Some(0), "expected leader to be node 0, got {:?}", leader); 38 | } 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /tests/tests/replication/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_append_entries_partial_success; 8 | mod t50_append_entries_backoff; 9 | mod t50_append_entries_backoff_rejoin; 10 | mod t51_append_entries_too_large; 11 | mod t60_feature_loosen_follower_log_revert; 12 | mod t61_allow_follower_log_revert; 13 | -------------------------------------------------------------------------------- /tests/tests/snapshot_building/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_build_snapshot; 8 | mod t35_building_snapshot_does_not_block_append; 9 | mod t35_building_snapshot_does_not_block_apply; 10 | mod t60_snapshot_policy_never; 11 | -------------------------------------------------------------------------------- /tests/tests/snapshot_streaming/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_api_install_snapshot; 8 | mod t10_api_install_snapshot_with_lower_vote; 9 | mod t20_startup_snapshot; 10 | mod t30_purge_in_snapshot_logs; 11 | mod t31_snapshot_overrides_membership; 12 | mod t32_snapshot_uses_prev_snap_membership; 13 | mod t33_snapshot_delete_conflict_logs; 14 | mod t34_replication_does_not_block_purge; 15 | mod t50_snapshot_line_rate_to_snapshot; 16 | mod t50_snapshot_when_lacking_log; 17 | mod t51_after_snapshot_add_learner_and_request_a_log; 18 | mod t60_snapshot_chunk_size; 19 | mod t90_issue_808_snapshot_to_unreachable_node_should_not_block; 20 | -------------------------------------------------------------------------------- /tests/tests/state_machine/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_total_order_apply; 11 | mod t20_state_machine_apply_membership; 12 | --------------------------------------------------------------------------------