├── change-log ├── v0.6.3.md ├── v0.6.4.md ├── v0.7.0.md ├── v0.6.2-alpha.7.md ├── v0.7.0-alpha.4.md ├── v0.7.0-alpha.5.md ├── v0.6.2-alpha.11.md ├── v0.6.2-alpha.12.md ├── v0.7.2.md ├── v0.8.1.md ├── v0.6.2-alpha.9.md ├── v0.7.0-alpha.2.md ├── v0.7.1.md ├── v0.6.2-alpha.6.md ├── v0.6.2-alpha.14.md ├── v0.6.2-alpha.13.md ├── v0.6.2-alpha.1.md ├── v0.7.0-alpha.3.md ├── v0.6.2-alpha.3.md ├── v0.6.2-alpha.8.md ├── v0.7.3.md └── v0.6.2-alpha.2.md ├── examples ├── memstore ├── multi-raft-kv │ ├── tests │ │ └── cluster │ │ │ └── main.rs │ └── Cargo.toml ├── raft-kv-memstore │ ├── .gitignore │ ├── tests │ │ └── cluster │ │ │ └── main.rs │ └── src │ │ ├── network │ │ ├── mod.rs │ │ └── raft.rs │ │ ├── app.rs │ │ ├── test.rs │ │ └── bin │ │ └── main.rs ├── raft-kv-rocksdb │ ├── .gitignore │ ├── src │ │ ├── network │ │ │ ├── mod.rs │ │ │ └── raft.rs │ │ ├── app.rs │ │ └── bin │ │ │ └── main.rs │ └── tests │ │ └── cluster │ │ └── main.rs ├── mem-log │ ├── test-cluster.sh │ ├── src │ │ └── lib.rs │ ├── Cargo.toml │ └── README.md ├── raft-kv-memstore-grpc │ ├── .gitignore │ ├── src │ │ ├── grpc │ │ │ └── mod.rs │ │ ├── pb_impl │ │ │ ├── impl_set_request.rs │ │ │ ├── mod.rs │ │ │ ├── impl_log_id.rs │ │ │ ├── impl_snapshot_request.rs │ │ │ ├── impl_vote_request.rs │ │ │ ├── impl_client_write_response.rs │ │ │ ├── impl_vote_response.rs │ │ │ ├── impl_vote.rs │ │ │ ├── impl_append_entries_request.rs │ │ │ └── impl_membership.rs │ │ ├── test_store.rs │ │ ├── bin │ │ │ └── main.rs │ │ └── lib.rs │ ├── proto │ │ └── app_types.proto │ └── build.rs ├── raft-kv-memstore-network-v2 │ ├── .gitignore │ ├── test-cluster.sh │ ├── tests │ │ └── cluster │ │ │ └── main.rs │ └── Cargo.toml ├── rocksstore │ ├── test-cluster.sh │ └── src │ │ └── test.rs ├── raft-kv-memstore-singlethreaded │ ├── .gitignore │ ├── test-cluster.sh │ ├── tests │ │ └── cluster │ │ │ └── main.rs │ └── Cargo.toml ├── raft-kv-memstore-opendal-snapshot-data │ ├── .gitignore │ ├── test-cluster.sh │ └── tests │ │ └── cluster │ │ └── main.rs ├── utils │ └── README.md ├── client-http │ └── Cargo.toml └── network-v1-http │ └── Cargo.toml ├── openraft ├── .gitignore ├── README.md └── src │ ├── quorum │ ├── bench │ │ └── mod.rs │ ├── joint_impl.rs │ ├── mod.rs │ └── quorum_set.rs │ ├── docs │ ├── faq │ │ ├── 04-storage │ │ │ ├── README.md │ │ │ ├── 01-state-machine-persistence.md │ │ │ └── 02-snapshot-building.md │ │ ├── 02-core-concepts │ │ │ ├── README.md │ │ │ ├── 02-log-id-tuple.md │ │ │ └── 01-differences-from-raft.md │ │ ├── 01-getting-started │ │ │ ├── README.md │ │ │ ├── 02-single-node.md │ │ │ ├── 03-initialize-multiple-nodes.md │ │ │ └── 01-initialize-cluster.md │ │ ├── 03-configuration │ │ │ ├── README.md │ │ │ ├── 04-slow-replication.md │ │ │ ├── 03-frequent-elections.md │ │ │ └── 02-snapshot-policy.md │ │ ├── 05-monitoring │ │ │ ├── README.md │ │ │ └── 04-minimize-error-logs.md │ │ ├── 06-operations │ │ │ ├── README.md │ │ │ ├── 02-remove-node.md │ │ │ ├── 01-node-restart.md │ │ │ ├── 05-forward-to-leader-missing.md │ │ │ ├── 06-error-after-shutdown.md │ │ │ └── 04-custom-node-info.md │ │ ├── 07-troubleshooting │ │ │ ├── README.md │ │ │ ├── 03-data-loss.md │ │ │ ├── 06-excessive-network-errors.md │ │ │ ├── 01-panic-is-following.md │ │ │ └── 05-incorrect-config.md │ │ ├── mod.rs │ │ ├── Makefile │ │ ├── build_faq.py │ │ └── README.md │ ├── getting_started │ │ └── mod.rs │ ├── feature_flags │ │ ├── mod.rs │ │ ├── Makefile │ │ └── feature-flags-toc.md │ ├── internal │ │ ├── mod.rs │ │ └── internal.md │ ├── components │ │ └── mod.rs │ ├── mod.rs │ ├── obsolete │ │ ├── mod.rs │ │ └── fast_commit.md │ ├── cluster_control │ │ └── mod.rs │ ├── protocol │ │ ├── replication.md │ │ └── mod.rs │ ├── upgrade_guide │ │ └── mod.rs │ └── data │ │ └── mod.rs │ ├── membership │ └── bench │ │ └── mod.rs │ ├── progress │ ├── bench │ │ ├── mod.rs │ │ └── vec_progress_update.rs │ ├── progress_stats.rs │ ├── id_val.rs │ ├── display_vec_progress.rs │ ├── inflight_id.rs │ └── stream_id.rs │ ├── raft │ ├── api │ │ └── mod.rs │ ├── responder │ │ ├── impls │ │ │ └── mod.rs │ │ └── core_responder.rs │ ├── linearizable_read │ │ └── mod.rs │ ├── core_state.rs │ ├── watch_handle.rs │ └── message │ │ ├── mod.rs │ │ └── stream_append_error.rs │ ├── core │ ├── heartbeat │ │ ├── mod.rs │ │ └── errors │ │ │ ├── mod.rs │ │ │ ├── raft_core_closed.rs │ │ │ └── stopped.rs │ ├── sm │ │ └── mod.rs │ ├── core_state.rs │ ├── replication_state.rs │ ├── io_flush_tracking │ │ └── mod.rs │ └── server_state.rs │ ├── network │ ├── v1 │ │ └── mod.rs │ ├── v2 │ │ └── mod.rs │ ├── rpc_type.rs │ └── backoff.rs │ ├── base │ ├── histogram │ │ ├── mod.rs │ │ └── percentile_stats.rs │ └── finalized.rs │ ├── compat │ └── mod.rs │ ├── engine │ ├── handler │ │ ├── mod.rs │ │ └── snapshot_handler │ │ │ └── trigger_snapshot_test.rs │ ├── command_kind.rs │ ├── replication_progress.rs │ └── respond_command.rs │ ├── vote │ ├── vote_status.rs │ ├── leader_id │ │ └── mod.rs │ └── raft_term │ │ └── mod.rs │ ├── try_as_ref.rs │ ├── entry │ ├── raft_payload.rs │ └── raft_entry_ext.rs │ ├── testing │ ├── log │ │ ├── mod.rs │ │ └── store_builder.rs │ ├── mod.rs │ └── common.rs │ ├── raft_state │ ├── vote_state_reader.rs │ └── tests │ │ └── validate_test.rs │ ├── proposer │ ├── mod.rs │ └── leader_state.rs │ ├── error │ ├── higher_vote.rs │ ├── replication_closed.rs │ ├── allow_next_revert_error.rs │ ├── replication_error.rs │ ├── node_not_found.rs │ ├── into_ok.rs │ ├── linearizable_read_error.rs │ ├── invalid_sm.rs │ └── membership_error.rs │ ├── type_config │ └── async_runtime │ │ ├── tokio_impls │ │ └── tokio_runtime │ │ │ ├── mutex.rs │ │ │ └── oneshot.rs │ │ ├── mpsc │ │ ├── send_error.rs │ │ └── try_recv_error.rs │ │ ├── mutex.rs │ │ └── watch │ │ └── watch_error.rs │ ├── replication │ ├── event_watcher.rs │ ├── snapshot_transmitter_handle.rs │ ├── replication_progress.rs │ └── stream_context.rs │ ├── log_id │ ├── option_ref_log_id_ext.rs │ ├── log_id_option_ext.rs │ ├── raft_log_id_ext.rs │ └── log_index_option_ext.rs │ ├── storage │ ├── log_state.rs │ ├── v2 │ │ └── mod.rs │ └── snapshot.rs │ ├── stats.rs │ ├── display_ext │ └── display_option.rs │ ├── metrics │ └── metric_display.rs │ └── raft_types.rs ├── rust-toolchain ├── .github ├── actions-rs │ └── grcov.yml ├── ISSUE_TEMPLATE │ ├── blank.md │ ├── feature_request.md │ ├── doc-request.md │ └── bug_report.md ├── dependabot.yml ├── workflows │ ├── gtp-translate.yml │ ├── devskim-analysis.yml │ └── coverage.yml └── pull_request_template.md ├── macros ├── tests │ ├── expand │ │ ├── expand │ │ │ ├── keyed_let_0.expanded.rs │ │ │ ├── keyed_let_1.expanded.rs │ │ │ ├── variable_empty.expanded.rs │ │ │ ├── keyed_let_dup_a.expanded.rs │ │ │ ├── keyed_let_0.rs │ │ │ ├── not_keyed_let_dup_a.expanded.rs │ │ │ ├── keyed_let_1.rs │ │ │ ├── variable_empty.rs │ │ │ ├── variable_type_attributes.expanded.rs │ │ │ ├── not_keyed_let_dup_a.rs │ │ │ ├── keyed_let_dup_a.rs │ │ │ └── variable_type_attributes.rs │ │ ├── fail │ │ │ ├── variable_absent.rs │ │ │ ├── variable_absent.stderr │ │ │ ├── arg0_invalid_keyed.rs │ │ │ └── arg0_invalid_keyed.stderr │ │ └── expand-unstable │ │ │ ├── type_param.rs │ │ │ └── type_param.expanded.rs │ ├── since │ │ ├── expand │ │ │ ├── valid_semver.expanded.rs │ │ │ ├── with_change.expanded.rs │ │ │ ├── with_date.expanded.rs │ │ │ ├── valid_semver.rs │ │ │ ├── with_date_change.expanded.rs │ │ │ ├── with_change.rs │ │ │ ├── with_date.rs │ │ │ ├── with_date_many_change.expanded.rs │ │ │ ├── with_date_change.rs │ │ │ ├── multi_since.expanded.rs │ │ │ ├── with_date_many_change.rs │ │ │ ├── multi_since.rs │ │ │ ├── with_doc.expanded.rs │ │ │ └── with_doc.rs │ │ └── fail │ │ │ ├── invalid_sem_ver.rs │ │ │ ├── invalid_date.rs │ │ │ ├── invalid_sem_ver.stderr │ │ │ └── invalid_date.stderr │ ├── test_since.rs │ └── test_expand.rs ├── README.md └── Cargo.toml ├── scripts ├── requirements.txt ├── change-types.yaml ├── check.sh ├── watch-doc.sh ├── mprocs-check.yaml ├── check.kdl ├── README.md └── check-parallel.sh ├── tests ├── src │ └── main.rs └── tests │ ├── management │ ├── main.rs │ └── t10_raft_config.rs │ ├── elect │ └── main.rs │ ├── log_store │ └── main.rs │ ├── state_machine │ └── main.rs │ ├── snapshot_building │ └── main.rs │ ├── replication │ └── main.rs │ ├── life_cycle │ ├── main.rs │ └── t90_issue_920_non_voter_leader_restart.rs │ ├── metrics │ ├── main.rs │ └── t10_current_leader.rs │ ├── client_api │ ├── main.rs │ └── t16_with_raft_state.rs │ ├── append_entries │ ├── main.rs │ └── t61_large_heartbeat.rs │ ├── snapshot_streaming │ └── main.rs │ ├── README.md │ ├── fixtures │ ├── pre_hook.rs │ ├── post_hook.rs │ └── rpc_error_type.rs │ └── membership │ ├── main.rs │ └── t52_change_membership_on_uninitialized_node.rs ├── clippy.toml ├── guide └── src │ ├── images │ └── raft-overview.png │ └── custom.css ├── benchmarks └── minimal │ ├── tests │ ├── benchmark │ │ ├── main.rs │ │ └── store_test.rs │ └── README.md │ └── src │ └── lib.rs ├── multiraft ├── src │ └── lib.rs └── Cargo.toml ├── rt-monoio ├── README.md ├── src │ ├── mutex.rs │ └── oneshot.rs └── Cargo.toml ├── rt-compio ├── README.md ├── src │ ├── mutex.rs │ └── oneshot.rs └── Cargo.toml ├── stores ├── memstore │ ├── README.md │ ├── src │ │ └── test.rs │ └── Cargo.toml └── README.md ├── .gitignore ├── book.toml ├── .mergify.yml ├── LICENSE-MIT ├── rustfmt.toml └── CONTRIBUTING.md /change-log/v0.6.3.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.6.4.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.7.0.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.7.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.4.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.5.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/memstore: -------------------------------------------------------------------------------- 1 | mem-log -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.11.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.12.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /openraft/.gitignore: -------------------------------------------------------------------------------- 1 | /_log/ 2 | -------------------------------------------------------------------------------- /openraft/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | nightly-2025-10-01 2 | -------------------------------------------------------------------------------- /openraft/src/quorum/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod is_quorum; 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/04-storage/README.md: -------------------------------------------------------------------------------- 1 | Storage 2 | -------------------------------------------------------------------------------- /openraft/src/membership/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod is_quorum; 2 | -------------------------------------------------------------------------------- /.github/actions-rs/grcov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "memstore*" 3 | -------------------------------------------------------------------------------- /openraft/src/progress/bench/mod.rs: -------------------------------------------------------------------------------- 1 | mod vec_progress_update; 2 | -------------------------------------------------------------------------------- /examples/multi-raft-kv/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | mod test_cluster; 2 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_0.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() {} 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/02-core-concepts/README.md: -------------------------------------------------------------------------------- 1 | Core Concepts 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/01-getting-started/README.md: -------------------------------------------------------------------------------- 1 | Getting Started 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/03-configuration/README.md: -------------------------------------------------------------------------------- 1 | Configuration & Tuning 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/05-monitoring/README.md: -------------------------------------------------------------------------------- 1 | Monitoring & Observability 2 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/README.md: -------------------------------------------------------------------------------- 1 | Operations & Maintenance 2 | -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | semantic_version 2 | toml 3 | jinja2 4 | PyYAML 5 | -------------------------------------------------------------------------------- /tests/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/07-troubleshooting/README.md: -------------------------------------------------------------------------------- 1 | Troubleshooting & Safety 2 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | *.db 5 | /*.log 6 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | too-many-arguments-threshold = 10 2 | cognitive-complexity-threshold = 25 3 | -------------------------------------------------------------------------------- /examples/mem-log/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for memstore" -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/valid_semver.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /openraft/src/docs/getting_started/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("getting-started.md")] 2 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/grpc/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod app_service; 2 | pub mod raft_service; 3 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | mod test_cluster; 2 | mod test_follower_read; 3 | -------------------------------------------------------------------------------- /examples/rocksstore/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for rocksstore" -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_1.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | } 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod management; 3 | pub mod raft; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod management; 3 | pub mod raft; 4 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0: add Foo 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | vendor 3 | .idea 4 | 5 | /*.log 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01) 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_empty.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | c; 3 | c; 4 | u8; 5 | } 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/valid_semver.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_sem_ver.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0..0")] 2 | fn main() {} 3 | -------------------------------------------------------------------------------- /openraft/src/raft/api/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod app; 2 | pub(crate) mod management; 3 | pub(crate) mod protocol; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /openraft/src/docs/faq/mod.rs: -------------------------------------------------------------------------------- 1 | //! # FAQ 2 | #![doc = include_str!("faq-toc.md")] 3 | #![doc = include_str!("faq.md")] 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/test-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "No shell test script for this example" -------------------------------------------------------------------------------- /guide/src/images/raft-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendlabs/openraft/HEAD/guide/src/images/raft-overview.png -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01): add Foo 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_date.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01--")] 2 | fn main() {} 3 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", change = "add Foo")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /benchmarks/minimal/tests/benchmark/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | mod store_test; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-opendal-snapshot-data/tests/cluster/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | mod test_cluster; 4 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_dup_a.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | let b: String = "foo".to_string(); 4 | } 5 | -------------------------------------------------------------------------------- /openraft/src/core/heartbeat/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod errors; 2 | pub(crate) mod event; 3 | pub(crate) mod handle; 4 | pub(crate) mod worker; 5 | -------------------------------------------------------------------------------- /examples/mem-log/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Provide storage layer implementation for examples. 2 | 3 | mod log_store; 4 | 5 | pub use log_store::LogStore; 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_many_change.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.0.0, Date(2021-01-01): add Foo; add Bar; add Baz 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01", change = "add Foo")] 2 | const A: i32 = 0; 3 | -------------------------------------------------------------------------------- /multiraft/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod network; 2 | 3 | pub use network::GroupNetworkAdapter; 4 | pub use network::GroupNetworkFactory; 5 | pub use network::GroupRouter; 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/blank.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Blank 3 | about: Unclassified issue 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_0.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | ); 6 | } 7 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/not_keyed_let_dup_a.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let a: u64 = 1; 3 | let b: String = "foo".to_string(); 4 | let a: u32 = 2; 5 | } 6 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/mod.rs: -------------------------------------------------------------------------------- 1 | //! # Feature flags 2 | 3 | #![doc = include_str!("feature-flags-toc.md")] 4 | // 5 | #![doc = include_str!("feature-flags.md")] 6 | -------------------------------------------------------------------------------- /openraft/src/core/heartbeat/errors/mod.rs: -------------------------------------------------------------------------------- 1 | mod raft_core_closed; 2 | mod stopped; 3 | 4 | pub(crate) use raft_core_closed::RaftCoreClosed; 5 | pub(crate) use stopped::Stopped; 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/multi_since.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Since: 1.1.0, Date(2021-02-01): add Foo; add Bar; add Baz 2 | /// 3 | /// Since: 1.0.0, Date(2021-01-01): Added 4 | const A: i32 = 0; 5 | -------------------------------------------------------------------------------- /openraft/src/network/v1/mod.rs: -------------------------------------------------------------------------------- 1 | //! Version 1 of the Raft network API. 2 | 3 | mod factory; 4 | mod network; 5 | 6 | pub use factory::RaftNetworkFactory; 7 | pub use network::RaftNetwork; 8 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_1.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | ); 7 | } 8 | -------------------------------------------------------------------------------- /openraft/src/core/heartbeat/errors/raft_core_closed.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 2 | #[error("RaftCore closed receiver")] 3 | pub(crate) struct RaftCoreClosed; 4 | -------------------------------------------------------------------------------- /change-log/v0.7.2.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [568ca470](https://github.com/databendlabs/openraft/commit/568ca470524f77bc721edb104206e1774e1555cc) add Raft::remove_learner(); by 张炎泼; 2022-09-02 4 | -------------------------------------------------------------------------------- /openraft/src/raft/responder/impls/mod.rs: -------------------------------------------------------------------------------- 1 | mod oneshot_responder; 2 | mod progress_responder; 3 | 4 | pub use oneshot_responder::OneshotResponder; 5 | pub use progress_responder::ProgressResponder; 6 | -------------------------------------------------------------------------------- /rt-monoio/README.md: -------------------------------------------------------------------------------- 1 | # openraft-rt-monoio 2 | 3 | monoio [`AsyncRuntime`][rt_link] support for Openraft. 4 | 5 | [rt_link]: https://docs.rs/openraft/latest/openraft/async_runtime/trait.AsyncRuntime.html -------------------------------------------------------------------------------- /openraft/src/base/histogram/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::module_inception)] 2 | mod histogram; 3 | mod percentile_stats; 4 | 5 | pub use histogram::Histogram; 6 | pub use percentile_stats::PercentileStats; 7 | -------------------------------------------------------------------------------- /rt-compio/README.md: -------------------------------------------------------------------------------- 1 | # openraft-rt-compio 2 | 3 | compio [`AsyncRuntime`][rt_link] support for Openraft. 4 | 5 | [rt_link]: https://docs.rs/openraft/latest/openraft/async_runtime/trait.AsyncRuntime.html 6 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/variable_absent.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {K; T; V;}, 5 | (c, , ), 6 | (c, , u8, ), 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_empty.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {K; T; V;}, 5 | (c, , ,), 6 | (c, , u8 ), 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | python ./build_faq.py 3 | # dependency: 4 | # https://github.com/jonschlinkert/markdown-toc#cli 5 | # brew install markdown-toc 6 | markdown-toc faq.md > faq-toc.md 7 | -------------------------------------------------------------------------------- /scripts/change-types.yaml: -------------------------------------------------------------------------------- 1 | - data-change 2 | - api-change 3 | - new-feature 4 | - improve 5 | - internal 6 | - dep 7 | - doc 8 | - test 9 | - ci 10 | - refactor 11 | - fixbug 12 | - fixdoc 13 | - other 14 | -------------------------------------------------------------------------------- /openraft/src/network/v2/mod.rs: -------------------------------------------------------------------------------- 1 | //! Version 2 of the Raft network API. 2 | 3 | #[cfg(all(feature = "tokio-rt", feature = "adapt-network-v1"))] 4 | mod adapt_v1; 5 | mod network; 6 | 7 | pub use network::RaftNetworkV2; 8 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_type_attributes.expanded.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let c: u8; 3 | #[allow(dead_code)] 4 | let c: u16; 5 | #[allow(dead_code)] 6 | #[allow(dead_code)] 7 | let c: u16; 8 | } 9 | -------------------------------------------------------------------------------- /openraft/src/raft/linearizable_read/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines the linearizable read protocol. 2 | 3 | mod linearize_state; 4 | mod linearizer; 5 | 6 | pub use linearize_state::LinearizeState; 7 | pub use linearizer::Linearizer; 8 | -------------------------------------------------------------------------------- /benchmarks/minimal/tests/README.md: -------------------------------------------------------------------------------- 1 | # Benchmark openraft cluster 2 | 3 | It builds a cluster locally with a **minimized** store and network layer, 4 | and is meant to benchmark and find performance bottleneck of the openraft framework. -------------------------------------------------------------------------------- /change-log/v0.8.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [b3c2ff7e](https://github.com/databendlabs/openraft/commit/b3c2ff7e37ce5996f572f43bc574cb50b2d0cdc2) add Membership methods: voter_ids(), learner_ids(), get_node(); by 张炎泼; 2023-02-28 4 | -------------------------------------------------------------------------------- /guide/src/custom.css: -------------------------------------------------------------------------------- 1 | svg.bob line { 2 | stroke: #888; 3 | } 4 | 5 | svg.bob path { 6 | stroke: #888; 7 | } 8 | 9 | svg.bob line.dashed { 10 | stroke: #ccc; 11 | } 12 | 13 | svg.bob circle { 14 | stroke: #bbb; 15 | } 16 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/variable_absent.stderr: -------------------------------------------------------------------------------- 1 | error: Expected the same number of arguments(`c, ,`) as template variables(`["K", "T", "V"]`) 2 | --> tests/expand/fail/variable_absent.rs:5:10 3 | | 4 | 5 | (c, , ), 5 | | ^ 6 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_date_many_change.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since( 2 | version = "1.0.0", 3 | date = "2021-01-01", 4 | change = "add Foo", 5 | change = "add Bar", 6 | change = "add Baz" 7 | )] 8 | const A: i32 = 0; 9 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.9.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [8b59966d](https://github.com/databendlabs/openraft/commit/8b59966dd0a6bf804eb0ba978b5375010bfbc3f3) MembershipConfig.member type is changed form HashSet BTreeSet; by drdr xp; 2021-08-17 4 | -------------------------------------------------------------------------------- /macros/tests/test_since.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn fail() { 3 | let t = trybuild::TestCases::new(); 4 | t.compile_fail("tests/since/fail/*.rs"); 5 | } 6 | 7 | #[test] 8 | fn pass() { 9 | macrotest::expand("tests/since/expand/*.rs"); 10 | } 11 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | # dependency: 3 | # https://github.com/jonschlinkert/markdown-toc#cli 4 | # brew install markdown-toc 5 | markdown-toc feature-flags.md > feature-flags-toc.md 6 | echo "" >> feature-flags-toc.md 7 | -------------------------------------------------------------------------------- /openraft/src/docs/internal/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("internal.md")] 2 | 3 | pub mod architecture { 4 | #![doc = include_str!("architecture.md")] 5 | } 6 | 7 | pub mod threading { 8 | #![doc = include_str!("threading.md")] 9 | } 10 | -------------------------------------------------------------------------------- /macros/tests/test_expand.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn fail() { 3 | let t = trybuild::TestCases::new(); 4 | t.compile_fail("tests/expand/fail/*.rs"); 5 | } 6 | 7 | #[test] 8 | fn pass() { 9 | macrotest::expand("tests/expand/expand/*.rs"); 10 | } 11 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_sem_ver.stderr: -------------------------------------------------------------------------------- 1 | error: `version`(`1.0.0..0`) is not valid semver. 2 | --> tests/since/fail/invalid_sem_ver.rs:1:36 3 | | 4 | 1 | #[openraft_macros::since(version = "1.0.0..0")] 5 | | ^^^^^^^^^^ 6 | -------------------------------------------------------------------------------- /openraft/src/compat/mod.rs: -------------------------------------------------------------------------------- 1 | //! This mod is an upgrade helper that provides functionalities for a newer openraft application to 2 | //! read data written by an older application. 3 | 4 | mod upgrade; 5 | 6 | pub use upgrade::Compat; 7 | pub use upgrade::Upgrade; 8 | -------------------------------------------------------------------------------- /openraft/src/progress/progress_stats.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone, Debug, Default)] 2 | #[derive(PartialEq, Eq)] 3 | pub(crate) struct ProgressStats { 4 | pub(crate) update_count: u64, 5 | pub(crate) move_count: u64, 6 | pub(crate) is_quorum_count: u64, 7 | } 8 | -------------------------------------------------------------------------------- /macros/tests/expand/expand-unstable/type_param.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (T, ATTR, V) => {ATTR type T = V;}, 5 | (Responder, , crate::impls::OneshotResponder where T: Send,), 6 | ); 7 | } 8 | -------------------------------------------------------------------------------- /scripts/check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | cargo fmt 6 | cargo test --lib 7 | cargo test --test '*' 8 | cargo clippy --no-deps --all-targets -- -D warnings 9 | RUSTDOCFLAGS="-D warnings" cargo doc --document-private-items --all --no-deps 10 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/not_keyed_let_dup_a.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | (b, String, "foo".to_string()), 7 | (a, u32, 2), 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /openraft/src/docs/internal/internal.md: -------------------------------------------------------------------------------- 1 | # Openraft internal design and implementation 2 | 3 | - [Architecture](`crate::docs::internal::architecture`) : The overall architecture of Openraft. 4 | - [Threading](`crate::docs::internal::threading`) : The threading model of Openraft. 5 | -------------------------------------------------------------------------------- /stores/memstore/README.md: -------------------------------------------------------------------------------- 1 | # openraft-memstore 2 | 3 | This is an in-memory example `RaftLogStorage` and `RaftStateMachine` implementation based on [openraft](https://github.com/databendlabs/openraft/). 4 | 5 | This crate is built mainly for testing or demonstrating purpose.:) 6 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/arg0_invalid_keyed.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !FOO, 4 | (K, T, V) => {K; T; V;}, 5 | ); 6 | 7 | openraft_macros::expand!( 8 | FOO, 9 | (K, T, V) => {K; T; V;}, 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /benchmarks/minimal/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(unused_crate_dependencies)] 2 | #![deny(unused_qualifications)] 3 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 4 | 5 | // Used by bin/bench.rs 6 | use clap as _; 7 | use maplit as _; 8 | 9 | pub mod network; 10 | pub mod store; 11 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/keyed_let_dup_a.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | KEYED, 4 | (K, T, V) => {let K: T = V;}, 5 | (a, u64, 1), 6 | (b, String, "foo".to_string()), 7 | (a, u32, 2), // duplicate `a` will be ignored 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Directory Ignores ########################################################## 3 | guide/book 4 | target 5 | vendor 6 | .idea 7 | tests/_log 8 | 9 | # File Ignores ############################################################### 10 | **/*.rs.bk 11 | Cargo.lock 12 | .DS_Store 13 | .vscode 14 | -------------------------------------------------------------------------------- /macros/tests/expand/expand/variable_type_attributes.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | openraft_macros::expand!( 3 | !KEYED, 4 | (K, M, T) => {M let K: T;}, 5 | (c, , u8, ), 6 | (c, #[allow(dead_code)] , u16), 7 | (c, #[allow(dead_code)] #[allow(dead_code)] , u16), 8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /macros/tests/expand/fail/arg0_invalid_keyed.stderr: -------------------------------------------------------------------------------- 1 | error: Expected KEYED 2 | --> tests/expand/fail/arg0_invalid_keyed.rs:3:10 3 | | 4 | 3 | !FOO, 5 | | ^^^ 6 | 7 | error: Expected KEYED 8 | --> tests/expand/fail/arg0_invalid_keyed.rs:8:9 9 | | 10 | 8 | FOO, 11 | | ^^^ 12 | -------------------------------------------------------------------------------- /macros/tests/since/fail/invalid_date.stderr: -------------------------------------------------------------------------------- 1 | error: `date`(`2021-01--`) is not valid date string. Expected format: yyyy-mm-dd 2 | --> tests/since/fail/invalid_date.rs:1:52 3 | | 4 | 1 | #[openraft_macros::since(version = "1.0.0", date = "2021-01--")] 5 | | ^^^^^^^^^^^ 6 | -------------------------------------------------------------------------------- /openraft/src/engine/handler/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod establish_handler; 2 | pub(crate) mod following_handler; 3 | pub(crate) mod leader_handler; 4 | pub(crate) mod log_handler; 5 | pub(crate) mod replication_handler; 6 | pub(crate) mod server_state_handler; 7 | pub(crate) mod snapshot_handler; 8 | pub(crate) mod vote_handler; 9 | -------------------------------------------------------------------------------- /stores/README.md: -------------------------------------------------------------------------------- 1 | Example Storage implementations. 2 | 3 | - `memstore` is in-memory storage and is used by the test cases `./tests`. 4 | 5 | If a crate has different feature flags enabled, it must not be members of the workspace. 6 | A feature flag will be enabled for the entire workspace if a member crate enables it. 7 | -------------------------------------------------------------------------------- /macros/tests/expand/expand-unstable/type_param.expanded.rs: -------------------------------------------------------------------------------- 1 | #![feature(prelude_import)] 2 | #[prelude_import] 3 | use std::prelude::rust_2024::*; 4 | #[macro_use] 5 | extern crate std; 6 | fn main() { 7 | type Responder 8 | = crate::impls::OneshotResponder 9 | where 10 | T: Send; 11 | } 12 | -------------------------------------------------------------------------------- /macros/tests/since/expand/multi_since.rs: -------------------------------------------------------------------------------- 1 | #[openraft_macros::since( 2 | version = "1.1.0", 3 | date = "2021-02-01", 4 | change = "add Foo", 5 | change = "add Bar", 6 | change = "add Baz" 7 | )] 8 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01", change = "Added")] 9 | const A: i32 = 0; 10 | -------------------------------------------------------------------------------- /tests/tests/management/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_raft_config; 11 | -------------------------------------------------------------------------------- /openraft/src/vote/vote_status.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::vote::committed::CommittedVote; 3 | use crate::vote::non_committed::UncommittedVote; 4 | 5 | #[derive(Debug, Clone, PartialEq, Eq)] 6 | pub(crate) enum VoteStatus { 7 | Committed(CommittedVote), 8 | Pending(UncommittedVote), 9 | } 10 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/02-remove-node.md: -------------------------------------------------------------------------------- 1 | ### How to remove node-2 safely from a cluster `{1, 2, 3}`? 2 | 3 | Call `Raft::change_membership(btreeset!{1, 3})` to exclude node-2 from 4 | the cluster. Then wipe out node-2 data. 5 | **NEVER** modify/erase the data of any node that is still in a raft cluster, unless you know what you are doing. 6 | -------------------------------------------------------------------------------- /openraft/src/try_as_ref.rs: -------------------------------------------------------------------------------- 1 | /// Similar to [`AsRef`], it does a cheap reference-to-reference conversion, but with the ability 2 | /// to return `None` if it is unable to perform the conversion to `&T`. 3 | pub trait TryAsRef { 4 | /// Try to convert this type into a shared reference of `T`. 5 | fn try_as_ref(&self) -> Option<&T>; 6 | } 7 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_set_request.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Formatter; 3 | 4 | use crate::protobuf as pb; 5 | 6 | impl fmt::Display for pb::SetRequest { 7 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 8 | write!(f, "SetRequest {{ key: {}, value: {} }}", self.key, self.value) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /openraft/src/entry/raft_payload.rs: -------------------------------------------------------------------------------- 1 | use crate::Membership; 2 | use crate::RaftTypeConfig; 3 | /// Defines operations on an entry payload. 4 | pub trait RaftPayload 5 | where C: RaftTypeConfig 6 | { 7 | /// Return `Some(Membership)` if the entry payload contains a membership payload. 8 | fn get_membership(&self) -> Option>; 9 | } 10 | -------------------------------------------------------------------------------- /macros/README.md: -------------------------------------------------------------------------------- 1 | # openraft_macros 2 | 3 | This crate provides a set of macros to support Openraft. 4 | 5 | # Test 6 | 7 | - Macro expansions are tested with [`macrotest`] crate. 8 | - Macro compile errors are tested with [`trybuild`] crate. 9 | 10 | 11 | [`macrotest`]: https://crates.io/crates/macrotest 12 | [`trybuild`]: https://crates.io/crates/trybuild -------------------------------------------------------------------------------- /openraft/src/testing/log/mod.rs: -------------------------------------------------------------------------------- 1 | //! Suite for testing implementations of [`RaftLogStorage`] and [`RaftStateMachine`]. 2 | //! 3 | //! [`RaftLogStorage`]: crate::storage::RaftLogStorage 4 | //! [`RaftStateMachine`]: crate::storage::RaftStateMachine 5 | 6 | mod store_builder; 7 | mod suite; 8 | 9 | pub use store_builder::StoreBuilder; 10 | pub use suite::Suite; 11 | -------------------------------------------------------------------------------- /openraft/src/raft_state/vote_state_reader.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::type_config::alias::VoteOf; 3 | 4 | // TODO: remove it? 5 | /// APIs to get vote. 6 | #[allow(dead_code)] 7 | pub(crate) trait VoteStateReader 8 | where C: RaftTypeConfig 9 | { 10 | /// Get a reference to the current vote. 11 | fn vote_ref(&self) -> &VoteOf; 12 | } 13 | -------------------------------------------------------------------------------- /scripts/watch-doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Appearence order is disabled 4 | # RUSTDOCFLAGS='-Z unstable-options --sort-modules-by-appearance' cargo watch -x 'doc --document-private-items --all --no-deps' 5 | 6 | if [ ".$1" = ".-p" ]; then 7 | cargo watch -x 'doc --document-private-items --all --no-deps' 8 | else 9 | cargo watch -x 'doc --all --no-deps' 10 | fi 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "17:00" 8 | open-pull-requests-limit: 2 9 | 10 | - package-ecosystem: cargo 11 | directory: "/examples/raft-kv-memstore" 12 | schedule: 13 | interval: daily 14 | time: "17:00" 15 | open-pull-requests-limit: 2 16 | -------------------------------------------------------------------------------- /openraft/src/docs/components/mod.rs: -------------------------------------------------------------------------------- 1 | //! Components and sub systems of the openraft project. 2 | //! 3 | //! - [`Engine and Runtime`](engine_runtime) 4 | //! - [`StateMachine`](state_machine) 5 | 6 | pub mod engine_runtime { 7 | #![doc = include_str!("engine-runtime.md")] 8 | } 9 | 10 | pub mod state_machine { 11 | #![doc = include_str!("state-machine.md")] 12 | } 13 | -------------------------------------------------------------------------------- /tests/tests/elect/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_elect_compare_last_log; 11 | mod t11_elect_seize_leadership; 12 | -------------------------------------------------------------------------------- /tests/tests/log_store/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_save_committed; 11 | -------------------------------------------------------------------------------- /tests/tests/state_machine/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_total_order_apply; 11 | mod t20_state_machine_apply_membership; 12 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/07-troubleshooting/03-data-loss.md: -------------------------------------------------------------------------------- 1 | ### What will happen when data gets lost? 2 | 3 | Raft operates on the presumption that the storage medium (i.e., the disk) is 4 | secure and reliable. 5 | 6 | If this presumption is violated, e.g., the raft logs are lost or the snapshot is 7 | damaged, no predictable outcome can be assured. In other words, the resulting 8 | behavior is **undefined**. 9 | -------------------------------------------------------------------------------- /openraft/src/core/heartbeat/errors/stopped.rs: -------------------------------------------------------------------------------- 1 | use crate::core::heartbeat::errors::raft_core_closed::RaftCoreClosed; 2 | 3 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 4 | pub enum Stopped { 5 | #[error("HeartbeatWorkerStopped: {0}")] 6 | RaftCoreClosed(#[from] RaftCoreClosed), 7 | 8 | #[error("HeartbeatWorkerStopped: received shutdown signal")] 9 | ReceivedShutdown, 10 | } 11 | -------------------------------------------------------------------------------- /openraft/src/docs/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(rustdoc::redundant_explicit_links)] 2 | #![doc = include_str!("docs.md")] 3 | 4 | pub mod faq; 5 | 6 | pub mod getting_started; 7 | 8 | pub mod cluster_control; 9 | 10 | pub mod feature_flags; 11 | 12 | pub mod data; 13 | 14 | pub mod components; 15 | 16 | pub mod protocol; 17 | 18 | pub mod internal; 19 | 20 | pub mod upgrade_guide; 21 | 22 | pub mod obsolete; 23 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/mod.rs: -------------------------------------------------------------------------------- 1 | //! Implements traits for protobuf types 2 | 3 | mod impl_append_entries_request; 4 | mod impl_append_entries_response; 5 | mod impl_client_write_response; 6 | mod impl_entry; 7 | mod impl_leader_id; 8 | mod impl_log_id; 9 | mod impl_membership; 10 | mod impl_set_request; 11 | mod impl_snapshot_request; 12 | mod impl_vote; 13 | mod impl_vote_request; 14 | mod impl_vote_response; 15 | -------------------------------------------------------------------------------- /openraft/src/proposer/mod.rs: -------------------------------------------------------------------------------- 1 | //! A proposer includes the Candidate(phase-1) state and Leader(phase-2) state. 2 | 3 | pub(crate) mod candidate; 4 | pub(crate) mod leader; 5 | pub(crate) mod leader_state; 6 | 7 | pub(crate) use candidate::Candidate; 8 | pub(crate) use leader::Leader; 9 | pub(crate) use leader_state::CandidateState; 10 | pub(crate) use leader_state::LeaderQuorumSet; 11 | pub(crate) use leader_state::LeaderState; 12 | -------------------------------------------------------------------------------- /tests/tests/snapshot_building/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_build_snapshot; 8 | mod t11_snapshot_builder_control; 9 | mod t35_building_snapshot_does_not_block_append; 10 | mod t35_building_snapshot_does_not_block_apply; 11 | mod t60_snapshot_policy_never; 12 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/app.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use crate::NodeId; 4 | use crate::Raft; 5 | use crate::StateMachineStore; 6 | 7 | // Representation of an application state. This struct can be shared around to share 8 | // instances of raft, store and more. 9 | pub struct App { 10 | pub id: NodeId, 11 | pub addr: String, 12 | pub raft: Raft, 13 | pub state_machine_store: Arc, 14 | } 15 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/01-getting-started/02-single-node.md: -------------------------------------------------------------------------------- 1 | ### Are there any issues with running a single node service? 2 | 3 | Not at all. 4 | 5 | Running a cluster with just one node is a standard approach for testing or as an initial step in setting up a cluster. 6 | 7 | A single node functions exactly the same as cluster mode. 8 | It will consistently maintain the `Leader` status and never transition to `Candidate` or `Follower` states. 9 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_doc.expanded.rs: -------------------------------------------------------------------------------- 1 | /// Doc 2 | /// 3 | /// By default, `Send` bounds will be added to the trait and to the return bounds of any async 4 | /// functions defined within the trait. 5 | /// 6 | /// If the `singlethreaded` feature is enabled, the trait definition remains the same without any 7 | /// added `Send` bounds. 8 | /// 9 | /// # Example 10 | /// 11 | /// - list 12 | /// 13 | /// Since: 1.0.0, Date(2021-01-01) 14 | const A: i32 = 0; 15 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.2.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [30058c03](https://github.com/databendlabs/openraft/commit/30058c036de06e9d0d66dd290dc75cf06831e12e) #424 wrong range when searching for membership entries: `[end-step, end)`.; by 张炎泼; 2022-07-03 4 | 5 | The iterating range searching for membership log entries should be 6 | `[end-step, end)`, not `[start, end)`. 7 | With this bug it will return duplicated membership entries. 8 | 9 | - Bug: #424 10 | -------------------------------------------------------------------------------- /openraft/src/error/higher_vote.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::type_config::alias::VoteOf; 3 | 4 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 5 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 6 | #[error("seen a higher vote: {higher} GT mine: {sender_vote}")] 7 | pub(crate) struct HigherVote { 8 | pub(crate) higher: VoteOf, 9 | pub(crate) sender_vote: VoteOf, 10 | } 11 | -------------------------------------------------------------------------------- /macros/tests/since/expand/with_doc.rs: -------------------------------------------------------------------------------- 1 | /// Doc 2 | /// 3 | /// By default, `Send` bounds will be added to the trait and to the return bounds of any async 4 | /// functions defined within the trait. 5 | /// 6 | /// If the `singlethreaded` feature is enabled, the trait definition remains the same without any 7 | /// added `Send` bounds. 8 | /// 9 | /// # Example 10 | /// 11 | /// - list 12 | #[openraft_macros::since(version = "1.0.0", date = "2021-01-01")] 13 | const A: i32 = 0; 14 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/01-getting-started/03-initialize-multiple-nodes.md: -------------------------------------------------------------------------------- 1 | ### Can I call `initialize()` on multiple nodes? 2 | 3 | Calling `initialize()` on multiple nodes with **identical configuration** is 4 | acceptable and will not cause consistency issues — the Raft voting protocol 5 | ensures that only one leader will be elected. 6 | 7 | However, calling `initialize()` with **different configurations** on different 8 | nodes may lead to a split-brain condition and must be avoided. 9 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/02-core-concepts/02-log-id-tuple.md: -------------------------------------------------------------------------------- 1 | ### Why is log id a tuple of `(term, node_id, log_index)`? 2 | 3 | In standard Raft log id is `(term, log_index)`, in Openraft he log id `(term, 4 | node_id, log_index)` is used to minimize the chance of election conflicts. 5 | This way in every term there could be more than one leader elected, and the last one is valid. 6 | See: [`leader-id`](`crate::docs::data::leader_id`) for details. 7 | 8 | [`leader-id`]: `crate::docs::data::leader_id` 9 | -------------------------------------------------------------------------------- /openraft/src/proposer/leader_state.rs: -------------------------------------------------------------------------------- 1 | use crate::proposer::Candidate; 2 | use crate::proposer::Leader; 3 | use crate::quorum::Joint; 4 | use crate::type_config::alias::NodeIdOf; 5 | 6 | /// The quorum set type used by `Leader`. 7 | pub(crate) type LeaderQuorumSet = Joint, Vec>, Vec>>>; 8 | 9 | pub(crate) type LeaderState = Option>>>; 10 | pub(crate) type CandidateState = Option>>; 11 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_log_id.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::LogId; 3 | 4 | impl From for pb::LogId { 5 | fn from(log_id: LogId) -> Self { 6 | pb::LogId { 7 | term: *log_id.committed_leader_id(), 8 | index: log_id.index(), 9 | } 10 | } 11 | } 12 | 13 | impl From for LogId { 14 | fn from(proto_log_id: pb::LogId) -> Self { 15 | LogId::new(proto_log_id.term, proto_log_id.index) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/tests/replication/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_append_entries_partial_success; 8 | mod t50_append_entries_backoff; 9 | mod t50_append_entries_backoff_rejoin; 10 | mod t60_feature_loosen_follower_log_revert; 11 | mod t61_allow_follower_log_revert; 12 | mod t62_follower_clear_restart_recover; 13 | mod t99_issue_1500_heartbeat_cause_reversion_panic; 14 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/app.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::sync::Arc; 3 | 4 | use openraft::Config; 5 | use tokio::sync::RwLock; 6 | 7 | use crate::typ::Raft; 8 | use crate::NodeId; 9 | 10 | // Representation of an application state. This struct can be shared around to share 11 | // instances of raft, store and more. 12 | pub struct App { 13 | pub id: NodeId, 14 | pub addr: String, 15 | pub raft: Raft, 16 | pub key_values: Arc>>, 17 | pub config: Arc, 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/core/sm/mod.rs: -------------------------------------------------------------------------------- 1 | //! State machine worker and its supporting types. 2 | //! 3 | //! This worker runs in a separate task and is the only one that can mutate the state machine. 4 | //! It is responsible for applying log entries, building/receiving snapshot and sending responses 5 | //! to the RaftCore. 6 | 7 | pub(crate) mod command; 8 | pub(crate) mod handle; 9 | pub(crate) mod response; 10 | pub(crate) mod worker; 11 | 12 | pub(crate) use command::Command; 13 | pub(crate) use response::CommandResult; 14 | pub(crate) use response::Response; 15 | -------------------------------------------------------------------------------- /openraft/src/docs/obsolete/mod.rs: -------------------------------------------------------------------------------- 1 | //! Obsolete Designs 2 | //! 3 | //! Several designs in Openraft have been discarded due to the problems they caused for 4 | //! applications. These designs were attempts at optimization or simplification, but ultimately 5 | //! proved to be inappropriate. They are included in this chapter as an archive to explain why they 6 | //! were discarded. 7 | 8 | pub mod heartbeat { 9 | #![doc = include_str!("blank-log-heartbeat.md")] 10 | } 11 | 12 | pub mod fast_commit { 13 | #![doc = include_str!("fast_commit.md")] 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/gtp-translate.yml: -------------------------------------------------------------------------------- 1 | name: GPT refine markdown 2 | 3 | on: 4 | issue_comment: 5 | types: [ created ] 6 | 7 | jobs: 8 | gpt_translate: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Run GPT Translate 15 | if: | 16 | contains(github.event.comment.body, '/gpt-translate') || 17 | contains(github.event.comment.body, '/gt') 18 | 19 | uses: drmingdrmer/gpt-refine-md@master 20 | with: 21 | apikey: ${{ secrets.OPENAI_API_KEY }} 22 | -------------------------------------------------------------------------------- /change-log/v0.7.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [ea696474](https://github.com/databendlabs/openraft/commit/ea696474191b82069fae465bb064a2e599537ede) add feature-flag: `bt` enables backtrace; by 张炎泼; 2022-03-12 4 | 5 | `--features bt` enables backtrace when generating errors. 6 | By default errors does not contain backtrace info. 7 | 8 | Thus openraft can be built on stable rust by default. 9 | 10 | To use on stable rust with backtrace, set `RUSTC_BOOTSTRAP=1`, e.g.: 11 | ``` 12 | RUSTUP_TOOLCHAIN=stable RUSTC_BOOTSTRAP=1 make test 13 | ``` 14 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/proto/app_types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package openraftpb; 4 | 5 | // SetRequest represents a key-value pair to be stored 6 | message SetRequest { 7 | string key = 1; // Key to store 8 | string value = 2; // Value to associate with the key 9 | } 10 | 11 | // GetRequest represents a key lookup request 12 | message GetRequest { 13 | string key = 1; // Key to look up 14 | } 15 | 16 | // GetResponse contains the value associated with the requested key 17 | message Response { 18 | optional string value = 1; // Retrieved value 19 | } 20 | -------------------------------------------------------------------------------- /openraft/src/docs/cluster_control/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("cluster-control.md")] 2 | 3 | pub mod cluster_formation { 4 | #![doc = include_str!("cluster-formation.md")] 5 | } 6 | 7 | pub mod dynamic_membership { 8 | #![doc = include_str!("dynamic-membership.md")] 9 | } 10 | 11 | pub mod joint_consensus { 12 | #![doc = include_str!("joint-consensus.md")] 13 | } 14 | 15 | pub mod node_lifecycle { 16 | #![doc = include_str!("node-lifecycle.md")] 17 | } 18 | 19 | pub mod monitoring_maintenance { 20 | #![doc = include_str!("monitoring-maintenance.md")] 21 | } 22 | -------------------------------------------------------------------------------- /openraft/src/engine/command_kind.rs: -------------------------------------------------------------------------------- 1 | /// Command kind is used to categorize commands. 2 | /// 3 | /// Commands of different kinds can be parallelized. 4 | #[allow(dead_code)] 5 | #[derive(Debug, Clone, Copy)] 6 | #[derive(PartialEq, Eq)] 7 | pub(crate) enum CommandKind { 8 | /// Log IO command 9 | Log, 10 | /// Network IO command 11 | Network, 12 | /// State machine IO command 13 | StateMachine, 14 | /// Command handled by RaftCore main thread. 15 | Main, 16 | /// Respond to caller. Can be executed in parallel with other commands. 17 | Respond, 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/tokio_impls/tokio_runtime/mutex.rs: -------------------------------------------------------------------------------- 1 | use std::future::Future; 2 | 3 | use crate::OptionalSend; 4 | use crate::async_runtime::mutex; 5 | 6 | pub type TokioMutex = tokio::sync::Mutex; 7 | 8 | impl mutex::Mutex for TokioMutex 9 | where T: OptionalSend + 'static 10 | { 11 | type Guard<'a> = tokio::sync::MutexGuard<'a, T>; 12 | 13 | fn new(value: T) -> Self { 14 | TokioMutex::new(value) 15 | } 16 | 17 | fn lock(&self) -> impl Future> + OptionalSend { 18 | self.lock() 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/vote/leader_id/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod leader_id_adv; 2 | pub mod leader_id_std; 3 | 4 | pub(crate) mod leader_id_cmp; 5 | pub(crate) mod raft_committed_leader_id; 6 | pub(crate) mod raft_leader_id; 7 | 8 | #[cfg(feature = "single-term-leader")] 9 | compile_error!( 10 | r#"`single-term-leader` is removed. 11 | To enable standard Raft mode: 12 | - either add `LeaderId = openraft::impls::leader_id_std::LeaderId` to `declare_raft_types!(YourTypeConfig)` statement, 13 | - or add `type LeaderId: opernaft::impls::leader_id_std::LeaderId` to the `RaftTypeConfig` implementation."# 14 | ); 15 | -------------------------------------------------------------------------------- /openraft/src/entry/raft_entry_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::entry::RaftEntry; 3 | use crate::log_id::ref_log_id::RefLogId; 4 | 5 | pub(crate) trait RaftEntryExt: RaftEntry 6 | where C: RaftTypeConfig 7 | { 8 | /// Returns a lightweight [`RefLogId`] that contains the log id information. 9 | fn ref_log_id(&self) -> RefLogId<'_, C> { 10 | let (leader_id, index) = self.log_id_parts(); 11 | RefLogId::new(leader_id, index) 12 | } 13 | } 14 | 15 | impl RaftEntryExt for T 16 | where 17 | C: RaftTypeConfig, 18 | T: RaftEntry, 19 | { 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mpsc/send_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by the `Sender`. 4 | #[derive(PartialEq, Eq, Clone, Copy)] 5 | pub struct SendError(pub T); 6 | 7 | impl fmt::Debug for SendError { 8 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 9 | f.debug_struct("SendError").finish_non_exhaustive() 10 | } 11 | } 12 | 13 | impl fmt::Display for SendError { 14 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 15 | write!(fmt, "channel closed") 16 | } 17 | } 18 | 19 | impl std::error::Error for SendError {} 20 | -------------------------------------------------------------------------------- /openraft/src/replication/event_watcher.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::raft_state::IOId; 3 | use crate::replication::replicate::Replicate; 4 | use crate::type_config::alias::LogIdOf; 5 | use crate::type_config::alias::WatchReceiverOf; 6 | 7 | #[derive(Clone)] 8 | pub(crate) struct EventWatcher 9 | where C: RaftTypeConfig 10 | { 11 | pub(crate) replicate_rx: WatchReceiverOf>, 12 | pub(crate) committed_rx: WatchReceiverOf>>, 13 | 14 | pub(crate) io_accepted_rx: WatchReceiverOf>, 15 | pub(crate) io_submitted_rx: WatchReceiverOf>, 16 | } 17 | -------------------------------------------------------------------------------- /rt-compio/src/mutex.rs: -------------------------------------------------------------------------------- 1 | use std::future::Future; 2 | 3 | use openraft::type_config::async_runtime::mutex; 4 | use openraft::OptionalSend; 5 | 6 | pub struct FlumeMutex(futures::lock::Mutex); 7 | 8 | impl mutex::Mutex for FlumeMutex 9 | where T: OptionalSend + 'static 10 | { 11 | type Guard<'a> = futures::lock::MutexGuard<'a, T>; 12 | 13 | #[inline] 14 | fn new(value: T) -> Self { 15 | FlumeMutex(futures::lock::Mutex::new(value)) 16 | } 17 | 18 | #[inline] 19 | fn lock(&self) -> impl Future> + OptionalSend { 20 | self.0.lock() 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/05-monitoring/04-minimize-error-logs.md: -------------------------------------------------------------------------------- 1 | ### How to minimize error logging when a follower is offline 2 | 3 | Excessive error logging, like `ERROR openraft::replication: 248: RPCError err=NetworkError: ...`, occurs when a follower node becomes unresponsive. To alleviate this, implement a mechanism within [`RaftNetwork`][] that returns a [`Unreachable`][] error instead of a [`NetworkError`][] when immediate replication retries to the affected node are not advised. 4 | 5 | [`RaftNetwork`]: `crate::network::RaftNetwork` 6 | [`Unreachable`]: `crate::error::Unreachable` 7 | [`NetworkError`]: `crate::error::NetworkError` 8 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/01-node-restart.md: -------------------------------------------------------------------------------- 1 | ### What actions are required when a node restarts? 2 | 3 | None. No calls, e.g., to either [`add_learner()`][] or [`change_membership()`][] 4 | are necessary. 5 | 6 | Openraft maintains the membership configuration in [`Membership`][] for all 7 | nodes in the cluster, including voters and non-voters (learners). When a 8 | `follower` or `learner` restarts, the leader will automatically re-establish 9 | replication. 10 | 11 | [`add_learner()`]: `crate::Raft::add_learner` 12 | [`change_membership()`]: `crate::Raft::change_membership` 13 | [`Membership`]: `crate::Membership` 14 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.6.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [82a3f2f9](https://github.com/databendlabs/openraft/commit/82a3f2f9c7ac37a0f24c6e0e8993c8d3bcee5666) use LogId to track last applied instead of using just an index.; by drdr xp; 2021-07-19 4 | 5 | It provides more info by Using LogId to track last applied log. 6 | E.g. when creating a snapshot, it need to walk through logs to find the 7 | term of the last applied log, just like it did in memstore impl. 8 | 9 | Using LogId{term, index} is a more natural way in every aspect. 10 | 11 | changes: RaftCore: change type of `last_applied` from u64 to LogId. 12 | -------------------------------------------------------------------------------- /openraft/src/docs/protocol/replication.md: -------------------------------------------------------------------------------- 1 | # Replication 2 | 3 | Appending entries is indeed the primary RPC for replicating logs from the leader to followers or learners in the Raft consensus algorithm. 4 | Installing a snapshot can be considered a special form of **appending logs** since it serves a similar purpose: 5 | ensuring that all nodes have a consistent state, particularly when log entries become too numerous or when a node has fallen far behind. 6 | 7 | - [Replication by Append-Entries](`crate::docs::protocol::replication::log_replication`). 8 | - [Replication by Snapshot](`crate::docs::protocol::replication::snapshot_replication`). 9 | -------------------------------------------------------------------------------- /openraft/src/replication/snapshot_transmitter_handle.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::type_config::alias::JoinHandleOf; 3 | use crate::type_config::alias::WatchSenderOf; 4 | 5 | /// Handle to a running `SnapshotTransmitter` task. 6 | /// 7 | /// Dropping this handle cancels the snapshot transmission. 8 | pub(crate) struct SnapshotTransmitterHandle 9 | where C: RaftTypeConfig 10 | { 11 | /// The spawn handle of the `SnapshotTransmitter` task. 12 | pub(crate) _join_handle: JoinHandleOf, 13 | 14 | /// Dropping this sender signals the task to cancel. 15 | pub(crate) _tx_cancel: WatchSenderOf, 16 | } 17 | -------------------------------------------------------------------------------- /openraft/src/log_id/option_ref_log_id_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::log_id::ref_log_id::RefLogId; 3 | use crate::type_config::alias::LogIdOf; 4 | 5 | pub(crate) trait OptionRefLogIdExt 6 | where C: RaftTypeConfig 7 | { 8 | /// Creates a new owned [`LogId`] from the reference log ID. 9 | /// 10 | /// [`LogId`]: crate::log_id::LogId 11 | fn to_log_id(&self) -> Option>; 12 | } 13 | 14 | impl OptionRefLogIdExt for Option> 15 | where C: RaftTypeConfig 16 | { 17 | fn to_log_id(&self) -> Option> { 18 | self.as_ref().map(|r| r.into_log_id()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/error/replication_closed.rs: -------------------------------------------------------------------------------- 1 | /// Replication is closed intentionally. 2 | /// 3 | /// No further replication action should be taken. 4 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 5 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 6 | #[error("Replication is closed: {reason}")] 7 | pub struct ReplicationClosed { 8 | reason: String, 9 | } 10 | 11 | impl ReplicationClosed { 12 | /// Create a new ReplicationClosed error with the given reason. 13 | pub fn new(reason: impl ToString) -> Self { 14 | Self { 15 | reason: reason.to_string(), 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/storage/log_state.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::type_config::alias::LogIdOf; 3 | 4 | /// The state about logs. 5 | /// 6 | /// Invariance: last_purged_log_id <= last_applied <= last_log_id 7 | #[derive(Clone, Debug, Default, PartialEq, Eq)] 8 | pub struct LogState { 9 | /// The greatest log id that has been purged after being applied to state machine. 10 | pub last_purged_log_id: Option>, 11 | 12 | /// The log id of the last present entry if there are any entries. 13 | /// Otherwise, the same value as `last_purged_log_id`. 14 | pub last_log_id: Option>, 15 | } 16 | -------------------------------------------------------------------------------- /openraft/src/core/core_state.rs: -------------------------------------------------------------------------------- 1 | use crate::LogId; 2 | use crate::RaftTypeConfig; 3 | #[cfg(doc)] 4 | use crate::core::RaftCore; 5 | 6 | /// State for [`RaftCore`] that does not directly affect consensus. 7 | /// 8 | /// Handles behavior not in [`Engine`](crate::engine::Engine), such as snapshot triggering and log 9 | /// purging. 10 | #[derive(Debug, Default, Clone)] 11 | pub(crate) struct CoreState 12 | where C: RaftTypeConfig 13 | { 14 | /// LogId of the last snapshot attempt. 15 | /// 16 | /// Prevents repeated attempts when the state machine declines to build a snapshot. 17 | pub(crate) snapshot_tried_at: Option>, 18 | } 19 | -------------------------------------------------------------------------------- /openraft/src/docs/upgrade_guide/mod.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("upgrade.md")] 2 | 3 | pub mod upgrade_06_07 { 4 | //! See: [Guide for upgrading v0.6 to v0.7](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_06_07) 5 | } 6 | pub mod upgrade_07_08 { 7 | //! See: [Guide for upgrading v0.7 to v0.8](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_07_08) 8 | } 9 | pub mod upgrade_083_084 { 10 | //! See: [Guide for upgrading v0.8.3 to v0.8.4](https://docs.rs/openraft/0.8.9/openraft/docs/upgrade_guide/upgrade_083_084) 11 | } 12 | pub mod upgrade_08_09 { 13 | #![doc = include_str!("upgrade-v08-v09.md")] 14 | } 15 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_snapshot_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb::snapshot_request::Payload; 2 | use crate::protobuf as pb; 3 | impl pb::SnapshotRequest { 4 | pub fn into_meta(self) -> Option { 5 | let p = self.payload?; 6 | match p { 7 | Payload::Meta(meta) => Some(meta), 8 | Payload::Chunk(_) => None, 9 | } 10 | } 11 | 12 | pub fn into_data_chunk(self) -> Option> { 13 | let p = self.payload?; 14 | match p { 15 | Payload::Meta(_) => None, 16 | Payload::Chunk(chunk) => Some(chunk), 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /rt-monoio/src/mutex.rs: -------------------------------------------------------------------------------- 1 | //! Mutex wrapper type and its trait impl. 2 | 3 | use std::future::Future; 4 | 5 | use openraft::type_config::async_runtime::mutex; 6 | use openraft::OptionalSend; 7 | 8 | pub struct TokioMutex(tokio::sync::Mutex); 9 | 10 | impl mutex::Mutex for TokioMutex 11 | where T: OptionalSend + 'static 12 | { 13 | type Guard<'a> = tokio::sync::MutexGuard<'a, T>; 14 | 15 | #[inline] 16 | fn new(value: T) -> Self { 17 | TokioMutex(tokio::sync::Mutex::new(value)) 18 | } 19 | 20 | #[inline] 21 | fn lock(&self) -> impl Future> + OptionalSend { 22 | self.0.lock() 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /openraft/src/quorum/joint_impl.rs: -------------------------------------------------------------------------------- 1 | use crate::quorum::AsJoint; 2 | use crate::quorum::Joint; 3 | use crate::quorum::QuorumSet; 4 | 5 | /// Use a vec of some implementation of `QuorumSet` as a joint quorum set. 6 | impl<'d, ID, QS> AsJoint<'d, ID, QS, &'d [QS]> for Vec 7 | where 8 | ID: 'static, 9 | QS: QuorumSet, 10 | { 11 | fn as_joint(&'d self) -> Joint 12 | where &'d [QS]: 'd { 13 | Joint::new(self) 14 | } 15 | } 16 | 17 | impl From> for Joint> 18 | where 19 | ID: 'static, 20 | QS: QuorumSet, 21 | { 22 | fn from(v: Vec) -> Self { 23 | Joint::new(v) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /openraft/src/docs/feature_flags/feature-flags-toc.md: -------------------------------------------------------------------------------- 1 | - [feature-flag `adapt-network-v1`](#feature-flag-adapt-network-v1) 2 | - [feature-flag `bench`](#feature-flag-bench) 3 | - [feature-flag `bt`](#feature-flag-bt) 4 | - [feature-flag `compat`](#feature-flag-compat) 5 | - [feature-flag `runtime-stats`](#feature-flag-runtime-stats) 6 | - [feature-flag `serde`](#feature-flag-serde) 7 | - [feature-flag `single-term-leader`](#feature-flag-single-term-leader) 8 | - [feature-flag `singlethreaded`](#feature-flag-singlethreaded) 9 | - [feature-flag `tokio-rt`](#feature-flag-tokio-rt) 10 | - [feature-flag `tracing-log`](#feature-flag-tracing-log) 11 | - [feature-flag `type-alias`](#feature-flag-type-alias) 12 | -------------------------------------------------------------------------------- /openraft/src/network/rpc_type.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Types of RPC requests in the Raft protocol. 4 | #[derive(Debug, Clone, Copy)] 5 | #[derive(PartialEq, Eq)] 6 | #[derive(Hash)] 7 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 8 | pub enum RPCTypes { 9 | /// Vote request RPC. 10 | Vote, 11 | /// AppendEntries request RPC. 12 | AppendEntries, 13 | /// InstallSnapshot request RPC. 14 | InstallSnapshot, 15 | /// TransferLeader request RPC. 16 | TransferLeader, 17 | } 18 | 19 | impl fmt::Display for RPCTypes { 20 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 21 | write!(f, "{:?}", self) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tests/tests/life_cycle/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_initialization; 11 | mod t11_shutdown; 12 | mod t50_follower_restart_does_not_interrupt; 13 | mod t50_leader_restart_clears_state; 14 | mod t50_single_follower_restart; 15 | mod t50_single_leader_restart_re_apply_logs; 16 | mod t90_issue_607_single_restart; 17 | mod t90_issue_881_transient_state_machine; 18 | mod t90_issue_920_non_voter_leader_restart; 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | authors = ["databendlabs"] 3 | language = "en" 4 | multilingual = false 5 | src = "guide/src" 6 | title = "openraft" 7 | description = "The openraft user guide." 8 | 9 | [build] 10 | build-dir = "guide/book" 11 | create-missing = false 12 | 13 | [preprocessor.svgbob] 14 | text_width = 8.0 15 | text_height = 16.0 16 | class = "bob" 17 | font_family = "arial" 18 | font_size = 14.0 19 | stroke_width = 2.0 20 | # there's using css-variables from theme: 21 | stroke_color = "var(--fg)" # see default theme / variables.css 22 | background_color = "transparent" # also useful `var(--bg)` 23 | # all properties are optional. 24 | 25 | 26 | [output.html] 27 | additional-css = ["guide/src/custom.css"] 28 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::VoteRequest; 3 | 4 | impl From for pb::VoteRequest { 5 | fn from(vote_req: VoteRequest) -> Self { 6 | pb::VoteRequest { 7 | vote: Some(vote_req.vote), 8 | last_log_id: vote_req.last_log_id.map(|log_id| log_id.into()), 9 | } 10 | } 11 | } 12 | 13 | impl From for VoteRequest { 14 | fn from(proto_vote_req: pb::VoteRequest) -> Self { 15 | let vote = proto_vote_req.vote.unwrap(); 16 | let last_log_id = proto_vote_req.last_log_id.map(|log_id| log_id.into()); 17 | VoteRequest::new(vote, last_log_id) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /scripts/mprocs-check.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env mprocs --config 2 | 3 | 4 | # run local check in parallel with mprocs 5 | # 6 | # Usage: 7 | # mprocs --config ./scripts/check.yaml 8 | # 9 | # Install: 10 | # cargo install mprocs 11 | # 12 | # See: https://github.com/pvolok/mprocs 13 | 14 | 15 | procs: 16 | test-lib: 17 | cmd: ["cargo", "test", "--lib"] 18 | it: 19 | cmd: ["cargo", "test", "--test", "*"] 20 | clippy: 21 | cmd: ["cargo", "clippy", "--no-deps", "--all-targets", "--", "-D", "warnings"] 22 | 23 | # # keeps examples: 24 | # xx: 25 | # shell: "nodemon server.js" 26 | # webpack: "webpack serve" 27 | # tests: 28 | # shell: "jest -w" 29 | # env: 30 | # NODE_ENV: test 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request for documentation 3 | about: Suggest more doc to explain something. 4 | title: '' 5 | labels: 'documentation' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What kind of doc do you ask for?** 11 | - [ ] Code comments. 12 | - [ ] Architecture design. 13 | - [ ] Algorithm explanation. 14 | - [ ] Guide. 15 | - [ ] Examples. 16 | 17 | **Describe the information you want** 18 | A clear and concise description of what you want. 19 | 20 | **Describe the status of the current doc** 21 | - Is there any doc yet?: 22 | - doc-1: [e.g. url] 23 | - doc-2: [e.g. url] 24 | - What other information should be added? 25 | - [e.g. ] 26 | 27 | - [ ] I'll open a pull request for it:) 28 | -------------------------------------------------------------------------------- /tests/tests/metrics/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_current_leader; 11 | mod t10_leader_last_ack; 12 | mod t10_purged; 13 | mod t10_server_metrics_and_data_metrics; 14 | mod t20_metrics_state_machine_consistency; 15 | mod t30_leader_metrics; 16 | mod t40_metrics_wait; 17 | mod t50_apply_progress_api; 18 | mod t50_commit_progress_api; 19 | mod t50_log_progress_api; 20 | mod t50_snapshot_progress_api; 21 | mod t50_watch_leader_api; 22 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mutex.rs: -------------------------------------------------------------------------------- 1 | //! Async mutex trait. 2 | 3 | use std::future::Future; 4 | use std::ops::DerefMut; 5 | 6 | use crate::OptionalSend; 7 | use crate::OptionalSync; 8 | 9 | /// Represents an implementation of an asynchronous Mutex. 10 | pub trait Mutex: OptionalSend + OptionalSync { 11 | /// Handle to an acquired lock, should release it when dropped. 12 | type Guard<'a>: DerefMut + OptionalSend 13 | where Self: 'a; 14 | 15 | /// Creates a new lock. 16 | #[track_caller] 17 | fn new(value: T) -> Self; 18 | 19 | /// Locks this Mutex. 20 | #[track_caller] 21 | fn lock(&self) -> impl Future> + OptionalSend; 22 | } 23 | -------------------------------------------------------------------------------- /multiraft/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-multi" 3 | description = "Multi-Raft adapters for connection sharing across Raft groups" 4 | documentation = "https://docs.rs/openraft-multiraft" 5 | readme = "README.md" 6 | version = "0.10.0" 7 | edition = "2021" 8 | authors = ["Databend Authors "] 9 | categories = ["network", "asynchronous", "data-structures"] 10 | homepage = "https://github.com/databendlabs/openraft" 11 | keywords = ["consensus", "raft", "multi-raft"] 12 | license = "MIT OR Apache-2.0" 13 | repository = "https://github.com/databendlabs/openraft" 14 | 15 | [dependencies] 16 | openraft = { path = "../openraft", version = "0.10.0", default-features = false } 17 | anyerror = { version = "0.1" } 18 | 19 | -------------------------------------------------------------------------------- /openraft/src/error/allow_next_revert_error.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::error::ForwardToLeader; 3 | use crate::error::NodeNotFound; 4 | 5 | /// Error related to setting the allow_next_revert flag. 6 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 7 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 8 | pub enum AllowNextRevertError { 9 | /// The target node was not found. 10 | #[error("cannot set allow_next_revert; error: {0}")] 11 | NodeNotFound(#[from] NodeNotFound), 12 | /// Request must be forwarded to the leader. 13 | #[error("cannot set allow_next_revert; error: {0}")] 14 | ForwardToLeader(#[from] ForwardToLeader), 15 | } 16 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | queue_rules: 2 | - name: feature_queue 3 | conditions: 4 | # - '#check-pending=0' 5 | - '#check-success>=2' 6 | - check-success=check-subject 7 | - check-success=openraft-test (stable, 0) 8 | - check-success~=openraft-test 9 | 10 | pull_request_rules: 11 | 12 | - name: put into queue if approved 13 | conditions: 14 | - "#approved-reviews-by>=1" 15 | - "#changes-requested-reviews-by=0" 16 | - check-success=check-subject 17 | - check-success=openraft-test (stable, 0) 18 | actions: 19 | queue: 20 | name: feature_queue 21 | 22 | - name: Delete head branch after merge 23 | conditions: 24 | - merged 25 | actions: 26 | delete_head_branch: 27 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.14.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [eee8e534](https://github.com/databendlabs/openraft/commit/eee8e534e0b0b9abdb37dd94aeb64dc1affd3ef7) snapshot replication does not need to send a last 0 size chunk; by drdr xp; 2021-08-22 4 | 5 | - Fixed: [8cd24ba0](https://github.com/databendlabs/openraft/commit/8cd24ba0f0212e94e61f21a7be0ce0806fcc66d5) RaftCore.entries_cache is inconsistent with storage. removed it.; by drdr xp; 2021-08-23 6 | 7 | - When leader changes, `entries_cache` is cleared. 8 | Thus there may be cached entries wont be applied to state machine. 9 | 10 | - When applying finished, the applied entries are not removed from the 11 | cache. 12 | Thus there could be entries being applied more than once. 13 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/test_store.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | 6 | use crate::store::LogStore; 7 | use crate::store::StateMachineStore; 8 | use crate::typ::*; 9 | use crate::TypeConfig; 10 | 11 | struct MemKVStoreBuilder {} 12 | 13 | impl StoreBuilder, ()> for MemKVStoreBuilder { 14 | async fn build(&self) -> Result<((), LogStore, Arc), StorageError> { 15 | Ok(((), LogStore::default(), Arc::default())) 16 | } 17 | } 18 | 19 | #[tokio::test] 20 | pub async fn test_mem_store() -> Result<(), StorageError> { 21 | Suite::test_all(MemKVStoreBuilder {}).await?; 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/05-forward-to-leader-missing.md: -------------------------------------------------------------------------------- 1 | ### Write returns `ForwardToLeader` but leader info is missing 2 | 3 | **Symptom**: [`ClientWriteError::ForwardToLeader`][] is returned but the `leader_id` field is `None` 4 | 5 | **Cause**: The current leader is no longer in the cluster's membership configuration. 6 | This occurs after a membership change that removes the leader node. 7 | 8 | **Solution**: If `leader_id` is `None`, query [`RaftMetrics::current_leader`][] to find the new 9 | leader, or retry the membership query after the new leader is elected. 10 | 11 | [`ClientWriteError::ForwardToLeader`]: `crate::error::ClientWriteError::ForwardToLeader` 12 | [`RaftMetrics::current_leader`]: `crate::metrics::RaftMetrics::current_leader` 13 | -------------------------------------------------------------------------------- /openraft/src/error/replication_error.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::StorageError; 3 | use crate::error::RPCError; 4 | use crate::error::higher_vote::HigherVote; 5 | use crate::error::replication_closed::ReplicationClosed; 6 | 7 | /// Error variants related to the Replication. 8 | #[derive(Debug, thiserror::Error)] 9 | #[allow(clippy::large_enum_variant)] 10 | pub(crate) enum ReplicationError 11 | where C: RaftTypeConfig 12 | { 13 | #[error(transparent)] 14 | HigherVote(#[from] HigherVote), 15 | 16 | #[error(transparent)] 17 | Closed(#[from] ReplicationClosed), 18 | 19 | #[error(transparent)] 20 | StorageError(#[from] StorageError), 21 | 22 | #[error(transparent)] 23 | RPCError(#[from] RPCError), 24 | } 25 | -------------------------------------------------------------------------------- /openraft/src/replication/replication_progress.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::type_config::alias::LogIdOf; 3 | 4 | /// Tracks the log replication state for a follower from the leader's perspective. 5 | #[derive(Clone, Debug)] 6 | #[derive(PartialEq, Eq)] 7 | pub(crate) struct ReplicationProgress 8 | where C: RaftTypeConfig 9 | { 10 | /// The leader's committed log id to replicate to the follower. 11 | pub(crate) local_committed: Option>, 12 | 13 | /// The last log id known to match on the follower. 14 | /// 15 | /// All logs up to and including this id are confirmed to exist on the follower. 16 | pub(crate) remote_matched: Option>, 17 | } 18 | 19 | impl ReplicationProgress where C: RaftTypeConfig {} 20 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/07-troubleshooting/06-excessive-network-errors.md: -------------------------------------------------------------------------------- 1 | ### Excessive "RPCError err=NetworkError" in logs when a node is offline 2 | 3 | **Symptom**: Continuous error logs `ERROR openraft::replication: RPCError err=NetworkError` 4 | when a follower is unreachable 5 | 6 | **Cause**: Openraft retries replication aggressively. Each failed RPC logs an error. 7 | 8 | **Solution**: In your [`RaftNetwork`][] implementation, when a node is known to be unreachable, 9 | return [`Unreachable`][] error instead of [`NetworkError`][]. Openraft backs off longer for 10 | `Unreachable` errors, reducing log spam. 11 | 12 | [`RaftNetwork`]: `crate::network::RaftNetwork` 13 | [`Unreachable`]: `crate::error::Unreachable` 14 | [`NetworkError`]: `crate::error::NetworkError` 15 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::testing::log::StoreBuilder; 4 | use openraft::testing::log::Suite; 5 | use openraft::StorageError; 6 | 7 | use crate::store::LogStore; 8 | use crate::store::StateMachineStore; 9 | use crate::TypeConfig; 10 | 11 | struct MemKVStoreBuilder {} 12 | 13 | impl StoreBuilder, ()> for MemKVStoreBuilder { 14 | async fn build(&self) -> Result<((), LogStore, Arc), StorageError> { 15 | Ok(((), LogStore::default(), Arc::default())) 16 | } 17 | } 18 | 19 | #[tokio::test] 20 | pub async fn test_mem_store() -> Result<(), StorageError> { 21 | Suite::test_all(MemKVStoreBuilder {}).await?; 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /openraft/src/quorum/mod.rs: -------------------------------------------------------------------------------- 1 | //! A quorum is a set of nodes a vote request or append-entries request has to contact to. 2 | //! The most common quorum is **majority**. 3 | //! A quorum set is a collection of quorums, e.g., the quorum set of the majority of `{a,b,c}` is 4 | //! `{a,b}, {b,c}, {a,c}`. 5 | 6 | mod coherent; 7 | mod coherent_impl; 8 | mod joint; 9 | mod joint_impl; 10 | mod quorum_set; 11 | mod quorum_set_impl; 12 | 13 | #[cfg(feature = "bench")] 14 | #[cfg(test)] 15 | mod bench; 16 | 17 | #[cfg(test)] 18 | mod coherent_test; 19 | #[cfg(test)] 20 | mod quorum_set_test; 21 | 22 | pub(crate) use coherent::Coherent; 23 | pub(crate) use coherent::FindCoherent; 24 | pub(crate) use joint::AsJoint; 25 | pub(crate) use joint::Joint; 26 | pub(crate) use quorum_set::QuorumSet; 27 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_client_write_response.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::*; 3 | 4 | impl From for ClientWriteResponse { 5 | fn from(r: pb::ClientWriteResponse) -> Self { 6 | ClientWriteResponse { 7 | log_id: r.log_id.unwrap().into(), 8 | data: r.data.unwrap(), 9 | membership: r.membership.map(|mem| mem.into()), 10 | } 11 | } 12 | } 13 | 14 | impl From for pb::ClientWriteResponse { 15 | fn from(r: ClientWriteResponse) -> Self { 16 | pb::ClientWriteResponse { 17 | log_id: Some(r.log_id.into()), 18 | data: Some(r.data), 19 | membership: r.membership.map(|mem| mem.into()), 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /openraft/src/network/backoff.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::OptionalSend; 4 | use crate::base::BoxIterator; 5 | 6 | /// A backoff instance that is an infinite iterator of durations to sleep before next retry, when a 7 | /// [`Unreachable`](`crate::error::Unreachable`) occurs. 8 | pub struct Backoff { 9 | inner: BoxIterator<'static, Duration>, 10 | } 11 | 12 | impl Backoff { 13 | /// Create a new Backoff from an iterator of durations. 14 | pub fn new(iter: impl Iterator + OptionalSend + 'static) -> Self { 15 | Self { inner: Box::new(iter) } 16 | } 17 | } 18 | 19 | impl Iterator for Backoff { 20 | type Item = Duration; 21 | 22 | fn next(&mut self) -> Option { 23 | self.inner.next() 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/tests/client_api/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // See ./README.md 9 | 10 | mod t10_client_write_many; 11 | mod t10_client_writes; 12 | mod t11_client_reads; 13 | mod t12_trigger_purge_log; 14 | mod t13_begin_receiving_snapshot; 15 | mod t13_get_snapshot; 16 | mod t13_install_full_snapshot; 17 | mod t13_trigger_snapshot; 18 | mod t14_transfer_leader; 19 | mod t15_client_write_with_twoshot; 20 | mod t16_with_raft_state; 21 | mod t16_with_state_machine; 22 | mod t20_raft_api; 23 | mod t50_lagging_network_write; 24 | mod t51_write_when_leader_quit; 25 | -------------------------------------------------------------------------------- /tests/tests/append_entries/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | // The number indicate the preferred running order for these case. 8 | // The later tests may depend on the earlier ones. 9 | 10 | mod t10_conflict_with_empty_entries; 11 | mod t10_see_higher_vote; 12 | mod t10_stream_append; 13 | mod t11_append_conflicts; 14 | mod t11_append_entries_with_bigger_term; 15 | mod t11_append_inconsistent_log; 16 | mod t11_append_updates_membership; 17 | mod t30_replication_1_voter_to_isolated_learner; 18 | mod t60_enable_heartbeat; 19 | mod t61_heartbeat_reject_vote; 20 | mod t61_large_heartbeat; 21 | mod t90_issue_216_stale_last_log_id; 22 | -------------------------------------------------------------------------------- /tests/tests/snapshot_streaming/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | #[macro_use] 4 | #[path = "../fixtures/mod.rs"] 5 | mod fixtures; 6 | 7 | mod t10_api_install_snapshot; 8 | mod t10_api_install_snapshot_with_lower_vote; 9 | mod t20_startup_snapshot; 10 | mod t30_purge_in_snapshot_logs; 11 | mod t31_snapshot_overrides_membership; 12 | mod t32_snapshot_uses_prev_snap_membership; 13 | mod t33_snapshot_delete_conflict_logs; 14 | mod t34_replication_does_not_block_purge; 15 | mod t50_snapshot_line_rate_to_snapshot; 16 | mod t50_snapshot_when_lacking_log; 17 | mod t51_after_snapshot_add_learner_and_request_a_log; 18 | mod t60_snapshot_chunk_size; 19 | mod t90_issue_808_snapshot_to_unreachable_node_should_not_block; 20 | -------------------------------------------------------------------------------- /stores/memstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use openraft::StorageError; 4 | use openraft::testing::log::StoreBuilder; 5 | use openraft::testing::log::Suite; 6 | 7 | use crate::MemLogStore; 8 | use crate::MemStateMachine; 9 | use crate::TypeConfig; 10 | 11 | struct MemStoreBuilder {} 12 | 13 | impl StoreBuilder, Arc, ()> for MemStoreBuilder { 14 | async fn build(&self) -> Result<((), Arc, Arc), StorageError> { 15 | let (log_store, sm) = crate::new_mem_store(); 16 | Ok(((), log_store, sm)) 17 | } 18 | } 19 | 20 | #[tokio::test] 21 | pub async fn test_mem_store() -> Result<(), StorageError> { 22 | Suite::test_all(MemStoreBuilder {}).await?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.13.md: -------------------------------------------------------------------------------- 1 | ### Fixed: 2 | 3 | - Fixed: [2eccb9e1](https://github.com/databendlabs/openraft/commit/2eccb9e1f82f1bf71f6a2cf9ef6da7bf6232fa84) install snapshot req with offset GE 0 should not start a new session.; by drdr xp; 2021-08-22 4 | 5 | A install-snapshot always ends with a req with data len to be 0 and 6 | offset GE 0. 7 | If such a req is re-sent, e.g., when timeout, the receiver will try to 8 | install a snapshot with empty data, if it just finished the previous 9 | install snapshot req(`snapshot_state` is None) and do not reject a 10 | install snapshot req with offset GE 0. 11 | Which results in a `fatal storage error`, since the storage tries to 12 | decode an empty snapshot data. 13 | 14 | - feature: add config `install_snapshot_timeout`. 15 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote_response.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::VoteResponse; 3 | 4 | impl From for pb::VoteResponse { 5 | fn from(vote_resp: VoteResponse) -> Self { 6 | pb::VoteResponse { 7 | vote: Some(vote_resp.vote), 8 | vote_granted: vote_resp.vote_granted, 9 | last_log_id: vote_resp.last_log_id.map(|log_id| log_id.into()), 10 | } 11 | } 12 | } 13 | 14 | impl From for VoteResponse { 15 | fn from(proto_vote_resp: pb::VoteResponse) -> Self { 16 | let vote = proto_vote_resp.vote.unwrap(); 17 | let last_log_id = proto_vote_resp.last_log_id.map(|log_id| log_id.into()); 18 | VoteResponse::new(vote, last_log_id, proto_vote_resp.vote_granted) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/vote/raft_term/mod.rs: -------------------------------------------------------------------------------- 1 | mod raft_term_impls; 2 | 3 | use std::fmt::Debug; 4 | use std::fmt::Display; 5 | 6 | use openraft_macros::since; 7 | 8 | use crate::base::OptionalFeatures; 9 | 10 | /// Type representing a Raft term number. 11 | /// 12 | /// A term is a logical clock in Raft that is used to detect obsolete information, 13 | /// such as old leaders. It must be totally ordered and monotonically increasing. 14 | /// 15 | /// Common implementations are provided for standard integer types like `u64`, `i64`, etc. 16 | #[since(version = "0.10.0")] 17 | pub trait RaftTerm 18 | where Self: OptionalFeatures + Ord + Debug + Display + Copy + Default + 'static 19 | { 20 | /// Returns the next term. 21 | /// 22 | /// Must satisfy: `self < self.next()` 23 | fn next(&self) -> Self; 24 | } 25 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/02-core-concepts/01-differences-from-raft.md: -------------------------------------------------------------------------------- 1 | ### What are the differences between Openraft and standard Raft? 2 | 3 | - Optionally, In one term there could be more than one leader to be established, in order to reduce election conflict. See: std mode and adv mode leader id: [`leader_id`][]; 4 | - Openraft stores committed log id: See: [`RaftLogStorage::save_committed()`][]; 5 | - Openraft optimized `ReadIndex`: no `blank log` check: [`Linearizable Read`][]. 6 | - A restarted Leader will stay in Leader state if possible; 7 | - Does not support single step membership change. Only joint is supported. 8 | 9 | [`Linearizable Read`]: `crate::docs::protocol::read` 10 | [`leader_id`]: `crate::docs::data::leader_id` 11 | [`RaftLogStorage::save_committed()`]: `crate::storage::RaftLogStorage::save_committed` 12 | -------------------------------------------------------------------------------- /openraft/src/stats.rs: -------------------------------------------------------------------------------- 1 | //! Runtime statistics for monitoring Raft operations. 2 | //! 3 | //! This module provides types for tracking runtime statistics such as 4 | //! batch sizes for apply and append operations. 5 | //! 6 | //! # Example 7 | //! 8 | //! ```ignore 9 | //! let raft = Raft::new(...).await?; 10 | //! let stats = raft.runtime_stats().await?; 11 | //! println!("{}", stats.display()); 12 | //! ``` 13 | 14 | pub use crate::base::histogram::Histogram; 15 | pub use crate::base::histogram::PercentileStats; 16 | pub use crate::core::NotificationName; 17 | pub use crate::core::RuntimeStats; 18 | pub use crate::core::RuntimeStatsDisplay; 19 | pub use crate::core::raft_msg::ExternalCommandName; 20 | pub use crate::core::raft_msg::RaftMsgName; 21 | pub use crate::engine::CommandName; 22 | pub use crate::engine::SMCommandName; 23 | -------------------------------------------------------------------------------- /openraft/src/docs/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | //! The protocol used by Openraft to replicate data. 2 | 3 | pub mod commit { 4 | #![doc = include_str!("commit.md")] 5 | } 6 | 7 | pub mod io_ordering { 8 | #![doc = include_str!("io_ordering.md")] 9 | } 10 | 11 | pub mod read { 12 | #![doc = include_str!("read.md")] 13 | } 14 | 15 | pub mod replication { 16 | #![doc = include_str!("replication.md")] 17 | 18 | pub mod leader_lease { 19 | #![doc = include_str!("leader_lease.md")] 20 | } 21 | 22 | pub mod log_replication { 23 | #![doc = include_str!("log_replication.md")] 24 | } 25 | 26 | pub mod log_stream { 27 | #![doc = include_str!("log_stream.md")] 28 | } 29 | 30 | pub mod snapshot_replication { 31 | #![doc = include_str!("snapshot_replication.md")] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/tests/README.md: -------------------------------------------------------------------------------- 1 | # Openraft integration tests 2 | 3 | The integration tests for Openraft are stored in this directory and rely on 4 | `memstore` with `serde` enabled. 5 | Certain tests in Openraft require the `serde` feature to be disabled. 6 | To avoid enabling `serde` for all tests in Openraft, we must relocate the 7 | integration tests to a separate crate. 8 | 9 | 10 | ## Case naming convention 11 | 12 | A test file name starts with `t[\d\d]_`, where `\d\d` is the test case number indicating priority. 13 | 14 | - `t00`: not used. 15 | - `t10`: basic behaviors. 16 | - `t20`: life cycle test cases. 17 | - `t30`: special cases for an API. 18 | - `t40`: not used. 19 | - `t50`: environment depended behaviors. 20 | - `t60`: config related behaviors. 21 | - `t70`: not used. 22 | - `t80`: not used. 23 | - `t90`: issue fixes. 24 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 15 | 16 | **Checklist** 17 | 18 | - [ ] Updated guide with pertinent info (may not always apply). 19 | - [ ] Squash down commits to one or two logical commits which clearly describe the work you've done. 20 | - [ ] Unittest is a friend:) 21 | -------------------------------------------------------------------------------- /examples/mem-log/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mem-log" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example in-memory storage for `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", default-features = false, features = ["type-alias", "tokio-rt"] } 19 | 20 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 21 | 22 | [features] 23 | 24 | serde = ["openraft/serde"] 25 | 26 | [package.metadata.docs.rs] 27 | all-features = true 28 | -------------------------------------------------------------------------------- /openraft/src/error/node_not_found.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::error::Operation; 3 | 4 | /// Error indicating a node was not found in the cluster. 5 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 6 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 7 | #[error("Node {node_id} not found when: ({operation})")] 8 | pub struct NodeNotFound { 9 | /// The node ID that was not found. 10 | pub node_id: C::NodeId, 11 | /// The operation that was being attempted when the node was not found. 12 | pub operation: Operation, 13 | } 14 | 15 | impl NodeNotFound { 16 | /// Create a new NodeNotFound error. 17 | pub fn new(node_id: C::NodeId, operation: Operation) -> Self { 18 | Self { node_id, operation } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/06-error-after-shutdown.md: -------------------------------------------------------------------------------- 1 | ### Error logs after `raft.shutdown()` completes 2 | 3 | **Symptom**: After calling [`Raft::shutdown`][] which returns successfully, logs show 4 | `ERROR openraft::raft::raft_inner: failure sending RaftMsg to RaftCore; message: AppendEntries ... core_result=Err(Stopped)` 5 | 6 | **Cause**: Other nodes in the cluster continue sending RPCs to this node. The `Raft` handle still 7 | exists and receives these RPCs, but the internal Raft core has stopped, so forwarding fails. 8 | 9 | **Solution**: This is expected behavior. These errors are harmless - they indicate the node has 10 | shut down as requested. You can ignore them or filter these specific error logs after shutdown. 11 | 12 | See: 13 | 14 | [`Raft::shutdown`]: `crate::Raft::shutdown` 15 | -------------------------------------------------------------------------------- /openraft/src/engine/replication_progress.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::RaftTypeConfig; 4 | use crate::progress::entry::ProgressEntry; 5 | 6 | /// Replication progress state for a single target node. 7 | #[derive(Debug)] 8 | #[derive(PartialEq, Eq)] 9 | pub(crate) struct TargetProgress { 10 | /// The node ID of the replication target. 11 | pub(crate) target: C::NodeId, 12 | 13 | /// The node info of the replication target. 14 | pub(crate) target_node: C::Node, 15 | 16 | /// The current replication progress to this target. 17 | pub(crate) progress: ProgressEntry, 18 | } 19 | 20 | impl fmt::Display for TargetProgress 21 | where C: RaftTypeConfig 22 | { 23 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 24 | write!(f, "TargetProgress({}={})", self.target, self.progress) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/07-troubleshooting/01-panic-is-following.md: -------------------------------------------------------------------------------- 1 | ### Panic: "assertion failed: self.internal_server_state.is_following()" 2 | 3 | **Symptom**: Node crashes with `panicked at 'assertion failed: self.internal_server_state.is_following()'` 4 | 5 | **Cause**: [`RaftNetworkFactory`][] creates a connection from a node to itself. When this node 6 | becomes leader, it sends replication messages to itself, but Openraft expects only followers to 7 | receive replication messages. 8 | 9 | **Solution**: In [`RaftNetworkFactory::new_client`][], ensure the target node ID never equals 10 | the local node's ID. Each node ID in [`Membership`][] must map to a different node. 11 | 12 | [`RaftNetworkFactory`]: `crate::network::RaftNetworkFactory` 13 | [`RaftNetworkFactory::new_client`]: `crate::network::RaftNetworkFactory::new_client` 14 | [`Membership`]: `crate::Membership` 15 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/03-configuration/04-slow-replication.md: -------------------------------------------------------------------------------- 1 | ### Slow replication performance with RocksDB or disk storage 2 | 3 | **Symptom**: Write throughput is much lower than expected when using disk-based [`RaftLogStorage`][] 4 | 5 | **Cause**: Synchronous writes to disk block the Raft thread. Additionally, HTTP client connection 6 | pooling (in libraries like `reqwest`) can add 40ms+ latency spikes. 7 | 8 | **Solution**: 9 | - Use non-blocking I/O in your [`RaftLogStorage::append`][] implementation 10 | - Consider batching writes in your storage layer 11 | - For network layer, prefer WebSocket or connection pooling that doesn't introduce latency 12 | - With RocksDB: disable `sync` on writes if you can tolerate some data loss on crash 13 | 14 | [`RaftLogStorage`]: `crate::storage::RaftLogStorage` 15 | [`RaftLogStorage::append`]: `crate::storage::RaftLogStorage::append` 16 | -------------------------------------------------------------------------------- /tests/tests/fixtures/pre_hook.rs: -------------------------------------------------------------------------------- 1 | //! Pre-hook types for intercepting RPC calls before they are sent. 2 | 3 | use openraft::base::BoxFuture; 4 | use openraft::error::Infallible; 5 | use openraft::error::RPCError; 6 | use openraft_memstore::MemNodeId; 7 | use openraft_memstore::TypeConfig; 8 | 9 | use crate::fixtures::TypedRaftRouter; 10 | use crate::fixtures::rpc_request::RpcRequest; 11 | 12 | /// Hook function called before an RPC is sent to a target node. 13 | /// 14 | /// Arguments: `(router, rpc, from_id, to_id)` 15 | pub type PreHook = 16 | Box, MemNodeId, MemNodeId) -> PreHookResult + Send + 'static>; 17 | 18 | /// Result type for pre-hook functions. 19 | /// 20 | /// Pre-hooks cannot return remote errors, only local errors. 21 | pub type PreHookResult = BoxFuture<'static, Result<(), RPCError>>; 22 | -------------------------------------------------------------------------------- /benchmarks/minimal/tests/benchmark/store_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use bench_minimal::store::LogStore; 4 | use bench_minimal::store::StateMachineStore; 5 | use bench_minimal::store::TypeConfig; 6 | use openraft::testing::log::StoreBuilder; 7 | use openraft::testing::log::Suite; 8 | use openraft::StorageError; 9 | 10 | struct Builder {} 11 | 12 | impl StoreBuilder, Arc> for Builder { 13 | async fn build(&self) -> Result<((), Arc, Arc), StorageError> { 14 | let log_store = LogStore::new_async().await; 15 | let sm = Arc::new(StateMachineStore::new()); 16 | Ok(((), log_store, sm)) 17 | } 18 | } 19 | 20 | #[tokio::test] 21 | pub async fn test_store() -> Result<(), StorageError> { 22 | Suite::test_all(Builder {}).await?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use raft_kv_memstore_grpc::app::start_raft_app; 3 | 4 | #[derive(Parser, Clone, Debug)] 5 | #[clap(author, version, about, long_about = None)] 6 | pub struct Opt { 7 | #[clap(long)] 8 | pub id: u64, 9 | 10 | #[clap(long)] 11 | /// Network address to bind the server to (e.g., "127.0.0.1:50051") 12 | pub addr: String, 13 | } 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), Box> { 17 | // Initialize tracing first, before any logging happens 18 | tracing_subscriber::fmt() 19 | .with_max_level(tracing::Level::INFO) 20 | .with_file(true) 21 | .with_line_number(true) 22 | .init(); 23 | 24 | // Parse the parameters passed by arguments. 25 | let options = Opt::parse(); 26 | 27 | start_raft_app(options.id, options.addr).await 28 | } 29 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use raft_kv_memstore::start_example_raft_node; 3 | use tracing_subscriber::EnvFilter; 4 | 5 | #[derive(Parser, Clone, Debug)] 6 | #[clap(author, version, about, long_about = None)] 7 | pub struct Opt { 8 | #[clap(long)] 9 | pub id: u64, 10 | 11 | #[clap(long)] 12 | pub http_addr: String, 13 | } 14 | 15 | #[actix_web::main] 16 | async fn main() -> std::io::Result<()> { 17 | // Setup the logger 18 | tracing_subscriber::fmt() 19 | .with_target(true) 20 | .with_thread_ids(true) 21 | .with_level(true) 22 | .with_ansi(false) 23 | .with_env_filter(EnvFilter::from_default_env()) 24 | .init(); 25 | 26 | // Parse the parameters passed by arguments. 27 | let options = Opt::parse(); 28 | 29 | start_example_raft_node(options.id, options.http_addr).await 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/07-troubleshooting/05-incorrect-config.md: -------------------------------------------------------------------------------- 1 | ### Is Openraft resilient to incorrectly configured clusters? 2 | 3 | No, Openraft, like standard raft, cannot identify errors in cluster configuration. 4 | 5 | A common error is the assigning incorrect network addresses to a node. In such 6 | a scenario, if this node becomes the leader, it will attempt to replicate 7 | logs to itself. This will cause Openraft to panic because replication 8 | messages can only be received by a follower. 9 | 10 | ```text 11 | thread 'main' panicked at openraft/src/engine/engine_impl.rs:793:9: 12 | assertion failed: self.internal_server_state.is_following() 13 | ``` 14 | 15 | ```ignore 16 | // openraft/src/engine/engine_impl.rs:793 17 | pub(crate) fn following_handler(&mut self) -> FollowingHandler { 18 | debug_assert!(self.internal_server_state.is_following()); 19 | // ... 20 | } 21 | ``` 22 | -------------------------------------------------------------------------------- /openraft/src/engine/respond_command.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::engine::Respond; 3 | 4 | /// A respond waiting for an IO condition to be satisfied. 5 | /// 6 | /// Stores the expected progress value that must be reached before sending the respond. 7 | #[derive(Debug)] 8 | pub(crate) struct PendingRespond 9 | where C: RaftTypeConfig 10 | { 11 | /// The expected progress value that must be reached before sending the respond. 12 | wait_for: V, 13 | respond: Respond, 14 | } 15 | 16 | impl PendingRespond 17 | where C: RaftTypeConfig 18 | { 19 | pub(crate) fn new(wait_for: V, respond: Respond) -> Self { 20 | Self { wait_for, respond } 21 | } 22 | 23 | pub(crate) fn wait_for(&self) -> &V { 24 | &self.wait_for 25 | } 26 | 27 | pub(crate) fn into_respond(self) -> Respond { 28 | self.respond 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.1.md: -------------------------------------------------------------------------------- 1 | ### Added: 2 | 3 | - Added: [1ad17e8e](https://github.com/databendlabs/openraft/commit/1ad17e8edf18d98eeb687f00a65ebf528ab3aeb7) move wait_for_xxx util into metrics.; by drdr xp; 2021-06-16 4 | 5 | Introduce struct `Wait` as a wrapper of the metrics channel to impl 6 | wait-for utils: 7 | - `log()`: wait for log to apply. 8 | - `current_leader()`: wait for known leader. 9 | - `state()`: wait for the role. 10 | - `members()`: wait for membership_config.members. 11 | - `next_members()`: wait for membership_config.members_after_consensus. 12 | 13 | E.g.: 14 | 15 | ```rust 16 | // wait for ever for raft node's current leader to become 3: 17 | r.wait(None).current_leader(2).await?; 18 | ``` 19 | 20 | The timeout is now an option arg to all wait_for_xxx functions in 21 | fixtures. wait_for_xxx_timeout are all removed. 22 | -------------------------------------------------------------------------------- /openraft/src/log_id/log_id_option_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::log_id::raft_log_id::RaftLogId; 3 | 4 | /// This helper trait extracts information from an `Option`. 5 | pub trait LogIdOptionExt 6 | where C: RaftTypeConfig 7 | { 8 | /// Returns the log index if it is not a `None`. 9 | fn index(&self) -> Option; 10 | 11 | /// Returns the next log index. 12 | /// 13 | /// If self is `None`, it returns 0. 14 | fn next_index(&self) -> u64; 15 | } 16 | 17 | impl LogIdOptionExt for Option 18 | where 19 | C: RaftTypeConfig, 20 | T: RaftLogId, 21 | { 22 | fn index(&self) -> Option { 23 | self.as_ref().map(|x| x.index()) 24 | } 25 | 26 | fn next_index(&self) -> u64 { 27 | match self { 28 | None => 0, 29 | Some(log_id) => log_id.index() + 1, 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tests/tests/fixtures/post_hook.rs: -------------------------------------------------------------------------------- 1 | //! Post-hook types for intercepting RPC responses after they are received. 2 | 3 | use openraft::base::BoxFuture; 4 | use openraft::error::RPCError; 5 | use openraft_memstore::MemNodeId; 6 | use openraft_memstore::TypeConfig; 7 | 8 | use crate::fixtures::TypedRaftRouter; 9 | use crate::fixtures::rpc_request::RpcRequest; 10 | use crate::fixtures::rpc_response::RpcResponse; 11 | 12 | /// Hook function called after an RPC response is received from a target node. 13 | /// 14 | /// Arguments: `(router, request, response, from_id, to_id)` 15 | pub type PostHook = Box< 16 | dyn Fn(&TypedRaftRouter, RpcRequest, RpcResponse, MemNodeId, MemNodeId) -> PostHookResult 17 | + Send 18 | + 'static, 19 | >; 20 | 21 | /// Result type for post-hook functions. 22 | pub type PostHookResult = BoxFuture<'static, Result<(), RPCError>>; 23 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/mpsc/try_recv_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by `try_recv`. 4 | #[derive(PartialEq, Eq, Clone, Copy, Debug)] 5 | pub enum TryRecvError { 6 | /// This **channel** is currently empty, but the **Sender**(s) have not yet 7 | /// disconnected, so data may yet become available. 8 | Empty, 9 | /// The **channel**'s sending half has become disconnected, and there will 10 | /// never be any more data received on it. 11 | Disconnected, 12 | } 13 | 14 | impl fmt::Display for TryRecvError { 15 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 16 | match *self { 17 | TryRecvError::Empty => "receiving on an empty channel".fmt(fmt), 18 | TryRecvError::Disconnected => "receiving on a closed channel".fmt(fmt), 19 | } 20 | } 21 | } 22 | 23 | impl std::error::Error for TryRecvError {} 24 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::uninlined_format_args)] 2 | 3 | pub mod app; 4 | pub mod grpc; 5 | pub mod network; 6 | pub mod store; 7 | 8 | pub mod protobuf { 9 | tonic::include_proto!("openraftpb"); 10 | } 11 | 12 | #[path = "../../utils/declare_types.rs"] 13 | pub mod typ; 14 | 15 | mod pb_impl; 16 | 17 | #[cfg(test)] 18 | mod test_store; 19 | 20 | use crate::protobuf as pb; 21 | 22 | openraft::declare_raft_types!( 23 | /// Declare the type configuration for example K/V store. 24 | pub TypeConfig: 25 | D = pb::SetRequest, 26 | R = pb::Response, 27 | LeaderId = pb::LeaderId, 28 | Vote = pb::Vote, 29 | Entry = pb::Entry, 30 | Node = pb::Node, 31 | SnapshotData = Vec, 32 | ); 33 | 34 | pub type NodeId = u64; 35 | pub type LogStore = store::LogStore; 36 | pub type StateMachineStore = store::StateMachineStore; 37 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/tokio_impls/tokio_runtime/oneshot.rs: -------------------------------------------------------------------------------- 1 | use crate::OptionalSend; 2 | use crate::async_runtime::oneshot; 3 | use crate::type_config::OneshotSender; 4 | 5 | pub struct TokioOneshot; 6 | 7 | impl oneshot::Oneshot for TokioOneshot { 8 | type Sender = tokio::sync::oneshot::Sender; 9 | type Receiver = tokio::sync::oneshot::Receiver; 10 | type ReceiverError = tokio::sync::oneshot::error::RecvError; 11 | 12 | #[inline] 13 | fn channel() -> (Self::Sender, Self::Receiver) 14 | where T: OptionalSend { 15 | let (tx, rx) = tokio::sync::oneshot::channel(); 16 | (tx, rx) 17 | } 18 | } 19 | 20 | impl OneshotSender for tokio::sync::oneshot::Sender 21 | where T: OptionalSend 22 | { 23 | #[inline] 24 | fn send(self, t: T) -> Result<(), T> { 25 | self.send(t) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use raft_kv_rocksdb::start_example_raft_node; 3 | use tracing_subscriber::EnvFilter; 4 | 5 | #[derive(Parser, Clone, Debug)] 6 | #[clap(author, version, about, long_about = None)] 7 | pub struct Opt { 8 | #[clap(long)] 9 | pub id: u64, 10 | 11 | #[clap(long)] 12 | pub addr: String, 13 | } 14 | 15 | #[actix_web::main] 16 | async fn main() -> std::io::Result<()> { 17 | // Setup the logger 18 | tracing_subscriber::fmt() 19 | .with_target(true) 20 | .with_thread_ids(true) 21 | .with_level(true) 22 | .with_ansi(false) 23 | .with_env_filter(EnvFilter::from_default_env()) 24 | .init(); 25 | 26 | // Parse the parameters passed by arguments. 27 | let options = Opt::parse(); 28 | 29 | start_example_raft_node(options.id, format!("{}.db", options.addr), options.addr).await 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/core/replication_state.rs: -------------------------------------------------------------------------------- 1 | use crate::log_id::LogIndexOptionExt; 2 | 3 | /// Calculate the distance between the matched log index on a replication target and the locally 4 | /// known last log index. 5 | pub(crate) fn replication_lag(matched_log_index: &Option, last_log_index: &Option) -> u64 { 6 | last_log_index.next_index().saturating_sub(matched_log_index.next_index()) 7 | } 8 | 9 | #[cfg(test)] 10 | mod test { 11 | use crate::core::replication_state::replication_lag; 12 | 13 | #[test] 14 | fn test_replication_lag() -> anyhow::Result<()> { 15 | assert_eq!(0, replication_lag(&None, &None)); 16 | assert_eq!(4, replication_lag(&None, &Some(3))); 17 | assert_eq!(1, replication_lag(&Some(2), &Some(3))); 18 | assert_eq!(0, replication_lag(&Some(3), &Some(3))); 19 | assert_eq!(0, replication_lag(&Some(4), &Some(3))); 20 | Ok(()) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/build_faq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from pathlib import Path 4 | 5 | def get_section_name(dir_path): 6 | readme = dir_path / "README.md" 7 | return readme.read_text().strip() 8 | 9 | def build_faq(): 10 | output = [] 11 | 12 | category_dirs = sorted([d for d in Path(".").iterdir() if d.is_dir() and d.name[0].isdigit()]) 13 | 14 | for dir_path in category_dirs: 15 | section_name = get_section_name(dir_path) 16 | output.append(f"## {section_name}\n\n") 17 | 18 | md_files = sorted([f for f in dir_path.glob("*.md") if f.name != "README.md"]) 19 | 20 | for md_file in md_files: 21 | content = md_file.read_text() 22 | output.append(content) 23 | output.append("\n\n") 24 | 25 | Path("faq.md").write_text("".join(output)) 26 | print("Generated faq.md successfully") 27 | 28 | if __name__ == "__main__": 29 | build_faq() 30 | -------------------------------------------------------------------------------- /rt-monoio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-rt-monoio" 3 | description = "monoio AsyncRuntime support for Openraft" 4 | documentation = "https://docs.rs/openraft-rt-monoio" 5 | readme = "README.md" 6 | version = "0.10.0" 7 | edition = "2021" 8 | authors = [ 9 | "Databend Authors ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../openraft", version = "0.10.0", default-features = false, features = ["singlethreaded"] } 19 | 20 | futures = { version = "0.3" } 21 | local-sync = { version = "0.1.1" } 22 | monoio = { version = "0.2.3" } 23 | rand = { version = "0.9" } 24 | tokio = { version = "1.22", features = ["sync"] } 25 | -------------------------------------------------------------------------------- /examples/rocksstore/src/test.rs: -------------------------------------------------------------------------------- 1 | use openraft::testing::log::StoreBuilder; 2 | use openraft::testing::log::Suite; 3 | use openraft::StorageError; 4 | use tempfile::TempDir; 5 | 6 | use crate::log_store::RocksLogStore; 7 | use crate::RocksStateMachine; 8 | use crate::TypeConfig; 9 | 10 | struct RocksBuilder {} 11 | 12 | impl StoreBuilder, RocksStateMachine, TempDir> for RocksBuilder { 13 | async fn build(&self) -> Result<(TempDir, RocksLogStore, RocksStateMachine), StorageError> { 14 | let td = TempDir::new().map_err(|e| StorageError::read(&e))?; 15 | let (log_store, sm) = crate::new(td.path()).await.map_err(|e| StorageError::read(&e))?; 16 | Ok((td, log_store, sm)) 17 | } 18 | } 19 | 20 | #[tokio::test] 21 | pub async fn test_rocks_store() -> Result<(), StorageError> { 22 | Suite::test_all(RocksBuilder {}).await?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /examples/utils/README.md: -------------------------------------------------------------------------------- 1 | # Example Utilities 2 | 3 | Shared utilities for Openraft examples, reducing boilerplate across example implementations. 4 | 5 | ## Contents 6 | 7 | - **`declare_types.rs`** - Type declarations for [`RaftTypeConfig`] implementations 8 | - Provides type aliases for common Raft types (Node, Entry, Response, etc.) 9 | - Reduces repetitive type definitions across examples 10 | - Usage: Import and use the generated type aliases 11 | 12 | ## Purpose 13 | 14 | This crate centralizes common type declarations used by multiple examples, making example code: 15 | - More concise and readable 16 | - Easier to maintain 17 | - Consistent across different examples 18 | 19 | ## Usage 20 | 21 | ```rust 22 | use example_utils::declare_raft_types; 23 | 24 | declare_raft_types!( 25 | pub TypeConfig: 26 | D = Request, 27 | R = Response, 28 | ); 29 | ``` 30 | 31 | Used by all Openraft examples for type configuration. -------------------------------------------------------------------------------- /openraft/src/error/into_ok.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for converting `Result` into `T`. 2 | 3 | use crate::error::Infallible; 4 | 5 | /// Trait to convert `Result` to `T`, if `E` is a `never` type. 6 | pub(crate) trait UnwrapInfallible { 7 | fn into_ok(self) -> T; 8 | } 9 | 10 | impl UnwrapInfallible for Result 11 | where E: Into 12 | { 13 | fn into_ok(self) -> T { 14 | match self { 15 | Ok(t) => t, 16 | Err(e) => { 17 | // NOTE: `allow` required because of buggy reachability detection by rust compiler 18 | #[allow(unreachable_code)] 19 | match e.into() {} 20 | } 21 | } 22 | } 23 | } 24 | 25 | /// Convert `Result` to `T`, if `E` is a `never` type. 26 | pub(crate) fn into_ok(result: Result) -> T 27 | where E: Into { 28 | UnwrapInfallible::into_ok(result) 29 | } 30 | -------------------------------------------------------------------------------- /openraft/src/log_id/raft_log_id_ext.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::log_id::raft_log_id::RaftLogId; 3 | use crate::log_id::ref_log_id::RefLogId; 4 | use crate::type_config::alias::LogIdOf; 5 | 6 | pub(crate) trait RaftLogIdExt 7 | where 8 | C: RaftTypeConfig, 9 | Self: RaftLogId, 10 | { 11 | /// Creates a new owned [`LogId`] from this log ID implementation. 12 | /// 13 | /// [`LogId`]: crate::log_id::LogId 14 | fn to_log_id(&self) -> LogIdOf { 15 | self.to_ref().into_log_id() 16 | } 17 | 18 | /// Creates a reference view of this log ID implementation via a [`RefLogId`]. 19 | fn to_ref(&self) -> RefLogId<'_, C> { 20 | RefLogId { 21 | leader_id: self.committed_leader_id(), 22 | index: self.index(), 23 | } 24 | } 25 | } 26 | 27 | impl RaftLogIdExt for T 28 | where 29 | C: RaftTypeConfig, 30 | T: RaftLogId, 31 | { 32 | } 33 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_vote.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use openraft::vote::RaftVote; 4 | 5 | use crate::pb; 6 | use crate::typ::LeaderId; 7 | use crate::TypeConfig; 8 | 9 | impl RaftVote for pb::Vote { 10 | fn from_leader_id(leader_id: LeaderId, committed: bool) -> Self { 11 | pb::Vote { 12 | leader_id: Some(leader_id), 13 | committed, 14 | } 15 | } 16 | 17 | fn leader_id(&self) -> &LeaderId { 18 | self.leader_id.as_ref().expect("Vote must have a leader_id") 19 | } 20 | 21 | fn is_committed(&self) -> bool { 22 | self.committed 23 | } 24 | } 25 | 26 | impl fmt::Display for pb::Vote { 27 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 28 | write!( 29 | f, 30 | "<{}:{}>", 31 | self.leader_id(), 32 | if self.is_committed() { "Q" } else { "-" } 33 | ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-macros" 3 | 4 | version = { workspace = true } 5 | edition = { workspace = true } 6 | authors = { workspace = true } 7 | categories = { workspace = true } 8 | description = { workspace = true } 9 | documentation = { workspace = true } 10 | homepage = { workspace = true } 11 | keywords = { workspace = true } 12 | license = { workspace = true } 13 | repository = { workspace = true } 14 | 15 | [lib] 16 | proc-macro = true 17 | 18 | [dependencies] 19 | chrono = { workspace = true } 20 | proc-macro2 = { workspace = true } 21 | quote = { workspace = true } 22 | semver = { workspace = true } 23 | syn = { workspace = true, features = ["full", "extra-traits"] } 24 | 25 | 26 | [dev-dependencies] 27 | macrotest = { version = "1" } 28 | trybuild = { version = "1.0" } 29 | 30 | 31 | [features] 32 | 33 | # Do not add `Send` bounds. 34 | singlethreaded = [] 35 | -------------------------------------------------------------------------------- /tests/tests/membership/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "bt", feature(error_generic_member_access))] 2 | #![allow(clippy::uninlined_format_args)] 3 | 4 | #[macro_use] 5 | #[path = "../fixtures/mod.rs"] 6 | mod fixtures; 7 | 8 | // The number indicate the preferred running order for these case. 9 | // The later tests may depend on the earlier ones. 10 | 11 | mod t10_learner_restart; 12 | mod t10_single_node; 13 | mod t11_add_learner; 14 | mod t12_concurrent_write_and_add_learner; 15 | mod t20_change_membership; 16 | mod t21_change_membership_cases; 17 | mod t30_commit_joint_config; 18 | mod t30_elect_with_new_config; 19 | mod t31_add_remove_follower; 20 | mod t31_remove_leader; 21 | mod t31_removed_follower; 22 | mod t51_remove_unreachable_follower; 23 | mod t52_change_membership_on_uninitialized_node; 24 | mod t99_issue_471_adding_learner_uses_uninit_leader_id; 25 | mod t99_issue_584_replication_state_reverted; 26 | mod t99_new_leader_auto_commit_uniform_config; 27 | -------------------------------------------------------------------------------- /change-log/v0.7.0-alpha.3.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [f99ade30](https://github.com/databendlabs/openraft/commit/f99ade30a7f806f18ed19ace12e226cd62fd43ec) API: move default impl methods in RaftStorage to StorageHelper; by 张炎泼; 2022-07-04 4 | 5 | ### Fixed: 6 | 7 | - Fixed: [44381b0c](https://github.com/databendlabs/openraft/commit/44381b0c776cfbb7dfc7789de27346110776b7f6) when handling append-entries, if prev_log_id is purged, it should not delete any logs.; by 张炎泼; 2022-08-14 8 | 9 | When handling append-entries, if the local log at `prev_log_id.index` is 10 | purged, a follower should not believe it is a **conflict** and should 11 | not delete all logs. It will get committed log lost. 12 | 13 | To fix this issue, use `last_applied` instead of `committed`: 14 | `last_applied` is always the committed log id, while `committed` is not 15 | persisted and may be smaller than the actually applied, when a follower 16 | is restarted. 17 | -------------------------------------------------------------------------------- /openraft/src/display_ext/display_option.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Implement `Display` for `Option` if T is `Display`. 4 | /// 5 | /// It outputs a literal string `"None"` if it is None, otherwise it invokes the Display 6 | /// implementation for T. 7 | pub(crate) struct DisplayOption<'a, T: fmt::Display>(pub &'a Option); 8 | 9 | impl fmt::Display for DisplayOption<'_, T> { 10 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 11 | match &self.0 { 12 | None => { 13 | write!(f, "None") 14 | } 15 | Some(x) => x.fmt(f), 16 | } 17 | } 18 | } 19 | 20 | pub(crate) trait DisplayOptionExt<'a, T: fmt::Display> { 21 | fn display(&'a self) -> DisplayOption<'a, T>; 22 | } 23 | 24 | impl DisplayOptionExt<'_, T> for Option 25 | where T: fmt::Display 26 | { 27 | fn display(&self) -> DisplayOption<'_, T> { 28 | DisplayOption(self) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/raft/core_state.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::error::Fatal; 3 | use crate::error::Infallible; 4 | use crate::type_config::alias::JoinHandleOf; 5 | use crate::type_config::alias::WatchReceiverOf; 6 | 7 | /// The running state of RaftCore 8 | pub(in crate::raft) enum CoreState 9 | where C: RaftTypeConfig 10 | { 11 | /// The RaftCore task is still running. 12 | Running(JoinHandleOf>>), 13 | 14 | /// The RaftCore task is waiting for a signal to finish joining. 15 | Joining(WatchReceiverOf), 16 | 17 | /// The RaftCore task has finished. The return value of the task is stored. 18 | Done(Result>), 19 | } 20 | 21 | impl CoreState 22 | where C: RaftTypeConfig 23 | { 24 | /// Returns `true` if the RaftCore task is still running. 25 | pub(in crate::raft) fn is_running(&self) -> bool { 26 | matches!(self, CoreState::Running(_)) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /openraft/src/replication/stream_context.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use crate::RaftTypeConfig; 4 | use crate::replication::inflight_append_queue::InflightAppendQueue; 5 | use crate::replication::stream_state::StreamState; 6 | use crate::storage::RaftLogStorage; 7 | use crate::type_config::alias::MutexOf; 8 | 9 | /// Context passed through the AppendEntries request stream. 10 | /// 11 | /// This struct is used with `futures::stream::unfold` to generate 12 | /// AppendEntries requests. It holds both the mutable state for reading 13 | /// log entries and a queue for tracking in-flight requests. 14 | pub(crate) struct StreamContext 15 | where 16 | C: RaftTypeConfig, 17 | LS: RaftLogStorage, 18 | { 19 | /// Shared state for generating the next request. 20 | pub(crate) stream_state: Arc>>, 21 | 22 | /// Tracks in-flight requests for RTT measurement. 23 | pub(crate) inflight_append_queue: InflightAppendQueue, 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/devskim-analysis.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | name: DevSkim 7 | 8 | on: 9 | push: 10 | branches: [ master ] 11 | pull_request: 12 | branches: [ master ] 13 | schedule: 14 | - cron: '35 13 * * 5' 15 | 16 | jobs: 17 | lint: 18 | name: DevSkim 19 | runs-on: ubuntu-20.04 20 | permissions: 21 | actions: read 22 | contents: read 23 | security-events: write 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v4 27 | 28 | - name: Run DevSkim scanner 29 | uses: microsoft/DevSkim-Action@v1 30 | 31 | - name: Upload DevSkim scan results to GitHub Security tab 32 | uses: github/codeql-action/upload-sarif@v1 33 | with: 34 | sarif_file: devskim-results.sarif 35 | -------------------------------------------------------------------------------- /openraft/src/storage/v2/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines [`RaftLogStorage`] and [`RaftStateMachine`] trait. 2 | //! 3 | //! [`RaftLogStorage`] is responsible for storing logs, 4 | //! and [`RaftStateMachine`] is responsible for storing state machine and snapshot. 5 | 6 | mod apply_responder; 7 | mod apply_responder_inner; 8 | pub(crate) mod entry_responder; 9 | mod raft_log_reader; 10 | mod raft_log_storage; 11 | mod raft_log_storage_ext; 12 | mod raft_snapshot_builder; 13 | mod raft_state_machine; 14 | 15 | pub use self::apply_responder::ApplyResponder; 16 | pub use self::entry_responder::EntryResponder; 17 | pub use self::raft_log_reader::LeaderBoundedStreamError; 18 | pub use self::raft_log_reader::LeaderBoundedStreamResult; 19 | pub use self::raft_log_reader::RaftLogReader; 20 | pub use self::raft_log_storage::RaftLogStorage; 21 | pub use self::raft_log_storage_ext::RaftLogStorageExt; 22 | pub use self::raft_snapshot_builder::RaftSnapshotBuilder; 23 | pub use self::raft_state_machine::RaftStateMachine; 24 | -------------------------------------------------------------------------------- /openraft/src/raft/watch_handle.rs: -------------------------------------------------------------------------------- 1 | //! Watch handle for async watch tasks. 2 | 3 | use crate::RaftTypeConfig; 4 | use crate::type_config::alias::JoinHandleOf; 5 | use crate::type_config::alias::OneshotSenderOf; 6 | 7 | /// Handle to control an async watch task. 8 | /// 9 | /// Use [`close()`](`Self::close`) to stop watching and wait for the task to complete. 10 | pub struct WatchChangeHandle 11 | where C: RaftTypeConfig 12 | { 13 | pub(crate) cancel_tx: Option>, 14 | pub(crate) join_handle: Option>, 15 | } 16 | 17 | impl WatchChangeHandle 18 | where C: RaftTypeConfig 19 | { 20 | /// Stop watching and wait for the task to complete. 21 | pub async fn close(&mut self) { 22 | // Drop the sender to signal shutdown 23 | drop(self.cancel_tx.take()); 24 | 25 | // Wait for task to finish 26 | if let Some(handle) = self.join_handle.take() { 27 | handle.await.ok(); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/quorum/quorum_set.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | /// A set of quorums is a collection of quorum. 4 | /// 5 | /// A quorum is a collection of nodes that a read or write operation in a distributed system has to 6 | /// contact. See: 7 | pub(crate) trait QuorumSet { 8 | type Iter: Iterator; 9 | 10 | /// Check if a series of ID constitute a quorum that is defined by this quorum set. 11 | fn is_quorum<'a, I: Iterator + Clone>(&self, ids: I) -> bool; 12 | 13 | /// Returns all ids in this QuorumSet 14 | fn ids(&self) -> Self::Iter; 15 | } 16 | 17 | impl> QuorumSet for Arc { 18 | type Iter = T::Iter; 19 | 20 | fn is_quorum<'a, I: Iterator + Clone>(&self, ids: I) -> bool { 21 | self.as_ref().is_quorum(ids) 22 | } 23 | 24 | fn ids(&self) -> Self::Iter { 25 | self.as_ref().ids() 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.3.md: -------------------------------------------------------------------------------- 1 | ### Dependency: 2 | 3 | - Dependency: [b351c87f](https://github.com/databendlabs/openraft/commit/b351c87f0adfd0a6f1105f55cf1223c6045ecf41) upgrade tokio from 1.7 to 1.8; by drdr xp; 2021-07-08 4 | 5 | ### Fixed: 6 | 7 | - Fixed: [cf4badd0](https://github.com/databendlabs/openraft/commit/cf4badd0d762757519e2db5ed2f2fc65c2f49d02) leader should re-create and send snapshot when `threshold/2 < last_log_index - snapshot < threshold`; by drdr xp; 2021-07-08 8 | 9 | The problem: 10 | 11 | If `last_log_index` advances `snapshot.applied_index` too many, i.e.: 12 | `threshold/2 < last_log_index - snapshot < threshold` 13 | (e.g., `10/2 < 16-10 < 20` in the test that reproduce this bug), the leader 14 | tries to re-create a new snapshot. But when 15 | `last_log_index < threshold`, it won't create, which result in a dead 16 | loop. 17 | 18 | Solution: 19 | 20 | In such case, force to create a snapshot without considering the 21 | threshold. 22 | -------------------------------------------------------------------------------- /openraft/src/storage/snapshot.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use openraft_macros::since; 4 | 5 | use crate::RaftTypeConfig; 6 | use crate::storage::SnapshotMeta; 7 | 8 | /// The data associated with the current snapshot. 9 | #[since(version = "0.10.0", change = "SnapshotData without Box")] 10 | #[derive(Debug, Clone)] 11 | pub struct Snapshot 12 | where C: RaftTypeConfig 13 | { 14 | /// metadata of a snapshot 15 | pub meta: SnapshotMeta, 16 | 17 | /// A read handle to the associated snapshot. 18 | pub snapshot: C::SnapshotData, 19 | } 20 | 21 | impl Snapshot 22 | where C: RaftTypeConfig 23 | { 24 | #[allow(dead_code)] 25 | pub(crate) fn new(meta: SnapshotMeta, snapshot: C::SnapshotData) -> Self { 26 | Self { meta, snapshot } 27 | } 28 | } 29 | 30 | impl fmt::Display for Snapshot 31 | where C: RaftTypeConfig 32 | { 33 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 34 | write!(f, "Snapshot{{meta: {}}}", self.meta) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/client-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "client-http" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example network implementation v1 built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus", "network"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 19 | 20 | reqwest = { version = "0.12.5", features = ["json"] } 21 | serde = { version = "1.0.114", features = ["derive"] } 22 | serde_json = { version = "1.0.57" } 23 | tokio = { version = "1.35.1", features = ["full"] } 24 | tracing = { version = "0.1.40" } 25 | 26 | [features] 27 | default = [] 28 | 29 | [package.metadata.docs.rs] 30 | all-features = true -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_append_entries_request.rs: -------------------------------------------------------------------------------- 1 | use crate::pb; 2 | use crate::typ::AppendEntriesRequest; 3 | 4 | impl From for AppendEntriesRequest { 5 | fn from(proto_req: pb::AppendEntriesRequest) -> Self { 6 | AppendEntriesRequest { 7 | vote: proto_req.vote.unwrap(), 8 | prev_log_id: proto_req.prev_log_id.map(|log_id| log_id.into()), 9 | entries: proto_req.entries, 10 | leader_commit: proto_req.leader_commit.map(|log_id| log_id.into()), 11 | } 12 | } 13 | } 14 | 15 | impl From for pb::AppendEntriesRequest { 16 | fn from(value: AppendEntriesRequest) -> Self { 17 | pb::AppendEntriesRequest { 18 | vote: Some(value.vote), 19 | prev_log_id: value.prev_log_id.map(|log_id| log_id.into()), 20 | entries: value.entries, 21 | leader_commit: value.leader_commit.map(|log_id| log_id.into()), 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/network-v1-http/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "network-v1-http" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example network implementation v1 built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus", "network"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 19 | 20 | reqwest = { version = "0.12.5", features = ["json"] } 21 | serde = { version = "1.0.114", features = ["derive"] } 22 | serde_json = { version = "1.0.57" } 23 | tokio = { version = "1.35.1", features = ["full"] } 24 | tracing = { version = "0.1.40" } 25 | 26 | [features] 27 | default = [] 28 | 29 | [package.metadata.docs.rs] 30 | all-features = true -------------------------------------------------------------------------------- /examples/raft-kv-rocksdb/src/network/raft.rs: -------------------------------------------------------------------------------- 1 | use actix_web::post; 2 | use actix_web::web::Data; 3 | use actix_web::web::Json; 4 | use actix_web::Responder; 5 | use openraft::error::decompose::DecomposeResult; 6 | 7 | use crate::app::App; 8 | use crate::typ::*; 9 | 10 | // --- Raft communication 11 | 12 | #[post("/vote")] 13 | pub async fn vote(app: Data, req: Json) -> actix_web::Result { 14 | let res = app.raft.vote(req.0).await.decompose().unwrap(); 15 | Ok(Json(res)) 16 | } 17 | 18 | #[post("/append")] 19 | pub async fn append(app: Data, req: Json) -> actix_web::Result { 20 | let res = app.raft.append_entries(req.0).await.decompose().unwrap(); 21 | Ok(Json(res)) 22 | } 23 | 24 | #[post("/snapshot")] 25 | pub async fn snapshot(app: Data, req: Json) -> actix_web::Result { 26 | let res = app.raft.install_snapshot(req.0).await.decompose().unwrap(); 27 | Ok(Json(res)) 28 | } 29 | -------------------------------------------------------------------------------- /openraft/src/docs/data/mod.rs: -------------------------------------------------------------------------------- 1 | //! Data structures used by the Openraft protocol, such log, vote, snapshot, membership etc. 2 | 3 | pub mod leader_id { 4 | #![doc = include_str!("leader_id.md")] 5 | } 6 | 7 | pub mod vote { 8 | #![doc = include_str!("vote.md")] 9 | } 10 | 11 | pub mod log_pointers { 12 | #![doc = include_str!("log_pointers.md")] 13 | } 14 | 15 | pub mod io_id { 16 | #![doc = include_str!("io_id.md")] 17 | } 18 | 19 | pub mod log_io_id { 20 | #![doc = include_str!("log_io_id.md")] 21 | } 22 | 23 | pub mod log_io_progress { 24 | #![doc = include_str!("log_io_progress.md")] 25 | } 26 | 27 | pub mod leader_lease { 28 | #![doc = include_str!("leader-lease.md")] 29 | } 30 | 31 | pub mod extended_membership { 32 | #![doc = include_str!("extended-membership.md")] 33 | } 34 | 35 | pub mod effective_membership { 36 | #![doc = include_str!("effective-membership.md")] 37 | } 38 | 39 | pub mod replication_session { 40 | #![doc = include_str!("replication-session.md")] 41 | } 42 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/03-configuration/03-frequent-elections.md: -------------------------------------------------------------------------------- 1 | ### Frequent leader elections and timeouts 2 | 3 | **Symptom**: Logs show repeated leader elections, or [`RaftMetrics::current_leader`][] changes frequently 4 | 5 | **Cause**: [`Config::election_timeout_min`][] is too small for your storage or network latency. 6 | If [`RaftLogStorage::append`][] takes longer than the election timeout, heartbeats time out and 7 | trigger elections. 8 | 9 | **Solution**: Increase both [`Config::election_timeout_min`][] and [`Config::election_timeout_max`][]. 10 | Ensure `heartbeat_interval < election_timeout_min / 2` and that election timeout is at least 11 | 10× your typical [`RaftLogStorage::append`][] latency. 12 | 13 | [`RaftMetrics::current_leader`]: `crate::metrics::RaftMetrics::current_leader` 14 | [`Config::election_timeout_min`]: `crate::config::Config::election_timeout_min` 15 | [`Config::election_timeout_max`]: `crate::config::Config::election_timeout_max` 16 | [`RaftLogStorage::append`]: `crate::storage::RaftLogStorage::append` 17 | -------------------------------------------------------------------------------- /openraft/src/testing/mod.rs: -------------------------------------------------------------------------------- 1 | //! Testing utilities for Openraft applications. 2 | //! 3 | //! This module provides test utilities and suite runners to verify Openraft implementations. 4 | //! 5 | //! ## Modules 6 | //! 7 | //! - [`common`] - Common test utilities and assertions 8 | //! - [`log`] - Log storage test suite 9 | //! - [`runtime`] - Runtime test utilities 10 | //! 11 | //! ## Overview 12 | //! 13 | //! Test suites help verify that custom implementations of storage and network traits 14 | //! behave correctly according to Raft protocol requirements. 15 | //! 16 | //! ## Usage 17 | //! 18 | //! Import test utilities to verify your implementations: 19 | //! 20 | //! ```ignore 21 | //! use openraft::testing::log::Suite; 22 | //! 23 | //! #[test] 24 | //! fn test_log_storage() { 25 | //! Suite::test_all(MyLogStore::new()); 26 | //! } 27 | //! ``` 28 | //! 29 | //! These tests help ensure correctness and catch subtle protocol violations. 30 | 31 | pub mod common; 32 | pub mod log; 33 | pub mod runtime; 34 | 35 | pub use common::*; 36 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/03-configuration/02-snapshot-policy.md: -------------------------------------------------------------------------------- 1 | ### How to customize snapshot-building policy? 2 | 3 | OpenRaft provides a default snapshot building policy that triggers snapshots 4 | when the log count exceeds a threshold. Configure this via [`Config::snapshot_policy`] 5 | set to [`SnapshotPolicy::LogsSinceLast(n)`][`SnapshotPolicy::LogsSinceLast`]. 6 | 7 | To customize snapshot behavior: 8 | 9 | - **Disable automatic snapshots**: Set [`Config::snapshot_policy`] to [`SnapshotPolicy::Never`] 10 | - **Manual snapshot triggers**: Use [`Raft::trigger().snapshot()`][`Trigger::snapshot`] to build snapshots on demand 11 | 12 | This allows full control over when snapshots are created based on your application's specific requirements. 13 | 14 | [`Config::snapshot_policy`]: `crate::config::Config::snapshot_policy` 15 | [`SnapshotPolicy::LogsSinceLast`]: `crate::config::SnapshotPolicy::LogsSinceLast` 16 | [`SnapshotPolicy::Never`]: `crate::config::SnapshotPolicy::Never` 17 | [`Trigger::snapshot`]: `crate::raft::trigger::Trigger::snapshot` 18 | -------------------------------------------------------------------------------- /openraft/src/progress/id_val.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::LogId; 4 | use crate::progress::entry; 5 | 6 | /// An ID and its associated value. 7 | #[derive(Clone, Debug, PartialEq, Eq)] 8 | pub(crate) struct IdVal { 9 | pub(crate) id: ID, 10 | pub(crate) val: Val, 11 | } 12 | 13 | impl fmt::Display for IdVal 14 | where 15 | ID: fmt::Display, 16 | Val: fmt::Display, 17 | { 18 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 19 | write!(f, "{}: {}", self.id, self.val) 20 | } 21 | } 22 | 23 | impl IdVal { 24 | pub(crate) fn new(id: ID, val: Val) -> Self { 25 | Self { id, val } 26 | } 27 | } 28 | 29 | impl IdVal> 30 | where 31 | C: crate::RaftTypeConfig, 32 | ID: Clone, 33 | { 34 | /// Return a tuple of (cloned id, cloned matching log id). 35 | pub(crate) fn to_matching_tuple(&self) -> (ID, Option>) { 36 | (self.id.clone(), self.val.matching().cloned()) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /scripts/check.kdl: -------------------------------------------------------------------------------- 1 | // Use zellij to run test in parallel. 2 | // 3 | // Install: 4 | // cargo install zellij 5 | // 6 | // Usage: 7 | // zellij action new-tab --layout check.kdl 8 | // zellij --layout check.kdl 9 | 10 | simplified_ui true 11 | 12 | layout { 13 | 14 | tab name="3m1q" { 15 | // tab-bar 16 | pane size=1 borderless=true { 17 | plugin location="zellij:tab-bar" 18 | } 19 | 20 | pane split_direction="vertical" { 21 | pane { 22 | command "cargo" 23 | args "test" "--lib" 24 | } 25 | pane { 26 | command "cargo" 27 | args "test" "--test" "*" 28 | } 29 | pane { 30 | command "cargo" 31 | args "clippy" "--no-deps" "--all-targets" "--" "-D" "warnings" 32 | } 33 | } 34 | // status-bar 35 | pane size=2 borderless=true { 36 | plugin location="zellij:status-bar" 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/04-storage/01-state-machine-persistence.md: -------------------------------------------------------------------------------- 1 | ### Does the state machine need to be persisted to disk? 2 | 3 | **Question**: Should I persist the state machine to disk, or just rely on snapshots and log replay? 4 | 5 | **Answer**: The state machine does not need to be persisted separately, since snapshots are periodically saved. On startup, rebuild the state machine from the latest snapshot. 6 | 7 | Whether to re-apply raft logs after loading a snapshot depends on whether your application stores the committed log id using [`RaftLogStorage::save_committed()`][]: 8 | 9 | - **If `save_committed()` is implemented**: Re-apply logs from the snapshot's last included log up to the saved committed log id on startup 10 | - **If `save_committed()` is NOT implemented**: No log replay needed - the snapshot represents the committed state 11 | 12 | This avoids the redundancy of persisting both the full state machine and its snapshot representation. 13 | 14 | [`RaftLogStorage::save_committed()`]: `crate::storage::RaftLogStorage::save_committed` 15 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.8.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [adc24f55](https://github.com/databendlabs/openraft/commit/adc24f55d75d9c7c01fcd0f4f9e35dd5aae679aa) pass all logs to apply_entry_to_state_machine(), not just Normal logs.; by drdr xp; 2021-08-16 4 | 5 | Pass `Entry` to `apply_entry_to_state_machine()`, not just the only 6 | `EntryPayload::Normal(normal_log)`. 7 | 8 | Thus the state machine is able to save the membership changes if it 9 | prefers to. 10 | 11 | Why: 12 | 13 | In practice, a snapshot contains info about all applied logs, including 14 | the membership config log. 15 | Before this change, the state machine does not receive any membership 16 | log thus when making a snapshot, one needs to walk through all applied 17 | logs to get the last membership that is included in state machine. 18 | 19 | By letting the state machine remember the membership log applied, 20 | the snapshto creation becomes more convenient and intuitive: it does not 21 | need to scan the applied logs any more. 22 | -------------------------------------------------------------------------------- /openraft/src/metrics/metric_display.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Formatter; 3 | 4 | use crate::RaftTypeConfig; 5 | use crate::display_ext::DisplayOption; 6 | use crate::metrics::Metric; 7 | 8 | /// Display the value of a metric. 9 | pub(crate) struct MetricDisplay<'a, C> 10 | where C: RaftTypeConfig 11 | { 12 | pub(crate) metric: &'a Metric, 13 | } 14 | 15 | impl fmt::Display for MetricDisplay<'_, C> 16 | where C: RaftTypeConfig 17 | { 18 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 19 | match self.metric { 20 | Metric::Term(v) => write!(f, "{}", v), 21 | Metric::Vote(v) => write!(f, "{}", v), 22 | Metric::LastLogIndex(v) => write!(f, "{}", DisplayOption(v)), 23 | Metric::Applied(v) => write!(f, "{}", DisplayOption(v)), 24 | Metric::AppliedIndex(v) => write!(f, "{}", DisplayOption(v)), 25 | Metric::Snapshot(v) => write!(f, "{}", DisplayOption(v)), 26 | Metric::Purged(v) => write!(f, "{}", DisplayOption(v)), 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /rt-compio/src/oneshot.rs: -------------------------------------------------------------------------------- 1 | //! Oneshot channel wrapper types and their trait impl. 2 | 3 | use openraft::type_config::async_runtime::oneshot; 4 | use openraft::OptionalSend; 5 | 6 | pub struct FuturesOneshot; 7 | 8 | pub struct FuturesOneshotSender(futures::channel::oneshot::Sender); 9 | 10 | impl oneshot::Oneshot for FuturesOneshot { 11 | type Sender = FuturesOneshotSender; 12 | type Receiver = futures::channel::oneshot::Receiver; 13 | type ReceiverError = futures::channel::oneshot::Canceled; 14 | 15 | #[inline] 16 | fn channel() -> (Self::Sender, Self::Receiver) 17 | where T: OptionalSend { 18 | let (tx, rx) = futures::channel::oneshot::channel(); 19 | let tx_wrapper = FuturesOneshotSender(tx); 20 | 21 | (tx_wrapper, rx) 22 | } 23 | } 24 | 25 | impl oneshot::OneshotSender for FuturesOneshotSender 26 | where T: OptionalSend 27 | { 28 | #[inline] 29 | fn send(self, t: T) -> Result<(), T> { 30 | self.0.send(t) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /change-log/v0.7.3.md: -------------------------------------------------------------------------------- 1 | ### Changed: 2 | 3 | - Changed: [25e94c36](https://github.com/databendlabs/openraft/commit/25e94c36e5c8ae640044196070f9a067d5f105a3) InstallSnapshotResponse: replies the last applied log id; Do not install a smaller snapshot; by 张炎泼; 2022-09-22 4 | 5 | A snapshot may not be installed by a follower if it already has a higher 6 | `last_applied` log id locally. 7 | In such a case, it just ignores the snapshot and respond with its local 8 | `last_applied` log id. 9 | 10 | This way the applied state(i.e., `last_applied`) will never revert back. 11 | 12 | ### Fixed: 13 | 14 | - Fixed: [21684bbd](https://github.com/databendlabs/openraft/commit/21684bbdfdc54b18daa68f623afc2b0be6718c72) potential inconsistency when installing snapshot; by 张炎泼; 2022-09-22 15 | 16 | The conflicting logs that are before `snapshot_meta.last_log_id` should 17 | be deleted before installing a snapshot. 18 | 19 | Otherwise there is chance the snapshot is installed but conflicting logs 20 | are left in the store, when a node crashes. 21 | -------------------------------------------------------------------------------- /examples/mem-log/README.md: -------------------------------------------------------------------------------- 1 | # mem-log 2 | 3 | A minimal in-memory implementation of [`RaftLogStorage`](https://docs.rs/openraft/latest/openraft/storage/trait.RaftLogStorage.html). 4 | 5 | ## Overview 6 | 7 | This crate provides only the log storage component for Raft. It implements: 8 | - **`RaftLogStorage`**: Stores and manages Raft log entries in memory 9 | 10 | ## What's NOT included 11 | 12 | - **`RaftStateMachine`**: For a complete working example with state machine implementation, see: 13 | - [`raft-kv-memstore`](../raft-kv-memstore/) - Key-value store example 14 | - [`memstore`](../../stores/memstore/) - Full in-memory storage implementation 15 | 16 | ## Usage 17 | 18 | This crate is used as a storage component by other Openraft examples. It's intentionally minimal to demonstrate log storage in isolation. 19 | 20 | ```rust 21 | use openraft_memlog::LogStore; 22 | 23 | // Create an in-memory log store 24 | let log_store = LogStore::default(); 25 | ``` 26 | 27 | For production use, consider using a persistent storage backend instead of in-memory storage. -------------------------------------------------------------------------------- /openraft/src/progress/bench/vec_progress_update.rs: -------------------------------------------------------------------------------- 1 | extern crate test; 2 | 3 | use test::Bencher; 4 | use test::black_box; 5 | 6 | use crate::progress::Progress; 7 | use crate::progress::VecProgress; 8 | use crate::quorum::Joint; 9 | 10 | #[bench] 11 | fn progress_update_01234_567(b: &mut Bencher) { 12 | let membership: Vec> = vec![vec![0, 1, 2, 3, 4], vec![5, 6, 7]]; 13 | let quorum_set = Joint::from(membership); 14 | let mut progress = VecProgress::::new(quorum_set, 0..=7, || 0); 15 | 16 | let mut id = 0u64; 17 | let mut values = [0, 1, 2, 3, 4, 5, 6, 7]; 18 | b.iter(|| { 19 | id = (id + 1) & 7; 20 | values[id as usize] += 1; 21 | let v = values[id as usize]; 22 | 23 | progress.update(&black_box(id), black_box(v)).ok(); 24 | }); 25 | 26 | // It shows that is_quorum() is called at a rate of about 1/4 of update() 27 | // `Stat { update_count: 42997501, move_count: 10749381, is_quorum_count: 10749399 }` 28 | // println!("progress stat: {:?}", progress.stat()); 29 | } 30 | -------------------------------------------------------------------------------- /rt-monoio/src/oneshot.rs: -------------------------------------------------------------------------------- 1 | //! Oneshot channel wrapper types and their trait impl. 2 | 3 | use local_sync::oneshot as monoio_oneshot; 4 | use openraft::type_config::async_runtime::oneshot; 5 | use openraft::OptionalSend; 6 | 7 | pub struct MonoioOneshot; 8 | 9 | pub struct MonoioOneshotSender(monoio_oneshot::Sender); 10 | 11 | impl oneshot::Oneshot for MonoioOneshot { 12 | type Sender = MonoioOneshotSender; 13 | type Receiver = monoio_oneshot::Receiver; 14 | type ReceiverError = monoio_oneshot::error::RecvError; 15 | 16 | #[inline] 17 | fn channel() -> (Self::Sender, Self::Receiver) 18 | where T: OptionalSend { 19 | let (tx, rx) = monoio_oneshot::channel(); 20 | let tx_wrapper = MonoioOneshotSender(tx); 21 | 22 | (tx_wrapper, rx) 23 | } 24 | } 25 | 26 | impl oneshot::OneshotSender for MonoioOneshotSender 27 | where T: OptionalSend 28 | { 29 | #[inline] 30 | fn send(self, t: T) -> Result<(), T> { 31 | self.0.send(t) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/tests/fixtures/rpc_error_type.rs: -------------------------------------------------------------------------------- 1 | //! RPC error type definitions for test fixtures. 2 | 3 | use anyerror::AnyError; 4 | use openraft::RaftTypeConfig; 5 | use openraft::error::NetworkError; 6 | use openraft::error::RPCError; 7 | use openraft::error::Unreachable; 8 | 9 | use crate::fixtures::Direction; 10 | 11 | /// Type of RPC error to inject during testing. 12 | #[derive(Debug, Clone, Copy)] 13 | pub enum RpcErrorType { 14 | /// Returns [`Unreachable`](`openraft::error::Unreachable`). 15 | Unreachable, 16 | /// Returns [`NetworkError`](`openraft::error::NetworkError`). 17 | NetworkError, 18 | } 19 | 20 | impl RpcErrorType { 21 | pub fn make_error(&self, id: C::NodeId, dir: Direction) -> RPCError 22 | where C: RaftTypeConfig { 23 | let msg = format!("error {} id={}", dir, id); 24 | 25 | match self { 26 | RpcErrorType::Unreachable => Unreachable::new(&AnyError::error(msg)).into(), 27 | RpcErrorType::NetworkError => NetworkError::new(&AnyError::error(msg)).into(), 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /openraft/src/raft_state/tests/validate_test.rs: -------------------------------------------------------------------------------- 1 | use validit::Validate; 2 | 3 | use crate::RaftState; 4 | use crate::engine::LogIdList; 5 | use crate::engine::testing::UTConfig; 6 | use crate::storage::SnapshotMeta; 7 | use crate::type_config::alias::LogIdOf; 8 | 9 | fn log_id(term: u64, index: u64) -> LogIdOf { 10 | crate::engine::testing::log_id(term, 0, index) 11 | } 12 | 13 | #[test] 14 | fn test_raft_state_validate_snapshot_is_none() -> anyhow::Result<()> { 15 | // Some app does not persist snapshot, when restarted, purged is not None but snapshot_last_log_id 16 | // is None. This is a valid state and should not emit error. 17 | 18 | let mut rs = RaftState:: { 19 | log_ids: LogIdList::new(vec![log_id(1, 1), log_id(3, 4)]), 20 | purged_next: 2, 21 | purge_upto: Some(log_id(1, 1)), 22 | snapshot_meta: SnapshotMeta::default(), 23 | ..Default::default() 24 | }; 25 | 26 | rs.apply_progress_mut().accept(log_id(1, 1)); 27 | 28 | assert!(rs.validate().is_ok()); 29 | 30 | Ok(()) 31 | } 32 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/01-getting-started/01-initialize-cluster.md: -------------------------------------------------------------------------------- 1 | ### How to initialize a cluster? 2 | 3 | The simplest and most appropriate way to initialize a cluster is to call 4 | `Raft::initialize()` on **exactly one node**. The other nodes should remain 5 | empty and wait for the initialized node to replicate logs to them. 6 | 7 | Assuming there are three nodes `n1, n2, n3`, there are two approaches: 8 | 9 | 1. **Single-step method**: 10 | Call `Raft::initialize()` on one node (e.g., `n1`) with the configuration of 11 | all three nodes: `n1.initialize(btreeset! {1,2,3})`. 12 | The initialized node will then replicate the membership to the other nodes. 13 | 14 | 2. **Incremental method**: 15 | First, call `Raft::initialize()` on `n1` with configuration containing only `n1` 16 | itself: `n1.initialize(btreeset! {1})`. 17 | Subsequently use `Raft::change_membership()` on `n1` to add `n2` and `n3` 18 | into the cluster. 19 | 20 | The incremental method provides flexibility to start with a single-node 21 | cluster for testing and expand it later for production. 22 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> Result<(), Box> { 2 | println!("cargo:rerun-if-changed=src/*"); 3 | let mut config = prost_build::Config::new(); 4 | config.protoc_arg("--experimental_allow_proto3_optional"); 5 | let proto_files = ["proto/raft.proto", "proto/app_types.proto", "proto/app.proto"]; 6 | 7 | // TODO: remove serde 8 | 9 | tonic_build::configure() 10 | .btree_map(["."]) 11 | .type_attribute("openraftpb.Node", "#[derive(Eq)]") 12 | .type_attribute("openraftpb.SetRequest", "#[derive(Eq)]") 13 | .type_attribute("openraftpb.Response", "#[derive(Eq)]") 14 | .type_attribute("openraftpb.LeaderId", "#[derive(Eq)]") 15 | .type_attribute("openraftpb.Vote", "#[derive(Eq)]") 16 | .type_attribute("openraftpb.NodeIdSet", "#[derive(Eq)]") 17 | .type_attribute("openraftpb.Membership", "#[derive(Eq)]") 18 | .type_attribute("openraftpb.Entry", "#[derive(Eq)]") 19 | .compile_protos_with_config(config, &proto_files, &["proto"])?; 20 | Ok(()) 21 | } 22 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | ### `build_change_log.py` 2 | 3 | Build change log since a previous git-tag 4 | 5 | - Usage: `./scripts/build_change_log.py`: build change log since last tag. 6 | - Usage: `./scripts/build_change_log.py `: build since specified commit. 7 | - Install: `pip install -r requirements.txt`. 8 | 9 | This script builds change log from git commit messages, outputs a versioned 10 | change log to `./change-log/.md` and concatenates all versioned change log 11 | into `./change-log.md`. 12 | 13 | If `./change-log/.md` exists, it skips re-building it from git-log. 14 | Thus you can generate semi-automatic change-logs by editing `./change-log/.md` and running the script again, which will just only the `./change-log.md`. 15 | 16 | 17 | ### `check.kdl` 18 | 19 | Run daily tests in parallel: 20 | 21 | - Usage: `zellij --layout ./scripts/check.kdl`. 22 | - Install: `cargo install zellij`. 23 | 24 | It opens 3 panes to run: 25 | - `cargo test --libs`: run only lib tests 26 | - `cargo test --test *`: run only integration tests. 27 | - `cargo clippy`: lint 28 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore/src/network/raft.rs: -------------------------------------------------------------------------------- 1 | use actix_web::post; 2 | use actix_web::web::Data; 3 | use actix_web::web::Json; 4 | use actix_web::Responder; 5 | use openraft::raft::AppendEntriesRequest; 6 | use openraft::raft::InstallSnapshotRequest; 7 | use openraft::raft::VoteRequest; 8 | 9 | use crate::app::App; 10 | use crate::TypeConfig; 11 | 12 | // --- Raft communication 13 | 14 | #[post("/vote")] 15 | pub async fn vote(app: Data, req: Json>) -> actix_web::Result { 16 | let res = app.raft.vote(req.0).await; 17 | Ok(Json(res)) 18 | } 19 | 20 | #[post("/append")] 21 | pub async fn append(app: Data, req: Json>) -> actix_web::Result { 22 | let res = app.raft.append_entries(req.0).await; 23 | Ok(Json(res)) 24 | } 25 | 26 | #[post("/snapshot")] 27 | pub async fn snapshot( 28 | app: Data, 29 | req: Json>, 30 | ) -> actix_web::Result { 31 | let res = app.raft.install_snapshot(req.0).await; 32 | Ok(Json(res)) 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /stores/memstore/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-memstore" 3 | description = "A in-memory implementation of the `openraft::RaftLogStorage` and `openraft::RaftStateMachine` trait." 4 | documentation = "https://docs.rs/openraft-memstore" 5 | readme = "README.md" 6 | 7 | version = { workspace = true } 8 | edition = { workspace = true } 9 | authors = { workspace = true } 10 | categories = { workspace = true } 11 | homepage = { workspace = true } 12 | keywords = { workspace = true } 13 | license = { workspace = true } 14 | repository = { workspace = true } 15 | 16 | [dependencies] 17 | openraft = { path = "../../openraft", version = "0.10.0", features = ["serde", "type-alias"] } 18 | 19 | derive_more = { workspace = true } 20 | futures = { workspace = true } 21 | serde = { workspace = true } 22 | serde_json = { workspace = true } 23 | tokio = { workspace = true } 24 | tracing = { workspace = true } 25 | 26 | [dev-dependencies] 27 | 28 | [features] 29 | bt = ["openraft/bt"] 30 | single-term-leader = [] 31 | 32 | [package.metadata.docs.rs] 33 | all-features = true 34 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/06-operations/04-custom-node-info.md: -------------------------------------------------------------------------------- 1 | ### How do I store additional information about nodes in Openraft? 2 | 3 | By default, Openraft provide a [`BasicNode`] as the node type in a cluster. 4 | To store more information about each node in Openraft, define a custom struct 5 | with the desired fields and use it in place of `BasicNode`. Here's a brief 6 | guide: 7 | 8 | 1. Define your custom node struct: 9 | 10 | ```rust,ignore 11 | #[derive(...)] 12 | struct MyNode { 13 | ipv4: String, 14 | ipv6: String, 15 | port: u16, 16 | // Add additional fields as needed 17 | } 18 | ``` 19 | 20 | 2. Register the custom node type with `declare_raft_types!` macro: 21 | 22 | ```rust,ignore 23 | openraft::declare_raft_types!( 24 | pub MyRaftConfig: 25 | // ... 26 | NodeId = u64, // Use the appropriate type for NodeId 27 | Node = MyNode, // Replace BasicNode with your custom node type 28 | // ... other associated types 29 | ); 30 | ``` 31 | 32 | Use `MyRaftConfig` in your Raft setup to utilize the custom node structure. 33 | 34 | [`BasicNode`]: `crate::node::BasicNode` 35 | -------------------------------------------------------------------------------- /openraft/src/core/io_flush_tracking/mod.rs: -------------------------------------------------------------------------------- 1 | //! I/O flush progress tracking. 2 | //! 3 | //! This module provides watch-based notification channels for tracking when Raft I/O operations 4 | //! (vote saves and log appends) are flushed to storage. It enables applications to: 5 | //! 6 | //! - Wait for specific log entries to be durably written 7 | //! - Track vote changes across leader elections 8 | //! - Ensure data persistence before responding to clients 9 | //! 10 | //! The tracking is based on monotonically increasing [`crate::raft_state::IOId`] values that 11 | //! identify each I/O operation. When storage completes an operation, it notifies RaftCore, which 12 | //! updates the progress channels. 13 | 14 | mod flush_point; 15 | mod sender; 16 | mod watch_progress; 17 | mod watcher; 18 | 19 | pub use flush_point::FlushPoint; 20 | pub(crate) use sender::IoProgressSender; 21 | pub use watch_progress::AppliedProgress; 22 | pub use watch_progress::CommitProgress; 23 | pub use watch_progress::LogProgress; 24 | pub use watch_progress::SnapshotProgress; 25 | pub use watch_progress::VoteProgress; 26 | pub(crate) use watcher::IoProgressWatcher; 27 | -------------------------------------------------------------------------------- /openraft/src/error/linearizable_read_error.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::TryAsRef; 3 | use crate::error::ForwardToLeader; 4 | use crate::error::QuorumNotEnough; 5 | 6 | /// An error related to an is_leader request. 7 | #[derive(Debug, Clone, thiserror::Error, derive_more::TryInto)] 8 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 9 | pub enum LinearizableReadError 10 | where C: RaftTypeConfig 11 | { 12 | /// This node is not the leader; request should be forwarded to the leader. 13 | #[error(transparent)] 14 | ForwardToLeader(#[from] ForwardToLeader), 15 | 16 | /// Cannot finish a request, such as elect or replicate, because a quorum is not available. 17 | #[error(transparent)] 18 | QuorumNotEnough(#[from] QuorumNotEnough), 19 | } 20 | 21 | impl TryAsRef> for LinearizableReadError 22 | where C: RaftTypeConfig 23 | { 24 | fn try_as_ref(&self) -> Option<&ForwardToLeader> { 25 | match self { 26 | Self::ForwardToLeader(f) => Some(f), 27 | _ => None, 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Unit test coverage 2 | on: 3 | push: 4 | branches: 5 | - "release-*" 6 | - "main" 7 | 8 | jobs: 9 | coverage: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - uses: actions-rust-lang/setup-rust-toolchain@v1 15 | with: 16 | toolchain: nightly 17 | components: llvm-tools-preview 18 | 19 | 20 | - name: Install cargo-llvm-cov 21 | uses: taiki-e/install-action@cargo-llvm-cov 22 | 23 | 24 | - name: Install cargo-expand 25 | run: cargo install cargo-expand 26 | 27 | 28 | - name: Generate coverage report 29 | # To test with all features: 30 | # run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info 31 | run: cargo llvm-cov --workspace --lcov --output-path lcov.info 32 | env: 33 | RUST_TEST_THREADS: 2 34 | 35 | 36 | - name: Upload to Coveralls 37 | uses: coverallsapp/github-action@v2 38 | with: 39 | github-token: ${{ secrets.GITHUB_TOKEN }} 40 | path-to-lcov: lcov.info 41 | -------------------------------------------------------------------------------- /openraft/src/raft/message/mod.rs: -------------------------------------------------------------------------------- 1 | //! Raft protocol messages and types. 2 | //! 3 | //! Request and response types for an application to talk to the Raft, 4 | //! and are also used by network layer to talk to other Raft nodes. 5 | 6 | mod append_entries_request; 7 | mod append_entries_response; 8 | mod install_snapshot; 9 | mod stream_append_error; 10 | mod transfer_leader; 11 | mod vote; 12 | mod write; 13 | 14 | mod client_write; 15 | mod write_request; 16 | 17 | pub use append_entries_request::AppendEntriesRequest; 18 | pub use append_entries_response::AppendEntriesResponse; 19 | pub use client_write::ClientWriteResponse; 20 | pub use client_write::ClientWriteResult; 21 | pub use install_snapshot::InstallSnapshotRequest; 22 | pub use install_snapshot::InstallSnapshotResponse; 23 | pub use install_snapshot::SnapshotResponse; 24 | pub use stream_append_error::StreamAppendError; 25 | pub use transfer_leader::TransferLeaderRequest; 26 | pub use vote::VoteRequest; 27 | pub use vote::VoteResponse; 28 | pub use write::WriteResponse; 29 | pub use write::WriteResult; 30 | pub(crate) use write::into_write_result; 31 | pub use write_request::WriteRequest; 32 | -------------------------------------------------------------------------------- /rt-compio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "openraft-rt-compio" 3 | description = "compio AsyncRuntime support for Openraft" 4 | documentation = "https://docs.rs/openraft-rt-compio" 5 | readme = "README.md" 6 | version = "0.10.0" 7 | edition = "2021" 8 | authors = ["Databend Authors "] 9 | categories = ["algorithms", "asynchronous", "data-structures"] 10 | homepage = "https://github.com/databendlabs/openraft" 11 | keywords = ["consensus", "raft"] 12 | license = "MIT OR Apache-2.0" 13 | repository = "https://github.com/databendlabs/openraft" 14 | 15 | [dependencies] 16 | openraft = { path = "../openraft", version = "0.10.0", default-features = false, features = ["singlethreaded"] } 17 | 18 | compio = { version = ">=0.14.0", features = ["runtime", "time"] } 19 | flume = { version = "0.11.1", default-features = false, features = ["async"] } 20 | futures = { version = "0.3" } 21 | pin-project-lite = { version = "0.2.16" } 22 | rand = { version = "0.9" } 23 | see = { version = "0.1.1" } 24 | 25 | [dev-dependencies] 26 | compio = { version = ">=0.14.0", features = ["macros"] } 27 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/04-storage/02-snapshot-building.md: -------------------------------------------------------------------------------- 1 | ### How does Openraft handle snapshot building and transfer? 2 | 3 | Openraft calls [`RaftStateMachine::get_snapshot_builder`][] to create snapshots. The builder runs 4 | concurrently with [`RaftStateMachine::apply`][], so your implementation must handle concurrent access 5 | to the state machine data. 6 | 7 | When a follower is more than [`Config::replication_lag_threshold`][] entries behind, the leader 8 | sends a snapshot instead of individual log entries. 9 | 10 | For large snapshots that timeout during transfer, increase [`Config::install_snapshot_timeout`][]. 11 | The snapshot is sent in chunks of [`Config::snapshot_max_chunk_size`][] bytes. 12 | 13 | [`RaftStateMachine::get_snapshot_builder`]: `crate::storage::RaftStateMachine::get_snapshot_builder` 14 | [`RaftStateMachine::apply`]: `crate::storage::RaftStateMachine::apply` 15 | [`Config::replication_lag_threshold`]: `crate::config::Config::replication_lag_threshold` 16 | [`Config::install_snapshot_timeout`]: `crate::config::Config::install_snapshot_timeout` 17 | [`Config::snapshot_max_chunk_size`]: `crate::config::Config::snapshot_max_chunk_size` 18 | -------------------------------------------------------------------------------- /openraft/src/error/invalid_sm.rs: -------------------------------------------------------------------------------- 1 | /// Error indicating that a state machine operation was attempted with an incompatible type. 2 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 3 | #[error( 4 | "User-defined function on the state machine failed to run; \ 5 | It may have used a different type \ 6 | of state machine from the one in RaftCore (`{actual_type}`)" 7 | )] 8 | pub struct InvalidStateMachineType { 9 | /// The actual type name of the state machine that was used. 10 | pub actual_type: &'static str, 11 | } 12 | 13 | impl InvalidStateMachineType { 14 | pub(crate) fn new() -> Self { 15 | Self { 16 | actual_type: std::any::type_name::(), 17 | } 18 | } 19 | } 20 | 21 | #[cfg(test)] 22 | mod tests { 23 | #[test] 24 | fn test_invalid_state_machine_type_to_string() { 25 | let err = super::InvalidStateMachineType::new::(); 26 | assert_eq!( 27 | err.to_string(), 28 | "User-defined function on the state machine failed to run; It may have used a different type of state machine from the one in RaftCore (`u32`)" 29 | ); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /openraft/src/testing/log/store_builder.rs: -------------------------------------------------------------------------------- 1 | use openraft_macros::add_async_trait; 2 | 3 | use crate::RaftTypeConfig; 4 | use crate::StorageError; 5 | use crate::storage::RaftLogStorage; 6 | use crate::storage::RaftStateMachine; 7 | 8 | /// The trait to build a [`RaftLogStorage`] and [`RaftStateMachine`] implementation. 9 | /// 10 | /// The generic parameter `C` is type config for a `RaftLogStorage` and `RaftStateMachine` 11 | /// implementation, 12 | /// `LS` is the type that implements `RaftLogStorage`, 13 | /// `SM` is the type that implements `RaftStateMachine`, 14 | /// and `G` is a guard type that cleans up resources when being dropped. 15 | /// 16 | /// By default, `G` is a trivial guard `()`. To test a store that is backed by a folder on disk, `G` 17 | /// could be the dropper of the temp-dir that stores data. 18 | #[add_async_trait] 19 | pub trait StoreBuilder: Send + Sync 20 | where 21 | C: RaftTypeConfig, 22 | LS: RaftLogStorage, 23 | SM: RaftStateMachine, 24 | { 25 | /// Build a [`RaftLogStorage`] and [`RaftStateMachine`] implementation 26 | async fn build(&self) -> Result<(G, LS, SM), StorageError>; 27 | } 28 | -------------------------------------------------------------------------------- /openraft/src/raft_types.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | use std::fmt::Formatter; 3 | 4 | /// Id of a snapshot stream. 5 | /// 6 | /// Everytime a snapshot is created, it is assigned with a globally unique id. 7 | /// Such an id is used by followers to distinguish different install-snapshot streams. 8 | pub type SnapshotId = String; 9 | 10 | /// The identity of a segment of a snapshot. 11 | #[derive(Debug, Clone, PartialOrd, PartialEq, Eq)] 12 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 13 | pub struct SnapshotSegmentId { 14 | /// The unique identifier of the snapshot stream. 15 | pub id: SnapshotId, 16 | 17 | /// The offset of this segment in the entire snapshot data. 18 | pub offset: u64, 19 | } 20 | 21 | impl From<(D, u64)> for SnapshotSegmentId { 22 | fn from(v: (D, u64)) -> Self { 23 | SnapshotSegmentId { 24 | id: v.0.to_string(), 25 | offset: v.1, 26 | } 27 | } 28 | } 29 | 30 | impl Display for SnapshotSegmentId { 31 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 32 | write!(f, "{}+{}", self.id, self.offset) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /openraft/src/base/histogram/percentile_stats.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Percentile statistics for a histogram. 4 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 5 | pub struct PercentileStats { 6 | /// Number of samples recorded 7 | pub samples: u64, 8 | /// 0.1th percentile (99.9% of values >= this) 9 | pub p0_1: u64, 10 | /// 1st percentile (99% of values >= this) 11 | pub p1: u64, 12 | /// 5th percentile (95% of values >= this) 13 | pub p5: u64, 14 | /// 10th percentile (90% of values >= this) 15 | pub p10: u64, 16 | /// 50th percentile (median) 17 | pub p50: u64, 18 | /// 90th percentile 19 | pub p90: u64, 20 | /// 99th percentile 21 | pub p99: u64, 22 | /// 99.9th percentile 23 | pub p99_9: u64, 24 | } 25 | 26 | impl fmt::Display for PercentileStats { 27 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 28 | write!( 29 | f, 30 | "[samples: {}, P0.1: {}, P1: {}, P5: {}, P10: {}, P50: {}, P90: {}, P99: {}, P99.9: {}]", 31 | self.samples, self.p0_1, self.p1, self.p5, self.p10, self.p50, self.p90, self.p99, self.p99_9 32 | ) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /openraft/src/type_config/async_runtime/watch/watch_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error returned by the `WatchSender`. 4 | #[derive(PartialEq, Eq, Clone, Copy)] 5 | pub struct SendError(pub T); 6 | 7 | impl fmt::Debug for SendError { 8 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 9 | f.debug_struct("SendError").finish_non_exhaustive() 10 | } 11 | } 12 | 13 | impl fmt::Display for SendError { 14 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 15 | write!(fmt, "watch channel closed") 16 | } 17 | } 18 | 19 | impl std::error::Error for SendError {} 20 | 21 | /// Error returned by the `WatchReceiver`. 22 | #[derive(PartialEq, Eq, Clone, Copy)] 23 | pub struct RecvError(pub ()); 24 | 25 | impl fmt::Debug for RecvError { 26 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 27 | f.debug_struct("RecvError").finish_non_exhaustive() 28 | } 29 | } 30 | 31 | impl fmt::Display for RecvError { 32 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 33 | write!(fmt, "watch channel closed") 34 | } 35 | } 36 | 37 | impl std::error::Error for RecvError {} 38 | -------------------------------------------------------------------------------- /change-log/v0.6.2-alpha.2.md: -------------------------------------------------------------------------------- 1 | ### Dependency: 2 | 3 | - Dependency: [70e1773e](https://github.com/databendlabs/openraft/commit/70e1773edcf5e2bc7369c7afe47bb1348bc2a274) adapt to changes of rand-0.8: gen_range() accepts a range instead of two args; by drdr xp; 2021-06-21 4 | 5 | ### Added: 6 | 7 | - Added: [32a67e22](https://github.com/databendlabs/openraft/commit/32a67e228cf26f9207593805a0386cf6aa4fe294) add metrics about leader; by drdr xp; 2021-06-29 8 | 9 | In LeaderState it also report metrics about the replication to other node when report metrics. 10 | 11 | When switched to other state, LeaderState will be destroyed as long as 12 | the cached replication metrics. 13 | 14 | Other state report an `None` to raft core to override the previous 15 | metrics data. 16 | 17 | At some point the raft core, without knonwning the state, just report 18 | metrics with an `Update::Ignore`, to indicate that leave replication 19 | metrics intact. 20 | 21 | ### Fixed: 22 | 23 | - Fixed: [d60f1e85](https://github.com/databendlabs/openraft/commit/d60f1e852d3e5b9455589593067599d261f695b2) client_read has using wrong quorum=majority-1; by drdr xp; 2021-07-02 24 | -------------------------------------------------------------------------------- /tests/tests/membership/t52_change_membership_on_uninitialized_node.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreemap; 5 | use openraft::ChangeMembers; 6 | use openraft::Config; 7 | 8 | use crate::fixtures::RaftRouter; 9 | use crate::fixtures::ut_harness; 10 | 11 | /// Call `Raft::change_membership()` on an uninitialized node should not panic due to empty 12 | /// membership. 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn change_membership_on_uninitialized_node() -> Result<()> { 16 | let config = Arc::new( 17 | Config { 18 | enable_heartbeat: false, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | router.new_raft_node(0).await; 26 | 27 | let n0 = router.get_raft_handle(&0)?; 28 | let res = n0.change_membership(ChangeMembers::AddVoters(btreemap! {0=>()}), false).await; 29 | tracing::info!("{:?}", res); 30 | 31 | let err = res.unwrap_err(); 32 | tracing::info!("{}", err); 33 | 34 | assert!(err.to_string().contains("forward request to")); 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /scripts/check-parallel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LOG_DIR=".check-logs" 4 | mkdir -p "$LOG_DIR" 5 | 6 | echo "Running checks in parallel..." 7 | 8 | cargo test --lib > "$LOG_DIR/test-lib.log" 2>&1 & 9 | PID1=$! 10 | 11 | cargo test --test '*' > "$LOG_DIR/test-integration.log" 2>&1 & 12 | PID2=$! 13 | 14 | cargo clippy --no-deps --all-targets -- -D warnings > "$LOG_DIR/clippy.log" 2>&1 & 15 | PID3=$! 16 | 17 | wait $PID1; EXIT1=$? 18 | wait $PID2; EXIT2=$? 19 | wait $PID3; EXIT3=$? 20 | 21 | echo "" 22 | echo "=== Results ===" 23 | if [ $EXIT1 -eq 0 ]; then 24 | echo "✓ Library tests passed" 25 | else 26 | echo "✗ Library tests failed (exit $EXIT1). See $LOG_DIR/test-lib.log" 27 | fi 28 | 29 | if [ $EXIT2 -eq 0 ]; then 30 | echo "✓ Integration tests passed" 31 | else 32 | echo "✗ Integration tests failed (exit $EXIT2). See $LOG_DIR/test-integration.log" 33 | fi 34 | 35 | if [ $EXIT3 -eq 0 ]; then 36 | echo "✓ Clippy passed" 37 | else 38 | echo "✗ Clippy failed (exit $EXIT3). See $LOG_DIR/clippy.log" 39 | fi 40 | 41 | TOTAL_EXIT=$((EXIT1 + EXIT2 + EXIT3)) 42 | if [ $TOTAL_EXIT -eq 0 ]; then 43 | echo "" 44 | echo "All checks passed!" 45 | fi 46 | 47 | exit $TOTAL_EXIT 48 | -------------------------------------------------------------------------------- /openraft/src/docs/faq/README.md: -------------------------------------------------------------------------------- 1 | # FAQ Structure 2 | 3 | Individual markdown files organized by numbered category directories. 4 | 5 | ## Structure 6 | 7 | ``` 8 | NN-category-name/ 9 | ├── README.md # Section name (e.g., "Getting Started") 10 | └── NN-*.md # FAQ entries 11 | ``` 12 | 13 | Categories auto-discovered by `build_faq.py` (sorted by directory name). 14 | 15 | ## Build 16 | 17 | ```bash 18 | make 19 | ``` 20 | 21 | Generates `faq.md` and `faq-toc.md` from individual files. 22 | 23 | ## Add FAQ Entry 24 | 25 | 1. Create `NN-my-faq.md` in appropriate category directory 26 | 2. Write FAQ with `###` header 27 | 3. Add link definitions at bottom: 28 | ```markdown 29 | ### My FAQ Title 30 | 31 | Content with [`SomeType`][] references. 32 | 33 | [`SomeType`]: `crate::path::SomeType` 34 | ``` 35 | 4. Run `make` 36 | 37 | ## Add Category 38 | 39 | 1. Create `NN-category-name/` directory 40 | 2. Add `README.md` with section name 41 | 3. Add FAQ markdown files 42 | 4. Run `make` 43 | 44 | ## Notes 45 | 46 | - Each FAQ file is self-contained with its own link definitions 47 | - Never edit `faq.md` or `faq-toc.md` directly 48 | - Number prefix controls ordering 49 | -------------------------------------------------------------------------------- /openraft/src/progress/display_vec_progress.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Borrow; 2 | use std::fmt::Display; 3 | use std::fmt::Formatter; 4 | 5 | use super::Progress; 6 | use super::vec_progress::VecProgress; 7 | use crate::quorum::QuorumSet; 8 | 9 | pub(crate) struct DisplayVecProgress<'a, ID, Ent, Prog, QS, Fmt> 10 | where 11 | ID: 'static, 12 | QS: QuorumSet, 13 | Fmt: Fn(&mut Formatter<'_>, &ID, &Ent) -> std::fmt::Result, 14 | { 15 | pub(crate) inner: &'a VecProgress, 16 | pub(crate) f: Fmt, 17 | } 18 | 19 | impl Display for DisplayVecProgress<'_, ID, Ent, Prog, QS, Fmt> 20 | where 21 | ID: PartialEq + 'static, 22 | Ent: Borrow, 23 | Prog: PartialOrd + Copy, 24 | QS: QuorumSet, 25 | Fmt: Fn(&mut Formatter<'_>, &ID, &Ent) -> std::fmt::Result, 26 | { 27 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 28 | write!(f, "{{")?; 29 | for (i, item) in self.inner.iter().enumerate() { 30 | if i > 0 { 31 | write!(f, ", ")?; 32 | } 33 | (self.f)(f, &item.id, &item.val)?; 34 | } 35 | write!(f, "}}")?; 36 | 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/tests/append_entries/t61_large_heartbeat.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use anyhow::Result; 5 | use maplit::btreeset; 6 | use openraft::Config; 7 | 8 | use crate::fixtures::RaftRouter; 9 | use crate::fixtures::ut_harness; 10 | 11 | /// Large heartbeat should not block replication. 12 | /// I.e., replication should not be driven by heartbeat. 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn large_heartbeat() -> Result<()> { 16 | // Setup test dependencies. 17 | let config = Arc::new( 18 | Config { 19 | heartbeat_interval: 10_000, 20 | election_timeout_min: 20_000, 21 | election_timeout_max: 30_000, 22 | max_payload_entries: 2, 23 | ..Default::default() 24 | } 25 | .validate()?, 26 | ); 27 | let mut router = RaftRouter::new(config.clone()); 28 | 29 | let mut log_index = router.new_cluster(btreeset! {0}, btreeset! {1}).await?; 30 | 31 | router.client_request_many(0, "foo", 10).await?; 32 | log_index += 10; 33 | 34 | router.wait(&1, Some(Duration::from_millis(3_000))).applied_index(Some(log_index), "").await?; 35 | 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /openraft/src/raft/message/stream_append_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::RaftTypeConfig; 4 | use crate::type_config::alias::LogIdOf; 5 | use crate::type_config::alias::VoteOf; 6 | 7 | /// Error type for stream append entries. 8 | /// 9 | /// When this error is returned, the stream is terminated. 10 | #[derive(Debug, Clone, PartialEq, Eq)] 11 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 12 | pub enum StreamAppendError { 13 | /// Log conflict at the given prev_log_id. 14 | /// 15 | /// The follower's log at this position does not match the leader's. 16 | Conflict(LogIdOf), 17 | 18 | /// Seen a higher vote from another leader. 19 | HigherVote(VoteOf), 20 | } 21 | 22 | impl fmt::Display for StreamAppendError 23 | where C: RaftTypeConfig 24 | { 25 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 26 | match self { 27 | StreamAppendError::Conflict(log_id) => { 28 | write!(f, "Conflict({})", log_id) 29 | } 30 | StreamAppendError::HigherVote(vote) => { 31 | write!(f, "HigherVote({})", vote) 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # unstable_features = true 2 | # edition = "2018" 3 | 4 | # fn_args_layout = "Compressed" 5 | # use_small_heuristics = "Default" 6 | # use_try_shorthand = true 7 | 8 | # # pre-unstable 9 | 10 | # single_line_if_else_max_width = 75 11 | # space_around_attr_eq = false 12 | # struct_lit_width = 50 13 | 14 | # # unstable 15 | 16 | # condense_wildcard_suffixes = true 17 | # format_code_in_doc_comments = true 18 | # format_strings = true 19 | # match_block_trailing_comma = false 20 | # normalize_comments = true 21 | # normalize_doc_attributes = true 22 | # reorder_impl_items = true 23 | # struct_lit_single_line = true 24 | # use_field_init_shorthand = true 25 | 26 | 27 | reorder_imports = true 28 | imports_granularity = "Item" 29 | group_imports = "StdExternalCrate" 30 | where_single_line = true 31 | trailing_comma = "Vertical" 32 | overflow_delimited_expr = true 33 | wrap_comments = true 34 | comment_width = 100 35 | max_width = 120 36 | inline_attribute_width = 0 37 | 38 | merge_derives = false 39 | 40 | # pre-unstable 41 | 42 | chain_width = 100 43 | -------------------------------------------------------------------------------- /openraft/src/progress/inflight_id.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Formatter; 3 | use std::ops::Deref; 4 | 5 | /// Unique identifier for an inflight replication request. 6 | /// 7 | /// Each time the leader sends logs or a snapshot to a follower, it assigns an `InflightId`. 8 | /// When the follower responds, the response carries the same `InflightId`, allowing the leader 9 | /// to correctly match responses to their corresponding requests. 10 | /// 11 | /// This prevents stale responses from incorrectly updating the progress state. For example, 12 | /// if a slow response from a previous request arrives after a new request has been sent, 13 | /// the mismatched `InflightId` causes the stale response to be ignored. 14 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 15 | pub(crate) struct InflightId { 16 | id: u64, 17 | } 18 | 19 | impl InflightId { 20 | pub(crate) fn new(id: u64) -> Self { 21 | Self { id } 22 | } 23 | } 24 | 25 | impl Deref for InflightId { 26 | type Target = u64; 27 | 28 | fn deref(&self) -> &Self::Target { 29 | &self.id 30 | } 31 | } 32 | 33 | impl fmt::Display for InflightId { 34 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 35 | write!(f, "{}", self.id) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /openraft/src/progress/stream_id.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::Formatter; 3 | use std::ops::Deref; 4 | 5 | /// Unique identifier for an inflight replication request. 6 | /// 7 | /// Each time the leader sends logs or a snapshot to a follower, it assigns an `InflightId`. 8 | /// When the follower responds, the response carries the same `InflightId`, allowing the leader 9 | /// to correctly match responses to their corresponding requests. 10 | /// 11 | /// This prevents stale responses from incorrectly updating the progress state. For example, 12 | /// if a slow response from a previous request arrives after a new request has been sent, 13 | /// the mismatched `InflightId` causes the stale response to be ignored. 14 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 15 | pub(crate) struct StreamId { 16 | id: u64, 17 | } 18 | 19 | impl StreamId { 20 | pub(crate) fn new(id: u64) -> Self { 21 | Self { id } 22 | } 23 | } 24 | 25 | impl Deref for StreamId { 26 | type Target = u64; 27 | 28 | fn deref(&self) -> &Self::Target { 29 | &self.id 30 | } 31 | } 32 | 33 | impl fmt::Display for StreamId { 34 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 35 | write!(f, "StreamId({})", self.id) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-grpc/src/pb_impl/impl_membership.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::collections::BTreeSet; 3 | 4 | use openraft::Membership; 5 | 6 | use crate::pb; 7 | use crate::TypeConfig; 8 | 9 | impl From for Membership { 10 | fn from(value: pb::Membership) -> Self { 11 | let mut configs = vec![]; 12 | for c in value.configs { 13 | let config: BTreeSet = c.node_ids.keys().copied().collect(); 14 | configs.push(config); 15 | } 16 | let nodes = value.nodes; 17 | // TODO: do not unwrap() 18 | Membership::new(configs, nodes).unwrap() 19 | } 20 | } 21 | 22 | impl From> for pb::Membership { 23 | fn from(value: Membership) -> Self { 24 | let mut configs = vec![]; 25 | for c in value.get_joint_config() { 26 | let mut node_ids = BTreeMap::new(); 27 | for nid in c.iter() { 28 | node_ids.insert(*nid, ()); 29 | } 30 | configs.push(pb::NodeIdSet { node_ids }); 31 | } 32 | let nodes = value.nodes().map(|(nid, n)| (*nid, n.clone())).collect(); 33 | pb::Membership { configs, nodes } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /openraft/src/raft/responder/core_responder.rs: -------------------------------------------------------------------------------- 1 | use crate::LogId; 2 | use crate::RaftTypeConfig; 3 | use crate::impls::ProgressResponder; 4 | use crate::raft::ClientWriteResult; 5 | use crate::raft::responder::Responder; 6 | use crate::type_config::alias::WriteResponderOf; 7 | 8 | /// The responder used in RaftCore. 9 | /// 10 | /// RaftCore use this responder to send response to the caller. 11 | /// It is either a progress responder or a user-defined responder. 12 | pub(crate) enum CoreResponder 13 | where C: RaftTypeConfig 14 | { 15 | Progress(ProgressResponder>), 16 | UserDefined(WriteResponderOf), 17 | } 18 | 19 | impl Responder> for CoreResponder 20 | where C: RaftTypeConfig 21 | { 22 | fn on_commit(&mut self, log_id: LogId) { 23 | match self { 24 | Self::Progress(responder) => responder.on_commit(log_id), 25 | Self::UserDefined(responder) => responder.on_commit(log_id), 26 | } 27 | } 28 | 29 | fn on_complete(self, res: ClientWriteResult) { 30 | match self { 31 | Self::Progress(responder) => responder.on_complete(res), 32 | Self::UserDefined(responder) => responder.on_complete(res), 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-singlethreaded/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-singlethreaded" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | description = "An example distributed key-value store built upon `openraft`." 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | [dependencies] 19 | openraft = { path = "../../openraft", features = ["serde", "singlethreaded", "type-alias"] } 20 | 21 | futures = { version = "0.3" } 22 | serde = { version = "1.0.114", features = ["derive"] } 23 | serde_json = { version = "1.0.57" } 24 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 25 | tracing = { version = "0.1.29" } 26 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 27 | 28 | [dev-dependencies] 29 | maplit = { version = "1.0.2" } 30 | 31 | [features] 32 | 33 | [package.metadata.docs.rs] 34 | all-features = true 35 | -------------------------------------------------------------------------------- /examples/multi-raft-kv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "multi-raft-kv" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "AriesDevil ", 9 | ] 10 | categories = ["algorithms", "asynchronous", "data-structures"] 11 | description = "An example Multi-Raft distributed key-value store with 3 groups built upon `openraft`." 12 | homepage = "https://github.com/databendlabs/openraft" 13 | keywords = ["raft", "consensus", "multi-raft"] 14 | license = "MIT OR Apache-2.0" 15 | repository = "https://github.com/databendlabs/openraft" 16 | 17 | [dependencies] 18 | mem-log = { path = "../mem-log", features = [] } 19 | openraft = { path = "../../openraft", default-features = false, features = ["serde", "type-alias"] } 20 | openraft-multi = { path = "../../multiraft" } 21 | 22 | futures = { version = "0.3" } 23 | serde = { version = "1", features = ["derive"] } 24 | serde_json = { version = "1" } 25 | tokio = { version = "1", default-features = false, features = ["sync"] } 26 | tracing = { version = "0.1" } 27 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 28 | 29 | 30 | [features] 31 | 32 | [package.metadata.docs.rs] 33 | all-features = true 34 | 35 | -------------------------------------------------------------------------------- /openraft/src/docs/obsolete/fast_commit.md: -------------------------------------------------------------------------------- 1 | ### Fast Commit(obsolete) 2 | 3 | **This algorithm is no longer used**, because it would be more flexible treating 4 | local-IO the same as replication to a remote node. As a result, applying a log does not 5 | need to wait for it to be flushed on local disk. 6 | 7 | #### Fast Commit Algorithm 8 | 9 | The algorithm assumes that an entry will be committed as soon as it is appended 10 | if the cluster has only one voter. 11 | 12 | However, if there is a membership log in the middle of the input entries, the 13 | condition to commit will change, and entries before and after the membership 14 | entry must be dealt with differently. 15 | 16 | When a membership entry is seen, update progress for all former entries. 17 | Then upgrade the quorum set for the Progress. 18 | 19 | E.g., if the input entries are `2..6`, entry 4 changes membership from `a` to `abc`. 20 | Then it will output a LeaderCommit command to commit entries `2,3`. 21 | 22 | ```text 23 | 1 2 3 4 5 6 24 | a x x a y y 25 | b 26 | c 27 | ``` 28 | 29 | If the input entries are `2..6`, entry 4 changes membership from `abc` to `a`. 30 | Then it will output a LeaderCommit command to commit entries `2,3,4,5,6`. 31 | 32 | ```text 33 | 1 2 3 4 5 6 34 | a x x a y y 35 | b 36 | c 37 | ``` 38 | -------------------------------------------------------------------------------- /openraft/src/error/membership_error.rs: -------------------------------------------------------------------------------- 1 | use crate::RaftTypeConfig; 2 | use crate::error::ChangeMembershipError; 3 | use crate::error::EmptyMembership; 4 | use crate::error::LearnerNotFound; 5 | use crate::error::NodeNotFound; 6 | 7 | /// Errors occur when building a [`Membership`]. 8 | /// 9 | /// [`Membership`]: crate::membership::Membership 10 | #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] 11 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(bound = ""))] 12 | pub enum MembershipError { 13 | /// The membership configuration is empty. 14 | #[error(transparent)] 15 | EmptyMembership(#[from] EmptyMembership), 16 | 17 | /// A required node was not found. 18 | #[error(transparent)] 19 | NodeNotFound(#[from] NodeNotFound), 20 | } 21 | 22 | impl From> for ChangeMembershipError 23 | where C: RaftTypeConfig 24 | { 25 | fn from(me: MembershipError) -> Self { 26 | match me { 27 | MembershipError::EmptyMembership(e) => ChangeMembershipError::EmptyMembership(e), 28 | MembershipError::NodeNotFound(e) => { 29 | ChangeMembershipError::LearnerNotFound(LearnerNotFound { node_id: e.node_id }) 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/tests/management/t10_raft_config.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::RaftRouter; 8 | use crate::fixtures::ut_harness; 9 | 10 | /// Get config via [`Raft::config`](openraft::Raft::config) 11 | #[tracing::instrument] 12 | #[test_harness::test(harness = ut_harness)] 13 | async fn raft_config() -> Result<()> { 14 | let config = Arc::new( 15 | Config { 16 | enable_tick: false, 17 | election_timeout_min: 123, 18 | election_timeout_max: 124, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | 26 | tracing::info!("--- initializing cluster"); 27 | let log_index = router.new_cluster(btreeset! {0}, btreeset! {}).await?; 28 | 29 | tracing::info!(log_index, "--- get config"); 30 | { 31 | let n0 = router.get_raft_handle(&0)?; 32 | let c = n0.config(); 33 | 34 | #[allow(clippy::bool_assert_comparison)] 35 | { 36 | assert_eq!(c.enable_tick, false); 37 | } 38 | assert_eq!(c.election_timeout_min, 123); 39 | assert_eq!(c.election_timeout_max, 124); 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /examples/raft-kv-memstore-network-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft-kv-memstore-network-v2" 3 | version = "0.1.0" 4 | readme = "README.md" 5 | 6 | edition = "2021" 7 | authors = [ 8 | "drdr xp ", 9 | "Pedro Paulo de Amorim ", 10 | ] 11 | categories = ["algorithms", "asynchronous", "data-structures"] 12 | description = "An example distributed key-value store built upon `openraft`." 13 | homepage = "https://github.com/databendlabs/openraft" 14 | keywords = ["raft", "consensus"] 15 | license = "MIT OR Apache-2.0" 16 | repository = "https://github.com/databendlabs/openraft" 17 | 18 | [dependencies] 19 | mem-log = { path = "../mem-log", features = [] } 20 | openraft = { path = "../../openraft", features = ["serde", "type-alias"] } 21 | 22 | futures = { version = "0.3" } 23 | serde = { version = "1.0.114", features = ["derive"] } 24 | serde_json = { version = "1.0.57" } 25 | tokio = { version = "1.0", default-features = false, features = ["sync"] } 26 | tracing = { version = "0.1.29" } 27 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 28 | 29 | [dev-dependencies] 30 | maplit = { version = "1.0.2" } 31 | 32 | [features] 33 | 34 | [package.metadata.docs.rs] 35 | all-features = true 36 | -------------------------------------------------------------------------------- /openraft/src/testing/common.rs: -------------------------------------------------------------------------------- 1 | //! Testing utilities used by all kinds of tests. 2 | 3 | use std::collections::BTreeSet; 4 | 5 | use crate::RaftTypeConfig; 6 | use crate::entry::RaftEntry; 7 | use crate::type_config::alias::LogIdOf; 8 | use crate::vote::RaftLeaderIdExt; 9 | 10 | /// Builds a log id, for testing purposes. 11 | pub fn log_id(term: u64, node_id: C::NodeId, index: u64) -> LogIdOf 12 | where 13 | C: RaftTypeConfig, 14 | C::Term: From, 15 | { 16 | LogIdOf::::new(C::LeaderId::new_committed(term.into(), node_id), index) 17 | } 18 | 19 | /// Create a blank log entry for tests. 20 | pub fn blank_ent(term: u64, node_id: C::NodeId, index: u64) -> crate::Entry 21 | where 22 | C: RaftTypeConfig, 23 | C::Term: From, 24 | { 25 | crate::Entry::::new_blank(log_id::(term, node_id, index)) 26 | } 27 | 28 | /// Create a membership log entry without learner config for test. 29 | pub fn membership_ent(term: u64, node_id: C::NodeId, index: u64, config: Vec>) -> crate::Entry 30 | where 31 | C: RaftTypeConfig, 32 | C::Term: From, 33 | C::Node: Default, 34 | { 35 | crate::Entry::new_membership( 36 | log_id::(term, node_id, index), 37 | crate::Membership::new_with_defaults(config, []), 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /openraft/src/core/server_state.rs: -------------------------------------------------------------------------------- 1 | /// All possible states of a Raft node. 2 | #[derive(Debug, Clone, Copy, Default)] 3 | #[derive(PartialEq, Eq)] 4 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 5 | pub enum ServerState { 6 | /// The node is completely passive; replicating entries, but neither voting nor timing out. 7 | #[default] 8 | Learner, 9 | /// The node is replicating logs from the leader. 10 | Follower, 11 | /// The node is campaigning to become the cluster leader. 12 | Candidate, 13 | /// The node is the Raft cluster leader. 14 | Leader, 15 | /// The Raft node is shutting down. 16 | Shutdown, 17 | } 18 | 19 | impl ServerState { 20 | /// Check if currently in learner state. 21 | pub fn is_learner(&self) -> bool { 22 | matches!(self, Self::Learner) 23 | } 24 | 25 | /// Check if currently in follower state. 26 | pub fn is_follower(&self) -> bool { 27 | matches!(self, Self::Follower) 28 | } 29 | 30 | /// Check if currently in candidate state. 31 | pub fn is_candidate(&self) -> bool { 32 | matches!(self, Self::Candidate) 33 | } 34 | 35 | /// Check if currently in leader state. 36 | pub fn is_leader(&self) -> bool { 37 | matches!(self, Self::Leader) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /openraft/src/log_id/log_index_option_ext.rs: -------------------------------------------------------------------------------- 1 | /// This helper trait extracts information from an `Option`. 2 | /// 3 | /// In openraft, `LogIndex` is a `u64`. 4 | pub trait LogIndexOptionExt { 5 | /// Return the next log index. 6 | /// 7 | /// If self is `None`, it returns 0. 8 | fn next_index(&self) -> u64; 9 | 10 | /// Return the previous log index. 11 | /// 12 | /// If self is `None`, it panics. 13 | fn prev_index(&self) -> Self; 14 | 15 | // TODO: unused, remove it 16 | /// Performs an "add" operation. 17 | fn add(&self, v: u64) -> Self; 18 | } 19 | 20 | impl LogIndexOptionExt for Option { 21 | fn next_index(&self) -> u64 { 22 | match self { 23 | None => 0, 24 | Some(v) => v + 1, 25 | } 26 | } 27 | 28 | fn prev_index(&self) -> Self { 29 | match self { 30 | None => { 31 | panic!("None has no previous value"); 32 | } 33 | Some(v) => { 34 | if *v == 0 { 35 | None 36 | } else { 37 | Some(*v - 1) 38 | } 39 | } 40 | } 41 | } 42 | 43 | fn add(&self, v: u64) -> Self { 44 | Some(self.next_index() + v).prev_index() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # CONTRIBUTING 2 | 3 | This is a Rust project, so [rustup](https://rustup.rs/) is the best place to start. 4 | 5 | ## The guide 6 | 7 | The guide for this project is built using [mdBook](https://rust-lang-nursery.github.io/mdBook/index.html). 8 | Review their guide for more details on how to work with mdBook. Here are a few of the pertinents: 9 | 10 | ``` 11 | # Install the CLI. 12 | cargo install mdbook 13 | 14 | # Build the guide. 15 | mdbook build 16 | 17 | # Watch the FS for changes & rebuild. 18 | mdbook watch 19 | ``` 20 | 21 | ## Working with git 22 | 23 | - **Write the commit message like writing an email to your friends**. There is a great [git commit format guide](https://cbea.ms/git-commit/). 24 | 25 | - Do **rebase** and **squash** the branch onto the latest `main` branch before publishing a PR. 26 | 27 | - Do **NOT** **rebase** after publishing PR. Only merge. 28 | 29 | 30 | ## Release checklist 31 | 32 | - `make`: pass the unit test, format and clippy check. 33 | 34 | - Any documentation updates should also be reflected in the guide. 35 | 36 | - Ensure the Cargo.toml version for openraft or memstore has been updated, depending on which is being released. 37 | 38 | - Once the release CI has been finished, navigate to the release page, update the release info and publish the release. 39 | -------------------------------------------------------------------------------- /tests/tests/metrics/t10_current_leader.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | 7 | use crate::fixtures::RaftRouter; 8 | use crate::fixtures::ut_harness; 9 | 10 | /// Current leader tests. 11 | /// 12 | /// What does this test do? 13 | /// 14 | /// - create a stable 3-node cluster. 15 | /// - call the current_leader interface on the all nodes, and assert success. 16 | #[tracing::instrument] 17 | #[test_harness::test(harness = ut_harness)] 18 | async fn current_leader() -> Result<()> { 19 | let config = Arc::new( 20 | Config { 21 | enable_heartbeat: false, 22 | ..Default::default() 23 | } 24 | .validate()?, 25 | ); 26 | 27 | let mut router = RaftRouter::new(config.clone()); 28 | 29 | let _log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 30 | 31 | // Get the ID of the leader, and assert that current_leader succeeds. 32 | let leader = router.leader().expect("leader not found"); 33 | assert_eq!(leader, 0, "expected leader to be node 0, got {}", leader); 34 | 35 | for i in 0..3 { 36 | let leader = router.current_leader(i).await; 37 | assert_eq!(leader, Some(0), "expected leader to be node 0, got {:?}", leader); 38 | } 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /tests/tests/client_api/t16_with_raft_state.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use anyhow::Result; 4 | use maplit::btreeset; 5 | use openraft::Config; 6 | use openraft::error::Fatal; 7 | 8 | use crate::fixtures::RaftRouter; 9 | use crate::fixtures::log_id; 10 | use crate::fixtures::ut_harness; 11 | 12 | /// Access Raft state via [`Raft::with_raft_state()`](openraft::Raft::with_raft_state) 13 | #[tracing::instrument] 14 | #[test_harness::test(harness = ut_harness)] 15 | async fn with_raft_state() -> Result<()> { 16 | let config = Arc::new( 17 | Config { 18 | enable_heartbeat: false, 19 | ..Default::default() 20 | } 21 | .validate()?, 22 | ); 23 | 24 | let mut router = RaftRouter::new(config.clone()); 25 | 26 | tracing::info!("--- initializing cluster"); 27 | let log_index = router.new_cluster(btreeset! {0,1,2}, btreeset! {}).await?; 28 | 29 | let n0 = router.get_raft_handle(&0)?; 30 | 31 | let committed = n0.with_raft_state(|st| st.committed().cloned()).await?; 32 | assert_eq!(committed, Some(log_id(1, 0, log_index))); 33 | 34 | tracing::info!("--- shutting down node 0"); 35 | n0.shutdown().await?; 36 | 37 | let res = n0.with_raft_state(|st| st.committed().cloned()).await; 38 | assert_eq!(Err(Fatal::Stopped), res); 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /openraft/src/engine/handler/snapshot_handler/trigger_snapshot_test.rs: -------------------------------------------------------------------------------- 1 | use pretty_assertions::assert_eq; 2 | 3 | use crate::core::sm; 4 | use crate::engine::Command; 5 | use crate::engine::Engine; 6 | use crate::engine::testing::UTConfig; 7 | 8 | fn eng() -> Engine { 9 | let mut eng = Engine::testing_default(0); 10 | eng.state.enable_validation(false); // Disable validation for incomplete state 11 | 12 | eng 13 | } 14 | 15 | #[test] 16 | fn test_trigger_snapshot() -> anyhow::Result<()> { 17 | let mut eng = eng(); 18 | 19 | assert_eq!( 20 | false, 21 | eng.state.io_state_mut().building_snapshot(), 22 | "initially, snapshot is not triggered" 23 | ); 24 | 25 | // Trigger snapshot. 26 | 27 | let got = eng.snapshot_handler().trigger_snapshot(); 28 | 29 | assert_eq!(true, got); 30 | assert_eq!(true, eng.state.io_state_mut().building_snapshot()); 31 | assert_eq!( 32 | vec![ 33 | // 34 | Command::from(sm::Command::build_snapshot()), 35 | ], 36 | eng.output.take_commands() 37 | ); 38 | 39 | // Trigger twice will not trigger again. 40 | 41 | let got = eng.snapshot_handler().trigger_snapshot(); 42 | assert_eq!(false, got, "snapshot is already triggered"); 43 | assert_eq!(0, eng.output.take_commands().len()); 44 | 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use openraft::Config; 5 | use openraft::ServerState; 6 | use openraft::Vote; 7 | use openraft::storage::RaftLogStorage; 8 | 9 | use crate::fixtures::RaftRouter; 10 | use crate::fixtures::ut_harness; 11 | 12 | /// Special case: A leader that is not a member(neither a voter or non-voter) should be started too, 13 | /// as a learner. 14 | #[tracing::instrument] 15 | #[test_harness::test(harness = ut_harness)] 16 | async fn issue_920_non_member_leader_restart() -> anyhow::Result<()> { 17 | let config = Arc::new( 18 | Config { 19 | enable_heartbeat: false, 20 | ..Default::default() 21 | } 22 | .validate()?, 23 | ); 24 | 25 | let mut router = RaftRouter::new(config.clone()); 26 | 27 | let (mut log_store, sm) = router.new_store(); 28 | // Set committed vote that believes node 0 is the leader. 29 | log_store.save_vote(&Vote::new_committed(1, 0)).await?; 30 | router.new_raft_node_with_sto(0, log_store, sm).await; 31 | 32 | router 33 | .wait(&0, timeout()) 34 | .state(ServerState::Learner, "node 0 becomes learner when startup") 35 | .await?; 36 | 37 | Ok(()) 38 | } 39 | 40 | fn timeout() -> Option { 41 | Some(Duration::from_millis(1000)) 42 | } 43 | -------------------------------------------------------------------------------- /openraft/src/base/finalized.rs: -------------------------------------------------------------------------------- 1 | //! Provides a marker trait to prevent external implementation of trait methods. 2 | 3 | /// A marker trait used to prevent specific already auto-implemented trait methods from being 4 | /// re-implemented outside their defining crate. 5 | /// 6 | /// This is achieved by adding this non-referencable marker trait to a trait method signature. 7 | /// 8 | /// # Example 9 | /// 10 | /// The following code demonstrates how `Final` prevents external implementation: 11 | /// 12 | /// ```ignore 13 | /// pub trait Trait { 14 | /// // This method cannot be implemented by users because it requires 15 | /// // the private `Final` trait in its bounds 16 | /// fn unimplementable(&self) where Self: Final { 17 | /// self.user_impl_this(); 18 | /// } 19 | /// 20 | /// // This method can be implemented by users 21 | /// fn user_impl_this(&self); 22 | /// } 23 | /// 24 | /// pub struct MyType; 25 | /// 26 | /// impl Trait for MyType { 27 | /// // Attempting to implement this method will fail to compile 28 | /// // because `Final` is not accessible from outside the crate 29 | /// fn unimplementable(&self) where Self: Final {} 30 | /// 31 | /// fn user_impl_this(&self) { 32 | /// println!("This implementation is allowed"); 33 | /// } 34 | /// } 35 | /// ``` 36 | pub trait Final {} 37 | 38 | impl Final for T {} 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. 'cargo ...' 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Actual behavior** 22 | applicable, add screenshots to help explain your problem. 23 | 24 | **Env (please complete the following information):** 25 | - Openraft version [e.g., a tag such as `v0.7.2` or a commit hash] 26 | - Does the bug still there in the latest version?(`main` for the latest release branch, or `release-0.7` for 0.7 branch)? 27 | - Rust-toolchain: [e.g. `cargo 1.65.0-nightly (ce40690a5 2022-08-09)`; Find toolchain with `cargo version` in a crate dir] 28 | - OS: [e.g. mac os 12.0.1, centos 7.0] 29 | - CPU: [e.g. mac m1, x86_64] 30 | 31 | **Additional files:** 32 | - Logs: Attach logs generate by unittest(e.g., `./openraft/_log/*`), or logs output by examples(`examples/raft-kv-*/test-cluster.sh` will print logs in `n1.log`, `n2.log`, `n3.log`) 33 | - Cargo.lock: [e.g. `./openraft/Cargo.lock`, `./examples/raft-kv-memstore/Cargo.lock`] 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | --------------------------------------------------------------------------------