├── Cargo.toml ├── rust-toolchain.toml ├── epaxos ├── src │ ├── lib.rs │ ├── error.rs │ ├── types │ │ ├── mod.rs │ │ ├── lb.rs │ │ ├── id.rs │ │ ├── cmd.rs │ │ ├── instance.rs │ │ ├── space.rs │ │ └── replica.rs │ ├── config.rs │ ├── util.rs │ ├── message.rs │ ├── client.rs │ ├── execute.rs │ └── server.rs ├── Cargo.toml └── examples │ └── kv.rs ├── pro-macro ├── Cargo.toml ├── tests │ └── success.rs └── src │ └── lib.rs ├── .gitignore └── .github └── workflows └── ci.yml /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "epaxos", 4 | "pro-macro", 5 | ] -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | profile = "minimal" 3 | channel = "1.57.0" 4 | components = ["clippy", "rustfmt", "rustc", "cargo", "rust-std"] 5 | -------------------------------------------------------------------------------- /epaxos/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | pub mod config; 3 | pub mod error; 4 | mod execute; 5 | pub mod message; 6 | pub mod server; 7 | mod types; 8 | mod util; 9 | 10 | pub use types::{Command, CommandExecutor}; 11 | -------------------------------------------------------------------------------- /pro-macro/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pro-macro" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | proc-macro = true 9 | 10 | [dependencies] 11 | proc-macro2 = "1.0" 12 | syn = "1.0" 13 | quote = "1.0" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | -------------------------------------------------------------------------------- /epaxos/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use thiserror::Error; 3 | 4 | #[derive(Error, Debug)] 5 | pub enum CommitError {} 6 | 7 | #[derive(Error, Debug)] 8 | pub enum ExecuteError { 9 | #[error("invalid command {0} ")] 10 | InvalidCommand(String), 11 | #[error("meet io related error")] 12 | IoError(#[from] io::Error), 13 | } 14 | 15 | #[derive(Error, Debug)] 16 | pub enum RpcError { 17 | #[error("meet io related error")] 18 | IoError(#[from] io::Error), 19 | } 20 | -------------------------------------------------------------------------------- /epaxos/src/types/mod.rs: -------------------------------------------------------------------------------- 1 | pub use self::cmd::{Command, CommandExecutor}; 2 | pub(crate) use self::{ 3 | id::{Ballot, InstanceId, LeaderId, LocalInstanceId, ReplicaId, Seq}, 4 | instance::{Instance, InstanceStatus, ResultSender, SharedInstance}, 5 | lb::LeaderBook, 6 | replica::Replica, 7 | space::{InstanceSpace, VecInstanceSpace}, 8 | }; 9 | 10 | #[cfg(test)] 11 | pub(crate) use self::{ 12 | cmd::{MockCommand, MockCommandExecutor, MockCommandId}, 13 | space::MockInstanceSpace, 14 | }; 15 | 16 | mod cmd; 17 | mod id; 18 | mod instance; 19 | mod lb; 20 | mod replica; 21 | mod space; 22 | -------------------------------------------------------------------------------- /epaxos/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | edition = "2021" 3 | name = "rpaxos" 4 | version = "0.1.0" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | async-trait = "0.1.51" 10 | bincode = "1.3.3" 11 | env_logger = "0.9.0" 12 | futures = "0.3.17" 13 | itertools = "0.10.1" 14 | log = "0.4.14" 15 | mockall = "0.11.0" 16 | petgraph = "0.6.0" 17 | pro-macro = {path = "../pro-macro"} 18 | serde = {version = "1.0.130", features = ["derive"]} 19 | thiserror = "1.0.30" 20 | tokio = {version = "1.14", features = ["net", "rt", "io-util", "sync", "macros", "rt-multi-thread", "time"]} 21 | uuid = {version = "1.0.0", features = ["v4"]} 22 | yaml-rust = "0.4.5" 23 | -------------------------------------------------------------------------------- /epaxos/src/types/lb.rs: -------------------------------------------------------------------------------- 1 | use crate::config::Configure; 2 | 3 | use super::id::{Ballot, ReplicaId}; 4 | 5 | #[derive(Debug)] 6 | #[cfg_attr(test, derive(Default))] 7 | pub(crate) struct LeaderBook { 8 | pub(crate) accept_ok: usize, 9 | pub(crate) preaccept_ok: usize, 10 | pub(crate) nack: usize, 11 | pub(crate) max_ballot: Ballot, 12 | pub(crate) all_equal: bool, 13 | } 14 | 15 | impl LeaderBook { 16 | pub(crate) fn new(replica: ReplicaId, conf: &Configure) -> Self { 17 | LeaderBook { 18 | accept_ok: 0, 19 | preaccept_ok: 0, 20 | nack: 0, 21 | max_ballot: Ballot::new(replica, conf), 22 | all_equal: true, 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pro-macro/tests/success.rs: -------------------------------------------------------------------------------- 1 | use pro_macro::FromInner; 2 | 3 | #[test] 4 | fn named_struct() { 5 | #[derive(FromInner)] 6 | struct NamedStruct { 7 | inner: usize, 8 | } 9 | 10 | // unref 11 | let mut a = NamedStruct { inner: 0 }; 12 | assert_eq!(*a, 0); 13 | 14 | // into 15 | let b = Into::::into(0); 16 | assert_eq!(*a, *b); 17 | 18 | // value add assign 19 | a += 1; 20 | assert_eq!(*a, 1); 21 | 22 | // value add 23 | let mut a = a + 1; 24 | assert_eq!(*a, 2); 25 | 26 | // ref add 27 | let a_ref = &a + 1; 28 | assert_eq!(*a_ref, 3); 29 | 30 | // mut ref add 31 | { 32 | let a_mut_ref = &mut a; 33 | assert_eq!(*(a_mut_ref + 1), 3); 34 | assert_eq!(*a, 2); 35 | } 36 | 37 | // mut ref add assign 38 | { 39 | let mut va_mut_ref = &mut a; 40 | va_mut_ref += 1; 41 | assert_eq!(**va_mut_ref, 3); 42 | assert_eq!(*a, 3); 43 | } 44 | } 45 | 46 | #[test] 47 | fn unnamed_struct() { 48 | #[derive(FromInner)] 49 | struct UnnamedStruct(usize); 50 | 51 | // unref 52 | let mut a = UnnamedStruct(0); 53 | assert_eq!(*a, 0); 54 | 55 | // into 56 | let b = Into::::into(0); 57 | assert_eq!(*a, *b); 58 | 59 | // value add assign 60 | a += 1; 61 | assert_eq!(*a, 1); 62 | 63 | // value add 64 | let mut a = a + 1; 65 | assert_eq!(*a, 2); 66 | 67 | // ref add 68 | let a_ref = &a + 1; 69 | assert_eq!(*a_ref, 3); 70 | 71 | // mut ref add 72 | { 73 | let a_mut_ref = &mut a; 74 | assert_eq!(*(a_mut_ref + 1), 3); 75 | assert_eq!(*a, 2); 76 | } 77 | 78 | // mut ref add assign 79 | { 80 | let mut va_mut_ref = &mut a; 81 | va_mut_ref += 1; 82 | assert_eq!(**va_mut_ref, 3); 83 | assert_eq!(*a, 3); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | branches: [main] 5 | 6 | env: 7 | RUST_TOOLCHAIN_VERSION: stable 8 | RUST_TOOLCHAIN_COMPONENET: clippy, rustfmt, rustc, cargo, rust-std 9 | 10 | jobs: 11 | fmt: 12 | name: Fmt 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: install rust toolchain 17 | uses: actions-rs/toolchain@v1 18 | with: 19 | profile: minimal 20 | toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} 21 | override: true 22 | components: ${{ env.RUST_TOOLCHAIN_COMPONENET }} 23 | - name: code format check 24 | uses: actions-rs/cargo@v1 25 | with: 26 | command: fmt 27 | args: --all -- --check 28 | 29 | clippy: 30 | name: Clippy 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v2 34 | - name: install rust toolchain 35 | uses: actions-rs/toolchain@v1 36 | with: 37 | profile: minimal 38 | toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} 39 | override: true 40 | components: ${{ env.RUST_TOOLCHAIN_COMPONENET }} 41 | - uses: actions-rs/cargo@v1 42 | with: 43 | command: clippy 44 | args: --all-features --all-targets -- -D warnings 45 | 46 | test: 47 | name: Test (UT/IT) 48 | runs-on: ubuntu-latest 49 | env: 50 | RUST_LOG: TRACE 51 | steps: 52 | - uses: actions/checkout@v2 53 | - name: install rust toolchain 54 | uses: actions-rs/toolchain@v1 55 | with: 56 | profile: minimal 57 | toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} 58 | override: true 59 | components: ${{ env.RUST_TOOLCHAIN_COMPONENET }} 60 | - uses: actions-rs/cargo@v1 61 | with: 62 | command: test 63 | - name: change directory to epaxos 64 | run: cd epaxos 65 | # Fixme: Should not run example as it 66 | - name: Run example as test 67 | uses: actions-rs/cargo@v1 68 | with: 69 | command: run 70 | args: --example kv -------------------------------------------------------------------------------- /epaxos/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Index; 2 | 3 | use yaml_rust::YamlLoader; 4 | 5 | #[derive(Clone)] 6 | pub struct Configure { 7 | pub(crate) peer_cnt: usize, 8 | pub(crate) peer: Vec, 9 | pub(crate) index: usize, 10 | pub(crate) epoch: usize, 11 | } 12 | 13 | impl Configure { 14 | pub fn new(peer_cnt: usize, peer: Vec, index: usize, epoch: usize) -> Self { 15 | if (peer_cnt % 2) == 0 { 16 | panic!("The peer count should be odd, but we got {}", peer_cnt); 17 | } 18 | 19 | Self { 20 | peer_cnt, 21 | peer, 22 | index, 23 | epoch, 24 | } 25 | } 26 | 27 | pub fn index(&self) -> usize { 28 | self.index 29 | } 30 | } 31 | 32 | impl Index for Configure { 33 | type Output = str; 34 | 35 | fn index(&self, index: usize) -> &Self::Output { 36 | &self.peer[index] 37 | } 38 | } 39 | 40 | pub trait ConfigureSrc { 41 | fn get_configure(&self) -> Configure; 42 | } 43 | 44 | /// Read Configure from regular file 45 | pub struct YamlConfigureSrc { 46 | yaml: String, 47 | } 48 | 49 | impl YamlConfigureSrc { 50 | pub fn new(yaml: &str) -> Self { 51 | Self { 52 | yaml: yaml.to_owned(), 53 | } 54 | } 55 | } 56 | 57 | impl ConfigureSrc for YamlConfigureSrc { 58 | fn get_configure(&self) -> Configure { 59 | let yaml = YamlLoader::load_from_str(&self.yaml).unwrap(); 60 | if yaml.len() != 1 { 61 | panic!("We should only pass in a yaml file"); 62 | } 63 | 64 | // have checked length 65 | let yaml = yaml.get(0).unwrap(); 66 | 67 | // TODO: put string to const 68 | let peer_cnt = yaml["peer_cnt"].as_i64().unwrap() as usize; 69 | 70 | let peer = yaml["peer"] 71 | .as_vec() 72 | .unwrap() 73 | .iter() 74 | .map(|y| y.as_str().unwrap().to_owned()) 75 | .collect(); 76 | 77 | let index = yaml["index"].as_i64().unwrap() as usize; 78 | 79 | let epoch: usize = yaml["epoch"].as_i64().unwrap() as usize; 80 | 81 | Configure { 82 | peer_cnt, 83 | peer, 84 | index, 85 | epoch, 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /epaxos/src/types/id.rs: -------------------------------------------------------------------------------- 1 | use pro_macro::FromInner; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | use crate::config::Configure; 5 | 6 | /// The Replica Id 7 | #[derive(Debug, Copy, Clone, FromInner, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 8 | pub(crate) struct ReplicaId(usize); 9 | 10 | /// The local instance id 11 | #[derive(Debug, Copy, Clone, FromInner, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 12 | pub(crate) struct LocalInstanceId(usize); 13 | 14 | /// The global instance id 15 | #[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] 16 | pub(crate) struct InstanceId { 17 | pub(crate) replica: ReplicaId, 18 | pub(crate) local: LocalInstanceId, 19 | } 20 | 21 | impl InstanceId { 22 | pub(crate) fn new(replica: ReplicaId, local: LocalInstanceId) -> Self { 23 | Self { replica, local } 24 | } 25 | } 26 | 27 | /// The seq num, which break the dependent circle while executing 28 | #[derive(Debug, Copy, Clone, FromInner, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 29 | pub(crate) struct Seq(usize); 30 | 31 | /// Ballot number 32 | #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] 33 | pub(crate) struct Ballot { 34 | epoch: usize, 35 | base: usize, 36 | replica: ReplicaId, 37 | } 38 | 39 | impl Ballot { 40 | pub(crate) fn new(replica: ReplicaId, conf: &Configure) -> Ballot { 41 | Ballot { 42 | replica, 43 | epoch: conf.epoch, 44 | base: 0, 45 | } 46 | } 47 | 48 | pub(crate) fn is_init(&self) -> bool { 49 | self.base == 0 50 | } 51 | 52 | #[cfg(test)] 53 | pub(crate) fn new_with_epoch(replica: ReplicaId, epoch: usize) -> Ballot { 54 | Ballot { 55 | replica, 56 | epoch, 57 | base: 0, 58 | } 59 | } 60 | } 61 | 62 | #[cfg(test)] 63 | impl Default for Ballot { 64 | fn default() -> Self { 65 | Self { 66 | epoch: Default::default(), 67 | base: Default::default(), 68 | replica: ReplicaId(0), 69 | } 70 | } 71 | } 72 | 73 | /// The leader id for a instance propose, used in the message passing 74 | #[derive(Debug, FromInner, Serialize, Deserialize)] 75 | pub(crate) struct LeaderId(usize); 76 | 77 | /// The acceptor id 78 | #[derive(FromInner, Serialize, Deserialize)] 79 | pub(crate) struct AcceptorId(usize); 80 | 81 | impl From for LeaderId { 82 | fn from(r: ReplicaId) -> Self { 83 | Self(*r) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /epaxos/examples/kv.rs: -------------------------------------------------------------------------------- 1 | use std::{io, time::Duration}; 2 | 3 | use log::{debug, info}; 4 | use rpaxos::{ 5 | client::{RpcClient, TcpRpcClient}, 6 | config::Configure, 7 | error, 8 | server::DefaultServer, 9 | Command, CommandExecutor, 10 | }; 11 | use serde::{Deserialize, Serialize}; 12 | use tokio::time::timeout; 13 | 14 | #[derive(Debug, Clone, Serialize, Deserialize)] 15 | enum TestCommandOp { 16 | Read, 17 | Write, 18 | } 19 | 20 | #[derive(Debug, Clone, Serialize, Deserialize)] 21 | struct TestCommand { 22 | key: String, 23 | value: Option, 24 | op: TestCommandOp, 25 | } 26 | 27 | #[async_trait::async_trait] 28 | impl Command for TestCommand { 29 | type K = String; 30 | type ER = String; 31 | 32 | fn key(&self) -> &Self::K { 33 | &self.key 34 | } 35 | } 36 | 37 | #[derive(Clone, Copy, Debug)] 38 | struct TestCommandExecutor {} 39 | 40 | #[async_trait::async_trait] 41 | impl CommandExecutor for TestCommandExecutor { 42 | async fn execute(&self, cmd: &TestCommand) -> Result { 43 | info!("execute command {:?}", cmd); 44 | Ok(cmd.value.as_ref().map_or(String::new(), |v| v.to_owned())) 45 | } 46 | } 47 | 48 | #[tokio::main] 49 | async fn main() -> io::Result<()> { 50 | env_logger::init(); 51 | let peer = vec![ 52 | "localhost:9000".to_owned(), 53 | "localhost:9001".to_owned(), 54 | "localhost:9002".to_owned(), 55 | ]; 56 | 57 | let cmd_exe = TestCommandExecutor {}; 58 | 59 | let mut server = Vec::with_capacity(3); 60 | for c in (0..3).map(|id| Configure::new(3, peer.to_vec(), id, 0)) { 61 | server.push(DefaultServer::::new(c, cmd_exe).await); 62 | } 63 | 64 | let handles: Vec<_> = server 65 | .into_iter() 66 | .map(|s| { 67 | tokio::spawn(timeout(Duration::from_secs(10), async move { 68 | s.run().await; 69 | })) 70 | }) 71 | .collect(); 72 | 73 | debug!("spawn servers"); 74 | 75 | let mut client = TcpRpcClient::::new(Configure::new(3, peer, 0, 0), 0).await; 76 | let write_result = client 77 | .propose(vec![TestCommand { 78 | key: "k1".to_owned(), 79 | value: Some("v1".to_owned()), 80 | op: TestCommandOp::Write, 81 | }]) 82 | .await; 83 | 84 | debug!("write result is {:?}", write_result); 85 | debug!("after client propose"); 86 | // The server will never end until someone kills is 87 | for h in handles { 88 | let _ = h.await?; 89 | } 90 | 91 | Ok(()) 92 | } 93 | -------------------------------------------------------------------------------- /epaxos/src/types/cmd.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use serde::de::DeserializeOwned; 3 | 4 | use crate::error::ExecuteError; 5 | use std::hash::Hash; 6 | 7 | #[cfg(test)] 8 | use super::InstanceId; 9 | #[cfg(test)] 10 | use tokio::sync::mpsc::Sender; 11 | 12 | #[async_trait] 13 | pub trait Command: Sized { 14 | /// K is used to tell confliction 15 | type K: Eq + Hash + Send + Sync + Clone + 'static; 16 | 17 | /// Execution result 18 | type ER: std::fmt::Debug + Send + serde::Serialize + DeserializeOwned + Sync + Clone; 19 | 20 | fn key(&self) -> &Self::K; 21 | 22 | async fn execute(&self, e: &E) -> Result 23 | where 24 | E: CommandExecutor + Send + Sync, 25 | { 26 | >::execute(e, self).await 27 | } 28 | } 29 | 30 | #[async_trait] 31 | pub trait CommandExecutor 32 | where 33 | C: Command, 34 | { 35 | async fn execute(&self, cmd: &C) -> Result; 36 | } 37 | 38 | #[cfg(test)] 39 | #[derive(PartialEq, Eq, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] 40 | pub(crate) struct MockCommandId { 41 | instance_id: InstanceId, 42 | cmd_id: usize, 43 | } 44 | 45 | #[cfg(test)] 46 | impl MockCommandId { 47 | pub(crate) fn new(instance_id: InstanceId, cmd_id: usize) -> Self { 48 | Self { 49 | instance_id, 50 | cmd_id, 51 | } 52 | } 53 | } 54 | 55 | #[cfg(test)] 56 | #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] 57 | pub(crate) struct MockCommand { 58 | key: String, 59 | value: MockCommandId, 60 | } 61 | 62 | #[cfg(test)] 63 | impl MockCommand { 64 | pub(crate) fn new(key: &str, value: MockCommandId) -> Self { 65 | Self { 66 | key: key.to_owned(), 67 | value, 68 | } 69 | } 70 | pub(crate) fn value(&self) -> MockCommandId { 71 | self.value 72 | } 73 | } 74 | 75 | #[cfg(test)] 76 | #[async_trait] 77 | impl Command for MockCommand { 78 | type K = String; 79 | type ER = String; 80 | 81 | fn key(&self) -> &Self::K { 82 | &self.key 83 | } 84 | } 85 | 86 | #[cfg(test)] 87 | #[derive(Clone, Debug)] 88 | pub(crate) struct MockCommandExecutor { 89 | cmd_sender: Option>, 90 | } 91 | 92 | #[cfg(test)] 93 | impl MockCommandExecutor { 94 | pub(crate) fn new(cmd_sender: Option>) -> Self { 95 | Self { cmd_sender } 96 | } 97 | } 98 | 99 | #[cfg(test)] 100 | #[async_trait] 101 | impl CommandExecutor for MockCommandExecutor { 102 | async fn execute(&self, cmd: &MockCommand) -> Result { 103 | if self.cmd_sender.is_some() { 104 | let _ = self.cmd_sender.as_ref().unwrap().send(cmd.value()).await; 105 | } 106 | Ok(String::new()) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /epaxos/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::{ops::DerefMut, sync::Arc}; 2 | 3 | use serde::{de::DeserializeOwned, Serialize}; 4 | use tokio::{ 5 | io::{AsyncReadExt, AsyncWriteExt}, 6 | net::TcpStream, 7 | sync::Mutex, 8 | }; 9 | 10 | use crate::types::{Command, SharedInstance}; 11 | 12 | pub(crate) async fn send_message(conn: &mut T, message: &M) 13 | where 14 | M: Serialize, 15 | T: AsyncWriteExt + Unpin, 16 | { 17 | // TODO: Report message content while meeting error 18 | let content = bincode::serialize(message) 19 | .map_err(|e| panic!("failed to serialize the message, {}", e)) 20 | .unwrap(); 21 | let len = (content.len() as u64).to_be_bytes(); 22 | 23 | // FIXME: handle network error 24 | let _ = conn.write(&len).await; 25 | let _ = conn.write(&content).await; 26 | } 27 | 28 | pub(crate) async fn send_message_arc(conn: &Arc>, message: &M) 29 | where 30 | M: Serialize, 31 | { 32 | let mut conn = conn.lock().await; 33 | let conn = conn.deref_mut(); 34 | 35 | send_message(conn, message).await; 36 | } 37 | 38 | pub(crate) async fn send_message_arc2(conn: &Arc>, message: &Arc) 39 | where 40 | M: Serialize, 41 | { 42 | send_message_arc(conn, message.as_ref()).await; 43 | } 44 | 45 | pub(crate) async fn recv_message(conn: &mut T) -> Option 46 | where 47 | M: DeserializeOwned, 48 | T: AsyncReadExt + Unpin, 49 | { 50 | let mut len_buf: [u8; 8] = [0; 8]; 51 | 52 | let read_size = read_from_stream(conn, &mut len_buf).await; 53 | if read_size != 8 { 54 | return None; 55 | } 56 | 57 | let expected_len = u64::from_be_bytes(len_buf); 58 | let mut buf: Vec = vec![0; expected_len as usize]; 59 | 60 | let read_size = read_from_stream(conn, &mut buf).await; 61 | if read_size != expected_len as usize { 62 | return None; 63 | } 64 | 65 | Some( 66 | bincode::deserialize(&buf) 67 | .map_err(|e| panic!("Deserialize message failed, {} ", e)) 68 | .unwrap(), 69 | ) 70 | } 71 | 72 | async fn read_from_stream(stream: &mut T, buf: &mut [u8]) -> usize 73 | where 74 | T: AsyncReadExt + Unpin, 75 | { 76 | let expect_len = buf.len(); 77 | let mut has_read: usize = 0; 78 | while has_read != expect_len { 79 | let read_size = stream 80 | .read(&mut buf[has_read..]) 81 | .await 82 | .map_err(|e| panic!("tcp link should read {} bytes message, {}", expect_len, e)) 83 | .unwrap(); 84 | // peer has closed 85 | if read_size == 0 { 86 | break; 87 | } 88 | 89 | has_read += read_size; 90 | } 91 | has_read 92 | } 93 | 94 | pub(crate) async fn instance_exist(ins: &Option>) -> bool 95 | where 96 | C: Command + Clone, 97 | { 98 | if ins.is_some() { 99 | let ins = ins.as_ref().unwrap(); 100 | let ins_read = ins.get_instance_read().await; 101 | ins_read.is_some() 102 | } else { 103 | false 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /epaxos/src/message.rs: -------------------------------------------------------------------------------- 1 | use crate::types::{Ballot, Command, InstanceId, LeaderId, LocalInstanceId, Seq}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Debug, Serialize, Deserialize)] 5 | pub(crate) struct PreAccept 6 | where 7 | C: Command, 8 | { 9 | pub(crate) leader_id: LeaderId, 10 | pub(crate) instance_id: InstanceId, 11 | pub(crate) seq: Seq, 12 | pub(crate) ballot: Ballot, 13 | pub(crate) cmds: Vec, 14 | pub(crate) deps: Vec>, 15 | } 16 | 17 | #[derive(Debug, Serialize, Deserialize)] 18 | pub(crate) struct PreAcceptReply { 19 | pub(crate) instance_id: InstanceId, 20 | pub(crate) seq: Seq, 21 | pub(crate) ballot: Ballot, 22 | pub(crate) ok: bool, 23 | pub(crate) deps: Vec>, 24 | pub(crate) committed_deps: Vec>, 25 | } 26 | 27 | #[derive(Debug, Serialize, Deserialize)] 28 | pub(crate) struct PreAcceptOk { 29 | pub(crate) instance_id: InstanceId, 30 | } 31 | 32 | #[derive(Debug, Serialize, Deserialize)] 33 | pub(crate) struct Accept { 34 | pub(crate) leader_id: LeaderId, 35 | pub(crate) instance_id: InstanceId, 36 | pub(crate) ballot: Ballot, 37 | pub(crate) seq: Seq, 38 | pub(crate) cmd_cnt: usize, 39 | pub(crate) deps: Vec>, 40 | } 41 | 42 | #[derive(Debug, Serialize, Deserialize)] 43 | pub(crate) struct AcceptReply { 44 | pub(crate) instance_id: InstanceId, 45 | pub(crate) ok: bool, 46 | pub(crate) ballot: Ballot, 47 | } 48 | 49 | #[derive(Debug, Serialize, Deserialize)] 50 | pub(crate) struct Commit 51 | where 52 | C: Command, 53 | { 54 | pub(crate) leader_id: LeaderId, 55 | pub(crate) instance_id: InstanceId, 56 | pub(crate) seq: Seq, 57 | pub(crate) cmds: Vec, 58 | pub(crate) deps: Vec>, 59 | } 60 | 61 | #[derive(Debug, Serialize, Deserialize)] 62 | pub(crate) struct CommitShort { 63 | leader_id: LeaderId, 64 | instance_id: InstanceId, 65 | seq: Seq, 66 | cmd_cnt: usize, 67 | deps: Vec, 68 | } 69 | 70 | #[derive(Debug, Clone, Serialize, Deserialize)] 71 | pub(crate) struct Propose 72 | where 73 | C: Command + Serialize, 74 | { 75 | pub(crate) cmd_id: String, 76 | pub(crate) cmds: Vec, 77 | } 78 | 79 | #[derive(Debug, Clone, Serialize, Deserialize)] 80 | pub(crate) enum SingleCmdResult 81 | where 82 | C: Command + Serialize, 83 | { 84 | ExecuteResult(C::ER), 85 | Error(String), 86 | } 87 | 88 | #[derive(Debug, Clone, Serialize, Deserialize)] 89 | pub(crate) struct ProposeResponse 90 | where 91 | C: Command + Serialize, 92 | { 93 | pub(crate) cmd_id: String, 94 | pub(crate) results: Vec>, 95 | } 96 | 97 | #[derive(Debug, Serialize, Deserialize)] 98 | pub(crate) enum Message 99 | where 100 | C: Command + Serialize, 101 | { 102 | PreAccept(PreAccept), 103 | PreAcceptReply(PreAcceptReply), 104 | PreAcceptOk(PreAcceptOk), 105 | Accept(Accept), 106 | AcceptReply(AcceptReply), 107 | Commit(Commit), 108 | CommitShort(CommitShort), 109 | Propose(Propose), 110 | ProposeResponse(ProposeResponse), 111 | } 112 | -------------------------------------------------------------------------------- /epaxos/src/client.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashMap, 3 | fmt::Debug, 4 | marker::PhantomData, 5 | sync::{Arc, Mutex}, 6 | }; 7 | 8 | use crate::{ 9 | config::Configure, 10 | message::{Message, Propose, SingleCmdResult}, 11 | types::Command, 12 | util, 13 | }; 14 | use async_trait::async_trait; 15 | use log::trace; 16 | use serde::{de::DeserializeOwned, Serialize}; 17 | use tokio::{io::WriteHalf, net::TcpStream, sync::oneshot, task::JoinHandle}; 18 | 19 | #[async_trait] 20 | pub trait RpcClient 21 | where 22 | C: Command, 23 | { 24 | async fn propose(&mut self, cmds: Vec) -> Vec>; 25 | } 26 | 27 | struct ResultSender(oneshot::Sender>>); 28 | struct ResultReceiver(oneshot::Receiver>>); 29 | 30 | pub struct TcpRpcClient 31 | where 32 | C: Command, 33 | { 34 | #[allow(dead_code)] 35 | // this filed will be used in the membership change 36 | conf: Configure, 37 | stream: WriteHalf, 38 | phantom: PhantomData, 39 | req_map: Arc>>>, 40 | handle: JoinHandle<()>, 41 | } 42 | 43 | impl TcpRpcClient 44 | where 45 | C: Command + Serialize + DeserializeOwned + Debug + Send + Sync + 'static, 46 | { 47 | fn register(&self) -> (String, ResultReceiver) { 48 | let mut map = self.req_map.lock().unwrap(); 49 | let (tx, rx) = oneshot::channel(); 50 | loop { 51 | let uuid = uuid::Uuid::new_v4(); 52 | let uuid_str = uuid.urn().to_string(); 53 | if map.contains_key(&uuid_str) { 54 | continue; 55 | } 56 | map.insert(uuid_str.to_owned(), ResultSender(tx)); 57 | return (uuid_str, ResultReceiver(rx)); 58 | } 59 | } 60 | 61 | pub async fn new(conf: Configure, id: usize) -> Self { 62 | let conn_str = conf 63 | .peer 64 | .get(id) 65 | .or_else(|| panic!("id {} is not in the configure scope", id)) 66 | .unwrap(); 67 | let stream = TcpStream::connect(conn_str) 68 | .await 69 | .map_err(|_e| panic!("connec to {} epaxos peer failed", id)) 70 | .unwrap(); 71 | 72 | let req_map: Arc>>> = 73 | Arc::new(Mutex::new(HashMap::new())); 74 | let req_map_clone = req_map.clone(); 75 | let (mut read_stream, write_stream) = tokio::io::split(stream); 76 | let handle = tokio::spawn(async move { 77 | loop { 78 | let response: Message = util::recv_message(&mut read_stream).await.unwrap_or_else(||panic!("Failed to receive mesasge from server because server has closed connection")); 79 | if let Message::ProposeResponse(pr) = response { 80 | let mut locked_map = req_map_clone.lock().unwrap(); 81 | let tx = locked_map.remove(&pr.cmd_id); 82 | 83 | let results = pr 84 | .results 85 | .into_iter() 86 | .map(|r| match r { 87 | SingleCmdResult::ExecuteResult(execute_result) => Ok(execute_result), 88 | SingleCmdResult::Error(error_msg) => Err(error_msg), 89 | }) 90 | .collect(); 91 | tx.map(|tx| tx.0.send(results)); 92 | } 93 | } 94 | }); 95 | 96 | Self { 97 | conf, 98 | stream: write_stream, 99 | phantom: PhantomData, 100 | req_map, 101 | handle, 102 | } 103 | } 104 | } 105 | 106 | impl Drop for TcpRpcClient 107 | where 108 | C: Command, 109 | { 110 | fn drop(&mut self) { 111 | self.handle.abort(); 112 | } 113 | } 114 | 115 | #[async_trait] 116 | impl RpcClient for TcpRpcClient 117 | where 118 | C: Command + Serialize + DeserializeOwned + Debug + Send + Sync + 'static, 119 | { 120 | async fn propose(&mut self, cmds: Vec) -> Vec> { 121 | trace!("start propose"); 122 | let (id, rx) = self.register(); 123 | let propose = Message::Propose(Propose { cmd_id: id, cmds }); 124 | util::send_message(&mut self.stream, &propose).await; 125 | rx.0.await.unwrap() 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /epaxos/src/types/instance.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::{fmt::Debug, hash::Hash}; 3 | 4 | use serde::{Deserialize, Serialize}; 5 | use tokio::sync::{ 6 | oneshot, Notify, RwLock, RwLockMappedWriteGuard, RwLockReadGuard, RwLockWriteGuard, 7 | }; 8 | 9 | use crate::error::ExecuteError; 10 | 11 | use super::{ 12 | cmd::Command, 13 | id::{Ballot, InstanceId, LocalInstanceId, Seq}, 14 | lb::LeaderBook, 15 | }; 16 | 17 | #[derive(Debug)] 18 | /// The instance stored in the instance space 19 | pub struct Instance 20 | where 21 | C: Command, 22 | { 23 | pub(crate) id: InstanceId, 24 | pub(crate) seq: Seq, 25 | pub(crate) ballot: Ballot, 26 | pub(crate) cmds: Vec, 27 | pub(crate) deps: Vec>, 28 | pub(crate) status: InstanceStatus, 29 | pub(crate) lb: LeaderBook, 30 | } 31 | 32 | impl Instance 33 | where 34 | C: Command, 35 | { 36 | pub(crate) fn local_id(&self) -> LocalInstanceId { 37 | self.id.local 38 | } 39 | } 40 | 41 | #[derive(Debug)] 42 | pub(crate) struct ResultSender( 43 | pub(crate) oneshot::Sender>>, 44 | ); 45 | 46 | #[derive(Debug)] 47 | pub(crate) struct SharedInstanceInner 48 | where 49 | C: Command + Clone, 50 | { 51 | ins: Option>, 52 | notify: Option>>, 53 | execute_complete: Option>, 54 | } 55 | 56 | /// Th shared instance stored in the instance space 57 | #[derive(Clone, Debug)] 58 | pub(crate) struct SharedInstance 59 | where 60 | C: Command + Clone, 61 | { 62 | inner: Arc>>, 63 | } 64 | 65 | impl PartialEq for SharedInstance 66 | where 67 | C: Command + Clone, 68 | { 69 | fn eq(&self, other: &Self) -> bool { 70 | Arc::ptr_eq(&self.inner, &other.inner) 71 | } 72 | } 73 | 74 | impl Eq for SharedInstance where C: Command + Clone {} 75 | 76 | impl Hash for SharedInstance 77 | where 78 | C: Command + Clone, 79 | { 80 | fn hash(&self, state: &mut H) { 81 | Arc::as_ptr(&self.inner).hash(state); 82 | } 83 | } 84 | 85 | impl SharedInstance 86 | where 87 | C: Command + Clone, 88 | { 89 | pub(crate) fn none() -> Self { 90 | Self::new(None, None) 91 | } 92 | 93 | pub(crate) fn new(instance: Option>, notify: Option>>) -> Self { 94 | Self { 95 | inner: Arc::new(RwLock::new(SharedInstanceInner { 96 | ins: instance, 97 | notify, 98 | execute_complete: None, 99 | })), 100 | } 101 | } 102 | 103 | pub(crate) fn new_execute_complete( 104 | instance: Option>, 105 | notify: Option>>, 106 | execute_complete: ResultSender, 107 | ) -> Self { 108 | Self { 109 | inner: Arc::new(RwLock::new(SharedInstanceInner { 110 | ins: instance, 111 | notify, 112 | execute_complete: Some(execute_complete), 113 | })), 114 | } 115 | } 116 | 117 | pub(crate) async fn match_status(&self, status: &[InstanceStatus]) -> bool { 118 | let d_ins_read = self.get_instance_read().await; 119 | if d_ins_read.is_none() { 120 | return false; 121 | } 122 | 123 | let d_ins_read_inner = d_ins_read.as_ref().unwrap(); 124 | status.contains(&d_ins_read_inner.status) 125 | } 126 | 127 | pub(crate) async fn get_instance_read(&'_ self) -> RwLockReadGuard<'_, Option>> { 128 | RwLockReadGuard::map(self.inner.read().await, |i| &i.ins) 129 | } 130 | 131 | pub(crate) async fn get_instance_write( 132 | &'_ self, 133 | ) -> RwLockMappedWriteGuard<'_, Option>> { 134 | RwLockWriteGuard::map(self.inner.write().await, |i| &mut i.ins) 135 | } 136 | 137 | pub(crate) fn get_raw_read( 138 | option_ins: RwLockReadGuard<'_, Option>>, 139 | ) -> RwLockReadGuard<'_, Instance> { 140 | RwLockReadGuard::>>::map(option_ins, |f| f.as_ref().unwrap()) 141 | } 142 | 143 | pub(crate) async fn get_notify_read(&'_ self) -> RwLockReadGuard<'_, Option>>> { 144 | RwLockReadGuard::map(self.inner.read().await, |i| &i.notify) 145 | } 146 | 147 | pub(crate) async fn clear_notify(&self) { 148 | let mut inner = self.inner.write().await; 149 | inner.notify = None; 150 | } 151 | 152 | pub(crate) async fn add_notify(&self, notify: Arc) { 153 | let mut inner = self.inner.write().await; 154 | if inner.notify.is_none() { 155 | inner.notify = Some(vec![notify]); 156 | } else if let Some(v) = inner.notify.as_mut() { 157 | v.push(notify); 158 | } 159 | } 160 | 161 | pub(crate) async fn notify_commit(&self) { 162 | // We have check the bound in the if branch 163 | let notify_vec = self.get_notify_read().await; 164 | if let Some(vec) = notify_vec.as_ref() { 165 | vec.iter().for_each(|notify| notify.notify_one()); 166 | } 167 | drop(notify_vec); 168 | self.clear_notify().await; 169 | } 170 | 171 | pub(crate) async fn notify_execute(&self, exe_result: Vec>) { 172 | let mut inner = self.inner.write().await; 173 | if inner.execute_complete.is_some() { 174 | let chan = inner.execute_complete.take(); 175 | chan.map(|chan| chan.0.send(exe_result)); 176 | } 177 | } 178 | } 179 | 180 | /// The status of the instance, which is recorded in the instance space 181 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] 182 | pub(crate) enum InstanceStatus { 183 | PreAccepted, 184 | PreAcceptedEq, 185 | Accepted, 186 | Committed, 187 | Executed, 188 | } 189 | -------------------------------------------------------------------------------- /epaxos/src/types/space.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use async_trait::async_trait; 4 | #[cfg(test)] 5 | use mockall::automock; 6 | use tokio::sync::{Notify, RwLock}; 7 | 8 | use crate::util::instance_exist; 9 | 10 | use super::{ 11 | cmd::Command, 12 | id::{LocalInstanceId, ReplicaId}, 13 | instance::{InstanceStatus, SharedInstance}, 14 | }; 15 | 16 | #[cfg_attr(test, automock)] 17 | #[async_trait] 18 | pub(crate) trait InstanceSpace 19 | where 20 | C: Command + Clone + Send + Sync + 'static, 21 | { 22 | /// Constructor to create a instance space 23 | fn new(peer_cnt: usize) -> Self; 24 | 25 | /// Get the instance, if the it returns a Notify it means the instance 26 | /// is not ready, and the Notify is stored in the Notify Vec. The 27 | /// returned Notify is always fresh new, so we don't reuse it to 28 | /// avoid racing. 29 | async fn get_instance_or_notify( 30 | &self, 31 | replica: &ReplicaId, 32 | instance_id: &LocalInstanceId, 33 | ) -> (Option>, Option>); 34 | 35 | /// Get the instance if it's created, otherwise None is returned 36 | /// TODO: unify `get_instance` and `get_instance_or_notify` 37 | async fn get_instance( 38 | &self, 39 | replica: &ReplicaId, 40 | instance_id: &LocalInstanceId, 41 | ) -> Option>; 42 | 43 | /// Helper function to insert instance into instance space. Because there might be hole 44 | /// in the space, we fill the hole as None for the later get operation. 45 | async fn insert_instance( 46 | &self, 47 | replica: &ReplicaId, 48 | instance_id: &LocalInstanceId, 49 | instance: SharedInstance, 50 | ); 51 | } 52 | 53 | // FIXME: maybe hashmap is more fit in this case. 54 | // A (replica_id, instance_id) pair to instance mapping. 55 | // And a big rwlock is not efficient here. 56 | pub struct VecInstanceSpace 57 | where 58 | C: Command + Clone + Send + Sync + 'static, 59 | { 60 | inner: RwLock>>>, 61 | } 62 | 63 | #[async_trait] 64 | impl InstanceSpace for VecInstanceSpace 65 | where 66 | C: Command + Clone + Send + Sync + 'static, 67 | { 68 | fn new(peer_cnt: usize) -> Self { 69 | let mut peer_vec = Vec::with_capacity(peer_cnt); 70 | (0..peer_cnt).for_each(|_| { 71 | peer_vec.push(vec![]); 72 | }); 73 | 74 | Self { 75 | inner: RwLock::new(peer_vec), 76 | } 77 | } 78 | 79 | /// Get the instance, if the it returns a Notify it means the instance 80 | /// is not ready, and the Notify is stored in the Notify Vec. The 81 | /// returned Notify is always fresh new, we don't reuse notify to 82 | /// avoid racing. 83 | async fn get_instance_or_notify( 84 | &self, 85 | replica: &ReplicaId, 86 | instance_id: &LocalInstanceId, 87 | ) -> (Option>, Option>) { 88 | // TODO: this lock is too big 89 | let mut space = self.inner.write().await; 90 | let instance_id = **instance_id; 91 | let replica_id = **replica; 92 | 93 | let space_len = space[replica_id].len(); 94 | if instance_id >= space_len { 95 | if Self::need_notify(&None).await { 96 | space[replica_id] 97 | .extend((space_len..(instance_id + 1)).map(|_| SharedInstance::none())); 98 | let notify = Arc::new(Notify::new()); 99 | space[replica_id][instance_id] 100 | .add_notify(notify.clone()) 101 | .await; 102 | (None, Some(notify)) 103 | } else { 104 | (None, None) 105 | } 106 | } else { 107 | // We have check the bound in the if branch 108 | let instance = space[replica_id][instance_id].clone(); 109 | if Self::need_notify(&Some(instance.clone())).await { 110 | let notify = Arc::new(Notify::new()); 111 | instance.add_notify(notify.clone()).await; 112 | (Some(instance), Some(notify)) 113 | } else { 114 | (Some(instance), None) 115 | } 116 | } 117 | } 118 | 119 | /// Get the instance if it's created, otherwise None is returned 120 | /// TODO: unify `get_instance` and `get_instance_or_notify` 121 | async fn get_instance( 122 | &self, 123 | replica: &ReplicaId, 124 | instance_id: &LocalInstanceId, 125 | ) -> Option> { 126 | let space = self.inner.read().await; 127 | let instance_id = **instance_id; 128 | let replica_id = **replica; 129 | 130 | if instance_id >= space[replica_id].len() { 131 | None 132 | } else { 133 | // We have check the bound in the if branch 134 | Some(space[replica_id][instance_id].clone()) 135 | } 136 | } 137 | 138 | /// Helper function to insert instance into instance space. Because there might be hole 139 | /// in the space, we fill the hole as None for the later get operation. 140 | async fn insert_instance( 141 | &self, 142 | replica: &ReplicaId, 143 | instance_id: &LocalInstanceId, 144 | instance: SharedInstance, 145 | ) { 146 | let mut space = self.inner.write().await; 147 | let instance_id = **instance_id; 148 | let replica_id = **replica; 149 | let space_len = space[replica_id].len(); 150 | match instance_id.partial_cmp(&space_len) { 151 | Some(std::cmp::Ordering::Greater) => { 152 | space[replica_id] 153 | .extend((space_len..(instance_id + 1)).map(|_| SharedInstance::none())); 154 | } 155 | Some(std::cmp::Ordering::Less) => { 156 | space[replica_id][instance_id] = instance; 157 | } 158 | Some(std::cmp::Ordering::Equal) => { 159 | space[replica_id].push(instance); 160 | } 161 | None => {} 162 | } 163 | } 164 | } 165 | 166 | impl VecInstanceSpace 167 | where 168 | C: Command + Clone + Send + Sync + 'static, 169 | { 170 | async fn need_notify(ins: &Option>) -> bool { 171 | if !instance_exist(ins).await { 172 | true 173 | } else { 174 | let d_ins = ins.as_ref().unwrap(); 175 | let d_ins_read = d_ins.get_instance_read().await; 176 | let d_ins_read_inner = d_ins_read.as_ref().unwrap(); 177 | 178 | !matches!( 179 | d_ins_read_inner.status, 180 | InstanceStatus::Committed | InstanceStatus::Executed 181 | ) 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /pro-macro/src/lib.rs: -------------------------------------------------------------------------------- 1 | use quote::quote; 2 | use syn::{parse_macro_input, spanned::Spanned, Data, DeriveInput, Error, Fields}; 3 | 4 | #[proc_macro_derive(FromInner)] 5 | pub fn from_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 6 | let input = parse_macro_input!(input as DeriveInput); 7 | let name = &input.ident; 8 | 9 | proc_macro::TokenStream::from(match input.data { 10 | Data::Struct(ref data) => match data.fields { 11 | Fields::Named(ref fields) => { 12 | if fields.named.len() != 1 { 13 | Error::new(input.span(), "Expected only one field").to_compile_error() 14 | } else { 15 | fields 16 | .named 17 | .first() 18 | .map(|f| { 19 | let t = &f.ty; 20 | f.ident.as_ref().map(|ident| { 21 | quote! { 22 | impl From<#t> for #name { 23 | fn from(value: #t) -> Self { 24 | Self { 25 | #ident: value, 26 | } 27 | } 28 | } 29 | 30 | impl std::ops::Deref for #name { 31 | type Target = #t; 32 | fn deref(&self) -> &Self::Target { 33 | &self.#ident 34 | } 35 | } 36 | 37 | impl std::ops::Add for #name { 38 | type Output = #name; 39 | 40 | fn add(self, rhs: usize) -> Self::Output { 41 | Self { 42 | #ident: self.#ident + rhs 43 | } 44 | } 45 | } 46 | 47 | impl std::ops::AddAssign for #name { 48 | fn add_assign(&mut self, rhs: usize) { 49 | self.#ident += rhs; 50 | } 51 | } 52 | 53 | impl<'a> std::ops::Add for &'a mut #name { 54 | type Output = #name; 55 | 56 | fn add(self, rhs: usize) -> Self::Output { 57 | #name { 58 | #ident: self.#ident + rhs, 59 | } 60 | } 61 | } 62 | 63 | impl<'a> std::ops::Add for &'a #name { 64 | type Output = #name; 65 | 66 | fn add(self, rhs: usize) -> Self::Output { 67 | #name { 68 | #ident: self.#ident + rhs, 69 | } 70 | } 71 | } 72 | 73 | impl std::ops::AddAssign for &mut #name { 74 | fn add_assign(&mut self, rhs: usize) { 75 | self.#ident += rhs; 76 | } 77 | } 78 | } 79 | }) 80 | }) 81 | .unwrap() 82 | .unwrap() 83 | } 84 | } 85 | Fields::Unnamed(ref fields) => { 86 | if fields.unnamed.len() != 1 { 87 | Error::new(input.span(), "Expected only one field").to_compile_error() 88 | } else { 89 | let t = &fields.unnamed.first().unwrap().ty; 90 | quote! { 91 | impl From<#t> for #name { 92 | fn from(value: #t) -> Self { 93 | Self(value) 94 | } 95 | } 96 | 97 | impl std::ops::Deref for #name { 98 | type Target = #t; 99 | fn deref(&self) -> &Self::Target { 100 | &self.0 101 | } 102 | } 103 | 104 | impl std::ops::Add for #name { 105 | type Output = #name; 106 | 107 | fn add(self, rhs: usize) -> Self::Output { 108 | Self(self.0 + rhs) 109 | } 110 | } 111 | 112 | impl std::ops::AddAssign for #name { 113 | fn add_assign(&mut self, rhs: usize) { 114 | self.0 += rhs; 115 | } 116 | } 117 | 118 | impl<'a> std::ops::Add for &'a mut #name { 119 | type Output = #name; 120 | 121 | fn add(self, rhs: usize) -> Self::Output { 122 | #name(self.0 + rhs) 123 | } 124 | } 125 | 126 | impl<'a> std::ops::Add for &'a #name { 127 | type Output = #name; 128 | 129 | fn add(self, rhs: usize) -> Self::Output { 130 | #name(self.0 + rhs) 131 | } 132 | } 133 | 134 | impl std::ops::AddAssign for &mut #name { 135 | fn add_assign(&mut self, rhs: usize) { 136 | self.0 += rhs; 137 | } 138 | } 139 | } 140 | } 141 | } 142 | _ => { 143 | Error::new(input.span(), "Only support named and unnamed struct").to_compile_error() 144 | } 145 | }, 146 | _ => Error::new(input.span(), "Only support named and unnamed struct").to_compile_error(), 147 | }) 148 | } 149 | -------------------------------------------------------------------------------- /epaxos/src/types/replica.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, fmt::Debug, iter, marker::PhantomData, sync::Arc}; 2 | 3 | use itertools::Itertools; 4 | use tokio::sync::mpsc; 5 | 6 | use crate::execute::Executor; 7 | 8 | use super::{ 9 | cmd::{Command, CommandExecutor}, 10 | id::{InstanceId, LocalInstanceId, ReplicaId, Seq}, 11 | instance::{Instance, SharedInstance}, 12 | space::InstanceSpace, 13 | }; 14 | 15 | type Conflict = Vec::K, SharedInstance>>; 16 | pub(crate) struct Replica 17 | where 18 | C: Command + Clone + Send + Sync + 'static, 19 | E: CommandExecutor + Clone, 20 | S: InstanceSpace, 21 | { 22 | pub(crate) id: ReplicaId, 23 | pub(crate) peer_cnt: usize, 24 | pub(crate) instance_space: Arc, 25 | /// Current max instance id for each replica, init values are 0s 26 | pub(crate) cur_instance: Vec, 27 | pub(crate) commited_upto: Vec>, 28 | pub(crate) conflict: Conflict, 29 | pub(crate) max_seq: Seq, 30 | pub(crate) exec_send: mpsc::Sender>, 31 | phantom: PhantomData, 32 | } 33 | 34 | impl Replica 35 | where 36 | C: Command + Debug + Clone + Sync + Send + 'static, 37 | E: CommandExecutor + Debug + Clone + Sync + Send + 'static, 38 | S: InstanceSpace + Send + Sync + 'static, 39 | { 40 | pub(crate) async fn new(id: usize, peer_cnt: usize, cmd_exe: E) -> Self { 41 | let instance_space = Arc::new(S::new(peer_cnt)); 42 | let mut cur_instance = Vec::with_capacity(peer_cnt); 43 | let mut commited_upto = Vec::with_capacity(peer_cnt); 44 | let mut conflict = Vec::with_capacity(peer_cnt); 45 | 46 | for _ in 0..peer_cnt { 47 | cur_instance.push(0.into()); 48 | commited_upto.push(None); 49 | conflict.push(HashMap::new()) 50 | } 51 | 52 | let (sender, receiver) = mpsc::channel(10); 53 | 54 | let space_clone = Clone::clone(&instance_space); 55 | tokio::spawn(async move { 56 | let executor = Executor::new(space_clone, cmd_exe); 57 | executor.execute(receiver).await; 58 | }); 59 | 60 | Self { 61 | id: id.into(), 62 | peer_cnt, 63 | instance_space, 64 | cur_instance, 65 | commited_upto, 66 | conflict, 67 | max_seq: 0.into(), 68 | exec_send: sender, 69 | phantom: PhantomData, 70 | } 71 | } 72 | 73 | // Helper function 74 | pub(crate) fn cur_instance(&self, r: &ReplicaId) -> LocalInstanceId { 75 | self.cur_instance[**r] 76 | } 77 | 78 | pub(crate) fn set_cur_instance(&mut self, inst: &InstanceId) { 79 | self.cur_instance[*inst.replica] = inst.local; 80 | } 81 | 82 | pub(crate) fn local_cur_instance(&self) -> &LocalInstanceId { 83 | self.cur_instance.get(*self.id).unwrap() 84 | } 85 | 86 | pub(crate) fn inc_local_cur_instance(&mut self) -> &LocalInstanceId { 87 | let mut ins_id = self.cur_instance.get_mut(*self.id).unwrap(); 88 | ins_id += 1; 89 | ins_id 90 | } 91 | 92 | // it's not a pure function 93 | pub(crate) fn merge_seq_deps( 94 | &self, 95 | ins: &mut Instance, 96 | new_seq: &Seq, 97 | new_deps: &[Option], 98 | ) -> bool { 99 | let mut equal = true; 100 | if &ins.seq != new_seq { 101 | equal = false; 102 | if new_seq > &ins.seq { 103 | ins.seq = *new_seq; 104 | } 105 | } 106 | 107 | ins.deps.iter_mut().zip(new_deps.iter()).for_each(|(o, n)| { 108 | if o != n { 109 | equal = false; 110 | if o.is_none() || (n.is_some() && o.as_ref().unwrap() < n.as_ref().unwrap()) { 111 | *o = *n; 112 | } 113 | } 114 | }); 115 | equal 116 | } 117 | 118 | pub(crate) async fn get_seq_deps(&self, cmds: &[C]) -> (Seq, Vec>) { 119 | let mut new_seq = 0.into(); 120 | let mut deps: Vec> = 121 | iter::repeat_with(|| None).take(self.peer_cnt).collect(); 122 | for (r_id, command) in (0..self.peer_cnt).cartesian_product(cmds.iter()) { 123 | if r_id != *self.id { 124 | if let Some(ins) = self.conflict[r_id].get(command.key()) { 125 | let conflict_ins = ins.get_instance_read().await; 126 | if conflict_ins.is_none() { 127 | continue; 128 | } 129 | let conflict_ins = SharedInstance::get_raw_read(conflict_ins); 130 | // update deps 131 | deps[r_id] = match deps[r_id] { 132 | Some(dep_ins_id) => { 133 | if conflict_ins.local_id() > dep_ins_id { 134 | Some(conflict_ins.local_id()) 135 | } else { 136 | Some(dep_ins_id) 137 | } 138 | } 139 | None => Some(conflict_ins.local_id()), 140 | }; 141 | // update seq 142 | let s = &conflict_ins.seq; 143 | if s >= &new_seq { 144 | new_seq = (**s + 1).into(); 145 | } 146 | } 147 | } 148 | } 149 | (new_seq, deps) 150 | } 151 | 152 | // TODO: merge with get_seq_deps 153 | pub(crate) async fn update_seq_deps( 154 | &self, 155 | mut seq: Seq, 156 | mut deps: Vec>, 157 | cmds: &[C], 158 | ) -> (Seq, Vec>, bool) { 159 | let mut changed = false; 160 | for (r_id, command) in (0..self.peer_cnt).cartesian_product(cmds.iter()) { 161 | if r_id != *self.id { 162 | if let Some(ins) = self.conflict[r_id].get(command.key()) { 163 | let conflict_ins = ins.get_instance_read().await; 164 | if conflict_ins.is_none() { 165 | continue; 166 | } 167 | let conflict_ins = SharedInstance::get_raw_read(conflict_ins); 168 | if deps[r_id].is_some() && deps[r_id].unwrap() < conflict_ins.local_id() { 169 | changed = true; 170 | 171 | // update deps 172 | deps[r_id] = match deps[r_id] { 173 | Some(dep_ins_id) => { 174 | if conflict_ins.local_id() > dep_ins_id { 175 | Some(conflict_ins.local_id()) 176 | } else { 177 | Some(dep_ins_id) 178 | } 179 | } 180 | None => Some(conflict_ins.local_id()), 181 | }; 182 | 183 | // update seq 184 | let conflict_seq = &conflict_ins.seq; 185 | if conflict_seq >= &seq { 186 | seq = (**conflict_seq + 1).into(); 187 | } 188 | } 189 | } 190 | } 191 | } 192 | (seq, deps, changed) 193 | } 194 | 195 | pub(crate) async fn update_conflict( 196 | &mut self, 197 | replica: &ReplicaId, 198 | cmds: &[C], 199 | new_inst: SharedInstance, 200 | ) { 201 | for c in cmds { 202 | let new_inst = match self.conflict[**replica].get(c.key()) { 203 | None => Some(new_inst.clone()), 204 | Some(ins) => { 205 | let ins = ins.get_instance_read().await; 206 | if ins.is_some() 207 | && SharedInstance::get_raw_read(ins).local_id() 208 | >= SharedInstance::get_raw_read(new_inst.get_instance_read().await) 209 | .local_id() 210 | { 211 | None 212 | } else { 213 | Some(new_inst.clone()) 214 | } 215 | } 216 | }; 217 | 218 | if new_inst.is_some() { 219 | self.conflict[*self.id].insert(c.key().clone(), new_inst.unwrap()); 220 | } 221 | } 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /epaxos/src/execute.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cmp::Ordering::{Equal, Greater, Less}, 3 | collections::{HashMap, VecDeque}, 4 | fmt::Debug, 5 | marker::PhantomData, 6 | sync::Arc, 7 | }; 8 | 9 | use petgraph::{ 10 | algo::tarjan_scc, 11 | graph::{DiGraph, NodeIndex}, 12 | }; 13 | use tokio::sync::mpsc; 14 | 15 | use crate::types::{Command, CommandExecutor, InstanceSpace, InstanceStatus, SharedInstance}; 16 | 17 | pub(crate) struct Executor 18 | where 19 | C: Command + Clone + Send + Sync + 'static, 20 | E: CommandExecutor, 21 | S: InstanceSpace, 22 | { 23 | space: Arc, 24 | cmd_exe: E, 25 | _phanto: PhantomData, 26 | } 27 | 28 | impl Executor 29 | where 30 | C: Command + Debug + Clone + Send + Sync + 'static, 31 | E: CommandExecutor + Debug + Clone + Send + Sync + 'static, 32 | S: InstanceSpace + Send + Sync + 'static, 33 | { 34 | pub(crate) fn new(space: Arc, cmd_exe: E) -> Self { 35 | Self { 36 | space, 37 | cmd_exe, 38 | _phanto: PhantomData, 39 | } 40 | } 41 | 42 | pub(crate) async fn execute(&self, mut recv: mpsc::Receiver>) { 43 | // A inifite loop to pull instance to execute 44 | loop { 45 | let ins = recv.recv().await; 46 | 47 | if ins.is_none() { 48 | // The channel has been closed, stop recving. 49 | break; 50 | } 51 | 52 | let mut inner = 53 | InnerExecutor::new(Arc::::clone(&self.space), &self.cmd_exe, ins.unwrap()); 54 | tokio::spawn(async move { 55 | let scc = inner.build_scc().await; 56 | if let Some(scc) = scc { 57 | inner.execute(scc).await; 58 | } 59 | }); 60 | } 61 | } 62 | } 63 | 64 | struct InnerExecutor 65 | where 66 | C: Command + Clone + Send + Sync + 'static, 67 | E: CommandExecutor, 68 | S: InstanceSpace, 69 | { 70 | space: Arc, 71 | cmd_exe: E, 72 | start_ins: SharedInstance, 73 | map: Option, NodeIndex>>, 74 | graph: Option, ()>>, 75 | } 76 | 77 | impl InnerExecutor 78 | where 79 | C: Command + Debug + Clone + Send + Sync + 'static, 80 | E: CommandExecutor + Clone + Send + Sync, 81 | S: InstanceSpace + Send + Sync + 'static, 82 | { 83 | fn new(space: Arc, cmd_exe: &E, start_ins: SharedInstance) -> Self { 84 | Self { 85 | space, 86 | cmd_exe: cmd_exe.clone(), 87 | start_ins, 88 | map: None, 89 | graph: None, 90 | } 91 | } 92 | 93 | /// Build the scc and generate the result vec from an instance. We'll stop inserting instance to 94 | /// the graph in the following condition: 95 | /// - the instance's status is EXECUTED, which means every following step is EXECUTED. 96 | /// 97 | /// We'll also wait for one instance in the following conditions: 98 | /// - the instance's status is NOT COMMITTED and NOT EXECUTED. 99 | /// - the instance is empty. 100 | /// 101 | /// The return value is None if there's no instance to execute. 102 | /// The return value is Some(Vec<...>), which is the scc vec, if there are instances to execute. 103 | async fn build_scc(&mut self) -> Option>> 104 | where 105 | C: Command + Clone + Send + Sync + 'static, 106 | { 107 | // the start_ins is at least in the stage of COMMITTED 108 | if self 109 | .start_ins 110 | .match_status(&[InstanceStatus::Executed]) 111 | .await 112 | { 113 | return None; 114 | } 115 | 116 | let mut queue = VecDeque::new(); 117 | queue.push_back(self.start_ins.clone()); 118 | 119 | // initialization for map and graph fields 120 | self.map = Some(HashMap::, NodeIndex>::new()); 121 | self.graph = Some(DiGraph::, ()>::new()); 122 | 123 | loop { 124 | let cur = queue.pop_front(); 125 | 126 | // queue is empty 127 | if cur.is_none() { 128 | break; 129 | } 130 | let cur = cur.unwrap(); 131 | 132 | // get node index 133 | let cur_index = self.get_or_insert_index(&cur); 134 | let cur_read = cur.get_instance_read().await; 135 | let cur_read_inner = cur_read.as_ref().unwrap(); 136 | 137 | for (r, d) in cur_read_inner.deps.iter().enumerate() { 138 | if d.is_none() { 139 | continue; 140 | } 141 | let r = r.into(); 142 | let d = d.as_ref().unwrap(); 143 | 144 | let (d_ins, notify) = self.space.get_instance_or_notify(&r, d).await; 145 | 146 | let d_ins = if let Some(n) = notify { 147 | n.notified().await; 148 | self.space.get_instance(&r, d).await 149 | } else { 150 | d_ins 151 | }; 152 | 153 | assert!( 154 | d_ins.is_some(), 155 | "instance should not be none after notification" 156 | ); 157 | let d_ins = d_ins.unwrap(); 158 | 159 | if d_ins.match_status(&[InstanceStatus::Committed]).await { 160 | // there might be cycle 161 | if !self.has_visited(&d_ins) { 162 | queue.push_back(d_ins.clone()); 163 | } 164 | 165 | let d_index = self.get_or_insert_index(&d_ins); 166 | self.add_edge(cur_index, d_index); 167 | } 168 | } 169 | } 170 | 171 | Some(self.generate_scc()) 172 | } 173 | 174 | /// Get the graph index for the instance, if the index is missing we 175 | /// insert the instance into graph and return the index, otherwise 176 | /// return the index in the map directly. 177 | fn get_or_insert_index(&mut self, ins: &SharedInstance) -> NodeIndex 178 | where 179 | C: Command + Clone + Send + Sync + 'static, 180 | { 181 | let map = self.map.as_mut().unwrap(); 182 | let g = self.graph.as_mut().unwrap(); 183 | if !HashMap::contains_key(map, ins) { 184 | let index = g.add_node(ins.clone()); 185 | map.insert(ins.clone(), index); 186 | index 187 | } else { 188 | *map.get(ins).unwrap() 189 | } 190 | } 191 | 192 | /// Tell whether we have visited the instance while building the dep graph 193 | fn has_visited(&self, ins: &SharedInstance) -> bool { 194 | let map = self.map.as_ref().unwrap(); 195 | map.contains_key(ins) 196 | } 197 | 198 | fn add_edge(&mut self, src: NodeIndex, dst: NodeIndex) { 199 | let g = self.graph.as_mut().unwrap(); 200 | g.add_edge(src, dst, ()); 201 | } 202 | 203 | fn generate_scc(&self) -> Vec> { 204 | let g = self.graph.as_ref().unwrap(); 205 | tarjan_scc(g) 206 | } 207 | 208 | async fn execute(&self, scc: Vec>) { 209 | let g = self.graph.as_ref().unwrap(); 210 | for each_scc in scc { 211 | let ins_vec = each_scc.iter().map(|index| &g[*index]); 212 | 213 | let mut sort_vec = Vec::with_capacity(each_scc.len()); 214 | for (id, ins) in ins_vec.enumerate() { 215 | let ins_read = ins.get_instance_read().await; 216 | let ins_read = ins_read.as_ref().unwrap(); 217 | sort_vec.push((id, (ins_read.id.replica, ins_read.seq))); 218 | } 219 | 220 | sort_vec.sort_by(|a, b| { 221 | // Compare seq 222 | match a.1 .1.partial_cmp(&b.1 .1) { 223 | Some(Greater) => return Greater, 224 | Some(Less) => return Less, 225 | _ => {} 226 | }; 227 | 228 | // Compare replica id 229 | match a.1 .0.partial_cmp(&b.1 .0) { 230 | Some(Greater) => Greater, 231 | Some(Less) => Less, 232 | _ => Equal, 233 | } 234 | }); 235 | 236 | for (id, _) in sort_vec { 237 | let mut results = vec![]; 238 | let ins = &g[each_scc[id]]; 239 | { 240 | let mut ins_write = ins.get_instance_write().await; 241 | let ins_write = ins_write.as_mut().unwrap(); 242 | 243 | // It may be executed by other execution tasks 244 | if matches!(ins_write.status, InstanceStatus::Committed) { 245 | for c in &ins_write.cmds { 246 | let exe_result = c.execute(&self.cmd_exe).await; 247 | results.push(exe_result); 248 | } 249 | ins_write.status = InstanceStatus::Executed; 250 | } 251 | } 252 | ins.notify_execute(results).await; 253 | } 254 | } 255 | } 256 | 257 | /// This function is used in the UTs to get the `SharedInstance` from the NodeIndex 258 | #[cfg(test)] 259 | fn get_node_from_graph_index(&self, index: NodeIndex) -> SharedInstance { 260 | self.graph.as_ref().unwrap()[index].clone() 261 | } 262 | } 263 | 264 | #[cfg(test)] 265 | mod test { 266 | use std::{collections::BTreeMap, sync::Arc}; 267 | 268 | use mockall::predicate; 269 | use tokio::sync::mpsc; 270 | 271 | use crate::types::{ 272 | Ballot, Instance, InstanceId, InstanceSpace, InstanceStatus, LeaderBook, MockCommand, 273 | MockCommandExecutor, MockCommandId, MockInstanceSpace, Seq, SharedInstance, 274 | }; 275 | 276 | use super::InnerExecutor; 277 | 278 | fn build_space( 279 | space: &mut MockInstanceSpace, 280 | nodes: BTreeMap)>, 281 | deps: BTreeMap>, 282 | ) -> Vec> { 283 | let mut shared_instance = vec![]; 284 | for (id, (status, seq, cmd)) in nodes { 285 | let deps = deps 286 | .get(&id) 287 | .unwrap_or_else(|| panic!("id {:?} has no deps setting", id)); 288 | let largest_replica = deps.iter().map(|ins| ins.replica).max(); 289 | 290 | let deps = match largest_replica { 291 | Some(top) => { 292 | let mut v = Vec::new(); 293 | v.extend((0..*top + 1).map(|_| None)); 294 | deps.iter() 295 | .for_each(|inst_id| v[*inst_id.replica] = Some(inst_id.local)); 296 | v 297 | } 298 | None => vec![], 299 | }; 300 | 301 | let ins = SharedInstance::new( 302 | Some(Instance { 303 | id: InstanceId::new(id.replica, id.local), 304 | seq, 305 | ballot: Ballot::new_with_epoch(0.into(), 0), 306 | cmds: cmd, 307 | deps: deps.to_owned(), 308 | status, 309 | lb: LeaderBook::default(), 310 | }), 311 | None, 312 | ); 313 | 314 | let ins_clone = ins.clone(); 315 | let ins_return = ins.clone(); 316 | 317 | space 318 | .expect_get_instance_or_notify() 319 | .with(predicate::eq(id.replica), predicate::eq(id.local)) 320 | .returning(move |_rep, _id| (Some(ins.clone()), None)); 321 | 322 | space 323 | .expect_get_instance() 324 | .with(predicate::eq(id.replica), predicate::eq(id.local)) 325 | .returning(move |_rep, _id| Some(ins_clone.clone())); 326 | 327 | shared_instance.push(ins_return); 328 | } 329 | 330 | shared_instance 331 | } 332 | 333 | #[tokio::test] 334 | async fn build_scc_from_executed() { 335 | let mut space = MockInstanceSpace::::default(); 336 | let cmd_exe = MockCommandExecutor::new(None); 337 | 338 | let mut deps = BTreeMap::new(); 339 | let mut status = BTreeMap::new(); 340 | deps.insert(InstanceId::new(0.into(), 0.into()), vec![]); 341 | status.insert( 342 | InstanceId::new(0.into(), 0.into()), 343 | (InstanceStatus::Executed, 0.into(), vec![]), 344 | ); 345 | build_space(&mut space, status, deps); 346 | 347 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 348 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 349 | assert_eq!(inner_exe.build_scc().await, None); 350 | } 351 | 352 | #[tokio::test] 353 | async fn build_scc_cycle() { 354 | let mut space = MockInstanceSpace::::default(); 355 | let cmd_exe = MockCommandExecutor::new(None); 356 | 357 | let mut deps = BTreeMap::new(); 358 | let mut nodes = BTreeMap::new(); 359 | deps.insert( 360 | InstanceId::new(0.into(), 0.into()), 361 | vec![InstanceId::new(1.into(), 0.into())], 362 | ); 363 | deps.insert( 364 | InstanceId::new(1.into(), 0.into()), 365 | vec![InstanceId::new(2.into(), 0.into())], 366 | ); 367 | deps.insert( 368 | InstanceId::new(2.into(), 0.into()), 369 | vec![InstanceId::new(0.into(), 0.into())], 370 | ); 371 | nodes.insert( 372 | InstanceId::new(0.into(), 0.into()), 373 | (InstanceStatus::Committed, 0.into(), vec![]), 374 | ); 375 | nodes.insert( 376 | InstanceId::new(1.into(), 0.into()), 377 | (InstanceStatus::Committed, 0.into(), vec![]), 378 | ); 379 | nodes.insert( 380 | InstanceId::new(2.into(), 0.into()), 381 | (InstanceStatus::Committed, 0.into(), vec![]), 382 | ); 383 | build_space(&mut space, nodes, deps); 384 | 385 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 386 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 387 | let scc = inner_exe.build_scc().await; 388 | 389 | assert!(scc.is_some()); 390 | let scc = scc.unwrap(); 391 | assert_eq!(scc.len(), 1); 392 | let cycle = &scc[0]; 393 | assert_eq!(cycle.len(), 3); 394 | } 395 | 396 | #[tokio::test] 397 | async fn build_scc_line() { 398 | let mut space = MockInstanceSpace::::default(); 399 | let cmd_exe = MockCommandExecutor::new(None); 400 | 401 | let mut deps = BTreeMap::new(); 402 | let mut nodes = BTreeMap::new(); 403 | deps.insert( 404 | InstanceId::new(0.into(), 0.into()), 405 | vec![InstanceId::new(1.into(), 0.into())], 406 | ); 407 | deps.insert( 408 | InstanceId::new(1.into(), 0.into()), 409 | vec![InstanceId::new(2.into(), 0.into())], 410 | ); 411 | deps.insert(InstanceId::new(2.into(), 0.into()), vec![]); 412 | nodes.insert( 413 | InstanceId::new(0.into(), 0.into()), 414 | (InstanceStatus::Committed, 0.into(), vec![]), 415 | ); 416 | nodes.insert( 417 | InstanceId::new(1.into(), 0.into()), 418 | (InstanceStatus::Committed, 0.into(), vec![]), 419 | ); 420 | nodes.insert( 421 | InstanceId::new(2.into(), 0.into()), 422 | (InstanceStatus::Committed, 0.into(), vec![]), 423 | ); 424 | let origin_nodes = build_space(&mut space, nodes, deps); 425 | 426 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 427 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 428 | let scc = inner_exe.build_scc().await; 429 | 430 | assert!(scc.is_some()); 431 | let scc = scc.unwrap(); 432 | assert_eq!(scc.len(), 3); 433 | assert_eq!(scc[0].len(), 1); 434 | assert_eq!(scc[1].len(), 1); 435 | assert_eq!(scc[2].len(), 1); 436 | 437 | assert_eq!( 438 | inner_exe.get_node_from_graph_index(scc[0][0]), 439 | origin_nodes[2] 440 | ); 441 | assert_eq!( 442 | inner_exe.get_node_from_graph_index(scc[1][0]), 443 | origin_nodes[1] 444 | ); 445 | assert_eq!( 446 | inner_exe.get_node_from_graph_index(scc[2][0]), 447 | origin_nodes[0] 448 | ); 449 | } 450 | 451 | #[tokio::test] 452 | async fn build_scc_line_executed() { 453 | let mut space = MockInstanceSpace::::default(); 454 | let cmd_exe = MockCommandExecutor::new(None); 455 | 456 | let mut deps = BTreeMap::new(); 457 | let mut nodes = BTreeMap::new(); 458 | deps.insert( 459 | InstanceId::new(0.into(), 0.into()), 460 | vec![InstanceId::new(1.into(), 0.into())], 461 | ); 462 | deps.insert( 463 | InstanceId::new(1.into(), 0.into()), 464 | vec![InstanceId::new(2.into(), 0.into())], 465 | ); 466 | deps.insert(InstanceId::new(2.into(), 0.into()), vec![]); 467 | nodes.insert( 468 | InstanceId::new(0.into(), 0.into()), 469 | (InstanceStatus::Committed, 0.into(), vec![]), 470 | ); 471 | nodes.insert( 472 | InstanceId::new(1.into(), 0.into()), 473 | (InstanceStatus::Committed, 0.into(), vec![]), 474 | ); 475 | nodes.insert( 476 | InstanceId::new(2.into(), 0.into()), 477 | (InstanceStatus::Executed, 0.into(), vec![]), 478 | ); 479 | let origin_nodes = build_space(&mut space, nodes, deps); 480 | 481 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 482 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 483 | let scc = inner_exe.build_scc().await; 484 | 485 | assert!(scc.is_some()); 486 | let scc = scc.unwrap(); 487 | assert_eq!(scc.len(), 2); 488 | assert_eq!(scc[0].len(), 1); 489 | assert_eq!(scc[1].len(), 1); 490 | 491 | assert_eq!( 492 | inner_exe.get_node_from_graph_index(scc[0][0]), 493 | origin_nodes[1] 494 | ); 495 | assert_eq!( 496 | inner_exe.get_node_from_graph_index(scc[1][0]), 497 | origin_nodes[0] 498 | ); 499 | } 500 | 501 | #[tokio::test] 502 | async fn executed_line() { 503 | let mut space = MockInstanceSpace::::default(); 504 | let (sender, mut receiver) = mpsc::channel(10); 505 | let cmd_exe = MockCommandExecutor::new(Some(sender)); 506 | 507 | let mut deps = BTreeMap::new(); 508 | let mut nodes = BTreeMap::new(); 509 | deps.insert( 510 | InstanceId::new(0.into(), 0.into()), 511 | vec![InstanceId::new(1.into(), 0.into())], 512 | ); 513 | deps.insert( 514 | InstanceId::new(1.into(), 0.into()), 515 | vec![InstanceId::new(2.into(), 0.into())], 516 | ); 517 | deps.insert(InstanceId::new(2.into(), 0.into()), vec![]); 518 | 519 | let mut mock_ids = vec![]; 520 | let ins_id = InstanceId::new(0.into(), 0.into()); 521 | let cmd_id = MockCommandId::new(ins_id, 0); 522 | mock_ids.push(cmd_id); 523 | nodes.insert( 524 | ins_id, 525 | ( 526 | InstanceStatus::Committed, 527 | 0.into(), 528 | vec![MockCommand::new("", cmd_id)], 529 | ), 530 | ); 531 | 532 | let ins_id = InstanceId::new(1.into(), 0.into()); 533 | let cmd_id = MockCommandId::new(ins_id, 0); 534 | mock_ids.push(cmd_id); 535 | nodes.insert( 536 | ins_id, 537 | ( 538 | InstanceStatus::Committed, 539 | 0.into(), 540 | vec![MockCommand::new("", cmd_id)], 541 | ), 542 | ); 543 | 544 | let ins_id = InstanceId::new(2.into(), 0.into()); 545 | let cmd_id = MockCommandId::new(ins_id, 0); 546 | mock_ids.push(cmd_id); 547 | nodes.insert( 548 | ins_id, 549 | ( 550 | InstanceStatus::Committed, 551 | 0.into(), 552 | vec![MockCommand::new("", cmd_id)], 553 | ), 554 | ); 555 | let _ = build_space(&mut space, nodes, deps); 556 | 557 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 558 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 559 | let scc = inner_exe.build_scc().await; 560 | 561 | inner_exe.execute(scc.unwrap()).await; 562 | 563 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[2]); 564 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[1]); 565 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[0]); 566 | } 567 | 568 | #[tokio::test] 569 | async fn executed_cycle() { 570 | let mut space = MockInstanceSpace::::default(); 571 | let (sender, mut receiver) = mpsc::channel(10); 572 | let cmd_exe = MockCommandExecutor::new(Some(sender)); 573 | 574 | let mut deps = BTreeMap::new(); 575 | let mut nodes = BTreeMap::new(); 576 | deps.insert( 577 | InstanceId::new(0.into(), 0.into()), 578 | vec![InstanceId::new(1.into(), 0.into())], 579 | ); 580 | deps.insert( 581 | InstanceId::new(1.into(), 0.into()), 582 | vec![InstanceId::new(2.into(), 0.into())], 583 | ); 584 | deps.insert( 585 | InstanceId::new(2.into(), 0.into()), 586 | vec![InstanceId::new(0.into(), 0.into())], 587 | ); 588 | 589 | let mut mock_ids = vec![]; 590 | let ins_id = InstanceId::new(0.into(), 0.into()); 591 | let cmd_id = MockCommandId::new(ins_id, 0); 592 | mock_ids.push(cmd_id); 593 | nodes.insert( 594 | ins_id, 595 | ( 596 | InstanceStatus::Committed, 597 | 0.into(), 598 | vec![MockCommand::new("", cmd_id)], 599 | ), 600 | ); 601 | 602 | let ins_id = InstanceId::new(1.into(), 0.into()); 603 | let cmd_id = MockCommandId::new(ins_id, 0); 604 | mock_ids.push(cmd_id); 605 | nodes.insert( 606 | ins_id, 607 | ( 608 | InstanceStatus::Committed, 609 | 0.into(), 610 | vec![MockCommand::new("", cmd_id)], 611 | ), 612 | ); 613 | 614 | let ins_id = InstanceId::new(2.into(), 0.into()); 615 | let cmd_id = MockCommandId::new(ins_id, 0); 616 | mock_ids.push(cmd_id); 617 | nodes.insert( 618 | ins_id, 619 | ( 620 | InstanceStatus::Committed, 621 | 0.into(), 622 | vec![MockCommand::new("", cmd_id)], 623 | ), 624 | ); 625 | let _ = build_space(&mut space, nodes, deps); 626 | 627 | let shared_ins = space.get_instance(&0.into(), &0.into()).await.unwrap(); 628 | let mut inner_exe = InnerExecutor::new(Arc::new(space), &cmd_exe, shared_ins); 629 | let scc = inner_exe.build_scc().await; 630 | 631 | inner_exe.execute(scc.unwrap()).await; 632 | 633 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[0]); 634 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[1]); 635 | assert_eq!(receiver.recv().await.unwrap(), mock_ids[2]); 636 | } 637 | } 638 | -------------------------------------------------------------------------------- /epaxos/src/server.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | config::Configure, 3 | error::RpcError, 4 | message::{ 5 | Accept, AcceptReply, Commit, Message, PreAccept, PreAcceptOk, PreAcceptReply, Propose, 6 | ProposeResponse, SingleCmdResult, 7 | }, 8 | types::{ 9 | Ballot, Command, CommandExecutor, Instance, InstanceId, InstanceSpace, InstanceStatus, 10 | LeaderBook, LeaderId, Replica, ReplicaId, SharedInstance, VecInstanceSpace, 11 | }, 12 | util::{self, instance_exist}, 13 | }; 14 | use futures::stream::{self, StreamExt}; 15 | use log::{debug, trace}; 16 | use serde::{de::DeserializeOwned, Serialize}; 17 | use std::fmt::Debug; 18 | use std::sync::Arc; 19 | use tokio::{ 20 | io::AsyncWriteExt, 21 | net::{TcpListener, TcpStream}, 22 | sync::{oneshot, Mutex, RwLock}, 23 | task::JoinHandle, 24 | }; 25 | 26 | /// Hide 27 | pub struct DefaultServer 28 | where 29 | C: Command + Clone + Send + Sync + 'static, 30 | E: CommandExecutor + Clone, 31 | { 32 | inner: Server>, 33 | } 34 | 35 | impl DefaultServer 36 | where 37 | C: Command + Debug + Clone + Send + Sync + Serialize + DeserializeOwned + 'static, 38 | E: CommandExecutor + Debug + Clone + Send + Sync + 'static, 39 | { 40 | pub async fn new(conf: Configure, cmd_exe: E) -> Self { 41 | Self { 42 | inner: Server::new(conf, cmd_exe).await, 43 | } 44 | } 45 | 46 | pub async fn run(&self) { 47 | self.inner.run().await; 48 | } 49 | } 50 | 51 | pub(crate) struct Server 52 | where 53 | C: Command + Clone + Send + Sync + 'static, 54 | E: CommandExecutor + Clone, 55 | S: InstanceSpace, 56 | { 57 | rpc_server: RpcServer, 58 | } 59 | 60 | impl Server 61 | where 62 | C: Command + Debug + Clone + Send + Sync + Serialize + DeserializeOwned + 'static, 63 | E: CommandExecutor + Debug + Clone + Send + Sync + 'static, 64 | S: InstanceSpace + Send + Sync + 'static, 65 | { 66 | pub async fn new(conf: Configure, cmd_exe: E) -> Self { 67 | let inner = Arc::new(InnerServer::new(conf, cmd_exe).await); 68 | let rpc_server = RpcServer::new(inner.conf(), inner.clone()).await; 69 | Self { rpc_server } 70 | } 71 | 72 | pub async fn run(&self) { 73 | let _ = self.rpc_server.serve().await; 74 | } 75 | } 76 | 77 | pub(crate) struct InnerServer 78 | where 79 | C: Command + Clone + Send + Sync + 'static, 80 | E: CommandExecutor + Clone, 81 | S: InstanceSpace, 82 | { 83 | conf: Configure, 84 | // Connection List is modified once on initialization, 85 | // and connection linked to self should be empty 86 | conns: RwLock>>>>, 87 | replica: Arc>>, 88 | } 89 | 90 | impl InnerServer 91 | where 92 | C: Command + Debug + Clone + Serialize + Send + Sync + 'static, 93 | E: CommandExecutor + Debug + Clone + Send + Sync + 'static, 94 | S: InstanceSpace + Send + Sync + 'static, 95 | { 96 | pub(crate) async fn new(conf: Configure, cmd_exe: E) -> Self { 97 | let peer_cnt = conf.peer_cnt; 98 | let id = conf.index; 99 | Self { 100 | conf, 101 | conns: RwLock::new(vec![]), 102 | replica: Arc::new(Mutex::new( 103 | Replica::new(id, peer_cnt as usize, cmd_exe).await, 104 | )), 105 | } 106 | } 107 | 108 | pub(crate) fn conf(&self) -> &Configure { 109 | &self.conf 110 | } 111 | 112 | pub(crate) fn new_leaderbook(&self, r_id: ReplicaId) -> LeaderBook { 113 | LeaderBook::new(r_id, &self.conf) 114 | } 115 | 116 | pub(crate) fn new_ballot(&self, r_id: ReplicaId) -> Ballot { 117 | Ballot::new(r_id, &self.conf) 118 | } 119 | 120 | pub(crate) async fn handle_message(&self, message: Message, write_stream: &mut T) 121 | where 122 | C: Command + Serialize, 123 | T: AsyncWriteExt + Unpin, 124 | { 125 | match message { 126 | Message::PreAccept(pa) => self.handle_preaccept(pa).await, 127 | Message::PreAcceptReply(par) => self.handle_preaccept_reply(par).await, 128 | Message::PreAcceptOk(pao) => self.handle_preaccept_ok(pao).await, 129 | Message::Accept(a) => self.handle_accept(a).await, 130 | Message::AcceptReply(ar) => self.handle_accept_reply(ar).await, 131 | Message::Commit(cm) => self.handle_commit(cm).await, 132 | Message::CommitShort(_) => todo!(), 133 | Message::Propose(p) => self.handle_propose(p, write_stream).await, 134 | Message::ProposeResponse(_) => unreachable!("This is response from server to client"), 135 | } 136 | } 137 | 138 | async fn handle_accept_reply(&self, ar: AcceptReply) { 139 | trace!("handle accept reply"); 140 | 141 | let r = self.replica.lock().await; 142 | let orig = r 143 | .instance_space 144 | .get_instance(&ar.instance_id.replica, &ar.instance_id.local) 145 | .await; 146 | 147 | if !instance_exist(&orig).await { 148 | panic!("The instance {:?} should exist", ar.instance_id); 149 | } 150 | 151 | let orig = orig.unwrap(); 152 | let mut ins_write = orig.get_instance_write().await; 153 | let ins = ins_write.as_mut().unwrap(); 154 | 155 | if !ar.ok { 156 | ins.lb.nack += 1; 157 | if ar.ballot > ins.lb.max_ballot { 158 | ins.lb.max_ballot = ar.ballot; 159 | } 160 | return; 161 | } 162 | 163 | ins.lb.accept_ok += 1; 164 | 165 | if ins.lb.accept_ok >= r.peer_cnt / 2 { 166 | ins.status = InstanceStatus::Committed; 167 | // TODO: sync to disk 168 | self.bdcst_message( 169 | r.id, 170 | Message::::Commit(Commit { 171 | leader_id: r.id.into(), 172 | instance_id: ar.instance_id, 173 | seq: ins.seq, 174 | cmds: ins.cmds.to_vec(), 175 | deps: ins.deps.to_vec(), 176 | }), 177 | ) 178 | .await; 179 | 180 | drop(ins_write); 181 | 182 | // TODO: sync to disk 183 | let _ = r.exec_send.send(orig.clone()).await; 184 | orig.notify_commit().await; 185 | } 186 | } 187 | 188 | async fn handle_accept(&self, a: Accept) { 189 | trace!("handle accept"); 190 | 191 | let mut r = self.replica.lock().await; 192 | 193 | if a.instance_id.local >= r.cur_instance(&a.instance_id.replica) { 194 | r.set_cur_instance(&InstanceId::new( 195 | a.instance_id.replica, 196 | (*a.instance_id.local + 1).into(), 197 | )); 198 | } 199 | 200 | let ins = r 201 | .instance_space 202 | .get_instance(&a.instance_id.replica, &a.instance_id.local) 203 | .await; 204 | 205 | let exist = instance_exist(&ins).await; 206 | if exist { 207 | let ins = ins.unwrap(); 208 | let ins_read = ins.get_instance_read().await; 209 | let ins_read_inner = ins_read.as_ref().unwrap(); 210 | if matches!( 211 | ins_read_inner.status, 212 | InstanceStatus::Committed | InstanceStatus::Executed 213 | ) { 214 | // We've translated to the later states 215 | return; 216 | } 217 | 218 | let ins_ballot = ins_read_inner.ballot; 219 | if a.ballot < ins_ballot { 220 | self.reply( 221 | r.id, 222 | &a.leader_id, 223 | Message::::AcceptReply(AcceptReply { 224 | instance_id: a.instance_id, 225 | ok: false, 226 | ballot: ins_ballot, 227 | }), 228 | ) 229 | .await; 230 | return; 231 | } 232 | 233 | drop(ins_read); 234 | 235 | let mut ins_write = ins.get_instance_write().await; 236 | let ins_write_inner = ins_write.as_mut().unwrap(); 237 | ins_write_inner.status = InstanceStatus::Accepted; 238 | ins_write_inner.seq = a.seq; 239 | ins_write_inner.deps = a.deps; 240 | } else { 241 | // Message reordering ? 242 | let new_instance = SharedInstance::new( 243 | Some(Instance { 244 | id: a.instance_id, 245 | seq: a.seq, 246 | ballot: a.ballot, 247 | cmds: vec![], 248 | deps: a.deps, 249 | status: InstanceStatus::Accepted, 250 | lb: self.new_leaderbook(r.id), 251 | }), 252 | None, 253 | ); 254 | r.instance_space 255 | .insert_instance(&a.instance_id.replica, &a.instance_id.local, new_instance) 256 | .await; 257 | } 258 | 259 | // TODO: sync to disk 260 | 261 | self.reply( 262 | r.id, 263 | &a.leader_id, 264 | Message::::AcceptReply(AcceptReply { 265 | instance_id: a.instance_id, 266 | ok: true, 267 | ballot: a.ballot, 268 | }), 269 | ) 270 | .await; 271 | } 272 | 273 | async fn handle_commit(&self, cm: Commit) { 274 | trace!("handle commit"); 275 | let mut r = self.replica.lock().await; 276 | 277 | if cm.instance_id.local >= r.cur_instance(&cm.instance_id.replica) { 278 | r.set_cur_instance(&InstanceId { 279 | replica: cm.instance_id.replica, 280 | local: (*cm.instance_id.local + 1).into(), 281 | }); 282 | } 283 | 284 | let ins = r 285 | .instance_space 286 | .get_instance(&cm.instance_id.replica, &cm.instance_id.local) 287 | .await; 288 | 289 | let exist = instance_exist(&ins).await; 290 | 291 | let ins = if exist { 292 | let orig = ins.unwrap(); 293 | let mut ins = orig.get_instance_write().await; 294 | let mut ins_inner = ins.as_mut().unwrap(); 295 | ins_inner.seq = cm.seq; 296 | ins_inner.deps = cm.deps; 297 | ins_inner.status = InstanceStatus::Committed; 298 | drop(ins); 299 | orig 300 | } else { 301 | let new_instance = SharedInstance::new( 302 | Some(Instance { 303 | id: cm.instance_id, 304 | seq: cm.seq, 305 | ballot: self.new_ballot(r.id), 306 | cmds: cm.cmds.to_vec(), 307 | deps: cm.deps, 308 | status: InstanceStatus::Committed, 309 | lb: self.new_leaderbook(r.id), 310 | }), 311 | None, 312 | ); 313 | r.instance_space 314 | .insert_instance( 315 | &cm.instance_id.replica, 316 | &cm.instance_id.local, 317 | new_instance.clone(), 318 | ) 319 | .await; 320 | r.update_conflict(&cm.instance_id.replica, &cm.cmds, new_instance.clone()) 321 | .await; 322 | new_instance 323 | }; 324 | 325 | // TODO: sync to disk 326 | 327 | // TODO: Handle Error 328 | let _ = r.exec_send.send(ins.clone()).await; 329 | ins.notify_commit().await; 330 | } 331 | 332 | async fn handle_preaccept_reply(&self, par: PreAcceptReply) { 333 | trace!("handle preaccept reply"); 334 | let r = self.replica.lock().await; 335 | 336 | let ins = r 337 | .instance_space 338 | .get_instance(&par.instance_id.replica, &par.instance_id.local) 339 | .await; 340 | 341 | if !instance_exist(&ins).await { 342 | panic!("This instance should already in the space"); 343 | } 344 | 345 | // We've checked the existence 346 | let orig = ins.unwrap(); 347 | let mut ins_write = orig.get_instance_write().await; 348 | let mut ins = ins_write.as_mut().unwrap(); 349 | 350 | if !matches!(ins.status, InstanceStatus::PreAccepted) { 351 | // We've translated to the later states 352 | return; 353 | } 354 | 355 | if ins.ballot != par.ballot { 356 | // Other advanced (larger ballot) leader is handling 357 | return; 358 | } 359 | 360 | if !par.ok { 361 | ins.lb.nack += 1; 362 | if par.ballot > ins.lb.max_ballot { 363 | ins.lb.max_ballot = par.ballot; 364 | } 365 | return; 366 | } 367 | 368 | ins.lb.preaccept_ok += 1; 369 | 370 | let equal = r.merge_seq_deps(ins, &par.seq, &par.deps); 371 | if ins.lb.preaccept_ok > 1 { 372 | ins.lb.all_equal = ins.lb.all_equal && equal; 373 | } 374 | 375 | if ins.lb.preaccept_ok >= r.peer_cnt / 2 && ins.lb.all_equal && ins.ballot.is_init() { 376 | ins.status = InstanceStatus::Committed; 377 | // TODO: sync to disk 378 | self.bdcst_message( 379 | r.id, 380 | Message::::Commit(Commit { 381 | leader_id: r.id.into(), 382 | instance_id: par.instance_id, 383 | seq: ins.seq, 384 | cmds: ins.cmds.to_vec(), 385 | deps: ins.deps.to_vec(), 386 | }), 387 | ) 388 | .await; 389 | 390 | drop(ins_write); 391 | let _ = r.exec_send.send(orig.clone()).await; 392 | orig.notify_commit().await; 393 | } else if ins.lb.preaccept_ok >= r.peer_cnt / 2 { 394 | ins.status = InstanceStatus::Accepted; 395 | self.bdcst_message( 396 | r.id, 397 | Message::::Accept(Accept { 398 | leader_id: r.id.into(), 399 | instance_id: par.instance_id, 400 | ballot: ins.ballot, 401 | seq: ins.seq, 402 | cmd_cnt: ins.cmds.len(), 403 | deps: ins.deps.to_vec(), 404 | }), 405 | ) 406 | .await; 407 | } 408 | } 409 | 410 | async fn handle_preaccept_ok(&self, pao: PreAcceptOk) { 411 | trace!("handle preaccept ok"); 412 | let r = self.replica.lock().await; 413 | 414 | let ins = r 415 | .instance_space 416 | .get_instance(&pao.instance_id.replica, &pao.instance_id.local) 417 | .await; 418 | 419 | if !instance_exist(&ins).await { 420 | panic!("This instance should already in the space"); 421 | } 422 | 423 | let ins = ins.unwrap(); 424 | let mut ins_write = ins.get_instance_write().await; 425 | let mut ins_write_inner = ins_write.as_mut().unwrap(); 426 | 427 | if !matches!(ins_write_inner.status, InstanceStatus::PreAccepted) { 428 | // We've translated to the later states 429 | return; 430 | } 431 | 432 | if !ins_write_inner.ballot.is_init() { 433 | // only the first leader can send ok 434 | return; 435 | } 436 | 437 | ins_write_inner.lb.preaccept_ok += 1; 438 | 439 | // TODO: remove duplicate code 440 | if ins_write_inner.lb.preaccept_ok >= r.peer_cnt / 2 441 | && ins_write_inner.lb.all_equal 442 | && ins_write_inner.ballot.is_init() 443 | { 444 | ins_write_inner.status = InstanceStatus::Committed; 445 | // TODO: sync to disk 446 | self.bdcst_message( 447 | r.id, 448 | Message::::Commit(Commit { 449 | leader_id: r.id.into(), 450 | instance_id: pao.instance_id, 451 | seq: ins_write_inner.seq, 452 | cmds: ins_write_inner.cmds.to_vec(), 453 | deps: ins_write_inner.deps.to_vec(), 454 | }), 455 | ) 456 | .await; 457 | drop(ins_write); 458 | let _ = r.exec_send.send(ins.clone()).await; 459 | ins.notify_commit().await; 460 | } else if ins_write_inner.lb.preaccept_ok >= r.peer_cnt / 2 { 461 | ins_write_inner.status = InstanceStatus::Accepted; 462 | self.bdcst_message( 463 | r.id, 464 | Message::::Accept(Accept { 465 | leader_id: r.id.into(), 466 | instance_id: pao.instance_id, 467 | ballot: ins_write_inner.ballot, 468 | seq: ins_write_inner.seq, 469 | cmd_cnt: ins_write_inner.cmds.len(), 470 | deps: ins_write_inner.deps.to_vec(), 471 | }), 472 | ) 473 | .await; 474 | } 475 | } 476 | 477 | async fn handle_preaccept(&self, pa: PreAccept) { 478 | trace!("handle preaccept {:?}", pa); 479 | let mut r = self.replica.lock().await; 480 | let ins = r 481 | .instance_space 482 | .get_instance(&pa.instance_id.replica, &pa.instance_id.local) 483 | .await; 484 | 485 | if instance_exist(&ins).await { 486 | // TODO: abstract to a macro 487 | let ins = ins.unwrap(); 488 | let ins_read = ins.get_instance_read().await; 489 | let ins_read_inner = ins_read.as_ref().unwrap(); 490 | 491 | // We've got accpet or commit before, don't reply 492 | if matches!( 493 | ins_read_inner.status, 494 | InstanceStatus::Committed | InstanceStatus::Accepted 495 | ) { 496 | // Later message may not contain commands, we should fill it here 497 | if ins_read_inner.cmds.is_empty() { 498 | drop(ins_read); 499 | // TODO: abstract to a macro 500 | let mut inst_write = ins.get_instance_write().await; 501 | let mut inst_write_inner = inst_write.as_mut().unwrap(); 502 | inst_write_inner.cmds = pa.cmds; 503 | } 504 | return; 505 | } 506 | 507 | // smaller Ballot number 508 | if pa.ballot < ins_read_inner.ballot { 509 | self.reply( 510 | r.id, 511 | &pa.leader_id, 512 | Message::::PreAcceptReply(PreAcceptReply { 513 | instance_id: pa.instance_id, 514 | seq: ins_read_inner.seq, 515 | ballot: ins_read_inner.ballot, 516 | ok: false, 517 | deps: ins_read_inner.deps.to_vec(), 518 | committed_deps: r.commited_upto.to_vec(), 519 | }), 520 | ) 521 | .await; 522 | return; 523 | } 524 | } 525 | 526 | if pa.instance_id.local > r.cur_instance(&pa.instance_id.replica) { 527 | r.set_cur_instance(&pa.instance_id); 528 | } 529 | 530 | // FIXME: We'd better not copy dep vec 531 | let (seq, deps, changed) = r.update_seq_deps(pa.seq, pa.deps.to_vec(), &pa.cmds).await; 532 | 533 | let status = if changed { 534 | InstanceStatus::PreAccepted 535 | } else { 536 | InstanceStatus::PreAcceptedEq 537 | }; 538 | 539 | let uncommited_deps = r 540 | .commited_upto 541 | .iter() 542 | .enumerate() 543 | .map(|cu| { 544 | if let Some(cu_id) = cu.1 { 545 | if let Some(d) = deps[cu.0] { 546 | if cu_id < &d { 547 | return true; 548 | } 549 | } 550 | } 551 | false 552 | }) 553 | .filter(|a| *a) 554 | .count() 555 | > 0; 556 | 557 | let new_instance = SharedInstance::new( 558 | Some(Instance { 559 | id: pa.instance_id, 560 | seq, 561 | ballot: pa.ballot, 562 | // FIXME: should not copy 563 | cmds: pa.cmds.to_vec(), 564 | // FIXME: Should not copy if we send reply ok later 565 | deps: deps.to_vec(), 566 | status, 567 | lb: self.new_leaderbook(r.id), 568 | }), 569 | None, 570 | ); 571 | 572 | r.instance_space 573 | .insert_instance( 574 | &pa.instance_id.replica, 575 | &pa.instance_id.local, 576 | new_instance.clone(), 577 | ) 578 | .await; 579 | 580 | r.update_conflict(&pa.instance_id.replica, &pa.cmds, new_instance) 581 | .await; 582 | 583 | // TODO: sync to disk 584 | 585 | // Send Reply 586 | if changed 587 | || uncommited_deps 588 | || *pa.instance_id.replica != *pa.leader_id 589 | || !pa.ballot.is_init() 590 | { 591 | self.reply( 592 | r.id, 593 | &pa.leader_id, 594 | Message::::PreAcceptReply(PreAcceptReply { 595 | instance_id: pa.instance_id, 596 | seq, 597 | ballot: pa.ballot, 598 | ok: true, 599 | deps, 600 | // Should not copy 601 | committed_deps: r.commited_upto.to_vec(), 602 | }), 603 | ) 604 | .await; 605 | } else { 606 | trace!("reply preaccept ok"); 607 | self.reply( 608 | r.id, 609 | &pa.leader_id, 610 | Message::::PreAcceptOk(PreAcceptOk { 611 | instance_id: pa.instance_id, 612 | }), 613 | ) 614 | .await; 615 | } 616 | } 617 | 618 | async fn handle_propose(&self, p: Propose, write_stream: &mut T) 619 | where 620 | T: AsyncWriteExt + Unpin, 621 | { 622 | trace!("handle propose"); 623 | let mut r = self.replica.lock().await; 624 | let inst_id = *r.local_cur_instance(); 625 | r.inc_local_cur_instance(); 626 | let (seq, deps) = r.get_seq_deps(&p.cmds).await; 627 | 628 | let (tx, rx) = oneshot::channel(); 629 | let new_ins = SharedInstance::new_execute_complete( 630 | Some(Instance { 631 | id: InstanceId { 632 | local: inst_id, 633 | replica: r.id, 634 | }, 635 | seq, 636 | ballot: self.new_ballot(r.id), 637 | cmds: p.cmds, 638 | deps, 639 | status: InstanceStatus::PreAccepted, 640 | lb: self.new_leaderbook(r.id), 641 | }), 642 | None, 643 | crate::types::ResultSender(tx), 644 | ); 645 | 646 | { 647 | let new_ins_read = new_ins.get_instance_read().await; 648 | let new_ins_read_inner = new_ins_read.as_ref().unwrap(); 649 | 650 | let r_id = r.id; 651 | r.update_conflict(&r_id, &new_ins_read_inner.cmds, new_ins.clone()) 652 | .await; 653 | 654 | let r_id = r.id; 655 | r.instance_space 656 | .insert_instance(&r_id, &inst_id, new_ins.clone()) 657 | .await; 658 | 659 | if seq > r.max_seq { 660 | r.max_seq = (*seq + 1).into(); 661 | } 662 | drop(r); 663 | 664 | // TODO: Flush the content to disk 665 | self.bdcst_message( 666 | r_id, 667 | Message::PreAccept(PreAccept { 668 | leader_id: r_id.into(), 669 | instance_id: InstanceId { 670 | replica: r_id, 671 | local: inst_id, 672 | }, 673 | seq, 674 | ballot: self.new_ballot(r_id), 675 | // FIXME: should not copy/clone vec 676 | cmds: new_ins_read_inner.cmds.to_vec(), 677 | deps: new_ins_read_inner.deps.to_vec(), 678 | }), 679 | ) 680 | .await; 681 | } 682 | 683 | trace!("waiting for execution"); 684 | 685 | let results = rx.await.unwrap(); 686 | let resp_vec = results 687 | .into_iter() 688 | .map(|result| match result { 689 | Ok(er) => SingleCmdResult::ExecuteResult(er), 690 | Err(err) => SingleCmdResult::Error(err.to_string()), 691 | }) 692 | .collect(); 693 | 694 | util::send_message( 695 | write_stream, 696 | &Message::::ProposeResponse(ProposeResponse { 697 | cmd_id: p.cmd_id, 698 | results: resp_vec, 699 | }), 700 | ) 701 | .await; 702 | } 703 | 704 | async fn reply(&self, replica: ReplicaId, leader: &LeaderId, message: Message) { 705 | let mut all_connect = self.conns.read().await; 706 | if all_connect.is_empty() { 707 | drop(all_connect); 708 | self.init_connections(&replica).await; 709 | all_connect = self.conns.read().await; 710 | } 711 | // Illeage leader id 712 | if **leader >= all_connect.len() { 713 | panic!( 714 | "all connection number is {}, but the leader id is {}", 715 | all_connect.len(), 716 | **leader 717 | ); 718 | } 719 | 720 | // Should not panic, leader is should be in the range 721 | let conn = all_connect.get(**leader).unwrap(); 722 | 723 | if conn.is_none() { 724 | // Should not reply to self 725 | panic!("should not reply to self"); 726 | } 727 | 728 | util::send_message_arc(conn.as_ref().unwrap(), &message).await; 729 | } 730 | 731 | async fn bdcst_message(&self, replica: ReplicaId, message: Message) { 732 | let mut conn = self.conns.read().await; 733 | if conn.is_empty() { 734 | drop(conn); 735 | self.init_connections(&replica).await; 736 | conn = self.conns.read().await; 737 | } 738 | let cnt = self.conf.peer.len(); 739 | 740 | let message = Arc::new(message); 741 | 742 | let tasks: Vec> = conn 743 | .iter() 744 | .filter(|c| c.is_some()) 745 | .map(|c| { 746 | let c = c.as_ref().unwrap().clone(); 747 | let message = message.clone(); 748 | tokio::spawn(async move { 749 | util::send_message_arc2(&c, &message).await; 750 | }) 751 | }) 752 | .collect(); 753 | 754 | let stream_of_futures = stream::iter(tasks); 755 | let mut buffered = stream_of_futures.buffer_unordered(self.conf.peer.len()); 756 | 757 | for _ in 0..cnt { 758 | buffered.next().await; 759 | } 760 | } 761 | 762 | async fn init_connections(&self, cur_replica: &ReplicaId) { 763 | let mut conn_write = self.conns.write().await; 764 | if conn_write.is_empty() { 765 | for (id, p) in self.conf.peer.iter().enumerate() { 766 | let stream = TcpStream::connect(p) 767 | .await 768 | .map_err(|e| panic!("connect to {} failed, {}", p, e)) 769 | .unwrap(); 770 | conn_write.push(if id == **cur_replica { 771 | None 772 | } else { 773 | Some(Arc::new(Mutex::new(stream))) 774 | }); 775 | } 776 | } 777 | } 778 | } 779 | 780 | pub(crate) struct RpcServer 781 | where 782 | C: Command + Clone + Send + Sync + 'static, 783 | E: CommandExecutor + Clone, 784 | S: InstanceSpace, 785 | { 786 | server: Arc>, 787 | listener: TcpListener, 788 | } 789 | 790 | impl RpcServer 791 | where 792 | C: Command + Debug + Clone + Serialize + DeserializeOwned + Send + Sync + 'static, 793 | E: CommandExecutor + Debug + Clone + Send + Sync + 'static, 794 | S: InstanceSpace + Send + Sync + 'static, 795 | { 796 | pub(crate) async fn new(config: &Configure, server: Arc>) -> Self { 797 | // Configure correctness has been checked 798 | let listener = TcpListener::bind(config.peer.get(config.index as usize).unwrap()) 799 | .await 800 | .map_err(|e| panic!("bind server address error, {}", e)) 801 | .unwrap(); 802 | 803 | Self { server, listener } 804 | } 805 | 806 | pub(crate) async fn serve(&self) -> Result<(), RpcError> { 807 | loop { 808 | let (stream, _) = self.listener.accept().await?; 809 | let server = self.server.clone(); 810 | tokio::spawn(async move { 811 | trace!("got a connection"); 812 | let (mut read_stream, mut write_stream) = tokio::io::split(stream); 813 | loop { 814 | let message_opt: Option> = 815 | util::recv_message(&mut read_stream).await; 816 | if let Some(message) = message_opt { 817 | server.handle_message(message, &mut write_stream).await; 818 | } else { 819 | debug!("Failed to receive message because peer has closed"); 820 | break; 821 | } 822 | } 823 | }); 824 | } 825 | } 826 | } 827 | --------------------------------------------------------------------------------