├── .github └── workflows │ ├── doc.yaml │ └── test.yaml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── README.md ├── labcodec ├── Cargo.toml ├── build.rs ├── demonstration │ ├── README.md │ └── expanded_fixture.rs ├── proto │ └── fixture.proto └── src │ └── lib.rs ├── labrpc ├── Cargo.toml ├── benches │ └── rpc.rs ├── examples │ └── echo.rs └── src │ ├── client.rs │ ├── error.rs │ ├── lib.rs │ ├── macros.rs │ ├── network.rs │ └── server.rs ├── linearizability ├── Cargo.toml ├── src │ ├── bitset.rs │ ├── lib.rs │ ├── model.rs │ └── models.rs └── test_data │ ├── c01-bad.txt │ ├── c01-ok.txt │ ├── c10-bad.txt │ ├── c10-ok.txt │ ├── c50-bad.txt │ └── c50-ok.txt ├── percolator ├── Cargo.toml ├── README.md ├── build.rs ├── proto │ └── msg.proto └── src │ ├── client.rs │ ├── lib.rs │ ├── server.rs │ ├── service.rs │ └── tests.rs ├── raft ├── Cargo.toml ├── README.md ├── build.rs └── src │ ├── kvraft │ ├── client.rs │ ├── config.rs │ ├── errors.rs │ ├── mod.rs │ ├── server.rs │ └── tests.rs │ ├── lib.rs │ ├── proto │ ├── kvraft.proto │ ├── mod.rs │ └── raft.proto │ └── raft │ ├── config.rs │ ├── errors.rs │ ├── mod.rs │ ├── persister.rs │ ├── states.rs │ └── tests.rs └── test-many.sh /.github/workflows/doc.yaml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Documentation 3 | 4 | jobs: 5 | docs: 6 | name: Doc 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | toolchain: nightly 13 | override: true 14 | - name: Run cargo doc 15 | uses: actions-rs/cargo@v1 16 | env: 17 | RUSTDOCFLAGS: --enable-index-page -Zunstable-options 18 | with: 19 | command: doc 20 | args: --no-deps --document-private-items 21 | - name: Deploy 22 | uses: peaceiris/actions-gh-pages@v3 23 | with: 24 | github_token: ${{ secrets.GITHUB_TOKEN }} 25 | publish_dir: target/doc/ 26 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Test 3 | 4 | jobs: 5 | test-2: 6 | name: Test 2 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | toolchain: stable 13 | - name: Check and Test 14 | env: 15 | RUST_TEST_THREADS: 8 16 | LOG_LEVEL: raft=error 17 | uses: nick-invision/retry@v2 18 | with: 19 | timeout_minutes: 10 20 | max_attempts: 3 21 | retry_on: timeout 22 | command: make test_2 23 | test-3: 24 | name: Test 3 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v2 28 | - uses: actions-rs/toolchain@v1 29 | with: 30 | toolchain: stable 31 | - name: Check and Test 32 | env: 33 | RUST_TEST_THREADS: 8 34 | LOG_LEVEL: raft=error 35 | uses: nick-invision/retry@v2 36 | with: 37 | timeout_minutes: 10 38 | max_attempts: 3 39 | retry_on: timeout 40 | command: make test_3 41 | test-percolator: 42 | name: Test Percolator 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v2 46 | - uses: actions-rs/toolchain@v1 47 | with: 48 | toolchain: stable 49 | - name: Check and Test 50 | env: 51 | RUST_TEST_THREADS: 8 52 | uses: nick-invision/retry@v2 53 | with: 54 | timeout_minutes: 10 55 | max_attempts: 3 56 | retry_on: timeout 57 | command: make test_percolator 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/target 2 | **/*.rs.bk 3 | 6.824-golabs-2018 4 | test-*.err 5 | test-*.log 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "labcodec", 4 | "labrpc", 5 | "linearizability", 6 | "raft", 7 | "percolator" 8 | ] 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export RUST_BACKTRACE=1 2 | 3 | RUST_TEST_THREADS ?= 1 4 | LOG_LEVEL ?= raft::kvraft=info,raft::raft=info,raft::kvraft::tests=debug,percolator=info 5 | 6 | check: 7 | cargo fmt --all -- --check 8 | cargo clippy --all --tests -- -D clippy::all 9 | 10 | test: test_others test_2 test_3 11 | 12 | test_23: test_2 test_3 13 | 14 | test_2: test_2a test_2b test_2c test_2d 15 | 16 | test_2a: cargo_test_2a 17 | 18 | test_2b: cargo_test_2b 19 | 20 | test_2c: cargo_test_2c 21 | 22 | test_2d: cargo_test_2d 23 | 24 | test_3: test_3a test_3b 25 | 26 | test_3a: cargo_test_3a 27 | 28 | test_3b: cargo_test_3b 29 | 30 | cargo_test_%: check 31 | RUST_LOG=${LOG_LEVEL} cargo test -p raft -- --nocapture --test $* 32 | 33 | test_others: check 34 | RUST_LOG=${LOG_LEVEL} cargo test -p labrpc -p labcodec -- --nocapture 35 | 36 | test_percolator: check 37 | RUST_LOG=${LOG_LEVEL} cargo test -p percolator -- --nocapture 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Raft 2 | 3 | [![Test](https://github.com/BugenZhao/Raft/actions/workflows/test.yaml/badge.svg)](https://github.com/BugenZhao/Raft/actions/workflows/test.yaml) 4 | [![Docs](https://img.shields.io/badge/Docs-detailed-orange)](https://bugenzhao.com/Raft/) 5 | 6 | A solution to _"6.824 Lab 2: Raft"_ and _"6.824 Lab 3: Fault-tolerant Key/Value Service"_ from MIT, with detailed documentations. Lab of Percolator transaction model is also included. 7 | 8 | Base code and lab parts marked with `*` are migrated from [pingcap/talent-plan](https://github.com/pingcap/talent-plan). 9 | 10 | ## Roadmap 11 | 12 | - [x] Lab 2 13 | - [x] Part 2A: leader election \* 14 | - [x] Part 2B: log \* 15 | - [x] Part 2C: persistence \* 16 | - [x] Part 2D: log compaction 17 | - [x] Lab 3 18 | - [x] Part 3A: Key/value service without snapshots \* 19 | - [x] Part 3B: Key/value service with snapshots (\*) 20 | - [x] Percolator \* 21 | 22 | # Distributed Systems in Rust 23 | 24 | A training course about the distributed systems in [Rust]. 25 | 26 | Subjects covered include: 27 | 28 | - [Raft consensus algorithm] (including a fault-tolerant key-value storage service 29 | using Raft) 30 | - [Percolator transaction model] 31 | 32 | After completing this course you will have the knowledge to implement a basic 33 | key-value storage service with transaction and fault-tolerant in Rust. 34 | 35 | **Important note: Distributed Systems in Rust is in an alpha state** 36 | It might contain bugs. Any feedback is greatly appreciated. Please [file issues] 37 | if you have any problem. And also You are encouraged to fix problems on your own 38 | and submit pull requests. 39 | 40 | ## The goal of this course 41 | 42 | The goal of this course is to teach the Rust programmers who are interested in 43 | distributed systems to know about how to make the distributed systems reliable 44 | and how to implement the distributed transaction. 45 | 46 | ## Who is this for? 47 | 48 | Distributed Systems in Rust is for experienced _Rust_ programmers, who are 49 | familiar with the Rust language. If you are not, you can first learn our [rust] 50 | lessons. 51 | 52 | ## A PingCAP-specific note 53 | 54 | This course, combined with [Deep Dive TiKV], is intended to be enough to enable 55 | programmers to meaningfully contribute to [TiKV]. It is most specifically 56 | designed to teach those in the Chinese Rust community enough Rust to work on 57 | TiKV. The language used is intended to be simple so that those who read only a 58 | little English can follow. If you find any of the language difficult to 59 | understand please [file issues]. 60 | 61 | ## License 62 | 63 | [CC-BY 4.0](https://opendefinition.org/licenses/cc-by/) 64 | 65 | 66 | 67 | [rust]: ../rust/README.md 68 | [file issues]: https://github.com/pingcap/talent-plan/issues/ 69 | [deep dive tikv]: https://tikv.github.io/deep-dive-tikv/overview/introduction.html 70 | [tikv]: https://github.com/tikv/tikv/ 71 | [rust]: https://www.rust-lang.org/ 72 | [raft consensus algorithm]: raft/README.md 73 | [percolator transaction model]: percolator/README.md 74 | -------------------------------------------------------------------------------- /labcodec/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "labcodec" 3 | version = "0.1.0" 4 | build = "build.rs" 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | prost = "0.8" 10 | 11 | [build-dependencies] 12 | prost-build = "0.8" 13 | -------------------------------------------------------------------------------- /labcodec/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | prost_build::compile_protos(&["proto/fixture.proto"], &["proto"]).unwrap(); 3 | println!("cargo:rerun-if-changed=proto"); 4 | } 5 | -------------------------------------------------------------------------------- /labcodec/demonstration/README.md: -------------------------------------------------------------------------------- 1 | # Demonstration 2 | 3 | Expanded version of generated rust files. Files in the folder are for the sake 4 | of understanding how procedural macro works, and they are not compilable. 5 | -------------------------------------------------------------------------------- /labcodec/demonstration/expanded_fixture.rs: -------------------------------------------------------------------------------- 1 | /// A simple protobuf message. 2 | pub struct Msg { 3 | #[prost(enumeration = "msg::Type", tag = "1")] 4 | pub r#type: i32, 5 | #[prost(uint64, tag = "2")] 6 | pub id: u64, 7 | #[prost(string, tag = "3")] 8 | pub name: std::string::String, 9 | #[prost(bytes, repeated, tag = "4")] 10 | pub paylad: ::std::vec::Vec>, 11 | } 12 | #[automatically_derived] 13 | #[allow(unused_qualifications)] 14 | impl ::core::clone::Clone for Msg { 15 | #[inline] 16 | fn clone(&self) -> Msg { 17 | match *self { 18 | Msg { 19 | r#type: ref __self_0_0, 20 | id: ref __self_0_1, 21 | name: ref __self_0_2, 22 | paylad: ref __self_0_3, 23 | } => Msg { 24 | r#type: ::core::clone::Clone::clone(&(*__self_0_0)), 25 | id: ::core::clone::Clone::clone(&(*__self_0_1)), 26 | name: ::core::clone::Clone::clone(&(*__self_0_2)), 27 | paylad: ::core::clone::Clone::clone(&(*__self_0_3)), 28 | }, 29 | } 30 | } 31 | } 32 | impl ::core::marker::StructuralPartialEq for Msg {} 33 | #[automatically_derived] 34 | #[allow(unused_qualifications)] 35 | impl ::core::cmp::PartialEq for Msg { 36 | #[inline] 37 | fn eq(&self, other: &Msg) -> bool { 38 | match *other { 39 | Msg { 40 | r#type: ref __self_1_0, 41 | id: ref __self_1_1, 42 | name: ref __self_1_2, 43 | paylad: ref __self_1_3, 44 | } => match *self { 45 | Msg { 46 | r#type: ref __self_0_0, 47 | id: ref __self_0_1, 48 | name: ref __self_0_2, 49 | paylad: ref __self_0_3, 50 | } => { 51 | (*__self_0_0) == (*__self_1_0) 52 | && (*__self_0_1) == (*__self_1_1) 53 | && (*__self_0_2) == (*__self_1_2) 54 | && (*__self_0_3) == (*__self_1_3) 55 | } 56 | }, 57 | } 58 | } 59 | #[inline] 60 | fn ne(&self, other: &Msg) -> bool { 61 | match *other { 62 | Msg { 63 | r#type: ref __self_1_0, 64 | id: ref __self_1_1, 65 | name: ref __self_1_2, 66 | paylad: ref __self_1_3, 67 | } => match *self { 68 | Msg { 69 | r#type: ref __self_0_0, 70 | id: ref __self_0_1, 71 | name: ref __self_0_2, 72 | paylad: ref __self_0_3, 73 | } => { 74 | (*__self_0_0) != (*__self_1_0) 75 | || (*__self_0_1) != (*__self_1_1) 76 | || (*__self_0_2) != (*__self_1_2) 77 | || (*__self_0_3) != (*__self_1_3) 78 | } 79 | }, 80 | } 81 | } 82 | } 83 | impl ::prost::Message for Msg { 84 | #[allow(unused_variables)] 85 | fn encode_raw(&self, buf: &mut B) 86 | where 87 | B: ::prost::bytes::BufMut, 88 | { 89 | if self.r#type != msg::Type::default() as i32 { 90 | ::prost::encoding::int32::encode(1u32, &self.r#type, buf); 91 | } 92 | if self.id != 0u64 { 93 | ::prost::encoding::uint64::encode(2u32, &self.id, buf); 94 | } 95 | if self.name != "" { 96 | ::prost::encoding::string::encode(3u32, &self.name, buf); 97 | } 98 | ::prost::encoding::bytes::encode_repeated(4u32, &self.paylad, buf); 99 | } 100 | #[allow(unused_variables)] 101 | fn merge_field( 102 | &mut self, 103 | tag: u32, 104 | wire_type: ::prost::encoding::WireType, 105 | buf: &mut B, 106 | ctx: ::prost::encoding::DecodeContext, 107 | ) -> ::std::result::Result<(), ::prost::DecodeError> 108 | where 109 | B: ::prost::bytes::Buf, 110 | { 111 | const STRUCT_NAME: &'static str = "Msg"; 112 | match tag { 113 | 1u32 => { 114 | let mut value = &mut self.r#type; 115 | ::prost::encoding::int32::merge(wire_type, value, buf, ctx).map_err(|mut error| { 116 | error.push(STRUCT_NAME, "r#type"); 117 | error 118 | }) 119 | } 120 | 2u32 => { 121 | let mut value = &mut self.id; 122 | ::prost::encoding::uint64::merge(wire_type, value, buf, ctx).map_err(|mut error| { 123 | error.push(STRUCT_NAME, "id"); 124 | error 125 | }) 126 | } 127 | 3u32 => { 128 | let mut value = &mut self.name; 129 | ::prost::encoding::string::merge(wire_type, value, buf, ctx).map_err(|mut error| { 130 | error.push(STRUCT_NAME, "name"); 131 | error 132 | }) 133 | } 134 | 4u32 => { 135 | let mut value = &mut self.paylad; 136 | ::prost::encoding::bytes::merge_repeated(wire_type, value, buf, ctx).map_err( 137 | |mut error| { 138 | error.push(STRUCT_NAME, "paylad"); 139 | error 140 | }, 141 | ) 142 | } 143 | _ => ::prost::encoding::skip_field(wire_type, tag, buf, ctx), 144 | } 145 | } 146 | #[inline] 147 | fn encoded_len(&self) -> usize { 148 | 0 + if self.r#type != msg::Type::default() as i32 { 149 | ::prost::encoding::int32::encoded_len(1u32, &self.r#type) 150 | } else { 151 | 0 152 | } + if self.id != 0u64 { 153 | ::prost::encoding::uint64::encoded_len(2u32, &self.id) 154 | } else { 155 | 0 156 | } + if self.name != "" { 157 | ::prost::encoding::string::encoded_len(3u32, &self.name) 158 | } else { 159 | 0 160 | } + ::prost::encoding::bytes::encoded_len_repeated(4u32, &self.paylad) 161 | } 162 | fn clear(&mut self) { 163 | self.r#type = msg::Type::default() as i32; 164 | self.id = 0u64; 165 | self.name.clear(); 166 | self.paylad.clear(); 167 | } 168 | } 169 | impl Default for Msg { 170 | fn default() -> Msg { 171 | Msg { 172 | r#type: msg::Type::default() as i32, 173 | id: 0u64, 174 | name: ::std::string::String::new(), 175 | paylad: ::std::vec::Vec::new(), 176 | } 177 | } 178 | } 179 | impl ::std::fmt::Debug for Msg { 180 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { 181 | let mut builder = f.debug_struct("Msg"); 182 | let builder = { 183 | let wrapper = { 184 | struct ScalarWrapper<'a>(&'a i32); 185 | impl<'a> ::std::fmt::Debug for ScalarWrapper<'a> { 186 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { 187 | match msg::Type::from_i32(*self.0) { 188 | None => ::std::fmt::Debug::fmt(&self.0, f), 189 | Some(en) => ::std::fmt::Debug::fmt(&en, f), 190 | } 191 | } 192 | } 193 | ScalarWrapper(&self.r#type) 194 | }; 195 | builder.field("r#type", &wrapper) 196 | }; 197 | let builder = { 198 | let wrapper = { 199 | fn ScalarWrapper(v: T) -> T { 200 | v 201 | } 202 | ScalarWrapper(&self.id) 203 | }; 204 | builder.field("id", &wrapper) 205 | }; 206 | let builder = { 207 | let wrapper = { 208 | fn ScalarWrapper(v: T) -> T { 209 | v 210 | } 211 | ScalarWrapper(&self.name) 212 | }; 213 | builder.field("name", &wrapper) 214 | }; 215 | let builder = { 216 | let wrapper = { 217 | struct ScalarWrapper<'a>(&'a ::std::vec::Vec<::std::vec::Vec>); 218 | impl<'a> ::std::fmt::Debug for ScalarWrapper<'a> { 219 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { 220 | let mut vec_builder = f.debug_list(); 221 | for v in self.0 { 222 | fn Inner(v: T) -> T { 223 | v 224 | } 225 | vec_builder.entry(&Inner(v)); 226 | } 227 | vec_builder.finish() 228 | } 229 | } 230 | ScalarWrapper(&self.paylad) 231 | }; 232 | builder.field("paylad", &wrapper) 233 | }; 234 | builder.finish() 235 | } 236 | } 237 | #[allow(dead_code)] 238 | impl Msg { 239 | ///Returns the enum value of `type`, or the default if the field is set to an invalid enum value. 240 | pub fn r#type(&self) -> msg::Type { 241 | msg::Type::from_i32(self.r#type).unwrap_or(msg::Type::default()) 242 | } 243 | ///Sets `type` to the provided enum value. 244 | pub fn set_type(&mut self, value: msg::Type) { 245 | self.r#type = value as i32; 246 | } 247 | } 248 | pub mod msg { 249 | #[repr(i32)] 250 | pub enum Type { 251 | Unknown = 0, 252 | Put = 1, 253 | Get = 2, 254 | Del = 3, 255 | } 256 | #[automatically_derived] 257 | #[allow(unused_qualifications)] 258 | impl ::core::clone::Clone for Type { 259 | #[inline] 260 | fn clone(&self) -> Type { 261 | { 262 | *self 263 | } 264 | } 265 | } 266 | #[automatically_derived] 267 | #[allow(unused_qualifications)] 268 | impl ::core::marker::Copy for Type {} 269 | #[automatically_derived] 270 | #[allow(unused_qualifications)] 271 | impl ::core::fmt::Debug for Type { 272 | fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { 273 | match (&*self,) { 274 | (&Type::Unknown,) => { 275 | let mut debug_trait_builder = f.debug_tuple("Unknown"); 276 | debug_trait_builder.finish() 277 | } 278 | (&Type::Put,) => { 279 | let mut debug_trait_builder = f.debug_tuple("Put"); 280 | debug_trait_builder.finish() 281 | } 282 | (&Type::Get,) => { 283 | let mut debug_trait_builder = f.debug_tuple("Get"); 284 | debug_trait_builder.finish() 285 | } 286 | (&Type::Del,) => { 287 | let mut debug_trait_builder = f.debug_tuple("Del"); 288 | debug_trait_builder.finish() 289 | } 290 | } 291 | } 292 | } 293 | impl ::core::marker::StructuralPartialEq for Type {} 294 | #[automatically_derived] 295 | #[allow(unused_qualifications)] 296 | impl ::core::cmp::PartialEq for Type { 297 | #[inline] 298 | fn eq(&self, other: &Type) -> bool { 299 | { 300 | let __self_vi = unsafe { ::core::intrinsics::discriminant_value(&*self) } as i32; 301 | let __arg_1_vi = unsafe { ::core::intrinsics::discriminant_value(&*other) } as i32; 302 | if true && __self_vi == __arg_1_vi { 303 | match (&*self, &*other) { 304 | _ => true, 305 | } 306 | } else { 307 | false 308 | } 309 | } 310 | } 311 | } 312 | impl ::core::marker::StructuralEq for Type {} 313 | #[automatically_derived] 314 | #[allow(unused_qualifications)] 315 | impl ::core::cmp::Eq for Type { 316 | #[inline] 317 | #[doc(hidden)] 318 | fn assert_receiver_is_total_eq(&self) -> () { 319 | {} 320 | } 321 | } 322 | #[automatically_derived] 323 | #[allow(unused_qualifications)] 324 | impl ::core::hash::Hash for Type { 325 | fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) -> () { 326 | match (&*self,) { 327 | _ => ::core::hash::Hash::hash( 328 | &unsafe { ::core::intrinsics::discriminant_value(self) }, 329 | state, 330 | ), 331 | } 332 | } 333 | } 334 | #[automatically_derived] 335 | #[allow(unused_qualifications)] 336 | impl ::core::cmp::PartialOrd for Type { 337 | #[inline] 338 | fn partial_cmp(&self, other: &Type) -> ::core::option::Option<::core::cmp::Ordering> { 339 | { 340 | let __self_vi = unsafe { ::core::intrinsics::discriminant_value(&*self) } as i32; 341 | let __arg_1_vi = unsafe { ::core::intrinsics::discriminant_value(&*other) } as i32; 342 | if true && __self_vi == __arg_1_vi { 343 | match (&*self, &*other) { 344 | _ => ::core::option::Option::Some(::core::cmp::Ordering::Equal), 345 | } 346 | } else { 347 | __self_vi.partial_cmp(&__arg_1_vi) 348 | } 349 | } 350 | } 351 | } 352 | #[automatically_derived] 353 | #[allow(unused_qualifications)] 354 | impl ::core::cmp::Ord for Type { 355 | #[inline] 356 | fn cmp(&self, other: &Type) -> ::core::cmp::Ordering { 357 | { 358 | let __self_vi = unsafe { ::core::intrinsics::discriminant_value(&*self) } as i32; 359 | let __arg_1_vi = unsafe { ::core::intrinsics::discriminant_value(&*other) } as i32; 360 | if true && __self_vi == __arg_1_vi { 361 | match (&*self, &*other) { 362 | _ => ::core::cmp::Ordering::Equal, 363 | } 364 | } else { 365 | __self_vi.cmp(&__arg_1_vi) 366 | } 367 | } 368 | } 369 | } 370 | impl Type { 371 | ///Returns `true` if `value` is a variant of `Type`. 372 | pub fn is_valid(value: i32) -> bool { 373 | match value { 374 | 0 => true, 375 | 1 => true, 376 | 2 => true, 377 | 3 => true, 378 | _ => false, 379 | } 380 | } 381 | ///Converts an `i32` to a `Type`, or `None` if `value` is not a valid variant. 382 | pub fn from_i32(value: i32) -> ::std::option::Option { 383 | match value { 384 | 0 => ::std::option::Option::Some(Type::Unknown), 385 | 1 => ::std::option::Option::Some(Type::Put), 386 | 2 => ::std::option::Option::Some(Type::Get), 387 | 3 => ::std::option::Option::Some(Type::Del), 388 | _ => ::std::option::Option::None, 389 | } 390 | } 391 | } 392 | impl ::std::default::Default for Type { 393 | fn default() -> Type { 394 | Type::Unknown 395 | } 396 | } 397 | impl ::std::convert::From for i32 { 398 | fn from(value: Type) -> i32 { 399 | value as i32 400 | } 401 | } 402 | } 403 | -------------------------------------------------------------------------------- /labcodec/proto/fixture.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fixture; 4 | 5 | // A simple protobuf message. 6 | message Msg { 7 | enum Type { 8 | UNKNOWN = 0; 9 | PUT = 1; 10 | GET = 2; 11 | DEL = 3; 12 | } 13 | 14 | Type type = 1; 15 | uint64 id = 2; 16 | string name = 3; 17 | repeated bytes paylad = 4; 18 | } 19 | -------------------------------------------------------------------------------- /labcodec/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A thin wrapper of [prost](https://docs.rs/prost/0.6.1/prost/) 2 | 3 | /// A labcodec message. 4 | pub trait Message: prost::Message + Default {} 5 | impl Message for T {} 6 | 7 | /// A message encoding error. 8 | pub type EncodeError = prost::EncodeError; 9 | /// A message decoding error. 10 | pub type DecodeError = prost::DecodeError; 11 | 12 | /// Encodes the message to a `Vec`. 13 | pub fn encode(message: &M, buf: &mut Vec) -> Result<(), EncodeError> { 14 | buf.reserve(message.encoded_len()); 15 | message.encode(buf)?; 16 | Ok(()) 17 | } 18 | 19 | /// Decodes an message from the buffer. 20 | pub fn decode(buf: &[u8]) -> Result { 21 | M::decode(buf) 22 | } 23 | 24 | #[cfg(test)] 25 | mod tests { 26 | mod fixture { 27 | // The generated rust file: 28 | // labs6824/target/debug/build/labcodec-hashhashhashhash/out/fixture.rs 29 | // 30 | // It looks like: 31 | // 32 | // ```no_run 33 | // /// A simple protobuf message. 34 | // #[derive(Clone, PartialEq, Message)] 35 | // pub struct Msg { 36 | // #[prost(enumeration="msg::Type", tag="1")] 37 | // pub type_: i32, 38 | // #[prost(uint64, tag="2")] 39 | // pub id: u64, 40 | // #[prost(string, tag="3")] 41 | // pub name: String, 42 | // #[prost(bytes, repeated, tag="4")] 43 | // pub paylad: ::std::vec::Vec>, 44 | // } 45 | // pub mod msg { 46 | // #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Enumeration)] 47 | // pub enum Type { 48 | // Unknown = 0, 49 | // Put = 1, 50 | // Get = 2, 51 | // Del = 3, 52 | // } 53 | // } 54 | // ``` 55 | include!(concat!(env!("OUT_DIR"), "/fixture.rs")); 56 | } 57 | 58 | use super::{decode, encode}; 59 | 60 | #[test] 61 | fn test_basic_encode_decode() { 62 | let msg = fixture::Msg { 63 | r#type: fixture::msg::Type::Put as _, 64 | id: 42, 65 | name: "the answer".to_owned(), 66 | paylad: vec![vec![7; 3]; 2], 67 | }; 68 | let mut buf = vec![]; 69 | encode(&msg, &mut buf).unwrap(); 70 | let msg1 = decode(&buf).unwrap(); 71 | assert_eq!(msg, msg1); 72 | } 73 | 74 | #[test] 75 | fn test_default() { 76 | let msg = fixture::Msg::default(); 77 | let msg1 = decode(&[]).unwrap(); 78 | assert_eq!(msg, msg1); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /labrpc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "labrpc" 3 | version = "0.1.0" 4 | edition = "2018" 5 | publish = false 6 | 7 | [dependencies] 8 | async-trait = "0.1" 9 | futures = { version = "0.3.16", features = ["thread-pool"] } 10 | futures-timer = "3.0" 11 | log = "0.4" 12 | prost = "0.8" 13 | rand = "0.7" 14 | 15 | labcodec = { path = "../labcodec" } 16 | 17 | [dev-dependencies] 18 | criterion = "0.3" 19 | env_logger = "0.7" 20 | prost-derive = "0.6" 21 | 22 | [[bench]] 23 | name = "rpc" 24 | path = "benches/rpc.rs" 25 | harness = false 26 | -------------------------------------------------------------------------------- /labrpc/benches/rpc.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 4 | use futures::executor::block_on; 5 | use prost_derive::Message; 6 | 7 | use labrpc::{service, Network, Result, Server, ServerBuilder}; 8 | 9 | service! { 10 | /// A simple bench-purpose service. 11 | service bench { 12 | /// Doc comments. 13 | rpc handler(BenchArgs) returns (BenchReply); 14 | } 15 | } 16 | use bench::{add_service, Client as BenchClient, Service}; 17 | 18 | // Hand-written protobuf messages. 19 | #[derive(Clone, PartialEq, Message)] 20 | pub struct BenchArgs { 21 | #[prost(int64, tag = "1")] 22 | pub x: i64, 23 | } 24 | 25 | #[derive(Clone, PartialEq, Message)] 26 | pub struct BenchReply { 27 | #[prost(string, tag = "1")] 28 | pub x: String, 29 | } 30 | 31 | #[derive(Default)] 32 | struct BenchInner { 33 | log2: Vec, 34 | } 35 | #[derive(Clone)] 36 | pub struct BenchService { 37 | inner: Arc>, 38 | } 39 | impl BenchService { 40 | fn new() -> BenchService { 41 | BenchService { 42 | inner: Arc::default(), 43 | } 44 | } 45 | } 46 | 47 | #[async_trait::async_trait] 48 | impl Service for BenchService { 49 | async fn handler(&self, args: BenchArgs) -> Result { 50 | self.inner.lock().unwrap().log2.push(args.x); 51 | Ok(BenchReply { 52 | x: format!("handler-{}", args.x), 53 | }) 54 | } 55 | } 56 | 57 | fn bench_suit() -> (Network, Server, BenchService) { 58 | let net = Network::new(); 59 | let server_name = "test_server".to_owned(); 60 | let mut builder = ServerBuilder::new(server_name); 61 | let bench_server = BenchService::new(); 62 | add_service(bench_server.clone(), &mut builder).unwrap(); 63 | let server = builder.build(); 64 | net.add_server(server.clone()); 65 | (net, server, bench_server) 66 | } 67 | 68 | fn bench_rpc(c: &mut Criterion) { 69 | let (net, server, _bench_server) = bench_suit(); 70 | let server_name = server.name(); 71 | let client_name = "client"; 72 | let client = BenchClient::new(net.create_client(client_name.to_owned())); 73 | net.connect(client_name, server_name); 74 | net.enable(client_name, true); 75 | 76 | c.bench_function("rpc", |b| { 77 | b.iter(|| { 78 | black_box(block_on(async { 79 | client.handler(&BenchArgs { x: 111 }).await.unwrap() 80 | })); 81 | }) 82 | // i7-8650U, 13 microseconds per RPC 83 | }); 84 | } 85 | 86 | criterion_group!(benches, bench_rpc); 87 | criterion_main!(benches); 88 | -------------------------------------------------------------------------------- /labrpc/examples/echo.rs: -------------------------------------------------------------------------------- 1 | use futures::executor::block_on; 2 | use prost_derive::Message; 3 | 4 | use labrpc::*; 5 | 6 | /// A Hand-written protobuf messages 7 | #[derive(Clone, PartialEq, Message)] 8 | pub struct Echo { 9 | #[prost(int64, tag = "1")] 10 | pub x: i64, 11 | } 12 | 13 | service! { 14 | service echo { 15 | rpc ping(Echo) returns (Echo); 16 | } 17 | } 18 | use echo::{add_service, Client, Service}; 19 | 20 | #[derive(Clone)] 21 | struct EchoService; 22 | 23 | #[async_trait::async_trait] 24 | impl Service for EchoService { 25 | async fn ping(&self, input: Echo) -> Result { 26 | Ok(input) 27 | } 28 | } 29 | 30 | fn main() { 31 | let rn = Network::new(); 32 | let server_name = "echo_server"; 33 | let mut builder = ServerBuilder::new(server_name.to_owned()); 34 | add_service(EchoService, &mut builder).unwrap(); 35 | let server = builder.build(); 36 | rn.add_server(server); 37 | 38 | let client_name = "client"; 39 | let client = Client::new(rn.create_client(client_name.to_owned())); 40 | rn.enable(client_name, true); 41 | rn.connect(client_name, server_name); 42 | 43 | let reply = block_on(async { client.ping(&Echo { x: 777 }).await.unwrap() }); 44 | assert_eq!(reply, Echo { x: 777 }); 45 | println!("{:?}", reply); 46 | } 47 | -------------------------------------------------------------------------------- /labrpc/src/client.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::sync::{Arc, Mutex}; 3 | 4 | use futures::channel::mpsc::UnboundedSender; 5 | use futures::channel::oneshot; 6 | use futures::executor::ThreadPool; 7 | use futures::future::{self, FutureExt}; 8 | 9 | use crate::error::{Error, Result}; 10 | use crate::server::RpcFuture; 11 | 12 | pub struct Rpc { 13 | pub(crate) client_name: String, 14 | pub(crate) fq_name: &'static str, 15 | pub(crate) req: Option>, 16 | pub(crate) resp: Option>>>, 17 | pub(crate) hooks: Arc>>>, 18 | } 19 | 20 | impl Rpc { 21 | pub(crate) fn take_resp_sender(&mut self) -> Option>>> { 22 | self.resp.take() 23 | } 24 | } 25 | 26 | impl fmt::Debug for Rpc { 27 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 28 | f.debug_struct("Rpc") 29 | .field("client_name", &self.client_name) 30 | .field("fq_name", &self.fq_name) 31 | .finish() 32 | } 33 | } 34 | 35 | pub trait RpcHooks: Sync + Send + 'static { 36 | fn before_dispatch(&self, fq_name: &str, req: &[u8]) -> Result<()>; 37 | fn after_dispatch(&self, fq_name: &str, resp: Result>) -> Result>; 38 | } 39 | 40 | #[derive(Clone)] 41 | pub struct Client { 42 | // this end-point's name 43 | pub(crate) name: String, 44 | // copy of Network.sender 45 | pub(crate) sender: UnboundedSender, 46 | pub(crate) hooks: Arc>>>, 47 | 48 | pub worker: ThreadPool, 49 | } 50 | 51 | impl Client { 52 | pub fn call(&self, fq_name: &'static str, req: &Req) -> RpcFuture> 53 | where 54 | Req: labcodec::Message, 55 | Rsp: labcodec::Message + 'static, 56 | { 57 | let mut buf = vec![]; 58 | if let Err(e) = labcodec::encode(req, &mut buf) { 59 | return Box::pin(future::err(Error::Encode(e))); 60 | } 61 | 62 | let (tx, rx) = oneshot::channel(); 63 | let rpc = Rpc { 64 | client_name: self.name.clone(), 65 | fq_name, 66 | req: Some(buf), 67 | resp: Some(tx), 68 | hooks: self.hooks.clone(), 69 | }; 70 | 71 | // Sends requests and waits responses. 72 | if self.sender.unbounded_send(rpc).is_err() { 73 | return Box::pin(future::err(Error::Stopped)); 74 | } 75 | 76 | Box::pin(rx.then(|res| async move { 77 | match res { 78 | Ok(Ok(resp)) => labcodec::decode(&resp).map_err(Error::Decode), 79 | Ok(Err(e)) => Err(e), 80 | Err(e) => Err(Error::Recv(e)), 81 | } 82 | })) 83 | } 84 | 85 | pub fn set_hooks(&self, hooks: Arc) { 86 | *self.hooks.lock().unwrap() = Some(hooks); 87 | } 88 | 89 | pub fn clear_hooks(&self) { 90 | *self.hooks.lock().unwrap() = None; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /labrpc/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | use futures::channel::oneshot::Canceled; 4 | 5 | use labcodec::{DecodeError, EncodeError}; 6 | 7 | #[derive(Clone, Debug, PartialEq, Eq)] 8 | pub enum Error { 9 | Unimplemented(String), 10 | Encode(EncodeError), 11 | Decode(DecodeError), 12 | Recv(Canceled), 13 | Timeout, 14 | Stopped, 15 | Other(String), 16 | } 17 | 18 | impl fmt::Display for Error { 19 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 20 | write!(f, "{:?}", self) 21 | } 22 | } 23 | 24 | impl error::Error for Error { 25 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 26 | match *self { 27 | Error::Encode(ref e) => Some(e), 28 | Error::Decode(ref e) => Some(e), 29 | Error::Recv(ref e) => Some(e), 30 | _ => None, 31 | } 32 | } 33 | } 34 | 35 | pub type Result = result::Result; 36 | -------------------------------------------------------------------------------- /labrpc/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::new_without_default)] 2 | 3 | mod client; 4 | mod error; 5 | #[macro_use] 6 | mod macros; 7 | mod network; 8 | mod server; 9 | 10 | pub use self::client::{Client, Rpc, RpcHooks}; 11 | pub use self::error::{Error, Result}; 12 | pub use self::network::Network; 13 | pub use self::server::{Handler, HandlerFactory, RpcFuture, Server, ServerBuilder}; 14 | 15 | #[cfg(test)] 16 | pub mod tests { 17 | use std::sync::atomic::{AtomicBool, Ordering}; 18 | use std::sync::{mpsc, Arc, Mutex, Once}; 19 | use std::thread; 20 | use std::time::{Duration, Instant}; 21 | 22 | use futures::channel::oneshot::Canceled; 23 | use futures::executor::{block_on, ThreadPool}; 24 | use futures::stream::StreamExt; 25 | use futures_timer::Delay; 26 | use prost_derive::Message; 27 | 28 | use super::*; 29 | 30 | service! { 31 | /// A simple test-purpose service. 32 | service junk { 33 | /// Doc comments. 34 | rpc handler2(JunkArgs) returns (JunkReply); 35 | rpc handler3(JunkArgs) returns (JunkReply); 36 | rpc handler4(JunkArgs) returns (JunkReply); 37 | } 38 | } 39 | use junk::{add_service, Client as JunkClient, Service as Junk}; 40 | 41 | // Hand-written protobuf messages. 42 | #[derive(Clone, PartialEq, Message)] 43 | pub struct JunkArgs { 44 | #[prost(int64, tag = "1")] 45 | pub x: i64, 46 | } 47 | #[derive(Clone, PartialEq, Message)] 48 | pub struct JunkReply { 49 | #[prost(string, tag = "1")] 50 | pub x: String, 51 | } 52 | 53 | #[derive(Default)] 54 | struct JunkInner { 55 | log2: Vec, 56 | } 57 | #[derive(Clone)] 58 | struct JunkService { 59 | inner: Arc>, 60 | } 61 | impl JunkService { 62 | fn new() -> JunkService { 63 | JunkService { 64 | inner: Arc::default(), 65 | } 66 | } 67 | } 68 | #[async_trait::async_trait] 69 | impl Junk for JunkService { 70 | async fn handler2(&self, args: JunkArgs) -> Result { 71 | self.inner.lock().unwrap().log2.push(args.x); 72 | Ok(JunkReply { 73 | x: format!("handler2-{}", args.x), 74 | }) 75 | } 76 | async fn handler3(&self, args: JunkArgs) -> Result { 77 | Delay::new(Duration::from_secs(20)).await; 78 | Ok(JunkReply { 79 | x: format!("handler3-{}", -args.x), 80 | }) 81 | } 82 | async fn handler4(&self, _: JunkArgs) -> Result { 83 | Ok(JunkReply { 84 | x: "pointer".to_owned(), 85 | }) 86 | } 87 | } 88 | 89 | fn init_logger() { 90 | static LOGGER_INIT: Once = Once::new(); 91 | LOGGER_INIT.call_once(env_logger::init); 92 | } 93 | 94 | #[test] 95 | fn test_service_dispatch() { 96 | init_logger(); 97 | 98 | let mut builder = ServerBuilder::new("test".to_owned()); 99 | let junk = JunkService::new(); 100 | add_service(junk.clone(), &mut builder).unwrap(); 101 | let prev_len = builder.services.len(); 102 | add_service(junk, &mut builder).unwrap_err(); 103 | assert_eq!(builder.services.len(), prev_len); 104 | let server = builder.build(); 105 | 106 | let buf = block_on(async { server.dispatch("junk.handler4", &[]).await.unwrap() }); 107 | let rsp = labcodec::decode(&buf).unwrap(); 108 | assert_eq!( 109 | JunkReply { 110 | x: "pointer".to_owned(), 111 | }, 112 | rsp, 113 | ); 114 | 115 | block_on(async { 116 | server 117 | .dispatch("junk.handler4", b"bad message") 118 | .await 119 | .unwrap_err(); 120 | 121 | server.dispatch("badjunk.handler4", &[]).await.unwrap_err(); 122 | 123 | server.dispatch("junk.badhandler", &[]).await.unwrap_err(); 124 | }); 125 | } 126 | 127 | #[test] 128 | fn test_network_client_rpc() { 129 | init_logger(); 130 | 131 | let mut builder = ServerBuilder::new("test".to_owned()); 132 | let junk = JunkService::new(); 133 | add_service(junk, &mut builder).unwrap(); 134 | let server = builder.build(); 135 | 136 | let (net, incoming) = Network::create(); 137 | net.add_server(server); 138 | 139 | let client = JunkClient::new(net.create_client("test_client".to_owned())); 140 | let (tx, rx) = mpsc::channel(); 141 | let cli = client.clone(); 142 | client.spawn(async move { 143 | let reply = cli.handler4(&JunkArgs { x: 777 }).await; 144 | tx.send(reply).unwrap(); 145 | }); 146 | let (mut rpc, incoming) = block_on(async { 147 | match incoming.into_future().await { 148 | (Some(rpc), s) => (rpc, s), 149 | _ => panic!("unexpected error"), 150 | } 151 | }); 152 | let reply = JunkReply { 153 | x: "boom!!!".to_owned(), 154 | }; 155 | let mut buf = vec![]; 156 | labcodec::encode(&reply, &mut buf).unwrap(); 157 | let resp = rpc.take_resp_sender().unwrap(); 158 | resp.send(Ok(buf)).unwrap(); 159 | assert_eq!(rpc.client_name, "test_client"); 160 | assert_eq!(rpc.fq_name, "junk.handler4"); 161 | assert!(!rpc.req.as_ref().unwrap().is_empty()); 162 | assert_eq!(rx.recv().unwrap(), Ok(reply)); 163 | 164 | let (tx, rx) = mpsc::channel(); 165 | let cli = client.clone(); 166 | client.spawn(async move { 167 | let reply = cli.handler4(&JunkArgs { x: 777 }).await; 168 | tx.send(reply).unwrap(); 169 | }); 170 | let (rpc, incoming) = block_on(async { 171 | match incoming.into_future().await { 172 | (Some(rpc), s) => (rpc, s), 173 | _ => panic!("unexpected error"), 174 | } 175 | }); 176 | drop(rpc.resp); 177 | assert_eq!(rx.recv().unwrap(), Err(Error::Recv(Canceled))); 178 | 179 | drop(incoming); 180 | assert_eq!( 181 | block_on(async { client.handler4(&JunkArgs::default()).await }), 182 | Err(Error::Stopped) 183 | ); 184 | } 185 | 186 | fn junk_suit() -> (Network, Server, JunkService) { 187 | let net = Network::new(); 188 | let server_name = "test_server".to_owned(); 189 | let mut builder = ServerBuilder::new(server_name); 190 | let junk_server = JunkService::new(); 191 | add_service(junk_server.clone(), &mut builder).unwrap(); 192 | let server = builder.build(); 193 | net.add_server(server.clone()); 194 | (net, server, junk_server) 195 | } 196 | 197 | #[test] 198 | fn test_basic() { 199 | init_logger(); 200 | 201 | let (net, _, _) = junk_suit(); 202 | 203 | let client = JunkClient::new(net.create_client("test_client".to_owned())); 204 | net.connect("test_client", "test_server"); 205 | net.enable("test_client", true); 206 | 207 | let rsp = block_on(async { client.handler4(&JunkArgs::default()).await.unwrap() }); 208 | assert_eq!( 209 | JunkReply { 210 | x: "pointer".to_owned(), 211 | }, 212 | rsp, 213 | ); 214 | } 215 | 216 | // does net.Enable(endname, false) really disconnect a client? 217 | #[test] 218 | fn test_disconnect() { 219 | init_logger(); 220 | 221 | let (net, _, _) = junk_suit(); 222 | 223 | let client = JunkClient::new(net.create_client("test_client".to_owned())); 224 | net.connect("test_client", "test_server"); 225 | 226 | block_on(async { client.handler4(&JunkArgs::default()).await.unwrap_err() }); 227 | 228 | net.enable("test_client", true); 229 | let rsp = block_on(async { client.handler4(&JunkArgs::default()).await.unwrap() }); 230 | 231 | assert_eq!( 232 | JunkReply { 233 | x: "pointer".to_owned(), 234 | }, 235 | rsp, 236 | ); 237 | } 238 | 239 | // test net.GetCount() 240 | #[test] 241 | fn test_count() { 242 | init_logger(); 243 | 244 | let (net, _, _) = junk_suit(); 245 | 246 | let client = JunkClient::new(net.create_client("test_client".to_owned())); 247 | net.connect("test_client", "test_server"); 248 | net.enable("test_client", true); 249 | 250 | for i in 0..=16 { 251 | let reply = block_on(async { client.handler2(&JunkArgs { x: i }).await.unwrap() }); 252 | assert_eq!(reply.x, format!("handler2-{}", i)); 253 | } 254 | 255 | assert_eq!(net.count("test_server"), 17); 256 | } 257 | 258 | // test RPCs from concurrent Clients 259 | #[test] 260 | fn test_concurrent_many() { 261 | init_logger(); 262 | 263 | let (net, server, _) = junk_suit(); 264 | let server_name = server.name(); 265 | 266 | let pool = ThreadPool::new().unwrap(); 267 | let (tx, rx) = mpsc::channel::(); 268 | 269 | let nclients = 20usize; 270 | let nrpcs = 10usize; 271 | for i in 0..nclients { 272 | let net = net.clone(); 273 | let sender = tx.clone(); 274 | let server_name = server_name.to_string(); 275 | 276 | pool.spawn_ok(async move { 277 | let mut n = 0; 278 | let client_name = format!("client-{}", i); 279 | let client = JunkClient::new(net.create_client(client_name.clone())); 280 | net.enable(&client_name, true); 281 | net.connect(&client_name, &server_name); 282 | 283 | for j in 0..nrpcs { 284 | let x = (i * 100 + j) as i64; 285 | let reply = client.handler2(&JunkArgs { x }).await.unwrap(); 286 | assert_eq!(reply.x, format!("handler2-{}", x)); 287 | n += 1; 288 | } 289 | 290 | sender.send(n).unwrap(); 291 | }); 292 | } 293 | 294 | let mut total = 0; 295 | for _ in 0..nclients { 296 | total += rx.recv().unwrap(); 297 | } 298 | assert_eq!(total, nrpcs * nclients); 299 | let n = net.count(server_name); 300 | assert_eq!(n, total); 301 | } 302 | 303 | #[test] 304 | fn test_unreliable() { 305 | init_logger(); 306 | 307 | let (net, server, _) = junk_suit(); 308 | let server_name = server.name(); 309 | net.set_reliable(false); 310 | 311 | let pool = ThreadPool::new().unwrap(); 312 | let (tx, rx) = mpsc::channel::(); 313 | let nclients = 300; 314 | for i in 0..nclients { 315 | let sender = tx.clone(); 316 | let mut n = 0; 317 | let server_name = server_name.to_owned(); 318 | let net = net.clone(); 319 | 320 | pool.spawn_ok(async move { 321 | let client_name = format!("client-{}", i); 322 | let client = JunkClient::new(net.create_client(client_name.clone())); 323 | net.enable(&client_name, true); 324 | net.connect(&client_name, &server_name); 325 | 326 | let x = i * 100; 327 | if let Ok(reply) = client.handler2(&JunkArgs { x }).await { 328 | assert_eq!(reply.x, format!("handler2-{}", x)); 329 | n += 1; 330 | } 331 | sender.send(n).unwrap(); 332 | }); 333 | } 334 | let mut total = 0; 335 | for _ in 0..nclients { 336 | total += rx.recv().unwrap(); 337 | } 338 | assert!( 339 | !(total == nclients as _ || total == 0), 340 | "all RPCs succeeded despite unreliable total {}, nclients {}", 341 | total, 342 | nclients 343 | ); 344 | } 345 | 346 | // test concurrent RPCs from a single Client 347 | #[test] 348 | fn test_concurrent_one() { 349 | init_logger(); 350 | 351 | let (net, server, junk_server) = junk_suit(); 352 | let server_name = server.name(); 353 | 354 | let pool = ThreadPool::new().unwrap(); 355 | let (tx, rx) = mpsc::channel::(); 356 | let nrpcs = 20; 357 | for i in 0..20 { 358 | let sender = tx.clone(); 359 | let client_name = format!("client-{}", i); 360 | let client = JunkClient::new(net.create_client(client_name.clone())); 361 | net.enable(&client_name, true); 362 | net.connect(&client_name, server_name); 363 | 364 | pool.spawn_ok(async move { 365 | let mut n = 0; 366 | let x = i + 100; 367 | let reply = client.handler2(&JunkArgs { x }).await.unwrap(); 368 | assert_eq!(reply.x, format!("handler2-{}", x)); 369 | n += 1; 370 | sender.send(n).unwrap() 371 | }); 372 | } 373 | 374 | let mut total = 0; 375 | for _ in 0..nrpcs { 376 | total += rx.recv().unwrap(); 377 | } 378 | assert_eq!( 379 | total, nrpcs, 380 | "wrong number of RPCs completed, got {}, expected {}", 381 | total, nrpcs 382 | ); 383 | 384 | assert_eq!( 385 | junk_server.inner.lock().unwrap().log2.len(), 386 | nrpcs, 387 | "wrong number of RPCs delivered" 388 | ); 389 | 390 | let n = net.count(server.name()); 391 | assert_eq!(n, total, "wrong count() {}, expected {}", n, total); 392 | } 393 | 394 | // regression: an RPC that's delayed during Enabled=false 395 | // should not delay subsequent RPCs (e.g. after Enabled=true). 396 | #[test] 397 | fn test_regression1() { 398 | init_logger(); 399 | 400 | let (net, server, junk_server) = junk_suit(); 401 | let server_name = server.name(); 402 | 403 | let client_name = "client"; 404 | let client = JunkClient::new(net.create_client(client_name.to_owned())); 405 | net.connect(client_name, server_name); 406 | 407 | // start some RPCs while the Client is disabled. 408 | // they'll be delayed. 409 | net.enable(client_name, false); 410 | 411 | let pool = ThreadPool::new().unwrap(); 412 | let (tx, rx) = mpsc::channel::(); 413 | let nrpcs = 20; 414 | for i in 0..20 { 415 | let sender = tx.clone(); 416 | let cli = client.clone(); 417 | pool.spawn_ok(async move { 418 | let x = i + 100; 419 | // this call ought to return false. 420 | let _ = cli.handler2(&JunkArgs { x }); 421 | sender.send(true).unwrap(); 422 | }); 423 | } 424 | 425 | // FIXME: have to sleep 300ms to pass the test 426 | // in my computer (i5-4200U, 4 threads). 427 | thread::sleep(Duration::from_millis(100 * 3)); 428 | 429 | let t0 = Instant::now(); 430 | net.enable(client_name, true); 431 | let x = 99; 432 | let reply = block_on(async { client.handler2(&JunkArgs { x }).await.unwrap() }); 433 | assert_eq!(reply.x, format!("handler2-{}", x)); 434 | let dur = t0.elapsed(); 435 | assert!( 436 | dur < Duration::from_millis(30), 437 | "RPC took too long ({:?}) after Enable", 438 | dur 439 | ); 440 | 441 | for _ in 0..nrpcs { 442 | rx.recv().unwrap(); 443 | } 444 | 445 | let len = junk_server.inner.lock().unwrap().log2.len(); 446 | assert_eq!( 447 | len, 1, 448 | "wrong number ({}) of RPCs delivered, expected 1", 449 | len 450 | ); 451 | 452 | let n = net.count(server.name()); 453 | assert_eq!(n, 1, "wrong count() {}, expected 1", n); 454 | } 455 | 456 | // if an RPC is stuck in a server, and the server 457 | // is killed with DeleteServer(), does the RPC 458 | // get un-stuck? 459 | #[test] 460 | fn test_killed() { 461 | init_logger(); 462 | 463 | let (net, server, _junk_server) = junk_suit(); 464 | let server_name = server.name(); 465 | 466 | let client_name = "client"; 467 | let client = JunkClient::new(net.create_client(client_name.to_owned())); 468 | net.connect(client_name, server_name); 469 | net.enable(client_name, true); 470 | let (tx, rx) = mpsc::channel(); 471 | let cli = client.clone(); 472 | client.spawn(async move { 473 | let reply = cli.handler3(&JunkArgs { x: 99 }).await; 474 | tx.send(reply).unwrap(); 475 | }); 476 | thread::sleep(Duration::from_secs(1)); 477 | rx.recv_timeout(Duration::from_millis(100)).unwrap_err(); 478 | 479 | net.delete_server(server_name); 480 | let reply = rx.recv_timeout(Duration::from_millis(100)).unwrap(); 481 | assert_eq!(reply, Err(Error::Stopped)); 482 | } 483 | 484 | struct Hooks { 485 | drop_req: AtomicBool, 486 | drop_resp: AtomicBool, 487 | } 488 | impl RpcHooks for Hooks { 489 | fn before_dispatch(&self, _: &str, _: &[u8]) -> Result<()> { 490 | if self.drop_req.load(Ordering::Relaxed) { 491 | Err(Error::Other("reqhook".to_owned())) 492 | } else { 493 | Ok(()) 494 | } 495 | } 496 | fn after_dispatch(&self, _: &str, resp: Result>) -> Result> { 497 | if self.drop_resp.load(Ordering::Relaxed) { 498 | Err(Error::Other("resphook".to_owned())) 499 | } else { 500 | resp 501 | } 502 | } 503 | } 504 | 505 | #[test] 506 | fn test_rpc_hooks() { 507 | init_logger(); 508 | let (net, _, _) = junk_suit(); 509 | 510 | let raw_cli = net.create_client("test_client".to_owned()); 511 | let hook = Arc::new(Hooks { 512 | drop_req: AtomicBool::new(false), 513 | drop_resp: AtomicBool::new(false), 514 | }); 515 | raw_cli.set_hooks(hook.clone()); 516 | 517 | let client = JunkClient::new(raw_cli); 518 | net.connect("test_client", "test_server"); 519 | net.enable("test_client", true); 520 | 521 | let i = 100; 522 | let reply = block_on(async { client.handler2(&JunkArgs { x: i }).await.unwrap() }); 523 | assert_eq!(reply.x, format!("handler2-{}", i)); 524 | hook.drop_req.store(true, Ordering::Relaxed); 525 | assert_eq!( 526 | block_on(async { client.handler2(&JunkArgs { x: i }).await.unwrap_err() }), 527 | Error::Other("reqhook".to_owned()) 528 | ); 529 | hook.drop_req.store(false, Ordering::Relaxed); 530 | hook.drop_resp.store(true, Ordering::Relaxed); 531 | assert_eq!( 532 | block_on(async { client.handler2(&JunkArgs { x: i }).await.unwrap_err() }), 533 | Error::Other("resphook".to_owned()) 534 | ); 535 | hook.drop_resp.store(false, Ordering::Relaxed); 536 | block_on(async { client.handler2(&JunkArgs { x: i }).await.unwrap() }); 537 | assert_eq!(reply.x, format!("handler2-{}", i)); 538 | } 539 | } 540 | -------------------------------------------------------------------------------- /labrpc/src/macros.rs: -------------------------------------------------------------------------------- 1 | #[macro_export] 2 | macro_rules! service { 3 | () => { 4 | compile_error!("empty service is not allowed"); 5 | }; 6 | ( 7 | $(#[$service_attr:meta])* 8 | service $svc_name:ident { 9 | $( 10 | $(#[$method_attr:meta])* 11 | rpc $method_name:ident($input:ty) returns ($output:ty); 12 | )* 13 | } 14 | ) => { 15 | $(#[$service_attr])* 16 | pub mod $svc_name { 17 | // In order to find input and output. 18 | use super::*; 19 | // $( use super::$input; )* 20 | // $( use super::$output;)* 21 | 22 | extern crate futures as __futures; 23 | 24 | #[async_trait::async_trait] 25 | pub trait Service: Clone + Send + Sync + 'static { 26 | $( 27 | $(#[$method_attr])* 28 | async fn $method_name(&self, req: $input) -> $crate::Result<$output>; 29 | )* 30 | } 31 | 32 | #[derive(Clone)] 33 | pub struct Client { 34 | client: $crate::Client, 35 | } 36 | impl Client { 37 | pub fn new(client: $crate::Client) -> Client { 38 | Client { client } 39 | } 40 | 41 | pub fn spawn(&self, f: F) 42 | where F: __futures::Future + Send + 'static 43 | { 44 | self.client.worker.spawn_ok(f); 45 | } 46 | 47 | $(pub fn $method_name(&self, args: &$input) -> $crate::RpcFuture<$crate::Result<$output>> { 48 | let fq_name = concat!(stringify!($svc_name), ".", stringify!($method_name)); 49 | self.client.call(fq_name, args) 50 | })* 51 | } 52 | 53 | pub fn add_service(svc: T, builder: &mut $crate::ServerBuilder) -> $crate::Result<()> { 54 | use ::std::sync::Mutex; 55 | struct Factory { 56 | svc: Mutex, 57 | } 58 | impl $crate::HandlerFactory for Factory { 59 | fn handler(&self, name: &'static str) -> Box<$crate::Handler> { 60 | let s = self.svc.lock().unwrap().clone(); 61 | Box::new(move |req| { 62 | match name { 63 | $(stringify!($method_name) => { 64 | let request = match labcodec::decode(req) { 65 | Ok(req) => req, 66 | Err(e) => return Box::pin(__futures::future::err( 67 | $crate::Error::Decode(e) 68 | )), 69 | }; 70 | Box::pin(async move { 71 | let f = s.$method_name(request); 72 | let resp = f.await; 73 | match resp { 74 | Ok(resp) => { 75 | let mut rsp = vec![]; 76 | labcodec::encode(&resp, &mut rsp).map_err($crate::Error::Encode)?; 77 | Ok(rsp) 78 | } 79 | Err(e) => Err(e), 80 | } 81 | }) 82 | })* 83 | other => { 84 | Box::pin(__futures::future::err( 85 | $crate::Error::Unimplemented( 86 | format!("unknown {} in {}", other, stringify!($svc_name)) 87 | ) 88 | )) 89 | } 90 | } 91 | }) 92 | } 93 | } 94 | 95 | let fact = Factory { 96 | svc: Mutex::new(svc), 97 | }; 98 | 99 | builder.add_service(stringify!($svc_name), Box::new(fact)) 100 | } 101 | } 102 | }; 103 | } 104 | -------------------------------------------------------------------------------- /labrpc/src/network.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::future::Future; 3 | use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; 4 | use std::sync::{Arc, Mutex}; 5 | use std::time::Duration; 6 | 7 | use futures::channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}; 8 | use futures::executor::ThreadPool; 9 | use futures::future::FutureExt; 10 | use futures::select; 11 | use futures::stream::StreamExt; 12 | use futures_timer::Delay; 13 | use log::{debug, error}; 14 | use rand::{thread_rng, Rng}; 15 | 16 | use crate::client::{Client, Rpc}; 17 | use crate::error::{Error, Result}; 18 | use crate::server::Server; 19 | 20 | #[derive(Debug)] 21 | struct EndInfo { 22 | enabled: bool, 23 | reliable: bool, 24 | long_reordering: bool, 25 | server: Option, 26 | } 27 | 28 | struct Endpoints { 29 | // by client name 30 | enabled: HashMap, 31 | // servers, by name 32 | servers: HashMap>, 33 | // client_name -> server_name 34 | connections: HashMap>, 35 | } 36 | 37 | struct NetworkCore { 38 | reliable: AtomicBool, 39 | // pause a long time on send on disabled connection 40 | long_delays: AtomicBool, 41 | // sometimes delay replies a long time 42 | long_reordering: AtomicBool, 43 | endpoints: Mutex, 44 | count: AtomicUsize, 45 | sender: UnboundedSender, 46 | poller: ThreadPool, 47 | worker: ThreadPool, 48 | } 49 | 50 | #[derive(Clone)] 51 | pub struct Network { 52 | core: Arc, 53 | } 54 | 55 | impl Network { 56 | pub fn new() -> Network { 57 | let (net, incoming) = Network::create(); 58 | net.start(incoming); 59 | net 60 | } 61 | 62 | pub fn create() -> (Network, UnboundedReceiver) { 63 | let (sender, incoming) = unbounded(); 64 | let net = Network { 65 | core: Arc::new(NetworkCore { 66 | reliable: AtomicBool::new(true), 67 | long_delays: AtomicBool::new(false), 68 | long_reordering: AtomicBool::new(false), 69 | endpoints: Mutex::new(Endpoints { 70 | enabled: HashMap::new(), 71 | servers: HashMap::new(), 72 | connections: HashMap::new(), 73 | }), 74 | count: AtomicUsize::new(0), 75 | poller: ThreadPool::builder().pool_size(2).create().unwrap(), 76 | worker: ThreadPool::new().unwrap(), 77 | sender, 78 | }), 79 | }; 80 | 81 | (net, incoming) 82 | } 83 | 84 | fn start(&self, mut incoming: UnboundedReceiver) { 85 | let network = self.clone(); 86 | self.core.poller.spawn_ok(async move { 87 | while let Some(mut rpc) = incoming.next().await { 88 | let resp = rpc.take_resp_sender().unwrap(); 89 | let net = network.clone(); 90 | network.core.poller.spawn_ok(async move { 91 | let res = net.process_rpc(rpc).await; 92 | if let Err(e) = resp.send(res) { 93 | error!("fail to send resp: {:?}", e); 94 | } 95 | }) 96 | } 97 | }); 98 | } 99 | 100 | pub fn add_server(&self, server: Server) { 101 | let mut eps = self.core.endpoints.lock().unwrap(); 102 | eps.servers.insert(server.core.name.clone(), Some(server)); 103 | } 104 | 105 | pub fn delete_server(&self, name: &str) { 106 | let mut eps = self.core.endpoints.lock().unwrap(); 107 | if let Some(s) = eps.servers.get_mut(name) { 108 | *s = None; 109 | } 110 | } 111 | 112 | pub fn create_client(&self, name: String) -> Client { 113 | let sender = self.core.sender.clone(); 114 | let mut eps = self.core.endpoints.lock().unwrap(); 115 | eps.enabled.insert(name.clone(), false); 116 | eps.connections.insert(name.clone(), None); 117 | Client { 118 | name, 119 | sender, 120 | worker: self.core.worker.clone(), 121 | hooks: Arc::new(Mutex::new(None)), 122 | } 123 | } 124 | 125 | /// Connects a Client to a server. 126 | /// a Client can only be connected once in its lifetime. 127 | pub fn connect(&self, client_name: &str, server_name: &str) { 128 | let mut eps = self.core.endpoints.lock().unwrap(); 129 | eps.connections 130 | .insert(client_name.to_owned(), Some(server_name.to_owned())); 131 | } 132 | 133 | /// Enable/disable a Client. 134 | pub fn enable(&self, client_name: &str, enabled: bool) { 135 | debug!( 136 | "client {} is {}", 137 | client_name, 138 | if enabled { "enabled" } else { "disabled" } 139 | ); 140 | let mut eps = self.core.endpoints.lock().unwrap(); 141 | eps.enabled.insert(client_name.to_owned(), enabled); 142 | } 143 | 144 | pub fn set_reliable(&self, yes: bool) { 145 | self.core.reliable.store(yes, Ordering::Release); 146 | } 147 | 148 | pub fn set_long_reordering(&self, yes: bool) { 149 | self.core.long_reordering.store(yes, Ordering::Release); 150 | } 151 | 152 | pub fn set_long_delays(&self, yes: bool) { 153 | self.core.long_delays.store(yes, Ordering::Release); 154 | } 155 | 156 | pub fn count(&self, server_name: &str) -> usize { 157 | let eps = self.core.endpoints.lock().unwrap(); 158 | eps.servers[server_name].as_ref().unwrap().count() 159 | } 160 | 161 | pub fn total_count(&self) -> usize { 162 | self.core.count.load(Ordering::Relaxed) 163 | } 164 | 165 | fn end_info(&self, client_name: &str) -> EndInfo { 166 | let eps = self.core.endpoints.lock().unwrap(); 167 | let mut server = None; 168 | if let Some(Some(server_name)) = eps.connections.get(client_name) { 169 | server = eps.servers[server_name].clone(); 170 | } 171 | EndInfo { 172 | enabled: eps.enabled[client_name], 173 | reliable: self.core.reliable.load(Ordering::Acquire), 174 | long_reordering: self.core.long_reordering.load(Ordering::Acquire), 175 | server, 176 | } 177 | } 178 | 179 | fn is_server_dead(&self, client_name: &str, server_name: &str, server_id: usize) -> bool { 180 | let eps = self.core.endpoints.lock().unwrap(); 181 | !eps.enabled[client_name] 182 | || eps.servers.get(server_name).map_or(true, |o| { 183 | o.as_ref().map(|s| s.core.id != server_id).unwrap_or(true) 184 | }) 185 | } 186 | 187 | async fn process_rpc(&self, rpc: Rpc) -> Result> { 188 | self.core.count.fetch_add(1, Ordering::Relaxed); 189 | let network = self.clone(); 190 | let end_info = self.end_info(&rpc.client_name); 191 | debug!("{:?} process with {:?}", rpc, end_info); 192 | let EndInfo { 193 | enabled, 194 | reliable, 195 | long_reordering, 196 | server, 197 | } = end_info; 198 | 199 | match (enabled, server) { 200 | (true, Some(server)) => { 201 | let short_delay = if !reliable { 202 | // short delay 203 | let ms = thread_rng().gen::() % 27; 204 | Some(ms) 205 | } else { 206 | None 207 | }; 208 | 209 | if !reliable && (thread_rng().gen::() % 1000) < 100 { 210 | // drop the request, return as if timeout 211 | Delay::new(Duration::from_secs(short_delay.unwrap())).await; 212 | return Err(Error::Timeout); 213 | } 214 | 215 | let drop_reply = !reliable && thread_rng().gen::() % 1000 < 100; 216 | let long_reordering = if long_reordering && thread_rng().gen_range(0, 900) < 600i32 217 | { 218 | // delay the response for a while 219 | let upper_bound: u64 = 1 + thread_rng().gen_range(0, 2000); 220 | Some(200 + thread_rng().gen_range(0, upper_bound)) 221 | } else { 222 | None 223 | }; 224 | 225 | // Dispatch 226 | process_rpc( 227 | short_delay, 228 | drop_reply, 229 | long_reordering, 230 | rpc, 231 | network, 232 | server, 233 | ) 234 | .await 235 | } 236 | _ => { 237 | // simulate no reply and eventual timeout. 238 | let ms = if self.core.long_delays.load(Ordering::Acquire) { 239 | // let Raft tests check that leader doesn't send 240 | // RPCs synchronously. 241 | thread_rng().gen::() % 7000 242 | } else { 243 | // many kv tests require the client to try each 244 | // server in fairly rapid succession. 245 | thread_rng().gen::() % 100 246 | }; 247 | 248 | debug!("{:?} delay {}ms then timeout", rpc, ms); 249 | Delay::new(Duration::from_millis(ms)).await; 250 | Err(Error::Timeout) 251 | } 252 | } 253 | } 254 | 255 | /// Spawns a future to run on this net framework. 256 | pub fn spawn(&self, f: F) 257 | where 258 | F: Future + Send + 'static, 259 | { 260 | self.core.worker.spawn_ok(f); 261 | } 262 | 263 | /// Spawns a future to run on this net framework. 264 | pub fn spawn_poller(&self, f: F) 265 | where 266 | F: Future + Send + 'static, 267 | { 268 | self.core.poller.spawn_ok(f); 269 | } 270 | } 271 | 272 | async fn process_rpc( 273 | mut delay: Option, 274 | drop_reply: bool, 275 | long_reordering: Option, 276 | mut rpc: Rpc, 277 | network: Network, 278 | server: Server, 279 | ) -> Result> { 280 | // Dispatch =============================================================== 281 | if let Some(delay) = delay { 282 | Delay::new(Duration::from_millis(delay)).await; 283 | } 284 | // We has finished the delay, take it out to prevent polling 285 | // twice. 286 | delay.take(); 287 | 288 | let fq_name = rpc.fq_name; 289 | let req = rpc.req.take().unwrap(); 290 | if let Some(hooks) = rpc.hooks.lock().unwrap().as_ref() { 291 | hooks.before_dispatch(fq_name, &req)?; 292 | } 293 | 294 | // Execute the request (call the RPC handler) in a separate thread so that 295 | // we can periodically check if the server has been killed and the RPC 296 | // should get a failure reply. 297 | // 298 | // do not reply if DeleteServer() has been called, i.e. the server has been killed. 299 | // this is needed to avoid situation in which a client gets a positive reply 300 | // to an Append, but the server persisted the update into the old Persister. 301 | // config.go is careful to call DeleteServer() before superseding the Persister. 302 | let resp = select! { 303 | res = server.dispatch(fq_name, &req).fuse() => res, 304 | _ = server_dead( 305 | Duration::from_millis(100), 306 | network.clone(), 307 | &rpc.client_name, 308 | &server.core.name, 309 | server.core.id, 310 | ).fuse() => Err(Error::Stopped), 311 | }; 312 | 313 | let resp = if let Some(hooks) = rpc.hooks.lock().unwrap().as_ref() { 314 | hooks.after_dispatch(fq_name, resp)? 315 | } else { 316 | resp? 317 | }; 318 | 319 | // Ongoing ================================================================ 320 | let client_name = &rpc.client_name; 321 | let server_name = &server.core.name; 322 | let server_id = server.core.id; 323 | if network.is_server_dead(client_name, server_name, server_id) { 324 | return Err(Error::Stopped); 325 | } 326 | if drop_reply { 327 | // drop the reply, return as if timeout. 328 | return Err(Error::Timeout); 329 | } 330 | 331 | // Reordering ============================================================= 332 | if let Some(reordering) = long_reordering { 333 | debug!("{:?} next long reordering {}ms", rpc, reordering); 334 | Delay::new(Duration::from_millis(reordering)).await; 335 | } 336 | Ok(resp) 337 | } 338 | 339 | /// Checks if the specified server killed. 340 | /// 341 | /// It will return when the server is killed. 342 | async fn server_dead( 343 | interval: Duration, 344 | net: Network, 345 | client_name: &str, 346 | server_name: &str, 347 | server_id: usize, 348 | ) { 349 | loop { 350 | Delay::new(interval).await; 351 | if net.is_server_dead(client_name, server_name, server_id) { 352 | debug!("{:?} is dead", server_name); 353 | return; 354 | } 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /labrpc/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hash_map::{Entry, HashMap}; 2 | use std::fmt; 3 | use std::sync::atomic::{AtomicUsize, Ordering}; 4 | use std::sync::Arc; 5 | 6 | use futures::future::{self, BoxFuture}; 7 | 8 | use crate::error::{Error, Result}; 9 | 10 | static ID_ALLOC: AtomicUsize = AtomicUsize::new(0); 11 | 12 | pub type RpcFuture = BoxFuture<'static, T>; 13 | 14 | pub type Handler = dyn FnOnce(&[u8]) -> RpcFuture>>; 15 | 16 | pub trait HandlerFactory: Sync + Send + 'static { 17 | fn handler(&self, name: &'static str) -> Box; 18 | } 19 | 20 | pub struct ServerBuilder { 21 | name: String, 22 | // Service name -> service methods 23 | pub(crate) services: HashMap<&'static str, Box>, 24 | } 25 | 26 | impl ServerBuilder { 27 | pub fn new(name: String) -> ServerBuilder { 28 | ServerBuilder { 29 | name, 30 | services: HashMap::new(), 31 | } 32 | } 33 | 34 | pub fn add_service( 35 | &mut self, 36 | service_name: &'static str, 37 | factory: Box, 38 | ) -> Result<()> { 39 | match self.services.entry(service_name) { 40 | Entry::Occupied(_) => Err(Error::Other(format!( 41 | "{} has already registered", 42 | service_name 43 | ))), 44 | Entry::Vacant(entry) => { 45 | entry.insert(factory); 46 | Ok(()) 47 | } 48 | } 49 | } 50 | 51 | pub fn build(self) -> Server { 52 | Server { 53 | core: Arc::new(ServerCore { 54 | name: self.name, 55 | services: self.services, 56 | id: ID_ALLOC.fetch_add(1, Ordering::Relaxed), 57 | count: AtomicUsize::new(0), 58 | }), 59 | } 60 | } 61 | } 62 | 63 | pub(crate) struct ServerCore { 64 | pub(crate) name: String, 65 | pub(crate) id: usize, 66 | 67 | pub(crate) services: HashMap<&'static str, Box>, 68 | pub(crate) count: AtomicUsize, 69 | } 70 | 71 | #[derive(Clone)] 72 | pub struct Server { 73 | pub(crate) core: Arc, 74 | } 75 | 76 | impl Server { 77 | pub fn count(&self) -> usize { 78 | self.core.count.load(Ordering::Relaxed) 79 | } 80 | 81 | pub fn name(&self) -> &str { 82 | &self.core.name 83 | } 84 | 85 | pub(crate) fn dispatch(&self, fq_name: &'static str, req: &[u8]) -> RpcFuture>> { 86 | self.core.count.fetch_add(1, Ordering::Relaxed); 87 | let mut names = fq_name.split('.'); 88 | let service_name = match names.next() { 89 | Some(n) => n, 90 | None => { 91 | return Box::pin(future::err(Error::Unimplemented(format!( 92 | "unknown {}", 93 | fq_name 94 | )))); 95 | } 96 | }; 97 | let method_name = match names.next() { 98 | Some(n) => n, 99 | None => { 100 | return Box::pin(future::err(Error::Unimplemented(format!( 101 | "unknown {}", 102 | fq_name 103 | )))); 104 | } 105 | }; 106 | if let Some(factory) = self.core.services.get(service_name) { 107 | let handle = factory.handler(method_name); 108 | handle(req) 109 | } else { 110 | Box::pin(future::err(Error::Unimplemented(format!( 111 | "unknown {}", 112 | fq_name 113 | )))) 114 | } 115 | } 116 | } 117 | 118 | impl fmt::Debug for Server { 119 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 120 | f.debug_struct("Server") 121 | .field("name", &self.core.name) 122 | .field("id", &self.core.id) 123 | .finish() 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /linearizability/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "linearizability" 3 | version = "0.1.0" 4 | edition = "2018" 5 | publish = false 6 | 7 | [dev-dependencies] 8 | regex = "1.3" 9 | lazy_static = "1.4" 10 | -------------------------------------------------------------------------------- /linearizability/src/bitset.rs: -------------------------------------------------------------------------------- 1 | use std::num::Wrapping; 2 | 3 | #[derive(Clone)] 4 | pub struct Bitset(Vec); 5 | 6 | impl Bitset { 7 | pub fn new(bits: usize) -> Self { 8 | let extra = if bits % 64 != 0 { 1 } else { 0 }; 9 | Bitset(vec![0; bits / 64 + extra]) 10 | } 11 | 12 | pub fn set(&mut self, pos: usize) { 13 | let (major, minor) = bitset_index(pos); 14 | self.0[major] |= 1 << minor; 15 | } 16 | 17 | pub fn clear(&mut self, pos: usize) { 18 | let (major, minor) = bitset_index(pos); 19 | self.0[major] &= !(1 << minor); 20 | } 21 | 22 | fn popcnt(&self) -> usize { 23 | let mut total = 0; 24 | for b in &self.0 { 25 | let mut v = *b; 26 | v = (v & 0x5555_5555_5555_5555) + ((v & 0xAAAA_AAAA_AAAA_AAAA) >> 1); 27 | v = (v & 0x3333_3333_3333_3333) + ((v & 0xCCCC_CCCC_CCCC_CCCC) >> 2); 28 | v = (v & 0x0F0F_0F0F_0F0F_0F0F) + ((v & 0xF0F0_F0F0_F0F0_F0F0) >> 4); 29 | v = (Wrapping(v) * Wrapping(0x0101_0101_0101_0101)).0; 30 | total += ((v >> 56) & 0xFF) as usize; 31 | } 32 | total 33 | } 34 | 35 | pub fn hash(&self) -> u64 { 36 | let mut hash = self.popcnt() as u64; 37 | for v in &self.0 { 38 | hash ^= v; 39 | } 40 | hash 41 | } 42 | 43 | pub fn equals(&self, b2: &Bitset) -> bool { 44 | let b = &self.0; 45 | let b2 = &b2.0; 46 | if b.len() != b2.len() { 47 | return false; 48 | } 49 | for i in 0..b.len() { 50 | if b[i] != b2[i] { 51 | return false; 52 | } 53 | } 54 | true 55 | } 56 | } 57 | 58 | fn bitset_index(pos: usize) -> (usize, usize) { 59 | (pos / 64, pos % 64) 60 | } 61 | -------------------------------------------------------------------------------- /linearizability/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod bitset; 2 | pub mod model; 3 | pub mod models; 4 | 5 | use std::cell::{Ref, RefCell}; 6 | use std::collections::HashMap; 7 | use std::fmt::Debug; 8 | use std::rc::Rc; 9 | use std::sync::atomic::{AtomicBool, Ordering}; 10 | use std::sync::mpsc::{channel, Receiver, RecvTimeoutError}; 11 | use std::sync::Arc; 12 | use std::thread; 13 | use std::time::Duration; 14 | 15 | use crate::bitset::Bitset; 16 | use crate::model::{Event, EventKind, Events, Model, Operations, Value}; 17 | 18 | enum EntryKind { 19 | CallEntry, 20 | ReturnEntry, 21 | } 22 | 23 | struct Entry { 24 | pub kind: EntryKind, 25 | pub value: T, 26 | pub id: usize, 27 | pub time: i64, 28 | } 29 | 30 | fn make_entries( 31 | history: Operations, 32 | ) -> Vec>> { 33 | let mut entries = Vec::new(); 34 | for (id, elem) in history.into_iter().enumerate() { 35 | entries.push(Entry { 36 | kind: EntryKind::CallEntry, 37 | value: Value::Input(elem.input), 38 | id, 39 | time: elem.call, 40 | }); 41 | entries.push(Entry { 42 | kind: EntryKind::ReturnEntry, 43 | value: Value::Output(elem.output), 44 | id, 45 | time: elem.finish, 46 | }) 47 | } 48 | entries.sort_by(|a, b| a.time.partial_cmp(&b.time).unwrap()); 49 | entries 50 | } 51 | 52 | struct LinkedNodes { 53 | head: Option>, 54 | } 55 | 56 | impl LinkedNodes { 57 | pub fn new() -> Self { 58 | LinkedNodes { head: None } 59 | } 60 | 61 | pub fn head(&self) -> Option> { 62 | self.head.clone() 63 | } 64 | 65 | pub fn from_entries(entries: Vec>) -> Self { 66 | let mut matches: HashMap> = HashMap::new(); 67 | let mut nodes = Self::new(); 68 | 69 | for entry in entries.into_iter().rev() { 70 | nodes.push_front(match entry.kind { 71 | EntryKind::CallEntry => Rc::new(RefCell::new(Node { 72 | value: entry.value, 73 | matched: matches.get(&entry.id).cloned(), 74 | id: entry.id, 75 | next: None, 76 | prev: None, 77 | })), 78 | EntryKind::ReturnEntry => { 79 | let node = Rc::new(RefCell::new(Node { 80 | value: entry.value, 81 | matched: None, 82 | id: entry.id, 83 | next: None, 84 | prev: None, 85 | })); 86 | matches.insert(entry.id, node.clone()); 87 | node 88 | } 89 | }) 90 | } 91 | 92 | nodes 93 | } 94 | 95 | pub fn len(&self) -> usize { 96 | let mut len = 0; 97 | let mut entry = self.head.clone(); 98 | while let Some(e) = entry { 99 | entry = e.borrow().next.clone(); 100 | len += 1; 101 | } 102 | len 103 | } 104 | 105 | pub fn push_front(&mut self, new_head: LinkNode) { 106 | match self.head.take() { 107 | Some(old_head) => { 108 | old_head.borrow_mut().prev = Some(new_head.clone()); 109 | new_head.borrow_mut().next = Some(old_head); 110 | self.head = Some(new_head); 111 | } 112 | None => { 113 | self.head = Some(new_head); 114 | } 115 | } 116 | } 117 | } 118 | 119 | type LinkNode = Rc>>; 120 | 121 | struct Node { 122 | pub value: T, 123 | pub matched: Option>, 124 | pub id: usize, 125 | pub next: Option>, 126 | pub prev: Option>, 127 | } 128 | 129 | fn renumber(events: Vec>) -> Vec> { 130 | let mut e = Vec::new(); 131 | let mut m: HashMap = HashMap::new(); // renumbering 132 | let mut id: usize = 0; 133 | for event in events { 134 | e.push(Event { 135 | kind: event.kind, 136 | value: event.value, 137 | id: *m.entry(event.id).or_insert_with(|| { 138 | id += 1; 139 | id - 1 140 | }), 141 | }); 142 | } 143 | e 144 | } 145 | 146 | fn convert_entries(events: Vec>) -> Vec> { 147 | let mut entries = Vec::new(); 148 | for event in events { 149 | entries.push(match event.kind { 150 | EventKind::CallEvent => Entry { 151 | kind: EntryKind::CallEntry, 152 | value: event.value, 153 | id: event.id, 154 | time: -1, 155 | }, 156 | EventKind::ReturnEvent => Entry { 157 | kind: EntryKind::ReturnEntry, 158 | value: event.value, 159 | id: event.id, 160 | time: -1, 161 | }, 162 | }) 163 | } 164 | entries 165 | } 166 | 167 | struct CacheEntry { 168 | linearized: Bitset, 169 | state: T, 170 | } 171 | 172 | fn cache_contains( 173 | model: &M, 174 | cache: &HashMap>>, 175 | entry: &CacheEntry, 176 | ) -> bool { 177 | if cache.contains_key(&entry.linearized.hash()) { 178 | for elem in &cache[&entry.linearized.hash()] { 179 | if entry.linearized.equals(&elem.linearized) && model.equal(&entry.state, &elem.state) { 180 | return true; 181 | } 182 | } 183 | } 184 | false 185 | } 186 | 187 | struct CallsEntry { 188 | entry: Option>, 189 | state: T, 190 | } 191 | 192 | fn lift(entry: &LinkNode) { 193 | let prev = Ref::map(entry.borrow(), |e| e.prev.as_ref().unwrap()); 194 | prev.borrow_mut().next = entry.borrow().next.clone(); 195 | let next = Ref::map(entry.borrow(), |e| e.next.as_ref().unwrap()); 196 | next.borrow_mut().prev = entry.borrow().prev.clone(); 197 | 198 | let matched = Ref::map(entry.borrow(), |e| e.matched.as_ref().unwrap()); 199 | let matched_prev = Ref::map(matched.borrow(), |e| e.prev.as_ref().unwrap()); 200 | matched_prev.borrow_mut().next = matched.borrow().next.clone(); 201 | if matched.borrow().next.is_some() { 202 | let matched_next = Ref::map(matched.borrow(), |e| e.next.as_ref().unwrap()); 203 | matched_next.borrow_mut().prev = matched.borrow().prev.clone(); 204 | } 205 | } 206 | 207 | fn unlift(entry: &LinkNode) { 208 | { 209 | let matched = Ref::map(entry.borrow(), |e| e.matched.as_ref().unwrap()); 210 | let matched_prev = Ref::map(matched.borrow(), |e| e.prev.as_ref().unwrap()); 211 | matched_prev.borrow_mut().next = Some(matched.clone()); 212 | if matched.borrow().next.is_some() { 213 | let matched_next = Ref::map(matched.borrow(), |e| e.next.as_ref().unwrap()); 214 | matched_next.borrow_mut().prev = Some(matched.clone()); 215 | } 216 | } 217 | 218 | let prev = Ref::map(entry.borrow(), |e| e.prev.as_ref().unwrap()); 219 | prev.borrow_mut().next = Some(entry.clone()); 220 | let next = Ref::map(entry.borrow(), |e| e.next.as_ref().unwrap()); 221 | next.borrow_mut().prev = Some(entry.clone()); 222 | } 223 | 224 | fn check_single( 225 | model: M, 226 | mut subhistory: LinkedNodes>, 227 | kill: Arc, 228 | ) -> bool { 229 | let n = subhistory.len() / 2; 230 | let mut linearized = Bitset::new(n); 231 | let mut cache = HashMap::new(); 232 | let mut calls = vec![]; 233 | 234 | let mut state = model.init(); 235 | subhistory.push_front(Rc::new(RefCell::new(Node { 236 | value: Value::None, 237 | matched: None, 238 | id: usize::max_value(), 239 | prev: None, 240 | next: None, 241 | }))); 242 | let head_entry = subhistory.head().unwrap(); 243 | let mut entry = head_entry.borrow().next.clone(); 244 | while head_entry.borrow().next.is_some() { 245 | if kill.load(Ordering::SeqCst) { 246 | return false; 247 | } 248 | let matched = entry.as_ref().unwrap().borrow().matched.clone(); 249 | entry = if let Some(matching) = matched { 250 | // the return entry 251 | let res = model.step( 252 | &state, 253 | entry.as_ref().unwrap().borrow().value.input(), 254 | matching.borrow().value.output(), 255 | ); 256 | match res { 257 | (true, new_state) => { 258 | let mut new_linearized = linearized.clone(); 259 | new_linearized.set(entry.as_ref().unwrap().borrow().id); 260 | let new_cache_entry = CacheEntry { 261 | linearized: new_linearized.clone(), 262 | state: new_state.clone(), 263 | }; 264 | if !cache_contains(&model, &cache, &new_cache_entry) { 265 | let hash = new_linearized.hash(); 266 | cache.entry(hash).or_default().push(new_cache_entry); 267 | calls.push(CallsEntry { 268 | entry: entry.clone(), 269 | state, 270 | }); 271 | state = new_state; 272 | linearized.set(entry.as_ref().unwrap().borrow().id); 273 | lift(entry.as_ref().unwrap()); 274 | head_entry.borrow().next.clone() 275 | } else { 276 | entry.as_ref().unwrap().borrow().next.clone() 277 | } 278 | } 279 | (false, _) => entry.as_ref().unwrap().borrow().next.clone(), 280 | } 281 | } else { 282 | if calls.is_empty() { 283 | return false; 284 | } 285 | let calls_top = calls.pop().unwrap(); 286 | entry = calls_top.entry; 287 | state = calls_top.state; 288 | linearized.clear(entry.as_ref().unwrap().borrow().id); 289 | unlift(entry.as_ref().unwrap()); 290 | entry.as_ref().unwrap().borrow().next.clone() 291 | } 292 | } 293 | true 294 | } 295 | 296 | pub fn check_operations(model: M, history: Operations) -> bool { 297 | check_operations_timeout(model, history, Duration::new(0, 0)) 298 | } 299 | 300 | // timeout = 0 means no timeout 301 | // if this operation times out, then a false positive is possible 302 | pub fn check_operations_timeout( 303 | model: M, 304 | history: Operations, 305 | timeout: Duration, 306 | ) -> bool { 307 | let partitions = model.partition(history); 308 | 309 | let (tx, rx) = channel(); 310 | let mut handles = vec![]; 311 | let kill = Arc::new(AtomicBool::new(false)); 312 | let count = partitions.len(); 313 | for subhistory in partitions { 314 | let tx = tx.clone(); 315 | let kill = Arc::clone(&kill); 316 | let m = model.clone(); 317 | let handle = thread::spawn(move || { 318 | let l = LinkedNodes::from_entries(make_entries(subhistory)); 319 | let _ = tx.send(check_single(m, l, kill)); 320 | }); 321 | handles.push(handle); 322 | } 323 | 324 | let res = wait_res(rx, kill, count, timeout); 325 | for handle in handles { 326 | handle.join().unwrap(); 327 | } 328 | res 329 | } 330 | 331 | pub fn check_events(model: M, history: Events) -> bool { 332 | check_events_timeout(model, history, Duration::new(0, 0)) 333 | } 334 | 335 | // timeout = 0 means no timeout 336 | // if this operation times out, then a false positive is possible 337 | pub fn check_events_timeout( 338 | model: M, 339 | history: Events, 340 | timeout: Duration, 341 | ) -> bool { 342 | let partitions = model.partition_event(history); 343 | 344 | let (tx, rx) = channel(); 345 | let mut handles = vec![]; 346 | let kill = Arc::new(AtomicBool::new(false)); 347 | let count = partitions.len(); 348 | for subhistory in partitions { 349 | let tx = tx.clone(); 350 | let kill = Arc::clone(&kill); 351 | let m = model.clone(); 352 | let handle = thread::spawn(move || { 353 | let l = LinkedNodes::from_entries(convert_entries(renumber(subhistory))); 354 | let _ = tx.send(check_single(m, l, kill)); 355 | }); 356 | handles.push(handle); 357 | } 358 | 359 | let res = wait_res(rx, kill, count, timeout); 360 | for handle in handles { 361 | handle.join().unwrap(); 362 | } 363 | res 364 | } 365 | 366 | fn wait_res( 367 | rx: Receiver, 368 | kill: Arc, 369 | mut count: usize, 370 | timeout: Duration, 371 | ) -> bool { 372 | let mut ok = true; 373 | loop { 374 | match if timeout.as_secs() == 0 && timeout.subsec_nanos() == 0 { 375 | rx.recv().map_err(From::from) 376 | } else { 377 | rx.recv_timeout(timeout) 378 | } { 379 | Ok(res) => { 380 | ok = ok && res; 381 | if !ok { 382 | kill.store(true, Ordering::SeqCst); 383 | break; 384 | } 385 | count -= 1; 386 | if count == 0 { 387 | break; 388 | } 389 | } 390 | Err(RecvTimeoutError::Timeout) => break, 391 | Err(e) => panic!("recv err: {}", e), 392 | } 393 | } 394 | ok 395 | } 396 | -------------------------------------------------------------------------------- /linearizability/src/model.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::PartialEq; 2 | use std::fmt::Debug; 3 | use std::fmt::Display; 4 | use std::marker::Send; 5 | 6 | #[derive(Debug)] 7 | pub enum Value { 8 | Input(I), 9 | Output(O), 10 | None, 11 | } 12 | 13 | impl Value { 14 | pub fn input(&self) -> &I { 15 | if let Value::Input(i) = self { 16 | i 17 | } else { 18 | panic!("Not a input") 19 | } 20 | } 21 | 22 | pub fn output(&self) -> &O { 23 | if let Value::Output(o) = self { 24 | o 25 | } else { 26 | panic!("Not a output") 27 | } 28 | } 29 | } 30 | 31 | #[derive(Debug, Clone)] 32 | pub struct Operation { 33 | pub input: I, 34 | pub call: i64, // invocation time 35 | pub output: O, 36 | pub finish: i64, // response time 37 | } 38 | 39 | pub enum EventKind { 40 | CallEvent, 41 | ReturnEvent, 42 | } 43 | 44 | pub struct Event { 45 | pub kind: EventKind, 46 | pub value: T, 47 | pub id: usize, 48 | } 49 | 50 | pub type Operations = Vec>; 51 | pub type Events = Vec>>; 52 | 53 | pub trait Model: Clone + Send + 'static { 54 | type State: Clone + Display + PartialEq; 55 | type Input: Send + Debug + Clone + 'static; 56 | type Output: Send + Debug + Clone + 'static; 57 | 58 | // Partition functions, such that a history is linearizable if an only 59 | // if each partition is linearizable. If you don't want to implement 60 | // this, you can always use the `NoPartition` functions implemented 61 | // below. 62 | fn partition( 63 | &self, 64 | history: Operations, 65 | ) -> Vec> { 66 | vec![history] 67 | } 68 | 69 | fn partition_event( 70 | &self, 71 | history: Events, 72 | ) -> Vec> { 73 | vec![history] 74 | } 75 | 76 | // Initial state of the system. 77 | fn init(&self) -> Self::State; 78 | 79 | // Step function for the system. Returns whether or not the system 80 | // could take this step with the given inputs and outputs and also 81 | // returns the new state. This should not mutate the existing state. 82 | fn step( 83 | &self, 84 | state: &Self::State, 85 | input: &Self::Input, 86 | output: &Self::Output, 87 | ) -> (bool, Self::State); 88 | 89 | // Equality on states. If you are using a simple data type for states, 90 | // you can use the `ShallowEqual` function implemented below. 91 | fn equal(&self, state1: &Self::State, state2: &Self::State) -> bool { 92 | state1 == state2 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /linearizability/src/models.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use super::model::{EventKind, Events, Model, Operations}; 4 | 5 | #[derive(Clone, Debug)] 6 | pub enum Op { 7 | GET, 8 | PUT, 9 | APPEND, 10 | } 11 | 12 | #[derive(Clone, Debug)] 13 | pub struct KvInput { 14 | pub op: Op, 15 | pub key: String, 16 | pub value: String, 17 | } 18 | 19 | #[derive(Clone, Debug)] 20 | pub struct KvOutput { 21 | pub value: String, 22 | } 23 | 24 | #[derive(Clone, Default)] 25 | pub struct KvModel {} 26 | 27 | impl Model for KvModel { 28 | type State = String; 29 | type Input = KvInput; 30 | type Output = KvOutput; 31 | 32 | fn partition( 33 | &self, 34 | history: Operations, 35 | ) -> Vec> { 36 | let mut map = HashMap::new(); 37 | for op in history { 38 | let v = map.entry(op.input.key.clone()).or_insert_with(Vec::new); 39 | (*v).push(op); 40 | } 41 | let mut ret = vec![]; 42 | for (_, ops) in map { 43 | ret.push(ops); 44 | } 45 | ret 46 | } 47 | 48 | fn partition_event( 49 | &self, 50 | history: Events, 51 | ) -> Vec> { 52 | let mut m = HashMap::new(); 53 | let mut matched: HashMap = HashMap::new(); 54 | for event in history { 55 | match event.kind { 56 | EventKind::CallEvent => { 57 | let key = event.value.input().key.clone(); 58 | matched.insert(event.id, key.clone()); 59 | m.entry(key).or_insert_with(Vec::new).push(event); 60 | } 61 | EventKind::ReturnEvent => { 62 | let key = matched[&event.id].clone(); 63 | m.entry(key).or_insert_with(Vec::new).push(event); 64 | } 65 | } 66 | } 67 | let mut ret = vec![]; 68 | for (_, v) in m { 69 | ret.push(v); 70 | } 71 | ret 72 | } 73 | 74 | fn init(&self) -> Self::State { 75 | // note: we are modeling a single key's value here; 76 | // we're partitioning by key, so this is okay 77 | "".to_string() 78 | } 79 | 80 | fn step( 81 | &self, 82 | state: &Self::State, 83 | input: &Self::Input, 84 | output: &Self::Output, 85 | ) -> (bool, Self::State) { 86 | match input.op { 87 | Op::GET => (&output.value == state, state.clone()), 88 | Op::PUT => (true, input.value.clone()), 89 | Op::APPEND => (true, state.clone() + &input.value), 90 | } 91 | } 92 | } 93 | 94 | #[cfg(test)] 95 | mod tests { 96 | use std::collections::HashMap; 97 | use std::fs::File; 98 | use std::io::{BufRead, BufReader, Result}; 99 | 100 | use super::super::check_events; 101 | use super::{KvInput, KvModel, KvOutput, Op}; 102 | use crate::model::{Event, EventKind, Events, Model, Value}; 103 | use regex::Regex; 104 | 105 | fn check_kv(log_name: String, correct: bool) { 106 | let model = KvModel {}; 107 | 108 | let file_name = format!("../linearizability/test_data/{}.txt", &log_name); 109 | let events = match parse_kv_log(&file_name) { 110 | Ok(events) => events, 111 | Err(e) => panic!("parse kv log {} failed: {}", &file_name, e), 112 | }; 113 | assert_eq!(check_events(model, events), correct); 114 | } 115 | 116 | fn parse_kv_log( 117 | file_name: &str, 118 | ) -> Result::Input, ::Output>> { 119 | lazy_static::lazy_static! { 120 | static ref INVOKE_GET: Regex = Regex::new( 121 | r#"\{:process (\d+), :type :invoke, :f :get, :key "(.*)", :value nil\}"# 122 | ) 123 | .unwrap(); 124 | static ref INVOKE_PUT: Regex = Regex::new( 125 | r#"\{:process (\d+), :type :invoke, :f :put, :key "(.*)", :value "(.*)"\}"# 126 | ) 127 | .unwrap(); 128 | static ref INVOKE_APPEND: Regex = Regex::new( 129 | r#"\{:process (\d+), :type :invoke, :f :append, :key "(.*)", :value "(.*)"\}"# 130 | ) 131 | .unwrap(); 132 | static ref RETURN_GET: Regex = 133 | Regex::new(r#"\{:process (\d+), :type :ok, :f :get, :key ".*", :value "(.*)"\}"#) 134 | .unwrap(); 135 | static ref RETURN_PUT: Regex = 136 | Regex::new(r#"\{:process (\d+), :type :ok, :f :put, :key ".*", :value ".*"\}"#) 137 | .unwrap(); 138 | static ref RETURN_APPEND: Regex = 139 | Regex::new(r#"\{:process (\d+), :type :ok, :f :append, :key ".*", :value ".*"\}"#) 140 | .unwrap(); 141 | } 142 | 143 | let f = File::open(file_name)?; 144 | let buf_reader = BufReader::new(f); 145 | let mut events = vec![]; 146 | let mut id = 0; 147 | let mut procid_map: HashMap = HashMap::new(); 148 | 149 | for line in buf_reader.lines() { 150 | let contents = line.unwrap(); 151 | if let Some(args) = INVOKE_GET.captures(&contents) { 152 | events.push(Event { 153 | kind: EventKind::CallEvent, 154 | value: Value::Input(KvInput { 155 | op: Op::GET, 156 | key: args[2].to_string(), 157 | value: "".to_string(), 158 | }), 159 | id, 160 | }); 161 | procid_map.insert(args[1].to_string().parse().unwrap(), id); 162 | id += 1; 163 | } else if let Some(args) = INVOKE_PUT.captures(&contents) { 164 | events.push(Event { 165 | kind: EventKind::CallEvent, 166 | value: Value::Input(KvInput { 167 | op: Op::PUT, 168 | key: args[2].to_string(), 169 | value: args[3].to_string(), 170 | }), 171 | id, 172 | }); 173 | procid_map.insert(args[1].to_string().parse().unwrap(), id); 174 | id += 1; 175 | } else if let Some(args) = INVOKE_APPEND.captures(&contents) { 176 | events.push(Event { 177 | kind: EventKind::CallEvent, 178 | value: Value::Input(KvInput { 179 | op: Op::APPEND, 180 | key: args[2].to_string(), 181 | value: args[3].to_string(), 182 | }), 183 | id, 184 | }); 185 | procid_map.insert(args[1].to_string().parse().unwrap(), id); 186 | id += 1; 187 | } else if let Some(args) = RETURN_GET.captures(&contents) { 188 | let match_id = procid_map 189 | .remove(&args[1].to_string().parse().unwrap()) 190 | .unwrap(); 191 | events.push(Event { 192 | kind: EventKind::ReturnEvent, 193 | value: Value::Output(KvOutput { 194 | value: args[2].to_string(), 195 | }), 196 | id: match_id, 197 | }); 198 | } else if let Some(args) = RETURN_PUT.captures(&contents) { 199 | let match_id = procid_map 200 | .remove(&args[1].to_string().parse().unwrap()) 201 | .unwrap(); 202 | events.push(Event { 203 | kind: EventKind::ReturnEvent, 204 | value: Value::Output(KvOutput { 205 | value: "".to_string(), 206 | }), 207 | id: match_id, 208 | }); 209 | } else if let Some(args) = RETURN_APPEND.captures(&contents) { 210 | let match_id = procid_map 211 | .remove(&args[1].to_string().parse().unwrap()) 212 | .unwrap(); 213 | events.push(Event { 214 | kind: EventKind::ReturnEvent, 215 | value: Value::Output(KvOutput { 216 | value: "".to_string(), 217 | }), 218 | id: match_id, 219 | }); 220 | } else { 221 | unreachable!(); 222 | } 223 | } 224 | 225 | for (_, match_id) in procid_map { 226 | events.push(Event { 227 | kind: EventKind::ReturnEvent, 228 | value: Value::Output(KvOutput { 229 | value: "".to_string(), 230 | }), 231 | id: match_id, 232 | }) 233 | } 234 | Ok(events) 235 | } 236 | 237 | #[test] 238 | fn test_kv_1client_ok() { 239 | check_kv("c01-ok".to_string(), true) 240 | } 241 | 242 | #[test] 243 | fn test_kv_1client_bad() { 244 | check_kv("c01-bad".to_string(), false) 245 | } 246 | 247 | #[test] 248 | fn test_kv_10client_ok() { 249 | check_kv("c10-ok".to_string(), true) 250 | } 251 | 252 | #[test] 253 | fn test_kv_10client_bad() { 254 | check_kv("c10-bad".to_string(), false) 255 | } 256 | 257 | #[test] 258 | fn test_kv_50client_ok() { 259 | check_kv("c50-ok".to_string(), true) 260 | } 261 | 262 | #[test] 263 | fn test_kv_50client_bad() { 264 | check_kv("c50-bad".to_string(), false) 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /linearizability/test_data/c01-bad.txt: -------------------------------------------------------------------------------- 1 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 0 y"} 2 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 0 y"} 3 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 4 | {:process 0, :type :ok, :f :get, :key "7", :value ""} 5 | {:process 0, :type :invoke, :f :get, :key "6", :value nil} 6 | {:process 0, :type :ok, :f :get, :key "6", :value ""} 7 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 8 | {:process 0, :type :ok, :f :get, :key "5", :value ""} 9 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 10 | {:process 0, :type :ok, :f :get, :key "2", :value ""} 11 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 1 y"} 12 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 1 y"} 13 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 14 | {:process 0, :type :ok, :f :get, :key "2", :value ""} 15 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 2 y"} 16 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 2 y"} 17 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 3 y"} 18 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 3 y"} 19 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 4 y"} 20 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 4 y"} 21 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 5 y"} 22 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 5 y"} 23 | {:process 0, :type :invoke, :f :get, :key "6", :value nil} 24 | {:process 0, :type :ok, :f :get, :key "6", :value ""} 25 | {:process 0, :type :invoke, :f :append, :key "1", :value "x 0 6 y"} 26 | {:process 0, :type :ok, :f :append, :key "1", :value "x 0 6 y"} 27 | {:process 0, :type :invoke, :f :get, :key "1", :value nil} 28 | {:process 0, :type :ok, :f :get, :key "1", :value "x 0 6 y"} 29 | {:process 0, :type :invoke, :f :get, :key "3", :value nil} 30 | {:process 0, :type :ok, :f :get, :key "3", :value ""} 31 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 7 y"} 32 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 7 y"} 33 | {:process 0, :type :invoke, :f :get, :key "0", :value nil} 34 | {:process 0, :type :ok, :f :get, :key "0", :value "x 0 0 yx 0 1 yx 0 2 yx 0 3 yx 0 4 y"} 35 | {:process 0, :type :invoke, :f :append, :key "1", :value "x 0 8 y"} 36 | {:process 0, :type :ok, :f :append, :key "1", :value "x 0 8 y"} 37 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 0 y"} 38 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 0 y"} 39 | {:process 0, :type :invoke, :f :get, :key "0", :value nil} 40 | {:process 0, :type :ok, :f :get, :key "0", :value "x 0 0 yx 0 1 yx 0 2 yx 0 3 yx 0 4 y"} 41 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 42 | {:process 0, :type :ok, :f :get, :key "2", :value "x 0 7 y"} 43 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 44 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 5 y"} 45 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 46 | {:process 0, :type :ok, :f :get, :key "2", :value "x 0 7 y"} 47 | {:process 0, :type :invoke, :f :append, :key "6", :value "x 0 1 y"} 48 | {:process 0, :type :ok, :f :append, :key "6", :value "x 0 1 y"} 49 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 50 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 5 y"} 51 | {:process 0, :type :invoke, :f :put, :key "1", :value "x 0 2 y"} 52 | {:process 0, :type :ok, :f :put, :key "1", :value "x 0 2 y"} 53 | {:process 0, :type :invoke, :f :get, :key "6", :value nil} 54 | {:process 0, :type :ok, :f :get, :key "6", :value "x 0 1 y"} 55 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 3 y"} 56 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 3 y"} 57 | {:process 0, :type :invoke, :f :append, :key "3", :value "x 0 4 y"} 58 | {:process 0, :type :ok, :f :append, :key "3", :value "x 0 4 y"} 59 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 60 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 0 y"} 61 | {:process 0, :type :invoke, :f :append, :key "3", :value "x 0 5 y"} 62 | {:process 0, :type :ok, :f :append, :key "3", :value "x 0 5 y"} 63 | {:process 0, :type :invoke, :f :append, :key "6", :value "x 0 6 y"} 64 | {:process 0, :type :ok, :f :append, :key "6", :value "x 0 6 y"} 65 | {:process 0, :type :invoke, :f :append, :key "4", :value "x 0 7 y"} 66 | {:process 0, :type :ok, :f :append, :key "4", :value "x 0 7 y"} 67 | {:process 0, :type :invoke, :f :append, :key "6", :value "x 0 8 y"} 68 | {:process 0, :type :ok, :f :append, :key "6", :value "x 0 8 y"} 69 | {:process 0, :type :invoke, :f :append, :key "3", :value "x 0 9 y"} 70 | {:process 0, :type :ok, :f :append, :key "3", :value "x 0 9 y"} 71 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 72 | {:process 0, :type :ok, :f :get, :key "2", :value "x 0 7 y"} 73 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 74 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 0 yx 0 3 y"} 75 | {:process 0, :type :invoke, :f :put, :key "2", :value "x 0 10 y"} 76 | {:process 0, :type :ok, :f :put, :key "2", :value "x 0 10 y"} -------------------------------------------------------------------------------- /linearizability/test_data/c01-ok.txt: -------------------------------------------------------------------------------- 1 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 0 y"} 2 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 0 y"} 3 | {:process 0, :type :invoke, :f :append, :key "4", :value "x 0 1 y"} 4 | {:process 0, :type :ok, :f :append, :key "4", :value "x 0 1 y"} 5 | {:process 0, :type :invoke, :f :append, :key "9", :value "x 0 2 y"} 6 | {:process 0, :type :ok, :f :append, :key "9", :value "x 0 2 y"} 7 | {:process 0, :type :invoke, :f :append, :key "4", :value "x 0 3 y"} 8 | {:process 0, :type :ok, :f :append, :key "4", :value "x 0 3 y"} 9 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 10 | {:process 0, :type :ok, :f :get, :key "5", :value ""} 11 | {:process 0, :type :invoke, :f :get, :key "0", :value nil} 12 | {:process 0, :type :ok, :f :get, :key "0", :value "x 0 0 y"} 13 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 4 y"} 14 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 4 y"} 15 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 16 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 4 y"} 17 | {:process 0, :type :invoke, :f :append, :key "9", :value "x 0 5 y"} 18 | {:process 0, :type :ok, :f :append, :key "9", :value "x 0 5 y"} 19 | {:process 0, :type :invoke, :f :get, :key "9", :value nil} 20 | {:process 0, :type :ok, :f :get, :key "9", :value "x 0 2 yx 0 5 y"} 21 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 6 y"} 22 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 6 y"} 23 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 7 y"} 24 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 7 y"} 25 | {:process 0, :type :invoke, :f :get, :key "1", :value nil} 26 | {:process 0, :type :ok, :f :get, :key "1", :value ""} 27 | {:process 0, :type :invoke, :f :append, :key "0", :value "x 0 8 y"} 28 | {:process 0, :type :ok, :f :append, :key "0", :value "x 0 8 y"} 29 | {:process 0, :type :invoke, :f :put, :key "9", :value "x 0 9 y"} 30 | {:process 0, :type :ok, :f :put, :key "9", :value "x 0 9 y"} 31 | {:process 0, :type :invoke, :f :get, :key "8", :value nil} 32 | {:process 0, :type :ok, :f :get, :key "8", :value ""} 33 | {:process 0, :type :invoke, :f :get, :key "8", :value nil} 34 | {:process 0, :type :ok, :f :get, :key "8", :value ""} 35 | {:process 0, :type :invoke, :f :get, :key "9", :value nil} 36 | {:process 0, :type :ok, :f :get, :key "9", :value "x 0 9 y"} 37 | {:process 0, :type :invoke, :f :append, :key "1", :value "x 0 10 y"} 38 | {:process 0, :type :ok, :f :append, :key "1", :value "x 0 10 y"} 39 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 11 y"} 40 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 11 y"} 41 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 42 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 4 y"} 43 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 44 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 4 y"} 45 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 12 y"} 46 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 12 y"} 47 | {:process 0, :type :invoke, :f :get, :key "6", :value nil} 48 | {:process 0, :type :ok, :f :get, :key "6", :value ""} 49 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 50 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 12 y"} 51 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 0 y"} 52 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 0 y"} 53 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 54 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 12 y"} 55 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 1 y"} 56 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 1 y"} 57 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 2 y"} 58 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 2 y"} 59 | {:process 0, :type :invoke, :f :get, :key "8", :value nil} 60 | {:process 0, :type :ok, :f :get, :key "8", :value ""} 61 | {:process 0, :type :invoke, :f :get, :key "9", :value nil} 62 | {:process 0, :type :ok, :f :get, :key "9", :value "x 0 9 y"} 63 | {:process 0, :type :invoke, :f :append, :key "8", :value "x 0 3 y"} 64 | {:process 0, :type :ok, :f :append, :key "8", :value "x 0 3 y"} 65 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 4 y"} 66 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 4 y"} 67 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 68 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 12 yx 0 4 y"} 69 | {:process 0, :type :invoke, :f :get, :key "5", :value nil} 70 | {:process 0, :type :ok, :f :get, :key "5", :value "x 0 12 yx 0 4 y"} 71 | {:process 0, :type :invoke, :f :append, :key "1", :value "x 0 5 y"} 72 | {:process 0, :type :ok, :f :append, :key "1", :value "x 0 5 y"} 73 | {:process 0, :type :invoke, :f :append, :key "3", :value "x 0 6 y"} 74 | {:process 0, :type :ok, :f :append, :key "3", :value "x 0 6 y"} 75 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 7 y"} 76 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 7 y"} 77 | {:process 0, :type :invoke, :f :get, :key "0", :value nil} 78 | {:process 0, :type :ok, :f :get, :key "0", :value "x 0 0 yx 0 8 y"} 79 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 8 y"} 80 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 8 y"} 81 | {:process 0, :type :invoke, :f :append, :key "4", :value "x 0 9 y"} 82 | {:process 0, :type :ok, :f :append, :key "4", :value "x 0 9 y"} 83 | {:process 0, :type :invoke, :f :put, :key "4", :value "x 0 10 y"} 84 | {:process 0, :type :ok, :f :put, :key "4", :value "x 0 10 y"} 85 | {:process 0, :type :invoke, :f :get, :key "1", :value nil} 86 | {:process 0, :type :ok, :f :get, :key "1", :value "x 0 10 yx 0 5 y"} 87 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 11 y"} 88 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 11 y"} 89 | {:process 0, :type :invoke, :f :append, :key "4", :value "x 0 12 y"} 90 | {:process 0, :type :ok, :f :append, :key "4", :value "x 0 12 y"} 91 | {:process 0, :type :invoke, :f :append, :key "7", :value "x 0 13 y"} 92 | {:process 0, :type :ok, :f :append, :key "7", :value "x 0 13 y"} 93 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 14 y"} 94 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 14 y"} 95 | {:process 0, :type :invoke, :f :get, :key "8", :value nil} 96 | {:process 0, :type :ok, :f :get, :key "8", :value "x 0 3 y"} 97 | {:process 0, :type :invoke, :f :get, :key "3", :value nil} 98 | {:process 0, :type :ok, :f :get, :key "3", :value "x 0 6 y"} 99 | {:process 0, :type :invoke, :f :get, :key "3", :value nil} 100 | {:process 0, :type :ok, :f :get, :key "3", :value "x 0 6 y"} 101 | {:process 0, :type :invoke, :f :get, :key "7", :value nil} 102 | {:process 0, :type :ok, :f :get, :key "7", :value "x 0 4 yx 0 0 yx 0 1 yx 0 7 yx 0 11 yx 0 13 y"} 103 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 15 y"} 104 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 15 y"} 105 | {:process 0, :type :invoke, :f :get, :key "3", :value nil} 106 | {:process 0, :type :ok, :f :get, :key "3", :value "x 0 6 y"} 107 | {:process 0, :type :invoke, :f :get, :key "2", :value nil} 108 | {:process 0, :type :ok, :f :get, :key "2", :value "x 0 6 yx 0 7 yx 0 11 yx 0 2 yx 0 8 y"} 109 | {:process 0, :type :invoke, :f :append, :key "2", :value "x 0 16 y"} 110 | {:process 0, :type :ok, :f :append, :key "2", :value "x 0 16 y"} 111 | {:process 0, :type :invoke, :f :append, :key "5", :value "x 0 17 y"} 112 | {:process 0, :type :ok, :f :append, :key "5", :value "x 0 17 y"} 113 | {:process 0, :type :invoke, :f :append, :key "6", :value "x 0 18 y"} 114 | {:process 0, :type :ok, :f :append, :key "6", :value "x 0 18 y"} 115 | {:process 0, :type :invoke, :f :append, :key "3", :value "x 0 19 y"} 116 | {:process 0, :type :ok, :f :append, :key "3", :value "x 0 19 y"} -------------------------------------------------------------------------------- /percolator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "percolator" 3 | version = "0.1.0" 4 | authors = ["Ryan Leung "] 5 | edition = "2018" 6 | build = "build.rs" 7 | publish = false 8 | 9 | [dependencies] 10 | async-trait = "0.1" 11 | futures = "0.3.16" 12 | futures-timer = "3.0" 13 | lazy_static = "1.4.0" 14 | log = "0.4" 15 | prost = "0.8" 16 | prost-derive = "0.6" 17 | tokio = { version = "1.5", features = ["time", "rt-multi-thread"] } 18 | 19 | labrpc = { path = "../labrpc" } 20 | labcodec = { path = "../labcodec" } 21 | 22 | [build-dependencies] 23 | prost-build = "0.8" 24 | 25 | [dev-dependencies] 26 | env_logger = "0.7" 27 | -------------------------------------------------------------------------------- /percolator/README.md: -------------------------------------------------------------------------------- 1 | # The Percolator lab 2 | 3 | ## What is Percolator 4 | 5 | Percolator is a system built by Google for incremental processing on a very 6 | large data set. Percolator also provides a distributed transaction protocol with 7 | ACID snapshot-isolation semantics. You can find more details in the paper: 8 | [Large-scale Incremental Processing Using Distributed Transactions and Notifications](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/36726.pdf). 9 | 10 | ## Lab prerequisites 11 | 12 | To start this lab, there are some prerequisites you need to: 13 | 14 | 1. be familiar with Rust (You can also learn something from our Rust training 15 | course) 16 | 17 | 2. know about how protobuf works 18 | 19 | 3. have basic knowledge of how RPC works 20 | 21 | 4. have basic knowledge of what is the distributed transaction 22 | 23 | ## Concepts of the lab 24 | 25 | ### Server 26 | 27 | There are two kinds of servers which provide different services in this lab: the 28 | TSO server and the storage server. 29 | 30 | #### TSO server 31 | 32 | Percolator relies on a service named *timestamp oracle*. The TSO server 33 | implemented by `TimestampOracle` can produce timestamps in a strictly increasing 34 | order. All transactions need to get the unique timestamp to indicate the 35 | execution order. 36 | 37 | #### Storage server 38 | 39 | Percolator is built upon the Bigtable, which presents a multi-dimensional sorted 40 | map to users. In this lab, the storage server implemented by `MemoryStorage`, 41 | which consists of three columns, is used to simulate the Bigtable. These columns 42 | which implemented by `BTreeMap` are similar to the column in the Bigtable. In 43 | particular, the `MemoryStorage` has three columns: `Write`, `Data`, `Lock` to 44 | keep consistent with the Bigtable. 45 | 46 | Besides, the storage also needs to provide the basic operations like `read`, 47 | `write` and `erase` to manipulate the data stored in it. 48 | 49 | ### Client 50 | 51 | The client will `begin` a transaction which contains a set of operations, like 52 | `get` and `set`, and call `commit` to commit a transaction. Also, the client 53 | will call `get_timestamp` to obtain a timestamp. 54 | 55 | More implementation details can be found in the paper. 56 | 57 | ## Writing your own implementation 58 | 59 | There are some comments leaving in this project such as "Your definitions here" 60 | or "Your code here". You need to write the code by yourself according to the paper. 61 | There are not many strict restrictions, and thus you can define as many variables 62 | in both *struct* and *proto* as required to implement the functionality. 63 | 64 | ## Testing your work 65 | 66 | You can directly run the following command in the current directory: 67 | 68 | ```sh 69 | make test_percolator 70 | ``` 71 | -------------------------------------------------------------------------------- /percolator/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | prost_build::compile_protos(&["proto/msg.proto"], &["proto"]).unwrap(); 3 | println!("cargo:rerun-if-changed=proto"); 4 | } 5 | -------------------------------------------------------------------------------- /percolator/proto/msg.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffers are a flexible, efficient, automated mechanism for 2 | // serializing structured data. You can define how you want your data to be 3 | // structured. Details can be found in 4 | // https://developers.google.com/protocol-buffers/docs/proto3. 5 | 6 | // Once you have defined the message, the `build.rs` will generate the 7 | // corresponding data structure in `OUT_DIR`. You can use the structure by 8 | // importing the `msg` module. Example: use crate::msg::CommitRequest; 9 | 10 | syntax = "proto3"; 11 | 12 | package msg; 13 | 14 | message TimestampRequest {} 15 | 16 | message TimestampResponse { uint64 ts = 1; } 17 | 18 | message GetRequest { 19 | bytes key = 1; 20 | uint64 start_ts = 2; 21 | } 22 | 23 | message GetResponse { bytes value = 1; } 24 | 25 | message Write { 26 | bytes key = 1; 27 | bytes value = 2; 28 | } 29 | 30 | message PrewriteRequest { 31 | Write write = 1; 32 | bytes primary_key = 2; 33 | uint64 start_ts = 3; 34 | } 35 | 36 | message PrewriteResponse { bool success = 1; } 37 | 38 | message CommitRequest { 39 | bool is_primary = 1; 40 | bytes key = 2; 41 | uint64 start_ts = 3; 42 | uint64 commit_ts = 4; 43 | } 44 | 45 | message CommitResponse { bool success = 1; } 46 | -------------------------------------------------------------------------------- /percolator/src/client.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::needless_collect)] 2 | 3 | use std::time::Duration; 4 | 5 | use futures::Future; 6 | use labrpc::*; 7 | 8 | use crate::msg::*; 9 | use crate::service::{TSOClient, TransactionClient}; 10 | use crate::RUNTIME; 11 | 12 | // BACKOFF_TIME_MS is the wait time before retrying to send the request. 13 | // It should be exponential growth. e.g. 14 | //| retry time | backoff time | 15 | //|--------------|----------------| 16 | //| 1 | 100 | 17 | //| 2 | 200 | 18 | //| 3 | 400 | 19 | const BACKOFF_TIME_MS: u64 = 100; 20 | // RETRY_TIMES is the maximum number of times a client attempts to send a request. 21 | const RETRY_TIMES: usize = 3; 22 | 23 | #[derive(Clone)] 24 | struct Transcation { 25 | ts: u64, 26 | writes: Vec, 27 | } 28 | 29 | impl Transcation { 30 | pub fn new(ts: u64) -> Self { 31 | Self { ts, writes: vec![] } 32 | } 33 | } 34 | 35 | /// Client mainly has two purposes: 36 | /// One is getting a monotonically increasing timestamp from TSO (Timestamp Oracle). 37 | /// The other is do the transaction logic. 38 | #[derive(Clone)] 39 | pub struct Client { 40 | // Your definitions here. 41 | tso_client: TSOClient, 42 | txn_client: TransactionClient, 43 | current_txn: Option, 44 | } 45 | 46 | impl Client { 47 | /// Creates a new Client. 48 | pub fn new(tso_client: TSOClient, txn_client: TransactionClient) -> Client { 49 | // Your code here. 50 | Client { 51 | tso_client, 52 | txn_client, 53 | current_txn: None, 54 | } 55 | } 56 | 57 | async fn get_timestamp_async(&self) -> Result { 58 | let response = back_off(|| self.tso_client.get_timestamp(&TimestampRequest {})).await; 59 | response.map(|r| r.ts) 60 | } 61 | 62 | /// Gets a timestamp from a TSO. 63 | pub fn get_timestamp(&self) -> Result { 64 | // Your code here. 65 | RUNTIME.block_on(self.get_timestamp_async()) 66 | } 67 | 68 | /// Begins a new transaction. 69 | pub fn begin(&mut self) { 70 | // Your code here. 71 | if self.current_txn.is_some() { 72 | panic!("already started another txn") 73 | } 74 | let ts = self.get_timestamp().unwrap(); 75 | self.current_txn.insert(Transcation::new(ts)); 76 | } 77 | 78 | /// Gets the value for a given key. 79 | pub fn get(&self, key: Vec) -> Result> { 80 | // Your code here. 81 | RUNTIME.block_on(async { 82 | let start_ts = match &self.current_txn { 83 | Some(t) => t.ts, 84 | None => self.get_timestamp_async().await.unwrap(), 85 | }; 86 | let response = self.txn_client.get(&GetRequest { key, start_ts }).await; 87 | response.map(|r| r.value) 88 | }) 89 | } 90 | 91 | /// Sets keys in a buffer until commit time. 92 | pub fn set(&mut self, key: Vec, value: Vec) { 93 | // Your code here. 94 | let txn = self.current_txn.as_mut().unwrap(); 95 | txn.writes.push(Write { key, value }); 96 | } 97 | 98 | /// Commits a transaction. 99 | pub fn commit(&mut self) -> Result { 100 | RUNTIME.block_on(self.commit_async()) 101 | } 102 | 103 | async fn commit_async(&mut self) -> Result { 104 | // Your code here. 105 | let Transcation { 106 | ts: start_ts, 107 | writes, 108 | } = self 109 | .current_txn 110 | .to_owned() 111 | .expect("no transaction to commit"); 112 | if writes.is_empty() { 113 | return Ok(true); 114 | } 115 | let primary_key = writes.get(0).unwrap().to_owned().key; 116 | 117 | let keys = writes.iter().map(|w| w.key.to_owned()).collect::>(); 118 | 119 | for write in writes.into_iter() { 120 | let args = &PrewriteRequest { 121 | write: Some(write), 122 | primary_key: primary_key.clone(), 123 | start_ts, 124 | }; 125 | let success = back_off(|| self.txn_client.prewrite(args)).await?.success; 126 | if !success { 127 | return Ok(false); 128 | } 129 | } 130 | 131 | let commit_ts = self.get_timestamp_async().await?; 132 | 133 | for (i, key) in keys.into_iter().enumerate() { 134 | let is_primary = i == 0; 135 | let args = &CommitRequest { 136 | is_primary, 137 | key, 138 | start_ts, 139 | commit_ts, 140 | }; 141 | let result = back_off(|| self.txn_client.commit(args)).await; 142 | if is_primary { 143 | match result { 144 | Ok(CommitResponse { success: false }) => return Ok(false), 145 | Ok(_) => {} 146 | Err(Error::Other(reason)) if reason == "reqhook" => return Ok(false), 147 | Err(e) => return Err(e), 148 | } 149 | } else { 150 | // must success, ignore any error 151 | } 152 | } 153 | 154 | self.current_txn.take(); 155 | Ok(true) 156 | } 157 | } 158 | 159 | async fn back_off(action: impl Fn() -> F) -> Result 160 | where 161 | F: Future>, 162 | { 163 | let mut result = action().await; 164 | 165 | for i in 1..=RETRY_TIMES { 166 | if result.is_ok() { 167 | return result; 168 | } 169 | let duration = Duration::from_millis(BACKOFF_TIME_MS << (i - 1)); 170 | tokio::time::sleep(duration).await; 171 | result = action().await; 172 | } 173 | 174 | result 175 | } 176 | -------------------------------------------------------------------------------- /percolator/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_imports)] 2 | #[macro_use] 3 | extern crate log; 4 | 5 | // After you finish the implementation, `#[allow(unused)]` should be removed. 6 | #[allow(unused)] 7 | mod client; 8 | mod server; 9 | mod service; 10 | #[cfg(test)] 11 | mod tests; 12 | 13 | // This is related to protobuf as described in `msg.proto`. 14 | mod msg { 15 | include!(concat!(env!("OUT_DIR"), "/msg.rs")); 16 | } 17 | 18 | use lazy_static::lazy_static; 19 | use tokio::runtime::Runtime; 20 | 21 | lazy_static! { 22 | static ref RUNTIME: Runtime = Runtime::new().unwrap(); 23 | } 24 | -------------------------------------------------------------------------------- /percolator/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::collections::btree_map::Entry; 2 | use std::collections::BTreeMap; 3 | use std::ops::{Bound, RangeBounds}; 4 | use std::sync::atomic::AtomicU64; 5 | use std::sync::{Arc, Mutex, MutexGuard}; 6 | use std::time::{Duration, Instant}; 7 | 8 | use crate::msg::*; 9 | use crate::service::*; 10 | 11 | // TTL is used for a lock key. 12 | // If the key's lifetime exceeds this value, it should be cleaned up. 13 | // Otherwise, the operation should back off. 14 | const TTL: Duration = Duration::from_millis(100); 15 | 16 | #[derive(Clone, Default)] 17 | pub struct TimestampOracle { 18 | // You definitions here if needed. 19 | next_ts: Arc, 20 | } 21 | 22 | #[async_trait::async_trait] 23 | impl timestamp::Service for TimestampOracle { 24 | // example get_timestamp RPC handler. 25 | async fn get_timestamp(&self, _: TimestampRequest) -> labrpc::Result { 26 | // Your code here. 27 | let ts = self 28 | .next_ts 29 | .fetch_add(1, std::sync::atomic::Ordering::SeqCst); 30 | 31 | Ok(TimestampResponse { ts }) 32 | } 33 | } 34 | 35 | // Key is a tuple (raw key, timestamp). 36 | pub type Key = (Vec, u64); 37 | 38 | #[derive(Clone, PartialEq)] 39 | pub enum Value { 40 | Timestamp(u64), 41 | Vector(Vec), 42 | } 43 | 44 | impl Value { 45 | fn as_ts(&self) -> u64 { 46 | match self { 47 | Value::Timestamp(ts) => *ts, 48 | Value::Vector(_) => panic!(), 49 | } 50 | } 51 | 52 | fn as_bytes(&self) -> &[u8] { 53 | match self { 54 | Value::Timestamp(_) => panic!(), 55 | Value::Vector(bytes) => bytes, 56 | } 57 | } 58 | } 59 | 60 | pub enum Column { 61 | Write, 62 | Data, 63 | Lock, 64 | } 65 | 66 | // KvTable is used to simulate Google's Bigtable. 67 | // It provides three columns: Write, Data, and Lock. 68 | #[derive(Clone, Default)] 69 | pub struct KvTable { 70 | write: BTreeMap, 71 | data: BTreeMap, 72 | lock: BTreeMap, 73 | } 74 | 75 | impl KvTable { 76 | fn column_ref(&self, column: Column) -> &BTreeMap { 77 | match column { 78 | Column::Write => &self.write, 79 | Column::Data => &self.data, 80 | Column::Lock => &self.lock, 81 | } 82 | } 83 | 84 | fn column_mut(&mut self, column: Column) -> &mut BTreeMap { 85 | match column { 86 | Column::Write => &mut self.write, 87 | Column::Data => &mut self.data, 88 | Column::Lock => &mut self.lock, 89 | } 90 | } 91 | 92 | // Reads the latest key-value record from a specified column 93 | // in MemoryStorage with a given key and a timestamp range. 94 | fn read( 95 | &self, 96 | key: &[u8], 97 | column: Column, 98 | ts_range: impl RangeBounds, 99 | ) -> Option<(&Key, &Value)> { 100 | // Your code here. 101 | let column = self.column_ref(column); 102 | let key_start = match ts_range.start_bound() { 103 | Bound::Included(ts) => Bound::Included((key.to_vec(), *ts)), 104 | Bound::Excluded(ts) => Bound::Excluded((key.to_vec(), *ts)), 105 | Bound::Unbounded => Bound::Included((key.to_vec(), 0)), 106 | }; 107 | let key_end = match ts_range.end_bound() { 108 | Bound::Included(ts) => Bound::Included((key.to_vec(), *ts)), 109 | Bound::Excluded(ts) => Bound::Excluded((key.to_vec(), *ts)), 110 | Bound::Unbounded => Bound::Included((key.to_vec(), u64::MAX)), 111 | }; 112 | 113 | column 114 | .range((key_start, key_end)) 115 | .last() 116 | .map(|(k, (v, _i))| (k, v)) 117 | } 118 | 119 | fn read_owned( 120 | &self, 121 | key: &[u8], 122 | column: Column, 123 | ts_range: impl RangeBounds, 124 | ) -> Option<(Key, Value)> { 125 | self.read(key, column, ts_range) 126 | .map(|(k, v)| (k.to_owned(), v.to_owned())) 127 | } 128 | 129 | // Writes a record to a specified column in MemoryStorage. 130 | fn write(&mut self, key: Vec, column: Column, ts: u64, value: Value) { 131 | // Your code here. 132 | let column = self.column_mut(column); 133 | column.insert((key, ts), (value, Instant::now())); 134 | } 135 | 136 | // Erases a record from a specified column in MemoryStorage. 137 | fn erase(&mut self, key: &[u8], column: Column, commit_ts: u64) -> Option { 138 | // Your code here. 139 | let column = self.column_mut(column); 140 | column.remove(&(key.to_vec(), commit_ts)).map(|(v, _i)| v) 141 | } 142 | 143 | fn try_erase_expired(&mut self, key: &[u8], column: Column, commit_ts: u64) -> Option { 144 | // Your code here. 145 | let column = self.column_mut(column); 146 | if let Entry::Occupied(o) = column.entry((key.to_vec(), commit_ts)) { 147 | let instant = o.get().1; 148 | if instant.elapsed() > TTL { 149 | return o.remove().0.into(); 150 | } 151 | } 152 | None 153 | } 154 | } 155 | 156 | // MemoryStorage is used to wrap a KvTable. 157 | // You may need to get a snapshot from it. 158 | #[derive(Clone, Default)] 159 | pub struct MemoryStorage { 160 | data: Arc>, 161 | } 162 | 163 | #[async_trait::async_trait] 164 | impl transaction::Service for MemoryStorage { 165 | // example get RPC handler. 166 | async fn get(&self, req: GetRequest) -> labrpc::Result { 167 | // Your code here. 168 | let GetRequest { key, start_ts } = req; 169 | 170 | let data = loop { 171 | let data = self.data.lock().unwrap(); 172 | let lock = data.read_owned(&key, Column::Lock, 0..=start_ts); 173 | 174 | if let Some(((key, ts), lock)) = lock { 175 | if let Some(data) = self.back_off_maybe_clean_up_lock(key, ts, lock, data) { 176 | break data; 177 | } 178 | } else { 179 | break data; 180 | } 181 | }; 182 | 183 | let value = if let Some((_, write)) = data.read(&key, Column::Write, 0..=start_ts) { 184 | let data_ts = write.as_ts(); 185 | let value = data.read(&key, Column::Data, data_ts..=data_ts).unwrap().1; 186 | value.as_bytes().to_vec() 187 | } else { 188 | vec![] 189 | }; 190 | 191 | Ok(GetResponse { value }) 192 | } 193 | 194 | // example prewrite RPC handler. 195 | async fn prewrite(&self, req: PrewriteRequest) -> labrpc::Result { 196 | // Your code here. 197 | 198 | let PrewriteRequest { 199 | write, 200 | primary_key, 201 | start_ts, 202 | } = req; 203 | let Write { key, value } = write.unwrap(); 204 | 205 | let success = { 206 | let mut data = self.data.lock().unwrap(); 207 | let lock = data.read(&key, Column::Lock, 0..); 208 | let write = data.read(&key, Column::Write, start_ts..); 209 | 210 | if lock.is_some() || write.is_some() { 211 | false 212 | } else { 213 | let lock = if key == primary_key { 214 | Value::Timestamp(0) 215 | } else { 216 | Value::Vector(primary_key) 217 | }; 218 | data.write(key.clone(), Column::Lock, start_ts, lock); 219 | data.write(key, Column::Data, start_ts, Value::Vector(value)); 220 | true 221 | } 222 | }; 223 | 224 | Ok(PrewriteResponse { success }) 225 | } 226 | 227 | // example commit RPC handler. 228 | async fn commit(&self, req: CommitRequest) -> labrpc::Result { 229 | // Your code here. 230 | let CommitRequest { 231 | is_primary, 232 | key, 233 | start_ts, 234 | commit_ts, 235 | } = req; 236 | 237 | let success = { 238 | let mut data = self.data.lock().unwrap(); 239 | let lock = data.erase(&key, Column::Lock, start_ts); 240 | if is_primary { 241 | if lock.is_some() { 242 | data.write(key, Column::Write, commit_ts, Value::Timestamp(start_ts)); 243 | true 244 | } else { 245 | false 246 | } 247 | } else { 248 | data.write(key, Column::Write, commit_ts, Value::Timestamp(start_ts)); 249 | true 250 | } 251 | }; 252 | 253 | Ok(CommitResponse { success }) 254 | } 255 | } 256 | 257 | impl MemoryStorage { 258 | fn back_off_maybe_clean_up_lock<'a>( 259 | &self, 260 | key: Vec, 261 | ts: u64, 262 | lock: Value, 263 | mut data: MutexGuard<'a, KvTable>, 264 | ) -> Option> { 265 | // Your code here. 266 | let may_rollback_primary = |key: &[u8], mut data: MutexGuard<'a, KvTable>| { 267 | if let Some(_lock) = data.try_erase_expired(key, Column::Lock, ts) { 268 | // may crashed, rollback it 269 | data.erase(key, Column::Data, ts); 270 | Some(data) 271 | } else { 272 | None 273 | } 274 | }; 275 | 276 | match lock { 277 | /* Primary */ 278 | Value::Timestamp(_) => may_rollback_primary(&key, data), 279 | /* Secondary */ 280 | Value::Vector(primary_key) => { 281 | let primary_lock = data.read_owned(&primary_key, Column::Lock, ts..=ts); 282 | let primary_write = data.read_owned(&primary_key, Column::Write, ts..); 283 | if primary_lock.is_none() { 284 | data.erase(&key, Column::Lock, ts).unwrap(); 285 | if let Some(((_, commit_ts), _)) = primary_write { 286 | // previous txn success, commit it 287 | data.write(key.to_vec(), Column::Write, commit_ts, Value::Timestamp(ts)); 288 | } else { 289 | // previous txn rollbacked, clean up it 290 | } 291 | Some(data) 292 | } else { 293 | may_rollback_primary(&primary_key, data) 294 | } 295 | } 296 | } 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /percolator/src/service.rs: -------------------------------------------------------------------------------- 1 | use crate::msg::{ 2 | CommitRequest, CommitResponse, GetRequest, GetResponse, PrewriteRequest, PrewriteResponse, 3 | TimestampRequest, TimestampResponse, 4 | }; 5 | 6 | labrpc::service! { 7 | service timestamp { 8 | rpc get_timestamp(TimestampRequest) returns (TimestampResponse); 9 | } 10 | } 11 | 12 | pub use timestamp::{add_service as add_tso_service, Client as TSOClient, Service}; 13 | 14 | labrpc::service! { 15 | service transaction { 16 | rpc get(GetRequest) returns (GetResponse); 17 | rpc prewrite(PrewriteRequest) returns (PrewriteResponse); 18 | rpc commit(CommitRequest) returns (CommitResponse); 19 | } 20 | } 21 | 22 | pub use transaction::{add_service as add_transaction_service, Client as TransactionClient}; 23 | -------------------------------------------------------------------------------- /percolator/src/tests.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{ 2 | atomic::{AtomicBool, Ordering}, 3 | Arc, 4 | }; 5 | use std::thread; 6 | use std::time::Duration; 7 | 8 | use labrpc::*; 9 | use prost::Message; 10 | 11 | use crate::client::Client; 12 | use crate::server::{MemoryStorage, TimestampOracle}; 13 | use crate::service::{add_transaction_service, add_tso_service, TSOClient, TransactionClient}; 14 | 15 | struct CommitHooks { 16 | drop_req: AtomicBool, 17 | drop_resp: AtomicBool, 18 | fail_primary: AtomicBool, 19 | } 20 | 21 | impl RpcHooks for CommitHooks { 22 | fn before_dispatch(&self, fq_name: &str, req: &[u8]) -> Result<()> { 23 | if self.drop_req.load(Ordering::Relaxed) && fq_name == "transaction.commit" { 24 | let m = crate::msg::CommitRequest::decode(req).unwrap(); 25 | if m.is_primary && !self.fail_primary.load(Ordering::Relaxed) { 26 | return Ok(()); 27 | } 28 | return Err(Error::Other("reqhook".to_owned())); 29 | } 30 | Ok(()) 31 | } 32 | fn after_dispatch(&self, fq_name: &str, resp: Result>) -> Result> { 33 | if self.drop_resp.load(Ordering::Relaxed) && fq_name == "transaction.commit" { 34 | return Err(Error::Other("resphook".to_owned())); 35 | } 36 | resp 37 | } 38 | } 39 | 40 | fn init_logger() { 41 | use std::sync::Once; 42 | static LOGGER_INIT: Once = Once::new(); 43 | LOGGER_INIT.call_once(env_logger::init); 44 | } 45 | 46 | fn init(num_clinet: usize) -> (Network, Vec, Arc) { 47 | init_logger(); 48 | 49 | let mut clients = vec![]; 50 | let rn = Network::new(); 51 | let tso_server_name = "tso_server"; 52 | let mut tso_server_builder = ServerBuilder::new(tso_server_name.to_owned()); 53 | let server_name = "server"; 54 | let mut server_builder = ServerBuilder::new(server_name.to_owned()); 55 | let tso: TimestampOracle = Default::default(); 56 | add_tso_service(tso, &mut tso_server_builder).unwrap(); 57 | let store: MemoryStorage = Default::default(); 58 | add_transaction_service(store, &mut server_builder).unwrap(); 59 | let tso_server = tso_server_builder.build(); 60 | let server = server_builder.build(); 61 | rn.add_server(tso_server); 62 | rn.add_server(server); 63 | let hook = Arc::new(CommitHooks { 64 | drop_req: AtomicBool::new(false), 65 | drop_resp: AtomicBool::new(false), 66 | fail_primary: AtomicBool::new(false), 67 | }); 68 | for i in 0..num_clinet { 69 | let txn_name_string = format!("txn{}", i); 70 | let txn_name = txn_name_string.as_str(); 71 | let cli = rn.create_client(txn_name.to_owned()); 72 | cli.set_hooks(hook.clone()); 73 | let txn_client = TransactionClient::new(cli); 74 | rn.enable(txn_name, true); 75 | rn.connect(txn_name, server_name); 76 | let tso_name_string = format!("tso{}", i); 77 | let tso_name = tso_name_string.as_str(); 78 | let cli = rn.create_client(tso_name.to_owned()); 79 | let tso_client = TSOClient::new(cli); 80 | rn.enable(tso_name, true); 81 | rn.connect(tso_name, tso_server_name); 82 | clients.push(crate::client::Client::new(tso_client, txn_client)); 83 | } 84 | 85 | (rn, clients, hook) 86 | } 87 | 88 | #[test] 89 | fn test_get_timestamp_under_unreliable_network() { 90 | let (rn, clients, _) = init(3); 91 | let mut children = vec![]; 92 | 93 | for (i, _) in clients.iter().enumerate() { 94 | let client = clients[i].to_owned(); 95 | let tso_name_string = format!("tso{}", i); 96 | rn.enable(tso_name_string.as_str(), false); 97 | children.push(thread::spawn(move || { 98 | let res = client.get_timestamp(); 99 | if i == 2 { 100 | assert_eq!(res, Err(Error::Timeout)); 101 | } else { 102 | assert!(res.is_ok()); 103 | } 104 | })); 105 | } 106 | 107 | thread::sleep(Duration::from_millis(150)); 108 | rn.enable("tso0", true); 109 | thread::sleep(Duration::from_millis(300)); 110 | rn.enable("tso1", true); 111 | thread::sleep(Duration::from_millis(600)); 112 | rn.enable("tso2", true); 113 | 114 | for child in children { 115 | child.join().unwrap(); 116 | } 117 | } 118 | 119 | #[test] 120 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#predicate-many-preceders-pmp 121 | fn test_predicate_many_preceders_read_predicates() { 122 | let (_, clients, _) = init(3); 123 | 124 | let mut client0 = clients[0].to_owned(); 125 | client0.begin(); 126 | client0.set(b"1".to_vec(), b"10".to_vec()); 127 | client0.set(b"2".to_vec(), b"20".to_vec()); 128 | assert_eq!(client0.commit(), Ok(true)); 129 | 130 | let mut client1 = clients[1].to_owned(); 131 | client1.begin(); 132 | assert_eq!(client1.get(b"3".to_vec()), Ok(Vec::new())); 133 | 134 | let mut client2 = clients[2].to_owned(); 135 | client2.begin(); 136 | client2.set(b"3".to_vec(), b"30".to_vec()); 137 | assert_eq!(client2.commit(), Ok(true)); 138 | 139 | assert_eq!(client1.get(b"3".to_vec()), Ok(Vec::new())); 140 | } 141 | 142 | #[test] 143 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#predicate-many-preceders-pmp 144 | fn test_predicate_many_preceders_write_predicates() { 145 | let (_, clients, _) = init(3); 146 | 147 | let mut client0 = clients[0].to_owned(); 148 | client0.begin(); 149 | client0.set(b"1".to_vec(), b"10".to_vec()); 150 | client0.set(b"2".to_vec(), b"20".to_vec()); 151 | assert_eq!(client0.commit(), Ok(true)); 152 | 153 | let mut client1 = clients[1].to_owned(); 154 | client1.begin(); 155 | 156 | let mut client2 = clients[2].to_owned(); 157 | client2.begin(); 158 | 159 | client1.set(b"1".to_vec(), b"20".to_vec()); 160 | client1.set(b"2".to_vec(), b"30".to_vec()); 161 | assert_eq!(client1.get(b"2".to_vec()), Ok(b"20".to_vec())); 162 | 163 | client2.set(b"2".to_vec(), b"40".to_vec()); 164 | assert_eq!(client1.commit(), Ok(true)); 165 | assert_eq!(client2.commit(), Ok(false)); 166 | } 167 | 168 | #[test] 169 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#lost-update-p4 170 | fn test_lost_update() { 171 | let (_, clients, _) = init(3); 172 | 173 | let mut client0 = clients[0].to_owned(); 174 | client0.begin(); 175 | client0.set(b"1".to_vec(), b"10".to_vec()); 176 | client0.set(b"2".to_vec(), b"20".to_vec()); 177 | assert_eq!(client0.commit(), Ok(true)); 178 | 179 | let mut client1 = clients[1].to_owned(); 180 | client1.begin(); 181 | 182 | let mut client2 = clients[2].to_owned(); 183 | client2.begin(); 184 | 185 | assert_eq!(client1.get(b"1".to_vec()), Ok(b"10".to_vec())); 186 | assert_eq!(client2.get(b"1".to_vec()), Ok(b"10".to_vec())); 187 | 188 | client1.set(b"1".to_vec(), b"11".to_vec()); 189 | client2.set(b"1".to_vec(), b"11".to_vec()); 190 | assert_eq!(client1.commit(), Ok(true)); 191 | assert_eq!(client2.commit(), Ok(false)); 192 | } 193 | 194 | #[test] 195 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#read-skew-g-single 196 | fn test_read_skew_read_only() { 197 | let (_, clients, _) = init(3); 198 | 199 | let mut client0 = clients[0].to_owned(); 200 | client0.begin(); 201 | client0.set(b"1".to_vec(), b"10".to_vec()); 202 | client0.set(b"2".to_vec(), b"20".to_vec()); 203 | assert_eq!(client0.commit(), Ok(true)); 204 | 205 | let mut client1 = clients[1].to_owned(); 206 | client1.begin(); 207 | 208 | let mut client2 = clients[2].to_owned(); 209 | client2.begin(); 210 | 211 | assert_eq!(client1.get(b"1".to_vec()), Ok(b"10".to_vec())); 212 | assert_eq!(client2.get(b"1".to_vec()), Ok(b"10".to_vec())); 213 | assert_eq!(client2.get(b"2".to_vec()), Ok(b"20".to_vec())); 214 | 215 | client2.set(b"1".to_vec(), b"12".to_vec()); 216 | client2.set(b"2".to_vec(), b"18".to_vec()); 217 | assert_eq!(client2.commit(), Ok(true)); 218 | 219 | assert_eq!(client1.get(b"2".to_vec()), Ok(b"20".to_vec())); 220 | } 221 | 222 | #[test] 223 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#read-skew-g-single 224 | fn test_read_skew_predicate_dependencies() { 225 | let (_, clients, _) = init(3); 226 | 227 | let mut client0 = clients[0].to_owned(); 228 | client0.begin(); 229 | client0.set(b"1".to_vec(), b"10".to_vec()); 230 | client0.set(b"2".to_vec(), b"20".to_vec()); 231 | assert_eq!(client0.commit(), Ok(true)); 232 | 233 | let mut client1 = clients[1].to_owned(); 234 | client1.begin(); 235 | 236 | let mut client2 = clients[2].to_owned(); 237 | client2.begin(); 238 | 239 | assert_eq!(client1.get(b"1".to_vec()), Ok(b"10".to_vec())); 240 | assert_eq!(client1.get(b"2".to_vec()), Ok(b"20".to_vec())); 241 | 242 | client2.set(b"3".to_vec(), b"30".to_vec()); 243 | assert_eq!(client2.commit(), Ok(true)); 244 | 245 | assert_eq!(client1.get(b"3".to_vec()), Ok(Vec::new())); 246 | } 247 | 248 | #[test] 249 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#read-skew-g-single 250 | fn test_read_skew_write_predicate() { 251 | let (_, clients, _) = init(3); 252 | 253 | let mut client0 = clients[0].to_owned(); 254 | client0.begin(); 255 | client0.set(b"1".to_vec(), b"10".to_vec()); 256 | client0.set(b"2".to_vec(), b"20".to_vec()); 257 | assert_eq!(client0.commit(), Ok(true)); 258 | 259 | let mut client1 = clients[1].to_owned(); 260 | client1.begin(); 261 | 262 | let mut client2 = clients[2].to_owned(); 263 | client2.begin(); 264 | 265 | assert_eq!(client1.get(b"1".to_vec()), Ok(b"10".to_vec())); 266 | assert_eq!(client2.get(b"1".to_vec()), Ok(b"10".to_vec())); 267 | assert_eq!(client2.get(b"2".to_vec()), Ok(b"20".to_vec())); 268 | 269 | client2.set(b"1".to_vec(), b"12".to_vec()); 270 | client2.set(b"2".to_vec(), b"18".to_vec()); 271 | assert_eq!(client2.commit(), Ok(true)); 272 | 273 | client1.set(b"2".to_vec(), b"30".to_vec()); 274 | assert_eq!(client1.commit(), Ok(false)); 275 | } 276 | 277 | #[test] 278 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#write-skew-g2-item 279 | fn test_write_skew() { 280 | let (_, clients, _) = init(3); 281 | 282 | let mut client0 = clients[0].to_owned(); 283 | client0.begin(); 284 | client0.set(b"1".to_vec(), b"10".to_vec()); 285 | client0.set(b"2".to_vec(), b"20".to_vec()); 286 | assert_eq!(client0.commit(), Ok(true)); 287 | 288 | let mut client1 = clients[1].to_owned(); 289 | client1.begin(); 290 | 291 | let mut client2 = clients[2].to_owned(); 292 | client2.begin(); 293 | 294 | assert_eq!(client1.get(b"1".to_vec()), Ok(b"10".to_vec())); 295 | assert_eq!(client1.get(b"2".to_vec()), Ok(b"20".to_vec())); 296 | assert_eq!(client2.get(b"1".to_vec()), Ok(b"10".to_vec())); 297 | assert_eq!(client2.get(b"2".to_vec()), Ok(b"20".to_vec())); 298 | 299 | client1.set(b"1".to_vec(), b"11".to_vec()); 300 | client2.set(b"2".to_vec(), b"21".to_vec()); 301 | 302 | assert_eq!(client1.commit(), Ok(true)); 303 | assert_eq!(client2.commit(), Ok(true)); 304 | } 305 | 306 | #[test] 307 | // https://github.com/ept/hermitage/blob/master/sqlserver.md#anti-dependency-cycles-g2 308 | fn test_anti_dependency_cycles() { 309 | let (_, clients, _) = init(4); 310 | 311 | let mut client0 = clients[0].to_owned(); 312 | client0.begin(); 313 | client0.set(b"1".to_vec(), b"10".to_vec()); 314 | client0.set(b"2".to_vec(), b"20".to_vec()); 315 | assert_eq!(client0.commit(), Ok(true)); 316 | 317 | let mut client1 = clients[1].to_owned(); 318 | client1.begin(); 319 | 320 | let mut client2 = clients[2].to_owned(); 321 | client2.begin(); 322 | 323 | client1.set(b"3".to_vec(), b"30".to_vec()); 324 | client2.set(b"4".to_vec(), b"42".to_vec()); 325 | 326 | assert_eq!(client1.commit(), Ok(true)); 327 | assert_eq!(client2.commit(), Ok(true)); 328 | 329 | let mut client3 = clients[3].to_owned(); 330 | client3.begin(); 331 | assert_eq!(client3.get(b"3".to_vec()), Ok(b"30".to_vec())); 332 | assert_eq!(client3.get(b"4".to_vec()), Ok(b"42".to_vec())); 333 | } 334 | 335 | #[test] 336 | fn test_commit_primary_drop_secondary_requests() { 337 | let (_, clients, hook) = init(2); 338 | 339 | let mut client0 = clients[0].to_owned(); 340 | client0.begin(); 341 | client0.set(b"3".to_vec(), b"30".to_vec()); 342 | client0.set(b"4".to_vec(), b"40".to_vec()); 343 | client0.set(b"5".to_vec(), b"50".to_vec()); 344 | hook.drop_req.store(true, Ordering::Relaxed); 345 | assert_eq!(client0.commit(), Ok(true)); 346 | 347 | let mut client1 = clients[1].to_owned(); 348 | client1.begin(); 349 | assert_eq!(client1.get(b"3".to_vec()).unwrap(), b"30"); 350 | assert_eq!(client1.get(b"4".to_vec()).unwrap(), b"40"); 351 | assert_eq!(client1.get(b"5".to_vec()).unwrap(), b"50"); 352 | } 353 | 354 | #[test] 355 | fn test_commit_primary_success() { 356 | let (_, clients, hook) = init(2); 357 | 358 | let mut client0 = clients[0].to_owned(); 359 | client0.begin(); 360 | client0.set(b"3".to_vec(), b"30".to_vec()); 361 | client0.set(b"4".to_vec(), b"40".to_vec()); 362 | client0.set(b"5".to_vec(), b"50".to_vec()); 363 | hook.drop_req.store(true, Ordering::Relaxed); 364 | assert_eq!(client0.commit(), Ok(true)); 365 | 366 | let mut client1 = clients[1].to_owned(); 367 | client1.begin(); 368 | assert_eq!(client1.get(b"3".to_vec()).unwrap(), b"30"); 369 | assert_eq!(client1.get(b"4".to_vec()).unwrap(), b"40"); 370 | assert_eq!(client1.get(b"5".to_vec()).unwrap(), b"50"); 371 | } 372 | 373 | #[test] 374 | fn test_commit_primary_success_without_response() { 375 | let (_, clients, hook) = init(2); 376 | 377 | let mut client0 = clients[0].to_owned(); 378 | client0.begin(); 379 | client0.set(b"3".to_vec(), b"30".to_vec()); 380 | client0.set(b"4".to_vec(), b"40".to_vec()); 381 | client0.set(b"5".to_vec(), b"50".to_vec()); 382 | hook.drop_resp.store(true, Ordering::Relaxed); 383 | assert_eq!(client0.commit(), Err(Error::Other("resphook".to_owned()))); 384 | 385 | let mut client1 = clients[1].to_owned(); 386 | client1.begin(); 387 | assert_eq!(client1.get(b"3".to_vec()).unwrap(), b"30"); 388 | assert_eq!(client1.get(b"4".to_vec()).unwrap(), b"40"); 389 | assert_eq!(client1.get(b"5".to_vec()).unwrap(), b"50"); 390 | } 391 | 392 | #[test] 393 | fn test_commit_primary_fail() { 394 | let (_, clients, hook) = init(2); 395 | 396 | let mut client0 = clients[0].to_owned(); 397 | client0.begin(); 398 | client0.set(b"3".to_vec(), b"30".to_vec()); 399 | client0.set(b"4".to_vec(), b"40".to_vec()); 400 | client0.set(b"5".to_vec(), b"50".to_vec()); 401 | hook.drop_req.store(true, Ordering::Relaxed); 402 | hook.fail_primary.store(true, Ordering::Relaxed); 403 | assert_eq!(client0.commit(), Ok(false)); 404 | 405 | let mut client1 = clients[1].to_owned(); 406 | client1.begin(); 407 | assert_eq!(client1.get(b"3".to_vec()), Ok(Vec::new())); 408 | assert_eq!(client1.get(b"4".to_vec()), Ok(Vec::new())); 409 | assert_eq!(client1.get(b"5".to_vec()), Ok(Vec::new())); 410 | } 411 | -------------------------------------------------------------------------------- /raft/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft" 3 | version = "0.1.0" 4 | authors = [ 5 | "Neil Shen ", 6 | "ShuNing ", 7 | "Connor ", 8 | "庄天翼 ", 9 | "MIT 6.824 <6824-staff@lists.csail.mit.edu>", 10 | ] 11 | edition = "2018" 12 | publish = false 13 | 14 | [dependencies] 15 | async-trait = "0.1" 16 | bincode = "1.3.3" 17 | futures = "0.3.16" 18 | futures-timer = "3.0" 19 | lazy_static = "1.4.0" 20 | log = "0.4" 21 | prost = "0.8" 22 | prost-derive = "0.6" 23 | rand = "0.7" 24 | serde = { version = "1.0.127", features = ["derive"] } 25 | uuid = { version = "0.8.2", features = ["v4", "serde"] } 26 | 27 | labcodec = { path = "../labcodec" } 28 | labrpc = { path = "../labrpc" } 29 | linearizability = { path = "../linearizability" } 30 | 31 | 32 | [dev-dependencies] 33 | env_logger = "0.7" 34 | 35 | [build-dependencies] 36 | prost-build = "0.8" 37 | -------------------------------------------------------------------------------- /raft/README.md: -------------------------------------------------------------------------------- 1 | # The Raft lab 2 | 3 | This is a series of labs on a key/value storage system built with the Raft 4 | consensus algorithm. These labs are derived from the [lab2:raft][6824lab2] and 5 | [lab3:kvraft][6824lab3] from the famous [MIT 6.824][6824] course but rewritten 6 | in Rust. The following text material is also very influenced by the text 7 | material there. 8 | 9 | [6824lab2]:http://nil.csail.mit.edu/6.824/2018/labs/lab-raft.html 10 | [6824lab3]:http://nil.csail.mit.edu/6.824/2018/labs/lab-kvraft.html 11 | [6824]:http://nil.csail.mit.edu/6.824/2018/index.html 12 | 13 | In these labs you will first implement the raft consensus algorithm in the 14 | lab2:raft, and then build a key/value service in the lab3:kvraft. 15 | 16 | Raft is a consensus algorithm that is designed to be easy to understand.You can 17 | read material about the Raft itself at [the Raft site][raftsite], including 18 | [the extended Raft paper][raftpaper], an interactive visualization of the Raft, 19 | and other resource. Those material should be helpful for you to complete this lab. 20 | 21 | [raftsite]:https://raft.github.io/ 22 | 23 | ## Getting started 24 | 25 | First, please clone this repository with `git` to get the source code of the labs. 26 | 27 | Then, make sure you have `rustup` installed. Also, to make things simpler you 28 | should have `make` installed. 29 | 30 | Now you can run `make test_others` to check that things are going right. You 31 | should see all tests passed. 32 | 33 | (If you are a Windows user, you may have to figure out how to use `make` on 34 | Windows, or type the commands from the makefile in the console manually, or just 35 | use the Windows Subsystem for Linux) 36 | 37 | ## The lab2: Raft 38 | 39 | In this lab you will implement the Raft consensus algorithm. This lab has 3 parts 40 | named with 2A, 2B and 2C. 41 | 42 | To run all the test in this lab, run `make test_2`. Please run the tests multiple 43 | times to mak sure your are not passing the tests just by luck. 44 | 45 | To run just a single test, run `make cargo_test_`. 46 | 47 | ### The code structure 48 | 49 | All your codes in this lab should be in the `src/proto/mod.rs`, 50 | `src/proto/raft.proto` and `src/raft/mod.rs`. 51 | 52 | The `src/raft/mod.rs` file should contains your main implementation of Raft. 53 | The tester (and your key/value server in lab3) will call methods in this file 54 | to use your Raft module. 55 | 56 | A service calls `Raft::new` to create a Raft peer, and then calls `Node::new` to 57 | start the Raft peer. The `Node::get_state`, `Node::is_leader` and `Node::term` 58 | will be called to get the current term of the node and whether it thinks it is 59 | leader. 60 | 61 | The `Node::start` will be called when the server need to append the command into 62 | the log. `Node::start` should returns immediately without waiting for the log 63 | appends to complete. A channel (`apply_ch`) is passed into `Raft::new` and you 64 | should send a `ApplyMsg` to the channel for each newly committed log entry. 65 | 66 | Your implements should use the provided `labrpc` crate to exchange RPCs. The 67 | `labrpc` use channels to simulate sockets internally. This makes it easy to test 68 | your code under challenging network conditions. Your definition of the RPCs 69 | should in `src/proto/mod.rs` and you should implement the RPC server in 70 | `impl RaftService for Node`. A set of RPC clients (`peers`) is pass into 71 | `Raft::new` for you to send RPCs to other nodes. 72 | 73 | ### Part 2A 74 | 75 | In this part you should implement the leader election and heartbeats 76 | (`AppendEntries` RPCs without log entries). You should make a single leader to be 77 | elected, make leader to remain leader when there are no failures, and have a new 78 | leader when old leader fails or packets to/from old leader is lost. 79 | 80 | To run all the test in this lab, run `make test_2a`. 81 | 82 | Here are some hints on this part: 83 | 84 | - Add any state you need to the `Raft` struct. 85 | - The `request_vote` RPC is already defined, you just need to fill the 86 | `RequestVoteArgs` and `RequestVoteReply` struct. The lab use `labcodec` crate to 87 | encode and decode messages in RPC, which internally use the `prost` external 88 | crate. See the [prost document][prost] to know how to define structs that is used 89 | as messages with `#[Derive(Message)]` and `#[prost(...)]`. 90 | - You need to define the `append_entries` RPC by yourself. `labrpc` use a 91 | `labrpc::service!` macro to define RPC service and generate server and client 92 | traits from your definition. There is an example in `labrpc/examples/echo.rs` 93 | which may help you to define new RPCs. 94 | - This lab use things from the `futures` external crate heavily like the channels 95 | and the `Future` trait. Read things about futures [here][futures]. 96 | - You need to make your code take actions periodically or after delays in time. 97 | You can use lots of threads and call `std::thread::sleep` ([doc][sleep]), but you 98 | can also use the `futures_timer::Delay` and other utilities from the 99 | `futures-timer` external crate ([doc][futures-timer]). 100 | - Don't forget that you need to make sure that the election timeouts in different 101 | peers don't always fire at the same time, or else all peers will vote only for 102 | themselves and no one will become the leader. You can generate random numbers 103 | using the `rand` external crate ([doc][rand]). 104 | - The tester limits the frequency of RPC calls to 10 times per second for each 105 | pair of sender and receiver. Please do not just send RPCs repeatedly without wait 106 | for a timeout. 107 | - The tester requires your Raft to elect a new leader within five seconds of the 108 | failure of the old leader (if a majority of peers can still communicate). However, 109 | leader election may require multiple rounds in case of a split vote (which can 110 | happen if packets are lost or if candidates unluckily choose the same random 111 | backoff times). You must pick election timeouts (and thus heartbeat intervals) 112 | that are short enough that it's very likely that an election will complete in less 113 | than five seconds even if it requires multiple rounds. 114 | - But also, since the tester limits the frequency of RPC calls, your election 115 | timeout should not be too small, and larger than the 150~300 milliseconds in the 116 | paper's Section 5.2. Choose the values wisely. 117 | - In Rust we lock data instead of code. Think carefully about what data should be 118 | in the same lock. 119 | - The Figure 2 in the [Raft paper][raftpaper] is useful when you are in trouble 120 | passing the test. 121 | - It's useful to debug by printing log messages. We use the external 122 | `log` crate ([doc][log]) to print logs in different levels. you can configure 123 | the log level and scope by set LOG_LEVEL environment variable like 124 | `LOG_LEVEL=labs6824=debug make test_2a`. This feature is provided by the 125 | `env_logger` external crate ([doc][env_logger]), read it's documentation for 126 | syntax of the log level. Also, you can collect the output in a file by redirecting 127 | the output to a file like `make test_2a 2>test.log` 128 | 129 | [prost]:https://github.com/danburkert/prost 130 | [futures]:https://docs.rs/futures/0.3/futures/index.html 131 | [sleep]:https://doc.rust-lang.org/std/thread/fn.sleep.html 132 | [futures-timer]:https://docs.rs/futures-timer/3.0/futures_timer/index.html 133 | [rand]:https://docs.rs/rand/0.7/rand/index.html 134 | [log]:https://docs.rs/log/0.4/log/ 135 | [env_logger]:https://docs.rs/env_logger/0.7/env_logger/ 136 | 137 | ### Part 2B 138 | 139 | In this part you should implement the log replication. You should implement the 140 | `Node::Start` method, complete the rest fields in the `append_entries` RPC and 141 | send them, and advance `commit_index` at leader. 142 | 143 | To run all the test in this lab, run `make test_2b`. You can try to pass the 144 | `test_basic_agree_2b` test first. 145 | 146 | Here are some hints on this part: 147 | 148 | - Don't forget election restriction, see section 5.4.1 in the 149 | [Raft paper][raftpaper]. 150 | - Every server commit new entries independently by write to `apply_ch` in the 151 | correct order. `apply_ch` is a `UnboundedSender` which will buffering the messages 152 | until out of memory, so it is not that easy to create deadlocks as the original 153 | go version. 154 | - Give yourself enough time to rewrite your implementation because only after 155 | writing a first implementation will you realize how to organize your code cleanly. 156 | - The `test_count_2b` requires your number of RPCs to be not too much when there 157 | is no failure. So you should optimize the number of RPCs to the minimum. 158 | - You may need to write code that waits for certain events to occur. In that case 159 | you can just use channels and wait on it. 160 | 161 | ### Part 2C 162 | 163 | In this part you should implement persistence by first adding code that saves and 164 | restores persistent state to `Persister`, like in `Raft::persist` and 165 | `Raft::restore` using `labcodec`. You also need to determine what and when to 166 | persist, and call `Raft::restore` in `Raft::new`. 167 | 168 | To run all the test in this lab, run `make test_2c`. You can try to pass the 169 | `test_persist1_2c` test first. 170 | 171 | Here are some hints on this part: 172 | - The usage of `labcodec` is covered in the hints of part 2A. 173 | - This part also introduce various challenging test that involving servers failing 174 | and the network losing RPC requests or replies. Check your implementation 175 | carefully to find bugs that only present in this situation. 176 | - In order to pass some of the challenging tests towards the end, such as those 177 | marked "unreliable", you will need to implement the optimization to allow a 178 | follower to back up the leader's nextIndex by more than one entry at a time. See 179 | the description in the [Raft paper][raftpaper] starting at the bottom of page 7 180 | and top of page 8 (marked by a gray line). However, the paper do not have many 181 | details on this. You will need to fill in the gaps, perhaps with the help of 182 | [this 6.824 Raft lectures][optimize-hint]. 183 | 184 | [optimize-hint]:http://nil.csail.mit.edu/6.824/2018/notes/l-raft2.txt 185 | 186 | ## The lab3: KvRaft 187 | 188 | In this lab you will build a fault-tolerant key-value storage service using the 189 | Raft module in lab 2. This lab has 2 parts named with 3A and 3B. 190 | 191 | To run all the test in this lab, run `make test_3`. Please run the tests multiple 192 | times to mak sure your are not passing the tests just by luck. 193 | 194 | To run just a single test, run `make cargo_test_`. 195 | 196 | ### The code structure 197 | 198 | All your codes in this lab should be in the `src/proto/mod.rs`, `src/proto/kvraft.proto`, 199 | `src/kvraft/server.rs` and `src/kvraft/client.rs`. The file name explains what 200 | they are. Also, you need to modify the files you touched in lab2:raft. 201 | 202 | ### Part 3A 203 | 204 | In this part you should first implement a solution that works when there are no 205 | dropped messages, and no failed servers. Your service must ensure that `get(...)` 206 | and `put_append(...)` are [linearizable][linearizable]. 207 | 208 | [linearizable]:https://en.wikipedia.org/wiki/Linearizability 209 | 210 | That means, completed application calls to the methods on the `Clerk` struct in 211 | `src/kvraft/client.rs` must appear to all clients to have affected the service in 212 | the same linear order, even in there are failures and leader changes. A 213 | `Clerk::Get` that starts after a completed `Clerk::Put` or `Clerk::Append` should 214 | see the value written by the most recent `Clerk::Put` or `Clerk::Append` in the 215 | linear order. Completed calls should have exactly-once semantics. 216 | 217 | A reasonable plan of implementation should be: 218 | 219 | - Client send RPC request in the `src/kvraft/client.rs` 220 | - Enter the operation from client into the Raft log by `raft::Node::start` in the 221 | RPC handler of `KvServer` 222 | - Execute the operation when the raft log committed, and then reply to RPC 223 | 224 | After implement that you should pass the basic one client test, run 225 | `make cargo_test_basic_3a` to check it. 226 | 227 | Here are some hints on this part: 228 | 229 | - You should receive commit message from Raft by `apply_ch` at the same time you 230 | receives RPC. 231 | - If a leader has called `Raft::start` for a Clerk's RPC, but loses its leadership 232 | before the request is committed to the log, you should make the client to re-send 233 | the RPC request to other servers until it finds the new leader. You can detect 234 | this by check things received from `apply_ch`. 235 | - The `Clerk` client should remember who is the last leader, and try the last 236 | leader first. This will avoid wasting time searching for the leader on every RPC. 237 | - The server should not complete a `get` RPC if it is not part of a majority and 238 | do not has up-to-date data. You can just put the get operation into the log, or 239 | implement the optimization for read-only operations that is described in Section 8 240 | in the [Raft paper][raftpaper]. 241 | - Think how to use lock at the beginning. 242 | 243 | Then, you should deal with duplicate client requests, including situations where 244 | the client sends a request to a server leader in one term, times out waiting for a 245 | reply, and re-sends the request to a new leader in another term. The request 246 | should always execute just once. 247 | 248 | After this you should pass all tests in this part. To run all the test in 249 | this lab, run `make test_3a`. 250 | 251 | Here are some hints on this part: 252 | 253 | - You will need to uniquely identify client operations to ensure that the key/ 254 | value service executes each one just once. 255 | - Your scheme for duplicate detection should free server memory quickly, like 256 | using a hashtable for and only for uncommitted logs. 257 | 258 | ### Part 3B 259 | 260 | In your current implementation, a rebooting server replays the complete Raft log 261 | in order to restore its state. However, it's not practical for a long-running 262 | server to remember the complete Raft log forever. 263 | 264 | Instead, you'll modify Raft and kvserver to cooperate to save space: from time to 265 | time kvserver will persistently store a "snapshot" of its current state, and Raft 266 | will discard log entries that precede the snapshot. When a server restarts (or 267 | falls far behind the leader and must catch up), the server first installs a 268 | snapshot and then replays log entries from after the point at which the snapshot 269 | was created. Section 7 of the [Raft paper][raftpaper] outlines the scheme; you 270 | will have to design the details. 271 | 272 | The tester passes a `maxraftstate` to the `KvServer::new` indicating the maximum 273 | allowed size of your persistent Raft state in bytes (including the log, but not 274 | including snapshots). You should check the size of Raft state and when Raft state 275 | size is approaching this threshold, it should save a snapshot, and tell the Raft 276 | library that it has snapshotted, so that Raft can discard old log entries. 277 | The `maxraftstate` is a `Option` and you do not have to snapshot when it 278 | is `None`. 279 | 280 | First you should modify the Raft implement to accept a compaction request and 281 | discard entries before the given index, and continue operating while storing only 282 | log entries after that index. The tests from lab2:raft should still pass. 283 | 284 | Then you should modify the kvserver so that it can hands snapshots to Raft and 285 | request compaction when Raft state grows too large. The snapshots should be saved 286 | in `raft::Persister` 287 | 288 | Here are some hints on this part: 289 | 290 | - You are allowed to add methods to your Raft so that kvserver can manage the 291 | process of trimming the Raft log and manage kvserver snapshots. 292 | - You can test your Raft and kvserver's ability to operate with a trimmed log, 293 | and its ability to re-start from the combination of a kvserver snapshot and 294 | persisted Raft state, by running the Lab 3A tests while overriding `maxraftstate` 295 | to `Some(1)`. 296 | - Think about what should be in the snapshot. You should save new snapshot and 297 | restore latest snapshot with `raft::Persister`. 298 | - Uncommitted logs can also in snapshots, so your kvserver must still be able to 299 | detect duplicated operations under this situation. 300 | 301 | After that, you should define the `install_snapshot` RPC and the leader should 302 | send this RPC when the leader has discarded the log entries the follower needs. 303 | When a follower receives an `install_snapshot` RPC, it should send the snapshot to 304 | kvserver (maybe in `apply_ch`). 305 | 306 | After this you should pass all tests in this part. To run all the test in 307 | this lab, run `make test_3b`. 308 | 309 | Here are some hints on this part: 310 | 311 | - You do not need to send the snapshots in multiple RPCs. Just send the entire 312 | snapshot in a single `install_snapshot` RPC and that should be enough for this lab. 313 | - You can try `test_snapshot_rpc_3b` first 314 | 315 | [raftpaper]:https://raft.github.io/raft.pdf 316 | -------------------------------------------------------------------------------- /raft/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let includes = &[std::path::PathBuf::from("src/proto")]; 3 | let mut protos = Vec::new(); 4 | for include in includes { 5 | for file in std::fs::read_dir(include).unwrap() { 6 | let file = file.unwrap(); 7 | if file.file_type().unwrap().is_dir() { 8 | continue; 9 | } 10 | let path = file.path(); 11 | if path.extension().unwrap() == "proto" { 12 | protos.push(path); 13 | } 14 | } 15 | } 16 | prost_build::Config::new() 17 | .type_attribute( 18 | "raftpb.Entry", 19 | "#[derive(::serde::Serialize, ::serde::Deserialize)]", 20 | ) 21 | .compile_protos(&protos, includes) 22 | .unwrap(); 23 | for p in protos { 24 | println!("cargo:rerun-if-changed={}", p.display()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /raft/src/kvraft/client.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt, 3 | sync::atomic::{AtomicU64, AtomicUsize, Ordering}, 4 | time::Duration, 5 | }; 6 | 7 | use futures::{ 8 | executor::block_on, 9 | future::{select, Either}, 10 | }; 11 | use uuid::Uuid; 12 | 13 | use crate::proto::kvraftpb::{self, *}; 14 | 15 | /// Macro for logging message combined with state and command of the kv clerk. 16 | macro_rules! clog { 17 | (level: $level:ident, $cl:expr, $args:expr, $($arg:tt)+) => { 18 | ::log::$level!("CL [{}] {} [while {:?}]", $cl.name, format_args!($($arg)+), $args) 19 | }; 20 | ($cl:expr, $args:expr, $($arg:tt)+) => { 21 | clog!(level: info, $cl, $args, $($arg)+) 22 | }; 23 | } 24 | 25 | /// A key-value clerk. 26 | pub struct Clerk { 27 | pub name: String, 28 | /// All accessible kv servers. 29 | pub servers: Vec, 30 | // You will have to modify this struct. 31 | id: String, 32 | /// Sequence number for next request. 33 | next_seq: AtomicU64, 34 | /// Index of the last remembered leader in `servers`. 35 | last_leader: AtomicUsize, 36 | } 37 | 38 | impl fmt::Debug for Clerk { 39 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 40 | f.debug_struct("Clerk").field("name", &self.name).finish() 41 | } 42 | } 43 | 44 | impl Clerk { 45 | pub fn new(name: String, servers: Vec) -> Clerk { 46 | // You'll have to add code here. 47 | let id = format!("{}-{:x}", name, Uuid::new_v4().as_fields().0); 48 | 49 | Clerk { 50 | name, 51 | servers, 52 | id, 53 | next_seq: 1.into(), 54 | last_leader: 0.into(), 55 | } 56 | } 57 | 58 | /// Get the next sequence number and increase it. 59 | fn next_seq(&self) -> u64 { 60 | self.next_seq.fetch_add(1, Ordering::SeqCst) 61 | } 62 | 63 | /// An infinite iterator of all servers, start from `last_leader`. 64 | fn cycle_servers(&self) -> impl Iterator { 65 | self.servers 66 | .iter() 67 | .enumerate() 68 | .cycle() 69 | .skip(self.last_leader.load(Ordering::Relaxed)) 70 | } 71 | 72 | /// Call servers for given `args` repeatedly until success. 73 | async fn call(&self, args: KvRequest) -> String { 74 | let mut iter = self.cycle_servers(); 75 | 76 | 'server: loop { 77 | // select next server 78 | let (i, server) = iter.next().unwrap(); 79 | 80 | 'retry: loop { 81 | clog!(level: debug, self, args, "request to #{}", i); 82 | let request = server.op(&args); 83 | let timeout = futures_timer::Delay::new(Duration::from_millis(1000)); 84 | match select(request, timeout).await { 85 | Either::Left((Ok(reply), _)) => { 86 | if reply.wrong_leader { 87 | continue 'server; 88 | } else { 89 | self.last_leader.store(i, Ordering::Relaxed); 90 | if !reply.err.is_empty() { 91 | clog!(level: warn, self, args, "retry: {}", reply.err); 92 | continue 'retry; 93 | } else { 94 | clog!(self, args, "successful request to #{}", i); 95 | break 'server reply.value; 96 | } 97 | } 98 | } 99 | Either::Left((Err(e), _)) => { 100 | clog!(level: warn, self, args, "try next server: {}", e); 101 | continue 'server; 102 | } 103 | Either::Right((_, _)) => { 104 | clog!(level: warn, self, args, "timeout"); 105 | continue 'server; 106 | } 107 | } 108 | } 109 | } 110 | } 111 | 112 | /// Async version of `get` request. 113 | /// 114 | /// Note that we cannot call `block_on` if we already entered a executor. 115 | /// Thus, this method should be called in any async context. 116 | pub async fn get_async(&self, key: String) -> String { 117 | let args = KvRequest { 118 | key, 119 | op: kvraftpb::Op::Get as i32, 120 | cid: self.id.clone(), 121 | seq: self.next_seq(), 122 | ..Default::default() 123 | }; 124 | self.call(args).await 125 | } 126 | 127 | /// fetch the current value for a key. 128 | /// returns "" if the key does not exist. 129 | /// keeps trying forever in the face of all other errors. 130 | // 131 | // you can send an RPC with code like this: 132 | // if let Some(reply) = self.servers[i].get(args).wait() { /* do something */ } 133 | pub fn get(&self, key: String) -> String { 134 | // You will have to modify this function. 135 | block_on(self.get_async(key)) 136 | } 137 | 138 | /// Async version of `put` request. 139 | /// 140 | /// Note that we cannot call `block_on` if we already entered a executor. 141 | /// Thus, this method should be called in any async context. 142 | pub async fn put_async(&self, key: String, value: String) { 143 | let args = KvRequest { 144 | key, 145 | value, 146 | op: kvraftpb::Op::Put as i32, 147 | cid: self.id.clone(), 148 | seq: self.next_seq(), 149 | }; 150 | self.call(args).await; 151 | } 152 | 153 | /// Put a key-value pair into kv servers. 154 | pub fn put(&self, key: String, value: String) { 155 | block_on(self.put_async(key, value)) 156 | } 157 | 158 | /// Async version of `append` request. 159 | /// 160 | /// Note that we cannot call `block_on` if we already entered a executor. 161 | /// Thus, this method should be called in any async context. 162 | pub async fn append_async(&self, key: String, value: String) { 163 | let args = KvRequest { 164 | key, 165 | value, 166 | op: kvraftpb::Op::Append as i32, 167 | cid: self.id.clone(), 168 | seq: self.next_seq(), 169 | }; 170 | self.call(args).await; 171 | } 172 | 173 | /// Append the `value` to the entry for `key`. 174 | /// Will do insertion if the entry not exists. 175 | pub fn append(&self, key: String, value: String) { 176 | block_on(self.append_async(key, value)) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /raft/src/kvraft/config.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::atomic::{AtomicUsize, Ordering}; 3 | use std::sync::{Arc, Mutex}; 4 | use std::time::{Duration, Instant}; 5 | 6 | use rand::seq::SliceRandom; 7 | 8 | use crate::kvraft::errors::{Error, Result}; 9 | use crate::kvraft::{client, server}; 10 | use crate::proto::kvraftpb::*; 11 | use crate::proto::raftpb::*; 12 | use crate::raft; 13 | use crate::raft::persister::*; 14 | 15 | static ID: AtomicUsize = AtomicUsize::new(300_000); 16 | 17 | fn uniqstring() -> String { 18 | format!("{}", ID.fetch_add(1, Ordering::Relaxed)) 19 | } 20 | 21 | struct Servers { 22 | kvservers: Vec>, 23 | saved: Vec>, 24 | endnames: Vec>, 25 | } 26 | 27 | fn init_logger() { 28 | use std::sync::Once; 29 | static LOGGER_INIT: Once = Once::new(); 30 | LOGGER_INIT.call_once(env_logger::init); 31 | } 32 | 33 | pub struct Config { 34 | pub net: labrpc::Network, 35 | pub n: usize, 36 | servers: Mutex, 37 | clerks: Mutex>>, 38 | next_client_id: AtomicUsize, 39 | maxraftstate: Option, 40 | 41 | // time at which the Config was created. 42 | start: Instant, 43 | 44 | // begin()/end() statistics 45 | // time at which test_test.go called cfg.begin() 46 | t0: Mutex, 47 | // rpc_total() at start of test 48 | rpcs0: AtomicUsize, 49 | // number of agreements 50 | ops: AtomicUsize, 51 | } 52 | 53 | impl Config { 54 | pub fn new(n: usize, unreliable: bool, maxraftstate: Option) -> Config { 55 | init_logger(); 56 | 57 | let servers = Servers { 58 | kvservers: vec![None; n], 59 | saved: (0..n).map(|_| Arc::new(SimplePersister::new())).collect(), 60 | endnames: vec![vec![String::new(); n]; n], 61 | }; 62 | let cfg = Config { 63 | n, 64 | net: labrpc::Network::new(), 65 | servers: Mutex::new(servers), 66 | clerks: Mutex::new(HashMap::new()), 67 | // client ids start 1000 above the highest serverid, 68 | next_client_id: AtomicUsize::new(n + 1000), 69 | maxraftstate, 70 | start: Instant::now(), 71 | t0: Mutex::new(Instant::now()), 72 | rpcs0: AtomicUsize::new(0), 73 | ops: AtomicUsize::new(0), 74 | }; 75 | 76 | // create a full set of KV servers. 77 | for i in 0..cfg.n { 78 | cfg.start_server(i); 79 | } 80 | 81 | cfg.connect_all(); 82 | 83 | cfg.net.set_reliable(!unreliable); 84 | 85 | cfg 86 | } 87 | 88 | pub fn op(&self) { 89 | self.ops.fetch_add(1, Ordering::Relaxed); 90 | } 91 | 92 | fn rpc_total(&self) -> usize { 93 | self.net.total_count() 94 | } 95 | 96 | pub fn check_timeout(&self) { 97 | // enforce a two minute real-time limit on each test 98 | if self.start.elapsed() > Duration::from_secs(120) { 99 | panic!("test took longer than 120 seconds"); 100 | } 101 | } 102 | 103 | /// Maximum log size across all servers 104 | pub fn log_size(&self) -> usize { 105 | let servers = self.servers.lock().unwrap(); 106 | let mut logsize = 0; 107 | for save in &servers.saved { 108 | let n = save.raft_state().len(); 109 | if n > logsize { 110 | logsize = n; 111 | } 112 | } 113 | logsize 114 | } 115 | 116 | /// Maximum snapshot size across all servers 117 | pub fn snapshot_size(&self) -> usize { 118 | let mut snapshotsize = 0; 119 | let servers = self.servers.lock().unwrap(); 120 | for save in &servers.saved { 121 | let n = save.snapshot().len(); 122 | if n > snapshotsize { 123 | snapshotsize = n; 124 | } 125 | } 126 | snapshotsize 127 | } 128 | 129 | /// Attach server i to servers listed in to 130 | fn connect(&self, i: usize, to: &[usize], servers: &Servers) { 131 | debug!("connect peer {} to {:?}", i, to); 132 | // outgoing socket files 133 | for j in to { 134 | let endname = &servers.endnames[i][*j]; 135 | self.net.enable(endname, true); 136 | } 137 | 138 | // incoming socket files 139 | for j in to { 140 | let endname = &servers.endnames[*j][i]; 141 | self.net.enable(endname, true); 142 | } 143 | } 144 | 145 | /// Detach server i from the servers listed in from 146 | fn disconnect(&self, i: usize, from: &[usize], servers: &Servers) { 147 | debug!("disconnect peer {} from {:?}", i, from); 148 | // outgoing socket files 149 | for j in from { 150 | if !servers.endnames[i].is_empty() { 151 | let endname = &servers.endnames[i][*j]; 152 | self.net.enable(endname, false); 153 | } 154 | } 155 | 156 | // incoming socket files 157 | for j in from { 158 | if !servers.endnames[*j].is_empty() { 159 | let endname = &servers.endnames[*j][i]; 160 | self.net.enable(endname, false); 161 | } 162 | } 163 | } 164 | 165 | pub fn all(&self) -> Vec { 166 | (0..self.n).collect() 167 | } 168 | 169 | pub fn connect_all(&self) { 170 | let servers = self.servers.lock().unwrap(); 171 | for i in 0..self.n { 172 | self.connect(i, &self.all(), &*servers); 173 | } 174 | } 175 | 176 | /// Sets up 2 partitions with connectivity between servers in each partition. 177 | pub fn partition(&self, p1: &[usize], p2: &[usize]) { 178 | debug!("partition servers into: {:?} {:?}", p1, p2); 179 | let servers = self.servers.lock().unwrap(); 180 | for i in p1 { 181 | self.disconnect(*i, p2, &*servers); 182 | self.connect(*i, p1, &*servers); 183 | } 184 | for i in p2 { 185 | self.disconnect(*i, p1, &*servers); 186 | self.connect(*i, p2, &*servers); 187 | } 188 | } 189 | 190 | // Create a clerk with clerk specific server names. 191 | // Give it connections to all of the servers, but for 192 | // now enable only connections to servers in to[]. 193 | pub fn make_client(&self, to: &[usize]) -> client::Clerk { 194 | // a fresh set of ClientEnds. 195 | let mut ends = Vec::with_capacity(self.n); 196 | let mut endnames = Vec::with_capacity(self.n); 197 | for j in 0..self.n { 198 | let name = uniqstring(); 199 | endnames.push(name.clone()); 200 | let cli = self.net.create_client(name.clone()); 201 | ends.push(KvClient::new(cli)); 202 | self.net.connect(&name, &format!("{}", j)); 203 | } 204 | 205 | ends.shuffle(&mut rand::thread_rng()); 206 | let ck_name = uniqstring(); 207 | let ck = client::Clerk::new(ck_name.clone(), ends); 208 | self.clerks.lock().unwrap().insert(ck_name, endnames); 209 | self.next_client_id.fetch_add(1, Ordering::Relaxed); 210 | self.connect_client(&ck, to); 211 | ck 212 | } 213 | 214 | pub fn delete_client(&self, ck: &client::Clerk) { 215 | self.clerks.lock().unwrap().remove(&ck.name); 216 | } 217 | 218 | pub fn connect_client(&self, ck: &client::Clerk, to: &[usize]) { 219 | self.connect_client_by_name(&ck.name, to); 220 | } 221 | 222 | pub fn connect_client_by_name(&self, ck_name: &str, to: &[usize]) { 223 | debug!("connect_client {:?} to {:?}", ck_name, to); 224 | let clerks = self.clerks.lock().unwrap(); 225 | let endnames = &clerks[ck_name]; 226 | for j in to { 227 | let s = &endnames[*j]; 228 | self.net.enable(s, true); 229 | } 230 | } 231 | 232 | /// Shutdown a server by isolating it 233 | pub fn shutdown_server(&self, i: usize) { 234 | let mut servers = self.servers.lock().unwrap(); 235 | self.disconnect(i, &self.all(), &*servers); 236 | 237 | // disable client connections to the server. 238 | // it's important to do this before creating 239 | // the new Persister in saved[i], to avoid 240 | // the possibility of the server returning a 241 | // positive reply to an Append but persisting 242 | // the result in the superseded Persister. 243 | self.net.delete_server(&format!("{}", i)); 244 | 245 | // a fresh persister, in case old instance 246 | // continues to update the Persister. 247 | // but copy old persister's content so that we always 248 | // pass Make() the last persisted state. 249 | let p = raft::persister::SimplePersister::new(); 250 | p.save_state_and_snapshot(servers.saved[i].raft_state(), servers.saved[i].snapshot()); 251 | servers.saved[i] = Arc::new(p); 252 | 253 | if let Some(kv) = servers.kvservers[i].take() { 254 | kv.kill(); 255 | } 256 | } 257 | 258 | /// Start a server i. 259 | /// If restart servers, first call shutdown_server 260 | pub fn start_server(&self, i: usize) { 261 | // a fresh set of outgoing ClientEnd names. 262 | let mut servers = self.servers.lock().unwrap(); 263 | servers.endnames[i] = (0..self.n).map(|_| uniqstring()).collect(); 264 | 265 | // a fresh set of ClientEnds. 266 | let mut ends = Vec::with_capacity(self.n); 267 | for (j, name) in servers.endnames[i].iter().enumerate() { 268 | let cli = self.net.create_client(name.clone()); 269 | ends.push(RaftClient::new(cli)); 270 | self.net.connect(name, &format!("{}", j)); 271 | } 272 | 273 | // a fresh persister, so old instance doesn't overwrite 274 | // new instance's persisted state. 275 | // give the fresh persister a copy of the old persister's 276 | // state, so that the spec is that we pass StartKVServer() 277 | // the last persisted state. 278 | let sp = raft::persister::SimplePersister::new(); 279 | sp.save_state_and_snapshot(servers.saved[i].raft_state(), servers.saved[i].snapshot()); 280 | let p = Arc::new(sp); 281 | servers.saved[i] = p.clone(); 282 | 283 | let (kv, apply_rx) = server::KvServer::new(ends, i, Box::new(p), self.maxraftstate); 284 | let rf_node = kv.raft.clone(); 285 | let kv_node = server::Node::new(kv, apply_rx); 286 | servers.kvservers[i] = Some(kv_node.clone()); 287 | 288 | let mut builder = labrpc::ServerBuilder::new(format!("{}", i)); 289 | add_raft_service(rf_node, &mut builder).unwrap(); 290 | add_kv_service(kv_node, &mut builder).unwrap(); 291 | let srv = builder.build(); 292 | self.net.add_server(srv); 293 | } 294 | 295 | pub fn leader(&self) -> Result { 296 | let servers = self.servers.lock().unwrap(); 297 | for (i, kv) in servers.kvservers.iter().enumerate() { 298 | if let Some(kv) = kv { 299 | if kv.is_leader() { 300 | return Ok(i); 301 | } 302 | } 303 | } 304 | Err(Error::NoLeader) 305 | } 306 | 307 | /// Partition servers into 2 groups and put current leader in minority 308 | pub fn make_partition(&self) -> (Vec, Vec) { 309 | let l = self.leader().unwrap_or(0); 310 | let mut p1 = Vec::with_capacity(self.n / 2 + 1); 311 | let mut p2 = Vec::with_capacity(self.n / 2); 312 | for i in 0..self.n { 313 | if i != l { 314 | if p1.len() < self.n / 2 + 1 { 315 | p1.push(i); 316 | } else { 317 | p2.push(i); 318 | } 319 | } 320 | } 321 | p2.push(l); 322 | (p1, p2) 323 | } 324 | 325 | /// Start a Test. 326 | /// print the Test message. 327 | /// e.g. cfg.begin("Test (2B): RPC counts aren't too high") 328 | pub fn begin(&self, description: &str) { 329 | println!(); // Force the log starts at a new line. 330 | info!("{} ...", description); 331 | *self.t0.lock().unwrap() = Instant::now(); 332 | self.rpcs0.store(self.rpc_total(), Ordering::Relaxed); 333 | self.ops.store(0, Ordering::Relaxed); 334 | } 335 | 336 | /// End a Test -- the fact that we got here means there 337 | /// was no failure. 338 | /// print the Passed message, 339 | /// and some performance numbers. 340 | pub fn end(&self) { 341 | self.check_timeout(); 342 | 343 | // real time 344 | let t = self.t0.lock().unwrap().elapsed(); 345 | // number of Raft peers 346 | let npeers = self.n; 347 | // number of RPC sends 348 | let nrpc = self.rpc_total() - self.rpcs0.load(Ordering::Relaxed); 349 | // number of clerk get/put/append calls 350 | let nops = self.ops.load(Ordering::Relaxed); 351 | 352 | info!(" ... Passed --"); 353 | info!(" {:?} {} {} {}", t, npeers, nrpc, nops); 354 | } 355 | } 356 | 357 | impl Drop for Config { 358 | fn drop(&mut self) { 359 | let servers = self.servers.lock().unwrap(); 360 | for s in servers.kvservers.iter().flatten() { 361 | s.kill(); 362 | } 363 | } 364 | } 365 | -------------------------------------------------------------------------------- /raft/src/kvraft/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | #[derive(Clone, Debug, PartialEq, Eq)] 4 | pub enum Error { 5 | Raft(crate::raft::errors::Error), 6 | NoLeader, 7 | } 8 | 9 | impl fmt::Display for Error { 10 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 11 | write!(f, "{:?}", self) 12 | } 13 | } 14 | 15 | impl error::Error for Error { 16 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 17 | match self { 18 | Error::Raft(e) => e.source(), 19 | _ => None, 20 | } 21 | } 22 | } 23 | 24 | pub type Result = result::Result; 25 | -------------------------------------------------------------------------------- /raft/src/kvraft/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | #[cfg(test)] 3 | pub mod config; 4 | pub mod errors; 5 | pub mod server; 6 | #[cfg(test)] 7 | mod tests; 8 | -------------------------------------------------------------------------------- /raft/src/kvraft/server.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::{Arc, RwLock}; 3 | 4 | use super::errors::{Error, Result}; 5 | use crate::proto::kvraftpb::*; 6 | use crate::raft::{self, ApplyMsg}; 7 | use crate::Executor; 8 | use futures::channel::mpsc::{unbounded, UnboundedReceiver}; 9 | use futures::channel::oneshot; 10 | use futures::task::SpawnExt; 11 | use futures::StreamExt; 12 | use serde::{Deserialize, Serialize}; 13 | 14 | /// Macro for logging message combined with state of the kv server. 15 | macro_rules! kvlog { 16 | (level: $level:ident, $kv:expr, $($arg:tt)+) => {{ 17 | let leader_desc = $kv.raft.is_leader().then(|| "Leader").unwrap_or("Non-leader"); 18 | ::log::$level!("KV [#{} @{} as {}] {}", $kv.me, $kv.raft.term(), leader_desc, format_args!($($arg)+)) 19 | }}; 20 | ($kv:expr, $($arg:tt)+) => { 21 | kvlog!(level: info, $kv, $($arg)+) 22 | }; 23 | } 24 | 25 | /// Reply for notifier. 26 | #[derive(Debug, Clone)] 27 | enum NotifyReply { 28 | Get { value: String }, 29 | PutAppend, 30 | } 31 | 32 | /// Notifier for applied requests. 33 | struct Notifier { 34 | term: u64, 35 | sender: oneshot::Sender, 36 | } 37 | 38 | type Command = KvRequest; 39 | type CommandIndex = u64; 40 | type ClerkId = String; 41 | 42 | /// State of a key-value server that should be included in the snapshots. 43 | #[derive(Serialize, Deserialize, Default)] 44 | struct PersistentState { 45 | /// Simple hash-based kv storage. 46 | store: HashMap, 47 | /// Sequence number of the latest applied request for each clerk. 48 | max_seqs: HashMap, 49 | } 50 | 51 | impl PersistentState { 52 | fn new() -> Self { 53 | Default::default() 54 | } 55 | } 56 | 57 | /// A key-value server. 58 | pub struct KvServer { 59 | /// Underlying Raft node. 60 | pub raft: raft::Node, 61 | /// Same as `raft.me`. 62 | me: usize, 63 | /// snapshot if log grows this big 64 | max_raft_state: Option, 65 | /// States that will be persisted into snapshots. 66 | p: PersistentState, 67 | /// Notifiers for each under-processing request. 68 | notifiers: HashMap, 69 | } 70 | 71 | impl KvServer { 72 | /// Create an instance of `KvServer`, 73 | /// returns the instance and the receiver of applies commands. 74 | pub fn new( 75 | servers: Vec, 76 | me: usize, 77 | persister: Box, 78 | max_raft_state: Option, 79 | ) -> (KvServer, UnboundedReceiver) { 80 | // You may need initialization code here. 81 | 82 | // restore persistent state from snapshot 83 | let p = 84 | bincode::deserialize(&persister.snapshot()).unwrap_or_else(|_| PersistentState::new()); 85 | 86 | let (apply_tx, apply_rx) = unbounded(); 87 | let raft_inner = raft::Raft::new(servers, me, persister, apply_tx); 88 | let raft = raft::Node::new(raft_inner); 89 | 90 | let kv = KvServer { 91 | raft, 92 | me, 93 | max_raft_state, 94 | p, 95 | notifiers: HashMap::new(), 96 | }; 97 | 98 | (kv, apply_rx) 99 | } 100 | } 101 | 102 | impl KvServer { 103 | /// Send and start a command to Raft layer. 104 | fn start(&mut self, command: Command) -> Result> { 105 | let (tx, rx) = oneshot::channel(); 106 | let (index, term) = self.raft.start(&command).map_err(Error::Raft)?; 107 | 108 | let new = self 109 | .notifiers 110 | .insert(index, Notifier { term, sender: tx }) 111 | .is_none(); 112 | assert!(new); 113 | 114 | Ok(rx) 115 | } 116 | 117 | /// Raft wants to apply a new command (request) or snapshot. 118 | fn apply(&mut self, msg: ApplyMsg) { 119 | match msg { 120 | ApplyMsg::Command { index, command } => { 121 | let req: KvRequest = labcodec::decode(&command).unwrap(); 122 | kvlog!(self, "apply command: {:?}", req); 123 | 124 | let to_apply = { 125 | let max_seq = self.p.max_seqs.entry(req.cid.clone()).or_insert(0); 126 | // only apply new requests, that is, with larger seq number 127 | if req.seq > *max_seq { 128 | *max_seq = req.seq; 129 | true 130 | } else { 131 | false 132 | } 133 | }; 134 | 135 | let reply = match req.op() { 136 | Op::Put => { 137 | if to_apply { 138 | kvlog!(self, "Put: {:?}", req); 139 | self.p.store.insert(req.key, req.value); 140 | } 141 | NotifyReply::PutAppend 142 | } 143 | Op::Append => { 144 | if to_apply { 145 | self.p 146 | .store 147 | .entry(req.key.clone()) 148 | .or_default() 149 | .push_str(&req.value); 150 | let value = self.p.store.get(&req.key).cloned().unwrap_or_default(); 151 | kvlog!(self, "After Append: {:?} => {:?}", req, value); 152 | } 153 | NotifyReply::PutAppend 154 | } 155 | Op::Get => { 156 | let value = self.p.store.get(&req.key).cloned().unwrap_or_default(); 157 | kvlog!(self, "Get: {:?} => {:?}", req, value); 158 | NotifyReply::Get { value } 159 | } 160 | Op::Unknown => unreachable!(), 161 | }; 162 | 163 | if let Some(Notifier { term, sender }) = self.notifiers.remove(&index) { 164 | if self.raft.is_leader() && term == self.raft.term() { 165 | match sender.send(reply) { 166 | Ok(_) => kvlog!(self, "send notification"), 167 | Err(_) => { 168 | kvlog!(level: error, self, "send notification error") 169 | } 170 | } 171 | } else { 172 | kvlog!(level: warn, self, "not THAT leader anymore"); 173 | } 174 | } 175 | 176 | self.may_snapshot(index); 177 | } 178 | // snapshot from peer (InstallSnapshot RPC) 179 | ApplyMsg::Snapshot { 180 | index, 181 | term, 182 | snapshot, 183 | } => { 184 | let p = bincode::deserialize(&snapshot).unwrap(); 185 | let success = self.raft.cond_install_snapshot(term, index, snapshot); 186 | if success { 187 | self.p = p; 188 | } 189 | } 190 | } 191 | } 192 | 193 | /// Check the log size and decide whether to ask Raft to compact its log. 194 | fn may_snapshot(&mut self, index: u64) { 195 | if self.max_raft_state.map(|max| self.raft.log_size() >= max) == Some(true) { 196 | kvlog!( 197 | self, 198 | "do snapshot with: log size: {}, last included index: {}", 199 | self.raft.log_size(), 200 | index 201 | ); 202 | let snapshot = bincode::serialize(&self.p).unwrap(); 203 | self.raft.snapshot(index, snapshot); 204 | } 205 | } 206 | } 207 | 208 | /// Key-value service which triggers command starting and applying for inner `KvServer` instance. 209 | #[derive(Clone)] 210 | pub struct Node { 211 | // Your definitions here. 212 | /// Inner server instance 213 | kv: Arc>, 214 | /// Thread pool executor shared with inner Raft instance. 215 | executor: Executor, 216 | } 217 | 218 | impl Node { 219 | /// Create a service with inner server and a receiver for applied commands. 220 | pub fn new(kv: KvServer, apply_rx: UnboundedReceiver) -> Node { 221 | // Your code here. 222 | let executor = kv.raft.executor.clone(); 223 | 224 | let node = Node { 225 | kv: Arc::new(RwLock::new(kv)), 226 | executor, 227 | }; 228 | node.start_applier(apply_rx); 229 | 230 | node 231 | } 232 | 233 | /// Spawn the main loop for kv server. 234 | /// Will poll applied commands from receiver and call `KvServer::apply`. 235 | pub fn start_applier(&self, mut apply_rx: UnboundedReceiver) { 236 | let kv = Arc::clone(&self.kv); 237 | 238 | self.executor 239 | .spawn(async move { 240 | loop { 241 | match apply_rx.next().await { 242 | Some(msg) => { 243 | kv.write().unwrap().apply(msg); 244 | } 245 | None => { 246 | let kv = kv.read().unwrap(); 247 | kvlog!(level: warn, kv, "applier exited"); 248 | break; 249 | } 250 | } 251 | } 252 | }) 253 | .expect("failed to spawn applier"); 254 | } 255 | 256 | /// the tester calls kill() when a KVServer instance won't 257 | /// be needed again. you are not required to do anything 258 | /// in kill(), but it might be convenient to (for example) 259 | /// turn off debug output from this instance. 260 | pub fn kill(&self) { 261 | // If you want to free some resources by `raft::Node::kill` method, 262 | // you should call `raft::Node::kill` here also to prevent resource leaking. 263 | // Since the test framework will call kvraft::Node::kill only. 264 | // self.server.kill(); 265 | 266 | // Your code here, if desired. 267 | self.kv.read().unwrap().raft.kill(); 268 | } 269 | 270 | /// The current term of this peer. 271 | pub fn term(&self) -> u64 { 272 | self.kv.read().unwrap().raft.term() 273 | } 274 | 275 | /// Whether this peer believes it is the leader. 276 | pub fn is_leader(&self) -> bool { 277 | self.kv.read().unwrap().raft.is_leader() 278 | } 279 | 280 | pub fn get_state(&self) -> raft::State { 281 | // Your code here. 282 | self.kv.read().unwrap().raft.get_state() 283 | } 284 | } 285 | 286 | #[async_trait::async_trait] 287 | impl KvService for Node { 288 | // CAVEATS: Please avoid locking or sleeping here, it may jam the network. 289 | 290 | /// RPC service handler for `Op`. 291 | async fn op(&self, req: KvRequest) -> labrpc::Result { 292 | let kv = Arc::clone(&self.kv); 293 | self.executor 294 | .spawn_with_handle(async move { 295 | let mut reply = KvReply::default(); 296 | let start_result = kv.write().unwrap().start(req); 297 | 298 | match start_result { 299 | Ok(notifier) => match notifier.await { 300 | Ok(NotifyReply::Get { value }) => reply.value = value, 301 | Ok(_r) => {} 302 | Err(e) => { 303 | reply.wrong_leader = true; 304 | reply.err = e.to_string(); 305 | } 306 | }, 307 | Err(Error::Raft(raft::errors::Error::NotLeader)) => reply.wrong_leader = true, 308 | Err(e) => reply.err = e.to_string(), 309 | } 310 | 311 | Ok(reply) 312 | }) 313 | .unwrap() 314 | .await 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /raft/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(clippy::all)] 2 | #![allow(clippy::single_match)] 3 | 4 | #[allow(unused_imports)] 5 | #[macro_use] 6 | extern crate log; 7 | #[allow(unused_imports)] 8 | #[macro_use] 9 | extern crate prost_derive; 10 | 11 | pub mod kvraft; 12 | mod proto; 13 | pub mod raft; 14 | 15 | pub type Executor = futures::executor::ThreadPool; 16 | lazy_static::lazy_static! { 17 | pub static ref SHARED_EXECUTOR: Executor = Executor::new().unwrap(); 18 | } 19 | -------------------------------------------------------------------------------- /raft/src/proto/kvraft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package kvraftpb; 4 | 5 | enum Op { 6 | Unknown = 0; 7 | Put = 1; 8 | Append = 2; 9 | Get = 3; 10 | } 11 | 12 | message KvRequest { 13 | string key = 1; 14 | string value = 2; 15 | Op op = 3; 16 | // You'll have to add definitions here. 17 | string cid = 4; 18 | uint64 seq = 5; 19 | } 20 | 21 | message KvReply { 22 | bool wrong_leader = 1; 23 | string err = 2; 24 | string value = 3; 25 | } 26 | -------------------------------------------------------------------------------- /raft/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod raftpb { 2 | include!(concat!(env!("OUT_DIR"), "/raftpb.rs")); 3 | 4 | labrpc::service! { 5 | service raft { 6 | rpc request_vote(RequestVoteArgs) returns (RequestVoteReply); 7 | 8 | // Your code here if more rpc desired. 9 | rpc append_entries(AppendEntriesArgs) returns (AppendEntriesReply); 10 | rpc install_snapshot(InstallSnapshotArgs) returns (InstallSnapshotReply); 11 | } 12 | } 13 | pub use self::raft::{ 14 | add_service as add_raft_service, Client as RaftClient, Service as RaftService, 15 | }; 16 | } 17 | 18 | pub mod kvraftpb { 19 | include!(concat!(env!("OUT_DIR"), "/kvraftpb.rs")); 20 | 21 | labrpc::service! { 22 | service kv { 23 | rpc op(KvRequest) returns (KvReply); 24 | 25 | // Your code here if more rpc desired. 26 | } 27 | } 28 | pub use self::kv::{add_service as add_kv_service, Client as KvClient, Service as KvService}; 29 | } 30 | -------------------------------------------------------------------------------- /raft/src/proto/raft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package raftpb; 4 | 5 | message Entry { 6 | uint64 term = 1; 7 | bytes data = 2; 8 | } 9 | 10 | message RequestVoteArgs { 11 | // Your data here (2A, 2B). 12 | uint64 term = 1; 13 | uint64 candidate_id = 2; 14 | uint64 last_log_index = 3; 15 | uint64 last_log_term = 4; 16 | } 17 | 18 | message RequestVoteReply { 19 | // Your data here (2A). 20 | uint64 term = 1; 21 | bool vote_granted = 2; 22 | } 23 | 24 | message AppendEntriesArgs { 25 | uint64 term = 1; 26 | uint64 leader_id = 2; 27 | uint64 prev_log_index = 3; 28 | uint64 prev_log_term = 4; 29 | repeated Entry entries = 5; 30 | uint64 leader_commit_index = 6; 31 | } 32 | 33 | message AppendEntriesReply { 34 | uint64 term = 1; 35 | bool success = 2; 36 | uint64 conflict_index = 3; 37 | } 38 | 39 | message InstallSnapshotArgs { 40 | uint64 term = 1; 41 | uint64 leader_id = 2; 42 | uint64 last_included_index = 3; 43 | uint64 last_included_term = 4; 44 | bytes data = 5; 45 | } 46 | 47 | message InstallSnapshotReply { uint64 term = 1; } 48 | -------------------------------------------------------------------------------- /raft/src/raft/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{error, fmt, result}; 2 | 3 | #[derive(Clone, Debug, PartialEq, Eq)] 4 | pub enum Error { 5 | Encode(labcodec::EncodeError), 6 | Decode(labcodec::DecodeError), 7 | Rpc(labrpc::Error), 8 | NotLeader, 9 | } 10 | 11 | impl fmt::Display for Error { 12 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 13 | write!(f, "{:?}", self) 14 | } 15 | } 16 | 17 | impl error::Error for Error { 18 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 19 | match *self { 20 | Error::Encode(ref e) => Some(e), 21 | Error::Decode(ref e) => Some(e), 22 | Error::Rpc(ref e) => Some(e), 23 | _ => None, 24 | } 25 | } 26 | } 27 | 28 | pub type Result = result::Result; 29 | -------------------------------------------------------------------------------- /raft/src/raft/persister.rs: -------------------------------------------------------------------------------- 1 | //! Support for Raft and kvraft to save persistent 2 | //! Raft state (log &c) and k/v server snapshots. 3 | //! 4 | //! we will use the original persister.rs to test your code for grading. 5 | //! so, while you can modify this code to help you debug, please 6 | //! test with the original before submitting. 7 | 8 | use std::sync::{Arc, Mutex}; 9 | 10 | pub trait Persister: Send + Sync + 'static { 11 | fn raft_state(&self) -> Vec; 12 | fn save_raft_state(&self, state: Vec); 13 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec); 14 | fn snapshot(&self) -> Vec; 15 | fn raft_state_size(&self) -> usize; 16 | fn snapshot_size(&self) -> usize; 17 | } 18 | 19 | impl Persister for Box { 20 | fn raft_state(&self) -> Vec { 21 | (**self).raft_state() 22 | } 23 | fn save_raft_state(&self, state: Vec) { 24 | (**self).save_raft_state(state) 25 | } 26 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 27 | (**self).save_state_and_snapshot(state, snapshot) 28 | } 29 | fn snapshot(&self) -> Vec { 30 | (**self).snapshot() 31 | } 32 | fn raft_state_size(&self) -> usize { 33 | (**self).raft_state_size() 34 | } 35 | fn snapshot_size(&self) -> usize { 36 | (**self).snapshot_size() 37 | } 38 | } 39 | 40 | impl Persister for Arc { 41 | fn raft_state(&self) -> Vec { 42 | (**self).raft_state() 43 | } 44 | fn save_raft_state(&self, state: Vec) { 45 | (**self).save_raft_state(state) 46 | } 47 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 48 | (**self).save_state_and_snapshot(state, snapshot) 49 | } 50 | fn snapshot(&self) -> Vec { 51 | (**self).snapshot() 52 | } 53 | fn raft_state_size(&self) -> usize { 54 | (**self).raft_state_size() 55 | } 56 | fn snapshot_size(&self) -> usize { 57 | (**self).snapshot_size() 58 | } 59 | } 60 | 61 | #[derive(Default)] 62 | pub struct SimplePersister { 63 | states: Mutex<( 64 | Vec, // raft state 65 | Vec, // snapshot 66 | )>, 67 | } 68 | 69 | impl SimplePersister { 70 | pub fn new() -> SimplePersister { 71 | SimplePersister { 72 | states: Mutex::default(), 73 | } 74 | } 75 | } 76 | 77 | impl Persister for SimplePersister { 78 | fn raft_state(&self) -> Vec { 79 | self.states.lock().unwrap().0.clone() 80 | } 81 | 82 | fn save_raft_state(&self, state: Vec) { 83 | self.states.lock().unwrap().0 = state; 84 | } 85 | 86 | fn save_state_and_snapshot(&self, state: Vec, snapshot: Vec) { 87 | self.states.lock().unwrap().0 = state; 88 | self.states.lock().unwrap().1 = snapshot; 89 | } 90 | 91 | fn snapshot(&self) -> Vec { 92 | self.states.lock().unwrap().1.clone() 93 | } 94 | 95 | fn raft_state_size(&self) -> usize { 96 | self.states.lock().unwrap().0.len() 97 | } 98 | 99 | fn snapshot_size(&self) -> usize { 100 | self.states.lock().unwrap().1.len() 101 | } 102 | } 103 | 104 | #[cfg(test)] 105 | mod tests { 106 | use super::*; 107 | 108 | #[test] 109 | fn test_object_safety() { 110 | let sp = SimplePersister::new(); 111 | sp.save_raft_state(vec![111]); 112 | let obj: Box = Box::new(sp); 113 | assert_eq!(obj.raft_state(), vec![111]); 114 | obj.save_state_and_snapshot(vec![222], vec![123]); 115 | assert_eq!(obj.raft_state(), vec![222]); 116 | assert_eq!(obj.snapshot(), vec![123]); 117 | 118 | let cloneable_obj: Arc = Arc::new(obj); 119 | assert_eq!(cloneable_obj.raft_state(), vec![222]); 120 | assert_eq!(cloneable_obj.snapshot(), vec![123]); 121 | 122 | let cloneable_obj_ = cloneable_obj.clone(); 123 | cloneable_obj.save_raft_state(vec![233]); 124 | assert_eq!(cloneable_obj_.raft_state(), vec![233]); 125 | assert_eq!(cloneable_obj_.snapshot(), vec![123]); 126 | 127 | let sp = SimplePersister::new(); 128 | let obj: Arc = Arc::new(sp); 129 | let _box_obj: Box = Box::new(obj); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /raft/src/raft/states.rs: -------------------------------------------------------------------------------- 1 | use crate::proto::raftpb::*; 2 | use serde::{Deserialize, Serialize}; 3 | use std::{ 4 | collections::{HashSet, VecDeque}, 5 | fmt::{self, Display}, 6 | ops, 7 | sync::Arc, 8 | }; 9 | 10 | use super::persister::Persister; 11 | 12 | /// State of a raft peer (for testing). 13 | #[derive(Default, Clone, Debug)] 14 | pub struct State { 15 | pub term: u64, 16 | pub is_leader: bool, 17 | } 18 | 19 | impl State { 20 | /// The current term of this peer. 21 | pub fn term(&self) -> u64 { 22 | self.term 23 | } 24 | /// Whether this peer believes it is the leader. 25 | pub fn is_leader(&self) -> bool { 26 | self.is_leader 27 | } 28 | } 29 | 30 | /// Role of a peer with its specific states. 31 | #[derive(Debug)] 32 | pub enum RoleState { 33 | Follower, 34 | Candidate { 35 | /// Received votes from peers. 36 | votes: HashSet, 37 | }, 38 | Leader { 39 | next_index: Vec, 40 | match_index: Vec, 41 | }, 42 | } 43 | 44 | impl Display for RoleState { 45 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 46 | let desc = match self { 47 | RoleState::Follower => "Follower", 48 | RoleState::Candidate { .. } => "Candidate", 49 | RoleState::Leader { .. } => "Leader", 50 | }; 51 | write!(f, "{}", desc) 52 | } 53 | } 54 | 55 | /// Represents the log of a Raft peer, with snapshot support. 56 | #[derive(Serialize, Deserialize)] 57 | pub struct Log { 58 | /// In memory log entries. 59 | inner: VecDeque, 60 | /// Length of entries in snapshot (i.e., not in `inner`). 61 | in_snapshot_len: usize, 62 | /// Term of the last entry in snapshot. 63 | snapshot_last_included_term: u64, 64 | } 65 | 66 | impl Log { 67 | /// Creates an empty log. 68 | pub fn new() -> Self { 69 | Self { 70 | inner: VecDeque::new(), 71 | in_snapshot_len: 1, // log index starts from 1 by this dummy snapshot 72 | snapshot_last_included_term: 0, 73 | } 74 | } 75 | 76 | /// Term of the last entry in snapshot. 77 | pub fn snapshot_last_included_term(&self) -> u64 { 78 | self.snapshot_last_included_term 79 | } 80 | 81 | /// Index of the last entry in snapshot. 82 | pub fn snapshot_last_included_index(&self) -> usize { 83 | self.in_snapshot_len - 1 84 | } 85 | 86 | /// Index of the next entry slot. 87 | pub fn next_index(&self) -> usize { 88 | self.inner.len() + self.in_snapshot_len 89 | } 90 | 91 | /// Index of the last entry. 92 | pub fn last_index(&self) -> usize { 93 | self.next_index() - 1 94 | } 95 | 96 | /// Term of the last entry. 97 | pub fn last_term(&self) -> u64 { 98 | self.inner 99 | .back() 100 | .map_or(self.snapshot_last_included_term, |e| e.term) 101 | } 102 | 103 | /// Get the ref to the entry at given index. 104 | /// 105 | /// The index is given as a absolute index of the log. 106 | /// Thus, the desired entry may be saved into the snapshot and not in the log anymore, 107 | /// which will be returned as `None`. 108 | fn get(&self, index: usize) -> Option<&Entry> { 109 | self.offset_index(index) 110 | .and_then(|index| self.inner.get(index)) 111 | } 112 | 113 | #[allow(dead_code)] 114 | pub fn replace(&mut self, index: usize, entry: Entry) -> Option { 115 | self.offset_index(index).and_then(move |index| { 116 | self.inner 117 | .get_mut(index) 118 | .map(|old| std::mem::replace(old, entry)) 119 | }) 120 | } 121 | 122 | /// Get the term of the entry at given index. 123 | /// 124 | /// Compared to `get`, this method is also able to get the last included term in the snapshot. 125 | pub fn term_at(&self, index: usize) -> Option { 126 | self.get(index).map(|e| e.term).or_else(|| { 127 | (index == self.in_snapshot_len - 1).then(|| self.snapshot_last_included_term) 128 | }) 129 | } 130 | 131 | /// Get the data of the entry at given index. 132 | pub fn data_at(&self, index: usize) -> Option<&Vec> { 133 | self.get(index).map(|e| &e.data) 134 | } 135 | 136 | /// Get an iterator of log starting at the given index. 137 | pub fn start_at(&self, index: usize) -> Option> { 138 | self.offset_index(index) 139 | .map(|index| self.inner.iter().skip(index)) 140 | } 141 | 142 | /// Convert an absolute index into `inner`'s index 143 | fn offset_index(&self, index: usize) -> Option { 144 | index.checked_sub(self.in_snapshot_len) 145 | } 146 | 147 | /// Push an entry to the log. 148 | pub fn push(&mut self, entry: Entry) { 149 | self.inner.push_back(entry); 150 | } 151 | 152 | /// Pop the last entry in log. 153 | pub fn pop_back(&mut self) -> Option { 154 | self.inner.pop_back() 155 | } 156 | 157 | /// Compact the log to the given index. 158 | /// This method is often called after a snapshot is created, 159 | /// when there'is no need to keep old entries in memory anymore. 160 | /// 161 | /// Returns whether the compaction is successful. 162 | pub fn compact_to(&mut self, included_index: usize, included_term: u64) -> bool { 163 | if included_index + 1 < self.in_snapshot_len { 164 | return false; 165 | } 166 | 167 | let n_compact = included_index + 1 - self.in_snapshot_len; 168 | for _ in 0..n_compact { 169 | self.inner.pop_front(); 170 | } 171 | 172 | if self.inner.len() >= 50 { 173 | warn!( 174 | "compacted {} entries, while {} entries still in memory!", 175 | n_compact, 176 | self.inner.len() 177 | ); 178 | } else { 179 | debug!("compacted {} entries", n_compact); 180 | } 181 | 182 | self.inner.shrink_to_fit(); 183 | self.in_snapshot_len = included_index + 1; 184 | self.snapshot_last_included_term = included_term; 185 | true 186 | } 187 | } 188 | 189 | impl fmt::Debug for Log { 190 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 191 | f.debug_struct("Log") 192 | .field("last_index", &self.last_index()) 193 | .finish_non_exhaustive() 194 | } 195 | } 196 | 197 | /// Persistent state of a raft peer. 198 | #[derive(Serialize, Deserialize, Debug)] 199 | pub struct PersistentState { 200 | pub current_term: u64, 201 | pub voted_for: Option, 202 | pub log: Log, 203 | } 204 | 205 | impl PersistentState { 206 | fn new() -> Self { 207 | Self { 208 | current_term: 0, 209 | voted_for: None, 210 | log: Log::new(), 211 | } 212 | } 213 | } 214 | 215 | /// Write guard for wrapped `PersistentState`, 216 | /// will persisted the state when dropped. 217 | pub struct PersistentStateGuard<'a> { 218 | inner: &'a mut PersistentState, 219 | on_drop: Option>, 220 | } 221 | 222 | impl<'a> PersistentStateGuard<'a> { 223 | fn new(inner: &'a mut PersistentState, persister: Arc) -> Self { 224 | Self { 225 | inner, 226 | on_drop: Some(Box::new(|state| { 227 | PersistentStateWrapper::do_persist(persister, state, None); 228 | })), 229 | } 230 | } 231 | } 232 | 233 | impl Drop for PersistentStateGuard<'_> { 234 | fn drop(&mut self) { 235 | if let Some(on_drop) = self.on_drop.take() { 236 | on_drop(self.inner) 237 | } 238 | } 239 | } 240 | 241 | impl ops::Deref for PersistentStateGuard<'_> { 242 | type Target = PersistentState; 243 | 244 | fn deref(&self) -> &Self::Target { 245 | self.inner 246 | } 247 | } 248 | 249 | impl ops::DerefMut for PersistentStateGuard<'_> { 250 | fn deref_mut(&mut self) -> &mut Self::Target { 251 | &mut self.inner 252 | } 253 | } 254 | 255 | /// Wrapper of `PersistentState` for persistence purpose. 256 | pub struct PersistentStateWrapper { 257 | /// The inner state. 258 | state: PersistentState, 259 | /// Object to serialize and deserialize this peer's persisted state. 260 | pub persister: Arc, 261 | } 262 | 263 | impl PersistentStateWrapper { 264 | /// restore previously persisted state. 265 | pub fn from(persister: Box) -> Self { 266 | let data = persister.raft_state(); 267 | let state = bincode::deserialize(&data).unwrap_or_else(|_| PersistentState::new()); 268 | Self { 269 | state, 270 | persister: persister.into(), 271 | } 272 | } 273 | 274 | /// Get a immutable reference to the state. 275 | pub fn read(&self) -> &PersistentState { 276 | &self.state 277 | } 278 | 279 | /// Acquire a write guard for wrapped `PersistentState`, 280 | /// and automatically persist it when dropped. 281 | pub fn write(&mut self) -> PersistentStateGuard { 282 | let persister = Arc::clone(&self.persister); 283 | PersistentStateGuard::new(&mut self.state, persister) 284 | } 285 | 286 | /// save Raft's persistent state to stable storage, 287 | /// where it can later be retrieved after a crash and restart. 288 | pub fn persist(&self, snapshot: Option>) { 289 | // Your code here (2C). 290 | Self::do_persist(&self.persister, &self.state, snapshot) 291 | } 292 | 293 | /// Do persist using `persister`. 294 | fn do_persist( 295 | persister: impl AsRef, 296 | state: &PersistentState, 297 | snapshot: Option>, 298 | ) { 299 | let persister = persister.as_ref(); 300 | 301 | let p = bincode::serialize(state).unwrap(); 302 | if let Some(ss) = snapshot { 303 | persister.save_state_and_snapshot(p, ss); 304 | } else { 305 | persister.save_raft_state(p); 306 | } 307 | } 308 | 309 | pub fn log_size(&self) -> usize { 310 | self.persister.raft_state_size() 311 | } 312 | } 313 | 314 | impl ops::Deref for PersistentStateWrapper { 315 | type Target = PersistentState; 316 | 317 | fn deref(&self) -> &Self::Target { 318 | self.read() 319 | } 320 | } 321 | 322 | /// Volatile state of a raft peer. 323 | #[derive(Debug)] 324 | pub struct VolatileState { 325 | pub commit_index: usize, 326 | pub last_applied: usize, 327 | } 328 | 329 | impl VolatileState { 330 | #[allow(dead_code)] 331 | pub fn new() -> Self { 332 | Self { 333 | commit_index: 0, 334 | last_applied: 0, 335 | } 336 | } 337 | } 338 | -------------------------------------------------------------------------------- /test-many.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Script for running `go test` a bunch of times, in parallel, storing the test 4 | # output as you go, and showing a nice status output telling you how you're 5 | # doing. 6 | # 7 | # Normally, you should be able to execute this script with 8 | # 9 | # ./go-test-many.sh 10 | # 11 | # and it should do The Right Thing(tm) by default. However, it does take some 12 | # arguments so that you can tweak it for your testing setup. To understand 13 | # them, we should first go quickly through what exactly this script does. 14 | # 15 | # First, it compiles your Go program (using go test -c) to ensure that all the 16 | # tests are run on the same codebase, and to speed up the testing. Then, it 17 | # runs the tester some number of times. It will run some number of testers in 18 | # parallel, and when that number of running testers has been reached, it will 19 | # wait for the oldest one it spawned to finish before spawning another. The 20 | # output from each test i is stored in test-$i.log and test-$i.err (STDOUT and 21 | # STDERR respectively). 22 | # 23 | # The options you can specify on the command line are: 24 | # 25 | # 1) how many times to run the tester (defaults to 100) 26 | # 2) how many testers to run in parallel (defaults to the number of CPUs) 27 | # 3) which subset of the tests to run (default to all tests) 28 | # 29 | # 3) is simply a regex that is passed to the tester under -test.run; any tests 30 | # matching the regex will be run. 31 | # 32 | # The script is smart enough to clean up after itself if you kill it 33 | # (in-progress tests are killed, their output is discarded, and no failure 34 | # message is printed), and will automatically continue from where it left off 35 | # if you kill it and then start it again. 36 | # 37 | # By now, you know everything that happens below. 38 | # If you still want to read the code, go ahead. 39 | 40 | if [ $# -eq 1 ] && [ "$1" = "--help" ]; then 41 | echo "Usage: $0 [RUNS=100] [PARALLELISM=#cpus] [TESTPATTERN='']" 42 | exit 1 43 | fi 44 | 45 | rm test-*.log test-*.err 46 | 47 | # # If the tests don't even build, don't bother. Also, this gives us a static 48 | # # tester binary for higher performance and higher reproducability. 49 | # if ! go test -c -o tester; then 50 | # echo -e "\e[1;31mERROR: Build failed\e[0m" 51 | # exit 1 52 | # fi 53 | 54 | # Default to 100 runs unless otherwise specified 55 | runs=100 56 | if [ $# -gt 0 ]; then 57 | runs="$1" 58 | fi 59 | 60 | # Default to one tester per CPU unless otherwise specified 61 | parallelism=16 62 | if [ $# -gt 1 ]; then 63 | parallelism="$2" 64 | fi 65 | 66 | # Default to no test filtering unless otherwise specified 67 | test="" 68 | if [ $# -gt 2 ]; then 69 | test="$3" 70 | fi 71 | 72 | # Figure out where we left off 73 | logs=$(gfind . -maxdepth 1 -name 'test-*.log' -type f -printf '.' | wc -c) 74 | success=$(grep -E '^PASS$' test-*.log | wc -l) 75 | ((failed = logs - success)) 76 | 77 | # Finish checks the exit status of the tester with the given PID, updates the 78 | # success/failed counters appropriately, and prints a pretty message. 79 | finish() { 80 | if ! wait "$1"; then 81 | if command -v notify-send >/dev/null 2>&1 && ((failed == 0)); then 82 | notify-send -i weather-storm "Tests started failing" \ 83 | "$(pwd)\n$(grep FAIL: -- *.log | sed -e 's/.*FAIL: / - /' -e 's/ (.*)//' | sort -u)" 84 | fi 85 | ((failed += 1)) 86 | else 87 | ((success += 1)) 88 | fi 89 | 90 | if [ "$failed" -eq 0 ]; then 91 | printf "\e[1;32m" 92 | else 93 | printf "\e[1;31m" 94 | fi 95 | 96 | printf "Done %03d/%d; %d ok, %d failed\n\e[0m" \ 97 | $((success + failed)) \ 98 | "$runs" \ 99 | "$success" \ 100 | "$failed" 101 | } 102 | 103 | waits=() # which tester PIDs are we waiting on? 104 | is=() # and which iteration does each one correspond to? 105 | 106 | # Cleanup is called when the process is killed. 107 | # It kills any remaining tests and removes their output files before exiting. 108 | cleanup() { 109 | for pid in "${waits[@]}"; do 110 | kill "$pid" 111 | wait "$pid" 112 | rm -rf "test-${is[0]}.err" "test-${is[0]}.log" 113 | is=("${is[@]:1}") 114 | done 115 | exit 0 116 | } 117 | trap cleanup SIGHUP SIGINT SIGTERM 118 | 119 | # Run remaining iterations (we may already have run some) 120 | for i in $(seq "$((success + failed + 1))" "$runs"); do 121 | # If we have already spawned the max # of testers, wait for one to 122 | # finish. We'll wait for the oldest one beause it's easy. 123 | if [[ ${#waits[@]} -eq "$parallelism" ]]; then 124 | finish "${waits[0]}" 125 | waits=("${waits[@]:1}") # this funky syntax removes the first 126 | is=("${is[@]:1}") # element from the array 127 | fi 128 | 129 | # Store this tester's iteration index 130 | # It's important that this happens before appending to waits(), 131 | # otherwise we could get an out-of-bounds in cleanup() 132 | is=("${is[@]}" $i) 133 | 134 | # Run the tester, passing -test.run if necessary 135 | if [[ -z "$test" ]]; then 136 | make LOG_LEVEL=error RUST_TEST_THREADS=16 test_23 2>"test-${i}.err" >"test-${i}.log" & 137 | pid=$! 138 | else 139 | make LOG_LEVEL=error RUST_TEST_THREADS=16 cargo_test_$test 2>"test-${i}.err" >"test-${i}.log" & 140 | pid=$! 141 | fi 142 | 143 | # Remember the tester's PID so we can wait on it later 144 | waits=("${waits[@]}" $pid) 145 | done 146 | 147 | # Wait for remaining testers 148 | for pid in "${waits[@]}"; do 149 | finish "$pid" 150 | done 151 | 152 | if ((failed > 0)); then 153 | exit 1 154 | fi 155 | exit 0 156 | --------------------------------------------------------------------------------