├── .DS_Store ├── .gitignore ├── .vscode └── launch.json ├── Cargo.lock ├── Cargo.toml ├── README.md ├── assets └── raft-rpc.png └── src ├── clock.rs ├── main.rs ├── storage ├── file_ops.rs └── mod.rs └── types.rs /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redixhumayun/raft/8b5f73a7e35aa82cf2aa14f4f467abbcec96c145/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | data_server_* 3 | temp_* 4 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "lldb", 9 | "request": "launch", 10 | "name": "Debug executable 'client'", 11 | "cargo": { 12 | "args": [ 13 | "build", 14 | "--bin=client", 15 | "--package=raft" 16 | ], 17 | "filter": { 18 | "name": "client", 19 | "kind": "bin" 20 | } 21 | }, 22 | "args": [], 23 | "cwd": "${workspaceFolder}" 24 | }, 25 | { 26 | "type": "lldb", 27 | "request": "launch", 28 | "name": "Debug unit tests in executable 'client'", 29 | "cargo": { 30 | "args": [ 31 | "test", 32 | "--no-run", 33 | "--bin=client", 34 | "--package=raft" 35 | ], 36 | "filter": { 37 | "name": "client", 38 | "kind": "bin" 39 | } 40 | }, 41 | "args": [], 42 | "cwd": "${workspaceFolder}" 43 | }, 44 | { 45 | "type": "lldb", 46 | "request": "launch", 47 | "name": "Debug executable 'raft'", 48 | "cargo": { 49 | "args": [ 50 | "build", 51 | "--bin=raft", 52 | "--package=raft" 53 | ], 54 | "filter": { 55 | "name": "raft", 56 | "kind": "bin" 57 | } 58 | }, 59 | "args": [], 60 | "cwd": "${workspaceFolder}" 61 | }, 62 | { 63 | "type": "lldb", 64 | "request": "launch", 65 | "name": "Debug unit tests in executable 'raft'", 66 | "cargo": { 67 | "args": [ 68 | "test", 69 | "--no-run", 70 | "--bin=raft", 71 | "--package=raft" 72 | ], 73 | "filter": { 74 | "name": "raft", 75 | "kind": "bin" 76 | } 77 | }, 78 | "args": [], 79 | "cwd": "${workspaceFolder}" 80 | } 81 | ] 82 | } -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.21.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "aho-corasick" 22 | version = "1.1.2" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" 25 | dependencies = [ 26 | "memchr", 27 | ] 28 | 29 | [[package]] 30 | name = "atty" 31 | version = "0.2.14" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 34 | dependencies = [ 35 | "hermit-abi 0.1.19", 36 | "libc", 37 | "winapi", 38 | ] 39 | 40 | [[package]] 41 | name = "autocfg" 42 | version = "1.1.0" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 45 | 46 | [[package]] 47 | name = "backtrace" 48 | version = "0.3.69" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" 51 | dependencies = [ 52 | "addr2line", 53 | "cc", 54 | "cfg-if", 55 | "libc", 56 | "miniz_oxide", 57 | "object", 58 | "rustc-demangle", 59 | ] 60 | 61 | [[package]] 62 | name = "base64" 63 | version = "0.21.7" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" 66 | 67 | [[package]] 68 | name = "bitflags" 69 | version = "1.3.2" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 72 | 73 | [[package]] 74 | name = "bitflags" 75 | version = "2.4.2" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" 78 | 79 | [[package]] 80 | name = "block-buffer" 81 | version = "0.10.4" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 84 | dependencies = [ 85 | "generic-array", 86 | ] 87 | 88 | [[package]] 89 | name = "bumpalo" 90 | version = "3.14.0" 91 | source = "registry+https://github.com/rust-lang/crates.io-index" 92 | checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" 93 | 94 | [[package]] 95 | name = "byteorder" 96 | version = "1.5.0" 97 | source = "registry+https://github.com/rust-lang/crates.io-index" 98 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 99 | 100 | [[package]] 101 | name = "bytes" 102 | version = "1.5.0" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" 105 | 106 | [[package]] 107 | name = "cc" 108 | version = "1.0.83" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" 111 | dependencies = [ 112 | "libc", 113 | ] 114 | 115 | [[package]] 116 | name = "cfg-if" 117 | version = "1.0.0" 118 | source = "registry+https://github.com/rust-lang/crates.io-index" 119 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 120 | 121 | [[package]] 122 | name = "core-foundation" 123 | version = "0.9.4" 124 | source = "registry+https://github.com/rust-lang/crates.io-index" 125 | checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" 126 | dependencies = [ 127 | "core-foundation-sys", 128 | "libc", 129 | ] 130 | 131 | [[package]] 132 | name = "core-foundation-sys" 133 | version = "0.8.6" 134 | source = "registry+https://github.com/rust-lang/crates.io-index" 135 | checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" 136 | 137 | [[package]] 138 | name = "cpufeatures" 139 | version = "0.2.12" 140 | source = "registry+https://github.com/rust-lang/crates.io-index" 141 | checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" 142 | dependencies = [ 143 | "libc", 144 | ] 145 | 146 | [[package]] 147 | name = "crypto-common" 148 | version = "0.1.6" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" 151 | dependencies = [ 152 | "generic-array", 153 | "typenum", 154 | ] 155 | 156 | [[package]] 157 | name = "data-encoding" 158 | version = "2.5.0" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" 161 | 162 | [[package]] 163 | name = "digest" 164 | version = "0.10.7" 165 | source = "registry+https://github.com/rust-lang/crates.io-index" 166 | checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 167 | dependencies = [ 168 | "block-buffer", 169 | "crypto-common", 170 | ] 171 | 172 | [[package]] 173 | name = "encoding_rs" 174 | version = "0.8.33" 175 | source = "registry+https://github.com/rust-lang/crates.io-index" 176 | checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" 177 | dependencies = [ 178 | "cfg-if", 179 | ] 180 | 181 | [[package]] 182 | name = "env_logger" 183 | version = "0.9.3" 184 | source = "registry+https://github.com/rust-lang/crates.io-index" 185 | checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" 186 | dependencies = [ 187 | "atty", 188 | "humantime", 189 | "log", 190 | "regex", 191 | "termcolor", 192 | ] 193 | 194 | [[package]] 195 | name = "equivalent" 196 | version = "1.0.1" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" 199 | 200 | [[package]] 201 | name = "errno" 202 | version = "0.3.8" 203 | source = "registry+https://github.com/rust-lang/crates.io-index" 204 | checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" 205 | dependencies = [ 206 | "libc", 207 | "windows-sys 0.52.0", 208 | ] 209 | 210 | [[package]] 211 | name = "fastrand" 212 | version = "2.0.1" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" 215 | 216 | [[package]] 217 | name = "fnv" 218 | version = "1.0.7" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 221 | 222 | [[package]] 223 | name = "foreign-types" 224 | version = "0.3.2" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 227 | dependencies = [ 228 | "foreign-types-shared", 229 | ] 230 | 231 | [[package]] 232 | name = "foreign-types-shared" 233 | version = "0.1.1" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 236 | 237 | [[package]] 238 | name = "form_urlencoded" 239 | version = "1.2.1" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 242 | dependencies = [ 243 | "percent-encoding", 244 | ] 245 | 246 | [[package]] 247 | name = "futures-channel" 248 | version = "0.3.30" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" 251 | dependencies = [ 252 | "futures-core", 253 | "futures-sink", 254 | ] 255 | 256 | [[package]] 257 | name = "futures-core" 258 | version = "0.3.30" 259 | source = "registry+https://github.com/rust-lang/crates.io-index" 260 | checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" 261 | 262 | [[package]] 263 | name = "futures-sink" 264 | version = "0.3.30" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" 267 | 268 | [[package]] 269 | name = "futures-task" 270 | version = "0.3.30" 271 | source = "registry+https://github.com/rust-lang/crates.io-index" 272 | checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" 273 | 274 | [[package]] 275 | name = "futures-util" 276 | version = "0.3.30" 277 | source = "registry+https://github.com/rust-lang/crates.io-index" 278 | checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" 279 | dependencies = [ 280 | "futures-core", 281 | "futures-sink", 282 | "futures-task", 283 | "pin-project-lite", 284 | "pin-utils", 285 | "slab", 286 | ] 287 | 288 | [[package]] 289 | name = "generic-array" 290 | version = "0.14.7" 291 | source = "registry+https://github.com/rust-lang/crates.io-index" 292 | checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 293 | dependencies = [ 294 | "typenum", 295 | "version_check", 296 | ] 297 | 298 | [[package]] 299 | name = "getrandom" 300 | version = "0.2.11" 301 | source = "registry+https://github.com/rust-lang/crates.io-index" 302 | checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" 303 | dependencies = [ 304 | "cfg-if", 305 | "libc", 306 | "wasi", 307 | ] 308 | 309 | [[package]] 310 | name = "gimli" 311 | version = "0.28.1" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" 314 | 315 | [[package]] 316 | name = "h2" 317 | version = "0.3.24" 318 | source = "registry+https://github.com/rust-lang/crates.io-index" 319 | checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" 320 | dependencies = [ 321 | "bytes", 322 | "fnv", 323 | "futures-core", 324 | "futures-sink", 325 | "futures-util", 326 | "http", 327 | "indexmap", 328 | "slab", 329 | "tokio", 330 | "tokio-util", 331 | "tracing", 332 | ] 333 | 334 | [[package]] 335 | name = "hashbrown" 336 | version = "0.14.3" 337 | source = "registry+https://github.com/rust-lang/crates.io-index" 338 | checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" 339 | 340 | [[package]] 341 | name = "headers" 342 | version = "0.3.9" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" 345 | dependencies = [ 346 | "base64", 347 | "bytes", 348 | "headers-core", 349 | "http", 350 | "httpdate", 351 | "mime", 352 | "sha1", 353 | ] 354 | 355 | [[package]] 356 | name = "headers-core" 357 | version = "0.2.0" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" 360 | dependencies = [ 361 | "http", 362 | ] 363 | 364 | [[package]] 365 | name = "hermit-abi" 366 | version = "0.1.19" 367 | source = "registry+https://github.com/rust-lang/crates.io-index" 368 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 369 | dependencies = [ 370 | "libc", 371 | ] 372 | 373 | [[package]] 374 | name = "hermit-abi" 375 | version = "0.3.4" 376 | source = "registry+https://github.com/rust-lang/crates.io-index" 377 | checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" 378 | 379 | [[package]] 380 | name = "http" 381 | version = "0.2.11" 382 | source = "registry+https://github.com/rust-lang/crates.io-index" 383 | checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" 384 | dependencies = [ 385 | "bytes", 386 | "fnv", 387 | "itoa", 388 | ] 389 | 390 | [[package]] 391 | name = "http-body" 392 | version = "0.4.6" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" 395 | dependencies = [ 396 | "bytes", 397 | "http", 398 | "pin-project-lite", 399 | ] 400 | 401 | [[package]] 402 | name = "httparse" 403 | version = "1.8.0" 404 | source = "registry+https://github.com/rust-lang/crates.io-index" 405 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" 406 | 407 | [[package]] 408 | name = "httpdate" 409 | version = "1.0.3" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" 412 | 413 | [[package]] 414 | name = "humantime" 415 | version = "2.1.0" 416 | source = "registry+https://github.com/rust-lang/crates.io-index" 417 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 418 | 419 | [[package]] 420 | name = "hyper" 421 | version = "0.14.28" 422 | source = "registry+https://github.com/rust-lang/crates.io-index" 423 | checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" 424 | dependencies = [ 425 | "bytes", 426 | "futures-channel", 427 | "futures-core", 428 | "futures-util", 429 | "h2", 430 | "http", 431 | "http-body", 432 | "httparse", 433 | "httpdate", 434 | "itoa", 435 | "pin-project-lite", 436 | "socket2", 437 | "tokio", 438 | "tower-service", 439 | "tracing", 440 | "want", 441 | ] 442 | 443 | [[package]] 444 | name = "hyper-tls" 445 | version = "0.5.0" 446 | source = "registry+https://github.com/rust-lang/crates.io-index" 447 | checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" 448 | dependencies = [ 449 | "bytes", 450 | "hyper", 451 | "native-tls", 452 | "tokio", 453 | "tokio-native-tls", 454 | ] 455 | 456 | [[package]] 457 | name = "idna" 458 | version = "0.5.0" 459 | source = "registry+https://github.com/rust-lang/crates.io-index" 460 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" 461 | dependencies = [ 462 | "unicode-bidi", 463 | "unicode-normalization", 464 | ] 465 | 466 | [[package]] 467 | name = "indexmap" 468 | version = "2.1.0" 469 | source = "registry+https://github.com/rust-lang/crates.io-index" 470 | checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" 471 | dependencies = [ 472 | "equivalent", 473 | "hashbrown", 474 | ] 475 | 476 | [[package]] 477 | name = "ipnet" 478 | version = "2.9.0" 479 | source = "registry+https://github.com/rust-lang/crates.io-index" 480 | checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" 481 | 482 | [[package]] 483 | name = "itoa" 484 | version = "1.0.10" 485 | source = "registry+https://github.com/rust-lang/crates.io-index" 486 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" 487 | 488 | [[package]] 489 | name = "js-sys" 490 | version = "0.3.67" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" 493 | dependencies = [ 494 | "wasm-bindgen", 495 | ] 496 | 497 | [[package]] 498 | name = "lazy_static" 499 | version = "1.4.0" 500 | source = "registry+https://github.com/rust-lang/crates.io-index" 501 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 502 | 503 | [[package]] 504 | name = "libc" 505 | version = "0.2.152" 506 | source = "registry+https://github.com/rust-lang/crates.io-index" 507 | checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" 508 | 509 | [[package]] 510 | name = "linux-raw-sys" 511 | version = "0.4.13" 512 | source = "registry+https://github.com/rust-lang/crates.io-index" 513 | checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" 514 | 515 | [[package]] 516 | name = "lock_api" 517 | version = "0.4.11" 518 | source = "registry+https://github.com/rust-lang/crates.io-index" 519 | checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" 520 | dependencies = [ 521 | "autocfg", 522 | "scopeguard", 523 | ] 524 | 525 | [[package]] 526 | name = "log" 527 | version = "0.4.20" 528 | source = "registry+https://github.com/rust-lang/crates.io-index" 529 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 530 | 531 | [[package]] 532 | name = "memchr" 533 | version = "2.7.1" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" 536 | 537 | [[package]] 538 | name = "mime" 539 | version = "0.3.17" 540 | source = "registry+https://github.com/rust-lang/crates.io-index" 541 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 542 | 543 | [[package]] 544 | name = "mime_guess" 545 | version = "2.0.4" 546 | source = "registry+https://github.com/rust-lang/crates.io-index" 547 | checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" 548 | dependencies = [ 549 | "mime", 550 | "unicase", 551 | ] 552 | 553 | [[package]] 554 | name = "miniz_oxide" 555 | version = "0.7.1" 556 | source = "registry+https://github.com/rust-lang/crates.io-index" 557 | checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" 558 | dependencies = [ 559 | "adler", 560 | ] 561 | 562 | [[package]] 563 | name = "mio" 564 | version = "0.8.10" 565 | source = "registry+https://github.com/rust-lang/crates.io-index" 566 | checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" 567 | dependencies = [ 568 | "libc", 569 | "wasi", 570 | "windows-sys 0.48.0", 571 | ] 572 | 573 | [[package]] 574 | name = "multer" 575 | version = "2.1.0" 576 | source = "registry+https://github.com/rust-lang/crates.io-index" 577 | checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" 578 | dependencies = [ 579 | "bytes", 580 | "encoding_rs", 581 | "futures-util", 582 | "http", 583 | "httparse", 584 | "log", 585 | "memchr", 586 | "mime", 587 | "spin", 588 | "version_check", 589 | ] 590 | 591 | [[package]] 592 | name = "native-tls" 593 | version = "0.2.11" 594 | source = "registry+https://github.com/rust-lang/crates.io-index" 595 | checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" 596 | dependencies = [ 597 | "lazy_static", 598 | "libc", 599 | "log", 600 | "openssl", 601 | "openssl-probe", 602 | "openssl-sys", 603 | "schannel", 604 | "security-framework", 605 | "security-framework-sys", 606 | "tempfile", 607 | ] 608 | 609 | [[package]] 610 | name = "num_cpus" 611 | version = "1.16.0" 612 | source = "registry+https://github.com/rust-lang/crates.io-index" 613 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 614 | dependencies = [ 615 | "hermit-abi 0.3.4", 616 | "libc", 617 | ] 618 | 619 | [[package]] 620 | name = "object" 621 | version = "0.32.2" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" 624 | dependencies = [ 625 | "memchr", 626 | ] 627 | 628 | [[package]] 629 | name = "once_cell" 630 | version = "1.19.0" 631 | source = "registry+https://github.com/rust-lang/crates.io-index" 632 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 633 | 634 | [[package]] 635 | name = "openssl" 636 | version = "0.10.63" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" 639 | dependencies = [ 640 | "bitflags 2.4.2", 641 | "cfg-if", 642 | "foreign-types", 643 | "libc", 644 | "once_cell", 645 | "openssl-macros", 646 | "openssl-sys", 647 | ] 648 | 649 | [[package]] 650 | name = "openssl-macros" 651 | version = "0.1.1" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" 654 | dependencies = [ 655 | "proc-macro2", 656 | "quote", 657 | "syn", 658 | ] 659 | 660 | [[package]] 661 | name = "openssl-probe" 662 | version = "0.1.5" 663 | source = "registry+https://github.com/rust-lang/crates.io-index" 664 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" 665 | 666 | [[package]] 667 | name = "openssl-sys" 668 | version = "0.9.99" 669 | source = "registry+https://github.com/rust-lang/crates.io-index" 670 | checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" 671 | dependencies = [ 672 | "cc", 673 | "libc", 674 | "pkg-config", 675 | "vcpkg", 676 | ] 677 | 678 | [[package]] 679 | name = "parking_lot" 680 | version = "0.12.1" 681 | source = "registry+https://github.com/rust-lang/crates.io-index" 682 | checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" 683 | dependencies = [ 684 | "lock_api", 685 | "parking_lot_core", 686 | ] 687 | 688 | [[package]] 689 | name = "parking_lot_core" 690 | version = "0.9.9" 691 | source = "registry+https://github.com/rust-lang/crates.io-index" 692 | checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" 693 | dependencies = [ 694 | "cfg-if", 695 | "libc", 696 | "redox_syscall", 697 | "smallvec", 698 | "windows-targets 0.48.5", 699 | ] 700 | 701 | [[package]] 702 | name = "percent-encoding" 703 | version = "2.3.1" 704 | source = "registry+https://github.com/rust-lang/crates.io-index" 705 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 706 | 707 | [[package]] 708 | name = "pin-project" 709 | version = "1.1.4" 710 | source = "registry+https://github.com/rust-lang/crates.io-index" 711 | checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" 712 | dependencies = [ 713 | "pin-project-internal", 714 | ] 715 | 716 | [[package]] 717 | name = "pin-project-internal" 718 | version = "1.1.4" 719 | source = "registry+https://github.com/rust-lang/crates.io-index" 720 | checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" 721 | dependencies = [ 722 | "proc-macro2", 723 | "quote", 724 | "syn", 725 | ] 726 | 727 | [[package]] 728 | name = "pin-project-lite" 729 | version = "0.2.13" 730 | source = "registry+https://github.com/rust-lang/crates.io-index" 731 | checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" 732 | 733 | [[package]] 734 | name = "pin-utils" 735 | version = "0.1.0" 736 | source = "registry+https://github.com/rust-lang/crates.io-index" 737 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 738 | 739 | [[package]] 740 | name = "pkg-config" 741 | version = "0.3.29" 742 | source = "registry+https://github.com/rust-lang/crates.io-index" 743 | checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" 744 | 745 | [[package]] 746 | name = "ppv-lite86" 747 | version = "0.2.17" 748 | source = "registry+https://github.com/rust-lang/crates.io-index" 749 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" 750 | 751 | [[package]] 752 | name = "proc-macro2" 753 | version = "1.0.78" 754 | source = "registry+https://github.com/rust-lang/crates.io-index" 755 | checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" 756 | dependencies = [ 757 | "unicode-ident", 758 | ] 759 | 760 | [[package]] 761 | name = "quote" 762 | version = "1.0.35" 763 | source = "registry+https://github.com/rust-lang/crates.io-index" 764 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" 765 | dependencies = [ 766 | "proc-macro2", 767 | ] 768 | 769 | [[package]] 770 | name = "raft" 771 | version = "0.1.0" 772 | dependencies = [ 773 | "env_logger", 774 | "log", 775 | "rand", 776 | "reqwest", 777 | "serde", 778 | "serde_json", 779 | "tokio", 780 | "uuid", 781 | "warp", 782 | ] 783 | 784 | [[package]] 785 | name = "rand" 786 | version = "0.8.5" 787 | source = "registry+https://github.com/rust-lang/crates.io-index" 788 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 789 | dependencies = [ 790 | "libc", 791 | "rand_chacha", 792 | "rand_core", 793 | ] 794 | 795 | [[package]] 796 | name = "rand_chacha" 797 | version = "0.3.1" 798 | source = "registry+https://github.com/rust-lang/crates.io-index" 799 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 800 | dependencies = [ 801 | "ppv-lite86", 802 | "rand_core", 803 | ] 804 | 805 | [[package]] 806 | name = "rand_core" 807 | version = "0.6.4" 808 | source = "registry+https://github.com/rust-lang/crates.io-index" 809 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 810 | dependencies = [ 811 | "getrandom", 812 | ] 813 | 814 | [[package]] 815 | name = "redox_syscall" 816 | version = "0.4.1" 817 | source = "registry+https://github.com/rust-lang/crates.io-index" 818 | checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" 819 | dependencies = [ 820 | "bitflags 1.3.2", 821 | ] 822 | 823 | [[package]] 824 | name = "regex" 825 | version = "1.10.3" 826 | source = "registry+https://github.com/rust-lang/crates.io-index" 827 | checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" 828 | dependencies = [ 829 | "aho-corasick", 830 | "memchr", 831 | "regex-automata", 832 | "regex-syntax", 833 | ] 834 | 835 | [[package]] 836 | name = "regex-automata" 837 | version = "0.4.4" 838 | source = "registry+https://github.com/rust-lang/crates.io-index" 839 | checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" 840 | dependencies = [ 841 | "aho-corasick", 842 | "memchr", 843 | "regex-syntax", 844 | ] 845 | 846 | [[package]] 847 | name = "regex-syntax" 848 | version = "0.8.2" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" 851 | 852 | [[package]] 853 | name = "reqwest" 854 | version = "0.11.23" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" 857 | dependencies = [ 858 | "base64", 859 | "bytes", 860 | "encoding_rs", 861 | "futures-core", 862 | "futures-util", 863 | "h2", 864 | "http", 865 | "http-body", 866 | "hyper", 867 | "hyper-tls", 868 | "ipnet", 869 | "js-sys", 870 | "log", 871 | "mime", 872 | "native-tls", 873 | "once_cell", 874 | "percent-encoding", 875 | "pin-project-lite", 876 | "serde", 877 | "serde_json", 878 | "serde_urlencoded", 879 | "system-configuration", 880 | "tokio", 881 | "tokio-native-tls", 882 | "tower-service", 883 | "url", 884 | "wasm-bindgen", 885 | "wasm-bindgen-futures", 886 | "web-sys", 887 | "winreg", 888 | ] 889 | 890 | [[package]] 891 | name = "rustc-demangle" 892 | version = "0.1.23" 893 | source = "registry+https://github.com/rust-lang/crates.io-index" 894 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" 895 | 896 | [[package]] 897 | name = "rustix" 898 | version = "0.38.30" 899 | source = "registry+https://github.com/rust-lang/crates.io-index" 900 | checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" 901 | dependencies = [ 902 | "bitflags 2.4.2", 903 | "errno", 904 | "libc", 905 | "linux-raw-sys", 906 | "windows-sys 0.52.0", 907 | ] 908 | 909 | [[package]] 910 | name = "rustls-pemfile" 911 | version = "1.0.4" 912 | source = "registry+https://github.com/rust-lang/crates.io-index" 913 | checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" 914 | dependencies = [ 915 | "base64", 916 | ] 917 | 918 | [[package]] 919 | name = "ryu" 920 | version = "1.0.16" 921 | source = "registry+https://github.com/rust-lang/crates.io-index" 922 | checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" 923 | 924 | [[package]] 925 | name = "schannel" 926 | version = "0.1.23" 927 | source = "registry+https://github.com/rust-lang/crates.io-index" 928 | checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" 929 | dependencies = [ 930 | "windows-sys 0.52.0", 931 | ] 932 | 933 | [[package]] 934 | name = "scoped-tls" 935 | version = "1.0.1" 936 | source = "registry+https://github.com/rust-lang/crates.io-index" 937 | checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" 938 | 939 | [[package]] 940 | name = "scopeguard" 941 | version = "1.2.0" 942 | source = "registry+https://github.com/rust-lang/crates.io-index" 943 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 944 | 945 | [[package]] 946 | name = "security-framework" 947 | version = "2.9.2" 948 | source = "registry+https://github.com/rust-lang/crates.io-index" 949 | checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" 950 | dependencies = [ 951 | "bitflags 1.3.2", 952 | "core-foundation", 953 | "core-foundation-sys", 954 | "libc", 955 | "security-framework-sys", 956 | ] 957 | 958 | [[package]] 959 | name = "security-framework-sys" 960 | version = "2.9.1" 961 | source = "registry+https://github.com/rust-lang/crates.io-index" 962 | checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" 963 | dependencies = [ 964 | "core-foundation-sys", 965 | "libc", 966 | ] 967 | 968 | [[package]] 969 | name = "serde" 970 | version = "1.0.195" 971 | source = "registry+https://github.com/rust-lang/crates.io-index" 972 | checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" 973 | dependencies = [ 974 | "serde_derive", 975 | ] 976 | 977 | [[package]] 978 | name = "serde_derive" 979 | version = "1.0.195" 980 | source = "registry+https://github.com/rust-lang/crates.io-index" 981 | checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" 982 | dependencies = [ 983 | "proc-macro2", 984 | "quote", 985 | "syn", 986 | ] 987 | 988 | [[package]] 989 | name = "serde_json" 990 | version = "1.0.111" 991 | source = "registry+https://github.com/rust-lang/crates.io-index" 992 | checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" 993 | dependencies = [ 994 | "itoa", 995 | "ryu", 996 | "serde", 997 | ] 998 | 999 | [[package]] 1000 | name = "serde_urlencoded" 1001 | version = "0.7.1" 1002 | source = "registry+https://github.com/rust-lang/crates.io-index" 1003 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 1004 | dependencies = [ 1005 | "form_urlencoded", 1006 | "itoa", 1007 | "ryu", 1008 | "serde", 1009 | ] 1010 | 1011 | [[package]] 1012 | name = "sha1" 1013 | version = "0.10.6" 1014 | source = "registry+https://github.com/rust-lang/crates.io-index" 1015 | checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" 1016 | dependencies = [ 1017 | "cfg-if", 1018 | "cpufeatures", 1019 | "digest", 1020 | ] 1021 | 1022 | [[package]] 1023 | name = "signal-hook-registry" 1024 | version = "1.4.1" 1025 | source = "registry+https://github.com/rust-lang/crates.io-index" 1026 | checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" 1027 | dependencies = [ 1028 | "libc", 1029 | ] 1030 | 1031 | [[package]] 1032 | name = "slab" 1033 | version = "0.4.9" 1034 | source = "registry+https://github.com/rust-lang/crates.io-index" 1035 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 1036 | dependencies = [ 1037 | "autocfg", 1038 | ] 1039 | 1040 | [[package]] 1041 | name = "smallvec" 1042 | version = "1.13.1" 1043 | source = "registry+https://github.com/rust-lang/crates.io-index" 1044 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" 1045 | 1046 | [[package]] 1047 | name = "socket2" 1048 | version = "0.5.5" 1049 | source = "registry+https://github.com/rust-lang/crates.io-index" 1050 | checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" 1051 | dependencies = [ 1052 | "libc", 1053 | "windows-sys 0.48.0", 1054 | ] 1055 | 1056 | [[package]] 1057 | name = "spin" 1058 | version = "0.9.8" 1059 | source = "registry+https://github.com/rust-lang/crates.io-index" 1060 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" 1061 | 1062 | [[package]] 1063 | name = "syn" 1064 | version = "2.0.48" 1065 | source = "registry+https://github.com/rust-lang/crates.io-index" 1066 | checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" 1067 | dependencies = [ 1068 | "proc-macro2", 1069 | "quote", 1070 | "unicode-ident", 1071 | ] 1072 | 1073 | [[package]] 1074 | name = "system-configuration" 1075 | version = "0.5.1" 1076 | source = "registry+https://github.com/rust-lang/crates.io-index" 1077 | checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" 1078 | dependencies = [ 1079 | "bitflags 1.3.2", 1080 | "core-foundation", 1081 | "system-configuration-sys", 1082 | ] 1083 | 1084 | [[package]] 1085 | name = "system-configuration-sys" 1086 | version = "0.5.0" 1087 | source = "registry+https://github.com/rust-lang/crates.io-index" 1088 | checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" 1089 | dependencies = [ 1090 | "core-foundation-sys", 1091 | "libc", 1092 | ] 1093 | 1094 | [[package]] 1095 | name = "tempfile" 1096 | version = "3.9.0" 1097 | source = "registry+https://github.com/rust-lang/crates.io-index" 1098 | checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" 1099 | dependencies = [ 1100 | "cfg-if", 1101 | "fastrand", 1102 | "redox_syscall", 1103 | "rustix", 1104 | "windows-sys 0.52.0", 1105 | ] 1106 | 1107 | [[package]] 1108 | name = "termcolor" 1109 | version = "1.4.1" 1110 | source = "registry+https://github.com/rust-lang/crates.io-index" 1111 | checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" 1112 | dependencies = [ 1113 | "winapi-util", 1114 | ] 1115 | 1116 | [[package]] 1117 | name = "thiserror" 1118 | version = "1.0.56" 1119 | source = "registry+https://github.com/rust-lang/crates.io-index" 1120 | checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" 1121 | dependencies = [ 1122 | "thiserror-impl", 1123 | ] 1124 | 1125 | [[package]] 1126 | name = "thiserror-impl" 1127 | version = "1.0.56" 1128 | source = "registry+https://github.com/rust-lang/crates.io-index" 1129 | checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" 1130 | dependencies = [ 1131 | "proc-macro2", 1132 | "quote", 1133 | "syn", 1134 | ] 1135 | 1136 | [[package]] 1137 | name = "tinyvec" 1138 | version = "1.6.0" 1139 | source = "registry+https://github.com/rust-lang/crates.io-index" 1140 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 1141 | dependencies = [ 1142 | "tinyvec_macros", 1143 | ] 1144 | 1145 | [[package]] 1146 | name = "tinyvec_macros" 1147 | version = "0.1.1" 1148 | source = "registry+https://github.com/rust-lang/crates.io-index" 1149 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 1150 | 1151 | [[package]] 1152 | name = "tokio" 1153 | version = "1.35.1" 1154 | source = "registry+https://github.com/rust-lang/crates.io-index" 1155 | checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" 1156 | dependencies = [ 1157 | "backtrace", 1158 | "bytes", 1159 | "libc", 1160 | "mio", 1161 | "num_cpus", 1162 | "parking_lot", 1163 | "pin-project-lite", 1164 | "signal-hook-registry", 1165 | "socket2", 1166 | "tokio-macros", 1167 | "windows-sys 0.48.0", 1168 | ] 1169 | 1170 | [[package]] 1171 | name = "tokio-macros" 1172 | version = "2.2.0" 1173 | source = "registry+https://github.com/rust-lang/crates.io-index" 1174 | checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" 1175 | dependencies = [ 1176 | "proc-macro2", 1177 | "quote", 1178 | "syn", 1179 | ] 1180 | 1181 | [[package]] 1182 | name = "tokio-native-tls" 1183 | version = "0.3.1" 1184 | source = "registry+https://github.com/rust-lang/crates.io-index" 1185 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" 1186 | dependencies = [ 1187 | "native-tls", 1188 | "tokio", 1189 | ] 1190 | 1191 | [[package]] 1192 | name = "tokio-stream" 1193 | version = "0.1.14" 1194 | source = "registry+https://github.com/rust-lang/crates.io-index" 1195 | checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" 1196 | dependencies = [ 1197 | "futures-core", 1198 | "pin-project-lite", 1199 | "tokio", 1200 | ] 1201 | 1202 | [[package]] 1203 | name = "tokio-tungstenite" 1204 | version = "0.20.1" 1205 | source = "registry+https://github.com/rust-lang/crates.io-index" 1206 | checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" 1207 | dependencies = [ 1208 | "futures-util", 1209 | "log", 1210 | "tokio", 1211 | "tungstenite", 1212 | ] 1213 | 1214 | [[package]] 1215 | name = "tokio-util" 1216 | version = "0.7.10" 1217 | source = "registry+https://github.com/rust-lang/crates.io-index" 1218 | checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" 1219 | dependencies = [ 1220 | "bytes", 1221 | "futures-core", 1222 | "futures-sink", 1223 | "pin-project-lite", 1224 | "tokio", 1225 | "tracing", 1226 | ] 1227 | 1228 | [[package]] 1229 | name = "tower-service" 1230 | version = "0.3.2" 1231 | source = "registry+https://github.com/rust-lang/crates.io-index" 1232 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" 1233 | 1234 | [[package]] 1235 | name = "tracing" 1236 | version = "0.1.40" 1237 | source = "registry+https://github.com/rust-lang/crates.io-index" 1238 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" 1239 | dependencies = [ 1240 | "log", 1241 | "pin-project-lite", 1242 | "tracing-core", 1243 | ] 1244 | 1245 | [[package]] 1246 | name = "tracing-core" 1247 | version = "0.1.32" 1248 | source = "registry+https://github.com/rust-lang/crates.io-index" 1249 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" 1250 | dependencies = [ 1251 | "once_cell", 1252 | ] 1253 | 1254 | [[package]] 1255 | name = "try-lock" 1256 | version = "0.2.5" 1257 | source = "registry+https://github.com/rust-lang/crates.io-index" 1258 | checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" 1259 | 1260 | [[package]] 1261 | name = "tungstenite" 1262 | version = "0.20.1" 1263 | source = "registry+https://github.com/rust-lang/crates.io-index" 1264 | checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" 1265 | dependencies = [ 1266 | "byteorder", 1267 | "bytes", 1268 | "data-encoding", 1269 | "http", 1270 | "httparse", 1271 | "log", 1272 | "rand", 1273 | "sha1", 1274 | "thiserror", 1275 | "url", 1276 | "utf-8", 1277 | ] 1278 | 1279 | [[package]] 1280 | name = "typenum" 1281 | version = "1.17.0" 1282 | source = "registry+https://github.com/rust-lang/crates.io-index" 1283 | checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" 1284 | 1285 | [[package]] 1286 | name = "unicase" 1287 | version = "2.7.0" 1288 | source = "registry+https://github.com/rust-lang/crates.io-index" 1289 | checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" 1290 | dependencies = [ 1291 | "version_check", 1292 | ] 1293 | 1294 | [[package]] 1295 | name = "unicode-bidi" 1296 | version = "0.3.15" 1297 | source = "registry+https://github.com/rust-lang/crates.io-index" 1298 | checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" 1299 | 1300 | [[package]] 1301 | name = "unicode-ident" 1302 | version = "1.0.12" 1303 | source = "registry+https://github.com/rust-lang/crates.io-index" 1304 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 1305 | 1306 | [[package]] 1307 | name = "unicode-normalization" 1308 | version = "0.1.22" 1309 | source = "registry+https://github.com/rust-lang/crates.io-index" 1310 | checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" 1311 | dependencies = [ 1312 | "tinyvec", 1313 | ] 1314 | 1315 | [[package]] 1316 | name = "url" 1317 | version = "2.5.0" 1318 | source = "registry+https://github.com/rust-lang/crates.io-index" 1319 | checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" 1320 | dependencies = [ 1321 | "form_urlencoded", 1322 | "idna", 1323 | "percent-encoding", 1324 | ] 1325 | 1326 | [[package]] 1327 | name = "utf-8" 1328 | version = "0.7.6" 1329 | source = "registry+https://github.com/rust-lang/crates.io-index" 1330 | checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" 1331 | 1332 | [[package]] 1333 | name = "uuid" 1334 | version = "1.7.0" 1335 | source = "registry+https://github.com/rust-lang/crates.io-index" 1336 | checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" 1337 | dependencies = [ 1338 | "getrandom", 1339 | "serde", 1340 | ] 1341 | 1342 | [[package]] 1343 | name = "vcpkg" 1344 | version = "0.2.15" 1345 | source = "registry+https://github.com/rust-lang/crates.io-index" 1346 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1347 | 1348 | [[package]] 1349 | name = "version_check" 1350 | version = "0.9.4" 1351 | source = "registry+https://github.com/rust-lang/crates.io-index" 1352 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 1353 | 1354 | [[package]] 1355 | name = "want" 1356 | version = "0.3.1" 1357 | source = "registry+https://github.com/rust-lang/crates.io-index" 1358 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" 1359 | dependencies = [ 1360 | "try-lock", 1361 | ] 1362 | 1363 | [[package]] 1364 | name = "warp" 1365 | version = "0.3.6" 1366 | source = "registry+https://github.com/rust-lang/crates.io-index" 1367 | checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" 1368 | dependencies = [ 1369 | "bytes", 1370 | "futures-channel", 1371 | "futures-util", 1372 | "headers", 1373 | "http", 1374 | "hyper", 1375 | "log", 1376 | "mime", 1377 | "mime_guess", 1378 | "multer", 1379 | "percent-encoding", 1380 | "pin-project", 1381 | "rustls-pemfile", 1382 | "scoped-tls", 1383 | "serde", 1384 | "serde_json", 1385 | "serde_urlencoded", 1386 | "tokio", 1387 | "tokio-stream", 1388 | "tokio-tungstenite", 1389 | "tokio-util", 1390 | "tower-service", 1391 | "tracing", 1392 | ] 1393 | 1394 | [[package]] 1395 | name = "wasi" 1396 | version = "0.11.0+wasi-snapshot-preview1" 1397 | source = "registry+https://github.com/rust-lang/crates.io-index" 1398 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1399 | 1400 | [[package]] 1401 | name = "wasm-bindgen" 1402 | version = "0.2.90" 1403 | source = "registry+https://github.com/rust-lang/crates.io-index" 1404 | checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" 1405 | dependencies = [ 1406 | "cfg-if", 1407 | "wasm-bindgen-macro", 1408 | ] 1409 | 1410 | [[package]] 1411 | name = "wasm-bindgen-backend" 1412 | version = "0.2.90" 1413 | source = "registry+https://github.com/rust-lang/crates.io-index" 1414 | checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" 1415 | dependencies = [ 1416 | "bumpalo", 1417 | "log", 1418 | "once_cell", 1419 | "proc-macro2", 1420 | "quote", 1421 | "syn", 1422 | "wasm-bindgen-shared", 1423 | ] 1424 | 1425 | [[package]] 1426 | name = "wasm-bindgen-futures" 1427 | version = "0.4.40" 1428 | source = "registry+https://github.com/rust-lang/crates.io-index" 1429 | checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" 1430 | dependencies = [ 1431 | "cfg-if", 1432 | "js-sys", 1433 | "wasm-bindgen", 1434 | "web-sys", 1435 | ] 1436 | 1437 | [[package]] 1438 | name = "wasm-bindgen-macro" 1439 | version = "0.2.90" 1440 | source = "registry+https://github.com/rust-lang/crates.io-index" 1441 | checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" 1442 | dependencies = [ 1443 | "quote", 1444 | "wasm-bindgen-macro-support", 1445 | ] 1446 | 1447 | [[package]] 1448 | name = "wasm-bindgen-macro-support" 1449 | version = "0.2.90" 1450 | source = "registry+https://github.com/rust-lang/crates.io-index" 1451 | checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" 1452 | dependencies = [ 1453 | "proc-macro2", 1454 | "quote", 1455 | "syn", 1456 | "wasm-bindgen-backend", 1457 | "wasm-bindgen-shared", 1458 | ] 1459 | 1460 | [[package]] 1461 | name = "wasm-bindgen-shared" 1462 | version = "0.2.90" 1463 | source = "registry+https://github.com/rust-lang/crates.io-index" 1464 | checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" 1465 | 1466 | [[package]] 1467 | name = "web-sys" 1468 | version = "0.3.67" 1469 | source = "registry+https://github.com/rust-lang/crates.io-index" 1470 | checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" 1471 | dependencies = [ 1472 | "js-sys", 1473 | "wasm-bindgen", 1474 | ] 1475 | 1476 | [[package]] 1477 | name = "winapi" 1478 | version = "0.3.9" 1479 | source = "registry+https://github.com/rust-lang/crates.io-index" 1480 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1481 | dependencies = [ 1482 | "winapi-i686-pc-windows-gnu", 1483 | "winapi-x86_64-pc-windows-gnu", 1484 | ] 1485 | 1486 | [[package]] 1487 | name = "winapi-i686-pc-windows-gnu" 1488 | version = "0.4.0" 1489 | source = "registry+https://github.com/rust-lang/crates.io-index" 1490 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1491 | 1492 | [[package]] 1493 | name = "winapi-util" 1494 | version = "0.1.6" 1495 | source = "registry+https://github.com/rust-lang/crates.io-index" 1496 | checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" 1497 | dependencies = [ 1498 | "winapi", 1499 | ] 1500 | 1501 | [[package]] 1502 | name = "winapi-x86_64-pc-windows-gnu" 1503 | version = "0.4.0" 1504 | source = "registry+https://github.com/rust-lang/crates.io-index" 1505 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1506 | 1507 | [[package]] 1508 | name = "windows-sys" 1509 | version = "0.48.0" 1510 | source = "registry+https://github.com/rust-lang/crates.io-index" 1511 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 1512 | dependencies = [ 1513 | "windows-targets 0.48.5", 1514 | ] 1515 | 1516 | [[package]] 1517 | name = "windows-sys" 1518 | version = "0.52.0" 1519 | source = "registry+https://github.com/rust-lang/crates.io-index" 1520 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 1521 | dependencies = [ 1522 | "windows-targets 0.52.0", 1523 | ] 1524 | 1525 | [[package]] 1526 | name = "windows-targets" 1527 | version = "0.48.5" 1528 | source = "registry+https://github.com/rust-lang/crates.io-index" 1529 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 1530 | dependencies = [ 1531 | "windows_aarch64_gnullvm 0.48.5", 1532 | "windows_aarch64_msvc 0.48.5", 1533 | "windows_i686_gnu 0.48.5", 1534 | "windows_i686_msvc 0.48.5", 1535 | "windows_x86_64_gnu 0.48.5", 1536 | "windows_x86_64_gnullvm 0.48.5", 1537 | "windows_x86_64_msvc 0.48.5", 1538 | ] 1539 | 1540 | [[package]] 1541 | name = "windows-targets" 1542 | version = "0.52.0" 1543 | source = "registry+https://github.com/rust-lang/crates.io-index" 1544 | checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" 1545 | dependencies = [ 1546 | "windows_aarch64_gnullvm 0.52.0", 1547 | "windows_aarch64_msvc 0.52.0", 1548 | "windows_i686_gnu 0.52.0", 1549 | "windows_i686_msvc 0.52.0", 1550 | "windows_x86_64_gnu 0.52.0", 1551 | "windows_x86_64_gnullvm 0.52.0", 1552 | "windows_x86_64_msvc 0.52.0", 1553 | ] 1554 | 1555 | [[package]] 1556 | name = "windows_aarch64_gnullvm" 1557 | version = "0.48.5" 1558 | source = "registry+https://github.com/rust-lang/crates.io-index" 1559 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 1560 | 1561 | [[package]] 1562 | name = "windows_aarch64_gnullvm" 1563 | version = "0.52.0" 1564 | source = "registry+https://github.com/rust-lang/crates.io-index" 1565 | checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" 1566 | 1567 | [[package]] 1568 | name = "windows_aarch64_msvc" 1569 | version = "0.48.5" 1570 | source = "registry+https://github.com/rust-lang/crates.io-index" 1571 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 1572 | 1573 | [[package]] 1574 | name = "windows_aarch64_msvc" 1575 | version = "0.52.0" 1576 | source = "registry+https://github.com/rust-lang/crates.io-index" 1577 | checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" 1578 | 1579 | [[package]] 1580 | name = "windows_i686_gnu" 1581 | version = "0.48.5" 1582 | source = "registry+https://github.com/rust-lang/crates.io-index" 1583 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 1584 | 1585 | [[package]] 1586 | name = "windows_i686_gnu" 1587 | version = "0.52.0" 1588 | source = "registry+https://github.com/rust-lang/crates.io-index" 1589 | checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" 1590 | 1591 | [[package]] 1592 | name = "windows_i686_msvc" 1593 | version = "0.48.5" 1594 | source = "registry+https://github.com/rust-lang/crates.io-index" 1595 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 1596 | 1597 | [[package]] 1598 | name = "windows_i686_msvc" 1599 | version = "0.52.0" 1600 | source = "registry+https://github.com/rust-lang/crates.io-index" 1601 | checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" 1602 | 1603 | [[package]] 1604 | name = "windows_x86_64_gnu" 1605 | version = "0.48.5" 1606 | source = "registry+https://github.com/rust-lang/crates.io-index" 1607 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 1608 | 1609 | [[package]] 1610 | name = "windows_x86_64_gnu" 1611 | version = "0.52.0" 1612 | source = "registry+https://github.com/rust-lang/crates.io-index" 1613 | checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" 1614 | 1615 | [[package]] 1616 | name = "windows_x86_64_gnullvm" 1617 | version = "0.48.5" 1618 | source = "registry+https://github.com/rust-lang/crates.io-index" 1619 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 1620 | 1621 | [[package]] 1622 | name = "windows_x86_64_gnullvm" 1623 | version = "0.52.0" 1624 | source = "registry+https://github.com/rust-lang/crates.io-index" 1625 | checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" 1626 | 1627 | [[package]] 1628 | name = "windows_x86_64_msvc" 1629 | version = "0.48.5" 1630 | source = "registry+https://github.com/rust-lang/crates.io-index" 1631 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 1632 | 1633 | [[package]] 1634 | name = "windows_x86_64_msvc" 1635 | version = "0.52.0" 1636 | source = "registry+https://github.com/rust-lang/crates.io-index" 1637 | checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" 1638 | 1639 | [[package]] 1640 | name = "winreg" 1641 | version = "0.50.0" 1642 | source = "registry+https://github.com/rust-lang/crates.io-index" 1643 | checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" 1644 | dependencies = [ 1645 | "cfg-if", 1646 | "windows-sys 0.48.0", 1647 | ] 1648 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "raft" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tokio = { version = "1.35.1", features = ["full"] } 10 | serde = { version = "1.0", features = ["derive"] } 11 | serde_json = "1.0" 12 | log = "0.4" 13 | env_logger = "0.9" 14 | uuid = { version = "1.7.0", features=["v4", "serde"] } 15 | rand = "0.8.5" 16 | warp = "0.3" 17 | reqwest = "0.11" 18 | 19 | [[bin]] 20 | name = "client" 21 | path = "bin/client.rs" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Raft 2 | This repo is a minimalistic reproduction of Raft for educational purposes. 3 | 4 | The classic Raft paper - https://raft.github.io/raft.pdf. It's probably one of the more readable papers I've come across. Pair this with the [Raft visualisation](https://thesecretlivesofdata.com/raft/) which is a great tool to understand Raft. 5 | 6 | ## The Code Structure 7 | 8 | Most of the code lives in the `main.rs` file currently, which is by design since this is an educational project. The only things I placed outside of the `main.rs` file were the clock module and the storage module. There is also a `types.rs` file for the more complicated types. 9 | 10 | I separated out the storage layers and the clock layers because I was unsure what I was doing with it initially so I wanted to hide it behind an API and worry about the implementation later. 11 | 12 | The clock injection itself wasn't a big deal actually but I ended up doing a plain-text CSV file with no indexing for storage which will probably be a pretty big problem at scale. This is still something I need to fix. 13 | 14 | ## The Main Components 15 | 16 | There are 3 main components to a Raft cluster 17 | 18 | 1. Storage (represented in the storage module) 19 | 2. Clocks (represented in the `clock.rs` file) 20 | 3. Networking (represented as the RPCManager struct in `main.rs`) 21 | 4. Nodes or servers (represented as the RaftNode struct in `main.rs`) 22 | 23 | The node struct contains most of the logic around the RPC methods (shown in the screenshot below) 24 | 25 | ![](/assets/raft-rpc.png) 26 | 27 | ## Testing 28 | 29 | I've ended up creating a little bit of monstrosity in terms of testing because I ended up going with creating explicit scenarios. But, I've tried as far as I can to create a deterministic state machine that can be tested by mocking the networking layer and the clocks. This allows me to realiably forward time and drop messages between nodes to re-create real-world situation. 30 | 31 | The testing logic is housed within the `TestCluster` struct in `main.rs`. The tests using this struct attempt to re-create explicit scenarios like a concurrent leader election and network partition to ensure the logic works correctly. 32 | 33 | ## Where From Here 34 | 35 | Currently the code is able to handle: 36 | 37 | * Leader Election & Stability 38 | * Log Replication 39 | * Durable storage and recovery from storage 40 | * Network partition recovery 41 | 42 | What I have yet to do: 43 | 44 | * Log snapshots 45 | * Benchmarks 46 | 47 | ## Credits 48 | 49 | A huge part of accomplishing this was being able to reference existing repos that do a Raft implementation 50 | 51 | * Phil Eaton's Raft implementation in Rust (https://github.com/eatonphil/raft-rs) 52 | * Jacky Zhao's Raft implementation also in Rust (https://github.com/jackyzha0/miniraft) 53 | * TLA+ spec of Raft (https://github.com/ongardie/raft.tla/tree/master) 54 | 55 | A large part of the testing inspiration came from the second repo. -------------------------------------------------------------------------------- /assets/raft-rpc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redixhumayun/raft/8b5f73a7e35aa82cf2aa14f4f467abbcec96c145/assets/raft-rpc.png -------------------------------------------------------------------------------- /src/clock.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | pub trait Clock { 4 | fn now(&self) -> Instant; 5 | } 6 | 7 | pub struct RealClock; 8 | impl Clock for RealClock { 9 | fn now(&self) -> Instant { 10 | Instant::now() 11 | } 12 | } 13 | 14 | pub struct MockClock { 15 | pub current_time: Instant, 16 | } 17 | 18 | impl MockClock { 19 | pub fn advance(&mut self, duration: Duration) { 20 | self.current_time += duration; 21 | } 22 | } 23 | 24 | impl Clock for MockClock { 25 | fn now(&self) -> Instant { 26 | self.current_time 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] // TODO: Remove this later. It's getting too noisy now. 2 | 3 | use clock::{Clock, MockClock, RealClock}; 4 | use core::fmt; 5 | use serde::de::DeserializeOwned; 6 | use serde_json; 7 | use std::cell::{Ref, RefCell}; 8 | use std::io::{self, BufRead, Write}; 9 | use std::net::{TcpListener, TcpStream}; 10 | use std::str::FromStr; 11 | use std::sync::atomic::AtomicBool; 12 | use std::sync::mpsc::{Receiver, Sender}; 13 | use std::sync::{mpsc, Arc, Mutex, MutexGuard}; 14 | use std::thread; 15 | use std::time::{Duration, Instant}; 16 | use std::{collections::HashMap, io::BufReader}; 17 | use types::RaftTypeTrait; 18 | 19 | use log::{debug, error, info}; 20 | use rand::Rng; 21 | use serde::{Deserialize, Serialize}; 22 | 23 | mod clock; 24 | mod storage; 25 | mod types; 26 | 27 | use crate::types::{ServerId, Term}; 28 | use storage::RaftFileOps; 29 | 30 | /** 31 | * RPC Stuff 32 | */ 33 | 34 | trait Communication { 35 | fn start(&self); 36 | 37 | fn stop(&self); 38 | 39 | fn send_message(&self, to_address: String, message: MessageWrapper); 40 | } 41 | 42 | #[derive(Debug, Clone)] 43 | struct MessageWrapper { 44 | from_node_id: ServerId, 45 | to_node_id: ServerId, 46 | message: RPCMessage, 47 | } 48 | 49 | struct MockRPCManager { 50 | server_id: ServerId, 51 | to_node_sender: mpsc::Sender>, 52 | sent_messages: RefCell>>, 53 | } 54 | 55 | impl MockRPCManager { 56 | fn new(server_id: ServerId, to_node_sender: mpsc::Sender>) -> Self { 57 | MockRPCManager { 58 | server_id, 59 | to_node_sender, 60 | sent_messages: RefCell::new(vec![]), 61 | } 62 | } 63 | } 64 | 65 | impl MockRPCManager { 66 | fn start(&self) {} 67 | 68 | fn stop(&self) {} 69 | 70 | fn send_message(&self, _to_address: String, message: MessageWrapper) { 71 | self.sent_messages.borrow_mut().push(message); 72 | } 73 | 74 | fn get_messages_in_queue(&mut self) -> Vec> { 75 | let mut mock_messages_vector: Vec> = Vec::new(); 76 | for message in self.sent_messages.borrow_mut().drain(..) { 77 | mock_messages_vector.push(message.clone()); 78 | } 79 | mock_messages_vector 80 | } 81 | 82 | fn replay_messages_in_queue(&self) -> Ref>> { 83 | self.sent_messages.borrow() 84 | } 85 | } 86 | 87 | struct RPCManager { 88 | server_id: ServerId, 89 | server_address: String, 90 | port: u64, 91 | to_node_sender: mpsc::Sender>, 92 | is_running: Arc, 93 | } 94 | 95 | impl RPCManager { 96 | fn new( 97 | server_id: ServerId, 98 | server_address: String, 99 | port: u64, 100 | to_node_sender: mpsc::Sender>, 101 | ) -> Self { 102 | RPCManager { 103 | server_id, 104 | server_address, 105 | port, 106 | to_node_sender, 107 | is_running: Arc::new(AtomicBool::new(false)), 108 | } 109 | } 110 | 111 | fn start(&self) { 112 | let server_address = self.server_address.clone(); 113 | let server_id = self.server_id.clone(); 114 | let to_node_sender = self.to_node_sender.clone(); 115 | let is_running = self.is_running.clone(); 116 | is_running.store(true, std::sync::atomic::Ordering::SeqCst); 117 | thread::spawn(move || { 118 | let listener = match TcpListener::bind(&server_address) { 119 | Ok(tcp_listener) => tcp_listener, 120 | Err(e) => { 121 | panic!( 122 | "There was an error while binding to the address {} for server id {} {}", 123 | server_address, server_id, e 124 | ); 125 | } 126 | }; 127 | 128 | for stream in listener.incoming() { 129 | if !is_running.load(std::sync::atomic::Ordering::SeqCst) { 130 | break; 131 | } 132 | match stream { 133 | Ok(stream) => { 134 | let reader = BufReader::new(stream); 135 | for line in reader.lines() { 136 | let json = match line { 137 | Ok(l) => l, 138 | Err(e) => { 139 | panic!("There was an error while reading the line {}", e); 140 | } 141 | }; 142 | let rpc: RPCMessage = match serde_json::from_str(&json) { 143 | Ok(message) => message, 144 | Err(e) => { 145 | panic!( 146 | "There was an error while deserializing the message {}", 147 | e 148 | ); 149 | } 150 | }; 151 | match to_node_sender.send(rpc) { 152 | Ok(_) => (), 153 | Err(e) => { 154 | panic!( 155 | "There was an error while sending the rpc message to the node {}", 156 | e 157 | ); 158 | } 159 | } 160 | } 161 | } 162 | Err(e) => { 163 | if !is_running.load(std::sync::atomic::Ordering::SeqCst) { 164 | break; 165 | } 166 | error!("There was an error while accepting a connection {}", e); 167 | } 168 | } 169 | } 170 | }); 171 | } 172 | 173 | fn stop(&self) { 174 | self.is_running 175 | .store(false, std::sync::atomic::Ordering::SeqCst); 176 | } 177 | 178 | /** 179 | * This method is called from the raft node to allow it to communicate with other nodes 180 | * via the RPC manager. 181 | */ 182 | fn send_message(&self, to_address: String, message: MessageWrapper) { 183 | info!( 184 | "Sending a message from server {} to server {} and the message is {:?}", 185 | message.from_node_id, message.to_node_id, message.message 186 | ); 187 | let mut stream = match TcpStream::connect(to_address) { 188 | Ok(stream) => stream, 189 | Err(e) => { 190 | panic!( 191 | "There was an error while connecting to the server {} {}", 192 | message.to_node_id, e 193 | ); 194 | } 195 | }; 196 | 197 | let serialized_request = match serde_json::to_string(&message.message) { 198 | Ok(serialized_message) => serialized_message, 199 | Err(e) => { 200 | panic!("There was an error while serializing the message {}", e); 201 | } 202 | }; 203 | 204 | if let Err(e) = stream.write_all(serialized_request.as_bytes()) { 205 | panic!("There was an error while sending the message {}", e); 206 | } 207 | } 208 | } 209 | 210 | enum CommunicationLayer { 211 | MockRPCManager(MockRPCManager), 212 | RPCManager(RPCManager), 213 | } 214 | 215 | impl Communication for CommunicationLayer { 216 | fn start(&self) { 217 | match self { 218 | CommunicationLayer::MockRPCManager(manager) => manager.start(), 219 | CommunicationLayer::RPCManager(manager) => manager.start(), 220 | } 221 | } 222 | 223 | fn stop(&self) { 224 | match self { 225 | CommunicationLayer::MockRPCManager(manager) => manager.stop(), 226 | CommunicationLayer::RPCManager(manager) => manager.stop(), 227 | } 228 | } 229 | 230 | fn send_message(&self, to_address: String, message: MessageWrapper) { 231 | match self { 232 | CommunicationLayer::MockRPCManager(manager) => { 233 | manager.send_message(to_address, message) 234 | } 235 | CommunicationLayer::RPCManager(manager) => manager.send_message(to_address, message), 236 | } 237 | } 238 | } 239 | 240 | impl CommunicationLayer { 241 | fn get_messages(&mut self) -> Vec> { 242 | match self { 243 | CommunicationLayer::MockRPCManager(manager) => { 244 | return manager.get_messages_in_queue(); 245 | } 246 | CommunicationLayer::RPCManager(_) => { 247 | panic!("This method is not supported for the RPCManager"); 248 | } 249 | } 250 | } 251 | 252 | fn replay_messages(&self) -> Ref>> { 253 | match self { 254 | CommunicationLayer::MockRPCManager(manager) => { 255 | return manager.replay_messages_in_queue(); 256 | } 257 | CommunicationLayer::RPCManager(_) => { 258 | panic!("This method is not supported for the RPCManager"); 259 | } 260 | } 261 | } 262 | } 263 | 264 | #[derive(Serialize, Deserialize, Debug, Clone)] 265 | enum RPCMessage { 266 | VoteRequest(VoteRequest), 267 | VoteResponse(VoteResponse), 268 | AppendEntriesRequest(AppendEntriesRequest), 269 | AppendEntriesResponse(AppendEntriesResponse), 270 | } 271 | 272 | impl PartialEq for RPCMessage { 273 | fn eq(&self, other: &Self) -> bool { 274 | match self { 275 | RPCMessage::VoteRequest(request) => { 276 | if let RPCMessage::VoteRequest(other_request) = other { 277 | return request == other_request; 278 | } 279 | return false; 280 | } 281 | RPCMessage::VoteResponse(response) => { 282 | if let RPCMessage::VoteResponse(other_response) = other { 283 | return response == other_response; 284 | } 285 | return false; 286 | } 287 | RPCMessage::AppendEntriesRequest(request) => { 288 | if let RPCMessage::AppendEntriesRequest(other_request) = other { 289 | return request == other_request; 290 | } 291 | return false; 292 | } 293 | RPCMessage::AppendEntriesResponse(response) => { 294 | if let RPCMessage::AppendEntriesResponse(other_response) = other { 295 | return response == other_response; 296 | } 297 | return false; 298 | } 299 | } 300 | } 301 | } 302 | 303 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 304 | struct VoteRequest { 305 | request_id: u16, 306 | term: Term, 307 | candidate_id: ServerId, 308 | last_log_index: u64, 309 | last_log_term: Term, 310 | } 311 | 312 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 313 | struct VoteResponse { 314 | request_id: u16, 315 | term: Term, 316 | vote_granted: bool, 317 | candidate_id: ServerId, // the id of the server granting the vote, used for de-duplication 318 | } 319 | 320 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 321 | struct AppendEntriesRequest { 322 | request_id: u16, 323 | term: Term, 324 | leader_id: ServerId, 325 | prev_log_index: u64, 326 | prev_log_term: Term, 327 | entries: Vec>, 328 | leader_commit_index: u64, 329 | } 330 | 331 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 332 | struct AppendEntriesResponse { 333 | request_id: u16, 334 | server_id: ServerId, 335 | term: Term, 336 | success: bool, 337 | match_index: u64, 338 | } 339 | /** 340 | * End RPC stuff 341 | */ 342 | 343 | #[derive(Debug)] 344 | struct ServerConfig { 345 | election_timeout: Duration, // used to calculate the actual election timeout 346 | heartbeat_interval: Duration, 347 | address: String, 348 | port: u64, 349 | cluster_nodes: Vec, 350 | id_to_address_mapping: HashMap, 351 | } 352 | 353 | trait StateMachine { 354 | fn apply_set(&self, key: String, value: T); 355 | fn apply_get(&self, key: &str) -> Option; 356 | fn apply(&self, entries: Vec>); 357 | } 358 | 359 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 360 | enum LogEntryCommand { 361 | Set = 0, 362 | Delete = 1, 363 | } 364 | 365 | impl FromStr for LogEntryCommand { 366 | type Err = io::Error; 367 | 368 | fn from_str(s: &str) -> Result { 369 | let v: u64 = s 370 | .trim() 371 | .parse() 372 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 373 | match v { 374 | 0 => Ok(LogEntryCommand::Set), 375 | 1 => Ok(LogEntryCommand::Delete), 376 | _ => Err(io::Error::new( 377 | io::ErrorKind::InvalidData, 378 | "Invalid log entry command", 379 | )), 380 | } 381 | } 382 | } 383 | 384 | impl fmt::Display for LogEntryCommand { 385 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 386 | match self { 387 | LogEntryCommand::Set => write!(f, "0"), 388 | LogEntryCommand::Delete => write!(f, "1"), 389 | } 390 | } 391 | } 392 | 393 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 394 | struct LogEntry { 395 | term: Term, 396 | index: u64, 397 | command: LogEntryCommand, 398 | key: String, 399 | value: T, 400 | } 401 | 402 | #[derive(PartialEq, Debug)] 403 | enum RaftNodeStatus { 404 | Leader, 405 | Candidate, 406 | Follower, 407 | } 408 | 409 | enum RaftNodeClock { 410 | RealClock(RealClock), 411 | MockClock(MockClock), 412 | } 413 | 414 | impl RaftNodeClock { 415 | fn advance(&mut self, duration: Duration) { 416 | match self { 417 | RaftNodeClock::RealClock(_) => (), // this method should do nothing for a real clock 418 | RaftNodeClock::MockClock(clock) => clock.advance(duration), 419 | } 420 | } 421 | } 422 | 423 | impl Clock for RaftNodeClock { 424 | fn now(&self) -> Instant { 425 | match self { 426 | RaftNodeClock::RealClock(clock) => clock.now(), 427 | RaftNodeClock::MockClock(clock) => clock.now(), 428 | } 429 | } 430 | } 431 | 432 | struct RaftNodeState { 433 | // persistent state on all servers 434 | current_term: Term, 435 | voted_for: Option, 436 | log: Vec>, 437 | 438 | // volatile state on all servers 439 | commit_index: u64, 440 | last_applied: u64, 441 | status: RaftNodeStatus, 442 | 443 | // volatile state for leaders 444 | next_index: Vec, 445 | match_index: Vec, 446 | 447 | // election variables 448 | votes_received: HashMap, 449 | election_timeout: Duration, 450 | last_heartbeat: Instant, 451 | } 452 | 453 | struct RaftNode, F: RaftFileOps> { 454 | id: ServerId, 455 | state: Mutex>, 456 | state_machine: S, 457 | config: ServerConfig, 458 | peers: Vec, 459 | to_node_sender: mpsc::Sender>, 460 | from_rpc_receiver: mpsc::Receiver>, 461 | rpc_manager: CommunicationLayer, 462 | persistence_manager: F, 463 | clock: RaftNodeClock, 464 | } 465 | 466 | impl + Send, F: RaftFileOps + Send> RaftNode { 467 | /** 468 | * \* Message handlers 469 | \* i = recipient, j = sender, m = message 470 | 471 | \* Server i receives a RequestVote request from server j with 472 | \* m.mterm <= currentTerm[i]. 473 | HandleRequestVoteRequest(i, j, m) == 474 | LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) 475 | \/ /\ m.mlastLogTerm = LastTerm(log[i]) 476 | /\ m.mlastLogIndex >= Len(log[i]) 477 | grant == /\ m.mterm = currentTerm[i] 478 | /\ logOk 479 | /\ votedFor[i] \in {Nil, j} 480 | IN /\ m.mterm <= currentTerm[i] 481 | /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] 482 | \/ ~grant /\ UNCHANGED votedFor 483 | /\ Reply([mtype |-> RequestVoteResponse, 484 | mterm |-> currentTerm[i], 485 | mvoteGranted |-> grant, 486 | \* mlog is used just for the `elections' history variable for 487 | \* the proof. It would not exist in a real implementation. 488 | mlog |-> log[i], 489 | msource |-> i, 490 | mdest |-> j], 491 | m) 492 | /\ UNCHANGED <> 493 | */ 494 | fn handle_vote_request(&mut self, request: &VoteRequest) -> VoteResponse { 495 | let mut state_guard = self.state.lock().unwrap(); 496 | debug!("ENTER: handle_vote_request on node {}", self.id); 497 | 498 | // basic term check first - if the request term is lower ignore the request 499 | if request.term < state_guard.current_term { 500 | debug!("EXIT: handle_vote_request on node {}. Not granting the vote because request term is lower", self.id); 501 | return VoteResponse { 502 | request_id: request.request_id, 503 | term: state_guard.current_term, 504 | vote_granted: false, 505 | candidate_id: self.id, 506 | }; 507 | } 508 | 509 | // this node has already voted for someone else in this term and it is not the requesting node 510 | if state_guard.voted_for != None && state_guard.voted_for != Some(request.candidate_id) { 511 | debug!("EXIT: handle_vote_request on node {}. Not granting the vote because the node has already voted for someone else", self.id); 512 | return VoteResponse { 513 | request_id: request.request_id, 514 | term: state_guard.current_term, 515 | vote_granted: false, 516 | candidate_id: self.id, 517 | }; 518 | } 519 | 520 | let last_log_term = state_guard.log.last().map_or(0, |entry| entry.term); 521 | let last_log_index = state_guard.log.last().map_or(0, |entry| entry.index); 522 | 523 | let log_check = request.last_log_term > last_log_term 524 | || (request.last_log_term == last_log_term && request.last_log_index >= last_log_index); 525 | 526 | if !log_check { 527 | return VoteResponse { 528 | request_id: request.request_id, 529 | term: state_guard.current_term, 530 | vote_granted: false, 531 | candidate_id: self.id, 532 | }; 533 | } 534 | 535 | // all checks have passed - grant the vote but only after recording it. Also write it to disk 536 | state_guard.voted_for = Some(request.candidate_id); 537 | match self 538 | .persistence_manager 539 | .write_term_and_voted_for(state_guard.current_term, state_guard.voted_for) 540 | { 541 | Ok(()) => (), 542 | Err(e) => { 543 | error!("There was a problem writing the term and voted_for variables to stable storage for node {}: {}", self.id, e); 544 | } 545 | } 546 | debug!( 547 | "EXIT: handle_vote_request on node {}. Granting the vote for {}", 548 | self.id, request.candidate_id 549 | ); 550 | VoteResponse { 551 | request_id: request.request_id, 552 | term: state_guard.current_term, 553 | vote_granted: true, 554 | candidate_id: self.id, 555 | } 556 | } 557 | 558 | /** 559 | * \* Server i receives a RequestVote response from server j with 560 | \* m.mterm = currentTerm[i]. 561 | HandleRequestVoteResponse(i, j, m) == 562 | \* This tallies votes even when the current state is not Candidate, but 563 | \* they won't be looked at, so it doesn't matter. 564 | /\ m.mterm = currentTerm[i] 565 | /\ votesResponded' = [votesResponded EXCEPT ![i] = 566 | votesResponded[i] \cup {j}] 567 | /\ \/ /\ m.mvoteGranted 568 | /\ votesGranted' = [votesGranted EXCEPT ![i] = 569 | votesGranted[i] \cup {j}] 570 | /\ voterLog' = [voterLog EXCEPT ![i] = 571 | voterLog[i] @@ (j :> m.mlog)] 572 | \/ /\ ~m.mvoteGranted 573 | /\ UNCHANGED <> 574 | /\ Discard(m) 575 | /\ UNCHANGED <> 576 | */ 577 | fn handle_vote_response(&mut self, vote_response: VoteResponse) { 578 | let mut state_guard = self.state.lock().unwrap(); 579 | 580 | if state_guard.status != RaftNodeStatus::Candidate { 581 | return; 582 | } 583 | 584 | if !vote_response.vote_granted && vote_response.term > state_guard.current_term { 585 | // the term is different, update term, downgrade to follower and persist to local storage 586 | state_guard.current_term = vote_response.term; 587 | state_guard.status = RaftNodeStatus::Follower; 588 | match self 589 | .persistence_manager 590 | .write_term_and_voted_for(state_guard.current_term, Option::None) 591 | { 592 | Ok(()) => (), 593 | Err(e) => { 594 | error!("There was a problem writing the term and voted_for variables to stable storage for node {}: {}", self.id, e); 595 | } 596 | } 597 | return; 598 | } 599 | 600 | // the vote wasn't granted, the reason is unknown 601 | if !vote_response.vote_granted { 602 | return; 603 | } 604 | 605 | // the vote was granted - check if node achieved quorum 606 | state_guard 607 | .votes_received 608 | .insert(vote_response.candidate_id, vote_response.vote_granted); 609 | 610 | if self.can_become_leader(&state_guard) { 611 | self.become_leader(&mut state_guard); 612 | } 613 | } 614 | 615 | fn handle_append_entries_request( 616 | &mut self, 617 | request: &AppendEntriesRequest, 618 | ) -> AppendEntriesResponse { 619 | let mut state_guard = self.state.lock().unwrap(); 620 | debug!("ENTER: handle_append_entries_request for node {}", self.id); 621 | 622 | // the term check 623 | if request.term < state_guard.current_term { 624 | debug!("The term in the request is less than the current term, ignoring the request"); 625 | debug!("EXIT: handle_append_entries_request for node {}", self.id); 626 | return AppendEntriesResponse { 627 | request_id: request.request_id, 628 | server_id: self.id, 629 | term: state_guard.current_term, 630 | success: false, 631 | match_index: state_guard.log.last().map_or(0, |entry| entry.index), 632 | }; 633 | } 634 | 635 | self.reset_election_timeout(&mut state_guard); 636 | 637 | // log consistency check - check that the term and the index match at the log entry the leader expects 638 | let log_index_to_check = request.prev_log_index.saturating_sub(1) as usize; 639 | let prev_log_term = state_guard 640 | .log 641 | .get(log_index_to_check) 642 | .map_or(0, |entry| entry.term); 643 | let prev_log_index = state_guard 644 | .log 645 | .get(log_index_to_check) 646 | .map_or(0, |entry| entry.index); 647 | let log_ok = request.prev_log_index == 0 648 | || (request.prev_log_index > 0 649 | && request.prev_log_index <= state_guard.log.len() as u64 650 | && request.prev_log_term == prev_log_term); 651 | 652 | debug!("The message has prev_log_index: {} and prev_log_term: {} and the node's log has prev_log_index: {} and prev_log_term: {}", request.prev_log_index, request.prev_log_term, prev_log_index, prev_log_term); 653 | // the log check is not OK, give false response 654 | if !log_ok { 655 | debug!("The node's log is {:?}", state_guard.log); 656 | debug!("The log check failed because request has prev_log_index as {} and prev_log_term as {} and the node has prev_log_index as {} and prev_log_term as {}", request.prev_log_index, request.prev_log_term, state_guard.log.len(), prev_log_term); 657 | debug!("EXIT: handle_append_entries_request for node {}", self.id); 658 | return AppendEntriesResponse { 659 | request_id: request.request_id, 660 | server_id: self.id, 661 | term: state_guard.current_term, 662 | success: false, 663 | match_index: state_guard.log.last().map_or(0, |entry| entry.index), 664 | }; 665 | } 666 | 667 | if request.entries.len() == 0 { 668 | // this is a heartbeat message 669 | debug!("This is a heartbeat message, returning success"); 670 | state_guard.commit_index = request.leader_commit_index; 671 | self.apply_entries(&mut state_guard); 672 | debug!("EXIT: handle_append_entries_request for node {}", self.id); 673 | return AppendEntriesResponse { 674 | request_id: request.request_id, 675 | server_id: self.id, 676 | term: state_guard.current_term, 677 | success: true, 678 | match_index: state_guard.log.last().map_or(0, |entry| entry.index), 679 | }; 680 | } 681 | 682 | // if there are any subsequent logs on the follower, truncate them and append the new logs 683 | if self.len_as_u64(&state_guard.log) > request.prev_log_index { 684 | state_guard.log.truncate(request.prev_log_index as usize); 685 | state_guard.log.extend_from_slice(&request.entries); 686 | if let Err(e) = self 687 | .persistence_manager 688 | .append_logs_at(&request.entries, request.prev_log_index.saturating_sub(1)) 689 | { 690 | error!("There was a problem appending logs to stable storage for node {} at position {}: {}", self.id, request.prev_log_index - 1, e); 691 | } 692 | return AppendEntriesResponse { 693 | request_id: request.request_id, 694 | server_id: self.id, 695 | term: state_guard.current_term, 696 | success: true, 697 | match_index: state_guard.log.last().map_or(0, |entry| entry.index), 698 | }; 699 | } 700 | 701 | // There are no subsequent logs on the follower, so no possibility of conflicts. Which means the logs can just be appended 702 | // and the response can be returned 703 | state_guard.log.append(&mut request.entries.clone()); 704 | if let Err(e) = self.persistence_manager.append_logs(&request.entries) { 705 | error!( 706 | "There was a problem appending logs to stable storage for node {}: {}", 707 | self.id, e 708 | ); 709 | } 710 | 711 | debug!( 712 | "The new match index is {}", 713 | state_guard.log.last().map_or(0, |entry| entry.index) 714 | ); 715 | debug!("EXIT: handle_append_entries_request for node {}", self.id); 716 | AppendEntriesResponse { 717 | request_id: request.request_id, 718 | server_id: self.id, 719 | term: state_guard.current_term, 720 | success: true, 721 | match_index: state_guard.log.last().map_or(0, |entry| entry.index), 722 | } 723 | } 724 | 725 | fn handle_append_entries_response(&mut self, response: AppendEntriesResponse) { 726 | let mut state_guard = self.state.lock().unwrap(); 727 | if state_guard.status != RaftNodeStatus::Leader { 728 | return; 729 | } 730 | 731 | let server_index = response.server_id as usize; 732 | 733 | if !response.success { 734 | // reduce the next index for that server and try again 735 | debug!("The append entry request was not successful, decrementing next_index and retrying request"); 736 | state_guard.next_index[server_index] = state_guard.next_index[server_index] 737 | .saturating_sub(1) 738 | .max(1); 739 | 740 | self.retry_append_request(&mut state_guard, server_index, response.server_id); 741 | return; 742 | } 743 | 744 | // the response is successful 745 | debug!( 746 | "The response was successful, updating next index and match index for server {} and the response is {:?}", 747 | response.server_id, response 748 | ); 749 | debug!( 750 | "The old set of next indices {:?} and match indices{:?}", 751 | state_guard.next_index, state_guard.match_index 752 | ); 753 | state_guard.next_index[(response.server_id) as usize] = response.match_index + 1; 754 | state_guard.match_index[(response.server_id) as usize] = response.match_index; 755 | debug!( 756 | "The new set of next indices {:?} and match indices {:?}", 757 | state_guard.next_index, state_guard.match_index 758 | ); 759 | self.advance_commit_index(&mut state_guard); 760 | self.apply_entries(&mut state_guard); 761 | } 762 | 763 | // Utility functions for main RPC's 764 | /** 765 | * BecomeLeader(i) == 766 | /\ state[i] = Candidate 767 | /\ votesGranted[i] \in Quorum 768 | /\ state' = [state EXCEPT ![i] = Leader] 769 | /\ nextIndex' = [nextIndex EXCEPT ![i] = 770 | [j \in Server |-> Len(log[i]) + 1]] 771 | /\ matchIndex' = [matchIndex EXCEPT ![i] = 772 | [j \in Server |-> 0]] 773 | /\ elections' = elections \cup 774 | {[eterm |-> currentTerm[i], 775 | eleader |-> i, 776 | elog |-> log[i], 777 | evotes |-> votesGranted[i], 778 | evoterLog |-> voterLog[i]]} 779 | /\ UNCHANGED <> 780 | */ 781 | fn become_leader(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 782 | info!("Node {} has become the leader", self.id); 783 | state_guard.status = RaftNodeStatus::Leader; 784 | let last_log_index = state_guard.log.last().map_or(0, |entry| entry.index); 785 | state_guard.next_index = self 786 | .config 787 | .cluster_nodes 788 | .iter() 789 | .map(|_| last_log_index + 1) 790 | .collect(); 791 | state_guard.match_index = vec![0; self.config.cluster_nodes.len()]; 792 | } 793 | 794 | fn retry_append_request( 795 | &self, 796 | state_guard: &mut MutexGuard<'_, RaftNodeState>, 797 | server_index: usize, 798 | to_server_id: u64, 799 | ) { 800 | let next_index_of_follower = state_guard.next_index[server_index] as usize; 801 | let new_next_index_of_follower = next_index_of_follower.saturating_sub(1).max(1); 802 | state_guard.next_index[server_index] = new_next_index_of_follower as u64; 803 | 804 | debug!( 805 | "The next index of follower {} was {} and now it is {}", 806 | to_server_id, next_index_of_follower, new_next_index_of_follower 807 | ); 808 | let entries_to_send = state_guard 809 | .log 810 | .get(new_next_index_of_follower - 1..) 811 | .unwrap() 812 | .to_vec(); 813 | let prev_log_index = if new_next_index_of_follower > 1 { 814 | state_guard 815 | .log 816 | .get(new_next_index_of_follower - 1) 817 | .unwrap() 818 | .index 819 | } else { 820 | 0 821 | }; 822 | let prev_log_term = if new_next_index_of_follower > 1 { 823 | state_guard 824 | .log 825 | .get(new_next_index_of_follower - 1) 826 | .unwrap() 827 | .term 828 | } else { 829 | 0 830 | }; 831 | 832 | let request = AppendEntriesRequest { 833 | request_id: rand::thread_rng().gen(), 834 | term: state_guard.current_term, 835 | leader_id: self.id, 836 | prev_log_index, 837 | prev_log_term, 838 | entries: entries_to_send, 839 | leader_commit_index: state_guard.commit_index, 840 | }; 841 | debug!("Retrying append entries with request {:?}", request); 842 | let message = RPCMessage::::AppendEntriesRequest(request); 843 | let to_address = self 844 | .config 845 | .id_to_address_mapping 846 | .get(&to_server_id) 847 | .unwrap(); 848 | let message_wrapper = MessageWrapper { 849 | from_node_id: self.id, 850 | to_node_id: to_server_id, 851 | message, 852 | }; 853 | self.rpc_manager 854 | .send_message(to_address.clone(), message_wrapper); 855 | } 856 | 857 | /** 858 | * \* Leader i advances its commitIndex. 859 | \* This is done as a separate step from handling AppendEntries responses, 860 | \* in part to minimize atomic regions, and in part so that leaders of 861 | \* single-server clusters are able to mark entries committed. 862 | AdvanceCommitIndex(i) == 863 | /\ state[i] = Leader 864 | /\ LET \* The set of servers that agree up through index. 865 | Agree(index) == {i} \cup {k \in Server : 866 | matchIndex[i][k] >= index} 867 | \* The maximum indexes for which a quorum agrees 868 | agreeIndexes == {index \in 1..Len(log[i]) : 869 | Agree(index) \in Quorum} 870 | \* New value for commitIndex'[i] 871 | newCommitIndex == 872 | IF /\ agreeIndexes /= {} 873 | /\ log[i][Max(agreeIndexes)].term = currentTerm[i] 874 | THEN 875 | Max(agreeIndexes) 876 | ELSE 877 | commitIndex[i] 878 | IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] 879 | /\ UNCHANGED <> 880 | */ 881 | fn advance_commit_index(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 882 | debug!("ENTER: advance_commit_index for node {}", self.id); 883 | assert!(state_guard.status == RaftNodeStatus::Leader); 884 | 885 | // find all match indexes that have quorum 886 | let mut match_index_count: HashMap = HashMap::new(); 887 | for &server_match_index in &state_guard.match_index { 888 | *match_index_count.entry(server_match_index).or_insert(0) += 1; 889 | } 890 | debug!("The match index count: {:?}", match_index_count); 891 | 892 | let new_commit_index = match_index_count 893 | .iter() 894 | .filter(|(&match_index, &count)| { 895 | count >= self.quorum_size().try_into().unwrap() 896 | && state_guard 897 | .log 898 | .get((match_index as usize).saturating_sub(1)) 899 | .map_or(false, |entry| entry.term == state_guard.current_term) 900 | }) 901 | .map(|(&match_index, _)| match_index) 902 | .max(); 903 | 904 | debug!( 905 | "The old commit index was: {} and the new commit index is: {}", 906 | state_guard.commit_index, 907 | new_commit_index.unwrap_or(state_guard.commit_index) 908 | ); 909 | if let Some(max_index) = new_commit_index { 910 | state_guard.commit_index = max_index; 911 | } 912 | debug!("EXIT: advance_commit_index for node {}", self.id); 913 | } 914 | 915 | /// This function will check whether there are any more entries that can be applied to the state machine for the node 916 | fn apply_entries(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 917 | debug!("ENTER: apply_entries for node {}", self.id); 918 | let mut entries_to_apply: Vec> = Vec::new(); 919 | debug!( 920 | "last_applied: {}, commit_index: {}", 921 | state_guard.last_applied, state_guard.commit_index 922 | ); 923 | while state_guard.last_applied < state_guard.commit_index { 924 | debug!("Updating the state machine for node {}", self.id); 925 | debug!("The current log is {:?}", state_guard.log); 926 | state_guard.last_applied += 1; 927 | let entry_at_index = state_guard 928 | .log 929 | .get((state_guard.last_applied - 1) as usize) 930 | .unwrap(); 931 | entries_to_apply.push(entry_at_index.clone()); 932 | } 933 | 934 | if entries_to_apply.len() > 0 { 935 | self.state_machine.apply(entries_to_apply); 936 | } 937 | debug!("EXIT: apply_entries for node {}", self.id); 938 | } 939 | 940 | /** 941 | * \* Server i restarts from stable storage. 942 | \* It loses everything but its currentTerm, votedFor, and log. 943 | Restart(i) == 944 | /\ state' = [state EXCEPT ![i] = Follower] 945 | /\ votesResponded' = [votesResponded EXCEPT ![i] = {}] 946 | /\ votesGranted' = [votesGranted EXCEPT ![i] = {}] 947 | /\ voterLog' = [voterLog EXCEPT ![i] = [j \in {} |-> <<>>]] 948 | /\ nextIndex' = [nextIndex EXCEPT ![i] = [j \in Server |-> 1]] 949 | /\ matchIndex' = [matchIndex EXCEPT ![i] = [j \in Server |-> 0]] 950 | /\ commitIndex' = [commitIndex EXCEPT ![i] = 0] 951 | /\ UNCHANGED <> 952 | */ 953 | fn restart(&mut self) { 954 | let mut state_guard = self.state.lock().unwrap(); 955 | state_guard.status = RaftNodeStatus::Follower; 956 | state_guard.next_index = vec![1; self.config.cluster_nodes.len()]; 957 | state_guard.match_index = vec![0; self.config.cluster_nodes.len()]; 958 | state_guard.commit_index = 0; 959 | state_guard.votes_received = HashMap::new(); 960 | 961 | let (term, voted_for) = match self.persistence_manager.read_term_and_voted_for() { 962 | Ok((t, v)) => (t, v), 963 | Err(e) => { 964 | panic!( 965 | "There was an error while reading the term and voted for {}", 966 | e 967 | ); 968 | } 969 | }; 970 | state_guard.current_term = term; 971 | state_guard.voted_for = Some(voted_for); 972 | 973 | let log_entries = match self.persistence_manager.read_logs(1) { 974 | Ok(entries) => entries, 975 | Err(e) => { 976 | panic!("There was an error while reading the logs {}", e); 977 | } 978 | }; 979 | state_guard.log = log_entries; 980 | } 981 | 982 | /// Any RPC with a newer term should force the node to advance its own term, reset to a Follower and set its voted_for to None 983 | /// before continuing to process the message 984 | fn update_term(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>, new_term: Term) { 985 | debug!("ENTER: update_term for node {}", self.id); 986 | state_guard.current_term = new_term; 987 | state_guard.voted_for = None; 988 | state_guard.status = RaftNodeStatus::Follower; 989 | 990 | if let Err(e) = self 991 | .persistence_manager 992 | .write_term_and_voted_for(new_term, None) 993 | { 994 | // valid to panic here because this is a write to disk 995 | panic!( 996 | "There was an error while writing the term and voted for to stable storage {}", 997 | e 998 | ); 999 | } 1000 | debug!("EXIT: update_term for node {}", self.id); 1001 | } 1002 | 1003 | fn update_node_term_if_required(&mut self, message_wrapper: MessageWrapper) { 1004 | let mut state_guard = self.state.lock().unwrap(); 1005 | match message_wrapper.message { 1006 | RPCMessage::VoteRequest(message) => { 1007 | if message.term > state_guard.current_term { 1008 | self.update_term(&mut state_guard, message.term); 1009 | } 1010 | } 1011 | RPCMessage::VoteResponse(message) => { 1012 | if message.term > state_guard.current_term { 1013 | self.update_term(&mut state_guard, message.term); 1014 | } 1015 | } 1016 | RPCMessage::AppendEntriesRequest(message) => { 1017 | if message.term > state_guard.current_term { 1018 | self.update_term(&mut state_guard, message.term); 1019 | } 1020 | } 1021 | RPCMessage::AppendEntriesResponse(message) => { 1022 | if message.term > state_guard.current_term { 1023 | self.update_term(&mut state_guard, message.term); 1024 | } 1025 | } 1026 | } 1027 | } 1028 | 1029 | fn is_message_stale(&self, message_wrapper: MessageWrapper) -> bool { 1030 | let state_guard = self.state.lock().unwrap(); 1031 | match message_wrapper.message { 1032 | RPCMessage::VoteRequest(message) => message.term < state_guard.current_term, 1033 | RPCMessage::VoteResponse(message) => message.term < state_guard.current_term, 1034 | RPCMessage::AppendEntriesRequest(message) => message.term < state_guard.current_term, 1035 | RPCMessage::AppendEntriesResponse(message) => message.term < state_guard.current_term, 1036 | } 1037 | } 1038 | /** 1039 | * Receive(m) == 1040 | LET i == m.mdest 1041 | j == m.msource 1042 | IN \* Any RPC with a newer term causes the recipient to advance 1043 | \* its term first. Responses with stale terms are ignored. 1044 | \/ UpdateTerm(i, j, m) 1045 | \/ /\ m.mtype = RequestVoteRequest 1046 | /\ HandleRequestVoteRequest(i, j, m) 1047 | \/ /\ m.mtype = RequestVoteResponse 1048 | /\ \/ DropStaleResponse(i, j, m) 1049 | \/ HandleRequestVoteResponse(i, j, m) 1050 | \/ /\ m.mtype = AppendEntriesRequest 1051 | /\ HandleAppendEntriesRequest(i, j, m) 1052 | \/ /\ m.mtype = AppendEntriesResponse 1053 | /\ \/ DropStaleResponse(i, j, m) 1054 | \/ HandleAppendEntriesResponse(i, j, m) 1055 | */ 1056 | fn receive(&mut self, message_wrapper: MessageWrapper) -> Option<(RPCMessage, ServerId)> { 1057 | self.update_node_term_if_required(message_wrapper.clone()); 1058 | let message_wrapper_clone = message_wrapper.clone(); 1059 | match message_wrapper.message { 1060 | RPCMessage::VoteRequest(request) => { 1061 | let response = self.handle_vote_request(&request); 1062 | Some((RPCMessage::VoteResponse(response), request.candidate_id)) 1063 | } 1064 | RPCMessage::VoteResponse(response) => { 1065 | if self.is_message_stale(message_wrapper_clone.clone()) { 1066 | return None; 1067 | } 1068 | self.handle_vote_response(response); 1069 | None 1070 | } 1071 | RPCMessage::AppendEntriesRequest(request) => { 1072 | let response = self.handle_append_entries_request(&request); 1073 | Some(( 1074 | RPCMessage::AppendEntriesResponse(response), 1075 | request.leader_id, 1076 | )) 1077 | } 1078 | RPCMessage::AppendEntriesResponse(response) => { 1079 | if self.is_message_stale(message_wrapper_clone.clone()) { 1080 | return None; 1081 | } 1082 | self.handle_append_entries_response(response); 1083 | None 1084 | } 1085 | } 1086 | } 1087 | 1088 | fn can_become_leader(&self, state_guard: &MutexGuard<'_, RaftNodeState>) -> bool { 1089 | let sum_of_votes_received = state_guard 1090 | .votes_received 1091 | .iter() 1092 | .filter(|(_, vote_granted)| **vote_granted) 1093 | .count(); 1094 | let quorum = self.quorum_size(); 1095 | sum_of_votes_received >= quorum 1096 | } 1097 | 1098 | /// This method will cause a node to elevate itself to candidate, cast a vote for itself, write that 1099 | /// vote to storage and then send out messages to all other peers requesting votes 1100 | fn start_election(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 1101 | debug!("ENTER: start_election for node {}", self.id); 1102 | state_guard.status = RaftNodeStatus::Candidate; 1103 | state_guard.current_term += 1; 1104 | state_guard.voted_for = Some(self.id); 1105 | state_guard.votes_received.insert(self.id, true); 1106 | if let Err(e) = self 1107 | .persistence_manager 1108 | .write_term_and_voted_for(state_guard.current_term, state_guard.voted_for) 1109 | { 1110 | panic!( 1111 | "There was an error while writing the term and voted for to stable storage {}", 1112 | e 1113 | ); 1114 | } 1115 | 1116 | // Check to ensure that a single node system will be immediately elected a leader. Feels a bit hacky 1117 | if self.can_become_leader(state_guard) { 1118 | self.become_leader(state_guard); 1119 | debug!( 1120 | "EXIT: start_election and node {} has become the leader", 1121 | self.id 1122 | ); 1123 | return; 1124 | } 1125 | 1126 | for peer in &self.peers { 1127 | if peer == &self.id { 1128 | continue; 1129 | } 1130 | // send out a vote request to all the remaining nodes 1131 | let vote_request = VoteRequest { 1132 | request_id: rand::thread_rng().gen(), 1133 | term: state_guard.current_term, 1134 | candidate_id: self.id, 1135 | last_log_index: state_guard.log.last().map_or(0, |entry| entry.index), 1136 | last_log_term: state_guard.log.last().map_or(0, |entry| entry.term), 1137 | }; 1138 | let message = RPCMessage::::VoteRequest(vote_request); 1139 | let to_address = self.config.id_to_address_mapping.get(&peer).expect( 1140 | format!("Cannot find the id to address mapping for peer id {}", peer).as_str(), 1141 | ); 1142 | let message_wrapper = MessageWrapper { 1143 | from_node_id: self.id, 1144 | to_node_id: *peer, 1145 | message, 1146 | }; 1147 | self.rpc_manager 1148 | .send_message(to_address.clone(), message_wrapper); 1149 | } 1150 | debug!("EXIT: start_election for node {}", self.id); 1151 | } 1152 | 1153 | fn send_heartbeat(&self, state_guard: &MutexGuard<'_, RaftNodeState>) { 1154 | if state_guard.status != RaftNodeStatus::Leader { 1155 | return; 1156 | } 1157 | 1158 | for peer in &self.peers { 1159 | if *peer == self.id { 1160 | // ignore self referencing node 1161 | continue; 1162 | } 1163 | 1164 | let heartbeat_request = AppendEntriesRequest { 1165 | request_id: rand::thread_rng().gen(), 1166 | term: state_guard.current_term, 1167 | leader_id: self.id, 1168 | prev_log_index: state_guard.log.last().map_or(0, |entry| entry.index), 1169 | prev_log_term: state_guard.log.last().map_or(0, |entry| entry.term), 1170 | entries: Vec::>::new(), 1171 | leader_commit_index: state_guard.commit_index, 1172 | }; 1173 | let message = RPCMessage::::AppendEntriesRequest(heartbeat_request); 1174 | let to_address = self.config.id_to_address_mapping.get(peer).expect( 1175 | format!("Cannot find the id to address mapping for peer id {}", peer).as_str(), 1176 | ); 1177 | let message_wrapper = MessageWrapper { 1178 | from_node_id: self.id, 1179 | to_node_id: *peer, 1180 | message, 1181 | }; 1182 | self.rpc_manager 1183 | .send_message(to_address.clone(), message_wrapper); 1184 | } 1185 | } 1186 | 1187 | fn reset_election_timeout(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 1188 | state_guard.last_heartbeat = self.clock.now(); 1189 | } 1190 | 1191 | fn check_election_timeout(&self, state_guard: &mut MutexGuard<'_, RaftNodeState>) { 1192 | if (self.clock.now() - state_guard.last_heartbeat) >= state_guard.election_timeout { 1193 | match state_guard.status { 1194 | RaftNodeStatus::Leader => { 1195 | // do nothing 1196 | } 1197 | RaftNodeStatus::Candidate => { 1198 | state_guard.status = RaftNodeStatus::Follower; 1199 | self.reset_election_timeout(state_guard); 1200 | self.start_election(state_guard); 1201 | } 1202 | RaftNodeStatus::Follower => { 1203 | self.reset_election_timeout(state_guard); 1204 | self.start_election(state_guard); 1205 | } 1206 | } 1207 | } 1208 | } 1209 | 1210 | /// Returns the log entry at a given index 1211 | fn get_log_entry_at( 1212 | &self, 1213 | state_guard: &MutexGuard<'_, RaftNodeState>, 1214 | log_entry_index: u32, 1215 | ) -> LogEntry { 1216 | let log_entry = state_guard.log.get(log_entry_index as usize).unwrap(); 1217 | return log_entry.clone(); 1218 | } 1219 | 1220 | /// Returns the last log entry 1221 | fn get_last_log_entry(&self) -> LogEntry { 1222 | let state_guard = self.state.lock().unwrap(); 1223 | return self.get_log_entry_at(&state_guard, state_guard.log.len() as u32 - 1); 1224 | } 1225 | 1226 | /// Method exposed by the node to allow a client to set a key value pair on the state machine 1227 | fn set_key_value_pair(&mut self, key: String, value: T) -> Result { 1228 | if !self.is_leader() { 1229 | return Err("This node is not the leader".to_string()); 1230 | } 1231 | let mut state_guard = self.state.lock().unwrap(); 1232 | info!( 1233 | "Setting the key: {}, value: {} pair on node {}", 1234 | key, value, self.id 1235 | ); 1236 | let log_entry = LogEntry { 1237 | term: state_guard.current_term, 1238 | index: state_guard.log.last().map_or(1, |entry| entry.index + 1), 1239 | command: LogEntryCommand::Set, 1240 | key, 1241 | value, 1242 | }; 1243 | let prev_log_index = state_guard.log.last().map_or(0, |entry| entry.index); 1244 | let prev_log_term = state_guard.log.last().map_or(0, |entry| entry.term); 1245 | state_guard.log.push(log_entry.clone()); 1246 | let index = state_guard.log.len() - 1; 1247 | if let Err(e) = self 1248 | .persistence_manager 1249 | .append_logs(&vec![log_entry.clone()]) 1250 | { 1251 | return Err(format!( 1252 | "There was an error while appending logs to stable storage {}", 1253 | e 1254 | )); 1255 | } 1256 | // update the match index of the leader 1257 | // This isn't strictly required but it makes subsequent calculations to advance the 1258 | // commit_index easier 1259 | // state_guard.match_index[self.id as usize] += 1; 1260 | state_guard.match_index[self.id as usize] = state_guard.last_applied + 1; 1261 | for peer in &self.peers { 1262 | if *peer == self.id { 1263 | continue; 1264 | } 1265 | let append_entry_request = AppendEntriesRequest { 1266 | request_id: rand::thread_rng().gen(), 1267 | term: state_guard.current_term, 1268 | leader_id: self.id, 1269 | prev_log_index, 1270 | prev_log_term, 1271 | entries: vec![log_entry.clone()], 1272 | leader_commit_index: state_guard.commit_index, 1273 | }; 1274 | let message = RPCMessage::::AppendEntriesRequest(append_entry_request); 1275 | let to_address = self.config.id_to_address_mapping.get(peer).expect( 1276 | format!("Cannot find the id to address mapping for peer id {}", peer).as_str(), 1277 | ); 1278 | let message_wrapper = MessageWrapper { 1279 | from_node_id: self.id, 1280 | to_node_id: *peer, 1281 | message, 1282 | }; 1283 | self.rpc_manager 1284 | .send_message(to_address.clone(), message_wrapper); 1285 | } 1286 | Ok(index) 1287 | } 1288 | // End utility functions for main RPC's 1289 | 1290 | // Helper functions 1291 | /// The quorum size is (N/2) + 1, where N = number of servers in the cluster and N is odd 1292 | fn quorum_size(&self) -> usize { 1293 | (self.config.cluster_nodes.len() / 2) + 1 1294 | } 1295 | 1296 | fn len_as_u64(&self, v: &Vec>) -> u64 { 1297 | v.len() as u64 1298 | } 1299 | 1300 | fn is_leader(&self) -> bool { 1301 | let state_guard = self.state.lock().unwrap(); 1302 | state_guard.status == RaftNodeStatus::Leader 1303 | } 1304 | 1305 | fn advance_time_by(&mut self, duration: Duration) { 1306 | self.clock.advance(duration); 1307 | } 1308 | 1309 | fn query_state_machine<'a>(&'a self, keys: &Vec<&'a str>) -> Vec<(&str, Option)> { 1310 | let mut results: Vec<(&str, Option)> = vec![]; 1311 | for key in keys { 1312 | let result = self.state_machine.apply_get(key); 1313 | results.push((key, result)); 1314 | } 1315 | results 1316 | } 1317 | // End helper functions 1318 | 1319 | // From this point onward are all the starter methods to bring the node up and handle communication 1320 | // between the node and the RPC manager 1321 | 1322 | /// Method to construct a new raft node 1323 | fn new( 1324 | id: ServerId, 1325 | state_machine: S, 1326 | config: ServerConfig, 1327 | peers: Vec, 1328 | persistence_manager: F, 1329 | rpc_manager: CommunicationLayer, 1330 | to_node_sender: Sender>, 1331 | from_rpc_receiver: Receiver>, 1332 | clock: RaftNodeClock, 1333 | ) -> Self { 1334 | let election_jitter = rand::thread_rng().gen_range(0..100); 1335 | let election_timeout = config.election_timeout + Duration::from_millis(election_jitter); 1336 | let server_state = RaftNodeState { 1337 | current_term: 0, 1338 | voted_for: None, 1339 | log: Vec::new(), 1340 | status: RaftNodeStatus::Follower, 1341 | election_timeout, 1342 | votes_received: HashMap::new(), 1343 | commit_index: 0, 1344 | last_applied: 0, 1345 | next_index: Vec::new(), 1346 | match_index: Vec::new(), 1347 | last_heartbeat: Instant::now(), 1348 | }; 1349 | let server = RaftNode { 1350 | id, 1351 | state: Mutex::new(server_state), 1352 | state_machine, 1353 | config, 1354 | peers, 1355 | to_node_sender, 1356 | from_rpc_receiver, 1357 | rpc_manager, 1358 | persistence_manager, 1359 | clock, 1360 | }; 1361 | server 1362 | } 1363 | 1364 | fn listen_for_messages(&mut self) { 1365 | if let Ok(message_wrapper) = self.from_rpc_receiver.try_recv() { 1366 | // info!("Received a message on node {}: {:?}", self.id, message); 1367 | info!( 1368 | "Received message from node {} on node {} {:?}", 1369 | message_wrapper.from_node_id, self.id, message_wrapper.message 1370 | ); 1371 | if let Some((message_response, from_id)) = self.receive(message_wrapper.clone()) { 1372 | let to_address = self.config.id_to_address_mapping.get(&from_id).expect( 1373 | format!( 1374 | "Cannot find the id to address mapping for peer id {}", 1375 | from_id 1376 | ) 1377 | .as_str(), 1378 | ); 1379 | let message_wrapper = MessageWrapper { 1380 | from_node_id: self.id, 1381 | to_node_id: from_id, 1382 | message: message_response, 1383 | }; 1384 | self.rpc_manager 1385 | .send_message(to_address.clone(), message_wrapper); 1386 | } 1387 | } 1388 | } 1389 | 1390 | fn start(&mut self) { 1391 | let _state_guard = self.state.lock().unwrap(); 1392 | self.rpc_manager.start(); 1393 | } 1394 | 1395 | fn stop(&self) { 1396 | let mut state_guard = self.state.lock().unwrap(); 1397 | self.rpc_manager.stop(); 1398 | let address = &self.config.address; 1399 | let _ = TcpStream::connect(address); // doing this so that the listener actually closes (H/T to Phil Eaton https://github.com/eatonphil/raft-rs/blob/main/src/lib.rs#L1921) 1400 | 1401 | state_guard.current_term = 0; 1402 | state_guard.commit_index = 0; 1403 | state_guard.last_applied = 0; 1404 | state_guard.log.clear(); 1405 | state_guard.voted_for = None; 1406 | state_guard.votes_received.clear(); 1407 | state_guard.next_index.clear(); 1408 | state_guard.match_index.clear(); 1409 | } 1410 | 1411 | fn tick(&mut self) { 1412 | let mut state_guard = self.state.lock().unwrap(); 1413 | match state_guard.status { 1414 | RaftNodeStatus::Leader => { 1415 | // the leader should send heartbeats to all followers 1416 | self.send_heartbeat(&state_guard); 1417 | } 1418 | RaftNodeStatus::Candidate | RaftNodeStatus::Follower => { 1419 | // the candidate or follower should check if the election timeout has elapsed 1420 | self.check_election_timeout(&mut state_guard); 1421 | } 1422 | } 1423 | drop(state_guard); 1424 | 1425 | // listen to incoming messages from the RPC manager 1426 | self.listen_for_messages(); 1427 | self.advance_time_by(Duration::from_millis(1)); 1428 | } 1429 | } 1430 | 1431 | fn main() {} 1432 | 1433 | // An example state machine - a key value store 1434 | struct KeyValueStore { 1435 | store: RefCell>, 1436 | } 1437 | 1438 | impl KeyValueStore { 1439 | fn new() -> Self { 1440 | KeyValueStore { 1441 | store: RefCell::new(HashMap::new()), 1442 | } 1443 | } 1444 | } 1445 | 1446 | impl StateMachine for KeyValueStore { 1447 | fn apply_set(&self, key: String, value: T) { 1448 | self.store.borrow_mut().insert(key, value); 1449 | } 1450 | 1451 | fn apply_get(&self, key: &str) -> Option { 1452 | self.store.borrow().get(key).cloned() 1453 | } 1454 | 1455 | fn apply(&self, entries: Vec>) { 1456 | for entry in entries { 1457 | self.store.borrow_mut().insert(entry.key, entry.value); 1458 | } 1459 | } 1460 | } 1461 | 1462 | #[cfg(test)] 1463 | 1464 | mod tests { 1465 | use super::*; 1466 | mod common { 1467 | 1468 | use std::{ 1469 | collections::{BTreeMap, HashSet}, 1470 | iter::zip, 1471 | }; 1472 | 1473 | use super::*; 1474 | use storage::DirectFileOpsWriter; 1475 | #[derive(Clone)] 1476 | pub struct ClusterConfig { 1477 | pub election_timeout: Duration, 1478 | pub heartbeat_interval: Duration, 1479 | pub ports: Vec, 1480 | } 1481 | 1482 | pub struct TestCluster { 1483 | pub nodes: Vec, DirectFileOpsWriter>>, 1484 | pub nodes_map: 1485 | BTreeMap, DirectFileOpsWriter>>, 1486 | pub message_queue: Vec>, 1487 | pub connectivity: HashMap>, 1488 | pub config: ClusterConfig, 1489 | } 1490 | 1491 | impl TestCluster { 1492 | pub fn tick(&mut self) { 1493 | // Collect all messages from nodes and store them in the central queue 1494 | self.nodes.iter_mut().for_each(|node| { 1495 | let mut messages_from_node = node.rpc_manager.get_messages(); 1496 | self.message_queue.append(&mut messages_from_node); 1497 | }); 1498 | 1499 | // allow each node to tick 1500 | self.nodes.iter_mut().for_each(|node| { 1501 | node.tick(); 1502 | }); 1503 | 1504 | // deliver all messages from the central queue 1505 | self.message_queue 1506 | .drain(..) 1507 | .into_iter() 1508 | .for_each(|message| { 1509 | let node = self 1510 | .nodes 1511 | .iter() 1512 | .find(|node| node.id == message.to_node_id) 1513 | .unwrap(); 1514 | // check if these pair of nodes are partitioned 1515 | if self 1516 | .connectivity 1517 | .get_mut(&message.from_node_id) 1518 | .unwrap() 1519 | .contains(&message.to_node_id) 1520 | { 1521 | match node.to_node_sender.send(message) { 1522 | Ok(_) => (), 1523 | Err(e) => { 1524 | panic!( 1525 | "There was an error while sending the message to the node: {}", 1526 | e 1527 | ) 1528 | } 1529 | }; 1530 | } 1531 | }); 1532 | // since i am not mocking the network layer in tests, i need this for now to 1533 | // simulate actual passage of time so that the TCP layer can actually deliver the 1534 | // message 1535 | // thread::sleep(Duration::from_millis(10)); 1536 | } 1537 | 1538 | pub fn tick_by(&mut self, tick_interval: u64) { 1539 | for _ in 0..tick_interval { 1540 | self.tick(); 1541 | } 1542 | } 1543 | 1544 | pub fn advance_time_by_for_node(&mut self, node_id: ServerId, duration: Duration) { 1545 | let node = self 1546 | .nodes 1547 | .iter_mut() 1548 | .find(|node| node.id == node_id) 1549 | .expect(&format!("Could not find node with id: {}", node_id)); 1550 | node.advance_time_by(duration); 1551 | } 1552 | 1553 | pub fn advance_time_by_variably(&mut self, duration: Duration) { 1554 | // for each node in the cluster, advance it's mock clock by the duration + some random variation 1555 | for node in &mut self.nodes { 1556 | let jitter = rand::thread_rng().gen_range(0..50); 1557 | let new_duration = duration + Duration::from_millis(jitter); 1558 | node.advance_time_by(new_duration); 1559 | } 1560 | } 1561 | 1562 | pub fn advance_time_by(&mut self, duration: Duration) { 1563 | // for each node in the cluster, advance it's mock clock by the duration 1564 | for node in &mut self.nodes { 1565 | node.advance_time_by(duration); 1566 | } 1567 | } 1568 | 1569 | pub fn transmit_message_to_all_nodes(&mut self, message: MessageWrapper) { 1570 | for node in &mut self.nodes { 1571 | match node.to_node_sender.send(message.clone()) { 1572 | Ok(_) => (), 1573 | Err(e) => { 1574 | panic!( 1575 | "There was an error while sending the message to the node: {}", 1576 | e 1577 | ); 1578 | } 1579 | } 1580 | } 1581 | } 1582 | 1583 | pub fn send_message_to_all_nodes( 1584 | &mut self, 1585 | message: MessageWrapper, 1586 | ) -> Vec, ServerId)>> { 1587 | // send this message to every node 1588 | let mut responses: Vec, ServerId)>> = Vec::new(); 1589 | for node in &mut self.nodes { 1590 | let response = node.receive(message.clone()); 1591 | responses.push(response); 1592 | } 1593 | responses 1594 | } 1595 | 1596 | pub fn get_by_id( 1597 | &self, 1598 | id: ServerId, 1599 | ) -> &RaftNode, DirectFileOpsWriter> { 1600 | self.nodes.iter().find(|node| node.id == id).unwrap() 1601 | } 1602 | 1603 | pub fn get_by_id_mut( 1604 | &mut self, 1605 | id: ServerId, 1606 | ) -> &mut RaftNode, DirectFileOpsWriter> { 1607 | self.nodes.iter_mut().find(|node| node.id == id).unwrap() 1608 | } 1609 | 1610 | pub fn has_leader(&self) -> bool { 1611 | self.nodes.iter().filter(|node| node.is_leader()).count() == 1 1612 | } 1613 | 1614 | pub fn has_leader_in_partition(&self, group: &[ServerId]) -> bool { 1615 | let nodes: Vec<&RaftNode, DirectFileOpsWriter>> = group 1616 | .iter() 1617 | .map(|node_id| self.nodes.iter().find(|node| node.id == *node_id).unwrap()) 1618 | .collect(); 1619 | nodes.iter().filter(|node| node.is_leader()).count() == 1 1620 | } 1621 | 1622 | pub fn get_leader( 1623 | &self, 1624 | ) -> Option<&RaftNode, DirectFileOpsWriter>> { 1625 | self.nodes.iter().filter(|node| node.is_leader()).last() 1626 | } 1627 | 1628 | pub fn get_leader_mut( 1629 | &mut self, 1630 | ) -> Option<&mut RaftNode, DirectFileOpsWriter>> { 1631 | self.nodes.iter_mut().filter(|node| node.is_leader()).last() 1632 | } 1633 | 1634 | pub fn get_leader_in_cluster( 1635 | &self, 1636 | group: &[ServerId], 1637 | ) -> Option<&RaftNode, DirectFileOpsWriter>> { 1638 | self.nodes 1639 | .iter() 1640 | .filter(|node| group.contains(&node.id) && node.is_leader()) 1641 | .last() 1642 | } 1643 | 1644 | pub fn get_leader_mut_in_cluster( 1645 | &mut self, 1646 | group: &[ServerId], 1647 | ) -> Option<&mut RaftNode, DirectFileOpsWriter>> { 1648 | self.nodes 1649 | .iter_mut() 1650 | .filter(|node| group.contains(&node.id) && node.is_leader()) 1651 | .last() 1652 | } 1653 | 1654 | pub fn get_follower_mut( 1655 | &mut self, 1656 | ) -> Option<&mut RaftNode, DirectFileOpsWriter>> { 1657 | self.nodes 1658 | .iter_mut() 1659 | .filter(|node| !node.is_leader()) 1660 | .last() 1661 | } 1662 | 1663 | pub fn get_all_followers( 1664 | &self, 1665 | ) -> Vec<&RaftNode, DirectFileOpsWriter>> { 1666 | self.nodes.iter().filter(|node| !node.is_leader()).collect() 1667 | } 1668 | 1669 | pub fn drop_node(&mut self, id: ServerId) -> bool { 1670 | let old_length = self.nodes.len(); 1671 | self.nodes.retain(|node| node.id != id); 1672 | self.nodes.len() != old_length 1673 | } 1674 | 1675 | pub fn send_client_request_partition( 1676 | &mut self, 1677 | key: String, 1678 | value: i32, 1679 | command: LogEntryCommand, 1680 | group: &[ServerId], 1681 | ) -> usize { 1682 | if command == LogEntryCommand::Set { 1683 | let leader = self 1684 | .get_leader_mut_in_cluster(group) 1685 | .expect("Could not find leader in partitioned cluster"); 1686 | leader.set_key_value_pair(key, value).unwrap() 1687 | } else { 1688 | 0 1689 | } 1690 | } 1691 | 1692 | pub fn send_client_request( 1693 | &mut self, 1694 | key: String, 1695 | value: i32, 1696 | command: LogEntryCommand, 1697 | ) -> usize { 1698 | if command == LogEntryCommand::Set { 1699 | let leader = self.get_leader_mut().expect("Could not find leader"); 1700 | leader.set_key_value_pair(key, value).unwrap() 1701 | } else { 1702 | 0 1703 | } 1704 | } 1705 | 1706 | pub fn wait_for_stable_leader_partition( 1707 | &mut self, 1708 | max_ticks: u64, 1709 | group: &[ServerId], 1710 | ) -> bool { 1711 | let mut ticks = 0; 1712 | loop { 1713 | if ticks >= max_ticks { 1714 | break; 1715 | } 1716 | if self.has_leader_in_partition(group) { 1717 | return true; 1718 | } 1719 | self.tick(); 1720 | ticks += 1; 1721 | } 1722 | false 1723 | } 1724 | 1725 | pub fn wait_for_stable_leader(&mut self, max_ticks: u64) -> bool { 1726 | let mut ticks = 0; 1727 | loop { 1728 | if ticks >= max_ticks { 1729 | break; 1730 | } 1731 | if self.has_leader() { 1732 | return true; 1733 | } 1734 | self.tick(); 1735 | ticks += 1; 1736 | } 1737 | false 1738 | } 1739 | 1740 | pub fn wait_until_entry_applied_partition( 1741 | &mut self, 1742 | log_entry: &LogEntry, 1743 | expected_index: usize, 1744 | max_ticks: u64, 1745 | group: &[ServerId], 1746 | ) -> bool { 1747 | let mut ticks = 0; 1748 | while ticks < max_ticks { 1749 | let is_in_log = self 1750 | .nodes 1751 | .iter() 1752 | .filter(|node| group.contains(&node.id)) 1753 | .all(|node| { 1754 | let state_guard = node.state.lock().unwrap(); 1755 | state_guard.log.get(expected_index) == Some(&log_entry) 1756 | }); 1757 | let key = &log_entry.key; 1758 | let is_applied = self 1759 | .nodes 1760 | .iter() 1761 | .filter(|node| group.contains(&node.id)) 1762 | .all(|node| { 1763 | let r = node.state_machine.apply_get(key).is_some(); 1764 | r 1765 | }); 1766 | if is_in_log && is_applied { 1767 | info!( 1768 | "Took {} ticks to apply the entry across the partitioned cluster", 1769 | ticks 1770 | ); 1771 | return true; 1772 | } 1773 | self.tick(); 1774 | ticks += 1; 1775 | } 1776 | false 1777 | } 1778 | 1779 | pub fn wait_until_entry_applied( 1780 | &mut self, 1781 | log_entry: &LogEntry, 1782 | expected_index: usize, 1783 | max_ticks: u64, 1784 | ) -> bool { 1785 | let mut ticks = 0; 1786 | while ticks < max_ticks { 1787 | let is_in_log = self.nodes.iter().all(|node| { 1788 | let state_guard = node.state.lock().unwrap(); 1789 | state_guard.log.get(expected_index) == Some(&log_entry) 1790 | }); 1791 | let key = &log_entry.key; 1792 | let is_applied = self.nodes.iter().all(|node| { 1793 | let r = node.state_machine.apply_get(key).is_some(); 1794 | r 1795 | }); 1796 | if is_in_log && is_applied { 1797 | info!("Took {} ticks to apply the entry across the cluster", ticks); 1798 | return true; 1799 | } 1800 | self.tick(); 1801 | ticks += 1; 1802 | } 1803 | false 1804 | } 1805 | 1806 | pub fn apply_entries_across_cluster_partition( 1807 | &mut self, 1808 | log_entries: Vec<&LogEntry>, 1809 | group: &[ServerId], 1810 | max_ticks: u64, 1811 | ) { 1812 | for log_entry in log_entries { 1813 | let log_index = self.send_client_request_partition( 1814 | log_entry.key.clone(), 1815 | log_entry.value, 1816 | LogEntryCommand::Set, 1817 | group, 1818 | ); 1819 | 1820 | let applied = self 1821 | .wait_until_entry_applied_partition(log_entry, log_index, max_ticks, group); 1822 | if !applied { 1823 | panic!("Could not apply the log entry across the partitioned cluster"); 1824 | } 1825 | } 1826 | } 1827 | 1828 | pub fn apply_entries_across_cluster( 1829 | &mut self, 1830 | log_entries: Vec<&LogEntry>, 1831 | max_ticks: u64, 1832 | ) { 1833 | for log_entry in log_entries { 1834 | let log_index = self.send_client_request( 1835 | log_entry.key.clone(), 1836 | log_entry.value, 1837 | LogEntryCommand::Set, 1838 | ); 1839 | 1840 | let applied = self.wait_until_entry_applied(log_entry, log_index, max_ticks); 1841 | if !applied { 1842 | panic!("Could not apply the log entry across the cluster"); 1843 | } 1844 | } 1845 | } 1846 | 1847 | pub fn verify_logs_across_cluster_partition_for( 1848 | &mut self, 1849 | expected_log_entries: Vec<&LogEntry>, 1850 | max_ticks: u64, 1851 | group: &[ServerId], 1852 | ) -> bool { 1853 | let mut ticks = 0; 1854 | while ticks < max_ticks { 1855 | let is_in_log = self 1856 | .nodes 1857 | .iter() 1858 | .filter(|node| group.contains(&node.id)) 1859 | .all(|node| { 1860 | let state_guard = node.state.lock().unwrap(); 1861 | // state_guard.log == expected_log_entries 1862 | zip(&state_guard.log, &expected_log_entries) 1863 | .into_iter() 1864 | .all(|(log_1, expected_log_1)| log_1 == *expected_log_1) 1865 | }); 1866 | if is_in_log { 1867 | info!( 1868 | "Took {} ticks to verify the logs across the partitioned cluster", 1869 | ticks 1870 | ); 1871 | return true; 1872 | } 1873 | self.tick(); 1874 | ticks += 1; 1875 | } 1876 | false 1877 | } 1878 | 1879 | pub fn verify_logs_across_cluster_for( 1880 | &mut self, 1881 | expected_log_entries: Vec<&LogEntry>, 1882 | max_ticks: u64, 1883 | ) -> bool { 1884 | let mut ticks = 0; 1885 | while ticks < max_ticks { 1886 | let is_in_log = self.nodes.iter().all(|node| { 1887 | let state_guard = node.state.lock().unwrap(); 1888 | // state_guard.log == expected_log_entries 1889 | zip(&state_guard.log, &expected_log_entries) 1890 | .into_iter() 1891 | .all(|(log_1, expected_log_1)| log_1 == *expected_log_1) 1892 | }); 1893 | if is_in_log { 1894 | info!("Took {} ticks to verify the logs across the cluster", ticks); 1895 | return true; 1896 | } 1897 | self.tick(); 1898 | ticks += 1; 1899 | } 1900 | false 1901 | } 1902 | 1903 | /// Separates the cluster into two smaller clusters where only the nodes within 1904 | /// each cluster can communicate between themselves 1905 | pub fn partition(&mut self, group1: &[ServerId], group2: &[ServerId]) { 1906 | for &node_id in group1 { 1907 | self.connectivity 1908 | .get_mut(&node_id) 1909 | .unwrap() 1910 | .retain(|&id| group1.contains(&id)); 1911 | } 1912 | for &node_id in group2 { 1913 | self.connectivity 1914 | .get_mut(&node_id) 1915 | .unwrap() 1916 | .retain(|&id| group2.contains(&id)); 1917 | } 1918 | } 1919 | 1920 | /// Removes any partition in the cluster and restores full connectivity among all nodes 1921 | pub fn heal_partition(&mut self) { 1922 | let all_node_ids = self 1923 | .nodes 1924 | .iter() 1925 | .map(|node| node.id) 1926 | .collect::>(); 1927 | for node_id in all_node_ids.iter() { 1928 | self.connectivity.insert(*node_id, all_node_ids.clone()); 1929 | } 1930 | } 1931 | 1932 | pub fn query_state_machine_across_nodes<'a>( 1933 | &'a self, 1934 | keys: Vec<&'a str>, 1935 | ) -> HashMap)>> { 1936 | let mut hm: HashMap)>> = HashMap::new(); 1937 | for node in &self.nodes { 1938 | let key_value_pairs = node.query_state_machine(&keys); 1939 | hm.insert(node.id, key_value_pairs); 1940 | } 1941 | hm 1942 | } 1943 | 1944 | pub fn new(number_of_nodes: u64, config: ClusterConfig) -> Self { 1945 | let mut nodes: Vec, DirectFileOpsWriter>> = 1946 | Vec::new(); 1947 | let nodes_map: BTreeMap< 1948 | ServerId, 1949 | RaftNode, DirectFileOpsWriter>, 1950 | > = BTreeMap::new(); 1951 | 1952 | let node_ids: Vec = (0..number_of_nodes).collect(); 1953 | let addresses: Vec = config 1954 | .ports 1955 | .iter() 1956 | .map(|port| format!("127.0.0.1:{}", port)) 1957 | .collect(); 1958 | let mut id_to_address_mapping: HashMap = HashMap::new(); 1959 | for (node_id, address) in node_ids.iter().zip(addresses.iter()) { 1960 | id_to_address_mapping.insert(*node_id, address.clone()); 1961 | } 1962 | 1963 | let mut counter = 0; 1964 | for (node_id, address) in node_ids.iter().zip(addresses.iter()) { 1965 | let server_config = ServerConfig { 1966 | election_timeout: config.election_timeout, 1967 | heartbeat_interval: config.heartbeat_interval, 1968 | address: address.clone(), 1969 | port: config.ports[counter], 1970 | cluster_nodes: node_ids.clone(), 1971 | id_to_address_mapping: id_to_address_mapping.clone(), 1972 | }; 1973 | 1974 | let state_machine = KeyValueStore::::new(); 1975 | let persistence_manager = DirectFileOpsWriter::new("data", *node_id).unwrap(); 1976 | let (to_node_sender, from_rpc_receiver) = 1977 | mpsc::channel::>(); 1978 | let rpc_manager = CommunicationLayer::MockRPCManager(MockRPCManager::new( 1979 | *node_id, 1980 | to_node_sender.clone(), 1981 | )); 1982 | let mock_clock = RaftNodeClock::MockClock(MockClock { 1983 | current_time: Instant::now(), 1984 | }); 1985 | 1986 | let node = RaftNode::new( 1987 | *node_id, 1988 | state_machine, 1989 | server_config, 1990 | node_ids.clone(), 1991 | persistence_manager, 1992 | rpc_manager, 1993 | to_node_sender, 1994 | from_rpc_receiver, 1995 | mock_clock, 1996 | ); 1997 | nodes.push(node); 1998 | // nodes_map.insert(*node_id, node); // TODO: Cannot clone node here. Need to get rid of vector before BTreeMap can be used 1999 | counter += 1; 2000 | } 2001 | let message_queue = Vec::new(); 2002 | 2003 | let mut connectivity_hm: HashMap> = HashMap::new(); 2004 | for node_id in &node_ids { 2005 | connectivity_hm.insert(*node_id, node_ids.clone().into_iter().collect()); 2006 | } 2007 | 2008 | TestCluster { 2009 | nodes, 2010 | nodes_map, 2011 | message_queue, 2012 | connectivity: connectivity_hm, 2013 | config, 2014 | } 2015 | } 2016 | 2017 | pub fn start(&mut self) { 2018 | for node in &mut self.nodes { 2019 | node.start(); 2020 | } 2021 | } 2022 | 2023 | pub fn stop(&self) { 2024 | for node in &self.nodes { 2025 | node.stop(); 2026 | } 2027 | } 2028 | } 2029 | } 2030 | 2031 | mod single_node_cluster { 2032 | use super::common::*; 2033 | use super::*; 2034 | 2035 | const ELECTION_TIMEOUT: Duration = Duration::from_millis(150); 2036 | const HEARTBEAT_INTERVAL: Duration = Duration::from_millis(50); 2037 | const MAX_TICKS: u64 = 100; 2038 | 2039 | /// This test checks whether a node in a single cluster will become leader as soon as the election timeout is reached 2040 | #[test] 2041 | fn leader_election() { 2042 | let _ = env_logger::builder().is_test(true).try_init(); 2043 | // create a cluster with a single node first 2044 | let cluster_config = ClusterConfig { 2045 | election_timeout: ELECTION_TIMEOUT, 2046 | heartbeat_interval: HEARTBEAT_INTERVAL, 2047 | ports: vec![8000], 2048 | }; 2049 | let mut cluster = TestCluster::new(1, cluster_config); 2050 | cluster.start(); 2051 | cluster.advance_time_by(ELECTION_TIMEOUT + Duration::from_millis(100 + 5)); // picking 255 here because 150 + a max jitter of 100 guarantees that election has timed out 2052 | cluster.wait_for_stable_leader(MAX_TICKS); 2053 | cluster.stop(); 2054 | assert_eq!(cluster.has_leader(), true); 2055 | } 2056 | 2057 | /// This test checks the functionality of a node receiving a vote request 2058 | #[test] 2059 | fn request_vote_success() { 2060 | let _ = env_logger::builder().is_test(true).try_init(); 2061 | let cluster_config = ClusterConfig { 2062 | election_timeout: ELECTION_TIMEOUT, 2063 | heartbeat_interval: HEARTBEAT_INTERVAL, 2064 | ports: vec![8000], 2065 | }; 2066 | let mut cluster = TestCluster::new(1, cluster_config); 2067 | 2068 | let request = VoteRequest { 2069 | request_id: 1, 2070 | term: 1, 2071 | candidate_id: 1, 2072 | last_log_index: 0, 2073 | last_log_term: 0, 2074 | }; 2075 | let message = RPCMessage::VoteRequest(request); 2076 | let message_wrapper = MessageWrapper { 2077 | from_node_id: 1, 2078 | to_node_id: 0, 2079 | message, 2080 | }; 2081 | 2082 | let response = cluster.send_message_to_all_nodes(message_wrapper); 2083 | 2084 | let vote_response = match response.get(0).unwrap() { 2085 | Some((RPCMessage::VoteResponse(response), _)) => response, 2086 | _ => panic!("The response is not a vote response"), 2087 | }; 2088 | 2089 | assert_eq!( 2090 | *vote_response, 2091 | VoteResponse { 2092 | request_id: 1, 2093 | term: 1, 2094 | vote_granted: true, 2095 | candidate_id: 0 2096 | } 2097 | ); 2098 | } 2099 | 2100 | /// This test asserts that a node refuses to grant a vote because its own log 2101 | /// is ahead of the candidates log 2102 | #[test] 2103 | fn request_vote_fail_log_check() { 2104 | let _ = env_logger::builder().is_test(true).try_init(); 2105 | let cluster_config = ClusterConfig { 2106 | election_timeout: ELECTION_TIMEOUT, 2107 | heartbeat_interval: HEARTBEAT_INTERVAL, 2108 | ports: vec![8000], 2109 | }; 2110 | let mut cluster = TestCluster::new(1, cluster_config); 2111 | 2112 | let node = cluster.get_by_id_mut(0); 2113 | let mut node_state = node.state.lock().unwrap(); 2114 | node_state.current_term = 2; 2115 | node_state.log.push(LogEntry { 2116 | term: 1, 2117 | index: 1, 2118 | command: LogEntryCommand::Set, 2119 | key: "a".to_string(), 2120 | value: 1, 2121 | }); 2122 | node_state.log.push(LogEntry { 2123 | term: 2, 2124 | index: 2, 2125 | command: LogEntryCommand::Set, 2126 | key: "b".to_string(), 2127 | value: 2, 2128 | }); 2129 | drop(node_state); 2130 | 2131 | let request = VoteRequest { 2132 | request_id: 1, 2133 | term: 1, 2134 | candidate_id: 1, 2135 | last_log_index: 1, 2136 | last_log_term: 1, 2137 | }; 2138 | let message = RPCMessage::VoteRequest(request); 2139 | let message_wrapper = MessageWrapper { 2140 | from_node_id: 1, 2141 | to_node_id: 0, 2142 | message, 2143 | }; 2144 | 2145 | let response = cluster.send_message_to_all_nodes(message_wrapper); 2146 | let vote_response = match response.get(0).unwrap() { 2147 | Some((RPCMessage::VoteResponse(response), _)) => response, 2148 | _ => panic!("The response is not a vote response"), 2149 | }; 2150 | assert_eq!( 2151 | *vote_response, 2152 | VoteResponse { 2153 | request_id: 1, 2154 | term: 2, 2155 | vote_granted: false, 2156 | candidate_id: 0 2157 | } 2158 | ); 2159 | } 2160 | 2161 | #[test] 2162 | fn append_entries_success() { 2163 | let _ = env_logger::builder().is_test(true).try_init(); 2164 | let cluster_config = ClusterConfig { 2165 | election_timeout: ELECTION_TIMEOUT, 2166 | heartbeat_interval: HEARTBEAT_INTERVAL, 2167 | ports: vec![8000], 2168 | }; 2169 | let mut cluster = TestCluster::new(1, cluster_config); 2170 | 2171 | { 2172 | let node = cluster.get_by_id_mut(0); 2173 | let mut node_state = node.state.lock().unwrap(); 2174 | node_state.current_term = 1; 2175 | node_state.voted_for = Some(1); 2176 | } 2177 | 2178 | let request = AppendEntriesRequest { 2179 | request_id: 1, 2180 | term: 1, 2181 | leader_id: 1, 2182 | prev_log_index: 0, 2183 | prev_log_term: 0, 2184 | entries: vec![LogEntry { 2185 | term: 1, 2186 | index: 1, 2187 | command: LogEntryCommand::Set, 2188 | key: "a".to_string(), 2189 | value: 1, 2190 | }], 2191 | leader_commit_index: 0, 2192 | }; 2193 | let message = RPCMessage::AppendEntriesRequest(request); 2194 | let message_wrapper = MessageWrapper { 2195 | from_node_id: 1, 2196 | to_node_id: 0, 2197 | message, 2198 | }; 2199 | 2200 | let response = cluster.send_message_to_all_nodes(message_wrapper); 2201 | let append_response = match response.get(0).unwrap() { 2202 | Some((RPCMessage::AppendEntriesResponse(response), _)) => response, 2203 | _ => panic!("The response is not an append entries response"), 2204 | }; 2205 | assert_eq!( 2206 | *append_response, 2207 | AppendEntriesResponse { 2208 | request_id: 1, 2209 | term: 1, 2210 | success: true, 2211 | server_id: 0, 2212 | match_index: 1 2213 | } 2214 | ); 2215 | 2216 | { 2217 | let node = cluster.get_by_id_mut(0); 2218 | let node_state = node.state.lock().unwrap(); 2219 | assert_eq!(node_state.log.len(), 1); 2220 | } 2221 | } 2222 | 2223 | #[test] 2224 | fn restore_from_durable_storage() { 2225 | let _ = env_logger::builder().is_test(true).try_init(); 2226 | let cluster_config = ClusterConfig { 2227 | election_timeout: ELECTION_TIMEOUT, 2228 | heartbeat_interval: HEARTBEAT_INTERVAL, 2229 | ports: vec![8000], 2230 | }; 2231 | let mut cluster = TestCluster::new(1, cluster_config); 2232 | 2233 | let vote_request = VoteRequest { 2234 | request_id: 1, 2235 | term: 1, 2236 | candidate_id: 1, 2237 | last_log_index: 0, 2238 | last_log_term: 0, 2239 | }; 2240 | let vote_message: RPCMessage = RPCMessage::VoteRequest(vote_request); 2241 | let vote_message_wrapper = MessageWrapper { 2242 | from_node_id: 1, 2243 | to_node_id: 0, 2244 | message: vote_message, 2245 | }; 2246 | 2247 | cluster.send_message_to_all_nodes(vote_message_wrapper); 2248 | 2249 | let request = AppendEntriesRequest { 2250 | request_id: 1, 2251 | term: 1, 2252 | leader_id: 1, 2253 | prev_log_index: 0, 2254 | prev_log_term: 0, 2255 | entries: vec![LogEntry { 2256 | term: 1, 2257 | index: 1, 2258 | command: LogEntryCommand::Set, 2259 | key: "a".to_string(), 2260 | value: 1, 2261 | }], 2262 | leader_commit_index: 0, 2263 | }; 2264 | let message = RPCMessage::AppendEntriesRequest(request); 2265 | let message_wrapper = MessageWrapper { 2266 | from_node_id: 1, 2267 | to_node_id: 0, 2268 | message, 2269 | }; 2270 | 2271 | cluster.send_message_to_all_nodes(message_wrapper); 2272 | 2273 | let node = cluster.get_by_id_mut(0); 2274 | node.stop(); 2275 | node.restart(); 2276 | let log_length = node.state.lock().unwrap().log.len(); 2277 | let term = node.state.lock().unwrap().current_term; 2278 | let voted_for = node.state.lock().unwrap().voted_for.clone(); 2279 | cluster.stop(); 2280 | assert_eq!(log_length, 1); 2281 | assert_eq!(term, 1); 2282 | assert_eq!(voted_for, Some(1)); 2283 | } 2284 | 2285 | /// This test will determine whether a leader correctly advances its commit index 2286 | /// based on the list of match indexes it maintains for each follower 2287 | #[test] 2288 | fn advance_commit_index_of_leader() { 2289 | let _ = env_logger::builder().is_test(true).try_init(); 2290 | let cluster_config = ClusterConfig { 2291 | election_timeout: ELECTION_TIMEOUT, 2292 | heartbeat_interval: HEARTBEAT_INTERVAL, 2293 | ports: vec![8000], 2294 | }; 2295 | let mut cluster = TestCluster::new(1, cluster_config); 2296 | cluster.start(); 2297 | cluster.advance_time_by(ELECTION_TIMEOUT + Duration::from_millis(50)); 2298 | cluster.wait_for_stable_leader(MAX_TICKS); 2299 | assert_eq!(cluster.has_leader(), true); 2300 | let leader_node = cluster.get_leader().unwrap(); 2301 | { 2302 | let mut state_guard = leader_node.state.lock().unwrap(); 2303 | state_guard.log.push(LogEntry { 2304 | term: 1, 2305 | index: 1, 2306 | command: LogEntryCommand::Set, 2307 | key: "a".to_string(), 2308 | value: 1, 2309 | }); 2310 | state_guard.log.push(LogEntry { 2311 | term: 1, 2312 | index: 2, 2313 | command: LogEntryCommand::Set, 2314 | key: "b".to_string(), 2315 | value: 2, 2316 | }); 2317 | state_guard.commit_index = 0; 2318 | state_guard.match_index = vec![2]; 2319 | leader_node.advance_commit_index(&mut state_guard); 2320 | assert_eq!(state_guard.commit_index, 2); 2321 | } 2322 | } 2323 | 2324 | /// This test asserts that a node that receives a set of log entries that is in conflict with its own log entries, then it will find the conflict 2325 | /// point and truncate all log entries from that point onwards before writing the new log entries 2326 | #[test] 2327 | fn log_conflict() { 2328 | let _ = env_logger::builder().is_test(true).try_init(); 2329 | let cluster_config = ClusterConfig { 2330 | election_timeout: ELECTION_TIMEOUT, 2331 | heartbeat_interval: HEARTBEAT_INTERVAL, 2332 | ports: vec![8000], 2333 | }; 2334 | let entries_on_node = vec![ 2335 | LogEntry { 2336 | term: 1, 2337 | index: 1, 2338 | command: LogEntryCommand::Set, 2339 | key: "a".to_string(), 2340 | value: 1, 2341 | }, 2342 | LogEntry { 2343 | term: 1, 2344 | index: 2, 2345 | command: LogEntryCommand::Set, 2346 | key: "b".to_string(), 2347 | value: 1, 2348 | }, 2349 | LogEntry { 2350 | term: 1, 2351 | index: 3, 2352 | command: LogEntryCommand::Set, 2353 | key: "c".to_string(), 2354 | value: 2, 2355 | }, 2356 | ]; 2357 | let mut cluster = TestCluster::new(1, cluster_config); 2358 | { 2359 | let node = cluster.get_by_id_mut(0); 2360 | node.state.lock().unwrap().log = entries_on_node; 2361 | } 2362 | 2363 | let new_entries_sent_to_node = vec![ 2364 | LogEntry { 2365 | term: 2, 2366 | index: 2, 2367 | command: LogEntryCommand::Set, 2368 | key: "d".to_string(), 2369 | value: 1, 2370 | }, 2371 | LogEntry { 2372 | term: 2, 2373 | index: 3, 2374 | command: LogEntryCommand::Set, 2375 | key: "d".to_string(), 2376 | value: 2, 2377 | }, 2378 | ]; 2379 | let request = AppendEntriesRequest { 2380 | request_id: 1, 2381 | term: 2, 2382 | leader_id: 1, 2383 | prev_log_index: 1, 2384 | prev_log_term: 1, 2385 | entries: new_entries_sent_to_node, 2386 | leader_commit_index: 1, 2387 | }; 2388 | let message = RPCMessage::AppendEntriesRequest(request); 2389 | let message_wrapper = MessageWrapper { 2390 | from_node_id: 1, 2391 | to_node_id: 0, 2392 | message, 2393 | }; 2394 | cluster.send_message_to_all_nodes(message_wrapper); 2395 | 2396 | // check that the log of the node is correct here now 2397 | let resolved_log = vec![ 2398 | LogEntry { 2399 | term: 1, 2400 | index: 1, 2401 | command: LogEntryCommand::Set, 2402 | key: "a".to_string(), 2403 | value: 1, 2404 | }, 2405 | LogEntry { 2406 | term: 2, 2407 | index: 2, 2408 | command: LogEntryCommand::Set, 2409 | key: "d".to_string(), 2410 | value: 1, 2411 | }, 2412 | LogEntry { 2413 | term: 2, 2414 | index: 3, 2415 | command: LogEntryCommand::Set, 2416 | key: "d".to_string(), 2417 | value: 2, 2418 | }, 2419 | ]; 2420 | { 2421 | let node = cluster.get_by_id(0); 2422 | let log = &node.state.lock().unwrap().log; 2423 | info!("Log entries: {:?}", log); 2424 | assert_eq!(log.len(), 3); 2425 | assert_eq!(*log, resolved_log); 2426 | } 2427 | } 2428 | } 2429 | 2430 | mod two_node_cluster { 2431 | use super::common::*; 2432 | use super::*; 2433 | 2434 | const ELECTION_TIMEOUT: Duration = Duration::from_millis(150); 2435 | const HEARTBEAT_INTERVAL: Duration = Duration::from_millis(50); 2436 | const MAX_TICKS: u64 = 100; 2437 | 2438 | /// Assert that in a 2-node cluster, the more advanced node becomes leader 2439 | /// after a few ticks 2440 | #[test] 2441 | fn leader_election() { 2442 | let _ = env_logger::builder().is_test(true).try_init(); 2443 | let cluster_config = ClusterConfig { 2444 | election_timeout: ELECTION_TIMEOUT, 2445 | heartbeat_interval: HEARTBEAT_INTERVAL, 2446 | ports: vec![8000, 8001], 2447 | }; 2448 | let mut cluster = TestCluster::new(2, cluster_config); 2449 | cluster.start(); 2450 | let node = cluster.get_by_id_mut(0); 2451 | node.advance_time_by(ELECTION_TIMEOUT + Duration::from_millis(50)); 2452 | cluster.wait_for_stable_leader(MAX_TICKS); 2453 | cluster.stop(); 2454 | assert_eq!(cluster.has_leader(), true); 2455 | } 2456 | 2457 | /// This test will determine whether an elected leader sends heartbeats regularly during its term 2458 | /// It does so by checking that the leader and term haven't changed after the iterations are done 2459 | /// This indicates that a leader regularly sent heartbeats which prevented an election timeout 2460 | #[test] 2461 | fn leader_send_heartbeats() { 2462 | let _ = env_logger::builder().is_test(true).try_init(); 2463 | let cluster_config = ClusterConfig { 2464 | election_timeout: ELECTION_TIMEOUT, 2465 | heartbeat_interval: HEARTBEAT_INTERVAL, 2466 | ports: vec![8000, 8001], 2467 | }; 2468 | let mut cluster = TestCluster::new(2, cluster_config); 2469 | cluster.start(); 2470 | let node = cluster.get_by_id_mut(0); 2471 | node.advance_time_by(ELECTION_TIMEOUT + Duration::from_millis(100)); 2472 | cluster.wait_for_stable_leader(MAX_TICKS); 2473 | let (leader_id, leader_term) = { 2474 | let leader = cluster.get_leader().unwrap(); 2475 | let leader_term = leader.state.lock().unwrap().current_term; 2476 | (leader.id, leader_term) 2477 | }; 2478 | cluster.tick_by(MAX_TICKS); 2479 | let new_leader = cluster.get_leader().unwrap(); 2480 | let new_leader_term = new_leader.state.lock().unwrap().current_term; 2481 | cluster.stop(); 2482 | assert_eq!(new_leader.id, leader_id); 2483 | assert_eq!(new_leader_term, leader_term); 2484 | } 2485 | 2486 | /// This test will determine whether an elected leader will correctly send a key-value pair it receives from a client 2487 | /// to all its followers. The leader should also append the entry to its own log before that. 2488 | #[test] 2489 | fn apply_entries() { 2490 | let _ = env_logger::builder().is_test(true).try_init(); 2491 | let cluster_config = ClusterConfig { 2492 | election_timeout: ELECTION_TIMEOUT, 2493 | heartbeat_interval: HEARTBEAT_INTERVAL, 2494 | ports: vec![8000, 8001], 2495 | }; 2496 | let mut cluster = TestCluster::new(2, cluster_config); 2497 | cluster.start(); 2498 | cluster.advance_time_by(ELECTION_TIMEOUT); 2499 | cluster.wait_for_stable_leader(MAX_TICKS); 2500 | let (log_entry, entry_index) = { 2501 | let leader = cluster.get_leader_mut().unwrap(); 2502 | let entry_index = leader.set_key_value_pair("a".to_string(), 1).unwrap(); 2503 | let log_entry = LogEntry { 2504 | term: 1, 2505 | index: 1, 2506 | command: LogEntryCommand::Set, 2507 | key: "a".to_string(), 2508 | value: 1, 2509 | }; 2510 | let leader_state = leader.state.lock().unwrap(); 2511 | assert_eq!(leader_state.log.len(), 1); 2512 | assert_eq!(leader_state.log.get(0), Some(&log_entry)); 2513 | (log_entry, entry_index) 2514 | }; 2515 | let result = cluster.wait_until_entry_applied(&log_entry, entry_index, MAX_TICKS); 2516 | assert_eq!(result, true); 2517 | cluster.stop(); 2518 | } 2519 | } 2520 | 2521 | mod three_node_cluster { 2522 | use super::common::*; 2523 | use super::*; 2524 | 2525 | const ELECTION_TIMEOUT: Duration = Duration::from_millis(150); 2526 | const HEARTBEAT_INTERVAL: Duration = Duration::from_millis(50); 2527 | const MAX_TICKS: u64 = 100; 2528 | 2529 | /// Assert that in a 3-node cluster one of the nodes eventually becomes leader within a certain number of ticks 2530 | #[test] 2531 | fn leader_election() { 2532 | let _ = env_logger::builder().is_test(true).try_init(); 2533 | let cluster_config = ClusterConfig { 2534 | election_timeout: ELECTION_TIMEOUT, 2535 | heartbeat_interval: HEARTBEAT_INTERVAL, 2536 | ports: vec![8000, 8001, 8002], 2537 | }; 2538 | let mut cluster = TestCluster::new(3, cluster_config); 2539 | cluster.advance_time_by(ELECTION_TIMEOUT); 2540 | cluster.start(); 2541 | cluster.wait_for_stable_leader(MAX_TICKS); 2542 | cluster.stop(); 2543 | assert_eq!(cluster.has_leader(), true); 2544 | } 2545 | 2546 | /// This test will determine whether an elected leader sends heartbeats regularly during its term 2547 | /// It does so by checking that the leader and term haven't changed after the iterations are done 2548 | /// This indicates that a leader regularly sent heartbeats which prevented an election timeout 2549 | #[test] 2550 | fn leader_send_heartbeats() { 2551 | let _ = env_logger::builder().is_test(true).try_init(); 2552 | let cluster_config = ClusterConfig { 2553 | election_timeout: ELECTION_TIMEOUT, 2554 | heartbeat_interval: HEARTBEAT_INTERVAL, 2555 | ports: vec![8000, 8001, 8002], 2556 | }; 2557 | let mut cluster = TestCluster::new(3, cluster_config); 2558 | cluster.start(); 2559 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2560 | cluster.wait_for_stable_leader(MAX_TICKS); 2561 | let (current_leader_id, current_leader_term) = { 2562 | let leader = cluster.get_leader().unwrap(); 2563 | let term = leader.state.lock().unwrap().current_term; 2564 | (leader.id, term) 2565 | }; 2566 | cluster.tick_by(MAX_TICKS); 2567 | let new_leader = cluster.get_leader().unwrap(); 2568 | let new_leader_term = new_leader.state.lock().unwrap().current_term; 2569 | assert_eq!(current_leader_id, new_leader.id); 2570 | assert_eq!(current_leader_term, new_leader_term); 2571 | } 2572 | 2573 | /// This test will check whether a key value pair submitted by a client is appended to the logs of the nodes in the cluster 2574 | /// and replicated across the state machines of the cluster 2575 | #[test] 2576 | fn apply_entries() { 2577 | let _ = env_logger::builder().is_test(true).try_init(); 2578 | let cluster_config = ClusterConfig { 2579 | election_timeout: ELECTION_TIMEOUT, 2580 | heartbeat_interval: HEARTBEAT_INTERVAL, 2581 | ports: vec![8000, 8001, 8002], 2582 | }; 2583 | let mut cluster = TestCluster::new(3, cluster_config); 2584 | cluster.start(); 2585 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2586 | cluster.wait_for_stable_leader(MAX_TICKS); 2587 | let current_leader_term = cluster 2588 | .get_leader() 2589 | .unwrap() 2590 | .state 2591 | .lock() 2592 | .unwrap() 2593 | .current_term; 2594 | let last_log_index = cluster 2595 | .get_leader() 2596 | .unwrap() 2597 | .state 2598 | .lock() 2599 | .unwrap() 2600 | .log 2601 | .last() 2602 | .map_or(0, |entry| entry.index); 2603 | let log_entry = LogEntry { 2604 | term: current_leader_term, 2605 | index: last_log_index + 1, 2606 | command: LogEntryCommand::Set, 2607 | key: "a".to_string(), 2608 | value: 1, 2609 | }; 2610 | cluster.apply_entries_across_cluster(vec![&log_entry], MAX_TICKS); 2611 | cluster.verify_logs_across_cluster_for(vec![&log_entry], MAX_TICKS); 2612 | } 2613 | 2614 | /// This test models the scenario where a follower has a log entry that is out of sync with the leader. When the leader sends a heartbeat, it will 2615 | /// recognize this and attempt to fix the incorrect log entry by rewinding the log back to a point where there is no conflict 2616 | #[test] 2617 | fn retry_failed_append_entry_request() { 2618 | let _ = env_logger::builder().is_test(true).try_init(); 2619 | let cluster_config = ClusterConfig { 2620 | election_timeout: ELECTION_TIMEOUT, 2621 | heartbeat_interval: HEARTBEAT_INTERVAL, 2622 | ports: vec![8000, 8001, 8002], 2623 | }; 2624 | let mut cluster = TestCluster::new(3, cluster_config); 2625 | cluster.start(); 2626 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2627 | cluster.wait_for_stable_leader(MAX_TICKS); 2628 | assert_eq!(cluster.has_leader(), true); 2629 | 2630 | let current_leader_term = cluster 2631 | .get_leader() 2632 | .unwrap() 2633 | .state 2634 | .lock() 2635 | .unwrap() 2636 | .current_term; 2637 | let mut last_log_index = cluster 2638 | .get_leader() 2639 | .unwrap() 2640 | .state 2641 | .lock() 2642 | .unwrap() 2643 | .log 2644 | .last() 2645 | .map_or(0, |entry| entry.index); 2646 | last_log_index += 1; 2647 | let log_entry_1 = LogEntry { 2648 | term: current_leader_term, 2649 | index: last_log_index, 2650 | command: LogEntryCommand::Set, 2651 | key: "a".to_string(), 2652 | value: 1, 2653 | }; 2654 | last_log_index += 1; 2655 | let log_entry_2 = LogEntry { 2656 | term: current_leader_term, 2657 | index: last_log_index, 2658 | command: LogEntryCommand::Set, 2659 | key: "b".to_string(), 2660 | value: 1, 2661 | }; 2662 | 2663 | cluster.apply_entries_across_cluster(vec![&log_entry_1, &log_entry_2], MAX_TICKS); 2664 | cluster.verify_logs_across_cluster_for(vec![&log_entry_1, &log_entry_2], MAX_TICKS); 2665 | 2666 | // now change the log on one of the followers so that the log check fails 2667 | { 2668 | let follower_node = cluster.get_follower_mut().unwrap(); 2669 | let mut follower_node_state_guard = follower_node.state.lock().unwrap(); 2670 | follower_node_state_guard.log.pop(); 2671 | follower_node_state_guard.log.push(LogEntry { 2672 | term: 2, 2673 | index: 3, 2674 | command: LogEntryCommand::Set, 2675 | key: "c".to_string(), 2676 | value: 2, 2677 | }); 2678 | } 2679 | 2680 | // wait for the cluster to "heal" and assert that the logs are up to date 2681 | let expected_log_entries = vec![&log_entry_1, &log_entry_2]; 2682 | let result = cluster.verify_logs_across_cluster_for(expected_log_entries, MAX_TICKS); 2683 | cluster.stop(); 2684 | assert_eq!(result, true); 2685 | } 2686 | 2687 | /// This test models the scenario where the leader in a cluster is network partitioned from the 2688 | /// rest of the cluster and a new leader is elected 2689 | #[test] 2690 | fn network_partition_new_leader() { 2691 | let _ = env_logger::builder().is_test(true).try_init(); 2692 | let cluster_config = ClusterConfig { 2693 | election_timeout: ELECTION_TIMEOUT, 2694 | heartbeat_interval: HEARTBEAT_INTERVAL, 2695 | ports: vec![8000, 8001, 8002], 2696 | }; 2697 | let mut cluster = TestCluster::new(3, cluster_config); 2698 | cluster.start(); 2699 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2700 | cluster.wait_for_stable_leader(MAX_TICKS); 2701 | 2702 | // partition the leader from the rest of the group 2703 | let group1 = &[cluster.get_leader().unwrap().id]; 2704 | let group2 = &cluster 2705 | .get_all_followers() 2706 | .iter() 2707 | .map(|node| node.id) 2708 | .collect::>(); 2709 | cluster.partition(group1, group2); 2710 | cluster.advance_time_by_for_node(1, ELECTION_TIMEOUT + Duration::from_millis(100)); 2711 | cluster.wait_for_stable_leader_partition(MAX_TICKS, group2); 2712 | cluster.stop(); 2713 | assert_eq!(cluster.has_leader_in_partition(group2), true); 2714 | } 2715 | 2716 | /// The test models the following scenario 2717 | /// 1. A leader is elected 2718 | /// 2. Client requests are received and replicated 2719 | /// 3. A partition occurs and a new leader is elected 2720 | /// 4. Clients requests are processed by the new leader 2721 | /// 5. The partition heals and the old leader rejoins the cluster 2722 | /// 6. The old leader must recognize its a follower and get caught up with the new leader 2723 | #[test] 2724 | fn network_partition_log_healing() { 2725 | let _ = env_logger::builder().is_test(true).try_init(); 2726 | let cluster_config = ClusterConfig { 2727 | election_timeout: ELECTION_TIMEOUT, 2728 | heartbeat_interval: HEARTBEAT_INTERVAL, 2729 | ports: vec![8000, 8001, 8002], 2730 | }; 2731 | let mut cluster = TestCluster::new(3, cluster_config); 2732 | 2733 | // cluster starts and a leader is elected 2734 | cluster.start(); 2735 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2736 | cluster.wait_for_stable_leader(MAX_TICKS); 2737 | assert_eq!(cluster.has_leader(), true); 2738 | 2739 | // client requests are received and replicated across cluster 2740 | let current_leader_term = cluster 2741 | .get_leader() 2742 | .unwrap() 2743 | .state 2744 | .lock() 2745 | .unwrap() 2746 | .current_term; 2747 | let mut last_log_index = cluster 2748 | .get_leader() 2749 | .unwrap() 2750 | .state 2751 | .lock() 2752 | .unwrap() 2753 | .log 2754 | .last() 2755 | .map_or(0, |entry| entry.index); 2756 | last_log_index += 1; 2757 | let log_entry_1 = LogEntry { 2758 | term: current_leader_term, 2759 | index: last_log_index, 2760 | command: LogEntryCommand::Set, 2761 | key: "a".to_string(), 2762 | value: 1, 2763 | }; 2764 | last_log_index += 1; 2765 | let log_entry_2 = LogEntry { 2766 | term: current_leader_term, 2767 | index: last_log_index, 2768 | command: LogEntryCommand::Set, 2769 | key: "b".to_string(), 2770 | value: 2, 2771 | }; 2772 | cluster.apply_entries_across_cluster(vec![&log_entry_1, &log_entry_2], MAX_TICKS); 2773 | cluster.verify_logs_across_cluster_for(vec![&log_entry_1, &log_entry_2], MAX_TICKS); 2774 | 2775 | // partition and new leader election here 2776 | let group1 = &[cluster.get_leader().unwrap().id]; 2777 | let group2 = &cluster 2778 | .get_all_followers() 2779 | .iter() 2780 | .map(|node| node.id) 2781 | .collect::>(); 2782 | cluster.partition(group1, group2); 2783 | cluster.advance_time_by_for_node(1, ELECTION_TIMEOUT + Duration::from_millis(100)); 2784 | cluster.wait_for_stable_leader_partition(MAX_TICKS, group2); 2785 | assert_eq!(cluster.has_leader_in_partition(group2), true); 2786 | 2787 | // send requests to group2 leader 2788 | let log_entry_3 = { 2789 | let current_leader_term = cluster 2790 | .get_leader_in_cluster(group2) 2791 | .unwrap() 2792 | .state 2793 | .lock() 2794 | .unwrap() 2795 | .current_term; 2796 | let mut last_log_index = cluster 2797 | .get_leader() 2798 | .unwrap() 2799 | .state 2800 | .lock() 2801 | .unwrap() 2802 | .log 2803 | .last() 2804 | .map_or(0, |entry| entry.index); 2805 | last_log_index += 1; 2806 | let log_entry_3 = LogEntry { 2807 | term: current_leader_term, 2808 | index: last_log_index, 2809 | command: LogEntryCommand::Set, 2810 | key: "c".to_string(), 2811 | value: 3, 2812 | }; 2813 | cluster.apply_entries_across_cluster_partition( 2814 | vec![&log_entry_3], 2815 | group2, 2816 | MAX_TICKS, 2817 | ); 2818 | log_entry_3 2819 | }; 2820 | 2821 | // partition heals and old leader rejoins the cluster 2822 | // cluster needs to be verified to ensure all logs are up to date 2823 | let leader_id = cluster.get_leader_in_cluster(group2).unwrap().id; 2824 | cluster.heal_partition(); 2825 | cluster.tick_by(MAX_TICKS); 2826 | cluster.wait_for_stable_leader(MAX_TICKS); 2827 | assert_eq!(cluster.get_leader().unwrap().id, leader_id); 2828 | cluster.verify_logs_across_cluster_for( 2829 | vec![&log_entry_1, &log_entry_2, &log_entry_3], 2830 | MAX_TICKS, 2831 | ); 2832 | } 2833 | 2834 | /// This test forces all the nodes in a cluster to time out at the same time so that it can be asserted 2835 | /// that the cluster eventually converges on a single leader 2836 | #[test] 2837 | fn concurrent_leader_election() { 2838 | let _ = env_logger::builder().is_test(true).try_init(); 2839 | let cluster_config = ClusterConfig { 2840 | election_timeout: ELECTION_TIMEOUT, 2841 | heartbeat_interval: HEARTBEAT_INTERVAL, 2842 | ports: vec![8000, 8001, 8002], 2843 | }; 2844 | let mut cluster = TestCluster::new(3, cluster_config); 2845 | 2846 | cluster.advance_time_by(ELECTION_TIMEOUT + Duration::from_millis(100)); 2847 | cluster.wait_for_stable_leader(MAX_TICKS * 3); // use 3 times the MAX_TICKS to ensure that the cluster has enough time to converge on a node 2848 | assert_eq!(cluster.has_leader(), true); 2849 | } 2850 | 2851 | #[test] 2852 | fn bulk_client_requests_test() { 2853 | let _ = env_logger::builder().is_test(true).try_init(); 2854 | let cluster_config = ClusterConfig { 2855 | election_timeout: ELECTION_TIMEOUT, 2856 | heartbeat_interval: HEARTBEAT_INTERVAL, 2857 | ports: vec![8000, 8001, 8002], 2858 | }; 2859 | let mut cluster = TestCluster::new(3, cluster_config); 2860 | 2861 | // cluster starts and a leader is elected 2862 | cluster.start(); 2863 | cluster.advance_time_by_for_node(0, ELECTION_TIMEOUT + Duration::from_millis(50)); 2864 | assert!(cluster.wait_for_stable_leader(MAX_TICKS)); 2865 | 2866 | // Generate and send bulk client requests 2867 | let num_requests = 100; 2868 | let mut test_data: Vec<(String, Option)> = Vec::new(); 2869 | for i in 0..num_requests { 2870 | let key = format!("key{}", i); 2871 | let value = i; 2872 | test_data.push((key.clone(), Option::Some(value))); 2873 | cluster.send_client_request(key, value, LogEntryCommand::Set); 2874 | thread::sleep(Duration::from_millis(10)); 2875 | } 2876 | 2877 | // allow some time to pass 2878 | cluster.tick_by(MAX_TICKS * 4); 2879 | let keys = test_data.iter().map(|(key, _)| key.as_str()).collect(); 2880 | let results = cluster.query_state_machine_across_nodes(keys); 2881 | 2882 | // each node's state machine should have all of the test data 2883 | let expected_results: Vec<(&str, Option)> = test_data 2884 | .iter() 2885 | .map(|(key, value)| (key.as_str(), *value)) 2886 | .collect(); 2887 | for (_, state_machine_results) in results.iter() { 2888 | assert_eq!(state_machine_results, &expected_results); 2889 | } 2890 | } 2891 | } 2892 | } 2893 | -------------------------------------------------------------------------------- /src/storage/file_ops.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cell::RefCell, 3 | fmt::Display, 4 | fs::{self, File, OpenOptions}, 5 | io::{self, BufRead, BufReader, Seek, SeekFrom, Write}, 6 | str::FromStr, 7 | }; 8 | 9 | use crate::{ 10 | types::{ServerId, Term}, 11 | LogEntry, LogEntryCommand, 12 | }; 13 | 14 | pub trait RaftFileOps { 15 | fn read_term_and_voted_for(&self) -> Result<(Term, ServerId), io::Error>; 16 | fn write_term_and_voted_for( 17 | &self, 18 | term: Term, 19 | voted_for: Option, 20 | ) -> Result<(), io::Error>; 21 | fn read_logs(&self, log_index: u64) -> Result>, io::Error>; 22 | fn append_logs(&mut self, entries: &Vec>) -> Result<(), io::Error>; 23 | fn append_logs_at( 24 | &mut self, 25 | entries: &Vec>, 26 | log_index: u64, 27 | ) -> Result<(), io::Error>; 28 | } 29 | 30 | pub struct DirectFileOpsWriter { 31 | file_path: String, 32 | file: RefCell>, 33 | } 34 | 35 | impl DirectFileOpsWriter { 36 | pub fn new(file_path: &str, server_id: ServerId) -> Result { 37 | let file_name = format!("{}_server_{}", file_path, server_id); 38 | let file = OpenOptions::new() 39 | .read(true) 40 | .write(true) 41 | .create(true) 42 | .open(&file_name)?; 43 | 44 | Ok(DirectFileOpsWriter { 45 | file_path: file_name, 46 | file: RefCell::new(Some(file)), 47 | }) 48 | } 49 | } 50 | 51 | impl Drop for DirectFileOpsWriter { 52 | fn drop(&mut self) { 53 | match fs::remove_file(&self.file_path) { 54 | Ok(()) => (), 55 | Err(e) => { 56 | panic!("Error removing file: {}", e); 57 | } 58 | } 59 | } 60 | } 61 | 62 | impl RaftFileOps for DirectFileOpsWriter { 63 | fn read_term_and_voted_for(&self) -> Result<(Term, ServerId), io::Error> { 64 | // let mut file = self.file.as_ref().expect("File not found"); 65 | let binding = self.file.borrow_mut(); 66 | let mut file = binding.as_ref().expect("File not found"); 67 | file.seek(SeekFrom::Start(0))?; 68 | let mut buf_reader = BufReader::new(file); 69 | let mut line = String::new(); 70 | buf_reader.read_line(&mut line)?; 71 | let values: Vec<&str> = line.split(',').collect(); 72 | let term: u64 = values[0] 73 | .trim() 74 | .parse() 75 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 76 | let voted_for: u64 = values[1] 77 | .trim() 78 | .parse() 79 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 80 | return Ok((term, voted_for)); 81 | } 82 | 83 | /// This function has been a bit of a pain. The earlier implemenation read the file using 84 | /// the file param and tried to overwrite the first line but that doesn't seem to work because 85 | /// a writeln function will literally just replace the bytes - so if the first line was 1,1000 86 | /// and the replacement is 1,2 this will shift the 3 zeroes to the next line 87 | /// The solution for now is to read the entire file into memory, modify the content and write it back 88 | /// There are obviously better ways but this should be good enough for now 89 | fn write_term_and_voted_for( 90 | &self, 91 | term: Term, 92 | voted_for: Option, 93 | ) -> Result<(), io::Error> { 94 | let content = fs::read_to_string(&self.file_path)?; 95 | let mut lines: Vec<&str> = content.lines().collect(); 96 | let new_line = format!("{},{}\n", term, voted_for.unwrap_or(100000)); 97 | if !lines.is_empty() { 98 | lines[0] = new_line.as_str(); 99 | } else { 100 | lines.push(new_line.as_str()); 101 | } 102 | let new_content = lines.join("\n"); 103 | 104 | let mut file = OpenOptions::new() 105 | .write(true) 106 | .truncate(true) 107 | .open(&self.file_path)?; 108 | file.write_all(new_content.as_bytes())?; 109 | Ok(()) 110 | } 111 | 112 | fn read_logs(&self, log_index: u64) -> Result>, io::Error> { 113 | let mut binding = self.file.borrow_mut(); 114 | let file = binding.as_mut().expect("File not found"); 115 | file.seek(SeekFrom::Start(0))?; 116 | let mut buf_reader = BufReader::new(file); 117 | let mut line = String::new(); 118 | let mut entries: Vec> = Vec::new(); 119 | for _ in 0..log_index { 120 | if buf_reader.read_line(&mut line)? == 0 { 121 | return Err(io::Error::new( 122 | io::ErrorKind::InvalidInput, 123 | "log_index out of bounds", 124 | )); 125 | } 126 | line.clear(); 127 | } 128 | 129 | for line in buf_reader.lines() { 130 | let l = line?; 131 | let values: Vec<&str> = l.split(',').collect(); 132 | let term: Term = values[0].trim().parse().map_err(|e| { 133 | io::Error::new(io::ErrorKind::InvalidData, format!("Invalid term: {}", e)) 134 | })?; 135 | let index: u64 = values[1].trim().parse().map_err(|e| { 136 | io::Error::new(io::ErrorKind::InvalidData, format!("Invalid index: {}", e)) 137 | })?; 138 | let command: LogEntryCommand = values[2].trim().parse().map_err(|e| { 139 | io::Error::new( 140 | io::ErrorKind::InvalidData, 141 | format!("Invalid command: {}", e), 142 | ) 143 | })?; 144 | let key: String = values[3].trim().to_string(); 145 | let value: T = values[4].trim().parse().map_err(|_| { 146 | io::Error::new(io::ErrorKind::InvalidData, format!("Invalid value")) 147 | })?; 148 | entries.push(LogEntry { 149 | term, 150 | index, 151 | command, 152 | key, 153 | value, 154 | }); 155 | } 156 | return Ok(entries); 157 | } 158 | 159 | fn append_logs(&mut self, entries: &Vec>) -> Result<(), io::Error> { 160 | let mut binding = self.file.borrow_mut(); 161 | let file = binding.as_mut().expect("File not found"); 162 | file.seek(SeekFrom::End(0))?; 163 | for entry in entries { 164 | writeln!( 165 | file, 166 | "{},{},{},{},{}", 167 | entry.term, entry.index, entry.command, entry.key, entry.value 168 | )?; 169 | } 170 | file.flush()?; 171 | Ok(()) 172 | } 173 | 174 | /** 175 | * This method is used to truncate the logs starting at log_index+1 and then 176 | * append the new logs past that point 177 | * Unfortunately, this function needs to re-open the file because Rust's borrow checker refuses 178 | * to allow me to use the existing file handle 179 | */ 180 | fn append_logs_at( 181 | &mut self, 182 | entries: &Vec>, 183 | log_index: u64, 184 | ) -> Result<(), io::Error> { 185 | let mut file = OpenOptions::new() 186 | .read(true) 187 | .write(true) 188 | .open(&self.file_path)?; 189 | file.seek(SeekFrom::Start(0))?; 190 | let mut buf_reader = BufReader::new(&file); 191 | let mut line = String::new(); 192 | let mut position = 0; 193 | let mut lines_read = 0; 194 | 195 | while lines_read <= log_index { 196 | line.clear(); 197 | let bytes_read = buf_reader.read_line(&mut line)?; 198 | if bytes_read == 0 { 199 | return Err(io::Error::new( 200 | io::ErrorKind::InvalidInput, 201 | format!("log_index {} out of bounds", log_index), 202 | )); 203 | } 204 | position += bytes_read as u64; 205 | lines_read += 1; 206 | } 207 | 208 | drop(buf_reader); 209 | file.set_len(position)?; 210 | file.seek(SeekFrom::Start(position))?; 211 | for entry in entries { 212 | writeln!( 213 | file, 214 | "{},{},{},{},{}", 215 | entry.term, entry.index, entry.command, entry.key, entry.value 216 | )?; 217 | } 218 | file.flush()?; 219 | Ok(()) 220 | } 221 | } 222 | 223 | #[derive(Clone, Debug)] 224 | struct TestEntryData(String); 225 | 226 | impl std::fmt::Display for TestEntryData { 227 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 228 | write!(f, "{}", self.0) 229 | } 230 | } 231 | 232 | impl std::str::FromStr for TestEntryData { 233 | type Err = std::convert::Infallible; 234 | 235 | fn from_str(s: &str) -> Result { 236 | Ok(TestEntryData(s.to_string())) 237 | } 238 | } 239 | 240 | #[cfg(test)] 241 | mod tests { 242 | use log::info; 243 | 244 | use std::{ 245 | fs::remove_file, 246 | io::{self}, 247 | }; 248 | 249 | use crate::{LogEntry, LogEntryCommand}; 250 | 251 | use super::{DirectFileOpsWriter, RaftFileOps, TestEntryData}; 252 | 253 | #[test] 254 | fn test_term_and_voted_for_write_and_read() -> Result<(), io::Error> { 255 | info!("Starting test_term_and_voted_for_write_and_read"); 256 | let _ = env_logger::try_init(); 257 | let temp_file = "temp_file"; 258 | let mut ops = DirectFileOpsWriter::new(temp_file, 0)?; 259 | >::write_term_and_voted_for( 260 | &mut ops, 261 | 1, 262 | Option::Some(2), 263 | )?; 264 | let (term, voted_for) = 265 | >::read_term_and_voted_for(&ops)?; 266 | // remove_file(format!("{}_server_{}", temp_file, 0))?; 267 | assert_eq!(term, 1); 268 | assert_eq!(voted_for, 2); 269 | Ok(()) 270 | } 271 | 272 | #[test] 273 | fn test_log_entries_write_and_read() -> Result<(), io::Error> { 274 | info!("Starting test_log_entries_write_and_read"); 275 | let _ = env_logger::try_init(); 276 | let temp_file = "temp_file"; 277 | let mut ops = DirectFileOpsWriter::new(temp_file, 0)?; 278 | >::write_term_and_voted_for( 279 | &mut ops, 280 | 1, 281 | Option::Some(2), 282 | )?; 283 | 284 | // generate a bunch of log entries and write them 285 | let mut log_entries: Vec> = Vec::new(); 286 | let mut log_index: u64 = 1; 287 | for _ in 0..20 { 288 | let log_entry: LogEntry = LogEntry { 289 | term: 1, 290 | index: log_index, 291 | command: LogEntryCommand::Set, 292 | key: log_index.to_string(), 293 | value: TestEntryData(format!("value_{}", log_index)), 294 | }; 295 | log_entries.push(log_entry); 296 | log_index += 1; 297 | } 298 | ops.append_logs(&log_entries)?; 299 | 300 | // read your own writes here 301 | let read_log_entries = 302 | >::read_logs(&mut ops, 1)?; 303 | // remove_file(format!("{}_server_{}", temp_file, 0))?; 304 | assert_eq!(log_entries.len(), read_log_entries.len()); 305 | Ok(()) 306 | } 307 | 308 | #[test] 309 | fn test_log_entries_write_and_read_from_offset() -> Result<(), io::Error> { 310 | info!("Starting test_log_entries_write_and_read_from_offset"); 311 | let _ = env_logger::try_init(); 312 | let temp_file = "temp_file"; 313 | let mut ops = DirectFileOpsWriter::new(temp_file, 0)?; 314 | >::write_term_and_voted_for( 315 | &mut ops, 316 | 1, 317 | Option::Some(2), 318 | )?; 319 | 320 | // generate a bunch of logs and write them 321 | let mut log_entries: Vec> = Vec::new(); 322 | let mut log_index: u64 = 1; 323 | for i in 0..100 { 324 | let log_entry: LogEntry = LogEntry { 325 | term: 1, 326 | index: log_index, 327 | command: LogEntryCommand::Set, 328 | key: i.to_string(), 329 | value: TestEntryData(format!("value_{}", i)), 330 | }; 331 | log_entries.push(log_entry); 332 | log_index += 1; 333 | } 334 | ops.append_logs(&log_entries)?; 335 | 336 | // reading your own writes from some offset here 337 | let read_log_entries = 338 | >::read_logs(&mut ops, 50)?; 339 | remove_file(format!("{}_server_{}", temp_file, 0))?; 340 | assert_eq!(log_entries.len() - 49, read_log_entries.len()); 341 | Ok(()) 342 | } 343 | 344 | #[test] 345 | fn test_log_entries_write_at_offset_and_read() -> Result<(), io::Error> { 346 | info!("Starting test_log_entries_write_at_offset_and_read"); 347 | let _ = env_logger::try_init(); 348 | let temp_file = "temp_file"; 349 | let mut ops = DirectFileOpsWriter::new(temp_file, 0)?; 350 | >::write_term_and_voted_for( 351 | &mut ops, 352 | 1, 353 | Option::Some(2), 354 | )?; 355 | 356 | // generate a bunch of logs and write them 357 | let mut log_entries: Vec> = Vec::new(); 358 | let mut log_index: u64 = 1; 359 | for i in 0..100 { 360 | let log_entry: LogEntry = LogEntry { 361 | term: 1, 362 | index: log_index, 363 | command: LogEntryCommand::Set, 364 | key: i.to_string(), 365 | value: TestEntryData(format!("value_{}", i)), 366 | }; 367 | log_entries.push(log_entry); 368 | log_index += 1; 369 | } 370 | ops.append_logs(&log_entries)?; 371 | 372 | // generate a new set of logs 373 | let mut new_log_entries: Vec> = Vec::new(); 374 | let mut new_log_index: u64 = 20; 375 | for i in 0..20 { 376 | let log_entry: LogEntry = LogEntry { 377 | term: 1, 378 | index: new_log_index, 379 | command: LogEntryCommand::Set, 380 | key: i.to_string(), 381 | value: TestEntryData(format!("value_{}", i)), 382 | }; 383 | new_log_entries.push(log_entry); 384 | new_log_index += 1; 385 | } 386 | ops.append_logs_at(&new_log_entries, 20)?; 387 | 388 | // read the logs 389 | let read_log_entries = 390 | >::read_logs(&mut ops, 1)?; 391 | remove_file(format!("{}_server_{}", temp_file, 0))?; 392 | assert_eq!(read_log_entries.len(), 40); 393 | Ok(()) 394 | } 395 | 396 | #[test] 397 | fn test_overwrite_term_and_voted_for_with_logs_present() -> Result<(), io::Error> { 398 | info!("Starting test_log_entries_write_at_offset_and_read"); 399 | let _ = env_logger::try_init(); 400 | let temp_file = "temp_file"; 401 | let mut ops = DirectFileOpsWriter::new(temp_file, 0)?; 402 | >::write_term_and_voted_for( 403 | &mut ops, 1, None, 404 | )?; 405 | 406 | // generate a bunch of logs and write them 407 | let mut log_entries: Vec> = Vec::new(); 408 | let mut log_index: u64 = 1; 409 | for i in 0..100 { 410 | let log_entry: LogEntry = LogEntry { 411 | term: 1, 412 | index: log_index, 413 | command: LogEntryCommand::Set, 414 | key: i.to_string(), 415 | value: TestEntryData(format!("value_{}", i)), 416 | }; 417 | log_entries.push(log_entry); 418 | log_index += 1; 419 | } 420 | ops.append_logs(&log_entries)?; 421 | 422 | // read your own writes here 423 | let read_log_entries = 424 | >::read_logs(&mut ops, 1)?; 425 | assert_eq!(log_entries.len(), read_log_entries.len()); 426 | 427 | // overwrite the term and voted for 428 | >::write_term_and_voted_for( 429 | &mut ops, 430 | 2, 431 | Some(1), 432 | )?; 433 | let (term, voted_for) = 434 | >::read_term_and_voted_for(&ops)?; 435 | assert_eq!(term, 2); 436 | assert_eq!(voted_for, 1); 437 | 438 | Ok(()) 439 | } 440 | } 441 | -------------------------------------------------------------------------------- /src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod file_ops; 2 | 3 | pub use file_ops::{DirectFileOpsWriter, RaftFileOps}; 4 | -------------------------------------------------------------------------------- /src/types.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt::{Debug, Display}, 3 | str::FromStr, 4 | }; 5 | 6 | use serde::{de::DeserializeOwned, Deserialize, Serialize}; 7 | 8 | pub type Term = u64; 9 | 10 | pub type ServerId = u64; 11 | 12 | /** 13 | * Trait Explanations 14 | * 1. Serialize -> Serde json serialization 15 | * 2. DeserializeOwned -> Serde json deserialize to a new instance 16 | * 3. Send -> Can transfer safely across threads 17 | * 4. 'static -> Lifetime specifier. Will live as long as the program (unsure about this) 18 | * 5. Debug -> Used by std lib for printing 19 | * 6. Clone -> Can create a duplicate of itself 20 | */ 21 | pub trait RaftTypeTrait: 22 | Serialize 23 | + for<'de> Deserialize<'de> 24 | + DeserializeOwned 25 | + Send 26 | + 'static 27 | + Debug 28 | + Clone 29 | + FromStr 30 | + Display 31 | { 32 | } 33 | impl< 34 | T: Serialize 35 | + for<'de> Deserialize<'de> 36 | + DeserializeOwned 37 | + Send 38 | + 'static 39 | + Debug 40 | + Clone 41 | + FromStr 42 | + Display, 43 | > RaftTypeTrait for T 44 | { 45 | } 46 | --------------------------------------------------------------------------------