├── raft_config.json ├── report ├── report.pdf └── benchmarks │ ├── raft_config.json │ ├── no_failure │ ├── workload_c_ecc │ │ ├── workload_c_ecc_mem.png │ │ ├── Screen Shot 2021-08-10 at 11.30.08 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.14 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.16 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.21 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.25 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.28 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.31 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.30.34 PM (2).png │ │ └── Screen Shot 2021-08-10 at 11.30.37 PM (2).png │ ├── workload_c_raft │ │ ├── workload_c_raft_mem.png │ │ ├── Screen Shot 2021-08-10 at 11.42.55 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.42.58 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.02 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.07 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.09 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.13 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.15 PM (2).png │ │ ├── Screen Shot 2021-08-10 at 11.43.21 PM (2).png │ │ └── Screen Shot 2021-08-10 at 11.43.25 PM (2).png │ ├── workload_a_ecc │ │ ├── Screen Shot 2021-08-08 at 11.21.25 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.29 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.34 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.37 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.40 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.43 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.51 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.21.53 PM (2).png │ │ └── Screen Shot 2021-08-08 at 11.21.55 PM (2).png │ ├── workload_a_raft │ │ ├── Screen Shot 2021-08-08 at 11.43.10 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.12 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.13 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.17 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.19 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.23 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.24 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.43.27 PM (2).png │ │ └── Screen Shot 2021-08-08 at 11.43.29 PM (2).png │ ├── workload_b_ecc │ │ ├── Screen Shot 2021-08-08 at 11.30.32 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.34 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.38 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.45 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.47 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.50 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.53 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.30.57 PM (2).png │ │ └── Screen Shot 2021-08-08 at 11.30.59 PM (2).png │ └── workload_b_raft │ │ ├── Screen Shot 2021-08-08 at 11.52.32 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.33 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.35 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.40 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.43 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.46 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.48 PM (2).png │ │ ├── Screen Shot 2021-08-08 at 11.52.50 PM (2).png │ │ └── Screen Shot 2021-08-08 at 11.53.48 PM (2).png │ ├── ecc_config.json │ ├── one_failure │ ├── workload_b_raft_node_failure │ │ ├── raft1_cpu.png │ │ ├── raft1_mem.png │ │ ├── raft2_cpu.png │ │ ├── raft2_mem.png │ │ ├── raft3_cpu.png │ │ ├── raft3_mem.png │ │ ├── raft1_network.png │ │ ├── raft2_network.png │ │ ├── raft3_network.png │ │ ├── raft3_replacement_cpu.png │ │ ├── raft3_replacement_mem.png │ │ └── raft3_replacement_network.png │ ├── workload_a_raft_node_failure │ │ ├── raft1_network.png │ │ ├── raft2_network.png │ │ ├── raft3_network.png │ │ ├── raft3_replacement_network.png │ │ ├── Screen Shot 2021-08-10 at 1.13.21 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.24 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.29 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.31 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.35 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.37 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.13.44 AM (2).png │ │ └── Screen Shot 2021-08-10 at 1.13.46 AM (2).png │ ├── workload_a_ecc_node_failure │ │ ├── Screen Shot 2021-08-10 at 12.56.02 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.07 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.09 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.14 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.17 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.27 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.29 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.38 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.40 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.57 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 12.56.59 AM (2).png │ │ └── Screen Shot 2021-08-10 at 12.57.24 AM (2).png │ └── workload_b_ecc_node_failure │ │ ├── Screen Shot 2021-08-10 at 1.05.15 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.18 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.20 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.44 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.47 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.50 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.53 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.05.57 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.06.00 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.06.22 AM (2).png │ │ ├── Screen Shot 2021-08-10 at 1.06.26 AM (2).png │ │ └── Screen Shot 2021-08-10 at 1.07.02 AM (2).png │ ├── readme.md │ ├── docker-compose-ecc.yml │ ├── docker-compose.yml │ ├── docker-compose-raft.yml │ └── benchmark.go ├── ecc_config.json ├── Dockerfile ├── .gitignore ├── src ├── raft.rs ├── ecc.rs ├── raft │ ├── raft.rs │ ├── client.rs │ ├── network.rs │ ├── server.rs │ └── storage.rs ├── main.rs └── ecc │ ├── server.rs │ └── client.rs ├── docker-compose.yml ├── docker-compose-raft.yml ├── .github └── workflows │ └── server.yml ├── Cargo.toml ├── LICENSE ├── proto ├── ecc.proto └── raft.proto ├── notes.md └── README.md /raft_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": ["0.0.0.0:4000", "0.0.0.0:4001", "0.0.0.0:4002"] 3 | } -------------------------------------------------------------------------------- /report/report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/report.pdf -------------------------------------------------------------------------------- /report/benchmarks/raft_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": ["0.0.0.0:4000", "0.0.0.0:4001", "0.0.0.0:4002"] 3 | } -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/workload_c_ecc_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/workload_c_ecc_mem.png -------------------------------------------------------------------------------- /report/benchmarks/ecc_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": ["0.0.0.0:3000", "0.0.0.0:3001", "0.0.0.0:3002"], 3 | "k": 2, 4 | "n": 3, 5 | "block_size": 8, 6 | "heartbeat_timeout_ms": 1000 7 | } -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/workload_c_raft_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/workload_c_raft_mem.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_cpu.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_mem.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_cpu.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_mem.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_cpu.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_mem.png -------------------------------------------------------------------------------- /ecc_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": ["0.0.0.0:3000", "0.0.0.0:3001", "0.0.0.0:3002", "0.0.0.0:3003", "0.0.0.0:3004"], 3 | "k": 3, 4 | "n": 5, 5 | "block_size": 4, 6 | "heartbeat_timeout_ms": 1000 7 | } -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/raft1_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/raft1_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/raft2_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/raft2_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/raft3_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/raft3_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft1_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft2_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_cpu.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_mem.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/raft3_replacement_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/raft3_replacement_network.png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_raft_node_failure/raft3_replacement_network.png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.25 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.25 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.29 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.29 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.34 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.34 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.37 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.37 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.40 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.40 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.43 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.43 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.51 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.51 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.53 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.53 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.55 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_ecc/Screen Shot 2021-08-08 at 11.21.55 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.10 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.10 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.12 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.12 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.13 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.13 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.17 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.17 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.19 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.19 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.23 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.23 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.24 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.24 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.27 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.27 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.29 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_a_raft/Screen Shot 2021-08-08 at 11.43.29 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.32 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.32 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.34 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.34 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.38 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.38 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.45 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.45 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.47 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.47 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.50 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.50 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.53 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.53 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.57 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.57 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.59 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_ecc/Screen Shot 2021-08-08 at 11.30.59 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.32 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.32 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.33 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.33 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.35 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.35 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.40 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.40 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.43 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.43 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.46 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.46 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.48 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.48 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.50 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.52.50 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.53.48 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_b_raft/Screen Shot 2021-08-08 at 11.53.48 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.08 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.08 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.14 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.14 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.16 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.16 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.21 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.21 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.25 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.25 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.28 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.28 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.31 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.31 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.34 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.34 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.37 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_ecc/Screen Shot 2021-08-10 at 11.30.37 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.42.55 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.42.55 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.42.58 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.42.58 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.02 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.02 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.07 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.07 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.09 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.09 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.13 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.13 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.15 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.15 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.21 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.21 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.25 PM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/no_failure/workload_c_raft/Screen Shot 2021-08-10 at 11.43.25 PM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.02 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.02 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.07 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.07 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.09 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.09 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.14 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.14 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.17 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.17 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.27 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.27 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.29 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.29 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.38 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.38 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.40 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.40 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.57 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.57 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.59 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.56.59 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.57.24 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_ecc_node_failure/Screen Shot 2021-08-10 at 12.57.24 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.21 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.21 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.24 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.24 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.29 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.29 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.31 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.31 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.35 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.35 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.37 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.37 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.44 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.44 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.46 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_a_raft_node_failure/Screen Shot 2021-08-10 at 1.13.46 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.15 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.15 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.18 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.18 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.20 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.20 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.44 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.44 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.47 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.47 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.50 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.50 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.53 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.53 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.57 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.05.57 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.00 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.00 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.22 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.22 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.26 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.06.26 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.07.02 AM (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixjchen/ECC-Cache/HEAD/report/benchmarks/one_failure/workload_b_ecc_node_failure/Screen Shot 2021-08-10 at 1.07.02 AM (2).png -------------------------------------------------------------------------------- /report/benchmarks/readme.md: -------------------------------------------------------------------------------- 1 | ``` 2 | go run benchmark.go 3 | ``` 4 | 5 | I hard coded most of the benchmarks, selecting which workload and which type of cache. Set, get, kill, start, disconnect and connect commands are all generated, a user will just need to piece what they need together. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.54 as builder 2 | WORKDIR /usr/src/myapp 3 | RUN rustup component add rustfmt 4 | COPY . . 5 | RUN cargo install --path . 6 | 7 | FROM debian:buster-slim 8 | COPY ecc_config.json . 9 | COPY raft_config.json . 10 | COPY --from=builder /usr/local/cargo/bin/distributed_cache /usr/local/bin/distributed_cache 11 | ENTRYPOINT ["distributed_cache"] 12 | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | *target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | 14 | # Added by cargo 15 | 16 | /target 17 | 18 | # MAC 19 | .DS_Store 20 | -------------------------------------------------------------------------------- /src/raft.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | pub mod network; 3 | pub mod raft; 4 | pub mod server; 5 | pub mod storage; 6 | use async_raft::NodeId; 7 | 8 | pub fn get_raft_settings() -> (Vec, Vec) { 9 | let mut settings = config::Config::default(); 10 | settings 11 | .merge(config::File::with_name("raft_config")) 12 | .unwrap(); 13 | 14 | let servers = settings 15 | .get_array("servers") 16 | .unwrap() 17 | .iter() 18 | .map(|x| x.to_string()) 19 | .collect::>(); 20 | 21 | let mut node_ids = Vec::new(); 22 | for i in 0..servers.len() { 23 | let i = i as NodeId; 24 | node_ids.push(i); 25 | } 26 | 27 | (node_ids, servers) 28 | } 29 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" # optional since v1.27.0 2 | services: 3 | ecc1: 4 | ports: 5 | - "3000:3000" 6 | image: felixchen1998/distributed-cache-server:latest 7 | command: ecc server startOne 0.0.0.0:3000 8 | environment: 9 | DOCKER_HOSTNAME: host.docker.internal 10 | 11 | ecc2: 12 | ports: 13 | - "3001:3001" 14 | image: felixchen1998/distributed-cache-server:latest 15 | command: ecc server startOne 0.0.0.0:3001 16 | environment: 17 | DOCKER_HOSTNAME: host.docker.internal 18 | 19 | ecc3: 20 | ports: 21 | - "3002:3002" 22 | image: felixchen1998/distributed-cache-server:latest 23 | command: ecc server startOne 0.0.0.0:3002 24 | environment: 25 | DOCKER_HOSTNAME: host.docker.internal 26 | 27 | -------------------------------------------------------------------------------- /report/benchmarks/docker-compose-ecc.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" # optional since v1.27.0 2 | services: 3 | ecc1: 4 | ports: 5 | - "3000:3000" 6 | image: felixchen1998/distributed-cache-server:latest 7 | command: ecc server startOne 0.0.0.0:3000 8 | environment: 9 | DOCKER_HOSTNAME: host.docker.internal 10 | 11 | ecc2: 12 | ports: 13 | - "3001:3001" 14 | image: felixchen1998/distributed-cache-server:latest 15 | command: ecc server startOne 0.0.0.0:3001 16 | environment: 17 | DOCKER_HOSTNAME: host.docker.internal 18 | 19 | ecc3: 20 | ports: 21 | - "3002:3002" 22 | image: felixchen1998/distributed-cache-server:latest 23 | command: ecc server startOne 0.0.0.0:3002 24 | environment: 25 | DOCKER_HOSTNAME: host.docker.internal 26 | 27 | -------------------------------------------------------------------------------- /docker-compose-raft.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" # optional since v1.27.0 2 | services: 3 | raft1: 4 | ports: 5 | - "4000:4000" 6 | image: felixchen1998/distributed-cache-server:latest 7 | command: raft server startOne 0 8 | environment: 9 | DOCKER_HOSTNAME: host.docker.internal 10 | 11 | raft2: 12 | ports: 13 | - "4001:4001" 14 | image: felixchen1998/distributed-cache-server:latest 15 | command: raft server startOne 1 16 | environment: 17 | DOCKER_HOSTNAME: host.docker.internal 18 | 19 | # Raft3 should be a follower 20 | raft3: 21 | ports: 22 | - "4002:4002" 23 | image: felixchen1998/distributed-cache-server:latest 24 | command: raft server startOne 2 25 | environment: 26 | DOCKER_HOSTNAME: host.docker.internal 27 | depends_on: 28 | - raft1 29 | - raft2 30 | 31 | -------------------------------------------------------------------------------- /report/benchmarks/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | netdata: 4 | image: netdata/netdata 5 | container_name: netdata 6 | # network_mode: host 7 | ports: 8 | - 19999:19999 9 | restart: unless-stopped 10 | cap_add: 11 | - SYS_PTRACE 12 | environment: 13 | - DOCKER_USR=root 14 | security_opt: 15 | - apparmor:unconfined 16 | volumes: 17 | - netdataconfig:/etc/netdata 18 | - netdatalib:/var/lib/netdata 19 | - netdatacache:/var/cache/netdata 20 | - /etc/passwd:/host/etc/passwd:ro 21 | - /etc/group:/host/etc/group:ro 22 | - /proc:/host/proc:ro 23 | - /sys:/host/sys:ro 24 | - /etc/os-release:/host/etc/os-release:ro 25 | - /var/run/docker.sock:/var/run/docker.sock:ro 26 | 27 | volumes: 28 | netdataconfig: 29 | netdatalib: 30 | netdatacache: -------------------------------------------------------------------------------- /src/ecc.rs: -------------------------------------------------------------------------------- 1 | pub mod client; 2 | pub mod server; 3 | pub type StdError = Box; 4 | 5 | pub fn get_ecc_settings() -> (usize, usize, usize, usize, Vec) { 6 | let mut settings = config::Config::default(); 7 | settings 8 | .merge(config::File::with_name("ecc_config")) 9 | .unwrap(); 10 | 11 | let servers = settings 12 | .get_array("servers") 13 | .unwrap() 14 | .iter() 15 | .map(|x| x.to_string()) 16 | .collect::>(); 17 | 18 | let k = settings.get_int("k").unwrap() as usize; 19 | let n = settings.get_int("n").unwrap() as usize; 20 | let heartbeat_timeout_ms = settings.get_int("heartbeat_timeout_ms").unwrap() as usize; 21 | let block_size = settings.get_int("block_size").unwrap() as usize; 22 | 23 | (k, n, heartbeat_timeout_ms, block_size, servers) 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/server.yml: -------------------------------------------------------------------------------- 1 | name: ci server 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | 8 | jobs: 9 | docker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - 13 | name: Checkout 14 | uses: actions/checkout@v2 15 | - 16 | name: Set up QEMU 17 | uses: docker/setup-qemu-action@v1 18 | - 19 | name: Set up Docker Buildx 20 | uses: docker/setup-buildx-action@v1 21 | - 22 | name: Login to DockerHub 23 | uses: docker/login-action@v1 24 | with: 25 | username: ${{ secrets.DOCKER_USERNAME }} 26 | password: ${{ secrets.DOCKER_API_KEY }} 27 | - 28 | name: Build and push 29 | uses: docker/build-push-action@v2 30 | with: 31 | push: true 32 | tags: felixchen1998/distributed-cache-server:latest 33 | -------------------------------------------------------------------------------- /report/benchmarks/docker-compose-raft.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" # optional since v1.27.0 2 | services: 3 | raft1: 4 | ports: 5 | - "4000:4000" 6 | image: felixchen1998/distributed-cache-server:latest 7 | command: raft server startOne 0 8 | environment: 9 | DOCKER_HOSTNAME: host.docker.internal 10 | 11 | raft2: 12 | ports: 13 | - "4001:4001" 14 | image: felixchen1998/distributed-cache-server:latest 15 | command: raft server startOne 1 16 | environment: 17 | DOCKER_HOSTNAME: host.docker.internal 18 | 19 | # Raft3 should be a follower 20 | raft3: 21 | ports: 22 | - "4002:4002" 23 | image: felixchen1998/distributed-cache-server:latest 24 | command: raft server startOne 2 25 | environment: 26 | DOCKER_HOSTNAME: host.docker.internal 27 | depends_on: 28 | - raft1 29 | - raft2 30 | 31 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "distributed_cache" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [dependencies] 8 | anyhow = "1.0.32" 9 | async-raft = { version="0.6" } 10 | serde = { version="1.0.114", features=["derive"] } 11 | serde_json = "1.0.57" 12 | thiserror = "1.0.20" 13 | tokio = { version="1.0", default-features=false, features=["fs", "io-util", "macros", "rt", "rt-multi-thread", "sync", "time"] } 14 | tracing = { version = "0.1.17" } 15 | tracing-futures = "0.2.4" 16 | futures="0.3.15" 17 | tonic = "0.5" 18 | prost = "0.8" 19 | clap = "2.33.3" 20 | reed-solomon-erasure = { version = "4.0", features = [ "simd-accel" ] } 21 | simple-error = "0.2.3" 22 | config = {version = "0.11", features=["json"]} 23 | uuid = { version = "0.8", features = [ "v4"] } 24 | 25 | 26 | [build-dependencies] 27 | tonic-build = "0.5" 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 felixjchen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /proto/ecc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package ecc_proto; 3 | 4 | service EccRpc { 5 | rpc Heartbeat (HeartbeatRequest) returns (HeartbeatReply); 6 | rpc Prepare (PrepareRequest) returns (PrepareReply); 7 | rpc Commit (CommitRequest) returns (CommitReply); 8 | rpc Abort (AbortRequest) returns (AbortReply); 9 | rpc Get (GetRequest) returns (GetReply); 10 | rpc GetKeys (GetKeysRequest) returns (GetKeysReply); 11 | } 12 | 13 | // Heartbeat 14 | message HeartbeatRequest {} 15 | message HeartbeatReply { 16 | string state = 1; 17 | } 18 | 19 | // TPC 20 | message PrepareRequest { 21 | string key = 1; 22 | string value = 2; 23 | string tid = 3; 24 | } 25 | message PrepareReply { 26 | string healthy_servers = 1; 27 | bool lock_acquired = 2; 28 | } 29 | message CommitRequest { 30 | string tid = 1; 31 | string key = 2; 32 | } 33 | message CommitReply { 34 | bool success = 1; 35 | } 36 | message AbortRequest { 37 | string tid = 1; 38 | string key = 2; 39 | } 40 | message AbortReply { 41 | bool success = 1; 42 | } 43 | 44 | 45 | 46 | message GetRequest { 47 | string key = 1; 48 | } 49 | message GetReply { 50 | optional string value = 1; 51 | } 52 | message GetKeysRequest { 53 | } 54 | message GetKeysReply { 55 | optional string keys = 1; 56 | } -------------------------------------------------------------------------------- /proto/raft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package raft_proto; 3 | 4 | service RaftRpc { 5 | rpc AppendEntries (AppendEntriesRpcRequest) returns (AppendEntriesRpcReply); 6 | rpc VoteRequest (VoteRequestRpcRequest) returns (VoteRequestRpcReply); 7 | rpc InstallSnapshot (InstallSnapshotRpcRequest) returns (InstallSnapshotRpcReply); 8 | rpc ClientWrite (ClientWriteRpcRequest) returns (ClientWriteRpcReply); 9 | rpc ClientRead (ClientReadRpcRequest) returns (ClientReadRpcReply); 10 | } 11 | 12 | message AppendEntriesRpcRequest { 13 | string data = 1; 14 | } 15 | 16 | message AppendEntriesRpcReply { 17 | string data = 1; 18 | } 19 | 20 | message VoteRequestRpcRequest { 21 | string data = 1; 22 | } 23 | 24 | message VoteRequestRpcReply { 25 | string data = 1; 26 | } 27 | 28 | message InstallSnapshotRpcRequest { 29 | string data = 1; 30 | } 31 | 32 | message InstallSnapshotRpcReply { 33 | string data = 1; 34 | } 35 | 36 | message ClientWriteRpcRequest { 37 | string key = 1; 38 | string value = 2; 39 | } 40 | message ClientWriteRpcReply { 41 | optional uint64 leader_id = 1; 42 | } 43 | 44 | message ClientReadRpcRequest { 45 | string key = 1; 46 | } 47 | message ClientReadRpcReply { 48 | optional string value = 1; 49 | optional uint64 leader_id = 2; 50 | } -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | ## Standard 2 | https://en.wikipedia.org/wiki/Redis 3 | https://redis.io/topics/cluster-tutorial 4 | https://github.com/tikv/tikv 5 | https://tikv.org/deep-dive/scalability/multi-raft/ 6 | 7 | ## Shard Rebalancing 8 | https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/redis-cluster-resharding-online.html 9 | https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/Shards.html 10 | 11 | ## Optimistic Concurrency Control 12 | https://stackoverflow.com/questions/129329/optimistic-vs-pessimistic-locking 13 | https://docs.databricks.com/delta/concurrency-control.html 14 | https://en.wikipedia.org/wiki/Optimistic_concurrency_control 15 | 16 | ## Reed Solomon 17 | https://github.com/darrenldl/reed-solomon-erasure 18 | 19 | ## Rust Tokio Async 20 | https://tokio.rs/tokio/tutorial/async 21 | 22 | ## Await first k 23 | https://stackoverflow.com/questions/68448854/how-to-await-for-the-first-k-futures 24 | 25 | ## Str to base 256 26 | https://doc.rust-lang.org/stable/std/string/struct.String.html#method.into_bytes 27 | https://doc.rust-lang.org/std/str/fn.from_utf8.html 28 | 29 | ## Redis uses a map internally 30 | db.c => dbAdd => dictAdd... 31 | 32 | ## 33 | docker run -e DOCKER_HOSTNAME=host.docker.internal felixchen1998/distributed-cache-server:latest ecc client set key2345 testfdasfdas 34 | 35 | docker run -e DOCKER_HOSTNAME=host.docker.internal felixchen1998/distributed-cache-server:latest raft client set key2345 testfdasfdas 36 | 37 | ## Docker routing 38 | https://dev.to/natterstefan/docker-tip-how-to-get-host-s-ip-address-inside-a-docker-container-5anh -------------------------------------------------------------------------------- /src/raft/raft.rs: -------------------------------------------------------------------------------- 1 | use crate::raft::network; 2 | use crate::raft::server; 3 | use crate::raft::storage; 4 | use async_raft::config::{Config}; 5 | use async_raft::{NodeId, Raft}; 6 | use futures::future::join_all; 7 | use std::collections::{HashMap, HashSet}; 8 | use std::sync::Arc; 9 | 10 | pub async fn start_raft( 11 | id: u64, 12 | node_ids: Vec, 13 | servers: Vec, 14 | ) -> Result<(), Box> { 15 | let mut members = HashSet::new(); 16 | let mut routing_table = HashMap::new(); 17 | for (i, &id) in node_ids.iter().enumerate() { 18 | members.insert(id); 19 | routing_table.insert(id, servers[i].to_string()); 20 | } 21 | 22 | let network = Arc::new(network::TonicgRPCNetwork::new(routing_table.clone())); 23 | let storage = Arc::new(storage::MemStore::new(id)); 24 | let config = Arc::new( 25 | Config::build("primary-raft-group".into()) 26 | .validate() 27 | .expect("failed to build Raft config"), 28 | ); 29 | 30 | let raft = Raft::new(id, config.clone(), network.clone(), storage.clone()); 31 | raft.initialize(members.clone()).await?; 32 | server::start_server( 33 | raft, 34 | storage.clone(), 35 | routing_table.get(&id).unwrap().clone(), 36 | ) 37 | .await?; 38 | Ok(()) 39 | } 40 | 41 | pub async fn start_rafts( 42 | node_ids: Vec, 43 | servers: Vec, 44 | ) -> Result<(), Box> { 45 | let mut members = HashSet::new(); 46 | let mut routing_table = HashMap::new(); 47 | for (i, &id) in node_ids.iter().enumerate() { 48 | members.insert(id); 49 | routing_table.insert(id, servers[i].to_string()); 50 | } 51 | 52 | // Create storages and networks 53 | let mut networks = HashMap::new(); 54 | let mut storages = HashMap::new(); 55 | for id in node_ids.clone() { 56 | let network = Arc::new(network::TonicgRPCNetwork::new(routing_table.clone())); 57 | networks.insert(id, network); 58 | let storage = Arc::new(storage::MemStore::new(id)); 59 | storages.insert(id, storage); 60 | } 61 | 62 | // Start rafts 63 | let mut rafts: HashMap = HashMap::new(); 64 | let config = Arc::new( 65 | Config::build("primary-raft-group".into()) 66 | .validate() 67 | .expect("failed to build Raft config"), 68 | ); 69 | for id in node_ids { 70 | let raft = Raft::new( 71 | id, 72 | config.clone(), 73 | networks.get(&id).unwrap().clone(), 74 | storages.get(&id).unwrap().clone(), 75 | ); 76 | rafts.insert(id, raft); 77 | } 78 | 79 | // Intialize rafts 80 | let mut futures = Vec::new(); 81 | for raft in rafts.values() { 82 | let future = raft.initialize(members.clone()); 83 | futures.push(future); 84 | } 85 | join_all(futures).await; 86 | 87 | // Await all servers 88 | let mut futures = Vec::new(); 89 | for (id, raft) in rafts.into_iter() { 90 | let future = server::start_server( 91 | raft, 92 | storages.get(&id).unwrap().clone(), 93 | routing_table.get(&id).unwrap().clone(), 94 | ); 95 | futures.push(future); 96 | } 97 | 98 | join_all(futures).await; 99 | Ok(()) 100 | } 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed-Cache 2 | 3 | 🗞️ [Report](https://github.com/felixjchen/Distributed-Cache/blob/main/report/report.pdf) 4 | 5 | 📊 [Benchmarks](https://github.com/felixjchen/Distributed-Cache/tree/main/report/benchmarks) 6 | 7 | 🎓 [Course / Studies](https://github.com/felixjchen/D94/blob/main/README.md) 8 | 9 | ## Motivation 10 | Reduce redundancy in distributed caching by avoiding data replication and applying error correction codes. 11 | 12 | 13 | ## Result 14 | 15 | Created two distributed key value stores, using two strategies: 16 | 1. Raft replication 17 | 2. Error Correcting Codes (Reed Solomon) 18 | 19 | To store 10,000 unique key-value pairs: 20 | ### ECC cache uses 3.5 MiB 21 | ![](https://user-images.githubusercontent.com/31393977/129127326-b744db92-29ca-4881-8aee-98c308f8b958.png) 22 | ### Raft based cache uses 6.0 MiB 23 | ![](https://user-images.githubusercontent.com/31393977/129127327-3d3aedab-76d6-4240-8225-d92d7a13cc78.png) 24 | 25 | 26 | ## Running From Source 27 | 28 | ``` 29 | git clone https://github.com/felixjchen/Distributed-ECC-Cache 30 | cd Distributed-ECC-Cache 31 | cargo build 32 | 33 | cargo run ecc server startAll 34 | cargo run ecc client set fruit cherry 35 | cargo run ecc client get fruit 36 | ``` 37 | - The command tree can be found in week 2 38 | 39 | # Implementation 40 | 41 | ## Assumptions 42 | - clients are healthy for the duration of a transacation 43 | - for ecc cache, transactions are atomic 44 | - for ecc cache, let value = utf8 encoding of value, then |value| <= k * block_size 45 | 46 | ## Todo overall 47 | - error handling 48 | - integration tests 49 | - cleanup logging 50 | 51 | ## Todo raft 52 | - Storage hard state 53 | - Handling getting from dead servers 54 | 55 | ## Todo ecc 56 | - better client naming and code reuse 57 | - 2PC may be buggy / testing 58 | - better timeout https://tokio-rs.github.io/tokio/doc/tokio/time/fn.timeout.html 59 | - Handling getting from dead servers 60 | 61 | ## Week 1 62 | - sketched outline for ECC cache 63 | - project setup 64 | - learned about gRPCs (tokio), Raft in Rust (tikv rust, async rust...) 65 | - implemented the networking trait for async-rust, to create a raft k/v store that uses gRPCs 66 | 67 | ## Week 2 68 | - created ECC client / server 69 | - Servers contain key value maps 70 | - Each server contains a block, where k*block_size = |message|, **we only need k blocks to reconstruct the message** 71 | - Reed Solomon requires Galois field 2^8... our message needs to be base 256. UTF8 does this for us, common characters have 1 char - 1 number in base256, rare characters are less efficient 72 | - Can recover if told so 73 | - Client ECC code 74 | - Reads first k responses, constructs message 75 | - Write to as many servers as possible (?) (missing optomistic concurrency) 76 | - cleaned up Raft implementation 77 | - Created client.rs, instead of using BloomRPC to test rpcs 78 | - No more stale reads, all reads are fresh from leader 79 | - Writing finds the leader 80 | - Among other code improvements 81 | - unified CLI entrypoint 82 | - cargo run 83 | - ecc 84 | - server 85 | - startAll 86 | - startOne 87 | - recover 88 | - client 89 | - set 90 | - k 91 | - v 92 | - get k 93 | - raft 94 | - server 95 | - startAll 96 | - client 97 | - set 98 | - k 99 | - v 100 | - get 101 | - k 102 | 103 | ## Week 3 104 | - Heartbeats, 2PC , Better Restore 105 | - Raft start one / client improvements 106 | - Dockerize everything, docker-compose and compiling the project 107 | - Figured out cAdvisor for resource monitoring 108 | - Tried netstat for resource monitoring 109 | - Wrote benchmarking tool 110 | 111 | ## Week 4 112 | - Report 113 | - Benchmarking workload b 114 | - 2PC + Restore 115 | - Raft membership changes 116 | - Report 117 | - Benchmarking maybe on a fresh VM on digital ocean ? 118 | - trying to get benchmark with errors 119 | -------------------------------------------------------------------------------- /src/raft/client.rs: -------------------------------------------------------------------------------- 1 | use async_raft::NodeId; 2 | use raft_proto::raft_rpc_client::RaftRpcClient; 3 | use raft_proto::{ClientReadRpcRequest, ClientWriteRpcRequest}; 4 | use std::collections::HashMap; 5 | use std::env; 6 | use tonic::transport::Channel; 7 | 8 | pub mod raft_proto { 9 | tonic::include_proto!("raft_proto"); 10 | } 11 | 12 | pub struct RaftClient { 13 | client_table: HashMap>>, 14 | last_leader: NodeId, 15 | n: u64, 16 | } 17 | 18 | impl RaftClient { 19 | pub async fn new(node_ids: Vec, addresses: Vec) -> RaftClient { 20 | let mut client_table = HashMap::new(); 21 | let last_leader = node_ids[0].clone(); 22 | let n = client_table.len() as u64; 23 | 24 | for (id, addr) in node_ids.iter().zip(addresses.iter()) { 25 | let addr = match env::var_os("DOCKER_HOSTNAME") { 26 | Some(hostname) => format!( 27 | "http://{}", 28 | addr.replace("0.0.0.0", &hostname.into_string().unwrap()) 29 | ), 30 | None => format!("http://{}", addr), 31 | }; 32 | println!("{}", addr); 33 | 34 | let client = RaftRpcClient::connect(addr.clone()).await; 35 | let client = match client { 36 | Ok(i) => Some(i), 37 | _ => None, 38 | }; 39 | client_table.insert(id.clone(), client); 40 | } 41 | // println!("{:?} {:?}", client_table, routing_table); 42 | 43 | RaftClient { 44 | client_table, 45 | last_leader, 46 | n, 47 | } 48 | } 49 | 50 | pub async fn set( 51 | &mut self, 52 | key: String, 53 | value: String, 54 | ) -> Result<(), Box> { 55 | let mut write = false; 56 | while !write { 57 | let client = self 58 | .client_table 59 | .get(&self.last_leader) 60 | .map(|c| c.clone()) 61 | .unwrap(); 62 | 63 | match client { 64 | Some(mut client) => { 65 | let request = tonic::Request::new(ClientWriteRpcRequest { 66 | key: key.clone(), 67 | value: value.clone(), 68 | }); 69 | let response = client.client_write(request).await?; 70 | let response = response.into_inner(); 71 | 72 | // Check if leader has changed, if not we're good! 73 | println!("{:?}", response); 74 | match response.leader_id { 75 | Some(new_leader) => { 76 | self.last_leader = new_leader; 77 | } 78 | None => { 79 | write = true; 80 | } 81 | } 82 | } 83 | _ => { 84 | self.last_leader += 1; 85 | self.last_leader = self.last_leader % self.n; 86 | } 87 | } 88 | } 89 | 90 | Ok(()) 91 | } 92 | 93 | pub async fn get(&mut self, key: String) -> Result, Box> { 94 | let mut read = false; 95 | let mut res = None; 96 | 97 | while !read { 98 | let client = self 99 | .client_table 100 | .get(&self.last_leader) 101 | .map(|c| c.clone()) 102 | .unwrap(); 103 | 104 | match client { 105 | Some(mut client) => { 106 | let request = tonic::Request::new(ClientReadRpcRequest { key: key.clone() }); 107 | let response = client.client_read(request).await?; 108 | let response = response.into_inner(); 109 | 110 | // Check if leader has changed, if not we're good! 111 | println!("{:?}", response); 112 | match response.leader_id { 113 | Some(new_leader) => { 114 | self.last_leader = new_leader; 115 | } 116 | None => { 117 | res = response.value; 118 | read = true; 119 | } 120 | } 121 | } 122 | _ => { 123 | self.last_leader += 1; 124 | self.last_leader = self.last_leader % self.n; 125 | } 126 | } 127 | } 128 | 129 | Ok(res) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/raft/network.rs: -------------------------------------------------------------------------------- 1 | use crate::raft::storage::ClientRequest; 2 | use anyhow::Result; 3 | use async_raft::async_trait::async_trait; 4 | use async_raft::raft::{ 5 | AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, 6 | VoteRequest, VoteResponse, 7 | }; 8 | use async_raft::{NodeId, RaftNetwork}; 9 | use raft_proto::raft_rpc_client::RaftRpcClient; 10 | use raft_proto::{AppendEntriesRpcRequest, InstallSnapshotRpcRequest, VoteRequestRpcRequest}; 11 | use std::collections::HashMap; 12 | use std::env; 13 | use tokio::sync::RwLock; 14 | use tonic::transport::Channel; 15 | 16 | pub mod raft_proto { 17 | tonic::include_proto!("raft_proto"); 18 | } 19 | 20 | /// A type which emulates a network transport and implements the `RaftNetwork` trait. 21 | pub struct TonicgRPCNetwork { 22 | routing_table: RwLock>, 23 | client_table: RwLock>>, 24 | } 25 | 26 | impl TonicgRPCNetwork { 27 | /// Create a new instance. 28 | pub fn new(routing_table: HashMap) -> Self { 29 | let routing_table = RwLock::new(routing_table); 30 | let client_table = Default::default(); 31 | Self { 32 | routing_table, 33 | client_table, 34 | } 35 | } 36 | 37 | #[allow(dead_code)] 38 | pub async fn add_route(&self, peer: NodeId, address: String) { 39 | let mut routing_table = self.routing_table.write().await; 40 | routing_table.insert(peer, address); 41 | } 42 | 43 | pub async fn get_route(&self, peer: NodeId) -> Result { 44 | let routing_table = self.routing_table.write().await; 45 | Ok(routing_table.get(&peer).cloned().unwrap()) 46 | } 47 | 48 | pub async fn get_client(&self, peer: NodeId) -> RaftRpcClient { 49 | let mut client_table = self.client_table.write().await; 50 | 51 | // Need to create connection 52 | if !client_table.contains_key(&peer) { 53 | let address = self.get_route(peer).await.unwrap(); 54 | 55 | let address = match env::var_os("DOCKER_HOSTNAME") { 56 | Some(hostname) => format!( 57 | "http://{}", 58 | address.replace("0.0.0.0", &hostname.into_string().unwrap()) 59 | ), 60 | None => format!("http://{}", address), 61 | }; 62 | 63 | let client = RaftRpcClient::connect(address).await.unwrap(); 64 | client_table.insert(peer, client); 65 | } 66 | 67 | // Return connection 68 | client_table.get(&peer).map(|c| c.clone()).unwrap() 69 | } 70 | } 71 | 72 | #[async_trait] 73 | impl RaftNetwork for TonicgRPCNetwork { 74 | /// Send an AppendEntries RPC to the target Raft node (§5). 75 | async fn append_entries( 76 | &self, 77 | target: NodeId, 78 | rpc: AppendEntriesRequest, 79 | ) -> Result { 80 | let mut client = self.get_client(target).await; 81 | 82 | let serialized = serde_json::to_string(&rpc).unwrap(); 83 | let request = tonic::Request::new(AppendEntriesRpcRequest { 84 | data: serialized.into(), 85 | }); 86 | 87 | let response = client.append_entries(request).await?; 88 | let serialized = response.into_inner().data; 89 | let deserialized: AppendEntriesResponse = serde_json::from_str(&serialized).unwrap(); 90 | 91 | Ok(deserialized) 92 | } 93 | 94 | /// Send an InstallSnapshot RPC to the target Raft node (§7). 95 | async fn install_snapshot( 96 | &self, 97 | target: NodeId, 98 | rpc: InstallSnapshotRequest, 99 | ) -> Result { 100 | let mut client = self.get_client(target).await; 101 | 102 | let serialized = serde_json::to_string(&rpc).unwrap(); 103 | let request = tonic::Request::new(InstallSnapshotRpcRequest { 104 | data: serialized.into(), 105 | }); 106 | 107 | let response = client.install_snapshot(request).await?; 108 | let serialized = response.into_inner().data; 109 | let deserialized: InstallSnapshotResponse = serde_json::from_str(&serialized).unwrap(); 110 | 111 | Ok(deserialized) 112 | } 113 | 114 | /// Send a RequestVote RPC to the target Raft node (§5). 115 | async fn vote(&self, target: NodeId, rpc: VoteRequest) -> Result { 116 | let mut client = self.get_client(target).await; 117 | let serialized = serde_json::to_string(&rpc).unwrap(); 118 | 119 | let request = tonic::Request::new(VoteRequestRpcRequest { 120 | data: serialized.into(), 121 | }); 122 | let response = client.vote_request(request).await?; 123 | let serialized = response.into_inner().data; 124 | let deserialized: VoteResponse = serde_json::from_str(&serialized).unwrap(); 125 | 126 | Ok(deserialized) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/raft/server.rs: -------------------------------------------------------------------------------- 1 | use crate::raft::network::TonicgRPCNetwork; 2 | use crate::raft::storage::{ClientRequest, ClientResponse, MemStore}; 3 | use anyhow::Result; 4 | use async_raft::error::{ClientReadError, ClientWriteError}; 5 | use async_raft::raft::{AppendEntriesRequest, InstallSnapshotRequest, VoteRequest}; 6 | use async_raft::raft::{ClientWriteRequest, Raft}; 7 | use raft_proto::raft_rpc_server::{RaftRpc, RaftRpcServer}; 8 | use raft_proto::{ 9 | AppendEntriesRpcReply, AppendEntriesRpcRequest, ClientReadRpcReply, ClientReadRpcRequest, 10 | ClientWriteRpcReply, ClientWriteRpcRequest, InstallSnapshotRpcReply, InstallSnapshotRpcRequest, 11 | VoteRequestRpcReply, VoteRequestRpcRequest, 12 | }; 13 | use std::sync::Arc; 14 | use tonic::{transport::Server, Request, Response, Status}; 15 | 16 | pub mod raft_proto { 17 | tonic::include_proto!("raft_proto"); 18 | } 19 | 20 | pub type MyRaft = Raft; 21 | 22 | pub struct RaftRpcService { 23 | raft: MyRaft, 24 | storage: Arc, 25 | } 26 | 27 | impl RaftRpcService { 28 | pub fn new(raft: MyRaft, storage: Arc) -> RaftRpcService { 29 | RaftRpcService { raft, storage } 30 | } 31 | } 32 | 33 | #[tonic::async_trait] 34 | impl RaftRpc for RaftRpcService { 35 | async fn append_entries( 36 | &self, 37 | request: Request, 38 | ) -> Result, Status> { 39 | let serialized = request.into_inner().data; 40 | let deserialized: AppendEntriesRequest = 41 | serde_json::from_str(&serialized).unwrap(); 42 | 43 | println!("Got a append_entries request: {:?}", deserialized); 44 | 45 | let response = self.raft.append_entries(deserialized).await.unwrap(); 46 | let reply = AppendEntriesRpcReply { 47 | data: serde_json::to_string(&response).unwrap(), 48 | }; 49 | 50 | Ok(Response::new(reply)) 51 | } 52 | 53 | async fn vote_request( 54 | &self, 55 | request: Request, 56 | ) -> Result, Status> { 57 | let serialized = request.into_inner().data; 58 | let deserialized: VoteRequest = serde_json::from_str(&serialized).unwrap(); 59 | 60 | let response = self.raft.vote(deserialized).await.unwrap(); 61 | let reply = VoteRequestRpcReply { 62 | data: serde_json::to_string(&response).unwrap(), 63 | }; 64 | 65 | Ok(Response::new(reply)) 66 | } 67 | 68 | async fn install_snapshot( 69 | &self, 70 | request: Request, 71 | ) -> Result, Status> { 72 | let serialized = request.into_inner().data; 73 | let deserialized: InstallSnapshotRequest = serde_json::from_str(&serialized).unwrap(); 74 | 75 | let response = self.raft.install_snapshot(deserialized).await.unwrap(); 76 | let reply = InstallSnapshotRpcReply { 77 | data: serde_json::to_string(&response).unwrap(), 78 | }; 79 | 80 | Ok(Response::new(reply)) 81 | } 82 | 83 | async fn client_write( 84 | &self, 85 | request: Request, 86 | ) -> Result, Status> { 87 | let request = request.into_inner(); 88 | let key = request.key; 89 | let value = request.value; 90 | 91 | let new_log = ClientRequest { key, value }; 92 | let raft_request = ClientWriteRequest::new(new_log); 93 | let reply = match self.raft.client_write(raft_request).await { 94 | Ok(_) => ClientWriteRpcReply { leader_id: None }, 95 | Err(ClientWriteError::ForwardToLeader(_, leader_id)) => ClientWriteRpcReply { leader_id }, 96 | Err(_) => panic!("raft write error"), 97 | }; 98 | 99 | Ok(Response::new(reply)) 100 | } 101 | 102 | async fn client_read( 103 | &self, 104 | request: Request, 105 | ) -> Result, Status> { 106 | let request = request.into_inner(); 107 | let key = request.key; 108 | 109 | // Guard stale reads 110 | let state_machine = self.storage.read_state_machine().await; 111 | let reply = match self.raft.client_read().await { 112 | Ok(_) => ClientReadRpcReply { 113 | value: state_machine.kv_store.get(&key).cloned(), 114 | leader_id: None, 115 | }, 116 | Err(ClientReadError::ForwardToLeader(leader_id)) => ClientReadRpcReply { 117 | value: None, 118 | leader_id, 119 | }, 120 | Err(_) => panic!("raft write error"), 121 | }; 122 | Ok(Response::new(reply)) 123 | } 124 | } 125 | 126 | pub async fn start_server( 127 | raft: MyRaft, 128 | storage: Arc, 129 | address: String, 130 | ) -> Result<(), Box> { 131 | let addr = address.parse().unwrap(); 132 | let service = RaftRpcService::new(raft, storage); 133 | Server::builder() 134 | .add_service(RaftRpcServer::new(service)) 135 | .serve(addr) 136 | .await?; 137 | Ok(()) 138 | } 139 | -------------------------------------------------------------------------------- /report/benchmarks/benchmark.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os/exec" 7 | "strconv" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | // https://tikv.org/blog/tikv-3.0ga/ 13 | 14 | const ecc_prefix = "distributed_cache ecc client" 15 | const raft_prefix = "distributed_cache raft client" 16 | 17 | const records = 10000 18 | 19 | func get_stop_raft_node_command() []string { 20 | return strings.Split("docker kill benchmarks_raft3_1", " ") 21 | } 22 | func get_start_raft_node_command() []string { 23 | return strings.Split("docker run -d -p 4002:4002 -e DOCKER_HOSTNAME=host.docker.internal --name benchmarks_raft3_replacement_1 felixchen1998/distributed-cache-server:latest raft server startOne 2", " ") 24 | } 25 | 26 | func get_stop_ecc_node_command() []string { 27 | return strings.Split("docker kill benchmarks_ecc3_1", " ") 28 | } 29 | func get_start_ecc_node_command() []string { 30 | return strings.Split("docker run -d -p 3002:3002 -e DOCKER_HOSTNAME=host.docker.internal --name benchmarks_ecc3_replacement_1 felixchen1998/distributed-cache-server:latest ecc server startOne 0.0.0.0:3002 recover", " ") 31 | } 32 | 33 | func get_disconnect_ecc_node_command() []string { 34 | return strings.Split("docker network disconnect benchmarks_default benchmarks_ecc3_1", " ") 35 | } 36 | func get_connect_ecc_node_command() []string { 37 | return strings.Split("docker network connect benchmarks_default benchmarks_ecc3_1", " ") 38 | } 39 | 40 | func get_set_command(key_number int) string { 41 | set_command := "key" + strconv.Itoa(key_number) + " value" + strconv.Itoa(rand.Intn(records)) 42 | return set_command 43 | } 44 | 45 | func insert(a [][]string, index int, value []string) [][]string { 46 | a = append(a[:index+1], a[index:]...) // Step 1+2 47 | a[index] = value // Step 3 48 | return a 49 | } 50 | 51 | func get_get_command(key_number int) string { 52 | return "key" + strconv.Itoa(key_number) 53 | } 54 | 55 | func get_commands(prefix string, count int, command_fn func(int) string) [][]string { 56 | var res [][]string 57 | for i := 0; i < count; i++ { 58 | command := strings.Split(prefix+command_fn(i), " ") 59 | res = append(res, command) 60 | } 61 | return res 62 | } 63 | 64 | func shuffle_workload(workload [][]string) { 65 | rand.Shuffle(len(workload), func(i int, j int) { 66 | workload[i], workload[j] = workload[j], workload[i] 67 | }) 68 | } 69 | 70 | func get_workload_A(prefix string) [][]string { 71 | // 50 Read / 50 Read 72 | reads := 0.5 * records 73 | writes := 0.5 * records 74 | 75 | write_workload := get_commands(prefix+" set ", int(writes), get_set_command) 76 | read_workload := get_commands(prefix+" get ", int(reads), get_get_command) 77 | workload := append(write_workload, read_workload...) 78 | shuffle_workload(workload) 79 | 80 | return workload 81 | } 82 | 83 | func get_workload_B(prefix string) [][]string { 84 | // 95 Read / 5 Write 85 | reads := 0.95 * records 86 | writes := 0.05 * records 87 | 88 | write_workload := get_commands(prefix+" set ", int(writes), get_set_command) 89 | read_workload := get_commands(prefix+" get ", int(reads), get_get_command) 90 | workload := append(write_workload, read_workload...) 91 | shuffle_workload(workload) 92 | 93 | return workload 94 | } 95 | 96 | func get_workload_C(prefix string) [][]string { 97 | // 100 Write 98 | writes := records 99 | 100 | workload := get_commands(prefix+" set ", int(writes), get_set_command) 101 | shuffle_workload(workload) 102 | 103 | return workload 104 | } 105 | 106 | func run_ecc(workload [][]string) { 107 | cmd := exec.Command("docker", strings.Fields("compose -f docker-compose-ecc.yml up -d")...) 108 | stdout, _ := cmd.Output() 109 | fmt.Println(string(stdout)) 110 | fmt.Println("Started ecc cache") 111 | 112 | start := time.Now() 113 | for i, command := range workload { 114 | cmd := exec.Command(command[0], command[1:]...) 115 | stdout, _ := cmd.Output() 116 | fmt.Println(i, string(stdout)) 117 | } 118 | elapsed := time.Since(start) 119 | fmt.Printf("Benchmark took %s", elapsed) 120 | fmt.Printf("Average transaction time %s seconds", elapsed.Seconds()/records) 121 | 122 | cmd = exec.Command("docker", strings.Fields("compose -f docker-compose-ecc.yml down")...) 123 | stdout, _ = cmd.Output() 124 | fmt.Println(string(stdout)) 125 | fmt.Println("Stopped ecc cache") 126 | } 127 | 128 | func run_raft(workload [][]string) { 129 | cmd := exec.Command("docker", strings.Fields("compose -f docker-compose-raft.yml up -d")...) 130 | stdout, _ := cmd.Output() 131 | fmt.Println(string(stdout)) 132 | fmt.Println("Started raft cache") 133 | 134 | start := time.Now() 135 | for i, command := range workload { 136 | cmd := exec.Command(command[0], command[1:]...) 137 | stdout, _ := cmd.Output() 138 | fmt.Println(i, string(stdout)) 139 | } 140 | elapsed := time.Since(start) 141 | fmt.Printf("Benchmark took %s", elapsed) 142 | fmt.Printf("Average transaction time %s seconds", elapsed.Seconds()/records) 143 | 144 | cmd = exec.Command("docker", strings.Fields("compose -f docker-compose-raft.yml down")...) 145 | stdout, _ = cmd.Output() 146 | fmt.Println(string(stdout)) 147 | fmt.Println("Stopped raft cache") 148 | } 149 | 150 | func run_raft_test() { 151 | workload := get_workload_C(raft_prefix) 152 | fmt.Println("Generated workload") 153 | // insert(workload, 3333, get_stop_raft_node_command()) 154 | // insert(workload, 6666, get_start_raft_node_command()) 155 | 156 | run_raft(workload) 157 | // run_raft(workload) 158 | // run_raft(workload) 159 | } 160 | 161 | 162 | func run_ecc_test() { 163 | workload := get_workload_C(ecc_prefix) 164 | fmt.Println("Generated workload") 165 | // insert(workload, 3333, get_disconnect_ecc_node_command()) 166 | // insert(workload, 6666, get_connect_ecc_node_command()) 167 | 168 | run_ecc(workload) 169 | } 170 | 171 | func main() { 172 | run_raft_test() 173 | } 174 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, Arg, SubCommand}; 2 | mod ecc; 3 | mod raft; 4 | use ecc::client::EccClient; 5 | use ecc::server::{start_many_servers, start_server}; 6 | use simple_error::bail; 7 | 8 | #[tokio::main] 9 | async fn main() -> Result<(), Box> { 10 | let matches = App::new("Distributed Cache") 11 | .version(env!("CARGO_PKG_VERSION")) 12 | .author(env!("CARGO_PKG_AUTHORS")) 13 | .about("Distributed cache with two modes for fault tolerence. Raft or error correcting codes.") 14 | .subcommand( 15 | SubCommand::with_name("ecc") 16 | .about("ECC Cache") 17 | .subcommand( 18 | SubCommand::with_name("server") 19 | .about("ECC Cache Server") 20 | .subcommand(SubCommand::with_name("startAll").about("Start all nodes from config.json")) 21 | .subcommand( 22 | SubCommand::with_name("startOne") 23 | .about("Start a single node from config.json. Meant to test restoring nodes.") 24 | .arg(Arg::with_name("address").help("a").required(true).index(1)) 25 | .arg( 26 | Arg::with_name("recover") 27 | .help("a") 28 | .required(false) 29 | .index(2) 30 | .default_value("no"), 31 | ), 32 | ), 33 | ) 34 | .subcommand( 35 | SubCommand::with_name("client") 36 | .about("ECC Cache Client") 37 | .subcommand( 38 | SubCommand::with_name("set") 39 | .about("Set key with value") 40 | .arg(Arg::with_name("key").help("k").required(true).index(1)) 41 | .arg(Arg::with_name("value").help("v").required(true).index(2)), 42 | ) 43 | .subcommand( 44 | SubCommand::with_name("get") 45 | .about("Get key's value") 46 | .arg(Arg::with_name("key").help("k").required(true).index(1)), 47 | ), 48 | ), 49 | ) 50 | .subcommand( 51 | SubCommand::with_name("raft") 52 | .about("Raft Cache") 53 | .subcommand( 54 | SubCommand::with_name("server") 55 | .about("Raft Cache Server") 56 | .subcommand(SubCommand::with_name("startAll").about("Starts a n node raft kv cache")) 57 | .subcommand( 58 | SubCommand::with_name("startOne") 59 | .about("Add a new raft member") 60 | .arg(Arg::with_name("id").help("i").required(true).index(1)), 61 | ), 62 | ) 63 | .subcommand( 64 | SubCommand::with_name("client") 65 | .about("Raft Cache Client") 66 | .subcommand( 67 | SubCommand::with_name("set") 68 | .about("Set key with value") 69 | .arg(Arg::with_name("key").help("k").required(true).index(1)) 70 | .arg(Arg::with_name("value").help("v").required(true).index(2)), 71 | ) 72 | .subcommand( 73 | SubCommand::with_name("get") 74 | .about("Get key's value") 75 | .arg(Arg::with_name("key").help("k").required(true).index(1)), 76 | ), 77 | ), 78 | ) 79 | .get_matches(); 80 | 81 | // ECC 82 | if let Some(matches) = matches.subcommand_matches("ecc") { 83 | // Read ecc settings from config.json 84 | let (k, n, heartbeat_timeout_ms, block_size, servers) = ecc::get_ecc_settings(); 85 | // ECC Server CLI 86 | if let Some(matches) = matches.subcommand_matches("server") { 87 | // Start all ECC servers 88 | if let Some(_) = matches.subcommand_matches("startAll") { 89 | start_many_servers(servers.clone(), heartbeat_timeout_ms) 90 | .await 91 | .unwrap(); 92 | } 93 | // Start one ECC server node during restore 94 | if let Some(matches) = matches.subcommand_matches("startOne") { 95 | let addr = matches.value_of("address").unwrap().to_string(); 96 | let recover = matches.value_of("recover").unwrap().to_string() == "recover"; 97 | match servers.iter().position(|i| i.clone() == addr) { 98 | Some(id) => start_server(id, addr, servers.clone(), heartbeat_timeout_ms, recover) 99 | .await 100 | .unwrap(), 101 | None => bail!("server address not found in config"), 102 | } 103 | } 104 | } 105 | // ECC Client CLI 106 | if let Some(matches) = matches.subcommand_matches("client") { 107 | let mut client = EccClient::new().await; 108 | // SET KV 109 | if let Some(matches) = matches.subcommand_matches("set") { 110 | let key = matches.value_of("key").unwrap().to_string(); 111 | let value = matches.value_of("value").unwrap().to_string(); 112 | client.two_phase_commit(key, value).await.unwrap(); 113 | } 114 | // GET K 115 | if let Some(matches) = matches.subcommand_matches("get") { 116 | let key = matches.value_of("key").unwrap().to_string(); 117 | println!("{:?}", client.get(key).await.unwrap()); 118 | } 119 | } 120 | } 121 | 122 | // Raft 123 | if let Some(matches) = matches.subcommand_matches("raft") { 124 | let (node_ids, servers) = raft::get_raft_settings(); 125 | if let Some(matches) = matches.subcommand_matches("server") { 126 | if let Some(_) = matches.subcommand_matches("startAll") { 127 | raft::raft::start_rafts(node_ids.clone(), servers.clone()).await?; 128 | } 129 | if let Some(matches) = matches.subcommand_matches("startOne") { 130 | let id = matches.value_of("id").unwrap().to_string(); 131 | raft::raft::start_raft( 132 | id.parse::().unwrap(), 133 | node_ids.clone(), 134 | servers.clone(), 135 | ) 136 | .await?; 137 | } 138 | } 139 | if let Some(matches) = matches.subcommand_matches("client") { 140 | let mut client = raft::client::RaftClient::new(node_ids.clone(), servers.clone()).await; 141 | // SET KV 142 | if let Some(matches) = matches.subcommand_matches("set") { 143 | let key = matches.value_of("key").unwrap().to_string(); 144 | let value = matches.value_of("value").unwrap().to_string(); 145 | client.set(key, value).await.unwrap(); 146 | } 147 | // GET K 148 | if let Some(matches) = matches.subcommand_matches("get") { 149 | let key = matches.value_of("key").unwrap().to_string(); 150 | println!("{:?}", client.get(key).await.unwrap()); 151 | } 152 | } 153 | }; 154 | 155 | Ok(()) 156 | } 157 | -------------------------------------------------------------------------------- /src/ecc/server.rs: -------------------------------------------------------------------------------- 1 | use crate::ecc::client::EccClient; 2 | use crate::ecc::{get_ecc_settings, StdError}; 3 | use ecc_proto::ecc_rpc_server::{EccRpc, EccRpcServer}; 4 | use ecc_proto::*; 5 | use futures::future::join_all; 6 | use std::collections::{HashMap, HashSet}; 7 | use std::sync::Arc; 8 | use tokio::runtime::Handle; 9 | use tokio::sync::RwLock; 10 | use tokio::time::{sleep, timeout, Duration}; 11 | use tonic::{transport::Server, Code, Request, Response, Status}; 12 | 13 | pub mod ecc_proto { 14 | tonic::include_proto!("ecc_proto"); 15 | } 16 | 17 | #[derive(Clone, PartialEq, Debug)] 18 | enum State { 19 | Ready, 20 | NotReady, 21 | } 22 | 23 | // Write ahead log lock 24 | #[derive(Clone, Debug)] 25 | struct Lock { 26 | lock: String, 27 | value: String, 28 | } 29 | 30 | pub struct EccRpcService { 31 | id: usize, 32 | k: usize, 33 | servers: Vec, 34 | state: RwLock, 35 | storage: RwLock>, 36 | lock_table: RwLock>, 37 | healthy_servers: RwLock>, 38 | client: RwLock, 39 | } 40 | 41 | impl EccRpcService { 42 | pub async fn new( 43 | id: usize, 44 | servers: Vec, 45 | recover: bool, 46 | ) -> Result { 47 | let (k, n, heartbeat_timeout_ms, block_size, servers) = get_ecc_settings(); 48 | let client = EccClient::new().await; 49 | let state = if !recover { 50 | State::Ready 51 | } else { 52 | State::NotReady 53 | }; 54 | let res = EccRpcService { 55 | id, 56 | k, 57 | servers, 58 | healthy_servers: Default::default(), 59 | state: RwLock::new(state), 60 | storage: Default::default(), 61 | lock_table: Default::default(), 62 | client: RwLock::new(client), 63 | }; 64 | Ok(res) 65 | } 66 | 67 | async fn get_state(&self) -> State { 68 | let state = self.state.read().await; 69 | state.clone() 70 | } 71 | async fn set_state(&self, new_state: State) { 72 | let mut state = self.state.write().await; 73 | *state = new_state; 74 | } 75 | async fn assert_ready(&self) -> Result<(), Status> { 76 | let state = self.get_state().await; 77 | match state { 78 | State::Ready => Ok(()), 79 | State::NotReady => Err(Status::new( 80 | Code::Unavailable, 81 | format!("{:?} is not ready", self.id), 82 | )), 83 | } 84 | } 85 | // Delete all entries and lock_table, to be called when out of date 86 | async fn drain(&self) { 87 | let mut storage = self.storage.write().await; 88 | storage.drain(); 89 | let mut lock_table = self.lock_table.write().await; 90 | lock_table.drain(); 91 | } 92 | 93 | async fn recover(&self) -> Result<(), StdError> { 94 | let client = self.client.write().await; 95 | 96 | // Get all keys to fill 97 | let servers_len = self.servers.len(); 98 | let mut target = (self.id + 1) % servers_len; 99 | let mut found_keys = false; 100 | 101 | while !found_keys { 102 | println!( 103 | "RECOVER for {:?} target {:?}", 104 | self.id, 105 | self.servers[target].clone() 106 | ); 107 | // let keys_option = client.get_keys_once(self.servers[target].clone()).await?; 108 | let keys_option = match timeout( 109 | Duration::from_millis(1000), 110 | client.get_keys_once(self.servers[target].clone()), 111 | ) 112 | .await 113 | { 114 | Ok(Ok(keys)) => keys, 115 | _ => None, 116 | }; 117 | 118 | match keys_option { 119 | Some(keys) => { 120 | println!("RECOVER got keys {:?}", keys); 121 | // Get all values 122 | let mut storage = self.storage.write().await; 123 | let mut exclude_servers = HashSet::new(); 124 | exclude_servers.insert(self.servers[self.id].clone()); 125 | for key in keys { 126 | let codeword = client 127 | .get_codeword(key.clone(), exclude_servers.clone()) 128 | .await? 129 | .unwrap(); 130 | let parity = serde_json::to_string(&codeword[self.id]).unwrap(); 131 | storage.insert(key, parity); 132 | } 133 | 134 | // All is good 135 | found_keys = true; 136 | self.set_state(State::Ready).await; 137 | } 138 | None => { 139 | // Keep looking 140 | target = (target + 1) % servers_len; 141 | if target == self.id { 142 | target = (target + 1) % servers_len; 143 | } 144 | } 145 | } 146 | } 147 | Ok(()) 148 | } 149 | 150 | async fn get_cluster_status(&self) { 151 | let mut client = self.client.write().await; 152 | let healthy_servers_new = 153 | match timeout(Duration::from_millis(1000), client.send_heartbeats()).await { 154 | Err(_) => Default::default(), 155 | Ok(res) => res, 156 | }; 157 | let mut healthy_servers = self.healthy_servers.write().await; 158 | 159 | // Only if healthy servers changed 160 | println!("HEARTBEAT {:?} {:?} ", healthy_servers, healthy_servers_new); 161 | if healthy_servers_new.len() < self.k { 162 | println!( 163 | "{:?} has too few peers and is now outdated, discarding everything", 164 | self.id 165 | ); 166 | self.set_state(State::NotReady).await; 167 | self.drain().await; 168 | } 169 | *healthy_servers = healthy_servers_new; 170 | } 171 | } 172 | 173 | #[tonic::async_trait] 174 | impl EccRpc for Arc { 175 | async fn get(&self, request: Request) -> Result, Status> { 176 | self.assert_ready().await?; 177 | 178 | let request = request.into_inner(); 179 | // println!("Got a get request: {:?}", request.clone()); 180 | let key = request.key; 181 | 182 | let storage = self.storage.read().await; 183 | let value = storage.get(&key).map(|val| val.clone()); 184 | 185 | let reply = GetReply { value }; 186 | 187 | Ok(Response::new(reply)) 188 | } 189 | 190 | async fn get_keys(&self, _: Request) -> Result, Status> { 191 | self.assert_ready().await?; 192 | 193 | // Only give keys if nothing in locktable 194 | let lock_table = self.lock_table.read().await; 195 | 196 | if lock_table.keys().len() == 0 { 197 | let storage = self.storage.read().await; 198 | let keys = storage.keys().cloned().collect::>(); 199 | let keys = serde_json::to_string(&keys).unwrap(); 200 | let reply = GetKeysReply { keys: Some(keys) }; 201 | Ok(Response::new(reply)) 202 | } else { 203 | let reply = GetKeysReply { keys: None }; 204 | Ok(Response::new(reply)) 205 | } 206 | } 207 | 208 | async fn heartbeat( 209 | &self, 210 | request: Request, 211 | ) -> Result, Status> { 212 | let state = match self.get_state().await { 213 | State::NotReady => "NotReady".to_string(), 214 | State::Ready => "Ready".to_string(), 215 | }; 216 | Ok(Response::new(HeartbeatReply { state })) 217 | } 218 | 219 | async fn prepare( 220 | &self, 221 | request: Request, 222 | ) -> Result, Status> { 223 | self.assert_ready().await?; 224 | let request = request.into_inner(); 225 | // println!("Got a prepare request: {:?}", request.clone()); 226 | let tid = request.tid; 227 | let value = request.value; 228 | let key = request.key; 229 | 230 | let mut lock_table = self.lock_table.write().await; 231 | let healthy_servers = self.healthy_servers.read().await.clone(); 232 | let healthy_servers = serde_json::to_string(&healthy_servers).unwrap(); 233 | 234 | // Grant lock if : 1. Not locked OR 2. TID matches 235 | let grant_lock = match lock_table.get(&key) { 236 | Some(existing_lock) => { 237 | existing_lock.clone().lock == tid && existing_lock.clone().value == value 238 | } 239 | None => true, 240 | }; 241 | 242 | if grant_lock { 243 | let lock = Lock { lock: tid, value }; 244 | lock_table.insert(key, lock); 245 | 246 | let reply = PrepareReply { 247 | lock_acquired: true, 248 | healthy_servers, 249 | }; 250 | Ok(Response::new(reply)) 251 | } else { 252 | let reply = PrepareReply { 253 | lock_acquired: false, 254 | healthy_servers, 255 | }; 256 | Ok(Response::new(reply)) 257 | } 258 | } 259 | 260 | // Commit transacation to storage 261 | async fn commit(&self, request: Request) -> Result, Status> { 262 | self.assert_ready().await?; 263 | let request = request.into_inner(); 264 | // println!("Got a commit request: {:?}", request.clone()); 265 | let tid = request.tid; 266 | let key = request.key; 267 | 268 | let mut lock_table = self.lock_table.write().await; 269 | let mut storage = self.storage.write().await; 270 | 271 | // insert into kv store 272 | let new_value = lock_table.get(&key).unwrap().clone().value; 273 | storage.insert(key.clone(), new_value); 274 | 275 | // free lock 276 | lock_table.remove(&key); 277 | 278 | let reply = CommitReply { success: true }; 279 | Ok(Response::new(reply)) 280 | } 281 | 282 | // Abort new transaction 283 | async fn abort(&self, request: Request) -> Result, Status> { 284 | self.assert_ready().await?; 285 | let request = request.into_inner(); 286 | println!("Got an abort request: {:?}", request.clone()); 287 | let tid = request.tid; 288 | let key = request.key; 289 | 290 | let mut lock_table = self.lock_table.write().await; 291 | let can_abort = match lock_table.get(&key) { 292 | Some(existing_lock) => existing_lock.clone().lock == tid, 293 | None => false, 294 | }; 295 | 296 | if can_abort { 297 | lock_table.remove(&key); 298 | let reply = AbortReply { success: true }; 299 | Ok(Response::new(reply)) 300 | } else { 301 | let reply = AbortReply { success: false }; 302 | Ok(Response::new(reply)) 303 | } 304 | } 305 | } 306 | 307 | pub async fn start_server( 308 | id: usize, 309 | addr: String, 310 | servers: Vec, 311 | heartbeat_timeout_ms: usize, 312 | recover: bool, 313 | ) -> Result<(), StdError> { 314 | let addr = addr.parse().unwrap(); 315 | let service = EccRpcService::new(id, servers, recover).await?; 316 | println!("Starting ecc cache node at {:?}", addr); 317 | 318 | let service = Arc::new(service); 319 | 320 | let server_future = Server::builder() 321 | .add_service(EccRpcServer::new(service.clone())) 322 | .serve(addr); 323 | 324 | let handle = Handle::current(); 325 | handle.spawn(async move { 326 | loop { 327 | sleep(Duration::from_millis(heartbeat_timeout_ms as u64)).await; 328 | 329 | let state = service.get_state().await; 330 | println!("{:?}", state); 331 | if state == State::Ready { 332 | service.get_cluster_status().await; 333 | } else { 334 | service.recover().await; 335 | } 336 | } 337 | }); 338 | 339 | server_future.await?; 340 | Ok(()) 341 | } 342 | 343 | pub async fn start_many_servers( 344 | servers: Vec, 345 | heartbeat_timeout_ms: usize, 346 | ) -> Result<(), StdError> { 347 | let mut futures = Vec::new(); 348 | for (id, addr) in servers.clone().into_iter().enumerate() { 349 | let future = start_server(id, addr, servers.clone(), heartbeat_timeout_ms, false); 350 | futures.push(future); 351 | } 352 | join_all(futures).await; 353 | 354 | Ok(()) 355 | } 356 | -------------------------------------------------------------------------------- /src/ecc/client.rs: -------------------------------------------------------------------------------- 1 | use crate::ecc::{get_ecc_settings, StdError}; 2 | use ecc_proto::ecc_rpc_client::EccRpcClient; 3 | use ecc_proto::*; 4 | use futures::future::join_all; 5 | use futures::prelude::*; 6 | use futures::stream::FuturesUnordered; 7 | use reed_solomon_erasure::galois_8::ReedSolomon; 8 | use simple_error::bail; 9 | use std::collections::HashMap; 10 | use std::collections::HashSet; 11 | use std::env; 12 | use std::str; 13 | use std::time::Duration; 14 | use tokio::sync::RwLock; 15 | use tokio::time::sleep; 16 | use tonic::transport::Channel; 17 | use tonic::Request; 18 | use uuid::Uuid; 19 | 20 | pub mod ecc_proto { 21 | tonic::include_proto!("ecc_proto"); 22 | } 23 | 24 | pub struct EccClient { 25 | k: usize, 26 | n: usize, 27 | block_size: usize, 28 | message_size: usize, 29 | codeword_size: usize, 30 | ecc: reed_solomon_erasure::ReedSolomon, 31 | servers: Vec, 32 | index_table: HashMap, 33 | client_table: RwLock>>>, 34 | } 35 | 36 | impl EccClient { 37 | pub async fn new() -> EccClient { 38 | let (k, n, heartbeat_timeout_ms, block_size, servers) = get_ecc_settings(); 39 | let mut client_table = HashMap::new(); 40 | let mut index_table = HashMap::new(); 41 | for (i, addr) in servers.clone().into_iter().enumerate() { 42 | client_table.insert(addr.clone(), None); 43 | index_table.insert(addr, i); 44 | } 45 | let client_table = RwLock::new(client_table); 46 | let message_size = k * block_size; 47 | let codeword_size = n * block_size; 48 | let ecc = ReedSolomon::new(k, n - k).unwrap(); 49 | EccClient { 50 | k, 51 | n, 52 | block_size, 53 | message_size, 54 | ecc, 55 | codeword_size, 56 | client_table, 57 | index_table, 58 | servers, 59 | } 60 | } 61 | 62 | async fn get_client(&self, addr: String) -> Option> { 63 | // If dead try to reconnect 64 | let mut client_table = self.client_table.write().await; 65 | match client_table.get(&addr) { 66 | Some(client_option) => match client_option { 67 | Some(client) => Some(client.clone()), 68 | None => { 69 | let address = match env::var_os("DOCKER_HOSTNAME") { 70 | Some(hostname) => format!( 71 | "http://{}", 72 | addr.replace("0.0.0.0", &hostname.into_string().unwrap()) 73 | ), 74 | None => format!("http://{}", addr), 75 | }; 76 | 77 | let client_option = EccRpcClient::connect(address).await; 78 | match client_option { 79 | Ok(client) => { 80 | client_table.insert(addr.clone(), Some(client.clone())); 81 | Some(client) 82 | } 83 | _ => None, 84 | } 85 | } 86 | }, 87 | None => None, 88 | } 89 | } 90 | 91 | async fn get_once( 92 | &self, 93 | address: String, 94 | key: String, 95 | ) -> Result<(String, Option), StdError> { 96 | let client = self.get_client(address.clone()).await; 97 | 98 | match client { 99 | Some(mut client) => { 100 | let request = Request::new(GetRequest { key: key.clone() }); 101 | let response = client.get(request).await?; 102 | let value = response.into_inner().value; 103 | println!("Get {:?} {:?} {:?}", address, key, value); 104 | Ok((address, value)) 105 | } 106 | _ => { 107 | sleep(Duration::from_secs(5)).await; 108 | bail!("Ignoring {:?} as client could not connect", address) 109 | } 110 | } 111 | } 112 | 113 | pub async fn get_codeword( 114 | &self, 115 | key: String, 116 | exclude_servers: HashSet, 117 | ) -> Result>>, StdError> { 118 | // Get first k responses 119 | let mut futures = Vec::new(); 120 | for addr in self.servers.clone() { 121 | if !exclude_servers.contains(&addr) { 122 | let future = self.get_once(addr.clone(), key.clone()); 123 | futures.push(future); 124 | } 125 | } 126 | let futures = futures.into_iter().collect::>(); 127 | let first_k = futures.take(self.k).collect::>().await; 128 | 129 | // Empty codeword 130 | let mut codeword: Vec>> = vec![None; self.n]; 131 | println!("First k {:?}", first_k); 132 | 133 | // Fill in codeword 134 | let mut all_none = true; 135 | for response in first_k { 136 | match response { 137 | Ok((addr, result)) => { 138 | // Check if result is none 139 | all_none = all_none && result.is_none(); 140 | // Get server index 141 | let i = self.index_table.get(&addr).unwrap().clone(); 142 | println!("Response {:?}", result); 143 | // String to vec of u8 144 | let result: Option> = result.map(|x| serde_json::from_str(&x).unwrap()); 145 | codeword[i] = result; 146 | } 147 | _ => bail!("Didn't get k responses..."), 148 | } 149 | } 150 | 151 | if all_none { 152 | return Ok(None); 153 | } 154 | 155 | // Reconstruct message 156 | self.ecc.reconstruct(&mut codeword).unwrap(); 157 | let codeword: Vec<_> = codeword.into_iter().map(|x| x.unwrap()).collect(); 158 | Ok(Some(codeword)) 159 | } 160 | 161 | pub async fn get(&self, key: String) -> Result, StdError> { 162 | match self.get_codeword(key, HashSet::new()).await? { 163 | Some(codeword) => { 164 | // Process into string 165 | let flattened: Vec = codeword.into_iter().flatten().collect(); 166 | let mut flattened: Vec = (&flattened[..self.message_size]).to_vec(); 167 | // pop padding 168 | while let Some(0) = flattened.last() { 169 | flattened.pop(); 170 | } 171 | let message = str::from_utf8(&flattened).unwrap(); 172 | Ok(Some(message.to_string())) 173 | } 174 | None => Ok(None), 175 | } 176 | } 177 | 178 | pub async fn get_keys_once(&self, address: String) -> Result>, StdError> { 179 | let client = self.get_client(address.clone()).await; 180 | match client { 181 | Some(mut client) => { 182 | let request = Request::new(GetKeysRequest {}); 183 | let response = client.get_keys(request).await?.into_inner(); 184 | let keys = response.keys; 185 | 186 | match keys { 187 | Some(keys) => { 188 | let keys: Vec = serde_json::from_str(&keys).unwrap(); 189 | Ok(Some(keys)) 190 | } 191 | None => Ok(None), 192 | } 193 | } 194 | None => Ok(None), 195 | } 196 | } 197 | 198 | async fn send_heartbeat(&self, addr: String) -> (String, String) { 199 | let client = self.get_client(addr.clone()).await; 200 | 201 | match client { 202 | Some(mut client) => { 203 | let request = Request::new(HeartbeatRequest {}); 204 | let response = client.heartbeat(request).await; 205 | match response { 206 | Ok(response) => { 207 | let state = response.into_inner().state; 208 | (state, addr) 209 | } 210 | Err(_) => ("NotReady".to_string(), addr), 211 | } 212 | } 213 | _ => ("NotReady".to_string(), addr), 214 | } 215 | } 216 | 217 | pub async fn send_heartbeats(&mut self) -> HashSet { 218 | let mut futures = Vec::new(); 219 | for addr in self.servers.clone() { 220 | let future = self.send_heartbeat(addr); 221 | futures.push(future) 222 | } 223 | let res = join_all(futures).await; 224 | let mut healthy_servers = HashSet::new(); 225 | for (state, addr) in res { 226 | if state == "Ready" { 227 | healthy_servers.insert(addr); 228 | } 229 | } 230 | healthy_servers 231 | } 232 | 233 | pub async fn two_phase_commit(&self, key: String, value: String) -> Result<(), StdError> { 234 | let tid = Uuid::new_v4().to_string(); 235 | println!("Beginning 2pc for {:?} {:?} {:?}", tid, key, value); 236 | let mut bytes = value.into_bytes(); 237 | 238 | // too long ... 239 | if bytes.len() > self.message_size.into() { 240 | bail!( 241 | "message too long, {:?} larger then {:?}", 242 | bytes.len(), 243 | self.message_size 244 | ); 245 | } else { 246 | // pad with zeros 247 | let pad_size = self.codeword_size - bytes.len(); 248 | let mut pad = vec![0; pad_size]; 249 | bytes.append(&mut pad); 250 | 251 | // chunk into vec of vecs 252 | let mut codeword: Vec> = bytes.chunks(self.block_size).map(|x| x.to_vec()).collect(); 253 | 254 | // calculate parity 255 | self.ecc.encode(&mut codeword).unwrap(); 256 | 257 | // map to strings 258 | let codeword: Vec = codeword 259 | .into_iter() 260 | .map(|x| serde_json::to_string(&x).unwrap()) 261 | .collect(); 262 | 263 | println!("Constructed codeword {:?}", codeword); 264 | 265 | // Prepare, try to acquire H >= k locks, make sure the healthy server clique is agreed upon 266 | let mut futures = Vec::new(); 267 | for (i, addr) in self.servers.iter().enumerate() { 268 | let future = self.send_prepare(addr.clone(), tid.clone(), key.clone(), codeword[i].clone()); 269 | futures.push(future) 270 | } 271 | let res = join_all(futures).await; 272 | 273 | println!("Waited for locks {:?}", res); 274 | 275 | // Clique matches and all locks acquired 276 | // Acquired >= k locks 277 | let mut clique = true; 278 | let mut all_healthy_servers: HashSet = HashSet::new(); 279 | let mut first_healthy_servers = true; 280 | let mut all_locks_acquired = true; 281 | let mut num_locks_acquired = 0; 282 | for i in res { 283 | match i { 284 | Err(_) => (), 285 | Ok((healthy_servers, locks_acquired)) => { 286 | if first_healthy_servers { 287 | all_healthy_servers = healthy_servers.clone(); 288 | first_healthy_servers = false; 289 | } 290 | 291 | clique = clique && (all_healthy_servers == healthy_servers); 292 | all_locks_acquired = all_locks_acquired && locks_acquired; 293 | if locks_acquired { 294 | num_locks_acquired += 1 295 | }; 296 | } 297 | } 298 | } 299 | 300 | let go_commit = clique && num_locks_acquired >= self.k && all_locks_acquired; 301 | println!("Can commit {:?}", go_commit); 302 | 303 | if go_commit { 304 | let mut futures = Vec::new(); 305 | for addr in self.servers.clone() { 306 | let future = self.send_commit(addr.clone(), tid.clone(), key.clone()); 307 | futures.push(future); 308 | } 309 | let res = join_all(futures).await; 310 | println!("COMMITED {:?}", res); 311 | } else { 312 | let mut futures = Vec::new(); 313 | for addr in self.servers.clone() { 314 | let future = self.send_abort(addr.clone(), tid.clone(), key.clone()); 315 | futures.push(future); 316 | } 317 | let res = join_all(futures).await; 318 | println!("ABORTED {:?}", res); 319 | } 320 | Ok(()) 321 | } 322 | } 323 | 324 | async fn send_prepare( 325 | &self, 326 | addr: String, 327 | tid: String, 328 | key: String, 329 | value: String, 330 | ) -> Result<(HashSet, bool), StdError> { 331 | let client = self.get_client(addr.clone()).await; 332 | match client { 333 | Some(mut client) => { 334 | let request = Request::new(PrepareRequest { tid, key, value }); 335 | let response = client.prepare(request).await?.into_inner(); 336 | let lock_acquired = response.lock_acquired; 337 | let healthy_servers = response.healthy_servers; 338 | let healthy_servers: HashSet = serde_json::from_str(&healthy_servers).unwrap(); 339 | Ok((healthy_servers, lock_acquired)) 340 | } 341 | _ => { 342 | bail!("Ignoring {:?} as client could not connect", addr) 343 | } 344 | } 345 | } 346 | 347 | async fn send_commit(&self, addr: String, tid: String, key: String) -> Result { 348 | let client = self.get_client(addr.clone()).await; 349 | match client { 350 | Some(mut client) => { 351 | let request = Request::new(CommitRequest { key, tid }); 352 | let response = client.commit(request).await?.into_inner(); 353 | let success = response.success; 354 | Ok(success) 355 | } 356 | _ => { 357 | bail!("Ignoring {:?} as client could not connect", addr) 358 | } 359 | } 360 | } 361 | 362 | async fn send_abort(&self, addr: String, tid: String, key: String) -> Result { 363 | let client = self.get_client(addr.clone()).await; 364 | match client { 365 | Some(mut client) => { 366 | let request = Request::new(AbortRequest { key, tid }); 367 | let response = client.abort(request).await?.into_inner(); 368 | let success = response.success; 369 | Ok(success) 370 | } 371 | _ => { 372 | bail!("Ignoring {:?} as client could not connect", addr) 373 | } 374 | } 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /src/raft/storage.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use async_raft::async_trait::async_trait; 3 | use async_raft::raft::{Entry, EntryPayload, MembershipConfig}; 4 | use async_raft::storage::{CurrentSnapshotData, HardState, InitialState}; 5 | use async_raft::{AppData, AppDataResponse, NodeId, RaftStorage}; 6 | use serde::{Deserialize, Serialize}; 7 | use std::collections::{BTreeMap, HashMap}; 8 | use std::io::Cursor; 9 | use thiserror::Error; 10 | use tokio::sync::RwLock; 11 | use tokio::sync::{RwLockReadGuard, RwLockWriteGuard}; 12 | 13 | const ERR_INCONSISTENT_LOG: &str = 14 | "a query was received which was expecting data to be in place which does not exist in the log"; 15 | 16 | /// returning the previously recorded status. 17 | #[derive(Serialize, Deserialize, Debug, Clone)] 18 | pub struct ClientRequest { 19 | pub key: String, 20 | pub value: String, 21 | } 22 | 23 | impl AppData for ClientRequest {} 24 | 25 | /// The application data response type which the `MemStore` works with. 26 | #[derive(Serialize, Deserialize, Debug, Clone)] 27 | pub struct ClientResponse(Option); 28 | 29 | impl AppDataResponse for ClientResponse {} 30 | 31 | /// Error used to trigger Raft shutdown from storage. 32 | #[allow(dead_code)] 33 | #[derive(Clone, Debug, Error)] 34 | pub enum ShutdownError { 35 | #[error("unsafe storage error")] 36 | UnsafeStorageError, 37 | } 38 | 39 | /// The application snapshot type which the `MemStore` works with. 40 | #[derive(Serialize, Deserialize, Debug, Clone)] 41 | pub struct MemStoreSnapshot { 42 | /// The last index covered by this snapshot. 43 | pub index: u64, 44 | /// The term of the last index covered by this snapshot. 45 | pub term: u64, 46 | /// The last memberhsip config included in this snapshot. 47 | pub membership: MembershipConfig, 48 | /// The data of the state machine at the time of this snapshot. 49 | pub data: Vec, 50 | } 51 | 52 | /// The state machine of the `MemStore`. 53 | #[derive(Serialize, Deserialize, Debug, Default, Clone)] 54 | pub struct MemStoreStateMachine { 55 | pub last_applied_log: u64, 56 | /// A mapping of client IDs to their state info. 57 | // pub client_serial_responses: HashMap)>, 58 | /// The current status of a client by ID. 59 | pub kv_store: HashMap, 60 | } 61 | 62 | /// An in-memory storage system implementing the `async_raft::RaftStorage` trait. 63 | pub struct MemStore { 64 | /// The ID of the Raft node for which this memory storage instances is configured. 65 | id: NodeId, 66 | /// The Raft log. 67 | log: RwLock>>, 68 | /// The Raft state machine. 69 | sm: RwLock, 70 | /// The current hard state. 71 | hs: RwLock>, 72 | /// The current snapshot. 73 | current_snapshot: RwLock>, 74 | } 75 | 76 | impl MemStore { 77 | /// Create a new `MemStore` instance. 78 | pub fn new(id: NodeId) -> Self { 79 | let log = RwLock::new(BTreeMap::new()); 80 | let sm = RwLock::new(MemStoreStateMachine::default()); 81 | let hs = RwLock::new(None); 82 | let current_snapshot = RwLock::new(None); 83 | Self { 84 | id, 85 | log, 86 | sm, 87 | hs, 88 | current_snapshot, 89 | } 90 | } 91 | 92 | pub async fn read_state_machine(&self) -> MemStoreStateMachine { 93 | let state_machine = self.sm.read().await; 94 | let result = state_machine.clone(); 95 | result 96 | } 97 | 98 | /// Create a new `MemStore` instance with some existing state (for testing). 99 | #[cfg(test)] 100 | #[allow(dead_code)] 101 | pub fn new_with_state( 102 | id: NodeId, 103 | log: BTreeMap>, 104 | sm: MemStoreStateMachine, 105 | hs: Option, 106 | current_snapshot: Option, 107 | ) -> Self { 108 | let log = RwLock::new(log); 109 | let sm = RwLock::new(sm); 110 | let hs = RwLock::new(hs); 111 | let current_snapshot = RwLock::new(current_snapshot); 112 | Self { 113 | id, 114 | log, 115 | sm, 116 | hs, 117 | current_snapshot, 118 | } 119 | } 120 | // Get a handle to the log for testing purposes. 121 | #[allow(dead_code)] 122 | pub async fn get_log(&self) -> RwLockWriteGuard<'_, BTreeMap>> { 123 | self.log.write().await 124 | } 125 | /// Get a handle to the state machine for testing purposes. 126 | #[allow(dead_code)] 127 | pub async fn get_state_machine(&self) -> RwLockWriteGuard<'_, MemStoreStateMachine> { 128 | self.sm.write().await 129 | } 130 | /// Get a handle to the current hard state for testing purposes. 131 | #[allow(dead_code)] 132 | pub async fn read_hard_state(&self) -> RwLockReadGuard<'_, Option> { 133 | self.hs.read().await 134 | } 135 | } 136 | 137 | #[async_trait] 138 | impl RaftStorage for MemStore { 139 | type Snapshot = Cursor>; 140 | type ShutdownError = ShutdownError; 141 | 142 | #[tracing::instrument(level = "trace", skip(self))] 143 | async fn get_membership_config(&self) -> Result { 144 | let log = self.log.read().await; 145 | let cfg_opt = log.values().rev().find_map(|entry| match &entry.payload { 146 | EntryPayload::ConfigChange(cfg) => Some(cfg.membership.clone()), 147 | EntryPayload::SnapshotPointer(snap) => Some(snap.membership.clone()), 148 | _ => None, 149 | }); 150 | Ok(match cfg_opt { 151 | Some(cfg) => cfg, 152 | None => MembershipConfig::new_initial(self.id), 153 | }) 154 | } 155 | 156 | #[tracing::instrument(level = "trace", skip(self))] 157 | async fn get_initial_state(&self) -> Result { 158 | let membership = self.get_membership_config().await?; 159 | let mut hs = self.hs.write().await; 160 | let log = self.log.read().await; 161 | let sm = self.sm.read().await; 162 | match &mut *hs { 163 | Some(inner) => { 164 | let (last_log_index, last_log_term) = match log.values().rev().next() { 165 | Some(log) => (log.index, log.term), 166 | None => (0, 0), 167 | }; 168 | let last_applied_log = sm.last_applied_log; 169 | Ok(InitialState { 170 | last_log_index, 171 | last_log_term, 172 | last_applied_log, 173 | hard_state: inner.clone(), 174 | membership, 175 | }) 176 | } 177 | None => { 178 | let new = InitialState::new_initial(self.id); 179 | *hs = Some(new.hard_state.clone()); 180 | Ok(new) 181 | } 182 | } 183 | } 184 | 185 | #[tracing::instrument(level = "trace", skip(self, hs))] 186 | async fn save_hard_state(&self, hs: &HardState) -> Result<()> { 187 | *self.hs.write().await = Some(hs.clone()); 188 | Ok(()) 189 | } 190 | 191 | #[tracing::instrument(level = "trace", skip(self))] 192 | async fn get_log_entries(&self, start: u64, stop: u64) -> Result>> { 193 | // Invalid request, return empty vec. 194 | if start > stop { 195 | tracing::error!("invalid request, start > stop"); 196 | return Ok(vec![]); 197 | } 198 | let log = self.log.read().await; 199 | Ok(log.range(start..stop).map(|(_, val)| val.clone()).collect()) 200 | } 201 | 202 | #[tracing::instrument(level = "trace", skip(self))] 203 | async fn delete_logs_from(&self, start: u64, stop: Option) -> Result<()> { 204 | if stop.as_ref().map(|stop| &start > stop).unwrap_or(false) { 205 | tracing::error!("invalid request, start > stop"); 206 | return Ok(()); 207 | } 208 | let mut log = self.log.write().await; 209 | 210 | // If a stop point was specified, delete from start until the given stop point. 211 | if let Some(stop) = stop.as_ref() { 212 | for key in start..*stop { 213 | log.remove(&key); 214 | } 215 | return Ok(()); 216 | } 217 | // Else, just split off the remainder. 218 | log.split_off(&start); 219 | Ok(()) 220 | } 221 | 222 | #[tracing::instrument(level = "trace", skip(self, entry))] 223 | async fn append_entry_to_log(&self, entry: &Entry) -> Result<()> { 224 | let mut log = self.log.write().await; 225 | log.insert(entry.index, entry.clone()); 226 | println!("Appending to log: {:?}", entry.clone()); 227 | Ok(()) 228 | } 229 | 230 | #[tracing::instrument(level = "trace", skip(self, entries))] 231 | async fn replicate_to_log(&self, entries: &[Entry]) -> Result<()> { 232 | let mut log = self.log.write().await; 233 | println!("Replicating to log: {:?}", entries.clone()); 234 | for entry in entries { 235 | log.insert(entry.index, entry.clone()); 236 | } 237 | Ok(()) 238 | } 239 | 240 | #[tracing::instrument(level = "trace", skip(self, data))] 241 | async fn apply_entry_to_state_machine( 242 | &self, 243 | index: &u64, 244 | data: &ClientRequest, 245 | ) -> Result { 246 | let mut sm = self.sm.write().await; 247 | sm.last_applied_log = *index; 248 | let previous = sm.kv_store.insert(data.key.clone(), data.value.clone()); 249 | Ok(ClientResponse(previous)) 250 | } 251 | 252 | #[tracing::instrument(level = "trace", skip(self, entries))] 253 | async fn replicate_to_state_machine(&self, entries: &[(&u64, &ClientRequest)]) -> Result<()> { 254 | let mut sm = self.sm.write().await; 255 | for (index, data) in entries { 256 | sm.last_applied_log = **index; 257 | sm.kv_store.insert(data.key.clone(), data.value.clone()); 258 | } 259 | Ok(()) 260 | } 261 | 262 | #[tracing::instrument(level = "trace", skip(self))] 263 | async fn do_log_compaction(&self) -> Result> { 264 | let (data, last_applied_log); 265 | { 266 | // Serialize the data of the state machine. 267 | let sm = self.sm.read().await; 268 | data = serde_json::to_vec(&*sm)?; 269 | last_applied_log = sm.last_applied_log; 270 | } // Release state machine read lock. 271 | 272 | let membership_config; 273 | { 274 | // Go backwards through the log to find the most recent membership config <= the `through` index. 275 | let log = self.log.read().await; 276 | membership_config = log 277 | .values() 278 | .rev() 279 | .skip_while(|entry| entry.index > last_applied_log) 280 | .find_map(|entry| match &entry.payload { 281 | EntryPayload::ConfigChange(cfg) => Some(cfg.membership.clone()), 282 | _ => None, 283 | }) 284 | .unwrap_or_else(|| MembershipConfig::new_initial(self.id)); 285 | } // Release log read lock. 286 | 287 | let snapshot_bytes: Vec; 288 | let term; 289 | { 290 | let mut log = self.log.write().await; 291 | let mut current_snapshot = self.current_snapshot.write().await; 292 | term = log 293 | .get(&last_applied_log) 294 | .map(|entry| entry.term) 295 | .ok_or_else(|| anyhow::anyhow!(ERR_INCONSISTENT_LOG))?; 296 | *log = log.split_off(&last_applied_log); 297 | log.insert( 298 | last_applied_log, 299 | Entry::new_snapshot_pointer(last_applied_log, term, "".into(), membership_config.clone()), 300 | ); 301 | 302 | let snapshot = MemStoreSnapshot { 303 | index: last_applied_log, 304 | term, 305 | membership: membership_config.clone(), 306 | data, 307 | }; 308 | snapshot_bytes = serde_json::to_vec(&snapshot)?; 309 | *current_snapshot = Some(snapshot); 310 | } // Release log & snapshot write locks. 311 | 312 | tracing::trace!( 313 | { snapshot_size = snapshot_bytes.len() }, 314 | "log compaction complete" 315 | ); 316 | Ok(CurrentSnapshotData { 317 | term, 318 | index: last_applied_log, 319 | membership: membership_config.clone(), 320 | snapshot: Box::new(Cursor::new(snapshot_bytes)), 321 | }) 322 | } 323 | 324 | #[tracing::instrument(level = "trace", skip(self))] 325 | async fn create_snapshot(&self) -> Result<(String, Box)> { 326 | Ok((String::from(""), Box::new(Cursor::new(Vec::new())))) // Snapshot IDs are insignificant to this storage engine. 327 | } 328 | 329 | #[tracing::instrument(level = "trace", skip(self, snapshot))] 330 | async fn finalize_snapshot_installation( 331 | &self, 332 | index: u64, 333 | term: u64, 334 | delete_through: Option, 335 | id: String, 336 | snapshot: Box, 337 | ) -> Result<()> { 338 | tracing::trace!( 339 | { snapshot_size = snapshot.get_ref().len() }, 340 | "decoding snapshot for installation" 341 | ); 342 | let raw = serde_json::to_string_pretty(snapshot.get_ref().as_slice())?; 343 | println!("JSON SNAP:\n{}", raw); 344 | let new_snapshot: MemStoreSnapshot = serde_json::from_slice(snapshot.get_ref().as_slice())?; 345 | // Update log. 346 | { 347 | // Go backwards through the log to find the most recent membership config <= the `through` index. 348 | let mut log = self.log.write().await; 349 | let membership_config = log 350 | .values() 351 | .rev() 352 | .skip_while(|entry| entry.index > index) 353 | .find_map(|entry| match &entry.payload { 354 | EntryPayload::ConfigChange(cfg) => Some(cfg.membership.clone()), 355 | _ => None, 356 | }) 357 | .unwrap_or_else(|| MembershipConfig::new_initial(self.id)); 358 | 359 | match &delete_through { 360 | Some(through) => { 361 | *log = log.split_off(&(through + 1)); 362 | } 363 | None => log.clear(), 364 | } 365 | log.insert( 366 | index, 367 | Entry::new_snapshot_pointer(index, term, id, membership_config), 368 | ); 369 | } 370 | 371 | // Update the state machine. 372 | { 373 | let new_sm: MemStoreStateMachine = serde_json::from_slice(&new_snapshot.data)?; 374 | let mut sm = self.sm.write().await; 375 | *sm = new_sm; 376 | } 377 | 378 | // Update current snapshot. 379 | let mut current_snapshot = self.current_snapshot.write().await; 380 | *current_snapshot = Some(new_snapshot); 381 | Ok(()) 382 | } 383 | 384 | #[tracing::instrument(level = "trace", skip(self))] 385 | async fn get_current_snapshot(&self) -> Result>> { 386 | match &*self.current_snapshot.read().await { 387 | Some(snapshot) => { 388 | let reader = serde_json::to_vec(&snapshot)?; 389 | Ok(Some(CurrentSnapshotData { 390 | index: snapshot.index, 391 | term: snapshot.term, 392 | membership: snapshot.membership.clone(), 393 | snapshot: Box::new(Cursor::new(reader)), 394 | })) 395 | } 396 | None => Ok(None), 397 | } 398 | } 399 | } 400 | --------------------------------------------------------------------------------