├── .gitignore ├── art ├── bob.png ├── alice.png ├── amanita.png ├── banner.png ├── footer.png ├── george.png ├── phone.png ├── tablet.png ├── explain_rm.png ├── faded_cube.png ├── small_george.png ├── solid_cube.png ├── explain_replay.png ├── key_hierarchy.png ├── george_back_view.png ├── network_barrier.png ├── explain_replication.png ├── george.svg ├── amanita.svg ├── george_back_view.svg ├── explain_replication.svg └── explain_replay.svg ├── hermitdb_presentation.pdf ├── src ├── lib.rs ├── log.rs ├── db.rs ├── memory_log.rs ├── error.rs ├── encrypted_git_log.rs ├── crypto.rs ├── data.rs ├── map.rs └── git_log.rs ├── Cargo.toml ├── README.md ├── tests ├── db.rs ├── log_replication.rs └── map.rs └── benches └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target/ 3 | **/*.rs.bk 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /art/bob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/bob.png -------------------------------------------------------------------------------- /art/alice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/alice.png -------------------------------------------------------------------------------- /art/amanita.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/amanita.png -------------------------------------------------------------------------------- /art/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/banner.png -------------------------------------------------------------------------------- /art/footer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/footer.png -------------------------------------------------------------------------------- /art/george.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/george.png -------------------------------------------------------------------------------- /art/phone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/phone.png -------------------------------------------------------------------------------- /art/tablet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/tablet.png -------------------------------------------------------------------------------- /art/explain_rm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/explain_rm.png -------------------------------------------------------------------------------- /art/faded_cube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/faded_cube.png -------------------------------------------------------------------------------- /art/small_george.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/small_george.png -------------------------------------------------------------------------------- /art/solid_cube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/solid_cube.png -------------------------------------------------------------------------------- /art/explain_replay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/explain_replay.png -------------------------------------------------------------------------------- /art/key_hierarchy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/key_hierarchy.png -------------------------------------------------------------------------------- /art/george_back_view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/george_back_view.png -------------------------------------------------------------------------------- /art/network_barrier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/network_barrier.png -------------------------------------------------------------------------------- /hermitdb_presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/hermitdb_presentation.pdf -------------------------------------------------------------------------------- /art/explain_replication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hermits-grove/hermitdb/HEAD/art/explain_replication.png -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod crypto; 3 | pub mod db; 4 | pub mod map; 5 | pub mod data; 6 | pub mod log; 7 | pub mod memory_log; 8 | pub mod git_log; 9 | pub mod encrypted_git_log; 10 | 11 | pub use crdts; 12 | pub use crate::db::DB; 13 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hermitdb" 3 | version = "0.1.0" 4 | authors = ["David Rusu "] 5 | description = "A private decentralized database replicated over Git (or any other distributed log)" 6 | license="GPL-2.0" 7 | edition = "2018" 8 | 9 | [dependencies] 10 | git2 = "0.10.1" 11 | ring = "0.16.11" 12 | serde = "1.0.104" 13 | serde_derive = "1.0.104" 14 | 15 | [dependencies.sled] 16 | version = "0.31.0" 17 | # features = ["zstd"] 18 | # path = "../sled/crates/sled" 19 | 20 | [dependencies.crdts] 21 | git = "https://github.com/rust-crdt/rust-crdt" 22 | rev = "3cbdd3aeb0ceaac4e08fe541a822eea2eea93c3f" 23 | 24 | [dependencies.bincode] 25 | version = "1.2.1" 26 | features = ["i128"] 27 | 28 | [dev-dependencies] 29 | assert_matches = "1.3.0" 30 | quickcheck = "0.9.2" 31 | tempfile = "3.1.0" 32 | -------------------------------------------------------------------------------- /src/log.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | 3 | use crdts::{CmRDT, Actor}; 4 | 5 | use crate::error::Result; 6 | 7 | pub trait TaggedOp { 8 | type ID: Eq; 9 | 10 | fn id(&self) -> Self::ID; 11 | fn op(&self) -> &C::Op; 12 | } 13 | 14 | pub trait LogReplicable { 15 | type LoggedOp: Debug + TaggedOp; 16 | type Remote; 17 | 18 | fn next(&self) -> Result>; 19 | fn ack(&mut self, logged_op: &Self::LoggedOp) -> Result<()>; 20 | fn commit(&mut self, op: C::Op) -> Result; 21 | fn pull(&mut self, remote: &Self::Remote) -> Result<()>; 22 | fn push(&self, remote: &mut Self::Remote) -> Result<()>; 23 | 24 | fn sync(&mut self, remote: &mut Self::Remote) -> Result<()> { 25 | self.pull(&remote)?; 26 | self.push(remote) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/db.rs: -------------------------------------------------------------------------------- 1 | use crdts::{AddCtx, CmRDT, ReadCtx, RmCtx}; 2 | 3 | use crate::data::{Actor, Data, Kind, Op}; 4 | use crate::error::Result; 5 | use crate::log::{LogReplicable, TaggedOp}; 6 | use crate::map; 7 | 8 | pub type Map = map::Map<(String, Kind), Data, Actor>; 9 | pub type Entry = map::Entry; 10 | 11 | pub struct DB> { 12 | log: L, 13 | map: Map, 14 | } 15 | 16 | impl> DB { 17 | pub fn new(log: L, map: Map) -> Self { 18 | DB { log, map } 19 | } 20 | 21 | pub fn get(&self, key: &(String, Kind)) -> Result, Actor>> { 22 | self.map.get(key) 23 | } 24 | 25 | pub fn update( 26 | &mut self, 27 | key: (impl Into, Kind), 28 | ctx: AddCtx, 29 | f: F, 30 | ) -> Result<()> 31 | where 32 | F: FnOnce(&Data, AddCtx) -> O, 33 | O: Into, 34 | { 35 | let (key_str, key_kind) = key; 36 | let key = (key_str.into(), key_kind); 37 | 38 | let map_op = self.map.update(key, ctx, f)?; 39 | let tagged_op = self.log.commit(map_op)?; 40 | self.map.apply(tagged_op.op()); 41 | self.log.ack(&tagged_op) 42 | } 43 | 44 | pub fn rm(&mut self, key: (impl Into, Kind), ctx: RmCtx) -> Result<()> { 45 | let (key_str, key_kind) = key; 46 | let key = (key_str.into(), key_kind); 47 | 48 | let op = self.map.rm(key, ctx); 49 | let tagged_op = self.log.commit(op)?; 50 | self.map.apply(tagged_op.op()); 51 | self.log.ack(&tagged_op) 52 | } 53 | 54 | pub fn iter(&self) -> Result> { 55 | self.map.iter() 56 | } 57 | 58 | pub fn sync(&mut self, remote: &mut L::Remote) -> Result<()> { 59 | self.log.sync(remote)?; 60 | 61 | while let Some(tagged_op) = self.log.next()? { 62 | self.map.apply(tagged_op.op()); 63 | self.log.ack(&tagged_op)?; 64 | } 65 | Ok(()) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # HermitDB 6 | 7 | ###### A private decentralized database replicated over Git (or any other distributed log) 8 | 9 | The replicated log abstraction has popped up in many distributed systems over the years, we see it in Bitcoin as the blockchain, we see it in systems that rely on distributed logs like Kafka, and we see it in Git as the branch commit history. 10 | 11 | HermitDB uses the widespread deployment of these logs to give users the ability to replicate their data using a log that they have access to. 12 | 13 | The motivating idea behind HermitDB is that if you've built a password manager with HermitDB, users of this password manager can effortlesly sync their data across their devices using a git repo they control, meaning they keep control over their data. 14 | 15 | ``` rust 16 | extern crate hermitdb; 17 | 18 | use hermitdb::{sled, memory_log, map, DB}; 19 | 20 | fn main() { 21 | let actor = 32; 22 | let config = sled::ConfigBuilder::new().temporary(true).build(); 23 | let tree = sled::Tree::start(config).unwrap(); 24 | // use an in memory log for testing 25 | let log = memory_log::Log::new(actor); 26 | let map = map::Map::new(tree); 27 | let db = DB::new(log, map); 28 | } 29 | ``` 30 | 31 | ### If you've got some spare time... 32 | 33 | - **crypto** 34 | - **Reduce our reliance on a strong rng** 35 | - If an attacker controls our source of entropy, it increases chance of leak. 36 | - Nonce's are currently generated randomly. Since we have a seperate encryption key per log, and logs are immutable (are they? what if we add compaction?) we should be able to use a counter on log entries as a nonce. 37 | - **reduce our reliance on a strong password.** The users password is the weakest link in our crypto system. To improve security, we can look into adding an `entropy file`: a randomly generated file that is not synced through hermitdb. This would be similar to Keepass's composite key, the contents of the entropy file would be taken as input to the kdf giving us a lower bound of `len()` bits of entropy (assuming a strong rng was used to generate the entropy file). 38 | - **compressing op's in the log** 39 | - Look into zstd (we already have zstd as a dependency from sled). 40 | - **log compaction** 41 | - 1000 edits => 1000 log entries => 1000 commits (in the current git_log implementation). 42 | - Can we compact this log *and* preserve causality? 43 | - can we make `Op`'s themselves CRDT's? `let compacted_op = merge(op1, op2)` 44 | 45 | 46 | 47 | ### Prior Art 48 | 49 | - https://github.com/ff-notes/ff - a distributed notes app built with CRDT's + Git 50 | 51 |

52 | 53 |

54 | -------------------------------------------------------------------------------- /src/memory_log.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | use std::fmt::{self, Debug}; 3 | 4 | use crdts::{Actor, CmRDT}; 5 | 6 | use crate::error::Result; 7 | use crate::log::{LogReplicable, TaggedOp}; 8 | 9 | pub struct Log { 10 | actor: A, 11 | logs: BTreeMap)>, 12 | } 13 | 14 | pub struct LoggedOp { 15 | actor: A, 16 | index: u64, 17 | op: C::Op, 18 | } 19 | 20 | impl Debug for LoggedOp { 21 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 22 | write!( 23 | f, 24 | "LoggedOp {{ actor: {:?}, index: {:?}, op: {:?} }}", 25 | self.actor, self.index, self.op 26 | ) 27 | } 28 | } 29 | 30 | impl TaggedOp for LoggedOp { 31 | type ID = (A, u64); 32 | 33 | fn id(&self) -> Self::ID { 34 | (self.actor.clone(), self.index) 35 | } 36 | 37 | fn op(&self) -> &C::Op { 38 | &self.op 39 | } 40 | } 41 | 42 | impl LogReplicable for Log { 43 | type LoggedOp = LoggedOp; 44 | type Remote = Self; 45 | 46 | fn next(&self) -> Result> { 47 | let largest_lag = self 48 | .logs 49 | .iter() 50 | .max_by_key(|(_, (index, log))| (log.len() as u64) - *index); 51 | 52 | if let Some((actor, (index, log))) = largest_lag { 53 | if *index >= log.len() as u64 { 54 | Ok(None) 55 | } else { 56 | Ok(Some(LoggedOp { 57 | actor: actor.clone(), 58 | index: *index, 59 | op: log[*index as usize].clone(), 60 | })) 61 | } 62 | } else { 63 | Ok(None) 64 | } 65 | } 66 | 67 | fn ack(&mut self, logged_op: &Self::LoggedOp) -> Result<()> { 68 | // We can ack ops that are not present in the log 69 | 70 | let (actor, index) = logged_op.id(); 71 | 72 | let log = self.logs.entry(actor).or_insert_with(|| (0, Vec::new())); 73 | 74 | log.0 = index + 1; 75 | Ok(()) 76 | } 77 | 78 | fn commit(&mut self, op: C::Op) -> Result { 79 | let log = self 80 | .logs 81 | .entry(self.actor.clone()) 82 | .or_insert_with(|| (0, Vec::new())); 83 | 84 | log.1.push(op.clone()); 85 | 86 | Ok(LoggedOp { 87 | actor: self.actor.clone(), 88 | index: log.0, 89 | op, 90 | }) 91 | } 92 | 93 | fn pull(&mut self, remote: &Self::Remote) -> Result<()> { 94 | for (actor, (_, log)) in remote.logs.iter() { 95 | let entry = self 96 | .logs 97 | .entry(actor.clone()) 98 | .or_insert_with(|| (0, vec![])); 99 | 100 | if log.len() > entry.1.len() { 101 | for i in (entry.1.len())..log.len() { 102 | entry.1.push(log[i as usize].clone()); 103 | } 104 | } 105 | } 106 | Ok(()) 107 | } 108 | 109 | fn push(&self, remote: &mut Self::Remote) -> Result<()> { 110 | remote.pull(self) 111 | } 112 | } 113 | 114 | impl Log { 115 | pub fn new(actor: A) -> Self { 116 | Log { 117 | actor, 118 | logs: BTreeMap::new(), 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use std::{self, fmt}; 2 | 3 | use crate::data::Kind; 4 | 5 | pub type Result = std::result::Result; 6 | 7 | // TODO: audit usage of these error types, I have a feeling not all of these are used 8 | #[derive(Debug)] 9 | pub enum Error { 10 | UnexpectedKind(Kind, Kind), 11 | BranchNameEncodingError, 12 | BranchIsNotADirectReference, 13 | LogCommitDoesNotContainOp, 14 | Parse(String), 15 | Crypto(String), 16 | State(String), 17 | Bincode(bincode::Error), 18 | CRDT(crdts::Error), 19 | Git(git2::Error), 20 | IO(std::io::Error), 21 | SledGeneric(sled::Error) 22 | } 23 | 24 | impl fmt::Display for Error { 25 | fn fmt(&self, mut f: &mut fmt::Formatter) -> fmt::Result { 26 | match self { 27 | Error::UnexpectedKind(expected, got) => 28 | write!(f, "Unexpected kind! got: {:?}, expected: {:?}", got, expected), 29 | Error::BranchNameEncodingError => 30 | write!(f, "A branch name is not utf8 encoded"), 31 | Error::BranchIsNotADirectReference => 32 | write!(f, "A branch reference isn't a direct ref to an oid"), 33 | Error::LogCommitDoesNotContainOp => 34 | write!(f, "Trees attached to commits in git are expected to have an 'op' entry"), 35 | Error::Parse(s) => 36 | write!(f, "Parsing failed: {}", s), 37 | Error::Crypto(s) => 38 | write!(f, "Crypto failure: {}", s), 39 | Error::State(s) => 40 | write!(f, "Gitdb entered a bad state: {}", s), 41 | Error::Bincode(e) => e.fmt(&mut f), 42 | Error::CRDT(e) => e.fmt(&mut f), 43 | Error::Git(e) => e.fmt(&mut f), 44 | Error::IO(e) => e.fmt(&mut f), 45 | Error::SledGeneric(e) => e.fmt(&mut f) 46 | } 47 | } 48 | } 49 | 50 | impl std::error::Error for Error { 51 | fn description(&self) -> &str { 52 | match self { 53 | Error::UnexpectedKind(_, _) => 54 | "Unexpected kind, were you attempting to convert a Data into something it's not?", 55 | Error::BranchNameEncodingError => "A branch name is not utf8 encoded", 56 | Error::BranchIsNotADirectReference => 57 | "A branch reference isn't a direct ref to an oid", 58 | Error::LogCommitDoesNotContainOp => 59 | "Trees attached to commits in git are expected to have an 'op' entry", 60 | Error::Parse(_) => "Parsing failed", 61 | Error::Crypto(_) => "Crypto failure", 62 | Error::State(_) => "Gitdb entered a bad state", 63 | Error::Bincode(e) => e.description(), 64 | Error::CRDT(e) => e.description(), 65 | Error::Git(e) => e.description(), 66 | Error::IO(e) => e.description(), 67 | Error::SledGeneric(e) => e.description() 68 | } 69 | } 70 | fn cause(&self) -> Option<&dyn std::error::Error> { 71 | match self { 72 | Error::UnexpectedKind(_, _) => None, 73 | Error::BranchNameEncodingError => None, 74 | Error::BranchIsNotADirectReference => None, 75 | Error::LogCommitDoesNotContainOp => None, 76 | Error::Parse(_) => None, 77 | Error::Crypto(_) => None, 78 | Error::State(_) => None, 79 | Error::Bincode(e) => Some(e), 80 | Error::CRDT(e) => Some(e), 81 | Error::Git(e) => Some(e), 82 | Error::IO(e) => Some(e), 83 | Error::SledGeneric(e) => Some(e) 84 | } 85 | } 86 | } 87 | 88 | impl From for Error { 89 | fn from(err: crdts::Error) -> Self { 90 | Error::CRDT(err) 91 | } 92 | } 93 | 94 | impl From for Error { 95 | fn from(err: sled::Error) -> Self { 96 | Error::SledGeneric(err) 97 | } 98 | } 99 | 100 | impl From for Error { 101 | fn from(err: git2::Error) -> Self { 102 | Error::Git(err) 103 | } 104 | } 105 | 106 | impl From for Error { 107 | fn from(err: std::io::Error) -> Self { 108 | Error::IO(err) 109 | } 110 | } 111 | 112 | impl From for Error { 113 | fn from(err: bincode::Error) -> Self { 114 | Error::Bincode(err) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /tests/db.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use sled; 4 | use assert_matches::assert_matches; 5 | use hermitdb::{ 6 | data::{Prim, Data, Kind, Actor}, 7 | crdts, 8 | memory_log, 9 | map, 10 | db, 11 | DB 12 | }; 13 | 14 | fn mk_db(actor: Actor) -> DB> { 15 | let sled = sled::Config::new().temporary(true).open().unwrap(); 16 | DB::new(memory_log::Log::new(actor), map::Map::new(sled)) 17 | } 18 | 19 | #[test] 20 | fn test_write_read_set() { 21 | let mut db = mk_db(1); 22 | 23 | let read_ctx = db.get(&("x".into(), Kind::Set)).unwrap(); 24 | assert_matches!(read_ctx.val, None); 25 | 26 | assert_matches!( 27 | db.update(("x", Kind::Set), read_ctx.derive_add_ctx(1), |data, ctx| { 28 | let set = data.to_set().unwrap(); 29 | set.add(57i64, ctx) 30 | }), 31 | Ok(()) 32 | ); 33 | 34 | assert_eq!( 35 | db.get(&("x".into(), Kind::Set)).unwrap().val 36 | .and_then(|val| val.to_set().ok()) 37 | .map(|set| set.read().val), 38 | Some(vec![Prim::Int(57)].into_iter().collect()) 39 | ); 40 | } 41 | 42 | #[test] 43 | fn test_iter() { 44 | let actor = 1; 45 | let mut db = mk_db(actor); 46 | 47 | let add_ctx = db.get(&("x".into(), Kind::Reg)).unwrap().derive_add_ctx(actor); 48 | db.update(("x", Kind::Reg), add_ctx, |data, ctx| { 49 | let reg = data.to_reg().unwrap(); 50 | reg.write("x's val", ctx) 51 | }).unwrap(); 52 | 53 | let add_ctx = db.get(&("y".into(), Kind::Reg)).unwrap().derive_add_ctx(actor); 54 | db.update(("y", Kind::Reg), add_ctx, |data, ctx| { 55 | let reg = data.to_reg().unwrap(); 56 | reg.write("y's val", ctx) 57 | }).unwrap(); 58 | 59 | let add_ctx = db.get(&("z".into(), Kind::Reg)).unwrap().derive_add_ctx(actor); 60 | db.update(("z", Kind::Reg), add_ctx, |data, ctx| { 61 | let reg = data.to_reg().unwrap(); 62 | reg.write("z's val", ctx) 63 | }).unwrap(); 64 | 65 | let items: BTreeMap<(String, Kind), crdts::ReadCtx> = db.iter().unwrap() 66 | .map(|opt| opt.unwrap()) 67 | .collect(); 68 | 69 | assert_eq!(items.len(), 3); 70 | assert_eq!( 71 | items.get(&("x".into(), Kind::Reg)) 72 | .cloned() 73 | .and_then(|e| e.val.to_reg().ok()) 74 | .map(|r| r.read().val), 75 | Some(vec!["x's val".into()]) 76 | ); 77 | assert_eq!( 78 | items.get(&("y".into(), Kind::Reg)) 79 | .cloned() 80 | .and_then(|e| e.val.to_reg().ok()) 81 | .map(|r| r.read().val), 82 | Some(vec!["y's val".into()]) 83 | ); 84 | assert_eq!( 85 | items.get(&("z".into(), Kind::Reg)) 86 | .cloned() 87 | .and_then(|e| e.val.to_reg().ok()) 88 | .map(|r| r.read().val), 89 | Some(vec!["z's val".into()]) 90 | ); 91 | } 92 | 93 | #[test] 94 | fn test_sync() { 95 | let mut remote = memory_log::Log::new(0); 96 | let mut db_1 = mk_db(1); 97 | let mut db_2 = mk_db(2); 98 | 99 | let add_ctx = db_1.get(&("x".into(), Kind::Reg)).unwrap().derive_add_ctx(1); 100 | db_1.update(("x", Kind::Reg), add_ctx, |d, ctx| { 101 | let reg = d.to_reg().unwrap(); 102 | reg.write("this is a reg for value 'x'", ctx) 103 | }).unwrap(); 104 | 105 | let add_ctx = db_2.get(&("y".into(), Kind::Reg)).unwrap().derive_add_ctx(2); 106 | db_2.update(("y", Kind::Reg), add_ctx, |d, ctx| { 107 | let reg = d.to_reg().unwrap(); 108 | reg.write("this is a reg for value 'y'", ctx) 109 | }).unwrap(); 110 | 111 | db_1.sync(&mut remote).unwrap(); 112 | db_2.sync(&mut remote).unwrap(); 113 | db_1.sync(&mut remote).unwrap(); 114 | 115 | assert_eq!( 116 | db_1.get(&("x".into(), Kind::Reg)).unwrap().val 117 | .and_then(|data| data.to_reg().ok()) 118 | .map(|reg| reg.read().val), 119 | Some(vec!["this is a reg for value 'x'".into()]) 120 | ); 121 | assert_eq!( 122 | db_1.get(&("y".into(), Kind::Reg)).unwrap().val 123 | .and_then(|data| data.to_reg().ok()) 124 | .map(|reg| reg.read().val), 125 | Some(vec!["this is a reg for value 'y'".into()]) 126 | ); 127 | assert_eq!( 128 | db_2.get(&("x".into(), Kind::Reg)).unwrap().val 129 | .and_then(|data| data.to_reg().ok()) 130 | .map(|reg| reg.read().val), 131 | Some(vec!["this is a reg for value 'x'".into()]) 132 | ); 133 | assert_eq!( 134 | db_2.get(&("y".into(), Kind::Reg)).unwrap().val 135 | .and_then(|data| data.to_reg().ok()) 136 | .map(|reg| reg.read().val), 137 | Some(vec!["this is a reg for value 'y'".into()]) 138 | ); 139 | } 140 | -------------------------------------------------------------------------------- /benches/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | extern crate test; 3 | 4 | extern crate gitdb; 5 | extern crate tempfile; 6 | 7 | // #[bench] 8 | // fn time_to_first_write(b: &mut test::Bencher) { 9 | // b.iter(|| { 10 | // let dir = tempfile::tempdir().unwrap(); 11 | // let dir_path = dir.path().to_owned(); 12 | // let git_root = dir_path.join("db"); 13 | // let db = gitdb::DB::init(&git_root).unwrap(); 14 | // 15 | // let kdf = gitdb::crypto::KDF { 16 | // pbkdf2_iters: 100_000, 17 | // salt: gitdb::crypto::rand_256().unwrap(), 18 | // entropy: gitdb::crypto::create_entropy_file(&dir_path).unwrap() 19 | // }; 20 | // 21 | // let sess = gitdb::Session { 22 | // site_id: 0, 23 | // master_key: kdf.master_key(b"super secret") 24 | // }; 25 | // 26 | // db.create_key_salt(&sess).unwrap(); 27 | // 28 | // let val = gitdb::ditto::Register::new(12.into(), sess.site_id); 29 | // db.write_block("val", &gitdb::Block::Val(val), &sess).unwrap(); 30 | // }) 31 | // } 32 | // 33 | // #[bench] 34 | // fn write_100_blocks(b: &mut test::Bencher) { 35 | // let kdf = gitdb::crypto::KDF { 36 | // pbkdf2_iters: 100_000, 37 | // salt: gitdb::crypto::rand_256().unwrap(), 38 | // entropy: gitdb::crypto::rand_256().unwrap() 39 | // }; 40 | // 41 | // let sess = gitdb::Session { 42 | // site_id: 0, 43 | // master_key: kdf.master_key(b"super secret") 44 | // }; 45 | // 46 | // b.iter(|| { 47 | // let dir = tempfile::tempdir().unwrap(); 48 | // let dir_path = dir.path().to_owned(); 49 | // let git_root = dir_path.join("db"); 50 | // let db = gitdb::DB::init(&git_root).unwrap(); 51 | // db.create_key_salt(&sess).unwrap(); 52 | // 53 | // for i in 0..100 { 54 | // let val = gitdb::ditto::Register::new(i.into(), sess.site_id); 55 | // db.write_block(&format!("val#{}", i), &gitdb::Block::Val(val), &sess).unwrap(); 56 | // } 57 | // }) 58 | // } 59 | // 60 | // #[bench] 61 | // fn read_block(b: &mut test::Bencher) { 62 | // let kdf = gitdb::crypto::KDF { 63 | // pbkdf2_iters: 100_000, 64 | // salt: gitdb::crypto::rand_256().unwrap(), 65 | // entropy: gitdb::crypto::rand_256().unwrap() 66 | // }; 67 | // 68 | // let sess = gitdb::Session { 69 | // site_id: 0, 70 | // master_key: kdf.master_key(b"super secret") 71 | // }; 72 | // 73 | // let dir = tempfile::tempdir().unwrap(); 74 | // let dir_path = dir.path().to_owned(); 75 | // let git_root = dir_path.join("db"); 76 | // let db = gitdb::DB::init(&git_root).unwrap(); 77 | // db.create_key_salt(&sess).unwrap(); 78 | // 79 | // let val = gitdb::ditto::Register::new(37.into(), sess.site_id); 80 | // db.write_block("val_key", &gitdb::Block::Val(val), &sess).unwrap(); 81 | // 82 | // b.iter(|| { 83 | // db.read_block("val_key", &sess).unwrap(); 84 | // }) 85 | // } 86 | // 87 | // #[bench] 88 | // fn read_1000_blocks(b: &mut test::Bencher) { 89 | // let kdf = gitdb::crypto::KDF { 90 | // pbkdf2_iters: 100_000, 91 | // salt: gitdb::crypto::rand_256().unwrap(), 92 | // entropy: gitdb::crypto::rand_256().unwrap() 93 | // }; 94 | // 95 | // let sess = gitdb::Session { 96 | // site_id: 0, 97 | // master_key: kdf.master_key(b"super secret") 98 | // }; 99 | // 100 | // let dir = tempfile::tempdir().unwrap(); 101 | // let dir_path = dir.path().to_owned(); 102 | // let git_root = dir_path.join("db"); 103 | // let db = gitdb::DB::init(&git_root).unwrap(); 104 | // db.create_key_salt(&sess).unwrap(); 105 | // 106 | // for i in 0..1000 { 107 | // let val = gitdb::ditto::Register::new(i.into(), sess.site_id); 108 | // db.write_block(&format!("val#{}", i), &gitdb::Block::Val(val), &sess).unwrap(); 109 | // } 110 | // 111 | // b.iter(|| { 112 | // for i in 0..1000 { 113 | // let block = db.read_block(&format!("val#{}", i), &sess).unwrap(); 114 | // assert_eq!(block.to_val().unwrap().get(), &gitdb::Prim::U64(i)); 115 | // } 116 | // }) 117 | // } 118 | // 119 | // 120 | // #[bench] 121 | // fn prefix_scan_1000_blocks(b: &mut test::Bencher) { 122 | // let kdf = gitdb::crypto::KDF { 123 | // pbkdf2_iters: 100_000, 124 | // salt: gitdb::crypto::rand_256().unwrap(), 125 | // entropy: gitdb::crypto::rand_256().unwrap() 126 | // }; 127 | // 128 | // let sess = gitdb::Session { 129 | // site_id: 0, 130 | // master_key: kdf.master_key(b"super secret") 131 | // }; 132 | // 133 | // let dir = tempfile::tempdir().unwrap(); 134 | // let dir_path = dir.path().to_owned(); 135 | // let git_root = dir_path.join("db"); 136 | // let db = gitdb::DB::init(&git_root).unwrap(); 137 | // db.create_key_salt(&sess).unwrap(); 138 | // 139 | // for i in 0..1000 { 140 | // let val = gitdb::ditto::Register::new(gitdb::Prim::U64(i), sess.site_id); 141 | // db.write_block(&format!("val#{}", "x".repeat(i as usize)), &gitdb::Block::Val(val), &sess).unwrap(); 142 | // } 143 | // 144 | // b.iter(|| { 145 | // let mut i: u64 = 0; 146 | // for (_key, block) in db.prefix_scan("val", &sess).unwrap() { 147 | // assert_eq!(block.to_val().unwrap().get(), &gitdb::Prim::U64(i)); 148 | // i += 1; 149 | // } 150 | // 151 | // assert_eq!(i, 1000); 152 | // }) 153 | // } 154 | // 155 | -------------------------------------------------------------------------------- /src/encrypted_git_log.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Debug}; 2 | use std::marker::PhantomData; 3 | /// An Encrypted Git Log 4 | /// Implementation wraps the unencypted git log with an encryption layer. 5 | use std::str::FromStr; 6 | use std::string::ToString; 7 | 8 | use serde_derive::{Deserialize, Serialize}; 9 | 10 | use crdts::{Actor, CmRDT}; 11 | use git2; 12 | 13 | use crate::crypto::{rand_256, Encrypted, KeyHierarchy}; 14 | use crate::error::Result; 15 | use crate::git_log; 16 | use crate::log::{LogReplicable, TaggedOp}; 17 | 18 | struct EncryptedCRDT { 19 | phantom_crdt: PhantomData, 20 | } 21 | 22 | #[derive(Debug, Clone, Serialize, Deserialize)] 23 | struct EncryptedOp { 24 | salt: [u8; 256 / 8], 25 | op: Encrypted, 26 | } 27 | 28 | unsafe impl Send for EncryptedOp {} 29 | 30 | pub struct Log 31 | where 32 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 33 | { 34 | root_key: KeyHierarchy, 35 | actor_key: KeyHierarchy, 36 | log: git_log::Log>, 37 | } 38 | 39 | pub struct LoggedOp 40 | where 41 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 42 | { 43 | encrypted_logged_op: git_log::LoggedOp>, 44 | plaintext_op: C::Op, 45 | } 46 | 47 | impl Debug for LoggedOp 48 | where 49 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 50 | { 51 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 52 | write!( 53 | f, 54 | "LoggedOp {{ encrypted_logged_op: {:?}, plaintext_op: {:?} }}", 55 | self.encrypted_logged_op, self.plaintext_op 56 | ) 57 | } 58 | } 59 | 60 | impl CmRDT for EncryptedCRDT 61 | where 62 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 63 | { 64 | type Op = EncryptedOp; 65 | 66 | fn apply(&mut self, _op: &Self::Op) { 67 | panic!("this should never be called"); 68 | } 69 | } 70 | 71 | impl EncryptedOp { 72 | fn encrypt(op: &C::Op, root: &KeyHierarchy) -> Result 73 | where 74 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 75 | { 76 | let bytes = bincode::serialize(&op)?; 77 | let salt = rand_256()?; 78 | let crypto_key = root.key_for(&salt); 79 | let op = crypto_key.encrypt(&bytes)?; 80 | Ok(EncryptedOp { salt, op }) 81 | } 82 | 83 | fn decrypt(&self, root: &KeyHierarchy) -> Result 84 | where 85 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 86 | { 87 | let crypto_key = root.key_for(&self.salt); 88 | let bytes = crypto_key.decrypt(&self.op)?; 89 | let op = bincode::deserialize(&bytes)?; 90 | Ok(op) 91 | } 92 | } 93 | 94 | impl TaggedOp for LoggedOp 95 | where 96 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 97 | { 98 | type ID = as TaggedOp>::ID; 99 | 100 | fn id(&self) -> Self::ID { 101 | self.encrypted_logged_op.id() 102 | } 103 | 104 | fn op(&self) -> &C::Op { 105 | &self.plaintext_op 106 | } 107 | } 108 | 109 | impl LogReplicable for Log 110 | where 111 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 112 | A: Actor + ToString + FromStr + serde::Serialize, 113 | { 114 | type LoggedOp = LoggedOp; 115 | type Remote = git_log::Remote; 116 | 117 | fn next(&self) -> Result> { 118 | match self.log.next() { 119 | Ok(Some(encrypted_logged_op)) => { 120 | let actor_bytes = bincode::serialize(encrypted_logged_op.actor())?; 121 | let actor_key = self.root_key.derive_child(&actor_bytes); 122 | 123 | let plaintext_op = { 124 | let encrypted_op = encrypted_logged_op.op(); 125 | encrypted_op.decrypt::(&actor_key)? 126 | }; 127 | 128 | Ok(Some(LoggedOp { 129 | encrypted_logged_op, 130 | plaintext_op, 131 | })) 132 | } 133 | Ok(None) => Ok(None), 134 | Err(e) => Err(e), 135 | } 136 | } 137 | 138 | fn ack(&mut self, logged_op: &Self::LoggedOp) -> Result<()> { 139 | self.log.ack(&logged_op.encrypted_logged_op) 140 | } 141 | 142 | fn commit(&mut self, op: C::Op) -> Result { 143 | let encrypted_op = EncryptedOp::encrypt::(&op, &self.actor_key)?; 144 | 145 | let encrypted_logged_op = self.log.commit(encrypted_op)?; 146 | Ok(LoggedOp { 147 | encrypted_logged_op, 148 | plaintext_op: op, 149 | }) 150 | } 151 | 152 | fn pull(&mut self, remote: &Self::Remote) -> Result<()> { 153 | self.log.pull(remote) 154 | } 155 | 156 | fn push(&self, remote: &mut Self::Remote) -> Result<()> { 157 | self.log.push(remote) 158 | } 159 | } 160 | 161 | impl Log 162 | where 163 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 164 | A: Actor + serde::Serialize + serde::de::DeserializeOwned, 165 | { 166 | pub fn new(actor: A, repo: git2::Repository, root_key: KeyHierarchy) -> Self { 167 | let actor_bytes = bincode::serialize(&actor).unwrap(); 168 | let actor_key = root_key.derive_child(&actor_bytes); 169 | 170 | Log { 171 | root_key, 172 | actor_key, 173 | log: git_log::Log::new(actor, repo), 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/crypto.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Debug}; 2 | use std::num::NonZeroU32; 3 | 4 | use ring::rand::{SecureRandom, SystemRandom}; 5 | use ring::{aead, hkdf, pbkdf2}; 6 | use serde_derive::{Deserialize, Serialize}; 7 | 8 | use crate::error::{Error, Result}; 9 | 10 | #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] 11 | pub struct KDF { 12 | pub pbkdf2_iters: NonZeroU32, 13 | pub salt: [u8; 256 / 8], 14 | } 15 | 16 | pub struct KeyHierarchy { 17 | key: hkdf::Prk, 18 | } 19 | 20 | #[derive(Debug, PartialEq, Eq)] 21 | pub struct CryptoKey { 22 | key: [u8; 256 / 8], 23 | } 24 | 25 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 26 | pub struct Encrypted { 27 | pub nonce: [u8; 96 / 8], 28 | pub ciphertext: Vec, 29 | } 30 | 31 | impl KDF { 32 | pub fn derive_root(&self, pass: &[u8]) -> KeyHierarchy { 33 | let mut root_key = [0u8; 256 / 8]; 34 | 35 | pbkdf2::derive( 36 | pbkdf2::PBKDF2_HMAC_SHA256, 37 | self.pbkdf2_iters, 38 | &self.salt, 39 | &pass, 40 | &mut root_key, 41 | ); 42 | 43 | let key = hkdf::Salt::new(hkdf::HKDF_SHA256, &self.salt).extract(&root_key); 44 | 45 | KeyHierarchy { key } 46 | } 47 | } 48 | 49 | impl KeyHierarchy { 50 | pub fn derive_child(&self, namespace: &[u8]) -> KeyHierarchy { 51 | let mut salt = [0u8; 256 / 8]; 52 | self.key 53 | .expand(&[], hkdf::HKDF_SHA256) 54 | .and_then(|output_keying_mat| output_keying_mat.fill(&mut salt)) 55 | .unwrap(); 56 | 57 | KeyHierarchy { 58 | key: hkdf::Salt::new(hkdf::HKDF_SHA256, &salt).extract(namespace), 59 | } 60 | } 61 | 62 | pub fn key_for(&self, plaintext_unique_id: &[u8]) -> CryptoKey { 63 | let mut crypto_key = CryptoKey { 64 | key: [0u8; 256 / 8], 65 | }; 66 | 67 | self.key 68 | .expand(&[plaintext_unique_id], hkdf::HKDF_SHA256) 69 | .and_then(|okm| okm.fill(&mut crypto_key.key)) 70 | .unwrap(); 71 | 72 | crypto_key 73 | } 74 | } 75 | 76 | impl CryptoKey { 77 | pub fn encrypt(&self, plaintext: &[u8]) -> Result { 78 | let algo = &aead::CHACHA20_POLY1305; 79 | 80 | let mut cryptic = Encrypted { 81 | nonce: rand_nonce()?, 82 | ciphertext: vec![0u8; plaintext.len() + algo.tag_len()], 83 | }; 84 | 85 | cryptic 86 | .ciphertext 87 | .splice(0..plaintext.len(), plaintext.iter().cloned()); 88 | 89 | // TODO: sanity check, rm this once you've convinced yourself 90 | assert_eq!(&cryptic.ciphertext[0..plaintext.len()], plaintext); 91 | 92 | let unbound_key = aead::UnboundKey::new(algo, &self.key) 93 | .map_err(|_| Error::Crypto("Failed to create an unbound key for sealing".into()))?; 94 | 95 | let key = aead::LessSafeKey::new(unbound_key); 96 | 97 | key.seal_in_place_append_tag( 98 | aead::Nonce::assume_unique_for_key(cryptic.nonce), 99 | aead::Aad::empty(), 100 | &mut cryptic.ciphertext, // plaintext (encrypted in place) 101 | ) 102 | .map_err(|_| Error::Crypto("Failed to encrypt".into()))?; 103 | 104 | Ok(cryptic) 105 | } 106 | 107 | pub fn decrypt(&self, encrypted: &Encrypted) -> Result> { 108 | let algo = &aead::CHACHA20_POLY1305; 109 | 110 | let unbound_key = aead::UnboundKey::new(algo, &self.key) 111 | .map_err(|_| Error::Crypto("Failed to create an unbound key for opening".into()))?; 112 | let key = aead::LessSafeKey::new(unbound_key); 113 | 114 | let mut in_out = encrypted.ciphertext.clone(); 115 | let plain_with_tag = key 116 | .open_in_place( 117 | aead::Nonce::assume_unique_for_key(encrypted.nonce), 118 | aead::Aad::empty(), 119 | &mut in_out, // cyphertext (decrypted in place) 120 | ) 121 | .map_err(|_| Error::Crypto("Failed to decrypt".into()))?; 122 | 123 | let plain = plain_with_tag[0..plain_with_tag.len() - algo.tag_len()].to_vec(); 124 | Ok(plain) 125 | } 126 | } 127 | 128 | impl Debug for KeyHierarchy { 129 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 130 | write!(f, "KeyHierarchy") 131 | } 132 | } 133 | 134 | pub fn rand_nonce() -> Result<[u8; 96 / 8]> { 135 | let mut buf = [0u8; 96 / 8]; 136 | // TAI: Should this rng live in a session so we don't have to recreate it each time? 137 | SystemRandom::new() 138 | .fill(&mut buf) 139 | .map_err(|_| Error::Crypto("Failed to generate 96 bits of random".into()))?; 140 | Ok(buf) 141 | } 142 | 143 | pub fn rand_256() -> Result<[u8; 256 / 8]> { 144 | let mut buf = [0u8; 256 / 8]; 145 | // TAI: Should this rng live in a session so we don't have to recreate it each time? 146 | SystemRandom::new() 147 | .fill(&mut buf) 148 | .map_err(|_| Error::Crypto("Failed to generate 256 bits of random".into()))?; 149 | Ok(buf) 150 | } 151 | 152 | #[cfg(test)] 153 | mod test { 154 | use super::*; 155 | 156 | impl PartialEq for KeyHierarchy { 157 | fn eq(&self, other: &Self) -> bool { 158 | let mut self_expanded = [0u8; 256 / 8]; 159 | let mut other_expanded = [0u8; 256 / 8]; 160 | self.key 161 | .expand(&[], hkdf::HKDF_SHA256) 162 | .and_then(|okm| okm.fill(&mut self_expanded)) 163 | .unwrap(); 164 | other 165 | .key 166 | .expand(&[], hkdf::HKDF_SHA256) 167 | .and_then(|okm| okm.fill(&mut other_expanded)) 168 | .unwrap(); 169 | 170 | self_expanded == other_expanded 171 | } 172 | } 173 | 174 | #[test] 175 | fn kdf() { 176 | let kdf = KDF { 177 | pbkdf2_iters: NonZeroU32::new(1000).unwrap(), 178 | salt: rand_256().unwrap(), 179 | }; 180 | 181 | let root_key1 = kdf.derive_root(b"sssshh.. it's a secret"); 182 | let root_key2 = kdf.derive_root(b"sssshh.. it's a secret"); 183 | let imposter_key = kdf.derive_root(b"imposter!!"); 184 | 185 | assert_eq!(root_key1, root_key2); // proof: kdf is deterministic 186 | assert_ne!(root_key1, imposter_key) // proof: varied password => varied key 187 | } 188 | 189 | #[test] 190 | fn key_hierarchy() { 191 | let kdf = KDF { 192 | pbkdf2_iters: NonZeroU32::new(1000).unwrap(), 193 | salt: rand_256().unwrap(), 194 | }; 195 | 196 | let root_key = kdf.derive_root(b"pass"); 197 | let log_key = root_key.derive_child(b"log"); 198 | 199 | assert_ne!(root_key, log_key); 200 | 201 | let log_key2 = root_key.derive_child(b"log"); 202 | 203 | assert_eq!(log_key, log_key2); // proof: derive_child is deterministic. 204 | 205 | let storage_key = root_key.derive_child(b"storage"); 206 | 207 | assert_ne!(log_key, storage_key); // proof: varied child name => varied key 208 | 209 | let log_msg1_key = log_key.key_for(&[1u8]); 210 | 211 | let log_msg2_key = log_key.key_for(&[2u8]); 212 | 213 | let storage_block_key = storage_key.key_for(&[1u8]); 214 | 215 | assert_ne!(log_msg1_key, log_msg2_key); 216 | assert_ne!(storage_block_key, log_msg1_key); 217 | assert_ne!(storage_block_key, log_msg2_key); 218 | } 219 | 220 | #[test] 221 | fn plaintext_encrypt_decrypt() { 222 | let kdf = KDF { 223 | pbkdf2_iters: NonZeroU32::new(1000).unwrap(), 224 | salt: rand_256().unwrap(), 225 | }; 226 | 227 | let root_key = kdf.derive_root(b"password"); 228 | 229 | let msg = b"I kinda like you"; 230 | let msg_id = [0u8, 1u8]; 231 | 232 | let key = root_key.key_for(&msg_id); 233 | 234 | let cryptic = key.encrypt(msg).unwrap(); 235 | 236 | // can we do a better check than this? 237 | // maybe we can make some test vectors? 238 | assert_ne!(cryptic.ciphertext, msg); 239 | 240 | // Our encryption scheme must be probabilistic! 241 | // 242 | // The same msg encrypted under the same key should produce 243 | // different ciphertexts. 244 | let cryptic2 = key.encrypt(msg).unwrap(); 245 | 246 | assert_ne!(cryptic.nonce, cryptic2.nonce); // nonces must differ! 247 | assert_ne!(cryptic.ciphertext, cryptic2.ciphertext); // ciphertexts must differ! 248 | 249 | let decrypted_msg = key.decrypt(&cryptic).unwrap(); 250 | let decrypted_string = String::from_utf8(decrypted_msg).unwrap(); 251 | assert_eq!(decrypted_string, "I kinda like you"); 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /tests/log_replication.rs: -------------------------------------------------------------------------------- 1 | use std::num::NonZeroU32; 2 | 3 | use assert_matches::assert_matches; 4 | use git2; 5 | use hermitdb::{ 6 | crdts::{map, CmRDT, Map, Orswot}, 7 | crypto, encrypted_git_log, git_log, 8 | log::{LogReplicable, TaggedOp}, 9 | memory_log, 10 | }; 11 | use quickcheck::{quickcheck, Arbitrary, Gen, TestResult}; 12 | 13 | type TActor = u8; 14 | type TKey = u8; 15 | type TVal = Orswot; 16 | type TMap = Map; 17 | type TOp = map::Op; 18 | 19 | #[derive(Debug, Clone)] 20 | struct OpVec(TActor, Vec); 21 | 22 | impl Arbitrary for OpVec { 23 | fn arbitrary(g: &mut G) -> Self { 24 | let actor = TActor::arbitrary(g); 25 | let num_ops = u8::arbitrary(g); 26 | let mut map = TMap::new(); 27 | let mut ops = Vec::with_capacity(num_ops as usize); 28 | for _ in 0..num_ops { 29 | let die_roll = u8::arbitrary(g); 30 | let key = TKey::arbitrary(g); 31 | let read_ctx = map.get(&key); 32 | let add_ctx = read_ctx.derive_add_ctx(actor.clone()); 33 | let op = match die_roll % 3 { 34 | 0 => { 35 | // update Orswot 36 | map.update(key, add_ctx, |set, ctx| { 37 | let die_roll = u8::arbitrary(g); 38 | let member = u8::arbitrary(g); 39 | match die_roll % 2 { 40 | 0 => set.add(member, ctx), 41 | _ => { 42 | let rm_ctx = set.contains(&member).derive_rm_ctx(); 43 | set.remove(member, rm_ctx) 44 | } 45 | } 46 | }) 47 | } 48 | 1 => { 49 | // rm 50 | let rm_ctx = map.get(&key).derive_rm_ctx(); 51 | map.rm(key, rm_ctx) 52 | } 53 | _ => { 54 | // nop 55 | map::Op::Nop 56 | } 57 | }; 58 | map.apply(&op); 59 | ops.push(op); 60 | } 61 | OpVec(actor, ops) 62 | } 63 | 64 | fn shrink(&self) -> Box> { 65 | let mut shrunk: Vec = Vec::new(); 66 | for i in 0..self.1.len() { 67 | let mut vec = self.1.clone(); 68 | vec.remove(i); 69 | shrunk.push(OpVec(self.0, vec)) 70 | } 71 | Box::new(shrunk.into_iter()) 72 | } 73 | } 74 | 75 | fn p2p_converge>( 76 | mut a_log: L, 77 | mut b_log: L, 78 | mut remote: L::Remote, 79 | a_ops: Vec, 80 | b_ops: Vec, 81 | ) -> TMap { 82 | let mut a_map = TMap::new(); 83 | let mut b_map = TMap::new(); 84 | 85 | for op in a_ops { 86 | let tagged_op = a_log.commit(op).unwrap(); 87 | a_map.apply(tagged_op.op()); 88 | assert_matches!(a_log.ack(&tagged_op), Ok(())); 89 | } 90 | 91 | for op in b_ops { 92 | let tagged_op = b_log.commit(op).unwrap(); 93 | b_map.apply(tagged_op.op()); 94 | assert_matches!(b_log.ack(&tagged_op), Ok(())); 95 | } 96 | 97 | assert_matches!(b_log.push(&mut remote), Ok(_)); 98 | assert_matches!(a_log.push(&mut remote), Ok(_)); 99 | 100 | assert_matches!(b_log.pull(&remote), Ok(_)); 101 | assert_matches!(a_log.pull(&remote), Ok(_)); 102 | 103 | while let Some(tagged_op) = a_log.next().unwrap() { 104 | a_map.apply(tagged_op.op()); 105 | assert_matches!(a_log.ack(&tagged_op), Ok(())); 106 | } 107 | 108 | while let Some(tagged_op) = b_log.next().unwrap() { 109 | b_map.apply(tagged_op.op()); 110 | assert_matches!(b_log.ack(&tagged_op), Ok(())); 111 | } 112 | 113 | assert_eq!(a_map, b_map); 114 | a_map 115 | } 116 | 117 | fn log_preserves_order(mut log: impl LogReplicable, ops: Vec) { 118 | for op in ops.iter() { 119 | assert_matches!(log.commit(op.clone()), Ok(_)); 120 | } 121 | 122 | for op in ops.iter() { 123 | let tagged_op = log.next().unwrap().unwrap(); 124 | assert_eq!(op, tagged_op.op()); 125 | log.ack(&tagged_op).unwrap(); 126 | } 127 | assert_matches!(log.next(), Ok(None)); 128 | } 129 | 130 | quickcheck! { 131 | fn prop_replication_converges_memory(a_ops: OpVec, b_ops: OpVec) -> TestResult { 132 | let (actor1, a_ops) = (a_ops.0, a_ops.1); 133 | let (actor2, b_ops) = (b_ops.0, b_ops.1); 134 | 135 | if actor1 == actor2 { 136 | return TestResult::discard(); 137 | } 138 | 139 | let a_log = memory_log::Log::new(actor1); 140 | let b_log = memory_log::Log::new(actor2); 141 | let remote = memory_log::Log::new(actor1); 142 | 143 | p2p_converge(a_log, b_log, remote, a_ops, b_ops); 144 | TestResult::from_bool(true) 145 | } 146 | 147 | fn prop_replication_converge_git(a_ops: OpVec, b_ops: OpVec) -> TestResult { 148 | let (actor1, a_ops) = (a_ops.0, a_ops.1); 149 | let (actor2, b_ops) = (b_ops.0, b_ops.1); 150 | 151 | if actor1 == actor2 { 152 | return TestResult::discard(); 153 | } 154 | 155 | let a_log_dir = tempfile::tempdir().unwrap(); 156 | let b_log_dir = tempfile::tempdir().unwrap(); 157 | let remote_dir = tempfile::tempdir().unwrap(); 158 | 159 | let a_log_git = git2::Repository::init_bare( 160 | &a_log_dir.path() 161 | ).unwrap(); 162 | 163 | let b_log_git = git2::Repository::init_bare( 164 | &b_log_dir.path() 165 | ).unwrap(); 166 | 167 | let _remote_git = git2::Repository::init_bare( 168 | &remote_dir.path() 169 | ).unwrap(); 170 | 171 | let a_log = git_log::Log::new(actor1, a_log_git); 172 | let b_log = git_log::Log::new(actor2, b_log_git); 173 | 174 | let remote = git_log::Remote::no_auth( 175 | "remote".into(), 176 | remote_dir.path().to_str().unwrap().to_string() 177 | ); 178 | 179 | p2p_converge(a_log, b_log, remote, a_ops, b_ops); 180 | TestResult::from_bool(true) 181 | } 182 | 183 | fn prop_replication_converge_encrypted_git(a_ops: OpVec, b_ops: OpVec) -> TestResult { 184 | let (actor1, a_ops) = (a_ops.0, a_ops.1); 185 | let (actor2, b_ops) = (b_ops.0, b_ops.1); 186 | 187 | if actor1 == actor2 { 188 | return TestResult::discard(); 189 | } 190 | 191 | let root_key = crypto::KDF { 192 | pbkdf2_iters: NonZeroU32::new(1).unwrap(), 193 | salt: [0u8; 256 / 8] 194 | }.derive_root(b"password"); 195 | 196 | let a_log_dir = tempfile::tempdir().unwrap(); 197 | let b_log_dir = tempfile::tempdir().unwrap(); 198 | let remote_dir = tempfile::tempdir().unwrap(); 199 | 200 | let a_log_git = git2::Repository::init_bare( 201 | &a_log_dir.path() 202 | ).unwrap(); 203 | 204 | let b_log_git = git2::Repository::init_bare( 205 | &b_log_dir.path() 206 | ).unwrap(); 207 | 208 | let _remote_git = git2::Repository::init_bare( 209 | &remote_dir.path() 210 | ).unwrap(); 211 | 212 | let a_log = encrypted_git_log::Log::new( 213 | actor1, 214 | a_log_git, 215 | root_key.derive_child(b"git_log") 216 | ); 217 | 218 | let b_log = encrypted_git_log::Log::new( 219 | actor2, 220 | b_log_git, 221 | root_key.derive_child(b"git_log") 222 | ); 223 | 224 | let remote = git_log::Remote::no_auth( 225 | "remote".into(), 226 | remote_dir.path().to_str().unwrap().to_string() 227 | ); 228 | 229 | p2p_converge(a_log, b_log, remote, a_ops, b_ops); 230 | TestResult::from_bool(true) 231 | } 232 | 233 | fn prop_log_preserves_order_memory(ops: OpVec) -> bool { 234 | let log: memory_log::Log = memory_log::Log::new(ops.0); 235 | log_preserves_order(log, ops.1); 236 | true 237 | } 238 | 239 | fn prop_log_preserves_order_git(ops: OpVec) -> bool { 240 | let OpVec(actor, ops) = ops; 241 | let log_dir = tempfile::tempdir().unwrap(); 242 | let log_path = log_dir.path(); 243 | let log_git = git2::Repository::init_bare(&log_path).unwrap(); 244 | 245 | let log = git_log::Log::new(actor, log_git); 246 | 247 | log_preserves_order(log, ops); 248 | 249 | true 250 | } 251 | 252 | fn prop_log_preserves_order_encrypted_git(ops: OpVec) -> bool { 253 | let OpVec(actor, ops) = ops; 254 | let log_dir = tempfile::tempdir().unwrap(); 255 | let log_path = log_dir.path(); 256 | let log_git = git2::Repository::init_bare(&log_path).unwrap(); 257 | 258 | let root_key = crypto::KDF { 259 | pbkdf2_iters: NonZeroU32::new(1).unwrap(), 260 | salt: [0u8; 256 / 8] 261 | }.derive_root(b"password"); 262 | 263 | let log = encrypted_git_log::Log::new( 264 | actor, 265 | log_git, 266 | root_key.derive_child(b"log") 267 | ); 268 | 269 | log_preserves_order(log, ops); 270 | 271 | true 272 | } 273 | } 274 | -------------------------------------------------------------------------------- /src/data.rs: -------------------------------------------------------------------------------- 1 | use std::hash::{Hash, Hasher}; 2 | 3 | use crdts::{self, Causal, CmRDT, CvRDT}; 4 | use serde_derive::{Deserialize, Serialize}; 5 | 6 | use crate::error::{Error, Result}; 7 | 8 | pub type Actor = u128; 9 | 10 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 11 | pub enum Prim { 12 | Nil, 13 | Float(f64), 14 | Int(i64), 15 | Str(String), 16 | Blob(Vec), 17 | } 18 | 19 | impl Eq for Prim {} 20 | 21 | #[allow(clippy::derive_hash_xor_eq)] 22 | impl Hash for Prim { 23 | fn hash(&self, state: &mut H) { 24 | match self { 25 | Prim::Nil => 0u8.hash(state), 26 | Prim::Float(f) => f.to_bits().hash(state), 27 | Prim::Int(i) => i.hash(state), 28 | Prim::Str(s) => s.hash(state), 29 | Prim::Blob(b) => b.hash(state), 30 | } 31 | } 32 | } 33 | 34 | impl Default for Prim { 35 | fn default() -> Self { 36 | Prim::Nil 37 | } 38 | } 39 | 40 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] 41 | pub enum Kind { 42 | Nil, 43 | Reg, 44 | Set, 45 | Map, 46 | Float, 47 | Int, 48 | Str, 49 | Blob, 50 | } 51 | 52 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 53 | pub enum Data { 54 | Nil, 55 | Reg(crdts::MVReg), 56 | Set(crdts::Orswot), 57 | Map(crdts::Map<(String, Kind), Box, Actor>), 58 | } 59 | 60 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 61 | pub enum Op { 62 | Reg(crdts::mvreg::Op), 63 | Set(crdts::orswot::Op), 64 | Map(crdts::map::Op<(String, Kind), Box, Actor>), 65 | } 66 | 67 | impl Default for Data { 68 | fn default() -> Self { 69 | Data::Nil 70 | } 71 | } 72 | 73 | impl CvRDT for Data { 74 | fn merge(&mut self, other: &Self) { 75 | if &mut Data::Nil == self { 76 | *self = other.clone() 77 | } 78 | 79 | // compute kinds here in case the match falls to error case. 80 | // (the match will consume self and other) 81 | let kind = self.kind(); 82 | let other_kind = other.kind(); 83 | match (self, other) { 84 | (_, Data::Nil) => { /* nothing to do */ } 85 | (Data::Reg(a), Data::Reg(b)) => a.merge(b), 86 | (Data::Set(a), Data::Set(b)) => a.merge(b), 87 | (Data::Map(a), Data::Map(b)) => a.merge(b), 88 | _ => { 89 | // If this ever happens, we've violated our invariants, we can't recover. 90 | // TAI: can we move this invariant to the type level some how? 91 | panic!("Merge failed: invalid kinds {:?}, {:?}", kind, other_kind); 92 | } 93 | } 94 | } 95 | } 96 | 97 | impl CvRDT for Box { 98 | fn merge(&mut self, other: &Self) { 99 | Data::merge(self, other) 100 | } 101 | } 102 | 103 | impl CmRDT for Data { 104 | type Op = Op; 105 | 106 | fn apply(&mut self, op: &Self::Op) { 107 | if &mut Data::Nil == self { 108 | *self = op.kind().default_data(); 109 | } 110 | let kind = self.kind(); 111 | let op_kind = op.kind(); 112 | match (self, op) { 113 | (Data::Reg(crdt), Op::Reg(op)) => crdt.apply(op), 114 | (Data::Set(crdt), Op::Set(op)) => crdt.apply(op), 115 | (Data::Map(crdt), Op::Map(op)) => crdt.apply(op), 116 | _ => { 117 | // If this ever happens, we've violated our invariants, we can't recover. 118 | // TAI: can we move this to the type level some how? 119 | panic!("Apply failed: invalid kinds {:?}, {:?}", kind, op_kind); 120 | } 121 | } 122 | } 123 | } 124 | 125 | impl CmRDT for Box { 126 | type Op = Box; 127 | 128 | fn apply(&mut self, op: &Self::Op) { 129 | Data::apply(self, op) 130 | } 131 | } 132 | 133 | impl Causal for Data { 134 | fn truncate(&mut self, clock: &crdts::VClock) { 135 | match self { 136 | Data::Nil => (), 137 | Data::Reg(causal) => causal.truncate(&clock), 138 | Data::Set(causal) => causal.truncate(&clock), 139 | Data::Map(causal) => causal.truncate(&clock), 140 | } 141 | } 142 | } 143 | 144 | impl Causal for Box { 145 | fn truncate(&mut self, clock: &crdts::VClock) { 146 | Data::truncate(self, clock) 147 | } 148 | } 149 | 150 | impl Data { 151 | pub fn kind(&self) -> Kind { 152 | match self { 153 | Data::Nil => Kind::Nil, 154 | Data::Reg(_) => Kind::Reg, 155 | Data::Set(_) => Kind::Set, 156 | Data::Map(_) => Kind::Map, 157 | } 158 | } 159 | 160 | pub fn to_nil(&self) -> Result<()> { 161 | match self { 162 | Data::Nil => Ok(()), 163 | other => Err(Error::UnexpectedKind(Kind::Nil, other.kind())), 164 | } 165 | } 166 | pub fn to_reg(&self) -> Result> { 167 | match self { 168 | Data::Nil => Ok(crdts::MVReg::default()), 169 | Data::Reg(r) => Ok(r.clone()), 170 | other => Err(Error::UnexpectedKind(Kind::Reg, other.kind())), 171 | } 172 | } 173 | 174 | pub fn to_set(&self) -> Result> { 175 | match self { 176 | Data::Nil => Ok(crdts::Orswot::default()), 177 | Data::Set(s) => Ok(s.clone()), 178 | other => Err(Error::UnexpectedKind(Kind::Set, other.kind())), 179 | } 180 | } 181 | 182 | pub fn to_map(&self) -> Result, Actor>> { 183 | match self { 184 | Data::Nil => Ok(crdts::Map::default()), 185 | Data::Map(m) => Ok(m.clone()), 186 | other => Err(Error::UnexpectedKind(Kind::Map, other.kind())), 187 | } 188 | } 189 | } 190 | 191 | impl Prim { 192 | pub fn kind(&self) -> Kind { 193 | match self { 194 | Prim::Nil => Kind::Nil, 195 | Prim::Float(_) => Kind::Float, 196 | Prim::Int(_) => Kind::Int, 197 | Prim::Str(_) => Kind::Str, 198 | Prim::Blob(_) => Kind::Blob, 199 | } 200 | } 201 | 202 | pub fn to_nil(&self) -> Result<()> { 203 | match self { 204 | Prim::Nil => Ok(()), 205 | other => Err(Error::UnexpectedKind(Kind::Nil, other.kind())), 206 | } 207 | } 208 | 209 | pub fn to_float(&self) -> Result { 210 | match self { 211 | Prim::Float(p) => Ok(*p), 212 | other => Err(Error::UnexpectedKind(Kind::Float, other.kind())), 213 | } 214 | } 215 | 216 | pub fn to_int(&self) -> Result { 217 | match self { 218 | Prim::Int(p) => Ok(*p), 219 | other => Err(Error::UnexpectedKind(Kind::Int, other.kind())), 220 | } 221 | } 222 | 223 | pub fn to_str(&self) -> Result { 224 | match self { 225 | Prim::Str(p) => Ok(p.clone()), 226 | other => Err(Error::UnexpectedKind(Kind::Str, other.kind())), 227 | } 228 | } 229 | 230 | pub fn to_blob(&self) -> Result> { 231 | match self { 232 | Prim::Blob(p) => Ok(p.clone()), 233 | other => Err(Error::UnexpectedKind(Kind::Blob, other.kind())), 234 | } 235 | } 236 | } 237 | 238 | impl Op { 239 | pub fn kind(&self) -> Kind { 240 | match self { 241 | Op::Reg(_) => Kind::Reg, 242 | Op::Set(_) => Kind::Set, 243 | Op::Map(_) => Kind::Map, 244 | } 245 | } 246 | } 247 | 248 | impl Kind { 249 | pub fn default_data(&self) -> Data { 250 | match self { 251 | Kind::Nil => Data::Nil, 252 | Kind::Reg => Data::Reg(crdts::MVReg::default()), 253 | Kind::Set => Data::Set(crdts::Orswot::default()), 254 | Kind::Map => Data::Map(crdts::Map::default()), 255 | 256 | // TAI: does it make sense to implement these prim kinds as Reg(::default())? 257 | Kind::Float => panic!("attempted to call default_data on Kind::Float"), 258 | Kind::Int => panic!("attempted to call default_data on Kind::Int"), 259 | Kind::Str => panic!("attempted to call default_data on Kind::Str"), 260 | Kind::Blob => panic!("attempted to call default_data on Kind::Blob"), 261 | } 262 | } 263 | } 264 | 265 | impl From for Prim { 266 | fn from(p: f64) -> Self { 267 | Prim::Float(p) 268 | } 269 | } 270 | 271 | impl From for Prim { 272 | fn from(p: i64) -> Self { 273 | Prim::Int(p) 274 | } 275 | } 276 | 277 | impl From for Prim { 278 | fn from(p: String) -> Self { 279 | Prim::Str(p) 280 | } 281 | } 282 | 283 | impl<'a> From<&'a str> for Prim { 284 | fn from(p: &'a str) -> Self { 285 | Prim::from(p.to_string()) 286 | } 287 | } 288 | 289 | impl From> for Prim { 290 | fn from(p: Vec) -> Self { 291 | Prim::Blob(p) 292 | } 293 | } 294 | 295 | impl From> for Op { 296 | fn from(op: crdts::mvreg::Op) -> Self { 297 | Op::Reg(op) 298 | } 299 | } 300 | 301 | impl From> for Op { 302 | fn from(op: crdts::orswot::Op) -> Self { 303 | Op::Set(op) 304 | } 305 | } 306 | 307 | impl From, Actor>> for Op { 308 | fn from(op: crdts::map::Op<(String, Kind), Box, Actor>) -> Self { 309 | Op::Map(op) 310 | } 311 | } 312 | -------------------------------------------------------------------------------- /art/george.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 45 | 47 | 48 | 50 | image/svg+xml 51 | 53 | 54 | 55 | 56 | 57 | 62 | 65 | 71 | 78 | 84 | 90 | 96 | 102 | 110 | 116 | 124 | 130 | 138 | 143 | 151 | 157 | 162 | 170 | 178 | 183 | 188 | 196 | 204 | 205 | 206 | 207 | -------------------------------------------------------------------------------- /src/map.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{BTreeSet, HashMap}; 2 | use std::fmt::Debug; 3 | use std::marker::PhantomData; 4 | 5 | use bincode; 6 | use crdts::{Actor, AddCtx, Causal, CmRDT, CvRDT, Dot, ReadCtx, RmCtx, VClock}; 7 | use serde_derive::{Deserialize, Serialize}; 8 | use sled; 9 | 10 | use crate::error::{Error, Result}; 11 | 12 | /// Key Trait alias to reduce redundancy in type decl. 13 | pub trait Key: Debug + Ord + Clone + Send {} 14 | impl Key for T {} 15 | 16 | /// Val Trait alias to reduce redundancy in type decl. 17 | pub trait Val: Debug + Default + Clone + Send + Causal + CmRDT + CvRDT {} 18 | 19 | impl Val for T where T: Debug + Default + Clone + Send + Causal + CmRDT + CvRDT {} 20 | 21 | #[derive(Debug)] 22 | pub struct Map, A: Actor> { 23 | // This clock stores the current version of the Map, it should 24 | // be greator or equal to all Entry clock's in the Map. 25 | sled: sled::Db, 26 | phantom_key: PhantomData, 27 | phantom_val: PhantomData, 28 | phantom_actor: PhantomData, 29 | } 30 | 31 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 32 | pub struct Entry, A: Actor> { 33 | // The entry clock tells us which actors edited this entry. 34 | pub clock: VClock, 35 | 36 | // The nested CRDT 37 | pub val: V, 38 | } 39 | 40 | pub struct Iter, A: Actor> { 41 | iter: sled::Iter, 42 | clock: VClock, 43 | phantom_key: PhantomData, 44 | phantom_val: PhantomData, 45 | phantom_actor: PhantomData, 46 | } 47 | 48 | /// Operations which can be applied to the Map CRDT 49 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 50 | pub enum Op, A: Actor> { 51 | /// No change to the CRDT 52 | Nop, 53 | /// Remove a key from the map 54 | Rm { 55 | /// Remove context 56 | clock: VClock, 57 | /// Key to remove 58 | key: K, 59 | }, 60 | /// Update an entry in the map 61 | Up { 62 | /// Update context 63 | dot: Dot, 64 | /// Key of the value to update 65 | key: K, 66 | /// The operation to apply on the value under `key` 67 | op: V::Op, 68 | }, 69 | } 70 | 71 | impl Iterator for Iter 72 | where 73 | K: Key + serde::de::DeserializeOwned, 74 | A: Actor + serde::de::DeserializeOwned, 75 | V: Val + serde::de::DeserializeOwned, 76 | { 77 | type Item = Result<(K, ReadCtx)>; 78 | 79 | fn next(&mut self) -> Option { 80 | match self.iter.next() { 81 | Some(Ok((k, v))) => { 82 | let res = bincode::deserialize(&k[KEY_PREFIX.len()..]).and_then(|key: K| { 83 | let entry: Entry = bincode::deserialize(&v)?; 84 | Ok(( 85 | key, 86 | ReadCtx { 87 | add_clock: self.clock.clone(), 88 | rm_clock: entry.clock, 89 | val: entry.val, 90 | }, 91 | )) 92 | }); 93 | 94 | Some(res.map_err(Error::from)) 95 | } 96 | Some(Err(e)) => Some(Err(Error::from(e))), 97 | None => None, 98 | } 99 | } 100 | } 101 | 102 | impl CmRDT for Map 103 | where 104 | K: Key + Debug + serde::Serialize + serde::de::DeserializeOwned, 105 | A: Actor + serde::Serialize + serde::de::DeserializeOwned, 106 | V: Val + Debug + serde::Serialize + serde::de::DeserializeOwned, 107 | { 108 | type Op = Op; 109 | 110 | fn apply(&mut self, op: &Self::Op) { 111 | match op.clone() { 112 | Op::Nop => { /* do nothing */ } 113 | Op::Rm { clock, key } => { 114 | self.apply_rm(key, &clock).unwrap(); 115 | self.sled.flush().unwrap(); 116 | } 117 | Op::Up { dot, key, op } => { 118 | let mut map_clock = self.get_clock().unwrap(); 119 | if map_clock.get(&dot.actor) >= dot.counter { 120 | // we've seen this op already 121 | return; 122 | } 123 | 124 | let key_bytes = self.key_bytes(&key).unwrap(); 125 | 126 | let mut entry = match self.sled.get(&key_bytes).unwrap() { 127 | Some(bytes) => bincode::deserialize(&bytes).unwrap(), 128 | None => Entry { 129 | clock: VClock::new(), 130 | val: V::default(), 131 | }, 132 | }; 133 | 134 | entry.clock.apply(&dot); 135 | entry.val.apply(&op); 136 | let entry_bytes = bincode::serialize(&entry).unwrap(); 137 | self.sled.insert(key_bytes, entry_bytes).unwrap(); 138 | 139 | map_clock.apply(&dot); 140 | self.put_clock(map_clock).unwrap(); 141 | self.apply_deferred().unwrap(); 142 | self.sled.flush().unwrap(); 143 | } 144 | } 145 | } 146 | } 147 | 148 | /// Key prefix is added to the front of all user added keys 149 | const KEY_PREFIX: [u8; 1] = [1]; 150 | 151 | /// Meta prefix is added to the front of all housekeeping keys created by the database 152 | const META_PREFIX: [u8; 1] = [0]; 153 | 154 | impl Map 155 | where 156 | K: Key + Debug + serde::Serialize + serde::de::DeserializeOwned, 157 | A: Actor + serde::Serialize + serde::de::DeserializeOwned, 158 | V: Val + Debug + serde::Serialize + serde::de::DeserializeOwned, 159 | { 160 | /// Constructs an empty Map 161 | pub fn new(sled: sled::Db) -> Map { 162 | Map { 163 | sled, 164 | phantom_key: PhantomData, 165 | phantom_val: PhantomData, 166 | phantom_actor: PhantomData, 167 | } 168 | } 169 | 170 | pub fn key_bytes(&self, key: &K) -> Result> { 171 | let mut bytes = bincode::serialize(&key)?; 172 | bytes.splice(0..0, KEY_PREFIX.iter().cloned()); 173 | Ok(bytes) 174 | } 175 | 176 | pub fn meta_key_bytes(&self, mut key: Vec) -> Vec { 177 | key.splice(0..0, META_PREFIX.iter().cloned()); 178 | key 179 | } 180 | 181 | /// Get a value stored under a key 182 | pub fn get(&self, key: &K) -> Result, A>> { 183 | let key_bytes = self.key_bytes(&key)?; 184 | 185 | let entry_opt = if let Some(val_bytes) = self.sled.get(&key_bytes)? { 186 | let entry: Entry = bincode::deserialize(&val_bytes)?; 187 | Some(entry) 188 | } else { 189 | None 190 | }; 191 | 192 | Ok(ReadCtx { 193 | add_clock: self.get_clock()?, 194 | rm_clock: entry_opt 195 | .clone() 196 | .map(|map_entry| map_entry.clock) 197 | .unwrap_or_else(VClock::new), 198 | val: entry_opt.map(|map_entry| map_entry.val), 199 | }) 200 | } 201 | 202 | /// Update a value under some key, if the key is not present in the map, 203 | /// the updater will be given `None`, otherwise `Some(val)` is given. 204 | /// 205 | /// The updater must return Some(val) to have the updated val stored back in 206 | /// the Map. If None is returned, this entry is removed from the Map. 207 | pub fn update(&self, key: I, ctx: AddCtx, f: F) -> Result> 208 | where 209 | F: FnOnce(&V, AddCtx) -> O, 210 | O: Into, 211 | I: Into, 212 | { 213 | let key = key.into(); 214 | let op = if let Some(data) = self.get(&key)?.val { 215 | f(&data, ctx.clone()).into() 216 | } else { 217 | f(&V::default(), ctx.clone()).into() 218 | }; 219 | Ok(Op::Up { 220 | dot: ctx.dot, 221 | key, 222 | op, 223 | }) 224 | } 225 | 226 | /// Remove an entry from the Map 227 | pub fn rm(&self, key: impl Into, ctx: RmCtx) -> Op { 228 | Op::Rm { 229 | clock: ctx.clock, 230 | key: key.into(), 231 | } 232 | } 233 | 234 | pub fn iter(&self) -> Result> { 235 | Ok(Iter { 236 | iter: self.sled.range(KEY_PREFIX..), 237 | clock: self.get_clock()?, 238 | phantom_key: PhantomData, 239 | phantom_val: PhantomData, 240 | phantom_actor: PhantomData, 241 | }) 242 | } 243 | 244 | fn apply_deferred(&mut self) -> Result<()> { 245 | let deferred = self.get_deferred()?; 246 | // TODO: it would be good to not clear the deferred map if we can avoid it. 247 | // this could be a point of data loss if we have a failure before we 248 | // finish applying all the deferred removes 249 | self.put_deferred(HashMap::new())?; 250 | for (clock, keys) in deferred { 251 | for key in keys { 252 | self.apply_rm(key, &clock)?; 253 | } 254 | } 255 | Ok(()) 256 | } 257 | 258 | /// Apply a key removal given a context. 259 | fn apply_rm(&mut self, key: K, clock: &VClock) -> Result<()> { 260 | let map_clock = self.get_clock()?; 261 | if !(clock <= &map_clock) { 262 | let mut deferred = self.get_deferred()?; 263 | let deferred_set = deferred.entry(clock.clone()).or_insert_with(BTreeSet::new); 264 | deferred_set.insert(key.clone()); 265 | self.put_deferred(deferred)?; 266 | } 267 | 268 | let key_bytes = self.key_bytes(&key)?; 269 | if let Some(entry_bytes) = self.sled.remove(&key_bytes)? { 270 | let mut entry: Entry = bincode::deserialize(&entry_bytes)?; 271 | entry.clock.subtract(&clock); 272 | if !entry.clock.is_empty() { 273 | entry.val.truncate(&clock); 274 | let new_entry_bytes = bincode::serialize(&entry)?; 275 | self.sled.insert(key_bytes, new_entry_bytes)?; 276 | } 277 | } 278 | Ok(()) 279 | } 280 | 281 | fn get_clock(&self) -> Result> { 282 | let clock_key = self.meta_key_bytes(b"clock".to_vec()); 283 | let clock = if let Some(clock_bytes) = self.sled.get(&clock_key)? { 284 | bincode::deserialize(&clock_bytes)? 285 | } else { 286 | VClock::new() 287 | }; 288 | Ok(clock) 289 | } 290 | 291 | fn put_clock(&self, clock: VClock) -> Result<()> { 292 | let clock_key = self.meta_key_bytes(b"clock".to_vec()); 293 | let clock_bytes = bincode::serialize(&clock)?; 294 | self.sled.insert(clock_key, clock_bytes)?; 295 | Ok(()) 296 | } 297 | 298 | fn get_deferred(&self) -> Result, BTreeSet>> { 299 | let deferred_key = self.meta_key_bytes(b"deferred".to_vec()); 300 | if let Some(deferred_bytes) = self.sled.get(&deferred_key)? { 301 | let deferred = bincode::deserialize(&deferred_bytes)?; 302 | Ok(deferred) 303 | } else { 304 | Ok(HashMap::new()) 305 | } 306 | } 307 | 308 | fn put_deferred(&mut self, deferred: HashMap, BTreeSet>) -> Result<()> { 309 | let deferred_key = self.meta_key_bytes(b"deferred".to_vec()); 310 | let deferred_bytes = bincode::serialize(&deferred)?; 311 | self.sled.insert(deferred_key, deferred_bytes)?; 312 | Ok(()) 313 | } 314 | } 315 | 316 | #[cfg(test)] 317 | mod test { 318 | use super::*; 319 | use crdts::{self, map, mvreg, MVReg}; 320 | 321 | type TestActor = u8; 322 | type TestKey = u8; 323 | type TestVal = MVReg; 324 | type TestMap = Map, TestActor>; 325 | 326 | fn mk_map() -> TestMap { 327 | let sled = sled::Config::new().temporary(true).open().unwrap(); 328 | Map::new(sled) 329 | } 330 | 331 | #[test] 332 | fn test_op_exchange_converges_quickcheck1() { 333 | let op_actor1 = Op::Up { 334 | dot: Dot { 335 | actor: 0, 336 | counter: 3, 337 | }, 338 | key: 9, 339 | op: map::Op::Up { 340 | dot: Dot { 341 | actor: 0, 342 | counter: 3, 343 | }, 344 | key: 0, 345 | op: mvreg::Op::Put { 346 | clock: Dot { 347 | actor: 0, 348 | counter: 3, 349 | } 350 | .into(), 351 | val: 0, 352 | }, 353 | }, 354 | }; 355 | let op_1_actor2 = Op::Up { 356 | dot: Dot { 357 | actor: 1, 358 | counter: 1, 359 | }, 360 | key: 9, 361 | op: map::Op::Rm { 362 | clock: Dot { 363 | actor: 1, 364 | counter: 1, 365 | } 366 | .into(), 367 | key: 0, 368 | }, 369 | }; 370 | let op_2_actor2 = Op::Rm { 371 | clock: Dot { 372 | actor: 1, 373 | counter: 2, 374 | } 375 | .into(), 376 | key: 9, 377 | }; 378 | 379 | let mut m1: TestMap = mk_map(); 380 | let mut m2: TestMap = mk_map(); 381 | 382 | m1.apply(&op_actor1); 383 | m2.apply(&op_1_actor2); 384 | m2.apply(&op_2_actor2); 385 | 386 | // m1 <- m2 387 | m1.apply(&op_1_actor2); 388 | m1.apply(&op_2_actor2); 389 | 390 | // m2 <- m1 391 | m2.apply(&op_actor1); 392 | 393 | // m1 <- m2 == m2 <- m1 394 | assert_eq!( 395 | m1.iter().unwrap().map(|e| e.unwrap()).collect::>(), 396 | m2.iter().unwrap().map(|e| e.unwrap()).collect::>() 397 | ); 398 | } 399 | } 400 | -------------------------------------------------------------------------------- /src/git_log.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Debug}; 2 | use std::marker::PhantomData; 3 | use std::str::FromStr; 4 | use std::string::ToString; 5 | 6 | use crdts::{Actor, CmRDT}; 7 | use git2; 8 | use serde_derive::{Deserialize, Serialize}; 9 | 10 | use crate::error::{Error, Result}; 11 | use crate::log::{LogReplicable, TaggedOp}; 12 | 13 | pub struct Log { 14 | actor: A, 15 | repo: git2::Repository, 16 | phantom_crdt: PhantomData, 17 | } 18 | 19 | #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] 20 | pub enum Auth { 21 | None, 22 | UserPass { user: String, pass: String }, 23 | } 24 | 25 | #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] 26 | pub struct Remote { 27 | name: String, 28 | url: String, 29 | auth: Auth, 30 | } 31 | 32 | #[derive(PartialEq, Eq, Serialize, Deserialize)] 33 | pub struct LoggedOp { 34 | actor: A, 35 | oid: Vec, // the object id of the commit with this op 36 | op: C::Op, 37 | } 38 | 39 | impl Debug for LoggedOp 40 | where 41 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 42 | { 43 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 44 | write!( 45 | f, 46 | "LoggedOp {{ actor: {:?}, index: {:?}, op: {:?} }}", 47 | self.actor, 48 | self.id(), 49 | self.op 50 | ) 51 | } 52 | } 53 | 54 | impl TaggedOp for LoggedOp 55 | where 56 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 57 | { 58 | type ID = git2::Oid; 59 | 60 | fn id(&self) -> Self::ID { 61 | git2::Oid::from_bytes(&self.oid).unwrap() 62 | } 63 | 64 | fn op(&self) -> &C::Op { 65 | &self.op 66 | } 67 | } 68 | 69 | impl LoggedOp 70 | where 71 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 72 | { 73 | pub fn actor(&self) -> &A { 74 | &self.actor 75 | } 76 | 77 | fn from_commit(actor: A, repo: &git2::Repository, commit: &git2::Commit) -> Result { 78 | let tree = commit.tree()?; 79 | let tree_entry = tree 80 | .get_name("op") 81 | .ok_or(Error::LogCommitDoesNotContainOp)?; 82 | let id = tree_entry.id(); 83 | let blob = repo.find_blob(id)?; 84 | let bytes = blob.content(); 85 | let op = bincode::deserialize(bytes)?; 86 | let oid = commit.id().as_bytes().to_vec(); 87 | Ok(LoggedOp { actor, oid, op }) 88 | } 89 | 90 | fn next_from_branches( 91 | actor: A, 92 | repo: &git2::Repository, 93 | unacked: Option, 94 | acked: Option, 95 | ) -> Result>> { 96 | match (unacked, acked) { 97 | (Some(unacked), Some(acked)) => { 98 | let local_unacked_oid = unacked 99 | .get() 100 | .target() 101 | .ok_or(Error::BranchIsNotADirectReference)?; 102 | let local_acked_oid = acked 103 | .get() 104 | .target() 105 | .ok_or(Error::BranchIsNotADirectReference)?; 106 | 107 | if local_unacked_oid != local_acked_oid { 108 | let mut curr_oid = local_unacked_oid; 109 | let mut commit; 110 | loop { 111 | commit = repo.find_commit(curr_oid)?; 112 | let parents: Vec = commit.parent_ids().collect(); 113 | assert_eq!(parents.len(), 1); 114 | if parents[0] == local_acked_oid { 115 | break; 116 | } 117 | curr_oid = parents[0]; 118 | } 119 | 120 | let op = LoggedOp::from_commit(actor, &repo, &commit)?; 121 | Ok(Some(op)) 122 | } else { 123 | Ok(None) 124 | } 125 | } 126 | (Some(unacked), None) => { 127 | let mut curr_oid = unacked 128 | .get() 129 | .target() 130 | .ok_or(Error::BranchIsNotADirectReference)?; 131 | let mut commit; 132 | loop { 133 | commit = repo.find_commit(curr_oid)?; 134 | let parents: Vec = commit.parent_ids().collect(); 135 | if parents.is_empty() { 136 | break; 137 | } 138 | 139 | assert_eq!(parents.len(), 1); 140 | curr_oid = parents[0]; 141 | } 142 | 143 | let op = LoggedOp::from_commit(actor, &repo, &commit)?; 144 | Ok(Some(op)) 145 | } 146 | (None, Some(_)) => panic!("we have acked ops that were never unacked!"), 147 | _ => Ok(None), 148 | } 149 | } 150 | } 151 | 152 | impl LogReplicable for Log 153 | where 154 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 155 | A: Actor + ToString + FromStr, 156 | { 157 | type LoggedOp = LoggedOp; 158 | type Remote = Remote; 159 | 160 | fn next(&self) -> Result> { 161 | let local_name = format!("actor_{}", self.actor.to_string()); 162 | let local_acked = format!("acked_actor_{}", self.actor.to_string()); 163 | 164 | let unacked = self.repo.find_branch(&local_name, git2::BranchType::Local); 165 | let acked = self.repo.find_branch(&local_acked, git2::BranchType::Local); 166 | if let Some(op) = 167 | LoggedOp::next_from_branches(self.actor.clone(), &self.repo, unacked.ok(), acked.ok())? 168 | { 169 | return Ok(Some(op)); 170 | } 171 | 172 | // we have no local unacked ops, check for remote ops 173 | for branch in self.repo.branches(Some(git2::BranchType::Remote))? { 174 | let (remote_branch, _) = branch?; 175 | 176 | println!( 177 | "branch name: {}", 178 | remote_branch 179 | .name()? 180 | .ok_or(Error::BranchNameEncodingError)? 181 | ); 182 | 183 | let actor = { 184 | let branch_name = remote_branch 185 | .name()? 186 | .ok_or(Error::BranchNameEncodingError)?; 187 | let split: Vec<&str> = branch_name.split("/actor_").collect(); 188 | println!("branch_name split: {:?}", split); 189 | let actor: A = match split.as_slice() { 190 | [_, s] => s.parse().map_err(|_| { 191 | Error::Parse(format!("Failed to parse actor from branch: {}", s)) 192 | })?, 193 | _ => continue, 194 | }; 195 | println!("actor {:?}", actor.to_string()); 196 | actor 197 | }; 198 | 199 | let tracking_branch = self.repo.find_branch( 200 | &format!("actor_{}", actor.to_string()), 201 | git2::BranchType::Local, 202 | ); 203 | 204 | let next_op = LoggedOp::next_from_branches( 205 | actor, 206 | &self.repo, 207 | Some(remote_branch), 208 | tracking_branch.ok(), 209 | )?; 210 | 211 | if let Some(op) = next_op { 212 | return Ok(Some(op)); 213 | } 214 | } 215 | Ok(None) 216 | } 217 | 218 | fn ack(&mut self, logged_op: &Self::LoggedOp) -> Result<()> { 219 | match self.next()? { 220 | Some(expected) => { 221 | if expected.id() != logged_op.id() { 222 | return Err(Error::State( 223 | "Attempting to ack an op that is not the next op".into(), 224 | )); 225 | } 226 | } 227 | None => { 228 | return Err(Error::State( 229 | "Attempting to ack an op when no op has been committed".into(), 230 | )); 231 | } 232 | } 233 | 234 | let branch_name: String = if logged_op.actor == self.actor { 235 | format!("acked_actor_{}", logged_op.actor.to_string()) 236 | } else { 237 | format!("actor_{}", logged_op.actor.to_string()) 238 | }; 239 | 240 | let commit = self.repo.find_commit(logged_op.id())?; 241 | println!("updating commit on {}, to {:?}", branch_name, commit.id()); 242 | self.repo.branch(&branch_name, &commit, true)?; 243 | Ok(()) 244 | } 245 | 246 | fn commit(&mut self, op: C::Op) -> Result { 247 | let name = format!("actor_{}", self.actor.to_string()); 248 | let parent = match self.repo.find_branch(&name, git2::BranchType::Local) { 249 | Ok(branch) => { 250 | let target = branch 251 | .get() 252 | .target() 253 | .ok_or(Error::BranchIsNotADirectReference)?; 254 | let commit = self.repo.find_commit(target)?; 255 | Some(commit) 256 | } 257 | _ => None, 258 | }; 259 | 260 | let op_bytes = bincode::serialize(&op)?; 261 | let op_oid = self.repo.blob(&op_bytes)?; 262 | let mut builder = self.repo.treebuilder(None)?; 263 | builder.insert("op", op_oid, 0o100_644)?; // TODO: what is this constant? 264 | let tree_oid = builder.write()?; 265 | let tree = self.repo.find_tree(tree_oid)?; 266 | 267 | let sig = self.repo.signature()?; 268 | 269 | let mut parent_commits = Vec::new(); 270 | if let Some(ref commit) = parent { 271 | parent_commits.push(commit) 272 | } 273 | 274 | let branch_ref = format!("refs/heads/{}", name); 275 | println!("committing to branch ref: {}", branch_ref); 276 | 277 | let commit_oid = self.repo.commit( 278 | Some(&branch_ref), 279 | &sig, 280 | &sig, 281 | "db op", 282 | &tree, 283 | &parent_commits, 284 | )?; 285 | 286 | LoggedOp::from_commit( 287 | self.actor.clone(), 288 | &self.repo, 289 | &self.repo.find_commit(commit_oid)?, 290 | ) 291 | } 292 | 293 | fn pull(&mut self, remote: &Self::Remote) -> Result<()> { 294 | println!("fetching remote: {}", &remote.name); 295 | 296 | println!("searching for existing remote in repo"); 297 | let mut git_remote = match self.repo.find_remote(&remote.name) { 298 | Ok(git_remote) => git_remote, 299 | Err(_) => { 300 | eprintln!( 301 | "Failed to find remote '{}', adding remote to git", 302 | remote.name 303 | ); 304 | // this remote is not added to git yet, we add it 305 | self.repo.remote(&remote.name, &remote.url)? 306 | } 307 | }; 308 | 309 | println!("found a remote, starting fetch..."); 310 | 311 | let mut fetch_opt = git2::FetchOptions::new(); 312 | fetch_opt.remote_callbacks(remote.git_callbacks()); 313 | let refspec_iter = git_remote.fetch_refspecs()?; 314 | let refspecs: Vec<&str> = refspec_iter.iter().map(|r| r.unwrap()).collect(); 315 | git_remote.fetch(&refspecs, Some(&mut fetch_opt), None)?; 316 | println!("finished fetch"); 317 | Ok(()) 318 | } 319 | 320 | fn push(&self, remote: &mut Self::Remote) -> Result<()> { 321 | println!("searching for existing remote in repo"); 322 | let mut git_remote = match self.repo.find_remote(&remote.name) { 323 | Ok(git_remote) => git_remote, 324 | Err(_) => { 325 | eprintln!( 326 | "Failed to find remote '{}', adding remote to git", 327 | remote.name 328 | ); 329 | // this remote is not added to git yet, we add it 330 | self.repo.remote(&remote.name, &remote.url)? 331 | } 332 | }; 333 | 334 | let mut push_opt = git2::PushOptions::new(); 335 | push_opt.remote_callbacks(remote.git_callbacks()); 336 | 337 | let branches: Vec = self 338 | .repo 339 | .branches(Some(git2::BranchType::Local))? 340 | .map(|b| b.unwrap()) 341 | .map(|(branch, _)| branch) 342 | .map(|b| { 343 | let b = b.name().unwrap().unwrap(); 344 | format!("refs/heads/{}", b) 345 | }) 346 | .collect(); 347 | 348 | let borrowed: Vec<&str> = branches.iter().map(|s| s.as_ref()).collect(); 349 | 350 | println!("branches to push: {:?}", borrowed); 351 | git_remote.push(&borrowed, Some(&mut push_opt))?; 352 | eprintln!("Finish push"); 353 | Ok(()) 354 | } 355 | } 356 | 357 | impl Log 358 | where 359 | C::Op: serde::Serialize + serde::de::DeserializeOwned, 360 | { 361 | pub fn new(actor: A, repo: git2::Repository) -> Self { 362 | Log { 363 | actor, 364 | repo, 365 | phantom_crdt: PhantomData, 366 | } 367 | } 368 | } 369 | 370 | impl Remote { 371 | pub fn userpass_auth(name: String, url: String, user: String, pass: String) -> Self { 372 | Remote { 373 | name, 374 | url, 375 | auth: Auth::UserPass { user, pass }, 376 | } 377 | } 378 | 379 | pub fn no_auth(name: String, url: String) -> Self { 380 | Remote { 381 | name, 382 | url, 383 | auth: Auth::None, 384 | } 385 | } 386 | 387 | pub fn git_callbacks(&self) -> git2::RemoteCallbacks { 388 | let mut cbs = git2::RemoteCallbacks::new(); 389 | cbs.credentials(move |_, _, _| match self.auth { 390 | Auth::None => panic!("It's a bug if this is ever called!"), 391 | Auth::UserPass { ref user, ref pass } => git2::Cred::userpass_plaintext(user, pass), 392 | }); 393 | cbs 394 | } 395 | } 396 | -------------------------------------------------------------------------------- /art/amanita.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 22 | 24 | 48 | 50 | 51 | 53 | image/svg+xml 54 | 56 | 57 | 58 | 59 | 60 | 65 | 71 | 77 | 83 | 90 | 93 | 101 | 104 | 110 | 116 | 122 | 128 | 134 | 140 | 146 | 152 | 158 | 164 | 165 | 168 | 174 | 180 | 186 | 192 | 198 | 204 | 210 | 216 | 222 | 228 | 229 | 230 | 231 | 232 | -------------------------------------------------------------------------------- /art/george_back_view.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 22 | 24 | 27 | 34 | 35 | 36 | 63 | 65 | 66 | 68 | image/svg+xml 69 | 71 | 72 | 73 | 74 | 75 | 80 | 83 | 90 | 96 | 102 | 110 | 118 | 124 | 132 | 139 | 147 | 154 | 161 | 169 | 177 | 185 | 191 | 199 | 207 | 215 | 223 | 231 | 239 | 247 | 255 | 263 | 271 | 279 | 285 | 293 | 301 | 307 | 313 | 319 | 325 | 333 | 339 | 346 | 347 | 348 | 349 | -------------------------------------------------------------------------------- /tests/map.rs: -------------------------------------------------------------------------------- 1 | use hermitdb::crdts::{map, mvreg, Causal, CmRDT, CvRDT, Dot, MVReg, Map, VClock}; 2 | use quickcheck::{quickcheck, TestResult}; 3 | 4 | type TestActor = u8; 5 | type TestKey = u8; 6 | type TestVal = MVReg; 7 | type TestOp = map::Op, TestActor>; 8 | type TestMap = Map, TestActor>; 9 | 10 | #[derive(Debug, Clone)] 11 | struct OpVec(TestActor, Vec); 12 | 13 | fn build_opvec(prims: (u8, Vec<(u8, u8, u8, u8, u8)>)) -> OpVec { 14 | let (actor, ops_data) = prims; 15 | 16 | let mut ops = Vec::new(); 17 | for (i, op_data) in ops_data.into_iter().enumerate() { 18 | let (choice, inner_choice, key, inner_key, val) = op_data; 19 | let clock: VClock<_> = Dot { 20 | actor, 21 | counter: i as u64, 22 | } 23 | .into(); 24 | 25 | let op = match choice % 3 { 26 | 0 => map::Op::Up { 27 | dot: clock.inc(actor), 28 | key, 29 | op: match inner_choice % 3 { 30 | 0 => map::Op::Up { 31 | dot: clock.inc(actor), 32 | key: inner_key, 33 | op: mvreg::Op::Put { clock, val }, 34 | }, 35 | 1 => map::Op::Rm { 36 | clock, 37 | key: inner_key, 38 | }, 39 | _ => map::Op::Nop, 40 | }, 41 | }, 42 | 1 => map::Op::Rm { clock, key }, 43 | _ => map::Op::Nop, 44 | }; 45 | ops.push(op); 46 | } 47 | OpVec(actor, ops) 48 | } 49 | 50 | #[test] 51 | fn test_new() { 52 | let m: TestMap = Map::new(); 53 | assert_eq!(m.len().val, 0); 54 | } 55 | 56 | #[test] 57 | fn test_update() { 58 | let mut m: TestMap = Map::new(); 59 | 60 | // constructs a default value if does not exist 61 | let ctx = m.get(&101).derive_add_ctx(1); 62 | let op = m.update(101, ctx, |map, ctx| { 63 | map.update(110, ctx, |reg, ctx| reg.write(2, ctx)) 64 | }); 65 | 66 | assert_eq!( 67 | op, 68 | map::Op::Up { 69 | dot: Dot { 70 | actor: 1, 71 | counter: 1 72 | }, 73 | key: 101, 74 | op: map::Op::Up { 75 | dot: Dot { 76 | actor: 1, 77 | counter: 1 78 | }, 79 | key: 110, 80 | op: mvreg::Op::Put { 81 | clock: Dot { 82 | actor: 1, 83 | counter: 1 84 | } 85 | .into(), 86 | val: 2 87 | } 88 | } 89 | } 90 | ); 91 | 92 | assert_eq!(m, Map::new()); 93 | 94 | m.apply(&op); 95 | 96 | assert_eq!( 97 | m.get(&101) 98 | .val 99 | .and_then(|m2| m2.get(&110).val) 100 | .map(|r| r.read().val), 101 | Some(vec![2]) 102 | ); 103 | 104 | // the map should give the latest val to the closure 105 | let op2 = m.update(101, m.get(&101).derive_add_ctx(1), |map, ctx| { 106 | map.update(110, ctx, |reg, ctx| { 107 | assert_eq!(reg.read().val, vec![2]); 108 | reg.write(6, ctx) 109 | }) 110 | }); 111 | m.apply(&op2); 112 | 113 | assert_eq!( 114 | m.get(&101) 115 | .val 116 | .and_then(|m2| m2.get(&110).val) 117 | .map(|r| r.read().val), 118 | Some(vec![6]) 119 | ); 120 | } 121 | 122 | #[test] 123 | fn test_remove() { 124 | let mut m: TestMap = Map::new(); 125 | let add_ctx = m.get(&101).derive_add_ctx(1); 126 | let op = m.update(101, add_ctx.clone(), |m, ctx| { 127 | m.update(110, ctx, |r, ctx| r.write(0, ctx)) 128 | }); 129 | let mut inner_map: Map = Map::new(); 130 | let inner_op = inner_map.update(110, add_ctx, |r, ctx| r.write(0, ctx)); 131 | inner_map.apply(&inner_op); 132 | 133 | m.apply(&op); 134 | 135 | let rm_op = { 136 | let read_ctx = m.get(&101); 137 | assert_eq!(read_ctx.val, Some(inner_map)); 138 | assert_eq!(m.len().val, 1); 139 | let rm_ctx = read_ctx.derive_rm_ctx(); 140 | m.rm(101, rm_ctx) 141 | }; 142 | m.apply(&rm_op); 143 | assert_eq!(m.get(&101).val, None); 144 | assert_eq!(m.len().val, 0); 145 | } 146 | 147 | #[test] 148 | fn test_reset_remove_semantics() { 149 | let mut m1 = TestMap::new(); 150 | let op1 = m1.update(101, m1.get(&101).derive_add_ctx(74), |map, ctx| { 151 | map.update(110, ctx, |reg, ctx| reg.write(32, ctx)) 152 | }); 153 | m1.apply(&op1); 154 | 155 | let mut m2 = m1.clone(); 156 | 157 | let read_ctx = m1.get(&101); 158 | 159 | let op2 = m1.rm(101, read_ctx.derive_rm_ctx()); 160 | m1.apply(&op2); 161 | 162 | let op3 = m2.update(101, m2.get(&101).derive_add_ctx(37), |map, ctx| { 163 | map.update(220, ctx, |reg, ctx| reg.write(5, ctx)) 164 | }); 165 | m2.apply(&op3); 166 | 167 | let m1_snapshot = m1.clone(); 168 | 169 | m1.merge(&m2); 170 | m2.merge(&m1_snapshot); 171 | assert_eq!(m1, m2); 172 | 173 | let inner_map = m1.get(&101).val.unwrap(); 174 | assert_eq!(inner_map.get(&220).val.map(|r| r.read().val), Some(vec![5])); 175 | assert_eq!(inner_map.get(&110).val, None); 176 | assert_eq!(inner_map.len().val, 1); 177 | } 178 | 179 | #[test] 180 | fn test_updating_with_current_clock_should_be_a_nop() { 181 | let mut m1: TestMap = Map::new(); 182 | 183 | m1.apply(&map::Op::Up { 184 | dot: Dot { 185 | actor: 1, 186 | counter: 0, 187 | }, 188 | key: 0, 189 | op: map::Op::Up { 190 | dot: Dot { 191 | actor: 1, 192 | counter: 0, 193 | }, 194 | key: 1, 195 | op: mvreg::Op::Put { 196 | clock: VClock::new(), 197 | val: 235, 198 | }, 199 | }, 200 | }); 201 | 202 | // the update should have been ignored 203 | assert_eq!(m1, Map::new()); 204 | } 205 | 206 | #[test] 207 | fn test_concurrent_update_and_remove_add_bias() { 208 | let mut m1 = TestMap::new(); 209 | let mut m2 = TestMap::new(); 210 | 211 | let op1 = map::Op::Rm { 212 | clock: Dot { 213 | actor: 1, 214 | counter: 1, 215 | } 216 | .into(), 217 | key: 102, 218 | }; 219 | let op2 = m2.update(102, m2.get(&102).derive_add_ctx(2), |_, _| map::Op::Nop); 220 | 221 | m1.apply(&op1); 222 | m2.apply(&op2); 223 | 224 | let mut m1_clone = m1.clone(); 225 | let mut m2_clone = m2.clone(); 226 | 227 | m1_clone.merge(&m2); 228 | m2_clone.merge(&m1); 229 | 230 | assert_eq!(m1_clone, m2_clone); 231 | 232 | m1.apply(&op2); 233 | m2.apply(&op1); 234 | 235 | assert_eq!(m1, m2); 236 | 237 | assert_eq!(m1, m1_clone); 238 | 239 | // we bias towards adds 240 | assert!(m1.get(&102).val.is_some()); 241 | } 242 | 243 | #[test] 244 | fn test_op_exchange_commutes_quickcheck1() { 245 | // THIS WILL NOT PASS IF WE SWAP OUT THE MVREG WITH AN LWWREG 246 | // we need a true causal register 247 | let mut m1: Map, u8> = Map::new(); 248 | let mut m2: Map, u8> = Map::new(); 249 | 250 | let m1_op1 = m1.update(0, m1.get(&0).derive_add_ctx(1), |reg, ctx| { 251 | reg.write(0, ctx) 252 | }); 253 | m1.apply(&m1_op1); 254 | 255 | let m1_op2 = m1.rm(0, m1.get(&0).derive_rm_ctx()); 256 | m1.apply(&m1_op2); 257 | 258 | let m2_op1 = m2.update(0, m2.get(&0).derive_add_ctx(2), |reg, ctx| { 259 | reg.write(0, ctx) 260 | }); 261 | m2.apply(&m2_op1); 262 | 263 | // m1 <- m2 264 | m1.apply(&m2_op1); 265 | 266 | // m2 <- m1 267 | m2.apply(&m1_op1); 268 | m2.apply(&m1_op2); 269 | 270 | assert_eq!(m1, m2); 271 | } 272 | 273 | #[test] 274 | fn test_op_deferred_remove() { 275 | let mut m1: Map, u8> = Map::new(); 276 | let mut m2 = m1.clone(); 277 | let mut m3 = m1.clone(); 278 | 279 | let m1_up1 = m1.update(0, m1.get(&0).derive_add_ctx(1), |reg, ctx| { 280 | reg.write(0, ctx) 281 | }); 282 | m1.apply(&m1_up1); 283 | 284 | let m1_up2 = m1.update(1, m1.get(&1).derive_add_ctx(1), |reg, ctx| { 285 | reg.write(1, ctx) 286 | }); 287 | m1.apply(&m1_up2); 288 | 289 | m2.apply(&m1_up1); 290 | m2.apply(&m1_up2); 291 | 292 | let read_ctx = m2.get(&0); 293 | let m2_rm = m2.rm(0, read_ctx.derive_rm_ctx()); 294 | m2.apply(&m2_rm); 295 | 296 | assert_eq!(m2.get(&0).val, None); 297 | m3.apply(&m2_rm); 298 | m3.apply(&m1_up1); 299 | m3.apply(&m1_up2); 300 | 301 | m1.apply(&m2_rm); 302 | 303 | assert_eq!(m2.get(&0).val, None); 304 | assert_eq!(m3.get(&1).val.map(|r| r.read().val), Some(vec![1])); 305 | 306 | assert_eq!(m2, m3); 307 | assert_eq!(m1, m2); 308 | assert_eq!(m1, m3); 309 | } 310 | 311 | #[test] 312 | fn test_merge_deferred_remove() { 313 | let mut m1 = TestMap::new(); 314 | let mut m2 = TestMap::new(); 315 | let mut m3 = TestMap::new(); 316 | 317 | let m1_up1 = m1.update(0, m1.get(&0).derive_add_ctx(1), |map, ctx| { 318 | map.update(0, ctx, |reg, ctx| reg.write(0, ctx)) 319 | }); 320 | m1.apply(&m1_up1); 321 | 322 | let m1_up2 = m1.update(1, m1.get(&1).derive_add_ctx(1), |map, ctx| { 323 | map.update(1, ctx, |reg, ctx| reg.write(1, ctx)) 324 | }); 325 | m1.apply(&m1_up2); 326 | 327 | m2.apply(&m1_up1); 328 | m2.apply(&m1_up2); 329 | 330 | let m2_rm = m2.rm(0, m2.get(&0).derive_rm_ctx()); 331 | m2.apply(&m2_rm); 332 | 333 | m3.merge(&m2); 334 | m3.merge(&m1); 335 | m1.merge(&m2); 336 | 337 | assert_eq!(m2.get(&0).val, None); 338 | assert_eq!( 339 | m3.get(&1) 340 | .val 341 | .and_then(|inner| inner.get(&1).val) 342 | .map(|r| r.read().val), 343 | Some(vec![1]) 344 | ); 345 | 346 | assert_eq!(m2, m3); 347 | assert_eq!(m1, m2); 348 | assert_eq!(m1, m3); 349 | } 350 | 351 | #[test] 352 | fn test_commute_quickcheck_bug() { 353 | let ops = vec![ 354 | map::Op::Rm { 355 | clock: Dot { 356 | actor: 45, 357 | counter: 1, 358 | } 359 | .into(), 360 | key: 0, 361 | }, 362 | map::Op::Up { 363 | dot: Dot { 364 | actor: 45, 365 | counter: 2, 366 | }, 367 | key: 0, 368 | op: map::Op::Up { 369 | dot: Dot { 370 | actor: 45, 371 | counter: 1, 372 | }, 373 | key: 0, 374 | op: mvreg::Op::Put { 375 | clock: VClock::new(), 376 | val: 0, 377 | }, 378 | }, 379 | }, 380 | ]; 381 | 382 | let mut m = Map::new(); 383 | apply_ops(&mut m, &ops); 384 | 385 | let m_snapshot = m.clone(); 386 | 387 | let mut empty_m = Map::new(); 388 | m.merge(&empty_m); 389 | empty_m.merge(&m_snapshot); 390 | 391 | assert_eq!(m, empty_m); 392 | } 393 | 394 | #[test] 395 | fn test_idempotent_quickcheck_bug1() { 396 | let ops = vec![ 397 | map::Op::Up { 398 | dot: Dot { 399 | actor: 21, 400 | counter: 5, 401 | }, 402 | key: 0, 403 | op: map::Op::Nop, 404 | }, 405 | map::Op::Up { 406 | dot: Dot { 407 | actor: 21, 408 | counter: 6, 409 | }, 410 | key: 1, 411 | op: map::Op::Up { 412 | dot: Dot { 413 | actor: 21, 414 | counter: 1, 415 | }, 416 | key: 0, 417 | op: mvreg::Op::Put { 418 | clock: VClock::new(), 419 | val: 0, 420 | }, 421 | }, 422 | }, 423 | ]; 424 | 425 | let mut m = Map::new(); 426 | apply_ops(&mut m, &ops); 427 | 428 | let m_snapshot = m.clone(); 429 | m.merge(&m_snapshot); 430 | 431 | assert_eq!(m, m_snapshot); 432 | } 433 | 434 | #[test] 435 | fn test_idempotent_quickcheck_bug2() { 436 | let mut m: TestMap = Map::new(); 437 | m.apply(&map::Op::Up { 438 | dot: Dot { 439 | actor: 32, 440 | counter: 5, 441 | }, 442 | key: 0, 443 | op: map::Op::Up { 444 | dot: Dot { 445 | actor: 32, 446 | counter: 5, 447 | }, 448 | key: 0, 449 | op: mvreg::Op::Put { 450 | clock: VClock::new(), 451 | val: 0, 452 | }, 453 | }, 454 | }); 455 | 456 | let m_snapshot = m.clone(); 457 | 458 | // m ^ m 459 | m.merge(&m_snapshot); 460 | 461 | // m ^ m = m 462 | assert_eq!(m, m_snapshot); 463 | } 464 | 465 | #[test] 466 | fn test_nop_on_new_map_should_remain_a_new_map() { 467 | let mut map = TestMap::new(); 468 | map.apply(&map::Op::Nop); 469 | assert_eq!(map, TestMap::new()); 470 | } 471 | 472 | #[test] 473 | fn test_op_exchange_same_as_merge_quickcheck1() { 474 | let op1 = map::Op::Up { 475 | dot: Dot { 476 | actor: 38, 477 | counter: 4, 478 | }, 479 | key: 216, 480 | op: map::Op::Nop, 481 | }; 482 | let op2 = map::Op::Up { 483 | dot: Dot { 484 | actor: 91, 485 | counter: 9, 486 | }, 487 | key: 216, 488 | op: map::Op::Up { 489 | dot: Dot { 490 | actor: 91, 491 | counter: 1, 492 | }, 493 | key: 37, 494 | op: mvreg::Op::Put { 495 | clock: Dot { 496 | actor: 91, 497 | counter: 1, 498 | } 499 | .into(), 500 | val: 94, 501 | }, 502 | }, 503 | }; 504 | let mut m1: TestMap = Map::new(); 505 | let mut m2: TestMap = Map::new(); 506 | m1.apply(&op1); 507 | m2.apply(&op2); 508 | 509 | let mut m1_merge = m1.clone(); 510 | m1_merge.merge(&m2); 511 | 512 | let mut m2_merge = m2.clone(); 513 | m2_merge.merge(&m1); 514 | 515 | m1.apply(&op2); 516 | m2.apply(&op1); 517 | 518 | assert_eq!(m1, m2); 519 | assert_eq!(m1_merge, m2_merge); 520 | assert_eq!(m1, m1_merge); 521 | assert_eq!(m2, m2_merge); 522 | assert_eq!(m1, m2_merge); 523 | assert_eq!(m2, m1_merge); 524 | } 525 | 526 | #[test] 527 | fn test_idempotent_quickcheck1() { 528 | let ops = vec![ 529 | map::Op::Up { 530 | dot: Dot { 531 | actor: 62, 532 | counter: 9, 533 | }, 534 | key: 47, 535 | op: map::Op::Up { 536 | dot: Dot { 537 | actor: 62, 538 | counter: 1, 539 | }, 540 | key: 65, 541 | op: mvreg::Op::Put { 542 | clock: Dot { 543 | actor: 62, 544 | counter: 1, 545 | } 546 | .into(), 547 | val: 240, 548 | }, 549 | }, 550 | }, 551 | map::Op::Up { 552 | dot: Dot { 553 | actor: 62, 554 | counter: 11, 555 | }, 556 | key: 60, 557 | op: map::Op::Up { 558 | dot: Dot { 559 | actor: 62, 560 | counter: 1, 561 | }, 562 | key: 193, 563 | op: mvreg::Op::Put { 564 | clock: Dot { 565 | actor: 62, 566 | counter: 1, 567 | } 568 | .into(), 569 | val: 28, 570 | }, 571 | }, 572 | }, 573 | ]; 574 | let mut m: TestMap = Map::new(); 575 | apply_ops(&mut m, &ops); 576 | let m_snapshot = m.clone(); 577 | 578 | // m ^ m 579 | m.merge(&m_snapshot); 580 | 581 | // m ^ m = m 582 | assert_eq!(m, m_snapshot); 583 | } 584 | 585 | fn apply_ops(map: &mut TestMap, ops: &[TestOp]) { 586 | for op in ops.iter().cloned() { 587 | map.apply(&op); 588 | } 589 | } 590 | 591 | quickcheck! { 592 | // TODO: add test to show equivalence of merge and Op exchange 593 | fn prop_op_exchange_same_as_merge( 594 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 595 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 596 | ) -> TestResult { 597 | let ops1 = build_opvec(ops1_prim); 598 | let ops2 = build_opvec(ops2_prim); 599 | 600 | if ops1.0 == ops2.0 { 601 | return TestResult::discard(); 602 | } 603 | 604 | let mut m1: TestMap = Map::new(); 605 | let mut m2: TestMap = Map::new(); 606 | 607 | apply_ops(&mut m1, &ops1.1); 608 | apply_ops(&mut m2, &ops2.1); 609 | 610 | let mut m_merged = m1.clone(); 611 | m_merged.merge(&m2); 612 | 613 | apply_ops(&mut m1, &ops2.1); 614 | apply_ops(&mut m2, &ops1.1); 615 | 616 | TestResult::from_bool(m1 == m_merged && m2 == m_merged) 617 | } 618 | 619 | fn prop_op_exchange_converges( 620 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 621 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 622 | ) -> TestResult { 623 | let ops1 = build_opvec(ops1_prim); 624 | let ops2 = build_opvec(ops2_prim); 625 | 626 | if ops1.0 == ops2.0 { 627 | return TestResult::discard(); 628 | } 629 | 630 | let mut m1: TestMap = Map::new(); 631 | let mut m2: TestMap = Map::new(); 632 | 633 | apply_ops(&mut m1, &ops1.1); 634 | apply_ops(&mut m2, &ops2.1); 635 | 636 | // m1 <- m2 637 | apply_ops(&mut m1, &ops2.1); 638 | 639 | // m2 <- m1 640 | apply_ops(&mut m2, &ops1.1); 641 | 642 | // m1 <- m2 == m2 <- m1 643 | assert_eq!(m1, m2); 644 | TestResult::from_bool(true) 645 | } 646 | 647 | fn prop_op_exchange_associative( 648 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 649 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 650 | ops3_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 651 | ) -> TestResult { 652 | let ops1 = build_opvec(ops1_prim); 653 | let ops2 = build_opvec(ops2_prim); 654 | let ops3 = build_opvec(ops3_prim); 655 | 656 | if ops1.0 == ops2.0 || ops1.0 == ops3.0 || ops2.0 == ops3.0 { 657 | return TestResult::discard(); 658 | } 659 | 660 | let mut m1: TestMap = Map::new(); 661 | let mut m2: TestMap = Map::new(); 662 | let mut m3: TestMap = Map::new(); 663 | 664 | apply_ops(&mut m1, &ops1.1); 665 | apply_ops(&mut m2, &ops2.1); 666 | apply_ops(&mut m3, &ops3.1); 667 | 668 | // (m1 <- m2) <- m3 669 | apply_ops(&mut m1, &ops2.1); 670 | apply_ops(&mut m1, &ops3.1); 671 | 672 | // (m2 <- m3) <- m1 673 | apply_ops(&mut m2, &ops3.1); 674 | apply_ops(&mut m2, &ops1.1); 675 | 676 | // (m1 <- m2) <- m3 = (m2 <- m3) <- m1 677 | TestResult::from_bool(m1 == m2) 678 | } 679 | 680 | fn prop_op_idempotent( 681 | ops_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 682 | ) -> bool { 683 | let ops = build_opvec(ops_prim); 684 | let mut m = TestMap::new(); 685 | 686 | apply_ops(&mut m, &ops.1); 687 | let m_snapshot = m.clone(); 688 | apply_ops(&mut m, &ops.1); 689 | 690 | m == m_snapshot 691 | } 692 | 693 | fn prop_op_associative( 694 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 695 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 696 | ops3_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 697 | ) -> TestResult { 698 | let ops1 = build_opvec(ops1_prim); 699 | let ops2 = build_opvec(ops2_prim); 700 | let ops3 = build_opvec(ops3_prim); 701 | 702 | if ops1.0 == ops2.0 || ops1.0 == ops3.0 || ops2.0 == ops3.0 { 703 | return TestResult::discard(); 704 | } 705 | 706 | let mut m1: TestMap = Map::new(); 707 | let mut m2: TestMap = Map::new(); 708 | let mut m3: TestMap = Map::new(); 709 | 710 | apply_ops(&mut m1, &ops1.1); 711 | apply_ops(&mut m2, &ops2.1); 712 | apply_ops(&mut m3, &ops3.1); 713 | 714 | // (m1 <- m2) <- m3 715 | apply_ops(&mut m1, &ops2.1); 716 | apply_ops(&mut m1, &ops3.1); 717 | 718 | // (m2 <- m3) <- m1 719 | apply_ops(&mut m2, &ops3.1); 720 | apply_ops(&mut m2, &ops1.1); 721 | 722 | // (m1 ^ m2) ^ m3 = m1 ^ (m2 ^ m3) 723 | TestResult::from_bool(m1 == m2) 724 | } 725 | 726 | 727 | fn prop_merge_associative( 728 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 729 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 730 | ops3_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 731 | ) -> TestResult { 732 | let ops1 = build_opvec(ops1_prim); 733 | let ops2 = build_opvec(ops2_prim); 734 | let ops3 = build_opvec(ops3_prim); 735 | if ops1.0 == ops2.0 || ops1.0 == ops3.0 || ops2.0 == ops3.0 { 736 | return TestResult::discard(); 737 | } 738 | 739 | let mut m1: TestMap = Map::new(); 740 | let mut m2: TestMap = Map::new(); 741 | let mut m3: TestMap = Map::new(); 742 | 743 | apply_ops(&mut m1, &ops1.1); 744 | apply_ops(&mut m2, &ops2.1); 745 | apply_ops(&mut m3, &ops3.1); 746 | 747 | let mut m1_snapshot = m1.clone(); 748 | 749 | // (m1 ^ m2) ^ m3 750 | m1.merge(&m2); 751 | m1.merge(&m3); 752 | 753 | // m1 ^ (m2 ^ m3) 754 | m2.merge(&m3); 755 | m1_snapshot.merge(&m2); 756 | 757 | // (m1 ^ m2) ^ m3 = m1 ^ (m2 ^ m3) 758 | TestResult::from_bool(m1 == m1_snapshot) 759 | } 760 | 761 | fn prop_merge_commutative( 762 | ops1_prim: (u8, Vec<(u8, u8, u8, u8, u8)>), 763 | ops2_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 764 | ) -> TestResult { 765 | let ops1 = build_opvec(ops1_prim); 766 | let ops2 = build_opvec(ops2_prim); 767 | 768 | if ops1.0 == ops2.0 { 769 | return TestResult::discard(); 770 | } 771 | 772 | let mut m1: TestMap = Map::new(); 773 | let mut m2: TestMap = Map::new(); 774 | 775 | apply_ops(&mut m1, &ops1.1); 776 | apply_ops(&mut m2, &ops2.1); 777 | 778 | let m1_snapshot = m1.clone(); 779 | // m1 ^ m2 780 | m1.merge(&m2); 781 | 782 | // m2 ^ m1 783 | m2.merge(&m1_snapshot); 784 | 785 | // m1 ^ m2 = m2 ^ m1 786 | TestResult::from_bool(m1 == m2) 787 | } 788 | 789 | fn prop_merge_idempotent( 790 | ops_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 791 | ) -> bool { 792 | let ops = build_opvec(ops_prim); 793 | 794 | let mut m: TestMap = Map::new(); 795 | apply_ops(&mut m, &ops.1); 796 | let m_snapshot = m.clone(); 797 | 798 | // m ^ m 799 | m.merge(&m_snapshot); 800 | 801 | // m ^ m = m 802 | m == m_snapshot 803 | } 804 | 805 | fn prop_truncate_with_empty_vclock_is_nop( 806 | ops_prim: (u8, Vec<(u8, u8, u8, u8, u8)>) 807 | ) -> bool { 808 | let ops = build_opvec(ops_prim); 809 | 810 | let mut m: TestMap = Map::new(); 811 | apply_ops(&mut m, &ops.1); 812 | 813 | let m_snapshot = m.clone(); 814 | m.truncate(&VClock::new()); 815 | 816 | m == m_snapshot 817 | } 818 | } 819 | -------------------------------------------------------------------------------- /art/explain_replication.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 22 | 30 | 36 | 37 | 45 | 51 | 52 | 60 | 66 | 67 | 75 | 81 | 82 | 90 | 96 | 97 | 105 | 109 | 114 | 119 | 124 | 129 | 134 | 139 | 140 | 141 | 149 | 155 | 156 | 164 | 170 | 171 | 174 | 178 | 179 | 182 | 188 | 189 | 190 | 213 | 215 | 216 | 218 | image/svg+xml 219 | 221 | 222 | 223 | 224 | 225 | 230 | 232 | 235 | 237 | 243 | 249 | 250 | 253 | 261 | 267 | 268 | 269 | 271 | 274 | 280 | 286 | 287 | 290 | 298 | 304 | 312 | 313 | 314 | 317 | 320 | 326 | 332 | 333 | 336 | 341 | 347 | 355 | 363 | 371 | 372 | 373 | 374 | 377 | 380 | 388 | 394 | 395 | 398 | 404 | 410 | 411 | 414 | 420 | 426 | 427 | 430 | 436 | 442 | 443 | 444 | 450 | 456 | 462 | 469 | 472 | 475 | 483 | db.sync(); 494 | 143 505 | 510 | 511 | 512 | 513 | 514 | -------------------------------------------------------------------------------- /art/explain_replay.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 22 | 30 | 33 | 39 | 45 | 51 | 52 | 53 | 61 | 67 | 68 | 76 | 80 | 85 | 90 | 95 | 100 | 105 | 110 | 111 | 112 | 120 | 126 | 127 | 135 | 141 | 142 | 150 | 156 | 157 | 165 | 171 | 172 | 180 | 186 | 187 | 195 | 199 | 204 | 209 | 214 | 219 | 224 | 229 | 230 | 231 | 239 | 245 | 246 | 254 | 260 | 261 | 264 | 268 | 269 | 272 | 278 | 279 | 280 | 303 | 305 | 306 | 308 | image/svg+xml 309 | 311 | 312 | 313 | 314 | 315 | 320 | 323 | 332 | 337 | 343 | 350 | 356 | RM("x", {...}) 367 | UPDATE("y", {...}) 378 | 383 | 388 | 393 | 394 | 397 | 406 | 412 | 418 | 421 | 429 | "x" => 32.01 440 | 441 | 444 | 452 | "y" => 278.9 463 | 464 | 468 | 477 | 486 | 487 | 490 | 498 | "v" => 0.15 509 | 510 | 511 | 517 | 518 | 519 | --------------------------------------------------------------------------------