├── .gitignore ├── Cargo.lock ├── Cargo.toml └── src ├── dht_arc.rs └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "ahash" 5 | version = "0.4.7" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" 8 | 9 | [[package]] 10 | name = "autocfg" 11 | version = "1.0.1" 12 | source = "registry+https://github.com/rust-lang/crates.io-index" 13 | checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" 14 | 15 | [[package]] 16 | name = "bitflags" 17 | version = "1.2.1" 18 | source = "registry+https://github.com/rust-lang/crates.io-index" 19 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 20 | 21 | [[package]] 22 | name = "byteorder" 23 | version = "1.4.3" 24 | source = "registry+https://github.com/rust-lang/crates.io-index" 25 | checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" 26 | 27 | [[package]] 28 | name = "cfg-if" 29 | version = "1.0.0" 30 | source = "registry+https://github.com/rust-lang/crates.io-index" 31 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 32 | 33 | [[package]] 34 | name = "fallible-iterator" 35 | version = "0.2.0" 36 | source = "registry+https://github.com/rust-lang/crates.io-index" 37 | checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" 38 | 39 | [[package]] 40 | name = "fallible-streaming-iterator" 41 | version = "0.1.9" 42 | source = "registry+https://github.com/rust-lang/crates.io-index" 43 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 44 | 45 | [[package]] 46 | name = "getrandom" 47 | version = "0.2.2" 48 | source = "registry+https://github.com/rust-lang/crates.io-index" 49 | checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" 50 | dependencies = [ 51 | "cfg-if", 52 | "libc", 53 | "wasi", 54 | ] 55 | 56 | [[package]] 57 | name = "hashbrown" 58 | version = "0.9.1" 59 | source = "registry+https://github.com/rust-lang/crates.io-index" 60 | checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" 61 | dependencies = [ 62 | "ahash", 63 | ] 64 | 65 | [[package]] 66 | name = "hashlink" 67 | version = "0.6.0" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" 70 | dependencies = [ 71 | "hashbrown", 72 | ] 73 | 74 | [[package]] 75 | name = "libc" 76 | version = "0.2.94" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" 79 | 80 | [[package]] 81 | name = "libsqlite3-sys" 82 | version = "0.22.1" 83 | source = "registry+https://github.com/rust-lang/crates.io-index" 84 | checksum = "19cb1effde5f834799ac5e5ef0e40d45027cd74f271b1de786ba8abb30e2164d" 85 | dependencies = [ 86 | "pkg-config", 87 | "vcpkg", 88 | ] 89 | 90 | [[package]] 91 | name = "memchr" 92 | version = "2.4.0" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" 95 | 96 | [[package]] 97 | name = "num-traits" 98 | version = "0.2.14" 99 | source = "registry+https://github.com/rust-lang/crates.io-index" 100 | checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" 101 | dependencies = [ 102 | "autocfg", 103 | ] 104 | 105 | [[package]] 106 | name = "pkg-config" 107 | version = "0.3.19" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" 110 | 111 | [[package]] 112 | name = "ppv-lite86" 113 | version = "0.2.10" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" 116 | 117 | [[package]] 118 | name = "proc-macro2" 119 | version = "1.0.26" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" 122 | dependencies = [ 123 | "unicode-xid", 124 | ] 125 | 126 | [[package]] 127 | name = "quote" 128 | version = "1.0.9" 129 | source = "registry+https://github.com/rust-lang/crates.io-index" 130 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 131 | dependencies = [ 132 | "proc-macro2", 133 | ] 134 | 135 | [[package]] 136 | name = "rand" 137 | version = "0.8.3" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" 140 | dependencies = [ 141 | "libc", 142 | "rand_chacha", 143 | "rand_core", 144 | "rand_hc", 145 | ] 146 | 147 | [[package]] 148 | name = "rand_chacha" 149 | version = "0.3.0" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" 152 | dependencies = [ 153 | "ppv-lite86", 154 | "rand_core", 155 | ] 156 | 157 | [[package]] 158 | name = "rand_core" 159 | version = "0.6.2" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" 162 | dependencies = [ 163 | "getrandom", 164 | ] 165 | 166 | [[package]] 167 | name = "rand_hc" 168 | version = "0.3.0" 169 | source = "registry+https://github.com/rust-lang/crates.io-index" 170 | checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" 171 | dependencies = [ 172 | "rand_core", 173 | ] 174 | 175 | [[package]] 176 | name = "rmp" 177 | version = "0.8.10" 178 | source = "registry+https://github.com/rust-lang/crates.io-index" 179 | checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" 180 | dependencies = [ 181 | "byteorder", 182 | "num-traits", 183 | ] 184 | 185 | [[package]] 186 | name = "rmp-serde" 187 | version = "0.15.4" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "839395ef53057db96b84c9238ab29e1a13f2e5c8ec9f66bef853ab4197303924" 190 | dependencies = [ 191 | "byteorder", 192 | "rmp", 193 | "serde", 194 | ] 195 | 196 | [[package]] 197 | name = "rusqlite" 198 | version = "0.25.1" 199 | source = "registry+https://github.com/rust-lang/crates.io-index" 200 | checksum = "fbc783b7ddae608338003bac1fa00b6786a75a9675fbd8e87243ecfdea3f6ed2" 201 | dependencies = [ 202 | "bitflags", 203 | "fallible-iterator", 204 | "fallible-streaming-iterator", 205 | "hashlink", 206 | "libsqlite3-sys", 207 | "memchr", 208 | "smallvec", 209 | ] 210 | 211 | [[package]] 212 | name = "serde" 213 | version = "1.0.125" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" 216 | dependencies = [ 217 | "serde_derive", 218 | ] 219 | 220 | [[package]] 221 | name = "serde_derive" 222 | version = "1.0.125" 223 | source = "registry+https://github.com/rust-lang/crates.io-index" 224 | checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" 225 | dependencies = [ 226 | "proc-macro2", 227 | "quote", 228 | "syn", 229 | ] 230 | 231 | [[package]] 232 | name = "smallvec" 233 | version = "1.6.1" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" 236 | 237 | [[package]] 238 | name = "spike-arc-sql" 239 | version = "0.1.0" 240 | dependencies = [ 241 | "rand", 242 | "rmp-serde", 243 | "rusqlite", 244 | "serde", 245 | ] 246 | 247 | [[package]] 248 | name = "syn" 249 | version = "1.0.72" 250 | source = "registry+https://github.com/rust-lang/crates.io-index" 251 | checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" 252 | dependencies = [ 253 | "proc-macro2", 254 | "quote", 255 | "unicode-xid", 256 | ] 257 | 258 | [[package]] 259 | name = "unicode-xid" 260 | version = "0.2.2" 261 | source = "registry+https://github.com/rust-lang/crates.io-index" 262 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 263 | 264 | [[package]] 265 | name = "vcpkg" 266 | version = "0.2.12" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" 269 | 270 | [[package]] 271 | name = "wasi" 272 | version = "0.10.2+wasi-snapshot-preview1" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" 275 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "spike-arc-sql" 3 | version = "0.1.0" 4 | authors = ["neonphog "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | rand = "0.8.3" 9 | rmp-serde = "0.15.4" 10 | rusqlite = { version = "0.25.1", features = ["sqlcipher"] } 11 | serde = { version = "1", features = ["derive", "rc"] } 12 | -------------------------------------------------------------------------------- /src/dht_arc.rs: -------------------------------------------------------------------------------- 1 | //! A type for indicating ranges on the dht arc 2 | 3 | use std::num::Wrapping; 4 | use std::ops::Bound; 5 | use std::ops::RangeBounds; 6 | 7 | #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq)] 8 | /// Type for representing a location that can wrap around 9 | /// a u32 dht arc 10 | pub struct DhtLocation(pub Wrapping); 11 | 12 | impl From> for DhtLocation { 13 | fn from(f: Wrapping) -> Self { 14 | Self(f) 15 | } 16 | } 17 | 18 | /// The maximum you can hold either side of the hash location 19 | /// is half the circle. 20 | /// This is half of the furthest index you can hold 21 | /// 1 is added for rounding 22 | /// 1 more is added to represent the middle point of an odd length array 23 | pub const MAX_HALF_LENGTH: u32 = (u32::MAX / 2) + 1 + 1; 24 | 25 | /// Maximum number of values that a u32 can represent. 26 | const U32_LEN: u64 = u32::MAX as u64 + 1; 27 | 28 | /// Number of copies of a given hash available at any given time. 29 | const REDUNDANCY_TARGET: usize = 50; 30 | 31 | /// If the redundancy drops due to inaccurate estimation we can't 32 | /// go lower then this level of redundancy. 33 | /// Note this can only be tested and not proved. 34 | const REDUNDANCY_FLOOR: usize = 20; 35 | 36 | /// Default assumed up time for nodes. 37 | const DEFAULT_UPTIME: f64 = 0.5; 38 | 39 | /// The minimum number of peers before sharding can begin. 40 | /// This factors in the expected uptime to reach the redundancy target. 41 | pub const MIN_PEERS: usize = (REDUNDANCY_TARGET as f64 / DEFAULT_UPTIME) as usize; 42 | 43 | /// The minimum number of peers we can consider acceptable to see in our arc 44 | /// during testing. 45 | pub const MIN_REDUNDANCY: usize = (REDUNDANCY_FLOOR as f64 / DEFAULT_UPTIME) as usize; 46 | 47 | /// The amount "change in arc" is scaled to prevent rapid changes. 48 | /// This also represents the maximum coverage change in a single update 49 | /// as a difference of 1.0 would scale to 0.2. 50 | const DELTA_SCALE: f64 = 0.2; 51 | 52 | /// The minimal "change in arc" before we stop scaling. 53 | /// This prevents never reaching the target arc coverage. 54 | const DELTA_THRESHOLD: f64 = 0.01; 55 | 56 | /// Due to estimation noise we don't want a very small difference 57 | /// between observed coverage and estimated coverage to 58 | /// amplify when scaled to by the estimated total peers. 59 | /// This threshold must be reached before an estimated coverage gap 60 | /// is calculated. 61 | const NOISE_THRESHOLD: f64 = 0.01; 62 | 63 | // TODO: Use the [`f64::clamp`] when we switch to rustc 1.50 64 | fn clamp(min: f64, max: f64, mut x: f64) -> f64 { 65 | if x < min { 66 | x = min; 67 | } 68 | if x > max { 69 | x = max; 70 | } 71 | x 72 | } 73 | 74 | /// The ideal coverage if all peers were holding the same sized 75 | /// arcs and our estimated total peers is close. 76 | fn coverage_target(est_total_peers: usize) -> f64 { 77 | if est_total_peers <= REDUNDANCY_TARGET { 78 | 1.0 79 | } else { 80 | REDUNDANCY_TARGET as f64 / est_total_peers as f64 81 | } 82 | } 83 | 84 | /// Calculate the target arc length given a peer density. 85 | fn target(density: PeerDensity) -> f64 { 86 | // Get the estimated coverage gap based on our observed peer density. 87 | let est_gap = density.est_gap(); 88 | // If we haven't observed at least our redundancy target number 89 | // of peers (adjusted for expected uptime) then we know that the data 90 | // in our arc is under replicated and we should start aiming for full coverage. 91 | if density.expected_count() < REDUNDANCY_TARGET { 92 | 1.0 93 | } else { 94 | // Get the estimated gap. We don't care about negative gaps 95 | // or gaps we can't fill (> 1.0) 96 | let est_gap = clamp(0.0, 1.0, est_gap); 97 | // Get the ideal coverage target for the size of that we estimate 98 | // the network to be. 99 | let ideal_target = coverage_target(density.est_total_peers()); 100 | // Take whichever is larger. We prefer nodes to target the ideal 101 | // coverage but if there is a larger gap then it needs to be filled. 102 | let target = est_gap.max(ideal_target); 103 | 104 | clamp(0.0, 1.0, target) 105 | } 106 | } 107 | 108 | /// The convergence algorithm that moves an arc towards 109 | /// our estimated target. 110 | /// 111 | /// Note the rate of convergence is dependant of the rate 112 | /// that [`DhtArc::update_length`] is called. 113 | fn converge(current: f64, density: PeerDensity) -> f64 { 114 | let target = target(density); 115 | // The change in arc we'd need to make to get to the target. 116 | let delta = target - current; 117 | // If this is below our threshold then apply that delta. 118 | if delta.abs() < DELTA_THRESHOLD { 119 | current + delta 120 | // Other wise scale the delta to avoid rapid change. 121 | } else { 122 | current + (delta * DELTA_SCALE) 123 | } 124 | } 125 | 126 | #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq)] 127 | /// Represents how much of a dht arc is held 128 | /// center_loc is where the hash is. 129 | /// The center_loc is the center of the arc 130 | /// The half length is the length of items held 131 | /// from the center in both directions 132 | /// half_length 0 means nothing is held 133 | /// half_length 1 means just the center_loc is held 134 | /// half_length n where n > 1 will hold those positions out 135 | /// half_length u32::MAX / 2 + 1 covers all positions 136 | /// on either side of center_loc. 137 | /// Imagine an bidirectional array: 138 | /// ```text 139 | /// [4][3][2][1][0][1][2][3][4] 140 | // half length of 3 will give you 141 | /// [2][1][0][1][2] 142 | /// ``` 143 | pub struct DhtArc { 144 | /// The center location of this dht arc 145 | pub center_loc: DhtLocation, 146 | 147 | /// The "half-length" of this dht arc 148 | pub half_length: u32, 149 | } 150 | 151 | #[derive(Debug, Clone, Copy)] 152 | /// The average density of peers at a location in the u32 space. 153 | pub struct PeerDensity { 154 | /// The arc that filtered the bucket that generated this density. 155 | filter: DhtArc, 156 | /// The average coverage of peers in the bucket. 157 | average_coverage: f64, 158 | /// The number of peers in the bucket. 159 | count: usize, 160 | } 161 | 162 | /// When sampling a section of the arc we can 163 | /// collect all the other peer [`DhtArc`]s into a 164 | /// DhtBucket. 165 | /// All the peer arcs arc contained within the buckets filter arc. 166 | /// The filter is this peer's "view" into their section of the dht arc. 167 | pub struct DhtArcBucket { 168 | /// The arc used to filter this bucket. 169 | filter: DhtArc, 170 | /// The arcs in this bucket. 171 | arcs: Vec, 172 | } 173 | 174 | impl DhtArc { 175 | /// Create an Arc from a hash location plus a length on either side 176 | /// half length is (0..(u32::Max / 2 + 1)) 177 | pub fn new>(center_loc: I, half_length: u32) -> Self { 178 | let half_length = std::cmp::min(half_length, MAX_HALF_LENGTH); 179 | Self { 180 | center_loc: center_loc.into(), 181 | half_length, 182 | } 183 | } 184 | 185 | /// Update the half length based on a density reading. 186 | /// This will converge on a new target instead of jumping directly 187 | /// to the new target and is designed to be called at a given rate 188 | /// with more recent peer density readings. 189 | pub fn update_length(&mut self, density: PeerDensity) { 190 | self.half_length = (MAX_HALF_LENGTH as f64 * converge(self.coverage(), density)) as u32; 191 | } 192 | 193 | /// Check if a location is contained in this arc 194 | pub fn contains>(&self, other_location: I) -> bool { 195 | let other_location = other_location.into(); 196 | let do_hold_something = self.half_length != 0; 197 | let only_hold_self = self.half_length == 1 && self.center_loc == other_location; 198 | // Add one to convert to "array length" from math distance 199 | let dist_as_array_len = shortest_arc_distance(self.center_loc, other_location.0) + 1; 200 | // Check for any other dist and the special case of the maximum array len 201 | let within_range = self.half_length > 1 && dist_as_array_len <= self.half_length; 202 | // Have to hold something and hold ourself or something within range 203 | do_hold_something && (only_hold_self || within_range) 204 | } 205 | 206 | /// Get the range of the arc 207 | pub fn range(&self) -> ArcRange { 208 | if self.half_length == 0 { 209 | ArcRange { 210 | start: Bound::Excluded(self.center_loc.into()), 211 | end: Bound::Excluded(self.center_loc.into()), 212 | } 213 | } else if self.half_length == 1 { 214 | ArcRange { 215 | start: Bound::Included(self.center_loc.into()), 216 | end: Bound::Included(self.center_loc.into()), 217 | } 218 | // In order to make sure the arc covers the full range we need some overlap at the 219 | // end to account for division rounding. 220 | } else if self.half_length == MAX_HALF_LENGTH || self.half_length == MAX_HALF_LENGTH - 1 { 221 | ArcRange { 222 | start: Bound::Included( 223 | (self.center_loc.0 - DhtLocation::from(MAX_HALF_LENGTH - 1).0).0, 224 | ), 225 | end: Bound::Included( 226 | (self.center_loc.0 + DhtLocation::from(MAX_HALF_LENGTH).0 - Wrapping(2)).0, 227 | ), 228 | } 229 | } else { 230 | ArcRange { 231 | start: Bound::Included( 232 | (self.center_loc.0 - DhtLocation::from(self.half_length - 1).0).0, 233 | ), 234 | end: Bound::Included( 235 | (self.center_loc.0 + DhtLocation::from(self.half_length).0 - Wrapping(1)).0, 236 | ), 237 | } 238 | } 239 | } 240 | 241 | /// The absolute length that this arc will hold. 242 | pub fn absolute_length(&self) -> u64 { 243 | self.range().len() 244 | } 245 | 246 | /// The percentage of the full circle that is covered 247 | /// by this arc. 248 | pub fn coverage(&self) -> f64 { 249 | self.absolute_length() as f64 / U32_LEN as f64 250 | } 251 | } 252 | 253 | impl PeerDensity { 254 | /// Create a new peer density reading from the: 255 | /// - The filter used to create the bucket. 256 | /// - Average coverage of all peers in the bucket. 257 | /// - Count of peers in the bucket. 258 | pub fn new(filter: DhtArc, average_coverage: f64, count: usize) -> Self { 259 | Self { 260 | filter, 261 | average_coverage, 262 | count, 263 | } 264 | } 265 | 266 | /// The expected number of peers for this arc over time. 267 | pub fn expected_count(&self) -> usize { 268 | (self.count as f64 * DEFAULT_UPTIME) as usize 269 | } 270 | 271 | /// Estimate the gap in coverage that needs to be filled. 272 | /// If the gap is negative that means we are over covered. 273 | pub fn est_gap(&self) -> f64 { 274 | let est_total_peers = self.est_total_peers(); 275 | let ideal_target = coverage_target(est_total_peers); 276 | let gap = ideal_target - self.average_coverage; 277 | // We want to check the ratio between the gap and the target 278 | // because small targets will have small gaps. 279 | let gap_ratio = gap.abs() / ideal_target; 280 | if gap_ratio < NOISE_THRESHOLD { 281 | 0.0 282 | } else { 283 | gap * est_total_peers as f64 284 | } 285 | } 286 | 287 | /// Estimate total peers. 288 | pub fn est_total_peers(&self) -> usize { 289 | let coverage = self.filter.coverage(); 290 | if coverage > 0.0 { 291 | (1.0 / coverage * self.expected_count() as f64) as usize 292 | } else { 293 | // If we had no coverage when we collected these 294 | // peers then we can't make a good guess at the total. 295 | 0 296 | } 297 | } 298 | 299 | /// Estimated total redundant coverage. 300 | pub fn est_total_redundancy(&self) -> usize { 301 | (self.est_total_peers() as f64 * self.average_coverage) as usize 302 | } 303 | } 304 | 305 | impl From for DhtLocation { 306 | fn from(a: u32) -> Self { 307 | Self(Wrapping(a)) 308 | } 309 | } 310 | 311 | impl From for u32 { 312 | fn from(l: DhtLocation) -> Self { 313 | (l.0).0 314 | } 315 | } 316 | 317 | /// Finds the shortest distance between two points on a circle 318 | fn shortest_arc_distance, B: Into>(a: A, b: B) -> u32 { 319 | // Turn into wrapped u32s 320 | let a = a.into().0; 321 | let b = b.into().0; 322 | std::cmp::min(a - b, b - a).0 323 | } 324 | 325 | #[derive(Debug, Clone, Eq, PartialEq)] 326 | /// This represents the range of values covered by an arc 327 | pub struct ArcRange { 328 | /// The start bound of an arc range 329 | pub start: Bound, 330 | 331 | /// The end bound of an arc range 332 | pub end: Bound, 333 | } 334 | 335 | impl ArcRange { 336 | /// Show if the bound is empty 337 | /// Useful before using as an index 338 | pub fn is_empty(&self) -> bool { 339 | matches!((self.start_bound(), self.end_bound()), (Bound::Excluded(a), Bound::Excluded(b)) if a == b) 340 | } 341 | 342 | /// Length of this range. Remember this range can be a wrapping range. 343 | /// Must be u64 because the length of possible values in a u32 is u32::MAX + 1. 344 | pub fn len(&self) -> u64 { 345 | match (self.start_bound(), self.end_bound()) { 346 | // Range has wrapped around. 347 | (Bound::Included(start), Bound::Included(end)) if end < start => { 348 | U32_LEN - *start as u64 + *end as u64 + 1 349 | } 350 | (Bound::Included(start), Bound::Included(end)) if start == end => 1, 351 | (Bound::Included(start), Bound::Included(end)) => (end - start) as u64 + 1, 352 | (Bound::Excluded(_), Bound::Excluded(_)) => 0, 353 | _ => unreachable!("Ranges are either completely inclusive or completely exclusive"), 354 | } 355 | } 356 | } 357 | 358 | impl RangeBounds for ArcRange { 359 | fn start_bound(&self) -> Bound<&u32> { 360 | match &self.start { 361 | Bound::Included(i) => Bound::Included(i), 362 | Bound::Excluded(i) => Bound::Excluded(i), 363 | Bound::Unbounded => unreachable!("No unbounded ranges for arcs"), 364 | } 365 | } 366 | 367 | fn end_bound(&self) -> Bound<&u32> { 368 | match &self.end { 369 | Bound::Included(i) => Bound::Included(i), 370 | Bound::Excluded(i) => Bound::Excluded(i), 371 | Bound::Unbounded => unreachable!("No unbounded ranges for arcs"), 372 | } 373 | } 374 | 375 | fn contains(&self, _item: &U) -> bool 376 | where 377 | u32: PartialOrd, 378 | U: ?Sized + PartialOrd, 379 | { 380 | unimplemented!("Contains doesn't make sense for this type of range due to redundant holding near the bounds. Use DhtArc::contains") 381 | } 382 | } 383 | 384 | impl DhtArcBucket { 385 | /// Select only the arcs that fit into the bucket. 386 | pub fn new>(filter: DhtArc, arcs: I) -> Self { 387 | let arcs = arcs 388 | .into_iter() 389 | .filter(|a| filter.contains(a.center_loc)) 390 | .collect(); 391 | Self { filter, arcs } 392 | } 393 | 394 | /// Same as new but doesn't check if arcs fit into the bucket. 395 | pub fn new_unchecked(bucket: DhtArc, arcs: Vec) -> Self { 396 | Self { 397 | filter: bucket, 398 | arcs, 399 | } 400 | } 401 | 402 | /// Get the density of this bucket. 403 | pub fn density(&self) -> PeerDensity { 404 | let (total, count) = self 405 | .arcs 406 | .iter() 407 | .fold((0u64, 0usize), |(total, count), arc| { 408 | (total + arc.half_length as u64, count + 1) 409 | }); 410 | let average = if count > 0 { 411 | (total as f64 / count as f64) / MAX_HALF_LENGTH as f64 412 | } else { 413 | 0.0 414 | }; 415 | PeerDensity::new(self.filter, average, count) 416 | } 417 | } 418 | 419 | impl std::fmt::Display for DhtArcBucket { 420 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 421 | for a in &self.arcs { 422 | writeln!(f, "{}", a)?; 423 | } 424 | writeln!(f, "{} <- Bucket arc", self.filter) 425 | } 426 | } 427 | 428 | impl std::fmt::Display for DhtArc { 429 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 430 | let mut out = ["_"; 100]; 431 | let half_cov = (self.coverage() * 50.0) as isize; 432 | let center = self.center_loc.0 .0 as f64 / U32_LEN as f64; 433 | let center = (center * 100.0) as isize; 434 | for mut i in (center - half_cov)..(center + half_cov) { 435 | if i >= 100 { 436 | i -= 100; 437 | } 438 | if i < 0 { 439 | i += 100; 440 | } 441 | out[i as usize] = "#"; 442 | } 443 | out[center as usize] = "|"; 444 | let out: String = out.iter().map(|a| a.chars()).flatten().collect(); 445 | writeln!(f, "[{}]", out) 446 | } 447 | } 448 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use rusqlite::*; 2 | 3 | pub mod dht_arc; 4 | use dht_arc::*; 5 | 6 | #[derive(Debug)] 7 | pub struct SplitArc { 8 | pub start_1: Option, 9 | pub end_1: Option, 10 | pub start_2: Option, 11 | pub end_2: Option, 12 | } 13 | 14 | impl From for SplitArc { 15 | fn from(f: DhtArc) -> Self { 16 | use std::ops::{Bound, RangeBounds}; 17 | let r = f.range(); 18 | let s = r.start_bound(); 19 | let e = r.end_bound(); 20 | match (s, e) { 21 | (Bound::Excluded(_), Bound::Excluded(_)) => { 22 | // DhtArc only returns excluded bounds in the zero len case 23 | Self { 24 | start_1: None, 25 | end_1: None, 26 | start_2: None, 27 | end_2: None, 28 | } 29 | } 30 | (Bound::Included(s), Bound::Included(e)) => { 31 | if s > e { 32 | Self { 33 | start_1: Some(u32::MIN), 34 | end_1: Some(*e), 35 | start_2: Some(*s), 36 | end_2: Some(u32::MAX), 37 | } 38 | } else { 39 | Self { 40 | start_1: Some(*s), 41 | end_1: Some(*e), 42 | start_2: None, 43 | end_2: None, 44 | } 45 | } 46 | } 47 | // this is a quirc of how DhtArc is implemented 48 | _ => unreachable!(), 49 | } 50 | } 51 | } 52 | 53 | #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] 54 | pub struct AgentInfo { 55 | key: [u8; 32], 56 | signed_at_ms: u64, 57 | storage_arc: DhtArc, 58 | } 59 | 60 | impl AgentInfo { 61 | pub fn new_rand() -> Self { 62 | use rand::Rng; 63 | let mut rng = rand::thread_rng(); 64 | 65 | let mut key = [0; 32]; 66 | rng.fill(&mut key[..]); 67 | 68 | let signed_at_ms: u32 = rng.gen(); 69 | let center_loc: u32 = rng.gen(); 70 | let half_length = rng.gen_range(0..u32::MAX / 2); 71 | 72 | Self { 73 | key, 74 | signed_at_ms: signed_at_ms as u64, 75 | storage_arc: DhtArc::new(center_loc, half_length), 76 | } 77 | } 78 | } 79 | 80 | fn insert(con: &Connection, agent_info: &AgentInfo) -> Result<()> { 81 | let key = &agent_info.key[..]; 82 | let blob = 83 | rmp_serde::to_vec_named(agent_info).map_err(|e| Error::ToSqlConversionFailure(e.into()))?; 84 | let signed_at_ms = agent_info.signed_at_ms; 85 | let center_loc: u32 = agent_info.storage_arc.center_loc.into(); 86 | let half_length = agent_info.storage_arc.half_length; 87 | let split_arc: SplitArc = agent_info.storage_arc.clone().into(); 88 | //println!("{:?}", split_arc); 89 | let SplitArc { 90 | start_1, 91 | end_1, 92 | start_2, 93 | end_2, 94 | } = split_arc; 95 | con.execute( 96 | "INSERT INTO p2p_store ( 97 | key, blob, signed_at_ms, center_loc, half_length, 98 | arc_start1, arc_end1, arc_start2, arc_end2 99 | ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9);", 100 | params![ 101 | key, 102 | blob, 103 | signed_at_ms, 104 | center_loc, 105 | half_length, 106 | start_1, 107 | end_1, 108 | start_2, 109 | end_2, 110 | ], 111 | )?; 112 | Ok(()) 113 | } 114 | 115 | fn dump(con: &Connection) -> Result<()> { 116 | let mut stmt = con.prepare("SELECT * FROM p2p_store")?; 117 | let _ = stmt 118 | .query_map([], |r| { 119 | let s1: Option = r.get(5)?; 120 | let e1: Option = r.get(6)?; 121 | let s2: Option = r.get(7)?; 122 | let e2: Option = r.get(8)?; 123 | println!("{:?}-{:?} + {:?}-{:?}", s1, e1, s2, e2); 124 | Ok(()) 125 | })? 126 | .collect::>(); 127 | Ok(()) 128 | } 129 | 130 | fn count_agents_covering_loc(con: &Connection, loc: u32) -> Result { 131 | let mut stmt = con.prepare( 132 | "SELECT COUNT(key) 133 | FROM p2p_store 134 | WHERE ( 135 | arc_start1 IS NOT NULL 136 | AND arc_end1 IS NOT NULL 137 | AND ?1 >= arc_start1 138 | AND ?1 <= arc_end1 139 | ) 140 | OR ( 141 | arc_start2 IS NOT NULL 142 | AND arc_end2 IS NOT NULL 143 | AND ?1 >= arc_start2 144 | AND ?1 <= arc_end2 145 | );", 146 | )?; 147 | stmt.query_row(params![loc], |r| r.get(0)) 148 | } 149 | 150 | fn count_agents_overlaping_arc(con: &Connection, arc: DhtArc) -> Result { 151 | let split_arc: SplitArc = arc.into(); 152 | let SplitArc { 153 | start_1, 154 | end_1, 155 | start_2, 156 | end_2, 157 | } = split_arc; 158 | let mut stmt = con.prepare( 159 | "SELECT COUNT(key) 160 | FROM p2p_store 161 | WHERE ( 162 | ?1 IS NOT NULL 163 | AND ?2 IS NOT NULL 164 | AND arc_start1 IS NOT NULL 165 | AND arc_end1 IS NOT NULL 166 | AND ?1 <= arc_end1 167 | AND ?2 >= arc_start1 168 | ) 169 | OR ( 170 | ?3 IS NOT NULL 171 | AND ?4 IS NOT NULL 172 | AND arc_start2 IS NOT NULL 173 | AND arc_end2 IS NOT NULL 174 | AND ?3 <= arc_end2 175 | AND ?4 >= arc_start2 176 | );", 177 | )?; 178 | stmt.query_row(params![start_1, end_1, start_2, end_2], |r| r.get(0)) 179 | } 180 | 181 | fn list_close_agents(con: &Connection, loc: u32) -> Result> { 182 | let mut stmt = con.prepare( 183 | "SELECT min( 184 | CASE WHEN arc_start1 IS NULL OR arc_end1 IS NULL 185 | THEN 4294967296 186 | ELSE 187 | CASE WHEN ?1 >= arc_start1 AND ?1 <= arc_end1 188 | THEN 0 189 | WHEN ?1 < arc_start1 THEN arc_start1 - ?1 190 | ELSE ?1 - arc_end1 191 | END 192 | END, 193 | CASE WHEN arc_start2 IS NULL OR arc_end2 IS NULL 194 | THEN 4294967296 195 | ELSE 196 | CASE WHEN ?1 >= arc_start2 AND ?1 <= arc_end2 197 | THEN 0 198 | WHEN ?1 < arc_start2 THEN arc_start2 - ?1 199 | ELSE ?1 - arc_end2 200 | END 201 | END 202 | ) AS distance 203 | FROM p2p_store 204 | ORDER BY distance 205 | ;", 206 | )?; 207 | let mut out = Vec::new(); 208 | for r in stmt.query_map(params![loc], |r| r.get(0))? { 209 | out.push(r?); 210 | } 211 | Ok(out) 212 | } 213 | 214 | fn main() -> Result<()> { 215 | let con = Connection::open_in_memory()?; 216 | 217 | con.execute( 218 | "CREATE TABLE IF NOT EXISTS p2p_store ( 219 | key BLOB PRIMARY KEY ON CONFLICT REPLACE, 220 | blob BLOB NOT NULL, 221 | signed_at_ms INTEGER NOT NULL, 222 | center_loc INTEGER NOT NULL, 223 | half_length INTEGER NOT NULL, 224 | arc_start1 INTEGER NULL, 225 | arc_end1 INTEGER NULL, 226 | arc_start2 INTEGER NULL, 227 | arc_end2 INTEGER NULL 228 | );", 229 | [], 230 | )?; 231 | 232 | let mut info_zero = AgentInfo::new_rand(); 233 | info_zero.storage_arc = DhtArc::new(0, 0); 234 | insert(&con, &info_zero)?; 235 | 236 | for _ in 0..10 { 237 | let agent_info = AgentInfo::new_rand(); 238 | insert(&con, &agent_info)?; 239 | } 240 | 241 | dump(&con)?; 242 | println!("agents covering 0: {}", count_agents_covering_loc(&con, 0)?); 243 | println!( 244 | "agents covering {}: {}", 245 | u32::MAX, 246 | count_agents_covering_loc(&con, u32::MAX)? 247 | ); 248 | let mid = u32::MAX / 2; 249 | println!( 250 | "agents covering {}: {}", 251 | mid, 252 | count_agents_covering_loc(&con, mid)? 253 | ); 254 | let overlap = DhtArc::new(mid, u32::MAX / 4); 255 | let res = count_agents_overlaping_arc(&con, overlap.clone())?; 256 | println!("agents overlapping {:?}: {}", &overlap, res); 257 | 258 | for r in list_close_agents(&con, mid)? { 259 | println!("-- {:?}", r); 260 | } 261 | 262 | Ok(()) 263 | } 264 | --------------------------------------------------------------------------------