├── .gitignore ├── graphs ├── String_lookup_hit.png ├── String_insert_remove.png ├── String_lookup_miss.png ├── dense_u64_sparse_u64_lookup_hit_16384.png ├── dense_u64_sparse_u64_insert_remove_16384.png ├── dense_u64_sparse_u64_lookup_hit_16777216.png ├── dense_u64_sparse_u64_lookup_miss_16384.png ├── dense_u64_sparse_u64_lookup_hit_268435456.png ├── dense_u64_sparse_u64_lookup_miss_16777216.png ├── dense_u64_sparse_u64_lookup_miss_268435456.png ├── dense_u64_sparse_u64_insert_remove_16777216.png ├── dense_u64_sparse_u64_insert_remove_268435456.png ├── make_csv └── make_graphs.r ├── Cargo.toml ├── src ├── lib.rs ├── macros.rs ├── common.rs ├── prefix_cache.rs ├── art_internal.rs └── art_impl.rs ├── Performance.md ├── README.md ├── benches └── set_bench.rs └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target 3 | **/*.rs.bk 4 | Cargo.lock 5 | results.csv 6 | -------------------------------------------------------------------------------- /graphs/String_lookup_hit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/String_lookup_hit.png -------------------------------------------------------------------------------- /graphs/String_insert_remove.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/String_insert_remove.png -------------------------------------------------------------------------------- /graphs/String_lookup_miss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/String_lookup_miss.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_hit_16384.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_hit_16384.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_insert_remove_16384.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_insert_remove_16384.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_hit_16777216.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_hit_16777216.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_miss_16384.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_miss_16384.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_hit_268435456.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_hit_268435456.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_miss_16777216.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_miss_16777216.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_lookup_miss_268435456.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_lookup_miss_268435456.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_insert_remove_16777216.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_insert_remove_16777216.png -------------------------------------------------------------------------------- /graphs/dense_u64_sparse_u64_insert_remove_268435456.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ezrosent/art-rs/HEAD/graphs/dense_u64_sparse_u64_insert_remove_268435456.png -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "radix-tree" 3 | version = "0.1.0" 4 | authors = ["eli"] 5 | 6 | [dependencies] 7 | byteorder = "1.2.1" 8 | simd = "0.2.1" 9 | smallvec = "0.6.0" 10 | fnv = "1.0.3" 11 | 12 | [features] 13 | default = [] 14 | print_cache_stats = [] 15 | 16 | [dev-dependencies] 17 | quickcheck = "0.6.1" 18 | rand = "0.4" 19 | criterion = "0.2" 20 | 21 | [[bench]] 22 | name = "set_bench" 23 | harness = false 24 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(swap_nonoverlapping)] 2 | #![feature(cfg_target_feature)] 3 | #![feature(stdsimd)] 4 | #[macro_use] 5 | mod macros; 6 | mod common; 7 | mod art_impl; 8 | mod art_internal; 9 | mod prefix_cache; 10 | 11 | extern crate byteorder; 12 | extern crate smallvec; 13 | 14 | pub use common::Digital; 15 | pub use art_impl::*; 16 | #[cfg(test)] 17 | #[macro_use] 18 | extern crate quickcheck; 19 | #[cfg(test)] 20 | extern crate rand; 21 | 22 | #[cfg(test)] 23 | mod tests { 24 | #[test] 25 | fn it_works() { 26 | assert_eq!(2 + 2, 4); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /graphs/make_csv: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # First run `cargo bench` (beware: this takes a long time and it uses a lot of memory). 4 | # Then run this script from the crate root and redirect output to graphs/results.csv. 5 | # Then run Rscript make_graphs.r to regenerate graphs. 6 | 7 | CATEGORIES="dense_u64 sparse_u64 String" 8 | DATA_STRUCTURES="ARTSet CachingARTSet HashSet BTreeSet" 9 | BENCHES="lookup_hit lookup_miss insert_remove" 10 | 11 | echo "data structure,data type,workload,number of elements,str number of elements,mean time per operation ns" 12 | for c in $CATEGORIES; do 13 | for d in $DATA_STRUCTURES; do 14 | for b in $BENCHES; do 15 | for sz in $(ls "target/criterion/${d}/${c}/${b}" | grep -Ev 'report|new|base|change'); do 16 | st_elts=$(echo "$sz" | gawk '{ 17 | r20=rshift($1, 20) 18 | r10=rshift($1, 10) 19 | if (r20 > 0) { 20 | print r20"M" 21 | } else if (r10 > 0) { 22 | print r10"K" 23 | } else if ($1 > 0) { 24 | print $1 25 | }}') 26 | avg_perf=$(cat "target/criterion/${d}/${c}/${b}/${sz}/report/index.html" | 27 | grep 'Mean' -A3 | 28 | tail -n2 | 29 | head -n1 | 30 | cut -d'>' -f2 | 31 | cut -d'<' -f1 | 32 | awk '{ if ($2 == "ns") { print $1 } if ($2 == "us") { print 1000 * $1 } }') 33 | echo "$d,$c,$b,$sz,$st_elts,$avg_perf" 34 | done 35 | done 36 | done 37 | done 38 | -------------------------------------------------------------------------------- /src/macros.rs: -------------------------------------------------------------------------------- 1 | macro_rules! with_node_inner { 2 | ($base_node: expr, $nod: ident, $body: expr, $r: tt) => { 3 | with_node_inner!($base_node, $nod, $body, $r, _) 4 | }; 5 | ($base_node: expr, $nod: ident, $body: expr, $r: tt, $ty: tt) => {{ 6 | let _b: $r<()> = $base_node; 7 | match _b.typ { 8 | NODE_4 => { 9 | #[allow(unused_unsafe)] 10 | let $nod = unsafe { mem::transmute::<$r<_>, $r>>(_b) }; 11 | $body 12 | } 13 | NODE_16 => { 14 | #[allow(unused_unsafe)] 15 | let $nod = unsafe { mem::transmute::<$r<_>, $r>>(_b) }; 16 | $body 17 | } 18 | NODE_48 => { 19 | #[allow(unused_unsafe)] 20 | let $nod = unsafe { mem::transmute::<$r<_>, $r>>(_b) }; 21 | $body 22 | } 23 | NODE_256 => { 24 | #[allow(unused_unsafe)] 25 | let $nod = unsafe { mem::transmute::<$r<_>, $r>>(_b) }; 26 | $body 27 | } 28 | _ => panic!("Found unrecognized node type {:?}", _b.typ), 29 | } 30 | }}; 31 | } 32 | 33 | macro_rules! with_node_mut { 34 | ($base_node: expr, $nod: ident, $body: expr) => { 35 | with_node_mut!($base_node, $nod, $body, _) 36 | }; 37 | ($base_node: expr, $nod: ident, $body: expr, $ty: tt) => { 38 | with_node_inner!($base_node, $nod, $body, RawMutRef, $ty) 39 | }; 40 | } 41 | 42 | macro_rules! with_node { 43 | ($base_node: expr, $nod: ident, $body: expr) => { 44 | with_node!($base_node, $nod, $body, _) 45 | }; 46 | ($base_node: expr, $nod: ident, $body: expr, $ty: tt) => { 47 | with_node_inner!($base_node, $nod, $body, RawRef, $ty) 48 | }; 49 | } 50 | 51 | macro_rules! trace { 52 | ($b:expr, $str:expr, $( $arg:expr ),+) => { 53 | #[cfg(debug_assertions)] 54 | { 55 | if $b { eprintln!("{} {} {}", file!(), line!(), format!($str, $( $arg ),*)) } 56 | } 57 | }; 58 | ($b:expr, $str:expr) => { trace!($b, "{}", $str) }; 59 | ($b:expr) => { trace!($b, "") }; 60 | } 61 | -------------------------------------------------------------------------------- /Performance.md: -------------------------------------------------------------------------------- 1 | # ART Performance 2 | 3 | We benchmarked lookups (keys within the set and keys not in the set) and 4 | insert/delete pairs for `ARTSet` (our ART implementation), `CachingARTSet` (an 5 | ART with a prefix cache), rust's `BTreeSet` and rust's `HashSet`. We use random 6 | integer keys where the keys are chosen from 0 to the size of the set ("dense") 7 | and where they are chosen from all possible 64-bit integers ("sparse"). We also 8 | include benchmarks for random UTF-8 strings. 9 | 10 | ### Integers 11 | 12 | Here we see that the ART generally does somewhere between the performance of 13 | the BTree and the hash table. The cache is little help for small tables or 14 | dense keys. This makes sense, as the dense keys will often share a prefix, 15 | making the likely depth of the tree fairly short, while prefix compression will 16 | ensure the absolute depth of the tree is quite low when there are few elements. 17 | Prefix caching *does*, however, make a substantial difference for sparse 18 | integers in larger tables. 19 | 20 | ![Integer Performance 16K](graphs/dense_u64_sparse_u64_lookup_miss_16384.png?raw=true) 21 | ![Integer Performance 16M](graphs/dense_u64_sparse_u64_lookup_miss_16777216.png?raw=true) 22 | ![Integer Performance 256M](graphs/dense_u64_sparse_u64_lookup_miss_268435456.png?raw=true) 23 | 24 | ![Integer Performance 16K](graphs/dense_u64_sparse_u64_lookup_hit_16384.png?raw=true) 25 | ![Integer Performance 16M](graphs/dense_u64_sparse_u64_lookup_hit_16777216.png?raw=true) 26 | ![Integer Performance 256M](graphs/dense_u64_sparse_u64_lookup_hit_268435456.png?raw=true) 27 | 28 | ![Integer Performance 16K](graphs/dense_u64_sparse_u64_insert_remove_16384.png?raw=true) 29 | ![Integer Performance 16M](graphs/dense_u64_sparse_u64_insert_remove_16777216.png?raw=true) 30 | ![Integer Performance 256M](graphs/dense_u64_sparse_u64_insert_remove_268435456.png?raw=true) 31 | 32 | ### Strings 33 | 34 | There is a similar story here as to the integer workloads above. The benefit of 35 | caching here is, however, more pronounced for both lookups and mutations. 36 | 37 | ![String Hits](graphs/String_lookup_hit.png?raw=true) 38 | ![String Misses](graphs/String_lookup_miss.png?raw=true) 39 | ![String Mutates](graphs/String_insert_remove.png?raw=true) 40 | -------------------------------------------------------------------------------- /graphs/make_graphs.r: -------------------------------------------------------------------------------- 1 | library(ggplot2) 2 | alldata = read.csv('graphs/results.csv') 3 | 4 | strgrapher <- function(exclude, ty, wl, title) { 5 | ftab = subset(subset(subset(alldata, data.type == ty), workload == wl), number.of.elements != exclude) 6 | tags=ftab$data.structure 7 | sizes=ftab$str.number.of.elements 8 | optimes=ftab$mean.time.per.operation.ns 9 | btab = data.frame(Size=sizes, Data.Structure=tags, ns.Per.Op=optimes) 10 | # print(btab) 11 | png(filename=paste('graphs/', paste(ty,wl,sep='_'), '.png', sep=''), width=600, height=600) 12 | posns <- c("16K", "1M", "16M") 13 | ggplot(btab, aes(fill=Data.Structure,y=ns.Per.Op, x=Size)) + 14 | scale_x_discrete(limits=posns) + 15 | geom_bar(position="dodge", stat="identity") + 16 | labs(title=title, x="Number of Elements", y="ns Per Operation") 17 | } 18 | 19 | intgrapher <- function(size, ty1, ty2, wl, title) { 20 | ftab1 = subset(subset(subset(alldata, number.of.elements == size), data.type == ty1), workload == wl) 21 | ftab2 = subset(subset(subset(alldata, number.of.elements == size), data.type == ty2), workload == wl) 22 | png(filename=paste('graphs/', paste(ty1,ty2,wl,size,sep='_'), '.png', sep=''), width=600, height=600) 23 | tags = c(rep('dense', length(ftab1$mean.time.per.operation.ns)), 24 | rep('sparse', length(ftab1$mean.time.per.operation.ns))) 25 | btab = data.frame(Data.Structure=rep(ftab1$data.structure, 2), 26 | ns.Per.Operation=c(ftab1$mean.time.per.operation.ns, ftab2$mean.time.per.operation.ns), 27 | workload=tags) 28 | ggplot(btab, aes(fill=workload, y=ns.Per.Operation, x=Data.Structure)) + 29 | geom_bar(position="dodge", stat="identity") + 30 | labs(title=title, x="Data Structure", y="ns Per Operation") 31 | } 32 | 33 | intgrapher(16384, 'dense_u64', 'sparse_u64', 'lookup_hit', 34 | 'Lookups for elements in the set with integer keys, 16K elements') 35 | intgrapher(16777216, 'dense_u64', 'sparse_u64', 'lookup_hit', 36 | 'Lookups for elements in the set with integer keys, 16M elements') 37 | intgrapher(268435456, 'dense_u64', 'sparse_u64', 'lookup_hit', 38 | 'Lookups for elements in the set with integer keys, 256M elements') 39 | 40 | intgrapher(16384, 'dense_u64', 'sparse_u64', 'lookup_miss', 41 | 'Lookups for elements not in the set with integer keys, 16K elements') 42 | intgrapher(16777216, 'dense_u64', 'sparse_u64', 'lookup_miss', 43 | 'Lookups for elements not in the set with integer keys, 16M elements') 44 | intgrapher(268435456, 'dense_u64', 'sparse_u64', 'lookup_miss', 45 | 'Lookups for elements not in the set with integer keys, 256M elements') 46 | 47 | intgrapher(16384, 'dense_u64', 'sparse_u64', 'insert_remove', 48 | 'Insert/Remove pairs with integer keys, 16K elements') 49 | intgrapher(16777216, 'dense_u64', 'sparse_u64', 'insert_remove', 50 | 'Insert/Remove pairs with integer keys, 16M elements') 51 | intgrapher(268435456, 'dense_u64', 'sparse_u64', 'insert_remove', 52 | 'Insert/Remove pairs with integer keys, 256M elements') 53 | 54 | strgrapher(67108864,'String', 'lookup_hit', 'Lookups in the set, UTF-8 Strings of mean length 10') 55 | strgrapher(67108864,'String', 'lookup_miss', 'Lookups not in the set, UTF-8 Strings of mean length 10') 56 | strgrapher(67108864,'String', 'insert_remove', 'Insert/Remove Pairs, UTF-8 Strings of mean length 10') 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # art-rs: Efficient ordered containers. 2 | The adaptive radix tree (ART) is an efficient radix tree (aka trie) design introduced by [Leis, Kemper 3 | and 4 | Neumann](https://15721.courses.cs.cmu.edu/spring2018/papers/09-oltpindexes2/leis-icde2013.pdf) in 2013. 5 | This includes an implementation of the ART data-structure described in that 6 | work, along with experimental support for *prefix-caching*. 7 | 8 | ## Overview 9 | 10 | ARTs operate on types that can be decomposed into sequences of bytes. If the 11 | types are ordered, and these byte sequences (with a lexicographic ordering) 12 | respect the ordering on the type, then ARTs also support efficient range scans. 13 | Because most ordered types can also be efficiently decomposed to byte 14 | sequences[1], this makes them a potential alternative to ordered tree 15 | data-structures, like Rust's `BTreeSet` or `std::set` in C++ (often a Red-Black 16 | Tree). 17 | 18 | Compared to the classic [Trie datastructure](https://en.wikipedia.org/wiki/Trie), 19 | the ART paper details lots of intricate optimizations to speed up lookups and 20 | insertions. The most important of these are: 21 | 22 | * *Prefix Compression*: A sequence of interior that do not point directly to a 23 | leaf can be compressed into a single node, thus reducing the length of the 24 | path that must be traversed. 25 | 26 | * *Lazy Expansion*: A sequence of interior nodes that only point to a single 27 | leaf can be elided entirely. 28 | 29 | * *Specialized Interior Nodes*: Inner nodes in the tree have specialized 30 | implementations for ones with up to 4, 16, 48 and 256 children. This 31 | balances space efficiency with the speed of lookups. 32 | 33 | See the ART paper for a more complete description of these features. 34 | 35 | 36 | ## Prefix-Caching 37 | 38 | Keys for this data-structure can be decomposed into byte sequences. Short byte 39 | sequences can be hashed efficiently. This repo provides variants of the ART 40 | that store a hash table mapping from key prefixes to *interior nodes* within 41 | the tree. This allows traversals for either mutation operations or lookups to 42 | skip several levels of the tree in their traversal. This sort of trick is much 43 | harder for ordered tree data-structures, as their keys do not necessarily have 44 | the needed structure, and they may have more complicated rebalancing operations 45 | which can make it more difficult to maintain the validity of the hash table. 46 | 47 | The length of the cached prefixes can be customized, allowing you to limit the 48 | maximum size of the cache. 49 | 50 | ## Performance 51 | 52 | While not complete, we have a number of benchmarks that compare the ART-based 53 | data-structures to rust's `HashSet` and `BTreeSet`. They show promising 54 | performance for the vanilla ART implementation, and demonstrate that prefix 55 | caching can improve performance even further when the set is large. See 56 | `Performance.md` for more information and measurements. 57 | 58 | ## TODOs 59 | 60 | This implementation if very rough, and likely contains bugs. On top of general 61 | code improvements, there is also still a lot to do to get it to feature-parity 62 | with the standard Rust container types. 63 | 64 | ### API Parity with `BTreeSet` 65 | This includes good implementations of set operations, as well as a proper 66 | iterator API. While we have a callback-based traversal API, we lack an idiomatic iterator 67 | implementation 68 | 69 | ### Bulk Insertions 70 | The ART paper describes a method for performing optimized bulk insertions of 71 | values, which is not yet implemented in this code-base. 72 | 73 | ### Multithreading 74 | While [follow-up work](https://db.in.tum.de/~leis/papers/artsync.pdf) 75 | implemented synchronization for the ART, this repo only includes a 76 | single-threaded implementation. I am interested in implementing a multithreaded 77 | version at some point in the future. 78 | 79 | ### Slab Allocation 80 | We currently do heap allocations for all new interior nodes and leaf nodes. At 81 | the very least interior nodes could probably benefit from slab allocation. 82 | 83 | ### Space-optimized Prefix Caching 84 | Because real-world map workloads are often skewed towards a small subset of the 85 | keys, it should be possible tune the prefix cache to store a small subset of 86 | keys. This would reduce the space overhead of the cache while still hopefully 87 | preserving most of the performance gains. 88 | 89 | 90 | [1]: See section 4 of the paper for more information on this. In this code, it 91 | is encapsulated by the `Digital` trait, which has implementation for common 92 | integer and string types. 93 | -------------------------------------------------------------------------------- /src/common.rs: -------------------------------------------------------------------------------- 1 | use std::iter::Iterator; 2 | use std::str; 3 | use super::byteorder::{BigEndian, ByteOrder}; 4 | 5 | /// `Digital` describes types that can be expressed as sequences of bytes. 6 | /// 7 | /// The type's `digits` should respect equality and ordering on the type. 8 | /// Furthermore, if the `digits` of one value are a prefix of the `digits` 9 | /// of another value of the same type, the two values must be equal. 10 | /// 11 | /// TODO implement floating point support. This is described in the ART paper but a couple details 12 | /// are left out. 13 | /// 14 | /// TODO implement macro/derive that will create a "digits" representation for any ordered type. 15 | pub trait Digital<'a> { 16 | // TODO: consider providing a more efficient interface here (e.g. passing a slice directly) 17 | type I: Iterator + 'a; 18 | const STOP_CHARACTER: Option = None; 19 | fn digits(&'a self) -> Self::I; 20 | } 21 | 22 | pub struct U32BytesIterator { 23 | cursor: usize, 24 | bytes: [u8; 4], 25 | } 26 | 27 | impl Iterator for U32BytesIterator { 28 | type Item = u8; 29 | fn next(&mut self) -> Option { 30 | if self.cursor < 4 { 31 | self.cursor += 1; 32 | Some(self.bytes[self.cursor - 1]) 33 | } else { 34 | None 35 | } 36 | } 37 | 38 | fn nth(&mut self, n: usize) -> Option { 39 | self.cursor += n; 40 | self.next() 41 | } 42 | } 43 | 44 | impl<'a> Digital<'a> for u32 { 45 | type I = U32BytesIterator; 46 | fn digits(&self) -> U32BytesIterator { 47 | let mut res = U32BytesIterator { 48 | cursor: 0, 49 | bytes: [0; 4], 50 | }; 51 | BigEndian::write_u32(&mut res.bytes, *self); 52 | res 53 | } 54 | } 55 | 56 | impl<'a> Digital<'a> for i32 { 57 | type I = U32BytesIterator; 58 | fn digits(&self) -> U32BytesIterator { 59 | let mut res = U32BytesIterator { 60 | cursor: 0, 61 | bytes: [0; 4], 62 | }; 63 | BigEndian::write_i32(&mut res.bytes, *self ^ (1 << 31)); 64 | res 65 | } 66 | } 67 | 68 | impl<'a> Digital<'a> for i64 { 69 | type I = U64BytesIterator; 70 | fn digits(&self) -> U64BytesIterator { 71 | let mut res = U64BytesIterator { 72 | cursor: 0, 73 | bytes: [0; 8], 74 | }; 75 | BigEndian::write_i64(&mut res.bytes, *self ^ (1 << 63)); 76 | res 77 | } 78 | } 79 | 80 | impl<'a> Digital<'a> for usize { 81 | // Just treat usize as u64. This should (inefficiently) support platforms with a smaller type, 82 | // and we debug-assert that usize <= u64 in size. 83 | type I = U64BytesIterator; 84 | fn digits(&self) -> Self::I { 85 | debug_assert!(::std::mem::size_of::() <= ::std::mem::size_of::()); 86 | (*self as u64).digits() 87 | } 88 | } 89 | 90 | impl<'a> Digital<'a> for isize { 91 | type I = U64BytesIterator; 92 | fn digits(&self) -> Self::I { 93 | debug_assert!(::std::mem::size_of::() <= ::std::mem::size_of::()); 94 | (*self as i64).digits() 95 | } 96 | } 97 | 98 | pub struct U64BytesIterator { 99 | cursor: usize, 100 | bytes: [u8; 8], 101 | } 102 | 103 | impl Iterator for U64BytesIterator { 104 | type Item = u8; 105 | fn next(&mut self) -> Option { 106 | if self.cursor < 8 { 107 | self.cursor += 1; 108 | Some(self.bytes[self.cursor - 1]) 109 | } else { 110 | None 111 | } 112 | } 113 | 114 | fn nth(&mut self, n: usize) -> Option { 115 | self.cursor += n; 116 | self.next() 117 | } 118 | } 119 | 120 | impl<'a> Digital<'a> for u64 { 121 | type I = U64BytesIterator; 122 | fn digits(&self) -> U64BytesIterator { 123 | let mut res = U64BytesIterator { 124 | cursor: 0, 125 | bytes: [0; 8], 126 | }; 127 | BigEndian::write_u64(&mut res.bytes, *self); 128 | res 129 | } 130 | } 131 | 132 | /// NullTerminate transforms iterator corresponding to the bytes of a valid UTF-8 string into an 133 | /// iterator suitable for use in a `Digital` implementation. This comes for free in languages using 134 | /// C-style ASCII strings by convention, because null-termination guarantees the "prefixes" 135 | /// property of the trait. 136 | /// 137 | /// In Rust, strings are most commonly encoded as UTF-8. For such strings, NUL characters are 138 | /// kosher in the middle of a string, and picking a different byte as a terminator character will 139 | /// ruin the compatibility with Ord[0]. To ensure that a null terminator is valid, we increase the 140 | /// value of all bytes emitted by `I` by 1. We are guaranteed no overflow by the fact that 255 is 141 | /// an invalid byte for UTF-8 strings. Given no overflow, equality and ordering are clearly 142 | /// conserved. 143 | /// 144 | /// [0]: To see why this is the case, consider the example of "" and "a". "" < "a", but "\u{255}" > 145 | /// "a\u{255}". 146 | pub struct NullTerminate { 147 | done: bool, 148 | i: I, 149 | } 150 | 151 | impl NullTerminate { 152 | fn new(i: I) -> Self { 153 | NullTerminate { done: false, i: i } 154 | } 155 | } 156 | 157 | impl> Iterator for NullTerminate { 158 | type Item = u8; 159 | fn next(&mut self) -> Option { 160 | if self.done { 161 | return None; 162 | } 163 | let res = self.i.next(); 164 | if let Some(s) = res { 165 | debug_assert!(s < 255); 166 | Some(s + 1) 167 | } else { 168 | self.done = true; 169 | Some(0) 170 | } 171 | } 172 | 173 | fn nth(&mut self, n: usize) -> Option { 174 | if self.done { 175 | return None; 176 | } 177 | let (remaining, _max) = self.i.size_hint(); 178 | debug_assert_eq!( 179 | Some(remaining), 180 | _max, 181 | "must use iterator with exact length for NullTerminate" 182 | ); 183 | if n + 1 == remaining { 184 | self.done = true; 185 | Some(0) 186 | } else { 187 | self.i.nth(n).map(|x| x + 1) 188 | } 189 | } 190 | } 191 | 192 | impl<'a> Digital<'a> for str { 193 | type I = NullTerminate>; 194 | const STOP_CHARACTER: Option = Some(0); 195 | fn digits(&'a self) -> Self::I { 196 | NullTerminate::new(self.bytes()) 197 | } 198 | } 199 | 200 | impl<'a> Digital<'a> for String { 201 | type I = NullTerminate>; 202 | const STOP_CHARACTER: Option = Some(0); 203 | fn digits(&'a self) -> Self::I { 204 | NullTerminate::new(self.as_str().bytes()) 205 | } 206 | } 207 | 208 | #[cfg(test)] 209 | mod tests { 210 | use super::*; 211 | 212 | fn test_digits_obey_order Digital<'a> + PartialOrd>(x: D, y: D) -> bool { 213 | let vx: Vec<_> = x.digits().collect(); 214 | let vy: Vec<_> = y.digits().collect(); 215 | if x < y { 216 | vx < vy 217 | } else { 218 | vx >= vy 219 | } 220 | } 221 | 222 | quickcheck! { 223 | fn digits_strings(x: String, y: String) -> bool { 224 | test_digits_obey_order(x, y) 225 | } 226 | 227 | fn digits_u64(x: u64, y: u64) -> bool { 228 | // why shift left? the RNG seems to generate numbers <256, so endianness bugs do not 229 | // get caught! 230 | test_digits_obey_order(x.wrapping_shl(20), y.wrapping_shl(20)) 231 | } 232 | 233 | fn digits_u32(x: u32, y: u32) -> bool { 234 | test_digits_obey_order(x.wrapping_shl(20), y.wrapping_shl(20)) 235 | } 236 | 237 | fn digits_i32(x: i32, y: i32) -> bool { 238 | test_digits_obey_order(x.wrapping_mul(1 << 10), y.wrapping_mul(1 << 10)) 239 | } 240 | 241 | fn digits_i64(x: i64, y: i64) -> bool { 242 | test_digits_obey_order(x.wrapping_mul(1 << 20), y.wrapping_mul(1 << 20)) 243 | } 244 | 245 | fn digits_isize(x: isize, y: isize) -> bool { 246 | test_digits_obey_order(x.wrapping_mul(1 << 20), y.wrapping_mul(1 << 20)) 247 | } 248 | 249 | fn digits_usize(x: usize, y: usize) -> bool { 250 | test_digits_obey_order(x.wrapping_shl(20), y.wrapping_shl(20)) 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /benches/set_bench.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | extern crate radix_tree; 4 | extern crate rand; 5 | 6 | use criterion::{Bencher, Criterion}; 7 | use rand::{Rng, SeedableRng, StdRng}; 8 | use std::collections::btree_set::BTreeSet; 9 | use std::collections::HashSet; 10 | use std::hash::Hash; 11 | 12 | use radix_tree::{ARTSet, ArtElement, CachingARTSet, Digital, PrefixCache, RawART}; 13 | 14 | /// We use a deterministic seed when generating random data to cut down on variance between 15 | /// different benchmark runs. 16 | const RAND_SEED: [usize; 32] = [1; 32]; 17 | 18 | /// Barebones set trait to abstract over various collections. 19 | trait Set { 20 | fn new() -> Self; 21 | fn contains(&self, t: &T) -> bool; 22 | fn insert(&mut self, t: T); 23 | fn delete(&mut self, t: &T) -> bool; 24 | } 25 | 26 | trait ARTArg { 27 | const PREFIX_LEN: usize; 28 | } 29 | 30 | impl ARTArg for u64 { 31 | const PREFIX_LEN: usize = 3; 32 | } 33 | 34 | impl ARTArg for String { 35 | const PREFIX_LEN: usize = 8; 36 | } 37 | 38 | impl Digital<'a> + Ord, C: PrefixCache>> Set 39 | for RawART, C> 40 | { 41 | fn new() -> Self { 42 | Self::with_prefix_buckets(T::PREFIX_LEN) 43 | } 44 | fn contains(&self, t: &T) -> bool { 45 | self.contains(t) 46 | } 47 | fn insert(&mut self, t: T) { 48 | self.replace(t); 49 | } 50 | fn delete(&mut self, t: &T) -> bool { 51 | self.remove(t) 52 | } 53 | } 54 | 55 | impl Set for HashSet { 56 | fn new() -> Self { 57 | HashSet::new() 58 | } 59 | fn contains(&self, t: &T) -> bool { 60 | self.get(t).is_some() 61 | } 62 | fn insert(&mut self, t: T) { 63 | self.replace(t); 64 | } 65 | fn delete(&mut self, t: &T) -> bool { 66 | self.remove(t) 67 | } 68 | } 69 | 70 | impl Set for BTreeSet { 71 | fn new() -> Self { 72 | BTreeSet::new() 73 | } 74 | fn contains(&self, t: &T) -> bool { 75 | self.get(t).is_some() 76 | } 77 | fn insert(&mut self, t: T) { 78 | self.replace(t); 79 | } 80 | fn delete(&mut self, t: &T) -> bool { 81 | self.remove(t) 82 | } 83 | } 84 | 85 | fn random_vec(len: usize, max_val: u64) -> Vec { 86 | let mut rng = StdRng::from_seed(&RAND_SEED[..]); 87 | (0..len.next_power_of_two()) 88 | .map(|_| rng.gen_range::(0, max_val)) 89 | .collect() 90 | } 91 | 92 | fn random_dense_vec(len: u64, bias: u64) -> Vec { 93 | let mut rng = StdRng::from_seed(&RAND_SEED[..]); 94 | let mut res = (0..len.next_power_of_two()) 95 | .map(|x| x + bias) 96 | .collect::>(); 97 | rng.shuffle(res.as_mut_slice()); 98 | res 99 | } 100 | 101 | fn random_string_vec(max_len: usize, len: usize) -> Vec { 102 | let mut rng = StdRng::from_seed(&RAND_SEED[..]); 103 | (0..len.next_power_of_two()) 104 | .map(|_| { 105 | let mlen = max_len as isize; 106 | let s_len = mlen + rng.gen_range::(-mlen / 2, mlen / 2); 107 | rng.gen_iter::() 108 | .take(s_len as usize) 109 | .collect::() 110 | }) 111 | .collect() 112 | } 113 | 114 | fn bench_set_rand_int_lookup Digital<'a>, S: Set>( 115 | b: &mut Bencher, 116 | contents: &S, 117 | lookups: &Vec, 118 | ) { 119 | assert!(lookups.len().is_power_of_two()); 120 | let mut ix = 0; 121 | b.iter(|| { 122 | contents.contains(&lookups[ix]); 123 | ix += 1; 124 | ix = ix & (lookups.len() - 1); 125 | }) 126 | } 127 | 128 | fn bench_set_insert_remove Digital<'a>, S: Set>( 129 | b: &mut Bencher, 130 | contents: &mut S, 131 | lookups: &Vec, 132 | ) { 133 | assert!(lookups.len().is_power_of_two()); 134 | let mut ix = 0; 135 | b.iter(|| { 136 | contents.insert(lookups[ix].clone()); 137 | ix += 1; 138 | ix = ix & (lookups.len() - 1); 139 | contents.delete(&lookups[ix]); 140 | // Why += 2? lookups has an even length, but we don't want all inserts to converge to 141 | // "replace" ops (similarly, deletes should sometimes succeed). 142 | // TODO: There's probably a more principled way of doing this. 143 | ix += 2; 144 | ix = ix & (lookups.len() - 1); 145 | }) 146 | } 147 | 148 | fn criterion_benchmark(c: &mut Criterion) { 149 | use std::fmt::{Debug, Error, Formatter}; 150 | #[derive(Clone)] 151 | struct SizeVec(Vec, Vec); 152 | impl Debug for SizeVec { 153 | fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 154 | write!(f, "{:?}", self.0.len()) 155 | } 156 | } 157 | fn make_bench Digital<'a>, S: Set + 'static>( 158 | c: &mut Criterion, 159 | desc: String, 160 | inp: &Vec>, 161 | ) { 162 | eprintln!("Generating for {} (1/3)", desc); 163 | struct Wrap(SizeVec, Box); 164 | impl Debug for Wrap { 165 | fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 166 | write!(f, "{:?}", self.0) 167 | } 168 | } 169 | let sets1 = inp.iter() 170 | .map(|sv| { 171 | let mut s = S::new(); 172 | for i in sv.0.iter() { 173 | s.insert(i.clone()); 174 | } 175 | Wrap(sv.clone(), Box::new(s)) 176 | }) 177 | .collect::>>(); 178 | c.bench_function_over_inputs( 179 | &format!("{}/lookup_hit", desc), 180 | |b, &Wrap(ref sv, ref s)| bench_set_rand_int_lookup::(b, &*s, &sv.0), 181 | sets1, 182 | ); 183 | eprintln!("Generating for {} (2/3)", desc); 184 | let sets2 = inp.iter() 185 | .map(|sv| { 186 | let mut s = S::new(); 187 | for i in sv.0.iter() { 188 | s.insert(i.clone()); 189 | } 190 | Wrap(sv.clone(), Box::new(s)) 191 | }) 192 | .collect::>>(); 193 | c.bench_function_over_inputs( 194 | &format!("{}/lookup_miss", desc), 195 | |b, &Wrap(ref sv, ref s)| bench_set_rand_int_lookup::(b, &*s, &sv.1), 196 | sets2, 197 | ); 198 | eprintln!("Generating for {} (3/3)", desc); 199 | use std::cell::UnsafeCell; 200 | let sets3 = inp.iter() 201 | .map(|sv| { 202 | let mut s = S::new(); 203 | for i in sv.0.iter() { 204 | s.insert(i.clone()); 205 | } 206 | Wrap(sv.clone(), Box::new(UnsafeCell::new(s))) 207 | }) 208 | .collect::>>(); 209 | unsafe { 210 | c.bench_function_over_inputs( 211 | &format!("{}/insert_remove", desc), 212 | |b, &Wrap(ref sv, ref s)| bench_set_insert_remove::(b, &mut *s.get(), &sv.0), 213 | sets3, 214 | ); 215 | } 216 | } 217 | macro_rules! bench_inner { 218 | ($c: expr, $container: tt, $ivec: expr, $ivec2: expr, $svec: expr) => {{ 219 | make_bench::>($c, format!("{}/sparse_u64", stringify!($container)), $ivec); 220 | make_bench::>($c, format!("{}/dense_u64", stringify!($container)), $ivec2); 221 | make_bench::>( 222 | $c, 223 | format!("{}/String", stringify!($container)), 224 | $svec, 225 | ); 226 | }}; 227 | } 228 | macro_rules! bench_all { 229 | ($c:expr, $ivec:expr, $ivec2:expr, $svec:expr, $( $container:tt ),+) => { 230 | $( 231 | bench_inner!($c, $container, $ivec, $ivec2, $svec); 232 | )+ 233 | } 234 | } 235 | eprintln!("Generating Ints"); 236 | let v1s: Vec> = [16 << 10, 16 << 20, 256 << 20] 237 | .iter() 238 | .map(|size: &usize| SizeVec(random_vec(*size, !0), random_vec(*size, !0))) 239 | .collect(); 240 | let v1_dense: Vec> = [16 << 10, 16 << 20, 256 << 20] 241 | .iter() 242 | .map(|size: &usize| { 243 | SizeVec( 244 | random_dense_vec(*size as u64, 0), 245 | random_dense_vec(*size as u64, *size as u64 * 2), 246 | ) 247 | }) 248 | .collect(); 249 | eprintln!("Generating Strings"); 250 | let v2s: Vec> = [16 << 10, 1 << 20, 16 << 20] 251 | .iter() 252 | // NB: random_string_vec will make random UTF8 strings, in practice asking for a string of 253 | // length 10 can give you far more than 10 bytes. 254 | .map(|size: &usize| SizeVec(random_string_vec(10, *size), random_string_vec(10, *size))) 255 | .collect(); 256 | 257 | bench_all!( 258 | c, 259 | &v1s, 260 | &v1_dense, 261 | &v2s, 262 | ARTSet, 263 | HashSet, 264 | BTreeSet, 265 | CachingARTSet 266 | ); 267 | } 268 | 269 | criterion_group!(benches, criterion_benchmark); 270 | criterion_main!(benches); 271 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/prefix_cache.rs: -------------------------------------------------------------------------------- 1 | extern crate fnv; 2 | #[cfg(feature = "print_cache_stats")] 3 | use std::cell::UnsafeCell; 4 | use std::cmp; 5 | use std::marker::PhantomData; 6 | use std::ptr; 7 | 8 | use super::art_internal::MarkedPtr; 9 | 10 | pub use self::dense_hash_set::HashSetPrefixCache; 11 | 12 | /// PrefixCache describes types that can cache pointers interior to an ART. 13 | pub trait PrefixCache { 14 | /// If true, the cache is used during ART set operations. If false, the cache is ignored. 15 | const ENABLED: bool; 16 | /// If true, lookup returning None indicates that no nodes with prefix `bs` are in the set. 17 | const COMPLETE: bool; 18 | fn new() -> Self; 19 | fn lookup(&self, bs: &[u8]) -> Option>; 20 | fn replace(&mut self, bs: &[u8], ptr: MarkedPtr) -> Option> { 21 | self.insert(bs, ptr); 22 | None 23 | } 24 | fn insert(&mut self, bs: &[u8], ptr: MarkedPtr) { 25 | let _ = self.replace(bs, ptr); 26 | } 27 | #[inline(always)] 28 | fn debug_assert_unreachable(&self, _ptr: MarkedPtr) {} 29 | } 30 | pub struct NullBuckets(PhantomData); 31 | 32 | impl PrefixCache for NullBuckets { 33 | const ENABLED: bool = false; 34 | const COMPLETE: bool = false; 35 | fn new() -> Self { 36 | NullBuckets(PhantomData) 37 | } 38 | fn lookup(&self, _: &[u8]) -> Option> { 39 | None 40 | } 41 | fn insert(&mut self, _: &[u8], _ptr: MarkedPtr) {} 42 | } 43 | 44 | mod dense_hash_set { 45 | use super::*; 46 | use super::fnv::FnvHasher; 47 | use super::super::Digital; 48 | use super::super::byteorder::{BigEndian, ByteOrder}; 49 | 50 | use std::hash::{Hash, Hasher}; 51 | use std::mem; 52 | 53 | fn read_u64(bs: &[u8]) -> u64 { 54 | debug_assert!(bs.len() <= 8); 55 | let mut arr = [0 as u8; 8]; 56 | unsafe { ptr::copy_nonoverlapping(&bs[0], &mut arr[0], cmp::min(bs.len(), 8)) }; 57 | BigEndian::read_u64(&arr[..]) 58 | } 59 | 60 | pub struct HashSetPrefixCache(DenseHashTable>); 61 | impl PrefixCache for HashSetPrefixCache { 62 | const ENABLED: bool = true; 63 | const COMPLETE: bool = true; 64 | fn new() -> Self { 65 | HashSetPrefixCache(DenseHashTable::new()) 66 | } 67 | 68 | #[cfg(debug_assertions)] 69 | fn debug_assert_unreachable(&self, ptr: MarkedPtr) { 70 | for elt in self.0.buckets.iter() { 71 | if elt.ptr == ptr { 72 | assert!( 73 | self.0.lookup(&elt.prefix).is_some(), 74 | "attempted to look up {:?}:{:?} but failed", 75 | elt.prefix, 76 | elt.ptr 77 | ); 78 | let l = self.0.lookup(&elt.prefix).unwrap(); 79 | assert!(l.ptr == elt.ptr, "got {:?} != elt {:?}", l, elt); 80 | assert!( 81 | elt.ptr != ptr, 82 | "Found ptr {:?} in elt with prefix {:?} [{:?}]", 83 | ptr, 84 | elt.prefix, 85 | elt.prefix.digits().collect::>().as_slice() 86 | ) 87 | } 88 | } 89 | } 90 | 91 | fn lookup(&self, bs: &[u8]) -> Option> { 92 | let prefix = read_u64(bs); 93 | let res = self.0.lookup(&prefix).map(|elt| elt.ptr.clone()); 94 | #[cfg(debug_assertions)] 95 | unsafe { 96 | if let Some(Err(inner)) = res.as_ref() 97 | .map(|x| x.get().expect("stored pointer should be non-null")) 98 | { 99 | assert!( 100 | inner.children != !0, 101 | "Returning an expired node {:?} (ty={:?})", 102 | res, 103 | inner.typ 104 | ); 105 | } 106 | } 107 | res 108 | } 109 | 110 | fn insert(&mut self, bs: &[u8], ptr: MarkedPtr) { 111 | let prefix = read_u64(bs); 112 | if ptr.is_null() { 113 | self.0.delete(&prefix); 114 | debug_assert!(self.lookup(bs).is_none()); 115 | } else { 116 | let _ = self.0.insert(MarkedElt { 117 | prefix: prefix, 118 | ptr: ptr, 119 | }); 120 | } 121 | } 122 | 123 | fn replace(&mut self, bs: &[u8], ptr: MarkedPtr) -> Option> { 124 | let prefix = read_u64(bs); 125 | if ptr.is_null() { 126 | self.0.delete(&prefix) 127 | } else { 128 | match self.0.insert(MarkedElt { 129 | prefix: prefix, 130 | ptr: ptr, 131 | }) { 132 | Ok(()) => None, 133 | Err(t) => Some(t), 134 | } 135 | }.map(|t| t.ptr) 136 | } 137 | } 138 | 139 | trait DHTE { 140 | type Key; 141 | fn null() -> Self; 142 | fn tombstone() -> Self; 143 | fn is_null(&self) -> bool; 144 | fn is_tombstone(&self) -> bool; 145 | fn key(&self) -> &Self::Key; 146 | } 147 | 148 | const MARKED_TOMBSTONE: usize = !0; 149 | struct MarkedElt { 150 | prefix: u64, 151 | ptr: MarkedPtr, 152 | } 153 | impl ::std::fmt::Debug for MarkedElt { 154 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { 155 | write!( 156 | f, 157 | "MarkedElt{{ {:?}, {:?} }}", 158 | self.prefix.digits().collect::>().as_slice(), 159 | self.ptr 160 | ) 161 | } 162 | } 163 | 164 | impl DHTE for MarkedElt { 165 | type Key = u64; 166 | fn null() -> Self { 167 | MarkedElt { 168 | prefix: 0, 169 | ptr: MarkedPtr::null(), 170 | } 171 | } 172 | fn tombstone() -> Self { 173 | MarkedElt { 174 | prefix: 0, 175 | ptr: MarkedPtr::from_leaf(MARKED_TOMBSTONE as *mut T), 176 | } 177 | } 178 | 179 | fn is_null(&self) -> bool { 180 | self.ptr.is_null() 181 | } 182 | fn is_tombstone(&self) -> bool { 183 | self.ptr.raw_eq(MARKED_TOMBSTONE) 184 | } 185 | fn key(&self) -> &Self::Key { 186 | &self.prefix 187 | } 188 | } 189 | 190 | /// A bare-bones implementation of Google's dense_hash_set. Not a full-featured map, but 191 | /// contains sufficient functionality to be used as a PrefixCache 192 | /// 193 | /// TODO: explore optimizing this more (for time or for space). 194 | struct DenseHashTable { 195 | buckets: Vec, 196 | len: usize, 197 | set: usize, 198 | } 199 | 200 | impl DenseHashTable 201 | where 202 | T::Key: Eq + Hash, 203 | { 204 | fn next_probe(hash: usize, i: usize) -> usize { 205 | // hash + i 206 | hash + (i + i * i) / 2 207 | } 208 | 209 | fn new() -> Self { 210 | DenseHashTable { 211 | buckets: Vec::new(), 212 | len: 0, 213 | set: 0, 214 | } 215 | } 216 | 217 | fn seek( 218 | &self, 219 | k: &T::Key, 220 | ) -> ( 221 | Option<*mut T>, /* first tombstone */ 222 | Option<*mut T>, /* matching or null */ 223 | ) { 224 | let mut tombstone = None; 225 | let l = self.buckets.len(); 226 | debug_assert!(l.is_power_of_two()); 227 | let hash = { 228 | let mut hasher = FnvHasher::default(); 229 | k.hash(&mut hasher); 230 | hasher.finish() as usize 231 | }; 232 | let mut ix = hash; 233 | let mut times = 0; 234 | while times < l { 235 | ix &= l - 1; 236 | debug_assert!(ix < self.buckets.len()); 237 | times += 1; 238 | let bucket = unsafe { self.buckets.get_unchecked(ix) }; 239 | let bucket_raw = bucket as *const T as *mut T; 240 | if tombstone.is_none() && bucket.is_tombstone() { 241 | tombstone = Some(bucket_raw); 242 | } else if bucket.is_null() || bucket.key() == k { 243 | return (tombstone, Some(bucket_raw)); 244 | } 245 | ix = Self::next_probe(hash, times); 246 | } 247 | (tombstone, None) 248 | } 249 | 250 | fn grow(&mut self) { 251 | debug_assert!(self.set >= self.len); 252 | let old_len = if self.buckets.len() == 0 { 253 | self.buckets.push(T::null()); 254 | return; 255 | } else if self.buckets.len() < 32 256 | || (self.set as i64) - (self.len as i64) < (self.buckets.len() as i64 / 4) 257 | { 258 | // actually grow. If this condition is not met, then we just re-hash 259 | let l = self.buckets.len(); 260 | self.buckets.extend((0..l).map(|_| T::null())); 261 | l 262 | } else { 263 | self.buckets.len() 264 | }; 265 | debug_assert!(self.buckets.len().is_power_of_two()); 266 | debug_assert!(old_len.is_power_of_two()); 267 | let mut v = Vec::with_capacity(self.len); 268 | for i in &mut self.buckets[0..old_len] { 269 | if i.is_null() { 270 | continue; 271 | } 272 | if i.is_tombstone() { 273 | *i = T::null(); 274 | continue; 275 | } 276 | let mut t = T::null(); 277 | mem::swap(i, &mut t); 278 | v.push(t); 279 | } 280 | self.set = 0; 281 | self.len = 0; 282 | for elt in v.into_iter() { 283 | let _res = self.insert(elt); 284 | debug_assert!(_res.is_ok()); 285 | } 286 | } 287 | 288 | fn lookup(&self, k: &T::Key) -> Option<&T> { 289 | if self.buckets.len() == 0 { 290 | return None; 291 | } 292 | let (_, b_opt) = self.seek(k); 293 | b_opt.and_then(|b| unsafe { 294 | if (*b).is_null() { 295 | None 296 | } else { 297 | Some(&*b) 298 | } 299 | }) 300 | } 301 | 302 | fn delete(&mut self, k: &T::Key) -> Option { 303 | if self.buckets.len() == 0 { 304 | return None; 305 | } 306 | let (_, b_opt) = self.seek(k); 307 | b_opt.and_then(|b| unsafe { 308 | if (*b).is_null() { 309 | None 310 | } else { 311 | let mut tomb = T::tombstone(); 312 | mem::swap(&mut *b, &mut tomb); 313 | self.len -= 1; 314 | Some(tomb) 315 | } 316 | }) 317 | } 318 | 319 | fn insert(&mut self, mut t: T) -> Result<(), T> { 320 | if self.set >= self.buckets.len() / 2 { 321 | self.grow(); 322 | } 323 | debug_assert!(!t.is_null()); 324 | debug_assert!(!t.is_tombstone()); 325 | let (tmb, b_opt) = self.seek(t.key()); 326 | unsafe { 327 | let bucket = b_opt.unwrap(); 328 | if (*bucket).is_null() { 329 | // t is not already in the table. We insert it somewhere 330 | if let Some(tombstone_bucket) = tmb { 331 | // there was a tombstone earlier in the probe chain. We overwrite its 332 | // value. 333 | *tombstone_bucket = t; 334 | } else { 335 | // we insert it into the new slot 336 | *bucket = t; 337 | self.set += 1; 338 | } 339 | self.len += 1; 340 | Ok(()) 341 | } else { 342 | // t is already in the table, we simply swap in the new value 343 | mem::swap(&mut *bucket, &mut t); 344 | Err(t) 345 | } 346 | } 347 | } 348 | } 349 | 350 | #[cfg(test)] 351 | mod tests { 352 | use super::*; 353 | use super::super::super::rand; 354 | use super::super::super::rand::Rng; 355 | fn random_vec(max_val: usize, len: usize) -> Vec { 356 | let mut rng = rand::thread_rng(); 357 | (0..len) 358 | .map(|_| rng.gen_range::(0, max_val)) 359 | .collect() 360 | } 361 | 362 | #[derive(Debug)] 363 | struct UsizeElt(usize, usize); 364 | impl DHTE for UsizeElt { 365 | type Key = usize; 366 | fn null() -> Self { 367 | UsizeElt(0, 0) 368 | } 369 | fn tombstone() -> Self { 370 | UsizeElt(0, 2) 371 | } 372 | fn is_null(&self) -> bool { 373 | self.1 == 0 374 | } 375 | fn is_tombstone(&self) -> bool { 376 | self.1 == 2 377 | } 378 | fn key(&self) -> &Self::Key { 379 | &self.0 380 | } 381 | } 382 | 383 | impl UsizeElt { 384 | fn new(u: usize) -> Self { 385 | UsizeElt(u, 1) 386 | } 387 | } 388 | 389 | #[test] 390 | fn dense_hash_set_smoke_test() { 391 | let mut s = DenseHashTable::::new(); 392 | let mut v1 = random_vec(!0, 1 << 18); 393 | for item in v1.iter() { 394 | let _ = s.insert(UsizeElt::new(*item)); 395 | assert!( 396 | s.lookup(item).is_some(), 397 | "lookup failed immediately for {:?}", 398 | *item 399 | ); 400 | } 401 | let mut missing = Vec::new(); 402 | for item in v1.iter() { 403 | if s.lookup(item).is_none() { 404 | missing.push(*item) 405 | } 406 | } 407 | assert_eq!(missing.len(), 0, "missing={:?}", missing); 408 | v1.sort(); 409 | v1.dedup_by_key(|x| *x); 410 | let mut v2 = Vec::new(); 411 | for _ in 0..(1 << 17) { 412 | if let Some(x) = v1.pop() { 413 | v2.push(x) 414 | } else { 415 | break; 416 | } 417 | } 418 | let mut failures = 0; 419 | for i in v2.iter() { 420 | let mut fail = 0; 421 | if s.lookup(i).is_none() { 422 | eprintln!("{:?} no longer in the set!", *i); 423 | fail = 1; 424 | } 425 | let res = s.delete(i); 426 | if res.is_none() { 427 | fail = 1; 428 | } 429 | if s.lookup(i).is_some() { 430 | fail = 1; 431 | } 432 | failures += fail; 433 | } 434 | assert_eq!(failures, 0); 435 | let mut failed = false; 436 | for i in v2.iter() { 437 | if s.lookup(i).is_some() { 438 | eprintln!("Deleted {:?}, but it's still there!", *i); 439 | failed = true; 440 | }; 441 | } 442 | assert!(!failed); 443 | for i in v1.iter() { 444 | assert!( 445 | s.lookup(i).is_some(), 446 | "Didn't delete {:?}, but it is gone!", 447 | *i 448 | ); 449 | } 450 | } 451 | } 452 | } 453 | -------------------------------------------------------------------------------- /src/art_internal.rs: -------------------------------------------------------------------------------- 1 | use std::cmp; 2 | use std::marker::PhantomData; 3 | use std::mem; 4 | use std::ptr; 5 | use super::common::Digital; 6 | 7 | extern crate simd; 8 | 9 | #[cfg(target_arch = "x86")] 10 | use std::arch::x86::_mm_movemask_epi8; 11 | #[cfg(target_arch = "x86_64")] 12 | use std::arch::x86_64::_mm_movemask_epi8; 13 | use super::smallvec::{Array, SmallVec}; 14 | 15 | pub const PREFIX_LEN: usize = 8; 16 | /// used by the `with_node_mut` macro 17 | pub type RawMutRef<'a, T> = &'a mut RawNode; 18 | 19 | /// used by the `with_node` macro 20 | pub type RawRef<'a, T> = &'a RawNode; 21 | 22 | /// a non-owning reference to a `ChildPtr` 23 | pub struct MarkedPtr(usize, PhantomData); 24 | pub use self::node_variants::{NODE_16, NODE_256, NODE_4, NODE_48, Node16, Node256, Node48, 25 | NodeType}; 26 | 27 | impl PartialEq for MarkedPtr { 28 | fn eq(&self, other: &MarkedPtr) -> bool { 29 | self.0 == other.0 30 | } 31 | } 32 | 33 | impl Eq for MarkedPtr {} 34 | 35 | pub trait Element { 36 | type Key: for<'a> Digital<'a> + PartialOrd; 37 | fn key(&self) -> &Self::Key; 38 | fn matches(&self, k: &Self::Key) -> bool; 39 | fn replace_matching(&mut self, other: &mut Self); 40 | } 41 | 42 | impl Clone for MarkedPtr { 43 | fn clone(&self) -> Self { 44 | MarkedPtr(self.0, PhantomData) 45 | } 46 | } 47 | pub struct ChildPtr(MarkedPtr); 48 | 49 | impl ::std::ops::Deref for ChildPtr { 50 | type Target = MarkedPtr; 51 | fn deref(&self) -> &MarkedPtr { 52 | &self.0 53 | } 54 | } 55 | 56 | impl ::std::ops::DerefMut for ChildPtr { 57 | fn deref_mut(&mut self) -> &mut MarkedPtr { 58 | &mut self.0 59 | } 60 | } 61 | 62 | impl Drop for ChildPtr { 63 | fn drop(&mut self) { 64 | unsafe { 65 | match self.get_mut() { 66 | None => return, 67 | Some(Ok(x)) => mem::drop(Box::from_raw(x)), 68 | // with_node_mut! will "un-erase" the actual type of the RawNode. We want to call drop 69 | // on that to ensure all children are dropped 70 | // ... and to avoid undefined behavior, as we could wind up passing free the wrong size :) 71 | Some(Err(x)) => with_node_mut!(x, nod, mem::drop(Box::from_raw(nod)), T), 72 | } 73 | } 74 | } 75 | } 76 | 77 | impl ::std::fmt::Debug for MarkedPtr { 78 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { 79 | unsafe { 80 | write!( 81 | f, 82 | "MarkedPtr({})", 83 | match self.get_raw() { 84 | None => String::from("null"), 85 | Some(Ok(leaf_ptr)) => format!("leaf:{:?}", leaf_ptr), 86 | Some(Err(inner)) => format!("inner:{:?}", *inner), 87 | } 88 | ) 89 | } 90 | } 91 | } 92 | 93 | impl ::std::fmt::Debug for ChildPtr { 94 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { 95 | write!(f, "ChildPtr({:?})", (self.0).0 as *mut ()) 96 | } 97 | } 98 | 99 | impl ChildPtr { 100 | pub fn null() -> Self { 101 | ChildPtr(MarkedPtr::null()) 102 | } 103 | 104 | pub fn from_node(p: *mut RawNode) -> Self { 105 | ChildPtr(MarkedPtr::from_node(p)) 106 | } 107 | 108 | pub fn from_leaf(p: *mut T) -> Self { 109 | ChildPtr(MarkedPtr::from_leaf(p)) 110 | } 111 | 112 | pub fn swap_null(&mut self) -> Self { 113 | let mut self_ptr = ChildPtr::null(); 114 | mem::swap(self, &mut self_ptr); 115 | self_ptr 116 | } 117 | 118 | pub unsafe fn to_marked(&self) -> MarkedPtr { 119 | ptr::read(&self.0) 120 | } 121 | } 122 | 123 | impl MarkedPtr { 124 | pub fn null() -> Self { 125 | MarkedPtr(0, PhantomData) 126 | } 127 | 128 | pub fn from_node(p: *mut RawNode) -> Self { 129 | debug_assert!(!p.is_null()); 130 | MarkedPtr(p as usize, PhantomData) 131 | } 132 | 133 | pub fn from_leaf(p: *mut T) -> Self { 134 | debug_assert!(!p.is_null()); 135 | MarkedPtr((p as usize) | 1, PhantomData) 136 | } 137 | 138 | pub fn is_null(&self) -> bool { 139 | self.0 == 0 140 | } 141 | 142 | pub fn raw_eq(&self, other: usize) -> bool { 143 | self.0 == other 144 | } 145 | 146 | pub unsafe fn get(&self) -> Option>> { 147 | if self.0 == 0 { 148 | None 149 | } else if self.0 & 1 == 1 { 150 | Some(Ok(&*((self.0 & !1) as *const T))) 151 | } else { 152 | Some(Err(&*(self.0 as *const RawNode<()>))) 153 | } 154 | } 155 | 156 | pub unsafe fn get_raw(&self) -> Option>> { 157 | if self.0 == 0 { 158 | None 159 | } else if self.0 & 1 == 1 { 160 | Some(Ok((self.0 & !1) as *mut T)) 161 | } else { 162 | Some(Err(self.0 as *mut RawNode<()>)) 163 | } 164 | } 165 | 166 | pub unsafe fn get_mut(&mut self) -> Option>> { 167 | if self.0 == 0 { 168 | None 169 | } else if self.0 & 1 == 1 { 170 | Some(Ok(&mut *((self.0 & !1) as *mut T))) 171 | } else { 172 | Some(Err(&mut *(self.0 as *mut RawNode<()>))) 173 | } 174 | } 175 | } 176 | 177 | unsafe fn place_in_hole_at(slice: &mut [T], at: usize, v: T, buff_len: usize) { 178 | let raw_p = slice.get_unchecked_mut(0) as *mut T; 179 | let target = raw_p.offset(at as isize); 180 | ptr::copy(target, raw_p.offset(at as isize + 1), buff_len - at - 1); 181 | ptr::write(target, v); 182 | } 183 | 184 | #[cfg(test)] 185 | mod place_test { 186 | use super::*; 187 | 188 | #[test] 189 | fn place_in_hole_test() { 190 | let mut v1 = vec![0, 1, 3, 4, 0]; 191 | let len = v1.len(); 192 | unsafe { 193 | place_in_hole_at(&mut v1[..], 2, 2, len); 194 | } 195 | assert_eq!(v1, vec![0, 1, 2, 3, 4]); 196 | } 197 | } 198 | 199 | #[repr(C)] 200 | #[derive(Debug)] 201 | pub struct RawNode