├── roaring ├── LICENSE-MIT ├── LICENSE-APACHE ├── tests │ ├── bitmapwithruns.bin │ ├── bitmapwithoutruns.bin │ ├── treemap_clone.rs │ ├── treemap_serialization.rs │ ├── clone.rs │ ├── treemap_select.rs │ ├── select.rs │ ├── push.rs │ ├── treemap_rank.rs │ ├── treemap_size_hint.rs │ ├── treemap_is_disjoint.rs │ ├── rank.rs │ ├── treemap_ops.rs │ ├── is_disjoint.rs │ ├── is_subset.rs │ ├── treemap_is_subset.rs │ ├── treemap_union_with.rs │ ├── treemap_intersect_with.rs │ ├── size_hint.rs │ ├── ops.rs │ ├── union_with.rs │ ├── intersect_with.rs │ ├── range_checks.rs │ ├── iter_range.rs │ ├── treemap_difference_with.rs │ ├── treemap_symmetric_difference_with.rs │ ├── treemap_lib.rs │ ├── symmetric_difference_with.rs │ ├── difference_with.rs │ ├── treemap_iter.rs │ ├── lib.rs │ ├── iter.rs │ ├── treemap_iter_advance_to.rs │ └── iter_advance_to.rs ├── src │ ├── treemap │ │ ├── fmt.rs │ │ ├── arbitrary.rs │ │ ├── mod.rs │ │ ├── serde.rs │ │ ├── util.rs │ │ ├── cmp.rs │ │ ├── serialization.rs │ │ └── multiops.rs │ ├── bitmap │ │ ├── fmt.rs │ │ ├── mod.rs │ │ ├── serde.rs │ │ ├── store │ │ │ └── array_store │ │ │ │ ├── visitor.rs │ │ │ │ └── scalar.rs │ │ ├── cmp.rs │ │ ├── statistics.rs │ │ ├── util.rs │ │ ├── arbitrary.rs │ │ └── ops_with_serialized.rs │ └── lib.rs └── Cargo.toml ├── rustfmt.toml ├── .gitignore ├── fuzz ├── .gitignore ├── rust-toolchain.toml ├── Cargo.toml ├── fuzz_targets │ └── against_croaring.rs └── Cargo.lock ├── benchmarks ├── .gitignore ├── Cargo.toml └── benches │ └── datasets │ └── mod.rs ├── .editorconfig ├── Cargo.toml ├── LICENSE-MIT ├── README.md ├── .github └── workflows │ └── test.yml └── LICENSE-APACHE /roaring/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /roaring/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | use_small_heuristics = "Max" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /proptest-regressions 4 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | -------------------------------------------------------------------------------- /fuzz/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly" 3 | -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /real-roaring-datasets 4 | -------------------------------------------------------------------------------- /roaring/tests/bitmapwithruns.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RoaringBitmap/roaring-rs/HEAD/roaring/tests/bitmapwithruns.bin -------------------------------------------------------------------------------- /roaring/tests/bitmapwithoutruns.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RoaringBitmap/roaring-rs/HEAD/roaring/tests/bitmapwithoutruns.bin -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.rs] 2 | end_of_line = lf 3 | trim_trailing_whitespace = true 4 | insert_final_newline = true 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 4 8 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "roaring-fuzz" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [dependencies] 11 | libfuzzer-sys = { version = "0.4.9", features = ["arbitrary-derive"] } 12 | roaring = { path = "../roaring" } 13 | croaring = "2.0" 14 | 15 | [features] 16 | simd = ["roaring/simd"] 17 | 18 | [[bin]] 19 | name = "against_croaring" 20 | path = "fuzz_targets/against_croaring.rs" 21 | test = false 22 | doc = false 23 | bench = false 24 | 25 | [workspace] 26 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ "benchmarks", "roaring" ] 3 | resolver = "2" 4 | 5 | [workspace.dependencies] 6 | roaring = { path = "roaring" } 7 | 8 | bytemuck = "1.21.0" 9 | byteorder = "1.5.0" 10 | criterion = "0.5" 11 | git2 = { version = "0.20", default-features = false } 12 | indicatif = "0.17" 13 | itertools = "0.14" 14 | once_cell = "1.20" 15 | postcard = { version = "1.1", features = [ "alloc" ] } 16 | proptest = "1.6.0" 17 | serde = "1.0.217" 18 | serde_json = "1.0.138" 19 | zip = { version = "0.6", default-features = false } 20 | 21 | [profile.test] 22 | opt-level = 2 23 | -------------------------------------------------------------------------------- /roaring/src/treemap/fmt.rs: -------------------------------------------------------------------------------- 1 | use core::fmt; 2 | 3 | use crate::RoaringTreemap; 4 | 5 | #[cfg(not(feature = "std"))] 6 | use alloc::vec::Vec; 7 | 8 | impl fmt::Debug for RoaringTreemap { 9 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 10 | if self.len() < 16 { 11 | write!(f, "RoaringTreemap<{:?}>", self.iter().collect::>()) 12 | } else { 13 | write!( 14 | f, 15 | "RoaringTreemap<{:?} values between {:?} and {:?}>", 16 | self.len(), 17 | self.min().unwrap(), 18 | self.max().unwrap() 19 | ) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /roaring/src/bitmap/fmt.rs: -------------------------------------------------------------------------------- 1 | use core::fmt; 2 | 3 | use crate::RoaringBitmap; 4 | 5 | #[cfg(not(feature = "std"))] 6 | use alloc::vec::Vec; 7 | 8 | impl fmt::Debug for RoaringBitmap { 9 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 10 | if self.len() < 16 { 11 | write!(f, "RoaringBitmap<{:?}>", self.iter().collect::>()) 12 | } else { 13 | write!( 14 | f, 15 | "RoaringBitmap<{:?} values between {:?} and {:?} in {:?} containers>", 16 | self.len(), 17 | self.min().unwrap(), 18 | self.max().unwrap(), 19 | self.containers.len(), 20 | ) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /roaring/src/treemap/arbitrary.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod test { 3 | use crate::{RoaringBitmap, RoaringTreemap}; 4 | use proptest::collection::btree_map; 5 | use proptest::prelude::*; 6 | 7 | impl RoaringTreemap { 8 | prop_compose! { 9 | pub(crate) fn arbitrary()(map in btree_map(0u32..=16, RoaringBitmap::arbitrary(), 0usize..=16)) -> RoaringTreemap { 10 | // we’re NEVER supposed to start with a treemap containing empty bitmaps 11 | // Since we can’t configure this in arbitrary we’re simply going to ignore the generated empty bitmaps 12 | let map = map.into_iter().filter(|(_, v)| !v.is_empty()).collect(); 13 | RoaringTreemap { map } 14 | } 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /benchmarks/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "benchmarks" 3 | description = "An external library for benchmarking the roaring crate with real datasets" 4 | version = "0.1.0" 5 | authors = ["Kerollmops "] 6 | edition = "2021" 7 | rust-version = "1.65.0" 8 | publish = false 9 | 10 | [dependencies] 11 | roaring = { workspace = true } 12 | 13 | [dev-dependencies] 14 | criterion = { workspace = true, features = ["html_reports"] } 15 | git2 = { workspace = true, default-features = false, features = ["https", "vendored-openssl"] } 16 | indicatif = { workspace = true } 17 | itertools = { workspace = true } 18 | once_cell = { workspace = true } 19 | zip = { workspace = true, default-features = false, features = ["deflate"] } 20 | 21 | [features] 22 | simd = ["roaring/simd"] 23 | 24 | [[bench]] 25 | name = "lib" 26 | harness = false 27 | -------------------------------------------------------------------------------- /roaring/src/treemap/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::RoaringBitmap; 2 | use alloc::collections::BTreeMap; 3 | 4 | mod fmt; 5 | mod multiops; 6 | mod util; 7 | 8 | // Order of these modules matters as it determines the `impl` blocks order in 9 | // the docs 10 | mod arbitrary; 11 | mod cmp; 12 | mod inherent; 13 | mod iter; 14 | mod ops; 15 | #[cfg(feature = "serde")] 16 | mod serde; 17 | #[cfg(feature = "std")] 18 | mod serialization; 19 | 20 | pub use self::iter::{BitmapIter, IntoIter, Iter}; 21 | 22 | /// A compressed bitmap with u64 values. 23 | /// Implemented as a `BTreeMap` of `RoaringBitmap`s. 24 | /// 25 | /// # Examples 26 | /// 27 | /// ```rust 28 | /// use roaring::RoaringTreemap; 29 | /// 30 | /// let mut rb = RoaringTreemap::new(); 31 | /// 32 | /// // insert all primes less than 10 33 | /// rb.insert(2); 34 | /// rb.insert(3); 35 | /// rb.insert(5); 36 | /// rb.insert(7); 37 | /// println!("total bits set to true: {}", rb.len()); 38 | /// ``` 39 | #[derive(PartialEq)] 40 | pub struct RoaringTreemap { 41 | map: BTreeMap, 42 | } 43 | -------------------------------------------------------------------------------- /roaring/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "roaring" 3 | version = "0.11.3" 4 | # When changing this value don't forget to change the MSRV test in `.github/workflows/test.yml`!! 5 | rust-version = "1.82.0" 6 | authors = ["Wim Looman ", "Kerollmops "] 7 | description = "A better compressed bitset - pure Rust implementation" 8 | 9 | documentation = "https://docs.rs/roaring" 10 | repository = "https://github.com/RoaringBitmap/roaring-rs" 11 | 12 | readme = "../README.md" 13 | keywords = ["roaring", "data-structure", "bitmap"] 14 | categories = ["data-structures"] 15 | edition = "2021" 16 | 17 | license = "MIT OR Apache-2.0" 18 | 19 | [dependencies] 20 | bytemuck = { workspace = true, optional = true } 21 | byteorder = { workspace = true, optional = true } 22 | serde = { workspace = true, optional = true } 23 | 24 | [features] 25 | default = ["std"] 26 | serde = ["dep:serde", "std"] 27 | simd = [] 28 | std = ["dep:bytemuck", "dep:byteorder"] 29 | 30 | [dev-dependencies] 31 | proptest = { workspace = true } 32 | serde_json = { workspace = true } 33 | postcard = { workspace = true } 34 | -------------------------------------------------------------------------------- /roaring/tests/treemap_clone.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | #[allow(clippy::redundant_clone)] 6 | fn array() { 7 | let original = (0..2000).collect::(); 8 | let clone = original.clone(); 9 | 10 | assert_eq!(clone, original); 11 | } 12 | 13 | #[test] 14 | #[allow(clippy::redundant_clone)] 15 | fn bitmap() { 16 | let original = (0..6000).collect::(); 17 | let clone = original.clone(); 18 | 19 | assert_eq!(clone, original); 20 | } 21 | 22 | #[test] 23 | #[allow(clippy::redundant_clone)] 24 | fn arrays() { 25 | let original = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) 26 | .collect::(); 27 | let clone = original.clone(); 28 | 29 | assert_eq!(clone, original); 30 | } 31 | 32 | #[test] 33 | #[allow(clippy::redundant_clone)] 34 | fn bitmaps() { 35 | let original = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) 36 | .collect::(); 37 | let clone = original.clone(); 38 | 39 | assert_eq!(clone, original); 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 2016 The roaring-rs developers. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 8 | of the Software, and to permit persons to whom the Software is furnished to do 9 | so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | -------------------------------------------------------------------------------- /roaring/tests/treemap_serialization.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "std")] 2 | 3 | use roaring::RoaringTreemap; 4 | 5 | fn serialize_deserialize(dataset: Dataset) 6 | where 7 | Dataset: IntoIterator, 8 | I: Iterator, 9 | { 10 | let rb = RoaringTreemap::from_iter(dataset); 11 | 12 | let mut buffer = vec![]; 13 | rb.serialize_into(&mut buffer).unwrap(); 14 | 15 | assert_eq!(buffer.len(), rb.serialized_size()); 16 | 17 | let new_rb = RoaringTreemap::deserialize_from(&buffer[..]).unwrap(); 18 | 19 | assert_eq!(rb, new_rb); 20 | } 21 | 22 | #[test] 23 | fn empty() { 24 | serialize_deserialize(vec![]) 25 | } 26 | 27 | #[test] 28 | fn basic() { 29 | serialize_deserialize(vec![1, 2, 3, 4, 5, 100, 1000]) 30 | } 31 | 32 | #[test] 33 | fn basic_2() { 34 | serialize_deserialize(vec![1, 2, 3, 4, 5, 100, 1000, 10000, 100000, 1000000]) 35 | } 36 | 37 | #[test] 38 | fn basic_3() { 39 | let u32max = u32::MAX as u64; 40 | serialize_deserialize( 41 | vec![1, 2, 3, 4, 5, 100, 1000, 10000, 100000, 1000000, u32max + 10, u32max << 10] 42 | .into_iter() 43 | .chain(u32max..(u32max + 2 * (1 << 16))), 44 | ) 45 | } 46 | -------------------------------------------------------------------------------- /roaring/src/bitmap/mod.rs: -------------------------------------------------------------------------------- 1 | mod arbitrary; 2 | mod container; 3 | mod fmt; 4 | mod multiops; 5 | mod proptests; 6 | mod statistics; 7 | mod store; 8 | mod util; 9 | 10 | // Order of these modules matters as it determines the `impl` blocks order in 11 | // the docs 12 | mod cmp; 13 | mod inherent; 14 | mod iter; 15 | mod ops; 16 | #[cfg(feature = "std")] 17 | mod ops_with_serialized; 18 | #[cfg(feature = "serde")] 19 | mod serde; 20 | #[cfg(feature = "std")] 21 | mod serialization; 22 | 23 | use self::cmp::Pairs; 24 | pub use self::iter::IntoIter; 25 | pub use self::iter::Iter; 26 | pub use self::statistics::Statistics; 27 | 28 | #[cfg(not(feature = "std"))] 29 | use alloc::vec::Vec; 30 | 31 | /// A compressed bitmap using the [Roaring bitmap compression scheme](https://roaringbitmap.org/). 32 | /// 33 | /// # Examples 34 | /// 35 | /// ```rust 36 | /// use roaring::RoaringBitmap; 37 | /// 38 | /// let mut rb = RoaringBitmap::new(); 39 | /// 40 | /// // insert all primes less than 10 41 | /// rb.insert(2); 42 | /// rb.insert(3); 43 | /// rb.insert(5); 44 | /// rb.insert(7); 45 | /// println!("total bits set to true: {}", rb.len()); 46 | /// ``` 47 | #[derive(PartialEq)] 48 | pub struct RoaringBitmap { 49 | containers: Vec, 50 | } 51 | -------------------------------------------------------------------------------- /roaring/tests/clone.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | #[allow(clippy::redundant_clone)] 6 | fn array() { 7 | let original = (0..2000).collect::(); 8 | let clone = original.clone(); 9 | 10 | assert_eq!(clone, original); 11 | } 12 | 13 | #[test] 14 | #[allow(clippy::redundant_clone)] 15 | fn bitmap() { 16 | let original = (0..6000).collect::(); 17 | let clone = original.clone(); 18 | 19 | assert_eq!(clone, original); 20 | } 21 | 22 | #[test] 23 | #[allow(clippy::redundant_clone)] 24 | fn arrays() { 25 | let original = (0..2000) 26 | .chain(1_000_000..1_002_000) 27 | .chain(2_000_000..2_001_000) 28 | .collect::(); 29 | let clone = original.clone(); 30 | 31 | assert_eq!(clone, original); 32 | } 33 | 34 | #[test] 35 | #[allow(clippy::redundant_clone)] 36 | fn bitmaps() { 37 | let original = (0..6000) 38 | .chain(1_000_000..1_012_000) 39 | .chain(2_000_000..2_010_000) 40 | .collect::(); 41 | let clone = original.clone(); 42 | 43 | assert_eq!(clone, original); 44 | } 45 | 46 | #[test] 47 | #[allow(clippy::redundant_clone)] 48 | fn runs() { 49 | let mut original = 50 | RoaringBitmap::from_iter((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)); 51 | original.optimize(); 52 | let clone = original.clone(); 53 | 54 | assert_eq!(clone, original); 55 | } 56 | -------------------------------------------------------------------------------- /roaring/tests/treemap_select.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | 3 | use proptest::collection::btree_set; 4 | use proptest::prelude::*; 5 | use roaring::RoaringTreemap; 6 | 7 | #[test] 8 | fn select() { 9 | let bitmap = (0..2000).collect::(); 10 | 11 | assert_eq!(bitmap.select(0), Some(0)); 12 | } 13 | 14 | #[test] 15 | fn select_multiple_bitmap() { 16 | let mut bitmap = (0..100_000).collect::(); 17 | bitmap.append(u32::MAX as u64..u32::MAX as u64 + 100_000).expect("sorted integers"); 18 | 19 | assert_eq!(bitmap.select(0), Some(0)); 20 | assert_eq!(bitmap.select(99_999), Some(99_999)); 21 | assert_eq!(bitmap.select(100_000), Some(u32::MAX as u64)); 22 | assert_eq!(bitmap.select(199_999), Some(u32::MAX as u64 + 99_999)); 23 | assert_eq!(bitmap.select(200_000), None); 24 | assert_eq!(bitmap.select(u64::MAX), None); 25 | } 26 | 27 | #[test] 28 | fn select_empty() { 29 | let bitmap = RoaringTreemap::new(); 30 | 31 | assert_eq!(bitmap.select(0), None); 32 | assert_eq!(bitmap.select(1024), None); 33 | assert_eq!(bitmap.select(u64::MAX), None); 34 | } 35 | 36 | proptest! { 37 | #[test] 38 | fn proptest_select(values in btree_set(any::(), 1000)) { 39 | let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 40 | for (i, value) in values.iter().cloned().enumerate() { 41 | prop_assert_eq!(bitmap.select(i as u64), Some(value)); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /roaring/tests/select.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | 3 | use proptest::collection::btree_set; 4 | use proptest::prelude::*; 5 | use roaring::RoaringBitmap; 6 | 7 | #[test] 8 | fn select() { 9 | let bitmap = (0..2000).collect::(); 10 | 11 | assert_eq!(bitmap.select(0), Some(0)); 12 | } 13 | 14 | #[test] 15 | fn select_array() { 16 | let bitmap = (0..2000).collect::(); 17 | 18 | assert_eq!(bitmap.select(0), Some(0)); 19 | assert_eq!(bitmap.select(100), Some(100)); 20 | assert_eq!(bitmap.select(1000), Some(1000)); 21 | assert_eq!(bitmap.select(1999), Some(1999)); 22 | assert_eq!(bitmap.select(2000), None); 23 | } 24 | 25 | #[test] 26 | fn select_bitmap() { 27 | let bitmap = (0..100_000).collect::(); 28 | 29 | assert_eq!(bitmap.select(0), Some(0)); 30 | assert_eq!(bitmap.select(63), Some(63)); 31 | assert_eq!(bitmap.select(1000), Some(1000)); 32 | assert_eq!(bitmap.select(65535), Some(65535)); 33 | } 34 | 35 | #[test] 36 | fn select_empty() { 37 | let bitmap = RoaringBitmap::new(); 38 | 39 | assert_eq!(bitmap.select(0), None); 40 | assert_eq!(bitmap.select(1024), None); 41 | assert_eq!(bitmap.select(u32::MAX), None); 42 | } 43 | 44 | proptest! { 45 | #[test] 46 | fn proptest_select(values in btree_set(any::(), 1000)) { 47 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 48 | for (i, value) in values.iter().cloned().enumerate() { 49 | prop_assert_eq!(bitmap.select(i as u32), Some(value)); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/against_croaring.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | mod arbitrary_ops; 4 | 5 | use libfuzzer_sys::arbitrary::{self, Arbitrary}; 6 | use libfuzzer_sys::fuzz_target; 7 | 8 | use crate::arbitrary_ops::{check_equal, BitmapIteratorOperation, CRoaringIterRange, Operation}; 9 | 10 | #[derive(Arbitrary, Debug)] 11 | struct FuzzInput<'a> { 12 | ops: Vec, 13 | iter_ops: Vec, 14 | initial_input: &'a [u8], 15 | } 16 | 17 | fuzz_target!(|input: FuzzInput| { 18 | let lhs_c = croaring::Bitmap::try_deserialize::(input.initial_input); 19 | let lhs_r = roaring::RoaringBitmap::deserialize_from(input.initial_input).ok(); 20 | 21 | let (mut lhs_c, mut lhs_r) = match (lhs_c, lhs_r) { 22 | (Some(lhs_c), Some(lhs_r)) => { 23 | check_equal(&lhs_c, &lhs_r); 24 | (lhs_c, lhs_r) 25 | } 26 | (None, None) => Default::default(), 27 | (Some(_), None) => panic!("croaring deserialized, but roaring failed"), 28 | (None, Some(_)) => panic!("roaring deserialized, but croaring failed"), 29 | }; 30 | 31 | let mut rhs_c = croaring::Bitmap::new(); 32 | let mut rhs_r = roaring::RoaringBitmap::new(); 33 | 34 | for op in input.ops { 35 | op.apply(&mut lhs_c, &mut rhs_c, &mut lhs_r, &mut rhs_r); 36 | } 37 | lhs_r.internal_validate().unwrap(); 38 | rhs_r.internal_validate().unwrap(); 39 | 40 | let mut lhs_c_iter = CRoaringIterRange::new(&lhs_c); 41 | let mut lhs_r_iter = lhs_r.iter(); 42 | 43 | for op in input.iter_ops { 44 | op.apply(&mut lhs_c_iter, &mut lhs_r_iter); 45 | } 46 | 47 | check_equal(&lhs_c, &lhs_r); 48 | check_equal(&rhs_c, &rhs_r); 49 | }); 50 | -------------------------------------------------------------------------------- /roaring/tests/push.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::{RoaringBitmap, RoaringTreemap}; 3 | 4 | /// macro created to reduce code duplication 5 | macro_rules! test_from_sorted_iter { 6 | ($values: expr, $class: ty) => {{ 7 | let rb1 = <$class>::from_iter($values.clone()); 8 | let rb2 = <$class>::from_sorted_iter($values).unwrap(); 9 | 10 | for (x, y) in rb1.iter().zip(rb2.iter()) { 11 | assert_eq!(x, y); 12 | } 13 | assert_eq!(rb1.len(), rb2.len()); 14 | assert_eq!(rb1.min(), rb2.min()); 15 | assert_eq!(rb1.max(), rb2.max()); 16 | assert_eq!(rb1.is_empty(), rb2.is_empty()); 17 | assert_eq!(rb1, rb2); 18 | }}; 19 | } 20 | 21 | #[test] 22 | fn append() { 23 | test_from_sorted_iter!((0..1_000_000).map(|x| 13 * x).collect::>(), RoaringBitmap); 24 | test_from_sorted_iter!(vec![1, 2, 4, 5, 7, 8, 9], RoaringBitmap); 25 | } 26 | 27 | #[test] 28 | fn append_empty() { 29 | assert_eq!(RoaringBitmap::new().append(vec![]), Ok(0u64)) 30 | } 31 | 32 | #[test] 33 | fn append_error() { 34 | match [100u32].iter().cloned().collect::().append(vec![10, 20, 0]) { 35 | Ok(_) => { 36 | panic!("The 0th element in the iterator was < the max of the bitmap") 37 | } 38 | Err(non_sorted_error) => { 39 | assert_eq!(non_sorted_error.valid_until(), 0) 40 | } 41 | } 42 | 43 | match [100u32].iter().cloned().collect::().append(vec![200, 201, 201]) { 44 | Ok(_) => { 45 | panic!("The 3rd element in the iterator was < 2nd") 46 | } 47 | Err(non_sorted_error) => { 48 | assert_eq!(non_sorted_error.valid_until(), 2) 49 | } 50 | } 51 | } 52 | 53 | #[test] 54 | fn append_tree() { 55 | test_from_sorted_iter!((0..1_000_000).map(|x| 13 * x).collect::>(), RoaringTreemap); 56 | test_from_sorted_iter!(vec![1, 2, 4, 5, 7, 8, 9], RoaringTreemap); 57 | } 58 | -------------------------------------------------------------------------------- /roaring/tests/treemap_rank.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | 3 | use core::ops::RangeInclusive; 4 | use proptest::collection::{btree_set, vec}; 5 | use proptest::prelude::*; 6 | use roaring::RoaringTreemap; 7 | 8 | const BITMAP_MAX: u64 = u32::MAX as u64; 9 | 10 | #[test] 11 | fn rank_roaring_bitmaps() { 12 | // A treemap with two roaring bitmaps. 13 | // The lower one contains one array container with the highest 1000 values 14 | // The higher one contains one bitmap at with the lowest 5000 values 15 | let treemap = RoaringTreemap::from_sorted_iter(BITMAP_MAX - 1000..BITMAP_MAX + 5000).unwrap(); 16 | 17 | // start of treemap 18 | assert_eq!(treemap.rank(0), 0); 19 | 20 | // low boundary 21 | assert_eq!(treemap.rank(BITMAP_MAX - 1002), 0); 22 | assert_eq!(treemap.rank(BITMAP_MAX - 1001), 0); 23 | assert_eq!(treemap.rank(BITMAP_MAX - 1000), 1); 24 | 25 | // middle range (spans two roaring bitmaps) 26 | assert_eq!(treemap.rank(BITMAP_MAX - 1), 1000); 27 | assert_eq!(treemap.rank(BITMAP_MAX), 1001); 28 | assert_eq!(treemap.rank(BITMAP_MAX + 1), 1002); 29 | 30 | // high boundary 31 | assert_eq!(treemap.rank(BITMAP_MAX + 4998), 5999); 32 | assert_eq!(treemap.rank(BITMAP_MAX + 4999), 6000); 33 | assert_eq!(treemap.rank(BITMAP_MAX + 5000), 6000); 34 | 35 | // end of treemap 36 | assert_eq!(treemap.rank(u64::MAX), 6000); 37 | } 38 | 39 | // A range that spans 2 roaring bitmaps with 2 containers each 40 | const PROP_RANGE: RangeInclusive = BITMAP_MAX - (1 << 17)..=BITMAP_MAX + (1 << 17); 41 | 42 | proptest! { 43 | #[test] 44 | fn proptest_rank( 45 | values in btree_set(PROP_RANGE, ..=1000), 46 | checks in vec(PROP_RANGE, ..=100) 47 | ){ 48 | let treemap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 49 | for i in checks { 50 | let expected = values.iter().take_while(|&&x| x <= i).count() as u64; 51 | assert_eq!(treemap.rank(i), expected); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /roaring/tests/treemap_size_hint.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array() { 6 | let bitmap = (0..2000).collect::(); 7 | let mut iter = bitmap.iter(); 8 | assert_eq!((2000, Some(2000)), iter.size_hint()); 9 | iter.by_ref().take(1000).for_each(drop); 10 | assert_eq!((1000, Some(1000)), iter.size_hint()); 11 | iter.by_ref().for_each(drop); 12 | assert_eq!((0, Some(0)), iter.size_hint()); 13 | } 14 | 15 | #[test] 16 | fn bitmap() { 17 | let bitmap = (0..6000).collect::(); 18 | let mut iter = bitmap.iter(); 19 | assert_eq!((6000, Some(6000)), iter.size_hint()); 20 | iter.by_ref().take(3000).for_each(drop); 21 | assert_eq!((3000, Some(3000)), iter.size_hint()); 22 | iter.by_ref().for_each(drop); 23 | assert_eq!((0, Some(0)), iter.size_hint()); 24 | } 25 | 26 | #[test] 27 | fn arrays() { 28 | let bitmap = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) 29 | .collect::(); 30 | let mut iter = bitmap.iter(); 31 | assert_eq!((5000, Some(5000)), iter.size_hint()); 32 | iter.by_ref().take(3000).for_each(drop); 33 | assert_eq!((2000, Some(2000)), iter.size_hint()); 34 | iter.by_ref().for_each(drop); 35 | assert_eq!((0, Some(0)), iter.size_hint()); 36 | } 37 | 38 | #[test] 39 | fn bitmaps() { 40 | let bitmap = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) 41 | .collect::(); 42 | let mut iter = bitmap.iter(); 43 | assert_eq!((28000, Some(28000)), iter.size_hint()); 44 | iter.by_ref().take(2000).for_each(drop); 45 | assert_eq!((26000, Some(26000)), iter.size_hint()); 46 | iter.by_ref().take(5000).for_each(drop); 47 | assert_eq!((21000, Some(21000)), iter.size_hint()); 48 | iter.by_ref().take(20000).for_each(drop); 49 | assert_eq!((1000, Some(1000)), iter.size_hint()); 50 | iter.by_ref().for_each(drop); 51 | assert_eq!((0, Some(0)), iter.size_hint()); 52 | } 53 | -------------------------------------------------------------------------------- /roaring/tests/treemap_is_disjoint.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array() { 6 | let bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (4000..6000).collect::(); 8 | assert!(bitmap1.is_disjoint(&bitmap2)); 9 | } 10 | 11 | #[test] 12 | fn array_not() { 13 | let bitmap1 = (0..4000).collect::(); 14 | let bitmap2 = (2000..6000).collect::(); 15 | assert!(!bitmap1.is_disjoint(&bitmap2)); 16 | } 17 | 18 | #[test] 19 | fn bitmap() { 20 | let bitmap1 = (0..6000).collect::(); 21 | let bitmap2 = (10000..16000).collect::(); 22 | assert!(bitmap1.is_disjoint(&bitmap2)); 23 | } 24 | 25 | #[test] 26 | fn bitmap_not() { 27 | let bitmap1 = (0..10000).collect::(); 28 | let bitmap2 = (5000..15000).collect::(); 29 | assert!(!bitmap1.is_disjoint(&bitmap2)); 30 | } 31 | 32 | #[test] 33 | fn arrays() { 34 | let bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_002_000)) 35 | .collect::(); 36 | let bitmap2 = ((100_000..102_000).chain(1_100_000..1_102_000)).collect::(); 37 | assert!(bitmap1.is_disjoint(&bitmap2)); 38 | } 39 | 40 | #[test] 41 | fn arrays_not() { 42 | let bitmap1 = ((0..2_000).chain(1_000_000..1_002_000).chain(2_000_000..2_002_000)) 43 | .collect::(); 44 | let bitmap2 = ((100_000..102_000).chain(1_001_000..1_003_000)).collect::(); 45 | assert!(!bitmap1.is_disjoint(&bitmap2)); 46 | } 47 | 48 | #[test] 49 | fn bitmaps() { 50 | let bitmap1 = ((0..6000).chain(1_000_000..1_006_000).chain(2_000_000..2_006_000)) 51 | .collect::(); 52 | let bitmap2 = ((100_000..106_000).chain(1_100_000..1_106_000)).collect::(); 53 | assert!(bitmap1.is_disjoint(&bitmap2)); 54 | } 55 | 56 | #[test] 57 | fn bitmaps_not() { 58 | let bitmap1 = ((0..6000).chain(1_000_000..1_006_000).chain(2_000_000..2_006_000)) 59 | .collect::(); 60 | let bitmap2 = ((100_000..106_000).chain(1_004_000..1_008_000)).collect::(); 61 | assert!(!bitmap1.is_disjoint(&bitmap2)); 62 | } 63 | -------------------------------------------------------------------------------- /roaring/tests/rank.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | 3 | use proptest::collection::{btree_set, vec}; 4 | use proptest::prelude::*; 5 | use roaring::RoaringBitmap; 6 | 7 | #[test] 8 | fn rank() { 9 | let mut bitmap = RoaringBitmap::from_sorted_iter(0..2000).unwrap(); 10 | bitmap.insert_range(200_000..210_000); 11 | 12 | // No matching container 13 | assert_eq!(bitmap.rank(80_000), 2000); 14 | assert_eq!(bitmap.rank(u32::MAX), 12_000); 15 | 16 | // Array container at key 17 | assert_eq!(bitmap.rank(0), 1); 18 | assert_eq!(bitmap.rank(100), 101); 19 | assert_eq!(bitmap.rank(2000), 2000); 20 | 21 | // Bitmap container at key 22 | assert_eq!(bitmap.rank(200_000), 2001); 23 | assert_eq!(bitmap.rank(210_000), 12_000); 24 | } 25 | 26 | #[test] 27 | fn rank_array() { 28 | let bitmap = RoaringBitmap::from_sorted_iter(0..2000).unwrap(); 29 | 30 | // No matching container 31 | assert_eq!(bitmap.rank(u32::MAX), 2000); 32 | 33 | // Has container (array) 34 | assert_eq!(bitmap.rank(0), 1); 35 | assert_eq!(bitmap.rank(100), 101); 36 | assert_eq!(bitmap.rank(2000), 2000); 37 | assert_eq!(bitmap.rank(3000), 2000); 38 | } 39 | 40 | #[test] 41 | fn rank_bitmap() { 42 | let bitmap = RoaringBitmap::from_sorted_iter(0..5000).unwrap(); 43 | 44 | // key: 0, bit: 0 45 | assert_eq!(bitmap.rank(0), 1); 46 | 47 | // key: 0, bit: 63 (mask of all ones) 48 | assert_eq!(bitmap.rank(63), 64); 49 | 50 | // key: 1023, bit: 0 51 | assert_eq!(bitmap.rank(65535), 5000); 52 | 53 | // key: 1023, bit: 63 (mask of all ones) 54 | assert_eq!(bitmap.rank(65472), 5000); 55 | 56 | assert_eq!(bitmap.rank(1), 2); 57 | assert_eq!(bitmap.rank(100), 101); 58 | assert_eq!(bitmap.rank(1000), 1001); 59 | assert_eq!(bitmap.rank(4999), 5000); 60 | } 61 | 62 | proptest! { 63 | #[test] 64 | fn proptest_rank( 65 | values in btree_set(..=262_143_u32, ..=1000), 66 | checks in vec(..=262_143_u32, ..=100) 67 | ){ 68 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 69 | for i in checks { 70 | let expected = values.iter().take_while(|&&x| x <= i).count() as u64; 71 | assert_eq!(bitmap.rank(i), expected); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /roaring/tests/treemap_ops.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn or() { 6 | let mut rb1 = (1..4).collect::(); 7 | let rb2 = (3..6).collect::(); 8 | let rb3 = (1..6).collect::(); 9 | 10 | assert_eq!(rb3, &rb1 | &rb2); 11 | assert_eq!(rb3, &rb1 | rb2.clone()); 12 | assert_eq!(rb3, rb1.clone() | &rb2); 13 | assert_eq!(rb3, rb1.clone() | rb2.clone()); 14 | assert_eq!(rb3.len(), rb1.union_len(&rb2)); 15 | 16 | rb1 |= &rb2; 17 | rb1 |= rb2; 18 | 19 | assert_eq!(rb3, rb1); 20 | } 21 | 22 | #[test] 23 | fn and() { 24 | let mut rb1 = (1..4).collect::(); 25 | let rb2 = (3..6).collect::(); 26 | let rb3 = (3..4).collect::(); 27 | 28 | assert_eq!(rb3, &rb1 & &rb2); 29 | assert_eq!(rb3, &rb1 & rb2.clone()); 30 | assert_eq!(rb3, rb1.clone() & &rb2); 31 | assert_eq!(rb3, rb1.clone() & rb2.clone()); 32 | assert_eq!(rb3.len(), rb1.intersection_len(&rb2)); 33 | 34 | rb1 &= &rb2; 35 | rb1 &= rb2; 36 | 37 | assert_eq!(rb3, rb1); 38 | } 39 | 40 | #[test] 41 | fn sub() { 42 | let mut rb1 = (1..4).collect::(); 43 | let rb2 = (3..6).collect::(); 44 | let rb3 = (1..3).collect::(); 45 | 46 | assert_eq!(rb3, &rb1 - &rb2); 47 | assert_eq!(rb3, &rb1 - rb2.clone()); 48 | assert_eq!(rb3, rb1.clone() - &rb2); 49 | assert_eq!(rb3, rb1.clone() - rb2.clone()); 50 | assert_eq!(rb3.len(), rb1.difference_len(&rb2)); 51 | 52 | rb1 -= &rb2; 53 | rb1 -= rb2; 54 | 55 | assert_eq!(rb3, rb1); 56 | } 57 | 58 | #[test] 59 | fn xor() { 60 | let mut rb1 = (1..4).collect::(); 61 | let rb2 = (3..6).collect::(); 62 | let rb3 = ((1..3).chain(4..6)).collect::(); 63 | let rb4 = (0..0).collect::(); 64 | 65 | assert_eq!(rb3, &rb1 ^ &rb2); 66 | assert_eq!(rb3, &rb1 ^ rb2.clone()); 67 | assert_eq!(rb3, rb1.clone() ^ &rb2); 68 | assert_eq!(rb3, rb1.clone() ^ rb2.clone()); 69 | assert_eq!(rb3.len(), rb1.symmetric_difference_len(&rb2)); 70 | 71 | rb1 ^= &rb2; 72 | 73 | assert_eq!(rb3, rb1); 74 | 75 | rb1 ^= rb3; 76 | 77 | assert_eq!(rb4, rb1); 78 | } 79 | -------------------------------------------------------------------------------- /roaring/tests/is_disjoint.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array() { 6 | let bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (4000..6000).collect::(); 8 | assert!(bitmap1.is_disjoint(&bitmap2)); 9 | } 10 | 11 | #[test] 12 | fn array_not() { 13 | let bitmap1 = (0..4000).collect::(); 14 | let bitmap2 = (2000..6000).collect::(); 15 | assert!(!bitmap1.is_disjoint(&bitmap2)); 16 | } 17 | 18 | #[test] 19 | fn bitmap() { 20 | let bitmap1 = (0..6000).collect::(); 21 | let bitmap2 = (10000..16000).collect::(); 22 | assert!(bitmap1.is_disjoint(&bitmap2)); 23 | } 24 | 25 | #[test] 26 | fn bitmap_not() { 27 | let bitmap1 = (0..10000).collect::(); 28 | let bitmap2 = (5000..15000).collect::(); 29 | assert!(!bitmap1.is_disjoint(&bitmap2)); 30 | } 31 | 32 | #[test] 33 | fn arrays() { 34 | let bitmap1 = (0..2000) 35 | .chain(1_000_000..1_002_000) 36 | .chain(2_000_000..2_002_000) 37 | .collect::(); 38 | let bitmap2 = (100_000..102_000).chain(1_100_000..1_102_000).collect::(); 39 | assert!(bitmap1.is_disjoint(&bitmap2)); 40 | } 41 | 42 | #[test] 43 | fn arrays_not() { 44 | let bitmap1 = (0..2_000) 45 | .chain(1_000_000..1_002_000) 46 | .chain(2_000_000..2_002_000) 47 | .collect::(); 48 | let bitmap2 = (100_000..102_000).chain(1_001_000..1_003_000).collect::(); 49 | assert!(!bitmap1.is_disjoint(&bitmap2)); 50 | } 51 | 52 | #[test] 53 | fn bitmaps() { 54 | let bitmap1 = (0..6000) 55 | .chain(1_000_000..1_006_000) 56 | .chain(2_000_000..2_006_000) 57 | .collect::(); 58 | let bitmap2 = (100_000..106_000).chain(1_100_000..1_106_000).collect::(); 59 | assert!(bitmap1.is_disjoint(&bitmap2)); 60 | } 61 | 62 | #[test] 63 | fn bitmaps_not() { 64 | let bitmap1 = (0..6000) 65 | .chain(1_000_000..1_006_000) 66 | .chain(2_000_000..2_006_000) 67 | .collect::(); 68 | let bitmap2 = (100_000..106_000).chain(1_004_000..1_008_000).collect::(); 69 | assert!(!bitmap1.is_disjoint(&bitmap2)); 70 | } 71 | -------------------------------------------------------------------------------- /roaring/tests/is_subset.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array_not() { 6 | let sup = (0..2000).collect::(); 7 | let sub = (1000..3000).collect::(); 8 | assert!(!sub.is_subset(&sup)); 9 | } 10 | 11 | #[test] 12 | fn array() { 13 | let sup = (0..4000).collect::(); 14 | let sub = (2000..3000).collect::(); 15 | assert!(sub.is_subset(&sup)); 16 | } 17 | 18 | #[test] 19 | fn array_bitmap_not() { 20 | let sup = (0..2000).collect::(); 21 | let sub = (1000..15000).collect::(); 22 | assert!(!sub.is_subset(&sup)); 23 | } 24 | 25 | #[test] 26 | fn bitmap_not() { 27 | let sup = (0..6000).collect::(); 28 | let sub = (4000..10000).collect::(); 29 | assert!(!sub.is_subset(&sup)); 30 | } 31 | 32 | #[test] 33 | fn bitmap() { 34 | let sup = (0..20000).collect::(); 35 | let sub = (5000..15000).collect::(); 36 | assert!(sub.is_subset(&sup)); 37 | } 38 | 39 | #[test] 40 | fn bitmap_array_not() { 41 | let sup = (0..20000).collect::(); 42 | let sub = (19000..21000).collect::(); 43 | assert!(!sub.is_subset(&sup)); 44 | } 45 | 46 | #[test] 47 | fn bitmap_array() { 48 | let sup = (0..20000).collect::(); 49 | let sub = (18000..20000).collect::(); 50 | assert!(sub.is_subset(&sup)); 51 | } 52 | 53 | #[test] 54 | fn arrays_not() { 55 | let sup = (0..2000).chain(1_000_000..1_002_000).collect::(); 56 | let sub = (100_000..102_000).chain(1_100_000..1_102_000).collect::(); 57 | assert!(!sub.is_subset(&sup)); 58 | } 59 | 60 | #[test] 61 | fn arrays() { 62 | let sup = (0..3000).chain(100_000..103_000).collect::(); 63 | let sub = (0..2000).chain(100_000..102_000).collect::(); 64 | assert!(sub.is_subset(&sup)); 65 | } 66 | 67 | #[test] 68 | fn bitmaps_not() { 69 | let sup = (0..6000) 70 | .chain(1_000_000..1_006_000) 71 | .chain(2_000_000..2_010_000) 72 | .collect::(); 73 | let sub = (100_000..106_000).chain(1_100_000..1_106_000).collect::(); 74 | assert!(!sub.is_subset(&sup)); 75 | } 76 | 77 | #[test] 78 | fn bitmaps() { 79 | let sup = (0..1_000_000).chain(2_000_000..2_010_000).collect::(); 80 | let sub = (0..10_000).chain(500_000..510_000).collect::(); 81 | assert!(sub.is_subset(&sup)); 82 | } 83 | -------------------------------------------------------------------------------- /roaring/tests/treemap_is_subset.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array_not() { 6 | let sup = (0..2000).collect::(); 7 | let sub = (1000..3000).collect::(); 8 | assert!(!sub.is_subset(&sup)); 9 | } 10 | 11 | #[test] 12 | fn array() { 13 | let sup = (0..4000).collect::(); 14 | let sub = (2000..3000).collect::(); 15 | assert!(sub.is_subset(&sup)); 16 | } 17 | 18 | #[test] 19 | fn array_bitmap_not() { 20 | let sup = (0..2000).collect::(); 21 | let sub = (1000..15000).collect::(); 22 | assert!(!sub.is_subset(&sup)); 23 | } 24 | 25 | #[test] 26 | fn bitmap_not() { 27 | let sup = (0..6000).collect::(); 28 | let sub = (4000..10000).collect::(); 29 | assert!(!sub.is_subset(&sup)); 30 | } 31 | 32 | #[test] 33 | fn bitmap() { 34 | let sup = (0..20000).collect::(); 35 | let sub = (5000..15000).collect::(); 36 | assert!(sub.is_subset(&sup)); 37 | } 38 | 39 | #[test] 40 | fn bitmap_array_not() { 41 | let sup = (0..20000).collect::(); 42 | let sub = (19000..21000).collect::(); 43 | assert!(!sub.is_subset(&sup)); 44 | } 45 | 46 | #[test] 47 | fn bitmap_array() { 48 | let sup = (0..20000).collect::(); 49 | let sub = (18000..20000).collect::(); 50 | assert!(sub.is_subset(&sup)); 51 | } 52 | 53 | #[test] 54 | fn arrays_not() { 55 | let sup = ((0..2000).chain(1_000_000..1_002_000)).collect::(); 56 | let sub = ((100_000..102_000).chain(1_100_000..1_102_000)).collect::(); 57 | assert!(!sub.is_subset(&sup)); 58 | } 59 | 60 | #[test] 61 | fn arrays() { 62 | let sup = ((0..3000).chain(100_000..103_000)).collect::(); 63 | let sub = ((0..2000).chain(100_000..102_000)).collect::(); 64 | assert!(sub.is_subset(&sup)); 65 | } 66 | 67 | #[test] 68 | fn bitmaps_not() { 69 | let sup = ((0..6000).chain(1_000_000..1_006_000).chain(2_000_000..2_010_000)) 70 | .collect::(); 71 | let sub = ((100_000..106_000).chain(1_100_000..1_106_000)).collect::(); 72 | assert!(!sub.is_subset(&sup)); 73 | } 74 | 75 | #[test] 76 | fn bitmaps() { 77 | let sup = ((0..1_000_000).chain(2_000_000..2_010_000)).collect::(); 78 | let sub = ((0..10_000).chain(500_000..510_000)).collect::(); 79 | assert!(sub.is_subset(&sup)); 80 | } 81 | -------------------------------------------------------------------------------- /roaring/src/bitmap/serde.rs: -------------------------------------------------------------------------------- 1 | use serde::de::SeqAccess; 2 | use serde::de::Visitor; 3 | use serde::Deserialize; 4 | use serde::Deserializer; 5 | use serde::Serialize; 6 | 7 | use crate::RoaringBitmap; 8 | 9 | impl<'de> Deserialize<'de> for RoaringBitmap { 10 | fn deserialize(deserializer: D) -> Result 11 | where 12 | D: Deserializer<'de>, 13 | { 14 | struct BitmapVisitor; 15 | 16 | impl<'de> Visitor<'de> for BitmapVisitor { 17 | type Value = RoaringBitmap; 18 | 19 | fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result { 20 | formatter.write_str("roaring bitmap") 21 | } 22 | 23 | fn visit_bytes(self, bytes: &[u8]) -> Result 24 | where 25 | E: serde::de::Error, 26 | { 27 | RoaringBitmap::deserialize_from(bytes).map_err(serde::de::Error::custom) 28 | } 29 | 30 | // in some case bytes will be serialized as a sequence thus we need to accept both 31 | // even if it means non optimal performance 32 | fn visit_seq(self, mut seq: A) -> Result 33 | where 34 | A: SeqAccess<'de>, 35 | { 36 | let mut bytes: Vec = Vec::new(); 37 | while let Some(el) = seq.next_element()? { 38 | bytes.push(el); 39 | } 40 | RoaringBitmap::deserialize_from(&*bytes).map_err(serde::de::Error::custom) 41 | } 42 | } 43 | 44 | deserializer.deserialize_bytes(BitmapVisitor) 45 | } 46 | } 47 | 48 | impl Serialize for RoaringBitmap { 49 | fn serialize(&self, serializer: S) -> Result 50 | where 51 | S: serde::Serializer, 52 | { 53 | let mut buf = Vec::new(); 54 | self.serialize_into(&mut buf).map_err(serde::ser::Error::custom)?; 55 | 56 | serializer.serialize_bytes(&buf) 57 | } 58 | } 59 | 60 | #[cfg(test)] 61 | mod test { 62 | use crate::RoaringBitmap; 63 | use proptest::prelude::*; 64 | 65 | proptest! { 66 | #[test] 67 | fn test_serde_json( 68 | bitmap in RoaringBitmap::arbitrary(), 69 | ) { 70 | let json = serde_json::to_vec(&bitmap).unwrap(); 71 | prop_assert_eq!(bitmap, serde_json::from_slice(&json).unwrap()); 72 | } 73 | 74 | #[test] 75 | fn test_postcard( 76 | bitmap in RoaringBitmap::arbitrary(), 77 | ) { 78 | let buffer = postcard::to_allocvec(&bitmap).unwrap(); 79 | prop_assert_eq!(bitmap, postcard::from_bytes(&buffer).unwrap()); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /roaring/src/treemap/serde.rs: -------------------------------------------------------------------------------- 1 | use serde::de::SeqAccess; 2 | use serde::de::Visitor; 3 | use serde::Deserialize; 4 | use serde::Deserializer; 5 | use serde::Serialize; 6 | 7 | use crate::RoaringTreemap; 8 | 9 | impl<'de> Deserialize<'de> for RoaringTreemap { 10 | fn deserialize(deserializer: D) -> Result 11 | where 12 | D: Deserializer<'de>, 13 | { 14 | struct TreemapVisitor; 15 | 16 | impl<'de> Visitor<'de> for TreemapVisitor { 17 | type Value = RoaringTreemap; 18 | 19 | fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result { 20 | formatter.write_str("roaring bitmap") 21 | } 22 | 23 | fn visit_bytes(self, bytes: &[u8]) -> Result 24 | where 25 | E: serde::de::Error, 26 | { 27 | RoaringTreemap::deserialize_from(bytes).map_err(serde::de::Error::custom) 28 | } 29 | 30 | // in some case bytes will be serialized as a sequence thus we need to accept both 31 | // even if it means non optimal performance 32 | fn visit_seq(self, mut seq: A) -> Result 33 | where 34 | A: SeqAccess<'de>, 35 | { 36 | let mut bytes: Vec = Vec::new(); 37 | while let Some(el) = seq.next_element()? { 38 | bytes.push(el); 39 | } 40 | RoaringTreemap::deserialize_from(&*bytes).map_err(serde::de::Error::custom) 41 | } 42 | } 43 | 44 | deserializer.deserialize_bytes(TreemapVisitor) 45 | } 46 | } 47 | 48 | impl Serialize for RoaringTreemap { 49 | fn serialize(&self, serializer: S) -> Result 50 | where 51 | S: serde::Serializer, 52 | { 53 | let mut buf = Vec::new(); 54 | self.serialize_into(&mut buf).map_err(serde::ser::Error::custom)?; 55 | 56 | serializer.serialize_bytes(&buf) 57 | } 58 | } 59 | 60 | #[cfg(test)] 61 | mod test { 62 | use crate::RoaringTreemap; 63 | use proptest::prelude::*; 64 | 65 | proptest! { 66 | #[test] 67 | fn test_serde_json( 68 | treemap in RoaringTreemap::arbitrary(), 69 | ) { 70 | let json = serde_json::to_vec(&treemap).unwrap(); 71 | prop_assert_eq!(treemap, serde_json::from_slice(&json).unwrap()); 72 | } 73 | 74 | #[test] 75 | fn test_postcard( 76 | treemap in RoaringTreemap::arbitrary(), 77 | ) { 78 | let buffer = postcard::to_allocvec(&treemap).unwrap(); 79 | prop_assert_eq!(treemap, postcard::from_bytes(&buffer).unwrap()); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /roaring/tests/treemap_union_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array_to_array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (0..3000).collect::(); 9 | 10 | bitmap1 |= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn array_to_bitmap() { 17 | let mut bitmap1 = (0..4000).collect::(); 18 | let bitmap2 = (4000..8000).collect::(); 19 | let bitmap3 = (0..8000).collect::(); 20 | 21 | bitmap1 |= bitmap2; 22 | 23 | assert_eq!(bitmap1, bitmap3); 24 | } 25 | 26 | #[test] 27 | fn array_and_bitmap() { 28 | let mut bitmap1 = (0..2000).collect::(); 29 | let bitmap2 = (1000..8000).collect::(); 30 | let bitmap3 = (0..8000).collect::(); 31 | 32 | bitmap1 |= bitmap2; 33 | 34 | assert_eq!(bitmap1, bitmap3); 35 | } 36 | 37 | #[test] 38 | fn bitmap() { 39 | let mut bitmap1 = (0..12000).collect::(); 40 | let bitmap2 = (6000..18000).collect::(); 41 | let bitmap3 = (0..18000).collect::(); 42 | 43 | bitmap1 |= bitmap2; 44 | 45 | assert_eq!(bitmap1, bitmap3); 46 | } 47 | 48 | #[test] 49 | fn bitmap_and_array() { 50 | let mut bitmap1 = (0..12000).collect::(); 51 | let bitmap2 = (10000..13000).collect::(); 52 | let bitmap3 = (0..13000).collect::(); 53 | 54 | bitmap1 |= bitmap2; 55 | 56 | assert_eq!(bitmap1, bitmap3); 57 | } 58 | 59 | #[test] 60 | fn arrays() { 61 | let mut bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(3_000_000..3_001_000)) 62 | .collect::(); 63 | let bitmap2 = ((1000..3000).chain(1_001_000..1_003_000).chain(2_000_000..2_001_000)) 64 | .collect::(); 65 | let bitmap3 = ((0..3000) 66 | .chain(1_000_000..1_003_000) 67 | .chain(2_000_000..2_001_000) 68 | .chain(3_000_000..3_001_000)) 69 | .collect::(); 70 | 71 | bitmap1 |= bitmap2; 72 | 73 | assert_eq!(bitmap1, bitmap3); 74 | } 75 | 76 | #[test] 77 | fn bitmaps() { 78 | let mut bitmap1 = ((0..6000).chain(1_000_000..1_012_000).chain(3_000_000..3_010_000)) 79 | .collect::(); 80 | let bitmap2 = ((3000..9000).chain(1_006_000..1_018_000).chain(2_000_000..2_010_000)) 81 | .collect::(); 82 | let bitmap3 = ((0..9000) 83 | .chain(1_000_000..1_018_000) 84 | .chain(2_000_000..2_010_000) 85 | .chain(3_000_000..3_010_000)) 86 | .collect::(); 87 | 88 | bitmap1 |= bitmap2; 89 | 90 | assert_eq!(bitmap1, bitmap3); 91 | } 92 | -------------------------------------------------------------------------------- /roaring/tests/treemap_intersect_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (1000..2000).collect::(); 9 | 10 | bitmap1 &= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_intersection() { 17 | let mut bitmap1 = (0..2).collect::(); 18 | let bitmap2 = (3..4).collect::(); 19 | 20 | bitmap1 &= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringTreemap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = (1000..2000).collect::(); 30 | 31 | bitmap1 &= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = (6000..12000).collect::(); 41 | 42 | bitmap1 &= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (3000..9000).collect::(); 51 | let bitmap3 = (3000..6000).collect::(); 52 | 53 | bitmap1 &= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (7000..9000).collect::(); 62 | let bitmap3 = (7000..9000).collect::(); 63 | 64 | bitmap1 &= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn arrays() { 71 | let mut bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(3_000_000..3_001_000)) 72 | .collect::(); 73 | let bitmap2 = ((1000..3000).chain(1_001_000..1_003_000).chain(2_000_000..2_001_000)) 74 | .collect::(); 75 | let bitmap3 = ((1000..2000).chain(1_001_000..1_002_000)).collect::(); 76 | 77 | bitmap1 &= bitmap2; 78 | 79 | assert_eq!(bitmap1, bitmap3); 80 | } 81 | 82 | #[test] 83 | fn bitmaps() { 84 | let mut bitmap1 = ((0..6000).chain(1_000_000..1_012_000).chain(3_000_000..3_010_000)) 85 | .collect::(); 86 | let bitmap2 = ((3000..9000).chain(1_006_000..1_018_000).chain(2_000_000..2_010_000)) 87 | .collect::(); 88 | let bitmap3 = ((3000..6000).chain(1_006_000..1_012_000)).collect::(); 89 | 90 | bitmap1 &= bitmap2; 91 | 92 | assert_eq!(bitmap1, bitmap3); 93 | } 94 | -------------------------------------------------------------------------------- /roaring/tests/size_hint.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array() { 6 | let bitmap = (0..2000).collect::(); 7 | let mut iter = bitmap.iter(); 8 | assert_eq!((2000, Some(2000)), iter.size_hint()); 9 | iter.by_ref().take(1000).for_each(drop); 10 | assert_eq!((1000, Some(1000)), iter.size_hint()); 11 | iter.by_ref().for_each(drop); 12 | assert_eq!((0, Some(0)), iter.size_hint()); 13 | } 14 | 15 | #[test] 16 | fn bitmap() { 17 | let bitmap = (0..6000).collect::(); 18 | let mut iter = bitmap.iter(); 19 | assert_eq!((6000, Some(6000)), iter.size_hint()); 20 | iter.by_ref().take(3000).for_each(drop); 21 | assert_eq!((3000, Some(3000)), iter.size_hint()); 22 | iter.by_ref().for_each(drop); 23 | assert_eq!((0, Some(0)), iter.size_hint()); 24 | } 25 | 26 | #[test] 27 | fn run() { 28 | let mut bitmap = RoaringBitmap::from_iter(0..6000); 29 | bitmap.optimize(); 30 | let mut iter = bitmap.iter(); 31 | assert_eq!((6000, Some(6000)), iter.size_hint()); 32 | iter.by_ref().take(3000).for_each(drop); 33 | assert_eq!((3000, Some(3000)), iter.size_hint()); 34 | iter.by_ref().for_each(drop); 35 | assert_eq!((0, Some(0)), iter.size_hint()); 36 | } 37 | 38 | #[test] 39 | fn arrays() { 40 | let bitmap = (0..2000) 41 | .chain(1_000_000..1_002_000) 42 | .chain(2_000_000..2_001_000) 43 | .collect::(); 44 | let mut iter = bitmap.iter(); 45 | assert_eq!((5000, Some(5000)), iter.size_hint()); 46 | iter.by_ref().take(3000).for_each(drop); 47 | assert_eq!((2000, Some(2000)), iter.size_hint()); 48 | iter.by_ref().for_each(drop); 49 | assert_eq!((0, Some(0)), iter.size_hint()); 50 | } 51 | 52 | #[test] 53 | fn bitmaps() { 54 | let bitmap = (0..6000) 55 | .chain(1_000_000..1_012_000) 56 | .chain(2_000_000..2_010_000) 57 | .collect::(); 58 | let mut iter = bitmap.iter(); 59 | assert_eq!((28000, Some(28000)), iter.size_hint()); 60 | iter.by_ref().take(2000).for_each(drop); 61 | assert_eq!((26000, Some(26000)), iter.size_hint()); 62 | iter.by_ref().take(5000).for_each(drop); 63 | assert_eq!((21000, Some(21000)), iter.size_hint()); 64 | iter.by_ref().take(20000).for_each(drop); 65 | assert_eq!((1000, Some(1000)), iter.size_hint()); 66 | iter.by_ref().for_each(drop); 67 | assert_eq!((0, Some(0)), iter.size_hint()); 68 | } 69 | 70 | #[test] 71 | fn runs() { 72 | let mut bitmap = 73 | RoaringBitmap::from_iter((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)); 74 | bitmap.optimize(); 75 | let mut iter = bitmap.iter(); 76 | assert_eq!((5000, Some(5000)), iter.size_hint()); 77 | iter.by_ref().take(3000).for_each(drop); 78 | assert_eq!((2000, Some(2000)), iter.size_hint()); 79 | iter.by_ref().for_each(drop); 80 | assert_eq!((0, Some(0)), iter.size_hint()); 81 | } 82 | -------------------------------------------------------------------------------- /roaring/tests/ops.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn or() { 6 | let mut rb1 = (1..4).collect::(); 7 | let rb2 = (3..6).collect::(); 8 | let rb3 = (1..6).collect::(); 9 | 10 | assert_eq!(rb3, &rb1 | &rb2); 11 | assert_eq!(rb3, &rb1 | rb2.clone()); 12 | assert_eq!(rb3, rb1.clone() | &rb2); 13 | assert_eq!(rb3, rb1.clone() | rb2.clone()); 14 | assert_eq!(rb3.len(), rb1.union_len(&rb2)); 15 | 16 | rb1 |= &rb2; 17 | rb1 |= rb2; 18 | 19 | assert_eq!(rb3, rb1); 20 | } 21 | 22 | #[test] 23 | fn and() { 24 | let mut rb1 = (1..4).collect::(); 25 | let rb2 = (3..6).collect::(); 26 | let rb3 = (3..4).collect::(); 27 | 28 | assert_eq!(rb3, &rb1 & &rb2); 29 | assert_eq!(rb3, &rb1 & rb2.clone()); 30 | assert_eq!(rb3, rb1.clone() & &rb2); 31 | assert_eq!(rb3, rb1.clone() & rb2.clone()); 32 | assert_eq!(rb3.len(), rb1.intersection_len(&rb2)); 33 | 34 | rb1 &= &rb2; 35 | rb1 &= rb2; 36 | 37 | assert_eq!(rb3, rb1); 38 | } 39 | 40 | #[test] 41 | fn sub() { 42 | let mut rb1 = (1..4000).collect::(); 43 | let rb2 = (3..5000).collect::(); 44 | let rb3 = (1..3).collect::(); 45 | 46 | assert_eq!(rb3, &rb1 - &rb2); 47 | assert_eq!(rb3, &rb1 - rb2.clone()); 48 | assert_eq!(rb3, rb1.clone() - &rb2); 49 | assert_eq!(rb3, rb1.clone() - rb2.clone()); 50 | assert_eq!(rb3.len(), rb1.difference_len(&rb2)); 51 | 52 | rb1 -= &rb2; 53 | rb1 -= rb2; 54 | 55 | assert_eq!(rb3, rb1); 56 | } 57 | 58 | // See issue #327 59 | #[test] 60 | fn subtraction_preserves_zero_element() { 61 | let mut a = RoaringBitmap::from([0, 35, 80, 104, 138, 214, 235, 258]); 62 | let b = RoaringBitmap::from([9, 35, 42, 51, 111, 134, 231, 239]); 63 | 64 | a -= b; 65 | 66 | // The bug: element 0 should still be present but was being removed 67 | assert!(a.contains(0), "Element 0 should be present after subtraction"); 68 | 69 | // Verify the complete result 70 | let expected: Vec = vec![0, 80, 104, 138, 214, 235, 258]; 71 | let actual: Vec = a.iter().collect(); 72 | assert_eq!(actual, expected, "Subtraction result should match expected values"); 73 | } 74 | 75 | #[test] 76 | fn xor() { 77 | let mut rb1 = (1..4).collect::(); 78 | let rb2 = (3..6).collect::(); 79 | let rb3 = (1..3).chain(4..6).collect::(); 80 | let rb4 = (0..0).collect::(); 81 | 82 | assert_eq!(rb3, &rb1 ^ &rb2); 83 | assert_eq!(rb3, &rb1 ^ rb2.clone()); 84 | assert_eq!(rb3, rb1.clone() ^ &rb2); 85 | assert_eq!(rb3, rb1.clone() ^ rb2.clone()); 86 | assert_eq!(rb3.len(), rb1.symmetric_difference_len(&rb2)); 87 | 88 | rb1 ^= &rb2; 89 | 90 | assert_eq!(rb3, rb1); 91 | 92 | rb1 ^= rb3; 93 | 94 | assert_eq!(rb4, rb1); 95 | } 96 | -------------------------------------------------------------------------------- /roaring/tests/union_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array_to_array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (0..3000).collect::(); 9 | 10 | bitmap1 |= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn array_to_bitmap() { 17 | let mut bitmap1 = (0..4000).collect::(); 18 | let bitmap2 = (4000..8000).collect::(); 19 | let bitmap3 = (0..8000).collect::(); 20 | 21 | bitmap1 |= bitmap2; 22 | 23 | assert_eq!(bitmap1, bitmap3); 24 | } 25 | 26 | #[test] 27 | fn array_and_bitmap() { 28 | let mut bitmap1 = (0..2000).collect::(); 29 | let bitmap2 = (1000..8000).collect::(); 30 | let bitmap3 = (0..8000).collect::(); 31 | 32 | bitmap1 |= bitmap2; 33 | 34 | assert_eq!(bitmap1, bitmap3); 35 | } 36 | 37 | #[test] 38 | fn bitmap() { 39 | let mut bitmap1 = (0..12000).collect::(); 40 | let bitmap2 = (6000..18000).collect::(); 41 | let bitmap3 = (0..18000).collect::(); 42 | 43 | bitmap1 |= bitmap2; 44 | 45 | assert_eq!(bitmap1, bitmap3); 46 | } 47 | 48 | #[test] 49 | fn bitmap_and_array() { 50 | let mut bitmap1 = (0..12000).collect::(); 51 | let bitmap2 = (10000..13000).collect::(); 52 | let bitmap3 = (0..13000).collect::(); 53 | 54 | bitmap1 |= bitmap2; 55 | 56 | assert_eq!(bitmap1, bitmap3); 57 | } 58 | 59 | #[test] 60 | fn arrays() { 61 | let mut bitmap1 = (0..2000) 62 | .chain(1_000_000..1_002_000) 63 | .chain(3_000_000..3_001_000) 64 | .collect::(); 65 | let bitmap2 = (1000..3000) 66 | .chain(1_001_000..1_003_000) 67 | .chain(2_000_000..2_001_000) 68 | .collect::(); 69 | let bitmap3 = (0..3000) 70 | .chain(1_000_000..1_003_000) 71 | .chain(2_000_000..2_001_000) 72 | .chain(3_000_000..3_001_000) 73 | .collect::(); 74 | 75 | bitmap1 |= bitmap2; 76 | 77 | assert_eq!(bitmap1, bitmap3); 78 | } 79 | 80 | #[test] 81 | fn bitmaps() { 82 | let mut bitmap1 = (0..6000) 83 | .chain(1_000_000..1_012_000) 84 | .chain(3_000_000..3_010_000) 85 | .collect::(); 86 | let bitmap2 = (3000..9000) 87 | .chain(1_006_000..1_018_000) 88 | .chain(2_000_000..2_010_000) 89 | .collect::(); 90 | let bitmap3 = (0..9000) 91 | .chain(1_000_000..1_018_000) 92 | .chain(2_000_000..2_010_000) 93 | .chain(3_000_000..3_010_000) 94 | .collect::(); 95 | 96 | bitmap1 |= bitmap2; 97 | 98 | assert_eq!(bitmap1, bitmap3); 99 | } 100 | -------------------------------------------------------------------------------- /roaring/tests/intersect_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (1000..2000).collect::(); 9 | 10 | bitmap1 &= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_intersection() { 17 | let mut bitmap1 = (0..2).collect::(); 18 | let bitmap2 = (3..4).collect::(); 19 | 20 | bitmap1 &= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringBitmap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = (1000..2000).collect::(); 30 | 31 | bitmap1 &= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = (6000..12000).collect::(); 41 | 42 | bitmap1 &= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (3000..9000).collect::(); 51 | let bitmap3 = (3000..6000).collect::(); 52 | 53 | bitmap1 &= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (7000..9000).collect::(); 62 | let bitmap3 = (7000..9000).collect::(); 63 | 64 | bitmap1 &= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn arrays() { 71 | let mut bitmap1 = (0..2000) 72 | .chain(1_000_000..1_002_000) 73 | .chain(3_000_000..3_001_000) 74 | .collect::(); 75 | let bitmap2 = (1000..3000) 76 | .chain(1_001_000..1_003_000) 77 | .chain(2_000_000..2_001_000) 78 | .collect::(); 79 | let bitmap3 = (1000..2000).chain(1_001_000..1_002_000).collect::(); 80 | 81 | bitmap1 &= bitmap2; 82 | 83 | assert_eq!(bitmap1, bitmap3); 84 | } 85 | 86 | #[test] 87 | fn bitmaps() { 88 | let mut bitmap1 = (0..6000) 89 | .chain(1_000_000..1_012_000) 90 | .chain(3_000_000..3_010_000) 91 | .collect::(); 92 | let bitmap2 = (3000..9000) 93 | .chain(1_006_000..1_018_000) 94 | .chain(2_000_000..2_010_000) 95 | .collect::(); 96 | let bitmap3 = (3000..6000).chain(1_006_000..1_012_000).collect::(); 97 | 98 | bitmap1 &= bitmap2; 99 | 100 | assert_eq!(bitmap1, bitmap3); 101 | } 102 | -------------------------------------------------------------------------------- /roaring/tests/range_checks.rs: -------------------------------------------------------------------------------- 1 | use proptest::collection::hash_set; 2 | use proptest::prelude::*; 3 | use roaring::RoaringBitmap; 4 | 5 | #[test] 6 | fn u32_max() { 7 | let mut bitmap = RoaringBitmap::new(); 8 | bitmap.insert(u32::MAX); 9 | assert!(bitmap.contains_range(u32::MAX..=u32::MAX)); 10 | assert!(!bitmap.contains_range(u32::MAX - 1..=u32::MAX)); 11 | 12 | bitmap.insert_range(4_000_000_000..); 13 | assert!(bitmap.contains_range(4_000_000_000..)); 14 | assert!(bitmap.contains_range(4_000_000_000..u32::MAX)); 15 | assert!(bitmap.contains_range(4_000_000_000..=u32::MAX)); 16 | assert!(bitmap.contains_range(4_100_000_000..=u32::MAX)); 17 | } 18 | 19 | proptest! { 20 | #[test] 21 | fn proptest_range( 22 | start in ..=262_143_u32, 23 | len in ..=262_143_u32, 24 | extra in hash_set(..=462_143_u32, ..=100), 25 | ){ 26 | let end = start + len; 27 | let range = start..end; 28 | let inverse_empty_range = (start+len)..start; 29 | 30 | let mut bitmap = RoaringBitmap::new(); 31 | bitmap.insert_range(range.clone()); 32 | assert!(bitmap.contains_range(range.clone())); 33 | assert!(bitmap.contains_range(inverse_empty_range.clone())); 34 | assert_eq!(bitmap.range_cardinality(range.clone()) as usize, range.len()); 35 | 36 | for &val in &extra { 37 | bitmap.insert(val); 38 | assert!(bitmap.contains_range(range.clone())); 39 | assert!(bitmap.contains_range(inverse_empty_range.clone())); 40 | assert_eq!(bitmap.range_cardinality(range.clone()) as usize, range.len()); 41 | } 42 | 43 | for (i, &val) in extra.iter().filter(|x| range.contains(x)).enumerate() { 44 | bitmap.remove(val); 45 | assert!(!bitmap.contains_range(range.clone())); 46 | assert!(bitmap.contains_range(inverse_empty_range.clone())); 47 | assert_eq!(bitmap.range_cardinality(range.clone()) as usize, range.len() - i - 1); 48 | } 49 | } 50 | 51 | #[test] 52 | fn proptest_range_boundaries( 53 | // Ensure we can always subtract one from start 54 | start in 1..=262_143_u32, 55 | len in 0..=262_143_u32, 56 | ) { 57 | let mut bitmap = RoaringBitmap::new(); 58 | let end = start + len; 59 | let half = start + len / 2; 60 | bitmap.insert_range(start..end); 61 | 62 | assert!(bitmap.contains_range(start..end)); 63 | 64 | assert!(bitmap.contains_range(start+1..end)); 65 | assert!(bitmap.contains_range(start..end - 1)); 66 | assert!(bitmap.contains_range(start+1..end - 1)); 67 | 68 | assert!(!bitmap.contains_range(start - 1..end)); 69 | assert!(!bitmap.contains_range(start - 1..end - 1)); 70 | assert!(!bitmap.contains_range(start..end + 1)); 71 | assert!(!bitmap.contains_range(start + 1..end + 1)); 72 | assert!(!bitmap.contains_range(start - 1..end + 1)); 73 | 74 | assert!(!bitmap.contains_range(start - 1..half)); 75 | assert!(!bitmap.contains_range(half..end + 1)); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /roaring/src/treemap/util.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds, RangeInclusive}; 2 | 3 | #[inline] 4 | pub fn split(value: u64) -> (u32, u32) { 5 | ((value >> 32) as u32, value as u32) 6 | } 7 | 8 | #[inline] 9 | pub fn join(high: u32, low: u32) -> u64 { 10 | (u64::from(high) << 32) | u64::from(low) 11 | } 12 | 13 | /// Convert a `RangeBounds` object to `RangeInclusive`, 14 | pub fn convert_range_to_inclusive(range: R) -> Option> 15 | where 16 | R: RangeBounds, 17 | { 18 | let start: u64 = match range.start_bound() { 19 | Bound::Included(&i) => i, 20 | Bound::Excluded(&u64::MAX) => return None, 21 | Bound::Excluded(&i) => i + 1, 22 | Bound::Unbounded => 0, 23 | }; 24 | let end: u64 = match range.end_bound() { 25 | Bound::Included(&i) => i, 26 | Bound::Excluded(&0) => return None, 27 | Bound::Excluded(&i) => i - 1, 28 | Bound::Unbounded => u64::MAX, 29 | }; 30 | if end < start { 31 | return None; 32 | } 33 | Some(start..=end) 34 | } 35 | 36 | #[cfg(test)] 37 | mod test { 38 | use super::{convert_range_to_inclusive, join, split}; 39 | 40 | #[test] 41 | fn test_split_u64() { 42 | assert_eq!((0x0000_0000u32, 0x0000_0000u32), split(0x0000_0000_0000_0000u64)); 43 | assert_eq!((0x0000_0000u32, 0x0000_0001u32), split(0x0000_0000_0000_0001u64)); 44 | assert_eq!((0x0000_0000u32, 0xFFFF_FFFEu32), split(0x0000_0000_FFFF_FFFEu64)); 45 | assert_eq!((0x0000_0000u32, 0xFFFF_FFFFu32), split(0x0000_0000_FFFF_FFFFu64)); 46 | assert_eq!((0x0000_0001u32, 0x0000_0000u32), split(0x0000_0001_0000_0000u64)); 47 | assert_eq!((0x0000_0001u32, 0x0000_0001u32), split(0x0000_0001_0000_0001u64)); 48 | assert_eq!((0xFFFF_FFFFu32, 0xFFFF_FFFEu32), split(0xFFFF_FFFF_FFFF_FFFEu64)); 49 | assert_eq!((0xFFFF_FFFFu32, 0xFFFF_FFFFu32), split(0xFFFF_FFFF_FFFF_FFFFu64)); 50 | } 51 | 52 | #[test] 53 | fn test_join_u64() { 54 | assert_eq!(0x0000_0000_0000_0000u64, join(0x0000_0000u32, 0x0000_0000u32)); 55 | assert_eq!(0x0000_0000_0000_0001u64, join(0x0000_0000u32, 0x0000_0001u32)); 56 | assert_eq!(0x0000_0000_FFFF_FFFEu64, join(0x0000_0000u32, 0xFFFF_FFFEu32)); 57 | assert_eq!(0x0000_0000_FFFF_FFFFu64, join(0x0000_0000u32, 0xFFFF_FFFFu32)); 58 | assert_eq!(0x0000_0001_0000_0000u64, join(0x0000_0001u32, 0x0000_0000u32)); 59 | assert_eq!(0x0000_0001_0000_0001u64, join(0x0000_0001u32, 0x0000_0001u32)); 60 | assert_eq!(0xFFFF_FFFF_FFFF_FFFEu64, join(0xFFFF_FFFFu32, 0xFFFF_FFFEu32)); 61 | assert_eq!(0xFFFF_FFFF_FFFF_FFFFu64, join(0xFFFF_FFFFu32, 0xFFFF_FFFFu32)); 62 | } 63 | 64 | #[test] 65 | fn test_convert_range_to_inclusive() { 66 | assert_eq!(Some(1..=5), convert_range_to_inclusive(1..6)); 67 | assert_eq!(Some(1..=u64::MAX), convert_range_to_inclusive(1..)); 68 | assert_eq!(Some(0..=u64::MAX), convert_range_to_inclusive(..)); 69 | assert_eq!(None, convert_range_to_inclusive(5..5)); 70 | assert_eq!(Some(16..=16), convert_range_to_inclusive(16..=16)) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /roaring/tests/iter_range.rs: -------------------------------------------------------------------------------- 1 | use proptest::collection::btree_set; 2 | use proptest::prelude::*; 3 | use roaring::RoaringBitmap; 4 | use std::ops::Bound; 5 | 6 | #[test] 7 | fn range_array() { 8 | let mut rb = RoaringBitmap::new(); 9 | rb.insert(0); 10 | rb.insert(1); 11 | rb.insert(10); 12 | rb.insert(100_000); 13 | rb.insert(999_999); 14 | rb.insert(1_000_000); 15 | 16 | let expected = vec![1, 10, 100_000, 999_999]; 17 | let actual: Vec = rb.range(1..=999_999).collect(); 18 | assert_eq!(expected, actual); 19 | } 20 | 21 | #[test] 22 | fn range_bitmap() { 23 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 24 | 25 | let expected = vec![10, 11, 12]; 26 | let actual: Vec = rb.range(0..13).collect(); 27 | assert_eq!(expected, actual); 28 | } 29 | 30 | #[test] 31 | fn empty_range() { 32 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 33 | 34 | let mut it = rb.range(0..0); 35 | assert_eq!(it.next(), None); 36 | let mut it = rb.range(..0); 37 | assert_eq!(it.next(), None); 38 | it = rb.range(13..13); 39 | assert_eq!(it.next(), None); 40 | it = rb.range((Bound::Excluded(1), Bound::Included(1))); 41 | assert_eq!(it.next(), None); 42 | it = rb.range(u32::MAX..u32::MAX); 43 | assert_eq!(it.next(), None); 44 | it = rb.range((Bound::Excluded(u32::MAX), Bound::Included(u32::MAX))); 45 | assert_eq!(it.next(), None); 46 | it = rb.range((Bound::Excluded(u32::MAX), Bound::Unbounded)); 47 | assert_eq!(it.next(), None); 48 | } 49 | 50 | #[test] 51 | #[should_panic(expected = "range start is greater than range end")] 52 | fn invalid_range() { 53 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 54 | #[allow(clippy::reversed_empty_ranges)] 55 | let _ = rb.range(13..0); 56 | } 57 | 58 | #[test] 59 | #[should_panic(expected = "range start and end are equal and excluded")] 60 | fn invalid_range_equal_excluded() { 61 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 62 | let _ = rb.range((Bound::Excluded(13), Bound::Excluded(13))); 63 | } 64 | 65 | #[test] 66 | #[should_panic(expected = "range start is greater than range end")] 67 | fn into_invalid_range() { 68 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 69 | #[allow(clippy::reversed_empty_ranges)] 70 | let _ = rb.into_range(13..0); 71 | } 72 | 73 | #[test] 74 | #[should_panic(expected = "range start and end are equal and excluded")] 75 | fn into_invalid_range_equal_excluded() { 76 | let rb = RoaringBitmap::from_sorted_iter(10..5000).unwrap(); 77 | let _ = rb.into_range((Bound::Excluded(13), Bound::Excluded(13))); 78 | } 79 | 80 | proptest! { 81 | #[test] 82 | fn proptest_range( 83 | values in btree_set(..=262_143_u32, ..=1000), 84 | range_a in 0u32..262_143, 85 | range_b in 0u32..262_143, 86 | ){ 87 | let range = range_a.min(range_b)..=range_a.max(range_b); 88 | 89 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 90 | let expected: Vec = values.range(range.clone()).copied().collect(); 91 | let actual: Vec = bitmap.range(range.clone()).collect(); 92 | 93 | assert_eq!(expected, actual); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RoaringBitmap [![github-actions-badge][]][github-actions] [![release-badge][]][cargo] [![docs-badge][]][docs] [![rust-version-badge][]][rust-version] 2 | 3 | This is a [Rust][] port of the [Roaring bitmap][] data structure, initially 4 | defined as a [Java library][roaring-java] and described in [_Better bitmap 5 | performance with Roaring bitmaps_][roaring-paper]. 6 | 7 | ## Rust version policy 8 | 9 | This crate only supports the current stable version of Rust, patch releases may 10 | use new features at any time. 11 | 12 | ## Developing 13 | 14 | This project uses [Clippy][], [rustfmt][], and denies warnings in CI builds. Available via 15 | `rustup component add clippy rustfmt`. 16 | 17 | To ensure your changes will be accepted please check them with: 18 | ``` 19 | cargo fmt -- --check 20 | cargo clippy --all-targets -- -D warnings 21 | ``` 22 | 23 | In addition, ensure all tests are passing with `cargo test` 24 | 25 | ### Benchmarking 26 | 27 | It is recommended to run the `cargo bench` command. 28 | The [benchmarks directory](./benchmarks) contains a library that is dedicated to benchmarking the 29 | Roaring library by using a set of [real-world datasets][]. It is also advised to run the benchmarks 30 | on a bare-metal machine, running them on the base branch and then on the contribution PR 31 | branch to better see the changes. 32 | 33 | Those benchmarks are designed on top of the Criterion library, 34 | you can read more about it [on the user guide][]. 35 | 36 | ## License 37 | 38 | Licensed under either of 39 | 40 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) 41 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT) 42 | 43 | at your option. 44 | 45 | ### Contribution 46 | 47 | Unless you explicitly state otherwise, any contribution intentionally submitted 48 | for inclusion in the work by you shall be dual licensed as above, without any 49 | additional terms or conditions. 50 | 51 | [github-actions-badge]: 52 | https://github.com/RoaringBitmap/roaring-rs/actions/workflows/test.yml/badge.svg 53 | [github-actions]: https://github.com/RoaringBitmap/roaring-rs/actions 54 | [release-badge]: https://img.shields.io/github/release/RoaringBitmap/roaring-rs.svg?style=flat-square 55 | [cargo]: https://crates.io/crates/roaring 56 | [docs-badge]: https://img.shields.io/badge/API-docs-blue.svg?style=flat-square 57 | [docs]: https://docs.rs/roaring 58 | [rust-version-badge]: https://img.shields.io/badge/rust-latest%20stable-blue.svg?style=flat-square 59 | [rust-version]: https://github.com/RoaringBitmap/roaring-rs#rust-version-policy 60 | 61 | [Rust]: https://www.rust-lang.org/ 62 | [Roaring bitmap]: https://roaringbitmap.org/ 63 | [roaring-java]: https://github.com/lemire/RoaringBitmap 64 | [roaring-paper]: https://arxiv.org/pdf/1402.6407v4 65 | [Clippy]: https://github.com/rust-lang/rust-clippy 66 | [rustfmt]: https://github.com/rust-lang/rustfmt 67 | 68 | [real-world datasets]: https://github.com/RoaringBitmap/real-roaring-datasets 69 | [on the user guide]: https://bheisler.github.io/criterion.rs/book/user_guide/user_guide.html 70 | 71 | ## Experimental features 72 | 73 | The `simd` feature is in active development. It has not been tested. If you would like to build with `simd` note that 74 | `std::simd` is only available in Rust nightly. 75 | -------------------------------------------------------------------------------- /roaring/src/bitmap/store/array_store/visitor.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "simd")] 2 | use crate::bitmap::store::array_store::vector::swizzle_to_front; 3 | 4 | #[cfg(not(feature = "std"))] 5 | use alloc::vec::Vec; 6 | 7 | /// This visitor pattern allows multiple different algorithms to be written over the same data 8 | /// For example: vectorized algorithms can pass a visitor off to a scalar algorithm to finish off 9 | /// a tail that is not a multiple of the vector width. 10 | /// 11 | /// Perhaps more importantly: it separates the set algorithms from the operations performed on 12 | /// their results. Future work can utilize the exiting algorithms to trivially implement 13 | /// computing the cardinality of an operation without materializng a new bitmap. 14 | pub trait BinaryOperationVisitor { 15 | #[cfg(feature = "simd")] 16 | fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8); 17 | fn visit_scalar(&mut self, value: u16); 18 | fn visit_slice(&mut self, values: &[u16]); 19 | } 20 | 21 | /// A simple visitor that stores the computation result to a Vec 22 | /// accessible by calling `into_inner()` 23 | pub struct VecWriter { 24 | vec: Vec, 25 | } 26 | 27 | impl VecWriter { 28 | pub fn new(capacity: usize) -> VecWriter { 29 | let vec = Vec::with_capacity(capacity); 30 | VecWriter { vec } 31 | } 32 | 33 | pub fn into_inner(self) -> Vec { 34 | // Consider shrinking the vec here. 35 | // Exactly len could be too aggressive. Len rounded up to next power of 2? 36 | // Related, but not exact issue: https://github.com/RoaringBitmap/roaring-rs/issues/136 37 | self.vec 38 | } 39 | } 40 | 41 | impl BinaryOperationVisitor for VecWriter { 42 | #[cfg(feature = "simd")] 43 | fn visit_vector(&mut self, value: core::simd::u16x8, mask: u8) { 44 | let result = swizzle_to_front(value, mask); 45 | 46 | // This idiom is better than subslicing result, as it compiles down to an unaligned vector 47 | // store instr. 48 | // A more straightforward, but unsafe way would be ptr::write_unaligned and Vec::set_len 49 | // Writing a vector at once is why the vectorized algorithms do not operate in place 50 | // first write the entire vector 51 | self.vec.extend_from_slice(&result.as_array()[..]); 52 | // next truncate the masked out values 53 | self.vec.truncate(self.vec.len() - (result.len() - mask.count_ones() as usize)); 54 | } 55 | 56 | fn visit_scalar(&mut self, value: u16) { 57 | self.vec.push(value) 58 | } 59 | 60 | fn visit_slice(&mut self, values: &[u16]) { 61 | self.vec.extend_from_slice(values); 62 | } 63 | } 64 | 65 | pub struct CardinalityCounter { 66 | count: usize, 67 | } 68 | 69 | impl CardinalityCounter { 70 | pub fn new() -> CardinalityCounter { 71 | CardinalityCounter { count: 0 } 72 | } 73 | 74 | pub fn into_inner(self) -> u64 { 75 | self.count as u64 76 | } 77 | } 78 | 79 | impl BinaryOperationVisitor for CardinalityCounter { 80 | #[cfg(feature = "simd")] 81 | fn visit_vector(&mut self, _value: core::simd::u16x8, mask: u8) { 82 | self.count += mask.count_ones() as usize; 83 | } 84 | 85 | fn visit_scalar(&mut self, _value: u16) { 86 | self.count += 1; 87 | } 88 | 89 | fn visit_slice(&mut self, values: &[u16]) { 90 | self.count += values.len(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /roaring/src/bitmap/store/array_store/scalar.rs: -------------------------------------------------------------------------------- 1 | //! Scalar arithmetic binary set operations on `ArrayStore`'s inner types 2 | 3 | use crate::bitmap::store::array_store::visitor::BinaryOperationVisitor; 4 | use core::cmp::Ordering::*; 5 | 6 | #[inline] 7 | pub fn or(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { 8 | // Traverse both arrays 9 | let mut i = 0; 10 | let mut j = 0; 11 | while i < lhs.len() && j < rhs.len() { 12 | let a = unsafe { lhs.get_unchecked(i) }; 13 | let b = unsafe { rhs.get_unchecked(j) }; 14 | match a.cmp(b) { 15 | Less => { 16 | visitor.visit_scalar(*a); 17 | i += 1; 18 | } 19 | Greater => { 20 | visitor.visit_scalar(*b); 21 | j += 1; 22 | } 23 | Equal => { 24 | visitor.visit_scalar(*a); 25 | i += 1; 26 | j += 1; 27 | } 28 | } 29 | } 30 | 31 | // Store remaining elements of the arrays 32 | visitor.visit_slice(&lhs[i..]); 33 | visitor.visit_slice(&rhs[j..]); 34 | } 35 | 36 | #[inline] 37 | pub fn and(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { 38 | // Traverse both arrays 39 | let mut i = 0; 40 | let mut j = 0; 41 | while i < lhs.len() && j < rhs.len() { 42 | let a = unsafe { lhs.get_unchecked(i) }; 43 | let b = unsafe { rhs.get_unchecked(j) }; 44 | match a.cmp(b) { 45 | Less => i += 1, 46 | Greater => j += 1, 47 | Equal => { 48 | visitor.visit_scalar(*a); 49 | i += 1; 50 | j += 1; 51 | } 52 | } 53 | } 54 | } 55 | 56 | #[inline] 57 | pub fn sub(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { 58 | // Traverse both arrays 59 | let mut i = 0; 60 | let mut j = 0; 61 | while i < lhs.len() && j < rhs.len() { 62 | let a = unsafe { lhs.get_unchecked(i) }; 63 | let b = unsafe { rhs.get_unchecked(j) }; 64 | match a.cmp(b) { 65 | Less => { 66 | visitor.visit_scalar(*a); 67 | i += 1; 68 | } 69 | Greater => j += 1, 70 | Equal => { 71 | i += 1; 72 | j += 1; 73 | } 74 | } 75 | } 76 | 77 | // Store remaining elements of the left array 78 | visitor.visit_slice(&lhs[i..]); 79 | } 80 | 81 | #[inline] 82 | pub fn xor(lhs: &[u16], rhs: &[u16], visitor: &mut impl BinaryOperationVisitor) { 83 | // Traverse both arrays 84 | let mut i = 0; 85 | let mut j = 0; 86 | while i < lhs.len() && j < rhs.len() { 87 | let a = unsafe { lhs.get_unchecked(i) }; 88 | let b = unsafe { rhs.get_unchecked(j) }; 89 | match a.cmp(b) { 90 | Less => { 91 | visitor.visit_scalar(*a); 92 | i += 1; 93 | } 94 | Greater => { 95 | visitor.visit_scalar(*b); 96 | j += 1; 97 | } 98 | Equal => { 99 | i += 1; 100 | j += 1; 101 | } 102 | } 103 | } 104 | 105 | // Store remaining elements of the arrays 106 | visitor.visit_slice(&lhs[i..]); 107 | visitor.visit_slice(&rhs[j..]); 108 | } 109 | -------------------------------------------------------------------------------- /roaring/tests/treemap_difference_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (0..1000).collect::(); 9 | 10 | bitmap1 -= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_difference() { 17 | let mut bitmap1 = (1..3).collect::(); 18 | let bitmap2 = (1..3).collect::(); 19 | 20 | bitmap1 -= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringTreemap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = (0..1000).collect::(); 30 | 31 | bitmap1 -= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = (0..6000).collect::(); 41 | 42 | bitmap1 -= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (3000..9000).collect::(); 51 | let bitmap3 = (0..3000).collect::(); 52 | 53 | bitmap1 -= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array_to_bitmap() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (9000..12000).collect::(); 62 | let bitmap3 = (0..9000).collect::(); 63 | 64 | bitmap1 -= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn bitmap_and_array_to_array() { 71 | let mut bitmap1 = (0..6000).collect::(); 72 | let bitmap2 = (3000..6000).collect::(); 73 | let bitmap3 = (0..3000).collect::(); 74 | 75 | bitmap1 -= bitmap2; 76 | 77 | assert_eq!(bitmap1, bitmap3); 78 | } 79 | 80 | #[test] 81 | fn arrays() { 82 | let mut bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) 83 | .collect::(); 84 | let bitmap2 = ((1000..3000).chain(1_001_000..1_003_000).chain(2_000_000..2_001_000)) 85 | .collect::(); 86 | let bitmap3 = ((0..1000).chain(1_000_000..1_001_000)).collect::(); 87 | 88 | bitmap1 -= bitmap2; 89 | 90 | assert_eq!(bitmap1, bitmap3); 91 | } 92 | 93 | #[test] 94 | fn arrays_removing_one_whole_container() { 95 | let mut bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) 96 | .collect::(); 97 | let bitmap2 = ((0..3000).chain(1_001_000..1_003_000).chain(2_000_000..2_001_000)) 98 | .collect::(); 99 | let bitmap3 = (1_000_000..1_001_000).collect::(); 100 | 101 | bitmap1 -= bitmap2; 102 | 103 | assert_eq!(bitmap1, bitmap3); 104 | } 105 | 106 | #[test] 107 | fn bitmaps() { 108 | let mut bitmap1 = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) 109 | .collect::(); 110 | let bitmap2 = ((3000..9000).chain(1_006_000..1_018_000).chain(2_000_000..2_010_000)) 111 | .collect::(); 112 | let bitmap3 = ((0..3000).chain(1_000_000..1_006_000)).collect::(); 113 | 114 | bitmap1 -= bitmap2; 115 | 116 | assert_eq!(bitmap1, bitmap3); 117 | } 118 | -------------------------------------------------------------------------------- /roaring/tests/treemap_symmetric_difference_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = ((0..1000).chain(2000..3000)).collect::(); 9 | 10 | bitmap1 ^= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_symmetric_difference() { 17 | let mut bitmap1 = (0..2).collect::(); 18 | let bitmap2 = (0..2).collect::(); 19 | 20 | bitmap1 ^= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringTreemap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = ((0..1000).chain(2000..8000)).collect::(); 30 | 31 | bitmap1 ^= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = ((0..6000).chain(12000..18000)).collect::(); 41 | 42 | bitmap1 ^= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (2000..7000).collect::(); 51 | let bitmap3 = ((0..2000).chain(6000..7000)).collect::(); 52 | 53 | bitmap1 ^= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array_to_bitmap() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (11000..14000).collect::(); 62 | let bitmap3 = ((0..11000).chain(12000..14000)).collect::(); 63 | 64 | bitmap1 ^= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn bitmap_and_array_to_array() { 71 | let mut bitmap1 = (0..6000).collect::(); 72 | let bitmap2 = (3000..7000).collect::(); 73 | let bitmap3 = ((0..3000).chain(6000..7000)).collect::(); 74 | 75 | bitmap1 ^= bitmap2; 76 | 77 | assert_eq!(bitmap1, bitmap3); 78 | } 79 | 80 | #[test] 81 | fn arrays() { 82 | let mut bitmap1 = ((0..2000).chain(1_000_000..1_002_000).chain(3_000_000..3_001_000)) 83 | .collect::(); 84 | let bitmap2 = ((1000..3000).chain(1_001_000..1_003_000).chain(2_000_000..2_000_001)) 85 | .collect::(); 86 | let bitmap3 = ((0..1000) 87 | .chain(1_000_000..1_001_000) 88 | .chain(2000..3000) 89 | .chain(1_002_000..1_003_000) 90 | .chain(2_000_000..2_000_001) 91 | .chain(3_000_000..3_001_000)) 92 | .collect::(); 93 | 94 | bitmap1 ^= bitmap2; 95 | 96 | assert_eq!(bitmap1, bitmap3); 97 | } 98 | 99 | #[test] 100 | fn bitmaps() { 101 | let mut bitmap1 = ((0..6000).chain(1_000_000..1_012_000).chain(3_000_000..3_010_000)) 102 | .collect::(); 103 | let bitmap2 = ((3000..7000).chain(1_006_000..1_018_000).chain(2_000_000..2_010_000)) 104 | .collect::(); 105 | let bitmap3 = ((0..3000) 106 | .chain(1_000_000..1_006_000) 107 | .chain(6000..7000) 108 | .chain(1_012_000..1_018_000) 109 | .chain(2_000_000..2_010_000) 110 | .chain(3_000_000..3_010_000)) 111 | .collect::(); 112 | 113 | bitmap1 ^= bitmap2; 114 | 115 | assert_eq!(bitmap1, bitmap3); 116 | } 117 | -------------------------------------------------------------------------------- /roaring/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This is a [Rust][] port of the [Roaring bitmap][] data structure, initially 2 | //! defined as a [Java library][roaring-java] and described in [_Better bitmap 3 | //! performance with Roaring bitmaps_][roaring-paper]. 4 | //! 5 | //! [Rust]: https://www.rust-lang.org/ 6 | //! [Roaring bitmap]: https://roaringbitmap.org/ 7 | //! [roaring-java]: https://github.com/lemire/RoaringBitmap 8 | //! [roaring-paper]: https://arxiv.org/pdf/1402.6407v4 9 | 10 | #![cfg_attr(not(feature = "std"), no_std)] 11 | #![cfg_attr(feature = "simd", feature(portable_simd))] 12 | #![warn(missing_docs)] 13 | #![warn(unsafe_op_in_unsafe_fn)] 14 | #![warn(variant_size_differences)] 15 | #![allow(unknown_lints)] // For clippy 16 | #![allow(clippy::doc_overindented_list_items)] 17 | #![deny(unnameable_types)] 18 | 19 | #[cfg(feature = "std")] 20 | extern crate byteorder; 21 | 22 | #[macro_use] 23 | extern crate alloc; 24 | 25 | use core::fmt; 26 | 27 | /// A compressed bitmap using the [Roaring bitmap compression scheme](https://roaringbitmap.org/). 28 | pub mod bitmap; 29 | 30 | /// A compressed bitmap with u64 values. Implemented as a `BTreeMap` of `RoaringBitmap`s. 31 | pub mod treemap; 32 | 33 | pub use bitmap::RoaringBitmap; 34 | pub use treemap::RoaringTreemap; 35 | 36 | /// An error type that is returned when a `try_push` in a bitmap did not succeed. 37 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 38 | pub struct IntegerTooSmall; 39 | 40 | impl fmt::Display for IntegerTooSmall { 41 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 42 | f.write_str("inserted integer is smaller than the largest integer") 43 | } 44 | } 45 | 46 | /// An error type that is returned when an iterator isn't sorted. 47 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 48 | pub struct NonSortedIntegers { 49 | valid_until: u64, 50 | } 51 | 52 | impl NonSortedIntegers { 53 | /// Returns the number of elements that were 54 | pub fn valid_until(&self) -> u64 { 55 | self.valid_until 56 | } 57 | } 58 | 59 | impl fmt::Display for NonSortedIntegers { 60 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 61 | write!(f, "integers are ordered up to the {}th element", self.valid_until()) 62 | } 63 | } 64 | 65 | #[cfg(feature = "std")] 66 | impl std::error::Error for NonSortedIntegers {} 67 | 68 | /// A [`Iterator::collect`] blanket implementation that provides extra methods for [`RoaringBitmap`] 69 | /// and [`RoaringTreemap`]. 70 | /// 71 | /// When merging multiple bitmap with the same operation it's usually faster to call the 72 | /// method in this trait than to write your own for loop and merging the bitmaps yourself. 73 | /// 74 | /// # Examples 75 | /// ``` 76 | /// use roaring::{MultiOps, RoaringBitmap}; 77 | /// 78 | /// let bitmaps = [ 79 | /// RoaringBitmap::from_iter(0..10), 80 | /// RoaringBitmap::from_iter(10..20), 81 | /// RoaringBitmap::from_iter(20..30), 82 | /// ]; 83 | /// 84 | /// // Stop doing this 85 | /// let naive = bitmaps.clone().into_iter().reduce(|a, b| a | b).unwrap_or_default(); 86 | /// 87 | /// // And start doing this instead, it will be much faster! 88 | /// let iter = bitmaps.union(); 89 | /// 90 | /// assert_eq!(naive, iter); 91 | /// ``` 92 | pub trait MultiOps: IntoIterator { 93 | /// The type of output from operations. 94 | type Output; 95 | 96 | /// The `union` between all elements. 97 | fn union(self) -> Self::Output; 98 | 99 | /// The `intersection` between all elements. 100 | fn intersection(self) -> Self::Output; 101 | 102 | /// The `difference` between all elements. 103 | fn difference(self) -> Self::Output; 104 | 105 | /// The `symmetric difference` between all elements. 106 | fn symmetric_difference(self) -> Self::Output; 107 | } 108 | -------------------------------------------------------------------------------- /roaring/tests/treemap_lib.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn smoke() { 6 | let mut bitmap = RoaringTreemap::new(); 7 | assert_eq!(bitmap.len(), 0); 8 | assert!(bitmap.is_empty()); 9 | bitmap.remove(0); 10 | assert_eq!(bitmap.len(), 0); 11 | assert!(bitmap.is_empty()); 12 | bitmap.insert(1); 13 | assert!(bitmap.contains(1)); 14 | assert_eq!(bitmap.len(), 1); 15 | assert!(!bitmap.is_empty()); 16 | bitmap.insert(u64::MAX - 2); 17 | assert!(bitmap.contains(u64::MAX - 2)); 18 | assert_eq!(bitmap.len(), 2); 19 | bitmap.insert(u64::MAX); 20 | assert!(bitmap.contains(u64::MAX)); 21 | assert_eq!(bitmap.len(), 3); 22 | bitmap.insert(2); 23 | assert!(bitmap.contains(2)); 24 | assert_eq!(bitmap.len(), 4); 25 | bitmap.remove(2); 26 | assert!(!bitmap.contains(2)); 27 | assert_eq!(bitmap.len(), 3); 28 | assert!(!bitmap.contains(0)); 29 | assert!(bitmap.contains(1)); 30 | assert!(!bitmap.contains(100)); 31 | assert!(bitmap.contains(u64::MAX - 2)); 32 | assert!(!bitmap.contains(u64::MAX - 1)); 33 | assert!(bitmap.contains(u64::MAX)); 34 | } 35 | 36 | #[test] 37 | fn insert_range() { 38 | let ranges = 0..0x1000; 39 | const SIGMA: u64 = u32::MAX as u64; 40 | 41 | let mut bitmap = RoaringTreemap::new(); 42 | assert_eq!(bitmap.insert_range(ranges), 0x1000); 43 | assert_eq!(bitmap.len(), 0x1000); 44 | assert_eq!(bitmap.max(), Some(0xFFF)); 45 | 46 | assert_eq!(bitmap.insert_range(u32::MAX as u64 - 1..u32::MAX as u64 + 1), 2); 47 | assert!(bitmap.contains(2)); 48 | assert!(bitmap.contains(0xFFF)); 49 | assert!(!bitmap.contains(0x1000)); 50 | 51 | bitmap.clear(); 52 | bitmap.insert_range(2 * SIGMA..=4 * SIGMA); 53 | 54 | assert_eq!(bitmap.min(), Some(2 * SIGMA)); 55 | assert_eq!(bitmap.max(), Some(4 * SIGMA)); 56 | 57 | assert!(bitmap.contains(3 * SIGMA)); 58 | } 59 | 60 | #[test] 61 | fn remove_range() { 62 | let ranges = [0u64, 1, 63, 64, 65, 100, 4096 - 1, 4096, 4096 + 1, 65536 - 1]; 63 | for (i, &a) in ranges.iter().enumerate() { 64 | for &b in &ranges[i..] { 65 | let mut bitmap = (0..=65536).collect::(); 66 | assert_eq!(bitmap.remove_range(a..b), (b - a)); 67 | assert_eq!(bitmap, ((0..a).chain(b..=65536)).collect::()); 68 | } 69 | } 70 | } 71 | 72 | #[test] 73 | fn test_max() { 74 | let mut bitmap = RoaringTreemap::new(); 75 | assert_eq!(bitmap.max(), None); 76 | bitmap.insert(0); 77 | assert_eq!(bitmap.max(), Some(0)); 78 | bitmap.insert(1); 79 | assert_eq!(bitmap.max(), Some(1)); 80 | bitmap.insert(u64::MAX); 81 | assert_eq!(bitmap.max(), Some(u64::MAX)); 82 | } 83 | 84 | #[test] 85 | fn test_min() { 86 | let mut bitmap = RoaringTreemap::new(); 87 | assert_eq!(bitmap.min(), None); 88 | bitmap.insert(u64::MAX); 89 | assert_eq!(bitmap.min(), Some(u64::MAX)); 90 | bitmap.insert(1); 91 | assert_eq!(bitmap.min(), Some(1)); 92 | bitmap.insert(0); 93 | assert_eq!(bitmap.min(), Some(0)); 94 | } 95 | 96 | #[test] 97 | fn to_bitmap() { 98 | let bitmap = (0..5000).collect::(); 99 | assert_eq!(bitmap.len(), 5000); 100 | for i in 1..5000 { 101 | assert!(bitmap.contains(i)); 102 | } 103 | assert!(!bitmap.contains(5001)); 104 | } 105 | 106 | #[test] 107 | fn to_array() { 108 | let mut bitmap = (0..5000).collect::(); 109 | for i in 3000..5000 { 110 | bitmap.remove(i); 111 | } 112 | assert_eq!(bitmap.len(), 3000); 113 | for i in 0..3000 { 114 | assert!(bitmap.contains(i)); 115 | } 116 | for i in 3000..5000 { 117 | assert!(!bitmap.contains(i)); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /roaring/tests/symmetric_difference_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (0..1000).chain(2000..3000).collect::(); 9 | 10 | bitmap1 ^= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_symmetric_difference() { 17 | let mut bitmap1 = (0..2).collect::(); 18 | let bitmap2 = (0..2).collect::(); 19 | 20 | bitmap1 ^= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringBitmap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = (0..1000).chain(2000..8000).collect::(); 30 | 31 | bitmap1 ^= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = (0..6000).chain(12000..18000).collect::(); 41 | 42 | bitmap1 ^= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (2000..7000).collect::(); 51 | let bitmap3 = (0..2000).chain(6000..7000).collect::(); 52 | 53 | bitmap1 ^= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array_to_bitmap() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (11000..14000).collect::(); 62 | let bitmap3 = (0..11000).chain(12000..14000).collect::(); 63 | 64 | bitmap1 ^= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn bitmap_and_array_to_array() { 71 | let mut bitmap1 = (0..6000).collect::(); 72 | let bitmap2 = (3000..7000).collect::(); 73 | let bitmap3 = (0..3000).chain(6000..7000).collect::(); 74 | 75 | bitmap1 ^= bitmap2; 76 | 77 | assert_eq!(bitmap1, bitmap3); 78 | } 79 | 80 | #[test] 81 | fn arrays() { 82 | let mut bitmap1 = (0..2000) 83 | .chain(1_000_000..1_002_000) 84 | .chain(3_000_000..3_001_000) 85 | .collect::(); 86 | let bitmap2 = (1000..3000) 87 | .chain(1_001_000..1_003_000) 88 | .chain(2_000_000..2_000_001) 89 | .collect::(); 90 | let bitmap3 = (0..1000) 91 | .chain(1_000_000..1_001_000) 92 | .chain(2000..3000) 93 | .chain(1_002_000..1_003_000) 94 | .chain(2_000_000..2_000_001) 95 | .chain(3_000_000..3_001_000) 96 | .collect::(); 97 | 98 | bitmap1 ^= bitmap2; 99 | 100 | assert_eq!(bitmap1, bitmap3); 101 | } 102 | 103 | #[test] 104 | fn bitmaps() { 105 | let mut bitmap1 = (0..6000) 106 | .chain(1_000_000..1_012_000) 107 | .chain(3_000_000..3_010_000) 108 | .collect::(); 109 | let bitmap2 = (3000..7000) 110 | .chain(1_006_000..1_018_000) 111 | .chain(2_000_000..2_010_000) 112 | .collect::(); 113 | let bitmap3 = (0..3000) 114 | .chain(1_000_000..1_006_000) 115 | .chain(6000..7000) 116 | .chain(1_012_000..1_018_000) 117 | .chain(2_000_000..2_010_000) 118 | .chain(3_000_000..3_010_000) 119 | .collect::(); 120 | 121 | bitmap1 ^= bitmap2; 122 | 123 | assert_eq!(bitmap1, bitmap3); 124 | } 125 | -------------------------------------------------------------------------------- /roaring/tests/difference_with.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn array() { 6 | let mut bitmap1 = (0..2000).collect::(); 7 | let bitmap2 = (1000..3000).collect::(); 8 | let bitmap3 = (0..1000).collect::(); 9 | 10 | bitmap1 -= bitmap2; 11 | 12 | assert_eq!(bitmap1, bitmap3); 13 | } 14 | 15 | #[test] 16 | fn no_difference() { 17 | let mut bitmap1 = (1..3).collect::(); 18 | let bitmap2 = (1..3).collect::(); 19 | 20 | bitmap1 -= bitmap2; 21 | 22 | assert_eq!(bitmap1, RoaringBitmap::new()); 23 | } 24 | 25 | #[test] 26 | fn array_and_bitmap() { 27 | let mut bitmap1 = (0..2000).collect::(); 28 | let bitmap2 = (1000..8000).collect::(); 29 | let bitmap3 = (0..1000).collect::(); 30 | 31 | bitmap1 -= bitmap2; 32 | 33 | assert_eq!(bitmap1, bitmap3); 34 | } 35 | 36 | #[test] 37 | fn bitmap_to_bitmap() { 38 | let mut bitmap1 = (0..12000).collect::(); 39 | let bitmap2 = (6000..18000).collect::(); 40 | let bitmap3 = (0..6000).collect::(); 41 | 42 | bitmap1 -= bitmap2; 43 | 44 | assert_eq!(bitmap1, bitmap3); 45 | } 46 | 47 | #[test] 48 | fn bitmap_to_array() { 49 | let mut bitmap1 = (0..6000).collect::(); 50 | let bitmap2 = (3000..9000).collect::(); 51 | let bitmap3 = (0..3000).collect::(); 52 | 53 | bitmap1 -= bitmap2; 54 | 55 | assert_eq!(bitmap1, bitmap3); 56 | } 57 | 58 | #[test] 59 | fn bitmap_and_array_to_bitmap() { 60 | let mut bitmap1 = (0..12000).collect::(); 61 | let bitmap2 = (9000..12000).collect::(); 62 | let bitmap3 = (0..9000).collect::(); 63 | 64 | bitmap1 -= bitmap2; 65 | 66 | assert_eq!(bitmap1, bitmap3); 67 | } 68 | 69 | #[test] 70 | fn bitmap_and_array_to_array() { 71 | let mut bitmap1 = (0..6000).collect::(); 72 | let bitmap2 = (3000..6000).collect::(); 73 | let bitmap3 = (0..3000).collect::(); 74 | 75 | bitmap1 -= bitmap2; 76 | 77 | assert_eq!(bitmap1, bitmap3); 78 | } 79 | 80 | #[test] 81 | fn arrays() { 82 | let mut bitmap1 = (0..2000) 83 | .chain(1_000_000..1_002_000) 84 | .chain(2_000_000..2_001_000) 85 | .collect::(); 86 | let bitmap2 = (1000..3000) 87 | .chain(1_001_000..1_003_000) 88 | .chain(2_000_000..2_001_000) 89 | .collect::(); 90 | let bitmap3 = (0..1000).chain(1_000_000..1_001_000).collect::(); 91 | 92 | bitmap1 -= bitmap2; 93 | 94 | assert_eq!(bitmap1, bitmap3); 95 | } 96 | 97 | #[test] 98 | fn arrays_removing_one_whole_container() { 99 | let mut bitmap1 = (0..2000) 100 | .chain(1_000_000..1_002_000) 101 | .chain(2_000_000..2_001_000) 102 | .collect::(); 103 | let bitmap2 = (0..3000) 104 | .chain(1_001_000..1_003_000) 105 | .chain(2_000_000..2_001_000) 106 | .collect::(); 107 | let bitmap3 = (1_000_000..1_001_000).collect::(); 108 | 109 | bitmap1 -= bitmap2; 110 | 111 | assert_eq!(bitmap1, bitmap3); 112 | } 113 | 114 | #[test] 115 | fn bitmaps() { 116 | let mut bitmap1 = (0..6000) 117 | .chain(1_000_000..1_012_000) 118 | .chain(2_000_000..2_010_000) 119 | .collect::(); 120 | let bitmap2 = (3000..9000) 121 | .chain(1_006_000..1_018_000) 122 | .chain(2_000_000..2_010_000) 123 | .collect::(); 124 | let bitmap3 = (0..3000).chain(1_000_000..1_006_000).collect::(); 125 | 126 | bitmap1 -= bitmap2; 127 | 128 | assert_eq!(bitmap1, bitmap3); 129 | } 130 | -------------------------------------------------------------------------------- /roaring/src/treemap/cmp.rs: -------------------------------------------------------------------------------- 1 | use alloc::collections::btree_map; 2 | use core::iter::Peekable; 3 | 4 | use crate::RoaringBitmap; 5 | use crate::RoaringTreemap; 6 | 7 | pub(crate) struct Pairs<'a>( 8 | Peekable>, 9 | Peekable>, 10 | ); 11 | 12 | impl RoaringTreemap { 13 | pub(crate) fn pairs<'a>(&'a self, other: &'a RoaringTreemap) -> Pairs<'a> { 14 | Pairs(self.map.iter().peekable(), other.map.iter().peekable()) 15 | } 16 | 17 | /// Returns true if the set has no elements in common with other. This is equivalent to 18 | /// checking for an empty intersection. 19 | /// 20 | /// # Examples 21 | /// 22 | /// ```rust 23 | /// use roaring::RoaringTreemap; 24 | /// 25 | /// let mut rb1 = RoaringTreemap::new(); 26 | /// let mut rb2 = RoaringTreemap::new(); 27 | /// 28 | /// rb1.insert(1); 29 | /// 30 | /// assert_eq!(rb1.is_disjoint(&rb2), true); 31 | /// 32 | /// rb2.insert(1); 33 | /// 34 | /// assert_eq!(rb1.is_disjoint(&rb2), false); 35 | /// 36 | /// ``` 37 | pub fn is_disjoint(&self, other: &Self) -> bool { 38 | self.pairs(other) 39 | .filter(|&(c1, c2)| c1.is_some() && c2.is_some()) 40 | .all(|(c1, c2)| c1.unwrap().is_disjoint(c2.unwrap())) 41 | } 42 | 43 | /// Returns `true` if this set is a subset of `other`. 44 | /// 45 | /// # Examples 46 | /// 47 | /// ```rust 48 | /// use roaring::RoaringTreemap; 49 | /// 50 | /// let mut rb1 = RoaringTreemap::new(); 51 | /// let mut rb2 = RoaringTreemap::new(); 52 | /// 53 | /// rb1.insert(1); 54 | /// 55 | /// assert_eq!(rb1.is_subset(&rb2), false); 56 | /// 57 | /// rb2.insert(1); 58 | /// 59 | /// assert_eq!(rb1.is_subset(&rb2), true); 60 | /// 61 | /// rb1.insert(2); 62 | /// 63 | /// assert_eq!(rb1.is_subset(&rb2), false); 64 | /// ``` 65 | pub fn is_subset(&self, other: &Self) -> bool { 66 | for pair in self.pairs(other) { 67 | match pair { 68 | (None, _) => (), 69 | (_, None) => { 70 | return false; 71 | } 72 | (Some(c1), Some(c2)) => { 73 | if !c1.is_subset(c2) { 74 | return false; 75 | } 76 | } 77 | } 78 | } 79 | true 80 | } 81 | 82 | /// Returns `true` if this set is a superset of `other`. 83 | /// 84 | /// # Examples 85 | /// 86 | /// ```rust 87 | /// use roaring::RoaringTreemap; 88 | /// 89 | /// let mut rb1 = RoaringTreemap::new(); 90 | /// let mut rb2 = RoaringTreemap::new(); 91 | /// 92 | /// rb1.insert(1); 93 | /// 94 | /// assert_eq!(rb2.is_superset(&rb1), false); 95 | /// 96 | /// rb2.insert(1); 97 | /// 98 | /// assert_eq!(rb2.is_superset(&rb1), true); 99 | /// 100 | /// rb1.insert(2); 101 | /// 102 | /// assert_eq!(rb2.is_superset(&rb1), false); 103 | /// ``` 104 | pub fn is_superset(&self, other: &Self) -> bool { 105 | other.is_subset(self) 106 | } 107 | } 108 | 109 | impl<'a> Iterator for Pairs<'a> { 110 | type Item = (Option<&'a RoaringBitmap>, Option<&'a RoaringBitmap>); 111 | 112 | fn next(&mut self) -> Option { 113 | enum Which { 114 | Left, 115 | Right, 116 | Both, 117 | None, 118 | } 119 | let which = match (self.0.peek(), self.1.peek()) { 120 | (None, None) => Which::None, 121 | (Some(_), None) => Which::Left, 122 | (None, Some(_)) => Which::Right, 123 | (Some(c1), Some(c2)) => match (c1.0, c2.0) { 124 | (key1, key2) if key1 == key2 => Which::Both, 125 | (key1, key2) if key1 < key2 => Which::Left, 126 | (key1, key2) if key1 > key2 => Which::Right, 127 | (_, _) => unreachable!(), 128 | }, 129 | }; 130 | match which { 131 | Which::Left => Some((self.0.next().map(|e| e.1), None)), 132 | Which::Right => Some((None, self.1.next().map(|e| e.1))), 133 | Which::Both => Some((self.0.next().map(|e| e.1), self.1.next().map(|e| e.1))), 134 | Which::None => None, 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /roaring/tests/treemap_iter.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | mod iter; 3 | use roaring::RoaringTreemap; 4 | 5 | use iter::outside_in; 6 | use proptest::arbitrary::any; 7 | use proptest::collection::btree_set; 8 | use proptest::proptest; 9 | 10 | #[test] 11 | fn range() { 12 | let original = (0..2000).collect::(); 13 | let clone = RoaringTreemap::from_iter(&original); 14 | let clone2 = RoaringTreemap::from_iter(original.clone()); 15 | 16 | assert_eq!(clone, original); 17 | assert_eq!(clone2, original); 18 | } 19 | 20 | #[test] 21 | fn array() { 22 | let original = (0..5).collect::(); 23 | let clone = RoaringTreemap::from([0, 1, 2, 3, 4]); 24 | 25 | assert_eq!(clone, original); 26 | } 27 | 28 | #[test] 29 | fn bitmap() { 30 | let original = (0..6000).collect::(); 31 | let clone = RoaringTreemap::from_iter(&original); 32 | let clone2 = RoaringTreemap::from_iter(original.clone()); 33 | 34 | assert_eq!(clone, original); 35 | assert_eq!(clone2, original); 36 | } 37 | 38 | #[test] 39 | fn arrays() { 40 | let original = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) 41 | .collect::(); 42 | let clone = RoaringTreemap::from_iter(&original); 43 | let clone2 = RoaringTreemap::from_iter(original.clone()); 44 | 45 | assert_eq!(clone, original); 46 | assert_eq!(clone2, original); 47 | } 48 | 49 | #[test] 50 | fn bitmaps() { 51 | let original = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) 52 | .collect::(); 53 | let clone = RoaringTreemap::from_iter(&original); 54 | let clone2 = RoaringTreemap::from_iter(original.clone()); 55 | 56 | assert_eq!(clone, original); 57 | assert_eq!(clone2, original); 58 | } 59 | 60 | #[test] 61 | fn bitmaps_iterator() { 62 | let original = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) 63 | .collect::(); 64 | let clone = RoaringTreemap::from_bitmaps(original.bitmaps().map(|(p, b)| (p, b.clone()))); 65 | let clone2 = original.bitmaps().map(|(p, b)| (p, b.clone())).collect::(); 66 | 67 | assert_eq!(clone, original); 68 | assert_eq!(clone2, original); 69 | } 70 | 71 | proptest! { 72 | #[test] 73 | fn iter(values in btree_set(any::(), ..=10_000)) { 74 | let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 75 | 76 | assert!(values.into_iter().eq(bitmap)); 77 | } 78 | } 79 | 80 | proptest! { 81 | #[test] 82 | fn fold(values in btree_set(any::(), ..=10_000)) { 83 | let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 84 | let mut val_iter = values.into_iter(); 85 | // `Iterator::all` uses currently unimplementable `try_fold`, we test `fold` 86 | #[allow(clippy::unnecessary_fold)] 87 | let r = bitmap.into_iter().fold(true, |b, i| { 88 | b && i == val_iter.next().unwrap() 89 | }); 90 | assert!(r) 91 | } 92 | } 93 | 94 | #[test] 95 | fn rev() { 96 | let values = (1..3) 97 | .chain(1_000_000..1_012_003) 98 | .chain(2_000_001..2_000_003) 99 | .chain(2_000_000_000_001..2_000_000_000_003); 100 | let bitmap = RoaringTreemap::from_iter(values.clone()); 101 | 102 | assert!(values.into_iter().rev().eq(bitmap.iter().rev())); 103 | } 104 | 105 | proptest! { 106 | #[test] 107 | fn rev_iter(values in btree_set(any::(), ..=10_000)) { 108 | let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 109 | 110 | assert!(values.into_iter().rev().eq(bitmap.iter().rev())); 111 | } 112 | } 113 | 114 | #[test] 115 | fn from_iter() { 116 | // This test verifies that the public API allows conversion from iterators 117 | // with u64 as well as &u64 elements. 118 | let vals = vec![1, 5, 1_000_000_000_000_000]; 119 | let a = RoaringTreemap::from_iter(vals.iter()); 120 | let b = RoaringTreemap::from_iter(vals); 121 | assert_eq!(a, b); 122 | } 123 | 124 | #[test] 125 | fn interleaved() { 126 | let values = (1..3) 127 | .chain(1_000_000..1_012_003) 128 | .chain(2_000_001..2_000_003) 129 | .chain(2_000_000_000_001..2_000_000_000_003); 130 | let bitmap = RoaringTreemap::from_iter(values.clone()); 131 | 132 | assert!(outside_in(values).eq(outside_in(bitmap))); 133 | } 134 | 135 | proptest! { 136 | #[test] 137 | fn interleaved_iter(values in btree_set(any::(), 50_000..=100_000)) { 138 | let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); 139 | 140 | assert!(outside_in(values).eq(outside_in(bitmap))); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /roaring/src/treemap/serialization.rs: -------------------------------------------------------------------------------- 1 | use super::RoaringTreemap; 2 | use crate::RoaringBitmap; 3 | use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; 4 | use std::{io, mem::size_of}; 5 | 6 | impl RoaringTreemap { 7 | /// Return the size in bytes of the serialized output. 8 | /// This is compatible with the official C/C++, Java and Go implementations. 9 | /// 10 | /// # Examples 11 | /// 12 | /// ```rust 13 | /// use roaring::RoaringTreemap; 14 | /// 15 | /// let rb1: RoaringTreemap = (1..4).collect(); 16 | /// let mut bytes = Vec::with_capacity(rb1.serialized_size()); 17 | /// rb1.serialize_into(&mut bytes).unwrap(); 18 | /// let rb2 = RoaringTreemap::deserialize_from(&bytes[..]).unwrap(); 19 | /// 20 | /// assert_eq!(rb1, rb2); 21 | /// ``` 22 | pub fn serialized_size(&self) -> usize { 23 | self.map 24 | .values() 25 | .fold(size_of::(), |acc, bitmap| acc + size_of::() + bitmap.serialized_size()) 26 | } 27 | 28 | /// Serialize this bitmap. 29 | /// This is compatible with the official C/C++, Java and Go implementations. 30 | /// 31 | /// # Examples 32 | /// 33 | /// ```rust 34 | /// use roaring::RoaringTreemap; 35 | /// 36 | /// let rb1: RoaringTreemap = (1..4).collect(); 37 | /// let mut bytes = vec![]; 38 | /// rb1.serialize_into(&mut bytes).unwrap(); 39 | /// let rb2 = RoaringTreemap::deserialize_from(&bytes[..]).unwrap(); 40 | /// 41 | /// assert_eq!(rb1, rb2); 42 | /// ``` 43 | pub fn serialize_into(&self, mut writer: W) -> io::Result<()> { 44 | writer.write_u64::(self.map.len() as u64)?; 45 | 46 | for (key, bitmap) in &self.map { 47 | writer.write_u32::(*key)?; 48 | bitmap.serialize_into(&mut writer)?; 49 | } 50 | 51 | Ok(()) 52 | } 53 | 54 | /// Deserialize a bitmap into memory. 55 | /// 56 | /// This is compatible with the official C/C++, Java and Go implementations. 57 | /// This method checks that all of the internal values are valid. 58 | /// 59 | /// # Examples 60 | /// 61 | /// ```rust 62 | /// use roaring::RoaringTreemap; 63 | /// 64 | /// let rb1: RoaringTreemap = (1..4).collect(); 65 | /// let mut bytes = vec![]; 66 | /// rb1.serialize_into(&mut bytes).unwrap(); 67 | /// let rb2 = RoaringTreemap::deserialize_from(&bytes[..]).unwrap(); 68 | /// 69 | /// assert_eq!(rb1, rb2); 70 | /// ``` 71 | pub fn deserialize_from(reader: R) -> io::Result { 72 | RoaringTreemap::deserialize_from_impl(reader, |reader| { 73 | RoaringBitmap::deserialize_from(reader) 74 | }) 75 | } 76 | 77 | /// Deserialize a bitmap into memory. 78 | /// 79 | /// This is compatible with the official C/C++, Java and Go implementations. 80 | /// This method is memory safe but will not check if the data is a valid bitmap. 81 | /// 82 | /// # Examples 83 | /// 84 | /// ```rust 85 | /// use roaring::RoaringTreemap; 86 | /// 87 | /// let rb1: RoaringTreemap = (1..4).collect(); 88 | /// let mut bytes = vec![]; 89 | /// rb1.serialize_into(&mut bytes).unwrap(); 90 | /// let rb2 = RoaringTreemap::deserialize_unchecked_from(&bytes[..]).unwrap(); 91 | /// 92 | /// assert_eq!(rb1, rb2); 93 | /// ``` 94 | pub fn deserialize_unchecked_from(reader: R) -> io::Result { 95 | RoaringTreemap::deserialize_from_impl(reader, |reader| { 96 | RoaringBitmap::deserialize_unchecked_from(reader) 97 | }) 98 | } 99 | 100 | fn deserialize_from_impl(mut reader: R, mut deserialize_bitmap: F) -> io::Result 101 | where 102 | R: io::Read, 103 | F: FnMut(&mut R) -> io::Result, 104 | { 105 | let size = reader.read_u64::()?; 106 | 107 | let mut s = Self::new(); 108 | 109 | for _ in 0..size { 110 | let key = reader.read_u32::()?; 111 | let bitmap = deserialize_bitmap(&mut reader)?; 112 | 113 | s.map.insert(key, bitmap); 114 | } 115 | 116 | Ok(s) 117 | } 118 | } 119 | 120 | #[cfg(test)] 121 | mod test { 122 | use crate::RoaringTreemap; 123 | use proptest::prelude::*; 124 | 125 | proptest! { 126 | #[test] 127 | fn test_serialization( 128 | treemap in RoaringTreemap::arbitrary(), 129 | ) { 130 | let mut buffer = Vec::new(); 131 | treemap.serialize_into(&mut buffer).unwrap(); 132 | prop_assert_eq!(treemap, RoaringTreemap::deserialize_from(buffer.as_slice()).unwrap()); 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /roaring/src/bitmap/cmp.rs: -------------------------------------------------------------------------------- 1 | use core::borrow::Borrow; 2 | use core::cmp::Ordering; 3 | use core::iter::Peekable; 4 | 5 | use super::container::Container; 6 | use crate::RoaringBitmap; 7 | 8 | impl RoaringBitmap { 9 | /// Returns true if the set has no elements in common with other. This is equivalent to 10 | /// checking for an empty intersection. 11 | /// 12 | /// # Examples 13 | /// 14 | /// ```rust 15 | /// use roaring::RoaringBitmap; 16 | /// 17 | /// let mut rb1 = RoaringBitmap::new(); 18 | /// let mut rb2 = RoaringBitmap::new(); 19 | /// 20 | /// rb1.insert(1); 21 | /// 22 | /// assert_eq!(rb1.is_disjoint(&rb2), true); 23 | /// 24 | /// rb2.insert(1); 25 | /// 26 | /// assert_eq!(rb1.is_disjoint(&rb2), false); 27 | /// 28 | /// ``` 29 | pub fn is_disjoint(&self, other: &Self) -> bool { 30 | Pairs::new(&self.containers, &other.containers) 31 | .filter_map(|(c1, c2)| c1.zip(c2)) 32 | .all(|(c1, c2)| c1.is_disjoint(c2)) 33 | } 34 | 35 | /// Returns `true` if this set is a subset of `other`. 36 | /// 37 | /// # Examples 38 | /// 39 | /// ```rust 40 | /// use roaring::RoaringBitmap; 41 | /// 42 | /// let mut rb1 = RoaringBitmap::new(); 43 | /// let mut rb2 = RoaringBitmap::new(); 44 | /// 45 | /// rb1.insert(1); 46 | /// 47 | /// assert_eq!(rb1.is_subset(&rb2), false); 48 | /// 49 | /// rb2.insert(1); 50 | /// 51 | /// assert_eq!(rb1.is_subset(&rb2), true); 52 | /// 53 | /// rb1.insert(2); 54 | /// 55 | /// assert_eq!(rb1.is_subset(&rb2), false); 56 | /// ``` 57 | pub fn is_subset(&self, other: &Self) -> bool { 58 | for pair in Pairs::new(&self.containers, &other.containers) { 59 | match pair { 60 | (None, _) => (), 61 | (_, None) => return false, 62 | (Some(c1), Some(c2)) => { 63 | if !c1.is_subset(c2) { 64 | return false; 65 | } 66 | } 67 | } 68 | } 69 | true 70 | } 71 | 72 | /// Returns `true` if this set is a superset of `other`. 73 | /// 74 | /// # Examples 75 | /// 76 | /// ```rust 77 | /// use roaring::RoaringBitmap; 78 | /// 79 | /// let mut rb1 = RoaringBitmap::new(); 80 | /// let mut rb2 = RoaringBitmap::new(); 81 | /// 82 | /// rb1.insert(1); 83 | /// 84 | /// assert_eq!(rb2.is_superset(&rb1), false); 85 | /// 86 | /// rb2.insert(1); 87 | /// 88 | /// assert_eq!(rb2.is_superset(&rb1), true); 89 | /// 90 | /// rb1.insert(2); 91 | /// 92 | /// assert_eq!(rb2.is_superset(&rb1), false); 93 | /// ``` 94 | pub fn is_superset(&self, other: &Self) -> bool { 95 | other.is_subset(self) 96 | } 97 | } 98 | 99 | /// An helping Iterator over pairs of containers. 100 | /// 101 | /// Returns the smallest container according to its key 102 | /// or both if the key is the same. It is useful when you need 103 | /// to iterate over two containers to do operations on them. 104 | pub(crate) struct Pairs 105 | where 106 | I: Iterator, 107 | J: Iterator, 108 | L: Borrow, 109 | R: Borrow, 110 | { 111 | left: Peekable, 112 | right: Peekable, 113 | } 114 | 115 | impl Pairs 116 | where 117 | I: Iterator, 118 | J: Iterator, 119 | L: Borrow, 120 | R: Borrow, 121 | { 122 | pub fn new(left: A, right: B) -> Pairs 123 | where 124 | A: IntoIterator, 125 | B: IntoIterator, 126 | { 127 | Pairs { left: left.into_iter().peekable(), right: right.into_iter().peekable() } 128 | } 129 | } 130 | 131 | impl Iterator for Pairs 132 | where 133 | I: Iterator, 134 | J: Iterator, 135 | L: Borrow, 136 | R: Borrow, 137 | { 138 | type Item = (Option, Option); 139 | 140 | fn next(&mut self) -> Option { 141 | match (self.left.peek(), self.right.peek()) { 142 | (None, None) => None, 143 | (Some(_), None) => Some((self.left.next(), None)), 144 | (None, Some(_)) => Some((None, self.right.next())), 145 | (Some(c1), Some(c2)) => match c1.borrow().key.cmp(&c2.borrow().key) { 146 | Ordering::Equal => Some((self.left.next(), self.right.next())), 147 | Ordering::Less => Some((self.left.next(), None)), 148 | Ordering::Greater => Some((None, self.right.next())), 149 | }, 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /roaring/src/bitmap/statistics.rs: -------------------------------------------------------------------------------- 1 | use core::mem; 2 | 3 | use crate::bitmap::container::Container; 4 | use crate::RoaringBitmap; 5 | 6 | use super::store::Store; 7 | 8 | /// Detailed statistics on the composition of a bitmap. 9 | #[derive(Clone, Copy, PartialEq, Debug)] 10 | #[non_exhaustive] 11 | pub struct Statistics { 12 | /// Number of containers in the bitmap 13 | pub n_containers: u32, 14 | /// Number of array containers in the bitmap 15 | pub n_array_containers: u32, 16 | /// Number of run containers in the bitmap 17 | pub n_run_containers: u32, 18 | /// Number of bitset containers in the bitmap 19 | pub n_bitset_containers: u32, 20 | /// Number of values stored in array containers 21 | pub n_values_array_containers: u32, 22 | /// Number of values stored in run containers 23 | pub n_values_run_containers: u32, 24 | /// Number of values stored in bitset containers 25 | pub n_values_bitset_containers: u64, 26 | /// Number of bytes used by array containers 27 | pub n_bytes_array_containers: u64, 28 | /// Number of bytes used by run containers 29 | pub n_bytes_run_containers: u64, 30 | /// Number of bytes used by bitset containers 31 | pub n_bytes_bitset_containers: u64, 32 | /// Maximum value stored in the bitmap 33 | pub max_value: Option, 34 | /// Minimum value stored in the bitmap 35 | pub min_value: Option, 36 | /// Number of values stored in the bitmap 37 | pub cardinality: u64, 38 | } 39 | 40 | impl RoaringBitmap { 41 | /// Returns statistics about the composition of a roaring bitmap. 42 | /// 43 | /// ``` 44 | /// use roaring::RoaringBitmap; 45 | /// 46 | /// let mut bitmap: RoaringBitmap = (1..100).collect(); 47 | /// let statistics = bitmap.statistics(); 48 | /// 49 | /// assert_eq!(statistics.n_containers, 1); 50 | /// assert_eq!(statistics.n_array_containers, 1); 51 | /// assert_eq!(statistics.n_run_containers, 0); 52 | /// assert_eq!(statistics.n_bitset_containers, 0); 53 | /// assert_eq!(statistics.n_values_array_containers, 99); 54 | /// assert_eq!(statistics.n_values_run_containers, 0); 55 | /// assert_eq!(statistics.n_values_bitset_containers, 0); 56 | /// assert_eq!(statistics.n_bytes_array_containers, 512); 57 | /// assert_eq!(statistics.n_bytes_run_containers, 0); 58 | /// assert_eq!(statistics.n_bytes_bitset_containers, 0); 59 | /// assert_eq!(statistics.max_value, Some(99)); 60 | /// assert_eq!(statistics.min_value, Some(1)); 61 | /// assert_eq!(statistics.cardinality, 99); 62 | /// ``` 63 | pub fn statistics(&self) -> Statistics { 64 | let mut n_containers = 0; 65 | let mut n_array_containers = 0; 66 | let mut n_bitset_containers = 0; 67 | let mut n_run_containers = 0; 68 | let mut n_values_array_containers = 0; 69 | let mut n_values_bitset_containers = 0; 70 | let mut n_values_run_containers = 0; 71 | let mut n_bytes_array_containers = 0; 72 | let mut n_bytes_bitset_containers = 0; 73 | let mut n_bytes_run_containers = 0; 74 | let mut cardinality = 0; 75 | 76 | for Container { key: _, store } in &self.containers { 77 | match store { 78 | Store::Array(array) => { 79 | cardinality += array.len(); 80 | n_values_array_containers += array.len() as u32; 81 | n_bytes_array_containers += (array.capacity() * mem::size_of::()) as u64; 82 | n_array_containers += 1; 83 | } 84 | Store::Bitmap(bitmap) => { 85 | cardinality += bitmap.len(); 86 | n_values_bitset_containers += bitmap.len(); 87 | n_bytes_bitset_containers += bitmap.capacity() as u64; 88 | n_bitset_containers += 1; 89 | } 90 | Store::Run(runs) => { 91 | cardinality += runs.len(); 92 | n_values_run_containers += runs.len() as u32; 93 | n_bytes_run_containers += runs.byte_size() as u64; 94 | n_run_containers += 1; 95 | } 96 | } 97 | n_containers += 1; 98 | } 99 | 100 | Statistics { 101 | n_containers, 102 | n_array_containers, 103 | n_run_containers, 104 | n_bitset_containers, 105 | n_values_array_containers, 106 | n_values_run_containers, 107 | n_values_bitset_containers, 108 | n_bytes_array_containers, 109 | n_bytes_run_containers, 110 | n_bytes_bitset_containers, 111 | max_value: self.max(), 112 | min_value: self.min(), 113 | cardinality, 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /roaring/tests/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringBitmap; 3 | 4 | #[test] 5 | fn smoke() { 6 | let mut bitmap = RoaringBitmap::new(); 7 | assert_eq!(bitmap.len(), 0); 8 | assert!(bitmap.is_empty()); 9 | bitmap.remove(0); 10 | assert_eq!(bitmap.len(), 0); 11 | assert!(bitmap.is_empty()); 12 | bitmap.insert(1); 13 | assert!(bitmap.contains(1)); 14 | assert_eq!(bitmap.len(), 1); 15 | assert!(!bitmap.is_empty()); 16 | bitmap.insert(u32::MAX - 2); 17 | assert!(bitmap.contains(u32::MAX - 2)); 18 | assert_eq!(bitmap.len(), 2); 19 | bitmap.insert(u32::MAX); 20 | assert!(bitmap.contains(u32::MAX)); 21 | assert_eq!(bitmap.len(), 3); 22 | bitmap.insert(2); 23 | assert!(bitmap.contains(2)); 24 | assert_eq!(bitmap.len(), 4); 25 | bitmap.remove(2); 26 | assert!(!bitmap.contains(2)); 27 | assert_eq!(bitmap.len(), 3); 28 | assert!(!bitmap.contains(0)); 29 | assert!(bitmap.contains(1)); 30 | assert!(!bitmap.contains(100)); 31 | assert!(bitmap.contains(u32::MAX - 2)); 32 | assert!(!bitmap.contains(u32::MAX - 1)); 33 | assert!(bitmap.contains(u32::MAX)); 34 | } 35 | 36 | #[test] 37 | fn remove_range() { 38 | let ranges = [0u32, 1, 63, 64, 65, 100, 4096 - 1, 4096, 4096 + 1, 65536 - 1, 65536, 65536 + 1]; 39 | for (i, &a) in ranges.iter().enumerate() { 40 | for &b in &ranges[i..] { 41 | let mut bitmap = (0..=65536).collect::(); 42 | assert_eq!(bitmap.remove_range(a..b), u64::from(b - a)); 43 | assert_eq!(bitmap, (0..a).chain(b..=65536).collect::()); 44 | } 45 | } 46 | } 47 | 48 | #[test] 49 | #[allow(clippy::range_plus_one)] // remove_range needs an exclusive range 50 | fn remove_range_array() { 51 | let mut bitmap = (0..1000).collect::(); 52 | for i in 0..1000 { 53 | assert_eq!(bitmap.remove_range(i..i), 0); 54 | assert_eq!(bitmap.remove_range(i..i + 1), 1); 55 | } 56 | 57 | // insert 0, 2, 4, .. 58 | // remove [0, 2), [2, 4), .. 59 | let mut bitmap = (0..1000).map(|x| x * 2).collect::(); 60 | for i in 0..1000 { 61 | assert_eq!(bitmap.remove_range(i * 2..(i + 1) * 2), 1); 62 | } 63 | 64 | // remove [0, 2), [2, 4), .. 65 | let mut bitmap = (0..1000).collect::(); 66 | for i in 0..1000 / 2 { 67 | assert_eq!(bitmap.remove_range(i * 2..(i + 1) * 2), 2); 68 | } 69 | } 70 | 71 | #[test] 72 | #[allow(clippy::range_plus_one)] // remove_range needs an exclusive range 73 | fn remove_range_bitmap() { 74 | let mut bitmap = (0..4096 + 1000).collect::(); 75 | for i in 0..1000 { 76 | assert_eq!(bitmap.remove_range(i..i), 0); 77 | assert_eq!(bitmap.remove_range(i..i + 1), 1); 78 | } 79 | 80 | // insert 0, 2, 4, .. 81 | // remove [0, 2), [2, 4), .. 82 | let mut bitmap = ((0..4096 + 1000).map(|x| x * 2)).collect::(); 83 | for i in 0..1000 { 84 | assert_eq!(bitmap.remove_range(i * 2..(i + 1) * 2), 1); 85 | } 86 | 87 | // remove [0, 2), [2, 4), .. 88 | let mut bitmap = (0..4096 + 1000).collect::(); 89 | for i in 0..1000 / 2 { 90 | assert_eq!(bitmap.remove_range(i * 2..(i + 1) * 2), 2); 91 | } 92 | 93 | // remove [1, 3), [3, 5), .. 94 | let mut bitmap = (0..4096 + 1000).collect::(); 95 | for i in 0..1000 / 2 { 96 | assert_eq!(bitmap.remove_range(i * 2 + 1..(i + 1) * 2 + 1), 2); 97 | } 98 | } 99 | 100 | #[test] 101 | fn to_bitmap() { 102 | let bitmap = (0..5000).collect::(); 103 | assert_eq!(bitmap.len(), 5000); 104 | for i in 1..5000 { 105 | assert!(bitmap.contains(i)); 106 | } 107 | assert!(!bitmap.contains(5001)); 108 | } 109 | 110 | #[test] 111 | fn to_array() { 112 | let mut bitmap = (0..5000).collect::(); 113 | for i in 3000..5000 { 114 | bitmap.remove(i); 115 | } 116 | assert_eq!(bitmap.len(), 3000); 117 | for i in 0..3000 { 118 | assert!(bitmap.contains(i)); 119 | } 120 | for i in 3000..5000 { 121 | assert!(!bitmap.contains(i)); 122 | } 123 | } 124 | 125 | #[test] 126 | fn optimize_array() { 127 | let mut bitmap = RoaringBitmap::from_iter(0..1000); 128 | assert!(bitmap.optimize()); 129 | let mut bitmap = RoaringBitmap::from_iter(0..5000); 130 | assert!(bitmap.optimize()); 131 | } 132 | 133 | #[test] 134 | fn optimize_bitmap() { 135 | let mut bitmap = RoaringBitmap::from_iter(0..5000); 136 | assert!(bitmap.optimize()); 137 | } 138 | 139 | #[test] 140 | fn remove_run_compression() { 141 | let mut bitmap = RoaringBitmap::from_iter(0..5000); 142 | assert!(bitmap.optimize()); 143 | assert!(bitmap.remove_run_compression()); 144 | } 145 | 146 | #[test] 147 | fn optimize_run() { 148 | let mut bitmap = RoaringBitmap::from_iter(0..1000); 149 | assert!(bitmap.optimize()); 150 | // Calling optimize a second time should return false as no changes will be made 151 | assert!(!bitmap.optimize()); 152 | } 153 | -------------------------------------------------------------------------------- /roaring/src/bitmap/util.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Bound, RangeBounds, RangeInclusive}; 2 | 3 | /// Returns the container key and the index 4 | /// in this container for a given integer. 5 | #[inline] 6 | pub fn split(value: u32) -> (u16, u16) { 7 | ((value >> 16) as u16, value as u16) 8 | } 9 | 10 | /// Returns the original integer from the container 11 | /// key and the index of it in the container. 12 | #[inline] 13 | pub fn join(high: u16, low: u16) -> u32 { 14 | (u32::from(high) << 16) + u32::from(low) 15 | } 16 | 17 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 18 | pub enum ConvertRangeError { 19 | Empty, 20 | StartGreaterThanEnd, 21 | StartAndEndEqualExcluded, 22 | } 23 | 24 | /// Convert a `RangeBounds` object to `RangeInclusive`, 25 | pub fn convert_range_to_inclusive(range: R) -> Result, ConvertRangeError> 26 | where 27 | R: RangeBounds, 28 | { 29 | let start_bound = range.start_bound().cloned(); 30 | let end_bound = range.end_bound().cloned(); 31 | match (start_bound, end_bound) { 32 | (Bound::Excluded(s), Bound::Excluded(e)) if s == e => { 33 | Err(ConvertRangeError::StartAndEndEqualExcluded) 34 | } 35 | (Bound::Included(s) | Bound::Excluded(s), Bound::Included(e) | Bound::Excluded(e)) 36 | if s > e => 37 | { 38 | Err(ConvertRangeError::StartGreaterThanEnd) 39 | } 40 | _ => { 41 | let start = match start_bound { 42 | Bound::Included(s) => s, 43 | Bound::Excluded(s) => s.checked_add(1).ok_or(ConvertRangeError::Empty)?, 44 | Bound::Unbounded => 0, 45 | }; 46 | 47 | let end = match end_bound { 48 | Bound::Included(e) => e, 49 | Bound::Excluded(e) => e.checked_sub(1).ok_or(ConvertRangeError::Empty)?, 50 | Bound::Unbounded => u32::MAX, 51 | }; 52 | 53 | if start > end { 54 | // This handles e.g. `x..x`: we've ruled out `start > end` overall, so a value must 55 | // have been changed via exclusion. 56 | Err(ConvertRangeError::Empty) 57 | } else { 58 | Ok(start..=end) 59 | } 60 | } 61 | } 62 | } 63 | 64 | #[cfg(test)] 65 | mod test { 66 | use super::{convert_range_to_inclusive, join, split, ConvertRangeError}; 67 | use core::ops::Bound; 68 | 69 | #[test] 70 | fn test_split_u32() { 71 | assert_eq!((0x0000u16, 0x0000u16), split(0x0000_0000u32)); 72 | assert_eq!((0x0000u16, 0x0001u16), split(0x0000_0001u32)); 73 | assert_eq!((0x0000u16, 0xFFFEu16), split(0x0000_FFFEu32)); 74 | assert_eq!((0x0000u16, 0xFFFFu16), split(0x0000_FFFFu32)); 75 | assert_eq!((0x0001u16, 0x0000u16), split(0x0001_0000u32)); 76 | assert_eq!((0x0001u16, 0x0001u16), split(0x0001_0001u32)); 77 | assert_eq!((0xFFFFu16, 0xFFFEu16), split(0xFFFF_FFFEu32)); 78 | assert_eq!((0xFFFFu16, 0xFFFFu16), split(0xFFFF_FFFFu32)); 79 | } 80 | 81 | #[test] 82 | fn test_join_u32() { 83 | assert_eq!(0x0000_0000u32, join(0x0000u16, 0x0000u16)); 84 | assert_eq!(0x0000_0001u32, join(0x0000u16, 0x0001u16)); 85 | assert_eq!(0x0000_FFFEu32, join(0x0000u16, 0xFFFEu16)); 86 | assert_eq!(0x0000_FFFFu32, join(0x0000u16, 0xFFFFu16)); 87 | assert_eq!(0x0001_0000u32, join(0x0001u16, 0x0000u16)); 88 | assert_eq!(0x0001_0001u32, join(0x0001u16, 0x0001u16)); 89 | assert_eq!(0xFFFF_FFFEu32, join(0xFFFFu16, 0xFFFEu16)); 90 | assert_eq!(0xFFFF_FFFFu32, join(0xFFFFu16, 0xFFFFu16)); 91 | } 92 | 93 | #[test] 94 | #[allow(clippy::reversed_empty_ranges)] 95 | fn test_convert_range_to_inclusive() { 96 | assert_eq!(Ok(1..=5), convert_range_to_inclusive(1..6)); 97 | assert_eq!(Ok(1..=u32::MAX), convert_range_to_inclusive(1..)); 98 | assert_eq!(Ok(0..=u32::MAX), convert_range_to_inclusive(..)); 99 | assert_eq!(Ok(16..=16), convert_range_to_inclusive(16..=16)); 100 | assert_eq!( 101 | Ok(11..=19), 102 | convert_range_to_inclusive((Bound::Excluded(10), Bound::Excluded(20))) 103 | ); 104 | 105 | assert_eq!(Err(ConvertRangeError::Empty), convert_range_to_inclusive(0..0)); 106 | assert_eq!(Err(ConvertRangeError::Empty), convert_range_to_inclusive(5..5)); 107 | assert_eq!(Err(ConvertRangeError::StartGreaterThanEnd), convert_range_to_inclusive(1..0)); 108 | assert_eq!(Err(ConvertRangeError::StartGreaterThanEnd), convert_range_to_inclusive(10..5)); 109 | assert_eq!( 110 | Err(ConvertRangeError::Empty), 111 | convert_range_to_inclusive((Bound::Excluded(u32::MAX), Bound::Included(u32::MAX))) 112 | ); 113 | assert_eq!( 114 | Err(ConvertRangeError::StartAndEndEqualExcluded), 115 | convert_range_to_inclusive((Bound::Excluded(u32::MAX), Bound::Excluded(u32::MAX))) 116 | ); 117 | assert_eq!( 118 | Err(ConvertRangeError::Empty), 119 | convert_range_to_inclusive((Bound::Excluded(0), Bound::Included(0))) 120 | ); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - staging 5 | - trying 6 | pull_request: 7 | branches: 8 | - main 9 | merge_group: 10 | 11 | name: Continuous integration 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | rust: 19 | - stable 20 | - beta 21 | - nightly 22 | # When changing this value don't forget to change the `package.rust-version` field in 23 | # `roaring/Cargo.toml`!!! 24 | - 1.82.0 25 | env: 26 | RUSTFLAGS: "-C target-cpu=native -C opt-level=3" 27 | 28 | steps: 29 | - name: Checkout roaring-rs 30 | uses: actions/checkout@v4 31 | 32 | - name: Initialize rust toolchain 33 | uses: dtolnay/rust-toolchain@master 34 | with: 35 | toolchain: ${{ matrix.rust }} 36 | components: rustfmt, clippy 37 | 38 | - name: Caching 39 | uses: Swatinem/rust-cache@v2 40 | 41 | - name: Check 42 | # clippy will also do a build check 43 | # so we don't need to run `cargo check` or `cargo build` 44 | # use different features to check if everything is fine 45 | # the incremental compilation will make this faster 46 | # We disallow todo!s in the code too. 47 | run: | 48 | cargo clippy -p roaring --all-targets --no-default-features -- -D warnings 49 | cargo clippy -p roaring --all-targets --features serde -- -Dclippy::todo -D warnings 50 | 51 | - name: Check SIMD 52 | if: matrix.rust == 'nightly' 53 | run: cargo clippy -p roaring --all-targets --all-features -- -D warnings 54 | 55 | - name: Check formatting 56 | run: cargo fmt --all -- --check 57 | 58 | test: 59 | runs-on: ubuntu-latest 60 | needs: build 61 | strategy: 62 | matrix: 63 | rust: 64 | - stable 65 | - beta 66 | - nightly 67 | - 1.82.0 68 | features: 69 | - default 70 | - no-std 71 | include: 72 | - rust: nightly 73 | features: simd 74 | env: 75 | RUSTFLAGS: "-C target-cpu=native -C opt-level=3" 76 | ROARINGRS_BENCH_OFFLINE: "true" 77 | 78 | steps: 79 | - name: Checkout roaring-rs 80 | uses: actions/checkout@v4 81 | 82 | - name: Checkout benchmark datasets 83 | uses: actions/checkout@v4 84 | with: 85 | repository: "RoaringBitmap/real-roaring-datasets" 86 | path: "benchmarks/real-roaring-datasets" 87 | 88 | - name: Caching 89 | uses: Swatinem/rust-cache@v2 90 | 91 | - name: Initialize rust toolchain 92 | uses: dtolnay/rust-toolchain@master 93 | with: 94 | toolchain: ${{ matrix.rust }} 95 | 96 | - name: Test 97 | if: matrix.features == 'default' 98 | run: cargo test -p roaring --features serde 99 | 100 | - name: Test Benches 101 | if: matrix.rust != '1.82.0' && matrix.features == 'default' 102 | run: cargo test -p benchmarks --benches 103 | 104 | - name: Test no default features 105 | if: matrix.features == 'no-std' 106 | run: cargo test -p roaring --no-default-features 107 | 108 | - name: SIMD test 109 | if: matrix.rust == 'nightly' && matrix.features == 'simd' 110 | run: cargo +nightly test -p roaring --features simd 111 | 112 | miri: 113 | runs-on: ubuntu-latest 114 | needs: build 115 | env: 116 | # warning: Miri does not support optimizations: the opt-level is ignored. 117 | RUSTFLAGS: "-C target-cpu=native" 118 | 119 | steps: 120 | - name: Checkout roaring-rs 121 | uses: actions/checkout@v4 122 | 123 | - name: Initialize rust toolchain 124 | uses: dtolnay/rust-toolchain@master 125 | with: 126 | toolchain: nightly 127 | components: miri 128 | 129 | - name: Caching 130 | uses: Swatinem/rust-cache@v2 131 | 132 | - name: Setup miri 133 | run: cargo miri setup 134 | 135 | - name: Test bit endian 136 | run: cargo miri test --target s390x-unknown-linux-gnu -p roaring --lib -- bitmap::serialization::test::test_from_lsb0_bytes 137 | 138 | fuzz: 139 | runs-on: ubuntu-latest 140 | needs: build 141 | env: 142 | RUSTFLAGS: "-C target-cpu=native -C opt-level=3" 143 | 144 | steps: 145 | - name: Checkout roaring-rs 146 | uses: actions/checkout@v4 147 | 148 | - name: Initialize rust toolchain 149 | uses: dtolnay/rust-toolchain@master 150 | with: 151 | toolchain: nightly 152 | 153 | - name: Caching 154 | uses: Swatinem/rust-cache@v2 155 | 156 | - name: Install cargo fuzz 157 | run: cargo install cargo-fuzz 158 | 159 | - name: Setup Cache for corpus and artifacts 160 | uses: actions/cache@v4 161 | with: 162 | key: always 163 | path: | 164 | fuzz/artifacts 165 | fuzz/corpus 166 | 167 | - name: Run Fuzzer vs croaring for 15 minutes 168 | run: cargo fuzz run against_croaring -s none -- -timeout=5 -max_total_time=900 169 | 170 | - name: Run Fuzzer (with simd) vs croaring for 15 minutes 171 | run: cargo fuzz run --features=simd against_croaring -- -timeout=5 -max_total_time=900 172 | -------------------------------------------------------------------------------- /fuzz/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "arbitrary" 7 | version = "1.4.1" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" 10 | dependencies = [ 11 | "derive_arbitrary", 12 | ] 13 | 14 | [[package]] 15 | name = "bitflags" 16 | version = "2.9.1" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" 19 | 20 | [[package]] 21 | name = "bytemuck" 22 | version = "1.23.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" 25 | 26 | [[package]] 27 | name = "byteorder" 28 | version = "1.5.0" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 31 | 32 | [[package]] 33 | name = "cc" 34 | version = "1.2.25" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" 37 | dependencies = [ 38 | "jobserver", 39 | "libc", 40 | "shlex", 41 | ] 42 | 43 | [[package]] 44 | name = "cfg-if" 45 | version = "1.0.0" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 48 | 49 | [[package]] 50 | name = "croaring" 51 | version = "2.3.1" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "0a7378e8f3ede464bd5d6dbdb1b6f2ed907c0dd27dcbe465a7991c4bb78b5ddd" 54 | dependencies = [ 55 | "croaring-sys", 56 | ] 57 | 58 | [[package]] 59 | name = "croaring-sys" 60 | version = "4.3.3" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "5008a00afde0b8493eae0f33975f1d0af95f2e654a7c9938c27e654c09119dcd" 63 | dependencies = [ 64 | "cc", 65 | ] 66 | 67 | [[package]] 68 | name = "derive_arbitrary" 69 | version = "1.4.1" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" 72 | dependencies = [ 73 | "proc-macro2", 74 | "quote", 75 | "syn", 76 | ] 77 | 78 | [[package]] 79 | name = "getrandom" 80 | version = "0.3.3" 81 | source = "registry+https://github.com/rust-lang/crates.io-index" 82 | checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" 83 | dependencies = [ 84 | "cfg-if", 85 | "libc", 86 | "r-efi", 87 | "wasi", 88 | ] 89 | 90 | [[package]] 91 | name = "jobserver" 92 | version = "0.1.33" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" 95 | dependencies = [ 96 | "getrandom", 97 | "libc", 98 | ] 99 | 100 | [[package]] 101 | name = "libc" 102 | version = "0.2.172" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" 105 | 106 | [[package]] 107 | name = "libfuzzer-sys" 108 | version = "0.4.9" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" 111 | dependencies = [ 112 | "arbitrary", 113 | "cc", 114 | ] 115 | 116 | [[package]] 117 | name = "proc-macro2" 118 | version = "1.0.95" 119 | source = "registry+https://github.com/rust-lang/crates.io-index" 120 | checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" 121 | dependencies = [ 122 | "unicode-ident", 123 | ] 124 | 125 | [[package]] 126 | name = "quote" 127 | version = "1.0.40" 128 | source = "registry+https://github.com/rust-lang/crates.io-index" 129 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" 130 | dependencies = [ 131 | "proc-macro2", 132 | ] 133 | 134 | [[package]] 135 | name = "r-efi" 136 | version = "5.2.0" 137 | source = "registry+https://github.com/rust-lang/crates.io-index" 138 | checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" 139 | 140 | [[package]] 141 | name = "roaring" 142 | version = "0.11.2" 143 | dependencies = [ 144 | "bytemuck", 145 | "byteorder", 146 | ] 147 | 148 | [[package]] 149 | name = "roaring-fuzz" 150 | version = "0.0.0" 151 | dependencies = [ 152 | "croaring", 153 | "libfuzzer-sys", 154 | "roaring", 155 | ] 156 | 157 | [[package]] 158 | name = "shlex" 159 | version = "1.3.0" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 162 | 163 | [[package]] 164 | name = "syn" 165 | version = "2.0.101" 166 | source = "registry+https://github.com/rust-lang/crates.io-index" 167 | checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" 168 | dependencies = [ 169 | "proc-macro2", 170 | "quote", 171 | "unicode-ident", 172 | ] 173 | 174 | [[package]] 175 | name = "unicode-ident" 176 | version = "1.0.18" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 179 | 180 | [[package]] 181 | name = "wasi" 182 | version = "0.14.2+wasi-0.2.4" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" 185 | dependencies = [ 186 | "wit-bindgen-rt", 187 | ] 188 | 189 | [[package]] 190 | name = "wit-bindgen-rt" 191 | version = "0.39.0" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" 194 | dependencies = [ 195 | "bitflags", 196 | ] 197 | -------------------------------------------------------------------------------- /roaring/src/bitmap/arbitrary.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod test { 3 | use crate::bitmap::container::{Container, RUN_MAX_SIZE}; 4 | use crate::bitmap::store::{ArrayStore, BitmapStore, IntervalStore, Store}; 5 | use crate::RoaringBitmap; 6 | use core::fmt::{Debug, Formatter}; 7 | use proptest::bits::{BitSetLike, SampledBitSetStrategy}; 8 | use proptest::collection::{vec, SizeRange}; 9 | use proptest::prelude::*; 10 | 11 | #[cfg(not(feature = "std"))] 12 | use alloc::vec::Vec; 13 | 14 | impl Debug for BitmapStore { 15 | fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { 16 | if self.len() < 16 { 17 | write!(f, "BitmapStore<{:?}>", self.iter().collect::>()) 18 | } else { 19 | write!( 20 | f, 21 | "BitmapStore<{:?} values between {:?} and {:?}>", 22 | self.len(), 23 | self.min().unwrap(), 24 | self.max().unwrap() 25 | ) 26 | } 27 | } 28 | } 29 | 30 | impl BitSetLike for BitmapStore { 31 | fn new_bitset(max: usize) -> Self { 32 | assert!(max <= BitmapStore::MAX + 1); 33 | BitmapStore::new() 34 | } 35 | 36 | fn len(&self) -> usize { 37 | BitmapStore::MAX + 1 38 | } 39 | 40 | fn test(&self, bit: usize) -> bool { 41 | assert!(bit <= BitmapStore::MAX); 42 | self.contains(bit as u16) 43 | } 44 | 45 | fn set(&mut self, bit: usize) { 46 | assert!(bit <= BitmapStore::MAX); 47 | self.insert(bit as u16); 48 | } 49 | 50 | fn clear(&mut self, bit: usize) { 51 | assert!(bit <= BitmapStore::MAX); 52 | self.remove(bit as u16); 53 | } 54 | 55 | fn count(&self) -> usize { 56 | self.len() as usize 57 | } 58 | } 59 | 60 | impl BitmapStore { 61 | const MAX: usize = u16::MAX as usize; 62 | 63 | pub fn sampled( 64 | size: impl Into, 65 | bits: impl Into, 66 | ) -> SampledBitSetStrategy { 67 | SampledBitSetStrategy::new(size.into(), bits.into()) 68 | } 69 | } 70 | 71 | impl BitSetLike for IntervalStore { 72 | fn new_bitset(max: usize) -> Self { 73 | assert!(max <= IntervalStore::MAX + 1); 74 | IntervalStore::new() 75 | } 76 | 77 | fn len(&self) -> usize { 78 | IntervalStore::MAX + 1 79 | } 80 | 81 | fn test(&self, bit: usize) -> bool { 82 | assert!(bit <= IntervalStore::MAX); 83 | self.contains(bit as u16) 84 | } 85 | 86 | fn set(&mut self, bit: usize) { 87 | assert!(bit <= IntervalStore::MAX); 88 | self.insert(bit as u16); 89 | } 90 | 91 | fn clear(&mut self, bit: usize) { 92 | assert!(bit <= IntervalStore::MAX); 93 | self.remove(bit as u16); 94 | } 95 | 96 | fn count(&self) -> usize { 97 | self.len() as usize 98 | } 99 | } 100 | 101 | impl IntervalStore { 102 | const MAX: usize = u16::MAX as usize; 103 | 104 | pub fn sampled( 105 | size: impl Into, 106 | bits: impl Into, 107 | ) -> SampledBitSetStrategy { 108 | SampledBitSetStrategy::new(size.into(), bits.into()) 109 | } 110 | } 111 | 112 | impl Debug for ArrayStore { 113 | fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { 114 | if self.len() < 16 { 115 | write!(f, "ArrayStore<{:?}>", self.as_slice()) 116 | } else { 117 | write!( 118 | f, 119 | "ArrayStore<{:?} values between {:?} and {:?}>", 120 | self.len(), 121 | self.min().unwrap(), 122 | self.max().unwrap() 123 | ) 124 | } 125 | } 126 | } 127 | 128 | impl BitSetLike for ArrayStore { 129 | fn new_bitset(max: usize) -> Self { 130 | assert!(max <= ArrayStore::MAX + 1); 131 | ArrayStore::new() 132 | } 133 | 134 | fn len(&self) -> usize { 135 | ArrayStore::MAX + 1 136 | } 137 | 138 | fn test(&self, bit: usize) -> bool { 139 | assert!(bit <= ArrayStore::MAX); 140 | self.contains(bit as u16) 141 | } 142 | 143 | fn set(&mut self, bit: usize) { 144 | assert!(bit <= ArrayStore::MAX); 145 | self.insert(bit as u16); 146 | } 147 | 148 | fn clear(&mut self, bit: usize) { 149 | assert!(bit <= ArrayStore::MAX); 150 | self.remove(bit as u16); 151 | } 152 | 153 | fn count(&self) -> usize { 154 | self.len() as usize 155 | } 156 | } 157 | 158 | impl ArrayStore { 159 | const MAX: usize = u16::MAX as usize; 160 | 161 | pub fn sampled( 162 | size: impl Into, 163 | bits: impl Into, 164 | ) -> SampledBitSetStrategy { 165 | SampledBitSetStrategy::new(size.into(), bits.into()) 166 | } 167 | } 168 | 169 | impl Debug for Store { 170 | fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { 171 | match self { 172 | Store::Array(a) => write!(f, "Store({a:?})"), 173 | Store::Bitmap(b) => write!(f, "Store({b:?})"), 174 | Store::Run(c) => write!(f, "Store({c:?})"), 175 | } 176 | } 177 | } 178 | 179 | impl Store { 180 | fn arbitrary() -> impl Strategy { 181 | prop_oneof![ 182 | ArrayStore::sampled(1..=4096, ..=u16::MAX as usize).prop_map(Store::Array), 183 | BitmapStore::sampled(4097..u16::MAX as usize, ..=u16::MAX as usize) 184 | .prop_map(Store::Bitmap), 185 | IntervalStore::sampled(1..=RUN_MAX_SIZE as usize, ..=u16::MAX as usize) 186 | .prop_map(Store::Run), 187 | ] 188 | } 189 | } 190 | 191 | prop_compose! { 192 | fn containers(n: usize) 193 | (keys in ArrayStore::sampled(..=n, ..=n), 194 | stores in vec(Store::arbitrary(), n)) -> RoaringBitmap { 195 | let containers = keys.into_iter().zip(stores).map(|(key, store)| { 196 | let mut container = Container { key, store }; 197 | container.ensure_correct_store(); 198 | container 199 | }).collect::>(); 200 | RoaringBitmap { containers } 201 | } 202 | } 203 | 204 | impl RoaringBitmap { 205 | prop_compose! { 206 | pub(crate) fn arbitrary()(bitmap in (0usize..=16).prop_flat_map(containers)) -> RoaringBitmap { 207 | bitmap 208 | } 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /benchmarks/benches/datasets/mod.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::fs::File; 3 | use std::io::BufReader; 4 | use std::path::{Path, PathBuf}; 5 | 6 | use git2::FetchOptions; 7 | use once_cell::sync::OnceCell as SyncOnceCell; 8 | 9 | use roaring::RoaringBitmap; 10 | 11 | static INSTANCE: SyncOnceCell> = SyncOnceCell::new(); 12 | 13 | pub struct Datasets; 14 | 15 | pub struct DatasetsIter { 16 | iter: std::slice::Iter<'static, Dataset>, 17 | } 18 | 19 | impl Iterator for DatasetsIter { 20 | type Item = &'static Dataset; 21 | 22 | fn next(&mut self) -> Option { 23 | self.iter.next() 24 | } 25 | } 26 | 27 | impl IntoIterator for Datasets { 28 | type Item = &'static Dataset; 29 | type IntoIter = DatasetsIter; 30 | 31 | fn into_iter(self) -> Self::IntoIter { 32 | DatasetsIter { 33 | iter: INSTANCE 34 | .get_or_init(|| { 35 | init_datasets().and_then(parse_datasets).expect("a collection of datasets") 36 | }) 37 | .iter(), 38 | } 39 | } 40 | } 41 | 42 | pub struct Dataset { 43 | pub name: String, 44 | pub bitmaps: Vec, 45 | } 46 | 47 | fn init_datasets() -> Result> { 48 | let out_dir = env::var_os("CARGO_MANIFEST_DIR").ok_or(env::VarError::NotPresent)?; 49 | 50 | let out_path = Path::new(&out_dir); 51 | let repo_path = out_path.join("real-roaring-datasets"); 52 | 53 | // Check if in offline mode 54 | 55 | let offline = env::var("ROARINGRS_BENCH_OFFLINE"); 56 | match offline { 57 | Ok(value) => { 58 | if value.parse::()? { 59 | return Ok(repo_path); 60 | } 61 | } 62 | Err(ref err) => match err { 63 | env::VarError::NotPresent => (), 64 | _ => { 65 | offline?; 66 | } 67 | }, 68 | }; 69 | 70 | // Setup progress callbacks 71 | 72 | let pb_cell = once_cell::unsync::OnceCell::new(); 73 | let mut cb = git2::RemoteCallbacks::new(); 74 | 75 | cb.transfer_progress(|progress| { 76 | let pb = pb_cell.get_or_init(|| { 77 | indicatif::ProgressBar::new(progress.total_objects() as u64) 78 | .with_style( 79 | indicatif::ProgressStyle::default_bar() 80 | .template(&format!( 81 | "{{prefix}}{{msg:.cyan/blue}} [{{bar}}] {{pos}}/{}", 82 | progress.total_objects() 83 | )) 84 | .expect("template string invalid") 85 | .progress_chars("#> "), 86 | ) 87 | .with_prefix(" ") 88 | .with_message("Receiving objects") 89 | }); 90 | 91 | pb.set_position((progress.local_objects() + progress.received_objects()) as u64); 92 | true 93 | }); 94 | 95 | let mut fetch_opts = FetchOptions::new(); 96 | fetch_opts.remote_callbacks(cb); 97 | 98 | // Do update 99 | 100 | if !Path::new(&repo_path).exists() { 101 | git2::build::RepoBuilder::new() 102 | .fetch_options(fetch_opts) 103 | .clone("https://github.com/RoaringBitmap/real-roaring-datasets.git", &repo_path)?; 104 | } else { 105 | let repo = git2::Repository::open(&repo_path)?; 106 | repo.find_remote("origin")?.fetch(&["master"], Some(&mut fetch_opts), None)?; 107 | 108 | let head = repo.head()?.peel_to_commit()?; 109 | let origin_master_head = repo 110 | .find_branch("origin/master", git2::BranchType::Remote)? 111 | .into_reference() 112 | .peel_to_commit()?; 113 | 114 | if head.id() != origin_master_head.id() { 115 | repo.reset(origin_master_head.as_object(), git2::ResetType::Hard, None)?; 116 | } 117 | } 118 | 119 | if let Some(pb) = pb_cell.get() { 120 | pb.finish() 121 | } 122 | 123 | Ok(repo_path) 124 | } 125 | 126 | fn parse_datasets>(path: P) -> Result, Box> { 127 | const DATASET_FILENAME_WHITELIST: &[&str] = &[ 128 | "census-income.zip", 129 | "census-income_srt.zip", 130 | "census1881.zip", 131 | "census1881_srt.zip", 132 | "weather_sept_85.zip", 133 | "weather_sept_85_srt.zip", 134 | "wikileaks-noquotes.zip", 135 | "wikileaks-noquotes_srt.zip", 136 | ]; 137 | 138 | use indicatif::{ProgressBar, ProgressStyle}; 139 | use std::io::BufRead; 140 | use zip::ZipArchive; 141 | 142 | let dir = path.as_ref().read_dir()?; 143 | 144 | let mut datasets = Vec::new(); 145 | 146 | // Future work: Reuse this buffer to parse croaring bitmaps for comparison 147 | let mut numbers = Vec::new(); 148 | 149 | for dir_entry_result in dir { 150 | let dir_entry = dir_entry_result?; 151 | let metadata = dir_entry.metadata()?; 152 | let file_name = dir_entry.file_name(); 153 | // TODO dont panic 154 | let file_name_str = file_name.to_str().expect("utf-8 filename"); 155 | 156 | if metadata.is_file() && DATASET_FILENAME_WHITELIST.contains(&file_name_str) { 157 | let file = File::open(dir_entry.path())?; 158 | let name = file_name_str.split_at(file_name_str.len() - ".zip".len()).0.to_string(); 159 | 160 | let mut zip = ZipArchive::new(file)?; 161 | 162 | let mut total_size = 0; 163 | for i in 0..zip.len() { 164 | let file = zip.by_index(i)?; 165 | total_size += file.size(); 166 | } 167 | 168 | let pb = ProgressBar::new(total_size) 169 | .with_style( 170 | ProgressStyle::default_bar() 171 | .template(" {prefix:.green} [{bar}] {msg}") 172 | .expect("template string invalid") 173 | .progress_chars("#> "), 174 | ) 175 | .with_prefix("Parsing") 176 | .with_message(name.clone()); 177 | 178 | let mut bitmaps = Vec::with_capacity(zip.len()); 179 | for i in 0..zip.len() { 180 | let file = zip.by_index(i)?; 181 | let size = file.size(); 182 | let buf = BufReader::new(file); 183 | 184 | for bytes in buf.split(b',') { 185 | let bytes = bytes?; 186 | let str = String::from_utf8(bytes)?; 187 | let n = str.trim().parse::()?; 188 | numbers.push(n); 189 | } 190 | 191 | let bitmap = RoaringBitmap::from_sorted_iter(numbers.iter().copied())?; 192 | numbers.clear(); 193 | bitmaps.push(bitmap); 194 | 195 | pb.set_position(pb.position() + size); 196 | } 197 | 198 | pb.finish(); 199 | datasets.push(Dataset { name, bitmaps }); 200 | } 201 | } 202 | datasets.sort_unstable_by(|a, b| a.name.cmp(&b.name)); 203 | println!(); 204 | Ok(datasets) 205 | } 206 | -------------------------------------------------------------------------------- /roaring/tests/iter.rs: -------------------------------------------------------------------------------- 1 | use proptest::arbitrary::any; 2 | use proptest::collection::btree_set; 3 | use proptest::proptest; 4 | 5 | use roaring::RoaringBitmap; 6 | 7 | #[test] 8 | fn range() { 9 | let original = (0..2000).collect::(); 10 | let clone = RoaringBitmap::from_iter(&original); 11 | let clone2 = RoaringBitmap::from_iter(original.clone()); 12 | 13 | assert_eq!(clone, original); 14 | assert_eq!(clone2, original); 15 | } 16 | 17 | #[test] 18 | fn array() { 19 | let original = (0..5).collect::(); 20 | let clone = RoaringBitmap::from([0, 1, 2, 3, 4]); 21 | 22 | assert_eq!(clone, original); 23 | } 24 | 25 | #[test] 26 | fn bitmap() { 27 | let original = (0..100_000).collect::(); 28 | let clone = RoaringBitmap::from_iter(&original); 29 | let clone2 = RoaringBitmap::from_iter(original.clone()); 30 | 31 | assert_eq!(clone, original); 32 | assert_eq!(clone2, original); 33 | } 34 | 35 | #[test] 36 | fn arrays() { 37 | let original = (0..2000) 38 | .chain(1_000_000..1_002_000) 39 | .chain(2_000_000..2_001_000) 40 | .collect::(); 41 | let clone = RoaringBitmap::from_iter(&original); 42 | let clone2 = RoaringBitmap::from_iter(original.clone()); 43 | 44 | assert_eq!(clone, original); 45 | assert_eq!(clone2, original); 46 | } 47 | 48 | #[test] 49 | fn bitmaps() { 50 | let original = (0..100_000) 51 | .chain(1_000_000..1_012_000) 52 | .chain(2_000_000..2_010_000) 53 | .collect::(); 54 | let clone = RoaringBitmap::from_iter(&original); 55 | let clone2 = RoaringBitmap::from_iter(original.clone()); 56 | 57 | assert_eq!(clone, original); 58 | assert_eq!(clone2, original); 59 | } 60 | 61 | proptest! { 62 | #[test] 63 | fn iter(values in btree_set(any::(), ..=10_000)) { 64 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 65 | // Iterator::eq != PartialEq::eq - cannot use assert_eq macro 66 | assert!(values.into_iter().eq(bitmap)); 67 | } 68 | } 69 | 70 | proptest! { 71 | #[test] 72 | fn fold(values in btree_set(any::(), ..=10_000)) { 73 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 74 | let mut val_iter = values.into_iter(); 75 | // `Iterator::all` uses currently unimplementable `try_fold`, we test `fold` 76 | #[allow(clippy::unnecessary_fold)] 77 | let r = bitmap.into_iter().fold(true, |b, i| { 78 | b && i == val_iter.next().unwrap() 79 | }); 80 | assert!(r) 81 | } 82 | } 83 | 84 | proptest! { 85 | #[test] 86 | fn nth(values in btree_set(any::(), ..=10_000), nth in 0..10_005usize) { 87 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 88 | let mut orig_iter = bitmap.iter().fuse(); 89 | let mut iter = bitmap.iter(); 90 | 91 | for _ in 0..nth { 92 | if orig_iter.next().is_none() { 93 | break; 94 | } 95 | } 96 | let expected = orig_iter.next(); 97 | assert_eq!(expected, iter.nth(nth)); 98 | let expected_next = orig_iter.next(); 99 | assert_eq!(expected_next, iter.next()); 100 | 101 | let mut val_iter = values.into_iter(); 102 | assert_eq!(expected, val_iter.nth(nth)); 103 | assert_eq!(expected_next, val_iter.next()); 104 | } 105 | } 106 | 107 | #[test] 108 | fn huge_nth() { 109 | let bitmap = RoaringBitmap::new(); 110 | let mut iter = bitmap.iter(); 111 | assert_eq!(None, iter.nth(usize::MAX)); 112 | } 113 | 114 | proptest! { 115 | #[test] 116 | fn count(values in btree_set(any::(), ..=10_000), skip in 0..10_005usize) { 117 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 118 | let mut iter = bitmap.iter(); 119 | 120 | if let Some(n) = skip.checked_sub(1) { 121 | iter.nth(n); 122 | } 123 | let expected_count = values.len().saturating_sub(skip); 124 | let size_hint = iter.size_hint(); 125 | assert_eq!(expected_count, size_hint.0); 126 | assert_eq!(Some(expected_count), size_hint.1); 127 | assert_eq!(expected_count, iter.count()); 128 | } 129 | } 130 | 131 | #[test] 132 | fn rev_array() { 133 | let values = 0..100; 134 | let bitmap = values.clone().collect::(); 135 | 136 | assert!(values.into_iter().rev().eq(bitmap.iter().rev())); 137 | } 138 | 139 | #[test] 140 | fn rev_bitmap() { 141 | let values = 0..=100_000; 142 | let bitmap = values.clone().collect::(); 143 | 144 | assert!(values.into_iter().rev().eq(bitmap.iter().rev())); 145 | } 146 | 147 | proptest! { 148 | #[test] 149 | fn rev_iter(values in btree_set(any::(), ..=10_000)) { 150 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 151 | 152 | assert!(values.into_iter().rev().eq(bitmap.iter().rev())); 153 | } 154 | } 155 | 156 | #[test] 157 | fn from_iter() { 158 | // This test verifies that the public API allows conversion from iterators 159 | // with u32 as well as &u32 elements. 160 | let vals = vec![1, 5, 10000]; 161 | let a = RoaringBitmap::from_iter(vals.iter()); 162 | let b = RoaringBitmap::from_iter(vals); 163 | assert_eq!(a, b); 164 | } 165 | 166 | #[derive(Clone, Debug)] 167 | pub struct OutsideInIter(bool, T); 168 | 169 | impl Iterator for OutsideInIter 170 | where 171 | I: DoubleEndedIterator, 172 | { 173 | type Item = T; 174 | 175 | fn next(&mut self) -> Option { 176 | let res = if self.0 { self.1.next() } else { self.1.next_back() }; 177 | self.0 = !self.0; 178 | res 179 | } 180 | } 181 | 182 | pub fn outside_in(into_iter: I) -> OutsideInIter 183 | where 184 | U: DoubleEndedIterator, 185 | I: IntoIterator, 186 | { 187 | OutsideInIter(true, into_iter.into_iter()) 188 | } 189 | 190 | // Sanity check that outside_in does what we expect 191 | #[test] 192 | fn outside_in_iterator() { 193 | let values = 0..10; 194 | assert!(outside_in(values).eq(vec![0, 9, 1, 8, 2, 7, 3, 6, 4, 5])); 195 | } 196 | 197 | #[test] 198 | fn interleaved_array() { 199 | let values = 0..100; 200 | let bitmap = values.clone().collect::(); 201 | 202 | assert!(outside_in(values).eq(outside_in(bitmap))); 203 | } 204 | 205 | #[test] 206 | fn interleaved_bitmap() { 207 | let values = 0..=4097; 208 | let bitmap = values.clone().collect::(); 209 | 210 | assert!(outside_in(values).eq(outside_in(bitmap))); 211 | } 212 | 213 | #[test] 214 | fn run_nth_max() { 215 | let mut bitmap = RoaringBitmap::new(); 216 | bitmap.insert_range(0..0x1_0000); 217 | let mut iter = bitmap.iter(); 218 | assert_eq!(iter.nth(0x0_FFFF), Some(0x0_FFFF)); 219 | assert_eq!(iter.len(), 0); 220 | #[allow(clippy::iter_nth_zero)] 221 | { 222 | assert_eq!(iter.nth(0), None); 223 | } 224 | assert_eq!(iter.next(), None); 225 | } 226 | 227 | #[test] 228 | fn run_nth_back_max() { 229 | let mut bitmap = RoaringBitmap::new(); 230 | bitmap.insert_range(0..0x1_0000); 231 | let mut iter = bitmap.iter(); 232 | assert_eq!(iter.nth_back(0x0_FFFF), Some(0)); 233 | assert_eq!(iter.len(), 0); 234 | assert_eq!(iter.nth_back(0), None); 235 | assert_eq!(iter.next_back(), None); 236 | } 237 | 238 | proptest! { 239 | #[test] 240 | fn interleaved_iter(values in btree_set(any::(), 50_000..=100_000)) { 241 | let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); 242 | 243 | assert!(outside_in(values).eq(outside_in(bitmap))); 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /roaring/tests/treemap_iter_advance_to.rs: -------------------------------------------------------------------------------- 1 | extern crate roaring; 2 | use roaring::RoaringTreemap; 3 | 4 | #[test] 5 | fn iter_basic() { 6 | let bm = RoaringTreemap::from([1, 2, 3, 4, 11, 12, 13, 14]); 7 | let mut i = bm.iter(); 8 | i.advance_to(10); 9 | for n in 11..=14 { 10 | assert_eq!(i.next(), Some(n)) 11 | } 12 | assert_eq!(i.next(), None); 13 | } 14 | 15 | #[test] 16 | fn to_missing_container() { 17 | let bm = RoaringTreemap::from([1, 0x2_0001, 0x2_0002]); 18 | let mut i = bm.iter(); 19 | i.advance_to(0x1_0000); 20 | assert_eq!(i.next(), Some(0x2_0001)); 21 | assert_eq!(i.next(), Some(0x2_0002)); 22 | assert_eq!(i.next(), None); 23 | } 24 | 25 | #[test] 26 | fn to_next_bitmap() { 27 | let bm = 28 | RoaringTreemap::from([1u64, 0x2_0001u64 + u32::MAX as u64, 0x2_0002u64 + u32::MAX as u64]); 29 | let mut i = bm.iter(); 30 | i.advance_to(0x1_0000); 31 | assert_eq!(i.next(), Some(0x2_0001u64 + u32::MAX as u64)); 32 | assert_eq!(i.next(), Some(0x2_0002u64 + u32::MAX as u64)); 33 | assert_eq!(i.next(), None); 34 | } 35 | 36 | #[test] 37 | fn iter_back_basic() { 38 | let bm = RoaringTreemap::from([1, 2, 3, 4, 11, 12, 13, 14]); 39 | let mut i = bm.iter(); 40 | i.advance_back_to(10); 41 | assert_eq!(i.next(), Some(1)); 42 | assert_eq!(i.next(), Some(2)); 43 | assert_eq!(i.next_back(), Some(4)); 44 | assert_eq!(i.next_back(), Some(3)); 45 | 46 | assert_eq!(i.next(), None); 47 | assert_eq!(i.next_back(), None); 48 | } 49 | 50 | #[test] 51 | fn iter_advance_past_end() { 52 | let bm = RoaringTreemap::from([1, 2, 3, 4, 11, 12, 13, 14]); 53 | let mut i = bm.iter(); 54 | i.advance_to(15); 55 | assert_eq!(i.next(), None); 56 | assert_eq!(i.size_hint(), (0, Some(0))); 57 | } 58 | 59 | #[test] 60 | fn iter_multi_container() { 61 | let bm = RoaringTreemap::from([1, 2, 3, 100000, 100001]); 62 | let mut i = bm.iter(); 63 | i.advance_to(3); 64 | assert_eq!(i.size_hint(), (3, Some(3))); 65 | assert_eq!(i.next(), Some(3)); 66 | assert_eq!(i.size_hint(), (2, Some(2))); 67 | assert_eq!(i.next(), Some(100000)); 68 | assert_eq!(i.size_hint(), (1, Some(1))); 69 | assert_eq!(i.next(), Some(100001)); 70 | assert_eq!(i.size_hint(), (0, Some(0))); 71 | assert_eq!(i.next(), None); 72 | assert_eq!(i.size_hint(), (0, Some(0))); 73 | } 74 | 75 | #[test] 76 | fn iter_multi_container_multi_bitmap() { 77 | let bm = RoaringTreemap::from([ 78 | 1, 79 | 2, 80 | 3, 81 | 100000, 82 | 100001, 83 | 1u64 + u32::MAX as u64, 84 | 2u64 + u32::MAX as u64, 85 | 3u64 + u32::MAX as u64, 86 | 100000u64 + u32::MAX as u64, 87 | 100001u64 + u32::MAX as u64, 88 | ]); 89 | let mut i = bm.iter(); 90 | i.advance_to(3); 91 | assert_eq!(i.size_hint(), (8, Some(8))); 92 | assert_eq!(i.next(), Some(3)); 93 | assert_eq!(i.size_hint(), (7, Some(7))); 94 | assert_eq!(i.next(), Some(100000)); 95 | assert_eq!(i.size_hint(), (6, Some(6))); 96 | assert_eq!(i.next(), Some(100001)); 97 | assert_eq!(i.size_hint(), (5, Some(5))); 98 | assert_eq!(i.next(), Some(1u64 + u32::MAX as u64)); 99 | assert_eq!(i.size_hint(), (4, Some(4))); 100 | assert_eq!(i.next(), Some(2u64 + u32::MAX as u64)); 101 | assert_eq!(i.size_hint(), (3, Some(3))); 102 | assert_eq!(i.next(), Some(3u64 + u32::MAX as u64)); 103 | assert_eq!(i.size_hint(), (2, Some(2))); 104 | assert_eq!(i.next(), Some(100000u64 + u32::MAX as u64)); 105 | assert_eq!(i.size_hint(), (1, Some(1))); 106 | assert_eq!(i.next(), Some(100001u64 + u32::MAX as u64)); 107 | assert_eq!(i.size_hint(), (0, Some(0))); 108 | assert_eq!(i.next(), None); 109 | assert_eq!(i.size_hint(), (0, Some(0))); 110 | } 111 | 112 | #[test] 113 | fn iter_empty() { 114 | let bm = RoaringTreemap::new(); 115 | let mut i = bm.iter(); 116 | i.advance_to(31337); 117 | assert_eq!(i.size_hint(), (0, Some(0))); 118 | assert_eq!(i.next(), None) 119 | } 120 | 121 | #[test] 122 | fn iter_back_empty() { 123 | let bm = RoaringTreemap::new(); 124 | let mut i = bm.iter(); 125 | i.advance_back_to(31337); 126 | assert_eq!(i.size_hint(), (0, Some(0))); 127 | assert_eq!(i.next(), None) 128 | } 129 | 130 | /*#[test] 131 | fn into_iter_basic() { 132 | let bm = RoaringTreemap::from([1, 2, 3, 4, 11, 12, 13, 14]); 133 | let mut i = bm.into_iter(); 134 | i.advance_to(10); 135 | let mut expected_size_hint = 4; 136 | assert_eq!(i.size_hint(), (expected_size_hint, Some(expected_size_hint))); 137 | for n in 11..=14 { 138 | assert_eq!(i.next(), Some(n)); 139 | expected_size_hint -= 1; 140 | assert_eq!(i.size_hint(), (expected_size_hint, Some(expected_size_hint))); 141 | } 142 | assert_eq!(i.next(), None); 143 | }*/ 144 | 145 | /*#[test] 146 | fn into_iter_multi_container() { 147 | let bm = RoaringTreemap::from([1, 2, 3, 100000, 100001]); 148 | let mut i = bm.into_iter(); 149 | i.advance_to(3); 150 | assert_eq!(i.size_hint(), (3, Some(3))); 151 | assert_eq!(i.next(), Some(3)); 152 | assert_eq!(i.next(), Some(100000)); 153 | assert_eq!(i.next(), Some(100001)); 154 | assert_eq!(i.next(), None); 155 | }*/ 156 | 157 | /*#[test] 158 | fn into_iter_empty() { 159 | let bm = RoaringTreemap::new(); 160 | let mut i = bm.into_iter(); 161 | i.advance_to(31337); 162 | assert_eq!(i.size_hint(), (0, Some(0))); 163 | assert_eq!(i.next(), None) 164 | }*/ 165 | 166 | /*#[test] 167 | fn into_iter_back_empty() { 168 | let bm = RoaringTreemap::new(); 169 | let mut i = bm.into_iter(); 170 | i.advance_back_to(31337); 171 | assert_eq!(i.size_hint(), (0, Some(0))); 172 | assert_eq!(i.next(), None) 173 | }*/ 174 | 175 | #[test] 176 | fn advance_to_with_tail_iter() { 177 | let bm = RoaringTreemap::from([1, 2, 3, 100000, 100001]); 178 | let mut i = bm.iter(); 179 | i.next_back(); 180 | i.advance_to(100000); 181 | assert_eq!(i.size_hint(), (1, Some(1))); 182 | assert_eq!(i.next(), Some(100000)); 183 | assert_eq!(i.size_hint(), (0, Some(0))); 184 | assert_eq!(i.next(), None); 185 | } 186 | 187 | #[test] 188 | fn advance_to_end() { 189 | let bitmap = RoaringTreemap::from([u64::MAX]); 190 | let mut iter = bitmap.iter(); 191 | iter.advance_to(u64::MAX); 192 | assert_eq!(Some(u64::MAX), iter.next()); 193 | assert_eq!(None, iter.next()); 194 | } 195 | 196 | #[test] 197 | fn advance_bitset() { 198 | let mut bitmap = RoaringTreemap::new(); 199 | for i in (0..=0x2_0000).step_by(2) { 200 | bitmap.insert(i); 201 | } 202 | let mut iter = bitmap.iter(); 203 | iter.advance_to(0x1_0000 - 4); 204 | // 0x1_0000 + 5 is not in the bitmap, so the next value will be the first value less than that 205 | iter.advance_back_to(0x1_0000 + 5); 206 | assert_eq!(iter.next(), Some(0x1_0000 - 4)); 207 | assert_eq!(iter.next_back(), Some(0x1_0000 + 4)); 208 | 209 | assert_eq!(iter.next(), Some(0x1_0000 - 2)); 210 | assert_eq!(iter.next(), Some(0x1_0000)); 211 | assert_eq!(iter.next(), Some(0x1_0000 + 2)); 212 | assert_eq!(iter.next(), None); 213 | assert_eq!(iter.next_back(), None); 214 | } 215 | 216 | #[test] 217 | fn advance_bitset_current_word() { 218 | let mut bitmap = RoaringTreemap::new(); 219 | for i in (0..=0x2_0000).step_by(2) { 220 | bitmap.insert(i); 221 | } 222 | let mut iter = bitmap.iter(); 223 | iter.advance_to(4); 224 | iter.advance_back_to(0x2_0000 - 4); 225 | for i in (4..=(0x2_0000 - 4)).step_by(2) { 226 | assert_eq!(iter.next(), Some(i)); 227 | } 228 | assert_eq!(iter.next(), None); 229 | } 230 | 231 | #[test] 232 | fn advance_bitset_to_end_word() { 233 | let mut bitmap = RoaringTreemap::new(); 234 | for i in (0..=0x2_0000).step_by(2) { 235 | bitmap.insert(i); 236 | } 237 | let mut iter = bitmap.iter(); 238 | iter.advance_to(0x1_0000 - 4); 239 | for i in ((0x1_0000 - 4)..=0x2_0000).step_by(2) { 240 | assert_eq!(iter.next(), Some(i)); 241 | } 242 | assert_eq!(iter.next(), None); 243 | } 244 | 245 | #[test] 246 | fn advance_bitset_back_to_start_word() { 247 | let mut bitmap = RoaringTreemap::new(); 248 | for i in (0..=0x2_0000).step_by(2) { 249 | bitmap.insert(i); 250 | } 251 | let mut iter = bitmap.iter(); 252 | iter.advance_back_to(0x1_0000 - 4); 253 | for i in (0..=(0x1_0000 - 4)).step_by(2) { 254 | assert_eq!(iter.next(), Some(i)); 255 | } 256 | assert_eq!(iter.next(), None); 257 | } 258 | -------------------------------------------------------------------------------- /roaring/tests/iter_advance_to.rs: -------------------------------------------------------------------------------- 1 | use roaring::RoaringBitmap; 2 | 3 | #[test] 4 | fn iter_basic() { 5 | let bm = RoaringBitmap::from([1, 2, 3, 4, 11, 12, 13, 14]); 6 | let mut i = bm.iter(); 7 | i.advance_to(10); 8 | for n in 11..=14 { 9 | assert_eq!(i.next(), Some(n)) 10 | } 11 | assert_eq!(i.next(), None); 12 | } 13 | 14 | #[test] 15 | fn to_missing_container() { 16 | let bm = RoaringBitmap::from([1, 0x2_0001, 0x2_0002]); 17 | let mut i = bm.iter(); 18 | i.advance_to(0x1_0000); 19 | assert_eq!(i.next(), Some(0x2_0001)); 20 | assert_eq!(i.next(), Some(0x2_0002)); 21 | assert_eq!(i.next(), None); 22 | } 23 | 24 | #[test] 25 | fn iter_back_basic() { 26 | let bm = RoaringBitmap::from([1, 2, 3, 4, 11, 12, 13, 14]); 27 | let mut i = bm.iter(); 28 | i.advance_back_to(10); 29 | assert_eq!(i.next(), Some(1)); 30 | assert_eq!(i.next(), Some(2)); 31 | assert_eq!(i.next_back(), Some(4)); 32 | assert_eq!(i.next_back(), Some(3)); 33 | 34 | assert_eq!(i.next(), None); 35 | assert_eq!(i.next_back(), None); 36 | } 37 | 38 | #[test] 39 | fn iter_advance_past_end() { 40 | let bm = RoaringBitmap::from([1, 2, 3, 4, 11, 12, 13, 14]); 41 | let mut i = bm.iter(); 42 | i.advance_to(15); 43 | assert_eq!(i.size_hint(), (0, Some(0))); 44 | assert_eq!(i.next(), None); 45 | } 46 | 47 | #[test] 48 | fn iter_multi_container() { 49 | let bm = RoaringBitmap::from([1, 2, 3, 100000, 100001]); 50 | let mut i = bm.iter(); 51 | i.advance_to(3); 52 | assert_eq!(i.size_hint(), (3, Some(3))); 53 | assert_eq!(i.next(), Some(3)); 54 | assert_eq!(i.size_hint(), (2, Some(2))); 55 | assert_eq!(i.next(), Some(100000)); 56 | assert_eq!(i.size_hint(), (1, Some(1))); 57 | assert_eq!(i.next(), Some(100001)); 58 | assert_eq!(i.size_hint(), (0, Some(0))); 59 | assert_eq!(i.next(), None); 60 | assert_eq!(i.size_hint(), (0, Some(0))); 61 | } 62 | 63 | #[test] 64 | fn iter_empty() { 65 | let bm = RoaringBitmap::new(); 66 | let mut i = bm.iter(); 67 | i.advance_to(31337); 68 | assert_eq!(i.size_hint(), (0, Some(0))); 69 | assert_eq!(i.next(), None) 70 | } 71 | 72 | #[test] 73 | fn iter_back_empty() { 74 | let bm = RoaringBitmap::new(); 75 | let mut i = bm.iter(); 76 | i.advance_back_to(31337); 77 | assert_eq!(i.size_hint(), (0, Some(0))); 78 | assert_eq!(i.next(), None) 79 | } 80 | 81 | #[test] 82 | fn into_iter_basic() { 83 | let bm = RoaringBitmap::from([1, 2, 3, 4, 11, 12, 13, 14]); 84 | let mut i = bm.into_iter(); 85 | i.advance_to(10); 86 | let mut expected_size_hint = 4; 87 | assert_eq!(i.size_hint(), (expected_size_hint, Some(expected_size_hint))); 88 | for n in 11..=14 { 89 | assert_eq!(i.next(), Some(n)); 90 | expected_size_hint -= 1; 91 | assert_eq!(i.size_hint(), (expected_size_hint, Some(expected_size_hint))); 92 | } 93 | assert_eq!(i.next(), None); 94 | } 95 | 96 | #[test] 97 | fn into_iter_multi_container() { 98 | let bm = RoaringBitmap::from([1, 2, 3, 100000, 100001]); 99 | let mut i = bm.into_iter(); 100 | i.advance_to(3); 101 | assert_eq!(i.size_hint(), (3, Some(3))); 102 | assert_eq!(i.next(), Some(3)); 103 | assert_eq!(i.next(), Some(100000)); 104 | assert_eq!(i.next(), Some(100001)); 105 | assert_eq!(i.next(), None); 106 | } 107 | 108 | #[test] 109 | fn into_iter_empty() { 110 | let bm = RoaringBitmap::new(); 111 | let mut i = bm.into_iter(); 112 | i.advance_to(31337); 113 | assert_eq!(i.size_hint(), (0, Some(0))); 114 | assert_eq!(i.next(), None) 115 | } 116 | 117 | #[test] 118 | fn into_iter_back_empty() { 119 | let bm = RoaringBitmap::new(); 120 | let mut i = bm.into_iter(); 121 | i.advance_back_to(31337); 122 | assert_eq!(i.size_hint(), (0, Some(0))); 123 | assert_eq!(i.next(), None) 124 | } 125 | 126 | #[test] 127 | fn advance_to_with_tail_iter() { 128 | let bm = RoaringBitmap::from([1, 2, 3, 100000, 100001]); 129 | let mut i = bm.iter(); 130 | i.next_back(); 131 | i.advance_to(100000); 132 | assert_eq!(i.size_hint(), (1, Some(1))); 133 | assert_eq!(i.next(), Some(100000)); 134 | assert_eq!(i.size_hint(), (0, Some(0))); 135 | assert_eq!(i.next(), None); 136 | } 137 | 138 | #[test] 139 | fn advance_to_end() { 140 | let bitmap = RoaringBitmap::from([u32::MAX]); 141 | let mut iter = bitmap.iter(); 142 | iter.advance_to(u32::MAX); 143 | assert_eq!(Some(u32::MAX), iter.next()); 144 | assert_eq!(None, iter.next()); 145 | } 146 | 147 | #[test] 148 | fn advance_bitset() { 149 | let mut bitmap = RoaringBitmap::new(); 150 | for i in (0..=0x2_0000).step_by(2) { 151 | bitmap.insert(i); 152 | } 153 | let mut iter = bitmap.iter(); 154 | iter.advance_to(0x1_0000 - 4); 155 | // 0x1_0000 + 5 is not in the bitmap, so the next value will be the first value less than that 156 | iter.advance_back_to(0x1_0000 + 5); 157 | assert_eq!(iter.next(), Some(0x1_0000 - 4)); 158 | assert_eq!(iter.next_back(), Some(0x1_0000 + 4)); 159 | 160 | assert_eq!(iter.next(), Some(0x1_0000 - 2)); 161 | assert_eq!(iter.next(), Some(0x1_0000)); 162 | assert_eq!(iter.next(), Some(0x1_0000 + 2)); 163 | assert_eq!(iter.next(), None); 164 | assert_eq!(iter.next_back(), None); 165 | } 166 | 167 | #[test] 168 | fn advance_bitset_current_word() { 169 | let mut bitmap = RoaringBitmap::new(); 170 | for i in (0..=0x2_0000).step_by(2) { 171 | bitmap.insert(i); 172 | } 173 | let mut iter = bitmap.iter(); 174 | iter.advance_to(4); 175 | iter.advance_back_to(0x2_0000 - 4); 176 | for i in (4..=(0x2_0000 - 4)).step_by(2) { 177 | assert_eq!(iter.next(), Some(i)); 178 | } 179 | assert_eq!(iter.next(), None); 180 | } 181 | 182 | #[test] 183 | fn advance_bitset_to_end_word() { 184 | let mut bitmap = RoaringBitmap::new(); 185 | for i in (0..=0x2_0000).step_by(2) { 186 | bitmap.insert(i); 187 | } 188 | let mut iter = bitmap.iter(); 189 | iter.advance_to(0x1_0000 - 4); 190 | for i in ((0x1_0000 - 4)..=0x2_0000).step_by(2) { 191 | assert_eq!(iter.next(), Some(i)); 192 | } 193 | assert_eq!(iter.next(), None); 194 | } 195 | 196 | #[test] 197 | fn advance_bitset_back_to_start_word() { 198 | let mut bitmap = RoaringBitmap::new(); 199 | for i in (0..=0x2_0000).step_by(2) { 200 | bitmap.insert(i); 201 | } 202 | let mut iter = bitmap.iter(); 203 | iter.advance_back_to(0x1_0000 - 4); 204 | for i in (0..=(0x1_0000 - 4)).step_by(2) { 205 | assert_eq!(iter.next(), Some(i)); 206 | } 207 | assert_eq!(iter.next(), None); 208 | } 209 | 210 | #[test] 211 | fn advance_run_past_the_end() { 212 | let mut bitmap = RoaringBitmap::new(); 213 | bitmap.insert_range(0..=0x35B00); 214 | let mut iter = bitmap.iter(); 215 | iter.advance_to(0x35B01); 216 | assert_eq!(iter.next(), None); 217 | } 218 | 219 | #[test] 220 | fn advance_run_back_before_start() { 221 | let mut bitmap = RoaringBitmap::new(); 222 | bitmap.insert_range(500..=0x35B00); 223 | let mut iter = bitmap.iter(); 224 | iter.advance_back_to(499); 225 | assert_eq!(iter.next_back(), None); 226 | } 227 | 228 | #[test] 229 | fn advance_run_back_reduces_forward_iter() { 230 | let mut bitmap = RoaringBitmap::new(); 231 | bitmap.insert_range(0..=0x4000); 232 | let mut iter = bitmap.iter(); 233 | iter.advance_back_to(1); 234 | 235 | assert_eq!(iter.next(), Some(0)); 236 | assert_eq!(iter.next(), Some(1)); 237 | assert_eq!(iter.next(), None); 238 | } 239 | 240 | #[test] 241 | fn advance_run_front_and_back_past_each_other() { 242 | let mut bitmap = RoaringBitmap::new(); 243 | bitmap.insert_range(0..=0x4000); 244 | let mut iter = bitmap.iter(); 245 | iter.advance_back_to(100); 246 | iter.advance_to(300); 247 | assert_eq!(iter.next(), None); 248 | } 249 | 250 | #[test] 251 | fn advance_run_both_sides_past_each_other() { 252 | let mut bitmap = RoaringBitmap::new(); 253 | bitmap.insert_range(0..0x1000); 254 | let mut iter = bitmap.iter(); 255 | iter.advance_back_to(100); 256 | iter.advance_to(0xFFFF); 257 | assert_eq!(iter.len(), 0); 258 | assert_eq!(iter.nth_back(0), None); 259 | } 260 | 261 | #[test] 262 | fn advance_run_with_nth() { 263 | let mut bitmap = RoaringBitmap::new(); 264 | bitmap.insert_range(36141..=224407); 265 | let mut iter = bitmap.iter(); 266 | iter.advance_back_to(101779); 267 | assert_eq!(iter.nth(100563), None); 268 | } 269 | 270 | #[test] 271 | fn advance_to_with_next_len() { 272 | let mut bitmap = RoaringBitmap::new(); 273 | bitmap.insert_range(100..0x4000); 274 | let mut iter = bitmap.iter(); 275 | iter.advance_back_to(100); 276 | assert_eq!(iter.next(), Some(100)); 277 | assert_eq!(iter.len(), 0); 278 | assert_eq!(iter.nth_back(0), None); 279 | } 280 | 281 | #[test] 282 | fn tmp() { 283 | let mut bitmap = RoaringBitmap::new(); 284 | bitmap.insert_range(196363..=262143); 285 | let mut iter = bitmap.iter(); 286 | assert_eq!(iter.next_back(), Some(262143)); 287 | iter.advance_to(228960); 288 | assert_eq!(iter.nth(36643), None); 289 | } 290 | 291 | #[test] 292 | fn advance_bitset_front_and_back_past_each_other() { 293 | let mut bitmap = RoaringBitmap::new(); 294 | bitmap.insert_range(0..=0x4000); 295 | bitmap.remove_run_compression(); 296 | let mut iter = bitmap.iter(); 297 | iter.advance_back_to(100); 298 | iter.advance_to(300); 299 | assert_eq!(iter.next(), None); 300 | } 301 | 302 | #[test] 303 | fn combine_with_nth() { 304 | let mut bitmap = RoaringBitmap::new(); 305 | bitmap.insert_range(0..=0xFFFF); 306 | bitmap.remove_run_compression(); 307 | let mut iter = bitmap.iter(); 308 | 309 | // Use nth to skip to a specific position 310 | assert_eq!(iter.nth(100), Some(100)); 311 | iter.advance_back_to(50); 312 | assert_eq!(iter.next_back(), None); 313 | } 314 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /roaring/src/treemap/multiops.rs: -------------------------------------------------------------------------------- 1 | use alloc::collections::{binary_heap::PeekMut, BTreeMap, BinaryHeap}; 2 | use core::{borrow::Borrow, cmp::Ordering, mem}; 3 | 4 | use crate::{MultiOps, RoaringBitmap, RoaringTreemap}; 5 | 6 | #[cfg(not(feature = "std"))] 7 | use alloc::vec::Vec; 8 | 9 | impl MultiOps for I 10 | where 11 | I: IntoIterator, 12 | { 13 | type Output = RoaringTreemap; 14 | 15 | fn union(self) -> Self::Output { 16 | try_simple_multi_op_owned::<_, _, UnionOp>( 17 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 18 | ) 19 | .unwrap() 20 | } 21 | 22 | fn intersection(self) -> Self::Output { 23 | try_ordered_multi_op_owned::<_, _, IntersectionOp>( 24 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 25 | ) 26 | .unwrap() 27 | } 28 | 29 | fn difference(self) -> Self::Output { 30 | try_ordered_multi_op_owned::<_, _, DifferenceOp>( 31 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 32 | ) 33 | .unwrap() 34 | } 35 | 36 | fn symmetric_difference(self) -> Self::Output { 37 | try_simple_multi_op_owned::<_, _, SymmetricDifferenceOp>( 38 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 39 | ) 40 | .unwrap() 41 | } 42 | } 43 | 44 | impl MultiOps> for I 45 | where 46 | I: IntoIterator>, 47 | { 48 | type Output = Result; 49 | 50 | fn union(self) -> Self::Output { 51 | try_simple_multi_op_owned::<_, _, UnionOp>(self) 52 | } 53 | 54 | fn intersection(self) -> Self::Output { 55 | try_ordered_multi_op_owned::<_, _, IntersectionOp>(self) 56 | } 57 | 58 | fn difference(self) -> Self::Output { 59 | try_ordered_multi_op_owned::<_, _, DifferenceOp>(self) 60 | } 61 | 62 | fn symmetric_difference(self) -> Self::Output { 63 | try_simple_multi_op_owned::<_, _, SymmetricDifferenceOp>(self) 64 | } 65 | } 66 | 67 | #[inline] 68 | fn try_simple_multi_op_owned(treemaps: I) -> Result 69 | where 70 | I: IntoIterator>, 71 | { 72 | let treemaps = treemaps.into_iter().collect::, _>>()?; 73 | 74 | let mut heap: BinaryHeap<_> = treemaps 75 | .into_iter() 76 | .filter_map(|treemap| { 77 | let mut iter = treemap.map.into_iter(); 78 | iter.next().map(|(key, bitmap)| PeekedRoaringBitmap { key, bitmap, iter }) 79 | }) 80 | .collect(); 81 | 82 | let mut bitmaps = Vec::new(); 83 | let mut map = BTreeMap::new(); 84 | 85 | while let Some(mut peek) = heap.peek_mut() { 86 | let (key, bitmap) = match peek.iter.next() { 87 | Some((next_key, next_bitmap)) => { 88 | let key = peek.key; 89 | peek.key = next_key; 90 | let bitmap = mem::replace(&mut peek.bitmap, next_bitmap); 91 | (key, bitmap) 92 | } 93 | None => { 94 | let popped = PeekMut::pop(peek); 95 | (popped.key, popped.bitmap) 96 | } 97 | }; 98 | 99 | if let Some((first_key, _)) = bitmaps.first() { 100 | if *first_key != key { 101 | let current_key = *first_key; 102 | let computed_bitmap = O::op_owned(bitmaps.drain(..).map(|(_, rb)| rb)); 103 | if !computed_bitmap.is_empty() { 104 | map.insert(current_key, computed_bitmap); 105 | } 106 | } 107 | } 108 | 109 | bitmaps.push((key, bitmap)); 110 | } 111 | 112 | if let Some((first_key, _)) = bitmaps.first() { 113 | let current_key = *first_key; 114 | let computed_bitmap = O::op_owned(bitmaps.drain(..).map(|(_, rb)| rb)); 115 | if !computed_bitmap.is_empty() { 116 | map.insert(current_key, computed_bitmap); 117 | } 118 | } 119 | 120 | Ok(RoaringTreemap { map }) 121 | } 122 | 123 | #[inline] 124 | fn try_ordered_multi_op_owned(treemaps: I) -> Result 125 | where 126 | I: IntoIterator>, 127 | { 128 | let mut treemaps = treemaps.into_iter(); 129 | let mut treemap = match treemaps.next().transpose()? { 130 | Some(treemap) => treemap, 131 | None => return Ok(RoaringTreemap::new()), 132 | }; 133 | let mut treemaps = treemaps.collect::, _>>()?; 134 | 135 | // for each key in the first treemap we're going to find and 136 | // accumulate all the corresponding bitmaps 137 | let keys: Vec<_> = treemap.map.keys().copied().collect(); 138 | for k in keys { 139 | // the unwrap is safe since we're iterating on our keys 140 | let current_bitmap = treemap.map.remove(&k).unwrap(); 141 | let new_bitmap = 142 | O::op_owned(core::iter::once(current_bitmap).chain( 143 | treemaps.iter_mut().map(|treemap| treemap.map.remove(&k).unwrap_or_default()), 144 | )); 145 | if !new_bitmap.is_empty() { 146 | treemap.map.insert(k, new_bitmap); 147 | } 148 | } 149 | 150 | Ok(treemap) 151 | } 152 | 153 | #[inline] 154 | fn try_ordered_multi_op_ref<'a, E: 'a, I, O: Op>(treemaps: I) -> Result 155 | where 156 | I: IntoIterator>, 157 | { 158 | let mut treemaps = treemaps.into_iter(); 159 | let treemap = match treemaps.next().transpose()? { 160 | Some(treemap) => treemap, 161 | None => return Ok(RoaringTreemap::new()), 162 | }; 163 | let treemaps = treemaps.collect::, _>>()?; 164 | 165 | let mut ret = RoaringTreemap::new(); 166 | 167 | // for each keys in the first treemap we're going find and accumulate all the corresponding bitmaps 168 | let keys: Vec<_> = treemap.map.keys().copied().collect(); 169 | let empty_bitmap = RoaringBitmap::new(); 170 | for k in keys { 171 | // the unwrap is safe since we're iterating on our keys 172 | let current_bitmap = treemap.map.get(&k).unwrap(); 173 | let new_bitmap = O::op_ref( 174 | core::iter::once(current_bitmap) 175 | .chain(treemaps.iter().map(|treemap| treemap.map.get(&k).unwrap_or(&empty_bitmap))), 176 | ); 177 | if !new_bitmap.is_empty() { 178 | ret.map.insert(k, new_bitmap); 179 | } 180 | } 181 | 182 | Ok(ret) 183 | } 184 | 185 | #[inline] 186 | fn try_simple_multi_op_ref<'a, E: 'a, I, O: Op>(treemaps: I) -> Result 187 | where 188 | I: IntoIterator>, 189 | { 190 | let treemaps = treemaps.into_iter().collect::, E>>()?; 191 | 192 | let mut heap: BinaryHeap<_> = treemaps 193 | .into_iter() 194 | .filter_map(|treemap| { 195 | let mut iter = treemap.map.iter(); 196 | iter.next().map(|(&key, bitmap)| PeekedRoaringBitmap { key, bitmap, iter }) 197 | }) 198 | .collect(); 199 | 200 | let mut bitmaps = Vec::new(); 201 | let mut map = BTreeMap::new(); 202 | 203 | while let Some(mut peek) = heap.peek_mut() { 204 | let (key, bitmap) = match peek.iter.next() { 205 | Some((&next_key, next_bitmap)) => { 206 | let key = peek.key; 207 | peek.key = next_key; 208 | let bitmap = mem::replace(&mut peek.bitmap, next_bitmap); 209 | (key, bitmap) 210 | } 211 | None => { 212 | let popped = PeekMut::pop(peek); 213 | (popped.key, popped.bitmap) 214 | } 215 | }; 216 | 217 | if let Some((first_key, _)) = bitmaps.first() { 218 | if *first_key != key { 219 | let current_key = *first_key; 220 | let computed_bitmap = O::op_ref(bitmaps.drain(..).map(|(_, rb)| rb)); 221 | if !computed_bitmap.is_empty() { 222 | map.insert(current_key, computed_bitmap); 223 | } 224 | } 225 | } 226 | 227 | bitmaps.push((key, bitmap)); 228 | } 229 | 230 | if let Some((first_key, _)) = bitmaps.first() { 231 | let current_key = *first_key; 232 | let computed_bitmap = O::op_ref(bitmaps.drain(..).map(|(_, rb)| rb)); 233 | if !computed_bitmap.is_empty() { 234 | map.insert(current_key, computed_bitmap); 235 | } 236 | } 237 | 238 | Ok(RoaringTreemap { map }) 239 | } 240 | 241 | trait Op { 242 | fn op_owned>(iter: I) -> RoaringBitmap; 243 | fn op_ref<'a, I: IntoIterator>(iter: I) -> RoaringBitmap; 244 | } 245 | 246 | enum UnionOp {} 247 | 248 | impl Op for UnionOp { 249 | fn op_owned>(iter: J) -> RoaringBitmap { 250 | iter.union() 251 | } 252 | 253 | fn op_ref<'a, J: IntoIterator>(iter: J) -> RoaringBitmap { 254 | iter.union() 255 | } 256 | } 257 | 258 | enum IntersectionOp {} 259 | 260 | impl Op for IntersectionOp { 261 | fn op_owned>(iter: J) -> RoaringBitmap { 262 | iter.intersection() 263 | } 264 | 265 | fn op_ref<'a, J: IntoIterator>(iter: J) -> RoaringBitmap { 266 | iter.intersection() 267 | } 268 | } 269 | 270 | enum DifferenceOp {} 271 | 272 | impl Op for DifferenceOp { 273 | fn op_owned>(iter: J) -> RoaringBitmap { 274 | iter.difference() 275 | } 276 | 277 | fn op_ref<'a, J: IntoIterator>(iter: J) -> RoaringBitmap { 278 | iter.difference() 279 | } 280 | } 281 | 282 | enum SymmetricDifferenceOp {} 283 | 284 | impl Op for SymmetricDifferenceOp { 285 | fn op_owned>(iter: J) -> RoaringBitmap { 286 | iter.symmetric_difference() 287 | } 288 | 289 | fn op_ref<'a, J: IntoIterator>(iter: J) -> RoaringBitmap { 290 | iter.symmetric_difference() 291 | } 292 | } 293 | 294 | impl<'a, I> MultiOps<&'a RoaringTreemap> for I 295 | where 296 | I: IntoIterator, 297 | { 298 | type Output = RoaringTreemap; 299 | 300 | fn union(self) -> Self::Output { 301 | try_simple_multi_op_ref::<_, _, UnionOp>( 302 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 303 | ) 304 | .unwrap() 305 | } 306 | 307 | fn intersection(self) -> Self::Output { 308 | try_ordered_multi_op_ref::<_, _, IntersectionOp>( 309 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 310 | ) 311 | .unwrap() 312 | } 313 | 314 | fn difference(self) -> Self::Output { 315 | try_ordered_multi_op_ref::<_, _, DifferenceOp>( 316 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 317 | ) 318 | .unwrap() 319 | } 320 | 321 | fn symmetric_difference(self) -> Self::Output { 322 | try_simple_multi_op_ref::<_, _, SymmetricDifferenceOp>( 323 | self.into_iter().map(Ok::<_, core::convert::Infallible>), 324 | ) 325 | .unwrap() 326 | } 327 | } 328 | 329 | impl<'a, I, E: 'a> MultiOps> for I 330 | where 331 | I: IntoIterator>, 332 | { 333 | type Output = Result; 334 | 335 | fn union(self) -> Self::Output { 336 | try_simple_multi_op_ref::<_, _, UnionOp>(self) 337 | } 338 | 339 | fn intersection(self) -> Self::Output { 340 | try_ordered_multi_op_ref::<_, _, IntersectionOp>(self) 341 | } 342 | 343 | fn difference(self) -> Self::Output { 344 | try_ordered_multi_op_ref::<_, _, DifferenceOp>(self) 345 | } 346 | 347 | fn symmetric_difference(self) -> Self::Output { 348 | try_simple_multi_op_ref::<_, _, SymmetricDifferenceOp>(self) 349 | } 350 | } 351 | 352 | struct PeekedRoaringBitmap { 353 | key: u32, 354 | bitmap: R, 355 | iter: I, 356 | } 357 | 358 | impl, I> Ord for PeekedRoaringBitmap { 359 | fn cmp(&self, other: &Self) -> Ordering { 360 | self.key.cmp(&other.key).reverse() 361 | } 362 | } 363 | 364 | impl, I> PartialOrd for PeekedRoaringBitmap { 365 | fn partial_cmp(&self, other: &Self) -> Option { 366 | Some(self.cmp(other)) 367 | } 368 | } 369 | 370 | impl, I> Eq for PeekedRoaringBitmap {} 371 | 372 | impl, I> PartialEq for PeekedRoaringBitmap { 373 | fn eq(&self, other: &Self) -> bool { 374 | self.key == other.key 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /roaring/src/bitmap/ops_with_serialized.rs: -------------------------------------------------------------------------------- 1 | use bytemuck::cast_slice_mut; 2 | use byteorder::{LittleEndian, ReadBytesExt}; 3 | use core::convert::Infallible; 4 | use std::error::Error; 5 | use std::io::{self, SeekFrom}; 6 | use std::mem; 7 | use std::ops::RangeInclusive; 8 | 9 | use crate::bitmap::container::Container; 10 | use crate::bitmap::serialization::{ 11 | NO_OFFSET_THRESHOLD, SERIAL_COOKIE, SERIAL_COOKIE_NO_RUNCONTAINER, 12 | }; 13 | use crate::RoaringBitmap; 14 | 15 | use super::container::ARRAY_LIMIT; 16 | use super::store::{ArrayStore, BitmapStore, Store, BITMAP_LENGTH}; 17 | 18 | impl RoaringBitmap { 19 | /// Computes the intersection between a materialized [`RoaringBitmap`] and a serialized one. 20 | /// 21 | /// This is faster and more space efficient when you only need the intersection result. 22 | /// It reduces the number of deserialized internal container and therefore 23 | /// the number of allocations and copies of bytes. 24 | /// 25 | /// # Examples 26 | /// 27 | /// ```rust 28 | /// use roaring::RoaringBitmap; 29 | /// use std::io::Cursor; 30 | /// 31 | /// let rb1: RoaringBitmap = (1..4).collect(); 32 | /// let rb2: RoaringBitmap = (3..5).collect(); 33 | /// 34 | /// // Let's say the rb2 bitmap is serialized 35 | /// let mut bytes = Vec::new(); 36 | /// rb2.serialize_into(&mut bytes).unwrap(); 37 | /// let rb2_bytes = Cursor::new(bytes); 38 | /// 39 | /// assert_eq!( 40 | /// rb1.intersection_with_serialized_unchecked(rb2_bytes).unwrap(), 41 | /// rb1 & rb2, 42 | /// ); 43 | /// ``` 44 | pub fn intersection_with_serialized_unchecked(&self, other: R) -> io::Result 45 | where 46 | R: io::Read + io::Seek, 47 | { 48 | RoaringBitmap::intersection_with_serialized_impl::( 49 | self, 50 | other, 51 | |values| Ok(ArrayStore::from_vec_unchecked(values)), 52 | |len, values| Ok(BitmapStore::from_unchecked(len, values)), 53 | ) 54 | } 55 | 56 | fn intersection_with_serialized_impl( 57 | &self, 58 | mut reader: R, 59 | a: A, 60 | b: B, 61 | ) -> io::Result 62 | where 63 | R: io::Read + io::Seek, 64 | A: Fn(Vec) -> Result, 65 | AErr: Error + Send + Sync + 'static, 66 | B: Fn(u64, Box<[u64; 1024]>) -> Result, 67 | BErr: Error + Send + Sync + 'static, 68 | { 69 | // First read the cookie to determine which version of the format we are reading 70 | let (size, has_offsets, has_run_containers) = { 71 | let cookie = reader.read_u32::()?; 72 | if cookie == SERIAL_COOKIE_NO_RUNCONTAINER { 73 | (reader.read_u32::()? as usize, true, false) 74 | } else if (cookie as u16) == SERIAL_COOKIE { 75 | let size = ((cookie >> 16) + 1) as usize; 76 | (size, size >= NO_OFFSET_THRESHOLD, true) 77 | } else { 78 | return Err(io::Error::other("unknown cookie value")); 79 | } 80 | }; 81 | 82 | // Read the run container bitmap if necessary 83 | let run_container_bitmap = if has_run_containers { 84 | let mut bitmap = vec![0u8; size.div_ceil(8)]; 85 | reader.read_exact(&mut bitmap)?; 86 | Some(bitmap) 87 | } else { 88 | None 89 | }; 90 | 91 | if size > u16::MAX as usize + 1 { 92 | return Err(io::Error::other("size is greater than supported")); 93 | } 94 | 95 | // Read the container descriptions 96 | let mut descriptions = vec![[0; 2]; size]; 97 | reader.read_exact(cast_slice_mut(&mut descriptions))?; 98 | descriptions.iter_mut().for_each(|[ref mut key, ref mut len]| { 99 | *key = u16::from_le(*key); 100 | *len = u16::from_le(*len); 101 | }); 102 | 103 | if has_offsets { 104 | let mut offsets = vec![0; size]; 105 | reader.read_exact(cast_slice_mut(&mut offsets))?; 106 | offsets.iter_mut().for_each(|offset| *offset = u32::from_le(*offset)); 107 | return self.intersection_with_serialized_impl_with_offsets( 108 | reader, 109 | a, 110 | b, 111 | &descriptions, 112 | &offsets, 113 | run_container_bitmap.as_deref(), 114 | ); 115 | } 116 | 117 | // Read each container and skip the useless ones 118 | let mut containers = Vec::new(); 119 | for (i, &[key, len_minus_one]) in descriptions.iter().enumerate() { 120 | let container = match self.containers.binary_search_by_key(&key, |c| c.key) { 121 | Ok(index) => self.containers.get(index), 122 | Err(_) => None, 123 | }; 124 | let cardinality = u64::from(len_minus_one) + 1; 125 | 126 | // If the run container bitmap is present, check if this container is a run container 127 | let is_run_container = 128 | run_container_bitmap.as_ref().is_some_and(|bm| bm[i / 8] & (1 << (i % 8)) != 0); 129 | 130 | let store = if is_run_container { 131 | let runs = reader.read_u16::()?; 132 | match container { 133 | Some(_) => { 134 | let mut intervals = vec![[0, 0]; runs as usize]; 135 | reader.read_exact(cast_slice_mut(&mut intervals))?; 136 | intervals.iter_mut().for_each(|[s, len]| { 137 | *s = u16::from_le(*s); 138 | *len = u16::from_le(*len); 139 | }); 140 | 141 | let cardinality = intervals.iter().map(|[_, len]| *len as usize).sum(); 142 | let mut store = Store::with_capacity(cardinality); 143 | intervals.into_iter().try_for_each( 144 | |[s, len]| -> Result<(), io::ErrorKind> { 145 | let end = s.checked_add(len).ok_or(io::ErrorKind::InvalidData)?; 146 | store.insert_range(RangeInclusive::new(s, end)); 147 | Ok(()) 148 | }, 149 | )?; 150 | store 151 | } 152 | None => { 153 | let runs_size = mem::size_of::() * 2 * runs as usize; 154 | reader.seek(SeekFrom::Current(runs_size as i64))?; 155 | continue; 156 | } 157 | } 158 | } else if cardinality <= ARRAY_LIMIT { 159 | match container { 160 | Some(_) => { 161 | let mut values = vec![0; cardinality as usize]; 162 | reader.read_exact(cast_slice_mut(&mut values))?; 163 | values.iter_mut().for_each(|n| *n = u16::from_le(*n)); 164 | let array = 165 | a(values).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 166 | Store::Array(array) 167 | } 168 | None => { 169 | let array_size = mem::size_of::() * cardinality as usize; 170 | reader.seek(SeekFrom::Current(array_size as i64))?; 171 | continue; 172 | } 173 | } 174 | } else { 175 | match container { 176 | Some(_) => { 177 | let mut values = Box::new([0; BITMAP_LENGTH]); 178 | reader.read_exact(cast_slice_mut(&mut values[..]))?; 179 | values.iter_mut().for_each(|n| *n = u64::from_le(*n)); 180 | let bitmap = b(cardinality, values) 181 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 182 | Store::Bitmap(bitmap) 183 | } 184 | None => { 185 | let bitmap_size = mem::size_of::() * BITMAP_LENGTH; 186 | reader.seek(SeekFrom::Current(bitmap_size as i64))?; 187 | continue; 188 | } 189 | } 190 | }; 191 | 192 | if let Some(container) = container { 193 | let mut other_container = Container { key, store }; 194 | other_container &= container; 195 | if !other_container.is_empty() { 196 | containers.push(other_container); 197 | } 198 | } 199 | } 200 | 201 | Ok(RoaringBitmap { containers }) 202 | } 203 | 204 | fn intersection_with_serialized_impl_with_offsets( 205 | &self, 206 | mut reader: R, 207 | a: A, 208 | b: B, 209 | descriptions: &[[u16; 2]], 210 | offsets: &[u32], 211 | run_container_bitmap: Option<&[u8]>, 212 | ) -> io::Result 213 | where 214 | R: io::Read + io::Seek, 215 | A: Fn(Vec) -> Result, 216 | AErr: Error + Send + Sync + 'static, 217 | B: Fn(u64, Box<[u64; 1024]>) -> Result, 218 | BErr: Error + Send + Sync + 'static, 219 | { 220 | let mut containers = Vec::new(); 221 | for container in &self.containers { 222 | let i = match descriptions.binary_search_by_key(&container.key, |[k, _]| *k) { 223 | Ok(index) => index, 224 | Err(_) => continue, 225 | }; 226 | 227 | // Seek to the bytes of the container we want. 228 | reader.seek(SeekFrom::Start(offsets[i] as u64))?; 229 | 230 | let [key, len_minus_one] = descriptions[i]; 231 | let cardinality = u64::from(len_minus_one) + 1; 232 | 233 | // If the run container bitmap is present, check if this container is a run container 234 | let is_run_container = 235 | run_container_bitmap.as_ref().is_some_and(|bm| bm[i / 8] & (1 << (i % 8)) != 0); 236 | 237 | let store = if is_run_container { 238 | let runs = reader.read_u16::().unwrap(); 239 | let mut intervals = vec![[0, 0]; runs as usize]; 240 | reader.read_exact(cast_slice_mut(&mut intervals)).unwrap(); 241 | intervals.iter_mut().for_each(|[s, len]| { 242 | *s = u16::from_le(*s); 243 | *len = u16::from_le(*len); 244 | }); 245 | 246 | let cardinality = intervals.iter().map(|[_, len]| *len as usize).sum(); 247 | let mut store = Store::with_capacity(cardinality); 248 | intervals.into_iter().try_for_each(|[s, len]| -> Result<(), io::ErrorKind> { 249 | let end = s.checked_add(len).ok_or(io::ErrorKind::InvalidData)?; 250 | store.insert_range(RangeInclusive::new(s, end)); 251 | Ok(()) 252 | })?; 253 | store 254 | } else if cardinality <= ARRAY_LIMIT { 255 | let mut values = vec![0; cardinality as usize]; 256 | reader.read_exact(cast_slice_mut(&mut values)).unwrap(); 257 | values.iter_mut().for_each(|n| *n = u16::from_le(*n)); 258 | let array = a(values).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 259 | Store::Array(array) 260 | } else { 261 | let mut values = Box::new([0; BITMAP_LENGTH]); 262 | reader.read_exact(cast_slice_mut(&mut values[..])).unwrap(); 263 | values.iter_mut().for_each(|n| *n = u64::from_le(*n)); 264 | let bitmap = b(cardinality, values) 265 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; 266 | Store::Bitmap(bitmap) 267 | }; 268 | 269 | let mut other_container = Container { key, store }; 270 | other_container &= container; 271 | if !other_container.is_empty() { 272 | containers.push(other_container); 273 | } 274 | } 275 | 276 | Ok(RoaringBitmap { containers }) 277 | } 278 | } 279 | 280 | #[cfg(test)] 281 | mod test { 282 | use crate::RoaringBitmap; 283 | use proptest::prelude::*; 284 | use std::io::Cursor; 285 | 286 | // fast count tests 287 | proptest! { 288 | #[test] 289 | fn intersection_with_serialized_eq_materialized_intersection( 290 | a in RoaringBitmap::arbitrary(), 291 | b in RoaringBitmap::arbitrary() 292 | ) { 293 | let mut serialized_bytes_b = Vec::new(); 294 | b.serialize_into(&mut serialized_bytes_b).unwrap(); 295 | let serialized_bytes_b = &serialized_bytes_b[..]; 296 | 297 | prop_assert_eq!(a.intersection_with_serialized_unchecked(Cursor::new(serialized_bytes_b)).unwrap(), a & b); 298 | } 299 | } 300 | } 301 | --------------------------------------------------------------------------------