├── .github └── workflows │ ├── ci.yml │ ├── tests.yml │ └── tests_impl.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── README.md ├── bench.bat ├── benches ├── common.rs ├── insert.rs ├── intersection.rs ├── iteration.rs └── union.rs ├── doc.bat ├── doc ├── hisparsebitset-50.png ├── hisparsebitset-bg-white-50.png ├── hisparsebitset-bg-white.png ├── hisparsebitset-dark-50.png ├── hisparsebitset-dark.png └── hisparsebitset.png ├── examples ├── custom_bitset.rs ├── custom_bitset_simple.rs ├── traverse_w_cursor.rs └── usage.rs ├── src ├── apply.rs ├── bit_block.rs ├── bit_queue.rs ├── bit_utils.rs ├── bitset.rs ├── bitset_interface.rs ├── block.rs ├── cache.rs ├── compact_block.rs ├── config.rs ├── derive_raw.rs ├── internals.rs ├── iter │ ├── caching.rs │ ├── mod.rs │ └── simple.rs ├── level.rs ├── lib.rs ├── ops.rs ├── primitive.rs ├── primitive_array.rs ├── raw.rs ├── reduce.rs ├── small_bitset.rs └── test.rs ├── test_128.bat ├── test_256.bat └── test_64.bat /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main, dev ] 7 | pull_request: 8 | branches: [ main, dev ] 9 | 10 | env: 11 | CARGO_TERM_COLOR: always 12 | 13 | jobs: 14 | build: 15 | name: build 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - run: cargo build --no-default-features 20 | - run: RUSTFLAGS="--deny warnings" cargo build 21 | - run: RUSTFLAGS="--deny warnings" cargo build --all-features 22 | 23 | tests: 24 | uses: ./.github/workflows/tests.yml 25 | with: 26 | rustflags: '--deny warnings' 27 | 28 | benchmarks: 29 | name: benchmarks build 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v4 33 | - run: RUSTFLAGS="--deny warnings" cargo build --benches --all-features 34 | 35 | doc: 36 | name: doc build 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v4 40 | - run: RUSTFLAGS="--deny warnings" cargo doc --lib --all-features 41 | 42 | docrs: 43 | name: docrs build 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: dtolnay/rust-toolchain@nightly 47 | - uses: actions/checkout@v4 48 | - run: 49 | RUSTFLAGS="--deny warnings" 50 | RUSTDOCFLAGS="--cfg docsrs" 51 | cargo +nightly doc --lib --all-features -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | workflow_dispatch: 5 | workflow_call: 6 | inputs: 7 | rustflags: 8 | required: false 9 | type: string 10 | 11 | env: 12 | CARGO_TERM_COLOR: always 13 | 14 | jobs: 15 | tests: 16 | strategy: 17 | matrix: 18 | cache_type_flag: [ 19 | hisparsebitset_test_NoCache, 20 | hisparsebitset_test_FixedCache, 21 | hisparsebitset_test_DynamicCache 22 | ] 23 | config_type_flag: [ 24 | hisparsebitset_test_64, 25 | hisparsebitset_test_128, 26 | hisparsebitset_test_256 27 | ] 28 | bitset: [ 29 | hisparsebitset_test_bitset, 30 | hisparsebitset_test_smallbitset 31 | ] 32 | uses: ./.github/workflows/tests_impl.yml 33 | with: 34 | rustflags: ${{ inputs.rustflags }} 35 | cache_type_flag: ${{ matrix.cache_type_flag }} 36 | config_type_flag: ${{ matrix.config_type_flag }} 37 | bitset: ${{ matrix.bitset }} -------------------------------------------------------------------------------- /.github/workflows/tests_impl.yml: -------------------------------------------------------------------------------- 1 | name: tests_impl 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | rustflags: 7 | required: false 8 | type: string 9 | cache_type_flag: 10 | required: true 11 | type: string 12 | config_type_flag: 13 | required: true 14 | type: string 15 | bitset: 16 | required: true 17 | type: string 18 | 19 | env: 20 | CARGO_TERM_COLOR: always 21 | 22 | jobs: 23 | tests: 24 | name: careful tests 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: dtolnay/rust-toolchain@nightly 28 | - uses: taiki-e/install-action@v2 29 | with: 30 | tool: cargo-careful 31 | - uses: actions/checkout@v4 32 | - run: RUSTFLAGS=" 33 | ${{ inputs.rustflags }} 34 | --cfg ${{ inputs.cache_type_flag }} 35 | --cfg ${{ inputs.config_type_flag }} 36 | --cfg ${{ inputs.bitset }} 37 | " cargo +nightly careful test --all-features 38 | 39 | miri: 40 | name: miri tests 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: dtolnay/rust-toolchain@nightly 44 | with: 45 | toolchain: nightly 46 | components: miri 47 | - uses: taiki-e/install-action@v2 48 | with: 49 | tool: cargo-nextest 50 | - uses: actions/checkout@v4 51 | - run: RUSTFLAGS=" 52 | ${{ inputs.rustflags }} 53 | --cfg ${{ inputs.cache_type_flag }} 54 | --cfg ${{ inputs.config_type_flag }} 55 | --cfg ${{ inputs.bitset }} 56 | " cargo +nightly miri nextest run -j6 --all-features -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /.idea 4 | /.vscode -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.6.1 4 | ### Fix 5 | - `is_empty()` worked wrong in non-`TRUSTED_HIERARCHY`. 6 | 7 | ## 0.6.0 8 | ### Changed 9 | - `config::Config::max_value()` moved to `BitSet::max_capacity()`. 10 | - `config::Config` changed (simplified). 11 | 12 | ### Added 13 | - `SmallBitset` - `BitSet` with minimized memory overhead of hierarchy indirection. 14 | - `internals::PrimitiveArray`. 15 | 16 | ## 0.5.1 17 | ### Fix 18 | - Intersection operation `ops::And` does **not** produce `TRUSTED_HIERARCHY`. 19 | 20 | ## 0.5.0 21 | ### Fix 22 | - `NoCache` reduce version was unsound for iterators which had to be dropped. 23 | 24 | ### Optimization 25 | - On each level, instead of empty block indices Vec, an intrusive single-linked list is now used. 26 | This completely eliminates this kind of memory overhead. Previously, if you would fill `_256bit` bitset, 27 | and then clear it - you would end up with additional 132Kb memory overhead from the list of free blocks. 28 | Considering that preallocated bitblocks themselves took 2Mb, this saves more than 5% of memory. 29 | - Minor `BitSet::remove()` optimization. 30 | 31 | ### Changed 32 | - `BitSetInterface` now have default implementation. 33 | - `BitSet` no longer implements `BitSetInterface`. 34 | But `&BitSet` still does. This prevents accidental sending container by value. 35 | - `config::with_cache::*` moved to `config::*` with additional default generic argument. 36 | - `crate::bit_queue` moved to `internals::bit_queue`. 37 | - `crate::Primitive` moved to `internals::Primitive`. 38 | - `Config` bitblocks no longer require `Default`. 39 | 40 | ### Added 41 | - `BitBlock::as_array()`. 42 | - `BitBlock::as_array_mut()`. 43 | - Some `BitBlock` methods now have default implementations. 44 | - `BitSetOp::HIERARCHY_OPERANDS_CONTAIN_RESULT` marker, for intersection-like 45 | optimization in user-defined operations. 46 | - Machinery, which allows to implement custom bitsets. Enabled with `impl` flag. 47 | - `internals` module, with implementation details that end user can use for deep customization. 48 | 49 | ### Removed 50 | - `num_traits` dependency removed. 51 | 52 | ## 0.4.0 53 | ### Fix 54 | - `Eq` did not work correctly between !`TrustedHierarchy` bitsets. 55 | 56 | ### Changed 57 | - `BitSetInterface` changed (simplified). 58 | - `BitSetOp` renamed to `Apply`. 59 | - `BinaryOp` renamed to `BitSetOp`. 60 | - `binary_op` module renamed to `ops`. 61 | - All former `binary_op` operations renamed. 62 | 63 | ### Added 64 | - `BitSet`, `BitSetOp`, `Reduce` now duplicate part of `BitSetInterface` in 65 | order to prevent the need of `BitSetInterface` import. 66 | - `CachingIndexIter` now have former `IndexIterator` functionality. 67 | - `CachingBlockIter` now have former `BlockIterator` functionality. 68 | - `BitSetInterface::is_empty()`. 69 | - `BitSetBase::TRUSTED_HIERARCHY`. 70 | 71 | ### Removed 72 | - `IndexIterator` removed. 73 | - `BlockIterator` removed. 74 | 75 | ## 0.3.0 76 | ### Fix 77 | - `IndexIter::move_to` to the empty bitset area fix. 78 | 79 | ### Changed 80 | - General minor performance improvement (removed index check in bit-manipulation). 81 | - `BitSetInterface`'s `IntoIterator` now implement `IndexIter`. 82 | - `BlockIterCursor` renamed to `BlockCursor`. 83 | - `IndexIterCursor` renamed to `IndexCursor`. 84 | - `BlockCursor` and `IndexCursor` now have `Conf` generic parameter. 85 | - both cursors now <= 64bit in size. 86 | - `BlockIter::as_indices` renamed to `BlockIter::into_indices`. 87 | 88 | ### Added 89 | - `BlockCursor` now implements `Copy`. 90 | - `IndexCursor` now implements `Copy`. 91 | - `BlockCursor::start()`. 92 | - `BlockCursor::end()`. 93 | - `BlockCursor::from(usize)`. 94 | - `BlockCursor::from(&DataBlock)`. 95 | - `IndexCursor::start()`. 96 | - `IndexCursor::end()`. 97 | - `IndexCursor::from(usize)`. 98 | - `IndexCursor::from(&DataBlock)`. 99 | - `CachingBlockIter` now implements `Clone`. 100 | - `CachingIndexIter` now implements `Clone`. 101 | - `CachingBlockIter::traverse`. 102 | - `CachingIndexIter::traverse`. 103 | - `CachingBlockIter` specialized `for_each` implementation. 104 | - `CachingIndexIter` specialized `for_each` implementation. 105 | - `DataBlockIter` now implements `Clone`. 106 | - `DataBlockIter::traverse`. 107 | - `DataBlockIter` specialized `for_each` implementation. 108 | - `DataBlock` now implements `Eq`. 109 | - All `BitSetInterface`s now implement `Eq`. 110 | - All `BitSetInterface`s now implement `Debug`. 111 | - `BitSet` (without &) now implements `op`s too. 112 | 113 | ### Removed 114 | - `IndexIterator::as_blocks()` removed as redundant, the same can be achieved with cursor move. 115 | And the very need of moving from index iterator to block iterator is questionable. 116 | 117 | ## 0.2.0 118 | ### Changed 119 | - `IConfig` renamed to `Config`. 120 | - `max_range()` moved to `Config`. 121 | 122 | ### Added 123 | - `BitBlock` trait for implementing custom `BitSet` bitblock. 124 | - `_256bit` `Config` added. 125 | 126 | ## 0.1.0 127 | 128 | Initial version -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hi_sparse_bitset" 3 | authors = ["Andrey Diduh "] 4 | license = "MIT OR Apache-2.0" 5 | version = "0.6.1" 6 | edition = "2021" 7 | repository = "https://github.com/tower120/hi_sparse_bitset" 8 | keywords = ["hierarchical", "sparse", "bitset", "bitmap", "container"] 9 | categories = ["data-structures"] 10 | description = "Hierarchical sparse bitset. Incredibly high performance. Compact memory usage." 11 | exclude = ["/doc", "/.github"] 12 | 13 | [features] 14 | default = ["simd"] 15 | # Makes LevelMasks, LevelMasksIterExt and impl_bitset! visible. 16 | # Having them hidden by default prevents your code completion tool 17 | # from showing you irrelevant implementation methods 18 | # (even if you did not import them). 19 | impl = [] 20 | # You don't need this. Original legacy iterator. 21 | simple_iter = [] 22 | # simd bitblocks support (128,256 bit configs) 23 | simd = ["dep:wide"] 24 | 25 | [dependencies] 26 | wide = { version = "0.7.13", optional = true } 27 | 28 | [dev-dependencies] 29 | rand = "0.8" 30 | itertools = "0.11" 31 | criterion = "0.5.1" 32 | cfg-if = "1" 33 | # for benchmarks 34 | hibitset = "0.6.4" 35 | roaring = "0.10.2" 36 | 37 | [lints.rust.unexpected_cfgs] 38 | level = "warn" 39 | check-cfg = [ 40 | 'cfg(hisparsebitset_test_NoCache)', 41 | 'cfg(hisparsebitset_test_FixedCache)', 42 | 'cfg(hisparsebitset_test_DynamicCache)', 43 | 44 | 'cfg(hisparsebitset_test_64)', 45 | 'cfg(hisparsebitset_test_128)', 46 | 'cfg(hisparsebitset_test_256)', 47 | 48 | 'cfg(hisparsebitset_test_bitset)', 49 | 'cfg(hisparsebitset_test_smallbitset)', 50 | ] 51 | 52 | [package.metadata.docs.rs] 53 | features = ["impl"] 54 | rustdoc-args = ["--cfg", "docsrs"] 55 | 56 | [[example]] 57 | name = "custom_bitset" 58 | required-features = ["impl"] 59 | 60 | [[example]] 61 | name = "custom_bitset_simple" 62 | required-features = ["impl"] 63 | 64 | [[bench]] 65 | name = "iteration_bench" 66 | path = "benches/iteration.rs" 67 | harness = false 68 | 69 | [[bench]] 70 | name = "intersection_bench" 71 | path = "benches/intersection.rs" 72 | harness = false 73 | 74 | [[bench]] 75 | name = "union_bench" 76 | path = "benches/union.rs" 77 | harness = false 78 | 79 | [[bench]] 80 | name = "insert_bench" 81 | path = "benches/insert.rs" 82 | harness = false 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hierarchical sparse bitset 2 | 3 | [![crates.io](https://img.shields.io/crates/v/hi_sparse_bitset.svg)](https://crates.io/crates/hi_sparse_bitset) 4 | [![license](https://img.shields.io/badge/license-Apache--2.0_OR_MIT-blue?style=flat-square)](#license) 5 | [![Docs](https://docs.rs/hi_sparse_bitset/badge.svg)](https://docs.rs/hi_sparse_bitset) 6 | [![CI](https://github.com/tower120/hi_sparse_bitset/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/tower120/hi_sparse_bitset/actions/workflows/ci.yml) 7 | 8 | High performance of operations between bitsets (intersection, union, etc.). 9 | Low memory usage. 10 | 11 | Think of [hibitset](https://crates.io/crates/hibitset), but with lower memory consumption. 12 | Unlike hibitset - it is actually sparse - it's memory usage does not depend on max index in set. 13 | Only amount of used bitblocks matters (or elements, to put it simply). 14 | And like hibitset, it also utilizes hierarchical bitmask acceleration structure to reduce 15 | algorithmic complexity on operations between bitsets. 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | ## Usage 24 | 25 | ```rust 26 | use hi_sparse_bitset::reduce; 27 | use hi_sparse_bitset::ops::*; 28 | type BitSet = hi_sparse_bitset::BitSet; 29 | 30 | let bitset1 = BitSet::from([1,2,3,4]); 31 | let bitset2 = BitSet::from([3,4,5,6]); 32 | let bitset3 = BitSet::from([3,4,7,8]); 33 | let bitset4 = BitSet::from([4,9,10]); 34 | let bitsets = [bitset1, bitset2, bitset3]; 35 | 36 | // reduce on bitsets iterator 37 | let intersection = reduce(And, bitsets.iter()).unwrap(); 38 | assert_equal(&intersection, [3,4]); 39 | 40 | // operation between different types 41 | let union = intersection | &bitset4; 42 | assert_equal(&union, [3,4,9,10]); 43 | 44 | // partially traverse iterator, and save position to cursor. 45 | let mut iter = union.iter(); 46 | assert_equal(iter.by_ref().take(2), [3,4]); 47 | let cursor = iter.cursor(); 48 | 49 | // resume iteration from cursor position 50 | let iter = union.iter().move_to(cursor); 51 | assert_equal(iter, [9,10]); 52 | ``` 53 | 54 | ## Memory footprint 55 | 56 | Being truly sparse, `hi_sparse_bitset` allocate memory only for bitblocks in use. 57 | `hi_sparse_bitset::BitSet` has tri-level hierarchy, with first and second levels 58 | containing bit-masks and indirection information, and third level - actual bit data. 59 | Currently, whole first level (which is one block itself) and one block from the 60 | second level are always allocated. 61 | 62 | Hierarchy-wise memory overhead, for `config::_128bit`: 63 | minimal(initial) = 416 bytes, maximum = 35 Kb. 64 | 65 | ### SmallBitSet 66 | 67 | `hi_sparse_bitset::SmallBitSet` instead of full-sized array for block pointers 68 | use technique we call "SparseBitMap", which allows to store pointers only to 69 | non-empty blocks. 70 | Thou, this tehcnique introduce some additional performance overhead, 71 | all operations still have O(1) complexity, like `BitSet`. 72 | 73 | ## Performance 74 | 75 | It is faster than hashsets and pure bitsets for all inter-bitset operations 76 | and all cases in orders of magnitude. It is even faster than 77 | hibitset. See benchmarks. 78 | 79 | ### Against `hibitset` 80 | 81 | Despite the fact that `hi_sparse_bitset` have layer of indirection for accessing 82 | each level, it is faster (sometimes significantly) then `hibitset` for all operations. 83 | 84 | On top of that, it is also **algorithmically** faster than `hibitset` in 85 | non-intersection inter-bitset operations due to caching iterator, which 86 | can skip bitsets with empty level1 blocks. 87 | 88 | ### Against `roaring` 89 | 90 | `roaring` is a hybrid bitset, that use sorted array of bitblocks for set with large integers, 91 | and big fixed-sized bitset for a small ones. 92 | Let's consider case for intersecting `roaring` sets, that contain large integers. 93 | In order to find intersection, it binary search for bitblocks with the same start index, 94 | then intersect each bitblock. Operation of binary searching matching bitblock 95 | is algorithmically more complex O(log N), then directly traversing intersected 96 | bitblock in hierarchy, which is close to O(1) for each resulted bitblock. 97 | Plus, hierarchical bitset discard groups of non-intersected blocks 98 | early, due to its tree-like nature. 99 | 100 | ## DataBlock operations 101 | 102 | In order to speed up things even more, you can work directly with 103 | `DataBlock`s. `DataBlock`s - is a bit-blocks (relatively small in size), 104 | which you can store and iterate latter. 105 | 106 | _In future versions, you can also insert DataBlocks into BitSet._ 107 | 108 | ## Reduce on iterator of bitsets 109 | 110 | In addition to "the usual" bitset-to-bitset(binary) operations, 111 | you can apply operation to iterator of bitsets (reduce/fold). 112 | In this way, you not only apply operation to the arbitrary 113 | number of bitsets, but also have the same result type, 114 | for any bitsets count. Which allows to have nested reduce 115 | operations. 116 | 117 | ## Ordered/sorted 118 | 119 | Iteration always return sorted sequences. 120 | 121 | ## Suspend-resume iterator with cursor 122 | 123 | Iterators of `BitSetInterface` (any kind of bitset) can return cursor, 124 | and can rewind to cursor. Cursor is like integer index in `Vec`. 125 | Which means, that you can use it even if container was mutated. 126 | 127 | ### Multi-session iteration 128 | 129 | With cursor you can suspend and later resume your iteration 130 | session. For example, you can create an intersection between several bitsets, iterate it 131 | to a certain point, and obtain an iterator cursor. Then, later, 132 | you can make an intersection between the same bitsets (but possibly in different state), 133 | and resume iteration from the last point you stopped, using cursor. 134 | 135 | ### Thread safe multi-session iteration 136 | 137 | You can use "multi-session iteration" in multithreaded env too. 138 | _(By wrapping bitsets in Mutex(es))_ 139 | 140 | #### Invariant intersection 141 | 142 | If intersection of bitsets _(or any other operation)_ does not change with possible bitsets mutations - you're guaranteed to correctly traverse all of its elements. 143 | 144 | #### Bitsets mutations narrows intersection/union 145 | 146 | If in intersection, only `remove` operation mutates bitsets - this guarantees that you will not loose any valid elements at the end of "multi-session iteration". 147 | 148 | #### Speculative iteration 149 | 150 | For other cases - you're guaranteed to proceed forward, without repeated elements. 151 | _(In each iteration session you'll see initial valid elements + some valid new ones)_ 152 | You can use this if you don't need to traverse EXACT intersection. For example, if you 153 | process intersection of the same bitsets over and over in a loop. 154 | 155 | ## Changelog 156 | 157 | See [CHANGELOG.md](CHANGELOG.md) for version differences. 158 | 159 | ## Known alternatives 160 | 161 | * [hibitset](https://crates.io/crates/hibitset) - hierarchical dense bitset. 162 | If you'll insert one index = 16_000_000, it will allocate 2Mb of RAM. 163 | It uses 4-level hierarchy, and being dense - does not use indirection. 164 | This makes it hierarchy overhead smaller, and on intersection operations it SHOULD perform 165 | better - but it doesn't (probably because of additional level of hierarchy, or some 166 | implementation details). 167 | 168 | * [bitvec](https://crates.io/crates/bitvec) - pure dense bitset. Plain operations (insert/contains) 169 | should be reasonably faster (not at magnitude scale). 170 | Inter-bitset operations - super-linearly slower for the worst case (which is almost always), 171 | and have approx. same performance for the best case (when each bitset block used). 172 | Have no memory overhead per-se, but memory usage depends on max int in bitset, 173 | so if you do not need to perform inter-bitset operations, 174 | and know that your indices are relatively small numbers, or expect bitset to be 175 | densely populated - this is a good choice. 176 | 177 | * `HashSet` - you should use it only if you work with a relatively small 178 | set with extremely large numbers. 179 | It is orders of magnitude slower for inter-set operations. 180 | And "just" slower for the rest ones. 181 | 182 | * [roaring](https://crates.io/crates/roaring) - compressed hybrid bitset. 183 | Higher algorithmic complexity of operations, but theoretically unlimited range. 184 | It is still super-linearly faster than pure dense bitsets and hashsets in inter-set 185 | operations. See [performance section](#against-roaring) for detais. -------------------------------------------------------------------------------- /bench.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | set RUSTFLAGS=-C target-feature=+popcnt,+bmi1 4 | cargo bench --bench %1 --all-features 5 | endlocal -------------------------------------------------------------------------------- /benches/common.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, black_box}; 2 | use criterion::measurement::Measurement; 3 | 4 | pub fn bench<'a, M, P, I, F, R>( 5 | group: &mut criterion::BenchmarkGroup<'a, M>, 6 | case: &str, 7 | param: P, 8 | input: &I, 9 | f: F 10 | ) where 11 | M: Measurement, 12 | P: std::fmt::Display, 13 | I: ?Sized, 14 | F: Fn(&I) -> R, 15 | { 16 | group.bench_with_input( 17 | BenchmarkId::new(case, param), 18 | input, 19 | |b, i| b.iter(|| f(black_box(i))) 20 | ); 21 | } -------------------------------------------------------------------------------- /benches/insert.rs: -------------------------------------------------------------------------------- 1 | use criterion::{black_box, Criterion, criterion_group, criterion_main}; 2 | 3 | type HiSparseBitset = hi_sparse_bitset::SmallBitSet; 4 | 5 | fn hi_sparse_bitset_insert(in_block: usize) -> HiSparseBitset{ 6 | let mut set: HiSparseBitset = Default::default(); 7 | for lvl0 in 0..128 { 8 | for lvl1 in 0..6 { 9 | let offset = lvl0*128*128 + lvl1*128; 10 | for i in 0..in_block{ 11 | set.insert(offset + i); 12 | } 13 | } 14 | } 15 | set 16 | } 17 | 18 | pub fn bench_iter(c: &mut Criterion) { 19 | c.bench_function("hi_sparse_bitset insert", |b| b.iter(|| hi_sparse_bitset_insert(black_box(80)))); 20 | } 21 | 22 | criterion_group!(benches_iter, bench_iter); 23 | criterion_main!(benches_iter); -------------------------------------------------------------------------------- /benches/intersection.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_imports)] 2 | 3 | mod common; 4 | 5 | use std::ops::ControlFlow; 6 | use std::collections::HashSet; 7 | use criterion::{AxisScale, Criterion, criterion_group, criterion_main, PlotConfiguration}; 8 | use hi_sparse_bitset::{BitSet, BitSetInterface, reduce}; 9 | use hi_sparse_bitset::ops::And; 10 | use hi_sparse_bitset::iter::{BlockCursor, IndexCursor, SimpleBlockIter, SimpleIndexIter}; 11 | use ControlFlow::*; 12 | use criterion::measurement::Measurement; 13 | use roaring::RoaringBitmap; 14 | use hi_sparse_bitset::config::Config; 15 | use crate::common::bench; 16 | 17 | // TODO: consider bench different Cache modes instead. 18 | 19 | // ---- REDUCE ----- 20 | // === Block iter === 21 | fn hi_sparse_bitset_reduce_and_simple_block_iter(sets: &[BitSet]) -> usize { 22 | let reduce = reduce(And, sets.iter()).unwrap(); 23 | SimpleBlockIter::new(reduce).count() 24 | } 25 | 26 | fn hi_sparse_bitset_reduce_and_caching_block_iter(sets: &[BitSet]) -> usize { 27 | let reduce = reduce(And, sets.iter()).unwrap(); 28 | reduce.into_block_iter().count() 29 | } 30 | 31 | // === Traverse === 32 | fn hi_sparse_bitset_reduce_and_simple_traverse(sets: &[BitSet]) -> usize { 33 | let reduce = reduce(And, sets.iter()).unwrap(); 34 | 35 | let mut counter = 0; 36 | for block in SimpleBlockIter::new(reduce) { 37 | block.traverse(|_|{ 38 | counter += 1; 39 | Continue(()) 40 | }); 41 | } 42 | counter 43 | } 44 | 45 | fn hi_sparse_bitset_reduce_and_caching_traverse(sets: &[BitSet]) -> usize { 46 | let reduce = reduce(And, sets.iter()).unwrap(); 47 | 48 | let mut counter = 0; 49 | reduce.iter().traverse(|_|{ 50 | counter += 1; 51 | Continue(()) 52 | }); 53 | counter 54 | } 55 | 56 | // === Iter === 57 | fn hi_sparse_bitset_reduce_and_simple_iter(sets: &[BitSet]) -> usize { 58 | let reduce = reduce(And, sets.iter()).unwrap(); 59 | SimpleIndexIter::new(SimpleBlockIter::new(reduce)).count() 60 | } 61 | 62 | fn hi_sparse_bitset_reduce_and_caching_iter(sets: &[BitSet]) -> usize { 63 | let reduce = reduce(And, sets.iter()).unwrap(); 64 | reduce.into_iter().count() 65 | } 66 | 67 | 68 | // ---- OP ----- 69 | // === Block iter === 70 | fn hi_sparse_bitset_op_and_simple_block_iter(sets: &[BitSet]) -> usize{ 71 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 72 | SimpleBlockIter::new(intersection).count() 73 | } 74 | 75 | fn hi_sparse_bitset_op_and_caching_block_iter(sets: &[BitSet]) -> usize{ 76 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 77 | intersection.into_block_iter().count() 78 | } 79 | 80 | // === Traverse === 81 | fn hi_sparse_bitset_op_and_simple_traverse(sets: &[BitSet]) -> usize { 82 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 83 | 84 | let mut counter = 0; 85 | for block in SimpleBlockIter::new(intersection) { 86 | block.traverse(|_|{ 87 | counter += 1; 88 | Continue(()) 89 | }); 90 | } 91 | counter 92 | } 93 | 94 | fn hi_sparse_bitset_op_and_caching_traverse(sets: &[BitSet]) -> usize { 95 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 96 | 97 | let mut counter = 0; 98 | intersection.iter().traverse(|_|{ 99 | counter += 1; 100 | Continue(()) 101 | }); 102 | counter 103 | } 104 | 105 | // === Iter === 106 | fn hi_sparse_bitset_op_and_simple_iter(sets: &[BitSet]) -> usize { 107 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 108 | SimpleIndexIter::new(SimpleBlockIter::new(intersection)).count() 109 | } 110 | 111 | fn hi_sparse_bitset_op_and_caching_iter(sets: &[BitSet]) -> usize { 112 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 113 | intersection.into_iter().count() 114 | } 115 | 116 | fn hibitset_intersection(sets: &[hibitset::BitSet]) -> usize{ 117 | // Looks like this is the best possible way of doing multi intersection with hibitset. 118 | let intersection = &sets[0] & &sets[1] & &sets[2] & &sets[3] & &sets[4]; 119 | 120 | let mut counter = 0; 121 | for _ in intersection{ 122 | counter += 1; 123 | } 124 | counter 125 | } 126 | 127 | fn hashset_intersection(sets: &[HashSet]) -> usize { 128 | let mut counter = 0; 129 | 130 | let (first, other) = sets.split_first().unwrap(); 131 | for i in first.iter(){ 132 | let mut intersects = true; 133 | for o in other{ 134 | if !o.contains(i){ 135 | intersects = false; 136 | break; 137 | } 138 | } 139 | 140 | if intersects{ 141 | counter += 1; 142 | } 143 | } 144 | 145 | counter 146 | } 147 | 148 | fn roaring_intersection(roarings: &[RoaringBitmap]) -> usize{ 149 | // There is no equivalent in RoaringBitmap for multiple set intersection. 150 | // Constructing a new one for each intersection would not be fare for the underlying algorithm. 151 | // This is probably the closest one computation-wise, since all input sets actually fully intersects. 152 | 153 | let len = 154 | roarings[0].intersection_len(&roarings[1]) + 155 | roarings[0].intersection_len(&roarings[2]) + 156 | roarings[0].intersection_len(&roarings[3]) + 157 | roarings[0].intersection_len(&roarings[4]); 158 | 159 | len as usize 160 | } 161 | 162 | pub fn bench_iter(c: &mut Criterion) { 163 | type HiSparseBitset = hi_sparse_bitset::BitSet; 164 | const SETS: usize = 5; 165 | 166 | fn generate_data(size: usize, index_mul: usize, sets: usize) 167 | -> (Vec, Vec, Vec>, Vec) 168 | { 169 | let mut random_indices: Vec> = Default::default(); 170 | for _ in 0..sets{ 171 | random_indices.push(Default::default()); 172 | let random_indices = random_indices.last_mut().unwrap(); 173 | for i in 0..size{ 174 | random_indices.push(i*index_mul); 175 | } 176 | } 177 | 178 | let mut hi_sparse_sets = Vec::new(); 179 | for set_indices in &random_indices{ 180 | let mut set = HiSparseBitset::default(); 181 | for &index in set_indices.iter(){ 182 | set.insert(index); 183 | } 184 | hi_sparse_sets.push(set); 185 | } 186 | 187 | let mut hibitsets = Vec::new(); 188 | for set_indices in &random_indices{ 189 | let mut set = hibitset::BitSet::default(); 190 | for &index in set_indices.iter(){ 191 | set.add(index as _); 192 | } 193 | hibitsets.push(set); 194 | } 195 | 196 | let mut hash_sets = Vec::new(); 197 | for set_indices in &random_indices{ 198 | let mut set: HashSet = HashSet::default(); 199 | for &index in set_indices.iter(){ 200 | set.insert(index); 201 | } 202 | hash_sets.push(set); 203 | } 204 | 205 | let mut roarings = Vec::new(); 206 | for set_indices in &random_indices{ 207 | let mut set: RoaringBitmap = Default::default(); 208 | for &index in set_indices.iter(){ 209 | set.insert(index as u32); 210 | } 211 | roarings.push(set); 212 | } 213 | 214 | (hi_sparse_sets, hibitsets, hash_sets, roarings) 215 | } 216 | 217 | fn do_bench<'a, M: Measurement>(group: &mut criterion::BenchmarkGroup<'a, M>, index_mul: usize){ 218 | let datas = [ 219 | (100, generate_data(100, index_mul, SETS)), 220 | (1000, generate_data(1000, index_mul, SETS)), 221 | (4000, generate_data(4000, index_mul, SETS)), 222 | ]; 223 | 224 | for (name, (hi_sparse_sets, hibitsets, hash_sets, roarings)) in &datas { 225 | let hi_sparse_sets = hi_sparse_sets.as_slice(); 226 | 227 | // ---- REDUCE ---- 228 | // === Block iter === 229 | bench(group, "hi_sparse_bitset_reduce_and_simple_block_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_simple_block_iter); 230 | bench(group, "hi_sparse_bitset_reduce_and_caching_block_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_caching_block_iter); 231 | // === Traverse === 232 | bench(group, "hi_sparse_bitset_reduce_and_simple_traverse", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_simple_traverse); 233 | bench(group, "hi_sparse_bitset_reduce_and_caching_traverse", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_caching_traverse); 234 | // === Iter === 235 | bench(group, "hi_sparse_bitset_reduce_and_simple_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_simple_iter); 236 | bench(group, "hi_sparse_bitset_reduce_and_caching_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_and_caching_iter); 237 | 238 | // ---- OP ---- 239 | // === Block iter === 240 | bench(group, "hi_sparse_bitset_op_and_simple_block_iter", name, hi_sparse_sets, hi_sparse_bitset_op_and_simple_block_iter); 241 | bench(group, "hi_sparse_bitset_op_and_caching_block_iter", name, hi_sparse_sets, hi_sparse_bitset_op_and_caching_block_iter); 242 | // === Traverse === 243 | bench(group, "hi_sparse_bitset_op_and_simple_traverse", name, hi_sparse_sets, hi_sparse_bitset_op_and_simple_traverse); 244 | bench(group, "hi_sparse_bitset_op_and_caching_traverse", name, hi_sparse_sets, hi_sparse_bitset_op_and_caching_traverse); 245 | // === Iter === 246 | bench(group, "hi_sparse_bitset_op_and_simple_iter", name, hi_sparse_sets, hi_sparse_bitset_op_and_simple_iter); 247 | bench(group, "hi_sparse_bitset_op_and_caching_iter", name, hi_sparse_sets, hi_sparse_bitset_op_and_caching_iter); 248 | 249 | // ---- Third party ---- 250 | bench(group, "hibitset_intersection", name, hibitsets.as_slice(), hibitset_intersection); 251 | bench(group, "hashset_intersection", name, hash_sets.as_slice(), hashset_intersection); 252 | bench(group, "roaring_intersection", name, roarings.as_slice(), roaring_intersection); 253 | } 254 | } 255 | 256 | { 257 | let mut group = c.benchmark_group("Intersection - index step 20"); 258 | /*group.plot_config( 259 | PlotConfiguration::default().summary_scale(AxisScale::Logarithmic) 260 | );*/ 261 | do_bench(&mut group, 20); 262 | } 263 | { 264 | let mut group = c.benchmark_group("Intersection - index step 200"); 265 | /*group.plot_config( 266 | PlotConfiguration::default().summary_scale(AxisScale::Logarithmic) 267 | );*/ 268 | do_bench(&mut group, 200); 269 | } 270 | } 271 | 272 | criterion_group!(benches_iter, bench_iter); 273 | criterion_main!(benches_iter); -------------------------------------------------------------------------------- /benches/iteration.rs: -------------------------------------------------------------------------------- 1 | use criterion::{black_box, Criterion, criterion_group, criterion_main}; 2 | 3 | type HiSparseBitset = hi_sparse_bitset::BitSet; 4 | 5 | fn iteration(set: &HiSparseBitset) -> u64{ 6 | let mut s = 0; 7 | for data in set.block_iter(){ 8 | s += data.bit_block; 9 | } 10 | s 11 | } 12 | 13 | pub fn bench_iter(c: &mut Criterion) { 14 | let mut set: HiSparseBitset = Default::default(); 15 | for i in 0..3000{ 16 | set.insert(i*64); 17 | } 18 | 19 | c.bench_function("hi_sparse_bitset iter", |b| b.iter(|| iteration(black_box(&set)))); 20 | } 21 | 22 | criterion_group!(benches_iter, bench_iter); 23 | criterion_main!(benches_iter); -------------------------------------------------------------------------------- /benches/union.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_imports)] 2 | 3 | mod common; 4 | 5 | use criterion::{AxisScale, Criterion, criterion_group, criterion_main, PlotConfiguration}; 6 | use criterion::measurement::Measurement; 7 | use hi_sparse_bitset::{BitSet, BitSetInterface, reduce}; 8 | use hi_sparse_bitset::ops::*; 9 | use hi_sparse_bitset::config::Config; 10 | use hi_sparse_bitset::iter::{SimpleBlockIter, SimpleIndexIter}; 11 | use crate::common::bench; 12 | 13 | // ---- REDUCE ---- 14 | fn hi_sparse_bitset_reduce_or_simple_block_iter(sets: &[BitSet]) -> usize { 15 | let union = reduce(Or, sets.iter()).unwrap(); 16 | SimpleBlockIter::new(union).count() 17 | } 18 | 19 | fn hi_sparse_bitset_reduce_or_caching_block_iter(sets: &[BitSet]) -> usize { 20 | let union = reduce(Or, sets.iter()).unwrap(); 21 | union.into_block_iter().count() 22 | } 23 | 24 | fn hi_sparse_bitset_reduce_or_simple_iter(sets: &[BitSet]) -> usize { 25 | let union = reduce(Or, sets.iter()).unwrap(); 26 | SimpleIndexIter::new(SimpleBlockIter::new(union)).count() 27 | } 28 | 29 | fn hi_sparse_bitset_reduce_or_caching_iter(sets: &[BitSet]) -> usize { 30 | let union = reduce(Or, sets.iter()).unwrap(); 31 | union.into_iter().count() 32 | } 33 | 34 | 35 | // ---- OP ---- 36 | fn hi_sparse_bitset_op_or_simple_block_iter(sets: &[BitSet]) -> usize { 37 | let union = &sets[0] | &sets[1] | &sets[2]; 38 | SimpleBlockIter::new(union).count() 39 | } 40 | 41 | fn hi_sparse_bitset_op_or_caching_block_iter(sets: &[BitSet]) -> usize { 42 | let union = &sets[0] | &sets[1] | &sets[2]; 43 | union.into_block_iter().count() 44 | } 45 | 46 | fn hi_sparse_bitset_op_or_simple_iter(sets: &[BitSet]) -> usize { 47 | let union = &sets[0] | &sets[1] | &sets[2]; 48 | SimpleIndexIter::new(SimpleBlockIter::new(union)).count() 49 | } 50 | 51 | fn hi_sparse_bitset_op_or_caching_iter(sets: &[BitSet]) -> usize { 52 | let union = &sets[0] | &sets[1] | &sets[2]; 53 | union.into_iter().count() 54 | } 55 | 56 | // ---- Third party ---- 57 | fn hibitset_union(sets: &[hibitset::BitSet]) -> usize{ 58 | // Looks like this is the best possible way of doing multi intersection with hibitset. 59 | let intersection = &sets[0] | &sets[1] | &sets[2]; 60 | 61 | let mut counter = 0; 62 | for _ in intersection{ 63 | counter += 1; 64 | } 65 | counter 66 | } 67 | 68 | 69 | /// Bench worst case scenario for hibitset and default iter. 70 | /// All sets does not have intersections. 71 | pub fn bench_iter(c: &mut Criterion) { 72 | type HiSparseBitset = hi_sparse_bitset::BitSet; 73 | const SETS: usize = 3; 74 | 75 | fn generate_data(size: usize, index_mul: usize, sets: usize) -> (Vec, Vec){ 76 | let mut random_indices: Vec> = Default::default(); 77 | for s in 0..sets{ 78 | let offset = s * (size - size/5) * index_mul; 79 | random_indices.push(Default::default()); 80 | let random_indices = random_indices.last_mut().unwrap(); 81 | for i in 0..size{ 82 | random_indices.push(offset + i*index_mul); 83 | } 84 | } 85 | 86 | let mut hi_sparse_sets = Vec::new(); 87 | for set_indices in &random_indices{ 88 | let mut set = HiSparseBitset::default(); 89 | for &index in set_indices.iter(){ 90 | set.insert(index); 91 | } 92 | hi_sparse_sets.push(set); 93 | } 94 | 95 | let mut hibitsets = Vec::new(); 96 | for set_indices in &random_indices{ 97 | let mut set = hibitset::BitSet::default(); 98 | for &index in set_indices.iter(){ 99 | set.add(index as _); 100 | } 101 | hibitsets.push(set); 102 | } 103 | 104 | (hi_sparse_sets, hibitsets) 105 | } 106 | 107 | fn do_bench<'a, M: Measurement>(group: &mut criterion::BenchmarkGroup<'a, M>, index_mul: usize){ 108 | let datas = [ 109 | (100, generate_data(100, index_mul, SETS)), 110 | (1000, generate_data(1000, index_mul, SETS)), 111 | (4000, generate_data(4000, index_mul, SETS)), 112 | ]; 113 | 114 | for (name, (hi_sparse_sets, hibitsets)) in &datas { 115 | let hi_sparse_sets = hi_sparse_sets.as_slice(); 116 | let hibitsets = hibitsets.as_slice(); 117 | 118 | // ---- REDUCE ---- 119 | bench(group, "hi_sparse_bitset_reduce_or_simple_block_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_or_simple_block_iter); 120 | bench(group, "hi_sparse_bitset_reduce_or_caching_block_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_or_caching_block_iter); 121 | bench(group, "hi_sparse_bitset_reduce_or_simple_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_or_simple_iter); 122 | bench(group, "hi_sparse_bitset_reduce_or_caching_iter", name, hi_sparse_sets, hi_sparse_bitset_reduce_or_caching_iter); 123 | 124 | // ---- OP ---- 125 | bench(group, "hi_sparse_bitset_op_or_simple_block_iter", name, hi_sparse_sets, hi_sparse_bitset_op_or_simple_block_iter); 126 | bench(group, "hi_sparse_bitset_op_or_caching_block_iter", name, hi_sparse_sets, hi_sparse_bitset_op_or_caching_block_iter); 127 | bench(group, "hi_sparse_bitset_op_or_simple_iter", name, hi_sparse_sets, hi_sparse_bitset_op_or_simple_iter); 128 | bench(group, "hi_sparse_bitset_op_or_caching_iter", name, hi_sparse_sets, hi_sparse_bitset_op_or_caching_iter); 129 | 130 | // ---- Third party ---- 131 | bench(group, "hibitset_union", name, hibitsets, hibitset_union); 132 | } 133 | } 134 | 135 | { 136 | let mut group = c.benchmark_group("Union - index step 20"); 137 | /*group.plot_config( 138 | PlotConfiguration::default().summary_scale(AxisScale::Logarithmic) 139 | );*/ 140 | do_bench(&mut group, 20); 141 | } 142 | { 143 | let mut group = c.benchmark_group("Union - index step 200"); 144 | /*group.plot_config( 145 | PlotConfiguration::default().summary_scale(AxisScale::Logarithmic) 146 | );*/ 147 | do_bench(&mut group, 200); 148 | } 149 | } 150 | 151 | criterion_group!(benches_iter, bench_iter); 152 | criterion_main!(benches_iter); -------------------------------------------------------------------------------- /doc.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | set RUSTDOCFLAGS=--cfg docsrs 4 | cargo +nightly doc --lib --features impl --no-deps %1 5 | endlocal -------------------------------------------------------------------------------- /doc/hisparsebitset-50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset-50.png -------------------------------------------------------------------------------- /doc/hisparsebitset-bg-white-50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset-bg-white-50.png -------------------------------------------------------------------------------- /doc/hisparsebitset-bg-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset-bg-white.png -------------------------------------------------------------------------------- /doc/hisparsebitset-dark-50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset-dark-50.png -------------------------------------------------------------------------------- /doc/hisparsebitset-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset-dark.png -------------------------------------------------------------------------------- /doc/hisparsebitset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tower120/hi_sparse_bitset/db2b9eb3c4f5e9d9cb77d20ea451ab74730cd79d/doc/hisparsebitset.png -------------------------------------------------------------------------------- /examples/custom_bitset.rs: -------------------------------------------------------------------------------- 1 | //! This example shows how to make custom bitset, leveraging LevelMasksIterExt 2 | //! to achieve maximum performance. See [examples/custom_bitset_simple] for 3 | //! simpler version. 4 | //! 5 | //! Requires `impl` feature to build. 6 | 7 | use std::marker::PhantomData; 8 | use std::mem::{ManuallyDrop, MaybeUninit}; 9 | use hi_sparse_bitset::config::Config; 10 | use hi_sparse_bitset::{BitBlock, BitSetBase, impl_bitset}; 11 | use hi_sparse_bitset::internals::{LevelMasks, LevelMasksIterExt}; 12 | 13 | #[derive(Default)] 14 | struct Empty(PhantomData); 15 | 16 | impl BitSetBase for Empty { 17 | type Conf = Conf; 18 | const TRUSTED_HIERARCHY: bool = true; 19 | } 20 | 21 | impl LevelMasks for Empty { 22 | fn level0_mask(&self) -> ::Level0BitBlock { 23 | BitBlock::zero() 24 | } 25 | 26 | unsafe fn level1_mask(&self, _level0_index: usize) 27 | -> ::Level1BitBlock 28 | { 29 | BitBlock::zero() 30 | } 31 | 32 | unsafe fn data_mask(&self, _level0_index: usize, _level1_index: usize) 33 | -> ::DataBitBlock 34 | { 35 | BitBlock::zero() 36 | } 37 | } 38 | 39 | impl LevelMasksIterExt for Empty { 40 | type IterState = (); 41 | type Level1BlockData = (); 42 | 43 | fn make_iter_state(&self) -> Self::IterState { () } 44 | unsafe fn drop_iter_state(&self, _state: &mut ManuallyDrop) {} 45 | 46 | unsafe fn init_level1_block_data( 47 | &self, 48 | _state: &mut Self::IterState, 49 | _level1_block_data: &mut MaybeUninit, 50 | _level0_index: usize 51 | ) -> (::Level1BitBlock, bool) { 52 | (BitBlock::zero(), false) 53 | } 54 | 55 | unsafe fn data_mask_from_block_data( 56 | _level1_block_data: &Self::Level1BlockData, _level1_index: usize 57 | ) -> ::DataBitBlock { 58 | BitBlock::zero() 59 | } 60 | } 61 | 62 | impl_bitset!( 63 | impl for Empty where Conf: Config 64 | ); 65 | 66 | fn main(){ 67 | type Conf = hi_sparse_bitset::config::_64bit; 68 | let empty = Empty::::default(); 69 | assert!(empty.is_empty()); 70 | assert!(!empty.contains(10)); 71 | assert!(empty.iter().next().is_none()); 72 | } -------------------------------------------------------------------------------- /examples/custom_bitset_simple.rs: -------------------------------------------------------------------------------- 1 | //! This example shows how to make custom bitset in simple form. 2 | //! 3 | //! Requires `impl` feature to build. 4 | 5 | use std::marker::PhantomData; 6 | use std::mem::{ManuallyDrop, MaybeUninit}; 7 | use hi_sparse_bitset::config::Config; 8 | use hi_sparse_bitset::{BitBlock, BitSetBase, impl_bitset_simple}; 9 | use hi_sparse_bitset::internals::LevelMasks; 10 | 11 | #[derive(Default)] 12 | struct Empty(PhantomData); 13 | 14 | impl BitSetBase for Empty { 15 | type Conf = Conf; 16 | const TRUSTED_HIERARCHY: bool = true; 17 | } 18 | 19 | impl LevelMasks for Empty { 20 | fn level0_mask(&self) -> ::Level0BitBlock { 21 | BitBlock::zero() 22 | } 23 | 24 | unsafe fn level1_mask(&self, _level0_index: usize) 25 | -> ::Level1BitBlock 26 | { 27 | BitBlock::zero() 28 | } 29 | 30 | unsafe fn data_mask(&self, _level0_index: usize, _level1_index: usize) 31 | -> ::DataBitBlock 32 | { 33 | BitBlock::zero() 34 | } 35 | } 36 | 37 | impl_bitset_simple!( 38 | impl for ref Empty where Conf: Config 39 | ); 40 | 41 | fn main(){ 42 | type Conf = hi_sparse_bitset::config::_64bit; 43 | let empty = Empty::::default(); 44 | assert!(empty.is_empty()); 45 | assert!(!empty.contains(10)); 46 | assert!(empty.iter().next().is_none()); 47 | } -------------------------------------------------------------------------------- /examples/traverse_w_cursor.rs: -------------------------------------------------------------------------------- 1 | //! Example of using traverse() with cursor. 2 | 3 | fn main(){ 4 | use hi_sparse_bitset::iter::IndexCursor; 5 | use std::ops::ControlFlow::*; 6 | 7 | type BitSet = hi_sparse_bitset::BitSet; 8 | let bitset = BitSet::from([1,2,3,4]); 9 | 10 | // copy elements from bitset in 2 element sessions. 11 | let mut output = Vec::new(); 12 | let mut cursor = IndexCursor::start(); 13 | 14 | loop{ 15 | let mut counter = 0; 16 | let ctrl = 17 | bitset 18 | .iter().move_to(cursor) 19 | .traverse(|i|{ 20 | if counter == 2{ 21 | cursor = i.into(); 22 | return Break(()); 23 | } 24 | counter += 1; 25 | 26 | output.push(i); 27 | Continue(()) 28 | }); 29 | if ctrl.is_continue(){ 30 | break; 31 | } 32 | } 33 | 34 | assert_eq!(output, vec![1,2,3,4]); 35 | } -------------------------------------------------------------------------------- /examples/usage.rs: -------------------------------------------------------------------------------- 1 | fn main(){ 2 | use itertools::assert_equal; 3 | 4 | use hi_sparse_bitset::reduce; 5 | use hi_sparse_bitset::ops::*; 6 | 7 | type BitSet = hi_sparse_bitset::BitSet; 8 | let bitset1 = BitSet::from([1,2,3,4]); 9 | let bitset2 = BitSet::from([3,4,5,6]); 10 | let bitset3 = BitSet::from([3,4,7,8]); 11 | let bitset4 = BitSet::from([4,9,10]); 12 | let bitsets = [bitset1, bitset2, bitset3]; 13 | 14 | // reduce on bitsets iterator 15 | let intersection = reduce(And, bitsets.iter()).unwrap(); 16 | assert_equal(&intersection, [3,4]); 17 | 18 | // operation between different types 19 | let union = intersection | &bitset4; 20 | assert_equal(&union, [3,4,9,10]); 21 | 22 | // partially traverse iterator, and save position to cursor. 23 | let mut iter = union.iter(); 24 | assert_equal(iter.by_ref().take(2), [3,4]); 25 | let cursor = iter.cursor(); 26 | 27 | // resume iteration from cursor position 28 | let iter = union.iter().move_to(cursor); 29 | assert_equal(iter, [9,10]); 30 | } -------------------------------------------------------------------------------- /src/apply.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | use std::mem; 3 | use std::mem::{ManuallyDrop, MaybeUninit}; 4 | use std::ptr::addr_of_mut; 5 | use crate::ops::*; 6 | use crate::BitSetInterface; 7 | use crate::internals::impl_bitset; 8 | use crate::bitset_interface::{BitSetBase, LevelMasks, LevelMasksIterExt}; 9 | use crate::config::Config; 10 | 11 | /// Binary operation application, as lazy bitset. 12 | /// 13 | /// Created by [apply], or by applying [BitOr], [BitAnd], [BitXor], 14 | /// [Sub] operations on [BitSetInterface]s. 15 | /// 16 | /// [BitOr]: std::ops::BitOr 17 | /// [BitAnd]: std::ops::BitAnd 18 | /// [BitXor]: std::ops::BitXor 19 | /// [Sub]: std::ops::Sub 20 | /// [apply]: crate::apply() 21 | /// [BitSetInterface]: crate::BitSetInterface 22 | #[derive(Clone)] 23 | pub struct Apply{ 24 | pub(crate) s1: S1, 25 | pub(crate) s2: S2, 26 | pub(crate) phantom: PhantomData 27 | } 28 | impl Apply{ 29 | #[inline] 30 | pub(crate) fn new(_:Op, s1:S1, s2:S2) -> Self{ 31 | Apply { s1, s2, phantom:PhantomData } 32 | } 33 | } 34 | 35 | impl BitSetBase for Apply 36 | where 37 | Op: BitSetOp, 38 | S1: LevelMasks, 39 | S2: LevelMasks, 40 | { 41 | type Conf = S1::Conf; 42 | 43 | /// true if S1, S2 and Op are `TrustedHierarchy`. 44 | const TRUSTED_HIERARCHY: bool = 45 | Op::TRUSTED_HIERARCHY 46 | & S1::TRUSTED_HIERARCHY & S2::TRUSTED_HIERARCHY; 47 | } 48 | 49 | impl LevelMasks for Apply 50 | where 51 | Op: BitSetOp, 52 | S1: LevelMasks, 53 | S2: LevelMasks, 54 | { 55 | #[inline] 56 | fn level0_mask(&self) -> ::Level0BitBlock { 57 | Op::hierarchy_op(self.s1.level0_mask(), self.s2.level0_mask()) 58 | } 59 | 60 | #[inline] 61 | unsafe fn level1_mask(&self, level0_index: usize) 62 | -> ::Level1BitBlock 63 | { 64 | Op::hierarchy_op( 65 | self.s1.level1_mask(level0_index), 66 | self.s2.level1_mask(level0_index) 67 | ) 68 | } 69 | 70 | #[inline] 71 | unsafe fn data_mask(&self, level0_index: usize, level1_index: usize) 72 | -> ::DataBitBlock 73 | { 74 | Op::data_op( 75 | self.s1.data_mask(level0_index, level1_index), 76 | self.s2.data_mask(level0_index, level1_index) 77 | ) 78 | } 79 | } 80 | 81 | impl LevelMasksIterExt for Apply 82 | where 83 | Op: BitSetOp, 84 | S1: LevelMasksIterExt, 85 | S2: LevelMasksIterExt, 86 | { 87 | type Level1BlockData = (S1::Level1BlockData, S2::Level1BlockData); 88 | 89 | type IterState = (S1::IterState, S2::IterState); 90 | 91 | #[inline] 92 | fn make_iter_state(&self) -> Self::IterState { 93 | (self.s1.make_iter_state(), self.s2.make_iter_state()) 94 | } 95 | 96 | #[inline] 97 | unsafe fn drop_iter_state(&self, state: &mut ManuallyDrop) { 98 | unsafe{ 99 | self.s1.drop_iter_state(mem::transmute(&mut state.0)); 100 | self.s2.drop_iter_state(mem::transmute(&mut state.1)); 101 | } 102 | } 103 | 104 | #[inline] 105 | unsafe fn init_level1_block_data( 106 | &self, 107 | state: &mut Self::IterState, 108 | level1_block_data: &mut MaybeUninit, 109 | level0_index: usize 110 | ) -> (::Level1BitBlock, bool) { 111 | // &mut MaybeUninit<(T0, T1)> = (&mut MaybeUninit, &mut MaybeUninit) 112 | let (level1_block_data0, level1_block_data1) = { 113 | let ptr = level1_block_data.as_mut_ptr(); 114 | let ptr0 = addr_of_mut!((*ptr).0); 115 | let ptr1 = addr_of_mut!((*ptr).1); 116 | ( 117 | &mut*mem::transmute::<_, *mut MaybeUninit>(ptr0), 118 | &mut*mem::transmute::<_, *mut MaybeUninit>(ptr1) 119 | ) 120 | }; 121 | 122 | let (mask1, v1) = self.s1.init_level1_block_data( 123 | &mut state.0, level1_block_data0, level0_index 124 | ); 125 | let (mask2, v2) = self.s2.init_level1_block_data( 126 | &mut state.1, level1_block_data1, level0_index 127 | ); 128 | 129 | let mask = Op::hierarchy_op(mask1, mask2); 130 | (mask, v1 | v2) 131 | } 132 | 133 | #[inline] 134 | unsafe fn data_mask_from_block_data( 135 | level1_blocks: &Self::Level1BlockData, level1_index: usize 136 | ) -> ::DataBitBlock { 137 | let m0 = S1::data_mask_from_block_data( 138 | &level1_blocks.0, level1_index 139 | ); 140 | let m1 = S2::data_mask_from_block_data( 141 | &level1_blocks.1, level1_index 142 | ); 143 | Op::data_op(m0, m1) 144 | } 145 | } 146 | 147 | impl_bitset!( 148 | impl for Apply 149 | where 150 | Op: BitSetOp, 151 | S1: BitSetInterface, 152 | S2: BitSetInterface 153 | ); 154 | 155 | #[cfg(test)] 156 | mod test{ 157 | use std::collections::HashSet; 158 | use itertools::assert_equal; 159 | use rand::Rng; 160 | use crate::reduce; 161 | use super::*; 162 | 163 | type HiSparseBitset = crate::BitSet; 164 | 165 | #[test] 166 | fn ops_test(){ 167 | cfg_if::cfg_if! { 168 | if #[cfg(miri)] { 169 | const MAX_RANGE: usize = 10_000; 170 | const AMOUNT : usize = 100; 171 | const INDEX_MUL: usize = 5; 172 | } else { 173 | const MAX_RANGE: usize = 10_000; 174 | const AMOUNT : usize = 1000; 175 | const INDEX_MUL: usize = 5; 176 | } 177 | } 178 | 179 | let mut rng = rand::thread_rng(); 180 | let mut v1 = Vec::new(); 181 | let mut v2 = Vec::new(); 182 | let mut v3 = Vec::new(); 183 | let mut v4 = Vec::new(); 184 | for _ in 0..AMOUNT{ 185 | v1.push(rng.gen_range(0..MAX_RANGE)*INDEX_MUL); 186 | v2.push(rng.gen_range(0..MAX_RANGE)*INDEX_MUL); 187 | v3.push(rng.gen_range(0..MAX_RANGE)*INDEX_MUL); 188 | v4.push(rng.gen_range(0..MAX_RANGE)*INDEX_MUL); 189 | } 190 | 191 | /* 192 | // This is incredibly slow with MIRI 193 | let v1 = (0..MAX_RANGE).map(|i|i*INDEX_MUL).choose_multiple(&mut rng, AMOUNT); 194 | let v2 = (0..MAX_RANGE).map(|i|i*INDEX_MUL).choose_multiple(&mut rng, AMOUNT); 195 | let v3 = (0..MAX_RANGE).map(|i|i*INDEX_MUL).choose_multiple(&mut rng, AMOUNT); 196 | let v4 = (0..MAX_RANGE).map(|i|i*INDEX_MUL).choose_multiple(&mut rng, AMOUNT); 197 | */ 198 | 199 | let hiset1: HiSparseBitset = v1.iter().copied().collect(); 200 | let hiset2: HiSparseBitset = v2.iter().copied().collect(); 201 | let hiset3: HiSparseBitset = v3.iter().copied().collect(); 202 | let hiset4: HiSparseBitset = v4.iter().copied().collect(); 203 | 204 | let set1: HashSet = v1.iter().copied().collect(); 205 | let set2: HashSet = v2.iter().copied().collect(); 206 | let set3: HashSet = v3.iter().copied().collect(); 207 | let set4: HashSet = v4.iter().copied().collect(); 208 | 209 | fn test(h: Apply, s: HashSet) 210 | where 211 | Op: BitSetOp, 212 | S1: BitSetInterface, 213 | S2: BitSetInterface, 214 | { 215 | let hv: Vec = h.block_iter() 216 | .flat_map(|block| block.iter()) 217 | .collect(); 218 | 219 | let mut s: Vec = s.into_iter().collect(); 220 | s.sort(); 221 | assert_equal(hv, s); 222 | } 223 | 224 | // &HiSet <-> &HiSet 225 | test(&hiset1 & &hiset2, &set1 & &set2); 226 | test(&hiset1 | &hiset2, &set1 | &set2); 227 | test(&hiset1 ^ &hiset2, &set1 ^ &set2); 228 | test(&hiset1 - &hiset2, &set1 - &set2); 229 | 230 | // Reduce <-> Reduce 231 | let group1 = [&hiset1, &hiset2]; 232 | let group2 = [&hiset3, &hiset4]; 233 | let reduce1 = reduce(Or, group1.iter().copied()).unwrap(); 234 | let reduce2 = reduce(Or, group2.iter().copied()).unwrap(); 235 | let set_or1 = &set1 | &set2; 236 | let set_or2 = &set3 | &set4; 237 | test( 238 | reduce1.clone() & reduce2.clone(), 239 | &set_or1 & &set_or2 240 | ); 241 | test( 242 | reduce1.clone() | reduce2.clone(), 243 | &set_or1 | &set_or2 244 | ); 245 | test( 246 | reduce1.clone() ^ reduce2.clone(), 247 | &set_or1 ^ &set_or2 248 | ); 249 | test( 250 | reduce1.clone() - reduce2.clone(), 251 | &set_or1 - &set_or2 252 | ); 253 | 254 | // &Reduce <-> &Reduce 255 | test( 256 | &reduce1 & &reduce2, 257 | &set_or1 & &set_or2 258 | ); 259 | test( 260 | &reduce1 | &reduce2, 261 | &set_or1 | &set_or2 262 | ); 263 | test( 264 | &reduce1 ^ &reduce2, 265 | &set_or1 ^ &set_or2 266 | ); 267 | test( 268 | &reduce1 - &reduce2, 269 | &set_or1 - &set_or2 270 | ); 271 | 272 | // Op <-> Op 273 | let hiset_or1 = &hiset1 | &hiset2; 274 | let hiset_or2 = &hiset3 | &hiset4; 275 | test(hiset_or1.clone() & hiset_or2.clone(), &set_or1 & &set_or2); 276 | test(hiset_or1.clone() | hiset_or2.clone(), &set_or1 | &set_or2); 277 | test(hiset_or1.clone() ^ hiset_or2.clone(), &set_or1 ^ &set_or2); 278 | test(hiset_or1.clone() - hiset_or2.clone(), &set_or1 - &set_or2); 279 | 280 | // &Op <-> &Op 281 | test(&hiset_or1 & &hiset_or2, &set_or1 & &set_or2); 282 | test(&hiset_or1 | &hiset_or2, &set_or1 | &set_or2); 283 | test(&hiset_or1 ^ &hiset_or2, &set_or1 ^ &set_or2); 284 | test(&hiset_or1 - &hiset_or2, &set_or1 - &set_or2); 285 | } 286 | } -------------------------------------------------------------------------------- /src/bit_block.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | use std::ops::{BitAnd, BitOr, BitXor, ControlFlow}; 3 | use crate::bit_utils; 4 | use crate::bit_queue::{ArrayBitQueue, BitQueue, PrimitiveBitQueue}; 5 | 6 | // TODO: consider removing copy 7 | /// Bit block. 8 | /// 9 | /// Used in [Config], to define bit blocks [BitSet] is built from. 10 | /// 11 | /// [Config]: crate::config::Config 12 | /// [BitSet]: crate::BitSet 13 | pub trait BitBlock 14 | : BitAnd + BitOr + BitXor 15 | + Eq + PartialEq 16 | + Sized + Copy + Clone 17 | { 18 | /// 2^N bits 19 | const SIZE_POT_EXPONENT: usize; 20 | 21 | /// Size in bits 22 | #[inline] 23 | /*const*/ fn size() -> usize { 24 | 1 << Self::SIZE_POT_EXPONENT 25 | } 26 | 27 | fn zero() -> Self; 28 | 29 | #[inline] 30 | fn is_zero(&self) -> bool { 31 | self == &Self::zero() 32 | } 33 | 34 | /// Returns previous bit 35 | /// 36 | /// `bit_index` is guaranteed to be valid 37 | #[inline] 38 | fn set_bit(&mut self, bit_index: usize) -> bool { 39 | unsafe{ 40 | bit_utils::set_array_bit_unchecked::(self.as_array_mut(), bit_index) 41 | } 42 | } 43 | 44 | /// `bit_index` is guaranteed to be valid 45 | #[inline] 46 | fn get_bit(&self, bit_index: usize) -> bool{ 47 | unsafe{ 48 | bit_utils::get_array_bit_unchecked(self.as_array(), bit_index) 49 | } 50 | } 51 | 52 | // TODO: This can be removed, since there is BitQueue::traverse 53 | // which do the same and perform the same in optimized build. 54 | /// Returns [Break] if traverse was interrupted (`f` returns [Break]). 55 | /// 56 | /// [Break]: ControlFlow::Break 57 | #[inline] 58 | fn traverse_bits(&self, f: F) -> ControlFlow<()> 59 | where 60 | F: FnMut(usize) -> ControlFlow<()> 61 | { 62 | bit_utils::traverse_array_one_bits(self.as_array(), f) 63 | } 64 | 65 | type BitsIter: BitQueue; 66 | fn into_bits_iter(self) -> Self::BitsIter; 67 | 68 | fn as_array(&self) -> &[u64]; 69 | fn as_array_mut(&mut self) -> &mut [u64]; 70 | 71 | #[inline] 72 | fn count_ones(&self) -> usize { 73 | let mut sum = 0; 74 | // will be unrolled at compile time 75 | for &i in self.as_array(){ 76 | sum += u64::count_ones(i); 77 | } 78 | sum as usize 79 | } 80 | } 81 | 82 | impl BitBlock for u64{ 83 | const SIZE_POT_EXPONENT: usize = 6; 84 | 85 | #[inline] 86 | fn zero() -> Self{ 87 | 0 88 | } 89 | 90 | #[inline] 91 | fn set_bit(&mut self, bit_index: usize) -> bool{ 92 | unsafe{ 93 | bit_utils::set_bit_unchecked::(self, bit_index) 94 | } 95 | } 96 | 97 | #[inline] 98 | fn get_bit(&self, bit_index: usize) -> bool { 99 | unsafe{ 100 | bit_utils::get_bit_unchecked(*self, bit_index) 101 | } 102 | } 103 | 104 | #[inline] 105 | fn traverse_bits(&self, f: F) -> ControlFlow<()> 106 | where 107 | F: FnMut(usize) -> ControlFlow<()> 108 | { 109 | bit_utils::traverse_one_bits(*self, f) 110 | } 111 | 112 | type BitsIter = PrimitiveBitQueue; 113 | #[inline] 114 | fn into_bits_iter(self) -> Self::BitsIter { 115 | PrimitiveBitQueue::new(self) 116 | } 117 | 118 | #[inline] 119 | fn as_array(&self) -> &[u64] { 120 | unsafe { 121 | mem::transmute::<&u64, &[u64; 1]>(self) 122 | } 123 | } 124 | 125 | #[inline] 126 | fn as_array_mut(&mut self) -> &mut [u64] { 127 | unsafe { 128 | mem::transmute::<&mut u64, &mut [u64; 1]>(self) 129 | } 130 | } 131 | } 132 | 133 | #[cfg(feature = "simd")] 134 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 135 | impl BitBlock for wide::u64x2{ 136 | const SIZE_POT_EXPONENT: usize = 7; 137 | 138 | #[inline] 139 | fn zero() -> Self { 140 | wide::u64x2::ZERO 141 | } 142 | 143 | #[inline] 144 | fn is_zero(&self) -> bool { 145 | // this should be faster then loading from memory into simd register, 146 | // and testz(if supported). 147 | let array = self.as_array_ref(); 148 | (array[0] | array[1]) == 0 149 | } 150 | 151 | type BitsIter = ArrayBitQueue; 152 | #[inline] 153 | fn into_bits_iter(self) -> Self::BitsIter { 154 | Self::BitsIter::new(self.to_array()) 155 | } 156 | 157 | #[inline] 158 | fn as_array(&self) -> &[u64] { 159 | self.as_array_ref() 160 | } 161 | 162 | #[inline] 163 | fn as_array_mut(&mut self) -> &mut [u64] { 164 | self.as_array_mut() 165 | } 166 | } 167 | 168 | #[cfg(feature = "simd")] 169 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 170 | impl BitBlock for wide::u64x4{ 171 | const SIZE_POT_EXPONENT: usize = 8; 172 | 173 | #[inline] 174 | fn zero() -> Self { 175 | wide::u64x4::ZERO 176 | } 177 | 178 | type BitsIter = ArrayBitQueue; 179 | #[inline] 180 | fn into_bits_iter(self) -> Self::BitsIter { 181 | Self::BitsIter::new(self.to_array()) 182 | } 183 | 184 | #[inline] 185 | fn as_array(&self) -> &[u64] { 186 | self.as_array_ref() 187 | } 188 | 189 | #[inline] 190 | fn as_array_mut(&mut self) -> &mut [u64] { 191 | self.as_array_mut() 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/bit_queue.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | use std::mem::{ManuallyDrop, size_of}; 3 | use std::ops::ControlFlow; 4 | 5 | use crate::bit_utils::{one_bits_iter, OneBitsIter, self}; 6 | use crate::Primitive; 7 | 8 | /// Return 0 if n > BITS 9 | #[inline] 10 | fn saturating_shl(p: P, n: usize) -> P { 11 | let bits = size_of::

() * 8; 12 | if n >= bits{ 13 | P::ZERO 14 | } else { 15 | p << n 16 | } 17 | } 18 | 19 | #[inline] 20 | fn trailing_zeroes(bit_block_iter: &OneBitsIter

) -> usize{ 21 | let block: &P = unsafe{ 22 | mem::transmute(bit_block_iter) 23 | }; 24 | block.trailing_zeros() as usize 25 | } 26 | 27 | /*#[inline] 28 | fn is_empty(bit_block_iter: &OneBitsIter

) -> bool{ 29 | let block: &P = unsafe{ 30 | mem::transmute(bit_block_iter) 31 | }; 32 | block.is_zero() 33 | }*/ 34 | 35 | /// Queue of 1 bits. 36 | /// 37 | /// Pop first set bit on iteration. "Consumed" bit replaced with zero. 38 | pub trait BitQueue: Iterator + Clone{ 39 | /// All bits 0. Iterator returns None. 40 | fn empty() -> Self; 41 | 42 | /// All bits 1. 43 | fn filled() -> Self; 44 | 45 | /* /// Remove first n bits. (Set 0) 46 | /// 47 | /// # Safety 48 | /// 49 | /// n is not checked. 50 | unsafe fn zero_first_n_unchecked(&mut self, n: usize); */ 51 | 52 | /// Remove first n bits. (Set 0) 53 | /// 54 | /// If n >= BitQueue len - make it empty. 55 | fn zero_first_n(&mut self, n: usize); 56 | 57 | /// Current index. Equals len - if iteration finished. 58 | fn current(&self) -> usize; 59 | 60 | fn traverse(self, f: F) -> ControlFlow<()> 61 | where 62 | F: FnMut(usize) -> ControlFlow<()>; 63 | 64 | /* // TODO: remove ? 65 | fn is_empty(&self) -> bool;*/ 66 | } 67 | 68 | /// [BitQueue] for [Primitive]. 69 | #[derive(Clone)] 70 | pub struct PrimitiveBitQueue

{ 71 | bit_block_iter: OneBitsIter

72 | } 73 | 74 | impl

PrimitiveBitQueue

{ 75 | #[inline] 76 | pub fn new(value: P) -> Self { 77 | Self{ 78 | bit_block_iter: one_bits_iter(value) 79 | } 80 | } 81 | } 82 | 83 | impl

BitQueue for PrimitiveBitQueue

84 | where 85 | P: Primitive 86 | { 87 | #[inline] 88 | fn empty() -> Self { 89 | Self::new(P::ZERO) 90 | } 91 | 92 | #[inline] 93 | fn filled() -> Self { 94 | Self::new(P::MAX) 95 | } 96 | 97 | /* #[inline] 98 | unsafe fn zero_first_n_unchecked(&mut self, n: usize) { 99 | zero_first_n(&mut self.bit_block_iter, n); 100 | } */ 101 | 102 | #[inline] 103 | fn zero_first_n(&mut self, n: usize) { 104 | let block: &mut P = unsafe{ 105 | mem::transmute(&mut self.bit_block_iter) 106 | }; 107 | let mask = saturating_shl(P::MAX, n); 108 | *block &= mask; 109 | } 110 | 111 | #[inline] 112 | fn current(&self) -> usize { 113 | trailing_zeroes(&self.bit_block_iter) 114 | } 115 | 116 | #[inline] 117 | fn traverse(self, f: F) -> ControlFlow<()> where F: FnMut(usize) -> ControlFlow<()> { 118 | let block: P = unsafe{ 119 | mem::transmute_copy(&self.bit_block_iter) 120 | }; 121 | bit_utils::traverse_one_bits(block, f) 122 | } 123 | 124 | /*fn is_empty(&self) -> bool { 125 | is_empty(&self.bit_block_iter) 126 | }*/ 127 | } 128 | 129 | 130 | impl

Iterator for PrimitiveBitQueue

131 | where 132 | P: Primitive 133 | { 134 | type Item = usize; 135 | 136 | #[inline] 137 | fn next(&mut self) -> Option { 138 | self.bit_block_iter.next() 139 | } 140 | } 141 | 142 | /// [BitQueue] for array of [Primitive]s. 143 | #[derive(Clone)] 144 | pub struct ArrayBitQueue{ 145 | /// first element - always active one. 146 | /// (copy of bit_block_iters[bit_block_index]). 147 | bit_block_iters: [OneBitsIter

; N], 148 | bit_block_index: usize, 149 | } 150 | 151 | impl ArrayBitQueue 152 | where 153 | P: Primitive 154 | { 155 | #[inline] 156 | pub fn new(array: [P;N]) -> Self{ 157 | Self{ 158 | bit_block_iters: unsafe{ 159 | // transmute is safe since OneBitsIter

transparent to P. 160 | // Should be just mem::transmute(array). 161 | mem::transmute_copy(&ManuallyDrop::new(array)) 162 | }, 163 | bit_block_index: 0, 164 | } 165 | } 166 | } 167 | 168 | impl BitQueue for ArrayBitQueue 169 | where 170 | P: Primitive 171 | { 172 | #[inline] 173 | fn empty() -> Self { 174 | Self{ 175 | bit_block_iters: [one_bits_iter(P::ZERO); N], 176 | bit_block_index: N-1, 177 | } 178 | } 179 | 180 | #[inline] 181 | fn filled() -> Self { 182 | Self::new([P::MAX; N]) 183 | } 184 | 185 | /* #[inline] 186 | unsafe fn zero_first_n_unchecked(&mut self, n: usize) { 187 | let element_index = n / (size_of::

() * 8); // compile-time math optimization 188 | let bit_index = n % (size_of::

() * 8); // compile-time math optimization 189 | 190 | // Fill all first elements with 0 191 | for i in 0..element_index{ 192 | *self.bit_block_iters.get_unchecked_mut(i) = one_bits_iter(P::zero()); 193 | } 194 | 195 | // Mask out last one 196 | zero_first_n(&mut self.bit_block_iters.get_unchecked_mut(element_index), bit_index); 197 | } */ 198 | 199 | #[inline] 200 | fn zero_first_n(&mut self, n: usize) { 201 | let element_index = n / (size_of::

() * 8); // compile-time math optimization 202 | 203 | // clamp to empty 204 | if element_index >= N { 205 | //*self = Self::empty(); 206 | self.bit_block_iters[0] = one_bits_iter(P::ZERO); 207 | self.bit_block_index = N-1; 208 | return; 209 | } 210 | 211 | // are we ahead of n block-wise? 212 | if element_index < self.bit_block_index { 213 | return; 214 | } 215 | 216 | 217 | 218 | /* // 2.0 219 | unsafe { 220 | self.bit_block_index = element_index; 221 | 222 | let active_block_iter = unsafe { 223 | self.bit_block_iters.get_unchecked_mut(element_index) 224 | }; 225 | 226 | // Mask out block 227 | let bit_index = n % (size_of::

() * 8); // compile-time math optimization 228 | unsafe /* zero_first_n */ { 229 | let block: &mut P = mem::transmute(active_block_iter); 230 | *block &= P::max_value() << bit_index; 231 | } 232 | 233 | // copy to active 234 | self.bit_block_iters[0] = *active_block_iter; 235 | 236 | return; 237 | }*/ 238 | 239 | 240 | // update active block 241 | if element_index != self.bit_block_index { 242 | self.bit_block_index = element_index; 243 | self.bit_block_iters[0] = unsafe { 244 | *self.bit_block_iters.get_unchecked_mut(element_index) 245 | }; 246 | } 247 | 248 | // Mask out active block 249 | let bit_index = n % (size_of::

() * 8); // compile-time math optimization 250 | unsafe /* zero_first_n */ { 251 | let active_block_iter = &mut self.bit_block_iters[0]; 252 | let block: &mut P = mem::transmute(active_block_iter); 253 | *block &= P::MAX << bit_index; 254 | } 255 | } 256 | 257 | #[inline] 258 | fn current(&self) -> usize { 259 | let active_block_iter = &self.bit_block_iters[0]; 260 | self.bit_block_index * size_of::

() * 8 + trailing_zeroes(active_block_iter) 261 | } 262 | 263 | #[inline] 264 | fn traverse(mut self, mut f: F) -> ControlFlow<()> 265 | where 266 | F: FnMut(usize) -> ControlFlow<()> 267 | { 268 | // This is faster, then iterating active value, then the rest ones 269 | unsafe{ 270 | // copy active back to its place. 271 | // compiler should optimize away this for newly constructed BitQueue. 272 | *self.bit_block_iters.get_unchecked_mut(self.bit_block_index) = self.bit_block_iters[0]; 273 | 274 | let slice: &[P] = std::slice::from_raw_parts( 275 | // cast is safe because OneBitsIter

transmutable to P. 276 | self.bit_block_iters.as_ptr().add(self.bit_block_index).cast(), 277 | N - self.bit_block_index 278 | ); 279 | 280 | let start_index = self.bit_block_index*size_of::

()*8; 281 | return bit_utils::traverse_array_one_bits( slice, |i|f(start_index + i)); 282 | } 283 | } 284 | 285 | 286 | /* #[inline] 287 | fn is_empty(&self) -> bool { 288 | let active_block_iter = &self.bit_block_iters[0]; 289 | is_empty(active_block_iter) 290 | }*/ 291 | } 292 | 293 | impl Iterator for ArrayBitQueue 294 | where 295 | P: Primitive 296 | { 297 | type Item = usize; 298 | 299 | #[inline] 300 | fn next(&mut self) -> Option { 301 | loop { 302 | if let Some(index) = self.bit_block_iters[0].next() { 303 | return Some(self.bit_block_index * size_of::

() * 8 + index); 304 | } 305 | if self.bit_block_index == N-1 { 306 | return None; 307 | } 308 | self.bit_block_index += 1; 309 | 310 | self.bit_block_iters[0] = unsafe { 311 | *self.bit_block_iters.get_unchecked_mut(self.bit_block_index) 312 | }; 313 | } 314 | } 315 | 316 | #[inline] 317 | fn for_each(self, mut f: F) 318 | where 319 | F: FnMut(usize) 320 | { 321 | self.traverse(|i|{ 322 | f(i); 323 | ControlFlow::Continue(()) 324 | }); 325 | } 326 | } -------------------------------------------------------------------------------- /src/bit_utils.rs: -------------------------------------------------------------------------------- 1 | use std::mem::size_of; 2 | use std::ops::ControlFlow; 3 | use crate::Primitive; 4 | 5 | /// Block ordering undefined. But same as [get_array_bit]. 6 | /// 7 | /// # Safety 8 | /// 9 | /// `index` validity is not checked. 10 | #[inline] 11 | pub unsafe fn set_array_bit_unchecked(blocks: &mut [T], index: usize) -> bool 12 | where 13 | T: Primitive 14 | { 15 | let bits_size: usize = size_of::() * 8; // compile-time known value 16 | let block_index = index / bits_size; 17 | 18 | // index % size 19 | // From https://stackoverflow.com/a/27589182 20 | let bit_index = index & (bits_size -1); 21 | 22 | set_bit_unchecked::(blocks.get_unchecked_mut(block_index), bit_index) 23 | } 24 | 25 | 26 | /// In machine endian. 27 | /// 28 | /// # Safety 29 | /// 30 | /// `bit_index` validity is not checked. 31 | #[inline] 32 | pub unsafe fn set_bit_unchecked(block: &mut T, bit_index: usize) -> bool 33 | where 34 | T: Primitive 35 | { 36 | let block_mask: T = T::ONE << bit_index; 37 | let masked_block = *block & block_mask; 38 | 39 | if FLAG { 40 | *block |= block_mask; 41 | } else { 42 | *block &= !block_mask; 43 | } 44 | 45 | !masked_block.is_zero() 46 | } 47 | 48 | /// Block ordering undefined. But same as [set_array_bit]. 49 | /// 50 | /// # Safety 51 | /// 52 | /// `index` validity is not checked. 53 | #[inline] 54 | pub unsafe fn get_array_bit_unchecked(blocks: &[T], index: usize) -> bool 55 | where 56 | T: Primitive 57 | { 58 | let bits_size: usize = size_of::() * 8; // compile-time known value 59 | let block_index = index / bits_size; 60 | 61 | // index % size 62 | // From https://stackoverflow.com/a/27589182 63 | let bit_index = index & (bits_size -1); 64 | 65 | get_bit_unchecked(*blocks.get_unchecked(block_index), bit_index) 66 | } 67 | 68 | /// In machine endian. 69 | /// 70 | /// # Safety 71 | /// 72 | /// `bit_index` validity is not checked. 73 | #[inline] 74 | pub unsafe fn get_bit_unchecked(block: T, bit_index: usize) -> bool { 75 | let block_mask: T = T::ONE << bit_index; 76 | let masked_block = block & block_mask; 77 | !masked_block.is_zero() 78 | } 79 | 80 | /// Blocks traversed in the same order as [set_array_bit], [get_array_bit]. 81 | #[inline] 82 | pub fn traverse_array_one_bits(array: &[P], mut f: F) -> ControlFlow<()> 83 | where 84 | P: Primitive, 85 | F: FnMut(usize) -> ControlFlow<()> 86 | { 87 | let len = array.len(); 88 | for i in 0..len{ 89 | let element = unsafe{*array.get_unchecked(i)}; 90 | let control = traverse_one_bits( 91 | element, 92 | |r|{ 93 | let index = i*size_of::

()*8 + r; 94 | f(index) 95 | } 96 | ); 97 | if control.is_break(){ 98 | return ControlFlow::Break(()); 99 | } 100 | } 101 | ControlFlow::Continue(()) 102 | } 103 | 104 | #[inline] 105 | pub fn traverse_one_bits(mut element: P, mut f: F) -> ControlFlow<()> 106 | where 107 | P: Primitive, 108 | F: FnMut(usize) -> ControlFlow<()> 109 | { 110 | // from https://lemire.me/blog/2018/03/08/iterating-over-set-bits-quickly-simd-edition/ 111 | // https://github.com/lemire/Code-used-on-Daniel-Lemire-s-blog/blob/master/2018/03/07/simdbitmapdecode.c#L45 112 | while !element.is_zero() { 113 | let index = element.trailing_zeros() as usize; 114 | 115 | let control = f(index); 116 | if control.is_break(){ 117 | return ControlFlow::Break(()); 118 | } 119 | 120 | // Returns an integer having just the least significant bit of 121 | // bitset turned on, all other bits are off. 122 | let t: P = element & element.wrapping_neg(); 123 | 124 | element ^= t; 125 | } 126 | ControlFlow::Continue(()) 127 | } 128 | 129 | /// This is 15% slower then "traverse" version 130 | #[inline] 131 | pub fn one_bits_iter

(element: P) -> OneBitsIter

{ 132 | OneBitsIter {element} 133 | } 134 | 135 | /// Can be safely casted to its original bit block type. 136 | /// 137 | /// "Consumed"/iterated one bits replaced with zero. 138 | #[repr(transparent)] 139 | #[derive(Copy, Clone)] 140 | pub struct OneBitsIter

{ 141 | element: P 142 | } 143 | impl

Iterator for OneBitsIter

144 | where 145 | P: Primitive, 146 | { 147 | type Item = usize; 148 | 149 | #[inline(always)] 150 | fn next(&mut self) -> Option { 151 | // from https://lemire.me/blog/2018/03/08/iterating-over-set-bits-quickly-simd-edition/ 152 | // https://github.com/lemire/Code-used-on-Daniel-Lemire-s-blog/blob/master/2018/03/07/simdbitmapdecode.c#L45 153 | if !self.element.is_zero() { 154 | let index = self.element.trailing_zeros() as usize; 155 | 156 | // Returns an integer having just the least significant bit of 157 | // bitset turned on, all other bits are off. 158 | let t: P = self.element & self.element.wrapping_neg(); 159 | self.element ^= t; 160 | 161 | Some(index) 162 | } else { 163 | None 164 | } 165 | } 166 | } 167 | 168 | -------------------------------------------------------------------------------- /src/bitset.rs: -------------------------------------------------------------------------------- 1 | use crate::config::Config; 2 | use crate::block::Block; 3 | use crate::derive_raw::derive_raw; 4 | use crate::{BitSetBase, raw}; 5 | 6 | type Level0Block = Block< 7 | ::Level0BitBlock, 8 | ::Level0BlockIndices 9 | >; 10 | type Level1Block = Block< 11 | ::Level1BitBlock, 12 | ::Level1BlockIndices 13 | >; 14 | type LevelDataBlock = Block< 15 | ::DataBitBlock, [usize;0] 16 | >; 17 | 18 | type RawBitSet = raw::RawBitSet< 19 | Conf, 20 | Level0Block, 21 | Level1Block, 22 | LevelDataBlock 23 | >; 24 | 25 | /// Hierarchical sparse bitset. 26 | /// 27 | /// Tri-level hierarchy. Highest uint it can hold 28 | /// is [Level0BitBlock]::size() * [Level1BitBlock]::size() * [DataBitBlock]::size(). 29 | /// 30 | /// Only last level contains blocks of actual data. Empty(skipped) data blocks 31 | /// are not allocated. 32 | /// 33 | /// Structure optimized for intersection speed. 34 | /// _(Other inter-bitset operations are in fact fast too - but intersection has lowest algorithmic complexity.)_ 35 | /// Insert/remove/contains is fast O(1) too. 36 | /// 37 | /// [Level0BitBlock]: crate::config::Config::Level0BitBlock 38 | /// [Level1BitBlock]: crate::config::Config::Level1BitBlock 39 | /// [DataBitBlock]: crate::config::Config::DataBitBlock 40 | pub struct BitSet( 41 | RawBitSet 42 | ); 43 | impl BitSetBase for BitSet { 44 | type Conf = Conf; 45 | const TRUSTED_HIERARCHY: bool = true; 46 | } 47 | derive_raw!( 48 | impl BitSet as RawBitSet where Conf: Config 49 | ); -------------------------------------------------------------------------------- /src/bitset_interface.rs: -------------------------------------------------------------------------------- 1 | use std::mem::{ManuallyDrop, MaybeUninit}; 2 | use std::ops::ControlFlow; 3 | use crate::{assume, level_indices}; 4 | use crate::bit_block::BitBlock; 5 | use crate::config::{DefaultBlockIterator, Config, DefaultIndexIterator}; 6 | 7 | // We have this separate trait with Config, to avoid making LevelMasks public. 8 | pub trait BitSetBase { 9 | type Conf: Config; 10 | 11 | /// Does each raised bit in hierarchy bitblock 12 | /// correspond to non-empty data block? 13 | /// 14 | /// Currently has effect for [Eq]/[internals::is_eq()] and 15 | /// [BitSetInterface::is_empty()]/[internals::is_empty()]. 16 | /// 17 | /// [internals::is_eq()]: crate::internals::is_eq 18 | /// [internals::is_empty()]: crate::internals::is_empty 19 | const TRUSTED_HIERARCHY: bool; 20 | } 21 | 22 | /// Basic interface for accessing block masks. Can work with `SimpleIter`. 23 | pub trait LevelMasks: BitSetBase{ 24 | fn level0_mask(&self) -> ::Level0BitBlock; 25 | 26 | /// # Safety 27 | /// 28 | /// index is not checked 29 | unsafe fn level1_mask(&self, level0_index: usize) 30 | -> ::Level1BitBlock; 31 | 32 | /// # Safety 33 | /// 34 | /// indices are not checked 35 | unsafe fn data_mask(&self, level0_index: usize, level1_index: usize) 36 | -> ::DataBitBlock; 37 | } 38 | 39 | /// More sophisticated masks interface, optimized for iteration speed of 40 | /// generative/lazy bitset. 41 | /// 42 | /// For example, in [Reduce] this achieved through 43 | /// caching level1(pre-data level) block pointers of all sets. Which also allows to discard 44 | /// bitsets with empty level1 blocks in final stage of getting data blocks. 45 | /// Properly implementing this gave [Reduce] and [Apply] 25-100% performance boost. 46 | /// 47 | /// NOTE: This interface is somewhat icky and initially was intended for internal use. 48 | /// I don't know if it will be actually used, so no work is done on top of that. 49 | /// If you do use it, and want it better - open an issue. 50 | /// 51 | /// # How it is used 52 | /// 53 | /// See [CachingBlockIter::next()] code to see how it used. 54 | /// 55 | /// ```ignore 56 | /// let mut state = bitset.make_iter_state(); 57 | /// let mut level1_block_data = MaybeUninit::new(Default::default()); 58 | /// 59 | /// fn next() { 60 | /// ... 61 | /// level1_block_data.assume_init_drop(); 62 | /// let (level1_mask, is_not_empty) = bitset.update_level1_block_data(state, level1_block_data, level0_index); 63 | /// ... 64 | /// let bitblock = data_mask_from_block_data(level1_block_data, level1_index); 65 | /// 66 | /// return bitblock; 67 | /// } 68 | /// 69 | /// level1_block_data.assume_init_drop(); 70 | /// bitset.drop_iter_state(state); 71 | /// ``` 72 | /// 73 | /// [Reduce]: crate::Reduce 74 | /// [Apply]: crate::Apply 75 | /// [CachingBlockIter::next()]: crate::iter::CachingBlockIter::next() 76 | pub trait LevelMasksIterExt: LevelMasks{ 77 | /// Consists from child states (if any) + Self state. 78 | /// 79 | /// Use `()` for stateless. 80 | /// 81 | /// [Level1BlockData]: Self::Level1BlockData 82 | type IterState; 83 | 84 | /// Level1 block related data, used to speed up data_mask access. 85 | /// 86 | /// Prefer POD, or any kind of drop-less, since it will be dropped 87 | /// before iteration of each next level1 block. 88 | /// 89 | /// In library, used to cache Level1Block(s) for faster DataBlock access, 90 | /// without traversing whole hierarchy for getting each block during iteration. 91 | type Level1BlockData: Default; 92 | 93 | fn make_iter_state(&self) -> Self::IterState; 94 | 95 | /// Having separate function for drop not strictly necessary, since 96 | /// IterState can actually drop itself. But! This allows not to store cache 97 | /// size within IterState. Which makes FixedCache CacheData ZST, if its childs 98 | /// are ZSTs, and which makes cache construction and destruction noop. Which is 99 | /// important for short iteration sessions. 100 | /// 101 | /// P.S. This can be done at compile-time by opting out "len" counter, 102 | /// but stable Rust does not allow to do that yet. 103 | /// 104 | /// # Safety 105 | /// 106 | /// - `state` must not be used after this. 107 | /// - Must be called exactly once for each `state`. 108 | unsafe fn drop_iter_state(&self, state: &mut ManuallyDrop); 109 | 110 | /// Init `level1_block_data` and return (Level1Mask, is_not_empty). 111 | /// 112 | /// `level1_block_data` will come in undefined state - rewrite it completely. 113 | /// 114 | /// `is_not_empty` is not used by iterator itself, but can be used by other 115 | /// generative bitsets (namely [Reduce]) - we expect compiler to optimize away non-used code. 116 | /// It exists - because sometimes you may have faster ways of checking emptiness, 117 | /// then checking simd register (bitblock) for zero in general case. 118 | /// For example, in BitSet - it is done by checking of block indirection index for zero. 119 | /// 120 | /// # Safety 121 | /// 122 | /// indices are not checked. 123 | /// 124 | /// [Reduce]: crate::Reduce 125 | // Performance-wise it is important to use this in-place construct style, 126 | // instead of just returning Level1BlockData. Even if we return Level1BlockData, 127 | // and then immoderately write it to MaybeUninit - compiler somehow still can't 128 | // optimize it as direct memory write without intermediate bitwise copy. 129 | unsafe fn init_level1_block_data( 130 | &self, 131 | state: &mut Self::IterState, 132 | level1_block_data: &mut MaybeUninit, 133 | level0_index: usize 134 | ) -> (::Level1BitBlock, bool); 135 | 136 | /// # Safety 137 | /// 138 | /// indices are not checked. 139 | unsafe fn data_mask_from_block_data( 140 | level1_block_data: &Self::Level1BlockData, level1_index: usize 141 | ) -> ::DataBitBlock; 142 | } 143 | 144 | impl<'a, T: LevelMasks> BitSetBase for &'a T { 145 | type Conf = T::Conf; 146 | const TRUSTED_HIERARCHY: bool = T::TRUSTED_HIERARCHY; 147 | } 148 | impl<'a, T: LevelMasks> LevelMasks for &'a T { 149 | #[inline] 150 | fn level0_mask(&self) -> ::Level0BitBlock { 151 | ::level0_mask(self) 152 | } 153 | 154 | #[inline] 155 | unsafe fn level1_mask(&self, level0_index: usize) 156 | -> ::Level1BitBlock 157 | { 158 | ::level1_mask(self, level0_index) 159 | } 160 | 161 | #[inline] 162 | unsafe fn data_mask(&self, level0_index: usize, level1_index: usize) 163 | -> ::DataBitBlock 164 | { 165 | ::data_mask(self, level0_index, level1_index) 166 | } 167 | } 168 | impl<'a, T: LevelMasksIterExt> LevelMasksIterExt for &'a T { 169 | type Level1BlockData = T::Level1BlockData; 170 | 171 | type IterState = T::IterState; 172 | 173 | #[inline] 174 | fn make_iter_state(&self) -> Self::IterState { 175 | ::make_iter_state(self) 176 | } 177 | 178 | #[inline] 179 | unsafe fn drop_iter_state(&self, cache: &mut ManuallyDrop) { 180 | ::drop_iter_state(self, cache) 181 | } 182 | 183 | #[inline] 184 | unsafe fn init_level1_block_data( 185 | &self, 186 | state: &mut Self::IterState, 187 | level1_blocks: &mut MaybeUninit, 188 | level0_index: usize 189 | ) -> (::Level1BitBlock, bool) { 190 | ::init_level1_block_data( 191 | self, state, level1_blocks, level0_index 192 | ) 193 | } 194 | 195 | #[inline] 196 | unsafe fn data_mask_from_block_data( 197 | level1_blocks: &Self::Level1BlockData, level1_index: usize 198 | ) -> ::DataBitBlock { 199 | ::data_mask_from_block_data( 200 | level1_blocks, level1_index 201 | ) 202 | } 203 | } 204 | 205 | // User-side interface 206 | /// Bitset interface. 207 | /// 208 | /// Implemented for bitset references and optionally for values. 209 | /// So as argument - accept BitSetInterface by value. 210 | /// _(Act as kinda forwarding reference in C++)_ 211 | /// 212 | /// # Traversing 213 | /// 214 | /// [CachingBlockIter] and [CachingIndexIter] have specialized `for_each()` implementation and `traverse()`. 215 | /// 216 | /// Like with most Rust iterators, traversing[^traverse_def] is somewhat faster 217 | /// then iteration. In this particular case, it has noticeable difference in micro-benchmarks. 218 | /// Remember, that iteration is already super-fast, and any tiny change become important at that scale. 219 | /// Hence, this will have effect in really tight loops (like incrementing counter). 220 | /// 221 | /// [^traverse_def]: Under "traverse" we understand function application for 222 | /// each element of bitset. 223 | /// 224 | /// # Implementation 225 | /// 226 | /// Consider using [impl_bitset!] instead of implementing it manually. 227 | /// 228 | /// Implementing BitSetInterface for T will make it passable by value to [apply], [reduce]. 229 | /// That may be not what you want, if your type contains heavy data, or your 230 | /// [LevelMasksIterExt] implementation depends on *Self being stable during iteration. 231 | /// If that is the case - implement only for &T. 232 | /// 233 | /// [CachingBlockIter]: crate::iter::CachingBlockIter 234 | /// [CachingIndexIter]: crate::iter::CachingIndexIter 235 | /// [LevelMasksIterExt]: crate::internals::LevelMasksIterExt 236 | /// [impl_bitset!]: crate::impl_bitset! 237 | /// [apply]: crate::apply() 238 | /// [reduce]: crate::reduce() 239 | pub unsafe trait BitSetInterface 240 | : BitSetBase 241 | + LevelMasksIterExt 242 | + IntoIterator> 243 | + Sized 244 | { 245 | #[inline] 246 | fn block_iter(&self) -> DefaultBlockIterator<&'_ Self> { 247 | DefaultBlockIterator::new(self) 248 | } 249 | 250 | #[inline] 251 | fn iter(&self) -> DefaultIndexIterator<&'_ Self> { 252 | DefaultIndexIterator::new(self) 253 | } 254 | 255 | #[inline] 256 | fn into_block_iter(self) -> DefaultBlockIterator { 257 | DefaultBlockIterator::new(self) 258 | } 259 | 260 | #[inline] 261 | fn contains(&self, index: usize) -> bool { 262 | bitset_contains(self, index) 263 | } 264 | 265 | /// O(1) if [TRUSTED_HIERARCHY], O(N) otherwise. 266 | /// 267 | /// [TRUSTED_HIERARCHY]: BitSetBase::TRUSTED_HIERARCHY 268 | #[inline] 269 | fn is_empty(&self) -> bool { 270 | bitset_is_empty(self) 271 | } 272 | } 273 | 274 | #[inline] 275 | pub(crate) fn bitset_contains(bitset: S, index: usize) -> bool { 276 | let (level0_index, level1_index, data_index) = 277 | level_indices::(index); 278 | unsafe{ 279 | let data_block = bitset.data_mask(level0_index, level1_index); 280 | data_block.get_bit(data_index) 281 | } 282 | } 283 | 284 | pub(crate) fn bitset_is_empty(bitset: S) -> bool { 285 | if S::TRUSTED_HIERARCHY{ 286 | return bitset.level0_mask().is_zero(); 287 | } 288 | 289 | use ControlFlow::*; 290 | DefaultBlockIterator::new(bitset).traverse(|block|{ 291 | if !block.is_empty(){ 292 | Break(()) 293 | } else { 294 | Continue(()) 295 | } 296 | }).is_continue() 297 | } 298 | 299 | /// Optimistic depth-first check. 300 | /// 301 | /// This traverse-based implementation is faster than using two iterators. 302 | pub(crate) fn bitsets_eq(left: L, right: R) -> bool 303 | where 304 | L: LevelMasksIterExt, 305 | R: LevelMasksIterExt, 306 | { 307 | let left_level0_mask = left.level0_mask(); 308 | let right_level0_mask = right.level0_mask(); 309 | 310 | // We can do early return with TrustedHierarchy. 311 | /*const*/ let is_trusted_hierarchy = L::TRUSTED_HIERARCHY & R::TRUSTED_HIERARCHY; 312 | 313 | let level0_mask = 314 | if is_trusted_hierarchy{ 315 | if left_level0_mask != right_level0_mask { 316 | return false; 317 | } 318 | left_level0_mask 319 | } else { 320 | // skip only 0's on both sides 321 | left_level0_mask | right_level0_mask 322 | }; 323 | 324 | let mut left_cache_data = left.make_iter_state(); 325 | let mut right_cache_data = right.make_iter_state(); 326 | 327 | let mut left_level1_blocks = MaybeUninit::new(Default::default()); 328 | let mut right_level1_blocks = MaybeUninit::new(Default::default()); 329 | 330 | use ControlFlow::*; 331 | let is_eq = level0_mask.traverse_bits(|level0_index|{ 332 | let (left_level1_mask, left_valid) = unsafe { 333 | left_level1_blocks.assume_init_drop(); 334 | left.init_level1_block_data(&mut left_cache_data, &mut left_level1_blocks, level0_index) 335 | }; 336 | let (right_level1_mask, right_valid) = unsafe { 337 | right_level1_blocks.assume_init_drop(); 338 | right.init_level1_block_data(&mut right_cache_data, &mut right_level1_blocks, level0_index) 339 | }; 340 | 341 | if is_trusted_hierarchy { 342 | unsafe{ 343 | assume!(left_valid); 344 | assume!(right_valid); 345 | } 346 | if left_level1_mask != right_level1_mask { 347 | return Break(()); 348 | } 349 | } 350 | 351 | if is_trusted_hierarchy || (left_valid & right_valid) { 352 | let level1_mask = 353 | if is_trusted_hierarchy { 354 | left_level1_mask 355 | } else{ 356 | left_level1_mask | right_level1_mask 357 | }; 358 | 359 | level1_mask.traverse_bits(|level1_index|{ 360 | let left_data = unsafe { 361 | L::data_mask_from_block_data(left_level1_blocks.assume_init_ref(), level1_index) 362 | }; 363 | let right_data = unsafe { 364 | R::data_mask_from_block_data(right_level1_blocks.assume_init_ref(), level1_index) 365 | }; 366 | 367 | if left_data == right_data{ 368 | Continue(()) 369 | } else { 370 | Break(()) 371 | } 372 | }) 373 | } else if left_valid /*right is zero*/ { 374 | if L::TRUSTED_HIERARCHY{ 375 | return if left_level1_mask.is_zero() { 376 | Continue(()) 377 | } else { 378 | Break(()) 379 | } 380 | } 381 | 382 | left_level1_mask.traverse_bits(|level1_index|{ 383 | let left_data = unsafe{ 384 | L::data_mask_from_block_data(left_level1_blocks.assume_init_ref(), level1_index) 385 | }; 386 | if left_data.is_zero() { 387 | Continue(()) 388 | } else { 389 | Break(()) 390 | } 391 | }) 392 | } else if right_valid /*left is zero*/ { 393 | if R::TRUSTED_HIERARCHY{ 394 | return if right_level1_mask.is_zero() { 395 | Continue(()) 396 | } else { 397 | Break(()) 398 | } 399 | } 400 | 401 | right_level1_mask.traverse_bits(|level1_index|{ 402 | let right_data = unsafe{ 403 | R::data_mask_from_block_data(right_level1_blocks.assume_init_ref(), level1_index) 404 | }; 405 | if right_data.is_zero() { 406 | Continue(()) 407 | } else { 408 | Break(()) 409 | } 410 | }) 411 | } else { 412 | // both are empty - its ok - just move on. 413 | Continue(()) 414 | } 415 | }).is_continue(); 416 | 417 | unsafe { 418 | left_level1_blocks.assume_init_drop(); 419 | right_level1_blocks.assume_init_drop(); 420 | } 421 | 422 | is_eq 423 | } -------------------------------------------------------------------------------- /src/block.rs: -------------------------------------------------------------------------------- 1 | use std::mem::{MaybeUninit, size_of}; 2 | use crate::bit_block::BitBlock; 3 | use crate::{Primitive, PrimitiveArray}; 4 | use crate::level::IBlock; 5 | 6 | #[derive(Clone)] 7 | pub struct Block { 8 | mask: Mask, 9 | /// Next level block indices 10 | block_indices: BlockIndices, 11 | } 12 | 13 | impl Default for Block 14 | where 15 | Mask: BitBlock, 16 | BlockIndices: PrimitiveArray 17 | { 18 | #[inline] 19 | fn default() -> Self { 20 | Self { 21 | mask: Mask::zero(), 22 | // All indices 0. 23 | block_indices: unsafe{MaybeUninit::zeroed().assume_init()} 24 | } 25 | } 26 | } 27 | 28 | impl IBlock for Block 29 | where 30 | Mask: BitBlock, 31 | BlockIndices: PrimitiveArray 32 | { 33 | type Mask = Mask; 34 | 35 | #[inline] 36 | fn mask(&self) -> &Self::Mask { 37 | &self.mask 38 | } 39 | 40 | #[inline] 41 | unsafe fn mask_mut(&mut self) -> &mut Self::Mask { 42 | &mut self.mask 43 | } 44 | 45 | type Item = BlockIndices::Item; 46 | 47 | /// Same as `get_unchecked` 48 | #[inline] 49 | unsafe fn get_or_zero(&self, index: usize) -> Self::Item { 50 | let block_indices = self.block_indices.as_ref(); 51 | *block_indices.get_unchecked(index) 52 | } 53 | 54 | #[inline] 55 | unsafe fn get_or_insert(&mut self, index: usize, mut f: impl FnMut() -> Self::Item) -> Self::Item { 56 | // mask 57 | let exists = self.mask.set_bit::(index); 58 | 59 | // indices 60 | let block_indices = self.block_indices.as_mut(); 61 | if exists { 62 | *block_indices.get_unchecked(index) 63 | } else { 64 | let block_index = f(); 65 | *block_indices.get_unchecked_mut(index) = block_index; 66 | block_index 67 | } 68 | } 69 | 70 | #[inline] 71 | unsafe fn remove_unchecked(&mut self, index: usize) { 72 | // mask 73 | self.mask.set_bit::(index); 74 | // If we have block_indices section (compile-time check) 75 | if !size_of::().is_zero(){ 76 | let block_indices = self.block_indices.as_mut(); 77 | *block_indices.get_unchecked_mut(index) = Primitive::ZERO; 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /src/cache.rs: -------------------------------------------------------------------------------- 1 | //! Cache used by [CachingBlockIter] for [reduce] operations. 2 | //! 3 | //! # Memory footprint 4 | //! 5 | //! Cache for one [BitSet] costs 2 pointers. 6 | //! 7 | //! [BitSet]: crate::BitSet 8 | //! [CachingBlockIter]: crate::iter::CachingBlockIter 9 | //! [reduce]: crate::reduce() 10 | 11 | use crate::ops::BitSetOp; 12 | use crate::bitset_interface::{BitSetBase, LevelMasksIterExt}; 13 | use crate::reduce::{DynamicCacheImpl, FixedCacheImpl, NonCachedImpl, ReduceCacheImpl}; 14 | 15 | /// Cache is not used. 16 | /// 17 | /// If reduced iterator contains other nested reduce operations - all of them 18 | /// WILL NOT use cache as well. 19 | /// 20 | /// # Example 1 21 | /// 22 | /// ``` 23 | /// # use itertools::assert_equal; 24 | /// # use hi_sparse_bitset::{reduce, reduce_w_cache}; 25 | /// # use hi_sparse_bitset::ops::{And, Or}; 26 | /// # use hi_sparse_bitset::cache::NoCache; 27 | /// # type BitSet = hi_sparse_bitset::BitSet; 28 | /// let su1 = [BitSet::from([1,2]), BitSet::from([5,6])]; 29 | /// let union1 = reduce(Or, su1.iter()).unwrap(); 30 | /// 31 | /// let su2 = [BitSet::from([1,3]), BitSet::from([4,6])]; 32 | /// let union2 = reduce(Or, su2.iter()).unwrap(); 33 | /// 34 | /// let si = [union1, union2]; 35 | /// let intersection = reduce_w_cache(And, si.iter(), NoCache).unwrap(); 36 | /// 37 | /// // Not only `intersection` will be computed without cache, 38 | /// // but also `union1` and `union2`. 39 | /// assert_equal(intersection, [1,6]); 40 | /// 41 | /// ``` 42 | /// 43 | /// # Example 2 44 | /// 45 | /// ``` 46 | /// # use itertools::assert_equal; 47 | /// # use hi_sparse_bitset::{reduce, reduce_w_cache}; 48 | /// # use hi_sparse_bitset::ops::{And, Or}; 49 | /// # use hi_sparse_bitset::cache::NoCache; 50 | /// # type BitSet = hi_sparse_bitset::BitSet; 51 | /// let su1 = [BitSet::from([1,2]), BitSet::from([5,6])]; 52 | /// let union1 = reduce_w_cache(Or, su1.iter(), NoCache).unwrap(); 53 | /// 54 | /// let su2 = [BitSet::from([1,3]), BitSet::from([4,6])]; 55 | /// let union2 = reduce_w_cache(Or, su2.iter(), NoCache).unwrap(); 56 | /// 57 | /// let si = [union1, union2]; 58 | /// let intersection = reduce(And, si.iter()).unwrap(); 59 | /// 60 | /// // Only `union1` and `union2` will not use cache, `intersection` 61 | /// // will be computed with cache. 62 | /// assert_equal(intersection, [1,6]); 63 | /// 64 | /// ``` 65 | /// 66 | /// [reduce]: crate::reduce() 67 | #[derive(Default, Copy, Clone)] 68 | pub struct NoCache; 69 | 70 | /// Cache with fixed capacity. 71 | /// 72 | /// This cache is noop to construct. 73 | /// Should be your default choice. 74 | /// 75 | /// N.B. Pay attention to stack-mem usage when working with 76 | /// reduce on reduce on reduce ... 77 | #[derive(Default, Copy, Clone)] 78 | pub struct FixedCache; 79 | 80 | /// Dynamically built in-heap cache. 81 | /// 82 | /// You want this, when your cache doesn't fit stack. 83 | /// This can happened, when you work with enormously large number of sets, 84 | /// and/or work with deep [reduce] operations. Alternatively, you 85 | /// can use [NoCache]. 86 | /// 87 | /// [reduce]: crate::reduce() 88 | #[derive(Default, Copy, Clone)] 89 | pub struct DynamicCache; 90 | 91 | pub trait ReduceCache: Default + 'static{ 92 | /// usize::MAX - if unlimited. 93 | const MAX_LEN: usize; 94 | type Impl 95 | : ReduceCacheImpl< 96 | Sets = S, 97 | Conf = ::Conf 98 | > 99 | where 100 | Op: BitSetOp, 101 | S: Iterator + Clone, 102 | S::Item: LevelMasksIterExt; 103 | } 104 | 105 | impl ReduceCache for NoCache{ 106 | const MAX_LEN: usize = usize::MAX; 107 | type Impl = NonCachedImpl 108 | where 109 | Op: BitSetOp, 110 | S: Iterator + Clone, 111 | S::Item: LevelMasksIterExt; 112 | } 113 | 114 | impl ReduceCache for FixedCache{ 115 | const MAX_LEN: usize = N; 116 | type Impl = FixedCacheImpl 117 | where 118 | Op: BitSetOp, 119 | S: Iterator + Clone, 120 | S::Item: LevelMasksIterExt; 121 | } 122 | 123 | impl ReduceCache for DynamicCache{ 124 | const MAX_LEN: usize = usize::MAX; 125 | type Impl = DynamicCacheImpl 126 | where 127 | Op: BitSetOp, 128 | S: Iterator + Clone, 129 | S::Item: LevelMasksIterExt; 130 | } -------------------------------------------------------------------------------- /src/compact_block.rs: -------------------------------------------------------------------------------- 1 | use std::mem::{ManuallyDrop, MaybeUninit}; 2 | use std::ops::{Deref, DerefMut}; 3 | use std::ops::ControlFlow::Continue; 4 | use std::ptr; 5 | use crate::BitBlock; 6 | use crate::level::IBlock; 7 | use crate::primitive::Primitive; 8 | use crate::primitive_array::{PrimitiveArray, UninitPrimitiveArray}; 9 | 10 | #[repr(C)] 11 | union BigSmallArray 12 | where 13 | BlockIndices: PrimitiveArray, 14 | SmallBlockIndices: PrimitiveArray, 15 | MaskU64Populations: PrimitiveArray, 16 | { 17 | big: (u8, ManuallyDrop>), 18 | 19 | /// First element in `MaskU64Populations` is always 0. 20 | /// 21 | /// SmallBlockIndices len = MaskU64Populations.last() + mask.last().count_ones(). 22 | small: (MaskU64Populations, SmallBlockIndices::UninitArray) 23 | } 24 | 25 | impl From> for BigSmallArray 26 | where 27 | BlockIndices: PrimitiveArray, 28 | SmallBlockIndices: PrimitiveArray, 29 | MaskU64Populations: PrimitiveArray 30 | { 31 | #[inline] 32 | fn from(array: Box) -> Self { 33 | Self{ 34 | big: (1, ManuallyDrop::new(array)) 35 | } 36 | } 37 | } 38 | 39 | impl From<(MaskU64Populations, SmallBlockIndices::UninitArray)> for BigSmallArray 40 | where 41 | BlockIndices: PrimitiveArray, 42 | SmallBlockIndices: PrimitiveArray, 43 | MaskU64Populations: PrimitiveArray 44 | { 45 | #[inline] 46 | fn from(small: (MaskU64Populations, SmallBlockIndices::UninitArray)) -> Self { 47 | debug_assert!(small.0.as_ref().first().unwrap().is_zero()); 48 | Self{ small } 49 | } 50 | } 51 | 52 | impl Clone for BigSmallArray 53 | where 54 | BlockIndices: PrimitiveArray, 55 | SmallBlockIndices: PrimitiveArray, 56 | MaskU64Populations: PrimitiveArray 57 | { 58 | #[inline] 59 | fn clone(&self) -> Self { 60 | unsafe{ 61 | if self.is_big(){ 62 | Self{big: (1, self.big.1.clone())} 63 | } else { 64 | Self{small: self.small} 65 | } 66 | } 67 | } 68 | } 69 | 70 | impl BigSmallArray 71 | where 72 | BlockIndices: PrimitiveArray, 73 | SmallBlockIndices: PrimitiveArray, 74 | MaskU64Populations: PrimitiveArray, 75 | { 76 | #[inline] 77 | fn is_small(&self) -> bool { 78 | unsafe{ self.big.0 == 0 } 79 | } 80 | #[inline] 81 | fn is_big(&self) -> bool { 82 | !self.is_small() 83 | } 84 | } 85 | 86 | impl Drop for BigSmallArray 87 | where 88 | BlockIndices: PrimitiveArray, 89 | SmallBlockIndices: PrimitiveArray, 90 | MaskU64Populations: PrimitiveArray 91 | { 92 | #[inline] 93 | fn drop(&mut self) { 94 | if self.is_big(){ 95 | unsafe{ ManuallyDrop::drop(&mut self.big.1); } 96 | } 97 | } 98 | } 99 | 100 | #[derive(Clone)] 101 | pub struct CompactBlock 102 | where 103 | BlockIndices: PrimitiveArray, 104 | SmallBlockIndices: PrimitiveArray, 105 | MaskU64Populations: PrimitiveArray, 106 | { 107 | mask: Mask, 108 | big_small: BigSmallArray 109 | } 110 | 111 | impl Default for CompactBlock 112 | where 113 | Mask: BitBlock, 114 | BlockIndices: PrimitiveArray, 115 | SmallBlockIndices: PrimitiveArray, 116 | MaskU64Populations: PrimitiveArray, 117 | { 118 | #[inline] 119 | fn default() -> Self { 120 | Self{ 121 | mask: Mask::zero(), 122 | big_small: 123 | BigSmallArray::from( 124 | ( 125 | /*mask_u64_populations:*/ unsafe{MaybeUninit::zeroed().assume_init()}, 126 | /*array:*/ SmallBlockIndices::UninitArray::uninit_array() 127 | ) 128 | ) 129 | } 130 | } 131 | } 132 | 133 | impl CompactBlock 134 | where 135 | Mask: BitBlock, 136 | MaskU64Populations: PrimitiveArray, 137 | BlockIndices: PrimitiveArray, 138 | SmallBlockIndices: PrimitiveArray, 139 | { 140 | /// number of 1 bits in mask before `index` bit. 141 | /// 142 | /// # Safety 143 | /// 144 | /// * small must be active. 145 | /// * `index` must be set. 146 | #[inline] 147 | unsafe fn small_array_index(mask_u64_populations: &MaskU64Populations, mask: &Mask, index: usize) 148 | -> usize 149 | { 150 | let u64_index = index / 64; 151 | let bit_index = index % 64; 152 | let mut block = *mask.as_array().get_unchecked(u64_index); 153 | let mask = !(u64::MAX << bit_index); 154 | block &= mask; 155 | let offset = *mask_u64_populations.as_ref().get_unchecked(u64_index); 156 | offset as usize + block.count_ones() as usize 157 | } 158 | 159 | /// # Safety 160 | /// 161 | /// * `index` must not be set. 162 | /// * `mask`'s corresponding bit must be 0. 163 | #[inline] 164 | unsafe fn insert_unchecked(&mut self, index: usize, value: BlockIndices::Item){ 165 | if self.big_small.is_big(){ 166 | let array = self.big_small.big.1.deref_mut(); 167 | *array.deref_mut().as_mut().get_unchecked_mut(index) = value; 168 | } else { 169 | let (mask_u64_populations, array) = &mut self.big_small.small; 170 | let len = *mask_u64_populations.as_ref().last().unwrap_unchecked() as usize + self.mask.as_array().last().unwrap_unchecked().count_ones() as usize; 171 | if len == SmallBlockIndices::CAP { 172 | // TODO: as non-inline function? 173 | // move to Big 174 | let mut big: Box = Box::new(unsafe{MaybeUninit::zeroed().assume_init()}); 175 | let big_array = big.deref_mut().as_mut(); 176 | let mut i = 0; 177 | self.mask.traverse_bits(|index|{ 178 | let value = array.as_ref().get_unchecked(i).assume_init_read(); 179 | i += 1; 180 | 181 | *big_array.get_unchecked_mut(index) = value; 182 | Continue(()) 183 | }); 184 | *big_array.get_unchecked_mut(index) = value; 185 | self.big_small = BigSmallArray::from(big); 186 | } else { 187 | let inner_index = Self::small_array_index(mask_u64_populations, &self.mask, index); 188 | unsafe{ 189 | let p: *mut _ = array.as_mut().as_mut_ptr().add(inner_index); 190 | // Shift everything over to make space. (Duplicating the 191 | // `index`th element into two consecutive places.) 192 | ptr::copy(p, p.offset(1), len - inner_index); 193 | // Write it in, overwriting the first copy of the `index`th 194 | // element. 195 | ptr::write(p, MaybeUninit::new(value)); 196 | } 197 | 198 | for i in (index/64)+1..Mask::size()/64 { 199 | *mask_u64_populations.as_mut().get_unchecked_mut(i) += 1; 200 | } 201 | } 202 | } 203 | self.mask.set_bit::(index); 204 | } 205 | } 206 | 207 | impl IBlock for CompactBlock 208 | where 209 | Mask: BitBlock, 210 | MaskU64Populations: PrimitiveArray, 211 | BlockIndices: PrimitiveArray, 212 | SmallBlockIndices: PrimitiveArray, 213 | { 214 | type Mask = Mask; 215 | 216 | #[inline] 217 | fn mask(&self) -> &Self::Mask { 218 | &self.mask 219 | } 220 | 221 | #[inline] 222 | unsafe fn mask_mut(&mut self) -> &mut Self::Mask { 223 | &mut self.mask 224 | } 225 | 226 | type Item = BlockIndices::Item; 227 | 228 | #[inline] 229 | unsafe fn get_or_zero(&self, index: usize) -> Self::Item { 230 | if self.big_small.is_big(){ 231 | let array = self.big_small.big.1.deref(); 232 | *array.deref().as_ref().get_unchecked(index) 233 | } else { 234 | let (mask_u64_populations, array) = &self.big_small.small; 235 | let u64_index = index / 64; 236 | let bit_index = index % 64; 237 | let mut block = *self.mask.as_array().get_unchecked(u64_index); 238 | 239 | { 240 | let block_mask: u64 = 1 << bit_index; 241 | let masked_block = block & block_mask; 242 | if masked_block.is_zero(){ 243 | return Primitive::ZERO; 244 | } 245 | } 246 | 247 | let mask = !(u64::MAX << bit_index); 248 | block &= mask; 249 | 250 | let offset = *mask_u64_populations.as_ref().get_unchecked(u64_index); 251 | let small_array_index = offset as usize + block.count_ones() as usize; 252 | array.as_ref().get_unchecked(small_array_index).assume_init_read() 253 | } 254 | } 255 | 256 | #[inline] 257 | unsafe fn get_or_insert(&mut self, index: usize, mut f: impl FnMut() -> Self::Item) -> Self::Item { 258 | let mut block_index = self.get_or_zero(index); 259 | if block_index.is_zero(){ 260 | block_index = f(); 261 | self.insert_unchecked(index, block_index); 262 | } 263 | block_index 264 | } 265 | 266 | #[inline] 267 | unsafe fn remove_unchecked(&mut self, index: usize) { 268 | let prev = self.mask.set_bit::(index); 269 | debug_assert!(prev); 270 | 271 | if self.big_small.is_big(){ 272 | let array = self.big_small.big.1.deref_mut(); 273 | // TODO: go back to small at small/2 size? 274 | *array.deref_mut().as_mut().get_unchecked_mut(index) = Primitive::ZERO; 275 | } else { 276 | let (mask_u64_populations, array) = &mut self.big_small.small; 277 | let len = *mask_u64_populations.as_ref().last().unwrap_unchecked() as usize + self.mask.as_array().last().unwrap_unchecked().count_ones() as usize; 278 | let inner_index = Self::small_array_index(mask_u64_populations, &self.mask, index); 279 | 280 | unsafe{ 281 | let p: *mut _ = array.as_mut().as_mut_ptr().add(inner_index); 282 | ptr::copy(p.offset(1), p, len - inner_index); 283 | } 284 | 285 | for i in (index/64)+1..Mask::size()/64 { 286 | *mask_u64_populations.as_mut().get_unchecked_mut(i) -= 1; 287 | } 288 | } 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | //! Configurations for [BitSet]. 2 | //! 3 | //! Increasing block size will increase max index [BitSet] can hold. 4 | //! Decreasing block size will lower memory footprint. 5 | //! 6 | //! For your task, you can make specialized config. For example, if you're 7 | //! not limited by MAX index, and know that your indices will be dense, 8 | //! you can try 64/64/256 bit levels. 9 | //! 10 | //! [BitSet]: crate::BitSet 11 | 12 | use std::marker::PhantomData; 13 | use crate::bit_block::BitBlock; 14 | use crate::cache; 15 | use crate::cache::ReduceCache; 16 | use crate::primitive_array::PrimitiveArray; 17 | use crate::iter::{CachingBlockIter, CachingIndexIter}; 18 | 19 | type DefaultCache = cache::FixedCache<32>; 20 | pub(crate) type DefaultBlockIterator = CachingBlockIter; 21 | pub(crate) type DefaultIndexIterator = CachingIndexIter; 22 | 23 | /// [BitSet] configuration 24 | /// 25 | /// [BitSet]: crate::BitSet 26 | pub trait Config: 'static { 27 | // Level 0 28 | /// BitBlock used as bitmask for level 0. 29 | type Level0BitBlock: BitBlock; 30 | 31 | /// Contiguous container, used as indirection storage for level 0. 32 | /// 33 | /// * Must be big enough to accommodate at least [Level0BitBlock]::size(). 34 | /// * Item should be able to store [Level0BitBlock]::size() integer. 35 | /// 36 | /// [Level0BitBlock]: Self::Level0BitBlock 37 | type Level0BlockIndices: PrimitiveArray; 38 | 39 | // Level 1 40 | // There can be maximum [Level0BitBlock]::size() level1 blocks 41 | 42 | /// BitBlock used as bitmask for level 1 block. 43 | type Level1BitBlock: BitBlock; 44 | 45 | /// Contiguous container, used as indirection storage for level 1 block. 46 | /// 47 | /// * Must be big enough to accommodate at least [Level1BitBlock]::size(). 48 | /// * Item should be able to store [Level0BitBlock]::size() * [Level1BitBlock]::size() integer. 49 | /// 50 | /// [Level0BitBlock]: Self::Level0BitBlock 51 | /// [Level1BitBlock]: Self::Level1BitBlock 52 | type Level1BlockIndices: PrimitiveArray; 53 | 54 | // Level data 55 | // There can be maximum [Level0BitBlock]::SIZE * [Level1BitBlock]::SIZE data level blocks 56 | 57 | /// BitBlock used as bitmask for data level block. 58 | type DataBitBlock: BitBlock; 59 | 60 | // Other 61 | 62 | /// Cache used be [reduce()]. 63 | /// 64 | /// [reduce()]: crate::reduce() 65 | type DefaultCache: ReduceCache; 66 | } 67 | 68 | #[inline] 69 | pub(crate) const fn max_addressable_index() -> usize { 70 | (1 << Conf::Level0BitBlock::SIZE_POT_EXPONENT) 71 | * (1 << Conf::Level1BitBlock::SIZE_POT_EXPONENT) 72 | * (1 << Conf::DataBitBlock::SIZE_POT_EXPONENT) 73 | } 74 | 75 | /// [SmallBitSet] configuration. 76 | /// 77 | /// Try to keep level1 block small. Remember that [Level1BitBlock] has huge align. 78 | /// Try to keep [Level1MaskU64Populations] + [Level1SmallBlockIndices] size within 79 | /// SIMD align. 80 | /// 81 | /// [SmallBitSet]: crate::SmallBitSet 82 | /// [Level1BitBlock]: Config::Level1BitBlock 83 | /// [Level1MaskU64Populations]: Self::Level1MaskU64Populations 84 | /// [Level1SmallBlockIndices]: Self::Level1SmallBlockIndices 85 | pub trait SmallConfig: Config { 86 | /// Small(inlined) buffer for level 1 block indices. 87 | type Level1SmallBlockIndices: PrimitiveArray< 88 | Item = <::Level1BlockIndices as PrimitiveArray>::Item 89 | >; 90 | 91 | /// Mask's bit-population at the start of each u64 block. 92 | /// Should be [u8; Self::Mask::size()/64]. 93 | /// 94 | /// P.S. It is OK to use u8 even for bitblocks bigger than 256 bits, 95 | /// since this array is used only when "small buffer" in use, which should 96 | /// be less than 256 elements anyway. 97 | /// 98 | /// P.P.S. Should be deductible from Mask, but THE RUST... 99 | type Level1MaskU64Populations: PrimitiveArray; 100 | } 101 | 102 | /// MAX = 262_144 103 | #[derive(Default)] 104 | pub struct _64bit(PhantomData); 105 | impl Config for _64bit { 106 | type Level0BitBlock = u64; 107 | type Level0BlockIndices = [u8; 64]; 108 | 109 | type Level1BitBlock = u64; 110 | type Level1BlockIndices = [u16; 64]; 111 | 112 | type DataBitBlock = u64; 113 | 114 | type DefaultCache = DefaultCache; 115 | } 116 | impl SmallConfig for _64bit { 117 | type Level1SmallBlockIndices = [u16;7]; 118 | type Level1MaskU64Populations = [u8;1]; 119 | } 120 | 121 | /// MAX = 2_097_152 122 | #[cfg(feature = "simd")] 123 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 124 | #[derive(Default)] 125 | pub struct _128bit(PhantomData); 126 | #[cfg(feature = "simd")] 127 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 128 | impl Config for _128bit { 129 | type Level0BitBlock = wide::u64x2; 130 | type Level0BlockIndices = [u8; 128]; 131 | 132 | type Level1BitBlock = wide::u64x2; 133 | type Level1BlockIndices = [u16; 128]; 134 | 135 | type DataBitBlock = wide::u64x2; 136 | 137 | type DefaultCache = DefaultCache; 138 | } 139 | #[cfg(feature = "simd")] 140 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 141 | impl SmallConfig for _128bit { 142 | type Level1SmallBlockIndices = [u16;7]; 143 | type Level1MaskU64Populations = [u8;2]; 144 | } 145 | 146 | /// MAX = 16_777_216 147 | #[cfg(feature = "simd")] 148 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 149 | #[derive(Default)] 150 | pub struct _256bit(PhantomData); 151 | #[cfg(feature = "simd")] 152 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 153 | impl Config for _256bit { 154 | type Level0BitBlock = wide::u64x4; 155 | type Level0BlockIndices = [u8; 256]; 156 | 157 | type Level1BitBlock = wide::u64x4; 158 | type Level1BlockIndices = [u16; 256]; 159 | 160 | type DataBitBlock = wide::u64x4; 161 | 162 | type DefaultCache = DefaultCache; 163 | } 164 | #[cfg(feature = "simd")] 165 | #[cfg_attr(docsrs, doc(cfg(feature = "simd")))] 166 | impl SmallConfig for _256bit { 167 | type Level1SmallBlockIndices = [u16;14]; 168 | type Level1MaskU64Populations = [u8;4]; 169 | } -------------------------------------------------------------------------------- /src/derive_raw.rs: -------------------------------------------------------------------------------- 1 | //! New type idiom wrapping for RawBitSet. 2 | 3 | /// * `$t` Must be Self(RawBitSet) 4 | /// * `$t` Must implement BitSetBase 5 | macro_rules! derive_raw { 6 | (impl <$($generics:tt),*> 7 | $t:ty as 8 | $raw:ty 9 | where $($where_bounds:tt)* 10 | ) => { 11 | impl<$($generics),*> $t 12 | where 13 | $($where_bounds)* 14 | { 15 | #[inline] 16 | pub fn new() -> Self { 17 | Default::default() 18 | } 19 | 20 | /// Max usize, bitset with this `Conf` can hold. 21 | #[inline] 22 | pub const fn max_capacity() -> usize { 23 | <$raw>::max_capacity() 24 | } 25 | 26 | /// # Safety 27 | /// 28 | /// Will panic, if `index` is out of range. 29 | #[inline] 30 | pub fn insert(&mut self, index: usize){ 31 | self.0.insert(index) 32 | } 33 | 34 | /// Returns false if index is invalid/not in bitset. 35 | #[inline] 36 | pub fn remove(&mut self, index: usize) -> bool { 37 | self.0.remove(index) 38 | } 39 | 40 | /// # Safety 41 | /// 42 | /// `index` MUST exists in HiSparseBitset! 43 | #[inline] 44 | pub unsafe fn remove_unchecked(&mut self, index: usize) { 45 | // TODO: make sure compiler actually get rid of unused code. 46 | let ok = self.remove(index); 47 | unsafe{ $crate::assume!(ok); } 48 | } 49 | } 50 | 51 | impl<$($generics),*> Clone for $t 52 | where 53 | $($where_bounds)* 54 | { 55 | #[inline] 56 | fn clone(&self) -> Self { 57 | Self(self.0.clone()) 58 | } 59 | } 60 | 61 | impl<$($generics),*> Default for $t 62 | where 63 | $($where_bounds)* 64 | { 65 | #[inline] 66 | fn default() -> Self{ 67 | Self(Default::default()) 68 | } 69 | } 70 | 71 | impl<$($generics),*> FromIterator for $t 72 | where 73 | $($where_bounds)* 74 | { 75 | #[inline] 76 | fn from_iter>(iter: T) -> Self { 77 | Self(<$raw>::from_iter(iter)) 78 | } 79 | } 80 | 81 | impl<$($generics),* , const N: usize> From<[usize; N]> for $t 82 | where 83 | $($where_bounds)* 84 | { 85 | #[inline] 86 | fn from(value: [usize; N]) -> Self { 87 | Self(<$raw>::from(value)) 88 | } 89 | } 90 | 91 | crate::derive_raw::derive_raw_levelmasks!( 92 | impl<$($generics),*> $t as $raw where $($where_bounds)* 93 | ); 94 | 95 | crate::internals::impl_bitset!(impl<$($generics),*> for ref $t where $($where_bounds)*); 96 | } 97 | } 98 | pub(crate) use derive_raw; 99 | 100 | 101 | /// * `$t` Must be Self(RawBitSet) 102 | /// * `$t` Must implement BitSetBase 103 | macro_rules! derive_raw_levelmasks { 104 | (impl <$($generics:tt),*> 105 | $t:ty as 106 | $raw:ty 107 | where $($where_bounds:tt)* 108 | ) => { 109 | impl<$($generics),*> $crate::internals::LevelMasks for $t 110 | where 111 | $($where_bounds)* 112 | { 113 | #[inline] 114 | fn level0_mask(&self) -> ::Level0BitBlock { 115 | self.0.level0_mask() 116 | } 117 | 118 | #[inline] 119 | unsafe fn level1_mask(&self, level0_index: usize) -> ::Level1BitBlock { 120 | self.0.level1_mask(level0_index) 121 | } 122 | 123 | #[inline] 124 | unsafe fn data_mask(&self, level0_index: usize, level1_index: usize) -> ::DataBitBlock { 125 | self.0.data_mask(level0_index, level1_index) 126 | } 127 | } 128 | 129 | impl<$($generics),*> $crate::internals::LevelMasksIterExt for $t 130 | where 131 | $($where_bounds)* 132 | { 133 | type IterState = <$raw as $crate::internals::LevelMasksIterExt>::IterState; 134 | type Level1BlockData = <$raw as $crate::internals::LevelMasksIterExt>::Level1BlockData; 135 | 136 | #[inline] 137 | fn make_iter_state(&self) -> Self::IterState { 138 | self.0.make_iter_state() 139 | } 140 | 141 | #[inline] 142 | unsafe fn drop_iter_state(&self, state: &mut std::mem::ManuallyDrop) { 143 | self.0.drop_iter_state(state) 144 | } 145 | 146 | #[inline] 147 | unsafe fn init_level1_block_data( 148 | &self, 149 | state: &mut Self::IterState, 150 | level1_block_data: &mut std::mem::MaybeUninit, 151 | level0_index: usize 152 | ) -> (::Level1BitBlock, bool) { 153 | self.0.init_level1_block_data(state, level1_block_data, level0_index) 154 | } 155 | 156 | #[inline] 157 | unsafe fn data_mask_from_block_data( 158 | level1_block_data: &Self::Level1BlockData, 159 | level1_index: usize 160 | ) -> ::DataBitBlock { 161 | <$raw>::data_mask_from_block_data(level1_block_data, level1_index) 162 | } 163 | } 164 | } 165 | } 166 | pub(crate) use derive_raw_levelmasks; -------------------------------------------------------------------------------- /src/internals.rs: -------------------------------------------------------------------------------- 1 | //! Implementation details for customization. 2 | //! 3 | //! # Custom bitblock 4 | //! 5 | //! If your target architecture have some specific SIMD registers, which 6 | //! you want to use as bitblocks, or you just want to have wider bitblocks to 7 | //! increase [BitSet] range - you can do this: 8 | //! * Implement [BitBlock] for your type. 9 | //! * Make [Config] with your bitblocks. 10 | //! * Use that config with [BitSet]. 11 | //! 12 | //! [BitSet]: crate::BitSet 13 | //! [BitBlock]: crate::BitBlock 14 | //! [Config]: crate::config::Config 15 | //! 16 | //! # Custom bitset 17 | //! 18 | //! You can make generative bitsets, like 19 | //! "empty", "full", "range-fill", etc. with virtually zero memory overhead 20 | //! and instant construction. 21 | //! 22 | //! Use [impl_bitset!] to make bitset from [LevelMasksIterExt]. 23 | //! Use [impl_simple_bitset!] to make bitset from [LevelMasks]. 24 | //! Otherwise, you can manually implement [BitSetInterface], and optionally other traits. 25 | //! 26 | //! [BitSetInterface]: crate::BitSetInterface 27 | //! [impl_bitset!]: crate::impl_bitset 28 | //! [impl_simple_bitset!]: crate::impl_bitset_simple 29 | //! 30 | //! ``` 31 | //! # use std::marker::PhantomData; 32 | //! # use std::mem::{ManuallyDrop, MaybeUninit}; 33 | //! # use hi_sparse_bitset::config::Config; 34 | //! # use hi_sparse_bitset::{BitBlock, BitSetBase, BitSetInterface, impl_bitset}; 35 | //! # use hi_sparse_bitset::internals::*; 36 | //! #[derive(Default)] 37 | //! struct Empty(PhantomData); 38 | //! 39 | //! impl BitSetBase for Empty { 40 | //! type Conf = Conf; 41 | //! const TRUSTED_HIERARCHY: bool = true; 42 | //! } 43 | //! 44 | //! impl LevelMasks for Empty { 45 | //! fn level0_mask(&self) -> ::Level0BitBlock { 46 | //! BitBlock::zero() 47 | //! } 48 | //! 49 | //! unsafe fn level1_mask(&self, _: usize) 50 | //! -> ::Level1BitBlock 51 | //! { 52 | //! BitBlock::zero() 53 | //! } 54 | //! 55 | //! unsafe fn data_mask(&self, _: usize, _: usize) 56 | //! -> ::DataBitBlock 57 | //! { 58 | //! BitBlock::zero() 59 | //! } 60 | //! } 61 | //! 62 | //! // This is not needed with impl_bitset_simple! 63 | //! impl LevelMasksIterExt for Empty { 64 | //! type IterState = (); 65 | //! type Level1BlockData = (); 66 | //! 67 | //! fn make_iter_state(&self) -> Self::IterState { () } 68 | //! unsafe fn drop_iter_state(&self, _: &mut ManuallyDrop) {} 69 | //! 70 | //! unsafe fn init_level1_block_data( 71 | //! &self, 72 | //! _: &mut Self::IterState, 73 | //! _: &mut MaybeUninit, 74 | //! _: usize 75 | //! ) -> (::Level1BitBlock, bool) { 76 | //! (BitBlock::zero(), false) 77 | //! } 78 | //! 79 | //! unsafe fn data_mask_from_block_data( 80 | //! _: &Self::Level1BlockData, _: usize 81 | //! ) -> ::DataBitBlock { 82 | //! BitBlock::zero() 83 | //! } 84 | //! } 85 | //! 86 | //! impl_bitset!( 87 | //! impl for Empty where Conf: Config 88 | //! ); 89 | //! ``` 90 | //! 91 | //! See: 92 | //! * examples/custom_bitset_simple.rs 93 | //! * examples/custom_bitset.rs 94 | 95 | use crate::bitset_interface::{bitset_is_empty, bitsets_eq, bitset_contains}; 96 | use crate::config::{DefaultBlockIterator, DefaultIndexIterator}; 97 | use crate::bitset_interface::BitSetInterface; 98 | 99 | #[cfg_attr(docsrs, doc(cfg(feature = "impl")))] 100 | #[cfg(feature = "impl")] 101 | pub use crate::bitset_interface::LevelMasks; 102 | #[cfg(not(feature = "impl"))] 103 | pub(crate) use crate::bitset_interface::LevelMasks; 104 | 105 | #[cfg_attr(docsrs, doc(cfg(feature = "impl")))] 106 | #[cfg(feature = "impl")] 107 | pub use crate::bitset_interface::LevelMasksIterExt; 108 | #[cfg(not(feature = "impl"))] 109 | pub(crate) use crate::bitset_interface::LevelMasksIterExt; 110 | 111 | pub use crate::primitive::Primitive; 112 | pub use crate::primitive_array::PrimitiveArray; 113 | 114 | pub mod bit_queue{ 115 | pub use crate::bit_queue::*; 116 | } 117 | 118 | #[inline] 119 | pub fn into_index_iter(set: T) -> DefaultIndexIterator 120 | where 121 | T: BitSetInterface 122 | { 123 | DefaultIndexIterator::new(set) 124 | } 125 | 126 | #[inline] 127 | pub fn index_iter<'a, T>(set: &'a T) -> DefaultIndexIterator<&'a T> 128 | where 129 | &'a T: BitSetInterface 130 | { 131 | DefaultIndexIterator::new(set) 132 | } 133 | 134 | #[allow(dead_code)] 135 | #[inline] 136 | pub fn into_block_iter(set: T) -> DefaultBlockIterator 137 | where 138 | T: BitSetInterface 139 | { 140 | DefaultBlockIterator::new(set) 141 | } 142 | 143 | #[inline] 144 | pub fn block_iter<'a, T>(set: &'a T) -> DefaultBlockIterator<&'a T> 145 | where 146 | &'a T: BitSetInterface 147 | { 148 | DefaultBlockIterator::new(set) 149 | } 150 | 151 | /// Can detect inequality earlier with [TRUSTED_HIERARCHY]. 152 | /// 153 | /// [TRUSTED_HIERARCHY]: crate::BitSetBase::TRUSTED_HIERARCHY 154 | #[inline] 155 | pub fn is_eq(left: L, right: R) -> bool 156 | where 157 | L: LevelMasksIterExt, 158 | R: LevelMasksIterExt 159 | { 160 | bitsets_eq(left, right) 161 | } 162 | 163 | /// O(1) for [TRUSTED_HIERARCHY]. 164 | /// 165 | /// [TRUSTED_HIERARCHY]: crate::BitSetBase::TRUSTED_HIERARCHY 166 | #[inline] 167 | pub fn is_empty(bitset: S) -> bool { 168 | bitset_is_empty(bitset) 169 | } 170 | 171 | #[inline] 172 | pub fn contains(bitset: S, index: usize) -> bool { 173 | bitset_contains(bitset, index) 174 | } 175 | 176 | /// Same as [impl_bitset], but for [LevelMasks]. 177 | /// 178 | /// Implements [LevelMasksIterExt] by routing all operations to [LevelMasks]. 179 | /// 180 | /// # Safety 181 | /// 182 | /// **DO NOT** implement [BitSetInterface] for `$t`, since `impl_bitset_simple`'s 183 | /// [LevelMasksIterExt] implementation stores pointer to Self in [Level1BlockData]. 184 | /// If "drain iterator" will move during iteration - that will invalidate 185 | /// [Level1BlockData]. 186 | /// You have to use [impl_bitset!] if you need `$t` to be [BitSetInterface]. 187 | /// 188 | /// [BitSetInterface]: crate::BitSetInterface 189 | /// [Level1BlockData]: crate::internals::LevelMasksIterExt::Level1BlockData 190 | #[cfg_attr(docsrs, doc(cfg(feature = "impl")))] 191 | #[cfg(feature = "impl")] 192 | #[macro_export] 193 | macro_rules! impl_bitset_simple { 194 | (impl <$($generics:tt),*> for ref $t:ty where $($where_bounds:tt)*) => { 195 | impl<$($generics),*> $crate::internals::LevelMasksIterExt for $t 196 | where 197 | $($where_bounds)* 198 | { 199 | type IterState = (); 200 | 201 | type Level1BlockData = (Option>, usize); 202 | 203 | fn make_iter_state(&self) -> Self::IterState { () } 204 | unsafe fn drop_iter_state(&self, state: &mut ManuallyDrop) {} 205 | 206 | #[inline] 207 | unsafe fn init_level1_block_data( 208 | &self, 209 | state: &mut Self::IterState, 210 | level1_block_data: &mut MaybeUninit, 211 | level0_index: usize 212 | ) -> (::Level1BitBlock, bool) { 213 | level1_block_data.write((Some(self.into()), level0_index)); 214 | (self.level1_mask(level0_index), true) 215 | } 216 | 217 | #[inline] 218 | unsafe fn data_mask_from_block_data( 219 | level1_block_data: &Self::Level1BlockData, level1_index: usize 220 | ) -> ::DataBitBlock { 221 | let this = unsafe{ level1_block_data.0.unwrap_unchecked() }.as_ref(); 222 | let level0_index = level1_block_data.1; 223 | this.data_mask(level0_index, level1_index) 224 | } 225 | } 226 | 227 | $crate::impl_bitset!(impl<$($generics),*> for ref $t where $($where_bounds)*); 228 | }; 229 | } 230 | //pub(crate) use impl_bitset_simple; 231 | 232 | /// Makes bitset from [LevelMasksIterExt]. 233 | /// 234 | /// Implements [BitSetInterface], [IntoIterator], [Eq], [Debug], [BitAnd], [BitOr], [BitXor], [Sub] 235 | /// for [LevelMasksIterExt]. Also duplicates part of BitSetInterface in struct impl, 236 | /// for ease of use. 237 | /// 238 | /// `ref` version will implement [BitSetInterface] for &T only. 239 | /// Otherwise - it will be implemented for both T and &T. 240 | /// Working only with refs will prevent T from being passed to apply/reduce 241 | /// as value, and will allow to store `&self` pointer safely inside [Level1BlockData]. 242 | /// 243 | /// [BitAnd]: std::ops::BitAnd 244 | /// [BitOr]: std::ops::BitOr 245 | /// [BitXor]: std::ops::BitXor 246 | /// [Sub]: std::ops::Sub 247 | /// [BitSetInterface]: crate::BitSetInterface 248 | /// [BitSet]: crate::BitSet 249 | /// [Level1BlockData]: LevelMasksIterExt::Level1BlockData 250 | #[cfg_attr(docsrs, doc(cfg(feature = "impl")))] 251 | #[cfg_attr(feature = "impl", macro_export)] 252 | macro_rules! impl_bitset { 253 | (impl <$($generics:tt),*> for $t:ty) => { 254 | impl_bitset!(impl<$($generics),*> for $t where) 255 | }; 256 | 257 | (impl <$($generics:tt),*> for $t:ty where $($where_bounds:tt)*) => { 258 | unsafe impl<$($generics),*> $crate::BitSetInterface for $t 259 | where 260 | $($where_bounds)* 261 | {} 262 | 263 | impl<$($generics),*> IntoIterator for $t 264 | where 265 | $($where_bounds)* 266 | { 267 | type Item = usize; 268 | type IntoIter = $crate::iter::CachingIndexIter; 269 | 270 | #[inline] 271 | fn into_iter(self) -> Self::IntoIter { 272 | $crate::internals::into_index_iter(self) 273 | } 274 | } 275 | 276 | impl<$($generics),*, Rhs> std::ops::BitAnd for $t 277 | where 278 | Rhs: $crate::BitSetInterface::Conf>, 279 | $($where_bounds)* 280 | { 281 | type Output = $crate::Apply<$crate::ops::And, Self, Rhs>; 282 | 283 | /// Returns intersection of self and rhs bitsets. 284 | #[inline] 285 | fn bitand(self, rhs: Rhs) -> Self::Output{ 286 | $crate::apply($crate::ops::And, self, rhs) 287 | } 288 | } 289 | 290 | impl<$($generics),*, Rhs> std::ops::BitOr for $t 291 | where 292 | Rhs: $crate::BitSetInterface::Conf>, 293 | $($where_bounds)* 294 | { 295 | type Output = $crate::Apply<$crate::ops::Or, Self, Rhs>; 296 | 297 | /// Returns union of self and rhs bitsets. 298 | #[inline] 299 | fn bitor(self, rhs: Rhs) -> Self::Output{ 300 | $crate::apply($crate::ops::Or, self, rhs) 301 | } 302 | } 303 | 304 | impl<$($generics),*, Rhs> std::ops::BitXor for $t 305 | where 306 | Rhs: $crate::BitSetInterface::Conf>, 307 | $($where_bounds)* 308 | { 309 | type Output = $crate::Apply<$crate::ops::Xor, Self, Rhs>; 310 | 311 | /// Returns symmetric difference of self and rhs bitsets. 312 | #[inline] 313 | fn bitxor(self, rhs: Rhs) -> Self::Output{ 314 | $crate::apply($crate::ops::Xor, self, rhs) 315 | } 316 | } 317 | 318 | impl<$($generics),*, Rhs> std::ops::Sub for $t 319 | where 320 | Rhs: $crate::BitSetInterface::Conf>, 321 | $($where_bounds)* 322 | { 323 | type Output = $crate::Apply<$crate::ops::Sub, Self, Rhs>; 324 | 325 | /// Returns difference of self and rhs bitsets. 326 | /// 327 | /// _Or relative complement of rhs in self._ 328 | #[inline] 329 | fn sub(self, rhs: Rhs) -> Self::Output{ 330 | $crate::apply($crate::ops::Sub, self, rhs) 331 | } 332 | } 333 | 334 | impl_bitset!(impl<$($generics),*> for ref $t where $($where_bounds)*); 335 | }; 336 | 337 | (impl <$($generics:tt),*> for ref $t:ty where $($where_bounds:tt)*) => { 338 | // -------------------------------- 339 | // BitsetInterface 340 | unsafe impl<$($generics),*> $crate::BitSetInterface for &$t 341 | where 342 | $($where_bounds)* 343 | {} 344 | 345 | // -------------------------------- 346 | // Duplicate BitsetInterface (not strictly necessary, but ergonomic) 347 | impl<$($generics),*> $t 348 | where 349 | $($where_bounds)* 350 | { 351 | #[inline] 352 | pub fn block_iter<'a>(&'a self) -> $crate::iter::CachingBlockIter<&'a Self> 353 | { 354 | $crate::internals::block_iter(self) 355 | } 356 | 357 | #[inline] 358 | pub fn iter<'a>(&'a self) -> $crate::iter::CachingIndexIter<&'a Self> 359 | { 360 | $crate::internals::index_iter(self) 361 | } 362 | 363 | #[inline] 364 | pub fn contains(&self, index: usize) -> bool { 365 | $crate::internals::contains(self, index) 366 | } 367 | 368 | /// See [BitSetInterface::is_empty()] 369 | /// 370 | /// [BitSetInterface::is_empty()]: crate::BitSetInterface::is_empty() 371 | #[inline] 372 | pub fn is_empty(&self) -> bool { 373 | $crate::internals::is_empty(self) 374 | } 375 | } 376 | 377 | // -------------------------------- 378 | // IntoIterator 379 | impl<$($generics),*> IntoIterator for &$t 380 | where 381 | $($where_bounds)* 382 | { 383 | type Item = usize; 384 | type IntoIter = $crate::iter::CachingIndexIter; 385 | 386 | #[inline] 387 | fn into_iter(self) -> Self::IntoIter { 388 | $crate::internals::into_index_iter(self) 389 | } 390 | } 391 | 392 | // -------------------------------- 393 | // Eq 394 | impl<$($generics),*,Rhs> PartialEq for $t 395 | where 396 | Rhs: $crate::internals::LevelMasksIterExt::Conf>, 397 | $($where_bounds)* 398 | { 399 | /// Works faster with [TRUSTED_HIERARCHY]. 400 | /// 401 | /// [TRUSTED_HIERARCHY]: crate::bitset_interface::BitSetBase::TRUSTED_HIERARCHY 402 | #[inline] 403 | fn eq(&self, other: &Rhs) -> bool { 404 | $crate::internals::is_eq(self, other) 405 | } 406 | } 407 | 408 | impl<$($generics),*> Eq for $t 409 | where 410 | $($where_bounds)* 411 | {} 412 | 413 | 414 | // -------------------------------- 415 | // Debug 416 | impl<$($generics),*> std::fmt::Debug for $t 417 | where 418 | $($where_bounds)* 419 | { 420 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 421 | f.debug_list().entries(self.iter()).finish() 422 | } 423 | } 424 | 425 | 426 | // --------------------------------- 427 | // And 428 | impl<$($generics),*, Rhs> std::ops::BitAnd for &$t 429 | where 430 | Rhs: $crate::BitSetInterface::Conf>, 431 | $($where_bounds)* 432 | { 433 | type Output = $crate::Apply<$crate::ops::And, Self, Rhs>; 434 | 435 | /// Returns intersection of self and rhs bitsets. 436 | #[inline] 437 | fn bitand(self, rhs: Rhs) -> Self::Output{ 438 | $crate::apply($crate::ops::And, self, rhs) 439 | } 440 | } 441 | 442 | // --------------------------------- 443 | // Or 444 | impl<$($generics),*, Rhs> std::ops::BitOr for &$t 445 | where 446 | Rhs: $crate::BitSetInterface::Conf>, 447 | $($where_bounds)* 448 | { 449 | type Output = $crate::Apply<$crate::ops::Or, Self, Rhs>; 450 | 451 | /// Returns union of self and rhs bitsets. 452 | #[inline] 453 | fn bitor(self, rhs: Rhs) -> Self::Output{ 454 | $crate::apply($crate::ops::Or, self, rhs) 455 | } 456 | } 457 | 458 | // --------------------------------- 459 | // Xor 460 | impl<$($generics),*, Rhs> std::ops::BitXor for &$t 461 | where 462 | Rhs: $crate::BitSetInterface::Conf>, 463 | $($where_bounds)* 464 | { 465 | type Output = $crate::Apply<$crate::ops::Xor, Self, Rhs>; 466 | 467 | /// Returns symmetric difference of self and rhs bitsets. 468 | #[inline] 469 | fn bitxor(self, rhs: Rhs) -> Self::Output{ 470 | $crate::apply($crate::ops::Xor, self, rhs) 471 | } 472 | } 473 | 474 | // --------------------------------- 475 | // Sub 476 | impl<$($generics),*, Rhs> std::ops::Sub for &$t 477 | where 478 | Rhs: $crate::BitSetInterface::Conf>, 479 | $($where_bounds)* 480 | { 481 | type Output = $crate::Apply<$crate::ops::Sub, Self, Rhs>; 482 | 483 | /// Returns difference of self and rhs bitsets. 484 | /// 485 | /// _Or relative complement of rhs in self._ 486 | #[inline] 487 | fn sub(self, rhs: Rhs) -> Self::Output{ 488 | $crate::apply($crate::ops::Sub, self, rhs) 489 | } 490 | } 491 | }; 492 | } 493 | pub(crate) use impl_bitset; -------------------------------------------------------------------------------- /src/iter/caching.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | use std::mem; 3 | use std::mem::{ManuallyDrop, MaybeUninit}; 4 | use std::ops::ControlFlow; 5 | 6 | use crate::bit_block::BitBlock; 7 | use crate::bit_queue::BitQueue; 8 | use crate::bitset_interface::{BitSetBase, LevelMasksIterExt}; 9 | use crate::{data_block_start_index, DataBlock, DataBlockIter, level_indices}; 10 | use crate::config::Config; 11 | use crate::iter::{BlockCursor, IndexCursor}; 12 | 13 | /// Caching block iterator. 14 | /// 15 | /// Constructed by [BitSetInterface]. 16 | /// 17 | /// Cache pre-data level block info, making data blocks access faster. 18 | /// This allows to have some additional logic - for example [Reduce] discard 19 | /// sets with empty level1 blocks. 20 | /// Since only intersection operation produce TrustedHierarchy, which exists in all input sets - 21 | /// all other operations eventually could traverse through empty level blocks across hierarchy. 22 | /// [Reduce] logic - eliminate this effect. 23 | /// 24 | /// # traverse / for_each 25 | /// 26 | /// Block [traverse]/[for_each] is up to 25% faster then iteration. 27 | /// 28 | /// # Empty blocks 29 | /// 30 | /// Block iterator may occasionally return empty blocks. 31 | /// This is for performance reasons - it is faster to just iterate/traverse empty 32 | /// blocks through, then to add adding additional `is_empty` check in the middle of the loop. 33 | /// 34 | /// TODO: consider changing this behavior. 35 | /// 36 | /// # Memory footprint 37 | /// 38 | /// This iterator may store some data in its internal state. 39 | /// Amount of memory used by cache depends on [cache] type. 40 | /// Cache affects only [reduce] operations. 41 | /// 42 | /// [BitSetInterface]: crate::BitSetInterface 43 | /// [Reduce]: crate::Reduce 44 | /// [cache]: crate::cache 45 | /// [reduce]: crate::reduce() 46 | /// [binary_op]: crate::ops 47 | /// [traverse]: Self::traverse 48 | /// [for_each]: std::iter::Iterator::for_each 49 | pub struct CachingBlockIter 50 | where 51 | T: LevelMasksIterExt, 52 | { 53 | virtual_set: T, 54 | 55 | level0_iter: <::Level0BitBlock as BitBlock>::BitsIter, 56 | level1_iter: <::Level1BitBlock as BitBlock>::BitsIter, 57 | level0_index: usize, 58 | 59 | state: ManuallyDrop, 60 | level1_block_data: MaybeUninit, 61 | } 62 | 63 | impl Clone for CachingBlockIter 64 | where 65 | T: LevelMasksIterExt + Clone 66 | { 67 | #[inline] 68 | fn clone(&self) -> Self { 69 | let state = self.virtual_set.make_iter_state(); 70 | 71 | let mut this = Self { 72 | virtual_set : self.virtual_set.clone(), 73 | level0_iter : self.level0_iter.clone(), 74 | level1_iter : self.level1_iter.clone(), 75 | level0_index: self.level0_index, 76 | state: ManuallyDrop::new(state), 77 | level1_block_data: MaybeUninit::uninit() 78 | }; 79 | 80 | /*const*/ let have_state = mem::size_of::() > 0; 81 | if !have_state { 82 | // bitwise-copy level1_block_data if have no IterState state. 83 | 84 | this.level1_block_data = unsafe{ std::ptr::read(&self.level1_block_data) }; 85 | } else { 86 | // update level1_block_data otherwise. 87 | // (because level1_block_data may depends on state) 88 | 89 | // Check if level0_index is valid. 90 | // level0_index can be only invalid in initial state and for "end". 91 | if this.level0_index < ::Level0BitBlock::size() 92 | { 93 | unsafe { 94 | // Do not drop level1_block_data, since it was never initialized before. 95 | this.virtual_set.init_level1_block_data( 96 | &mut this.state, 97 | &mut this.level1_block_data, 98 | this.level0_index 99 | ); 100 | } 101 | } 102 | } 103 | 104 | this 105 | } 106 | } 107 | 108 | impl CachingBlockIter 109 | where 110 | T: LevelMasksIterExt, 111 | { 112 | #[inline] 113 | pub(crate) fn new(virtual_set: T) -> Self { 114 | let level0_iter = virtual_set.level0_mask().into_bits_iter(); 115 | let state = virtual_set.make_iter_state(); 116 | Self{ 117 | virtual_set, 118 | 119 | level0_iter, 120 | level1_iter: BitQueue::empty(), 121 | // usize::MAX - is marker, that we in "intial state". 122 | // Which means that only level0_iter initialized, and in original state. 123 | level0_index: usize::MAX, 124 | 125 | state: ManuallyDrop::new(state), 126 | level1_block_data: MaybeUninit::new(Default::default()) 127 | } 128 | } 129 | 130 | /// Constructs cursor for BlockIterator, with current iterator position. 131 | /// 132 | /// This means that if you [move_to] iterator to cursor, 133 | /// iterator will be in the same position as now. IOW - cursor points 134 | /// to the NEXT element. 135 | /// 136 | /// [move_to]: Self::move_to 137 | #[inline] 138 | pub fn cursor(&self) -> BlockCursor { 139 | // "initial state"? 140 | if self.level0_index == usize::MAX /*almost never*/ { 141 | return BlockCursor::default(); 142 | } 143 | 144 | BlockCursor { 145 | level0_index : self.level0_index as u16, 146 | level1_next_index: self.level1_iter.current() as u16, 147 | phantom: PhantomData 148 | } 149 | } 150 | 151 | /// Into index iterator. 152 | /// 153 | /// Index iterator will start iteration from next block. 154 | #[inline] 155 | pub fn into_indices(mut self) -> CachingIndexIter { 156 | let data_block_iter = 157 | if let Some(data_block) = self.next(){ 158 | data_block.into_iter() 159 | } else { 160 | DataBlockIter { 161 | start_index : usize::MAX, 162 | bit_block_iter: BitQueue::empty() 163 | } 164 | }; 165 | 166 | CachingIndexIter{ 167 | block_iter: self, 168 | data_block_iter 169 | } 170 | } 171 | 172 | /// Move iterator to cursor position. 173 | /// 174 | /// Fast O(1) operation. 175 | #[must_use] 176 | #[inline] 177 | pub fn move_to(mut self, cursor: BlockCursor) -> Self{ 178 | // Reset level0 mask if we not in "initial state" 179 | if self.level0_index != usize::MAX{ 180 | self.level0_iter = self.virtual_set.level0_mask().into_bits_iter(); 181 | } 182 | 183 | // Mask out level0 mask 184 | let cursor_level0_index = cursor.level0_index as usize; 185 | self.level0_iter.zero_first_n(cursor_level0_index); 186 | 187 | if let Some(level0_index) = self.level0_iter.next(){ 188 | self.level0_index = level0_index; 189 | 190 | // generate level1 mask, and update cache. 191 | let level1_mask = unsafe { 192 | self.level1_block_data.assume_init_drop(); 193 | let (level1_mask, _) = self.virtual_set.init_level1_block_data( 194 | &mut self.state, 195 | &mut self.level1_block_data, 196 | level0_index 197 | ); 198 | level1_mask 199 | }; 200 | self.level1_iter = level1_mask.into_bits_iter(); 201 | 202 | // TODO: can we mask SIMD block directly? 203 | // mask out level1 mask, if this is block pointed by cursor 204 | if level0_index == cursor_level0_index{ 205 | self.level1_iter.zero_first_n(cursor.level1_next_index as usize); 206 | } 207 | } else { 208 | // absolutely empty 209 | self.level1_iter = BitQueue::empty(); 210 | self.level0_index = ::DataBitBlock::size(); 211 | } 212 | 213 | self 214 | } 215 | 216 | /// Stable [try_for_each] version. 217 | /// 218 | /// [try_for_each]: std::iter::Iterator::try_for_each 219 | #[inline] 220 | pub fn traverse(mut self, mut f: F) -> ControlFlow<()> 221 | where 222 | F: FnMut(DataBlock<::DataBitBlock>) -> ControlFlow<()> 223 | { 224 | // Self have Drop - hence we can't move out values from it. 225 | // We need level0_iter and level1_iter - we'll ptr::read them instead. 226 | // It is ok - since they does not participate in Self::Drop. 227 | // See https://github.com/Jules-Bertholet/rfcs/blob/manuallydrop-deref-move/text/3466-manuallydrop-deref-move.md#rationale-and-alternatives 228 | 229 | // compiler SHOULD be able to detect and opt-out this branch away for 230 | // traverse() after new() call. 231 | if self.level0_index != usize::MAX{ 232 | let level0_index = self.level0_index; 233 | 234 | let level1_iter = unsafe{ std::ptr::read(&self.level1_iter) }; 235 | let ctrl = level1_iter.traverse( 236 | |level1_index| level1_mask_traverse_fn::( 237 | level0_index, level1_index, &self.level1_block_data, |b| f(b) 238 | ) 239 | ); 240 | if ctrl.is_break(){ 241 | return ControlFlow::Break(()); 242 | } 243 | } 244 | 245 | let level0_iter = unsafe{ std::ptr::read(&self.level0_iter) }; 246 | level0_iter.traverse( 247 | |level0_index| level0_mask_traverse_fn( 248 | &self.virtual_set, 249 | level0_index, 250 | &mut self.state, 251 | &mut self.level1_block_data, 252 | |b| f(b) 253 | ) 254 | ) 255 | } 256 | } 257 | 258 | impl Iterator for CachingBlockIter 259 | where 260 | T: LevelMasksIterExt, 261 | { 262 | type Item = DataBlock<::DataBitBlock>; 263 | 264 | #[inline] 265 | fn next(&mut self) -> Option { 266 | let level1_index = loop { 267 | if let Some(index) = self.level1_iter.next() { 268 | break index; 269 | } else { 270 | //update level0 271 | if let Some(index) = self.level0_iter.next() { 272 | self.level0_index = index; 273 | 274 | let level1_mask = unsafe { 275 | self.level1_block_data.assume_init_drop(); 276 | let (level1_mask, _) = 277 | self.virtual_set.init_level1_block_data( 278 | &mut self.state, 279 | &mut self.level1_block_data, 280 | index 281 | ); 282 | level1_mask 283 | }; 284 | 285 | self.level1_iter = level1_mask.into_bits_iter(); 286 | } else { 287 | return None; 288 | } 289 | } 290 | }; 291 | 292 | let data_mask = unsafe { 293 | T::data_mask_from_block_data( 294 | self.level1_block_data.assume_init_ref(), level1_index 295 | ) 296 | }; 297 | 298 | let block_start_index = 299 | data_block_start_index::<::Conf>( 300 | self.level0_index, level1_index, 301 | ); 302 | 303 | Some(DataBlock { start_index: block_start_index, bit_block: data_mask }) 304 | } 305 | 306 | #[inline] 307 | fn for_each(self, mut f: F) 308 | where 309 | F: FnMut(Self::Item) 310 | { 311 | self.traverse(|block| { 312 | f(block); 313 | ControlFlow::Continue(()) 314 | }); 315 | } 316 | } 317 | 318 | impl Drop for CachingBlockIter 319 | where 320 | T: LevelMasksIterExt 321 | { 322 | #[inline] 323 | fn drop(&mut self) { 324 | unsafe{ 325 | self.level1_block_data.assume_init_drop(); 326 | self.virtual_set.drop_iter_state(&mut self.state); 327 | } 328 | } 329 | } 330 | 331 | 332 | /// Caching index iterator. 333 | /// 334 | /// Constructed by [BitSetInterface], or acquired from [CachingBlockIter::into_indices]. 335 | /// 336 | /// Same as [CachingBlockIter] but for indices. 337 | /// 338 | /// # traverse / for_each 339 | /// 340 | /// Index [traverse]/[for_each] is up to 2x faster then iteration. 341 | /// 342 | /// [BitSetInterface]: crate::BitSetInterface 343 | /// [traverse]: Self::traverse 344 | /// [for_each]: std::iter::Iterator::for_each 345 | pub struct CachingIndexIter 346 | where 347 | T: LevelMasksIterExt, 348 | { 349 | block_iter: CachingBlockIter, 350 | data_block_iter: DataBlockIter<::DataBitBlock>, 351 | } 352 | 353 | impl Clone for CachingIndexIter 354 | where 355 | T: LevelMasksIterExt + Clone 356 | { 357 | #[inline] 358 | fn clone(&self) -> Self { 359 | Self{ 360 | block_iter: self.block_iter.clone(), 361 | data_block_iter: self.data_block_iter.clone(), 362 | } 363 | } 364 | } 365 | 366 | impl CachingIndexIter 367 | where 368 | T: LevelMasksIterExt, 369 | { 370 | #[inline] 371 | pub(crate) fn new(virtual_set: T) -> Self { 372 | Self{ 373 | block_iter: CachingBlockIter::new(virtual_set), 374 | data_block_iter: DataBlockIter{ 375 | // do not calc `start_index` now - will be calculated in 376 | // iterator, or in move_to. 377 | start_index: 0, 378 | bit_block_iter: BitQueue::empty(), 379 | } 380 | } 381 | } 382 | 383 | /// Move iterator to cursor position. 384 | /// 385 | /// Fast O(1) operation. 386 | #[must_use] 387 | #[inline] 388 | pub fn move_to(mut self, cursor: IndexCursor) -> Self { 389 | self.block_iter = self.block_iter.move_to(cursor.block_cursor); 390 | 391 | self.data_block_iter = 392 | if let Some(data_block) = self.block_iter.next(){ 393 | let mut data_block_iter = data_block.into_iter(); 394 | 395 | // mask out, if this is block pointed by cursor 396 | let cursor_block_start_index = data_block_start_index::( 397 | cursor.block_cursor.level0_index as usize, 398 | cursor.block_cursor.level1_next_index /*this is current index*/ as usize, 399 | ); 400 | if data_block_iter.start_index == cursor_block_start_index{ 401 | data_block_iter.bit_block_iter.zero_first_n(cursor.data_next_index as usize); 402 | } 403 | 404 | data_block_iter 405 | } else { 406 | // absolutely empty 407 | // point to the end 408 | DataBlockIter{ 409 | start_index: usize::MAX, 410 | bit_block_iter: BitQueue::empty(), 411 | } 412 | }; 413 | 414 | self 415 | } 416 | 417 | /// Same as [CachingBlockIter::cursor], but for index. 418 | #[inline] 419 | pub fn cursor(&self) -> IndexCursor { 420 | if self.block_iter.level0_index == usize::MAX{ 421 | return IndexCursor::default(); 422 | } 423 | 424 | // Extract level0_index, level1_index from block_start_index 425 | let (level0_index, level1_index, _) = level_indices::(self.data_block_iter.start_index); 426 | 427 | IndexCursor { 428 | block_cursor: BlockCursor { 429 | level0_index: level0_index as u16, 430 | // This will actually point to current index, not to next one. 431 | level1_next_index: level1_index as u16, 432 | phantom: PhantomData 433 | }, 434 | data_next_index: self.data_block_iter.bit_block_iter.current() as u32, 435 | } 436 | } 437 | 438 | /// Stable [try_for_each] version. 439 | /// 440 | /// [try_for_each]: std::iter::Iterator::try_for_each 441 | #[inline] 442 | pub fn traverse(mut self, mut f: F) -> ControlFlow<()> 443 | where 444 | F: FnMut(usize) -> ControlFlow<()> 445 | { 446 | // See CachingBlockIter::traverse comments. 447 | 448 | if self.block_iter.level0_index != usize::MAX{ 449 | let level0_index = self.block_iter.level0_index; 450 | 451 | // 1. traverse data block 452 | let ctrl = self.data_block_iter.traverse(|i| f(i)); 453 | if ctrl.is_break(){ 454 | return ControlFlow::Break(()); 455 | } 456 | 457 | // 2. traverse rest of the level1 block 458 | let level1_iter = unsafe{ std::ptr::read(&self.block_iter.level1_iter) }; 459 | let ctrl = level1_iter.traverse( 460 | |level1_index| level1_mask_traverse_fn::( 461 | level0_index, level1_index, &self.block_iter.level1_block_data, 462 | |b| b.traverse(|i| f(i)) 463 | ) 464 | ); 465 | if ctrl.is_break(){ 466 | return ControlFlow::Break(()); 467 | } 468 | } 469 | 470 | let level0_iter = unsafe{ std::ptr::read(&self.block_iter.level0_iter) }; 471 | level0_iter.traverse( 472 | |level0_index| level0_mask_traverse_fn( 473 | &self.block_iter.virtual_set, 474 | level0_index, 475 | &mut self.block_iter.state, 476 | &mut self.block_iter.level1_block_data, 477 | |b| b.traverse(|i| f(i)) 478 | ) 479 | ) 480 | } 481 | } 482 | 483 | impl Iterator for CachingIndexIter 484 | where 485 | T: LevelMasksIterExt, 486 | { 487 | type Item = usize; 488 | 489 | #[inline] 490 | fn next(&mut self) -> Option { 491 | // looping, because BlockIter may return empty DataBlocks. 492 | loop{ 493 | if let Some(index) = self.data_block_iter.next(){ 494 | return Some(index); 495 | } 496 | 497 | if let Some(data_block) = self.block_iter.next(){ 498 | self.data_block_iter = data_block.into_iter(); 499 | } else { 500 | return None; 501 | } 502 | } 503 | } 504 | 505 | #[inline] 506 | fn for_each(self, mut f: F) 507 | where 508 | F: FnMut(Self::Item) 509 | { 510 | self.traverse(|index| { 511 | f(index); 512 | ControlFlow::Continue(()) 513 | }); 514 | } 515 | } 516 | 517 | 518 | #[inline] 519 | fn level1_mask_traverse_fn( 520 | level0_index: usize, 521 | level1_index: usize, 522 | level1_block_data: &MaybeUninit, 523 | mut f: F 524 | ) -> ControlFlow<()> 525 | where 526 | S: LevelMasksIterExt, 527 | F: FnMut(DataBlock<::DataBitBlock>) -> ControlFlow<()> 528 | { 529 | let data_mask = unsafe { 530 | S::data_mask_from_block_data(level1_block_data.assume_init_ref(), level1_index) 531 | }; 532 | 533 | let block_start_index = 534 | crate::data_block_start_index::<::Conf>( 535 | level0_index, level1_index 536 | ); 537 | 538 | f(DataBlock{ start_index: block_start_index, bit_block: data_mask }) 539 | } 540 | 541 | #[inline] 542 | fn level0_mask_traverse_fn( 543 | set: &S, 544 | level0_index: usize, 545 | state: &mut S::IterState, 546 | level1_blocks: &mut MaybeUninit, 547 | mut f: F 548 | ) -> ControlFlow<()> 549 | where 550 | S: LevelMasksIterExt, 551 | F: FnMut(DataBlock<::DataBitBlock>) -> ControlFlow<()> 552 | { 553 | let level1_mask = unsafe{ 554 | level1_blocks.assume_init_drop(); 555 | let (level1_mask, _) = 556 | set.init_level1_block_data(state, level1_blocks, level0_index); 557 | level1_mask 558 | }; 559 | 560 | level1_mask.traverse_bits(|level1_index|{ 561 | level1_mask_traverse_fn::(level0_index, level1_index, level1_blocks, |b| f(b)) 562 | }) 563 | } 564 | -------------------------------------------------------------------------------- /src/iter/mod.rs: -------------------------------------------------------------------------------- 1 | //! Iteration always return ordered (or sorted) index sequences. 2 | 3 | use std::marker::PhantomData; 4 | 5 | use crate::{DataBlock, level_indices}; 6 | use crate::bit_block::BitBlock; 7 | use crate::config::{Config, max_addressable_index}; 8 | 9 | mod caching; 10 | pub use caching::{CachingBlockIter, CachingIndexIter}; 11 | 12 | #[cfg(feature = "simple_iter")] 13 | mod simple; 14 | #[cfg(feature = "simple_iter")] 15 | pub use simple::{SimpleBlockIter, SimpleIndexIter}; 16 | 17 | /// Block iterator cursor, or position of iterable. 18 | /// 19 | /// Created by [CachingBlockIter::cursor()], used by [CachingBlockIter::move_to()]. 20 | /// Also can be built [from] index and DataBlock. 21 | /// 22 | /// [from]: Self::from 23 | /// 24 | /// Allows to resume iteration from the last position, even if the 25 | /// source was mutated. Can be used with any [BitSetInterface]. 26 | /// Default constructed cursor will traverse bitset from the very begin. 27 | /// 28 | /// # Use-case 29 | /// 30 | /// This can be used to split long iteration into a few sessions. 31 | /// You may want that in concurrent environment, when you can't process whole 32 | /// iteration sequence fast, and want not to keep lock 33 | /// on resource all the time you process iteration sequence. 34 | /// 35 | /// Example: you lock sets, make intersection iterator, read 40 blocks into 36 | /// buffer, take iterator to cursor, unlock sets, process buffer, lock sets, 37 | /// move iterator to cursor, and so on. 38 | /// 39 | /// [BitSetInterface]: crate::BitSetInterface 40 | // 41 | // Additional `Conf` generic argument helps with index safety, and enhance 42 | // type safety. 43 | pub struct BlockCursor { 44 | pub(crate) level0_index: u16, 45 | // We don't have current/last returned index in iterator 46 | pub(crate) level1_next_index: u16, 47 | pub(crate) phantom: PhantomData 48 | } 49 | 50 | impl Default for BlockCursor{ 51 | #[inline] 52 | fn default() -> Self { 53 | Self::start() 54 | } 55 | } 56 | 57 | impl BlockCursor{ 58 | /// Constructs cursor that points to the start of bitset. 59 | #[inline] 60 | pub fn start() -> Self{ 61 | unsafe{ std::mem::zeroed() } 62 | } 63 | 64 | /// Constructs cursor that points to the end of bitset. 65 | /// 66 | /// Iterator [moved to] this cursor will always return `None`. 67 | /// 68 | /// [moved to]: CachingBlockIter::move_to 69 | #[inline] 70 | pub fn end() -> Self{ 71 | Self{ 72 | level0_index: Conf::Level0BitBlock::size() as u16, 73 | level1_next_index: Conf::Level1BitBlock::size() as u16, 74 | phantom: Default::default(), 75 | } 76 | } 77 | } 78 | 79 | impl Clone for BlockCursor{ 80 | #[inline] 81 | fn clone(&self) -> Self { 82 | unsafe{ std::ptr::read(self) } 83 | } 84 | } 85 | impl Copy for BlockCursor{} 86 | 87 | impl From for BlockCursor{ 88 | /// Build cursor that points to the block, that contains `index`. 89 | #[inline] 90 | fn from(mut index: usize) -> Self { 91 | // It is ok to use max_addressable_index instead of max_value, 92 | // because we point past the actual bitset data anyway. 93 | index = std::cmp::min(index, max_addressable_index::()); 94 | 95 | let (level0, level1, _) = level_indices::(index); 96 | Self{ 97 | level0_index: level0 as u16, 98 | level1_next_index: level1 as u16, 99 | phantom: PhantomData, 100 | } 101 | } 102 | } 103 | 104 | impl From<&DataBlock> for BlockCursor{ 105 | /// Build cursor that points to the `block`. 106 | #[inline] 107 | fn from(block: &DataBlock) -> Self { 108 | Self::from(block.start_index) 109 | } 110 | } 111 | 112 | /// Index iterator cursor. 113 | /// 114 | /// Created by [CachingIndexIter::cursor()], used by [CachingIndexIter::move_to()]. 115 | /// Also can be built [from] index and DataBlock. 116 | /// 117 | /// [from]: Self::from 118 | /// 119 | /// Same as [BlockCursor], but for indices iterator. 120 | pub struct IndexCursor { 121 | pub(crate) block_cursor: BlockCursor, 122 | // use u32 instead of u16, to nicely fit 64bit register 123 | pub(crate) data_next_index: u32 124 | } 125 | 126 | impl Default for IndexCursor{ 127 | #[inline] 128 | fn default() -> Self { 129 | Self::start() 130 | } 131 | } 132 | 133 | impl IndexCursor{ 134 | /// Constructs cursor that points to the start of the bitset. 135 | #[inline] 136 | pub fn start() -> Self{ 137 | unsafe{ std::mem::zeroed() } 138 | } 139 | 140 | /// Constructs cursor that points to the end of the bitset. 141 | /// 142 | /// Iterator [moved to] this cursor will always return `None`. 143 | /// 144 | /// [moved to]: CachingIndexIter::move_to 145 | #[inline] 146 | pub fn end() -> Self{ 147 | Self{ 148 | block_cursor: BlockCursor::end(), 149 | data_next_index: Conf::DataBitBlock::size() as u32 150 | } 151 | } 152 | } 153 | 154 | impl Clone for IndexCursor{ 155 | #[inline] 156 | fn clone(&self) -> Self { 157 | unsafe{ std::ptr::read(self) } 158 | } 159 | } 160 | impl Copy for IndexCursor{} 161 | 162 | impl From for IndexCursor{ 163 | /// Build cursor that points to the `index`. 164 | #[inline] 165 | fn from(mut index: usize) -> Self { 166 | index = std::cmp::min(index, max_addressable_index::()); 167 | 168 | let (level0, level1, data) = level_indices::(index); 169 | Self{ 170 | block_cursor: BlockCursor { 171 | level0_index: level0 as u16, 172 | level1_next_index: level1 as u16, 173 | phantom: PhantomData 174 | }, 175 | data_next_index: data as u32, 176 | } 177 | } 178 | } 179 | 180 | impl From<&DataBlock> for IndexCursor{ 181 | /// Build cursor that points to the `block` start index. 182 | #[inline] 183 | fn from(block: &DataBlock) -> Self { 184 | Self::from(block.start_index) 185 | } 186 | } -------------------------------------------------------------------------------- /src/iter/simple.rs: -------------------------------------------------------------------------------- 1 | use crate::bitset_interface::{BitSetBase, LevelMasks}; 2 | use crate::bit_queue::BitQueue; 3 | use crate::{BitBlock, data_block_start_index, DataBlock, DataBlockIter}; 4 | use crate::config::Config; 5 | 6 | /// Simple iterator - access each data block, by traversing all hierarchy 7 | /// levels indirections each time. 8 | /// 9 | /// Does not cache intermediate level1 position - hence have smaller size. 10 | /// All Cache parameters will be ignored. Consider using [CachingBlockIter] 11 | /// with [cache::NoCache] instead. 12 | /// 13 | /// May have similar to [CachingBlockIter] performance on very sparse sets. 14 | /// 15 | /// [cache::NoCache]: crate::cache::NoCache 16 | pub struct SimpleBlockIter 17 | where 18 | T: LevelMasks, 19 | { 20 | virtual_set: T, 21 | 22 | level0_iter: <::Level0BitBlock as BitBlock>::BitsIter, 23 | level1_iter: <::Level1BitBlock as BitBlock>::BitsIter, 24 | level0_index: usize, 25 | } 26 | 27 | impl SimpleBlockIter 28 | where 29 | T: LevelMasks 30 | { 31 | #[inline] 32 | pub fn new(virtual_set: T) -> Self { 33 | let level0_iter = virtual_set.level0_mask().into_bits_iter(); 34 | Self{ 35 | virtual_set, 36 | level0_iter, 37 | level1_iter: BitQueue::empty(), 38 | level0_index: 0 39 | } 40 | } 41 | } 42 | 43 | 44 | impl Iterator for SimpleBlockIter 45 | where 46 | T: LevelMasks, 47 | { 48 | type Item = DataBlock<<::Conf as Config>::DataBitBlock>; 49 | 50 | #[inline] 51 | fn next(&mut self) -> Option { 52 | let level1_index = loop{ 53 | if let Some(index) = self.level1_iter.next(){ 54 | break index; 55 | } else { 56 | //update level0 57 | if let Some(index) = self.level0_iter.next(){ 58 | self.level0_index = index; 59 | 60 | // update level1 iter 61 | let level1_mask = unsafe { 62 | self.virtual_set.level1_mask(index) 63 | }; 64 | self.level1_iter = level1_mask.into_bits_iter(); 65 | } else { 66 | return None; 67 | } 68 | } 69 | }; 70 | 71 | let data_mask = unsafe { 72 | self.virtual_set.data_mask(self.level0_index, level1_index) 73 | }; 74 | 75 | let block_start_index = 76 | data_block_start_index::<::Conf>( 77 | self.level0_index, level1_index 78 | ); 79 | 80 | Some(DataBlock{ start_index: block_start_index, bit_block: data_mask }) 81 | } 82 | } 83 | 84 | // It's just flatmap across block iterator. 85 | pub struct SimpleIndexIter 86 | where 87 | T: LevelMasks 88 | { 89 | block_iter: SimpleBlockIter, 90 | data_block_iter: DataBlockIter<::DataBitBlock>, 91 | } 92 | impl SimpleIndexIter 93 | where 94 | T: LevelMasks 95 | { 96 | #[inline] 97 | pub fn new(block_iter: SimpleBlockIter) -> Self{ 98 | Self{ 99 | block_iter, 100 | data_block_iter: DataBlockIter{ 101 | start_index: 0, 102 | bit_block_iter: BitQueue::empty() 103 | } 104 | } 105 | } 106 | } 107 | impl Iterator for SimpleIndexIter 108 | where 109 | T: LevelMasks 110 | { 111 | type Item = usize; 112 | 113 | #[inline] 114 | fn next(&mut self) -> Option { 115 | // TODO: ?? Still empty blocks ?? 116 | // looping, because BlockIter may return empty DataBlocks. 117 | loop{ 118 | if let Some(index) = self.data_block_iter.next(){ 119 | return Some(index); 120 | } 121 | 122 | if let Some(data_block) = self.block_iter.next(){ 123 | self.data_block_iter = data_block.into_iter(); 124 | } else { 125 | return None; 126 | } 127 | } 128 | } 129 | } -------------------------------------------------------------------------------- /src/level.rs: -------------------------------------------------------------------------------- 1 | use crate::BitBlock; 2 | use crate::primitive::Primitive; 3 | 4 | pub trait IBlock: Sized + Default{ 5 | type Mask: BitBlock; 6 | 7 | fn mask(&self) -> &Self::Mask; 8 | unsafe fn mask_mut(&mut self) -> &mut Self::Mask; 9 | 10 | // TODO: BlockIndex 11 | type Item: Primitive; 12 | 13 | /*/// # Safety 14 | /// 15 | /// - index is not checked for out-of-bounds. 16 | /// - index is not checked for validity (must exist). 17 | unsafe fn get_unchecked(&self, index: usize) -> Self::Item;*/ 18 | 19 | /// Returns 0 if item does not exist at `index`. 20 | /// 21 | /// # Safety 22 | /// 23 | /// index is not checked for out-of-bounds. 24 | unsafe fn get_or_zero(&self, index: usize) -> Self::Item; 25 | 26 | /// # Safety 27 | /// 28 | /// index is not checked. 29 | unsafe fn get_or_insert( 30 | &mut self, 31 | index: usize, 32 | f: impl FnMut() -> Self::Item 33 | ) -> Self::Item; 34 | 35 | /// Return previous mask bit. 36 | /// 37 | /// # Safety 38 | /// 39 | /// * `index` must be set 40 | /// * `index` is not checked for out-of-bounds. 41 | unsafe fn remove_unchecked(&mut self, index: usize); 42 | 43 | #[inline] 44 | fn is_empty(&self) -> bool { 45 | Self::Mask::is_zero(self.mask()) 46 | } 47 | } 48 | 49 | #[derive(Clone)] 50 | pub struct Level{ 51 | blocks: Vec, 52 | 53 | /// Single linked list of empty block indices. 54 | /// Mask of empty block used as a "next free block". 55 | /// u64::MAX - terminator. 56 | root_empty_block: u64, 57 | } 58 | 59 | impl Default for Level { 60 | #[inline] 61 | fn default() -> Self { 62 | Self{ 63 | //Always have empty block at index 0. 64 | blocks:vec![Default::default()], 65 | root_empty_block: u64::MAX, 66 | } 67 | } 68 | } 69 | 70 | impl Level { 71 | #[inline] 72 | pub fn blocks(&self) -> &[Block] { 73 | self.blocks.as_slice() 74 | } 75 | 76 | #[inline] 77 | pub fn blocks_mut(&mut self) -> &mut [Block] { 78 | self.blocks.as_mut_slice() 79 | } 80 | 81 | /// Next empty block link 82 | /// 83 | /// Block's mask used as index to next empty block 84 | #[inline] 85 | unsafe fn next_empty_block_index(block: &mut Block) -> &mut u64 { 86 | block.mask_mut().as_array_mut().get_unchecked_mut(0) 87 | } 88 | 89 | #[inline] 90 | fn pop_empty_block(&mut self) -> Option { 91 | if self.root_empty_block == u64::MAX { 92 | return None; 93 | } 94 | 95 | let index = self.root_empty_block as usize; 96 | unsafe{ 97 | let empty_block = self.blocks.get_unchecked_mut(index); 98 | let next_empty_block_index = Self::next_empty_block_index(empty_block); 99 | 100 | // update list root 101 | self.root_empty_block = *next_empty_block_index; 102 | 103 | // restore original mask zero state 104 | *next_empty_block_index = 0; 105 | } 106 | Some(index) 107 | } 108 | 109 | /// # Safety 110 | /// 111 | /// block must be empty and not in use! 112 | #[inline] 113 | unsafe fn push_empty_block(&mut self, block_index: usize){ 114 | let empty_block = self.blocks.get_unchecked_mut(block_index); 115 | let next_empty_block_index = Self::next_empty_block_index(empty_block); 116 | *next_empty_block_index = self.root_empty_block; 117 | 118 | self.root_empty_block = block_index as u64; 119 | } 120 | 121 | #[inline] 122 | pub fn insert_block(&mut self) -> usize { 123 | if let Some(index) = self.pop_empty_block(){ 124 | index 125 | } else { 126 | let index = self.blocks.len(); 127 | self.blocks.push(Default::default()); 128 | index 129 | } 130 | } 131 | 132 | /// # Safety 133 | /// 134 | /// block_index and block emptiness are not checked. 135 | #[inline] 136 | pub unsafe fn remove_empty_block_unchecked(&mut self, block_index: usize) { 137 | self.push_empty_block(block_index); 138 | // Do not touch block itself - it should be already empty 139 | } 140 | } -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(miri, feature(alloc_layout_extra) )] 2 | #![cfg_attr(docsrs, feature(doc_cfg))] 3 | //! Hierarchical sparse bitset. 4 | //! 5 | //! Memory consumption does not depend on max index inserted. 6 | //! 7 | //! ```text 8 | //! Level0 128bit SIMD 9 | //! [u8;128] 10 | //! SmallBitSet 11 | //! ┌ ┐ │ ┌ ┐ 12 | //! Level1 Vec │128bit SIMD│ ┃ │ 128bit SIMD │ 13 | //! │ [u16;128] │ ┃ │[u16;7]/Box<[u16;128]>│ 14 | //! └ ┘ │ └ ┘ 15 | //! ┌ ┐ 16 | //! Data Vec │128bit SIMD│ 17 | //! └ ┘ 18 | //! ──────────────────────────────────────────────────── 19 | //! 1 0 1 ... 1 ◀══ bit-mask 20 | //! Level0 □ Ø □ □ ◀══ index-pointers 21 | //! ┗━│━━━│━━━━━━━━━│┛ 22 | //! ╭──╯ ╰──────╮ ╰───────────────────╮ 23 | //! 1 0 0 1 ▽ ▽ 1 ▽ 24 | //! Level1 □ Ø Ø □ ... ... □ ... 25 | //! ┗━│━━━━━│━━━━━━━┛ ┗━━━━━━━│━━━━━━┛ ┗━━━━━━━━━━━━━━┛ 26 | //! ╰──╮ ╰─────────────────╮ ╰───────────────╮ 27 | //! ▽ ▽ ▽ 28 | //! Data 1 0 0 0 1 ... 0 0 1 1 0 ... 0 1 0 1 0 ... ... 29 | //! ┗━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━┛ 30 | //! ``` 31 | //! 32 | //! The very structure of [BitSet] acts as acceleration structure for 33 | //! intersection operation. All operations are incredibly fast - see benchmarks. 34 | //! (insert/contains in "traditional bitset" ballpark, intersection/union - orders of magnitude faster) 35 | //! 36 | //! It is multi-level structure. Last level contains actual bit-data. Each previous level 37 | //! have bitmask, where each bit corresponds to `!is_empty` of bitblock in next level. 38 | //! 39 | //! In addition to "non-empty-marker" bitmasks, there is pointers(indices) to non-empty blocks in next level. 40 | //! In this way, only blocks with actual data allocated. 41 | //! 42 | //! For inter-bitset operations, for example intersection: 43 | //! * root level bitmasks AND-ed. 44 | //! * resulting bitmask traversed for bits with 1. 45 | //! * indexes of bits with 1, used for getting pointers to the next level for each bitset. 46 | //! * repeat for next level until the data level, then for each next 1 bit in each level. 47 | //! 48 | //! Bitmasks allow to cutoff empty tree/hierarchy branches early for intersection operation, 49 | //! and traverse only actual data during iteration. 50 | //! 51 | //! In addition to this, during the inter-bitset operation, level1 blocks of 52 | //! bitsets are cached for faster access. Empty blocks are skipped and not added 53 | //! to the cache container, which algorithmically speeds up bitblock computations 54 | //! at the data level. 55 | //! This has observable effect in a merge operation between N non-intersecting 56 | //! bitsets: without this optimization - the data level bitmask would be OR-ed N times; 57 | //! with it - only once. 58 | //! 59 | //! # SmallBitset 60 | //! 61 | //! [SmallBitSet] is like [BitSet], but have **significantly** lower memory footprint 62 | //! on sparse sets. If not some performance overhead - that would be the one and 63 | //! only container in this lib. 64 | //! 65 | //! # Config 66 | //! 67 | //! Max index [BitSet] can hold, depends on used bitblocks capacity. 68 | //! The bigger the bitblocks - the higher [BitSet] index range. 69 | //! The lower - the smaller memory footprint it has. 70 | //! 71 | //! Max index for 64bit blocks = 262_144; for 256bit blocks = 16_777_216. 72 | //! 73 | //! Use [BitSet] with predefined [config]: 74 | //! ``` 75 | //! type BitSet = hi_sparse_bitset::BitSet; 76 | //! ``` 77 | //! 78 | //! # Inter-bitset operations 79 | //! 80 | //! Inter-bitset operations can be applied between ANY [BitSetInterface]. 81 | //! Output of inter-bitset operations are lazy bitsets(which are [BitSetInterface]s too). 82 | //! This means that you can combine different operations however you want 83 | //! without ever materializing them into actual [BitSet]. 84 | //! 85 | //! Use [reduce()] to apply inter-bitset operation between elements of bitsets iterator. 86 | //! 87 | //! Use [apply()] to apply inter-bitset operation between two bitsets. Also [&], [|], [`^`], [-]. 88 | //! 89 | //! You can define your own inter-bitset operation, by implementing [BitSetOp]. 90 | //! 91 | //! [&]: std::ops::BitAnd 92 | //! [|]: std::ops::BitOr 93 | //! [`^`]: std::ops::BitXor 94 | //! [-]: std::ops::Sub 95 | //! 96 | //! # Cursor 97 | //! 98 | //! [BitSetInterface] iterators can return [cursor()], pointing to the current iterator position. 99 | //! You can use [Cursor] to move ANY [BitSetInterface] iterator to it's position with [move_to]. 100 | //! 101 | //! You can also build cursor from index. 102 | //! 103 | //! [cursor()]: crate::iter::CachingIndexIter::cursor 104 | //! [Cursor]: crate::iter::IndexCursor 105 | //! [move_to]: crate::iter::CachingIndexIter::move_to 106 | //! 107 | //! # Iterator::for_each 108 | //! 109 | //! [BitSetInterface] iterators have [for_each] specialization and stable [try_for_each] version - [traverse]. 110 | //! For tight loops, traversing is observably faster then iterating. 111 | //! 112 | //! [for_each]: std::iter::Iterator::for_each 113 | //! [try_for_each]: std::iter::Iterator::try_for_each 114 | //! [traverse]: crate::iter::CachingIndexIter::traverse 115 | //! 116 | //! # TrustedHierarchy 117 | //! 118 | //! TrustedHierarchy means that each raised bit in hierarchy bitblock 119 | //! is guaranteed to correspond to non-empty block. 120 | //! That may be not true for [difference] and [symmetric difference] operation result. 121 | //! 122 | //! You can check if bitset has TrustedHierarchy with [BitSetBase::TRUSTED_HIERARCHY]. 123 | //! 124 | //! Bitsets with TrustedHierarchy are faster to compare with [Eq] and 125 | //! have O(1) [is_empty()]. 126 | //! 127 | //! [difference]: ops::Sub 128 | //! [symmetric difference]: ops::Xor 129 | //! [is_empty()]: BitSetInterface::is_empty 130 | //! 131 | //! # DataBlocks 132 | //! 133 | //! You can iterate [DataBlock]s instead of individual indices. DataBlocks can be moved, cloned 134 | //! and iterated for indices. 135 | //! 136 | //! # Custom bitsets 137 | //! 138 | //! You can make your own bitsets - like 139 | //! generative sets (empty, full), specially packed sets (range-fill), 140 | //! adapters, etc. See [internals] module. You need `impl` feature for that. 141 | //! 142 | //! # CPU extensions 143 | //! 144 | //! Library uses `popcnt`/`count_ones` and `tzcnt`/`trailing_zeros` heavily. 145 | //! Make sure you compile with hardware support of these 146 | //! (on x86: `target-feature=+popcnt,+bmi1`). 147 | //! 148 | //! ## SIMD 149 | //! 150 | //! 128 and 256 bit configurations use SIMD, powered by [wide]. Make sure you compile with simd support 151 | //! enabled (on x86: `sse2` for _128bit, `avx` for _256bit) to achieve best performance. 152 | //! _sse2 enabled by default in Rust for most desktop environments_ 153 | //! 154 | //! If you want to use other SIMD types/registers - see [internals] module. 155 | //! If you don't need "wide" configurations, you may disable default feature `simd`. 156 | //! 157 | //! [wide]: https://crates.io/crates/wide 158 | 159 | #[cfg(test)] 160 | mod test; 161 | 162 | mod primitive; 163 | mod primitive_array; 164 | mod block; 165 | mod compact_block; 166 | mod level; 167 | mod bit_block; 168 | mod bit_queue; 169 | mod bit_utils; 170 | mod reduce; 171 | mod bitset_interface; 172 | mod apply; 173 | mod raw; 174 | mod derive_raw; 175 | mod bitset; 176 | mod small_bitset; 177 | 178 | pub mod config; 179 | pub mod ops; 180 | pub mod iter; 181 | pub mod cache; 182 | pub mod internals; 183 | 184 | pub use bitset_interface::{BitSetBase, BitSetInterface}; 185 | pub use apply::Apply; 186 | pub use reduce::Reduce; 187 | pub use bit_block::BitBlock; 188 | pub use bitset::BitSet; 189 | pub use small_bitset::SmallBitSet; 190 | 191 | use primitive::Primitive; 192 | use primitive_array::PrimitiveArray; 193 | use std::ops::ControlFlow; 194 | use config::Config; 195 | use ops::BitSetOp; 196 | use bit_queue::BitQueue; 197 | use cache::ReduceCache; 198 | 199 | macro_rules! assume { 200 | ($e: expr) => { 201 | if !($e){ 202 | std::hint::unreachable_unchecked(); 203 | } 204 | }; 205 | } 206 | pub(crate) use assume; 207 | 208 | #[inline] 209 | fn level_indices(index: usize) -> (usize/*level0*/, usize/*level1*/, usize/*data*/){ 210 | // this should be const and act as const. 211 | /*const*/ let data_block_capacity_pot_exp : usize = Conf::DataBitBlock::SIZE_POT_EXPONENT; 212 | /*const*/ let data_block_capacity : usize = 1 << data_block_capacity_pot_exp; 213 | 214 | /*const*/ let level1_block_capacity_pot_exp: usize = Conf::Level1BitBlock::SIZE_POT_EXPONENT 215 | + Conf::DataBitBlock::SIZE_POT_EXPONENT; 216 | /*const*/ let level1_block_capacity : usize = 1 << level1_block_capacity_pot_exp; 217 | 218 | // index / LEVEL1_BLOCK_CAP 219 | let level0 = index >> level1_block_capacity_pot_exp; 220 | // index % LEVEL1_BLOCK_CAP 221 | let level0_remainder = index & (level1_block_capacity - 1); 222 | 223 | // level0_remainder / DATA_BLOCK_CAP 224 | let level1 = level0_remainder >> data_block_capacity_pot_exp; 225 | 226 | // level0_remainder % DATA_BLOCK_CAP = index % LEVEL1_BLOCK_CAP % DATA_BLOCK_CAP 227 | let level1_remainder = index & ( 228 | (level1_block_capacity-1) & (data_block_capacity-1) 229 | ); 230 | 231 | let data = level1_remainder; 232 | 233 | (level0, level1, data) 234 | } 235 | 236 | #[inline] 237 | fn data_block_start_index(level0_index: usize, level1_index: usize) -> usize{ 238 | let level0_offset = level0_index << (Conf::DataBitBlock::SIZE_POT_EXPONENT + Conf::Level1BitBlock::SIZE_POT_EXPONENT); 239 | let level1_offset = level1_index << (Conf::DataBitBlock::SIZE_POT_EXPONENT); 240 | level0_offset + level1_offset 241 | } 242 | 243 | #[derive(Clone, Debug, Eq, PartialEq)] 244 | pub struct DataBlock{ 245 | pub start_index: usize, 246 | pub bit_block: Block 247 | } 248 | impl DataBlock{ 249 | // TODO: remove 250 | /// traverse approx. 15% faster then iterator 251 | #[inline] 252 | pub fn traverse(&self, mut f: F) -> ControlFlow<()> 253 | where 254 | F: FnMut(usize) -> ControlFlow<()> 255 | { 256 | self.bit_block.traverse_bits(|index| f(self.start_index + index)) 257 | } 258 | 259 | #[inline] 260 | pub fn iter(&self) -> DataBlockIter{ 261 | DataBlockIter{ 262 | start_index: self.start_index, 263 | bit_block_iter: self.bit_block.clone().into_bits_iter() 264 | } 265 | } 266 | 267 | /// Calculate elements count in DataBlock. 268 | /// 269 | /// On most platforms, this should be faster then manually traversing DataBlock 270 | /// and counting elements. It use hardware accelerated "popcnt", 271 | /// whenever possible. 272 | #[inline] 273 | pub fn len(&self) -> usize { 274 | self.bit_block.count_ones() 275 | } 276 | 277 | #[inline] 278 | pub fn is_empty(&self) -> bool { 279 | self.bit_block.is_zero() 280 | } 281 | } 282 | impl IntoIterator for DataBlock{ 283 | type Item = usize; 284 | type IntoIter = DataBlockIter; 285 | 286 | /// This is actually no-op fast. 287 | #[inline] 288 | fn into_iter(self) -> Self::IntoIter { 289 | DataBlockIter{ 290 | start_index: self.start_index, 291 | bit_block_iter: self.bit_block.into_bits_iter() 292 | } 293 | } 294 | } 295 | 296 | #[derive(Clone)] 297 | pub struct DataBlockIter{ 298 | start_index: usize, 299 | bit_block_iter: Block::BitsIter 300 | } 301 | impl DataBlockIter{ 302 | /// Stable version of [try_for_each]. 303 | /// 304 | /// traverse approx. 15% faster then iterator 305 | /// 306 | /// [try_for_each]: std::iter::Iterator::try_for_each 307 | #[inline] 308 | pub fn traverse(self, mut f: F) -> ControlFlow<()> 309 | where 310 | F: FnMut(usize) -> ControlFlow<()> 311 | { 312 | self.bit_block_iter.traverse(|index| f(self.start_index + index)) 313 | } 314 | } 315 | impl Iterator for DataBlockIter{ 316 | type Item = usize; 317 | 318 | #[inline] 319 | fn next(&mut self) -> Option { 320 | self.bit_block_iter.next().map(|index|self.start_index + index) 321 | } 322 | 323 | #[inline] 324 | fn for_each(self, mut f: F) 325 | where 326 | F: FnMut(Self::Item) 327 | { 328 | self.traverse(|index| { 329 | f(index); 330 | ControlFlow::Continue(()) 331 | }); 332 | } 333 | } 334 | 335 | /// Creates a lazy bitset, as [BitSetOp] application between two bitsets. 336 | #[inline] 337 | pub fn apply(op: Op, s1: S1, s2: S2) -> Apply 338 | where 339 | Op: BitSetOp, 340 | S1: BitSetInterface, 341 | S2: BitSetInterface::Conf>, 342 | { 343 | Apply::new(op, s1, s2) 344 | } 345 | 346 | /// Creates a lazy bitset, as bitsets iterator reduction. 347 | /// 348 | /// "Reduce" term used in Rust's [Iterator::reduce] sense. 349 | /// 350 | /// If the `bitsets` is empty - returns `None`; otherwise - returns the resulting 351 | /// lazy bitset. 352 | /// 353 | /// `bitsets` iterator must be cheap to clone (slice iterator is a good example). 354 | /// It will be cloned AT LEAST once for each returned [DataBlock] during iteration. 355 | /// 356 | /// # Safety 357 | /// 358 | /// Panics, if [Config::DefaultCache] capacity is smaller then sets len. 359 | #[inline] 360 | pub fn reduce(op: Op, bitsets: I) 361 | -> Option> 362 | where 363 | Conf: Config, 364 | Op: BitSetOp, 365 | I: Iterator + Clone, 366 | I::Item: BitSetInterface, 367 | { 368 | reduce_w_cache(op, bitsets, Default::default()) 369 | } 370 | 371 | /// [reduce], using specific [cache] for iteration. 372 | /// 373 | /// Cache applied to current operation only, so you can combine different cache 374 | /// types. 375 | /// 376 | /// N.B. Alternatively, you can change [Config::DefaultCache] and use [reduce]. 377 | /// 378 | /// # Safety 379 | /// 380 | /// Panics, if `Cache` capacity is smaller then sets len. 381 | /// 382 | /// [reduce]: reduce() 383 | #[inline] 384 | pub fn reduce_w_cache(_: Op, bitsets: I, _: Cache) 385 | -> Option> 386 | where 387 | Op: BitSetOp, 388 | I: Iterator + Clone, 389 | I::Item: BitSetInterface, 390 | Cache: ReduceCache 391 | { 392 | // Compile-time if 393 | if Cache::MAX_LEN != usize::MAX{ 394 | let len = bitsets.clone().count(); 395 | assert!(len<=Cache::MAX_LEN, "Cache is too small for this iterator."); 396 | if len == 0{ 397 | return None; 398 | } 399 | } else { 400 | if bitsets.clone().next().is_none(){ 401 | return None; 402 | } 403 | } 404 | 405 | Some(reduce::Reduce{ sets: bitsets, phantom: Default::default() }) 406 | } 407 | 408 | // TODO: Do we need fold as well? -------------------------------------------------------------------------------- /src/ops.rs: -------------------------------------------------------------------------------- 1 | //! Operations for [apply] and [reduce]. 2 | //! 3 | //! * [And] is the only operation that can discard blocks early 4 | //! on hierarchy level during traverse. Complexity-wise this is the fastest operation. 5 | //! * [Or] - does not need to discard any blocks, since it is a merge operation by definition. 6 | //! * [Xor] - have [Or] performance. 7 | //! * [Sub] - traverse all left operand bitset blocks. 8 | //! 9 | //! You can make your own operation by implementing [BitSetOp]. 10 | //! 11 | //! [apply]: crate::apply() 12 | //! [reduce]: crate::reduce() 13 | 14 | use std::ops::{BitAnd, BitOr, BitXor}; 15 | use crate::bit_block::BitBlock; 16 | 17 | // TODO: all operations should accept & instead? 18 | // To work with [u64;N] more flawlessly? 19 | /// Binary operation interface for [BitSetInterface]s. 20 | /// 21 | /// Implement this trait for creating your own operation. 22 | /// Pay attention to hierarchical nature of `hi_sparse_bitset` - you 23 | /// may need to apply "broader" operations to "hierarchical blocks", then 24 | /// to "data blocks". 25 | /// 26 | /// [BitSetInterface]: crate::BitSetInterface 27 | pub trait BitSetOp: Default + Copy + 'static{ 28 | /// Will operation between two [TRUSTED_HIERARCHY] bitsets produce 29 | /// [TRUSTED_HIERARCHY] as well? 30 | /// 31 | /// Enables some optimizations. False - is always safe value. 32 | /// 33 | /// [TRUSTED_HIERARCHY]: crate::BitSetBase::TRUSTED_HIERARCHY 34 | const TRUSTED_HIERARCHY: bool; 35 | 36 | /// Does [hierarchy_op] operands contain result? 37 | /// - left contains all bits from [hierarchy_op] result, 38 | /// - right contains all bits from [hierarchy_op] result, 39 | /// 40 | /// This is true for [intersection], or narrower. 41 | /// 42 | /// Enables some optimizations. False - is always safe value. 43 | /// 44 | /// [hierarchy_op]: Self::hierarchy_op 45 | /// [intersection]: And 46 | const HIERARCHY_OPERANDS_CONTAIN_RESULT: bool; 47 | 48 | /// Operation applied to indirection/hierarchy level bitblock 49 | fn hierarchy_op(left: T, right: T) -> T; 50 | 51 | /// Operation applied to data level bitblock 52 | fn data_op(left: T, right: T) -> T; 53 | } 54 | 55 | /// Intersection 56 | /// 57 | /// Will traverse only intersected blocks of left and right. 58 | #[derive(Default, Copy, Clone)] 59 | pub struct And; 60 | impl BitSetOp for And { 61 | const TRUSTED_HIERARCHY: bool = false; 62 | const HIERARCHY_OPERANDS_CONTAIN_RESULT: bool = true; 63 | 64 | #[inline] 65 | fn hierarchy_op(left: T, right: T) -> T { 66 | BitAnd::bitand(left, right) 67 | } 68 | 69 | #[inline] 70 | fn data_op(left: T, right: T) -> T { 71 | BitAnd::bitand(left, right) 72 | } 73 | } 74 | 75 | /// Union 76 | /// 77 | /// Will traverse all blocks of left and right. (Since all of them participate in merge) 78 | #[derive(Default, Copy, Clone)] 79 | pub struct Or; 80 | impl BitSetOp for Or { 81 | const TRUSTED_HIERARCHY: bool = true; 82 | const HIERARCHY_OPERANDS_CONTAIN_RESULT: bool = false; 83 | 84 | #[inline] 85 | fn hierarchy_op(left: T, right: T) -> T { 86 | BitOr::bitor(left, right) 87 | } 88 | 89 | #[inline] 90 | fn data_op(left: T, right: T) -> T { 91 | BitOr::bitor(left, right) 92 | } 93 | } 94 | 95 | /// Symmetric difference. 96 | /// 97 | /// Have performance of [Or]. 98 | #[derive(Default, Copy, Clone)] 99 | pub struct Xor; 100 | impl BitSetOp for Xor { 101 | const TRUSTED_HIERARCHY: bool = false; 102 | const HIERARCHY_OPERANDS_CONTAIN_RESULT: bool = false; 103 | 104 | #[inline] 105 | fn hierarchy_op(left: T, right: T) -> T { 106 | BitOr::bitor(left, right) 107 | } 108 | 109 | #[inline] 110 | fn data_op(left: T, right: T) -> T { 111 | BitXor::bitxor(left, right) 112 | } 113 | } 114 | 115 | /// Difference (relative complement) left\right. 116 | /// 117 | /// Have performance of traversing left operand. 118 | #[derive(Default, Copy, Clone)] 119 | pub struct Sub; 120 | impl BitSetOp for Sub { 121 | const TRUSTED_HIERARCHY: bool = false; 122 | const HIERARCHY_OPERANDS_CONTAIN_RESULT: bool = false; 123 | 124 | #[inline] 125 | fn hierarchy_op(left: T, _right: T) -> T { 126 | left 127 | } 128 | 129 | #[inline] 130 | fn data_op(left: T, right: T) -> T { 131 | left & (left ^ right) 132 | } 133 | } -------------------------------------------------------------------------------- /src/primitive.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Shl, ShlAssign/*, Shr, ShrAssign*/}; 3 | 4 | // num_traits was just **TOO** hard to use with primitives... 5 | // Cast from/to concrete primitive was a final nail into num_trait's coffin. 6 | pub trait Primitive: 7 | Default 8 | + BitAnd 9 | + BitAndAssign 10 | + BitOr 11 | + BitOrAssign 12 | + BitXor 13 | + BitXorAssign 14 | + Shl 15 | + Shl 16 | + ShlAssign 17 | /* + Shr 18 | + Shr 19 | + ShrAssign */ 20 | + Not 21 | + Copy 22 | + Ord 23 | + Debug 24 | + 'static 25 | { 26 | const MIN: Self; 27 | const MAX: Self; 28 | 29 | const ZERO: Self; 30 | const ONE : Self; 31 | 32 | fn from_usize(i: usize) -> Self; 33 | fn as_usize(self) -> usize; 34 | 35 | fn trailing_zeros(self) -> u32; 36 | fn wrapping_neg(self) -> Self; 37 | 38 | fn is_zero(self) -> bool; 39 | } 40 | 41 | macro_rules! impl_primitive { 42 | ($x:ty) => { 43 | impl Primitive for $x{ 44 | const MIN: $x = <$x>::MIN; 45 | const MAX: $x = <$x>::MAX; 46 | 47 | const ZERO: Self = 0; 48 | const ONE : Self = 1; 49 | 50 | #[inline] 51 | fn from_usize(i: usize) -> Self { 52 | i as Self 53 | } 54 | 55 | #[inline] 56 | fn as_usize(self) -> usize { 57 | self as usize 58 | } 59 | 60 | #[inline] 61 | fn trailing_zeros(self) -> u32 { 62 | self.trailing_zeros() 63 | } 64 | 65 | #[inline] 66 | fn wrapping_neg(self) -> Self { 67 | self.wrapping_neg() 68 | } 69 | 70 | #[inline] 71 | fn is_zero(self) -> bool { 72 | self == 0 73 | } 74 | } 75 | }; 76 | } 77 | 78 | impl_primitive!(u8); 79 | impl_primitive!(u16); 80 | impl_primitive!(u32); 81 | impl_primitive!(u64); 82 | impl_primitive!(usize); -------------------------------------------------------------------------------- /src/primitive_array.rs: -------------------------------------------------------------------------------- 1 | use std::mem::MaybeUninit; 2 | use crate::internals::Primitive; 3 | 4 | pub trait PrimitiveArray: AsRef<[Self::Item]> + AsMut<[Self::Item]> + Copy{ 5 | type Item: Primitive; 6 | const CAP: usize; 7 | 8 | type UninitArray: UninitPrimitiveArray; 9 | } 10 | impl PrimitiveArray for [T; N] 11 | where 12 | T: Primitive 13 | { 14 | type Item = T; 15 | const CAP: usize = N; 16 | type UninitArray = [MaybeUninit; N]; 17 | } 18 | 19 | #[allow(dead_code)] // Because not publicly visibile 20 | pub trait UninitPrimitiveArray 21 | : AsRef<[MaybeUninit]> 22 | + AsMut<[MaybeUninit]> 23 | + Copy 24 | { 25 | //type Item? 26 | type UninitItem: Primitive; 27 | const CAP: usize; 28 | 29 | fn uninit_array() -> Self; 30 | } 31 | impl UninitPrimitiveArray for [MaybeUninit; N] 32 | where 33 | T: Primitive 34 | { 35 | type UninitItem = T; 36 | const CAP: usize = N; 37 | 38 | #[inline] 39 | fn uninit_array() -> Self{ 40 | // From Rust MaybeUninit::uninit_array() : 41 | // SAFETY: An uninitialized `[MaybeUninit<_>; LEN]` is valid. 42 | unsafe { MaybeUninit::<[MaybeUninit; N]>::uninit().assume_init() } 43 | } 44 | } -------------------------------------------------------------------------------- /src/raw.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | use std::mem::{ManuallyDrop, MaybeUninit}; 3 | use std::ptr::NonNull; 4 | use crate::config::{Config, max_addressable_index}; 5 | use crate::{BitBlock, BitSetBase, level_indices}; 6 | use crate::bitset_interface::{LevelMasks, LevelMasksIterExt}; 7 | use crate::level::{IBlock, Level}; 8 | use crate::primitive::Primitive; 9 | 10 | pub struct RawBitSet 11 | where 12 | Level0Block: IBlock, 13 | Level1Block: IBlock, 14 | LevelDataBlock: IBlock, 15 | { 16 | level0: Level0Block, 17 | level1: Level, 18 | data : Level, 19 | phantom: PhantomData 20 | } 21 | 22 | impl Clone for RawBitSet 23 | where 24 | Conf: Config, 25 | Level0Block: IBlock + Clone, 26 | Level1Block: IBlock + Clone, 27 | LevelDataBlock: IBlock + Clone, 28 | { 29 | #[inline] 30 | fn clone(&self) -> Self { 31 | Self{ 32 | level0: self.level0.clone(), 33 | level1: self.level1.clone(), 34 | data: self.data.clone(), 35 | phantom: Default::default(), 36 | } 37 | } 38 | } 39 | 40 | impl Default for RawBitSet 41 | where 42 | Conf: Config, 43 | Level0Block: IBlock, 44 | Level1Block: IBlock, 45 | LevelDataBlock: IBlock, 46 | { 47 | #[inline] 48 | fn default() -> Self { 49 | Self{ 50 | level0: Default::default(), 51 | level1: Default::default(), 52 | data: Default::default(), 53 | phantom: PhantomData 54 | } 55 | } 56 | } 57 | 58 | impl FromIterator for RawBitSet 59 | where 60 | Conf: Config, 61 | Level0Block: IBlock, 62 | Level1Block: IBlock, 63 | LevelDataBlock: IBlock, 64 | { 65 | fn from_iter>(iter: T) -> Self { 66 | let mut this = Self::default(); 67 | for i in iter{ 68 | this.insert(i); 69 | } 70 | this 71 | } 72 | } 73 | 74 | impl From<[usize; N]> for RawBitSet 75 | where 76 | Conf: Config, 77 | Level0Block: IBlock, 78 | Level1Block: IBlock, 79 | LevelDataBlock: IBlock, 80 | { 81 | #[inline] 82 | fn from(value: [usize; N]) -> Self { 83 | Self::from_iter(value.into_iter()) 84 | } 85 | } 86 | 87 | impl RawBitSet 88 | where 89 | Conf: Config, 90 | Level0Block: IBlock, 91 | Level1Block: IBlock, 92 | LevelDataBlock: IBlock, 93 | { 94 | #[inline] 95 | fn level_indices(index: usize) -> (usize/*level0*/, usize/*level1*/, usize/*data*/){ 96 | level_indices::(index) 97 | } 98 | 99 | /// Max usize, [BitSet] with this `Config` can hold. 100 | /// 101 | /// [BitSet]: crate::BitSet 102 | #[inline] 103 | pub const fn max_capacity() -> usize { 104 | // We occupy one block for "empty" at each level, except root. 105 | max_addressable_index::() 106 | - (1 << Level1Block::Mask::SIZE_POT_EXPONENT) * (1 << LevelDataBlock::Mask::SIZE_POT_EXPONENT) 107 | - (1 << LevelDataBlock::Mask::SIZE_POT_EXPONENT) 108 | } 109 | 110 | #[inline] 111 | fn is_in_range(index: usize) -> bool{ 112 | index < Self::max_capacity() 113 | } 114 | 115 | #[inline] 116 | fn get_block_indices(&self, level0_index: usize, level1_index: usize) 117 | -> Option<(usize, usize)> 118 | { 119 | let level1_block_index = unsafe{ 120 | self.level0.get_or_zero(level0_index) 121 | }.as_usize(); 122 | 123 | // 2. Level1 124 | let data_block_index = unsafe{ 125 | let level1_block = self.level1.blocks().get_unchecked(level1_block_index); 126 | level1_block.get_or_zero(level1_index) 127 | }.as_usize(); 128 | 129 | return if data_block_index == 0 { 130 | // Block 0 - is preallocated empty block 131 | None 132 | } else { 133 | Some((level1_block_index, data_block_index)) 134 | }; 135 | } 136 | 137 | /// # Safety 138 | /// 139 | /// Will panic, if `index` is out of range. 140 | pub fn insert(&mut self, index: usize){ 141 | assert!(Self::is_in_range(index), "{index} index out of range!"); 142 | 143 | // That's indices to next level 144 | let (level0_index, level1_index, data_index) = Self::level_indices(index); 145 | 146 | // 1. Level0 147 | let level1_block_index = unsafe{ 148 | self.level0.get_or_insert(level0_index, ||{ 149 | let block_index = self.level1.insert_block(); 150 | Primitive::from_usize(block_index) 151 | }) 152 | }.as_usize(); 153 | 154 | // 2. Level1 155 | let data_block_index = unsafe{ 156 | let level1_block = self.level1.blocks_mut().get_unchecked_mut(level1_block_index); 157 | level1_block.get_or_insert(level1_index, ||{ 158 | let block_index = self.data.insert_block(); 159 | Primitive::from_usize(block_index) 160 | }) 161 | }.as_usize(); 162 | 163 | // 3. Data level 164 | unsafe{ 165 | let data_block = self.data.blocks_mut().get_unchecked_mut(data_block_index); 166 | data_block.mask_mut().set_bit::(data_index); 167 | } 168 | } 169 | 170 | /// Returns false if index is invalid/not in bitset. 171 | pub fn remove(&mut self, index: usize) -> bool { 172 | if !Self::is_in_range(index){ 173 | return false; 174 | } 175 | 176 | // 1. Resolve indices 177 | let (level0_index, level1_index, data_index) = Self::level_indices(index); 178 | let (level1_block_index, data_block_index) = match self.get_block_indices(level0_index, level1_index){ 179 | None => return false, 180 | Some(value) => value, 181 | }; 182 | 183 | unsafe{ 184 | // 2. Get Data block and set bit 185 | let data_block = self.data.blocks_mut().get_unchecked_mut(data_block_index); 186 | let existed = data_block.mask_mut().set_bit::(data_index); 187 | 188 | // TODO: fast check of mutated data_block's primitive == 0? 189 | //if existed{ 190 | // 3. Remove free blocks 191 | if data_block.is_empty(){ 192 | // remove data block 193 | self.data.remove_empty_block_unchecked(data_block_index); 194 | 195 | // remove pointer from level1 196 | let level1_block = self.level1.blocks_mut().get_unchecked_mut(level1_block_index); 197 | level1_block.remove_unchecked(level1_index); 198 | 199 | if level1_block.is_empty(){ 200 | // remove level1 block 201 | self.level1.remove_empty_block_unchecked(level1_block_index); 202 | 203 | // remove pointer from level0 204 | self.level0.remove_unchecked(level0_index); 205 | } 206 | } 207 | //} 208 | existed 209 | } 210 | } 211 | } 212 | 213 | impl BitSetBase 214 | for 215 | RawBitSet 216 | where 217 | Conf: Config, 218 | Level0Block: IBlock, 219 | Level1Block: IBlock, 220 | LevelDataBlock: IBlock, 221 | { 222 | type Conf = Conf; 223 | const TRUSTED_HIERARCHY: bool = true; 224 | } 225 | 226 | impl LevelMasks 227 | for 228 | RawBitSet 229 | where 230 | Conf: Config, 231 | Level0Block: IBlock, 232 | Level1Block: IBlock, 233 | LevelDataBlock: IBlock 234 | { 235 | #[inline] 236 | fn level0_mask(&self) -> Conf::Level0BitBlock { 237 | *self.level0.mask() 238 | } 239 | 240 | #[inline] 241 | unsafe fn level1_mask(&self, level0_index: usize) -> Conf::Level1BitBlock { 242 | let level1_block_index = self.level0.get_or_zero(level0_index).as_usize(); 243 | let level1_block = self.level1.blocks().get_unchecked(level1_block_index); 244 | *level1_block.mask() 245 | } 246 | 247 | #[inline] 248 | unsafe fn data_mask(&self, level0_index: usize, level1_index: usize) -> Conf::DataBitBlock { 249 | let level1_block_index = self.level0.get_or_zero(level0_index).as_usize(); 250 | let level1_block = self.level1.blocks().get_unchecked(level1_block_index); 251 | 252 | let data_block_index = level1_block.get_or_zero(level1_index).as_usize(); 253 | let data_block = self.data.blocks().get_unchecked(data_block_index); 254 | *data_block.mask() 255 | } 256 | } 257 | 258 | impl LevelMasksIterExt 259 | for 260 | RawBitSet 261 | where 262 | Conf: Config, 263 | Level0Block: IBlock, 264 | Level1Block: IBlock, 265 | LevelDataBlock: IBlock 266 | { 267 | /// Points to elements in heap. Guaranteed to be stable. 268 | /// This is just plain pointers with null in default: 269 | /// `(*const LevelDataBlock, *const Level1Block)` 270 | type Level1BlockData = ( 271 | Option>, /* data array pointer */ 272 | Option> /* block pointer */ 273 | ); 274 | 275 | type IterState = (); 276 | fn make_iter_state(&self) -> Self::IterState { () } 277 | unsafe fn drop_iter_state(&self, _: &mut ManuallyDrop) {} 278 | 279 | #[inline] 280 | unsafe fn init_level1_block_data( 281 | &self, 282 | _: &mut Self::IterState, 283 | level1_block_data: &mut MaybeUninit, 284 | level0_index: usize 285 | ) -> (::Level1BitBlock, bool){ 286 | let level1_block_index = self.level0.get_or_zero(level0_index); 287 | let level1_block = self.level1.blocks().get_unchecked(level1_block_index.as_usize()); 288 | level1_block_data.write( 289 | ( 290 | Some(NonNull::new_unchecked(self.data.blocks().as_ptr() as *mut _)), 291 | Some(NonNull::from(level1_block)) 292 | ) 293 | ); 294 | (*level1_block.mask(), !level1_block_index.is_zero()) 295 | } 296 | 297 | #[inline] 298 | unsafe fn data_mask_from_block_data( 299 | level1_blocks: &Self::Level1BlockData, level1_index: usize 300 | ) -> Conf::DataBitBlock { 301 | let array_ptr = level1_blocks.0.unwrap_unchecked().as_ptr().cast_const(); 302 | let level1_block = level1_blocks.1.unwrap_unchecked().as_ref(); 303 | 304 | let data_block_index = level1_block.get_or_zero(level1_index); 305 | let data_block = &*array_ptr.add(data_block_index.as_usize()); 306 | *data_block.mask() 307 | } 308 | } 309 | -------------------------------------------------------------------------------- /src/small_bitset.rs: -------------------------------------------------------------------------------- 1 | use crate::BitSetBase; 2 | use crate::block::Block; 3 | use crate::compact_block::CompactBlock; 4 | use crate::config::{Config, SmallConfig}; 5 | use crate::derive_raw::derive_raw; 6 | use crate::raw::RawBitSet; 7 | 8 | type Level0Block = Block< 9 | ::Level0BitBlock, 10 | ::Level0BlockIndices 11 | >; 12 | type Level1Block = CompactBlock< 13 | ::Level1BitBlock, 14 | ::Level1MaskU64Populations, 15 | ::Level1BlockIndices, 16 | ::Level1SmallBlockIndices, 17 | >; 18 | type LevelDataBlock = Block< 19 | ::DataBitBlock, [usize;0] 20 | >; 21 | 22 | type RawSmallBitSet = RawBitSet< 23 | Conf, 24 | Level0Block, 25 | Level1Block, 26 | LevelDataBlock 27 | >; 28 | 29 | /// Same as [BitSet], but sparsely populated hierarchy blocks 9 times smaller. 30 | /// 31 | /// Which means that sparse sets virtually do not have indirection-related memory overhead! 32 | /// 33 | /// # Memory 34 | /// 35 | /// For [_128bit] each Level1 block consumes just 32 bytes, if less 36 | /// than 8 data blocks pointed from. 256+32 bytes otherwise. 37 | /// 38 | /// # Performance 39 | /// 40 | /// All operations still have O(1) complexity. But in terms of raw performance, 41 | /// (due to additional layer of indirection) 42 | /// this is x1.5 - x2 slower than [BitSet]. Which is still very fast. 43 | /// 44 | /// # Implementation details 45 | /// 46 | /// ```text 47 | /// Level0 128bit SIMD 48 | /// [u8;128] 49 | /// 50 | /// ┌ 128bit SIMD ┐ ╭─ ── ── ── ── ── ── ── ── ── ╮ 51 | /// │ ╭──────────────╮ │ Act as SBO: │ 52 | /// Level1 Vec│ │ [u16;7] │ │ │- Inline SparseBitMap, for 53 | /// │ │ ──────────── │◁┼────┤small size. │ 54 | /// │ │Box<[u16;128]>│ │ - Boxed full-size array with │ 55 | /// └ ╰──────────────╯ ┘ │direct access for a big one. 56 | /// ┌ ┐ ╰ ── ── ── ── ── ── ── ── ── ─╯ 57 | /// Data Vec│ 128bit SIMD │ 58 | /// └ ┘ 59 | /// ``` 60 | /// SparseBitMap - `bit_block` acts as a sparse array: 61 | /// ```text 62 | /// 0 1 2 3 ◁═ popcnt before element 63 | /// (dense_array indices) 64 | /// bit_block 0 1 1 0 0 0 1 1 0 0 ... 65 | /// └───┬─┬───────┬─┬─────────┘ 66 | /// ┌──┘┌┘ ┌─────┘ │ 67 | /// │ │ │ ┌────┘ 68 | /// ▼ ▼ ▼ ▼ 69 | /// dense_array 1, 32, 4, 5 len = bit_block popcnt 70 | /// ``` 71 | /// As you can see, SparseBitMap has fast O(1) per-index access. 72 | /// Insert and remove - are O(N) operations, because 73 | /// `dense_array` must keep its element order. 74 | /// 75 | /// Why not just use SparseBitMap for everything? 76 | /// Big-sized `dense_array` should be placed somewhere outside of LevelBlock struct 77 | /// to be memory efficient (e.g., in dynamic-sized heap). Hence - it 78 | /// will be accessed through the pointer (which is yet another layer of indirection). 79 | /// Due to this, and the O(N) insert/remove - SparseBitMap is used only for small-sized arrays. 80 | /// We aim for a 16-element array - as with this size, data blocks pointed 81 | /// from level1 block will have the same size in total, as a full-sized level1 block 82 | /// indirection array. 83 | /// 84 | /// [BitSet]: crate::BitSet 85 | /// [_128bit]: crate::config::_128bit 86 | pub struct SmallBitSet( 87 | RawSmallBitSet 88 | ); 89 | impl BitSetBase for SmallBitSet { 90 | type Conf = Conf; 91 | const TRUSTED_HIERARCHY: bool = true; 92 | } 93 | derive_raw!( 94 | impl SmallBitSet as RawSmallBitSet where Conf: SmallConfig 95 | ); -------------------------------------------------------------------------------- /test_128.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | set RUSTFLAGS=--cfg hisparsebitset_test_128 4 | cargo test 5 | endlocal -------------------------------------------------------------------------------- /test_256.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | set RUSTFLAGS=--cfg hisparsebitset_test_256 4 | cargo test 5 | endlocal -------------------------------------------------------------------------------- /test_64.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | set RUSTFLAGS=--cfg hisparsebitset_test_64 4 | cargo test 5 | endlocal --------------------------------------------------------------------------------