├── .gitignore
├── external
    └── CMakeLists.txt
├── include
    ├── segment_tree
    │   ├── avx512
    │   │   ├── print_table.py
    │   │   ├── node128.hpp
    │   │   ├── node256.hpp
    │   │   ├── node512.hpp
    │   │   ├── segment_tree_macros.hpp
    │   │   └── segment_tree.hpp
    │   └── avx2
    │   │   ├── search_common.hpp
    │   │   ├── tables.hpp
    │   │   ├── node32.hpp
    │   │   ├── node128.hpp
    │   │   ├── node64.hpp
    │   │   └── segment_tree.hpp
    ├── hft
    │   ├── README.md
    │   ├── fenwick.hpp
    │   ├── fenwick
    │   │   ├── fixedf.hpp
    │   │   ├── fenwick_tree.hpp
    │   │   ├── hybrid.hpp
    │   │   ├── bytef.hpp
    │   │   ├── fixedl.hpp
    │   │   ├── bitf.hpp
    │   │   ├── bitl.hpp
    │   │   ├── bytel.hpp
    │   │   └── typef.hpp
    │   └── darray.hpp
    ├── mranisz
    │   └── README.md
    ├── poppy
    │   ├── README.md
    │   └── shared.h
    ├── util.hpp
    ├── types.hpp
    └── mutable_bitmap.hpp
├── src
    ├── collect_results.sh
    ├── collect_results_mutable_bitmap.py
    ├── CMakeLists.txt
    ├── perf_rank64.cpp
    ├── perf_select64.cpp
    ├── perf_rank512.cpp
    ├── perf_rank256.cpp
    ├── perf_select512.cpp
    ├── perf_select256.cpp
    ├── perf_popcnt_for_avx_select256.cpp
    ├── perf_popcnt_for_avx_select512.cpp
    ├── perf_popcnt256.cpp
    ├── perf_prefix_sum256.cpp
    ├── perf_search512.cpp
    └── perf_search256.cpp
├── .gitmodules
├── test
    ├── test_segment_tree.cpp
    ├── test_rank512.cpp
    ├── test_rank256.cpp
    ├── test_select256.cpp
    ├── test_select512.cpp
    ├── test_nodes.cpp
    ├── test_common.hpp
    ├── test_mutable_bitmap.cpp
    ├── test_mutable_bitmap.hpp
    ├── test_node.hpp
    ├── test_tree.hpp
    └── test_rank_select_algorithms.hpp
├── LICENSE
├── CMakeLists.txt
├── README.md
└── .clang-format


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | build*
3 | old


--------------------------------------------------------------------------------
/external/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | add_subdirectory(sdsl-lite EXCLUDE_FROM_ALL)
2 | add_subdirectory(succinct EXCLUDE_FROM_ALL)
3 | 


--------------------------------------------------------------------------------
/include/segment_tree/avx512/print_table.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | 
 3 | n = int(sys.argv[1])
 4 | val = sys.argv[2] # +1 or -1
 5 | for row in range(n+1):
 6 |     for col in range(row):
 7 |         print("0,", end='')
 8 |     for col in range(n - row):
 9 |         print(val+",", end='')
10 |     print("")


--------------------------------------------------------------------------------
/include/hft/README.md:
--------------------------------------------------------------------------------
1 | Hybrid Compact Fenwick Trees
2 | ----
3 | 
4 | Library implemented by Marchini and Vigna, available [here](https://github.com/pacman616/hybrid-fenwick-tree).
5 | 
6 | The data structures are decribed in the paper *[Compact Fenwick trees for dynamic ranking and selection](https://arxiv.org/pdf/1904.12370.pdf)* by the same authors.


--------------------------------------------------------------------------------
/include/mranisz/README.md:
--------------------------------------------------------------------------------
1 | Rank/Select by Marcin Raniszewski
2 | ----
3 | 
4 | Library implemented by Marcin Raniszewski, available [here](https://github.com/mranisz/rank-select).
5 | 
6 | The data structures are decribed in the paper, *[Rank and select: Another lesson learned](https://arxiv.org/abs/1605.01539)* by Szymon Grabowski and Marcin Raniszewski.


--------------------------------------------------------------------------------
/src/collect_results.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # $1 --> output file
3 | ./perf_prefix_sum avx2 sum 2>> $1.sum.txt
4 | ./perf_prefix_sum avx2 update 2>> $1.update.txt
5 | ./perf_prefix_sum avx2 search 2>> $1.search.txt
6 | ./perf_prefix_sum avx512 sum 2>> $1.sum.txt
7 | ./perf_prefix_sum avx512 update 2>> $1.update.txt
8 | ./perf_prefix_sum avx512 search 2>> $1.search.txt


--------------------------------------------------------------------------------
/include/hft/fenwick.hpp:
--------------------------------------------------------------------------------
 1 | #include "fenwick/fenwick_tree.hpp"
 2 | 
 3 | #include "fenwick/fixedf.hpp"
 4 | #include "fenwick/fixedl.hpp"
 5 | 
 6 | #include "fenwick/typef.hpp"
 7 | #include "fenwick/typel.hpp"
 8 | 
 9 | #include "fenwick/bytef.hpp"
10 | #include "fenwick/bytel.hpp"
11 | 
12 | #include "fenwick/bitf.hpp"
13 | #include "fenwick/bitl.hpp"
14 | 
15 | #include "fenwick/hybrid.hpp"
16 | 


--------------------------------------------------------------------------------
/include/poppy/README.md:
--------------------------------------------------------------------------------
1 | Poppy
2 | ----
3 | 
4 | Library implemented by Efficient Computing at Carnegie Mellon, available [here](https://github.com/efficient/rankselect).
5 | 
6 | The data structures are decribed in the paper *[Space-efficient, high-performance rank and select structures on uncompressed bit sequences](https://link.springer.com/chapter/10.1007/978-3-642-38527-8_15)* by Zhou, Andersen, and Kaminsky.
7 | 
8 | The implementation uses PTSelect described in the paper *[A Fast x86 Implementation of Select](http://arxiv.org/abs/1706.00990)* by Pandey, Bender, and Johnson.
9 | 


--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
 1 | [submodule "external/essentials"]
 2 | 	path = external/essentials
 3 | 	url = https://github.com/jermp/essentials.git
 4 | [submodule "external/doctest"]
 5 | 	path = external/doctest
 6 | 	url = https://github.com/onqtam/doctest.git
 7 | [submodule "external/cmd_line_parser"]
 8 | 	path = external/cmd_line_parser
 9 | 	url = https://github.com/jermp/cmd_line_parser.git
10 | [submodule "external/succinct"]
11 | 	path = external/succinct
12 | 	url = https://github.com/ot/succinct.git
13 | [submodule "external/sdsl-lite"]
14 | 	path = external/sdsl-lite
15 | 	url = https://github.com/simongog/sdsl-lite.git
16 | 


--------------------------------------------------------------------------------
/src/collect_results_mutable_bitmap.py:
--------------------------------------------------------------------------------
 1 | import sys, os
 2 | 
 3 | 
 4 | output_filename = sys.argv[1]
 5 | 
 6 | types = [
 7 | "avx2_256_a",
 8 | "avx512_256_a",
 9 | "avx2_256_b",
10 | "avx512_256_b",
11 | "avx2_512_a",
12 | "avx512_512_a",
13 | "avx2_512_b",
14 | "avx512_512_b",
15 | "avx2_256_c",
16 | "avx512_256_c"]
17 | 
18 | for t in types:
19 |     os.system("./perf_mutable_bitmap " + t + " flip 0.3 2>> " + output_filename + ".flip.txt")
20 |     os.system("./perf_mutable_bitmap " + t + " rank 0.3 2>> " + output_filename + ".rank.txt")
21 |     os.system("./perf_mutable_bitmap " + t + " select 0.3 2>> " + output_filename + ".select.txt")
22 | 


--------------------------------------------------------------------------------
/test/test_segment_tree.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_tree.hpp"
 3 | 
 4 | template <int i, template <uint32_t> typename Tree>
 5 | struct test {
 6 |     static void run() {
 7 |         const uint32_t n = sizes[i];
 8 |         const uint32_t height = Tree<1>::height(n);
 9 |         test_tree<Tree<height>>(n);
10 |         test<i + 1, Tree>::run();
11 |     }
12 | };
13 | 
14 | template <template <uint32_t> typename Tree>
15 | struct test<sizeof(sizes) / sizeof(sizes[0]), Tree> {
16 |     static void run() {}
17 | };
18 | 
19 | TEST_CASE("test avx2::segment_tree") {
20 |     test<0, avx2::segment_tree>::run();
21 | }
22 | 
23 | TEST_CASE("test avx512::segment_tree") {
24 |     test<0, avx512::segment_tree>::run();
25 | }


--------------------------------------------------------------------------------
/test/test_rank512.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_rank_select_algorithms.hpp"
 3 | 
 4 | TEST_CASE("test broadword_loop") {
 5 |     test_rank_512<rank_modes::broadword_loop>();
 6 | }
 7 | TEST_CASE("test broadword_unrolled") {
 8 |     test_rank_512<rank_modes::broadword_unrolled>();
 9 | }
10 | #ifdef __SSE4_2__
11 | TEST_CASE("test builtin_loop") {
12 |     test_rank_512<rank_modes::builtin_loop>();
13 | }
14 | TEST_CASE("test builtin_unrolled") {
15 |     test_rank_512<rank_modes::builtin_unrolled>();
16 | }
17 | #endif
18 | #ifdef __AVX512VL__
19 | TEST_CASE("test avx512_unrolled") {
20 |     test_rank_512<rank_modes::avx512_unrolled>();
21 | }
22 | TEST_CASE("test avx512_parallel") {
23 |     test_rank_512<rank_modes::avx512_parallel>();
24 | }
25 | #endif
26 | 


--------------------------------------------------------------------------------
/test/test_rank256.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_rank_select_algorithms.hpp"
 3 | 
 4 | TEST_CASE("test broadword_loop") {
 5 |     test_rank_256<rank_modes::broadword_loop>();
 6 | }
 7 | TEST_CASE("test broadword_unrolled") {
 8 |     test_rank_256<rank_modes::broadword_unrolled>();
 9 | }
10 | #ifdef __SSE4_2__
11 | TEST_CASE("test builtin_loop") {
12 |     test_rank_256<rank_modes::builtin_loop>();
13 | }
14 | TEST_CASE("test builtin_unrolled") {
15 |     test_rank_256<rank_modes::builtin_unrolled>();
16 | }
17 | #endif
18 | #ifdef __AVX2__
19 | TEST_CASE("test avx2_unrolled") {
20 |     test_rank_256<rank_modes::avx2_unrolled>();
21 | }
22 | #endif
23 | #ifdef __AVX512VL__
24 | TEST_CASE("test avx2_parallel") {
25 |     test_rank_256<rank_modes::avx2_parallel>();
26 | }
27 | #endif
28 | 


--------------------------------------------------------------------------------
/include/poppy/shared.h:
--------------------------------------------------------------------------------
 1 | #ifndef _SHARED_H_
 2 | #define _SHARED_H_
 3 | 
 4 | #include <sys/time.h>
 5 | #include <sys/types.h>
 6 | #include <stdint.h>
 7 | #include <inttypes.h>
 8 | 
 9 | namespace poppy {
10 | 
11 | typedef uint16_t uint16;
12 | typedef uint32_t uint32;
13 | typedef uint64_t uint64;
14 | 
15 | const int kCacheLineSize = 64;
16 | 
17 | inline double timeval_diff(const struct timeval* start,
18 |                            const struct timeval* end) {
19 |     double r = (end->tv_sec - start->tv_sec) * 1000000;
20 | 
21 |     if (end->tv_usec > start->tv_usec)
22 |         r += (end->tv_usec - start->tv_usec);
23 |     else if (end->tv_usec < start->tv_usec)
24 |         r -= (start->tv_usec - end->tv_usec);
25 | 
26 |     return (double)r / 1000000;
27 | }
28 | 
29 | }  // namespace poppy
30 | 
31 | #endif /* _SHARED_H_ */
32 | 


--------------------------------------------------------------------------------
/test/test_select256.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_rank_select_algorithms.hpp"
 3 | 
 4 | TEST_CASE("test broadword_loop_sdsl") {
 5 |     test_select_256<select_modes::broadword_loop_sdsl>();
 6 | }
 7 | TEST_CASE("test broadword_loop_succinct") {
 8 |     test_select_256<select_modes::broadword_loop_succinct>();
 9 | }
10 | #ifdef __SSE4_2__
11 | TEST_CASE("test builtin_loop_sdsl") {
12 |     test_select_256<select_modes::builtin_loop_sdsl>();
13 | }
14 | TEST_CASE("test builtin_loop_succinct") {
15 |     test_select_256<select_modes::builtin_loop_succinct>();
16 | }
17 | #endif
18 | #ifdef __BMI2__
19 | TEST_CASE("test builtin_loop_pdep") {
20 |     test_select_256<select_modes::builtin_loop_pdep>();
21 | }
22 | #endif
23 | #ifdef __AVX512VL__
24 | TEST_CASE("test builtin_avx512_pdep") {
25 |     test_select_256<select_modes::builtin_avx512_pdep>();
26 | }
27 | TEST_CASE("test avx2_avx512_pdep") {
28 |     test_select_256<select_modes::avx2_avx512_pdep>();
29 | }
30 | #endif


--------------------------------------------------------------------------------
/test/test_select512.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_rank_select_algorithms.hpp"
 3 | 
 4 | TEST_CASE("test broadword_loop_sdsl") {
 5 |     test_select_512<select_modes::broadword_loop_sdsl>();
 6 | }
 7 | TEST_CASE("test broadword_loop_succinct") {
 8 |     test_select_512<select_modes::broadword_loop_succinct>();
 9 | }
10 | #ifdef __SSE4_2__
11 | TEST_CASE("test builtin_loop_sdsl") {
12 |     test_select_512<select_modes::builtin_loop_sdsl>();
13 | }
14 | TEST_CASE("test builtin_loop_succinct") {
15 |     test_select_512<select_modes::builtin_loop_succinct>();
16 | }
17 | #endif
18 | #ifdef __BMI2__
19 | TEST_CASE("test builtin_loop_pdep") {
20 |     test_select_512<select_modes::builtin_loop_pdep>();
21 | }
22 | #endif
23 | #ifdef __AVX512VL__
24 | TEST_CASE("test builtin_avx512_pdep") {
25 |     test_select_512<select_modes::builtin_avx512_pdep>();
26 | }
27 | TEST_CASE("test avx512_avx512_pdep") {
28 |     test_select_512<select_modes::avx512_avx512_pdep>();
29 | }
30 | #endif


--------------------------------------------------------------------------------
/include/segment_tree/avx2/search_common.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace mrs::avx2 {
 4 | 
 5 | inline int ctzll(unsigned long long w) {
 6 |     // the if is necessary since result is undefined otherwise,
 7 |     // although on most architectures it correctly returns 64
 8 |     if (w == 0) return 64;
 9 |     return __builtin_ctzll(w);
10 | }
11 | 
12 | // find index of "first set" (fs) -- 128-bit version
13 | template <uint64_t word_bits>  // 8, 16, 32, 64
14 | inline static uint64_t index_fs(__m128i const& vec) {
15 |     uint64_t const* w = reinterpret_cast<uint64_t const*>(&vec);
16 |     uint64_t index = (ctzll(w[0]) + ctzll(w[1])) / word_bits;
17 |     return index;
18 | }
19 | 
20 | // find index of "first set" (fs) -- 256-bit version
21 | template <uint64_t word_bits>  // 8, 16, 32, 64
22 | inline static uint64_t index_fs(__m256i const& vec) {
23 |     uint64_t const* w = reinterpret_cast<uint64_t const*>(&vec);
24 |     uint64_t index =
25 |         (ctzll(w[0]) + ctzll(w[1]) + ctzll(w[2]) + ctzll(w[3])) / word_bits;
26 |     return index;
27 | }
28 | 
29 | }  // namespace mrs::avx2


--------------------------------------------------------------------------------
/test/test_nodes.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_node.hpp"
 3 | 
 4 | TEST_CASE("test avx2::node128") {
 5 |     test_node<avx2::node128, 1ULL << 8>();
 6 |     test_node<avx2::node128, 1ULL << 9>();
 7 | }
 8 | 
 9 | TEST_CASE("test avx2::node64") {
10 |     test_node<avx2::node64, 1ULL << 15>();
11 |     test_node<avx2::node64, 1ULL << 16>();
12 | }
13 | 
14 | TEST_CASE("test avx2::node64") {
15 |     test_node<avx2::node64, 1ULL << 21>();
16 |     test_node<avx2::node64, 1ULL << 22>();
17 | }
18 | 
19 | TEST_CASE("test avx2::node32") {
20 |     test_node<avx2::node32, 1ULL << 27>();
21 |     test_node<avx2::node32, 1ULL << 28>();
22 | }
23 | 
24 | TEST_CASE("test avx512::node512") {
25 |     test_node<avx512::node512, 1ULL << 8>();
26 |     test_node<avx512::node512, 1ULL << 9>();
27 | }
28 | 
29 | TEST_CASE("test avx512::node256") {
30 |     test_node<avx512::node256, 1ULL << 17>();
31 |     test_node<avx512::node256, 1ULL << 18>();
32 | }
33 | 
34 | TEST_CASE("test avx512::node128") {
35 |     test_node<avx512::node128, 1ULL << 25>();
36 |     test_node<avx512::node128, 1ULL << 26>();
37 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright 2020 Giulio Ermanno Pibiri and Shunsuke Kanda
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included
13 | in all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 | OTHER DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/src/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_executable(perf_prefix_sum perf_prefix_sum.cpp)
 2 | add_executable(perf_rank64 perf_rank64.cpp)
 3 | add_executable(perf_rank256 perf_rank256.cpp)
 4 | add_executable(perf_rank512 perf_rank512.cpp)
 5 | add_executable(perf_select64 perf_select64.cpp)
 6 | add_executable(perf_select256 perf_select256.cpp)
 7 | add_executable(perf_select512 perf_select512.cpp)
 8 | add_executable(perf_mutable_bitmap perf_mutable_bitmap.cpp)
 9 | add_executable(perf_hft perf_hft.cpp)
10 | add_executable(perf_immutable_bitmap perf_immutable_bitmap.cpp)
11 | target_link_libraries(perf_immutable_bitmap sdsl succinct ${Boost_LIBRARIES})
12 | add_executable(perf_compressed_bitmap perf_compressed_bitmap.cpp)
13 | 
14 | add_executable(perf_popcnt256 perf_popcnt256.cpp)
15 | add_executable(perf_popcnt512 perf_popcnt512.cpp)
16 | add_executable(perf_prefix_sum256 perf_prefix_sum256.cpp)
17 | add_executable(perf_prefix_sum512 perf_prefix_sum512.cpp)
18 | 
19 | add_executable(perf_search256 perf_search256.cpp)
20 | add_executable(perf_search512 perf_search512.cpp)
21 | 
22 | #add_executable(perf_popcnt_for_avx_select256 perf_popcnt_for_avx_select256.cpp)
23 | #add_executable(perf_popcnt_for_avx_select512 perf_popcnt_for_avx_select512.cpp)
24 | 


--------------------------------------------------------------------------------
/test/test_common.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
 4 | #include "../external/doctest/doctest/doctest.h"
 5 | 
 6 | #include <iostream>
 7 | #include <vector>
 8 | 
 9 | #include "../external/essentials/include/essentials.hpp"
10 | #include "util.hpp"
11 | #include "types.hpp"
12 | 
13 | namespace mrs::testing {
14 | static constexpr uint32_t sizes[] = {
15 |     5,       9,       13,      25,       37,       251,      316,     398,
16 |     501,     630,     794,     1000,     1258,     1584,     1995,    2511,
17 |     3162,    3981,    5011,    6309,     7943,     10000,    12589,   15848,
18 |     19952,   25118,   31622,   39810,    50118,    63095,    79432,   100000,
19 |     125892,  158489,  199526,  251188,   316227,   398107,   501187,  630957,
20 |     794328,  1000000, 1258925, 1584893,  1995262,  2511886,  3162277, 3981071,
21 |     5011872, 6309573, 7943282, 10000000, 12589254, 15848931,
22 |     16777216  // max value
23 | };
24 | static constexpr uint32_t logs2[] = {
25 |     8,  9,  10, 11, 12, 13, 14,
26 |     15, 16, 17, 18, 19, 20
27 |     // , 21, 22, 23, 24, 25,
28 |     // 26, 27, 28, 29, 30, 31, 32
29 | };
30 | }  // namespace mrs::testing
31 | 
32 | using namespace mrs;
33 | using namespace mrs::testing;


--------------------------------------------------------------------------------
/test/test_mutable_bitmap.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_common.hpp"
 2 | #include "test_mutable_bitmap.hpp"
 3 | 
 4 | template <int i, template <uint32_t> typename SearchablePrefixSums,
 5 |           typename BlockTypeTraits>
 6 | struct test {
 7 |     static void run() {
 8 |         const uint64_t num_bits = 1ULL << logs2[i];
 9 |         static constexpr uint64_t height = SearchablePrefixSums<1>::height(
10 |             (num_bits + BlockTypeTraits::block_size - 1) /
11 |             BlockTypeTraits::block_size);
12 |         typedef mutable_bitmap<SearchablePrefixSums<height>, BlockTypeTraits>
13 |             mutable_bitmap_type;
14 |         static constexpr double density = 0.3;
15 |         test_mutable_bitmap<mutable_bitmap_type>(num_bits, density);
16 |         test<i + 1, SearchablePrefixSums, BlockTypeTraits>::run();
17 |     }
18 | };
19 | 
20 | template <template <uint32_t> typename SearchablePrefixSums,
21 |           typename BlockTypeTraits>
22 | struct test<sizeof(logs2) / sizeof(logs2[0]), SearchablePrefixSums,
23 |             BlockTypeTraits> {
24 |     static void run() {}
25 | };
26 | 
27 | TEST_CASE("test avx2_mutable_bitmap_64") {
28 |     test<0, avx2::segment_tree, block64_type_default>::run();
29 | }
30 | 
31 | TEST_CASE("test avx512_mutable_bitmap_64") {
32 |     test<0, avx512::segment_tree, block64_type_default>::run();
33 | }
34 | 
35 | TEST_CASE("test avx2_mutable_bitmap_256") {
36 |     test<0, avx2::segment_tree, block256_type_a>::run();
37 | }
38 | 
39 | TEST_CASE("test avx512_mutable_bitmap_256") {
40 |     test<0, avx512::segment_tree, block256_type_a>::run();
41 | }
42 | 
43 | TEST_CASE("test avx2_mutable_bitmap_512") {
44 |     test<0, avx2::segment_tree, block512_type_a>::run();
45 | }
46 | 
47 | TEST_CASE("test avx512_mutable_bitmap_512") {
48 |     test<0, avx512::segment_tree, block512_type_a>::run();
49 | }


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8)
 2 | project(MRS) # Mutable Rank/Select
 3 | 
 4 | if (NOT CMAKE_BUILD_TYPE)
 5 |   set(CMAKE_BUILD_TYPE "Release")
 6 | endif()
 7 | 
 8 | MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
 9 | 
10 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
11 | 
12 | if (DISABLE_AVX)
13 |   MESSAGE(STATUS "SIMD AVX disabled")
14 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDISABLE_AVX")
15 | endif()
16 | 
17 | if (AVX512)
18 |   MESSAGE(STATUS "SIMD AVX512 enabled")
19 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mbmi2 -mavx512f -mavx512bw -mavx512vl -mavx512vpopcntdq -DAVX512")
20 | endif()
21 | 
22 | if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
23 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
24 | endif ()
25 | 
26 | if (UNIX)
27 | 
28 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
29 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
30 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
31 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
32 |   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-missing-braces")
33 | 
34 |   if (USE_SANITIZERS)
35 |     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
36 |   endif()
37 | 
38 | endif()
39 | 
40 | MESSAGE(STATUS "Compiling with ${CMAKE_CXX_FLAGS}")
41 | 
42 | # For succinct
43 | find_package(Boost 1.42.0 COMPONENTS unit_test_framework iostreams system filesystem REQUIRED)
44 | include_directories(${Boost_INCLUDE_DIRS})
45 | link_directories(${Boost_LIBRARY_DIRS})
46 | 
47 | include_directories(${MRS_SOURCE_DIR}/include)
48 | 
49 | add_subdirectory(external)
50 | add_subdirectory(src)
51 | 
52 | enable_testing()
53 | file(GLOB TEST_SOURCES test/test_*.cpp)
54 | foreach(TEST_SRC ${TEST_SOURCES})
55 |   get_filename_component (TEST_SRC_NAME ${TEST_SRC} NAME_WE) # without extension
56 |   add_executable(${TEST_SRC_NAME} ${TEST_SRC})
57 |   add_test(${TEST_SRC_NAME} ${TEST_SRC_NAME})
58 | endforeach(TEST_SRC)


--------------------------------------------------------------------------------
/include/util.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace mrs {
 4 | 
 5 | struct search_result {
 6 |     uint64_t position;
 7 |     uint64_t sum;  // sum of values up to (excluding) position
 8 | };
 9 | 
10 | template <typename SummaryType, typename KeyType>
11 | void build_node_prefix_sums(KeyType const* input, uint8_t* out,
12 |                             uint64_t segment_size, uint64_t num_segments,
13 |                             uint64_t bytes) {
14 |     std::fill(out, out + bytes, 0);
15 |     SummaryType* summary = reinterpret_cast<SummaryType*>(out);
16 |     uint64_t summary_bytes = num_segments * sizeof(SummaryType);
17 |     KeyType* keys = reinterpret_cast<KeyType*>(out + summary_bytes);
18 |     summary[0] = 0;
19 |     for (uint64_t i = 0; i != num_segments; ++i) {
20 |         keys[0] = input[0];
21 |         for (uint64_t j = 1; j != segment_size; ++j) {
22 |             keys[j] = keys[j - 1] + input[j];
23 |         }
24 |         if (i + 1 < num_segments) {
25 |             summary[i + 1] = summary[i] + keys[segment_size - 1];
26 |         }
27 |         input += segment_size;
28 |         keys += segment_size;
29 |     }
30 | }
31 | 
32 | // From http://xoroshiro.di.unimi.it/splitmix64.c
33 | struct splitmix64 {
34 |     splitmix64(uint64_t seed) : x(seed){};
35 | 
36 |     uint64_t next() {
37 |         uint64_t z = (x += uint64_t(0x9E3779B97F4A7C15));
38 |         z = (z ^ (z >> 30)) * uint64_t(0xBF58476D1CE4E5B9);
39 |         z = (z ^ (z >> 27)) * uint64_t(0x94D049BB133111EB);
40 |         return z ^ (z >> 31);
41 |     }
42 | 
43 | private:
44 |     uint64_t x;
45 | };
46 | 
47 | uint64_t create_random_bits(std::vector<uint64_t>& bits, uint64_t threshold,
48 |                             uint64_t seed = 13) {
49 |     uint64_t num_ones = 0;
50 |     splitmix64 hasher(seed);
51 |     for (uint64_t i = 0; i < bits.size() * 64; i++) {
52 |         if (hasher.next() < threshold) {
53 |             bits[i / 64] |= 1ULL << (i % 64);
54 |             num_ones++;
55 |         }
56 |     }
57 | 
58 |     // If we do not have at least one bit set in the first 256 bits,
59 |     // we cannot create queries for Select256.
60 |     // Therefore, we always set the first bit to 1.
61 |     bool allzero = true;
62 |     for (size_t i = 0; i < 4; i++) {
63 |         if (bits[i] != 0) {
64 |             allzero = false;
65 |             break;
66 |         }
67 |     }
68 |     if (allzero) {
69 |         bits[0] = 1ULL;
70 |         num_ones++;
71 |     }
72 | 
73 |     return num_ones;
74 | }
75 | 
76 | }  // namespace mrs


--------------------------------------------------------------------------------
/include/types.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "segment_tree/avx2/node128.hpp"
 4 | #include "segment_tree/avx2/node64.hpp"
 5 | #include "segment_tree/avx2/node32.hpp"
 6 | #include "segment_tree/avx2/segment_tree.hpp"
 7 | 
 8 | #include "segment_tree/avx512/node512.hpp"
 9 | #include "segment_tree/avx512/node256.hpp"
10 | #include "segment_tree/avx512/node128.hpp"
11 | #include "segment_tree/avx512/segment_tree.hpp"
12 | 
13 | #include "mutable_bitmap.hpp"
14 | 
15 | namespace mrs {
16 | 
17 | template <rank_modes RankMode, select_modes SelectMode>
18 | struct block64_type {
19 |     static constexpr uint64_t block_size = 64;
20 |     static constexpr rank_modes rank_mode = RankMode;
21 |     static constexpr select_modes select_mode = SelectMode;
22 | 
23 |     inline static uint64_t rank(const uint64_t* x, uint64_t i) {
24 |         i += 1;
25 |         uint64_t mask = i == 64 ? -1 : (i != 0) * (1ULL << i) - 1;
26 |         return popcount_u64<popcount_modes::builtin>(*x & mask);
27 |     }
28 |     inline static uint64_t select(const uint64_t* x, uint64_t i) {
29 |         return select_u64<select64_modes::pdep>(*x, i);
30 |     }
31 | };
32 | 
33 | template <rank_modes RankMode, select_modes SelectMode>
34 | struct block256_type {
35 |     static constexpr uint64_t block_size = 256;
36 |     static constexpr rank_modes rank_mode = RankMode;
37 |     static constexpr select_modes select_mode = SelectMode;
38 | 
39 |     inline static uint64_t rank(const uint64_t* x, uint64_t i) {
40 |         return rank_u256<RankMode>(x, i);
41 |     }
42 |     inline static uint64_t select(const uint64_t* x, uint64_t i) {
43 |         return select_u256<SelectMode>(x, i);
44 |     }
45 | };
46 | 
47 | template <rank_modes RankMode, select_modes SelectMode>
48 | struct block512_type {
49 |     static constexpr uint64_t block_size = 512;
50 |     static constexpr rank_modes rank_mode = RankMode;
51 |     static constexpr select_modes select_mode = SelectMode;
52 | 
53 |     inline static uint64_t rank(const uint64_t* x, uint64_t i) {
54 |         return rank_u512<RankMode>(x, i);
55 |     }
56 |     inline static uint64_t select(const uint64_t* x, uint64_t i) {
57 |         return select_u512<SelectMode>(x, i);
58 |     }
59 | };
60 | 
61 | typedef block64_type<rank_modes::builtin, select_modes::pdep>
62 |     block64_type_default;
63 | 
64 | typedef block256_type<rank_modes::builtin_unrolled,
65 |                       select_modes::builtin_loop_pdep>
66 |     block256_type_a;
67 | 
68 | typedef block256_type<rank_modes::builtin_loop, select_modes::builtin_loop_pdep>
69 |     block256_type_b;
70 | 
71 | typedef block512_type<rank_modes::builtin_unrolled,
72 |                       select_modes::builtin_loop_pdep>
73 |     block512_type_a;
74 | 
75 | typedef block512_type<rank_modes::builtin_loop, select_modes::builtin_loop_pdep>
76 |     block512_type_b;
77 | 
78 | #ifdef __AVX512VL__
79 | typedef block256_type<rank_modes::builtin_unrolled,
80 |                       select_modes::avx2_avx512_pdep>
81 |     block256_type_c;
82 | #endif
83 | 
84 | }  // namespace mrs


--------------------------------------------------------------------------------
/include/hft/fenwick/fixedf.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_FIXED_HPP__
  2 | #define __FENWICK_FIXED_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class FixedF - no compression and classical node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class FixedF : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   size_t Size;
 22 |   DArray<uint64_t> Tree;
 23 | 
 24 | public:
 25 |   FixedF(uint64_t sequence[], size_t size) : Size(size), Tree(pos(size) + 1) {
 26 |     for (size_t j = 1; j <= size; j++)
 27 |       Tree[pos(j)] = sequence[j - 1];
 28 | 
 29 |     for (size_t m = 2; m <= size; m <<= 1) {
 30 |       for (size_t idx = m; idx <= size; idx += m)
 31 |         Tree[pos(idx)] += Tree[pos(idx - m / 2)];
 32 |     }
 33 |   }
 34 | 
 35 |   virtual uint64_t prefix(size_t idx) const {
 36 |     uint64_t sum = 0;
 37 | 
 38 |     while (idx != 0) {
 39 |       sum += Tree[pos(idx)];
 40 |       idx = clear_rho(idx);
 41 |     }
 42 | 
 43 |     return sum;
 44 |   }
 45 | 
 46 |   virtual void add(size_t idx, int64_t inc) {
 47 |     while (idx <= Size) {
 48 |       Tree[pos(idx)] += inc;
 49 |       idx += mask_rho(idx);
 50 |     }
 51 |   }
 52 | 
 53 |   using FenwickTree::find;
 54 |   virtual size_t find(uint64_t *val) const {
 55 |     size_t node = 0;
 56 | 
 57 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 58 |       if (node + m > Size)
 59 |         continue;
 60 | 
 61 |       uint64_t value = Tree[pos(node + m)];
 62 | 
 63 |       if (*val >= value) {
 64 |         node += m;
 65 |         *val -= value;
 66 |       }
 67 |     }
 68 | 
 69 |     return node;
 70 |   }
 71 | 
 72 |   using FenwickTree::compFind;
 73 |   virtual size_t compFind(uint64_t *val) const {
 74 |     size_t node = 0;
 75 | 
 76 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 77 |       if (node + m > Size)
 78 |         continue;
 79 | 
 80 |       uint64_t value = (BOUND << rho(node + m)) - Tree[pos(node + m)];
 81 | 
 82 |       if (*val >= value) {
 83 |         node += m;
 84 |         *val -= value;
 85 |       }
 86 |     }
 87 | 
 88 |     return node;
 89 |   }
 90 | 
 91 |   virtual size_t size() const { return Size; }
 92 | 
 93 |   virtual size_t bitCount() const {
 94 |     return sizeof(FixedF<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree);
 95 |   }
 96 | 
 97 | private:
 98 |   static inline size_t holes(size_t idx) { return idx >> 14; }
 99 | 
100 |   static inline size_t pos(size_t idx) { return idx + holes(idx); }
101 | 
102 |   friend std::ostream &operator<<(std::ostream &os, const FixedF<BOUND> &ft) {
103 |     uint64_t nsize = hton((uint64_t)ft.Size);
104 |     os.write((char *)&nsize, sizeof(uint64_t));
105 | 
106 |     return os << ft.Tree;
107 |   }
108 | 
109 |   friend std::istream &operator>>(std::istream &is, FixedF<BOUND> &ft) {
110 |     uint64_t nsize;
111 |     is.read((char *)(&nsize), sizeof(uint64_t));
112 | 
113 |     ft.Size = ntoh(nsize);
114 |     return is >> ft.Tree;
115 |   }
116 | };
117 | 
118 | } // namespace hft::fenwick
119 | 
120 | #endif // __FENWICK_FIXED_HPP__
121 | 


--------------------------------------------------------------------------------
/test/test_mutable_bitmap.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace mrs::testing {
 4 | 
 5 | template <typename MutableBitmap>
 6 | void test_mutable_bitmap(uint64_t num_bits, double density) {
 7 |     assert(num_bits >= 256);
 8 |     std::cout << "== testing " << MutableBitmap::name() << " with " << num_bits
 9 |               << " bits ==" << std::endl;
10 | 
11 |     MutableBitmap bitmap;
12 |     uint64_t num_ones = 0;
13 |     {
14 |         std::vector<uint64_t> bits((num_bits + 63) / 64);
15 |         num_ones = create_random_bits(bits, UINT64_MAX * density,
16 |                                       essentials::get_random_seed());
17 |         bitmap.build(bits.data(), bits.size());
18 |     }
19 | 
20 |     auto rank = [&]() {
21 |         essentials::logger("testing rank queries...");
22 |         auto const& bits = bitmap.bits();
23 |         uint64_t bits_in_word = 64;
24 |         uint64_t word_index = 0;
25 |         uint64_t word = bits[word_index];
26 |         uint64_t pos = 0;
27 |         uint64_t expected = 0;
28 |         for (uint64_t i = 0; i != num_bits; ++i) {
29 |             while (true) {
30 |                 expected += word & 1;
31 |                 word >>= 1;
32 |                 --bits_in_word;
33 |                 if (bits_in_word == 0) {
34 |                     if (++word_index == bits.size()) break;
35 |                     word = bits[word_index];
36 |                     bits_in_word = 64;
37 |                 }
38 |                 if (pos++ == i) break;
39 |             }
40 |             uint64_t got = bitmap.rank(i);
41 |             REQUIRE_MESSAGE(got == expected,
42 |                             "error got rank(" << i << ") = " << got
43 |                                               << " but expected " << expected);
44 |         }
45 |     };
46 | 
47 |     auto select = [&]() {
48 |         essentials::logger("testing select queries...");
49 |         auto const& bits = bitmap.bits();
50 |         uint64_t bits_in_word = 64;
51 |         uint64_t word_index = 0;
52 |         uint64_t word = bits[word_index];
53 |         uint64_t expected = -1;
54 |         uint64_t ones = 0;
55 |         for (uint64_t i = 0; i != num_ones; ++i) {
56 |             while (true) {
57 |                 ones += word & 1;
58 |                 word >>= 1;
59 |                 ++expected;
60 |                 --bits_in_word;
61 |                 if (bits_in_word == 0) {
62 |                     if (++word_index == bits.size()) break;
63 |                     word = bits[word_index];
64 |                     bits_in_word = 64;
65 |                 }
66 |                 if (ones == i + 1) break;
67 |             }
68 |             uint64_t got = bitmap.select(i);
69 |             REQUIRE_MESSAGE(got == expected, "error got select("
70 |                                                  << i << ") = " << got
71 |                                                  << " but expected "
72 |                                                  << expected);
73 |         }
74 |     };
75 | 
76 |     rank();
77 |     select();
78 | 
79 |     static constexpr uint64_t num_flips = 5000;
80 |     splitmix64 distr(essentials::get_random_seed());
81 |     essentials::logger("flipping some bits...");
82 |     for (uint64_t i = 0; i != num_flips; ++i) {
83 |         uint64_t pos = distr.next() % bitmap.size();
84 |         bitmap.flip(pos);
85 |     }
86 | 
87 |     rank();
88 |     select();
89 | 
90 |     std::cout << "\teverything's good" << std::endl;
91 | }
92 | 
93 | }  // namespace mrs::testing


--------------------------------------------------------------------------------
/include/hft/fenwick/fenwick_tree.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_FENWICKTREE_HPP__
  2 | #define __FENWICK_FENWICKTREE_HPP__
  3 | 
  4 | #include "../common.hpp"
  5 | #include "../darray.hpp"
  6 | 
  7 | namespace hft::fenwick {
  8 | 
  9 | /**
 10 |  * FenwickTree - Fenwick Tree data structure interface.
 11 |  * @sequence: An integer vector.
 12 |  * @size: The length of the sequence.
 13 |  * @BOUND: maximum value that @sequence can store.
 14 |  *
 15 |  * This data structure indices starts from 1 and ends in @size.
 16 |  *
 17 |  */
 18 | class FenwickTree {
 19 | public:
 20 |   virtual ~FenwickTree() = default;
 21 | 
 22 |   /**
 23 |    * prefix() - Compute the prefix sum.
 24 |    * @idx: Length of the prefix sum.
 25 |    *
 26 |    * Sum the elements in the range (0 .. @idx], returns zero when @idx
 27 |    * is zero.
 28 |    *
 29 |    */
 30 |   virtual uint64_t prefix(size_t idx) const = 0;
 31 | 
 32 |   /**
 33 |    * add() - Increment an element of the sequence (not the tree).
 34 |    * @idx: Index (starting from 1) of the element.
 35 |    * @inc: Value to sum.
 36 |    *
 37 |    * You are allowed to use negative values for the increment, but keep in mind you should respect
 38 |    * the structure boundaries.
 39 |    *
 40 |    */
 41 |   virtual void add(size_t idx, int64_t inc) = 0;
 42 | 
 43 |   /**
 44 |    * find() - Search the index of the closest (less or equal than) prefix.
 45 |    * @val: Prefix to search.
 46 |    *
 47 |    * If @val is an l-value reference its value will be changed with the distance between the found
 48 |    * and the searched prefix (i.e. the difference between the prefix and @val).
 49 |    *
 50 |    * This method returns zero if such an element doesn't exists (i.e. there are no prefixes that are
 51 |    * greater or equal to @val).
 52 |    *
 53 |    */
 54 |   virtual size_t find(uint64_t *val) const = 0;
 55 |   size_t find(uint64_t val) const { return find(&val); }
 56 | 
 57 |   /**
 58 |    * compFind() - Complement find.
 59 |    * @val: Prefix to search.
 60 |    *
 61 |    * This method search the index whose its prefix its the closest to MAXVAL-@val. MAXVAL is the
 62 |    * maximum possibile value for such a prefix (@sequence is therefore bounded).
 63 |    *
 64 |    * The same considerations made for FenwickTree::find() holds.
 65 |    *
 66 |    */
 67 |   virtual size_t compFind(uint64_t *val) const = 0;
 68 |   size_t compFind(uint64_t val) const { return compFind(&val); }
 69 | 
 70 |   /**
 71 |    * size() - Returns the length of the sequence.
 72 |    *
 73 |    */
 74 |   virtual size_t size() const = 0;
 75 | 
 76 |   /**
 77 |    * bitCount() - Estimation of the size (in bits) of this structure.
 78 |    *
 79 |    */
 80 |   virtual size_t bitCount() const = 0;
 81 | 
 82 |   /**
 83 |    * Each FenwickTree is serializable and deserializable with:
 84 |    * - friend std::ostream &operator<<(std::ostream &os, const FenwickTree &ft);
 85 |    * - friend std::istream &operator>>(std::istream &is, FenwickTree &ft);
 86 |    *
 87 |    * The data is stored and loaded with the network (big-endian) byte order to guarantee
 88 |    * compatibility on different architectures.
 89 |    *
 90 |    * The serialized data follows the compression and node ordering of the specific Fenwick tree
 91 |    * without any compatibility layer (e.g. if you serialize a FixedF, you cannot deserialize the
 92 |    * very same data with a ByteL).
 93 |    *
 94 |    */
 95 | };
 96 | 
 97 | } // namespace hft::fenwick
 98 | 
 99 | #endif // __FENWICK_FENWICKTREE_HPP__
100 | 


--------------------------------------------------------------------------------
/test/test_node.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace mrs::testing {
 4 | 
 5 | template <typename Node, uint64_t max_int>
 6 | void test_node() {
 7 |     std::cout << "# " << Node::name() << " # fanout " << Node::fanout
 8 |               << " # max_int " << max_int << std::endl;
 9 | 
10 |     typedef typename Node::key_type uint_type;
11 |     std::vector<uint_type> A(Node::fanout);
12 |     {
13 |         essentials::uniform_int_rng<uint_type> distr(
14 |             0, max_int, essentials::get_random_seed());
15 |         std::generate(A.begin(), A.end(), [&] { return distr.gen(); });
16 |     }
17 | 
18 |     std::vector<uint8_t> out(Node::bytes);
19 |     Node::build(A.data(), out.data());
20 |     Node node(out.data());
21 | 
22 |     {
23 |         essentials::logger("testing sum queries...");
24 |         uint64_t expected = 0;
25 |         for (uint32_t i = 0; i != Node::fanout; ++i) {
26 |             uint64_t got = node.sum(i);
27 |             expected += A[i];
28 |             REQUIRE_MESSAGE(got == expected, "error during SUM: got sum("
29 |                                                  << i << ") = " << got
30 |                                                  << " but expected "
31 |                                                  << expected);
32 |         }
33 |     }
34 | 
35 |     auto update = [&](int8_t delta) {
36 |         bool sign = delta >> 7;
37 |         essentials::logger("testing update(" + std::to_string(delta) + ")...");
38 |         for (uint64_t run = 0; run != 100; ++run) {
39 |             uint64_t expected = 0;
40 |             for (uint64_t i = 0; i != Node::fanout; ++i) {
41 |                 node.update(i, sign);
42 |                 uint64_t got = node.sum(i);
43 |                 A[i] += delta;
44 |                 expected += A[i];
45 |                 REQUIRE_MESSAGE(got == expected,
46 |                                 "error during UPDATE("
47 |                                     << int(delta) << ") at run " << run
48 |                                     << ": got sum(" << i << ") = " << got
49 |                                     << " but expected " << expected);
50 |             }
51 |         }
52 |     };
53 | 
54 |     {
55 |         essentials::logger("testing search queries...");
56 |         uint64_t max_sum = node.sum(Node::fanout - 1);
57 |         essentials::uniform_int_rng<uint_type> distr(
58 |             0, max_sum - 1, essentials::get_random_seed());
59 |         for (uint32_t i = 0; i != 10000; ++i) {
60 |             uint64_t x = distr.gen();
61 |             uint64_t expected_position = 0;
62 |             uint64_t expected_sum = 0;
63 |             for (; expected_position != Node::fanout; ++expected_position) {
64 |                 if (expected_sum + A[expected_position] > x) break;
65 |                 expected_sum += A[expected_position];
66 |             }
67 |             auto got = node.search(x);
68 |             bool good = (got.position == expected_position) and
69 |                         (got.sum == expected_sum);
70 |             REQUIRE_MESSAGE(good, "error during SEARCH: got search("
71 |                                       << x << ") = (" << got.position << ","
72 |                                       << got.sum << ")"
73 |                                       << " but expected (" << expected_position
74 |                                       << "," << expected_sum << ")");
75 |         }
76 |     }
77 | 
78 |     update(+1);
79 |     update(-1);
80 | 
81 |     std::cout << "\teverything's good" << std::endl;
82 | }
83 | 
84 | }  // namespace mrs::testing


--------------------------------------------------------------------------------
/test/test_tree.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace mrs::testing {
 4 | 
 5 | template <typename Tree>
 6 | void test_tree(size_t n) {
 7 |     essentials::uniform_int_rng<uint16_t> distr(0, 256,
 8 |                                                 essentials::get_random_seed());
 9 |     std::cout << "== testing " << Tree::name() << " with " << n
10 |               << " leaves ==" << std::endl;
11 |     std::vector<uint16_t> A(n);
12 |     std::generate(A.begin(), A.end(), [&] { return distr.gen(); });
13 |     Tree tree;
14 |     essentials::logger("building tree...");
15 |     tree.build(A.data(), n);
16 | 
17 |     {
18 |         essentials::logger("testing sum queries...");
19 |         static constexpr uint32_t sum_queries = 5000;
20 |         uint32_t step = n / sum_queries;
21 |         if (step == 0) step = 1;
22 |         uint64_t expected = 0;
23 |         uint32_t k = 0;
24 |         for (uint32_t i = 0; i < n; i += step) {
25 |             uint64_t got = tree.sum(i);
26 |             for (; k != i + 1; ++k) expected += A[k];
27 |             REQUIRE_MESSAGE(got == expected, "got sum(" << i << ") = " << got
28 |                                                         << " but expected "
29 |                                                         << expected);
30 |         }
31 |     }
32 | 
33 |     auto update = [&](int8_t delta) {
34 |         bool sign = delta >> 7;
35 |         essentials::logger("testing update(" + std::to_string(delta) + ")...");
36 |         static constexpr uint32_t update_queries = 5000;
37 |         uint32_t step = n / update_queries;
38 |         if (step == 0) step = 1;
39 |         for (uint32_t run = 0; run != 100; ++run) {
40 |             uint64_t expected = 0;
41 |             uint32_t k = 0;
42 |             for (uint32_t i = 0; i < n; i += step) {
43 |                 tree.update(i, sign);
44 |                 uint64_t got = tree.sum(i);
45 |                 A[i] += delta;
46 |                 for (; k != i + 1; ++k) expected += A[k];
47 |                 REQUIRE_MESSAGE(got == expected,
48 |                                 "error during run "
49 |                                     << run << ": got sum(" << i << ") = " << got
50 |                                     << " but expected " << expected);
51 |             }
52 |         }
53 |     };
54 | 
55 |     {
56 |         essentials::logger("testing search queries...");
57 |         uint64_t max_sum = tree.sum(n - 1);
58 |         essentials::uniform_int_rng<uint64_t> distr(
59 |             0, max_sum - 1, essentials::get_random_seed());
60 |         for (uint32_t i = 0; i != 1000; ++i) {
61 |             uint64_t x = distr.gen();
62 |             uint64_t expected_position = 0;
63 |             uint64_t expected_sum = 0;
64 |             for (; expected_position != n; ++expected_position) {
65 |                 if (expected_sum + A[expected_position] > x) break;
66 |                 expected_sum += A[expected_position];
67 |             }
68 |             auto got = tree.search(x);
69 |             bool good = (got.position == expected_position) and
70 |                         (got.sum == expected_sum);
71 |             REQUIRE_MESSAGE(good, "error during SEARCH: got search("
72 |                                       << x << ") = (" << got.position << ","
73 |                                       << got.sum << ")"
74 |                                       << " but expected (" << expected_position
75 |                                       << "," << expected_sum << ")");
76 |         }
77 |     }
78 | 
79 |     update(+1);
80 |     update(-1);
81 | 
82 |     std::cout << "\teverything's good" << std::endl;
83 | }
84 | 
85 | }  // namespace mrs::testing


--------------------------------------------------------------------------------
/include/segment_tree/avx2/tables.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #if defined(_MSC_VER)
 4 | #define ALIGNED(x) __declspec(align(x))
 5 | #else
 6 | #if defined(__GNUC__)
 7 | #define ALIGNED(x) __attribute__((aligned(x)))
 8 | #endif
 9 | #endif
10 | 
11 | #define AVX2_ALIGNEMENT ALIGNED(32)
12 | 
13 | namespace mrs::avx2::tables {
14 | 
15 | // used by node32
16 | static const int64_t update_4_64[4 * 4 * 2] AVX2_ALIGNEMENT = {
17 | 
18 |     0, +1, +1, +1, 0, 0, +1, +1, 0,
19 |     0, 0,  +1, 0,  0, 0, 0,
20 | 
21 |     0, -1, -1, -1, 0, 0, -1, -1, 0,
22 |     0, 0,  -1, 0,  0, 0, 0
23 | 
24 | };
25 | 
26 | // used by node128, node64, and node32
27 | static const int32_t update_8_32[(8 + 1) * 8 * 2] AVX2_ALIGNEMENT = {
28 | 
29 |     +1, +1, +1, +1, +1, +1, +1, +1, 0,  +1, +1, +1, +1, +1, +1, +1, 0,  0,  +1,
30 |     +1, +1, +1, +1, +1, 0,  0,  0,  +1, +1, +1, +1, +1, 0,  0,  0,  0,  +1, +1,
31 |     +1, +1, 0,  0,  0,  0,  0,  +1, +1, +1, 0,  0,  0,  0,  0,  0,  +1, +1, 0,
32 |     0,  0,  0,  0,  0,  0,  +1, 0,  0,  0,  0,  0,  0,  0,  0,
33 | 
34 |     -1, -1, -1, -1, -1, -1, -1, -1, 0,  -1, -1, -1, -1, -1, -1, -1, 0,  0,  -1,
35 |     -1, -1, -1, -1, -1, 0,  0,  0,  -1, -1, -1, -1, -1, 0,  0,  0,  0,  -1, -1,
36 |     -1, -1, 0,  0,  0,  0,  0,  -1, -1, -1, 0,  0,  0,  0,  0,  0,  -1, -1, 0,
37 |     0,  0,  0,  0,  0,  0,  -1, 0,  0,  0,  0,  0,  0,  0,  0
38 | 
39 | };
40 | 
41 | // used by node128
42 | static const int16_t update_16_16[16 * 16 * 2] AVX2_ALIGNEMENT = {
43 | 
44 |     +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, 0,  +1, +1,
45 |     +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, 0,  0,  +1, +1, +1, +1,
46 |     +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, 0,  0,  0,  +1, +1, +1, +1, +1, +1,
47 |     +1, +1, +1, +1, +1, +1, +1, 0,  0,  0,  0,  +1, +1, +1, +1, +1, +1, +1, +1,
48 |     +1, +1, +1, +1, 0,  0,  0,  0,  0,  +1, +1, +1, +1, +1, +1, +1, +1, +1, +1,
49 |     +1, 0,  0,  0,  0,  0,  0,  +1, +1, +1, +1, +1, +1, +1, +1, +1, +1, 0,  0,
50 |     0,  0,  0,  0,  0,  +1, +1, +1, +1, +1, +1, +1, +1, +1, 0,  0,  0,  0,  0,
51 |     0,  0,  0,  +1, +1, +1, +1, +1, +1, +1, +1, 0,  0,  0,  0,  0,  0,  0,  0,
52 |     0,  +1, +1, +1, +1, +1, +1, +1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  +1,
53 |     +1, +1, +1, +1, +1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  +1, +1, +1,
54 |     +1, +1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  +1, +1, +1, +1, 0,
55 |     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  +1, +1, +1, 0,  0,  0,  0,
56 |     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  +1, +1, 0,  0,  0,  0,  0,  0,  0,
57 |     0,  0,  0,  0,  0,  0,  0,  0,  +1,
58 | 
59 |     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  -1, -1,
60 |     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  0,  -1, -1, -1, -1,
61 |     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  0,  0,  -1, -1, -1, -1, -1, -1,
62 |     -1, -1, -1, -1, -1, -1, -1, 0,  0,  0,  0,  -1, -1, -1, -1, -1, -1, -1, -1,
63 |     -1, -1, -1, -1, 0,  0,  0,  0,  0,  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
64 |     -1, 0,  0,  0,  0,  0,  0,  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  0,
65 |     0,  0,  0,  0,  0,  -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  0,  0,  0,  0,
66 |     0,  0,  0,  -1, -1, -1, -1, -1, -1, -1, -1, 0,  0,  0,  0,  0,  0,  0,  0,
67 |     0,  -1, -1, -1, -1, -1, -1, -1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  -1,
68 |     -1, -1, -1, -1, -1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  -1, -1, -1,
69 |     -1, -1, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  -1, -1, -1, -1, 0,
70 |     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  -1, -1, -1, 0,  0,  0,  0,
71 |     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  -1, -1, 0,  0,  0,  0,  0,  0,  0,
72 |     0,  0,  0,  0,  0,  0,  0,  0,  -1
73 | 
74 | };
75 | 
76 | }  // namespace mrs::avx2::tables


--------------------------------------------------------------------------------
/include/mutable_bitmap.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <vector>
  4 | #include "rank_select_algorithms/rank.hpp"
  5 | #include "rank_select_algorithms/select.hpp"
  6 | 
  7 | namespace mrs {
  8 | 
  9 | template <typename SearchablePrefixSums, typename BlockType>
 10 | struct mutable_bitmap {
 11 |     // currently supported block sizes
 12 |     static_assert(BlockType::block_size == 64 or BlockType::block_size == 256 or
 13 |                   BlockType::block_size == 512);
 14 |     static constexpr uint64_t block_size = BlockType::block_size;
 15 |     static constexpr uint64_t words_in_block = block_size / 64;
 16 | 
 17 |     mutable_bitmap() {}
 18 | 
 19 |     void build(uint64_t const* in, uint64_t num_64bit_words) {
 20 |         assert(num_64bit_words * 64 <= (1ULL << 24) * block_size);
 21 |         uint64_t num_blocks =
 22 |             std::ceil(static_cast<double>(num_64bit_words) / words_in_block);
 23 |         std::vector<uint16_t> prefix_sums_input;
 24 |         prefix_sums_input.reserve(num_blocks);
 25 |         m_bits.resize(words_in_block * num_blocks,
 26 |                       0);  // pad to a multiple of words_in_block
 27 |         std::memcpy(m_bits.data(), in, num_64bit_words * sizeof(uint64_t));
 28 |         for (uint64_t i = 0; i != m_bits.size(); i += words_in_block) {
 29 |             uint16_t val = 0;
 30 |             for (uint64_t j = 0; j != words_in_block; ++j) {
 31 |                 val += __builtin_popcountll(m_bits[i + j]);
 32 |             }
 33 |             assert(val <= block_size);
 34 |             prefix_sums_input.push_back(val);
 35 |         }
 36 |         m_index.build(prefix_sums_input.data(), prefix_sums_input.size());
 37 |     }
 38 | 
 39 |     static std::string name() {
 40 |         return "mutable_bitmap_" + std::to_string(block_size) + "<" +
 41 |                SearchablePrefixSums::name() + "," +
 42 |                print_rank_mode(BlockType::rank_mode) + "," +
 43 |                print_select_mode(BlockType::select_mode) + ">";
 44 |     }
 45 | 
 46 |     uint64_t size() const {
 47 |         return m_bits.size() * sizeof(m_bits.front()) * 8;
 48 |     }
 49 | 
 50 |     uint64_t bytes() const {
 51 |         return m_index.bytes() + size() / 8;
 52 |     }
 53 | 
 54 |     auto const& bits() const {
 55 |         return m_bits;
 56 |     }
 57 | 
 58 |     inline bool access(uint64_t i) const {
 59 |         assert(i < size());
 60 |         uint64_t r;
 61 |         uint64_t w = m_bits[i / 64];
 62 |         __asm volatile(
 63 |             "bt %2,%1\n"
 64 |             "sbb %0,%0"
 65 |             : "=r"(r)
 66 |             : "r"(w), "r"(i));
 67 |         return r;
 68 |     }
 69 | 
 70 |     void flip(uint64_t i) {
 71 |         assert(i < size());
 72 |         bool bit = access(i);
 73 |         uint64_t word = i / 64;
 74 |         uint64_t offset = i & 63;
 75 |         uint64_t block = i / block_size;
 76 |         m_bits[word] ^= 1ULL << offset;
 77 |         m_index.update(block, bit);
 78 |     }
 79 | 
 80 |     uint64_t rank(uint64_t i) const {
 81 |         assert(i < size());
 82 |         uint64_t block = i / block_size;
 83 |         uint64_t offset = i & (block_size - 1);
 84 |         return (block ? m_index.sum(block - 1) : 0) +
 85 |                BlockType::rank(&m_bits[words_in_block * block], offset);
 86 |     }
 87 | 
 88 |     uint64_t select(uint64_t i) const {
 89 |         assert(i < size());
 90 |         auto [block, sum] = m_index.search(i);
 91 |         uint64_t offset = i - sum;  // i - m_index.sum(block - 1);
 92 |         return block_size * block +
 93 |                BlockType::select(&m_bits[words_in_block * block], offset);
 94 |     }
 95 | 
 96 | private:
 97 |     SearchablePrefixSums m_index;
 98 |     std::vector<uint64_t> m_bits;
 99 | };
100 | 
101 | }  // namespace mrs


--------------------------------------------------------------------------------
/include/segment_tree/avx2/node32.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "search_common.hpp"
  8 | #include "util.hpp"
  9 | 
 10 | namespace mrs::avx2 {
 11 | 
 12 | struct node32 {
 13 |     typedef uint32_t key_type;
 14 |     typedef uint64_t summary_type;
 15 |     static constexpr uint64_t fanout = 32;
 16 |     static constexpr uint64_t segment_size = 8;
 17 |     static constexpr uint64_t num_segments = 4;
 18 |     static constexpr uint64_t bytes =
 19 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 20 | 
 21 |     node32() {}
 22 | 
 23 |     static void build(key_type const* input, uint8_t* out) {
 24 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 25 |                                                        num_segments, bytes);
 26 |     }
 27 | 
 28 |     static std::string name() {
 29 |         return "avx2::node32";
 30 |     }
 31 | 
 32 |     node32(uint8_t* ptr) {
 33 |         at(ptr);
 34 |     }
 35 | 
 36 |     inline void at(uint8_t* ptr) {
 37 |         summary = reinterpret_cast<summary_type*>(ptr);
 38 |         keys = reinterpret_cast<key_type*>(ptr +
 39 |                                            num_segments * sizeof(summary_type));
 40 |     }
 41 | 
 42 |     void update(uint64_t i, bool sign) {
 43 |         if (i == fanout) return;
 44 |         assert(i < fanout);
 45 |         uint64_t j = i / segment_size;
 46 |         uint64_t k = i % segment_size;
 47 | #ifdef DISABLE_AVX
 48 |         int8_t delta = sign ? -1 : +1;
 49 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 50 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 51 |             keys[base + z] += delta;
 52 |         }
 53 | #else
 54 |         __m256i s1 = _mm256_load_si256((__m256i const*)tables::update_4_64 + j +
 55 |                                        sign * num_segments);
 56 |         __m256i r1 =
 57 |             _mm256_add_epi64(_mm256_loadu_si256((__m256i const*)summary), s1);
 58 |         _mm256_storeu_si256((__m256i*)summary, r1);
 59 |         __m256i s2 = _mm256_load_si256((__m256i const*)tables::update_8_32 + k +
 60 |                                        sign * (segment_size + 1));
 61 |         __m256i r2 = _mm256_add_epi32(
 62 |             _mm256_loadu_si256((__m256i const*)(keys + j * segment_size)), s2);
 63 |         _mm256_storeu_si256((__m256i*)(keys + j * segment_size), r2);
 64 | #endif
 65 |     }
 66 | 
 67 |     uint64_t sum(uint64_t i) const {
 68 |         assert(i < fanout);
 69 |         return summary[i / segment_size] + keys[i];
 70 |     }
 71 | 
 72 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 73 |        if x >= sum(fanout-1), then fanout is returned */
 74 |     search_result search(uint64_t x) const {
 75 |         // if (x >= sum(fanout - 1)) return fanout;
 76 |         uint64_t i = 0;
 77 | #ifdef DISABLE_AVX
 78 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 79 |             if (summary[z] > x) break;
 80 |         }
 81 |         assert(i < num_segments);
 82 |         x -= summary[i];
 83 |         i *= segment_size;
 84 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 85 |             if (keys[i] > x) break;
 86 |         }
 87 | #else
 88 |         __m256i cmp1 = _mm256_cmpgt_epi64(
 89 |             _mm256_loadu_si256((__m256i const*)summary), _mm256_set1_epi64x(x));
 90 |         i = index_fs<64>(cmp1) - 1;
 91 |         assert(i < num_segments);
 92 |         x -= summary[i];
 93 |         i *= segment_size;
 94 |         __m256i cmp2 =
 95 |             _mm256_cmpgt_epi32(_mm256_loadu_si256((__m256i const*)(keys + i)),
 96 |                                _mm256_set1_epi32(x));
 97 |         i += index_fs<32>(cmp2);
 98 | #endif
 99 |         return {i, i ? sum(i - 1) : 0};
100 |     }
101 | 
102 | private:
103 |     summary_type* summary;
104 |     key_type* keys;
105 | };
106 | 
107 | }  // namespace mrs::avx2


--------------------------------------------------------------------------------
/include/segment_tree/avx2/node128.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "search_common.hpp"
  8 | #include "util.hpp"
  9 | 
 10 | namespace mrs::avx2 {
 11 | 
 12 | struct node128 {
 13 |     typedef uint16_t key_type;
 14 |     typedef uint32_t summary_type;
 15 |     static constexpr uint64_t fanout = 128;
 16 |     static constexpr uint64_t segment_size = 16;
 17 |     static constexpr uint64_t num_segments = 8;
 18 |     static constexpr uint64_t bytes =
 19 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 20 | 
 21 |     node128() {}
 22 | 
 23 |     static void build(key_type const* input, uint8_t* out) {
 24 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 25 |                                                        num_segments, bytes);
 26 |     }
 27 | 
 28 |     static std::string name() {
 29 |         return "avx2::node128";
 30 |     }
 31 | 
 32 |     node128(uint8_t* ptr) {
 33 |         at(ptr);
 34 |     }
 35 | 
 36 |     inline void at(uint8_t* ptr) {
 37 |         summary = reinterpret_cast<summary_type*>(ptr);
 38 |         keys = reinterpret_cast<key_type*>(ptr +
 39 |                                            num_segments * sizeof(summary_type));
 40 |     }
 41 | 
 42 |     void update(uint64_t i, bool sign) {
 43 |         if (i == fanout) return;
 44 |         assert(i < fanout);
 45 |         uint64_t j = i / segment_size;
 46 |         uint64_t k = i % segment_size;
 47 | #ifdef DISABLE_AVX
 48 |         int8_t delta = sign ? -1 : +1;
 49 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 50 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 51 |             keys[base + z] += delta;
 52 |         }
 53 | #else
 54 |         __m256i s1 = _mm256_load_si256((__m256i const*)tables::update_8_32 + j +
 55 |                                        1 + sign * (num_segments + 1));
 56 |         __m256i r1 =
 57 |             _mm256_add_epi32(_mm256_load_si256((__m256i const*)summary), s1);
 58 |         _mm256_storeu_si256((__m256i*)summary, r1);
 59 |         __m256i s2 = _mm256_load_si256((__m256i const*)tables::update_16_16 +
 60 |                                        k + sign * segment_size);
 61 |         __m256i r2 = _mm256_add_epi16(
 62 |             _mm256_loadu_si256((__m256i const*)(keys + j * segment_size)), s2);
 63 |         _mm256_storeu_si256((__m256i*)(keys + j * segment_size), r2);
 64 | #endif
 65 |     }
 66 | 
 67 |     uint64_t sum(uint64_t i) const {
 68 |         assert(i < fanout);
 69 |         return summary[i / segment_size] + keys[i];
 70 |     }
 71 | 
 72 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 73 |        in this case x is always < sum(fanout-1) by assumption */
 74 |     search_result search(uint64_t x) const {
 75 |         assert(x < sum(fanout - 1));
 76 |         uint64_t i = 0;
 77 | #ifdef DISABLE_AVX
 78 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 79 |             if (summary[z] > x) break;
 80 |         }
 81 |         assert(i < num_segments);
 82 |         x -= summary[i];
 83 |         i *= segment_size;
 84 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 85 |             if (keys[i] > x) break;
 86 |         }
 87 | #else
 88 |         __m256i cmp1 = _mm256_cmpgt_epi32(
 89 |             _mm256_loadu_si256((__m256i const*)summary), _mm256_set1_epi32(x));
 90 |         i = index_fs<32>(cmp1) - 1;
 91 |         assert(i < num_segments);
 92 |         x -= summary[i];
 93 |         i *= segment_size;
 94 |         __m256i cmp2 =
 95 |             _mm256_cmpgt_epi16(_mm256_loadu_si256((__m256i const*)(keys + i)),
 96 |                                _mm256_set1_epi16(x));
 97 |         i += index_fs<16>(cmp2);
 98 | #endif
 99 |         assert(i < fanout);
100 |         return {i, i ? sum(i - 1) : 0};
101 |     }
102 | 
103 | private:
104 |     summary_type* summary;
105 |     key_type* keys;
106 | };
107 | 
108 | }  // namespace mrs::avx2


--------------------------------------------------------------------------------
/include/segment_tree/avx512/node128.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "util.hpp"
  8 | 
  9 | namespace mrs::avx512 {
 10 | 
 11 | struct node128 {
 12 |     typedef uint32_t key_type;
 13 |     typedef uint64_t summary_type;
 14 |     static constexpr uint64_t fanout = 128;
 15 |     static constexpr uint64_t segment_size = 16;
 16 |     static constexpr uint64_t num_segments = 8;
 17 |     static constexpr uint64_t bytes =
 18 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 19 | 
 20 |     node128() {}
 21 | 
 22 |     static void build(key_type const* input, uint8_t* out) {
 23 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 24 |                                                        num_segments, bytes);
 25 |     }
 26 | 
 27 |     static std::string name() {
 28 |         return "avx512::node128";
 29 |     }
 30 | 
 31 |     node128(uint8_t* ptr) {
 32 |         at(ptr);
 33 |     }
 34 | 
 35 |     inline void at(uint8_t* ptr) {
 36 |         summary = reinterpret_cast<summary_type*>(ptr);
 37 |         keys = reinterpret_cast<key_type*>(ptr +
 38 |                                            num_segments * sizeof(summary_type));
 39 |     }
 40 | 
 41 |     void update(uint64_t i, bool sign) {
 42 |         if (i == fanout) return;
 43 |         assert(i < fanout);
 44 |         uint64_t j = i / segment_size;
 45 |         uint64_t k = i % segment_size;
 46 | #ifdef AVX512
 47 |         __m512i s1 = _mm512_load_si512((__m512i const*)tables::update_8_64 + j +
 48 |                                        sign * num_segments);
 49 |         __m512i r1 =
 50 |             _mm512_add_epi64(_mm512_loadu_si512((__m512i const*)summary), s1);
 51 |         _mm512_storeu_si512((__m512i*)summary, r1);
 52 |         __m512i s2 = _mm512_load_si512((__m512i const*)tables::update_16_32 +
 53 |                                        k + sign * (segment_size + 1));
 54 |         __m512i r2 = _mm512_add_epi32(
 55 |             _mm512_loadu_si512((__m512i const*)(keys + j * segment_size)), s2);
 56 |         _mm512_storeu_si512((__m512i*)(keys + j * segment_size), r2);
 57 | #else
 58 |         int8_t delta = sign ? -1 : +1;
 59 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 60 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 61 |             keys[base + z] += delta;
 62 |         }
 63 | #endif
 64 |     }
 65 | 
 66 |     uint64_t sum(uint64_t i) const {
 67 |         assert(i < fanout);
 68 |         return summary[i / segment_size] + keys[i];
 69 |     }
 70 | 
 71 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 72 |        if x >= sum(fanout-1), then fanout is returned */
 73 |     search_result search(uint64_t x) const {
 74 |         if (x >= sum(fanout - 1)) return {fanout, sum(fanout - 1)};
 75 |         uint64_t i = 0;
 76 | #ifdef AVX512
 77 |         __mmask8 cmp1 = _mm512_cmpgt_epi64_mask(
 78 |             _mm512_loadu_si512((__m512i const*)summary), _mm512_set1_epi64(x));
 79 |         i = cmp1 != 0 ? __builtin_ctz(cmp1) - 1 : num_segments - 1;
 80 |         assert(i < num_segments);
 81 |         x -= summary[i];
 82 |         i *= segment_size;
 83 |         __mmask16 cmp2 = _mm512_cmpgt_epi32_mask(
 84 |             _mm512_loadu_si512((__m512i const*)(keys + i)),
 85 |             _mm512_set1_epi32(x));
 86 |         assert(cmp2 > 0);
 87 |         i += __builtin_ctzll(cmp2);
 88 | #else
 89 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 90 |             if (summary[z] > x) break;
 91 |         }
 92 |         assert(i < num_segments);
 93 |         x -= summary[i];
 94 |         i *= segment_size;
 95 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 96 |             if (keys[i] > x) break;
 97 |         }
 98 | #endif
 99 |         return {i, i ? sum(i - 1) : 0};
100 |     }
101 | 
102 | private:
103 |     summary_type* summary;
104 |     key_type* keys;
105 | };
106 | 
107 | }  // namespace mrs::avx512


--------------------------------------------------------------------------------
/include/segment_tree/avx2/node64.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "search_common.hpp"
  8 | #include "util.hpp"
  9 | 
 10 | namespace mrs::avx2 {
 11 | 
 12 | struct node64 {
 13 |     typedef uint32_t key_type;
 14 |     typedef uint32_t summary_type;
 15 |     static constexpr uint64_t fanout = 64;
 16 |     static constexpr uint64_t segment_size = 8;
 17 |     static constexpr uint64_t num_segments = 8;
 18 |     static constexpr uint64_t bytes =
 19 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 20 | 
 21 |     node64() {}
 22 | 
 23 |     static void build(key_type const* input, uint8_t* out) {
 24 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 25 |                                                        num_segments, bytes);
 26 |     }
 27 | 
 28 |     static std::string name() {
 29 |         return "avx2::node64";
 30 |     }
 31 | 
 32 |     node64(uint8_t* ptr) {
 33 |         at(ptr);
 34 |     }
 35 | 
 36 |     inline void at(uint8_t* ptr) {
 37 |         summary = reinterpret_cast<summary_type*>(ptr);
 38 |         keys = reinterpret_cast<key_type*>(ptr +
 39 |                                            num_segments * sizeof(summary_type));
 40 |     }
 41 | 
 42 |     void update(uint64_t i, bool sign) {
 43 |         if (i == fanout) return;
 44 |         assert(i < fanout);
 45 |         uint64_t j = i / segment_size;
 46 |         uint64_t k = i % segment_size;
 47 | #ifdef DISABLE_AVX
 48 |         int8_t delta = sign ? -1 : +1;
 49 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 50 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 51 |             keys[base + z] += delta;
 52 |         }
 53 | #else
 54 |         __m256i s1 = _mm256_load_si256((__m256i const*)tables::update_8_32 + j +
 55 |                                        1 + sign * (num_segments + 1));
 56 |         __m256i r1 =
 57 |             _mm256_add_epi32(_mm256_loadu_si256((__m256i const*)summary), s1);
 58 |         _mm256_storeu_si256((__m256i*)summary, r1);
 59 |         __m256i s2 = _mm256_load_si256((__m256i const*)tables::update_8_32 + k +
 60 |                                        sign * (segment_size + 1));
 61 |         __m256i r2 = _mm256_add_epi32(
 62 |             _mm256_loadu_si256((__m256i const*)(keys + j * segment_size)), s2);
 63 |         _mm256_storeu_si256((__m256i*)(keys + j * segment_size), r2);
 64 | #endif
 65 |     }
 66 | 
 67 |     uint64_t sum(uint64_t i) const {
 68 |         assert(i < fanout);
 69 |         return summary[i / segment_size] + keys[i];
 70 |     }
 71 | 
 72 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 73 |        if x >= sum(fanout-1), then fanout is returned */
 74 |     search_result search(uint64_t x) const {
 75 |         // is it faster to return immediately?
 76 |         // if (x >= sum(fanout - 1)) return fanout;
 77 |         uint64_t i = 0;
 78 | #ifdef DISABLE_AVX
 79 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 80 |             if (summary[z] > x) break;
 81 |         }
 82 |         assert(i < num_segments);
 83 |         x -= summary[i];
 84 |         i *= segment_size;
 85 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 86 |             if (keys[i] > x) break;
 87 |         }
 88 | #else
 89 |         __m256i cmp1 = _mm256_cmpgt_epi32(
 90 |             _mm256_loadu_si256((__m256i const*)summary), _mm256_set1_epi32(x));
 91 |         i = index_fs<32>(cmp1) - 1;
 92 |         assert(i < num_segments);
 93 |         x -= summary[i];
 94 |         i *= segment_size;
 95 |         __m256i cmp2 =
 96 |             _mm256_cmpgt_epi32(_mm256_loadu_si256((__m256i const*)(keys + i)),
 97 |                                _mm256_set1_epi32(x));
 98 |         i += index_fs<32>(cmp2);
 99 | #endif
100 |         return {i, i ? sum(i - 1) : 0};
101 |     }
102 | 
103 | private:
104 |     summary_type* summary;
105 |     key_type* keys;
106 | };
107 | 
108 | }  // namespace mrs::avx2


--------------------------------------------------------------------------------
/include/segment_tree/avx512/node256.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "util.hpp"
  8 | 
  9 | namespace mrs::avx512 {
 10 | 
 11 | struct node256 {
 12 |     typedef uint32_t key_type;
 13 |     typedef uint32_t summary_type;
 14 |     static constexpr uint64_t fanout = 256;
 15 |     static constexpr uint64_t segment_size = 16;
 16 |     static constexpr uint64_t num_segments = 16;
 17 |     static constexpr uint64_t bytes =
 18 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 19 | 
 20 |     node256() {}
 21 | 
 22 |     static void build(key_type const* input, uint8_t* out) {
 23 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 24 |                                                        num_segments, bytes);
 25 |     }
 26 | 
 27 |     static std::string name() {
 28 |         return "avx512::node256";
 29 |     }
 30 | 
 31 |     node256(uint8_t* ptr) {
 32 |         at(ptr);
 33 |     }
 34 | 
 35 |     inline void at(uint8_t* ptr) {
 36 |         summary = reinterpret_cast<summary_type*>(ptr);
 37 |         keys = reinterpret_cast<key_type*>(ptr +
 38 |                                            num_segments * sizeof(summary_type));
 39 |     }
 40 | 
 41 |     void update(uint64_t i, bool sign) {
 42 |         if (i == fanout) return;
 43 |         assert(i < fanout);
 44 |         uint64_t j = i / segment_size;
 45 |         uint64_t k = i % segment_size;
 46 | #ifdef AVX512
 47 |         __m512i s1 = _mm512_load_si512((__m512i const*)tables::update_16_32 +
 48 |                                        j + 1 + sign * (num_segments + 1));
 49 |         __m512i r1 =
 50 |             _mm512_add_epi32(_mm512_loadu_si512((__m512i const*)summary), s1);
 51 |         _mm512_storeu_si512((__m512i*)summary, r1);
 52 |         __m512i s2 = _mm512_load_si512((__m512i const*)tables::update_16_32 +
 53 |                                        k + sign * (segment_size + 1));
 54 |         __m512i r2 = _mm512_add_epi32(
 55 |             _mm512_loadu_si512((__m512i const*)(keys + j * segment_size)), s2);
 56 |         _mm512_storeu_si512((__m512i*)(keys + j * segment_size), r2);
 57 | #else
 58 |         int8_t delta = sign ? -1 : +1;
 59 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 60 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 61 |             keys[base + z] += delta;
 62 |         }
 63 | #endif
 64 |     }
 65 | 
 66 |     uint64_t sum(uint64_t i) const {
 67 |         assert(i < fanout);
 68 |         return summary[i / segment_size] + keys[i];
 69 |     }
 70 | 
 71 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 72 |        if x >= sum(fanout-1), then fanout is returned */
 73 |     search_result search(uint64_t x) const {
 74 |         if (x >= sum(fanout - 1)) return {fanout, sum(fanout - 1)};
 75 |         uint64_t i = 0;
 76 | #ifdef AVX512
 77 |         __mmask16 cmp1 = _mm512_cmpgt_epi32_mask(
 78 |             _mm512_loadu_si512((__m512i const*)summary), _mm512_set1_epi32(x));
 79 |         i = cmp1 != 0 ? __builtin_ctz(cmp1) - 1 : num_segments - 1;
 80 |         assert(i < num_segments);
 81 |         x -= summary[i];
 82 |         i *= segment_size;
 83 |         __mmask16 cmp2 = _mm512_cmpgt_epi32_mask(
 84 |             _mm512_loadu_si512((__m512i const*)(keys + i)),
 85 |             _mm512_set1_epi32(x));
 86 |         assert(cmp2 > 0);
 87 |         i += __builtin_ctzll(cmp2);
 88 | #else
 89 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 90 |             if (summary[z] > x) break;
 91 |         }
 92 |         assert(i < num_segments);
 93 |         x -= summary[i];
 94 |         i *= segment_size;
 95 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 96 |             if (keys[i] > x) break;
 97 |         }
 98 | #endif
 99 |         return {i, i ? sum(i - 1) : 0};
100 |     }
101 | 
102 | private:
103 |     summary_type* summary;
104 |     key_type* keys;
105 | };
106 | 
107 | }  // namespace mrs::avx512


--------------------------------------------------------------------------------
/include/segment_tree/avx512/node512.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | 
  5 | #include "immintrin.h"
  6 | #include "tables.hpp"
  7 | #include "util.hpp"
  8 | 
  9 | namespace mrs::avx512 {
 10 | 
 11 | struct node512 {
 12 |     typedef uint16_t key_type;
 13 |     typedef uint32_t summary_type;
 14 |     static constexpr uint64_t fanout = 512;
 15 |     static constexpr uint64_t segment_size = 32;
 16 |     static constexpr uint64_t num_segments = 16;
 17 |     static constexpr uint64_t bytes =
 18 |         fanout * sizeof(key_type) + num_segments * sizeof(summary_type);
 19 | 
 20 |     node512() {}
 21 | 
 22 |     static void build(key_type const* input, uint8_t* out) {
 23 |         build_node_prefix_sums<summary_type, key_type>(input, out, segment_size,
 24 |                                                        num_segments, bytes);
 25 |     }
 26 | 
 27 |     static std::string name() {
 28 |         return "avx512::node512";
 29 |     }
 30 | 
 31 |     node512(uint8_t* ptr) {
 32 |         at(ptr);
 33 |     }
 34 | 
 35 |     inline void at(uint8_t* ptr) {
 36 |         summary = reinterpret_cast<summary_type*>(ptr);
 37 |         keys = reinterpret_cast<key_type*>(ptr +
 38 |                                            num_segments * sizeof(summary_type));
 39 |     }
 40 | 
 41 |     void update(uint64_t i, bool sign) {
 42 |         if (i == fanout) return;
 43 |         assert(i < fanout);
 44 |         uint64_t j = i / segment_size;
 45 |         uint64_t k = i % segment_size;
 46 | #ifdef AVX512
 47 |         __m512i s1 = _mm512_load_si512((__m512i const*)tables::update_16_32 +
 48 |                                        j + 1 + sign * (num_segments + 1));
 49 |         __m512i r1 =
 50 |             _mm512_add_epi32(_mm512_loadu_si512((__m512i const*)summary), s1);
 51 |         _mm512_storeu_si512((__m512i*)summary, r1);
 52 |         __m512i s2 = _mm512_load_si512((__m512i const*)tables::update_32_16 +
 53 |                                        k + sign * segment_size);
 54 |         __m512i r2 = _mm512_add_epi16(
 55 |             _mm512_loadu_si512((__m512i const*)(keys + j * segment_size)), s2);
 56 |         _mm512_storeu_si512((__m512i*)(keys + j * segment_size), r2);
 57 | #else
 58 |         int8_t delta = sign ? -1 : +1;
 59 |         for (uint64_t z = j + 1; z != num_segments; ++z) summary[z] += delta;
 60 |         for (uint64_t z = k, base = j * segment_size; z != segment_size; ++z) {
 61 |             keys[base + z] += delta;
 62 |         }
 63 | #endif
 64 |     }
 65 | 
 66 |     uint64_t sum(uint64_t i) const {
 67 |         assert(i < fanout);
 68 |         return summary[i / segment_size] + keys[i];
 69 |     }
 70 | 
 71 |     /* return the smallest i in [0,fanout-1] such that sum(i) > x;
 72 |        in this case x is always < sum(fanout-1) by assumption */
 73 |     search_result search(uint64_t x) const {
 74 |         assert(x < sum(fanout - 1));
 75 |         uint64_t i = 0;
 76 | #ifdef AVX512
 77 |         __mmask16 cmp1 = _mm512_cmpgt_epi32_mask(
 78 |             _mm512_loadu_si512((__m512i const*)summary), _mm512_set1_epi32(x));
 79 |         i = cmp1 != 0 ? __builtin_ctz(cmp1) - 1 : num_segments - 1;
 80 |         assert(i < num_segments);
 81 |         x -= summary[i];
 82 |         i *= segment_size;
 83 |         __mmask32 cmp2 = _mm512_cmpgt_epi16_mask(
 84 |             _mm512_loadu_si512((__m512i const*)(keys + i)),
 85 |             _mm512_set1_epi16(x));
 86 |         assert(cmp2 > 0);
 87 |         i += __builtin_ctz(cmp2);
 88 | #else
 89 |         for (uint64_t z = 1; z != num_segments; ++z, ++i) {
 90 |             if (summary[z] > x) break;
 91 |         }
 92 |         assert(i < num_segments);
 93 |         x -= summary[i];
 94 |         i *= segment_size;
 95 |         for (uint64_t z = 0; z != segment_size; ++z, ++i) {
 96 |             if (keys[i] > x) break;
 97 |         }
 98 | #endif
 99 |         assert(i < fanout);
100 |         return {i, i ? sum(i - 1) : 0};
101 |     }
102 | 
103 | private:
104 |     summary_type* summary;
105 |     key_type* keys;
106 | };
107 | 
108 | }  // namespace mrs::avx512


--------------------------------------------------------------------------------
/include/hft/fenwick/hybrid.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_HYBRID_HPP__
  2 | #define __FENWICK_HYBRID_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | #include <vector>
  6 | 
  7 | namespace hft::fenwick {
  8 | 
  9 | template <template <size_t> class TOP, template <size_t> class BOTTOM, size_t BOUND, size_t CUT>
 10 | class Hybrid : public FenwickTree {
 11 | private:
 12 |   size_t BottomSize = (1ULL << CUT) - 1;
 13 |   static constexpr size_t TOPBOUND = BOUND * (1ULL << CUT);
 14 | 
 15 | protected:
 16 |   size_t Size;
 17 |   TOP<TOPBOUND> TopFenwick;
 18 |   std::vector<BOTTOM<BOUND>> BottomForest;
 19 | 
 20 | public:
 21 |   Hybrid(uint64_t sequence[], size_t size) : Size(size), TopFenwick(buildTop(sequence, size)) {
 22 |     for (size_t i = 0; i <= TopFenwick.size(); i++) {
 23 |       const size_t next = (BottomSize + 1) * i;
 24 |       const size_t bsize = BottomSize + next <= size ? BottomSize : size & BottomSize;
 25 | 
 26 |       BottomForest.push_back(BOTTOM<BOUND>(sequence + next, bsize));
 27 |     }
 28 | 
 29 |     for (size_t i = 0; i < TopFenwick.size(); i++)
 30 |       TopFenwick.add(i + 1, BottomForest[i].prefix(BottomForest[i].size()));
 31 |   }
 32 | 
 33 |   virtual uint64_t prefix(size_t idx) const {
 34 |     const size_t top = idx >> CUT;
 35 |     const size_t bottom = idx & BottomSize;
 36 | 
 37 |     return TopFenwick.prefix(top) + BottomForest[top].prefix(bottom);
 38 |   }
 39 | 
 40 |   virtual void add(size_t idx, int64_t inc) {
 41 |     const size_t top = idx >> CUT;
 42 |     const size_t bottom = idx & BottomSize;
 43 | 
 44 |     if (bottom == 0) {
 45 |       TopFenwick.add(top, inc);
 46 |     } else {
 47 |       TopFenwick.add(top + 1, inc);
 48 |       BottomForest[top].add(bottom, inc);
 49 |     }
 50 |   }
 51 | 
 52 |   using FenwickTree::find;
 53 |   virtual size_t find(uint64_t *val) const {
 54 |     const size_t top = TopFenwick.size() != 0 ? TopFenwick.find(val) : 0;
 55 |     const size_t bottom = top < BottomForest.size() ? BottomForest[top].find(val) : 0;
 56 | 
 57 |     return (top << CUT) + bottom;
 58 |   }
 59 | 
 60 |   using FenwickTree::compFind;
 61 |   virtual size_t compFind(uint64_t *val) const {
 62 |     const size_t top = TopFenwick.size() != 0 ? TopFenwick.compFind(val) : 0;
 63 |     const size_t bottom = top < BottomForest.size() ? BottomForest[top].compFind(val) : 0;
 64 | 
 65 |     return (top << CUT) + bottom;
 66 |   }
 67 | 
 68 |   virtual size_t size() const { return Size; }
 69 | 
 70 |   virtual size_t bitCount() const {
 71 |     size_t bottomSize = 0;
 72 |     for (auto &t : BottomForest)
 73 |       bottomSize += t.bitCount();
 74 | 
 75 |     return bottomSize + sizeof(Hybrid<BOTTOM, TOP, BOUND, CUT>) + TopFenwick.bitCount();
 76 |   }
 77 | 
 78 | private:
 79 |   TOP<TOPBOUND> buildTop(const uint64_t sequence[], size_t size) const {
 80 |     const size_t topsize = size >> CUT;
 81 |     const std::unique_ptr<uint64_t[]> subseq = std::make_unique<uint64_t[]>(topsize);
 82 | 
 83 |     for (size_t i = 1; i <= topsize; i++)
 84 |       subseq[i - 1] = sequence[i * (BottomSize + 1) - 1];
 85 | 
 86 |     return TOP<TOPBOUND>(subseq.get(), topsize);
 87 |   }
 88 | 
 89 |   friend std::ostream &operator<<(std::ostream &os, const Hybrid<TOP, BOTTOM, BOUND, CUT> &ft) {
 90 |     const uint64_t nsize = hton(ft.Size);
 91 |     os.write((char *)&nsize, sizeof(uint64_t));
 92 | 
 93 |     const uint64_t nbottom = hton(ft.BottomForest.size());
 94 |     os.write((char *)&nbottom, sizeof(uint64_t));
 95 | 
 96 |     os << ft.TopFenwick;
 97 |     for (size_t i = 0; i < nbottom; ++i)
 98 |       os << ft.BottomForest[i];
 99 | 
100 |     return os;
101 |   }
102 | 
103 |   friend std::istream &operator>>(std::istream &is, Hybrid<TOP, BOTTOM, BOUND, CUT> &ft) {
104 |     uint64_t nsize;
105 |     is.read((char *)(&nsize), sizeof(uint64_t));
106 | 
107 |     uint64_t nbottom;
108 |     is.read((char *)(&nbottom), sizeof(uint64_t));
109 | 
110 |     is >> ft.TopFenwick;
111 | 
112 |     auto readNextBottom = [&]() -> BOTTOM<BOUND> {
113 |       BOTTOM<BOUND> bottom(nullptr, 0);
114 |       is >> bottom;
115 |       return bottom;
116 |     };
117 | 
118 |     ft.BottomForest.reserve(nbottom);
119 |     for (size_t i = 0; i < nbottom; ++i)
120 |       ft.BottomForest.push_back(readNextBottom());
121 | 
122 |     return is;
123 |   }
124 | };
125 | 
126 | } // namespace hft::fenwick
127 | 
128 | #endif // __FENWICK_HYBRID_HPP__
129 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Rank/Select Queries over Mutable Bitmaps
  2 | ========================================
  3 | 
  4 | A C++ library providing rank/select queries over mutable bitmaps.
  5 | 
  6 | Given a *mutable* bitmap B[0..u) where n bits are set, the *rank/select problem* asks for a data structure built from B that supports:
  7 | 
  8 | - rank(i) is the number of bits set in B[0..i], for 0 ≤ i < u.
  9 | - select(i) is the position of the i-th bit set, for 0 ≤ i < n.
 10 | - flip(i) toggles B[i], for 0 ≤ i < u.
 11 | - access(i) = B[i], for 0 ≤ i < u.
 12 | 
 13 | The input bitmap is partitioned into blocks and a tree index is built over them. 
 14 | The tree index implemented in the library is an optimized b-ary Segment-Tree with
 15 | SIMD AVX2/AVX-512 instructions.
 16 | You can test a block size of 256 or 512 bits, and various rank/select algorithms for the blocks such as broadword techniques, CPU intrinsics, and SIMD instructions.
 17 | 
 18 | 
 19 | For a description and anlysis of all these data structures, see the paper [Rank/Select Queries over Mutable Bitmaps](https://www.sciencedirect.com/science/article/pii/S0306437921000235), by Giulio Ermanno Pibiri and Shunsuke Kanda, Information Systems (INFOSYS), 2021.
 20 | 
 21 | Please cite this paper if you use the library.
 22 | 
 23 | Compiling the code <a name="compiling"></a>
 24 | ------------------
 25 | 
 26 | The code is tested on Linux with `gcc` 7.4 and 9.2.1; on Mac 10.14 with `clang` 10.0.0 and 11.0.0.
 27 | To build the code, [`CMake`](https://cmake.org/) is required.
 28 | 
 29 | Clone the repository with
 30 | 
 31 | 	git clone --recursive https://github.com/jermp/mutable_rank_select.git
 32 | 
 33 | If you have cloned the repository without `--recursive`, you will need to perform the following commands before
 34 | compiling:
 35 | 
 36 |     git submodule init
 37 |     git submodule update
 38 | 
 39 | To compile the code for a release environment (see file `CMakeLists.txt` for the used compilation flags), it is sufficient to do the following:
 40 | 
 41 |     mkdir build
 42 |     cd build
 43 |     cmake ..
 44 |     make -j
 45 | 
 46 | By default, SIMD AVX instructions are enabled (flag `-DDISABLE_AVX=Off`). If you want to
 47 | disable them (although your compiler has proper support), you can compile with
 48 | 
 49 | 	cmake .. -DDISABLE_AVX=On
 50 | 	make -j
 51 | 
 52 | The library also exploits the new AVX512 instruction set. If you have proper support,
 53 | you can enable those instructions with
 54 | 
 55 | 	cmake .. -DAVX512=On
 56 | 	make -j
 57 | 
 58 | For the best of performance, we recommend compiling with:
 59 | 
 60 | 	cmake .. -DCMAKE_BUILD_TYPE=Release -DUSE_SANITIZERS=Off -DDISABLE_AVX=Off -DAVX512=On
 61 | 	make -j
 62 | 
 63 | For a testing environment, use the following instead:
 64 | 
 65 |     mkdir debug_build
 66 |     cd debug_build
 67 |     cmake .. -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZERS=On
 68 |     make -j
 69 | 
 70 | Benchmarks
 71 | ---------
 72 | 
 73 | To benchmark the running time of rank, select, and flip for the disired data structure, use the program `src/perf_mutable_bitmap`. Running the program without arguments will show what arguments are required. (See also the file `src/perf_mutable_bitmap.cpp` for a list of available data structure types.)
 74 | 
 75 | Below we show some examples.
 76 | 
 77 | - The command
 78 | 
 79 | ```
 80 | ./perf_mutable_bitmap avx2_256_a rank 0.3
 81 | ```
 82 | 
 83 | will benchmark the speed of rank queries for the b-ary Segment-Tree data structure with AVX2 instructions for the bitmap of density 30%, where the block size is 256. The bitmap is tested by varying the size from 2<sup>9</sup> to 2<sup>32</sup>. The suffix `_a` indicates the type of rank algorithms for a small bitmap (See `include/types.hpp` for the details).
 84 | 
 85 | - The command
 86 | 
 87 | ```
 88 | ./perf_mutable_bitmap avx512_512_b select 0.5
 89 | ```
 90 | 
 91 | will benchmark the speed of select queries for the b-ary Segment-Tree data structure with AVX-512 instructions for the bitmap of density 50%, where the block size is 512.
 92 | 
 93 | - The command
 94 | 
 95 | ```
 96 | ./perf_mutable_bitmap avx512_256_c flip 0.8
 97 | ```
 98 | 
 99 | will benchmark the speed of flip queries for the b-ary Segment-Tree data structure with AVX-512 instructions for the bitmap of density 80%, where the block size is 256.
100 | 
101 | Unit tests <a name="testing"></a>
102 | -----------
103 | 
104 | The unit tests are written using [doctest](https://github.com/onqtam/doctest).
105 | 
106 | After compilation, it is advised
107 | to run the unit tests with:
108 | 
109 | 	make test
110 | 


--------------------------------------------------------------------------------
/include/hft/fenwick/bytef.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_BYTEF_HPP__
  2 | #define __FENWICK_BYTEF_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class ByteF - byte compression and classical node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class ByteF : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   size_t Size;
 22 |   DArray<uint8_t> Tree;
 23 | 
 24 | public:
 25 |   ByteF(uint64_t sequence[], size_t size) : Size(size), Tree(pos(size + 1) + 8) {
 26 |     for (size_t i = 1; i <= size; i++)
 27 |       bytewrite(&Tree[pos(i)], bytesize(i), sequence[i - 1]);
 28 | 
 29 |     for (size_t m = 2; m <= size; m <<= 1) {
 30 |       for (size_t idx = m; idx <= size; idx += m) {
 31 |         const uint64_t left = byteread(&Tree[pos(idx)], bytesize(idx));
 32 |         const uint64_t right = byteread(&Tree[pos(idx - m / 2)], bytesize(idx - m / 2));
 33 |         bytewrite(&Tree[pos(idx)], bytesize(idx), left + right);
 34 |       }
 35 |     }
 36 |   }
 37 | 
 38 |   virtual uint64_t prefix(size_t idx) const {
 39 |     uint64_t sum = 0;
 40 | 
 41 |     while (idx != 0) {
 42 |       sum += byteread(&Tree[pos(idx)], bytesize(idx));
 43 |       idx = clear_rho(idx);
 44 |     }
 45 | 
 46 |     return sum;
 47 |   }
 48 | 
 49 |   virtual void add(size_t idx, int64_t inc) {
 50 |     while (idx <= Size) {
 51 |       bytewrite_inc(&Tree[pos(idx)], inc);
 52 |       idx += mask_rho(idx);
 53 |     }
 54 |   }
 55 | 
 56 |   using FenwickTree::find;
 57 |   virtual size_t find(uint64_t *val) const {
 58 |     size_t node = 0;
 59 | 
 60 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 61 |       if (node + m > Size)
 62 |         continue;
 63 | 
 64 |       const uint64_t value = byteread(&Tree[pos(node + m)], bytesize(node + m));
 65 | 
 66 |       if (*val >= value) {
 67 |         node += m;
 68 |         *val -= value;
 69 |       }
 70 |     }
 71 | 
 72 |     return node;
 73 |   }
 74 | 
 75 |   using FenwickTree::compFind;
 76 |   virtual size_t compFind(uint64_t *val) const {
 77 |     size_t node = 0;
 78 | 
 79 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 80 |       if (node + m > Size)
 81 |         continue;
 82 | 
 83 |       const uint64_t value =
 84 |           (BOUND << rho(node + m)) - byteread(&Tree[pos(node + m)], bytesize(node + m));
 85 | 
 86 |       if (*val >= value) {
 87 |         node += m;
 88 |         *val -= value;
 89 |       }
 90 |     }
 91 | 
 92 |     return node;
 93 |   }
 94 | 
 95 |   virtual size_t size() const { return Size; }
 96 | 
 97 |   virtual size_t bitCount() const {
 98 |     return sizeof(ByteF<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree);
 99 |   }
100 | 
101 | private:
102 |   static inline size_t bytesize(size_t idx) { return ((rho(idx) + BOUNDSIZE - 1) >> 3) + 1; }
103 | 
104 |   static inline size_t holes(size_t idx) {
105 |     // Exhaustive benchmarking shows it is better to use no holes on (relatively) small trees,
106 |     // but we expect holes to be handy again in (very) big trees
107 |     if (BOUNDSIZE >= 32)
108 |       return 0;
109 | 
110 | #ifdef HFT_DISABLE_TRANSHUGE
111 |     return (idx >> (18 + (64 - BOUNDSIZE) % 8));
112 | #else
113 |     return (idx >> (28 + (64 - BOUNDSIZE) % 8));
114 | #endif
115 |   }
116 | 
117 |   static inline size_t pos(size_t idx) {
118 |     idx--;
119 |     constexpr size_t NEXTBYTE = ((BOUNDSIZE - 1) | (8 - 1)) + 1;
120 | 
121 |     constexpr size_t SMALL = ((BOUNDSIZE - 1) >> 3) + 1;
122 |     constexpr size_t MEDIUM = NEXTBYTE - BOUNDSIZE + 1;
123 |     constexpr size_t LARGE = MEDIUM + 8;
124 | 
125 |     constexpr size_t MULTIPLIER = 8 - SMALL - 1;
126 | 
127 |     return idx * SMALL + (idx >> MEDIUM) + (idx >> LARGE) * MULTIPLIER + holes(idx);
128 |   }
129 | 
130 |   friend std::ostream &operator<<(std::ostream &os, const ByteF<BOUND> &ft) {
131 |     uint64_t nsize = hton((uint64_t)ft.Size);
132 |     os.write((char *)&nsize, sizeof(uint64_t));
133 | 
134 |     return os << ft.Tree;
135 |   }
136 | 
137 |   friend std::istream &operator>>(std::istream &is, ByteF<BOUND> &ft) {
138 |     uint64_t nsize;
139 |     is.read((char *)(&nsize), sizeof(uint64_t));
140 |     ft.Size = ntoh(nsize);
141 | 
142 |     return is >> ft.Tree;
143 |   }
144 | };
145 | 
146 | } // namespace hft::fenwick
147 | 
148 | #endif // __FENWICK_BYTEF_HPP__
149 | 


--------------------------------------------------------------------------------
/include/hft/darray.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __DARRAY_HPP__
  2 | #define __DARRAY_HPP__
  3 | 
  4 | #include "common.hpp"
  5 | #include <assert.h>
  6 | #include <iostream>
  7 | #include <sys/mman.h>
  8 | 
  9 | namespace hft {
 10 | 
 11 | /**
 12 |  * class DArray - Dinamically-allocated fixed-sized array with hugepages support
 13 |  *
 14 |  * This class is a wrapper of mmap. It supports hugepages [1], transparent
 15 |  * hugepags [2]. It guarantees to return page-aligned pointers. It has four
 16 |  * behaviors you can choose by defining their name:
 17 |  * - HFT_FORCEHUGE: every request allocs hugepages;
 18 |  * - HFT_FORCENOHUGE: every request allocs standard 4k pages (non-transparent)
 19 |  * - HFT_HUGE: every request of at least 2MB allocs hugepages while smaller use
 20 |  * 4k pages;
 21 |  *
 22 |  * By default (without defining any of the above) every request of at least 2MB
 23 |  * allocs 4k pages, but they are transparently defragmented into hugepages by
 24 |  * khugepaged thanks to a call to madvice.
 25 |  *
 26 |  * [1] https://www.kernel.org/doc/html/latest/admin-guide/mm/hugetlbpage.html
 27 |  * [2] https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
 28 |  *
 29 |  */
 30 | template <typename T>
 31 | class DArray {
 32 | public:
 33 |     static constexpr int PROT = PROT_READ | PROT_WRITE;
 34 |     static constexpr int FLAGS = MAP_PRIVATE | MAP_ANONYMOUS;
 35 | 
 36 | private:
 37 |     size_t Size = 0;
 38 |     T* Buffer = nullptr;
 39 | 
 40 | public:
 41 |     DArray<T>() = default;
 42 | 
 43 |     explicit DArray<T>(size_t length) : Size(length) {
 44 |         const size_t space = page_aligned(Size);
 45 |         if (space) {
 46 |             // #ifdef HFT_FORCE_HUGETLBPAGE
 47 |             //       void *mem = mmap(nullptr, space, PROT, FLAGS | MAP_HUGETLB,
 48 |             //       -1, 0); assert(mem != MAP_FAILED && "mmap failed");
 49 |             // #elif HFT_DISABLE_TRANSHUGE
 50 |             //       void *mem = mmap(nullptr, space, PROT, FLAGS, -1, 0);
 51 |             //       assert(mem != MAP_FAILED && "mmap failed");
 52 |             // #else
 53 |             void* mem = mmap(nullptr, space, PROT, FLAGS, -1, 0);
 54 |             assert(mem != MAP_FAILED && "mmap failed");
 55 | 
 56 |             //       int adv = madvise(mem, space, MADV_HUGEPAGE);
 57 |             //       assert(adv == 0 && "madvise failed");
 58 |             // #endif
 59 | 
 60 |             Buffer = static_cast<T*>(mem);
 61 |         }
 62 |     }
 63 | 
 64 |     DArray(DArray<T>&& oth)
 65 |         : Size(std::exchange(oth.Size, 0))
 66 |         , Buffer(std::exchange(oth.Buffer, nullptr)) {}
 67 | 
 68 |     DArray<T>& operator=(DArray<T>&& oth) {
 69 |         swap(*this, oth);
 70 |         return *this;
 71 |     }
 72 | 
 73 |     ~DArray<T>() {
 74 |         if (Buffer != nullptr) {
 75 |             int result = munmap(Buffer, page_aligned(Size));
 76 |             assert(result == 0 && "mmunmap failed");
 77 |         }
 78 |     }
 79 | 
 80 |     friend void swap(DArray<T>& first, DArray<T>& second) noexcept {
 81 |         std::swap(first.Size, second.Size);
 82 |         std::swap(first.Buffer, second.Buffer);
 83 |     }
 84 | 
 85 |     inline T* get() const {
 86 |         return Buffer;
 87 |     }
 88 | 
 89 |     inline T& operator[](size_t i) const {
 90 |         return Buffer[i];
 91 |     };
 92 | 
 93 |     inline size_t size() const {
 94 |         return Size;
 95 |     }
 96 | 
 97 |     size_t bitCount() const {
 98 |         return sizeof(DArray<T>) * 8 + page_aligned(Size) * 8;
 99 |     }
100 | 
101 | private:
102 |     static size_t page_aligned(size_t size) {
103 | #ifdef HFT_FORCE_HUGETLBPAGE
104 |         return ((2 * 1024 * 1024 - 1) | (size * sizeof(T) - 1)) + 1;
105 | #else
106 |         return ((4 * 1024 - 1) | (size * sizeof(T) - 1)) + 1;
107 | #endif
108 |     }
109 | 
110 |     friend std::ostream& operator<<(std::ostream& os, const DArray<T>& darray) {
111 |         const uint64_t nsize = hton(static_cast<uint64_t>(darray.Size));
112 |         os.write((char*)&nsize, sizeof(uint64_t));
113 | 
114 |         for (size_t i = 0; i < darray.Size; ++i) {
115 |             const T value = hton(darray[i]);
116 |             os.write((char*)&value, sizeof(T));
117 |         }
118 | 
119 |         return os;
120 |     }
121 | 
122 |     friend std::istream& operator>>(std::istream& is, DArray<T>& darray) {
123 |         uint64_t nsize;
124 |         is.read((char*)&nsize, sizeof(uint64_t));
125 | 
126 |         darray = DArray<T>(ntoh(nsize));
127 | 
128 |         for (size_t i = 0; i < darray.size(); ++i) {
129 |             is.read((char*)&darray[i], sizeof(T));
130 |             darray[i] = ntoh(darray[i]);
131 |         }
132 | 
133 |         return is;
134 |     }
135 | };  // namespace hft
136 | 
137 | }  // namespace hft
138 | 
139 | #endif  // __DARRAY_HPP__
140 | 


--------------------------------------------------------------------------------
/src/perf_rank64.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | 
 19 | static constexpr std::array<uint64_t, 25> sizes = {
 20 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 21 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 22 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 23 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 24 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 25 | };
 26 | 
 27 | void test(const double density) {
 28 |     std::vector<uint64_t> bits(sizes.back() / 64);
 29 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 30 |     std::cout << "# number of ones: " << num_ones << " ("
 31 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 32 | 
 33 |     std::string json("{\"type\":\"builtin\", ");
 34 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 35 |     json += "\"timings\":[";
 36 | 
 37 |     for (const uint64_t n : sizes) {
 38 |         // !! Create queries in the same manner as rank256 !!
 39 |         splitmix64 hasher(query_seed);
 40 | 
 41 |         const auto num_buckets = n / 256;
 42 |         std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 43 | 
 44 |         for (uint64_t i = 0; i < num_queries; i++) {
 45 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 256 / 64];
 46 |             queries[i] = {x, hasher.next() % 64};
 47 |         }
 48 | 
 49 |         essentials::timer_type t;
 50 |         double min = 0.0, max = 0.0, avg = 0.0;
 51 | 
 52 |         auto measure = [&]() {
 53 |             uint64_t tmp = 0;  // to avoid the optimization
 54 |             for (int run = 0; run != runs; run++) {
 55 |                 t.start();
 56 |                 for (uint64_t i = 0; i < num_queries; i++) {
 57 |                     const uint64_t x = *queries[i].first;
 58 |                     const uint64_t k = queries[i].second;
 59 |                     const uint64_t mask = (k != 0) * (1ULL << k) - 1;
 60 |                     tmp += popcount_u64<popcount_modes::builtin>(x & mask);
 61 |                 }
 62 |                 t.stop();
 63 |             }
 64 |             std::cout << "# ignore: " << tmp << std::endl;
 65 |         };
 66 | 
 67 |         static constexpr int K = 10;
 68 | 
 69 |         // warm-up
 70 |         for (int k = 0; k != K; ++k) {
 71 |             measure();
 72 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 73 |             avg += avg_ns_query;
 74 |             t.reset();
 75 |         }
 76 |         std::cout << "# warm-up: " << avg / K << std::endl;
 77 |         avg = 0.0;
 78 | 
 79 |         for (int k = 0; k != K; ++k) {
 80 |             measure();
 81 |             t.discard_max();
 82 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 83 |             max += avg_ns_query;
 84 |             t.reset();
 85 |         }
 86 | 
 87 |         for (int k = 0; k != K; ++k) {
 88 |             measure();
 89 |             t.discard_min();
 90 |             t.discard_max();
 91 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 92 |             avg += avg_ns_query;
 93 |             t.reset();
 94 |         }
 95 | 
 96 |         for (int k = 0; k != K; ++k) {
 97 |             measure();
 98 |             t.discard_min();
 99 |             double avg_ns_query = (t.min() * 1000) / num_queries;
100 |             min += avg_ns_query;
101 |             t.reset();
102 |         }
103 | 
104 |         min /= K;
105 |         max /= K;
106 |         avg /= K;
107 |         std::vector<double> tt{min, avg, max};
108 |         std::sort(tt.begin(), tt.end());
109 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
110 | 
111 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
112 |                 "," + std::to_string(tt[2]) + "],";
113 |     }
114 | 
115 |     json.pop_back();
116 |     json += "]}";
117 |     std::cerr << json << std::endl;
118 | }
119 | 
120 | int main(int argc, char** argv) {
121 |     cmd_line_parser::parser parser(argc, argv);
122 |     parser.add("density", "Density of ones (in [0,1]).");
123 |     if (!parser.parse()) return 1;
124 | 
125 |     test(parser.get<double>("density"));
126 | 
127 |     return 0;
128 | }
129 | 


--------------------------------------------------------------------------------
/.clang-format:
--------------------------------------------------------------------------------
  1 | ---
  2 | Language:        Cpp
  3 | # BasedOnStyle:  Google
  4 | AccessModifierOffset: -4
  5 | AlignAfterOpenBracket: Align
  6 | AlignConsecutiveAssignments: false
  7 | AlignConsecutiveDeclarations: false
  8 | AlignEscapedNewlines: Left
  9 | AlignOperands:   true
 10 | AlignTrailingComments: true
 11 | AllowAllParametersOfDeclarationOnNextLine: true
 12 | AllowShortBlocksOnASingleLine: true
 13 | AllowShortCaseLabelsOnASingleLine: false
 14 | AllowShortFunctionsOnASingleLine: Empty
 15 | AllowShortIfStatementsOnASingleLine: true
 16 | AllowShortLoopsOnASingleLine: true
 17 | AlwaysBreakAfterDefinitionReturnType: None
 18 | AlwaysBreakAfterReturnType: None
 19 | AlwaysBreakBeforeMultilineStrings: true
 20 | AlwaysBreakTemplateDeclarations: Yes
 21 | BinPackArguments: true
 22 | BinPackParameters: true
 23 | BraceWrapping:
 24 |   AfterClass:      false
 25 |   AfterControlStatement: false
 26 |   AfterEnum:       false
 27 |   AfterFunction:   false
 28 |   AfterNamespace:  false
 29 |   AfterObjCDeclaration: false
 30 |   AfterStruct:     false
 31 |   AfterUnion:      false
 32 |   AfterExternBlock: false
 33 |   BeforeCatch:     false
 34 |   BeforeElse:      false
 35 |   IndentBraces:    false
 36 |   SplitEmptyFunction: true
 37 |   SplitEmptyRecord: true
 38 |   SplitEmptyNamespace: true
 39 | BreakBeforeBinaryOperators: None
 40 | BreakBeforeBraces: Attach
 41 | BreakBeforeInheritanceComma: false
 42 | BreakInheritanceList: BeforeComma
 43 | BreakBeforeTernaryOperators: true
 44 | BreakConstructorInitializersBeforeComma: true
 45 | BreakConstructorInitializers: BeforeComma
 46 | BreakAfterJavaFieldAnnotations: false
 47 | BreakStringLiterals: true
 48 | ColumnLimit:     80
 49 | CommentPragmas:  '^ IWYU pragma:'
 50 | CompactNamespaces: false
 51 | ConstructorInitializerAllOnOneLineOrOnePerLine: true
 52 | ConstructorInitializerIndentWidth: 4
 53 | ContinuationIndentWidth: 4
 54 | Cpp11BracedListStyle: true
 55 | DerivePointerAlignment: false
 56 | DisableFormat:   false
 57 | ExperimentalAutoDetectBinPacking: false
 58 | FixNamespaceComments: true
 59 | ForEachMacros:
 60 |   - foreach
 61 |   - Q_FOREACH
 62 |   - BOOST_FOREACH
 63 | IncludeBlocks:   Preserve
 64 | IncludeCategories:
 65 |   - Regex:           '^<ext/.*\.h>'
 66 |     Priority:        2
 67 |   - Regex:           '^<.*\.h>'
 68 |     Priority:        1
 69 |   - Regex:           '^<.*'
 70 |     Priority:        2
 71 |   - Regex:           '.*'
 72 |     Priority:        3
 73 | IncludeIsMainRegex: '([-_](test|unittest))?$'
 74 | IndentCaseLabels: true
 75 | IndentPPDirectives: None
 76 | IndentWidth:     4
 77 | IndentWrappedFunctionNames: false
 78 | JavaScriptQuotes: Leave
 79 | JavaScriptWrapImports: true
 80 | KeepEmptyLinesAtTheStartOfBlocks: false
 81 | MacroBlockBegin: ''
 82 | MacroBlockEnd:   ''
 83 | MaxEmptyLinesToKeep: 1
 84 | NamespaceIndentation: None
 85 | ObjCBinPackProtocolList: Never
 86 | ObjCBlockIndentWidth: 2
 87 | ObjCSpaceAfterProperty: false
 88 | ObjCSpaceBeforeProtocolList: true
 89 | PenaltyBreakAssignment: 2
 90 | PenaltyBreakBeforeFirstCallParameter: 1
 91 | PenaltyBreakComment: 300
 92 | PenaltyBreakFirstLessLess: 120
 93 | PenaltyBreakString: 1000
 94 | PenaltyBreakTemplateDeclaration: 10
 95 | PenaltyExcessCharacter: 1000000
 96 | PenaltyReturnTypeOnItsOwnLine: 200
 97 | PointerAlignment: Left
 98 | RawStringFormats:
 99 |   - Language:        Cpp
100 |     Delimiters:
101 |       - cc
102 |       - CC
103 |       - cpp
104 |       - Cpp
105 |       - CPP
106 |       - 'c++'
107 |       - 'C++'
108 |     CanonicalDelimiter: ''
109 |     BasedOnStyle:    google
110 |   - Language:        TextProto
111 |     Delimiters:
112 |       - pb
113 |       - PB
114 |       - proto
115 |       - PROTO
116 |     EnclosingFunctions:
117 |       - EqualsProto
118 |       - EquivToProto
119 |       - PARSE_PARTIAL_TEXT_PROTO
120 |       - PARSE_TEST_PROTO
121 |       - PARSE_TEXT_PROTO
122 |       - ParseTextOrDie
123 |       - ParseTextProtoOrDie
124 |     CanonicalDelimiter: ''
125 |     BasedOnStyle:    google
126 | ReflowComments:  true
127 | SortIncludes:    false
128 | SortUsingDeclarations: false
129 | SpaceAfterCStyleCast: false
130 | SpaceAfterTemplateKeyword: true
131 | SpaceBeforeAssignmentOperators: true
132 | SpaceBeforeCpp11BracedList: false
133 | SpaceBeforeCtorInitializerColon: true
134 | SpaceBeforeInheritanceColon: true
135 | SpaceBeforeParens: ControlStatements
136 | SpaceBeforeRangeBasedForLoopColon: true
137 | SpaceInEmptyParentheses: false
138 | SpacesBeforeTrailingComments: 2
139 | SpacesInAngles:  false
140 | SpacesInContainerLiterals: true
141 | SpacesInCStyleCastParentheses: false
142 | SpacesInParentheses: false
143 | SpacesInSquareBrackets: false
144 | Standard:        Auto
145 | StatementMacros:
146 |   - Q_UNUSED
147 |   - QT_REQUIRE_VERSION
148 | TabWidth:        8
149 | UseTab:          Never
150 | ...
151 | 
152 | 


--------------------------------------------------------------------------------
/include/hft/fenwick/fixedl.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_FIXEDL_HPP__
  2 | #define __FENWICK_FIXEDL_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class FixedL - no compression and level-ordered node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class FixedL : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   size_t Size, Levels;
 22 |   unique_ptr<size_t[]> Level;
 23 |   DArray<uint64_t> Tree;
 24 | 
 25 | public:
 26 |   FixedL(uint64_t sequence[], size_t size)
 27 |       : Size(size), Levels(size != 0 ? lambda(size) + 2 : 1), Level(make_unique<size_t[]>(Levels)) {
 28 |     Level[0] = 0;
 29 |     for (size_t i = 1; i < Levels; i++)
 30 |       Level[i] = ((size + (1ULL << (i - 1))) / (1ULL << i)) + Level[i - 1];
 31 | 
 32 |     Tree = DArray<uint64_t>(Level[Levels - 1]);
 33 | 
 34 |     for (size_t l = 0; l < Levels - 1; l++) {
 35 |       for (size_t node = 1ULL << l; node <= size; node += 1ULL << (l + 1)) {
 36 |         size_t sequence_idx = node - 1;
 37 |         uint64_t value = sequence[sequence_idx];
 38 |         for (size_t j = 0; j < l; j++) {
 39 |           sequence_idx >>= 1;
 40 |           value += Tree[Level[j] + sequence_idx];
 41 |         }
 42 | 
 43 |         Tree[Level[l] + (node >> (l + 1))] = value;
 44 |       }
 45 |     }
 46 |   }
 47 | 
 48 |   virtual uint64_t prefix(size_t idx) const {
 49 |     uint64_t sum = 0;
 50 | 
 51 |     while (idx != 0) {
 52 |       const int height = rho(idx);
 53 |       size_t level_idx = idx >> (1 + height);
 54 |       sum += Tree[Level[height] + level_idx];
 55 | 
 56 |       idx = clear_rho(idx);
 57 |     }
 58 | 
 59 |     return sum;
 60 |   }
 61 | 
 62 |   virtual void add(size_t idx, int64_t inc) {
 63 |     while (idx <= Size) {
 64 |       const int height = rho(idx);
 65 |       size_t level_idx = idx >> (1 + height);
 66 |       Tree[Level[height] + level_idx] += inc;
 67 | 
 68 |       idx += mask_rho(idx);
 69 |     }
 70 |   }
 71 | 
 72 |   using FenwickTree::find;
 73 |   virtual size_t find(uint64_t *val) const {
 74 |     size_t node = 0, idx = 0;
 75 | 
 76 |     for (size_t height = Levels - 2; height != SIZE_MAX; height--) {
 77 |       const size_t pos = Level[height] + idx;
 78 | 
 79 |       idx <<= 1;
 80 | 
 81 |       if (pos >= Level[height + 1])
 82 |         continue;
 83 | 
 84 |       uint64_t value = Tree[pos];
 85 |       if (*val >= value) {
 86 |         idx++;
 87 |         *val -= value;
 88 |         node += 1ULL << height;
 89 |       }
 90 |     }
 91 | 
 92 |     return min(node, Size);
 93 |   }
 94 | 
 95 |   using FenwickTree::compFind;
 96 |   virtual size_t compFind(uint64_t *val) const {
 97 |     size_t node = 0, idx = 0;
 98 | 
 99 |     for (size_t height = Levels - 2; height != SIZE_MAX; height--) {
100 |       const size_t pos = Level[height] + idx;
101 | 
102 |       idx <<= 1;
103 | 
104 |       if (pos >= Level[height + 1])
105 |         continue;
106 | 
107 |       uint64_t value = (BOUND << height) - Tree[pos];
108 |       if (*val >= value) {
109 |         idx++;
110 |         *val -= value;
111 |         node += 1ULL << height;
112 |       }
113 |     }
114 | 
115 |     return min(node, Size);
116 |   }
117 | 
118 |   virtual size_t size() const { return Size; }
119 | 
120 |   virtual size_t bitCount() const {
121 |     return sizeof(FixedL<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree) +
122 |            Levels * sizeof(size_t) * 8;
123 |   }
124 | 
125 | private:
126 |   friend std::ostream &operator<<(std::ostream &os, const FixedL<BOUND> &ft) {
127 |     const uint64_t nsize = hton((uint64_t)ft.Size);
128 |     os.write((char *)&nsize, sizeof(uint64_t));
129 | 
130 |     const uint64_t nlevels = hton((uint64_t)ft.Levels);
131 |     os.write((char *)&nlevels, sizeof(uint64_t));
132 | 
133 |     for (size_t i = 0; i < ft.Levels; ++i) {
134 |       const uint64_t nlevel = hton((uint64_t)ft.Level[i]);
135 |       os.write((char *)&nlevel, sizeof(uint64_t));
136 |     }
137 | 
138 |     return os << ft.Tree;
139 |   }
140 | 
141 |   friend std::istream &operator>>(std::istream &is, FixedL<BOUND> &ft) {
142 |     uint64_t nsize;
143 |     is.read((char *)(&nsize), sizeof(uint64_t));
144 |     ft.Size = ntoh(nsize);
145 | 
146 |     uint64_t nlevels;
147 |     is.read((char *)&nlevels, sizeof(uint64_t));
148 |     ft.Levels = ntoh(nlevels);
149 | 
150 |     ft.Level = make_unique<size_t[]>(ft.Levels);
151 |     for (size_t i = 0; i < ft.Levels; ++i) {
152 |       uint64_t nlevel;
153 |       is.read((char *)&nlevel, sizeof(uint64_t));
154 |       ft.Level[i] = ntoh(nlevel);
155 |     }
156 | 
157 |     return is >> ft.Tree;
158 |   }
159 | };
160 | 
161 | } // namespace hft::fenwick
162 | 
163 | #endif // __FENWICK_FIXEDL_HPP__
164 | 


--------------------------------------------------------------------------------
/include/hft/fenwick/bitf.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_BITF_HPP__
  2 | #define __FENWICK_BITF_HPP__
  3 | #include "fenwick_tree.hpp"
  4 | #include <cstring>
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class BitF - bit compression and classical node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class BitF : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static constexpr size_t STARTING_OFFSET = 1;
 19 |   static constexpr size_t END_PADDING = 56;
 20 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 55, "Some nodes will span on multiple words");
 21 | 
 22 | protected:
 23 |   size_t Size;
 24 |   DArray<uint8_t> Tree;
 25 | 
 26 | public:
 27 |   BitF(uint64_t sequence[], size_t size)
 28 |       : Size(size), Tree((first_bit_after(size) + END_PADDING + 7) >> 3) {
 29 |     for (size_t idx = 1; idx <= size; idx++)
 30 |       addToPartialFrequency(idx, sequence[idx - 1]);
 31 | 
 32 |     for (size_t m = 2; m <= size; m <<= 1)
 33 |       for (size_t idx = m; idx <= size; idx += m)
 34 |         addToPartialFrequency(idx, getPartialFrequency(idx - m / 2));
 35 |   }
 36 | 
 37 |   virtual uint64_t prefix(size_t idx) const {
 38 |     uint64_t sum = 0;
 39 | 
 40 |     while (idx != 0) {
 41 |       sum += getPartialFrequency(idx);
 42 |       idx = clear_rho(idx);
 43 |     }
 44 | 
 45 |     return sum;
 46 |   }
 47 | 
 48 |   virtual void add(size_t idx, int64_t inc) {
 49 |     while (idx <= Size) {
 50 |       addToPartialFrequency(idx, inc);
 51 |       idx += mask_rho(idx);
 52 |     }
 53 |   }
 54 | 
 55 |   using FenwickTree::find;
 56 |   virtual size_t find(uint64_t *val) const {
 57 |     size_t node = 0;
 58 | 
 59 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 60 |       if (node + m > Size)
 61 |         continue;
 62 | 
 63 |       const uint64_t value = getPartialFrequency(node + m);
 64 | 
 65 |       if (*val >= value) {
 66 |         node += m;
 67 |         *val -= value;
 68 |       }
 69 |     }
 70 | 
 71 |     return node;
 72 |   }
 73 | 
 74 |   using FenwickTree::compFind;
 75 |   virtual size_t compFind(uint64_t *val) const {
 76 |     size_t node = 0;
 77 | 
 78 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
 79 |       if (node + m > Size)
 80 |         continue;
 81 | 
 82 |       const int height = rho(node + m);
 83 |       const uint64_t value = (BOUND << height) - getPartialFrequency(node + m);
 84 | 
 85 |       if (*val >= value) {
 86 |         node += m;
 87 |         *val -= value;
 88 |       }
 89 |     }
 90 | 
 91 |     return node;
 92 |   }
 93 | 
 94 |   virtual size_t size() const { return Size; }
 95 | 
 96 |   virtual size_t bitCount() const {
 97 |     return sizeof(BitF<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree);
 98 |   }
 99 | 
100 | private:
101 |   // TODO: try the last micro-improvement (email 05/05/19 09:51)
102 |   inline static size_t holes(size_t idx) { return STARTING_OFFSET + (idx >> 14) * 64; }
103 | 
104 |   inline static size_t first_bit_after(size_t idx) {
105 |     return (BOUNDSIZE + 1) * idx - popcount(idx) + holes(idx);
106 |   }
107 | 
108 |   inline uint64_t getPartialFrequency(size_t idx) const {
109 |     const uint64_t mask = (UINT64_C(1) << (BOUNDSIZE + rho(idx))) - 1;
110 |     idx--;
111 |     const uint64_t prod = (BOUNDSIZE + 1) * idx;
112 |     const uint64_t pos = prod - popcount(idx) + holes(idx);
113 | 
114 |     uint64_t t;
115 |     if ((prod + (BOUNDSIZE + 1)) % 64 == 0) {
116 |       memcpy(&t, (uint64_t *)&Tree[0] + pos / 64, 8);
117 |       return t >> (pos % 64) & mask;
118 |     } else {
119 |       memcpy(&t, &Tree[0] + pos / 8, 8);
120 |       return t >> (pos % 8) & mask;
121 |     }
122 |   }
123 | 
124 |   inline void addToPartialFrequency(size_t idx, uint64_t value) {
125 |     idx--;
126 |     const uint64_t prod = (BOUNDSIZE + 1) * idx;
127 |     const uint64_t pos = prod - popcount(idx) + holes(idx);
128 | 
129 |     uint64_t t;
130 |     if ((prod + (BOUNDSIZE + 1)) % 64 == 0) {
131 |       uint64_t *const p = (uint64_t *)&Tree[0] + pos / 64;
132 |       memcpy(&t, p, 8);
133 |       t += value << (pos % 64);
134 |       memcpy(p, &t, 8);
135 |     } else {
136 |       uint8_t *const p = &Tree[0] + pos / 8;
137 |       memcpy(&t, p, 8);
138 |       t += value << (pos % 8);
139 |       memcpy(p, &t, 8);
140 |     }
141 |   }
142 | 
143 |   friend std::ostream &operator<<(std::ostream &os, const BitF<BOUND> &ft) {
144 |     const uint64_t nsize = hton((uint64_t)ft.Size);
145 |     os.write((char *)&nsize, sizeof(uint64_t));
146 | 
147 |     return os << ft.Tree;
148 |   }
149 | 
150 |   friend std::istream &operator>>(std::istream &is, BitF<BOUND> &ft) {
151 |     uint64_t nsize;
152 |     is.read((char *)(&nsize), sizeof(uint64_t));
153 |     ft.Size = ntoh(nsize);
154 | 
155 |     return is >> ft.Tree;
156 |   }
157 | };
158 | 
159 | } // namespace hft::fenwick
160 | 
161 | #endif // __FENWICK_BITF_HPP__
162 | 


--------------------------------------------------------------------------------
/test/test_rank_select_algorithms.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "rank_select_algorithms/rank.hpp"
  4 | #include "rank_select_algorithms/select.hpp"
  5 | 
  6 | namespace mrs::testing {
  7 | 
  8 | template <uint64_t Words>
  9 | uint64_t naive_rank_for_words(uint64_t const* x, uint64_t i) {
 10 |     assert(i < 64 * Words);
 11 |     uint64_t ones = 0;
 12 |     uint64_t pos = 0;
 13 |     for (uint64_t w = 0; w != Words; ++w) {
 14 |         uint64_t word = x[w];
 15 |         for (uint64_t bit = 0; bit != 64; ++bit) {
 16 |             ones += word & 1;
 17 |             if (pos == i) goto EXIT;
 18 |             word >>= 1;
 19 |             ++pos;
 20 |         }
 21 |     }
 22 | EXIT:
 23 |     assert(ones <= 64 * Words);
 24 |     return ones;
 25 | }
 26 | 
 27 | template <uint64_t Words>
 28 | uint64_t naive_select_for_words(uint64_t const* x, uint64_t i) {
 29 |     assert(i < naive_rank_for_words<Words>(x, 64 * Words - 1));
 30 |     uint64_t ones = 0;
 31 |     uint64_t pos = 0;
 32 |     i += 1;
 33 |     for (uint64_t w = 0; w != Words; ++w) {
 34 |         uint64_t word = x[w];
 35 |         for (uint64_t bit = 0; bit != 64; ++bit) {
 36 |             ones += word & 1;
 37 |             if (ones == i) goto EXIT;
 38 |             word >>= 1;
 39 |             ++pos;
 40 |         }
 41 |     }
 42 | EXIT:
 43 |     assert(pos < 64 * Words);
 44 |     return pos;
 45 | }
 46 | 
 47 | template <rank_modes Mode>
 48 | void test_rank_256() {
 49 |     std::cout << "----- Algorithm: " << print_rank_mode(Mode) << " -----"
 50 |               << std::endl;
 51 |     splitmix64 distr(essentials::get_random_seed());
 52 |     std::vector<uint64_t> bits{distr.next(), distr.next(), distr.next(),
 53 |                                distr.next()};
 54 |     for (uint32_t i = 0; i != 256; ++i) {
 55 |         uint64_t got = rank_u256<Mode>(bits.data(), i);
 56 |         uint64_t expected = naive_rank_for_words<4>(bits.data(), i);
 57 |         REQUIRE_MESSAGE(got == expected, "error got rank(" << i << ") = " << got
 58 |                                                            << " but expected "
 59 |                                                            << expected);
 60 |     }
 61 |     std::cout << "\teverything's good" << std::endl;
 62 | }
 63 | 
 64 | template <rank_modes Mode>
 65 | void test_rank_512() {
 66 |     std::cout << "----- Algorithm: " << print_rank_mode(Mode) << " -----"
 67 |               << std::endl;
 68 |     splitmix64 distr(essentials::get_random_seed());
 69 |     std::vector<uint64_t> bits{distr.next(), distr.next(), distr.next(),
 70 |                                distr.next(), distr.next(), distr.next(),
 71 |                                distr.next(), distr.next()};
 72 |     for (uint32_t i = 0; i != 512; ++i) {
 73 |         uint64_t got = rank_u512<Mode>(bits.data(), i);
 74 |         uint64_t expected = naive_rank_for_words<8>(bits.data(), i);
 75 |         REQUIRE_MESSAGE(got == expected, "error got rank(" << i << ") = " << got
 76 |                                                            << " but expected "
 77 |                                                            << expected);
 78 |     }
 79 |     std::cout << "\teverything's good" << std::endl;
 80 | }
 81 | 
 82 | template <select_modes Mode>
 83 | void test_select_256() {
 84 |     std::cout << "----- Algorithm: " << print_select_mode(Mode) << " -----"
 85 |               << std::endl;
 86 |     splitmix64 distr(essentials::get_random_seed());
 87 |     std::vector<uint64_t> bits{distr.next(), distr.next(), distr.next(),
 88 |                                distr.next()};
 89 |     uint64_t total_ones = naive_rank_for_words<4>(bits.data(), 255);
 90 |     for (uint32_t i = 0; i != total_ones; ++i) {
 91 |         uint64_t got = select_u256<Mode>(bits.data(), i);
 92 |         uint64_t expected = naive_select_for_words<4>(bits.data(), i);
 93 |         REQUIRE_MESSAGE(got == expected, "error got select("
 94 |                                              << i << ") = " << got
 95 |                                              << " but expected " << expected);
 96 |     }
 97 |     std::cout << "\teverything's good" << std::endl;
 98 | }
 99 | 
100 | template <select_modes Mode>
101 | void test_select_512() {
102 |     std::cout << "----- Algorithm: " << print_select_mode(Mode) << " -----"
103 |               << std::endl;
104 |     splitmix64 distr(essentials::get_random_seed());
105 |     std::vector<uint64_t> bits{distr.next(), distr.next(), distr.next(),
106 |                                distr.next(), distr.next(), distr.next(),
107 |                                distr.next(), distr.next()};
108 |     uint64_t total_ones = naive_rank_for_words<8>(bits.data(), 511);
109 |     for (uint32_t i = 0; i != total_ones; ++i) {
110 |         uint64_t got = select_u512<Mode>(bits.data(), i);
111 |         uint64_t expected = naive_select_for_words<8>(bits.data(), i);
112 |         REQUIRE_MESSAGE(got == expected, "error got select("
113 |                                              << i << ") = " << got
114 |                                              << " but expected " << expected);
115 |     }
116 |     std::cout << "\teverything's good" << std::endl;
117 | }
118 | 
119 | }  // namespace mrs::testing


--------------------------------------------------------------------------------
/include/segment_tree/avx512/segment_tree_macros.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #define DYRS_AVX512_SUM_H1      \
 4 |     node512 node1(m_ptr);       \
 5 |     uint64_t s1 = node1.sum(i); \
 6 |     return s1;
 7 | 
 8 | #define DYRS_AVX512_SUM_H2                                           \
 9 |     uint64_t child1 = i / node512::fanout;                           \
10 |     uint64_t child2 = i % node512::fanout;                           \
11 |     node256 node1(m_ptr);                                            \
12 |     node512 node2(m_ptr + node256::bytes + child1 * node512::bytes); \
13 |     uint64_t s1 = node1.sum(child1);                                 \
14 |     uint64_t s2 = node2.sum(child2);                                 \
15 |     return s1 + s2;
16 | 
17 | #define DYRS_AVX512_SUM_H3                                               \
18 |     constexpr uint64_t B1 = node512::fanout * node256::fanout;           \
19 |     uint64_t child1 = i / B1;                                            \
20 |     uint64_t child2 = (i % B1) / node512::fanout;                        \
21 |     uint64_t child3 = (i % B1) % node512::fanout;                        \
22 |     node128 node1(m_ptr);                                                \
23 |     node256 node2(m_ptr + node128::bytes + child1 * node256::bytes);     \
24 |     node512 node3(m_ptr + node128::bytes +                               \
25 |                   m_num_nodes_per_level[1] * node256::bytes +            \
26 |                   (child2 + child1 * node256::fanout) * node512::bytes); \
27 |     uint64_t s1 = node1.sum(child1);                                     \
28 |     uint64_t s2 = node2.sum(child2);                                     \
29 |     uint64_t s3 = node3.sum(child3);                                     \
30 |     return s1 + s2 + s3;
31 | 
32 | #define DYRS_AVX512_UPDATE_H1 \
33 |     node512 node1(m_ptr);     \
34 |     node1.update(i, sign);
35 | 
36 | #define DYRS_AVX512_UPDATE_H2                                        \
37 |     uint64_t child1 = i / node512::fanout;                           \
38 |     uint64_t child2 = i % node512::fanout;                           \
39 |     node256 node1(m_ptr);                                            \
40 |     node512 node2(m_ptr + node256::bytes + child1 * node512::bytes); \
41 |     node1.update(child1 + 1, sign);                                  \
42 |     node2.update(child2, sign);
43 | 
44 | #define DYRS_AVX512_UPDATE_H3                                            \
45 |     constexpr uint64_t B1 = node512::fanout * node256::fanout;           \
46 |     uint64_t child1 = i / B1;                                            \
47 |     uint64_t child2 = (i % B1) / node512::fanout;                        \
48 |     uint64_t child3 = (i % B1) % node512::fanout;                        \
49 |     node128 node1(m_ptr);                                                \
50 |     node256 node2(m_ptr + node128::bytes + child1 * node256::bytes);     \
51 |     node512 node3(m_ptr + node128::bytes +                               \
52 |                   m_num_nodes_per_level[1] * node256::bytes +            \
53 |                   (child2 + child1 * node256::fanout) * node512::bytes); \
54 |     node1.update(child1 + 1, sign);                                      \
55 |     node2.update(child2 + 1, sign);                                      \
56 |     node3.update(child3, sign);
57 | 
58 | #define DYRS_AVX512_SEARCH_H1  \
59 |     node512 node1(m_ptr);      \
60 |     auto p1 = node1.search(x); \
61 |     return p1;
62 | 
63 | #define DYRS_AVX512_SEARCH_H2                                    \
64 |     node256 node1(m_ptr);                                        \
65 |     auto p1 = node1.search(x);                                   \
66 |     x -= p1.sum;                                                 \
67 |     uint64_t i1 = p1.position - 1;                               \
68 |     node512 node2(m_ptr + node256::bytes + i1 * node512::bytes); \
69 |     auto p2 = node2.search(x);                                   \
70 |     return {i1 * node512::fanout + p2.position, p1.sum + p2.sum};
71 | 
72 | #define DYRS_AVX512_SEARCH_H3                                               \
73 |     node128 node1(m_ptr);                                                   \
74 |     auto p1 = node1.search(x);                                              \
75 |     x -= p1.sum;                                                            \
76 |     uint64_t i1 = p1.position - 1;                                          \
77 |     node256 node2(m_ptr + node128::bytes + i1 * node256::bytes);            \
78 |     auto p2 = node2.search(x);                                              \
79 |     x -= p2.sum;                                                            \
80 |     uint64_t i2 = p2.position - 1;                                          \
81 |     node512 node3(m_ptr + node128::bytes +                                  \
82 |                   m_num_nodes_per_level[1] * node256::bytes +               \
83 |                   (i2 + i1 * node256::fanout) * node512::bytes);            \
84 |     auto p3 = node3.search(x);                                              \
85 |     return {i1 * node512::fanout * node256::fanout + i2 * node512::fanout + \
86 |                 p3.position,                                                \
87 |             p1.sum + p2.sum + p3.sum};
88 | 


--------------------------------------------------------------------------------
/src/perf_select64.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | static constexpr std::array<uint64_t, 25> sizes = {
 21 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 22 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 23 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 24 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 25 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32};
 26 | 
 27 | template <select64_modes Mode>
 28 | void test(std::string type) {
 29 |     std::vector<uint64_t> bits(sizes.back() / 64);
 30 |     std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 31 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 32 |     std::cout << "# number of ones: " << num_ones << " ("
 33 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 34 | 
 35 |     std::string json("{\"type\":\"" + type + "\", ");
 36 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 37 |     json += "\"timings\":[";
 38 | 
 39 |     for (const uint64_t n : sizes) {
 40 |         splitmix64 hasher(query_seed);
 41 | 
 42 |         const auto num_buckets = n / 64;
 43 |         for (uint64_t i = 0; i < num_queries;) {
 44 |             const uint64_t* x = &bits[(hasher.next() % num_buckets)];
 45 |             const uint64_t r = popcount_u64<popcount_modes::builtin>(*x);
 46 |             if (r != 0) { queries[i++] = {x, hasher.next() % r}; }
 47 |         }
 48 | 
 49 |         essentials::timer_type t;
 50 |         double min = 0.0, max = 0.0, avg = 0.0;
 51 | 
 52 |         auto measure = [&]() {
 53 |             uint64_t tmp = 0;  // to avoid the optimization
 54 |             for (int run = 0; run != runs; run++) {
 55 |                 t.start();
 56 |                 for (uint64_t i = 0; i < num_queries; i++) {
 57 |                     const uint64_t x = *queries[i].first;
 58 |                     const uint64_t k = queries[i].second;
 59 |                     tmp += select_u64<Mode>(x, k);
 60 |                 }
 61 |                 t.stop();
 62 |             }
 63 |             std::cout << "# ignore: " << tmp << std::endl;
 64 |         };
 65 | 
 66 |         static constexpr int K = 10;
 67 | 
 68 |         // warm-up
 69 |         for (int k = 0; k != K; ++k) {
 70 |             measure();
 71 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 72 |             avg += avg_ns_query;
 73 |             t.reset();
 74 |         }
 75 |         std::cout << "# warm-up: " << avg / K << std::endl;
 76 |         avg = 0.0;
 77 | 
 78 |         for (int k = 0; k != K; ++k) {
 79 |             measure();
 80 |             t.discard_max();
 81 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 82 |             max += avg_ns_query;
 83 |             t.reset();
 84 |         }
 85 | 
 86 |         for (int k = 0; k != K; ++k) {
 87 |             measure();
 88 |             t.discard_min();
 89 |             t.discard_max();
 90 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 91 |             avg += avg_ns_query;
 92 |             t.reset();
 93 |         }
 94 | 
 95 |         for (int k = 0; k != K; ++k) {
 96 |             measure();
 97 |             t.discard_min();
 98 |             double avg_ns_query = (t.min() * 1000) / num_queries;
 99 |             min += avg_ns_query;
100 |             t.reset();
101 |         }
102 | 
103 |         min /= K;
104 |         max /= K;
105 |         avg /= K;
106 |         std::vector<double> tt{min, avg, max};
107 |         std::sort(tt.begin(), tt.end());
108 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
109 | 
110 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
111 |                 "," + std::to_string(tt[2]) + "],";
112 |     }
113 | 
114 |     json.pop_back();
115 |     json += "]}";
116 |     std::cerr << json << std::endl;
117 | }
118 | 
119 | int main(int argc, char** argv) {
120 |     cmd_line_parser::parser parser(argc, argv);
121 |     parser.add("mode", "Mode of select64 algorithm.");
122 |     if (!parser.parse()) return 1;
123 | 
124 |     auto mode = parser.get<std::string>("mode");
125 | 
126 |     if (mode == "broadword_sdsl") {
127 |         test<select64_modes::broadword_sdsl>(mode);
128 |     } else if (mode == "broadword_succinct") {
129 |         test<select64_modes::broadword_succinct>(mode);
130 |     }
131 | #ifdef __SSE4_2__
132 |     else if (mode == "sse4_sdsl") {
133 |         test<select64_modes::sse4_sdsl>(mode);
134 |     } else if (mode == "sse4_succinct") {
135 |         test<select64_modes::sse4_succinct>(mode);
136 |     }
137 | #endif
138 | #ifdef __BMI2__
139 |     else if (mode == "pdep") {
140 |         test<select64_modes::pdep>(mode);
141 |     }
142 | #endif
143 |     else {
144 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
145 |         return 1;
146 |     }
147 | 
148 |     return 0;
149 | }
150 | 


--------------------------------------------------------------------------------
/include/hft/fenwick/bitl.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_BITL_HPP__
  2 | #define __FENWICK_BITL_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class BitL - bit compression and level ordered node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class BitL : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   size_t Size, Levels;
 22 |   unique_ptr<size_t[]> Level;
 23 |   DArray<uint8_t> Tree;
 24 | 
 25 | public:
 26 |   BitL(uint64_t sequence[], size_t size)
 27 |       : Size(size), Levels(size != 0 ? lambda(size) + 2 : 1), Level(make_unique<size_t[]>(Levels)) {
 28 |     Level[0] = 0;
 29 |     for (size_t i = 1; i < Levels; i++)
 30 |       Level[i] = ((size + (1ULL << (i - 1))) / (1ULL << i)) * (BOUNDSIZE - 1 + i) + Level[i - 1];
 31 | 
 32 |     Tree = DArray<uint8_t>((Level[Levels - 1] >> 3) + 8); // +8 for safety
 33 | 
 34 |     for (size_t l = 0; l < Levels - 1; l++) {
 35 |       for (size_t node = 1ULL << l; node <= size; node += 1ULL << (l + 1)) {
 36 |         size_t sequence_idx = node - 1;
 37 |         uint64_t value = sequence[sequence_idx];
 38 | 
 39 |         for (size_t j = 0; j < l; j++) {
 40 |           sequence_idx >>= 1;
 41 |           const size_t lowpos = Level[j] + (BOUNDSIZE + j) * sequence_idx;
 42 |           value += bitread(&Tree[lowpos / 8], lowpos % 8, BOUNDSIZE + j);
 43 |         }
 44 | 
 45 |         const size_t highpos = Level[l] + (BOUNDSIZE + l) * (node >> (l + 1));
 46 |         bitwrite_inc(&Tree[highpos / 8], highpos % 8, BOUNDSIZE + l, value);
 47 |       }
 48 |     }
 49 |   }
 50 | 
 51 |   virtual uint64_t prefix(size_t idx) const {
 52 |     uint64_t sum = 0;
 53 | 
 54 |     while (idx != 0) {
 55 |       const int height = rho(idx);
 56 |       const size_t pos = Level[height] + (idx >> (1 + height)) * (BOUNDSIZE + height);
 57 |       sum += bitread(&Tree[pos / 8], pos % 8, BOUNDSIZE + height);
 58 | 
 59 |       idx = clear_rho(idx);
 60 |     }
 61 | 
 62 |     return sum;
 63 |   }
 64 | 
 65 |   virtual void add(size_t idx, int64_t inc) {
 66 |     while (idx <= Size) {
 67 |       const int height = rho(idx);
 68 |       const size_t pos = Level[height] + (idx >> (1 + height)) * (BOUNDSIZE + height);
 69 |       bitwrite_inc(&Tree[pos / 8], pos % 8, BOUNDSIZE + height, inc);
 70 | 
 71 |       idx += mask_rho(idx);
 72 |     }
 73 |   }
 74 | 
 75 |   using FenwickTree::find;
 76 |   virtual size_t find(uint64_t *val) const {
 77 |     size_t node = 0, idx = 0;
 78 | 
 79 |     for (size_t height = Levels - 2; height != SIZE_MAX; height--) {
 80 |       const size_t pos = Level[height] + idx * (BOUNDSIZE + height);
 81 | 
 82 |       idx <<= 1;
 83 | 
 84 |       if (pos >= Level[height + 1])
 85 |         continue;
 86 | 
 87 |       const uint64_t value = bitread(&Tree[pos / 8], pos % 8, BOUNDSIZE + height);
 88 | 
 89 |       if (*val >= value) {
 90 |         idx++;
 91 |         *val -= value;
 92 |         node += 1ULL << height;
 93 |       }
 94 |     }
 95 | 
 96 |     return min(node, Size);
 97 |   }
 98 | 
 99 |   using FenwickTree::compFind;
100 |   virtual size_t compFind(uint64_t *val) const {
101 |     size_t node = 0, idx = 0;
102 | 
103 |     for (size_t height = Levels - 2; height != SIZE_MAX; height--) {
104 |       const size_t pos = Level[height] + idx * (BOUNDSIZE + height);
105 | 
106 |       idx <<= 1;
107 | 
108 |       if (pos >= Level[height + 1])
109 |         continue;
110 | 
111 |       const uint64_t value =
112 |           (BOUND << height) - bitread(&Tree[pos / 8], pos % 8, BOUNDSIZE + height);
113 | 
114 |       if (*val >= value) {
115 |         idx++;
116 |         *val -= value;
117 |         node += 1ULL << height;
118 |       }
119 |     }
120 | 
121 |     return min(node, Size);
122 |   }
123 | 
124 |   virtual size_t size() const { return Size; }
125 | 
126 |   virtual size_t bitCount() const {
127 |     return sizeof(BitL<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree) +
128 |            Levels * sizeof(size_t) * 8;
129 |   }
130 | 
131 | private:
132 |   friend std::ostream &operator<<(std::ostream &os, const BitL<BOUND> &ft) {
133 |     const uint64_t nsize = hton((uint64_t)ft.Size);
134 |     os.write((char *)&nsize, sizeof(uint64_t));
135 | 
136 |     const uint64_t nlevels = hton((uint64_t)ft.Levels);
137 |     os.write((char *)&nlevels, sizeof(uint64_t));
138 | 
139 |     for (size_t i = 0; i < ft.Levels; ++i) {
140 |       const uint64_t nlevel = hton((uint64_t)ft.Level[i]);
141 |       os.write((char *)&nlevel, sizeof(uint64_t));
142 |     }
143 | 
144 |     return os << ft.Tree;
145 |   }
146 | 
147 |   friend std::istream &operator>>(std::istream &is, BitL<BOUND> &ft) {
148 |     uint64_t nsize;
149 |     is.read((char *)(&nsize), sizeof(uint64_t));
150 |     ft.Size = ntoh(nsize);
151 | 
152 |     uint64_t nlevels;
153 |     is.read((char *)&nlevels, sizeof(uint64_t));
154 |     ft.Levels = ntoh(nlevels);
155 | 
156 |     ft.Level = make_unique<size_t[]>(ft.Levels);
157 |     for (size_t i = 0; i < ft.Levels; ++i) {
158 |       uint64_t nlevel;
159 |       is.read((char *)&nlevel, sizeof(uint64_t));
160 |       ft.Level[i] = ntoh(nlevel);
161 |     }
162 | 
163 |     return is >> ft.Tree;
164 |   }
165 | };
166 | 
167 | } // namespace hft::fenwick
168 | 
169 | #endif // __FENWICK_BITL_HPP__
170 | 


--------------------------------------------------------------------------------
/include/hft/fenwick/bytel.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_BYTEL_HPP__
  2 | #define __FENWICK_BYTEL_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class ByteL - byte compression and level ordered node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class ByteL : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   size_t Size, Levels;
 22 |   unique_ptr<size_t[]> Level;
 23 |   DArray<uint8_t> Tree;
 24 | 
 25 | public:
 26 |   ByteL(uint64_t sequence[], size_t size)
 27 |       : Size(size), Levels(size != 0 ? lambda(size) + 2 : 1), Level(make_unique<size_t[]>(Levels)) {
 28 |     Level[0] = 0;
 29 |     for (size_t i = 1; i < Levels; i++)
 30 |       Level[i] = ((size + (1ULL << (i - 1))) / (1ULL << i)) * heightsize(i - 1) + Level[i - 1];
 31 | 
 32 |     Tree = DArray<uint8_t>(Level[Levels - 1] + 8); // +8 for safety
 33 | 
 34 |     for (size_t l = 0; l < Levels - 1; l++) {
 35 |       for (size_t node = 1ULL << l; node <= Size; node += 1ULL << (l + 1)) {
 36 |         size_t sequence_idx = node - 1;
 37 |         uint64_t value = sequence[sequence_idx];
 38 |         for (size_t j = 0; j < l; j++) {
 39 |           sequence_idx >>= 1;
 40 |           const size_t lowpos = Level[j] + heightsize(j) * sequence_idx;
 41 |           const uint64_t low = byteread(&Tree[lowpos], heightsize(j));
 42 | 
 43 |           value += low;
 44 |         }
 45 | 
 46 |         const size_t highpos = Level[l] + heightsize(l) * (node >> (l + 1));
 47 |         bytewrite(&Tree[highpos], heightsize(l), value);
 48 |       }
 49 |     }
 50 |   }
 51 | 
 52 |   virtual uint64_t prefix(size_t idx) const {
 53 |     uint64_t sum = 0;
 54 | 
 55 |     while (idx != 0) {
 56 |       const int height = rho(idx);
 57 |       const size_t isize = heightsize(height);
 58 |       const size_t pos = Level[height] + (idx >> (1 + height)) * isize;
 59 | 
 60 |       sum += byteread(&Tree[pos], isize);
 61 |       idx = clear_rho(idx);
 62 |     }
 63 | 
 64 |     return sum;
 65 |   }
 66 | 
 67 |   virtual void add(size_t idx, int64_t inc) {
 68 |     while (idx <= Size) {
 69 |       const int height = rho(idx);
 70 |       const size_t isize = heightsize(height);
 71 |       const size_t pos = Level[height] + (idx >> (1 + height)) * isize;
 72 | 
 73 |       bytewrite_inc(&Tree[pos], inc);
 74 |       idx += mask_rho(idx);
 75 |     }
 76 |   }
 77 | 
 78 |   using FenwickTree::find;
 79 |   virtual size_t find(uint64_t *val) const {
 80 |     size_t node = 0, idx = 0;
 81 | 
 82 |     for (int height = Levels - 2; height >= 0; --height) {
 83 |       const size_t isize = heightsize(height);
 84 |       const size_t pos = Level[height] + idx * heightsize(height);
 85 | 
 86 |       idx <<= 1;
 87 | 
 88 |       if (pos >= Level[height + 1])
 89 |         continue;
 90 | 
 91 |       const uint64_t value = byteread(&Tree[pos], isize);
 92 | 
 93 |       if (*val >= value) {
 94 |         idx++;
 95 |         *val -= value;
 96 |         node += 1ULL << height;
 97 |       }
 98 |     }
 99 | 
100 |     return min(node, Size);
101 |   }
102 | 
103 |   using FenwickTree::compFind;
104 |   virtual size_t compFind(uint64_t *val) const {
105 |     size_t node = 0, idx = 0;
106 | 
107 |     for (size_t height = Levels - 2; height != SIZE_MAX; height--) {
108 |       const size_t isize = heightsize(height);
109 |       const size_t pos = Level[height] + idx * heightsize(height);
110 | 
111 |       idx <<= 1;
112 | 
113 |       if (pos >= Level[height + 1])
114 |         continue;
115 | 
116 |       const uint64_t value = (BOUND << height) - byteread(&Tree[pos], isize);
117 | 
118 |       if (*val >= value) {
119 |         idx++;
120 |         *val -= value;
121 |         node += 1ULL << height;
122 |       }
123 |     }
124 | 
125 |     return min(node, Size);
126 |   }
127 | 
128 |   virtual size_t size() const { return Size; }
129 | 
130 |   virtual size_t bitCount() const {
131 |     return sizeof(ByteL<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree) +
132 |            Levels * sizeof(size_t) * 8;
133 |   }
134 | 
135 | private:
136 |   static inline size_t heightsize(size_t height) { return ((height + BOUNDSIZE - 1) >> 3) + 1; }
137 | 
138 |   friend std::ostream &operator<<(std::ostream &os, const ByteL<BOUND> &ft) {
139 |     const uint64_t nsize = hton((uint64_t)ft.Size);
140 |     os.write((char *)&nsize, sizeof(uint64_t));
141 | 
142 |     const uint64_t nlevels = hton((uint64_t)ft.Levels);
143 |     os.write((char *)&nlevels, sizeof(uint64_t));
144 | 
145 |     for (size_t i = 0; i < ft.Levels; ++i) {
146 |       const uint64_t nlevel = hton((uint64_t)ft.Level[i]);
147 |       os.write((char *)&nlevel, sizeof(uint64_t));
148 |     }
149 | 
150 |     return os << ft.Tree;
151 |   }
152 | 
153 |   friend std::istream &operator>>(std::istream &is, ByteL<BOUND> &ft) {
154 |     uint64_t nsize;
155 |     is.read((char *)(&nsize), sizeof(uint64_t));
156 |     ft.Size = ntoh(nsize);
157 | 
158 |     uint64_t nlevels;
159 |     is.read((char *)&nlevels, sizeof(uint64_t));
160 |     ft.Levels = ntoh(nlevels);
161 | 
162 |     ft.Level = make_unique<size_t[]>(ft.Levels);
163 |     for (size_t i = 0; i < ft.Levels; ++i) {
164 |       uint64_t nlevel;
165 |       is.read((char *)&nlevel, sizeof(uint64_t));
166 |       ft.Level[i] = ntoh(nlevel);
167 |     }
168 | 
169 |     return is >> ft.Tree;
170 |   }
171 | };
172 | 
173 | } // namespace hft::fenwick
174 | 
175 | #endif // __FENWICK_BYTEL_HPP__
176 | 


--------------------------------------------------------------------------------
/src/perf_rank512.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | 
 19 | static constexpr std::array<uint64_t, 24> sizes = {
 20 |     1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12, 1ULL << 13, 1ULL << 14,
 21 |     1ULL << 15, 1ULL << 16, 1ULL << 17, 1ULL << 18, 1ULL << 19, 1ULL << 20,
 22 |     1ULL << 21, 1ULL << 22, 1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26,
 23 |     1ULL << 27, 1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 24 | };
 25 | 
 26 | template <rank_modes Mode>
 27 | void test(const double density) {
 28 |     std::vector<uint64_t> bits(sizes.back() / 64);
 29 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 30 |     std::cout << "# number of ones: " << num_ones << " ("
 31 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 32 | 
 33 |     std::string json("{\"type\":\"" + print_rank_mode(Mode) + "\", ");
 34 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 35 |     json += "\"timings\":[";
 36 | 
 37 |     for (const uint64_t n : sizes) {
 38 |         splitmix64 hasher(query_seed);
 39 | 
 40 |         const auto num_buckets = n / 512;
 41 |         std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 42 | 
 43 |         for (uint64_t i = 0; i < num_queries; i++) {
 44 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 512 / 64];
 45 |             queries[i] = {x, hasher.next() % 512};
 46 |         }
 47 | 
 48 |         essentials::timer_type t;
 49 |         double min = 0.0, max = 0.0, avg = 0.0;
 50 | 
 51 |         auto measure = [&]() {
 52 |             uint64_t tmp = 0;  // to avoid the optimization
 53 |             for (int run = 0; run != runs; run++) {
 54 |                 t.start();
 55 |                 for (uint64_t i = 0; i < num_queries; i++) {
 56 |                     const uint64_t* x = queries[i].first;
 57 |                     const uint64_t k = queries[i].second;
 58 |                     tmp += rank_u512<Mode>(x, k);
 59 |                 }
 60 |                 t.stop();
 61 |             }
 62 |             std::cout << "# ignore: " << tmp << std::endl;
 63 |         };
 64 | 
 65 |         static constexpr int K = 10;
 66 | 
 67 |         // warm-up
 68 |         for (int k = 0; k != K; ++k) {
 69 |             measure();
 70 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 71 |             avg += avg_ns_query;
 72 |             t.reset();
 73 |         }
 74 |         std::cout << "# warm-up: " << avg / K << std::endl;
 75 |         avg = 0.0;
 76 | 
 77 |         for (int k = 0; k != K; ++k) {
 78 |             measure();
 79 |             t.discard_max();
 80 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 81 |             max += avg_ns_query;
 82 |             t.reset();
 83 |         }
 84 | 
 85 |         for (int k = 0; k != K; ++k) {
 86 |             measure();
 87 |             t.discard_min();
 88 |             t.discard_max();
 89 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 90 |             avg += avg_ns_query;
 91 |             t.reset();
 92 |         }
 93 | 
 94 |         for (int k = 0; k != K; ++k) {
 95 |             measure();
 96 |             t.discard_min();
 97 |             double avg_ns_query = (t.min() * 1000) / num_queries;
 98 |             min += avg_ns_query;
 99 |             t.reset();
100 |         }
101 | 
102 |         min /= K;
103 |         max /= K;
104 |         avg /= K;
105 |         std::vector<double> tt{min, avg, max};
106 |         std::sort(tt.begin(), tt.end());
107 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
108 | 
109 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
110 |                 "," + std::to_string(tt[2]) + "],";
111 |     }
112 | 
113 |     json.pop_back();
114 |     json += "]}";
115 |     std::cerr << json << std::endl;
116 | }
117 | 
118 | int main(int argc, char** argv) {
119 |     cmd_line_parser::parser parser(argc, argv);
120 |     parser.add("mode", "Mode of rank algorithm.");
121 |     parser.add("density", "Density of ones (in [0,1]).");
122 |     if (!parser.parse()) return 1;
123 | 
124 |     auto mode = parser.get<std::string>("mode");
125 |     auto density = parser.get<double>("density");
126 | 
127 |     if (mode == "broadword_loop") {
128 |         test<rank_modes::broadword_loop>(density);
129 |     } else if (mode == "broadword_unrolled") {
130 |         test<rank_modes::broadword_unrolled>(density);
131 |     }
132 | #ifdef __SSE4_2__
133 |     else if (mode == "builtin_loop") {
134 |         test<rank_modes::builtin_loop>(density);
135 |     } else if (mode == "builtin_unrolled") {
136 |         test<rank_modes::builtin_unrolled>(density);
137 |     }
138 | #endif
139 | #ifdef __AVX512VL__
140 |     else if (mode == "avx512_loop") {
141 |         test<rank_modes::avx512_loop>(density);
142 |     } else if (mode == "avx512_unrolled") {
143 |         test<rank_modes::avx512_unrolled>(density);
144 |     } else if (mode == "builtin_parallel") {
145 |         test<rank_modes::builtin_parallel>(density);
146 |     } else if (mode == "avx512_parallel") {
147 |         test<rank_modes::avx512_parallel>(density);
148 |     }
149 | #endif
150 |     else {
151 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
152 |         return 1;
153 |     }
154 | 
155 |     return 0;
156 | }
157 | 


--------------------------------------------------------------------------------
/src/perf_rank256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | 
 19 | static constexpr std::array<uint64_t, 25> sizes = {
 20 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 21 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 22 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 23 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 24 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 25 | };
 26 | 
 27 | template <rank_modes Mode>
 28 | void test(const double density) {
 29 |     std::vector<uint64_t> bits(sizes.back() / 64);
 30 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 31 |     std::cout << "# number of ones: " << num_ones << " ("
 32 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 33 | 
 34 |     std::string json("{\"type\":\"" + print_rank_mode(Mode) + "\", ");
 35 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 36 |     json += "\"timings\":[";
 37 | 
 38 |     for (const uint64_t n : sizes) {
 39 |         splitmix64 hasher(query_seed);
 40 | 
 41 |         const auto num_buckets = n / 256;
 42 |         std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 43 | 
 44 |         for (uint64_t i = 0; i < num_queries; i++) {
 45 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 256 / 64];
 46 |             queries[i] = {x, hasher.next() % 256};
 47 |         }
 48 | 
 49 |         essentials::timer_type t;
 50 |         double min = 0.0, max = 0.0, avg = 0.0;
 51 | 
 52 |         auto measure = [&]() {
 53 |             uint64_t tmp = 0;  // to avoid the optimization
 54 |             for (int run = 0; run != runs; run++) {
 55 |                 t.start();
 56 |                 for (uint64_t i = 0; i < num_queries; i++) {
 57 |                     const uint64_t* x = queries[i].first;
 58 |                     const uint64_t k = queries[i].second;
 59 |                     tmp += rank_u256<Mode>(x, k);
 60 |                 }
 61 |                 t.stop();
 62 |             }
 63 |             std::cout << "# ignore: " << tmp << std::endl;
 64 |         };
 65 | 
 66 |         static constexpr int K = 10;
 67 | 
 68 |         // warm-up
 69 |         for (int k = 0; k != K; ++k) {
 70 |             measure();
 71 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 72 |             avg += avg_ns_query;
 73 |             t.reset();
 74 |         }
 75 |         std::cout << "# warm-up: " << avg / K << std::endl;
 76 |         avg = 0.0;
 77 | 
 78 |         for (int k = 0; k != K; ++k) {
 79 |             measure();
 80 |             t.discard_max();
 81 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 82 |             max += avg_ns_query;
 83 |             t.reset();
 84 |         }
 85 | 
 86 |         for (int k = 0; k != K; ++k) {
 87 |             measure();
 88 |             t.discard_min();
 89 |             t.discard_max();
 90 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 91 |             avg += avg_ns_query;
 92 |             t.reset();
 93 |         }
 94 | 
 95 |         for (int k = 0; k != K; ++k) {
 96 |             measure();
 97 |             t.discard_min();
 98 |             double avg_ns_query = (t.min() * 1000) / num_queries;
 99 |             min += avg_ns_query;
100 |             t.reset();
101 |         }
102 | 
103 |         min /= K;
104 |         max /= K;
105 |         avg /= K;
106 |         std::vector<double> tt{min, avg, max};
107 |         std::sort(tt.begin(), tt.end());
108 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
109 | 
110 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
111 |                 "," + std::to_string(tt[2]) + "],";
112 |     }
113 | 
114 |     json.pop_back();
115 |     json += "]}";
116 |     std::cerr << json << std::endl;
117 | }
118 | 
119 | int main(int argc, char** argv) {
120 |     cmd_line_parser::parser parser(argc, argv);
121 |     parser.add("mode", "Mode of rank algorithm.");
122 |     parser.add("density", "Density of ones (in [0,1]).");
123 |     if (!parser.parse()) return 1;
124 | 
125 |     auto mode = parser.get<std::string>("mode");
126 |     auto density = parser.get<double>("density");
127 | 
128 |     if (mode == "broadword_loop") {
129 |         test<rank_modes::broadword_loop>(density);
130 |     } else if (mode == "broadword_unrolled") {
131 |         test<rank_modes::broadword_unrolled>(density);
132 |     }
133 | #ifdef __SSE4_2__
134 |     else if (mode == "builtin_loop") {
135 |         test<rank_modes::builtin_loop>(density);
136 |     } else if (mode == "builtin_unrolled") {
137 |         test<rank_modes::builtin_unrolled>(density);
138 |     }
139 | #endif
140 | #ifdef __AVX2__
141 |     else if (mode == "avx2_loop") {
142 |         test<rank_modes::avx2_loop>(density);
143 |     } else if (mode == "avx2_unrolled") {
144 |         test<rank_modes::avx2_unrolled>(density);
145 |     }
146 | #endif
147 | #ifdef __AVX512VL__
148 |     else if (mode == "builtin_parallel") {
149 |         test<rank_modes::builtin_parallel>(density);
150 |     } else if (mode == "avx2_parallel") {
151 |         test<rank_modes::avx2_parallel>(density);
152 |     }
153 | #endif
154 |     else {
155 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
156 |         return 1;
157 |     }
158 | 
159 |     return 0;
160 | }
161 | 


--------------------------------------------------------------------------------
/src/perf_select512.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | 
 19 | static constexpr std::array<uint64_t, 24> sizes = {
 20 |     1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12, 1ULL << 13, 1ULL << 14,
 21 |     1ULL << 15, 1ULL << 16, 1ULL << 17, 1ULL << 18, 1ULL << 19, 1ULL << 20,
 22 |     1ULL << 21, 1ULL << 22, 1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26,
 23 |     1ULL << 27, 1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 24 | };
 25 | 
 26 | template <select_modes Mode>
 27 | void test(const double density) {
 28 |     std::vector<uint64_t> bits(sizes.back() / 64);
 29 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 30 |     std::cout << "# number of ones: " << num_ones << " ("
 31 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 32 | 
 33 |     std::string json("{\"type\":\"" + print_select_mode(Mode) + "\", ");
 34 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 35 |     json += "\"timings\":[";
 36 | 
 37 |     for (const uint64_t n : sizes) {
 38 |         splitmix64 hasher(query_seed);
 39 | 
 40 |         const auto num_buckets = n / 512;
 41 |         std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 42 | 
 43 |         for (uint64_t i = 0; i < num_queries;) {
 44 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 512 / 64];
 45 |             const uint64_t r =
 46 |                 rank_u512<rank_modes::broadword_unrolled>(x, 511);
 47 |             if (r != 0) { queries[i++] = {x, hasher.next() % r}; }
 48 |         }
 49 | 
 50 |         essentials::timer_type t;
 51 |         double min = 0.0, max = 0.0, avg = 0.0;
 52 | 
 53 |         auto measure = [&]() {
 54 |             uint64_t tmp = 0;  // to avoid the optimization
 55 |             for (int run = 0; run != runs; run++) {
 56 |                 t.start();
 57 |                 for (uint64_t i = 0; i < num_queries; i++) {
 58 |                     const uint64_t* x = queries[i].first;
 59 |                     const uint64_t k = queries[i].second;
 60 |                     tmp += select_u512<Mode>(x, k);
 61 |                 }
 62 |                 t.stop();
 63 |             }
 64 |             std::cout << "# ignore: " << tmp << std::endl;
 65 |         };
 66 | 
 67 |         static constexpr int K = 10;
 68 | 
 69 |         // warm-up
 70 |         for (int k = 0; k != K; ++k) {
 71 |             measure();
 72 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 73 |             avg += avg_ns_query;
 74 |             t.reset();
 75 |         }
 76 |         std::cout << "# warm-up: " << avg / K << std::endl;
 77 |         avg = 0.0;
 78 | 
 79 |         for (int k = 0; k != K; ++k) {
 80 |             measure();
 81 |             t.discard_max();
 82 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 83 |             max += avg_ns_query;
 84 |             t.reset();
 85 |         }
 86 | 
 87 |         for (int k = 0; k != K; ++k) {
 88 |             measure();
 89 |             t.discard_min();
 90 |             t.discard_max();
 91 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 92 |             avg += avg_ns_query;
 93 |             t.reset();
 94 |         }
 95 | 
 96 |         for (int k = 0; k != K; ++k) {
 97 |             measure();
 98 |             t.discard_min();
 99 |             double avg_ns_query = (t.min() * 1000) / num_queries;
100 |             min += avg_ns_query;
101 |             t.reset();
102 |         }
103 | 
104 |         min /= K;
105 |         max /= K;
106 |         avg /= K;
107 |         std::vector<double> tt{min, avg, max};
108 |         std::sort(tt.begin(), tt.end());
109 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
110 | 
111 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
112 |                 "," + std::to_string(tt[2]) + "],";
113 |     }
114 | 
115 |     json.pop_back();
116 |     json += "]}";
117 |     std::cerr << json << std::endl;
118 | }
119 | 
120 | int main(int argc, char** argv) {
121 |     cmd_line_parser::parser parser(argc, argv);
122 |     parser.add("mode", "Mode of select algorithm.");
123 |     parser.add("density", "Density of ones (in [0,1]).");
124 |     if (!parser.parse()) return 1;
125 | 
126 |     auto mode = parser.get<std::string>("mode");
127 |     auto density = parser.get<double>("density");
128 | 
129 |     if (mode == "broadword_loop_sdsl") {
130 |         test<select_modes::broadword_loop_sdsl>(density);
131 |     } else if (mode == "broadword_loop_succinct") {
132 |         test<select_modes::broadword_loop_succinct>(density);
133 |     }
134 | #ifdef __SSE4_2__
135 |     else if (mode == "builtin_loop_sdsl") {
136 |         test<select_modes::builtin_loop_sdsl>(density);
137 |     } else if (mode == "builtin_loop_succinct") {
138 |         test<select_modes::builtin_loop_succinct>(density);
139 |     }
140 | #endif
141 | #ifdef __BMI2__
142 |     else if (mode == "builtin_loop_pdep") {
143 |         test<select_modes::builtin_loop_pdep>(density);
144 |     }
145 | #endif
146 | #ifdef __AVX512VL__
147 |     else if (mode == "builtin_avx512_pdep") {
148 |         test<select_modes::builtin_avx512_pdep>(density);
149 |     } else if (mode == "avx512_avx512_pdep") {
150 |         test<select_modes::avx512_avx512_pdep>(density);
151 |     }
152 | #endif
153 |     else {
154 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
155 |         return 1;
156 |     }
157 | 
158 |     return 0;
159 | }
160 | 


--------------------------------------------------------------------------------
/src/perf_select256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | 
 19 | static constexpr std::array<uint64_t, 25> sizes = {
 20 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 21 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 22 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 23 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 24 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 25 | };
 26 | 
 27 | template <select_modes Mode>
 28 | void test(const double density) {
 29 |     std::vector<uint64_t> bits(sizes.back() / 64);
 30 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 31 |     std::cout << "# number of ones: " << num_ones << " ("
 32 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 33 | 
 34 |     std::string json("{\"type\":\"" + print_select_mode(Mode) + "\", ");
 35 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 36 |     json += "\"timings\":[";
 37 | 
 38 |     for (const uint64_t n : sizes) {
 39 |         splitmix64 hasher(query_seed);
 40 | 
 41 |         const auto num_buckets = n / 256;
 42 |         std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 43 | 
 44 |         for (uint64_t i = 0; i < num_queries;) {
 45 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 256 / 64];
 46 |             const uint64_t r =
 47 |                 rank_u256<rank_modes::broadword_unrolled>(x, 255);
 48 |             if (r != 0) { queries[i++] = {x, hasher.next() % r}; }
 49 |         }
 50 | 
 51 |         essentials::timer_type t;
 52 |         double min = 0.0, max = 0.0, avg = 0.0;
 53 | 
 54 |         auto measure = [&]() {
 55 |             uint64_t tmp = 0;  // to avoid the optimization
 56 |             for (int run = 0; run != runs; run++) {
 57 |                 t.start();
 58 |                 for (uint64_t i = 0; i < num_queries; i++) {
 59 |                     const uint64_t* x = queries[i].first;
 60 |                     const uint64_t k = queries[i].second;
 61 |                     tmp += select_u256<Mode>(x, k);
 62 |                 }
 63 |                 t.stop();
 64 |             }
 65 |             std::cout << "# ignore: " << tmp << std::endl;
 66 |         };
 67 | 
 68 |         static constexpr int K = 10;
 69 | 
 70 |         // warm-up
 71 |         for (int k = 0; k != K; ++k) {
 72 |             measure();
 73 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 74 |             avg += avg_ns_query;
 75 |             t.reset();
 76 |         }
 77 |         std::cout << "# warm-up: " << avg / K << std::endl;
 78 |         avg = 0.0;
 79 | 
 80 |         for (int k = 0; k != K; ++k) {
 81 |             measure();
 82 |             t.discard_max();
 83 |             double avg_ns_query = (t.max() * 1000) / num_queries;
 84 |             max += avg_ns_query;
 85 |             t.reset();
 86 |         }
 87 | 
 88 |         for (int k = 0; k != K; ++k) {
 89 |             measure();
 90 |             t.discard_min();
 91 |             t.discard_max();
 92 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 93 |             avg += avg_ns_query;
 94 |             t.reset();
 95 |         }
 96 | 
 97 |         for (int k = 0; k != K; ++k) {
 98 |             measure();
 99 |             t.discard_min();
100 |             double avg_ns_query = (t.min() * 1000) / num_queries;
101 |             min += avg_ns_query;
102 |             t.reset();
103 |         }
104 | 
105 |         min /= K;
106 |         max /= K;
107 |         avg /= K;
108 |         std::vector<double> tt{min, avg, max};
109 |         std::sort(tt.begin(), tt.end());
110 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
111 | 
112 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
113 |                 "," + std::to_string(tt[2]) + "],";
114 |     }
115 | 
116 |     json.pop_back();
117 |     json += "]}";
118 |     std::cerr << json << std::endl;
119 | }
120 | 
121 | int main(int argc, char** argv) {
122 |     cmd_line_parser::parser parser(argc, argv);
123 |     parser.add("mode", "Mode of select algorithm.");
124 |     parser.add("density", "Density of ones (in [0,1]).");
125 |     if (!parser.parse()) return 1;
126 | 
127 |     auto mode = parser.get<std::string>("mode");
128 |     auto density = parser.get<double>("density");
129 | 
130 |     if (mode == "broadword_loop_sdsl") {
131 |         test<select_modes::broadword_loop_sdsl>(density);
132 |     } else if (mode == "broadword_loop_succinct") {
133 |         test<select_modes::broadword_loop_succinct>(density);
134 |     }
135 | #ifdef __SSE4_2__
136 |     else if (mode == "builtin_loop_sdsl") {
137 |         test<select_modes::builtin_loop_sdsl>(density);
138 |     } else if (mode == "builtin_loop_succinct") {
139 |         test<select_modes::builtin_loop_succinct>(density);
140 |     }
141 | #endif
142 | #ifdef __BMI2__
143 |     else if (mode == "builtin_loop_pdep") {
144 |         test<select_modes::builtin_loop_pdep>(density);
145 |     }
146 | #endif
147 | #ifdef __AVX512VL__
148 |     else if (mode == "builtin_avx512_pdep") {
149 |         test<select_modes::builtin_avx512_pdep>(density);
150 |     } else if (mode == "avx2_avx512_pdep") {
151 |         test<select_modes::avx2_avx512_pdep>(density);
152 |     }
153 | #endif
154 |     else {
155 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
156 |         return 1;
157 |     }
158 | 
159 |     return 0;
160 | }
161 | 


--------------------------------------------------------------------------------
/src/perf_popcnt_for_avx_select256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 1000000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | static constexpr std::array<uint64_t, 1> sizes = {
 21 |     1ULL << 8,
 22 | };
 23 | // static constexpr std::array<uint64_t, 25> sizes = {
 24 | //     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 25 | //     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 26 | //     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 27 | //     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 28 | //     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 29 | // };
 30 | 
 31 | template <popcount_modes>
 32 | inline __m256i popcount_256(const uint64_t*) {
 33 |     assert(false);  // should not come
 34 |     return __m256i{};
 35 | }
 36 | template <>
 37 | inline __m256i popcount_256<popcount_modes::builtin>(const uint64_t* x) {
 38 |     static uint64_t cnts[4];
 39 |     cnts[0] = popcount_u64<popcount_modes::builtin>(x[0]);
 40 |     cnts[1] = popcount_u64<popcount_modes::builtin>(x[1]);
 41 |     cnts[2] = popcount_u64<popcount_modes::builtin>(x[2]);
 42 |     cnts[3] = popcount_u64<popcount_modes::builtin>(x[3]);
 43 |     return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(cnts));
 44 | }
 45 | #ifdef __AVX2__
 46 | template <>
 47 | inline __m256i popcount_256<popcount_modes::avx2>(const uint64_t* x) {
 48 |     const __m256i mx = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x));
 49 |     return popcount_m256i(mx);
 50 | }
 51 | #endif
 52 | 
 53 | template <popcount_modes Mode>
 54 | void test(std::string type) {
 55 |     std::vector<uint64_t> bits(sizes.back() / 64);
 56 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 57 |     std::cout << "# number of ones: " << num_ones << " ("
 58 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 59 | 
 60 |     std::string json("{\"type\":\"" + type + "\", ");
 61 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 62 |     json += "\"timings\":[";
 63 | 
 64 |     for (const uint64_t n : sizes) {
 65 |         splitmix64 hasher(query_seed);
 66 | 
 67 |         const auto num_buckets = n / 256;
 68 |         std::vector<const uint64_t*> queries(num_queries);
 69 | 
 70 |         for (uint64_t i = 0; i < num_queries; i++) {
 71 |             queries[i] = &bits[(hasher.next() % num_buckets) * 256 / 64];
 72 |         }
 73 | 
 74 |         essentials::timer_type t;
 75 |         double min = 0.0, max = 0.0, avg = 0.0;
 76 | 
 77 |         auto measure = [&]() {
 78 |             __m256i tmp{};  // to avoid the optimization
 79 |             for (int run = 0; run != runs; run++) {
 80 |                 t.start();
 81 |                 for (uint64_t i = 0; i < num_queries; i++) {
 82 |                     const uint64_t* x = queries[i];
 83 |                     tmp = _mm256_add_epi64(tmp, popcount_256<Mode>(x));
 84 |                 }
 85 |                 t.stop();
 86 |             }
 87 |             uint64_t tmp2[4];
 88 |             _mm256_storeu_si256(reinterpret_cast<__m256i*>(tmp2), tmp);
 89 |             std::cout << "# ignore: " << tmp2[0] << std::endl;
 90 |         };
 91 | 
 92 |         static constexpr int K = 10;
 93 | 
 94 |         // warm-up
 95 |         for (int k = 0; k != K; ++k) {
 96 |             measure();
 97 |             double avg_ns_query = (t.average() * 1000) / num_queries;
 98 |             avg += avg_ns_query;
 99 |             t.reset();
100 |         }
101 |         std::cout << "# warm-up: " << avg / K << std::endl;
102 |         avg = 0.0;
103 | 
104 |         for (int k = 0; k != K; ++k) {
105 |             measure();
106 |             t.discard_max();
107 |             double avg_ns_query = (t.max() * 1000) / num_queries;
108 |             max += avg_ns_query;
109 |             t.reset();
110 |         }
111 | 
112 |         for (int k = 0; k != K; ++k) {
113 |             measure();
114 |             t.discard_min();
115 |             t.discard_max();
116 |             double avg_ns_query = (t.average() * 1000) / num_queries;
117 |             avg += avg_ns_query;
118 |             t.reset();
119 |         }
120 | 
121 |         for (int k = 0; k != K; ++k) {
122 |             measure();
123 |             t.discard_min();
124 |             double avg_ns_query = (t.min() * 1000) / num_queries;
125 |             min += avg_ns_query;
126 |             t.reset();
127 |         }
128 | 
129 |         min /= K;
130 |         max /= K;
131 |         avg /= K;
132 |         std::vector<double> tt{min, avg, max};
133 |         std::sort(tt.begin(), tt.end());
134 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
135 | 
136 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
137 |                 "," + std::to_string(tt[2]) + "],";
138 |     }
139 | 
140 |     json.pop_back();
141 |     json += "]}";
142 |     std::cerr << json << std::endl;
143 | }
144 | 
145 | int main(int argc, char** argv) {
146 |     cmd_line_parser::parser parser(argc, argv);
147 |     parser.add("mode", "Mode of popcnt algorithm.");
148 |     if (!parser.parse()) return 1;
149 | 
150 |     auto mode = parser.get<std::string>("mode");
151 | 
152 |     if (mode == "builtin") {
153 |         test<popcount_modes::builtin>(mode);
154 |     }
155 | #ifdef __AVX2__
156 |     else if (mode == "avx2") {
157 |         test<popcount_modes::avx2>(mode);
158 |     }
159 | #endif
160 |     else {
161 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
162 |         return 1;
163 |     }
164 |     return 0;
165 | }
166 | 


--------------------------------------------------------------------------------
/src/perf_popcnt_for_avx_select512.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 1000000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | static constexpr std::array<uint64_t, 1> sizes = {
 21 |     1ULL << 9,
 22 | };
 23 | // static constexpr std::array<uint64_t, 24> sizes = {
 24 | //     1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12, 1ULL << 13, 1ULL << 14,
 25 | //     1ULL << 15, 1ULL << 16, 1ULL << 17, 1ULL << 18, 1ULL << 19, 1ULL << 20,
 26 | //     1ULL << 21, 1ULL << 22, 1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26,
 27 | //     1ULL << 27, 1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32,
 28 | // };
 29 | 
 30 | template <popcount_modes>
 31 | inline __m512i popcount_512(const uint64_t*) {
 32 |     assert(false);  // should not come
 33 |     return __m512i{};
 34 | }
 35 | template <>
 36 | inline __m512i popcount_512<popcount_modes::builtin>(const uint64_t* x) {
 37 |     static uint64_t cnts[8];
 38 |     cnts[0] = popcount_u64<popcount_modes::builtin>(x[0]);
 39 |     cnts[1] = popcount_u64<popcount_modes::builtin>(x[1]);
 40 |     cnts[2] = popcount_u64<popcount_modes::builtin>(x[2]);
 41 |     cnts[3] = popcount_u64<popcount_modes::builtin>(x[3]);
 42 |     cnts[4] = popcount_u64<popcount_modes::builtin>(x[4]);
 43 |     cnts[5] = popcount_u64<popcount_modes::builtin>(x[5]);
 44 |     cnts[6] = popcount_u64<popcount_modes::builtin>(x[6]);
 45 |     cnts[7] = popcount_u64<popcount_modes::builtin>(x[7]);
 46 |     return _mm512_loadu_si512(reinterpret_cast<const __m512i*>(cnts));
 47 | }
 48 | #ifdef __AVX2__
 49 | template <>
 50 | inline __m512i popcount_512<popcount_modes::avx2>(const uint64_t* x) {
 51 |     const __m512i mx = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(x));
 52 |     return popcount_m512i(mx);
 53 | }
 54 | #endif
 55 | 
 56 | template <popcount_modes Mode>
 57 | void test(std::string type) {
 58 |     std::vector<uint64_t> bits(sizes.back() / 64);
 59 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 60 |     std::cout << "# number of ones: " << num_ones << " ("
 61 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 62 | 
 63 |     std::string json("{\"type\":\"" + type + "\", ");
 64 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 65 |     json += "\"timings\":[";
 66 | 
 67 |     for (const uint64_t n : sizes) {
 68 |         splitmix64 hasher(query_seed);
 69 | 
 70 |         const auto num_buckets = n / 512;
 71 |         std::vector<const uint64_t*> queries(num_queries);
 72 | 
 73 |         for (uint64_t i = 0; i < num_queries; i++) {
 74 |             queries[i] = &bits[(hasher.next() % num_buckets) * 512 / 64];
 75 |         }
 76 | 
 77 |         essentials::timer_type t;
 78 |         double min = 0.0, max = 0.0, avg = 0.0;
 79 | 
 80 |         auto measure = [&]() {
 81 |             __m512i tmp{};  // to avoid the optimization
 82 |             for (int run = 0; run != runs; run++) {
 83 |                 t.start();
 84 |                 for (uint64_t i = 0; i < num_queries; i++) {
 85 |                     const uint64_t* x = queries[i];
 86 |                     tmp = _mm512_add_epi64(tmp, popcount_512<Mode>(x));
 87 |                 }
 88 |                 t.stop();
 89 |             }
 90 |             uint64_t tmp2[8];
 91 |             _mm512_storeu_si512(reinterpret_cast<__m512i*>(tmp2), tmp);
 92 |             std::cout << "# ignore: " << tmp2[0] << std::endl;
 93 |         };
 94 | 
 95 |         static constexpr int K = 10;
 96 | 
 97 |         // warm-up
 98 |         for (int k = 0; k != K; ++k) {
 99 |             measure();
100 |             double avg_ns_query = (t.average() * 1000) / num_queries;
101 |             avg += avg_ns_query;
102 |             t.reset();
103 |         }
104 |         std::cout << "# warm-up: " << avg / K << std::endl;
105 |         avg = 0.0;
106 | 
107 |         for (int k = 0; k != K; ++k) {
108 |             measure();
109 |             t.discard_max();
110 |             double avg_ns_query = (t.max() * 1000) / num_queries;
111 |             max += avg_ns_query;
112 |             t.reset();
113 |         }
114 | 
115 |         for (int k = 0; k != K; ++k) {
116 |             measure();
117 |             t.discard_min();
118 |             t.discard_max();
119 |             double avg_ns_query = (t.average() * 1000) / num_queries;
120 |             avg += avg_ns_query;
121 |             t.reset();
122 |         }
123 | 
124 |         for (int k = 0; k != K; ++k) {
125 |             measure();
126 |             t.discard_min();
127 |             double avg_ns_query = (t.min() * 1000) / num_queries;
128 |             min += avg_ns_query;
129 |             t.reset();
130 |         }
131 | 
132 |         min /= K;
133 |         max /= K;
134 |         avg /= K;
135 |         std::vector<double> tt{min, avg, max};
136 |         std::sort(tt.begin(), tt.end());
137 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
138 | 
139 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
140 |                 "," + std::to_string(tt[2]) + "],";
141 |     }
142 | 
143 |     json.pop_back();
144 |     json += "]}";
145 |     std::cerr << json << std::endl;
146 | }
147 | 
148 | int main(int argc, char** argv) {
149 |     cmd_line_parser::parser parser(argc, argv);
150 |     parser.add("mode", "Mode of popcnt algorithm.");
151 |     if (!parser.parse()) return 1;
152 | 
153 |     auto mode = parser.get<std::string>("mode");
154 | 
155 |     if (mode == "builtin") {
156 |         test<popcount_modes::builtin>(mode);
157 |     }
158 | #ifdef __AVX2__
159 |     else if (mode == "avx2") {
160 |         test<popcount_modes::avx2>(mode);
161 |     }
162 | #endif
163 |     else {
164 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
165 |         return 1;
166 |     }
167 |     return 0;
168 | }
169 | 


--------------------------------------------------------------------------------
/include/segment_tree/avx512/segment_tree.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | #include <cmath>
  5 | #include <vector>
  6 | 
  7 | #include "util.hpp"
  8 | #include "segment_tree_macros.hpp"
  9 | 
 10 | namespace mrs::avx512 {
 11 | 
 12 | template <uint32_t Height = 1>
 13 | struct segment_tree {
 14 |     static_assert(Height > 0 and Height <= 3);
 15 | 
 16 |     static constexpr uint64_t height(const uint64_t n) {
 17 |         if (n <= (1ULL << 9)) return 1;
 18 |         if (n <= (1ULL << 17)) return 2;
 19 |         if (n <= (1ULL << 24)) return 3;
 20 |         return 0;
 21 |     }
 22 | 
 23 |     segment_tree()
 24 |         : m_size(0), m_num_nodes_per_level(nullptr), m_ptr(nullptr) {}
 25 | 
 26 |     void build(uint16_t const* input, uint64_t n) {
 27 |         assert(n > 0 and n <= (1ULL << 24));
 28 |         m_size = n;
 29 |         std::vector<uint32_t> num_nodes_per_level(Height);
 30 |         uint64_t m = n;
 31 |         uint64_t total_bytes = 0;
 32 |         int h = Height;
 33 | 
 34 |         {
 35 |             m = std::ceil(static_cast<double>(m) / node512::fanout);
 36 |             num_nodes_per_level[--h] = m;
 37 |             total_bytes += m * node512::bytes;
 38 |             if (m > 1) {
 39 |                 m = std::ceil(static_cast<double>(m) / node256::fanout);
 40 |                 num_nodes_per_level[--h] = m;
 41 |                 total_bytes += m * node256::bytes;
 42 |             }
 43 |             if (m > 1) {
 44 |                 m = std::ceil(static_cast<double>(m) / node128::fanout);
 45 |                 num_nodes_per_level[--h] = m;
 46 |                 total_bytes += m * node128::bytes;
 47 |             }
 48 |             assert(m == 1);
 49 |         }
 50 | 
 51 |         total_bytes += Height * sizeof(uint32_t);
 52 |         m_data.resize(total_bytes);
 53 | 
 54 |         m_num_nodes_per_level = reinterpret_cast<uint32_t*>(m_data.data());
 55 |         for (uint32_t i = 0; i != Height; ++i) {
 56 |             m_num_nodes_per_level[i] = num_nodes_per_level[i];
 57 |         }
 58 |         m_ptr = m_data.data() + Height * sizeof(uint32_t);
 59 | 
 60 |         uint8_t* begin = m_data.data() + total_bytes;  // end
 61 |         uint64_t step = 1;
 62 |         h = Height;
 63 | 
 64 |         {
 65 |             std::vector<node512::key_type> tmp(node512::fanout);
 66 |             uint64_t nodes = num_nodes_per_level[--h];
 67 |             begin -= nodes * node512::bytes;
 68 |             uint8_t* out = begin;
 69 |             for (uint64_t i = 0, base = 0; i != nodes;
 70 |                  ++i, base += node512::fanout) {
 71 |                 for (uint64_t j = 0; j != node512::fanout and base + j < n;
 72 |                      ++j) {
 73 |                     uint16_t val = input[base + j];
 74 |                     assert(val <= 256);
 75 |                     tmp[j] = val;
 76 |                 }
 77 |                 node512::build(tmp.data(), out);
 78 |                 out += node512::bytes;
 79 |             }
 80 |             step *= node512::fanout;
 81 |         }
 82 | 
 83 |         if (h > 0) {
 84 |             std::vector<node256::key_type> tmp(node256::fanout);
 85 |             uint64_t nodes = num_nodes_per_level[--h];
 86 |             begin -= nodes * node256::bytes;
 87 |             uint8_t* out = begin;
 88 |             for (uint64_t i = 0, base = 0; i != nodes; ++i) {
 89 |                 for (node256::key_type& x : tmp) {
 90 |                     node256::key_type sum = 0;
 91 |                     for (uint64_t k = 0; k != step and base + k < n; ++k) {
 92 |                         sum += input[base + k];
 93 |                     }
 94 |                     x = sum;
 95 |                     base += step;
 96 |                 }
 97 | 
 98 |                 // shift right by 1
 99 |                 for (int i = node256::fanout - 1; i > 0; --i) {
100 |                     tmp[i] = tmp[i - 1];
101 |                 }
102 |                 tmp[0] = 0;
103 | 
104 |                 node256::build(tmp.data(), out);
105 |                 out += node256::bytes;
106 |             }
107 |             step *= node256::fanout;
108 |         }
109 | 
110 |         if (h > 0) {
111 |             std::vector<node128::key_type> tmp(node128::fanout);
112 |             uint64_t nodes = num_nodes_per_level[--h];
113 |             begin -= nodes * node128::bytes;
114 |             uint8_t* out = begin;
115 |             for (uint64_t i = 0, base = 0; i != nodes; ++i) {
116 |                 for (node128::key_type& x : tmp) {
117 |                     node128::key_type sum = 0;
118 |                     for (uint64_t k = 0; k != step and base + k < n; ++k) {
119 |                         sum += input[base + k];
120 |                     }
121 |                     x = sum;
122 |                     base += step;
123 |                 }
124 | 
125 |                 // shift right by 1
126 |                 for (int i = node128::fanout - 1; i > 0; --i) {
127 |                     tmp[i] = tmp[i - 1];
128 |                 }
129 |                 tmp[0] = 0;
130 | 
131 |                 node128::build(tmp.data(), out);
132 |                 out += node128::bytes;
133 |             }
134 |         }
135 | 
136 |         assert(h == 0);
137 |     }
138 | 
139 |     static std::string name() {
140 |         return "avx512::segment_tree";
141 |     }
142 | 
143 |     uint64_t bytes() const {
144 |         return sizeof(m_size) + sizeof(m_num_nodes_per_level) + sizeof(m_ptr) +
145 |                essentials::vec_bytes(m_data);
146 |     }
147 | 
148 |     uint32_t size() const {
149 |         return m_size;
150 |     }
151 | 
152 |     uint64_t sum(uint64_t i) const {
153 |         assert(i < size());
154 |         if constexpr (Height == 1) { DYRS_AVX512_SUM_H1 }
155 |         if constexpr (Height == 2) { DYRS_AVX512_SUM_H2 }
156 |         if constexpr (Height == 3) { DYRS_AVX512_SUM_H3 }
157 |     }
158 | 
159 |     void update(uint64_t i,
160 |                 bool sign  // the sign of the update: 0 for +1, or 1 for -1
161 |     ) {
162 |         assert(i < size());
163 |         if constexpr (Height == 1) { DYRS_AVX512_UPDATE_H1 }
164 |         if constexpr (Height == 2) { DYRS_AVX512_UPDATE_H2 }
165 |         if constexpr (Height == 3) { DYRS_AVX512_UPDATE_H3 }
166 |     }
167 | 
168 |     search_result search(uint64_t x) const {
169 |         if constexpr (Height == 1) { DYRS_AVX512_SEARCH_H1 }
170 |         if constexpr (Height == 2) { DYRS_AVX512_SEARCH_H2 }
171 |         if constexpr (Height == 3) { DYRS_AVX512_SEARCH_H3 }
172 |     }
173 | 
174 | private:
175 |     uint32_t m_size;
176 |     uint32_t* m_num_nodes_per_level;
177 |     uint8_t* m_ptr;
178 |     std::vector<uint8_t> m_data;
179 | };
180 | 
181 | }  // namespace mrs::avx512


--------------------------------------------------------------------------------
/include/hft/fenwick/typef.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef __FENWICK_TYPEF_HPP__
  2 | #define __FENWICK_TYPEF_HPP__
  3 | 
  4 | #include "fenwick_tree.hpp"
  5 | 
  6 | namespace hft::fenwick {
  7 | 
  8 | /**
  9 |  * class TypeF - closer type compression and classical node layout.
 10 |  * @sequence: sequence of integers.
 11 |  * @size: number of elements.
 12 |  * @BOUND: maximum value that @sequence can store.
 13 |  *
 14 |  */
 15 | template <size_t BOUND> class TypeF : public FenwickTree {
 16 | public:
 17 |   static constexpr size_t BOUNDSIZE = ceil_log2_plus1(BOUND);
 18 |   static_assert(BOUNDSIZE >= 1 && BOUNDSIZE <= 64, "Leaves can't be stored in a 64-bit word");
 19 | 
 20 | protected:
 21 |   const size_t Size;
 22 |   DArray<uint8_t> Tree;
 23 | 
 24 | public:
 25 |   TypeF(uint64_t sequence[], size_t size) : Size(size), Tree(pos(size + 1)) {
 26 |     for (size_t i = 1; i <= Size; i++) {
 27 |       const size_t bytepos = pos(i);
 28 | 
 29 |       switch (BOUNDSIZE + rho(i)) {
 30 |       case 17 ... 64:
 31 |         reinterpret_cast<auint64_t &>(Tree[bytepos]) = sequence[i - 1];
 32 |         break;
 33 |       case 9 ... 16:
 34 |         reinterpret_cast<auint16_t &>(Tree[bytepos]) = sequence[i - 1];
 35 |         break;
 36 |       default:
 37 |         reinterpret_cast<auint8_t &>(Tree[bytepos]) = sequence[i - 1];
 38 |       }
 39 |     }
 40 | 
 41 |     for (size_t m = 2; m <= Size; m <<= 1) {
 42 |       for (size_t idx = m; idx <= Size; idx += m) {
 43 |         const size_t left = pos(idx);
 44 |         const size_t right = pos(idx - m / 2);
 45 | 
 46 |         switch (BOUNDSIZE + rho(idx)) {
 47 |         case 17 ... 64:
 48 |           switch (BOUNDSIZE + rho(idx - m / 2)) {
 49 |           case 17 ... 64:
 50 |             reinterpret_cast<auint64_t &>(Tree[left]) +=
 51 |                 *reinterpret_cast<auint64_t *>(&Tree[right]);
 52 |             break;
 53 |           case 9 ... 16:
 54 |             reinterpret_cast<auint64_t &>(Tree[left]) +=
 55 |                 *reinterpret_cast<auint16_t *>(&Tree[right]);
 56 |             break;
 57 |           default:
 58 |             reinterpret_cast<auint64_t &>(Tree[left]) +=
 59 |                 *reinterpret_cast<auint8_t *>(&Tree[right]);
 60 |           }
 61 |           break;
 62 |         case 9 ... 16:
 63 |           switch (BOUNDSIZE + rho(idx - m / 2)) {
 64 |           case 9 ... 16:
 65 |             reinterpret_cast<auint16_t &>(Tree[left]) +=
 66 |                 *reinterpret_cast<auint16_t *>(&Tree[right]);
 67 |             break;
 68 |           default:
 69 |             reinterpret_cast<auint16_t &>(Tree[left]) +=
 70 |                 *reinterpret_cast<auint8_t *>(&Tree[right]);
 71 |           }
 72 |           break;
 73 |         default:
 74 |           reinterpret_cast<auint8_t &>(Tree[left]) +=
 75 |             *reinterpret_cast<auint8_t *>(&Tree[right]);
 76 |         }
 77 |       }
 78 |     }
 79 |   }
 80 | 
 81 |   virtual uint64_t prefix(size_t idx) const {
 82 |     uint64_t sum = 0;
 83 | 
 84 |     while (idx != 0) {
 85 |       const size_t bytepos = pos(idx);
 86 | 
 87 |       switch (BOUNDSIZE + rho(idx)) {
 88 |       case 17 ... 64:
 89 |         sum += *reinterpret_cast<auint64_t *>(&Tree[bytepos]);
 90 |         break;
 91 |       case 9 ... 16:
 92 |         sum += *reinterpret_cast<auint16_t *>(&Tree[bytepos]);
 93 |         break;
 94 |       default:
 95 |         sum += *reinterpret_cast<auint8_t *>(&Tree[bytepos]);
 96 |       }
 97 | 
 98 |       idx = clear_rho(idx);
 99 |     }
100 | 
101 |     return sum;
102 |   }
103 | 
104 |   virtual void add(size_t idx, int64_t inc) {
105 |     while (idx <= Size) {
106 |       const size_t bytepos = pos(idx);
107 | 
108 |       switch (BOUNDSIZE + rho(idx)) {
109 |       case 17 ... 64:
110 |         *reinterpret_cast<auint64_t *>(&Tree[bytepos]) += inc;
111 |         break;
112 |       case 9 ... 16:
113 |         *reinterpret_cast<auint16_t *>(&Tree[bytepos]) += inc;
114 |         break;
115 |       default:
116 |         *reinterpret_cast<auint8_t *>(&Tree[bytepos]) += inc;
117 |       }
118 | 
119 |       idx += mask_rho(idx);
120 |     }
121 |   }
122 | 
123 |   using FenwickTree::find;
124 |   virtual size_t find(uint64_t *val) const {
125 |     size_t node = 0;
126 | 
127 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
128 |       if (node + m > Size)
129 |         continue;
130 | 
131 |       const size_t bytepos = pos(node + m);
132 |       const int bitlen = BOUNDSIZE + rho(node + m);
133 | 
134 |       uint64_t value;
135 |       switch (bitlen) {
136 |       case 17 ... 64:
137 |         value = *reinterpret_cast<auint64_t *>(&Tree[bytepos]);
138 |         break;
139 |       case 9 ... 16:
140 |         value = *reinterpret_cast<auint16_t *>(&Tree[bytepos]);
141 |         break;
142 |       default:
143 |         value = *reinterpret_cast<auint8_t *>(&Tree[bytepos]);
144 |       }
145 | 
146 |       if (*val >= value) {
147 |         node += m;
148 |         *val -= value;
149 |       }
150 |     }
151 | 
152 |     return node;
153 |   }
154 | 
155 |   using FenwickTree::compFind;
156 |   virtual size_t compFind(uint64_t *val) const {
157 |     size_t node = 0;
158 | 
159 |     for (size_t m = mask_lambda(Size); m != 0; m >>= 1) {
160 |       if (node + m > Size)
161 |         continue;
162 | 
163 |       const size_t bytepos = pos(node + m);
164 |       const int height = rho(node + m);
165 | 
166 |       uint64_t value = BOUND << height;
167 |       switch (BOUNDSIZE + height) {
168 |       case 17 ... 64:
169 |         value -= *reinterpret_cast<auint64_t *>(&Tree[bytepos]);
170 |         break;
171 |       case 9 ... 16:
172 |         value -= *reinterpret_cast<auint16_t *>(&Tree[bytepos]);
173 |         break;
174 |       default:
175 |         value -= *reinterpret_cast<auint8_t *>(&Tree[bytepos]);
176 |       }
177 | 
178 |       if (*val >= value) {
179 |         node += m;
180 |         *val -= value;
181 |       }
182 |     }
183 | 
184 |     return node;
185 |   }
186 | 
187 |   virtual size_t size() const { return Size; }
188 | 
189 |   virtual size_t bitCount() const {
190 |     return sizeof(TypeF<BOUNDSIZE>) * 8 + Tree.bitCount() - sizeof(Tree);
191 |   }
192 | 
193 | private:
194 |   inline static size_t pos(size_t idx) {
195 |     idx--;
196 |     return idx + (idx >> (BOUNDSIZE <= 8 ? (8 - BOUNDSIZE + 1) : 0)) +
197 |            (idx >> (BOUNDSIZE <= 16 ? (16 - BOUNDSIZE + 1) : 0)) * 6;
198 |   }
199 | 
200 |   friend std::ostream &operator<<(std::ostream &os, const TypeF<BOUND> &ft) {
201 |     const uint64_t nsize = hton((uint64_t)ft.Size);
202 |     os.write((char *)&nsize, sizeof(uint64_t));
203 | 
204 |     return os << ft.Tree;
205 |   }
206 | 
207 |   friend std::istream &operator>>(std::istream &is, TypeF<BOUND> &ft) {
208 |     uint64_t nsize;
209 |     is.read((char *)(&nsize), sizeof(uint64_t));
210 | 
211 |     ft.Size = ntoh(nsize);
212 |     return is >> ft.Tree;
213 |   }
214 | };
215 | 
216 | } // namespace hft::fenwick
217 | 
218 | #endif // __FENWICK_TYPEF_HPP__
219 | 


--------------------------------------------------------------------------------
/src/perf_popcnt256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | static constexpr std::array<uint64_t, 25> sizes = {
 21 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 22 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 23 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 24 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 25 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32};
 26 | 
 27 | template <popcount_modes>
 28 | inline uint64_t popcount_256(const uint64_t*) {
 29 |     assert(false);  // should not come
 30 |     return UINT64_MAX;
 31 | }
 32 | template <>
 33 | inline uint64_t popcount_256<popcount_modes::broadword>(const uint64_t* x) {
 34 |     uint64_t tmp = 0;
 35 |     tmp += popcount_u64<popcount_modes::broadword>(x[0]);
 36 |     tmp += popcount_u64<popcount_modes::broadword>(x[1]);
 37 |     tmp += popcount_u64<popcount_modes::broadword>(x[2]);
 38 |     tmp += popcount_u64<popcount_modes::broadword>(x[3]);
 39 |     return tmp;
 40 | }
 41 | template <>
 42 | inline uint64_t popcount_256<popcount_modes::builtin>(const uint64_t* x) {
 43 |     uint64_t tmp = 0;
 44 |     tmp += popcount_u64<popcount_modes::builtin>(x[0]);
 45 |     tmp += popcount_u64<popcount_modes::builtin>(x[1]);
 46 |     tmp += popcount_u64<popcount_modes::builtin>(x[2]);
 47 |     tmp += popcount_u64<popcount_modes::builtin>(x[3]);
 48 |     return tmp;
 49 | }
 50 | #ifdef __AVX2__
 51 | template <popcount_modes>
 52 | inline __m256i avx2_popcount_256(const uint64_t*) {
 53 |     assert(false);  // should not come
 54 |     return __m256i{};
 55 | }
 56 | template <>
 57 | inline __m256i avx2_popcount_256<popcount_modes::avx2>(const uint64_t* x) {
 58 |     return popcount_m256i(_mm256_loadu_si256((__m256i const*)x));
 59 | }
 60 | #endif
 61 | 
 62 | template <popcount_modes Mode>
 63 | void test(std::string type) {
 64 |     std::vector<uint64_t> bits(sizes.back() / 64);
 65 |     std::vector<const uint64_t*> queries(num_queries);
 66 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 67 |     std::cout << "# number of ones: " << num_ones << " ("
 68 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 69 | 
 70 |     std::string json("{\"type\":\"" + type + "\", ");
 71 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 72 |     json += "\"timings\":[";
 73 | 
 74 |     for (const uint64_t n : sizes) {
 75 |         splitmix64 hasher(query_seed);
 76 | 
 77 |         const auto num_buckets = n / 256;
 78 |         for (uint64_t i = 0; i != num_queries; ++i) {
 79 |             queries[i] = &bits[(hasher.next() % num_buckets) * 256 / 64];
 80 |         }
 81 | 
 82 |         essentials::timer_type t;
 83 |         double min = 0.0, max = 0.0, avg = 0.0;
 84 | 
 85 |         auto measure = [&]() {
 86 |             if constexpr (Mode != popcount_modes::avx2) {
 87 |                 uint64_t tmp = 0;  // to avoid the optimization
 88 |                 for (int run = 0; run != runs; run++) {
 89 |                     t.start();
 90 |                     for (uint64_t i = 0; i < num_queries; i++) {
 91 |                         tmp += popcount_256<Mode>(queries[i]);
 92 |                     }
 93 |                     t.stop();
 94 |                 }
 95 |                 std::cout << "# ignore: " << tmp << std::endl;
 96 |             } else {
 97 |                 __m256i tmp;  // to avoid the optimization
 98 |                 for (int run = 0; run != runs; run++) {
 99 |                     t.start();
100 |                     for (uint64_t i = 0; i < num_queries; i++) {
101 |                         tmp = avx2_popcount_256<Mode>(queries[i]);
102 |                     }
103 |                     t.stop();
104 |                 }
105 |                 uint64_t tmp2[4];
106 |                 _mm256_storeu_si256(reinterpret_cast<__m256i*>(tmp2), tmp);
107 |                 std::cout << "# ignore: " << tmp2[0] << std::endl;
108 |             }
109 |         };
110 | 
111 |         static constexpr int K = 10;
112 | 
113 |         // warm-up
114 |         for (int k = 0; k != K; ++k) {
115 |             measure();
116 |             double avg_ns_query = (t.average() * 1000) / num_queries;
117 |             avg += avg_ns_query;
118 |             t.reset();
119 |         }
120 |         std::cout << "# warm-up: " << avg / K << std::endl;
121 |         avg = 0.0;
122 | 
123 |         for (int k = 0; k != K; ++k) {
124 |             measure();
125 |             t.discard_max();
126 |             double avg_ns_query = (t.max() * 1000) / num_queries;
127 |             max += avg_ns_query;
128 |             t.reset();
129 |         }
130 | 
131 |         for (int k = 0; k != K; ++k) {
132 |             measure();
133 |             t.discard_min();
134 |             t.discard_max();
135 |             double avg_ns_query = (t.average() * 1000) / num_queries;
136 |             avg += avg_ns_query;
137 |             t.reset();
138 |         }
139 | 
140 |         for (int k = 0; k != K; ++k) {
141 |             measure();
142 |             t.discard_min();
143 |             double avg_ns_query = (t.min() * 1000) / num_queries;
144 |             min += avg_ns_query;
145 |             t.reset();
146 |         }
147 | 
148 |         min /= K;
149 |         max /= K;
150 |         avg /= K;
151 |         std::vector<double> tt{min, avg, max};
152 |         std::sort(tt.begin(), tt.end());
153 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
154 | 
155 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
156 |                 "," + std::to_string(tt[2]) + "],";
157 |     }
158 | 
159 |     json.pop_back();
160 |     json += "]}";
161 |     std::cerr << json << std::endl;
162 | }
163 | 
164 | int main(int argc, char** argv) {
165 |     cmd_line_parser::parser parser(argc, argv);
166 |     parser.add("mode", "Mode of popcnt algorithm.");
167 |     if (!parser.parse()) return 1;
168 | 
169 |     auto mode = parser.get<std::string>("mode");
170 | 
171 |     if (mode == "broadword") {
172 |         test<popcount_modes::broadword>(mode);
173 |     }
174 | #ifdef __SSE4_2__
175 |     else if (mode == "builtin") {
176 |         test<popcount_modes::builtin>(mode);
177 |     }
178 | #endif
179 | #ifdef __AVX2__
180 |     else if (mode == "avx2") {
181 |         test<popcount_modes::avx2>(mode);
182 |     }
183 | #endif
184 |     else {
185 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
186 |         return 1;
187 |     }
188 |     return 0;
189 | }
190 | 


--------------------------------------------------------------------------------
/include/segment_tree/avx2/segment_tree.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | #include <cmath>
  5 | #include <vector>
  6 | 
  7 | #include "util.hpp"
  8 | #include "segment_tree_macros.hpp"
  9 | 
 10 | namespace mrs::avx2 {
 11 | 
 12 | template <uint32_t Height = 1>
 13 | struct segment_tree {
 14 |     static_assert(Height > 0 and Height <= 4);
 15 | 
 16 |     static constexpr uint64_t height(const uint64_t n) {
 17 |         if (n <= (1ULL << 7)) return 1;
 18 |         if (n <= (1ULL << 13)) return 2;
 19 |         if (n <= (1ULL << 19)) return 3;
 20 |         if (n <= (1ULL << 24)) return 4;
 21 |         return 0;
 22 |     }
 23 | 
 24 |     segment_tree()
 25 |         : m_size(0), m_num_nodes_per_level(nullptr), m_ptr(nullptr) {}
 26 | 
 27 |     void build(uint16_t const* input, uint64_t n) {
 28 |         assert(n > 0 and n <= (1ULL << 24));
 29 |         m_size = n;
 30 |         std::vector<uint32_t> num_nodes_per_level(Height);
 31 |         uint64_t m = n;
 32 |         uint64_t total_bytes = 0;
 33 |         int h = Height;
 34 | 
 35 |         {
 36 |             m = std::ceil(static_cast<double>(m) / node128::fanout);
 37 |             num_nodes_per_level[--h] = m;
 38 |             total_bytes += m * node128::bytes;
 39 | 
 40 |             for (int twice = 0; twice != 2; ++twice) {
 41 |                 if (m > 1) {
 42 |                     m = std::ceil(static_cast<double>(m) / node64::fanout);
 43 |                     num_nodes_per_level[--h] = m;
 44 |                     total_bytes += m * node64::bytes;
 45 |                 }
 46 |             }
 47 | 
 48 |             if (m > 1) {
 49 |                 m = std::ceil(static_cast<double>(m) / node32::fanout);
 50 |                 num_nodes_per_level[--h] = m;
 51 |                 total_bytes += m * node32::bytes;
 52 |             }
 53 | 
 54 |             assert(m == 1);
 55 |         }
 56 | 
 57 |         total_bytes += Height * sizeof(uint32_t);
 58 |         m_data.resize(total_bytes);
 59 | 
 60 |         m_num_nodes_per_level = reinterpret_cast<uint32_t*>(m_data.data());
 61 |         for (uint32_t i = 0; i != Height; ++i) {
 62 |             m_num_nodes_per_level[i] = num_nodes_per_level[i];
 63 |         }
 64 |         m_ptr = m_data.data() + Height * sizeof(uint32_t);
 65 | 
 66 |         uint8_t* begin = m_data.data() + total_bytes;  // end
 67 |         uint64_t step = 1;
 68 |         h = Height;
 69 | 
 70 |         {
 71 |             std::vector<node128::key_type> tmp(node128::fanout);
 72 |             uint64_t nodes = num_nodes_per_level[--h];
 73 |             begin -= nodes * node128::bytes;
 74 |             uint8_t* out = begin;
 75 |             for (uint64_t i = 0, base = 0; i != nodes;
 76 |                  ++i, base += node128::fanout) {
 77 |                 for (uint64_t j = 0; j != node128::fanout and base + j < n;
 78 |                      ++j) {
 79 |                     uint16_t val = input[base + j];
 80 |                     assert(val <= 512);
 81 |                     tmp[j] = val;
 82 |                 }
 83 |                 node128::build(tmp.data(), out);
 84 |                 out += node128::bytes;
 85 |             }
 86 |             step *= node128::fanout;
 87 |         }
 88 | 
 89 |         for (int twice = 0; twice != 2; ++twice) {
 90 |             if (h > 0) {
 91 |                 std::vector<node64::key_type> tmp(node64::fanout);
 92 |                 uint64_t nodes = num_nodes_per_level[--h];
 93 |                 begin -= nodes * node64::bytes;
 94 |                 uint8_t* out = begin;
 95 |                 for (uint64_t i = 0, base = 0; i != nodes; ++i) {
 96 |                     for (node64::key_type& x : tmp) {
 97 |                         node64::key_type sum = 0;
 98 |                         for (uint64_t k = 0; k != step and base + k < n; ++k) {
 99 |                             sum += input[base + k];
100 |                         }
101 |                         x = sum;
102 |                         base += step;
103 |                     }
104 | 
105 |                     // shift right by 1
106 |                     for (int i = node64::fanout - 1; i > 0; --i) {
107 |                         tmp[i] = tmp[i - 1];
108 |                     }
109 |                     tmp[0] = 0;
110 | 
111 |                     node64::build(tmp.data(), out);
112 |                     out += node64::bytes;
113 |                 }
114 |                 step *= node64::fanout;
115 |             }
116 |         }
117 | 
118 |         if (h > 0) {
119 |             std::vector<node32::key_type> tmp(node32::fanout);
120 |             uint64_t nodes = num_nodes_per_level[--h];
121 |             begin -= nodes * node32::bytes;
122 |             uint8_t* out = begin;
123 |             for (uint64_t i = 0, base = 0; i != nodes; ++i) {
124 |                 for (node32::key_type& x : tmp) {
125 |                     node32::key_type sum = 0;
126 |                     for (uint64_t k = 0; k != step and base + k < n; ++k) {
127 |                         sum += input[base + k];
128 |                     }
129 |                     x = sum;
130 |                     base += step;
131 |                 }
132 | 
133 |                 // shift right by 1
134 |                 for (int i = node32::fanout - 1; i > 0; --i) {
135 |                     tmp[i] = tmp[i - 1];
136 |                 }
137 |                 tmp[0] = 0;
138 | 
139 |                 node32::build(tmp.data(), out);
140 |                 out += node32::bytes;
141 |             }
142 |         }
143 | 
144 |         assert(h == 0);
145 |     }
146 | 
147 |     static std::string name() {
148 |         return "avx2::segment_tree";
149 |     }
150 | 
151 |     uint64_t bytes() const {
152 |         return sizeof(m_size) + sizeof(m_num_nodes_per_level) + sizeof(m_ptr) +
153 |                essentials::vec_bytes(m_data);
154 |     }
155 | 
156 |     uint32_t size() const {
157 |         return m_size;
158 |     }
159 | 
160 |     uint64_t sum(uint64_t i) const {
161 |         assert(i < size());
162 |         if constexpr (Height == 1) { DYRS_AVX2_SUM_H1 }
163 |         if constexpr (Height == 2) { DYRS_AVX2_SUM_H2 }
164 |         if constexpr (Height == 3) { DYRS_AVX2_SUM_H3 }
165 |         if constexpr (Height == 4) { DYRS_AVX2_SUM_H4 }
166 |     }
167 | 
168 |     void update(uint64_t i,
169 |                 bool sign  // the sign of the update: 0 for +1, or 1 for -1
170 |     ) {
171 |         assert(i < size());
172 |         if constexpr (Height == 1) { DYRS_AVX2_UPDATE_H1 }
173 |         if constexpr (Height == 2) { DYRS_AVX2_UPDATE_H2 }
174 |         if constexpr (Height == 3) { DYRS_AVX2_UPDATE_H3 }
175 |         if constexpr (Height == 4) { DYRS_AVX2_UPDATE_H4 }
176 |     }
177 | 
178 |     search_result search(uint64_t x) const {
179 |         if constexpr (Height == 1) { DYRS_AVX2_SEARCH_H1 }
180 |         if constexpr (Height == 2) { DYRS_AVX2_SEARCH_H2 }
181 |         if constexpr (Height == 3) { DYRS_AVX2_SEARCH_H3 }
182 |         if constexpr (Height == 4) { DYRS_AVX2_SEARCH_H4 }
183 |     }
184 | 
185 | private:
186 |     uint32_t m_size;
187 |     uint32_t* m_num_nodes_per_level;
188 |     uint8_t* m_ptr;
189 |     std::vector<uint8_t> m_data;
190 | };
191 | 
192 | }  // namespace mrs::avx2


--------------------------------------------------------------------------------
/src/perf_prefix_sum256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | static constexpr std::array<uint64_t, 25> sizes = {
 21 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 22 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 23 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 24 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 25 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32};
 26 | 
 27 | template <prefixsum_modes>
 28 | inline uint64_t prefixsum_256(const uint64_t*, uint64_t) {
 29 |     assert(false);  // should not come
 30 |     return UINT64_MAX;
 31 | }
 32 | template <>
 33 | inline uint64_t prefixsum_256<prefixsum_modes::loop>(const uint64_t* x,
 34 |                                                      uint64_t k) {
 35 |     uint64_t sum = 0;
 36 |     for (uint64_t i = 0; i <= k; i++) { sum += x[i]; }
 37 |     return sum;
 38 | }
 39 | template <>
 40 | inline uint64_t prefixsum_256<prefixsum_modes::unrolled>(const uint64_t* x,
 41 |                                                          uint64_t k) {
 42 |     static uint64_t sums[4] = {};
 43 |     sums[0] = x[0];
 44 |     sums[1] = sums[0] + x[1];
 45 |     sums[2] = sums[1] + x[2];
 46 |     sums[3] = sums[2] + x[3];
 47 |     return sums[k];
 48 | }
 49 | #ifdef __AVX512VL__
 50 | template <prefixsum_modes>
 51 | inline uint64_t prefixsum_256(__m256i, uint64_t) {
 52 |     assert(false);  // should not come
 53 |     return UINT64_MAX;
 54 | }
 55 | template <>
 56 | inline uint64_t prefixsum_256<prefixsum_modes::parallel>(uint64_t const* x,
 57 |                                                          uint64_t k) {
 58 |     // volatile const __m256i y = prefixsum_m256i(x);
 59 |     // volatile const uint64_t* C = reinterpret_cast<volatile uint64_t
 60 |     // const*>(&y); return C[k];
 61 |     static uint64_t sums[4] = {};
 62 |     _mm256_storeu_si256((__m256i*)(sums),
 63 |                         prefixsum_m256i(_mm256_loadu_si256((__m256i const*)x)));
 64 |     return sums[k];
 65 | }
 66 | #endif
 67 | 
 68 | template <prefixsum_modes Mode>
 69 | void test(std::string type) {
 70 |     std::vector<uint64_t> bits(sizes.back() / 64);
 71 |     std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 72 | 
 73 |     auto num_ones = create_random_bits(bits, UINT64_MAX * density, bits_seed);
 74 |     std::cout << "# number of ones: " << num_ones << " ("
 75 |               << num_ones / double(sizes.back()) << ")" << std::endl;
 76 | 
 77 |     std::string json("{\"type\":\"" + type + "\", ");
 78 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 79 |     json += "\"timings\":[";
 80 | 
 81 |     for (const uint64_t n : sizes) {
 82 |         splitmix64 hasher(query_seed);
 83 | 
 84 |         const auto num_buckets = n / 256;
 85 |         for (uint64_t i = 0; i != num_queries; ++i) {
 86 |             const uint64_t* x = &bits[(hasher.next() % num_buckets) * 256 / 64];
 87 |             queries[i] = {x, hasher.next() % 4};
 88 |         }
 89 | 
 90 |         essentials::timer_type t;
 91 |         double min = 0.0, max = 0.0, avg = 0.0;
 92 | 
 93 |         auto measure = [&]() {
 94 |             uint64_t tmp = 0;  // to avoid the optimization
 95 |             if constexpr (Mode != prefixsum_modes::parallel) {
 96 |                 for (int run = 0; run != runs; run++) {
 97 |                     t.start();
 98 |                     for (uint64_t i = 0; i < num_queries; i++) {
 99 |                         const uint64_t* x = queries[i].first;
100 |                         const uint64_t k = queries[i].second;
101 |                         tmp += prefixsum_256<Mode>(x, k);
102 |                     }
103 |                     t.stop();
104 |                 }
105 |             } else {
106 |                 for (int run = 0; run != runs; run++) {
107 |                     t.start();
108 |                     for (uint64_t i = 0; i < num_queries; i++) {
109 |                         const uint64_t* x = queries[i].first;
110 |                         const uint64_t k = queries[i].second;
111 |                         tmp += prefixsum_256<Mode>(x, k);
112 |                     }
113 |                     t.stop();
114 |                 }
115 |             }
116 |             std::cout << "# ignore: " << tmp << std::endl;
117 |         };
118 | 
119 |         static constexpr int K = 10;
120 | 
121 |         // warm-up
122 |         for (int k = 0; k != K; ++k) {
123 |             measure();
124 |             double avg_ns_query = (t.average() * 1000) / num_queries;
125 |             avg += avg_ns_query;
126 |             t.reset();
127 |         }
128 |         std::cout << "# warm-up: " << avg / K << std::endl;
129 |         avg = 0.0;
130 | 
131 |         for (int k = 0; k != K; ++k) {
132 |             measure();
133 |             t.discard_max();
134 |             double avg_ns_query = (t.max() * 1000) / num_queries;
135 |             max += avg_ns_query;
136 |             t.reset();
137 |         }
138 | 
139 |         for (int k = 0; k != K; ++k) {
140 |             measure();
141 |             t.discard_min();
142 |             t.discard_max();
143 |             double avg_ns_query = (t.average() * 1000) / num_queries;
144 |             avg += avg_ns_query;
145 |             t.reset();
146 |         }
147 | 
148 |         for (int k = 0; k != K; ++k) {
149 |             measure();
150 |             t.discard_min();
151 |             double avg_ns_query = (t.min() * 1000) / num_queries;
152 |             min += avg_ns_query;
153 |             t.reset();
154 |         }
155 | 
156 |         min /= K;
157 |         max /= K;
158 |         avg /= K;
159 |         std::vector<double> tt{min, avg, max};
160 |         std::sort(tt.begin(), tt.end());
161 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
162 | 
163 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
164 |                 "," + std::to_string(tt[2]) + "],";
165 |     }
166 | 
167 |     json.pop_back();
168 |     json += "]}";
169 |     std::cerr << json << std::endl;
170 | }
171 | 
172 | int main(int argc, char** argv) {
173 |     cmd_line_parser::parser parser(argc, argv);
174 |     parser.add("mode", "Mode of perfixsum algorithm.");
175 |     if (!parser.parse()) return 1;
176 | 
177 |     auto mode = parser.get<std::string>("mode");
178 | 
179 |     if (mode == "loop") {
180 |         test<prefixsum_modes::loop>(mode);
181 |     } else if (mode == "unrolled") {
182 |         test<prefixsum_modes::unrolled>(mode);
183 |     }
184 | #ifdef __AVX512VL__
185 |     else if (mode == "parallel") {
186 |         test<prefixsum_modes::parallel>(mode);
187 |     }
188 | #endif
189 |     else {
190 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
191 |         return 1;
192 |     }
193 | 
194 |     return 0;
195 | }
196 | 


--------------------------------------------------------------------------------
/src/perf_search512.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | // static constexpr std::array<uint64_t, 1> sizes = {
 21 | //     1ULL << 9,
 22 | // };
 23 | static constexpr std::array<uint64_t, 24> sizes = {
 24 |     1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12, 1ULL << 13, 1ULL << 14,
 25 |     1ULL << 15, 1ULL << 16, 1ULL << 17, 1ULL << 18, 1ULL << 19, 1ULL << 20,
 26 |     1ULL << 21, 1ULL << 22, 1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26,
 27 |     1ULL << 27, 1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32};
 28 | 
 29 | template <search_modes>
 30 | inline uint64_t search_512(const uint64_t*, uint64_t) {
 31 |     assert(false);  // should not come
 32 |     return UINT64_MAX;
 33 | }
 34 | template <>
 35 | inline uint64_t search_512<search_modes::loop>(const uint64_t* x, uint64_t k) {
 36 |     uint64_t i = 0, cnt = 0;
 37 |     while (i < 8) {
 38 |         cnt = x[i];
 39 |         if (k < cnt) { break; }
 40 |         k -= cnt;
 41 |         i++;
 42 |     }
 43 |     assert(k < cnt);
 44 |     return k;
 45 | }
 46 | #ifdef __AVX512VL__
 47 | template <search_modes>
 48 | inline uint64_t search_512(__m512i, uint64_t) {
 49 |     assert(false);  // should not come
 50 |     return UINT64_MAX;
 51 | }
 52 | template <>
 53 | inline uint64_t search_512<search_modes::avx512>(uint64_t const* x,
 54 |                                                  uint64_t k) {
 55 |     const __m512i msums =
 56 |         prefixsum_m512i(_mm512_loadu_si512((__m512i const*)x));
 57 |     const __m512i mk = _mm512_set_epi64(k, k, k, k, k, k, k, k);
 58 |     const __mmask8 mask = _mm512_cmple_epi64_mask(msums, mk);  // 1 if mc <= mk
 59 |     const uint8_t i = lt_cnt[mask];
 60 |     static uint64_t sums[9] = {0ULL};  // the head element is a sentinel
 61 |     _mm512_storeu_si512(reinterpret_cast<__m512i*>(sums + 1), msums);
 62 |     return k - sums[i];
 63 | }
 64 | #endif
 65 | 
 66 | template <search_modes Mode>
 67 | void test(std::string type) {
 68 |     std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 69 | 
 70 |     std::string json("{\"type\":\"" + type + "\", ");
 71 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 72 |     json += "\"timings\":[";
 73 | 
 74 |     std::vector<uint64_t> cnts(sizes.back() / 64);  // popcnts for each word
 75 |     {
 76 |         std::vector<uint64_t> bits(sizes.back() / 64);
 77 |         auto num_ones =
 78 |             create_random_bits(bits, UINT64_MAX * density, bits_seed);
 79 |         std::cout << "# number of ones: " << num_ones << " ("
 80 |                   << num_ones / double(sizes.back()) << ")" << std::endl;
 81 |         for (uint64_t i = 0; i < cnts.size(); i++) {
 82 |             cnts[i] = popcount_u64<popcount_modes::builtin>(bits[i]);
 83 |         }
 84 |     }
 85 | 
 86 |     for (const uint64_t n : sizes) {
 87 |         splitmix64 hasher(query_seed);
 88 | 
 89 |         const auto num_buckets = n / 512;
 90 |         for (uint64_t i = 0; i < num_queries; i++) {
 91 |             const uint64_t* x = &cnts[(hasher.next() % num_buckets) * 512 / 64];
 92 |             const uint64_t r = std::accumulate(x, x + 8, 0ULL);
 93 |             queries[i] = {x, hasher.next() % r};
 94 |         }
 95 | 
 96 |         essentials::timer_type t;
 97 |         double min = 0.0, max = 0.0, avg = 0.0;
 98 | 
 99 |         auto measure = [&]() {
100 |             uint64_t tmp = 0;  // to avoid the optimization
101 |             if constexpr (Mode != search_modes::avx512) {
102 |                 for (int run = 0; run != runs; run++) {
103 |                     t.start();
104 |                     for (uint64_t i = 0; i < num_queries; i++) {
105 |                         const uint64_t* x = queries[i].first;
106 |                         const uint64_t k = queries[i].second;
107 |                         tmp += search_512<Mode>(x, k);
108 |                     }
109 |                     t.stop();
110 |                 }
111 |             } else {
112 |                 for (int run = 0; run != runs; run++) {
113 |                     t.start();
114 |                     for (uint64_t i = 0; i < num_queries; i++) {
115 |                         const uint64_t* x = queries[i].first;
116 |                         const uint64_t k = queries[i].second;
117 |                         tmp += search_512<Mode>(x, k);
118 |                     }
119 |                     t.stop();
120 |                 }
121 |             }
122 |             std::cout << "# ignore: " << tmp << std::endl;
123 |         };
124 | 
125 |         static constexpr int K = 10;
126 | 
127 |         // warm-up
128 |         for (int k = 0; k != K; ++k) {
129 |             measure();
130 |             double avg_ns_query = (t.average() * 1000) / num_queries;
131 |             avg += avg_ns_query;
132 |             t.reset();
133 |         }
134 |         std::cout << "# warm-up: " << avg / K << std::endl;
135 |         avg = 0.0;
136 | 
137 |         for (int k = 0; k != K; ++k) {
138 |             measure();
139 |             t.discard_max();
140 |             double avg_ns_query = (t.max() * 1000) / num_queries;
141 |             max += avg_ns_query;
142 |             t.reset();
143 |         }
144 | 
145 |         for (int k = 0; k != K; ++k) {
146 |             measure();
147 |             t.discard_min();
148 |             t.discard_max();
149 |             double avg_ns_query = (t.average() * 1000) / num_queries;
150 |             avg += avg_ns_query;
151 |             t.reset();
152 |         }
153 | 
154 |         for (int k = 0; k != K; ++k) {
155 |             measure();
156 |             t.discard_min();
157 |             double avg_ns_query = (t.min() * 1000) / num_queries;
158 |             min += avg_ns_query;
159 |             t.reset();
160 |         }
161 | 
162 |         min /= K;
163 |         max /= K;
164 |         avg /= K;
165 |         std::vector<double> tt{min, avg, max};
166 |         std::sort(tt.begin(), tt.end());
167 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
168 | 
169 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
170 |                 "," + std::to_string(tt[2]) + "],";
171 |     }
172 | 
173 |     json.pop_back();
174 |     json += "]}";
175 |     std::cerr << json << std::endl;
176 | }
177 | 
178 | int main(int argc, char** argv) {
179 |     cmd_line_parser::parser parser(argc, argv);
180 |     parser.add("mode", "Mode of search algorithm.");
181 |     if (!parser.parse()) return 1;
182 | 
183 |     auto mode = parser.get<std::string>("mode");
184 | 
185 |     if (mode == "loop") {
186 |         test<search_modes::loop>(mode);
187 |     }
188 | #ifdef __AVX512VL__
189 |     else if (mode == "avx512") {
190 |         test<search_modes::avx512>(mode);
191 |     }
192 | #endif
193 |     else {
194 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
195 |         return 1;
196 |     }
197 | 
198 |     return 0;
199 | }
200 | 


--------------------------------------------------------------------------------
/src/perf_search256.cpp:
--------------------------------------------------------------------------------
  1 | #include <vector>
  2 | #include <iostream>
  3 | #include <array>
  4 | 
  5 | #include "../external/essentials/include/essentials.hpp"
  6 | #include "../external/cmd_line_parser/include/parser.hpp"
  7 | 
  8 | #include "util.hpp"
  9 | #include "rank_select_algorithms/rank.hpp"
 10 | #include "rank_select_algorithms/select.hpp"
 11 | 
 12 | using namespace mrs;
 13 | 
 14 | static constexpr int runs = 100;
 15 | static constexpr uint32_t num_queries = 10000;
 16 | static constexpr uint64_t bits_seed = 13;
 17 | static constexpr uint64_t query_seed = 71;
 18 | static constexpr double density = 0.3;
 19 | 
 20 | // static constexpr std::array<uint64_t, 1> sizes = {
 21 | //     1ULL << 8,
 22 | // };
 23 | static constexpr std::array<uint64_t, 25> sizes = {
 24 |     1ULL << 8,  1ULL << 9,  1ULL << 10, 1ULL << 11, 1ULL << 12,
 25 |     1ULL << 13, 1ULL << 14, 1ULL << 15, 1ULL << 16, 1ULL << 17,
 26 |     1ULL << 18, 1ULL << 19, 1ULL << 20, 1ULL << 21, 1ULL << 22,
 27 |     1ULL << 23, 1ULL << 24, 1ULL << 25, 1ULL << 26, 1ULL << 27,
 28 |     1ULL << 28, 1ULL << 29, 1ULL << 30, 1ULL << 31, 1ULL << 32};
 29 | 
 30 | template <search_modes>
 31 | inline uint64_t search_256(const uint64_t*, uint64_t) {
 32 |     assert(false);  // should not come
 33 |     return UINT64_MAX;
 34 | }
 35 | template <>
 36 | inline uint64_t search_256<search_modes::loop>(const uint64_t* x, uint64_t k) {
 37 |     uint64_t i = 0, cnt = 0;
 38 |     while (i < 4) {
 39 |         cnt = x[i];
 40 |         if (k < cnt) { break; }
 41 |         k -= cnt;
 42 |         i++;
 43 |     }
 44 |     assert(k < cnt);
 45 |     return k;
 46 | }
 47 | #ifdef __AVX512VL__
 48 | template <search_modes>
 49 | inline uint64_t search_256(__m256i, uint64_t) {
 50 |     assert(false);  // should not come
 51 |     return UINT64_MAX;
 52 | }
 53 | template <>
 54 | inline uint64_t search_256<search_modes::avx512>(uint64_t const* x,
 55 |                                                  uint64_t k) {
 56 |     const __m256i msums =
 57 |         prefixsum_m256i(_mm256_loadu_si256((__m256i const*)x));
 58 |     const __m256i mk = _mm256_set_epi64x(k, k, k, k);
 59 |     const __mmask8 mask = _mm256_cmple_epi64_mask(msums, mk);  // 1 if mc <= mk
 60 |     const uint8_t i = lt_cnt[mask];
 61 | 
 62 |     static uint64_t sums[5] = {0ULL};  // the head element is a sentinel
 63 |     _mm256_storeu_si256(reinterpret_cast<__m256i*>(sums + 1), msums);
 64 |     return k - sums[i];
 65 | }
 66 | #endif
 67 | 
 68 | template <search_modes Mode>
 69 | void test(std::string type) {
 70 |     std::vector<std::pair<const uint64_t*, uint64_t>> queries(num_queries);
 71 | 
 72 |     std::string json("{\"type\":\"" + type + "\", ");
 73 |     json += "\"density\":\"" + std::to_string(density) + "\", ";
 74 |     json += "\"timings\":[";
 75 | 
 76 |     std::vector<uint64_t> cnts(sizes.back() / 64);  // popcnts for each word
 77 |     {
 78 |         std::vector<uint64_t> bits(sizes.back() / 64);
 79 |         auto num_ones =
 80 |             create_random_bits(bits, UINT64_MAX * density, bits_seed);
 81 |         std::cout << "# number of ones: " << num_ones << " ("
 82 |                   << num_ones / double(sizes.back()) << ")" << std::endl;
 83 |         for (uint64_t i = 0; i < cnts.size(); i++) {
 84 |             cnts[i] = popcount_u64<popcount_modes::builtin>(bits[i]);
 85 |         }
 86 |     }
 87 | 
 88 |     for (const uint64_t n : sizes) {
 89 |         splitmix64 hasher(query_seed);
 90 | 
 91 |         const auto num_buckets = n / 256;
 92 |         for (uint64_t i = 0; i < num_queries; i++) {
 93 |             const uint64_t* x = &cnts[(hasher.next() % num_buckets) * 256 / 64];
 94 |             const uint64_t r = std::accumulate(x, x + 4, 0ULL);
 95 |             queries[i] = {x, hasher.next() % r};
 96 |         }
 97 | 
 98 |         essentials::timer_type t;
 99 |         double min = 0.0, max = 0.0, avg = 0.0;
100 | 
101 |         auto measure = [&]() {
102 |             uint64_t tmp = 0;  // to avoid the optimization
103 |             if constexpr (Mode != search_modes::avx512) {
104 |                 for (int run = 0; run != runs; run++) {
105 |                     t.start();
106 |                     for (uint64_t i = 0; i < num_queries; i++) {
107 |                         const uint64_t* x = queries[i].first;
108 |                         const uint64_t k = queries[i].second;
109 |                         tmp += search_256<Mode>(x, k);
110 |                     }
111 |                     t.stop();
112 |                 }
113 |             } else {
114 |                 for (int run = 0; run != runs; run++) {
115 |                     t.start();
116 |                     for (uint64_t i = 0; i < num_queries; i++) {
117 |                         const uint64_t* x = queries[i].first;
118 |                         const uint64_t k = queries[i].second;
119 |                         tmp += search_256<Mode>(x, k);
120 |                     }
121 |                     t.stop();
122 |                 }
123 |             }
124 |             std::cout << "# ignore: " << tmp << std::endl;
125 |         };
126 | 
127 |         static constexpr int K = 10;
128 | 
129 |         // warm-up
130 |         for (int k = 0; k != K; ++k) {
131 |             measure();
132 |             double avg_ns_query = (t.average() * 1000) / num_queries;
133 |             avg += avg_ns_query;
134 |             t.reset();
135 |         }
136 |         std::cout << "# warm-up: " << avg / K << std::endl;
137 |         avg = 0.0;
138 | 
139 |         for (int k = 0; k != K; ++k) {
140 |             measure();
141 |             t.discard_max();
142 |             double avg_ns_query = (t.max() * 1000) / num_queries;
143 |             max += avg_ns_query;
144 |             t.reset();
145 |         }
146 | 
147 |         for (int k = 0; k != K; ++k) {
148 |             measure();
149 |             t.discard_min();
150 |             t.discard_max();
151 |             double avg_ns_query = (t.average() * 1000) / num_queries;
152 |             avg += avg_ns_query;
153 |             t.reset();
154 |         }
155 | 
156 |         for (int k = 0; k != K; ++k) {
157 |             measure();
158 |             t.discard_min();
159 |             double avg_ns_query = (t.min() * 1000) / num_queries;
160 |             min += avg_ns_query;
161 |             t.reset();
162 |         }
163 | 
164 |         min /= K;
165 |         max /= K;
166 |         avg /= K;
167 |         std::vector<double> tt{min, avg, max};
168 |         std::sort(tt.begin(), tt.end());
169 |         std::cout << "[" << tt[0] << "," << tt[1] << "," << tt[2] << "]\n";
170 | 
171 |         json += "[" + std::to_string(tt[0]) + "," + std::to_string(tt[1]) +
172 |                 "," + std::to_string(tt[2]) + "],";
173 |     }
174 | 
175 |     json.pop_back();
176 |     json += "]}";
177 |     std::cerr << json << std::endl;
178 | }
179 | 
180 | int main(int argc, char** argv) {
181 |     cmd_line_parser::parser parser(argc, argv);
182 |     parser.add("mode", "Mode of search algorithm.");
183 |     if (!parser.parse()) return 1;
184 | 
185 |     auto mode = parser.get<std::string>("mode");
186 | 
187 |     if (mode == "loop") {
188 |         test<search_modes::loop>(mode);
189 |     }
190 | #ifdef __AVX512VL__
191 |     else if (mode == "avx512") {
192 |         test<search_modes::avx512>(mode);
193 |     }
194 | #endif
195 |     else {
196 |         std::cout << "unknown mode \"" << mode << "\"" << std::endl;
197 |         return 1;
198 |     }
199 | 
200 |     return 0;
201 | }
202 | 


--------------------------------------------------------------------------------