├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── bench ├── CMakeLists.txt ├── benchmark_hash_map.hpp ├── benchmarks │ └── uint32_uint32 │ │ ├── erase_exists.hpp │ │ ├── find_exists.hpp │ │ ├── find_modify_eq.hpp │ │ ├── insert_or_assigns.hpp │ │ ├── inserts.hpp │ │ ├── random.hpp │ │ └── utils.hpp └── benchmarks_main.cpp ├── draw_bench.ipynb ├── enviroment.yml ├── include ├── common │ ├── atomic_buffer.hpp │ ├── bucket.hpp │ ├── bucket_container.hpp │ ├── data_storage.hpp │ ├── lock_container.hpp │ ├── seqlock.hpp │ └── utils.hpp └── hash_maps │ ├── cuckoo │ ├── cuckoo_bucket.hpp │ ├── cuckoo_bucket_container.hpp │ ├── cuckoohash_config.hpp │ ├── cuckoohash_map.hpp │ └── cuckoohash_util.hpp │ └── rh │ ├── rh_bucket.hpp │ ├── rh_bucket_container.hpp │ ├── rhhash_config.hpp │ └── rhhash_map.hpp ├── scripts ├── init_cmake.sh ├── run_bench.sh ├── run_stress_checked.sh ├── run_stress_unchecked.sh └── run_unit.sh └── tests └── cuckoo ├── CMakeLists.txt ├── stress_tests ├── CMakeLists.txt ├── stress_checked.cpp └── stress_unchecked.cpp ├── test_utils.hpp └── unit_tests ├── CMakeLists.txt ├── test_bucket_container.cpp ├── test_constructor.cpp ├── test_hash_properties.cpp ├── test_heterogeneous_compare.cpp ├── test_iterator.cpp ├── test_locked_table.cpp ├── test_maximum_hashpower.cpp ├── test_minimum_load_factor.cpp ├── test_resize.cpp ├── test_user_exceptions.cpp ├── unit_test_util.cpp └── unit_test_util.hpp /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .cache 3 | .vscode 4 | .clangd 5 | bench_out.json 6 | bench_plot.png 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set (CMAKE_C_COMPILER "clang") 4 | set (CMAKE_CXX_COMPILER "clang++") 5 | 6 | project (seqlock_concurrent_hash_map LANGUAGES CXX) 7 | 8 | set (CMAKE_EXPORT_COMPILE_COMMANDS ON) 9 | 10 | set (CMAKE_CXX_STANDARD 20) 11 | 12 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") 13 | 14 | if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++ -pthread") 16 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libstdc++") 17 | endif() 18 | 19 | set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Ofast") 20 | 21 | option(ENABLE_SANITIZERS "Enable sanitizers" OFF) 22 | if(ENABLE_SANITIZERS) 23 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fsanitize=thread -fno-sanitize-recover=all -gdwarf-4") 24 | set(CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fsanitize=thread") 25 | endif() 26 | 27 | include_directories(PUBLIC include/common include/hash_maps) 28 | 29 | add_subdirectory(bench) 30 | 31 | enable_testing() 32 | add_subdirectory(tests/cuckoo) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2023, Artyom Davydov 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Hash tables with seqlock-based synchronization 2 | ========= 3 | ## Overview 4 | This repository contains open-addressed concurrent hash tables with seqlock-based synchronization, tests for it, and mini benchmark framework for creation of custom benchmarks for concurrent hash tables. 5 | 6 | ## Hash tables 7 | Seqlock has been integrated into the libcuckoo hash table. New cuckoo hash map algorithm provides much better scalability for reads than original algorithm, without overhead on writes.\ 8 | Robin hood concurrent hash table with seqlock was implemented from scratch, so it has poor interface.\ 9 | Correctes idea was inspired by ["Can Seqlocks Get Along With Programming Language Memory Models?"](https://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf) paper 10 | 11 | ## Install dependencies 12 | Using conda: 13 | ```bash 14 | conda env create -f environment.yml 15 | conda activate seqlock_hash_map 16 | ``` 17 | ## Build & Run 18 | ```bash 19 | scripts/init_cmake.sh 20 | 21 | # run benchmarks 22 | scripts/run_bench.sh 23 | 24 | # run tests 25 | scripts/cuckoo/run_unit.sh 26 | scripts/cuckoo/run_stress_checked.sh 27 | scripts/cuckoo/run_stress_unchecked.sh 28 | ``` 29 | 30 | ## Plot results 31 | Check benchmark results with draw_bench.ipynb 32 | -------------------------------------------------------------------------------- /bench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(bench benchmarks_main.cpp) 2 | 3 | include(FetchContent) 4 | FetchContent_Declare( 5 | googletest 6 | GIT_REPOSITORY https://github.com/google/googletest.git 7 | GIT_TAG release-1.12.1 8 | ) 9 | FetchContent_Declare( 10 | googlebenchmark 11 | GIT_REPOSITORY https://github.com/google/benchmark.git 12 | GIT_TAG v1.7.1 13 | ) 14 | FetchContent_Declare( 15 | libcuckoo 16 | GIT_REPOSITORY https://github.com/efficient/libcuckoo.git 17 | GIT_TAG v0.3.1 18 | ) 19 | 20 | FetchContent_MakeAvailable( 21 | googletest 22 | googlebenchmark 23 | libcuckoo 24 | ) 25 | 26 | target_link_libraries(bench PRIVATE libcuckoo benchmark::benchmark) 27 | target_include_directories(bench PRIVATE include) 28 | target_compile_definitions(bench PRIVATE 29 | UIN32_UINT32_BENCHMARK_ERASE_EXISTS 30 | UIN32_UINT32_BENCHMARK_FIND_EXISTS 31 | UIN32_UINT32_BENCHMARK_FIND_MODIFY_EQ 32 | UIN32_UINT32_BENCHMARK_INSERT_OR_ASSIGNS 33 | UIN32_UINT32_BENCHMARK_INSERTS 34 | UIN32_UINT32_BENCHMARK_RANDOM 35 | ) 36 | -------------------------------------------------------------------------------- /bench/benchmark_hash_map.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "cuckoo/cuckoohash_map.hpp" 12 | #include "libcuckoo/cuckoohash_map.hh" 13 | #include "rh/rhhash_map.hpp" 14 | 15 | namespace OperationsInfo { 16 | enum class Types { 17 | FIND, 18 | INSERT, 19 | INSERT_OR_ASSIGN, 20 | ERASE, 21 | }; 22 | 23 | template 24 | struct Find { 25 | Key key; 26 | Find(const Key& key) 27 | : key(key) {}; 28 | }; 29 | 30 | template 31 | struct Insert { 32 | Key key; 33 | Value value; 34 | Insert(const Key& key, const Value& value) 35 | : key(key), value(value) {}; 36 | }; 37 | 38 | template 39 | struct InsertOrAssign { 40 | Key key; 41 | Value value; 42 | InsertOrAssign(const Key& key, const Value& value) 43 | : key(key), value(value) {}; 44 | }; 45 | 46 | template 47 | struct Erase { 48 | Key key; 49 | Erase(const Key& key) 50 | : key(key) {}; 51 | }; 52 | } 53 | 54 | using RawScenario = std::vector; 55 | 56 | template 57 | using Operation = std::variant< 58 | typename OperationsInfo::Find, 59 | typename OperationsInfo::Insert, 60 | typename OperationsInfo::InsertOrAssign, 61 | typename OperationsInfo::Erase>; 62 | 63 | template 64 | using Scenario = std::vector>; 65 | 66 | template 67 | class OperationGenerator { 68 | std::function key_gen; 69 | std::function value_gen; 70 | public: 71 | OperationGenerator(const std::function& key_gen, const std::function& value_gen) 72 | : key_gen(key_gen), value_gen(value_gen) {} 73 | 74 | Operation operator()(const typename OperationsInfo::Types& type) const { 75 | switch (type) { 76 | case OperationsInfo::Types::FIND: 77 | return Operation( 78 | std::in_place_type>, 79 | key_gen()); 80 | case OperationsInfo::Types::INSERT: 81 | return Operation( 82 | std::in_place_type>, 83 | key_gen(), value_gen()); 84 | case OperationsInfo::Types::INSERT_OR_ASSIGN: 85 | return Operation( 86 | std::in_place_type>, 87 | key_gen(), value_gen()); 88 | case OperationsInfo::Types::ERASE: 89 | return Operation( 90 | std::in_place_type>, 91 | key_gen()); 92 | default: 93 | assert(false); 94 | } 95 | } 96 | }; 97 | 98 | template typename Map> 99 | void do_operation(Map& map, const Operation& operation) { 100 | switch (operation.index()) { 101 | case 0: { 102 | const auto& find_info = std::get>(operation); 103 | Value value; 104 | map.find(find_info.key, value); 105 | break; 106 | } 107 | case 1: { 108 | auto& insert_info = std::get>(operation); 109 | map.insert(insert_info.key, insert_info.value); 110 | break; 111 | } 112 | case 2: { 113 | auto& insert_info = std::get>(operation); 114 | map.insert_or_assign(insert_info.key, insert_info.value); 115 | break; 116 | } 117 | case 3: { 118 | const auto& erase_info = std::get>(operation); 119 | map.erase(erase_info.key); 120 | break; 121 | } 122 | default: 123 | assert(false); 124 | } 125 | } 126 | 127 | inline RawScenario generate_raw_scenario( 128 | const std::unordered_map& operations_distr, 129 | const size_t& scale) { 130 | RawScenario result; 131 | for (const auto& [operation_type, count] : operations_distr) { 132 | for (size_t i = 0; i < count * scale; ++i) { 133 | result.push_back(operation_type); 134 | } 135 | } 136 | 137 | return result; 138 | } 139 | 140 | inline std::vector get_raw_scenarious_from_raw_scenario( 141 | size_t threads_count, 142 | const RawScenario& scenario) { 143 | std::vector scenarious(threads_count); 144 | std::mt19937 gen_32(222); 145 | for (auto& cur_scenario: scenarious) { 146 | cur_scenario = scenario; 147 | std::shuffle(cur_scenario.begin(), cur_scenario.end(), gen_32); 148 | } 149 | 150 | return scenarious; 151 | } 152 | 153 | inline std::vector split_raw_scenario_on_raw_scenarious( 154 | size_t threads_count, 155 | const RawScenario& scenario) { 156 | size_t scenario_size = scenario.size() / threads_count; 157 | std::vector scenarious(threads_count); 158 | std::mt19937 gen_32(111); 159 | 160 | auto it = scenario.begin(); 161 | for (auto& cur_scenario: scenarious) { 162 | cur_scenario = RawScenario(it, it + scenario_size); 163 | std::shuffle(cur_scenario.begin(), cur_scenario.end(), gen_32); 164 | it += scenario_size; 165 | } 166 | 167 | return scenarious; 168 | } 169 | 170 | template 171 | Scenario get_scenario_from_raw( 172 | const RawScenario& scenario, 173 | const OperationGenerator& operation_generator) { 174 | Scenario result; 175 | result.reserve(scenario.size()); 176 | for (const auto& operation_type : scenario) { 177 | result.push_back(operation_generator(operation_type)); 178 | } 179 | 180 | return result; 181 | } 182 | 183 | template 184 | std::vector> get_scenarious_from_raw( 185 | const std::vector& scenarious, 186 | const OperationGenerator& operation_generator) { 187 | std::vector> result; 188 | result.reserve(scenarious.size()); 189 | for (const auto& scenario : scenarious) { 190 | result.push_back(get_scenario_from_raw(scenario, operation_generator)); 191 | } 192 | 193 | return result; 194 | } 195 | 196 | template class Map> 197 | void execute_scenario( 198 | Map& map, 199 | const Scenario& scenario) { 200 | for (const auto& operation : scenario) { 201 | do_operation(map, operation); 202 | } 203 | } 204 | 205 | template typename Map> 206 | void run_pfor_benchmark( 207 | Map& map, 208 | const std::vector>& scenarious) { 209 | std::vector vec; 210 | vec.reserve(scenarious.size() - 1); 211 | 212 | for (size_t i = 0; i < scenarious.size() - 1; ++i) { 213 | vec.emplace_back( 214 | [i, &map, &scenarious] { 215 | execute_scenario(map, scenarious[i]); 216 | } 217 | ); 218 | } 219 | 220 | execute_scenario(map, scenarious.back()); 221 | 222 | for (auto& t : vec) { 223 | t.join(); 224 | } 225 | } 226 | 227 | template 228 | inline std::function( 229 | size_t threads_count, 230 | const RawScenario&)> get_scenarious_function(int64_t arg) { 231 | switch (arg) { 232 | case 0: 233 | return get_raw_scenarious_from_raw_scenario; 234 | case 1: 235 | return split_raw_scenario_on_raw_scenarious; 236 | default: 237 | assert(false); 238 | } 239 | return{}; 240 | } 241 | 242 | inline std::vector get_vector_for_key_function_all(uint32_t max_value) { 243 | std::vector keys(max_value + 1); 244 | for (uint32_t i = 0; i < max_value + 1; ++i) { 245 | keys[i] = i; 246 | } 247 | std::mt19937 gen_32(1337); 248 | std::shuffle(keys.begin(), keys.end(), gen_32); 249 | 250 | return keys; 251 | } 252 | 253 | inline std::function get_uint32_key_function(int64_t arg, int64_t max_value) { 254 | switch (arg) { 255 | case 0: 256 | return [max_value]{static std::mt19937 gen_32(1337); return gen_32() % max_value;}; 257 | case 1: 258 | { 259 | std::vector keys = get_vector_for_key_function_all(max_value); 260 | uint32_t index = 0; 261 | return [keys = std::move(keys), index]() mutable { 262 | return keys[index++]; 263 | }; 264 | } 265 | default: 266 | assert(false); 267 | } 268 | return {}; 269 | } 270 | 271 | template typename Map> 272 | void abstract_uint32_uint32_benchmark(benchmark::State& state) { 273 | const int64_t threads_count = state.range(0); 274 | 275 | const int64_t init_map_size = state.range(1); 276 | 277 | const int64_t init_scenario_size = state.range(2); 278 | const int64_t running_scenario_scale = state.range(3); 279 | 280 | const int64_t key_max_value = state.range(4); 281 | 282 | const int64_t running_find = state.range(5); 283 | const int64_t running_insert = state.range(6); 284 | const int64_t running_insert_or_assign = state.range(7); 285 | const int64_t running_erase = state.range(8); 286 | 287 | for (auto _ : state) { 288 | state.PauseTiming(); 289 | const auto init_key_generator = get_uint32_key_function(state.range(9), key_max_value); 290 | const auto running_key_generator = get_uint32_key_function(state.range(10), key_max_value); 291 | 292 | const auto scenarious_generator = get_scenarious_function(state.range(11)); 293 | 294 | const auto scenarious = get_scenarious_from_raw(scenarious_generator( 295 | threads_count, 296 | generate_raw_scenario({ 297 | {OperationsInfo::Types::FIND, running_find}, 298 | {OperationsInfo::Types::INSERT, running_insert}, 299 | {OperationsInfo::Types::INSERT_OR_ASSIGN, running_insert_or_assign}, 300 | {OperationsInfo::Types::ERASE, running_erase}}, 301 | running_scenario_scale)), 302 | OperationGenerator( 303 | running_key_generator, 304 | [](){return 0;})); 305 | char offset[64]; 306 | Map map(init_map_size); 307 | execute_scenario( 308 | map, 309 | get_scenario_from_raw(generate_raw_scenario( 310 | {{OperationsInfo::Types::INSERT, 1}}, 311 | init_scenario_size), 312 | OperationGenerator( 313 | init_key_generator, 314 | [](){return 0;}))); 315 | state.ResumeTiming(); 316 | 317 | run_pfor_benchmark(map, scenarious); 318 | } 319 | } 320 | 321 | struct Uint32Args{ 322 | int64_t threads_count; 323 | int64_t init_map_size; 324 | int64_t init_scenario_size; 325 | int64_t running_scenario_scale; 326 | int64_t key_max_value; 327 | int64_t running_find; 328 | int64_t running_insert; 329 | int64_t running_insert_or_assign; 330 | int64_t running_erase; 331 | int64_t init_key_generator; 332 | int64_t running_key_generator; 333 | int64_t scenarious_generator; 334 | }; 335 | 336 | inline std::vector get_uint32_benchmark_args(const Uint32Args& args) { 337 | return { 338 | args.threads_count, 339 | args.init_map_size, 340 | args.init_scenario_size, 341 | args.running_scenario_scale, 342 | args.key_max_value, 343 | args.running_find, 344 | args.running_insert, 345 | args.running_insert_or_assign, 346 | args.running_erase, 347 | args.init_key_generator, 348 | args.running_key_generator, 349 | args.scenarious_generator 350 | }; 351 | } 352 | 353 | struct uint_hash { 354 | size_t operator()(uint64_t x) const { 355 | x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); 356 | x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); 357 | x = x ^ (x >> 31); 358 | return x; 359 | } 360 | }; 361 | 362 | template 363 | class cuckoo_seqlock : public seqlock_lib::cuckoo::cuckoohash_map { 364 | using base = seqlock_lib::cuckoo::cuckoohash_map; 365 | public: 366 | using base::base; 367 | }; 368 | 369 | template 370 | class cuckoo : public libcuckoo::cuckoohash_map { 371 | using base = libcuckoo::cuckoohash_map; 372 | public: 373 | using base::base; 374 | }; 375 | 376 | template 377 | class robin_hood : public seqlock_lib::rh::rhhash_map { 378 | using base = seqlock_lib::rh::rhhash_map; 379 | public: 380 | using base::base; 381 | }; 382 | 383 | #define START_BENCHMARK(name, arguments_generator)\ 384 | BENCHMARK_TEMPLATE(abstract_uint32_uint32_benchmark, cuckoo_seqlock)\ 385 | ->Name(name + "-cuckoo_seqlock")->Apply(arguments_generator)->Unit(benchmark::kMillisecond)->Iterations(5)->UseRealTime()->MeasureProcessCPUTime();\ 386 | BENCHMARK_TEMPLATE(abstract_uint32_uint32_benchmark, robin_hood)\ 387 | ->Name(name + "-robin_hood")->Apply(arguments_generator)->Unit(benchmark::kMillisecond)->Iterations(5)->UseRealTime()->MeasureProcessCPUTime();\ 388 | BENCHMARK_TEMPLATE(abstract_uint32_uint32_benchmark, cuckoo)\ 389 | ->Name(name + "-cuckoo")->Apply(arguments_generator)->Unit(benchmark::kMillisecond)->Iterations(5)->UseRealTime()->MeasureProcessCPUTime();\ 390 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/erase_exists.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_ERASE_EXISTS 6 | static void erase_exists_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 20000000, 9 | .init_scenario_size = 20000000, 10 | .running_scenario_scale = 20000000, 11 | .key_max_value = 20000000, 12 | .running_find = 0, 13 | .running_insert = 0, 14 | .running_insert_or_assign = 0, 15 | .running_erase = 1, 16 | .init_key_generator = 1, 17 | .running_key_generator = 1, 18 | .scenarious_generator = 1 19 | }); 20 | } 21 | #endif 22 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/find_exists.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_FIND_EXISTS 6 | static void find_exists_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 1000000, 9 | .init_scenario_size = 1000000, 10 | .running_scenario_scale = 5000000, 11 | .key_max_value = 1000000, 12 | .running_find = 1, 13 | .running_insert = 0, 14 | .running_insert_or_assign = 0, 15 | .running_erase = 0, 16 | .init_key_generator = 1, 17 | .running_key_generator = 0, 18 | .scenarious_generator = 0 19 | }); 20 | } 21 | 22 | static void find_exists_high_contentions_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 23 | fill_args(b, { 24 | .init_map_size = 1000, 25 | .init_scenario_size = 1000, 26 | .running_scenario_scale = 5000000, 27 | .key_max_value = 1000, 28 | .running_find = 1, 29 | .running_insert = 0, 30 | .running_insert_or_assign = 0, 31 | .running_erase = 0, 32 | .init_key_generator = 1, 33 | .running_key_generator = 0, 34 | .scenarious_generator = 0 35 | }); 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/find_modify_eq.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_FIND_MODIFY_EQ 6 | static void find_modify_eq_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 0, 9 | .init_scenario_size = 0, 10 | .running_scenario_scale = 1000000, 11 | .key_max_value = 1000000, 12 | .running_find = 2, 13 | .running_insert = 0, 14 | .running_insert_or_assign = 1, 15 | .running_erase = 1, 16 | .init_key_generator = 0, 17 | .running_key_generator = 0, 18 | .scenarious_generator = 0 19 | }); 20 | } 21 | 22 | static void find_modify_eq_high_contention_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 23 | fill_args(b, { 24 | .init_map_size = 0, 25 | .init_scenario_size = 0, 26 | .running_scenario_scale = 1000000, 27 | .key_max_value = 1000, 28 | .running_find = 2, 29 | .running_insert = 0, 30 | .running_insert_or_assign = 1, 31 | .running_erase = 1, 32 | .init_key_generator = 0, 33 | .running_key_generator = 0, 34 | .scenarious_generator = 0 35 | }); 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/insert_or_assigns.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_INSERT_OR_ASSIGNS 6 | static void insert_or_assigns_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 0, 9 | .init_scenario_size = 0, 10 | .running_scenario_scale = 10000000, 11 | .key_max_value = 1000000, 12 | .running_find = 0, 13 | .running_insert = 0, 14 | .running_insert_or_assign = 1, 15 | .running_erase = 0, 16 | .init_key_generator = 0, 17 | .running_key_generator = 0, 18 | .scenarious_generator = 0 19 | }); 20 | } 21 | 22 | static void insert_or_assigns_high_contention_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 23 | fill_args(b, { 24 | .init_map_size = 0, 25 | .init_scenario_size = 0, 26 | .running_scenario_scale = 10000000, 27 | .key_max_value = 1000, 28 | .running_find = 0, 29 | .running_insert = 0, 30 | .running_insert_or_assign = 1, 31 | .running_erase = 0, 32 | .init_key_generator = 0, 33 | .running_key_generator = 0, 34 | .scenarious_generator = 0 35 | }); 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/inserts.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_INSERTS 6 | static void inserts_rehashing_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 0, 9 | .init_scenario_size = 0, 10 | .running_scenario_scale = 50000000, 11 | .key_max_value = 50000000, 12 | .running_find = 0, 13 | .running_insert = 1, 14 | .running_insert_or_assign = 0, 15 | .running_erase = 0, 16 | .init_key_generator = 0, 17 | .running_key_generator = 1, 18 | .scenarious_generator = 1 19 | }); 20 | } 21 | 22 | static void inserts_no_rehashing_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 23 | fill_args(b, { 24 | .init_map_size = 50000000, 25 | .init_scenario_size = 0, 26 | .running_scenario_scale = 5000000, 27 | .key_max_value = 50000000, 28 | .running_find = 4, 29 | .running_insert = 1, 30 | .running_insert_or_assign = 0, 31 | .running_erase = 0, 32 | .init_key_generator = 0, 33 | .running_key_generator = 1, 34 | .scenarious_generator = 1 35 | }); 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/random.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.hpp" 4 | 5 | #ifdef UIN32_UINT32_BENCHMARK_RANDOM 6 | static void random_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 7 | fill_args(b, { 8 | .init_map_size = 0, 9 | .init_scenario_size = 0, 10 | .running_scenario_scale = 1000000, 11 | .key_max_value = 1000000, 12 | .running_find = 1, 13 | .running_insert = 1, 14 | .running_insert_or_assign = 1, 15 | .running_erase = 1, 16 | .init_key_generator = 0, 17 | .running_key_generator = 0, 18 | .scenarious_generator = 0 19 | }); 20 | } 21 | 22 | static void random_high_contention_uint32_uint32_arguments(benchmark::internal::Benchmark* b) { 23 | fill_args(b, { 24 | .init_map_size = 0, 25 | .init_scenario_size = 0, 26 | .running_scenario_scale = 1000000, 27 | .key_max_value = 1000, 28 | .running_find = 1, 29 | .running_insert = 1, 30 | .running_insert_or_assign = 1, 31 | .running_erase = 1, 32 | .init_key_generator = 0, 33 | .running_key_generator = 0, 34 | .scenarious_generator = 0 35 | }); 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /bench/benchmarks/uint32_uint32/utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../../benchmark_hash_map.hpp" 4 | 5 | static int32_t threads_step = 1; 6 | static int32_t threads_count = 48; 7 | 8 | inline void fill_args(benchmark::internal::Benchmark* b, Uint32Args args) { 9 | if (threads_step > 1) { 10 | args.threads_count = 1; 11 | b->Args(get_uint32_benchmark_args(args)); 12 | } 13 | for (int32_t threads = threads_step; threads <= threads_count; threads += threads_step) { 14 | args.threads_count = threads; 15 | b->Args(get_uint32_benchmark_args(args)); 16 | } 17 | } -------------------------------------------------------------------------------- /bench/benchmarks_main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "benchmarks/uint32_uint32/erase_exists.hpp" 4 | #include "benchmarks/uint32_uint32/find_exists.hpp" 5 | #include "benchmarks/uint32_uint32/find_modify_eq.hpp" 6 | #include "benchmarks/uint32_uint32/insert_or_assigns.hpp" 7 | #include "benchmarks/uint32_uint32/inserts.hpp" 8 | #include "benchmarks/uint32_uint32/random.hpp" 9 | 10 | bool ParseInt32Flag(const char* arg, const char* flag, int32_t& value) { 11 | const size_t flag_len = strlen(flag); 12 | if (strncmp(flag, arg + 2, flag_len) != 0) { 13 | return false; 14 | } 15 | 16 | value = atoi(arg + (2 + flag_len + 1)); 17 | return true; 18 | } 19 | 20 | int main(int argc, char** argv) { 21 | char arg0_default[] = "benchmark"; 22 | char* args_default = arg0_default; 23 | if (!argv) { 24 | argc = 1; 25 | argv = &args_default; 26 | } 27 | benchmark::Initialize(&argc, argv); 28 | 29 | bool unrecognized = false; 30 | for (int i = 1; i < argc; ++i) { 31 | if (!ParseInt32Flag(argv[i], "benchmark_threads_step", threads_step) && 32 | !ParseInt32Flag(argv[i], "benchmark_threads_count", threads_count)) { 33 | fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], 34 | argv[i]); 35 | unrecognized = true; 36 | } 37 | } 38 | 39 | if (unrecognized) { 40 | return 1; 41 | } 42 | 43 | #ifdef UIN32_UINT32_BENCHMARK_ERASE_EXISTS 44 | START_BENCHMARK(std::string("erase_exists_uint32_uint32"), erase_exists_uint32_uint32_arguments) 45 | #endif 46 | #ifdef UIN32_UINT32_BENCHMARK_FIND_EXISTS 47 | START_BENCHMARK(std::string("find_exists_uint32_uint32"), find_exists_uint32_uint32_arguments) 48 | START_BENCHMARK(std::string("find_exists_high_contentions_uint32_uint32"), find_exists_high_contentions_uint32_uint32_arguments) 49 | #endif 50 | #ifdef UIN32_UINT32_BENCHMARK_FIND_MODIFY_EQ 51 | START_BENCHMARK(std::string("find_modify_eq_uint32_uint32"), find_modify_eq_uint32_uint32_arguments) 52 | START_BENCHMARK(std::string("find_modify_eq_high_contention_uint32_uint32"), find_modify_eq_high_contention_uint32_uint32_arguments) 53 | #endif 54 | #ifdef UIN32_UINT32_BENCHMARK_INSERT_OR_ASSIGNS 55 | START_BENCHMARK(std::string("insert_or_assigns_uint32_uint32"), insert_or_assigns_uint32_uint32_arguments) 56 | START_BENCHMARK(std::string("insert_or_assigns_high_contention_uint32_uint32"), insert_or_assigns_high_contention_uint32_uint32_arguments) 57 | #endif 58 | #ifdef UIN32_UINT32_BENCHMARK_INSERTS 59 | START_BENCHMARK(std::string("inserts_rehashing_uint32_uint32"), inserts_rehashing_uint32_uint32_arguments) 60 | START_BENCHMARK(std::string("inserts_no_rehashing_uint32_uint32"), inserts_no_rehashing_uint32_uint32_arguments) 61 | #endif 62 | #ifdef UIN32_UINT32_BENCHMARK_RANDOM 63 | START_BENCHMARK(std::string("random_uint32_uint32"), random_uint32_uint32_arguments) 64 | START_BENCHMARK(std::string("random_high_contention_uint32_uint32"), random_high_contention_uint32_uint32_arguments) 65 | #endif 66 | 67 | benchmark::RunSpecifiedBenchmarks(); 68 | benchmark::Shutdown(); 69 | 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /draw_bench.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import json\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "\n", 12 | "colors = {\n", 13 | " 'cuckoo' : 'g',\n", 14 | " 'robin_hood' : 'b',\n", 15 | " 'cuckoo_seqlock' : 'y',\n", 16 | "}\n", 17 | "\n", 18 | "def draw_plot(json_out):\n", 19 | " stats = {}\n", 20 | "\n", 21 | " for benchmark in json_out['benchmarks']:\n", 22 | " splitted_name = benchmark['name'].split('/')\n", 23 | " splitted_first = splitted_name[0].split('-')\n", 24 | "\n", 25 | " benchmark_name = splitted_first[0]\n", 26 | " map_name = splitted_first[1]\n", 27 | "\n", 28 | " threads = int(splitted_name[1])\n", 29 | " scenario_size = int(splitted_name[4])\n", 30 | " scenarious_generator = int(splitted_name[-4])\n", 31 | "\n", 32 | " operations_amount = scenario_size * threads if scenarious_generator == 0 else scenario_size\n", 33 | "\n", 34 | " key = int(splitted_name[1])\n", 35 | "\n", 36 | " if benchmark_name in stats:\n", 37 | " if map_name in stats[benchmark_name]:\n", 38 | " stats[benchmark_name][map_name].append((key, operations_amount / benchmark['real_time']))\n", 39 | " else:\n", 40 | " stats[benchmark_name][map_name] = [(key, operations_amount / benchmark['real_time'])]\n", 41 | " else:\n", 42 | " stats[benchmark_name] = {map_name : [(key, operations_amount / benchmark['real_time'])]}\n", 43 | "\n", 44 | " fig, axs = plt.subplots(nrows=len(stats))\n", 45 | " fig.set_size_inches(18.5, len(stats) * 8, forward=True)\n", 46 | " fig.set_dpi(100)\n", 47 | "\n", 48 | " plot_num = 0\n", 49 | " for benchmark_name, map_to_array in stats.items():\n", 50 | " axs[plot_num].plot([], [])\n", 51 | " axs[plot_num].set_xlabel('threads')\n", 52 | " axs[plot_num].set_ylabel('op/ms')\n", 53 | "\n", 54 | " for map_name, array in map_to_array.items():\n", 55 | " x = []\n", 56 | " y = []\n", 57 | " for (threads, real_time) in array:\n", 58 | " x.append(threads)\n", 59 | " y.append(real_time)\n", 60 | "\n", 61 | " axs[plot_num].plot(x, y, color=colors[map_name], label=map_name)\n", 62 | " axs[plot_num].legend()\n", 63 | " \n", 64 | " axs[plot_num].set_title(benchmark_name)\n", 65 | " plot_num += 1\n", 66 | " plt.savefig('bench_plot.png')\n", 67 | "\n", 68 | "def main():\n", 69 | " with open('bench_out.json', 'r') as data:\n", 70 | " draw_plot(json.load(data))\n", 71 | "\n", 72 | "main()" 73 | ] 74 | } 75 | ], 76 | "metadata": { 77 | "kernelspec": { 78 | "display_name": "Python 3.6.9 64-bit", 79 | "language": "python", 80 | "name": "python3" 81 | }, 82 | "language_info": { 83 | "codemirror_mode": { 84 | "name": "ipython", 85 | "version": 3 86 | }, 87 | "file_extension": ".py", 88 | "mimetype": "text/x-python", 89 | "name": "python", 90 | "nbconvert_exporter": "python", 91 | "pygments_lexer": "ipython3", 92 | "version": "3.10.6" 93 | }, 94 | "orig_nbformat": 4, 95 | "vscode": { 96 | "interpreter": { 97 | "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" 98 | } 99 | } 100 | }, 101 | "nbformat": 4, 102 | "nbformat_minor": 2 103 | } 104 | -------------------------------------------------------------------------------- /enviroment.yml: -------------------------------------------------------------------------------- 1 | name: seqlock_hash_map 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - make 6 | - cmake >= 3.15 7 | - compilers 8 | - clang==14 9 | - clangxx==14 10 | - libhwloc 11 | - gtest 12 | - benchmark 13 | -------------------------------------------------------------------------------- /include/common/atomic_buffer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace seqlock_lib { 6 | 7 | template 8 | using big_type = std::conditional_t>>>; 12 | 13 | template 14 | constexpr size_t big_type_count = sizeof(T) / sizeof(big_type); 15 | template 16 | constexpr size_t bytes_count = big_type_count * sizeof(big_type); 17 | 18 | template 19 | std::atomic& as_atomic(T& t) { 20 | return reinterpret_cast&>(t); 21 | } 22 | 23 | template 24 | const std::atomic& as_atomic(const T& t) { 25 | return reinterpret_cast&>(t); 26 | } 27 | 28 | template 29 | void atomic_load_memcpy(T* dest, const T& source) { 30 | using big_type_ = big_type; 31 | 32 | for (size_t i = 0; i < big_type_count; ++i) { 33 | auto& dst = reinterpret_cast(dest)[i]; 34 | auto& src = reinterpret_cast(&source)[i]; 35 | 36 | dst = as_atomic(src).load(std::memory_order_relaxed); 37 | } 38 | for (size_t i = bytes_count; i < sizeof(T); ++i) { 39 | auto& dst = reinterpret_cast(dest)[i]; 40 | auto& src = reinterpret_cast(&source)[i]; 41 | 42 | dst = as_atomic(src).load(std::memory_order_relaxed); 43 | } 44 | } 45 | 46 | template 47 | void atomic_store_memcpy(T* dest, const T& source) { 48 | using big_type_ = big_type; 49 | 50 | for (size_t i = 0; i < big_type_count; ++i) { 51 | auto& dst = reinterpret_cast(dest)[i]; 52 | auto& src = reinterpret_cast(&source)[i]; 53 | 54 | as_atomic(dst).store(src, std::memory_order_relaxed); 55 | } 56 | for (size_t i = bytes_count; i < sizeof(T); ++i) { 57 | auto& dst = reinterpret_cast(dest)[i]; 58 | auto& src = reinterpret_cast(&source)[i]; 59 | 60 | as_atomic(dst).store(src, std::memory_order_relaxed); 61 | } 62 | } 63 | 64 | } // namespace seqlock_lib 65 | -------------------------------------------------------------------------------- /include/common/bucket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "utils.hpp" 6 | 7 | namespace seqlock_lib { 8 | 9 | /* 10 | * The bucket type holds SLOT_PER_BUCKET key-value pairs, along with their 11 | * occupancy info. It uses aligned_storage arrays to store 12 | * the keys and values to allow constructing and destroying key-value pairs 13 | * in place. The lifetime of bucket data should be managed by the container. 14 | * It is the user's responsibility to confirm whether the data they are 15 | * accessing is live or not. 16 | */ 17 | template 18 | class bucket { 19 | public: 20 | using key_type = Key; 21 | using mapped_type = T; 22 | using value_type = std::pair; 23 | using storage_value_type = std::pair; 24 | 25 | using key_raw_type = aligned_storage_type; 26 | using mapped_raw_type = aligned_storage_type; 27 | using storage_value_raw_type = aligned_storage_type; 28 | 29 | bucket() noexcept : occupied_() {} 30 | 31 | const storage_value_type& storage_kvpair(uint32_t slot) const { 32 | return *reinterpret_cast(&values_[slot]); 33 | } 34 | storage_value_type& storage_kvpair(uint32_t slot) { 35 | return *reinterpret_cast(&values_[slot]); 36 | } 37 | 38 | const value_type& kvpair(uint32_t slot) const { 39 | return *reinterpret_cast(&values_[slot]); 40 | } 41 | value_type& kvpair(uint32_t slot) { 42 | return *reinterpret_cast(&values_[slot]); 43 | } 44 | 45 | const key_type& key(uint32_t slot) const { 46 | return storage_kvpair(slot).first; 47 | } 48 | key_type& key(uint32_t slot) { 49 | return storage_kvpair(slot).first; 50 | } 51 | 52 | const mapped_type& mapped(uint32_t slot) const { 53 | return storage_kvpair(slot).second; 54 | } 55 | mapped_type& mapped(uint32_t slot) { 56 | return storage_kvpair(slot).second; 57 | } 58 | 59 | const bool& occupied(uint32_t slot) const { return occupied_[slot]; } 60 | bool& occupied(uint32_t slot) { return occupied_[slot]; } 61 | 62 | private: 63 | std::array values_; 64 | std::array occupied_; 65 | }; 66 | 67 | } // namespace seqlock_lib 68 | -------------------------------------------------------------------------------- /include/common/bucket_container.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "atomic_buffer.hpp" 4 | #include "bucket.hpp" 5 | #include "data_storage.hpp" 6 | 7 | namespace seqlock_lib { 8 | 9 | template 10 | class bucket_container : public data_storage{ 11 | public: 12 | using typed_bucket = Bucket; 13 | 14 | using key_type = typename typed_bucket::key_type; 15 | using mapped_type = typename typed_bucket::mapped_type; 16 | using value_type = typename typed_bucket::value_type; 17 | 18 | using reference = value_type&; 19 | using const_reference = const value_type&; 20 | 21 | private: 22 | using data_storage_base_t = data_storage; 23 | 24 | protected: 25 | using traits = typename std::allocator_traits< 26 | Allocator>::template rebind_traits; 27 | 28 | public: 29 | using size_type = typename data_storage_base_t::size_type; 30 | using allocator_type = typename traits::allocator_type; 31 | using pointer = typename traits::pointer; 32 | using const_pointer = typename traits::const_pointer; 33 | 34 | private: 35 | // true here means the bucket allocator should be propagated 36 | void move_assign(bucket_container& other, std::true_type) { 37 | value_allocator_ = std::move(other.value_allocator_); 38 | this->data_allocator_ = value_allocator_; 39 | 40 | this->move_pointers(std::move(other)); 41 | } 42 | 43 | void move_assign(bucket_container& other, std::false_type) { 44 | if (value_allocator_ == other.value_allocator_) { 45 | this->move_pointers(std::move(other)); 46 | } else { 47 | this->change_size(other.hashpower()); 48 | copy(other); 49 | } 50 | } 51 | 52 | void copy(const bucket_container& bc) { 53 | auto it = this->begin(); 54 | 55 | for (const auto& b : bc) { 56 | for (uint16_t slot = 0; slot < typed_bucket::SLOT_PER_BUCKET; ++slot) { 57 | if (b.occupied(slot)) { 58 | Derived::copy_bucket_slot(this, slot, *it, b); 59 | } 60 | } 61 | ++it; 62 | } 63 | } 64 | 65 | public: 66 | bucket_container(size_type hp, const allocator_type& allocator) 67 | : data_storage_base_t(hp, allocator), value_allocator_(allocator) {} 68 | 69 | ~bucket_container() { 70 | deallocate(); 71 | } 72 | 73 | bucket_container(const bucket_container& other) 74 | : value_allocator_( 75 | traits::select_on_container_copy_construction(other.value_allocator_)), 76 | data_storage_base_t(other.hashpower(), value_allocator_) { 77 | copy(other); 78 | } 79 | 80 | bucket_container(const bucket_container& other, const allocator_type& allocator) 81 | : value_allocator_(allocator), data_storage_base_t(other.hashpower(), value_allocator_) { 82 | copy(other); 83 | } 84 | 85 | bucket_container(bucket_container&& other) 86 | : value_allocator_(std::move(other.value_allocator_)), data_storage_base_t(-1, value_allocator_) { 87 | this->move_pointers(std::move(other)); 88 | } 89 | 90 | bucket_container(bucket_container&& bc, 91 | const allocator_type& allocator) 92 | : value_allocator_(allocator), data_storage_base_t(-1, value_allocator_) { 93 | move_assign(bc, std::false_type()); 94 | } 95 | 96 | bucket_container& operator=(const bucket_container& other) { 97 | deallocate(); 98 | copy_allocator(value_allocator_, other.value_allocator_, 99 | typename traits::propagate_on_container_copy_assignment()); 100 | this->data_allocator_ = value_allocator_; 101 | 102 | this->change_size(other.hashpower()); 103 | copy(other); 104 | 105 | return *this; 106 | } 107 | 108 | bucket_container &operator=(bucket_container&& bc) { 109 | deallocate(); 110 | move_assign(bc, typename traits::propagate_on_container_move_assignment()); 111 | return *this; 112 | } 113 | 114 | void swap(bucket_container& other) noexcept { 115 | swap_allocator(value_allocator_, other.value_allocator_, 116 | typename traits::propagate_on_container_swap()); 117 | 118 | data_storage_base_t::swap(other); 119 | } 120 | 121 | template 122 | void set_field(Field& f, Args&& ...args) { 123 | aligned_storage_type storage; 124 | traits::construct( 125 | value_allocator_, 126 | reinterpret_cast(&storage), 127 | std::forward(args)...); 128 | atomic_store_memcpy(std::addressof(f), *reinterpret_cast(&storage)); 129 | } 130 | template 131 | static void set_field(Field& f, const Field& args) { 132 | atomic_store_memcpy(std::addressof(f), args); 133 | } 134 | 135 | static void deoccupy(typed_bucket& b, uint16_t slot) { 136 | assert(b.occupied(slot)); 137 | atomic_store_memcpy(&b.occupied(slot), false); 138 | } 139 | 140 | void deoccupy(size_type ind, uint16_t slot) { 141 | deoccupy((*this)[ind], slot); 142 | } 143 | 144 | void clear() noexcept { 145 | for (typed_bucket& b : *this) { 146 | for (uint16_t slot = 0; slot < typed_bucket::SLOT_PER_BUCKET; ++slot) { 147 | if (b.occupied(slot)) { 148 | deoccupy(b, slot); 149 | } 150 | } 151 | } 152 | } 153 | 154 | void deallocate() noexcept { 155 | if (this->is_deallocated()) { 156 | return; 157 | } 158 | this->change_size(-1); 159 | } 160 | 161 | allocator_type& get_allocator() { 162 | return value_allocator_; 163 | } 164 | 165 | const allocator_type& get_allocator() const { 166 | return value_allocator_; 167 | } 168 | 169 | allocator_type value_allocator_; 170 | }; 171 | 172 | } // namespace seqlock_lib 173 | -------------------------------------------------------------------------------- /include/common/data_storage.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "utils.hpp" 7 | 8 | namespace seqlock_lib { 9 | 10 | template 11 | class data_storage { 12 | protected: 13 | using traits = typename std::allocator_traits< 14 | Allocator>::template rebind_traits; 15 | using allocator_type = typename traits::allocator_type; 16 | 17 | using data_pointer = typename traits::pointer; 18 | using data_pointers_array = std::array; 19 | 20 | using size_type = typename traits::size_type; 21 | 22 | public: 23 | struct iterator { 24 | using array_iterator_t = typename data_pointers_array::iterator; 25 | 26 | iterator(const array_iterator_t& array_iterator, size_type data_index, int32_t array_index) 27 | : array_iterator(array_iterator), data_index(data_index), array_index(array_index) {} 28 | 29 | const Value& operator*() const { 30 | return (*array_iterator)[data_index]; 31 | } 32 | Value& operator*() { 33 | return (*array_iterator)[data_index]; 34 | } 35 | 36 | const Value* operator->() const { 37 | return &((*array_iterator)[data_index]); 38 | } 39 | Value* operator->() { 40 | return &((*array_iterator)[data_index]); 41 | } 42 | 43 | iterator& operator++() { 44 | if (data_index == size_by_index(array_index) - 1) { 45 | ++array_iterator; 46 | ++array_index; 47 | data_index = 0; 48 | } else { 49 | ++data_index; 50 | } 51 | 52 | return *this; 53 | } 54 | iterator operator++(int) { 55 | iterator result(*this); 56 | ++(*this); 57 | return result; 58 | } 59 | 60 | iterator& operator--() { 61 | if (data_index == 0) { 62 | --array_iterator; 63 | --array_index; 64 | data_index = size_by_index(array_index) - 1; 65 | } else { 66 | --data_index; 67 | } 68 | 69 | return *this; 70 | } 71 | iterator operator--(int) { 72 | iterator result(*this); 73 | --(*this); 74 | return result; 75 | } 76 | 77 | bool operator==(const iterator& other) const { 78 | return data_index == other.data_index && 79 | array_index == other.array_index; 80 | } 81 | bool operator!=(const iterator& other) const { 82 | return !(*this == other); 83 | } 84 | 85 | public: 86 | array_iterator_t array_iterator; 87 | size_type data_index; 88 | int32_t array_index; 89 | }; 90 | 91 | private: 92 | template 93 | static ResultT find_value(size_type i, F func) { 94 | static constexpr uint32_t type_size_ind = (sizeof(size_type) << 3); 95 | 96 | if (i == 0) { 97 | return func(0, 0); 98 | } 99 | 100 | const size_type left_bit_ind = type_size_ind - std::countl_zero(i); 101 | return func(left_bit_ind, i ^ (size_type(1) << (left_bit_ind - 1))); 102 | } 103 | 104 | void hashpower(int32_t hp) noexcept { 105 | hp_.store(hp, std::memory_order_release); 106 | } 107 | 108 | protected: 109 | static constexpr size_type size_by_index(int32_t index) { 110 | return index == 0 ? 1 : (size_type(1) << (index - 1)); 111 | } 112 | 113 | void move_pointers(data_storage&& other) { 114 | const int32_t hp = other.hashpower(); 115 | for (int32_t i = 0; i <= hp; ++i) { 116 | data_[i] = other.data_[i]; 117 | other.data_[i] = nullptr; 118 | } 119 | 120 | hashpower(other.hp_.exchange(-1, std::memory_order_acq_rel)); 121 | } 122 | 123 | void change_size(int32_t new_hp) { 124 | const int32_t hp = hashpower(); 125 | 126 | if (hp == new_hp) { 127 | return; 128 | } 129 | 130 | if (hp < new_hp) { 131 | for (int32_t i = hp + 1; i <= new_hp; ++i) { 132 | data_[i] = allocate_and_construct(i); 133 | } 134 | hp_.store(new_hp, std::memory_order_release); 135 | } else { 136 | hp_.store(new_hp, std::memory_order_release); 137 | for (int32_t i = hp; i > new_hp; --i) { 138 | deallocate(data_[i],i); 139 | } 140 | } 141 | } 142 | 143 | public: 144 | data_storage(int32_t hp, const allocator_type& allocator) 145 | : data_allocator_(allocator), hp_(-1) { 146 | change_size(hp); 147 | } 148 | 149 | virtual ~data_storage() { 150 | change_size(-1); 151 | } 152 | 153 | virtual void swap(data_storage &other) noexcept { 154 | swap_allocator(data_allocator_, other.data_allocator_, 155 | typename traits::propagate_on_container_swap()); 156 | // Regardless of whether we actually swapped the allocators or not, it will 157 | // always be okay to do the remainder of the swap. This is because if the 158 | // allocators were swapped, then the subsequent operations are okay. If the 159 | // allocators weren't swapped but compare equal, then we're okay. If they 160 | // weren't swapped and compare unequal, then behavior is undefined, so 161 | // we're okay. 162 | std::swap(hp_, other.hp_); 163 | for (int32_t i = 0; i <= std::max(hashpower(), other.hashpower()); ++i) { 164 | std::swap(data_[i], other.data_[i]); 165 | } 166 | } 167 | 168 | int32_t hashpower() const noexcept { 169 | return hp_.load(std::memory_order_acquire); 170 | } 171 | 172 | size_type size() const { 173 | const int32_t hp = hashpower(); 174 | return hp == -1 ? 0 : (size_type(1) << hp); 175 | } 176 | 177 | template 178 | data_pointer allocate_and_construct(int32_t index, Args&& ...args) { 179 | const size_type size = size_by_index(index); 180 | data_pointer pointer = data_allocator_.allocate(size); 181 | 182 | for (size_type i = 0; i < size; ++i) { 183 | traits::construct(data_allocator_, &pointer[i], std::forward(args)...); 184 | } 185 | return pointer; 186 | } 187 | 188 | void deallocate(data_pointer& pointer, size_type index) noexcept { 189 | data_allocator_.deallocate(pointer, size_by_index(index)); 190 | pointer = nullptr; 191 | } 192 | 193 | template 194 | void double_size(Args&& ...args) { 195 | const int32_t new_hp = hashpower() + 1; 196 | data_[new_hp] = allocate_and_construct(new_hp, std::forward(args)...); 197 | hashpower(new_hp); 198 | } 199 | 200 | void push_back(data_pointer pointer) noexcept { 201 | const int32_t new_hp = hashpower() + 1; 202 | data_[new_hp] = pointer; 203 | hashpower(new_hp); 204 | } 205 | 206 | Value& operator[](size_type i) { 207 | return find_value(i, 208 | [this](size_type array_index, size_type data_index) -> Value& { 209 | return data_[array_index][data_index]; 210 | } 211 | ); 212 | } 213 | const Value& operator[](size_type i) const { 214 | return find_value(i, 215 | [this](size_type array_index, size_type data_index) -> const Value& { 216 | return data_[array_index][data_index]; 217 | } 218 | ); 219 | } 220 | 221 | iterator get_iterator(size_type index) { 222 | return find_value(index, 223 | [this, index](size_type array_index, size_type data_index) -> iterator { 224 | return iterator(data_.begin() + array_index, data_index, array_index); 225 | } 226 | ); 227 | } 228 | 229 | iterator begin() const { 230 | return iterator(data_.begin(), 0, 0); 231 | } 232 | 233 | iterator end() const { 234 | const int32_t hp = hashpower(); 235 | const int32_t array_index = (hp == -1 ? 0 : hp + 1); 236 | return iterator(data_.begin() + array_index, 0, array_index); 237 | } 238 | 239 | bool is_deallocated() const noexcept { 240 | return data_[0] == nullptr; 241 | } 242 | 243 | protected: 244 | mutable data_pointers_array data_{nullptr}; 245 | allocator_type data_allocator_; 246 | CopyableAtomic hp_; 247 | }; 248 | 249 | } // namespace seqlock_lib 250 | -------------------------------------------------------------------------------- /include/common/lock_container.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "data_storage.hpp" 4 | 5 | namespace seqlock_lib { 6 | 7 | template 8 | class lock_container : public data_storage { 9 | using data_storage_base_t = data_storage; 10 | 11 | void copy(const lock_container& other) { 12 | auto it = this->begin(); 13 | 14 | for (const Lock& lock : other) { 15 | it->set_migrated(Lock::is_migrated(lock.get_epoch())); 16 | it->elem_counter() = lock.elem_counter(); 17 | 18 | ++it; 19 | } 20 | } 21 | 22 | // true here means the allocator should be propagated 23 | void move_assign(lock_container& other, std::true_type) { 24 | this->data_allocator_ = std::move(other.data_allocator_); 25 | this->move_pointers(std::move(other)); 26 | } 27 | 28 | void move_assign(lock_container& other, std::false_type) { 29 | if (this->data_allocator_ == other.data_allocator_) { 30 | this->move_pointers(std::move(other)); 31 | } else { 32 | this->change_size(other.hashpower()); 33 | copy(other); 34 | } 35 | } 36 | 37 | public: 38 | lock_container(size_t hp, const Allocator& allocator) 39 | : data_storage_base_t(hp, allocator) {} 40 | 41 | ~lock_container() = default; 42 | 43 | lock_container(const lock_container& other) 44 | : data_storage_base_t(other.hashpower(), other.data_allocator_) { 45 | copy(other); 46 | }; 47 | lock_container(const lock_container& other, const Allocator& allocator) 48 | : data_storage_base_t(other.hashpower(), allocator) { 49 | copy(other); 50 | } 51 | 52 | lock_container(lock_container&& other) 53 | : data_storage_base_t(-1, other.data_allocator_) { 54 | this->move_pointers(std::move(other)); 55 | }; 56 | lock_container(lock_container&& other, const Allocator& allocator) 57 | : data_storage_base_t(-1, allocator) { 58 | move_assign(other, std::false_type()); 59 | }; 60 | 61 | lock_container& operator=(const lock_container& other) { 62 | copy_allocator(this->data_allocator_, other.data_allocator_, 63 | typename data_storage_base_t::traits::propagate_on_container_copy_assignment()); 64 | this->change_size(other.hashpower()); 65 | copy(other); 66 | return *this; 67 | } 68 | lock_container& operator=(lock_container&& other) { 69 | this->change_size(-1); 70 | move_assign(other, typename data_storage_base_t::traits::propagate_on_container_move_assignment()); 71 | return *this; 72 | } 73 | }; 74 | 75 | } // namespace seqlock_lib 76 | -------------------------------------------------------------------------------- /include/common/seqlock.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "utils.hpp" 6 | 7 | namespace seqlock_lib { 8 | 9 | using seqlock_epoch_t = uint64_t; 10 | 11 | // Can be rewritten using bts assembler instruction, see 12 | // https://github.com/oneapi-src/oneTBB/pull/131 13 | template 14 | LIBCUCKOO_SQUELCH_PADDING_WARNING 15 | class LIBCUCKOO_ALIGNAS(Align) seqlock { 16 | private: 17 | static constexpr seqlock_epoch_t seqlock_type_size = sizeof(seqlock_epoch_t) << 3; 18 | 19 | static constexpr seqlock_epoch_t is_migrated_bit = seqlock_epoch_t(1) << (seqlock_type_size - 1); 20 | static constexpr seqlock_epoch_t is_migrated_bit_xor = 21 | std::numeric_limits::max() ^ is_migrated_bit; 22 | 23 | public: 24 | seqlock(bool blocked = false, bool is_migrated = true) noexcept 25 | : elem_counter_(0), cur_epoch_((blocked ? 1 : 0) + (is_migrated ? is_migrated_bit : 0)), 26 | epoch_(cur_epoch_) { 27 | // since C++20 default constructor initialize atomic_flag to clear state 28 | if (blocked) { 29 | lock_.test_and_set(std::memory_order_acquire); 30 | } 31 | } 32 | 33 | inline seqlock_epoch_t lock() noexcept { 34 | while (lock_.test_and_set(std::memory_order_acquire)) { 35 | cpu_relax(); 36 | } 37 | 38 | epoch_.store(++cur_epoch_, std::memory_order_relaxed); 39 | 40 | return cur_epoch_; 41 | } 42 | 43 | inline bool try_lock() noexcept { 44 | if (!lock_.test_and_set(std::memory_order_acquire)) { 45 | epoch_.store(++cur_epoch_, std::memory_order_relaxed); 46 | return true; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | inline void unlock() noexcept { 53 | ++cur_epoch_; 54 | unlock_atomics(); 55 | } 56 | 57 | inline void unlock_no_modified() noexcept { 58 | --cur_epoch_; 59 | unlock_atomics(); 60 | } 61 | 62 | inline seqlock_epoch_t get_epoch(std::memory_order m = std::memory_order_acquire) const noexcept { 63 | return epoch_.load(m); 64 | } 65 | 66 | constexpr static inline bool is_locked(seqlock_epoch_t value) noexcept { 67 | return (value & 1) != 0; 68 | } 69 | 70 | constexpr static inline bool is_migrated(seqlock_epoch_t value) noexcept { 71 | return (value & is_migrated_bit) != 0; 72 | } 73 | 74 | inline void set_migrated(bool is_migrated) noexcept { 75 | if (is_migrated) { 76 | cur_epoch_ |= is_migrated_bit; 77 | } else { 78 | cur_epoch_ &= is_migrated_bit_xor; 79 | } 80 | 81 | epoch_.store(cur_epoch_, std::memory_order_relaxed); 82 | } 83 | 84 | inline counter_type elem_counter() const noexcept { 85 | return elem_counter_; 86 | } 87 | inline counter_type& elem_counter() noexcept { 88 | return elem_counter_; 89 | } 90 | 91 | private: 92 | inline void unlock_atomics() noexcept { 93 | epoch_.store(cur_epoch_, std::memory_order_relaxed); 94 | lock_.clear(std::memory_order_release); 95 | } 96 | 97 | counter_type elem_counter_; 98 | 99 | seqlock_epoch_t cur_epoch_; 100 | std::atomic epoch_; 101 | std::atomic_flag lock_; 102 | }; 103 | 104 | } // namespace seqlock_lib 105 | -------------------------------------------------------------------------------- /include/common/utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "atomic_buffer.hpp" 5 | 6 | namespace seqlock_lib { 7 | 8 | /** 9 | * alignas() requires GCC >= 4.9, so we stick with the alignment attribute for 10 | * GCC. 11 | */ 12 | #ifdef __GNUC__ 13 | #define LIBCUCKOO_ALIGNAS(x) __attribute__((aligned(x))) 14 | #else 15 | #define LIBCUCKOO_ALIGNAS(x) alignas(x) 16 | #endif 17 | 18 | /** 19 | * At higher warning levels, MSVC produces an annoying warning that alignment 20 | * may cause wasted space: "structure was padded due to __declspec(align())". 21 | */ 22 | #ifdef _MSC_VER 23 | #define LIBCUCKOO_SQUELCH_PADDING_WARNING __pragma(warning(suppress : 4324)) 24 | #else 25 | #define LIBCUCKOO_SQUELCH_PADDING_WARNING 26 | #endif 27 | 28 | // Counter type 29 | using counter_type = int64_t; 30 | 31 | inline void cpu_relax() { 32 | #if defined(__x86_64__) 33 | asm volatile("pause" : : : "memory"); 34 | #elif defined(__aarch64__) 35 | asm volatile("yield" : : : "memory"); 36 | #endif 37 | } 38 | 39 | // true here means the allocators from `src` are propagated on libcuckoo_copy 40 | template 41 | void copy_allocator(A &dst, const A &src, std::true_type) { 42 | dst = src; 43 | } 44 | 45 | template 46 | void copy_allocator(A &, const A &, std::false_type) {} 47 | 48 | // true here means the allocators from `src` are propagated on libcuckoo_swap 49 | template void swap_allocator(A &dst, A &src, std::true_type) { 50 | std::swap(dst, src); 51 | } 52 | 53 | template void swap_allocator(A &, A &, std::false_type) {} 54 | 55 | // reserve_calc takes in a parameter specifying a certain number of slots 56 | // for a table and returns the smallest hashpower that will hold n elements. 57 | template 58 | static size_type reserve_calc_for_slots(const size_type n) { 59 | const size_type buckets = (n + SLOT_PER_BUCKET - 1) / SLOT_PER_BUCKET; 60 | size_type blog2; 61 | for (blog2 = 0; (size_type(1) << blog2) < buckets; ++blog2); 62 | 63 | assert(n <= buckets * SLOT_PER_BUCKET && buckets <= (size_type(1) << blog2)); 64 | return blog2; 65 | } 66 | 67 | template 68 | struct LockDeleter { 69 | void operator()(Lock* l) const {l->unlock(); } 70 | }; 71 | 72 | // This exception is thrown whenever we try to lock a bucket, but the 73 | // hashpower is not what was expected 74 | class hashpower_changed {}; 75 | 76 | 77 | // A small wrapper around std::atomic to make it copyable for constructors. 78 | template 79 | class CopyableAtomic : public std::atomic { 80 | public: 81 | using std::atomic::atomic; 82 | 83 | CopyableAtomic(const CopyableAtomic& other) noexcept 84 | : CopyableAtomic(other.load(std::memory_order_acquire)) {} 85 | 86 | CopyableAtomic& operator=(const CopyableAtomic& other) noexcept { 87 | this->store(other.load(std::memory_order_acquire), 88 | std::memory_order_release); 89 | return *this; 90 | } 91 | }; 92 | 93 | enum class operation_mode : bool { 94 | SAFE, 95 | UNSAFE 96 | }; 97 | 98 | template 99 | void update_safely(mapped_type& value, F fn) { 100 | if constexpr (OPERATION_MODE == operation_mode::SAFE) { 101 | mapped_type value_copy(value); 102 | fn(value_copy); 103 | atomic_store_memcpy(&value, value_copy); 104 | } else { 105 | fn(value); 106 | } 107 | } 108 | 109 | template 110 | using aligned_storage_type = 111 | std::aligned_storage_t; 112 | 113 | } // seqlock_lib 114 | -------------------------------------------------------------------------------- /include/hash_maps/cuckoo/cuckoo_bucket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "bucket.hpp" 4 | 5 | namespace seqlock_lib::cuckoo { 6 | 7 | template 8 | class cuckoo_bucket : public bucket { 9 | using base = bucket; 10 | 11 | public: 12 | static constexpr uint32_t SLOT_PER_BUCKET = _SLOT_PER_BUCKET; 13 | 14 | using base::base; 15 | 16 | using partial_t = Partial; 17 | 18 | const partial_t& partial(uint32_t slot) const { 19 | return partials_[slot]; 20 | } 21 | partial_t& partial(uint32_t slot) { 22 | return partials_[slot]; 23 | } 24 | 25 | private: 26 | std::array partials_; 27 | }; 28 | 29 | } // namespace seqlock_lib::cuckoo 30 | -------------------------------------------------------------------------------- /include/hash_maps/cuckoo/cuckoo_bucket_container.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "atomic_buffer.hpp" 4 | #include "bucket_container.hpp" 5 | 6 | #include "cuckoo_bucket.hpp" 7 | 8 | namespace seqlock_lib::cuckoo { 9 | 10 | template 11 | class cuckoo_bucket_container : public bucket_container< 12 | cuckoo_bucket_container, 13 | cuckoo_bucket, Allocator> { 14 | public: 15 | using typed_bucket = cuckoo_bucket; 16 | 17 | private: 18 | using base = bucket_container< 19 | cuckoo_bucket_container, 20 | typed_bucket, Allocator>; 21 | 22 | friend base; 23 | 24 | public: 25 | using traits = typename base::traits; 26 | 27 | using base::base; 28 | using base::operator=; 29 | 30 | using size_type = typename base::size_type; 31 | using partial_t = Partial; 32 | 33 | private: 34 | static void copy_bucket_slot(base* this_base, uint32_t slot, typed_bucket& dst, const typed_bucket& src) { 35 | setKV_by_this(this_base, dst, slot, src.partial(slot), src.key(slot), src.mapped(slot)); 36 | } 37 | 38 | // Constructs live data in a bucket 39 | template 40 | static void setKV_by_this(base* this_base, typed_bucket& b, uint32_t slot, 41 | partial_t p, K &&k, Args&& ...args) { 42 | assert(!b.occupied(slot)); 43 | 44 | atomic_store_memcpy(&b.partial(slot), p); 45 | this_base->set_field(b.key(slot), std::forward(k)); 46 | this_base->set_field(b.mapped(slot), std::forward(args)...); 47 | atomic_store_memcpy(&b.occupied(slot), true); 48 | } 49 | 50 | public: 51 | template 52 | void setKV(typed_bucket& b, uint32_t slot, partial_t p, 53 | K&& k, Args&& ...args) { 54 | setKV_by_this(static_cast(this), b, slot, p, std::forward(k), std::forward(args)...); 55 | } 56 | 57 | template 58 | void setKV(size_type ind, uint32_t slot, partial_t p, K&& k, 59 | Args&& ...args) { 60 | setKV((*this)[ind], slot, p, std::forward(k), std::forward(args)...); 61 | } 62 | }; 63 | 64 | } // seqlock_lib::cuckoo 65 | -------------------------------------------------------------------------------- /include/hash_maps/cuckoo/cuckoohash_config.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace seqlock_lib::cuckoo { 7 | 8 | //! The default maximum number of keys per bucket 9 | constexpr size_t DEFAULT_SLOT_PER_BUCKET = 4; 10 | 11 | //! The default number of elements in an empty hash table 12 | constexpr size_t DEFAULT_SIZE = 13 | (1U << 16) * DEFAULT_SLOT_PER_BUCKET; 14 | 15 | //! The default minimum load factor that the table allows for automatic 16 | //! expansion. It must be a number between 0.0 and 1.0. The table will throw 17 | //! load_factor_too_low if the load factor falls below this value 18 | //! during an automatic expansion. 19 | constexpr double DEFAULT_MINIMUM_LOAD_FACTOR = 0.05; 20 | 21 | //! An alias for the value that sets no limit on the maximum hashpower. If this 22 | //! value is set as the maximum hashpower limit, there will be no limit. This 23 | //! is also the default initial value for the maximum hashpower in a table. 24 | constexpr size_t NO_MAXIMUM_HASHPOWER = 25 | std::numeric_limits::max(); 26 | 27 | //! set LIBCUCKOO_DEBUG to 1 to enable debug output 28 | #define LIBCUCKOO_DEBUG 0 29 | 30 | } // namespace seqlock_lib::cuckoo 31 | -------------------------------------------------------------------------------- /include/hash_maps/cuckoo/cuckoohash_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "cuckoohash_config.hpp" // for LIBCUCKOO_DEBUG 7 | 8 | namespace seqlock_lib::cuckoo { 9 | 10 | #if LIBCUCKOO_DEBUG 11 | //! When \ref LIBCUCKOO_DEBUG is 0, LIBCUCKOO_DBG will printing out status 12 | //! messages in various situations 13 | #define LIBCUCKOO_DBG(fmt, ...) \ 14 | fprintf(stderr, "\x1b[32m" \ 15 | "[libcuckoo:%s:%d:%lu] " fmt "" \ 16 | "\x1b[0m", \ 17 | __FILE__, __LINE__, \ 18 | std::hash()(std::this_thread::get_id()), \ 19 | __VA_ARGS__) 20 | #else 21 | //! When \ref LIBCUCKOO_DEBUG is 0, LIBCUCKOO_DBG does nothing 22 | #define LIBCUCKOO_DBG(fmt, ...) \ 23 | do { \ 24 | } while (0) 25 | #endif 26 | 27 | /** 28 | * At higher warning levels, MSVC may issue a deadcode warning which depends on 29 | * the template arguments given. For certain other template arguments, the code 30 | * is not really "dead". 31 | */ 32 | #ifdef _MSC_VER 33 | #define LIBCUCKOO_SQUELCH_DEADCODE_WARNING_BEGIN \ 34 | do { \ 35 | __pragma(warning(push)); \ 36 | __pragma(warning(disable : 4702)) \ 37 | } while (0) 38 | #define LIBCUCKOO_SQUELCH_DEADCODE_WARNING_END __pragma(warning(pop)) 39 | #else 40 | #define LIBCUCKOO_SQUELCH_DEADCODE_WARNING_BEGIN 41 | #define LIBCUCKOO_SQUELCH_DEADCODE_WARNING_END 42 | #endif 43 | 44 | /** 45 | * Thrown when an automatic expansion is triggered, but the load factor of the 46 | * table is below a minimum threshold, which can be set by the \ref 47 | * cuckoohash_map::minimum_load_factor method. This can happen if the hash 48 | * function does not properly distribute keys, or for certain adversarial 49 | * workloads. 50 | */ 51 | class load_factor_too_low : public std::exception { 52 | public: 53 | /** 54 | * Constructor 55 | * 56 | * @param lf the load factor of the table when the exception was thrown 57 | */ 58 | load_factor_too_low(const double lf) noexcept : load_factor_(lf) {} 59 | 60 | /** 61 | * @return a descriptive error message 62 | */ 63 | virtual const char *what() const noexcept override { 64 | return "Automatic expansion triggered when load factor was below " 65 | "minimum threshold"; 66 | } 67 | 68 | /** 69 | * @return the load factor of the table when the exception was thrown 70 | */ 71 | double load_factor() const noexcept { return load_factor_; } 72 | 73 | private: 74 | const double load_factor_; 75 | }; 76 | 77 | /** 78 | * Thrown when an expansion is triggered, but the hashpower specified is greater 79 | * than the maximum, which can be set with the \ref 80 | * cuckoohash_map::maximum_hashpower method. 81 | */ 82 | class maximum_hashpower_exceeded : public std::exception { 83 | public: 84 | /** 85 | * Constructor 86 | * 87 | * @param hp the hash power we were trying to expand to 88 | */ 89 | maximum_hashpower_exceeded(const size_t hp) noexcept : hashpower_(hp) {} 90 | 91 | /** 92 | * @return a descriptive error message 93 | */ 94 | virtual const char *what() const noexcept override { 95 | return "Expansion beyond maximum hashpower"; 96 | } 97 | 98 | /** 99 | * @return the hashpower we were trying to expand to 100 | */ 101 | size_t hashpower() const noexcept { return hashpower_; } 102 | 103 | private: 104 | const size_t hashpower_; 105 | }; 106 | 107 | } // namespace seqlock_lib::cuckoo 108 | -------------------------------------------------------------------------------- /include/hash_maps/rh/rh_bucket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "bucket.hpp" 4 | #include "seqlock.hpp" 5 | 6 | namespace seqlock_lib::rh { 7 | 8 | using seqlock_t = seqlock<1>; 9 | 10 | template 11 | class rh_bucket : public seqlock_t, public bucket{ 12 | using base_bucket = bucket; 13 | 14 | public: 15 | static constexpr size_t SLOT_PER_BUCKET = _SLOT_PER_BUCKET; 16 | 17 | rh_bucket() = default; 18 | rh_bucket(bool locked) : seqlock(locked), base_bucket() {} 19 | 20 | const uint16_t& dist(size_t ind) const { 21 | return distances[ind]; 22 | } 23 | uint16_t& dist(size_t ind) { 24 | return distances[ind]; 25 | } 26 | private: 27 | std::array distances; 28 | }; 29 | 30 | } // namespace seqlock_lib::rh 31 | -------------------------------------------------------------------------------- /include/hash_maps/rh/rh_bucket_container.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "atomic_buffer.hpp" 4 | #include "bucket_container.hpp" 5 | 6 | #include "rh_bucket.hpp" 7 | #include 8 | #include 9 | 10 | namespace seqlock_lib::rh { 11 | 12 | template 13 | class rh_bucket_container : public bucket_container< 14 | rh_bucket_container, 15 | rh_bucket, Allocator> { 16 | public: 17 | using typed_bucket = rh_bucket; 18 | 19 | private: 20 | using base = bucket_container< 21 | rh_bucket_container, 22 | typed_bucket, Allocator>; 23 | 24 | friend base; 25 | 26 | using traits = typename base::traits; 27 | 28 | public: 29 | using base::base; 30 | 31 | using size_type = typename base::size_type; 32 | 33 | private: 34 | static void copy_bucket_slot(base* this_base, uint16_t slot, typed_bucket& dst, const typed_bucket& src) { 35 | setKV_by_this(this_base, dst, slot, src.dist(slot), src.key(slot), src.mapped(slot)); 36 | } 37 | 38 | // Constructs live data in a bucket 39 | template 40 | static void setKV_by_this(base* this_base, typed_bucket& b, uint16_t slot, 41 | uint16_t dist, K &&k, Args&& ...args) { 42 | atomic_store_memcpy(&b.dist(slot), dist); 43 | this_base->set_field(b.key(slot), std::forward(k)); 44 | this_base->set_field(b.mapped(slot), std::forward(args)...); 45 | atomic_store_memcpy(&b.occupied(slot), true); 46 | } 47 | 48 | public: 49 | template 50 | void setKV(typed_bucket& b, size_type slot, uint16_t dist, 51 | K &&k, Args&& ...args) { 52 | setKV_by_this(static_cast(this), b, slot, dist, std::forward(k), std::forward(args)...); 53 | } 54 | 55 | template 56 | void setKV(size_type ind, size_type slot, uint16_t dist, 57 | K&& k, Args&& ...args) { 58 | setKV((*this)[ind], slot, dist, std::forward(k), std::forward(args)...); 59 | } 60 | }; 61 | 62 | } // seqlock_lib::rh 63 | -------------------------------------------------------------------------------- /include/hash_maps/rh/rhhash_config.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | //! The default maximum number of keys per bucket 6 | constexpr size_t DEFAULT_SLOT_PER_BUCKET = 4; 7 | 8 | //! The default number of elements in an empty hash table 9 | constexpr size_t DEFAULT_SIZE = 10 | (1U << 16) * DEFAULT_SLOT_PER_BUCKET; -------------------------------------------------------------------------------- /include/hash_maps/rh/rhhash_map.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "atomic_buffer.hpp" 8 | #include "rhhash_config.hpp" 9 | #include "rh_bucket_container.hpp" 10 | #include "seqlock.hpp" 11 | #include "utils.hpp" 12 | 13 | namespace seqlock_lib::rh { 14 | 15 | template , 16 | class KeyEqual = std::equal_to, 17 | class Allocator = std::allocator>, 18 | std::size_t SLOT_PER_BUCKET = DEFAULT_SLOT_PER_BUCKET> 19 | class rhhash_map { 20 | static_assert(std::is_trivially_copyable_v && std::is_trivially_copyable_v, 21 | "Key and value should be trivially copyable due to seqlock requirements"); 22 | private: 23 | // The type of the buckets container 24 | using buckets_t = rh_bucket_container; 25 | 26 | public: 27 | // The type of the bucket 28 | using bucket_t = typename buckets_t::typed_bucket; 29 | 30 | /** @name Type Declarations */ 31 | /**@{*/ 32 | using key_type = typename buckets_t::key_type; 33 | using mapped_type = typename buckets_t::mapped_type; 34 | 35 | /** 36 | * This type is defined as an @c std::pair. Note that table behavior is 37 | * undefined if a user-defined specialization of @c std::pair or @c 38 | * std::pair exists. 39 | */ 40 | using value_type = typename buckets_t::value_type; 41 | using size_type = typename buckets_t::size_type; 42 | using difference_type = std::ptrdiff_t; 43 | using hasher = Hash; 44 | using key_equal = KeyEqual; 45 | using allocator_type = typename buckets_t::allocator_type; 46 | using reference = typename buckets_t::reference; 47 | using const_reference = typename buckets_t::const_reference; 48 | using pointer = typename buckets_t::pointer; 49 | using const_pointer = typename buckets_t::const_pointer; 50 | 51 | private: 52 | enum class table_mode : bool { 53 | LOCKED, 54 | UNLOCKED 55 | }; 56 | 57 | enum class collect_epochs_status : uint16_t { 58 | RETRY, 59 | FOUND, 60 | NOT_FOUND 61 | }; 62 | 63 | enum class cycle_status: uint16_t { 64 | OUT_OF_WINDOW, 65 | NOT_OCCUPIED, 66 | LESS_DIST, 67 | EQUAL 68 | }; 69 | 70 | struct cycle_checks { 71 | bool out_of_window = false; 72 | bool not_occupied = false; 73 | bool less_dist = false; 74 | bool equal = false; 75 | }; 76 | 77 | using lock_manager_t = std::unique_ptr>; 78 | using lock_list_t = std::vector; 79 | 80 | using epoch_array_t = std::vector; 81 | 82 | struct AllUnlocker { 83 | void operator()(rhhash_map* map) const { 84 | for (bucket_t& bucket : map->buckets_) { 85 | bucket.unlock(); 86 | } 87 | } 88 | }; 89 | using AllLocksManager = std::unique_ptr; 90 | 91 | using bucket_iterator = typename buckets_t::iterator; 92 | 93 | struct rh_data { 94 | bucket_iterator it; 95 | 96 | const size_type original_index; 97 | 98 | const size_t hv; 99 | const int16_t hp; 100 | 101 | const uint16_t window_size; 102 | uint16_t dist; 103 | uint16_t slot; 104 | 105 | bool out_of_window() const { 106 | return dist > window_size; 107 | } 108 | }; 109 | 110 | struct rh_path_data { 111 | bucket_iterator it; 112 | 113 | uint16_t current_dist; 114 | uint16_t dist; 115 | uint16_t slot; 116 | }; 117 | public: 118 | rhhash_map(size_type n = DEFAULT_SIZE, const Hash &hf = Hash(), 119 | const KeyEqual &equal = KeyEqual(), 120 | const Allocator &alloc = Allocator()) 121 | : hash_fn_(hf), eq_fn_(equal), 122 | buckets_(reserve_calc_for_slots(n + MAX_WINDOW_SIZE + 1), alloc){} 123 | 124 | size_type size() const { 125 | counter_type s = 0; 126 | for (bucket_t& bucket : buckets_) { 127 | s += bucket.elem_counter(); 128 | } 129 | assert(s >= 0); 130 | return static_cast(s); 131 | } 132 | 133 | template bool insert(K&& key, Args&& ...val) { 134 | return insert_fn([](mapped_type&) {}, std::forward(key), std::forward(val)...); 135 | } 136 | 137 | template bool insert_or_assign(K&& key, Val&& val) { 138 | return insert_fn([&val](mapped_type& value) { 139 | mapped_type v(std::forward(val)); 140 | atomic_store_memcpy(&value, v); 141 | }, std::forward(key), std::forward(val)); 142 | } 143 | template bool insert_or_assign(K&& key, const mapped_type& val) { 144 | return insert_fn([&val](mapped_type& value) { 145 | atomic_store_memcpy(&value, val); 146 | }, std::forward(key), val); 147 | } 148 | 149 | template 150 | bool upsert(K&& key, F fn, Args&& ...val) { 151 | return insert_fn([&fn](mapped_type& value) { 152 | update_safely(value, fn); 153 | }, std::forward(key), std::forward(val)...); 154 | } 155 | 156 | template 157 | bool update_fn(const K &key, F fn) { 158 | static constexpr cycle_checks update_fn_checks { 159 | .out_of_window = true, 160 | .not_occupied = true, 161 | .less_dist = true, 162 | .equal = true 163 | }; 164 | 165 | while (true) { 166 | rh_data data = get_rh_data(key); 167 | 168 | lock_list_t locks; 169 | locks.reserve(data.window_size); 170 | if (!lock_first(data, locks)) { 171 | continue; 172 | } 173 | 174 | switch (abstract_cycle(key, data, locks)) { 175 | case cycle_status::OUT_OF_WINDOW: 176 | case cycle_status::NOT_OCCUPIED: 177 | case cycle_status::LESS_DIST: 178 | return false; 179 | case cycle_status::EQUAL: 180 | update_safely(data.it->mapped(data.slot), fn); 181 | return true; 182 | } 183 | } 184 | } 185 | 186 | template bool update(const K& key, V&& val) { 187 | return update_fn(key, [&val](mapped_type &v) { v = std::forward(val); }); 188 | } 189 | 190 | template bool update(const K &key, const mapped_type& val) { 191 | return update_fn(key, [&val](mapped_type &v) { atomic_store_memcpy(&v, val); }); 192 | } 193 | 194 | template bool find(const K& key, mapped_type& val) const { 195 | while (true) { 196 | rh_data data = get_rh_data(key); 197 | 198 | epoch_array_t epochs; 199 | epochs.reserve(data.window_size); 200 | collect_epochs_status status = collect_epochs(key, data, epochs); 201 | 202 | if (status == collect_epochs_status::RETRY) { 203 | continue; 204 | } 205 | 206 | typename bucket_t::mapped_raw_type storage; 207 | if (status == collect_epochs_status::FOUND) { 208 | atomic_load_memcpy(reinterpret_cast(&storage), data.it->mapped(data.slot)); 209 | } 210 | std::atomic_thread_fence(std::memory_order_acquire); 211 | 212 | if (!check_epochs(data, epochs)) { 213 | continue; 214 | } 215 | 216 | if (status == collect_epochs_status::FOUND) { 217 | val = *reinterpret_cast(&storage); 218 | } 219 | 220 | return status == collect_epochs_status::FOUND; 221 | } 222 | } 223 | 224 | template mapped_type find(const K &key) const { 225 | mapped_type value; 226 | if (find(key, value)) { 227 | return value; 228 | } 229 | 230 | throw std::out_of_range("key not found in table"); 231 | } 232 | 233 | template bool erase(const K& key) { 234 | static constexpr cycle_checks erase_checks { 235 | .out_of_window = true, 236 | .not_occupied = true, 237 | .less_dist = true, 238 | .equal = true 239 | }; 240 | 241 | while (true) { 242 | rh_data data = get_rh_data(key); 243 | 244 | lock_list_t locks; 245 | locks.reserve(data.window_size); 246 | if (!lock_first(data, locks)) { 247 | continue; 248 | } 249 | 250 | switch (abstract_cycle(key, data, locks)) { 251 | case cycle_status::OUT_OF_WINDOW: 252 | case cycle_status::NOT_OCCUPIED: 253 | case cycle_status::LESS_DIST: 254 | return false; 255 | case cycle_status::EQUAL: 256 | del_from_bucket(*data.it, data.slot); 257 | data.slot = (data.original_index + data.dist) & BUCKET_MASK; 258 | ++data.dist; 259 | ++data.slot; 260 | 261 | while (true) { 262 | for (; data.slot < SLOT_PER_BUCKET; ++data.slot) { 263 | if (!data.it->occupied(data.slot) || 264 | data.it->dist(data.slot) == 0) { 265 | return true; 266 | } 267 | 268 | set_on_prev(data.it, data.slot); 269 | } 270 | 271 | before_next_bucket(data, locks); 272 | } 273 | return true; 274 | } 275 | } 276 | } 277 | 278 | private: 279 | template rh_data get_rh_data(const K& key) const { 280 | const int16_t hp = hashpower(); 281 | const size_t hv = hash_fn_(key); 282 | const size_type original_index = get_original_index(hp, hv); 283 | return { 284 | .it = buckets_.get_iterator(get_bucket_index(original_index)), 285 | .original_index = original_index, 286 | .hv = hv, 287 | .hp = hp, 288 | .window_size = calc_window_size(hp), 289 | .dist = 0, 290 | .slot = get_slot(original_index) 291 | }; 292 | } 293 | 294 | rh_path_data get_rh_path_data(const rh_data& data) const { 295 | return { 296 | .it = data.it, 297 | .current_dist = static_cast(data.it->dist(data.slot) + 1), 298 | .dist = static_cast(data.dist + 1), 299 | .slot = static_cast(data.slot + 1), 300 | }; 301 | } 302 | 303 | template 304 | cycle_status abstract_cycle(const K& key, Data& data, lock_list_t& lock_array) { 305 | while (true) { 306 | for (; data.slot < SLOT_PER_BUCKET; ++data.slot, ++data.dist) { 307 | if constexpr (Checks.out_of_window) { 308 | if (data.out_of_window()) { 309 | return cycle_status::OUT_OF_WINDOW; 310 | } 311 | } 312 | 313 | if constexpr (Checks.not_occupied) { 314 | if (!data.it->occupied(data.slot)) { 315 | return cycle_status::NOT_OCCUPIED; 316 | } 317 | } 318 | 319 | if constexpr (Checks.less_dist) { 320 | if (data.it->dist(data.slot) < data.dist) { 321 | return cycle_status::LESS_DIST; 322 | } 323 | } 324 | 325 | if constexpr (Checks.equal) { 326 | if (eq_fn_(data.it->key(data.slot), key)) { 327 | return cycle_status::EQUAL; 328 | } 329 | } 330 | } 331 | 332 | if constexpr (Checks.out_of_window) { 333 | if (data.out_of_window()) { 334 | return cycle_status::OUT_OF_WINDOW; 335 | } 336 | } 337 | 338 | before_next_bucket(data, lock_array); 339 | } 340 | } 341 | 342 | bool lock_first(rh_data& data, lock_list_t& locks) { 343 | data.it->lock(); 344 | if (hashpower_changed(data.hp, *data.it)) { 345 | return false; 346 | } 347 | locks.emplace_back(&(*data.it)); 348 | std::atomic_thread_fence(std::memory_order_release); 349 | 350 | return true; 351 | } 352 | 353 | size_type hashsize() const { 354 | return buckets_.size() * SLOT_PER_BUCKET_POW; 355 | } 356 | 357 | int16_t hashpower() const { 358 | return buckets_.hashpower() + SLOT_PER_BUCKET_POW; 359 | } 360 | 361 | inline static uint16_t calc_window_size(uint16_t hp) { 362 | return hp + 1; 363 | } 364 | 365 | size_type get_original_index(const size_type hp, const size_type hv) const { 366 | const size_type size_mask = (size_type(1) << hp) - 1; 367 | const size_type index = hv & size_mask; 368 | const size_type border = size_mask - MAX_WINDOW_SIZE; 369 | return index <= border ? index : index - border; 370 | } 371 | 372 | static constexpr size_type get_bucket_index(size_type original_index) { 373 | return original_index >> SLOT_PER_BUCKET_POW; 374 | } 375 | 376 | static constexpr uint16_t get_slot(size_type index) { 377 | return index & BUCKET_MASK; 378 | } 379 | 380 | static bool no_further_data(const rh_data& data, bool occupied, uint16_t dist) { 381 | return data.out_of_window() || 382 | !occupied || 383 | dist + 1 < data.dist; 384 | } 385 | 386 | template 387 | static void before_next_bucket(rh_data& data, lock_list_t& lock_list) { 388 | ++data.it; 389 | data.slot = 0; 390 | if constexpr (TABLE_MODE == table_mode::UNLOCKED) { 391 | data.it->lock(); 392 | lock_list.emplace_back(&(*data.it)); 393 | } 394 | } 395 | 396 | bool hashpower_changed(int32_t hp, bucket_t& bucket) const { 397 | if (hashpower() != hp) { 398 | bucket.unlock(); 399 | return true; 400 | } 401 | return false; 402 | } 403 | 404 | template 405 | collect_epochs_status collect_epochs(const K& key, rh_data& data, epoch_array_t& epochs) const { 406 | epochs.push_back(data.it->get_epoch()); 407 | if (seqlock_t::is_locked(epochs.back()) || data.hp != hashpower()) { 408 | return collect_epochs_status::RETRY; 409 | } 410 | 411 | while (true) { 412 | for (; data.slot < SLOT_PER_BUCKET; ++data.slot, ++data.dist) { 413 | bool cur_occupied; 414 | uint16_t cur_dist; 415 | 416 | atomic_load_memcpy(&cur_occupied, data.it->occupied(data.slot)); 417 | atomic_load_memcpy(&cur_dist, data.it->dist(data.slot)); 418 | 419 | if (no_further_data(data, cur_occupied, cur_dist)) { 420 | return collect_epochs_status::NOT_FOUND; 421 | } 422 | 423 | typename bucket_t::key_raw_type cur_key; 424 | atomic_load_memcpy(reinterpret_cast(&cur_key), data.it->key(data.slot)); 425 | if (eq_fn_(*reinterpret_cast(&cur_key), key)) { 426 | return collect_epochs_status::FOUND; 427 | } 428 | } 429 | 430 | if (data.out_of_window()) { 431 | return collect_epochs_status::NOT_FOUND; 432 | } 433 | 434 | ++data.it; 435 | epochs.push_back(data.it->get_epoch()); 436 | if (seqlock_t::is_locked(epochs.back())) { 437 | return collect_epochs_status::RETRY; 438 | } 439 | data.slot = 0; 440 | } 441 | } 442 | 443 | bool check_epochs(rh_data& data, const epoch_array_t& epochs) const { 444 | for (uint16_t i = epochs.size(); i > 0; --i) { 445 | if (epochs[i - 1] != data.it->get_epoch(std::memory_order_relaxed)) { 446 | return false; 447 | } 448 | 449 | if (i != 1) { 450 | --data.it; 451 | } 452 | } 453 | 454 | return true; 455 | } 456 | 457 | void set_on_prev(bucket_iterator& it, size_type slot) { 458 | if (slot == 0) { 459 | bucket_iterator prev(it); 460 | --prev; 461 | 462 | add_to_bucket(*prev, SLOT_PER_BUCKET - 1, it->dist(slot) - 1, 463 | it->key(slot), it->mapped(slot)); 464 | --it->elem_counter(); 465 | } else { 466 | buckets_.setKV(*it, slot - 1, it->dist(slot) - 1, it->key(slot), it->mapped(slot)); 467 | } 468 | buckets_.deoccupy(*it, slot); 469 | } 470 | 471 | template 472 | void add_to_bucket(bucket_t& bucket, const size_type slot, 473 | const uint16_t dist, K&& key, Args&& ...val) const { 474 | buckets_.setKV(bucket, slot, dist, std::forward(key), std::forward(val)...); 475 | ++bucket.elem_counter(); 476 | } 477 | 478 | void del_from_bucket(bucket_t& bucket, const size_type slot) const { 479 | buckets_.deoccupy(bucket, slot); 480 | --bucket.elem_counter(); 481 | } 482 | 483 | template 484 | bool rehash_required(const rh_data& data, lock_list_t& lock_array, F fn, K&& key, Args&& ...val) { 485 | lock_array.clear(); 486 | rh_fast_double(data.hp); 487 | return insert_fn(fn, std::forward(key), std::forward(val)...); 488 | } 489 | 490 | template 491 | bool insert_fn(F fn, K&& key, Args&& ...val) { 492 | static constexpr cycle_checks insert_fn_checks { 493 | .out_of_window = true, 494 | .not_occupied = true, 495 | .less_dist = true, 496 | .equal = true 497 | }; 498 | 499 | while (true) { 500 | rh_data data = get_rh_data(key); 501 | 502 | lock_list_t locks; 503 | locks.reserve(data.window_size); 504 | if constexpr (TABLE_MODE == table_mode::UNLOCKED) { 505 | if (!lock_first(data, locks)) { 506 | continue; 507 | } 508 | } 509 | 510 | switch (abstract_cycle(key, data, locks)) { 511 | case cycle_status::OUT_OF_WINDOW: 512 | return rehash_required(data, locks, fn, std::forward(key), std::forward(val)...); 513 | case cycle_status::NOT_OCCUPIED: 514 | add_to_bucket(*data.it, data.slot, 515 | data.dist, std::forward(key), std::forward(val)...); 516 | return true; 517 | case cycle_status::LESS_DIST: 518 | if constexpr (TABLE_MODE == table_mode::UNLOCKED) { 519 | if (!path_exists(data, locks)) { 520 | return rehash_required(data, locks, fn, std::forward(key), std::forward(val)...); 521 | } 522 | } 523 | 524 | move_path(data, std::forward(key), std::forward(val)...); 525 | return true; 526 | case cycle_status::EQUAL: 527 | fn(data.it->mapped(data.slot)); 528 | return false; 529 | } 530 | } 531 | } 532 | 533 | template 534 | bool path_exists(const rh_data& data, lock_list_t& lock_list) { 535 | rh_path_data path_data = get_rh_path_data(data); 536 | while (true) { 537 | for (; path_data.slot < SLOT_PER_BUCKET; ++path_data.current_dist, ++path_data.dist, ++path_data.slot) { 538 | if (path_data.dist > data.window_size) { 539 | return false; 540 | } 541 | 542 | if (!path_data.it->occupied(path_data.slot)) { 543 | return true; 544 | } else if (path_data.it->dist(path_data.slot) < path_data.current_dist) { 545 | path_data.current_dist = path_data.it->dist(path_data.slot); 546 | } 547 | } 548 | 549 | if (path_data.dist > data.window_size) { 550 | return false; 551 | } 552 | 553 | ++path_data.it; 554 | path_data.slot = 0; 555 | if constexpr (!LOCKED) { 556 | path_data.it->lock(); 557 | lock_list.emplace_back(&(*path_data.it)); 558 | } 559 | } 560 | } 561 | 562 | template 563 | void move_path(rh_data& data, K&& key, Args&& ...val) { 564 | typename bucket_t::storage_value_type storage(data.it->storage_kvpair(data.slot)); 565 | uint16_t tmp_dist = data.it->dist(data.slot); 566 | buckets_.deoccupy(*data.it, data.slot); 567 | buckets_.setKV(*data.it, data.slot, data.dist, std::forward(key), std::forward(val)...); 568 | 569 | ++data.slot; 570 | data.dist = tmp_dist + 1; 571 | while (true) { 572 | for (; data.slot < SLOT_PER_BUCKET; ++data.slot, ++data.dist) { 573 | if (!data.it->occupied(data.slot)) { 574 | add_to_bucket(*data.it, data.slot, data.dist, 575 | storage.first, storage.second); 576 | return; 577 | } else if (data.it->dist(data.slot) < data.dist) { 578 | typename bucket_t::storage_value_type tmp = storage; 579 | storage = data.it->storage_kvpair(data.slot); 580 | 581 | tmp_dist = data.it->dist(data.slot); 582 | buckets_.deoccupy(*data.it, data.slot); 583 | buckets_.setKV(*data.it, data.slot, data.dist, tmp.first, tmp.second); 584 | data.dist = tmp_dist; 585 | } 586 | } 587 | 588 | ++data.it; 589 | data.slot = 0; 590 | } 591 | } 592 | 593 | AllLocksManager lock_all() { 594 | bucket_iterator it = buckets_.begin(); 595 | it->lock(); 596 | ++it; 597 | for (; it != buckets_.end(); ++it) { 598 | it->lock(); 599 | } 600 | return AllLocksManager(this, AllUnlocker()); 601 | } 602 | 603 | AllLocksManager rh_fast_double(int16_t current_hp) { 604 | const int16_t new_hp = current_hp + 1; 605 | auto all_locks_manager = lock_all(); 606 | 607 | if (hashpower() != current_hp) { 608 | return nullptr; 609 | } 610 | 611 | buckets_.double_size(true/*locked*/); 612 | std::atomic_thread_fence(std::memory_order_release); 613 | 614 | bucket_iterator it = buckets_.begin(); 615 | uint16_t free_behind = 0; 616 | for (size_type index = 0; index < (size_type(1) << current_hp); ++it) { 617 | for (uint16_t slot = 0; slot < SLOT_PER_BUCKET; ++slot, ++index) { 618 | if (!it->occupied(slot)) { 619 | ++free_behind; 620 | continue; 621 | } 622 | 623 | size_type original_index = get_original_index(new_hp, hash_fn_(it->key(slot))); 624 | 625 | // change location 626 | if (original_index > index) { 627 | insert_fn([](mapped_type&) {}, it->key(slot), it->mapped(slot)); 628 | del_from_bucket(*it, slot); 629 | ++free_behind; 630 | } else if (it->dist(slot) != 0 && free_behind > 0) { 631 | size_type best_index = std::max(original_index, index - free_behind); 632 | size_type new_slot = get_slot(best_index); 633 | 634 | add_to_bucket(buckets_[get_bucket_index(best_index)], new_slot, 635 | best_index - original_index, it->key(slot), it->mapped(slot)); 636 | del_from_bucket(*it, slot); 637 | 638 | free_behind = index - best_index; 639 | } else { 640 | free_behind = 0; 641 | } 642 | } 643 | } 644 | 645 | return all_locks_manager; 646 | } 647 | 648 | private: 649 | static constexpr uint16_t SLOT_PER_BUCKET_POW = std::countr_zero(SLOT_PER_BUCKET); 650 | static_assert((size_t(1) << SLOT_PER_BUCKET_POW) == SLOT_PER_BUCKET); 651 | static constexpr uint16_t BUCKET_MASK = SLOT_PER_BUCKET - 1; 652 | 653 | static constexpr uint16_t MAX_WINDOW_SIZE = 64; 654 | 655 | hasher hash_fn_; 656 | key_equal eq_fn_; 657 | mutable buckets_t buckets_; 658 | }; 659 | 660 | } // seqlock_lib::rh 661 | -------------------------------------------------------------------------------- /scripts/init_cmake.sh: -------------------------------------------------------------------------------- 1 | rm -rf build/ 2 | mkdir build/ 3 | cd build/ 4 | 5 | mkdir debug/ 6 | mkdir release/ 7 | 8 | cd debug/ 9 | cmake -DCMAKE_BUILD_TYPE=Debug ../.. 10 | 11 | cd ../release 12 | cmake -DCMAKE_BUILD_TYPE=Release ../.. 13 | -------------------------------------------------------------------------------- /scripts/run_bench.sh: -------------------------------------------------------------------------------- 1 | cmake --build build/release --config Release --target bench 2 | build/release/bench/bench --benchmark_out=bench_out.json --benchmark_out_format=json --benchmark_threads_step=2 --benchmark_threads_count=16 3 | -------------------------------------------------------------------------------- /scripts/run_stress_checked.sh: -------------------------------------------------------------------------------- 1 | cmake --build build/debug --config Debug --target stress_checked 2 | build/debug/tests/cuckoo/stress_tests/stress_checked --power 16 --thread-num 16 --time 5 --use-big-objects 3 | -------------------------------------------------------------------------------- /scripts/run_stress_unchecked.sh: -------------------------------------------------------------------------------- 1 | cmake --build build/release --config Release --target stress_unchecked 2 | build/release/tests/cuckoo/stress_tests/stress_unchecked --power 26 --thread-num 16 --time 15 --use-big-objects 3 | -------------------------------------------------------------------------------- /scripts/run_unit.sh: -------------------------------------------------------------------------------- 1 | cmake --build build/debug --config Debug --target unit_tests 2 | build/debug/tests/cuckoo/unit_tests/unit_tests 3 | -------------------------------------------------------------------------------- /tests/cuckoo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(test_utils INTERFACE) 2 | target_include_directories(test_utils INTERFACE 3 | $ 4 | $ 5 | ) 6 | 7 | option (CUCKOO_SEQLOCK_BUILD_TESTS "build all cuckoo tests" ON) 8 | option (CUCKOO_SEQLOCK_BUILD_STRESS_TESTS "build the stress tests" ON) 9 | option (CUCKOO_SEQLOCK_BUILD_UNIT_TESTS "build the unit tests" ON) 10 | 11 | if (CUCKOO_SEQLOCK_BUILD_TESTS OR CUCKOO_SEQLOCK_BUILD_UNIT_TESTS) 12 | add_subdirectory(unit_tests) 13 | endif() 14 | 15 | if (CUCKOO_SEQLOCK_BUILD_TESTS OR CUCKOO_SEQLOCK_BUILD_STRESS_TESTS) 16 | add_subdirectory(stress_tests) 17 | endif() 18 | -------------------------------------------------------------------------------- /tests/cuckoo/stress_tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(stress_checked stress_checked.cpp) 2 | target_link_libraries(stress_checked PRIVATE test_utils GTest::gtest_main) 3 | 4 | add_executable(stress_unchecked stress_unchecked.cpp) 5 | target_link_libraries(stress_unchecked PRIVATE test_utils GTest::gtest_main) 6 | 7 | add_test(NAME stress_checked COMMAND stress_checked) 8 | add_test(NAME stress_unchecked COMMAND stress_unchecked) 9 | -------------------------------------------------------------------------------- /tests/cuckoo/stress_tests/stress_checked.cpp: -------------------------------------------------------------------------------- 1 | // Tests concurrent inserts, deletes, updates, and finds. The test makes sure 2 | // that multiple operations are not run on the same key, so that the accuracy of 3 | // the operations can be verified. 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "test_utils.hpp" 10 | 11 | using KeyType = uint32_t; 12 | using KeyType2 = big_object<5>; 13 | using ValType = uint32_t; 14 | using ValType2 = big_object<10>; 15 | 16 | // The number of keys to size the table with, expressed as a power of 17 | // 2. This can be set with the command line flag --power 18 | size_t g_power = 25; 19 | size_t g_numkeys; // Holds 2^power 20 | // The number of threads spawned for each type of operation. This can 21 | // be set with the command line flag --thread-num 22 | size_t g_thread_num = 4; 23 | // Whether to disable inserts or not. This can be set with the command 24 | // line flag --disable-inserts 25 | bool g_disable_inserts = false; 26 | // Whether to disable deletes or not. This can be set with the command 27 | // line flag --disable-deletes 28 | bool g_disable_deletes = false; 29 | // Whether to disable updates or not. This can be set with the command 30 | // line flag --disable-updates 31 | bool g_disable_updates = false; 32 | // Whether to disable finds or not. This can be set with the command 33 | // line flag --disable-finds 34 | bool g_disable_finds = false; 35 | // How many seconds to run the test for. This can be set with the 36 | // command line flag --time 37 | size_t g_test_len = 10; 38 | // The seed for the random number generator. If this isn't set to a 39 | // nonzero value with the --seed flag, the current time is used 40 | size_t g_seed = 0; 41 | // Whether to use big_objects as the value and key 42 | bool g_use_big_objects = false; 43 | 44 | std::atomic num_inserts; 45 | std::atomic num_deletes; 46 | std::atomic num_updates; 47 | std::atomic num_finds; 48 | 49 | template typename Map> 50 | class AllEnvironment { 51 | public: 52 | AllEnvironment() 53 | : table(64), table2(64), keys(g_numkeys), vals(g_numkeys), 54 | vals2(g_numkeys), in_table(new bool[g_numkeys]), 55 | in_use(new std::atomic_flag[g_numkeys]), 56 | val_dist(std::numeric_limits::min(), 57 | std::numeric_limits::max()), 58 | val_dist2(std::numeric_limits::min(), 59 | std::numeric_limits::max()), 60 | ind_dist(0, g_numkeys - 1), finished(false) { 61 | //table.max_num_worker_threads(16); 62 | // Sets up the random number generator 63 | if (g_seed == 0) { 64 | g_seed = std::chrono::system_clock::now().time_since_epoch().count(); 65 | } 66 | std::cout << "seed = " << g_seed << std::endl; 67 | gen_seed = g_seed; 68 | // Fills in all the vectors except vals, which will be filled 69 | // in by the insertion threads. 70 | for (size_t i = 0; i < g_numkeys; i++) { 71 | keys[i] = generateKey(i); 72 | in_table[i] = false; 73 | in_use[i].clear(); 74 | } 75 | } 76 | 77 | Map table; 78 | Map table2; 79 | std::vector keys; 80 | std::vector vals; 81 | std::vector vals2; 82 | std::unique_ptr in_table; 83 | std::unique_ptr in_use; 84 | std::uniform_int_distribution val_dist; 85 | std::uniform_int_distribution val_dist2; 86 | std::uniform_int_distribution ind_dist; 87 | size_t gen_seed; 88 | // When set to true, it signals to the threads to stop running 89 | std::atomic finished; 90 | }; 91 | 92 | template typename Map> 93 | void stress_insert_thread(AllEnvironment *env) { 94 | std::mt19937 gen(env->gen_seed); 95 | while (!env->finished.load()) { 96 | // Pick a random number between 0 and g_numkeys. If that slot is 97 | // not in use, lock the slot. Insert a random value into both 98 | // tables. The inserts should only be successful if the key 99 | // wasn't in the table. If the inserts succeeded, check that 100 | // the insertion were actually successful with another find 101 | // operation, and then store the values in their arrays and 102 | // set in_table to true and clear in_use 103 | size_t ind = env->ind_dist(gen); 104 | if (!env->in_use[ind].test_and_set()) { 105 | KType k = env->keys[ind]; 106 | ValType v = env->val_dist(gen); 107 | ValType2 v2 = env->val_dist2(gen); 108 | bool res = env->table.insert(k, v); 109 | bool res2 = env->table2.insert(k, v2); 110 | 111 | ASSERT_NE(res, env->in_table[ind]); 112 | ASSERT_NE(res2, env->in_table[ind]); 113 | if (res) { 114 | ASSERT_EQ(v, env->table.find(k)); 115 | ASSERT_EQ(v2, env->table2.find(k)); 116 | env->vals[ind] = v; 117 | env->vals2[ind] = v2; 118 | env->in_table[ind] = true; 119 | num_inserts.fetch_add(2, std::memory_order_relaxed); 120 | } 121 | env->in_use[ind].clear(); 122 | } 123 | } 124 | } 125 | 126 | template typename Map> 127 | void delete_thread(AllEnvironment *env) { 128 | std::mt19937 gen(env->gen_seed); 129 | while (!env->finished.load()) { 130 | // Run deletes on a random key, check that the deletes 131 | // succeeded only if the keys were in the table. If the 132 | // deletes succeeded, check that the keys are indeed not in 133 | // the tables anymore, and then set in_table to false 134 | size_t ind = env->ind_dist(gen); 135 | if (!env->in_use[ind].test_and_set()) { 136 | KType k = env->keys[ind]; 137 | bool res = env->table.erase(k); 138 | bool res2 = env->table2.erase(k); 139 | ASSERT_EQ(res, env->in_table[ind]); 140 | ASSERT_EQ(res2, env->in_table[ind]); 141 | if (res) { 142 | ValType find_v = 0; 143 | ValType2 find_v2 = 0; 144 | ASSERT_FALSE(env->table.find(k, find_v)); 145 | ASSERT_FALSE(env->table2.find(k, find_v2)); 146 | env->in_table[ind] = false; 147 | num_deletes.fetch_add(2, std::memory_order_relaxed); 148 | } 149 | env->in_use[ind].clear(); 150 | } 151 | } 152 | } 153 | 154 | template typename Map> 155 | void update_thread(AllEnvironment *env) { 156 | std::mt19937 gen(env->gen_seed); 157 | std::uniform_int_distribution third(0, 2); 158 | auto updatefn = [](ValType &v) { v += 3; }; 159 | auto updatefn2 = [](ValType2 &v) { v += 10; }; 160 | while (!env->finished.load()) { 161 | // Run updates, update_fns, or upserts on a random key, check 162 | // that the operations succeeded only if the keys were in the 163 | // table (or that they succeeded regardless if it's an 164 | // upsert). If successful, check that the keys are indeed in 165 | // the table with the new value, and then set in_table to true 166 | size_t ind = env->ind_dist(gen); 167 | if (!env->in_use[ind].test_and_set()) { 168 | KType k = env->keys[ind]; 169 | ValType v; 170 | ValType2 v2; 171 | bool res, res2; 172 | switch (third(gen)) { 173 | case 0: 174 | // update 175 | v = env->val_dist(gen); 176 | v2 = env->val_dist2(gen); 177 | res = env->table.update(k, v); 178 | res2 = env->table2.update(k, v2); 179 | ASSERT_EQ(res, env->in_table[ind]); 180 | ASSERT_EQ(res2, env->in_table[ind]); 181 | break; 182 | case 1: 183 | // update_fn 184 | v = env->vals[ind]; 185 | v2 = env->vals2[ind]; 186 | updatefn(v); 187 | updatefn2(v2); 188 | res = env->table.update_fn(k, updatefn); 189 | res2 = env->table2.update_fn(k, updatefn2); 190 | ASSERT_EQ(res, env->in_table[ind]); 191 | ASSERT_EQ(res2, env->in_table[ind]); 192 | break; 193 | case 2: 194 | // upsert 195 | if (env->in_table[ind]) { 196 | // Then it should run updatefn 197 | v = env->vals[ind]; 198 | v2 = env->vals2[ind]; 199 | updatefn(v); 200 | updatefn2(v2); 201 | } else { 202 | // Then it should run an insert 203 | v = env->val_dist(gen); 204 | v2 = env->val_dist2(gen); 205 | } 206 | // These upserts should always succeed, so set res and res2 to 207 | // true. 208 | env->table.upsert(k, updatefn, v); 209 | env->table2.upsert(k, updatefn2, v2); 210 | res = res2 = true; 211 | env->in_table[ind] = true; 212 | break; 213 | default: 214 | throw std::logic_error("Impossible"); 215 | } 216 | if (res) { 217 | ASSERT_EQ(v, env->table.find(k)); 218 | ASSERT_EQ(v2, env->table2.find(k)); 219 | env->vals[ind] = v; 220 | env->vals2[ind] = v2; 221 | num_updates.fetch_add(2, std::memory_order_relaxed); 222 | } 223 | env->in_use[ind].clear(); 224 | } 225 | } 226 | } 227 | 228 | template typename Map> 229 | void find_thread(AllEnvironment *env) { 230 | std::mt19937 gen(env->gen_seed); 231 | while (!env->finished.load()) { 232 | // Run finds on a random key and check that the presence of 233 | // the keys matches in_table 234 | size_t ind = env->ind_dist(gen); 235 | if (!env->in_use[ind].test_and_set()) { 236 | KType k = env->keys[ind]; 237 | try { 238 | ASSERT_EQ(env->vals[ind], env->table.find(k)); 239 | ASSERT_TRUE(env->in_table[ind]); 240 | } catch (const std::out_of_range &) { 241 | ASSERT_FALSE(env->in_table[ind]); 242 | } 243 | try { 244 | ASSERT_EQ(env->vals2[ind], env->table2.find(k)); 245 | ASSERT_TRUE(env->in_table[ind]); 246 | } catch (const std::out_of_range &) { 247 | ASSERT_FALSE(env->in_table[ind]); 248 | } 249 | num_finds.fetch_add(2, std::memory_order_relaxed); 250 | env->in_use[ind].clear(); 251 | } 252 | } 253 | } 254 | 255 | // Spawns g_thread_num insert, delete, update, and find threads 256 | template typename Map> 257 | void StressTest(AllEnvironment *env) { 258 | std::vector threads; 259 | for (size_t i = 0; i < g_thread_num; i++) { 260 | if (!g_disable_inserts) { 261 | threads.emplace_back(stress_insert_thread, env); 262 | } 263 | if (!g_disable_deletes) { 264 | threads.emplace_back(delete_thread, env); 265 | } 266 | if (!g_disable_updates) { 267 | threads.emplace_back(update_thread, env); 268 | } 269 | if (!g_disable_finds) { 270 | threads.emplace_back(find_thread, env); 271 | } 272 | } 273 | // Sleeps before ending the threads 274 | std::this_thread::sleep_for(std::chrono::seconds(g_test_len)); 275 | env->finished.store(true); 276 | for (size_t i = 0; i < threads.size(); i++) { 277 | threads[i].join(); 278 | } 279 | // Finds the number of slots that are filled 280 | size_t numfilled = 0; 281 | for (size_t i = 0; i < g_numkeys; i++) { 282 | if (env->in_table[i]) { 283 | numfilled++; 284 | } 285 | } 286 | ASSERT_EQ(numfilled, env->table.size()); 287 | std::cout << "----------Results----------" << std::endl; 288 | std::cout << "Number of inserts:\t" << num_inserts.load() << std::endl; 289 | std::cout << "Number of deletes:\t" << num_deletes.load() << std::endl; 290 | std::cout << "Number of updates:\t" << num_updates.load() << std::endl; 291 | std::cout << "Number of finds:\t" << num_finds.load() << std::endl; 292 | } 293 | 294 | template