├── LICENSE ├── README.md └── src ├── CMakeLists.txt ├── allocators ├── glib_allocator.h ├── intel.h ├── jemalloc_allocator.h └── super_malloc_allocator.h ├── bench ├── action_generator.h ├── allocators.cpp ├── allocators.h ├── arg_parsing.cpp ├── arg_parsing.h ├── benchmark_config.cpp ├── benchmark_config.h ├── benchmark_results.h ├── benchmark_summary.cpp ├── benchmark_summary.h ├── benchmark_table.h ├── random.cpp ├── random.h ├── table.cpp ├── table.h ├── thread_papi_wrapper.cpp ├── thread_papi_wrapper.h ├── thread_pinner.h ├── utils.cpp └── utils.h ├── cmake └── FindPAPI.cmake ├── hash-tables ├── common.h ├── hs_locked.h ├── hs_serial.h ├── hs_trans.h ├── hsbm_lf.h ├── hsbm_locked.h ├── main.cpp ├── ph_qp.h └── table_init.h ├── mem-reclaimer ├── leaky.h ├── reclaimer.cpp └── reclaimer.h ├── primitives ├── backoff.cpp ├── backoff.h ├── barrier.h ├── brown_kcas.h ├── brown_original_kcas.h ├── cache_utils.h ├── harris_kcas.h ├── locks.cpp ├── locks.h └── marked_pointers.h └── random ├── lcg.h ├── pcg_extras.h ├── pcg_random.h ├── pcg_uint128.h └── xorshift.h /README.md: -------------------------------------------------------------------------------- 1 | # Lock-Free Hopscotch Hashing 2 | 3 | ## Tables present 4 | 1. Lock-Free Hopscotch Hashing 5 | 2. Locked Hopscotch (Bit-map variant) 6 | 3. Locked Hopscotch (Relative offset variant) 7 | 4. Purcell-Harris lock-free quadratic programming. 8 | 9 | ## Build instruction 10 | These benchmarks require a number of dependencies. 11 | * [A CPU topology library for smart thread pinning](https://github.com/Maratyszcza/cpuinfo) 12 | * [JeMalloc](https://github.com/jemalloc/jemalloc) 13 | * CMake (Installed via system package manager) 14 | * PAPI (Installed via system package manager) 15 | * Boost (Installed via system package manager) 16 | 17 | Our code was tested and ran on Ubuntu 14.04, 16.04, and 18.04. The code itself uses the CMake build system. When testing make sure to compile the code in release! Or at least *our code*... 18 | 19 | ## Run instructions 20 | Once built the binary takes a number of arguments at command-line parameters and through standard input. 21 | 22 | Brief explanation of the command. 23 | * -T ==> Number of threads 24 | * -L ==> Load factor 1.0 is full, 0.0 is empty, 0.4 is 40% full, etc. 25 | * -S ==> Table size as a power of 2. 26 | * -D ==> Number of seconds to run the benchmark procedure for. 27 | * -U ==> Percentage updates 28 | * -B ==> Table to benchmark 29 | * -M ==> Memory reclaimer 30 | * -A ==> What allocator to use. 31 | * -P ==> Whether PAPI is turned on. 32 | * -R ==> Choice of random number generator 33 | 34 | Here are some example commands. All parameters have default values if none are provided. 35 | 36 | * ./concurrent_hash_tables -T 1 -L 0.6 -S 23 -D 10 -U 10 -P true -M leaky -A je -R xor_shift_64 -B hsbm_lf_set 37 | * ./concurrent_hash_tables -T 4 -L 0.6 -S 23 -D 10 -U 30 -P true -M leaky -A je -R xor_shift_64 -B hsbm_locked_set 38 | * ./concurrent_hash_tables -T 8 -L 0.8 -S 23 -D 10 -U 40 -P true -M leaky -A je -R xor_shift_64 -B ph_qp_set 39 | 40 | The results are put into two csv files, one containing the keys and the other containing the specific info. -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(concurrent_data_structures) 2 | cmake_minimum_required(VERSION 3.1) 3 | 4 | 5 | include_directories(.) 6 | 7 | find_package(Boost COMPONENTS system filesystem REQUIRED) 8 | FILE(GLOB_RECURSE HeaderCFiles "*.h") 9 | 10 | 11 | aux_source_directory(allocators COMMON_LIST) 12 | aux_source_directory(bench COMMON_LIST) 13 | aux_source_directory(mem-reclaimer COMMON_LIST) 14 | aux_source_directory(primitives COMMON_LIST) 15 | aux_source_directory(sets HASH_TABLE) 16 | aux_source_directory(random RANDOM_GENERATORS) 17 | 18 | 19 | 20 | set(HASH_TABLE_EXE "concurrent_hash_tables") 21 | 22 | set(LIBS 23 | ${Boost_FILESYSTEM_LIBRARY} 24 | ${Boost_SYSTEM_LIBRARY} 25 | -latomic 26 | -lpapi 27 | -ljemalloc 28 | -lsupermalloc 29 | -lrt 30 | -lcpuinfo 31 | -ltbb 32 | -ltbbmalloc 33 | -lpthread) 34 | 35 | add_executable(${HASH_TABLE_EXE} hash-tables/main.cpp ${COMMON_LIST} ${HeaderCFiles}) 36 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -mrtm -Wall -fno-omit-frame-pointer -flto") 37 | 38 | 39 | TARGET_LINK_LIBRARIES(${HASH_TABLE_EXE} ${LIBS}) 40 | -------------------------------------------------------------------------------- /src/allocators/glib_allocator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple wrapper around standard malloc. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | struct GlibcAllocator { 25 | static void *malloc(size_t size) { return std::malloc(size); } 26 | static void *aligned_alloc(size_t alignment, size_t size) { 27 | return ::aligned_alloc(alignment, size); 28 | } 29 | static void free(void *ptr) { std::free(ptr); } 30 | static size_t malloc_usable_size(void *ptr) { 31 | return ::malloc_usable_size(ptr); 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /src/allocators/intel.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple wrapper around Intel scalable allocator. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | struct IntelAllocator { 24 | static void *malloc(size_t size) { return scalable_malloc(size); } 25 | static void *aligned_alloc(size_t alignment, size_t size) { 26 | return scalable_aligned_malloc(alignment, size); 27 | } 28 | static void free(void *ptr) { scalable_free(ptr); } 29 | static size_t malloc_usable_size(void *ptr) { return scalable_msize(ptr); } 30 | }; 31 | -------------------------------------------------------------------------------- /src/allocators/jemalloc_allocator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple wrapper around the mangled JeMalloc allocator. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | struct JeMallocAllocator { 24 | static void *malloc(size_t size) { return je_malloc(size); } 25 | static void *aligned_alloc(size_t alignment, size_t size) { 26 | return je_aligned_alloc(alignment, size); 27 | } 28 | static void free(void *ptr) { je_free(ptr); } 29 | static size_t malloc_usable_size(void *ptr) { 30 | return je_malloc_usable_size(ptr); 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /src/allocators/super_malloc_allocator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple wrapper around the mangled SuperMalloc allocator. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | extern "C" void *__super_malloc(size_t); 24 | extern "C" void *__super_aligned_alloc(size_t, size_t); 25 | extern "C" void __super_free(void *); 26 | extern "C" size_t __super_malloc_usable_size(void *); 27 | 28 | struct SuperMallocAllocator { 29 | static void *malloc(size_t size) { return __super_malloc(size); } 30 | static void *aligned_alloc(size_t alignment, size_t size) { 31 | return __super_aligned_alloc(alignment, size); 32 | } 33 | static void free(void *ptr) { return __super_free(ptr); } 34 | static size_t malloc_usable_size(void *ptr) { 35 | return __super_malloc_usable_size(ptr); 36 | } 37 | }; 38 | -------------------------------------------------------------------------------- /src/bench/action_generator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Generates random actions for benchmarking tools. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "benchmark_config.h" 22 | #include "random/lcg.h" 23 | #include "random/pcg_random.h" 24 | #include "random/xorshift.h" 25 | #include 26 | #include 27 | 28 | namespace concurrent_data_structures { 29 | 30 | enum class SetAction { 31 | Contains, 32 | Add, 33 | Remove, 34 | }; 35 | 36 | template class SetActionGenerator { 37 | private: 38 | Random m_action_generator; 39 | Random m_key_generator; 40 | const std::uintptr_t m_key_mask; 41 | const std::uintptr_t m_read_limit, m_add_limit; 42 | 43 | public: 44 | SetActionGenerator(const SetBenchmarkConfig &config) 45 | : m_action_generator(std::random_device{}()), 46 | m_key_generator(std::random_device{}()), 47 | m_key_mask((config.table_size) - 1), 48 | m_read_limit((m_key_mask) - 49 | (static_cast(m_key_mask) * 50 | static_cast(config.updates) / 100.0)), 51 | m_add_limit(m_read_limit + 52 | (static_cast(m_key_mask) * 53 | static_cast(config.updates) / 200.0)) {} 54 | SetAction generate_action() { 55 | const std::uintptr_t action = m_action_generator.next_rand() & m_key_mask; 56 | if (action <= m_read_limit) { 57 | return SetAction::Contains; 58 | } else if (action <= m_add_limit) { 59 | return SetAction::Add; 60 | } else { 61 | return SetAction::Remove; 62 | } 63 | } 64 | Key generate_key() { 65 | const Key key = m_key_generator.next_rand() & m_key_mask; 66 | return key; 67 | } 68 | }; 69 | } 70 | -------------------------------------------------------------------------------- /src/bench/allocators.cpp: -------------------------------------------------------------------------------- 1 | #include "allocators.h" 2 | 3 | /* 4 | Maps allocators to human strings. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | namespace concurrent_data_structures { 23 | 24 | namespace { 25 | static const std::map allocator_map{ 26 | std::make_pair(Allocator::JeMalloc, "Je-Malloc"), 27 | std::make_pair(Allocator::Glibc, "Glibc"), 28 | std::make_pair(Allocator::SuperMalloc, "Super-Malloc"), 29 | std::make_pair(Allocator::Intel, "Intel")}; 30 | } 31 | 32 | const std::string get_allocator_name(const Allocator table) { 33 | std::string allocator_name = "ERROR: Incorrect allocator name."; 34 | auto it = allocator_map.find(table); 35 | if (it != allocator_map.end()) { 36 | allocator_name = it->second; 37 | } 38 | return allocator_name; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/bench/allocators.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Allocators used with a C++ template. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace concurrent_data_structures { 28 | 29 | enum class Allocator { 30 | JeMalloc, 31 | Glibc, 32 | SuperMalloc, 33 | Intel, 34 | }; 35 | 36 | template class AllocatorInterface { 37 | template struct allocator_type { typedef Z value_type; }; 38 | 39 | public: 40 | typedef typename allocator_type::value_type value_type; 41 | typedef value_type *pointer; 42 | typedef const value_type *const_pointer; 43 | typedef value_type &reference; 44 | typedef const value_type &const_reference; 45 | typedef std::size_t size_type; 46 | typedef std::ptrdiff_t difference_type; 47 | template struct rebind { 48 | typedef AllocatorInterface other; 49 | }; 50 | 51 | AllocatorInterface() throw() {} 52 | AllocatorInterface(const AllocatorInterface &) throw() {} 53 | template 54 | AllocatorInterface(const AllocatorInterface &) throw() {} 55 | 56 | pointer address(reference x) const { return &x; } 57 | const_pointer address(const_reference x) const { return &x; } 58 | 59 | pointer allocate(size_type n, const void * /*hint*/ = 0) { 60 | return static_cast(GivenAllocator::malloc(n * sizeof(value_type))); 61 | } 62 | 63 | void deallocate(pointer p, size_type) { GivenAllocator::free(p); } 64 | 65 | size_type max_size() const throw() { 66 | size_type absolutemax = static_cast(-1) / sizeof(value_type); 67 | return (absolutemax > 0 ? absolutemax : 1); 68 | } 69 | template 70 | void construct(U *p, Args &&... args) { 71 | ::new ((void *)p) U(std::forward(args)...); 72 | } 73 | void destroy(pointer p) { p->~value_type(); } 74 | }; 75 | 76 | const std::string get_allocator_name(const Allocator allocator); 77 | } 78 | -------------------------------------------------------------------------------- /src/bench/arg_parsing.cpp: -------------------------------------------------------------------------------- 1 | #include "arg_parsing.h" 2 | 3 | /* 4 | Body for parsing args to main. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "allocators.h" 22 | #include "random.h" 23 | #include "table.h" 24 | #include 25 | #include 26 | 27 | namespace concurrent_data_structures { 28 | 29 | namespace { 30 | // Table_Sync_(Opti/Method)_(Set/Map) 31 | static const std::map table_map{ 32 | // Hopscotch Hashing 33 | std::make_pair("hsbm_serial_set", HashTable::HSBM_SERIAL_SET), 34 | std::make_pair("hs_trans_set", HashTable::HS_TRANS_SET), 35 | std::make_pair("hs_locked_set", HashTable::HS_LOCKED_SET), 36 | std::make_pair("hsc_locked_set", HashTable::HSC_LOCKED_SET), 37 | std::make_pair("hsbm_locked_set", HashTable::HSBM_LOCKED_SET), 38 | std::make_pair("hsbm_lf_set", HashTable::HSBM_LF_SET), 39 | // Purcell Harris 40 | std::make_pair("ph_qp_set", HashTable::PH_QP_SET), 41 | }; 42 | 43 | static const std::map generator_map{ 44 | std::make_pair("lcg_bsd", RandomGenerator::LCGBSD), 45 | std::make_pair("xor_shift_32", RandomGenerator::XOR_SHIFT_32), 46 | std::make_pair("xor_shift_64", RandomGenerator::XOR_SHIFT_64), 47 | std::make_pair("pcg", RandomGenerator::PCG), 48 | }; 49 | 50 | static const std::map reclaimer_map{ 51 | std::make_pair("leaky", Reclaimer::Leaky), 52 | }; 53 | 54 | static const std::map allocator_map{ 55 | std::make_pair("je", Allocator::JeMalloc), 56 | std::make_pair("glibc", Allocator::Glibc), 57 | std::make_pair("intel", Allocator::Intel), 58 | }; 59 | 60 | static std::function S_PRINT_AND_EXIT_FUNC; 61 | } 62 | 63 | bool parse_base_arg(BenchmarkConfig &base, const int current_option, 64 | char *arg) { 65 | switch (current_option) { 66 | case 'R': { 67 | auto random_res = generator_map.find(std::string(arg)); 68 | if (generator_map.end() == random_res) { 69 | std::cout << "Invalid benchmark choice." << std::endl; 70 | S_PRINT_AND_EXIT_FUNC(); 71 | } else { 72 | base.generator = random_res->second; 73 | } 74 | return true; 75 | } 76 | case 'D': 77 | base.duration = std::chrono::seconds(std::atoi(arg)); 78 | return true; 79 | case 'T': 80 | base.num_threads = std::size_t(std::atoi(arg)); 81 | return true; 82 | case 'M': { 83 | auto reclaimer_res = reclaimer_map.find(std::string(arg)); 84 | if (reclaimer_map.end() == reclaimer_res) { 85 | std::cout << "Invalid reclaimer choice." << std::endl; 86 | S_PRINT_AND_EXIT_FUNC(); 87 | } else { 88 | base.reclaimer = reclaimer_res->second; 89 | } 90 | return true; 91 | } 92 | case 'A': { 93 | auto allocator_res = allocator_map.find(std::string(arg)); 94 | if (allocator_map.end() == allocator_res) { 95 | std::cout << "Invalid allocator choice." << std::endl; 96 | S_PRINT_AND_EXIT_FUNC(); 97 | } else { 98 | base.allocator = allocator_res->second; 99 | } 100 | return true; 101 | } 102 | case 'P': 103 | base.papi_active = std::string(optarg) == "true"; 104 | return true; 105 | case 'H': 106 | base.hyperthreading = std::string(optarg) == "true"; 107 | return true; 108 | case 'N': 109 | base.results_directory = std::string(optarg); 110 | return true; 111 | } 112 | return false; 113 | } 114 | 115 | SetBenchmarkConfig parse_set_args(std::int32_t argc, char *argv[]) { 116 | S_PRINT_AND_EXIT_FUNC = []() { set_print_help_and_exit(); }; 117 | SetBenchmarkConfig config = { 118 | BenchmarkConfig{1, std::chrono::seconds(1), 119 | boost::filesystem::path{"results"}, 120 | RandomGenerator::XOR_SHIFT_64, Reclaimer::Leaky, 121 | Allocator::JeMalloc, true, false}, 122 | 1 << 23, 10, 0.5, HashTable::HSBM_LF_SET}; 123 | int current_option; 124 | while ((current_option = getopt(argc, argv, ":R:L:S:D:T:U:B:M:P:V:A:H:N:")) != 125 | -1) { 126 | if (parse_base_arg(config.base, current_option, optarg)) { 127 | continue; 128 | } 129 | switch (current_option) { 130 | case 'L': 131 | config.load_factor = std::stod(optarg); 132 | break; 133 | case 'S': 134 | config.table_size = std::size_t(1 << std::atoi(optarg)); 135 | break; 136 | case 'U': 137 | config.updates = std::size_t(std::atoi(optarg)); 138 | break; 139 | case 'B': { // C++ 140 | auto table_res = table_map.find(std::string(optarg)); 141 | if (table_map.end() == table_res) { 142 | std::cout << "Invalid benchmark choice." << std::endl; 143 | S_PRINT_AND_EXIT_FUNC(); 144 | } else { 145 | config.table = table_res->second; 146 | } 147 | } break; 148 | case ':': 149 | default: 150 | S_PRINT_AND_EXIT_FUNC(); 151 | } 152 | } 153 | return config; 154 | } 155 | 156 | void base_print_help() { 157 | std::cout 158 | << "D: Duration of benchmark in seconds. Default = 1 second.\n" 159 | << "T: Number of concurrent threads. Default = 1.\n" 160 | << "M: Memory reclaimer using within table (if needed). Default = None.\n" 161 | << "A: Allocator used within the table. Default = JeMalloc.\n" 162 | << "R: Random number generator used during the benchmark. Default = XOR " 163 | "Shift 64.\n" 164 | << "P: Whether PAPI is turned on or not. Default = True.\n" 165 | << "H: Whether to schedule threads with a preference for " 166 | "Hyperthreading. Default = False.\n" 167 | << "N: Name of results directory. Default = results" << std::endl; 168 | } 169 | 170 | void set_print_help_and_exit() { 171 | base_print_help(); 172 | std::cout << "L: Load Factor. Default = 50%.\n" 173 | << "S: Power of two size. Default = 1 << 23.\n" 174 | << "U: Updates as a percentage of workload. Default = 10%.\n" 175 | << "B: Table being benchmarked. Default = kcas_rh_set.\n" 176 | << std::endl; 177 | exit(0); 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/bench/arg_parsing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Arg parser for main program. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "benchmark_config.h" 22 | #include 23 | #include 24 | #include 25 | 26 | extern int optind; 27 | extern char *optarg; 28 | 29 | namespace concurrent_data_structures { 30 | 31 | SetBenchmarkConfig parse_set_args(std::int32_t argc, char *argv[]); 32 | void set_print_help_and_exit(); 33 | void queue_print_help_and_exit(); 34 | void pqueue_print_help_and_exit(); 35 | void random_print_help_and_exit(); 36 | } 37 | -------------------------------------------------------------------------------- /src/bench/benchmark_config.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark_config.h" 2 | 3 | /* 4 | Configuration class for benchmark. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | void BenchmarkConfig::print(std::ostream &os) const { 26 | os << "Number of threads: " << num_threads << "\n" 27 | << "Benchmark duration: " << duration.count() << "\n" 28 | << "Results directory: " << results_directory.string() << "\n" 29 | << "Memory reclaimer: " << get_reclaimer_name(reclaimer) << "\n" 30 | << "Memory allocator: " << get_allocator_name(allocator) << "\n" 31 | << "Random generator: " << get_generator_name(generator) << "\n" 32 | << "PAPI Enabled: " << (papi_active ? "true" : "false") << "\n" 33 | << "Hypthreading before socket switch: " 34 | << (hyperthreading ? "true" : "false") << std::endl; 35 | } 36 | 37 | void SetBenchmarkConfig::print(std::ostream &os) const { 38 | base.print(os); 39 | os << "Load factor: " << load_factor << "\n" 40 | << "Table size: " << table_size << "\n" 41 | << "Update percentage: " << updates << "\n" 42 | << "Table name: " << get_table_name(table) << "\n"; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/bench/benchmark_config.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Configuration class for benchmarking. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "allocators.h" 22 | #include "mem-reclaimer/reclaimer.h" 23 | #include "random.h" 24 | #include "table.h" 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace concurrent_data_structures { 31 | 32 | struct BenchmarkConfig { 33 | std::size_t num_threads; 34 | std::chrono::seconds duration; 35 | boost::filesystem::path results_directory; 36 | RandomGenerator generator; 37 | Reclaimer reclaimer; 38 | Allocator allocator; 39 | bool papi_active, hyperthreading; 40 | void print(std::ostream &os) const; 41 | }; 42 | 43 | struct SetBenchmarkConfig { 44 | BenchmarkConfig base; 45 | std::size_t table_size, updates; 46 | double load_factor; 47 | HashTable table; 48 | void print(std::ostream &os) const; 49 | }; 50 | } 51 | -------------------------------------------------------------------------------- /src/bench/benchmark_results.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Encodes results of benchmarks. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "primitives/cache_utils.h" 22 | #include "thread_papi_wrapper.h" 23 | #include "thread_pinner.h" 24 | #include 25 | #include 26 | 27 | namespace concurrent_data_structures { 28 | 29 | struct SetThreadBenchmarkResult { 30 | std::uint64_t query_attempts, query_successes; 31 | std::uint64_t addition_attempts, addition_successes; 32 | std::uint64_t removal_attempts, removal_successes; 33 | PapiCounters papi_counters; 34 | SetThreadBenchmarkResult() 35 | : query_attempts(0), query_successes(0), addition_attempts(0), 36 | addition_successes(0), removal_attempts(0), removal_successes(0) {} 37 | }; 38 | 39 | struct SetBenchmarkResult { 40 | std::size_t num_threads; 41 | CachePadded **per_thread_benchmark_result; 42 | std::vector scheduling_info; 43 | SetBenchmarkResult(const std::size_t num_threads) 44 | : num_threads(num_threads), 45 | per_thread_benchmark_result( 46 | new CachePadded *[num_threads]) { 47 | for (std::size_t i = 0; i < num_threads; i++) { 48 | per_thread_benchmark_result[i] = 49 | new CachePadded(); 50 | } 51 | } 52 | 53 | const SetThreadBenchmarkResult collate_results() const { 54 | SetThreadBenchmarkResult results; 55 | for (std::size_t i = 0; i < num_threads; i++) { 56 | results.query_attempts += per_thread_benchmark_result[i]->query_attempts; 57 | results.query_successes += 58 | per_thread_benchmark_result[i]->query_successes; 59 | results.addition_attempts += 60 | per_thread_benchmark_result[i]->addition_attempts; 61 | results.addition_successes += 62 | per_thread_benchmark_result[i]->addition_successes; 63 | results.removal_attempts += 64 | per_thread_benchmark_result[i]->removal_attempts; 65 | results.removal_successes += 66 | per_thread_benchmark_result[i]->removal_successes; 67 | for (std::size_t event = 0; event < PAPI_EVENTS::TOTAL_PAPI_EVENTS; 68 | event++) { 69 | results.papi_counters.counters[event] += 70 | per_thread_benchmark_result[i]->papi_counters.counters[event]; 71 | } 72 | } 73 | return results; 74 | } 75 | }; 76 | } 77 | -------------------------------------------------------------------------------- /src/bench/benchmark_summary.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark_summary.h" 2 | 3 | /* 4 | Summarises benchmark data into files. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "mem-reclaimer/reclaimer.h" 22 | #include "random.h" 23 | #include "table.h" 24 | #include 25 | #include 26 | #include 27 | 28 | namespace concurrent_data_structures { 29 | 30 | template 31 | void write_field(std::ofstream &human_file, std::ofstream &csv_key_file, 32 | std::ofstream &csv_data_file, const std::string &field_name, 33 | const T &field_value, bool write_keys, bool last = false, 34 | bool last_key = false) { 35 | human_file << "(" << field_name << " -> " << field_value 36 | << (last ? ")" : ") "); 37 | if (write_keys) { 38 | csv_key_file << field_name << (last_key ? "" : ","); 39 | } 40 | csv_data_file << field_value << (last ? "" : ","); 41 | } 42 | 43 | void config_summary(const BenchmarkConfig &config, std::ofstream &human_file, 44 | std::ofstream &csv_key_file, std::ofstream &csv_data_file, 45 | bool write_keys) { 46 | write_field(human_file, csv_key_file, csv_data_file, "Num Threads", 47 | config.num_threads, write_keys); 48 | write_field(human_file, csv_key_file, csv_data_file, "Duration", 49 | config.duration.count(), write_keys); 50 | write_field(human_file, csv_key_file, csv_data_file, "Reclaimer Name", 51 | get_reclaimer_name(config.reclaimer), write_keys); 52 | write_field(human_file, csv_key_file, csv_data_file, "Allocator Name", 53 | get_allocator_name(config.allocator), write_keys); 54 | write_field(human_file, csv_key_file, csv_data_file, "Generator Name", 55 | get_generator_name(config.generator), write_keys); 56 | write_field(human_file, csv_key_file, csv_data_file, "Papi Enabled", 57 | config.papi_active ? "True" : "False", write_keys); 58 | write_field(human_file, csv_key_file, csv_data_file, "Hyperthreading Enabled", 59 | config.hyperthreading ? "True" : "False", write_keys); 60 | } 61 | 62 | void set_config_summary(const SetBenchmarkConfig &config, 63 | std::ofstream &human_file, std::ofstream &csv_key_file, 64 | std::ofstream &csv_data_file, bool write_keys) { 65 | write_field(human_file, csv_key_file, csv_data_file, "Table Name", 66 | get_table_name(config.table), write_keys); 67 | write_field(human_file, csv_key_file, csv_data_file, "Table Size", 68 | config.table_size, write_keys); 69 | write_field(human_file, csv_key_file, csv_data_file, "Load Factor", 70 | config.load_factor, write_keys); 71 | write_field(human_file, csv_key_file, csv_data_file, "Updates", 72 | config.updates, write_keys); 73 | config_summary(config.base, human_file, csv_key_file, csv_data_file, 74 | write_keys); 75 | } 76 | 77 | // static const double milliseconds = 1000.0; 78 | static const double microseconds = 1000000.0; 79 | 80 | void papi_summary(const std::size_t total_operations, 81 | const PapiCounters &papi_counters, std::ofstream &human_file, 82 | std::ofstream &csv_key_file, std::ofstream &csv_data_file, 83 | bool write_keys) { 84 | 85 | const double total_ops = static_cast(total_operations); 86 | const double level1_cache_misses_per_op = 87 | static_cast( 88 | papi_counters.counters[PAPI_EVENTS::L1_CACHE_MISSES]) / 89 | total_ops; 90 | 91 | const double level2_cache_misses_per_op = 92 | static_cast( 93 | papi_counters.counters[PAPI_EVENTS::L2_CACHE_MISSES]) / 94 | total_ops; 95 | 96 | const double instruction_stalls_per_op = 97 | static_cast( 98 | papi_counters.counters[PAPI_EVENTS::INSTRUCTION_STALLS]) / 99 | total_ops; 100 | 101 | const double total_instructions_per_op = 102 | static_cast( 103 | papi_counters.counters[PAPI_EVENTS::TOTAL_INSTRUCTIONS]) / 104 | total_ops; 105 | 106 | const double level1_data_cache_misses_per_op = 107 | static_cast( 108 | papi_counters.counters[PAPI_EVENTS::L1_DATA_CACHE_MISSES]) / 109 | total_ops; 110 | 111 | write_field(human_file, csv_key_file, csv_data_file, 112 | PAPI_EVENTS::get_event_name(PAPI_EVENTS::L1_DATA_CACHE_MISSES), 113 | level1_data_cache_misses_per_op, write_keys); 114 | write_field(human_file, csv_key_file, csv_data_file, 115 | PAPI_EVENTS::get_event_name(PAPI_EVENTS::L1_CACHE_MISSES), 116 | level1_cache_misses_per_op, write_keys); 117 | write_field(human_file, csv_key_file, csv_data_file, 118 | PAPI_EVENTS::get_event_name(PAPI_EVENTS::L2_CACHE_MISSES), 119 | level2_cache_misses_per_op, write_keys); 120 | write_field(human_file, csv_key_file, csv_data_file, 121 | PAPI_EVENTS::get_event_name(PAPI_EVENTS::INSTRUCTION_STALLS), 122 | instruction_stalls_per_op, write_keys); 123 | write_field(human_file, csv_key_file, csv_data_file, 124 | PAPI_EVENTS::get_event_name(PAPI_EVENTS::TOTAL_INSTRUCTIONS), 125 | total_instructions_per_op, write_keys); 126 | } 127 | 128 | void operations_per_thread_summary(const SetThreadBenchmarkResult &result, 129 | std::ofstream &human_file, 130 | std::ofstream &csv_key_file, 131 | std::ofstream &csv_data_file, 132 | const std::chrono::seconds &duration, 133 | bool print_papi, bool write_keys, 134 | bool last = false) { 135 | const std::size_t total_operations_attempted = result.query_attempts + 136 | result.addition_attempts + 137 | result.removal_attempts; 138 | 139 | if (print_papi) { 140 | papi_summary(total_operations_attempted, result.papi_counters, human_file, 141 | csv_key_file, csv_data_file, write_keys); 142 | human_file << std::endl; 143 | } 144 | 145 | const double attempted_ops_per_second = 146 | static_cast(total_operations_attempted) / 147 | static_cast(duration.count()); 148 | const double attempted_ops_per_microsecond = 149 | attempted_ops_per_second / microseconds; 150 | // Operations as a percentage of the work done. 151 | const double query_attempt_percentage = 152 | (static_cast(result.query_attempts) / 153 | static_cast(total_operations_attempted)) * 154 | 100.0; 155 | const double insertion_attempt_percentage = 156 | (static_cast(result.addition_attempts) / 157 | static_cast(total_operations_attempted)) * 158 | 100.0; 159 | const double removal_attempt_percentage = 160 | (static_cast(result.removal_attempts) / 161 | static_cast(total_operations_attempted)) * 162 | 100.0; 163 | 164 | const double query_successes_percentage = 165 | (static_cast(result.query_successes) / 166 | static_cast(std::max(result.query_attempts, std::uint64_t(1)))) * 167 | 100.0; 168 | const double insertion_successes_percentage = 169 | (static_cast(result.addition_successes) / 170 | static_cast( 171 | std::max(result.addition_attempts, std::uint64_t(1)))) * 172 | 100.0; 173 | const double removal_successes_percentage = 174 | (static_cast(result.removal_successes) / 175 | static_cast( 176 | std::max(result.removal_attempts, std::uint64_t(1)))) * 177 | 100.0; 178 | 179 | write_field(human_file, csv_key_file, csv_data_file, "Queries attempted", 180 | result.query_attempts, write_keys); 181 | human_file << "Queries attempted as percentage: " << query_attempt_percentage 182 | << "%" << std::endl; 183 | write_field(human_file, csv_key_file, csv_data_file, "Queries succeeded", 184 | result.query_successes, write_keys); 185 | human_file << "Queries succeeded as percentage: " 186 | << query_successes_percentage << "%" << std::endl; 187 | write_field(human_file, csv_key_file, csv_data_file, "Insertion attempted", 188 | result.addition_attempts, write_keys); 189 | human_file << "Insertions attempted as percentage: " 190 | << insertion_attempt_percentage << "%" << std::endl; 191 | write_field(human_file, csv_key_file, csv_data_file, "Insertion succeeded", 192 | result.addition_successes, write_keys); 193 | human_file << "Insertions succeeded as percentage: " 194 | << insertion_successes_percentage << "%" << std::endl; 195 | write_field(human_file, csv_key_file, csv_data_file, "Removes attempted", 196 | result.removal_attempts, write_keys); 197 | human_file << "Removes attempted as percentage: " 198 | << removal_attempt_percentage << "%" << std::endl; 199 | write_field(human_file, csv_key_file, csv_data_file, "Removes successes", 200 | result.removal_successes, write_keys); 201 | human_file << "Removes succeeded as percentage: " 202 | << removal_successes_percentage << "%" << std::endl; 203 | write_field(human_file, csv_key_file, csv_data_file, 204 | "Total Ops per microsecond", attempted_ops_per_microsecond, 205 | write_keys, last, true); 206 | } 207 | 208 | template 209 | void operations_summary(const Result &result, std::ofstream &human_file, 210 | std::ofstream &csv_key_file, 211 | std::ofstream &csv_data_file, 212 | const std::chrono::seconds &duration, bool papi) { 213 | human_file << "TOTAL OPERATIONS FOR ALL THREADS" << std::endl; 214 | operations_per_thread_summary(result.collate_results(), human_file, 215 | csv_key_file, csv_data_file, duration, papi, 216 | true); 217 | human_file << std::endl; 218 | human_file << "OPERATIONS INDIVIDUAL THREADS" << std::endl; 219 | for (std::size_t i = 0; i < result.num_threads; i++) { 220 | human_file << std::string(40, '*') << std::endl; 221 | human_file << "THREAD: " << result.scheduling_info[i].user_id 222 | << " operation summary." << std::endl; 223 | human_file << "Thread was pinned to core id: " 224 | << result.scheduling_info[i].linux_id << " on L3 cache index: " 225 | << result.scheduling_info[i].L3_cache_id 226 | << " with an L2 ID of: " << result.scheduling_info[i].L2_id 227 | << " and with L2 index of: " 228 | << result.scheduling_info[i].L2_index << std::endl; 229 | operations_per_thread_summary( 230 | *result.per_thread_benchmark_result[result.scheduling_info[i].user_id], 231 | human_file, csv_key_file, csv_data_file, duration, papi, false, 232 | (i == (result.num_threads - 1))); 233 | human_file << std::endl; 234 | } 235 | } 236 | 237 | void produce_summary(const SetBenchmarkConfig &config, 238 | const SetBenchmarkResult &result, 239 | const std::string &human_filename, 240 | const std::string &csv_key_filename, 241 | const std::string &csv_data_filename) { 242 | std::ofstream human_file(human_filename); 243 | std::ofstream csv_key_file(csv_key_filename, 244 | std::ofstream::out | std::ofstream::trunc); 245 | std::ofstream csv_data_file(csv_data_filename, 246 | std::ofstream::out | std::ofstream::app); 247 | human_file << "CONFIG." << std::endl; 248 | human_file << std::string(40, '*') << std::endl; 249 | // Setup. 250 | set_config_summary(config, human_file, csv_key_file, csv_data_file, true); 251 | human_file << std::endl; 252 | // Numbers produced. 253 | human_file << std::string(40, '*') << std::endl; 254 | human_file << "OPERATIONS." << std::endl; 255 | operations_summary(result, human_file, csv_key_file, csv_data_file, 256 | config.base.duration, config.base.papi_active); 257 | csv_key_file << std::endl; 258 | csv_data_file << std::endl; 259 | } 260 | 261 | } // namespace concurrent_hash_tables 262 | -------------------------------------------------------------------------------- /src/bench/benchmark_summary.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /* 3 | Summarises benchmark data into files. 4 | Copyright (C) 2018 Robert Kelly 5 | 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | 11 | This program is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License for more details. 15 | 16 | You should have received a copy of the GNU General Public License 17 | along with this program. If not, see . 18 | */ 19 | #include "benchmark_config.h" 20 | #include "benchmark_results.h" 21 | #include 22 | namespace concurrent_data_structures { 23 | // Produce summary for file. 24 | void produce_summary(const SetBenchmarkConfig &config, 25 | const SetBenchmarkResult &result, 26 | const std::string &human_filename, 27 | const std::string &csv_key_filename, 28 | const std::string &csv_data_filename); 29 | 30 | } // namespace concurrent_hash_tables 31 | -------------------------------------------------------------------------------- /src/bench/benchmark_table.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Hash table benchmarking class. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "action_generator.h" 22 | #include "benchmark_config.h" 23 | #include "benchmark_results.h" 24 | #include "hash-tables/table_init.h" 25 | #include "primitives/barrier.h" 26 | #include "primitives/cache_utils.h" 27 | #include "thread_papi_wrapper.h" 28 | #include "thread_pinner.h" 29 | #include "utils.h" 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace concurrent_data_structures { 38 | 39 | template 40 | class TableBenchmark { 41 | private: 42 | enum class BenchmarkState { RUNNING, STOPPED }; 43 | 44 | struct BenchmarkThreadData { 45 | const std::size_t thread_id; 46 | std::atomic *state; 47 | ThreadBarrierWrapper *thread_barrier; 48 | BenchmarkThreadData(const std::size_t thread_id, 49 | std::atomic *state, 50 | ThreadBarrierWrapper *thread_barrier) 51 | : thread_id(thread_id), state(state), thread_barrier(thread_barrier) {} 52 | }; 53 | 54 | const SetBenchmarkConfig m_config; 55 | SetBenchmarkResult m_results; 56 | Table *m_table; 57 | 58 | void benchmark_routine(CachePadded *thread_data) { 59 | SetActionGenerator action_generator(m_config); 60 | CachePadded *result = 61 | m_results.per_thread_benchmark_result[thread_data->thread_id]; 62 | ThreadPapiWrapper papi_wrapper(m_config.base.papi_active); 63 | bool init = m_table->thread_init(thread_data->thread_id); 64 | thread_data->thread_barrier->wait(); 65 | assert(init); 66 | bool papi_start = papi_wrapper.start(); 67 | assert(papi_start); 68 | while (thread_data->state->load(std::memory_order_relaxed) == 69 | BenchmarkState::RUNNING) { 70 | const SetAction current_action = action_generator.generate_action(); 71 | switch (current_action) { 72 | case SetAction::Contains: { 73 | const auto key = action_generator.generate_key(); 74 | result->query_attempts++; 75 | if (m_table->contains(key, thread_data->thread_id)) { 76 | result->query_successes++; 77 | } 78 | } break; 79 | case SetAction::Add: { 80 | auto key = action_generator.generate_key(); 81 | result->addition_attempts++; 82 | while (!m_table->add(key, thread_data->thread_id)) { 83 | result->addition_attempts++; 84 | key = action_generator.generate_key(); 85 | } 86 | result->addition_successes++; 87 | } break; 88 | case SetAction::Remove: { 89 | result->removal_attempts++; 90 | auto key = action_generator.generate_key(); 91 | while (!m_table->remove(key, thread_data->thread_id)) { 92 | result->removal_attempts++; 93 | key = action_generator.generate_key(); 94 | } 95 | result->removal_successes++; 96 | } break; 97 | } 98 | } 99 | bool papi_stop = papi_wrapper.stop(result->papi_counters); 100 | assert(papi_stop); 101 | } 102 | 103 | public: 104 | TableBenchmark(const SetBenchmarkConfig &config) 105 | : m_config(config), m_results(config.base.num_threads) { 106 | std::cout << "Initialising hash-table." << std::endl; 107 | m_table = TableInit(config); 108 | std::cout << "Hash-table initialised." << std::endl; 109 | } 110 | 111 | ~TableBenchmark() { 112 | m_table->~Table(); 113 | Allocator::free(m_table); 114 | } 115 | 116 | SetBenchmarkResult bench() { 117 | std::cout << "Running maintain benchmark...." << std::endl; 118 | ThreadBarrierWrapper barrier(m_config.base.num_threads + 1); 119 | std::atomic benchmark_state{BenchmarkState::RUNNING}; 120 | CachePadded *thread_data = 121 | static_cast *>( 122 | Allocator::aligned_alloc(S_CACHE_PADDING, 123 | sizeof(CachePadded) * 124 | m_config.base.num_threads)); 125 | for (std::size_t t = 0; t < m_config.base.num_threads; t++) { 126 | new (&thread_data[t]) 127 | CachePadded(t, &benchmark_state, &barrier); 128 | } 129 | ThreadPinner pinner(m_config.base.hyperthreading); 130 | std::vector threads; 131 | std::cout << "Launching threads." << std::endl; 132 | for (std::size_t t = 0; t < m_config.base.num_threads; t++) { 133 | threads.push_back(new std::thread(&TableBenchmark::benchmark_routine, 134 | this, &thread_data[t])); 135 | bool res = pinner.schedule_thread(threads[t], t); 136 | assert(res); 137 | } 138 | std::cout << "Waiting..." << std::endl; 139 | // Wait for other threads. 140 | barrier.wait(); 141 | // Sleep. 142 | sleep_ignore_signals(m_config.base.duration); 143 | // End benchmark. 144 | benchmark_state.store(BenchmarkState::STOPPED); 145 | std::cout << "Joining threads." << std::endl; 146 | m_results.scheduling_info = pinner.join(); 147 | for (std::size_t t = 0; t < m_config.base.num_threads; t++) { 148 | delete threads[t]; 149 | } 150 | std::cout << "Collating benchmark data." << std::endl; 151 | return m_results; 152 | } 153 | }; 154 | } 155 | -------------------------------------------------------------------------------- /src/bench/random.cpp: -------------------------------------------------------------------------------- 1 | #include "random.h" 2 | 3 | /* 4 | Mapping enums to string descriptions of random generators. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | namespace { 26 | static const std::map random_map{ 27 | std::make_pair(RandomGenerator::LCGBSD, "BSD LCG"), 28 | std::make_pair(RandomGenerator::PCG, "PCG"), 29 | std::make_pair(RandomGenerator::XOR_SHIFT_32, "XOR 32"), 30 | std::make_pair(RandomGenerator::XOR_SHIFT_64, "XOR 64"), 31 | }; 32 | } 33 | 34 | const std::string get_generator_name(const RandomGenerator random_generator) { 35 | std::string random_name = "ERROR: Incorrect random generator name."; 36 | auto it = random_map.find(random_generator); 37 | if (it != random_map.end()) { 38 | random_name = it->second; 39 | } 40 | return random_name; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/bench/random.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Collection of hash table names 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | enum class RandomGenerator { 26 | LCGBSD, 27 | XOR_SHIFT_32, 28 | XOR_SHIFT_64, 29 | PCG, 30 | }; 31 | 32 | const std::string get_generator_name(const RandomGenerator random_generator); 33 | } 34 | -------------------------------------------------------------------------------- /src/bench/table.cpp: -------------------------------------------------------------------------------- 1 | #include "table.h" 2 | 3 | /* 4 | Mapping enums to string descriptions of hash tables. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | namespace { 26 | static const std::map table_map{ 27 | std::make_pair(HashTable::HS_TRANS_SET, "Hopscotch Transactional Set"), 28 | std::make_pair(HashTable::HS_LOCKED_SET, "Hopscotch Locked Set"), 29 | std::make_pair(HashTable::HSC_LOCKED_SET, "Hopscotch Compact Locked Set"), 30 | std::make_pair(HashTable::HSBM_LOCKED_SET, "Hopscotch Bitmap Locked Set"), 31 | std::make_pair(HashTable::HSBM_SERIAL_SET, "Hopscotch Bitmap Serial Set"), 32 | std::make_pair(HashTable::HSBM_LF_SET, "Hopscotch Bitmap Lock-Free Set"), 33 | std::make_pair(HashTable::PH_QP_SET, "Purcell Harris QP Set"), 34 | }; 35 | } 36 | 37 | const std::string get_table_name(const HashTable table) { 38 | std::string table_name = "ERROR: Incorrect hash-table name."; 39 | auto it = table_map.find(table); 40 | if (it != table_map.end()) { 41 | table_name = it->second; 42 | } 43 | return table_name; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/bench/table.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Collection of hash table names 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | enum class HashTable { 26 | HSBM_SERIAL_SET, 27 | HS_TRANS_SET, 28 | HS_LOCKED_SET, 29 | HSC_LOCKED_SET, 30 | HSBM_LOCKED_SET, 31 | PH_QP_SET, 32 | HSBM_LF_SET, 33 | }; 34 | 35 | const std::string get_table_name(const HashTable table); 36 | } 37 | -------------------------------------------------------------------------------- /src/bench/thread_papi_wrapper.cpp: -------------------------------------------------------------------------------- 1 | #include "thread_papi_wrapper.h" 2 | 3 | /* 4 | Main body of per-thread PAPI wrapper. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace concurrent_data_structures { 31 | 32 | namespace { 33 | std::map event_map = { 34 | std::make_pair(PAPI_EVENTS::L1_CACHE_MISSES, "Level 1 Cache Misses"), 35 | std::make_pair(PAPI_EVENTS::L2_CACHE_MISSES, "Level 2 Cache Misses"), 36 | std::make_pair(PAPI_EVENTS::INSTRUCTION_STALLS, "Instruction Stalls"), 37 | std::make_pair(PAPI_EVENTS::TOTAL_INSTRUCTIONS, "Total Instructions"), 38 | std::make_pair(PAPI_EVENTS::L1_DATA_CACHE_MISSES, 39 | "Level 1 Data Cache Misses"), 40 | }; 41 | } 42 | 43 | const std::string PAPI_EVENTS::get_event_name(EVENTS event) { 44 | std::string event_name = "ERROR: Incorrect event name."; 45 | auto it = event_map.find(event); 46 | if (it != event_map.end()) { 47 | event_name = it->second; 48 | } 49 | return event_name; 50 | } 51 | 52 | static int PAPI_events[PAPI_EVENTS::TOTAL_PAPI_EVENTS] = { 53 | PAPI_L1_TCM, PAPI_L2_TCM, PAPI_STL_ICY, PAPI_TOT_INS, 54 | PAPI_L1_DCM /*PAPI_TOT_CYC*/}; 55 | 56 | PapiCounters::PapiCounters() { 57 | for (std::size_t i = 0; i < PAPI_EVENTS::TOTAL_PAPI_EVENTS; i++) { 58 | counters[i] = 0; 59 | } 60 | } 61 | 62 | ThreadPapiWrapper::ThreadPapiWrapper(bool active) : m_active(active) { 63 | if (active) { 64 | int res = PAPI_register_thread(); 65 | assert(res == PAPI_OK); 66 | } 67 | } 68 | 69 | bool ThreadPapiWrapper::start() { 70 | if (m_active) { 71 | int res = PAPI_start_counters(PAPI_events, PAPI_EVENTS::TOTAL_PAPI_EVENTS); 72 | if (res != PAPI_OK) { 73 | std::cout << res << std::endl; 74 | return false; 75 | } 76 | return true; 77 | } else { 78 | return true; 79 | } 80 | } 81 | 82 | bool ThreadPapiWrapper::stop(PapiCounters &counters) { 83 | if (m_active) { 84 | return PAPI_stop_counters(counters.counters, 85 | PAPI_EVENTS::TOTAL_PAPI_EVENTS) == PAPI_OK; 86 | } else { 87 | return true; 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/bench/thread_papi_wrapper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | A wrapper to collect per-thread info in PAPI. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace concurrent_data_structures { 26 | 27 | struct PAPI_EVENTS { 28 | enum EVENTS { 29 | L1_CACHE_MISSES = 0, 30 | L2_CACHE_MISSES = 1, 31 | INSTRUCTION_STALLS = 2, 32 | TOTAL_INSTRUCTIONS = 3, 33 | L1_DATA_CACHE_MISSES = 4, 34 | TOTAL_PAPI_EVENTS = 5, 35 | }; 36 | static const std::string get_event_name(EVENTS event); 37 | }; 38 | 39 | struct PapiCounters { 40 | long long counters[PAPI_EVENTS::TOTAL_PAPI_EVENTS]; 41 | PapiCounters(); 42 | }; 43 | 44 | class ThreadPapiWrapper { 45 | private: 46 | bool m_active; 47 | 48 | public: 49 | ThreadPapiWrapper(bool active); 50 | 51 | bool start(); 52 | bool stop(PapiCounters &counters); 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /src/bench/thread_pinner.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | A thread pinning class. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "cpuinfo.h" 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | namespace concurrent_data_structures { 34 | 35 | class ThreadPinner { 36 | public: 37 | struct ProcessorInfo { 38 | std::int32_t user_id, linux_id; 39 | std::uint32_t L3_cache_id, L2_id, L2_index; 40 | }; 41 | 42 | bool m_hyperthreading_before_socket_switch; 43 | std::unordered_map m_scheduled; 44 | std::uint32_t m_index; 45 | std::vector m_processors; 46 | 47 | public: 48 | ThreadPinner(bool hyperthreading_before_socket_switch) 49 | : m_hyperthreading_before_socket_switch( 50 | hyperthreading_before_socket_switch), 51 | m_index(0) { 52 | cpuinfo_initialize(); 53 | std::cout << "Processors..." << std::endl; 54 | const std::uint32_t num_processors = cpuinfo_get_processors_count(); 55 | m_processors.reserve(num_processors); 56 | const cpuinfo_processor *proc = cpuinfo_get_processors(); 57 | for (std::uint32_t i = 0; i < num_processors; i++) { 58 | const cpuinfo_processor *cur_proc = proc + i; 59 | // std::cout << "Linux id: " << cur_proc->linux_id << std::endl; 60 | m_processors[cur_proc->linux_id] = cur_proc; 61 | } 62 | std::cout << "Processors... End" << std::endl; 63 | } 64 | 65 | bool schedule_thread(std::thread *thread, std::int32_t user_id) { 66 | 67 | const cpuinfo_processor *cur_proc = m_processors[m_index++]; 68 | // std::cout << "Pinning to: " << cur_proc->linux_id << std::endl; 69 | cpu_set_t cpu_set; 70 | CPU_ZERO(&cpu_set); 71 | CPU_SET(cur_proc->linux_id, &cpu_set); 72 | if (pthread_setaffinity_np(thread->native_handle(), sizeof(cpu_set_t), 73 | &cpu_set) != 0) { 74 | return false; 75 | } 76 | 77 | ProcessorInfo *proc_info = new ProcessorInfo; 78 | proc_info->user_id = user_id; 79 | proc_info->linux_id = cur_proc->linux_id; 80 | proc_info->L2_id = 81 | std::distance(cpuinfo_get_l2_caches(), cur_proc->cache.l2); 82 | proc_info->L2_index = 83 | cur_proc->linux_id - cur_proc->cache.l2->processor_start; 84 | proc_info->L3_cache_id = 85 | std::distance(cpuinfo_get_l3_caches(), cur_proc->cache.l3); 86 | m_scheduled.insert(std::make_pair(thread, proc_info)); 87 | return true; 88 | } 89 | 90 | std::vector join() { 91 | std::vector info; 92 | for (auto it : m_scheduled) { 93 | it.first->join(); 94 | info.push_back(*it.second); 95 | } 96 | std::sort(info.begin(), info.end(), 97 | [](const ProcessorInfo &lhs, const ProcessorInfo &rhs) { 98 | if (lhs.L2_id < rhs.L2_id) { 99 | return true; 100 | } else if (lhs.L2_id == rhs.L2_id) { 101 | return lhs.L2_index < rhs.L2_index; 102 | } else { 103 | return false; 104 | } 105 | }); 106 | return info; 107 | } 108 | 109 | ~ThreadPinner() { cpuinfo_deinitialize(); } 110 | }; 111 | } 112 | -------------------------------------------------------------------------------- /src/bench/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | #include 4 | #include 5 | namespace concurrent_data_structures { 6 | 7 | void sleep_ignore_signals(const std::chrono::seconds &duration) { 8 | auto now = std::chrono::high_resolution_clock::now(); 9 | const auto sleep_to = now + duration; 10 | while (now < sleep_to) { 11 | std::this_thread::sleep_until(sleep_to); 12 | now = std::chrono::high_resolution_clock::now(); 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/bench/utils.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace concurrent_data_structures { 4 | 5 | void sleep_ignore_signals(const std::chrono::seconds &duration); 6 | } 7 | -------------------------------------------------------------------------------- /src/cmake/FindPAPI.cmake: -------------------------------------------------------------------------------- 1 | # Try to find PAPI headers and libraries. 2 | # 3 | # Usage of this module as follows: 4 | # 5 | # find_package(PAPI) 6 | # 7 | # Variables used by this module, they can change the default behaviour and need 8 | # to be set before calling find_package: 9 | # 10 | # PAPI_PREFIX Set this variable to the root installation of 11 | # libpapi if the module has problems finding the 12 | # proper installation path. 13 | # 14 | # Variables defined by this module: 15 | # 16 | # PAPI_FOUND System has PAPI libraries and headers 17 | # PAPI_LIBRARIES The PAPI library 18 | # PAPI_INCLUDE_DIRS The location of PAPI headers 19 | 20 | find_path(PAPI_PREFIX 21 | NAMES include/papi.h 22 | ) 23 | 24 | find_library(PAPI_LIBRARIES 25 | # Pick the static library first for easier run-time linking. 26 | NAMES libpapi.so libpapi.a papi 27 | HINTS ${PAPI_PREFIX}/lib ${HILTIDEPS}/lib 28 | ) 29 | 30 | find_path(PAPI_INCLUDE_DIRS 31 | NAMES papi.h 32 | HINTS ${PAPI_PREFIX}/include ${HILTIDEPS}/include 33 | ) 34 | 35 | include(FindPackageHandleStandardArgs) 36 | find_package_handle_standard_args(PAPI DEFAULT_MSG 37 | PAPI_LIBRARIES 38 | PAPI_INCLUDE_DIRS 39 | ) 40 | 41 | mark_as_advanced( 42 | PAPI_PREFIX_DIRS 43 | PAPI_LIBRARIES 44 | PAPI_INCLUDE_DIRS 45 | ) 46 | -------------------------------------------------------------------------------- /src/hash-tables/common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Some common hash table functions. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace concurrent_data_structures { 26 | 27 | // TODO: Please just destroy this file. 28 | 29 | struct SizetHash { 30 | SizetHash() {} 31 | std::size_t operator()(const std::size_t arg) const { 32 | std::size_t key = arg; 33 | key = (key ^ (key >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); 34 | key = (key ^ (key >> 27)) * UINT64_C(0x94d049bb133111eb); 35 | key = key ^ (key >> 31); 36 | return key; 37 | } 38 | }; 39 | 40 | struct IntelHashCompare { 41 | static std::size_t hash(const std::size_t arg) { return SizetHash{}(arg); } 42 | static bool equal(const std::size_t lhs, const std::size_t rhs) { 43 | return lhs == rhs; 44 | } 45 | }; 46 | 47 | template T nearest_power_of_two(T num) { 48 | std::size_t actual_size = sizeof(num); 49 | if (num == 0) 50 | return std::size_t(0); 51 | num--; 52 | // Single byte switches 53 | num |= num >> 1; 54 | num |= num >> 2; 55 | num |= num >> 4; 56 | std::size_t current_shift = 8; 57 | for (std::size_t i = 1; i < actual_size; i *= 2, current_shift *= 2) { 58 | num |= num >> current_shift; 59 | } 60 | return ++num; 61 | } 62 | 63 | template struct KeyTraits { 64 | typedef T Key; 65 | typedef typename std::hash Hash; 66 | static const Key NullKey = std::numeric_limits::max() >> 4; 67 | static const Key Tombstone = NullKey - 1; 68 | static std::size_t hash(T key) { return Hash{}(key); } 69 | static std::size_t hash2(T key) { return hash(hash(key)); } 70 | }; 71 | 72 | template struct ValueTraits { 73 | typedef T Value; 74 | static const T NullValue = std::numeric_limits::max(); 75 | }; 76 | 77 | template <> struct KeyTraits { 78 | typedef std::size_t Key; 79 | typedef typename std::hash Hash; 80 | // TODO: Change this please 81 | static const Key NullKey = std::numeric_limits::max() >> 2; 82 | static const Key Tombstone = NullKey - 1; 83 | static std::size_t hash(Key key) { 84 | key = (key ^ (key >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); 85 | key = (key ^ (key >> 27)) * UINT64_C(0x94d049bb133111eb); 86 | key = key ^ (key >> 31); 87 | return key; 88 | } 89 | static std::size_t hash2(Key key) { return hash(hash(key)); } 90 | }; 91 | 92 | template <> struct KeyTraits { 93 | typedef std::intptr_t Key; 94 | typedef typename std::hash Hash; 95 | // TODO: Change this please 96 | static const Key NullKey = std::numeric_limits::max() >> 2; 97 | static const Key Tombstone = NullKey - 1; 98 | static std::size_t hash(Key key) { 99 | key = (key ^ (key >> 30)) * INT64_C(0xbf58476d1ce4e5b9); 100 | key = (key ^ (key >> 27)) * INT64_C(0x94d049bb133111eb); 101 | key = key ^ (key >> 31); 102 | return key; 103 | } 104 | static std::size_t hash2(Key key) { return hash(hash(key)); } 105 | }; 106 | } 107 | -------------------------------------------------------------------------------- /src/hash-tables/hs_locked.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | //////////////////////////////////////////////////////////////////////////////// 4 | // ConcurrentHopscotchHashMap Class 5 | // 6 | //////////////////////////////////////////////////////////////////////////////// 7 | // TERMS OF USAGE 8 | //------------------------------------------------------------------------------ 9 | // 10 | // Permission to use, copy, modify and distribute this software and 11 | // its documentation for any purpose is hereby granted without fee, 12 | // provided that due acknowledgments to the authors are provided and 13 | // this permission notice appears in all copies of the software. 14 | // The software is provided "as is". There is no warranty of any kind. 15 | // 16 | // Authors: 17 | // Maurice Herlihy 18 | // Brown University 19 | // and 20 | // Nir Shavit 21 | // Tel-Aviv University 22 | // and 23 | // Moran Tzafrir 24 | // Tel-Aviv University 25 | // 26 | // Date: July 15, 2008. 27 | // 28 | //////////////////////////////////////////////////////////////////////////////// 29 | // Programmer : Moran Tzafrir (MoranTza@gmail.com) 30 | // 31 | //////////////////////////////////////////////////////////////////////////////// 32 | 33 | //////////////////////////////////////////////////////////////////////////////// 34 | // INCLUDE DIRECTIVES 35 | //////////////////////////////////////////////////////////////////////////////// 36 | #include "common.h" 37 | #include "math.h" 38 | #include "primitives/cache_utils.h" 39 | #include "primitives/locks.h" 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | 47 | //////////////////////////////////////////////////////////////////////////////// 48 | // CLASS: ConcurrentHopscotchHashMap 49 | //////////////////////////////////////////////////////////////////////////////// 50 | 51 | namespace concurrent_data_structures { 52 | 53 | template class Reclaimer, class Lock, 54 | class K, bool Compact, class KT = KeyTraits> 55 | class Hopscotch_Locked_Set { 56 | static const std::int32_t _NULL_DELTA = 57 | std::numeric_limits::min(); 58 | 59 | private: 60 | // Inner Classes ............................................................ 61 | struct Bucket { 62 | std::atomic _first_delta, _next_delta; 63 | std::atomic_size_t _hash; 64 | std::atomic _key; 65 | 66 | void init() { 67 | _first_delta = _NULL_DELTA; 68 | _next_delta = _NULL_DELTA; 69 | _hash = 0; 70 | _key = KT::NullKey; 71 | } 72 | }; 73 | 74 | struct Segment { 75 | std::atomic _timestamp; 76 | Lock _lock; 77 | 78 | void init() { 79 | new (&_lock) Lock(); 80 | _timestamp.store(0); 81 | } 82 | }; 83 | 84 | inline int first_msb_bit_indx(std::uint32_t x) { 85 | if (0 == x) 86 | return -1; 87 | return __builtin_clz(x) - 1; 88 | } 89 | 90 | // Fields ................................................................... 91 | std::size_t m_size_mask; 92 | std::uint32_t m_segment_shift; 93 | CachePadded *volatile _segments; 94 | Bucket *volatile _table; 95 | 96 | const int _cache_mask; 97 | 98 | // Constants ................................................................ 99 | static const std::uint32_t _INSERT_RANGE = 1024 * 4; 100 | // static const std::uint32_t _NUM_SEGMENTS = 1024*1; 101 | // static const std::uint32_t _SEGMENTS_MASK = _NUM_SEGMENTS-1; 102 | static const std::uint32_t _RESIZE_FACTOR = 2; 103 | 104 | // Small Utilities .......................................................... 105 | Bucket *get_start_cacheline_bucket(Bucket *const bucket) { 106 | return (bucket - ((bucket - _table) & _cache_mask)); // can optimize 107 | } 108 | 109 | void remove_key(Segment &segment, Bucket *const from_bucket, 110 | Bucket *const key_bucket, Bucket *const prev_key_bucket, 111 | const unsigned int hash) { 112 | key_bucket->_key.store(KT::NullKey); 113 | 114 | if (NULL == prev_key_bucket) { 115 | if (_NULL_DELTA == key_bucket->_next_delta.load()) 116 | from_bucket->_first_delta.store(_NULL_DELTA); 117 | else 118 | from_bucket->_first_delta.store((from_bucket->_first_delta.load() + 119 | key_bucket->_next_delta.load())); 120 | } else { 121 | if (_NULL_DELTA == key_bucket->_next_delta.load()) 122 | prev_key_bucket->_next_delta.store(_NULL_DELTA); 123 | else 124 | prev_key_bucket->_next_delta.store( 125 | (prev_key_bucket->_next_delta.load() + 126 | key_bucket->_next_delta.load())); 127 | } 128 | 129 | segment._timestamp.fetch_add(1); 130 | key_bucket->_next_delta = _NULL_DELTA; 131 | key_bucket->_hash.store(0); 132 | } 133 | void add_key_to_begining_of_list(Bucket *const keys_bucket, 134 | Bucket *const free_bucket, const K &key) { 135 | free_bucket->_key.store(key); 136 | 137 | if (0 == keys_bucket->_first_delta.load()) { 138 | if (_NULL_DELTA == keys_bucket->_next_delta.load()) 139 | free_bucket->_next_delta.store(_NULL_DELTA); 140 | else 141 | free_bucket->_next_delta.store((std::int32_t)( 142 | (keys_bucket + keys_bucket->_next_delta.load()) - free_bucket)); 143 | keys_bucket->_next_delta.store((std::int32_t)(free_bucket - keys_bucket)); 144 | } else { 145 | if (_NULL_DELTA == keys_bucket->_first_delta.load()) 146 | free_bucket->_next_delta.store(_NULL_DELTA); 147 | else 148 | free_bucket->_next_delta.store((std::int32_t)( 149 | (keys_bucket + keys_bucket->_first_delta.load()) - free_bucket)); 150 | keys_bucket->_first_delta.store( 151 | (std::int32_t)(free_bucket - keys_bucket)); 152 | } 153 | } 154 | 155 | void add_key_to_end_of_list(Bucket *const keys_bucket, 156 | Bucket *const free_bucket, const K &key, 157 | Bucket *const last_bucket) { 158 | free_bucket->_key.store(key); 159 | free_bucket->_next_delta.store(_NULL_DELTA); 160 | 161 | if (nullptr == last_bucket) 162 | keys_bucket->_first_delta.store( 163 | (std::int32_t)(free_bucket - keys_bucket)); 164 | else 165 | last_bucket->_next_delta.store((std::int32_t)(free_bucket - last_bucket)); 166 | } 167 | 168 | void optimize_cacheline_use(Segment &segment, Bucket *const free_bucket) { 169 | Bucket *const start_cacheline_bucket = 170 | get_start_cacheline_bucket(free_bucket); 171 | Bucket *const end_cacheline_bucket(start_cacheline_bucket + _cache_mask); 172 | Bucket *opt_bucket = start_cacheline_bucket; 173 | 174 | do { 175 | if (_NULL_DELTA != opt_bucket->_first_delta.load()) { 176 | Bucket *relocate_key_last(nullptr); 177 | int curr_delta(opt_bucket->_first_delta.load()); 178 | Bucket *relocate_key(opt_bucket + curr_delta); 179 | do { 180 | if (curr_delta < 0 || curr_delta > _cache_mask) { 181 | free_bucket->_key.store(relocate_key->_key.load()); 182 | free_bucket->_hash.store(relocate_key->_hash.load()); 183 | 184 | if (_NULL_DELTA == relocate_key->_next_delta.load()) 185 | free_bucket->_next_delta.store(_NULL_DELTA); 186 | else 187 | free_bucket->_next_delta.store((std::int32_t)( 188 | (relocate_key + relocate_key->_next_delta) - free_bucket)); 189 | 190 | if (nullptr == relocate_key_last) 191 | opt_bucket->_first_delta.store( 192 | (std::int32_t)(free_bucket - opt_bucket)); 193 | else 194 | relocate_key_last->_next_delta.store( 195 | (std::int32_t)(free_bucket - relocate_key_last)); 196 | 197 | segment._timestamp.fetch_add(1); 198 | relocate_key->_key.store(KT::NullKey); 199 | relocate_key->_next_delta.store(_NULL_DELTA); 200 | relocate_key->_hash.store(0); 201 | return; 202 | } 203 | 204 | if (_NULL_DELTA == relocate_key->_next_delta.load()) 205 | break; 206 | relocate_key_last = relocate_key; 207 | curr_delta += relocate_key->_next_delta.load(); 208 | relocate_key += relocate_key->_next_delta.load(); 209 | } while (true); // for on list 210 | } 211 | ++opt_bucket; 212 | } while (opt_bucket <= end_cacheline_bucket); 213 | } 214 | 215 | public 216 | : // Ctors ................................................................ 217 | Hopscotch_Locked_Set( 218 | std::uint32_t inCapacity = 32 * 1024, // init capacity 219 | std::uint32_t concurrencyLevel = 220 | std::thread::hardware_concurrency(), // num of updating threads 221 | std::uint32_t cache_line_size = S_CACHE_SIZE // Cache-line size of machine 222 | ) 223 | : _cache_mask((cache_line_size / sizeof(Bucket)) - 1) { 224 | concurrencyLevel = nearest_power_of_two(concurrencyLevel); 225 | 226 | std::size_t num_timestamps = concurrencyLevel; 227 | 228 | std::uint8_t num_timestamp_bits = 0; 229 | for (std::size_t timestamp = num_timestamps; timestamp > 0; 230 | timestamp >>= 1, num_timestamp_bits++) { 231 | } 232 | std::uint8_t num_size_bits = 0; 233 | for (std::size_t size = inCapacity; size > 0; size >>= 1, num_size_bits++) { 234 | } 235 | std::uint8_t timestamp_shift = num_size_bits - num_timestamp_bits; 236 | m_segment_shift = timestamp_shift; 237 | 238 | // ADJUST INPUT ............................ 239 | const std::uint32_t adjInitCap = (inCapacity); 240 | m_size_mask = inCapacity - 1; 241 | // const std::uint32_t adjConcurrencyLevel = 242 | // NearestPowerOfTwo(concurrencyLevel); 243 | const std::uint32_t num_buckets(adjInitCap + _INSERT_RANGE + 1); 244 | // ALLOCATE THE SEGMENTS ................... 245 | _segments = static_cast *>(Allocator::aligned_alloc( 246 | S_CACHE_PADDING, sizeof(CachePadded) * concurrencyLevel)); 247 | _table = static_cast( 248 | Allocator::aligned_alloc(S_CACHE_SIZE, sizeof(Bucket) * num_buckets)); 249 | 250 | CachePadded *curr_seg = _segments; 251 | for (std::uint32_t iSeg = 0; iSeg < concurrencyLevel; ++iSeg, ++curr_seg) { 252 | curr_seg->init(); 253 | } 254 | 255 | Bucket *curr_bucket = _table; 256 | for (std::uint32_t iElm = 0; iElm < num_buckets; ++iElm, ++curr_bucket) { 257 | curr_bucket->init(); 258 | } 259 | } 260 | 261 | ~Hopscotch_Locked_Set() { 262 | Allocator::free(_table); 263 | Allocator::free(_segments); 264 | } 265 | 266 | bool thread_init(const std::size_t thread_id) { return true; } 267 | 268 | // Query Operations ......................................................... 269 | bool contains(const K &key, const std::size_t thread_id) { 270 | 271 | // CALCULATE HASH .......................... 272 | const unsigned int hash(KT::hash(key)); 273 | 274 | // CHECK IF ALREADY CONTAIN ................ 275 | const Segment &segment(_segments[(hash & m_size_mask) >> m_segment_shift]); 276 | 277 | // go over the list and look for key 278 | unsigned int start_timestamp; 279 | do { 280 | start_timestamp = segment._timestamp.load(); 281 | const Bucket *curr_bucket(&(_table[hash & m_size_mask])); 282 | std::int32_t next_delta(curr_bucket->_first_delta.load()); 283 | while (_NULL_DELTA != next_delta) { 284 | curr_bucket += next_delta; 285 | if (key == curr_bucket->_key.load()) { 286 | return true; 287 | } 288 | next_delta = curr_bucket->_next_delta.load(); 289 | } 290 | } while (start_timestamp != segment._timestamp.load()); 291 | return false; 292 | } 293 | 294 | // modification Operations ................................................... 295 | inline bool add(const K &key, const std::size_t thread_id) { 296 | const unsigned int hash(KT::hash(key)); 297 | // Segment &segment(_segments[(hash & m_size_mask) >> m_segment_shift]); 298 | 299 | // go over the list and look for key 300 | std::lock_guard guard( 301 | this->_segments[(hash & m_size_mask) >> m_segment_shift]._lock); 302 | Bucket *const start_bucket(&(_table[hash & m_size_mask])); 303 | 304 | Bucket *last_bucket(nullptr); 305 | Bucket *compare_bucket = start_bucket; 306 | std::int32_t next_delta(compare_bucket->_first_delta.load()); 307 | while (_NULL_DELTA != next_delta) { 308 | compare_bucket += next_delta; 309 | if (hash == compare_bucket->_hash.load() && 310 | key == compare_bucket->_key.load()) { 311 | return false; 312 | } 313 | last_bucket = compare_bucket; 314 | next_delta = compare_bucket->_next_delta.load(); 315 | } 316 | 317 | // try to place the key in the same cache-line 318 | if (Compact) { 319 | Bucket *free_bucket = start_bucket; 320 | Bucket *start_cacheline_bucket = get_start_cacheline_bucket(start_bucket); 321 | Bucket *end_cacheline_bucket(start_cacheline_bucket + _cache_mask); 322 | do { 323 | std::size_t cur_hash = free_bucket->_hash.load(); 324 | if (0 == cur_hash) { 325 | if (!free_bucket->_hash.compare_exchange_weak(cur_hash, hash)) { 326 | continue; 327 | } 328 | add_key_to_begining_of_list(start_bucket, free_bucket, key); 329 | return true; 330 | } 331 | ++free_bucket; 332 | if (free_bucket > end_cacheline_bucket) 333 | free_bucket = start_cacheline_bucket; 334 | } while (start_bucket != free_bucket); 335 | } 336 | 337 | // place key in arbitrary free forward bucket 338 | Bucket *max_bucket(start_bucket + 339 | (std::numeric_limits::max() - 1)); 340 | Bucket *last_table_bucket(_table + m_size_mask); 341 | if (max_bucket > last_table_bucket) 342 | max_bucket = last_table_bucket; 343 | Bucket *free_max_bucket(start_bucket + (_cache_mask + 1)); 344 | while (free_max_bucket <= max_bucket) { 345 | std::size_t cur_hash = free_max_bucket->_hash.load(); 346 | if (0 == cur_hash) { 347 | if (!free_max_bucket->_hash.compare_exchange_weak(cur_hash, hash)) { 348 | continue; 349 | } 350 | add_key_to_end_of_list(start_bucket, free_max_bucket, key, last_bucket); 351 | return true; 352 | } 353 | ++free_max_bucket; 354 | } 355 | 356 | // place key in arbitrary free backward bucket 357 | Bucket *min_bucket(start_bucket - 358 | (std::numeric_limits::max() - 1)); 359 | if (min_bucket < _table) 360 | min_bucket = _table; 361 | Bucket *free_min_bucket(start_bucket - (_cache_mask + 1)); 362 | while (free_min_bucket >= min_bucket) { 363 | std::size_t cur_hash = free_min_bucket->_hash.load(); 364 | if (0 == cur_hash) { 365 | if (!free_min_bucket->_hash.compare_exchange_weak(cur_hash, hash)) { 366 | continue; 367 | } 368 | add_key_to_end_of_list(start_bucket, free_min_bucket, key, last_bucket); 369 | return true; 370 | } 371 | --free_min_bucket; 372 | } 373 | 374 | // NEED TO RESIZE .......................... 375 | fprintf(stderr, "ERROR - RESIZE is not implemented - size %u\n", size()); 376 | exit(1); 377 | return false; 378 | } 379 | 380 | inline bool remove(const K &key, const std::size_t thread_id) { 381 | // CALCULATE HASH .......................... 382 | const unsigned int hash(KT::hash(key)); 383 | 384 | // CHECK IF ALREADY CONTAIN ................ 385 | std::lock_guard guard( 386 | this->_segments[(hash & m_size_mask) >> m_segment_shift]._lock); 387 | Bucket *const start_bucket = &_table[hash & m_size_mask]; 388 | Bucket *last_bucket(nullptr); 389 | Bucket *curr_bucket = start_bucket; 390 | std::int32_t next_delta = curr_bucket->_first_delta.load(); 391 | do { 392 | if (_NULL_DELTA == next_delta) { 393 | return false; 394 | } 395 | curr_bucket += next_delta; 396 | 397 | if (hash == curr_bucket->_hash.load() && 398 | (key == curr_bucket->_key.load())) { 399 | remove_key(this->_segments[(hash & m_size_mask) >> m_segment_shift], 400 | start_bucket, curr_bucket, last_bucket, hash); 401 | if (Compact) 402 | optimize_cacheline_use( 403 | this->_segments[(hash & m_size_mask) >> m_segment_shift], 404 | curr_bucket); 405 | return true; 406 | } 407 | last_bucket = curr_bucket; 408 | next_delta = curr_bucket->_next_delta.load(); 409 | } while (true); 410 | return false; 411 | } 412 | 413 | // status Operations ......................................................... 414 | unsigned int size() { 415 | std::uint32_t counter = 0; 416 | const std::uint32_t num_elm(m_size_mask + _INSERT_RANGE); 417 | for (std::uint32_t iElm = 0; iElm < num_elm; ++iElm) { 418 | if (0 != _table[iElm]._hash) { 419 | ++counter; 420 | } 421 | } 422 | return counter; 423 | } 424 | 425 | double percentKeysInCacheline() { 426 | unsigned int total_in_cache(0); 427 | unsigned int total(0); 428 | 429 | Bucket *curr_bucket(_table); 430 | for (int iElm(0); iElm <= m_size_mask; ++iElm, ++curr_bucket) { 431 | 432 | if (_NULL_DELTA != curr_bucket->_first_delta) { 433 | Bucket *const startCacheLineBucket = 434 | get_start_cacheline_bucket(curr_bucket); 435 | Bucket *check_bucket = curr_bucket + curr_bucket->_first_delta; 436 | int currDist(curr_bucket->_first_delta); 437 | do { 438 | ++total; 439 | if ((check_bucket - startCacheLineBucket) >= 0 && 440 | (check_bucket - startCacheLineBucket) <= _cache_mask) 441 | ++total_in_cache; 442 | if (_NULL_DELTA == check_bucket->_next_delta) 443 | break; 444 | currDist += check_bucket->_next_delta; 445 | check_bucket += check_bucket->_next_delta; 446 | } while (true); 447 | } 448 | } 449 | 450 | // return percent in cache 451 | return (((double)total_in_cache) / ((double)total) * 100.0); 452 | } 453 | 454 | void print_table() {} 455 | 456 | private: 457 | // Private Static Utilities ................................................. 458 | static std::uint32_t NearestPowerOfTwo(const std::uint32_t value) { 459 | std::uint32_t rc(1); 460 | while (rc < value) { 461 | rc <<= 1; 462 | } 463 | return rc; 464 | } 465 | 466 | static unsigned int CalcDivideShift(const unsigned int _value) { 467 | unsigned int numShift(0); 468 | unsigned int curr(1); 469 | while (curr < _value) { 470 | curr <<= 1; 471 | ++numShift; 472 | } 473 | return numShift; 474 | } 475 | }; 476 | 477 | template class Reclaimer, class K> 478 | using Hopscotch_SpinLock_Set = 479 | Hopscotch_Locked_Set; 480 | 481 | template class Reclaimer, class K> 482 | using Hopscotch_Mutex_Set = 483 | Hopscotch_Locked_Set; 484 | 485 | template class Reclaimer, class K> 486 | using Hopscotch_TicketLock_Set = 487 | Hopscotch_Locked_Set; 488 | 489 | template class Reclaimer, class K> 490 | using HopscotchCompact_SpinLock_Set = 491 | Hopscotch_Locked_Set; 492 | 493 | template class Reclaimer, class K> 494 | using HopscotchCompact_Mutex_Set = 495 | Hopscotch_Locked_Set; 496 | 497 | template class Reclaimer, class K> 498 | using HopscotchCompact_TicketLock_Set = 499 | Hopscotch_Locked_Set; 500 | 501 | template class Reclaimer, class K> 502 | using Hopscotch_Transaction_Set = 503 | Hopscotch_Locked_Set; 504 | } 505 | -------------------------------------------------------------------------------- /src/hash-tables/hs_serial.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "common.h" 4 | #include "primitives/cache_utils.h" 5 | #include "primitives/locks.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace concurrent_data_structures { 13 | 14 | template class Reclaimer, class K, 15 | class KT = KeyTraits> 16 | class HopscotchBitMap_Serial_Set { 17 | private: 18 | typedef std::uint64_t bit_mask_t; 19 | struct Bucket { 20 | bit_mask_t bit_mask; 21 | K key; 22 | }; 23 | 24 | std::size_t m_size_mask; 25 | Bucket *m_table; 26 | 27 | static const std::size_t S_BIT_MASK_RANGE = 64, 28 | S_MAX_INSERTION_RANGE = 4 * 1024; 29 | 30 | bool find_closer_free_backet(std::size_t &free_bucket, 31 | std::size_t &free_distance) { 32 | std::size_t distance = S_BIT_MASK_RANGE - 1; 33 | for (std::size_t current_bucket = free_bucket - distance; 34 | current_bucket < free_bucket; current_bucket++, distance--) { 35 | bit_mask_t bit_mask = m_table[current_bucket].bit_mask; 36 | while (bit_mask > 0) { 37 | const std::size_t moved_offset = ffsll(bit_mask) - 1; 38 | const std::size_t index = current_bucket + moved_offset; 39 | if (index >= free_bucket) { 40 | break; 41 | } 42 | // Entry we want to push up has moved. 43 | const K current_key = m_table[index].key; 44 | m_table[free_bucket].key = current_key; 45 | m_table[current_bucket].bit_mask |= bit_mask_t(1U) << distance; 46 | m_table[current_bucket].bit_mask &= ~(bit_mask_t(1U) << moved_offset); 47 | free_distance -= (free_bucket - index); 48 | free_bucket = index; 49 | return true; 50 | } 51 | } 52 | return false; 53 | } 54 | 55 | public: 56 | HopscotchBitMap_Serial_Set(const std::size_t size, const std::size_t threads) 57 | : m_size_mask(nearest_power_of_two(size) - 1), 58 | m_table(static_cast(Allocator::aligned_alloc( 59 | S_CACHE_PADDING, 60 | (m_size_mask + 1 + S_MAX_INSERTION_RANGE) * sizeof(Bucket)))) { 61 | for (std::size_t b = 0; b <= (m_size_mask + S_MAX_INSERTION_RANGE); b++) { 62 | m_table[b].bit_mask = bit_mask_t(0); 63 | m_table[b].key = KT::NullKey; 64 | } 65 | } 66 | 67 | ~HopscotchBitMap_Serial_Set() { Allocator::free(m_table); } 68 | 69 | bool thread_init(const std::size_t thread_id) { return true; } 70 | 71 | bool contains(const K key, const std::size_t thread_id) { 72 | const std::size_t hash = KT::hash(key); 73 | const std::size_t original_bucket = hash & m_size_mask; 74 | 75 | bit_mask_t bit_mask = m_table[original_bucket].bit_mask; 76 | while (bit_mask > 0) { 77 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 78 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 79 | const K current_key = m_table[current_index].key; 80 | if (key == current_key) { 81 | return true; 82 | } 83 | bit_mask &= ~(std::size_t(1) << lowest_set); 84 | } 85 | return false; 86 | } 87 | 88 | bool add(const K key, const std::size_t thread_id) { 89 | const std::size_t hash = KT::hash(key); 90 | const std::size_t original_bucket = hash & m_size_mask; 91 | 92 | bit_mask_t bit_mask = m_table[original_bucket].bit_mask; 93 | while (bit_mask > 0) { 94 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 95 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 96 | const K current_key = m_table[current_index].key; 97 | if (key == current_key) { 98 | return false; 99 | } 100 | bit_mask &= ~(std::size_t(1) << lowest_set); 101 | } 102 | 103 | std::size_t offset = 0; 104 | std::size_t reserved_bucket = original_bucket; 105 | for (; offset < S_MAX_INSERTION_RANGE; reserved_bucket++, offset++) { 106 | // If not in use and when claiming the bucket it was still not in use... 107 | if (m_table[reserved_bucket].key == KT::NullKey) { 108 | break; 109 | } 110 | } 111 | 112 | if (offset < S_MAX_INSERTION_RANGE) { 113 | while (offset >= S_BIT_MASK_RANGE) { 114 | bool moved = find_closer_free_backet(reserved_bucket, offset); 115 | if (!moved) { 116 | m_table[reserved_bucket].key = KT::NullKey; 117 | return false; 118 | } 119 | } 120 | m_table[reserved_bucket].key = key; 121 | m_table[original_bucket].bit_mask |= bit_mask_t(1) << offset; 122 | return true; 123 | } else { 124 | return false; 125 | } 126 | } 127 | 128 | bool remove(const K key, const std::size_t thread_id) { 129 | const std::size_t hash = KT::hash(key); 130 | const std::size_t original_bucket = hash & m_size_mask; 131 | 132 | bit_mask_t bit_mask = m_table[original_bucket].bit_mask; 133 | while (bit_mask > 0) { 134 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 135 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 136 | const K current_key = m_table[current_index].key; 137 | if (key == current_key) { 138 | m_table[current_index].key = KT::NullKey; 139 | m_table[original_bucket].bit_mask ^= std::size_t(1) << lowest_set; 140 | return true; 141 | } 142 | bit_mask &= ~(std::size_t(1) << lowest_set); 143 | } 144 | return false; 145 | } 146 | 147 | std::size_t size() { 148 | std::size_t counter = 0; 149 | return counter; 150 | } 151 | }; 152 | } 153 | -------------------------------------------------------------------------------- /src/hash-tables/hs_trans.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | //////////////////////////////////////////////////////////////////////////////// 4 | // ConcurrentHopscotchHashMap Class 5 | // 6 | //////////////////////////////////////////////////////////////////////////////// 7 | // TERMS OF USAGE 8 | //------------------------------------------------------------------------------ 9 | // 10 | // Permission to use, copy, modify and distribute this software and 11 | // its documentation for any purpose is hereby granted without fee, 12 | // provided that due acknowledgments to the authors are provided and 13 | // this permission notice appears in all copies of the software. 14 | // The software is provided "as is". There is no warranty of any kind. 15 | // 16 | // Authors: 17 | // Maurice Herlihy 18 | // Brown University 19 | // and 20 | // Nir Shavit 21 | // Tel-Aviv University 22 | // and 23 | // Moran Tzafrir 24 | // Tel-Aviv University 25 | // 26 | // Date: July 15, 2008. 27 | // 28 | //////////////////////////////////////////////////////////////////////////////// 29 | // Programmer : Moran Tzafrir (MoranTza@gmail.com) 30 | // 31 | //////////////////////////////////////////////////////////////////////////////// 32 | 33 | //////////////////////////////////////////////////////////////////////////////// 34 | // INCLUDE DIRECTIVES 35 | //////////////////////////////////////////////////////////////////////////////// 36 | #include "common.h" 37 | #include "math.h" 38 | #include "primitives/locks.h" 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | 46 | //////////////////////////////////////////////////////////////////////////////// 47 | // CLASS: ConcurrentHopscotchHashMap 48 | //////////////////////////////////////////////////////////////////////////////// 49 | 50 | namespace concurrent_data_structures { 51 | 52 | template class Reclaimer, class K, 53 | class KT = KeyTraits> 54 | class Hopscotch_Trans_Set { 55 | static const std::int32_t _NULL_DELTA = INT32_MIN; 56 | 57 | private: 58 | // Inner Classes ............................................................ 59 | struct Bucket { 60 | std::int32_t _first_delta; 61 | std::int32_t _next_delta; 62 | std::size_t _hash; 63 | K _key; 64 | 65 | void init() { 66 | _first_delta = _NULL_DELTA; 67 | _next_delta = _NULL_DELTA; 68 | _hash = 0; 69 | _key = KT::NullKey; 70 | } 71 | }; 72 | 73 | ElidedLock _lock; 74 | 75 | inline int first_msb_bit_indx(std::uint32_t x) { 76 | if (0 == x) 77 | return -1; 78 | return __builtin_clz(x) - 1; 79 | } 80 | 81 | // Fields ................................................................... 82 | std::size_t m_size_mask; 83 | Bucket *volatile _table; 84 | 85 | const int _cache_mask; 86 | const bool _is_cacheline_alignment; 87 | 88 | // Constants ................................................................ 89 | static const std::uint32_t _INSERT_RANGE = 1024 * 4; 90 | // static const std::uint32_t _NUM_SEGMENTS = 1024*1; 91 | // static const std::uint32_t _SEGMENTS_MASK = _NUM_SEGMENTS-1; 92 | static const std::uint32_t _RESIZE_FACTOR = 2; 93 | 94 | // Small Utilities .......................................................... 95 | Bucket *get_start_cacheline_bucket(Bucket *const bucket) { 96 | return (bucket - ((bucket - _table) & _cache_mask)); // can optimize 97 | } 98 | 99 | void remove_key(Bucket *const from_bucket, Bucket *const key_bucket, 100 | Bucket *const prev_key_bucket, const unsigned int hash) { 101 | key_bucket->_hash = 0; 102 | key_bucket->_key = KT::NullKey; 103 | 104 | if (NULL == prev_key_bucket) { 105 | if (_NULL_DELTA == key_bucket->_next_delta) 106 | from_bucket->_first_delta = _NULL_DELTA; 107 | else 108 | from_bucket->_first_delta = 109 | (from_bucket->_first_delta + key_bucket->_next_delta); 110 | } else { 111 | if (_NULL_DELTA == key_bucket->_next_delta) 112 | prev_key_bucket->_next_delta = _NULL_DELTA; 113 | else 114 | prev_key_bucket->_next_delta = 115 | (prev_key_bucket->_next_delta + key_bucket->_next_delta); 116 | } 117 | key_bucket->_next_delta = _NULL_DELTA; 118 | } 119 | void add_key_to_begining_of_list(Bucket *const keys_bucket, 120 | Bucket *const free_bucket, 121 | const std::size_t hash, const K &key) { 122 | free_bucket->_key = key; 123 | free_bucket->_hash = hash; 124 | 125 | if (0 == keys_bucket->_first_delta) { 126 | if (_NULL_DELTA == keys_bucket->_next_delta) 127 | free_bucket->_next_delta = _NULL_DELTA; 128 | else 129 | free_bucket->_next_delta = 130 | (short)((keys_bucket + keys_bucket->_next_delta) - free_bucket); 131 | keys_bucket->_next_delta = (std::int32_t)(free_bucket - keys_bucket); 132 | } else { 133 | if (_NULL_DELTA == keys_bucket->_first_delta) 134 | free_bucket->_next_delta = _NULL_DELTA; 135 | else 136 | free_bucket->_next_delta = 137 | (short)((keys_bucket + keys_bucket->_first_delta) - free_bucket); 138 | keys_bucket->_first_delta = (std::int32_t)(free_bucket - keys_bucket); 139 | } 140 | } 141 | 142 | void add_key_to_end_of_list(Bucket *const keys_bucket, 143 | Bucket *const free_bucket, const std::size_t hash, 144 | const K &key, Bucket *const last_bucket) { 145 | free_bucket->_key = key; 146 | free_bucket->_hash = hash; 147 | free_bucket->_next_delta = _NULL_DELTA; 148 | 149 | if (NULL == last_bucket) 150 | keys_bucket->_first_delta = (std::int32_t)(free_bucket - keys_bucket); 151 | else 152 | last_bucket->_next_delta = (std::int32_t)(free_bucket - last_bucket); 153 | } 154 | 155 | void optimize_cacheline_use(Bucket *const free_bucket) { 156 | Bucket *const start_cacheline_bucket = 157 | get_start_cacheline_bucket(free_bucket); 158 | Bucket *const end_cacheline_bucket = (start_cacheline_bucket + _cache_mask); 159 | Bucket *opt_bucket = start_cacheline_bucket; 160 | 161 | do { 162 | if (_NULL_DELTA != opt_bucket->_first_delta) { 163 | Bucket *relocate_key_last = nullptr; 164 | int curr_delta(opt_bucket->_first_delta); 165 | Bucket *relocate_key(opt_bucket + curr_delta); 166 | do { 167 | if (curr_delta < 0 || curr_delta > _cache_mask) { 168 | free_bucket->_key = relocate_key->_key; 169 | free_bucket->_hash = relocate_key->_hash; 170 | 171 | if (_NULL_DELTA == relocate_key->_next_delta) 172 | free_bucket->_next_delta = _NULL_DELTA; 173 | else 174 | free_bucket->_next_delta = 175 | (short)((relocate_key + relocate_key->_next_delta) - 176 | free_bucket); 177 | 178 | if (nullptr == relocate_key_last) 179 | opt_bucket->_first_delta = 180 | (std::int32_t)(free_bucket - opt_bucket); 181 | else 182 | relocate_key_last->_next_delta = 183 | (std::int32_t)(free_bucket - relocate_key_last); 184 | 185 | relocate_key->_hash = 0; 186 | relocate_key->_key = KT::NullKey; 187 | relocate_key->_next_delta = _NULL_DELTA; 188 | return; 189 | } 190 | 191 | if (_NULL_DELTA == relocate_key->_next_delta) 192 | break; 193 | relocate_key_last = relocate_key; 194 | curr_delta += relocate_key->_next_delta; 195 | relocate_key += relocate_key->_next_delta; 196 | } while (true); // for on list 197 | } 198 | ++opt_bucket; 199 | } while (opt_bucket <= end_cacheline_bucket); 200 | } 201 | 202 | public 203 | : // Ctors ................................................................ 204 | Hopscotch_Trans_Set( 205 | std::uint32_t inCapacity = 32 * 1024, // init capacity 206 | std::uint32_t concurrencyLevel = 207 | std::thread::hardware_concurrency(), // num of updating threads 208 | std::uint32_t cache_line_size = 64, // Cache-line size of machine 209 | bool is_optimize_cacheline = true) 210 | : _cache_mask((cache_line_size / sizeof(Bucket)) - 1), 211 | _is_cacheline_alignment(is_optimize_cacheline) { 212 | concurrencyLevel = concurrencyLevel << 10; 213 | 214 | const std::uint32_t adjInitCap = (inCapacity); 215 | m_size_mask = inCapacity - 1; 216 | // const std::uint32_t adjConcurrencyLevel = 217 | // NearestPowerOfTwo(concurrencyLevel); 218 | const std::uint32_t num_buckets(adjInitCap + _INSERT_RANGE + 1); 219 | // ALLOCATE THE SEGMENTS ................... 220 | // _segments = static_cast( 221 | // Allocator::malloc(sizeof(Segment) * (_segmentMask + 1))); 222 | _table = 223 | static_cast(Allocator::malloc(sizeof(Bucket) * num_buckets)); 224 | 225 | // Segment *curr_seg = _segments; 226 | // for (std::uint32_t iSeg = 0; iSeg <= _segmentMask; ++iSeg, ++curr_seg) 227 | // { 228 | // curr_seg->init(); 229 | // } 230 | 231 | Bucket *curr_bucket = _table; 232 | for (std::uint32_t iElm = 0; iElm < num_buckets; ++iElm, ++curr_bucket) { 233 | curr_bucket->init(); 234 | } 235 | } 236 | 237 | bool thread_init(const std::size_t thread_id) { return true; } 238 | 239 | ~Hopscotch_Trans_Set() { 240 | Allocator::free(_table); 241 | // Allocator::free(_segments); 242 | } 243 | 244 | // Query Operations ......................................................... 245 | bool contains(const K &key, const std::size_t thread_id) { 246 | 247 | // CALCULATE HASH .......................... 248 | const std::size_t hash(KT::hash(key)); 249 | 250 | std::lock_guard guard(_lock); 251 | 252 | const Bucket *curr_bucket(&(_table[hash & m_size_mask])); 253 | short next_delta(curr_bucket->_first_delta); 254 | while (_NULL_DELTA != next_delta) { 255 | curr_bucket += next_delta; 256 | if (key == curr_bucket->_key) { 257 | return true; 258 | } 259 | next_delta = curr_bucket->_next_delta; 260 | } 261 | return false; 262 | } 263 | 264 | // modification Operations ................................................... 265 | inline bool add(const K &key, const std::size_t thread_id) { 266 | const unsigned int hash(KT::hash(key)); 267 | // go over the list and look for key 268 | std::lock_guard guard(_lock); 269 | Bucket *const start_bucket(&(_table[hash & m_size_mask])); 270 | 271 | Bucket *last_bucket = nullptr; 272 | Bucket *compare_bucket = start_bucket; 273 | short next_delta(compare_bucket->_first_delta); 274 | while (_NULL_DELTA != next_delta) { 275 | compare_bucket += next_delta; 276 | if (hash == compare_bucket->_hash && key == compare_bucket->_key) { 277 | return false; 278 | } 279 | last_bucket = compare_bucket; 280 | next_delta = compare_bucket->_next_delta; 281 | } 282 | 283 | // try to place the key in the same cache-line 284 | if (_is_cacheline_alignment) { 285 | Bucket *free_bucket = start_bucket; 286 | Bucket *start_cacheline_bucket = get_start_cacheline_bucket(start_bucket); 287 | Bucket *end_cacheline_bucket(start_cacheline_bucket + _cache_mask); 288 | do { 289 | if (0 == free_bucket->_hash) { 290 | add_key_to_begining_of_list(start_bucket, free_bucket, hash, key); 291 | return true; 292 | } 293 | ++free_bucket; 294 | if (free_bucket > end_cacheline_bucket) 295 | free_bucket = start_cacheline_bucket; 296 | } while (start_bucket != free_bucket); 297 | } 298 | 299 | // place key in arbitrary free forward bucket 300 | Bucket *max_bucket(start_bucket + (INT32_MAX - 1)); 301 | Bucket *last_table_bucket(_table + m_size_mask); 302 | if (max_bucket > last_table_bucket) 303 | max_bucket = last_table_bucket; 304 | Bucket *free_max_bucket(start_bucket + (_cache_mask + 1)); 305 | while (free_max_bucket <= max_bucket) { 306 | if (KT::NullKey == free_max_bucket->_key) { 307 | add_key_to_end_of_list(start_bucket, free_max_bucket, hash, key, 308 | last_bucket); 309 | return true; 310 | } 311 | ++free_max_bucket; 312 | } 313 | 314 | // place key in arbitrary free backward bucket 315 | Bucket *min_bucket(start_bucket - (INT32_MAX - 1)); 316 | if (min_bucket < _table) 317 | min_bucket = _table; 318 | Bucket *free_min_bucket(start_bucket - (_cache_mask + 1)); 319 | while (free_min_bucket >= min_bucket) { 320 | if (KT::NullKey == free_max_bucket->_key) { 321 | add_key_to_end_of_list(start_bucket, free_min_bucket, hash, key, 322 | last_bucket); 323 | return true; 324 | } 325 | --free_min_bucket; 326 | } 327 | 328 | // NEED TO RESIZE .......................... 329 | fprintf(stderr, "ERROR - RESIZE is not implemented - size %u\n", size()); 330 | exit(1); 331 | return false; 332 | } 333 | 334 | inline bool remove(const K &key, const std::size_t thread_id) { 335 | // CALCULATE HASH .......................... 336 | const unsigned int hash(KT::hash(key)); 337 | 338 | std::lock_guard guard( 339 | /*this->_segments[(hash & m_size_mask) >> m_segment_shift].*/ _lock); 340 | Bucket *const start_bucket(&(_table[hash & m_size_mask])); 341 | Bucket *last_bucket(NULL); 342 | Bucket *curr_bucket(start_bucket); 343 | short next_delta(curr_bucket->_first_delta); 344 | do { 345 | if (_NULL_DELTA == next_delta) { 346 | return false; 347 | } 348 | curr_bucket += next_delta; 349 | 350 | if (hash == curr_bucket->_hash && (key == curr_bucket->_key)) { 351 | remove_key(/*this->_segments[(hash & m_size_mask) >> m_segment_shift],*/ 352 | start_bucket, curr_bucket, last_bucket, hash); 353 | if (_is_cacheline_alignment) 354 | optimize_cacheline_use( 355 | // this->_segments[(hash & m_size_mask) >> 356 | // m_segment_shift], 357 | curr_bucket); 358 | return true; 359 | } 360 | last_bucket = curr_bucket; 361 | next_delta = curr_bucket->_next_delta; 362 | } while (true); 363 | return false; 364 | } 365 | 366 | // status Operations ......................................................... 367 | unsigned int size() { 368 | std::uint32_t counter = 0; 369 | const std::uint32_t num_elm(m_size_mask + _INSERT_RANGE); 370 | for (std::uint32_t iElm = 0; iElm < num_elm; ++iElm) { 371 | if (0 != _table[iElm]._hash) { 372 | ++counter; 373 | } 374 | } 375 | return counter; 376 | } 377 | 378 | double percentKeysInCacheline() { 379 | unsigned int total_in_cache(0); 380 | unsigned int total(0); 381 | 382 | Bucket *curr_bucket(_table); 383 | for (int iElm(0); iElm <= m_size_mask; ++iElm, ++curr_bucket) { 384 | 385 | if (_NULL_DELTA != curr_bucket->_first_delta) { 386 | Bucket *const startCacheLineBucket( 387 | get_start_cacheline_bucket(curr_bucket)); 388 | Bucket *check_bucket(curr_bucket + curr_bucket->_first_delta); 389 | int currDist(curr_bucket->_first_delta); 390 | do { 391 | ++total; 392 | if ((check_bucket - startCacheLineBucket) >= 0 && 393 | (check_bucket - startCacheLineBucket) <= _cache_mask) 394 | ++total_in_cache; 395 | if (_NULL_DELTA == check_bucket->_next_delta) 396 | break; 397 | currDist += check_bucket->_next_delta; 398 | check_bucket += check_bucket->_next_delta; 399 | } while (true); 400 | } 401 | } 402 | 403 | // return percent in cache 404 | return (((double)total_in_cache) / ((double)total) * 100.0); 405 | } 406 | 407 | void print_table() {} 408 | 409 | private: 410 | // Private Static Utilities ................................................. 411 | static std::uint32_t NearestPowerOfTwo(const std::uint32_t value) { 412 | std::uint32_t rc(1); 413 | while (rc < value) { 414 | rc <<= 1; 415 | } 416 | return rc; 417 | } 418 | 419 | static unsigned int CalcDivideShift(const unsigned int _value) { 420 | unsigned int numShift(0); 421 | unsigned int curr(1); 422 | while (curr < _value) { 423 | curr <<= 1; 424 | ++numShift; 425 | } 426 | return numShift; 427 | } 428 | }; 429 | } 430 | -------------------------------------------------------------------------------- /src/hash-tables/hsbm_locked.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "common.h" 4 | #include "primitives/cache_utils.h" 5 | #include "primitives/locks.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace concurrent_data_structures { 13 | 14 | template class Reclaimer, class Lock, 15 | class K, class KT = KeyTraits> 16 | class HopscotchBitMap_Locked_Set { 17 | private: 18 | struct Segment { 19 | CachePadded> timestamp; 20 | Lock lock; 21 | }; 22 | 23 | typedef std::uint64_t bit_mask_t; 24 | 25 | struct Bucket { 26 | std::atomic bit_mask; 27 | std::atomic_bool in_use; 28 | std::atomic key; 29 | }; 30 | 31 | std::size_t m_size_mask, m_segment_shift; 32 | CachePadded *m_segments; 33 | Bucket *m_table; 34 | 35 | static const std::size_t S_BIT_MASK_RANGE = 64, 36 | S_MAX_INSERTION_RANGE = 4 * 1024; 37 | 38 | bool find_closer_free_backet(const std::size_t free_segment, 39 | std::size_t &free_bucket, 40 | std::size_t &free_distance) { 41 | loopBegin: 42 | std::size_t distance = S_BIT_MASK_RANGE - 1; 43 | for (std::size_t current_bucket = free_bucket - distance; 44 | current_bucket < free_bucket; current_bucket++, distance--) { 45 | bit_mask_t bit_mask = 46 | m_table[current_bucket].bit_mask.load(std::memory_order_relaxed); 47 | while (bit_mask > 0) { 48 | const std::size_t moved_offset = ffsll(bit_mask) - 1; 49 | const std::size_t index = current_bucket + moved_offset; 50 | if (index >= free_bucket) { 51 | break; 52 | } 53 | const std::size_t current_segment = index >> m_segment_shift; 54 | if (free_segment != current_segment) { 55 | m_segments[current_segment].lock.lock(); 56 | } 57 | const std::size_t bit_mask_after = 58 | m_table[current_bucket].bit_mask.load(std::memory_order_relaxed); 59 | // Entry we want to push up has moved. 60 | if (bit_mask_after != bit_mask) { 61 | if (free_segment != current_segment) { 62 | m_segments[current_segment].lock.lock(); 63 | } 64 | goto loopBegin; 65 | } 66 | const K current_key = 67 | m_table[index].key.load(std::memory_order_relaxed); 68 | m_table[free_bucket].key.store(current_key, std::memory_order_relaxed); 69 | m_table[current_bucket].bit_mask.fetch_or(bit_mask_t(1U) << distance, 70 | std::memory_order_relaxed); 71 | m_segments[current_segment].timestamp.fetch_add( 72 | 1, std::memory_order_relaxed); 73 | // m_table[index].key.store(free_key, std::memory_order_relaxed); 74 | m_table[current_bucket].bit_mask.fetch_and( 75 | ~(bit_mask_t(1U) << moved_offset)); 76 | free_distance -= (free_bucket - index); 77 | free_bucket = index; 78 | if (free_segment != current_segment) { 79 | m_segments[current_segment].lock.unlock(); 80 | } 81 | return true; 82 | } 83 | } 84 | return false; 85 | } 86 | 87 | public: 88 | HopscotchBitMap_Locked_Set(const std::size_t size, const std::size_t threads) 89 | : m_size_mask(nearest_power_of_two(size) - 1), 90 | m_table(static_cast(Allocator::aligned_alloc( 91 | S_CACHE_PADDING, 92 | (m_size_mask + 1 + S_MAX_INSERTION_RANGE) * sizeof(Bucket)))) { 93 | std::size_t num_timestamps = nearest_power_of_two(threads); 94 | std::uint8_t num_timestamp_bits = 0; 95 | for (std::size_t timestamp = num_timestamps; timestamp > 0; 96 | timestamp >>= 1, num_timestamp_bits++) { 97 | } 98 | std::uint8_t num_size_bits = 0; 99 | for (std::size_t size = m_size_mask + 1; size > 0; 100 | size >>= 1, num_size_bits++) { 101 | } 102 | std::uint8_t timestamp_shift = num_size_bits - num_timestamp_bits; 103 | m_segment_shift = timestamp_shift; 104 | m_segments = static_cast *>(Allocator::aligned_alloc( 105 | S_CACHE_PADDING, num_timestamps * sizeof(CachePadded))); 106 | 107 | for (std::size_t s = 0; s < num_timestamps; s++) { 108 | m_segments[s].timestamp.store(bit_mask_t(0), std::memory_order_relaxed); 109 | new (&m_segments[s].lock) Lock(); 110 | } 111 | 112 | for (std::size_t b = 0; b <= (m_size_mask + S_MAX_INSERTION_RANGE); b++) { 113 | m_table[b].bit_mask.store(bit_mask_t(0), std::memory_order_relaxed); 114 | m_table[b].in_use.store(false, std::memory_order_relaxed); 115 | m_table[b].key.store(KT::NullKey, std::memory_order_relaxed); 116 | } 117 | } 118 | 119 | ~HopscotchBitMap_Locked_Set() { 120 | Allocator::free(m_table); 121 | Allocator::free(m_segments); 122 | } 123 | 124 | bool thread_init(const std::size_t thread_id) { return true; } 125 | 126 | bool contains(const K key, const std::size_t thread_id) { 127 | const std::size_t hash = KT::hash(key); 128 | const std::size_t original_bucket = hash & m_size_mask; 129 | const std::size_t original_segment = original_bucket >> m_segment_shift; 130 | 131 | std::intptr_t timestamp_before = 132 | m_segments[original_segment].timestamp.load(); 133 | while (true) { 134 | bit_mask_t bit_mask = 135 | m_table[original_bucket].bit_mask.load(std::memory_order_relaxed); 136 | while (bit_mask > 0) { 137 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 138 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 139 | const K current_key = 140 | m_table[current_index].key.load(std::memory_order_relaxed); 141 | if (key == current_key) { 142 | return true; 143 | } 144 | bit_mask &= ~(std::size_t(1) << lowest_set); 145 | } 146 | const std::intptr_t timestamp_after = 147 | m_segments[original_segment].timestamp.load(); 148 | if (timestamp_before != timestamp_after) { 149 | timestamp_before = timestamp_after; 150 | continue; 151 | } 152 | return false; 153 | } 154 | } 155 | 156 | bool add(const K key, const std::size_t thread_id) { 157 | const std::size_t hash = KT::hash(key); 158 | const std::size_t original_bucket = hash & m_size_mask; 159 | const std::size_t original_segment = original_bucket >> m_segment_shift; 160 | 161 | std::lock_guard guard(m_segments[original_segment].lock); 162 | 163 | bit_mask_t bit_mask = 164 | m_table[original_bucket].bit_mask.load(std::memory_order_relaxed); 165 | while (bit_mask > 0) { 166 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 167 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 168 | const K current_key = 169 | m_table[current_index].key.load(std::memory_order_relaxed); 170 | if (key == current_key) { 171 | return false; 172 | } 173 | bit_mask &= ~(std::size_t(1) << lowest_set); 174 | } 175 | 176 | std::size_t offset = 0; 177 | std::size_t reserved_bucket = original_bucket; 178 | for (; offset < S_MAX_INSERTION_RANGE; reserved_bucket++, offset++) { 179 | // If not in use and when claiming the bucket it was still not in use... 180 | if (!m_table[reserved_bucket].in_use.load(std::memory_order_relaxed) and 181 | m_table[reserved_bucket].in_use.exchange( 182 | true, std::memory_order_relaxed) == false) { 183 | break; 184 | } 185 | } 186 | 187 | if (offset < S_MAX_INSERTION_RANGE) { 188 | while (offset >= S_BIT_MASK_RANGE) { 189 | bool moved = 190 | find_closer_free_backet(original_segment, reserved_bucket, offset); 191 | if (!moved) { 192 | m_table[reserved_bucket].key.store(KT::NullKey, 193 | std::memory_order_relaxed); 194 | m_table[reserved_bucket].in_use.store(false, 195 | std::memory_order_release); 196 | return false; 197 | } 198 | } 199 | m_table[reserved_bucket].key.store(key, std::memory_order_relaxed); 200 | m_table[original_bucket].bit_mask.fetch_or(bit_mask_t(1) << offset); 201 | return true; 202 | } else { 203 | // std::cerr << "Couldn't find a bucket within the range." << 204 | // std::endl; 205 | return false; 206 | } 207 | } 208 | 209 | bool remove(const K key, const std::size_t thread_id) { 210 | const std::size_t hash = KT::hash(key); 211 | const std::size_t original_bucket = hash & m_size_mask; 212 | const std::size_t original_segment = original_bucket >> m_segment_shift; 213 | 214 | std::lock_guard guard(m_segments[original_segment].lock); 215 | bit_mask_t bit_mask = 216 | m_table[original_bucket].bit_mask.load(std::memory_order_relaxed); 217 | while (bit_mask > 0) { 218 | const std::size_t lowest_set = ffsll(bit_mask) - 1; 219 | const std::size_t current_index = (hash + lowest_set) & m_size_mask; 220 | const K current_key = 221 | m_table[current_index].key.load(std::memory_order_relaxed); 222 | if (key == current_key) { 223 | m_table[current_index].key.store(KT::NullKey, 224 | std::memory_order_relaxed); 225 | m_table[current_index].in_use.store(false, std::memory_order_release); 226 | m_table[original_bucket].bit_mask.fetch_xor( 227 | std::size_t(1) << lowest_set, std::memory_order_relaxed); 228 | return true; 229 | } 230 | bit_mask &= ~(std::size_t(1) << lowest_set); 231 | } 232 | return false; 233 | } 234 | 235 | std::size_t size() { 236 | std::size_t counter = 0; 237 | return counter; 238 | } 239 | }; 240 | 241 | template class Reclaimer, class K> 242 | using HopscotchBitMap_SpinLock_Set = 243 | HopscotchBitMap_Locked_Set; 244 | 245 | template class Reclaimer, class K> 246 | using HopscotchBitMap_Mutex_Set = 247 | HopscotchBitMap_Locked_Set; 248 | 249 | template class Reclaimer, class K> 250 | using HopscotchBitMap_TicketLock_Set = 251 | HopscotchBitMap_Locked_Set; 252 | } 253 | -------------------------------------------------------------------------------- /src/hash-tables/main.cpp: -------------------------------------------------------------------------------- 1 | #undef NDEBUG 2 | #include "allocators/glib_allocator.h" 3 | #include "allocators/intel.h" 4 | #include "allocators/jemalloc_allocator.h" 5 | #include "allocators/super_malloc_allocator.h" 6 | #include "bench/arg_parsing.h" 7 | #include "bench/benchmark_config.h" 8 | #include "bench/benchmark_summary.h" 9 | #include "bench/benchmark_table.h" 10 | #include "hash-tables/hs_locked.h" 11 | #include "hash-tables/hs_serial.h" 12 | #include "hash-tables/hs_trans.h" 13 | #include "hash-tables/hsbm_lf.h" 14 | #include "hash-tables/hsbm_lf_compact.h" 15 | #include "hash-tables/hsbm_locked.h" 16 | #include "hash-tables/ph_qp.h" 17 | #include "mem-reclaimer/leaky.h" 18 | #include "random/lcg.h" 19 | #include "random/xorshift.h" 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace concurrent_data_structures; 28 | 29 | namespace { 30 | 31 | static const boost::filesystem::path BENCH_PATH = 32 | boost::filesystem::path("hash_tables"); 33 | 34 | // https://stackoverflow.com/a/3418285 35 | void replaceAll(std::string &str, const std::string &from, 36 | const std::string &to) { 37 | if (from.empty()) 38 | return; 39 | size_t start_pos = 0; 40 | while ((start_pos = str.find(from, start_pos)) != std::string::npos) { 41 | str.replace(start_pos, from.length(), to); 42 | start_pos += to.length(); // In case 'to' contains 'from', like replacing 43 | // 'x' with 'yx' 44 | } 45 | } 46 | } // namespace 47 | 48 | const std::string generate_file_name(const SetBenchmarkConfig &config) { 49 | std::stringstream file_name; 50 | file_name << std::string("M:") + get_reclaimer_name(config.base.reclaimer) 51 | << " A:" << get_allocator_name(config.base.allocator) 52 | << std::string("R:") + get_generator_name(config.base.generator) 53 | << " T:" << config.base.num_threads << " S:" << config.table_size 54 | << " U:" << config.updates << " L:" << config.load_factor 55 | << std::string(".txt"); 56 | std::string human_file_name = file_name.str(); 57 | replaceAll(human_file_name, " ", "_"); 58 | return human_file_name; 59 | } 60 | 61 | // Sick 62 | template class, class...> class Table, 64 | class Allocator, template class Reclaimer> 65 | bool run_and_save(const SetBenchmarkConfig &config) { 66 | typedef std::intptr_t Key; 67 | typedef Table HashTable; 68 | 69 | const boost::filesystem::path prefix_path = 70 | config.base.results_directory / BENCH_PATH; 71 | std::string table_name = get_table_name(config.table); 72 | replaceAll(table_name, " ", ""); 73 | const boost::filesystem::path table_path = prefix_path / table_name; 74 | if (!boost::filesystem::exists(table_path)) { 75 | if (!boost::filesystem::create_directory(table_path)) { 76 | std::cout << "Failed to create directory: \"" << table_path.string() 77 | << "\"" << std::endl; 78 | set_print_help_and_exit(); 79 | } 80 | } 81 | TableBenchmark benchmark(config); 82 | produce_summary(config, benchmark.bench(), 83 | table_path.string() + "/" + generate_file_name(config), 84 | prefix_path.string() + "/" + "set_keys.csv", 85 | prefix_path.string() + "/" + "set_results.csv"); 86 | 87 | return true; 88 | } 89 | 90 | template class Reclaimer> 91 | bool fix_table(const SetBenchmarkConfig &config) { 92 | switch (config.table) { 93 | case HashTable::PH_QP_SET: 94 | return run_and_save(config); 95 | case HashTable::HSBM_SERIAL_SET: 96 | return run_and_save(config); 98 | case HashTable::HSBM_LF_SET: 99 | return run_and_save( 100 | config); 101 | case HashTable::HSC_LOCKED_SET: 102 | return run_and_save(config); 104 | case HashTable::HS_LOCKED_SET: 105 | return run_and_save( 106 | config); 107 | case HashTable::HSBM_LOCKED_SET: 108 | return run_and_save(config); 110 | case HashTable::HS_TRANS_SET: 111 | return run_and_save(config); 113 | default: 114 | return false; 115 | } 116 | } 117 | 118 | template class Reclaimer> 119 | bool fix_allocator(const SetBenchmarkConfig &config) { 120 | switch (config.base.allocator) { 121 | case Allocator::Glibc: 122 | return fix_table(config); 123 | case Allocator::JeMalloc: 124 | return fix_table(config); 125 | case Allocator::Intel: 126 | return fix_table(config); 127 | default: 128 | return false; 129 | } 130 | } 131 | 132 | template bool fix_reclaimer(const SetBenchmarkConfig &config) { 133 | return fix_allocator(config); 134 | } 135 | 136 | bool run(const SetBenchmarkConfig &config) { 137 | switch (config.base.generator) { 138 | case RandomGenerator::LCGBSD: 139 | return fix_reclaimer(config); 140 | case RandomGenerator::PCG: 141 | return fix_reclaimer(config); 142 | case RandomGenerator::XOR_SHIFT_32: 143 | return fix_reclaimer(config); 144 | case RandomGenerator::XOR_SHIFT_64: 145 | return fix_reclaimer(config); 146 | default: 147 | return false; 148 | } 149 | } 150 | 151 | int main(int argc, char *argv[]) { 152 | const SetBenchmarkConfig config = parse_set_args(argc, argv); 153 | if (config.base.papi_active) { 154 | assert(PAPI_library_init(PAPI_VER_CURRENT) == PAPI_VER_CURRENT and 155 | "Couldn't initialise PAPI library. Check installation."); 156 | int res = PAPI_thread_init(pthread_self); 157 | assert(res == PAPI_OK); 158 | } 159 | config.print(std::cout); 160 | if (!boost::filesystem::exists(config.base.results_directory)) { 161 | if (!boost::filesystem::create_directory(config.base.results_directory)) { 162 | std::cout << "Failed to create directory: \"" 163 | << config.base.results_directory.string() << "\"" << std::endl; 164 | set_print_help_and_exit(); 165 | } 166 | } 167 | if (!boost::filesystem::exists(config.base.results_directory / BENCH_PATH)) { 168 | if (!boost::filesystem::create_directory(config.base.results_directory / 169 | BENCH_PATH)) { 170 | std::cout << "Failed to create directory: \"" 171 | << (config.base.results_directory.string() + 172 | BENCH_PATH.string()) 173 | << "\"" << std::endl; 174 | set_print_help_and_exit(); 175 | } 176 | } 177 | if (!run(config)) { 178 | set_print_help_and_exit(); 179 | } 180 | std::cout << "Finished." << std::endl; 181 | 182 | return 0; 183 | } 184 | -------------------------------------------------------------------------------- /src/hash-tables/ph_qp.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Lock-Free quadratic probing with physical deletion based on 5 | "Non-blocking hash tables with open addressing" by Chris Purcell 6 | and Tim Harris. 7 | Copyright (C) 2018 Robert Kelly 8 | 9 | This program is free software: you can redistribute it and/or modify 10 | it under the terms of the GNU General Public License as published by 11 | the Free Software Foundation, either version 3 of the License, or 12 | (at your option) any later version. 13 | 14 | This program is distributed in the hope that it will be useful, 15 | but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | GNU General Public License for more details. 18 | 19 | You should have received a copy of the GNU General Public License 20 | along with this program. If not, see . 21 | */ 22 | 23 | #include "common.h" 24 | #include 25 | 26 | namespace concurrent_data_structures { 27 | 28 | template class Reclaimer, class K, 29 | class KT = KeyTraits> 30 | class PH_QP_Set { 31 | private: 32 | struct Bucket { 33 | std::atomic_uintptr_t vs; 34 | std::atomic key; 35 | }; 36 | std::size_t m_size, m_size_mask; 37 | std::atomic_size_t *m_probe_bounds; 38 | Bucket *m_table; 39 | 40 | static const std::uintptr_t S_EMPTY = 0, S_INSERTING = 1, S_MEMBER = 2, 41 | S_VISIBLE = 3, S_BUSY = 4, S_COLLIDED = 5; 42 | 43 | Bucket *get_bucket(std::size_t hash, std::size_t index) { 44 | return &m_table[(hash + index * (index + 1) / 2) & m_size_mask]; 45 | // return &m_table[(hash + index) & m_size_mask]; 46 | } 47 | 48 | std::uintptr_t get_version(std::uintptr_t vs) { return vs >> 32; } 49 | 50 | std::uintptr_t embed_version(std::uintptr_t vs, std::uintptr_t v) { 51 | return get_state(vs) | (v << 32); 52 | } 53 | 54 | std::uintptr_t get_state(std::uintptr_t vs) { 55 | return vs & ~(get_version(vs) << 32); 56 | } 57 | 58 | std::uintptr_t embed_state(std::uintptr_t v, std::uintptr_t s) { 59 | return (v << 32) | s; 60 | } 61 | 62 | bool contains_collision(std::size_t hash, std::size_t index) { 63 | std::uintptr_t vs1 = get_bucket(hash, index)->vs.load(); 64 | std::uintptr_t state1 = get_state(vs1); 65 | if (state1 == S_VISIBLE or state1 == S_INSERTING or state1 == S_MEMBER) { 66 | if ((KT::hash( 67 | get_bucket(hash, index)->key.load(std::memory_order_relaxed)) & 68 | m_size_mask) == hash) { 69 | std::uintptr_t vs2 = get_bucket(hash, index)->vs.load(); 70 | std::uintptr_t state2 = get_state(vs2); 71 | if (state2 == S_VISIBLE or state2 == S_INSERTING or 72 | state2 == S_MEMBER) { 73 | if (get_version(vs1) == get_version(vs2)) { 74 | return true; 75 | } 76 | } 77 | } 78 | } 79 | return false; 80 | } 81 | 82 | std::uintptr_t get_probe_bound(const std::uintptr_t pb) { return pb >> 1; } 83 | bool get_scanning(const std::uintptr_t pb) { return (pb & 1) == 1; } 84 | std::uintptr_t embed_scanning(const std::uintptr_t b, const bool scanning) { 85 | return (b << 1) | (scanning ? 1 : 0); 86 | } 87 | void cond_raise_bound(std::size_t hash, std::size_t index) { 88 | std::uintptr_t pb, new_pb; 89 | do { 90 | pb = m_probe_bounds[hash].load(); 91 | new_pb = std::max(get_probe_bound(pb), index); 92 | } while (!m_probe_bounds[hash].compare_exchange_strong( 93 | pb, embed_scanning(new_pb, false))); 94 | } 95 | 96 | void cond_lower_bound(std::size_t hash, std::size_t index) { 97 | std::uintptr_t pb = m_probe_bounds[hash].load(); 98 | if (get_scanning(pb)) { 99 | m_probe_bounds[hash].compare_exchange_strong( 100 | pb, embed_scanning(get_probe_bound(pb), false)); 101 | } 102 | if (index > 0) { 103 | pb = embed_scanning(index, false); 104 | while (m_probe_bounds[hash].compare_exchange_strong( 105 | pb, embed_scanning(index, true))) { 106 | std::size_t i = index - 1; 107 | while (i > 0 and !contains_collision(hash, i)) { 108 | i--; 109 | } 110 | std::size_t expected_index = embed_scanning(index, true); 111 | m_probe_bounds[hash].compare_exchange_strong(expected_index, 112 | embed_scanning(i, false)); 113 | } 114 | } 115 | } 116 | 117 | bool assist(const K &key, const std::size_t hash, const std::size_t index, 118 | std::size_t ver_i) { 119 | std::uintptr_t pb = m_probe_bounds[hash].load(); 120 | std::size_t max = get_probe_bound(pb); 121 | for (std::size_t j = 0; j <= max; j++) { 122 | if (j != index) { 123 | Bucket *bucket_j = get_bucket(hash, j); 124 | std::uintptr_t vs_j = bucket_j->vs.load(); 125 | std::uintptr_t s_j = get_state(vs_j); 126 | std::uintptr_t expected_ver_i = embed_state(ver_i, S_INSERTING); 127 | if (s_j == S_INSERTING and 128 | bucket_j->key.load(std::memory_order_relaxed) == key) { 129 | if (j < index) { 130 | if (bucket_j->vs.load() == vs_j) { 131 | /*bool _ =*/get_bucket(hash, index) 132 | ->vs.compare_exchange_strong(expected_ver_i, 133 | embed_state(ver_i, S_COLLIDED)); 134 | return assist(key, hash, j, get_version(vs_j)); 135 | } else { 136 | if (get_bucket(hash, index)->vs.load() == 137 | embed_state(ver_i, S_INSERTING)) { 138 | // bool _ = 139 | get_bucket(hash, index) 140 | ->vs.compare_exchange_strong( 141 | expected_ver_i, embed_state(ver_i, S_COLLIDED)); 142 | } 143 | } 144 | } 145 | } 146 | vs_j = bucket_j->vs.load(); 147 | std::uintptr_t v_j = get_version(vs_j); 148 | if (get_state(vs_j) == S_MEMBER and 149 | bucket_j->key.load(std::memory_order_relaxed) == key) { 150 | if (bucket_j->vs.load() == embed_state(v_j, S_MEMBER)) { 151 | /*bool _ = */ get_bucket(hash, index) 152 | ->vs.compare_exchange_strong(expected_ver_i, 153 | embed_state(ver_i, S_COLLIDED)); 154 | return false; 155 | } 156 | } 157 | } 158 | } 159 | ver_i = embed_state(ver_i, S_INSERTING); 160 | // bool _ = 161 | get_bucket(hash, index) 162 | ->vs.compare_exchange_strong(ver_i, embed_state(ver_i, S_MEMBER)); 163 | return true; 164 | } 165 | 166 | public: 167 | PH_QP_Set(const std::size_t size, const std::size_t threads) 168 | : m_size(nearest_power_of_two(size)), m_size_mask(m_size - 1), 169 | m_probe_bounds(static_cast( 170 | Allocator::malloc(sizeof(std::atomic_size_t) * m_size))), 171 | m_table( 172 | static_cast(Allocator::malloc(sizeof(Bucket) * m_size))) { 173 | for (std::size_t i = 0; i < m_size; i++) { 174 | m_table[i].vs.store(0, std::memory_order_relaxed); 175 | m_table[i].key.store(KT::NullKey, std::memory_order_relaxed); 176 | m_probe_bounds[i].store(0, std::memory_order_relaxed); 177 | } 178 | } 179 | ~PH_QP_Set() { 180 | Allocator::free(m_probe_bounds); 181 | Allocator::free(m_table); 182 | /*print_table(); */ 183 | } 184 | 185 | bool thread_init(const std::size_t thread_id) { return true; } 186 | 187 | bool contains(const K &key, std::size_t thread_id) { 188 | const std::size_t hash = KT::hash(key) & m_size_mask; 189 | std::uintptr_t pb = m_probe_bounds[hash].load(); 190 | std::size_t max = get_probe_bound(pb); 191 | for (std::size_t i = 0; i <= max; i++) { 192 | const Bucket *bucket = get_bucket(hash, i); 193 | std::uintptr_t vs = bucket->vs.load(); 194 | std::uintptr_t state = get_state(vs); 195 | if (state == S_MEMBER and 196 | bucket->key.load(std::memory_order_relaxed) == key) { 197 | if (bucket->vs.load() == vs) { 198 | return true; 199 | } 200 | } 201 | } 202 | return false; 203 | } 204 | 205 | bool add(const K &key, std::size_t thread_id) { 206 | const std::size_t hash = KT::hash(key) & m_size_mask; 207 | std::size_t i = 0; 208 | std::uintptr_t version; 209 | for (; i < m_size; i++) { 210 | Bucket *current_bucket = get_bucket(hash, i); 211 | std::uintptr_t vs = current_bucket->vs.load(); 212 | loadBegin: 213 | std::uintptr_t state = get_state(vs); 214 | version = get_version(vs); 215 | if (state != S_EMPTY) { 216 | continue; 217 | } else if (state == S_EMPTY) { 218 | if (current_bucket->vs.compare_exchange_strong( 219 | vs, embed_state(vs, S_BUSY))) { 220 | break; 221 | } else { 222 | goto loadBegin; 223 | } 224 | } 225 | } 226 | Bucket *current_bucket = get_bucket(hash, i); 227 | current_bucket->key.store(key, std::memory_order_relaxed); 228 | 229 | while (true) { 230 | current_bucket->vs.store(embed_state(version, S_VISIBLE)); 231 | cond_raise_bound(hash, i); 232 | current_bucket->vs.store(embed_state(version, S_INSERTING)); 233 | bool r = assist(key, hash, i, version); 234 | std::uintptr_t vs = current_bucket->vs.load(); 235 | std::uintptr_t expected_vs = embed_state(version, S_COLLIDED); 236 | if (vs != expected_vs) { 237 | return true; 238 | } 239 | if (!r) { 240 | cond_lower_bound(hash, i); 241 | current_bucket->key.store(KT::NullKey, std::memory_order_relaxed); 242 | current_bucket->vs.store(embed_state(version + 1, S_EMPTY)); 243 | return false; 244 | } 245 | version++; 246 | } 247 | } 248 | 249 | bool remove(const K &key, std::size_t thread_id) { 250 | const std::size_t hash = KT::hash(key) & m_size_mask; 251 | std::uintptr_t pb = m_probe_bounds[hash].load(); 252 | std::size_t max = get_probe_bound(pb); 253 | for (std::size_t i = 0; i <= max; i++) { 254 | Bucket *current_bucket = get_bucket(hash, i); 255 | std::uintptr_t vs = current_bucket->vs.load(); 256 | std::uintptr_t state = get_state(vs); 257 | const K cur_key = current_bucket->key.load(std::memory_order_relaxed); 258 | if (state == S_MEMBER and cur_key == key) { 259 | if (current_bucket->vs.compare_exchange_strong( 260 | vs, embed_state(vs, S_BUSY))) { 261 | cond_lower_bound(hash, i); 262 | std::uintptr_t version = get_version(vs); 263 | current_bucket->key.store(KT::NullKey, std::memory_order_relaxed); 264 | current_bucket->vs.store(embed_state(version + 1, S_EMPTY)); 265 | return true; 266 | } 267 | } 268 | } 269 | return false; 270 | } 271 | 272 | void print_table() { 273 | std::cout << "*********************" << std::endl; 274 | for (std::size_t i = 0; i < m_size; i++) { 275 | Bucket *current_bucket = &m_table[i]; 276 | const K key = current_bucket->key.load(); 277 | const std::uintptr_t vs = current_bucket->vs.load(); 278 | const std::size_t hash = (KT::hash(key) & m_size_mask); 279 | bool empty = key == KT::NullKey; 280 | const std::uintptr_t pb = m_probe_bounds[i].load(); 281 | if (empty) { 282 | std::cout << i << " - Key: Empty"; 283 | } else { 284 | std::cout << i << " - Key: " << key; 285 | } 286 | std::cout << " version: " << get_version(vs) 287 | << " state: " << get_state(vs); 288 | 289 | if (empty) { 290 | std::cout << " hash: NA"; 291 | } else { 292 | std::cout << " hash: " << hash; 293 | } 294 | std::cout << " raw bound: " << pb 295 | << " probe bound: " << get_probe_bound(pb) 296 | << " scanning: " << get_scanning(pb) << std::endl; 297 | } 298 | std::cout << "*********************" << std::endl; 299 | } 300 | }; 301 | } 302 | -------------------------------------------------------------------------------- /src/hash-tables/table_init.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple class to initialise a hash table. 5 | Copyright (C) 2018 Robert Kelly 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #include "bench/benchmark_config.h" 22 | #include "primitives/cache_utils.h" 23 | #include "random/pcg_random.h" 24 | #include 25 | #include 26 | 27 | namespace concurrent_data_structures { 28 | 29 | template 30 | static Table *TableInit(const SetBenchmarkConfig &config) { 31 | Table *table = new (Allocator::aligned_alloc(S_CACHE_PADDING, sizeof(Table))) 32 | Table(config.table_size, config.base.num_threads); 33 | std::size_t amount = 34 | static_cast(config.table_size * config.load_factor); 35 | std::vector keys(config.table_size); 36 | for (std::size_t i = 0; i < keys.size(); i++) { 37 | keys[i] = i; 38 | } 39 | pcg32 random = pcg_extras::seed_seq_from(); 40 | std::shuffle(keys.begin(), keys.end(), random); 41 | std::size_t total = 0; 42 | for (std::size_t i = 0; total < amount; i++) { 43 | if (table->add(keys[i], i % config.base.num_threads)) { 44 | total++; 45 | } 46 | } 47 | if (total < amount) { 48 | assert(false); 49 | } 50 | return table; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/mem-reclaimer/leaky.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | "Leaky" reclaimer. Does nothing but fit interface for reclaimers. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include "mem-reclaimer/reclaimer.h" 19 | #include "primitives/marked_pointers.h" 20 | #include 21 | 22 | namespace concurrent_data_structures { 23 | 24 | template 25 | class LeakyReclaimer : public ReclaimerAllocator { 26 | public: 27 | class LeakyBase {}; 28 | 29 | class LeakyHandle { 30 | public: 31 | LeakyHandle() {} 32 | ~LeakyHandle() {} 33 | LeakyHandle(const LeakyHandle &rhs) = delete; 34 | LeakyHandle &operator=(const LeakyHandle &rhs) = delete; 35 | LeakyHandle(LeakyHandle &&rhs) {} 36 | LeakyHandle &operator=(LeakyHandle &&rhs) { return *this; } 37 | 38 | void set(const LeakyBase *ptr) {} 39 | 40 | template 41 | bool try_protect(PtrType &ptr, const AtomicSourceType &src, Func f) { 42 | return true; 43 | } 44 | template 45 | bool try_protect(PtrType &ptr, const AtomicSourceType &src) noexcept { 46 | return this->try_protect(ptr, src, [](PtrType ptr) { return ptr; }); 47 | } 48 | 49 | template 50 | PtrType get_protected(const AtomicSourceType &src, Func f) {} 51 | template 52 | PtrType get_protected(const AtomicSourceType &src) noexcept { 53 | return this->get_protected(src, [](PtrType ptr) { return ptr; }); 54 | } 55 | friend class LeakyReclaimer; 56 | }; 57 | 58 | class LeakyArrayHandle { 59 | public: 60 | LeakyArrayHandle() {} 61 | ~LeakyArrayHandle() {} 62 | LeakyArrayHandle(const LeakyArrayHandle &rhs) = delete; 63 | LeakyArrayHandle &operator=(const LeakyArrayHandle &rhs) = delete; 64 | LeakyArrayHandle(LeakyArrayHandle &&rhs) {} 65 | LeakyArrayHandle &operator=(LeakyArrayHandle &&rhs) { return *this; } 66 | 67 | void set(const std::size_t index, LeakyBase *ptr) {} 68 | 69 | template 70 | bool try_protect(const std::size_t index, LeakyBase *ptr, 71 | const std::atomic &src, Func f) { 72 | return true; 73 | } 74 | 75 | template 76 | bool try_protect(const std::size_t index, MarkedPointer ptr, 77 | const AtomicMarkedPointer &src, Func f) { 78 | return true; 79 | } 80 | 81 | template 82 | bool try_protect(const std::size_t index, LeakyBase *ptr, 83 | const std::atomic &src) { 84 | return this->try_protect(index, ptr, src, 85 | [](LeakyBase *ptr) { return ptr; }); 86 | } 87 | 88 | template 89 | bool try_protect(const std::size_t index, MarkedPointer ptr, 90 | const AtomicMarkedPointer &src) { 91 | return this->try_protect(index, ptr, src, 92 | [](MarkedPointer ptr) { return ptr; }); 93 | } 94 | 95 | template 96 | PtrType get_protected(const std::size_t index, 97 | const std::atomic &src, Func f) { 98 | PtrType ptr = f(src.load()); 99 | while (!try_protect(index, ptr, src, f)) { 100 | ptr = f(src.load()); 101 | } 102 | return ptr; 103 | } 104 | template 105 | PtrType get_protected(const std::size_t index, 106 | const std::atomic &src) noexcept { 107 | return this->get_protected(index, src, [](PtrType ptr) { return ptr; }); 108 | } 109 | friend class LeakyReclaimer; 110 | }; 111 | 112 | typedef LeakyBase RecordBase; 113 | typedef LeakyHandle RecordHandle; 114 | typedef LeakyArrayHandle RecordArrayHandle; 115 | 116 | LeakyReclaimer(const std::size_t num_threads, 117 | const std::size_t refs_per_thread) {} 118 | ~LeakyReclaimer() {} 119 | bool thread_init(const std::size_t thread_id) { return true; } 120 | void enter(const std::size_t thread_id) {} 121 | void exit(const std::size_t thread_id) {} 122 | LeakyHandle get_rec(const std::size_t thread_id) { return LeakyHandle(); } 123 | LeakyArrayHandle get_bulk_rec(const std::size_t thread_id, 124 | const std::size_t size) { 125 | return LeakyArrayHandle(); 126 | } 127 | void retire(const RecordHandle &handle, const std::size_t thread_id) {} 128 | void retire(const RecordArrayHandle &handle, const std::size_t thread_id, 129 | const std::size_t index) {} 130 | }; 131 | } 132 | -------------------------------------------------------------------------------- /src/mem-reclaimer/reclaimer.cpp: -------------------------------------------------------------------------------- 1 | #include "reclaimer.h" 2 | 3 | /* 4 | Maps reclaimers to human readable strings. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | 20 | namespace concurrent_data_structures { 21 | 22 | namespace { 23 | static const std::map reclaimer_map{ 24 | std::make_pair(Reclaimer::Leaky, "Leaky"), 25 | }; 26 | } 27 | 28 | const std::string get_reclaimer_name(const Reclaimer reclaimer) { 29 | std::string reclaimer_name = "ERROR: Incorrect reclaimer name."; 30 | auto it = reclaimer_map.find(reclaimer); 31 | if (it != reclaimer_map.end()) { 32 | reclaimer_name = it->second; 33 | } 34 | return reclaimer_name; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/mem-reclaimer/reclaimer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Enums for reclaimers used. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | namespace concurrent_data_structures { 22 | 23 | enum class Reclaimer { 24 | Leaky, 25 | }; 26 | 27 | const std::string get_reclaimer_name(const Reclaimer reclaimer); 28 | 29 | template class ReclaimerAllocator { 30 | public: 31 | void *malloc(size_t size) { return Allocator::malloc(size); } 32 | void free(void *ptr) { Allocator::free(ptr); } 33 | size_t malloc_usable_size(void *ptr) { 34 | return Allocator::malloc_usable_size(ptr); 35 | } 36 | }; 37 | 38 | template class ReclaimerPin { 39 | private: 40 | typedef typename MemReclaimer::RecordHandle RecordHandle; 41 | typedef typename MemReclaimer::RecordArrayHandle RecordArrayHandle; 42 | 43 | MemReclaimer *m_reclaimer; 44 | std::size_t m_thread_id; 45 | 46 | public: 47 | ReclaimerPin(MemReclaimer *reclaimer, std::size_t thread_id) 48 | : m_reclaimer(reclaimer), m_thread_id(thread_id) { 49 | m_reclaimer->enter(m_thread_id); 50 | } 51 | ~ReclaimerPin() { m_reclaimer->exit(m_thread_id); } 52 | 53 | RecordHandle get_rec() { return m_reclaimer->get_rec(m_thread_id); } 54 | RecordArrayHandle get_bulk_rec(const std::size_t size) { 55 | return m_reclaimer->get_bulk_rec(m_thread_id, size); 56 | } 57 | 58 | void retire(const RecordHandle &handle) { 59 | m_reclaimer->retire(handle, m_thread_id); 60 | } 61 | 62 | void retire(const RecordArrayHandle &handle, const std::size_t index) { 63 | m_reclaimer->retire(handle, m_thread_id, index); 64 | } 65 | }; 66 | } 67 | -------------------------------------------------------------------------------- /src/primitives/backoff.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | An implementation of backoff algorithms. 3 | Copyright (C) 2018 Robert Kelly 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "backoff.h" 22 | 23 | namespace concurrent_data_structures { 24 | 25 | ExponentialBackoff::ExponentialBackoff() 26 | : m_sleep_generator(1), m_collision_count(0) {} 27 | 28 | void ExponentialBackoff::reset() { m_collision_count = 0; } 29 | void ExponentialBackoff::backoff() { 30 | const std::size_t sleep_amount = 31 | m_sleep_generator.next_rand() & 32 | ((std::size_t(1) << m_collision_count++) - 1); 33 | 34 | if (sleep_amount != 0) { 35 | std::this_thread::sleep_for( 36 | std::chrono::microseconds(S_SLEEP_EPOCH * sleep_amount)); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/primitives/backoff.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | An implementation of backoff algorithms. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include "random/xorshift.h" 19 | #include 20 | 21 | namespace concurrent_data_structures { 22 | 23 | class ExponentialBackoff { 24 | private: 25 | static const std::size_t S_MAX_COUNT = 10; 26 | static const std::size_t S_SLEEP_EPOCH = 5; 27 | XORShift64 m_sleep_generator; 28 | std::size_t m_collision_count; 29 | 30 | public: 31 | ExponentialBackoff(); 32 | void reset(); 33 | void backoff(); 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /src/primitives/barrier.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Simple C++ wrapper around pthread barrier. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | 20 | namespace concurrent_data_structures { 21 | 22 | class ThreadBarrierWrapper { 23 | private: 24 | pthread_barrier_t m_thread_barrier; 25 | 26 | public: 27 | ThreadBarrierWrapper(const std::size_t num_threads) { 28 | assert(pthread_barrier_init(&m_thread_barrier, nullptr, num_threads) == 29 | 0 and 30 | "Couldn't initialise thread barrier."); 31 | } 32 | ~ThreadBarrierWrapper() { pthread_barrier_destroy(&m_thread_barrier); } 33 | 34 | void wait() { pthread_barrier_wait(&m_thread_barrier); } 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /src/primitives/cache_utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Some utilities to cache align classes and structs. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | namespace concurrent_data_structures { 22 | 23 | static const std::size_t S_CACHE_PADDING = 128; 24 | static const std::size_t S_CACHE_SIZE = 64; 25 | 26 | namespace { 27 | 28 | template class DummyCachePadded { 29 | private: 30 | std::uint8_t padding[S_CACHE_PADDING - (sizeof(Object) % S_CACHE_PADDING)]; 31 | 32 | public: 33 | template 34 | DummyCachePadded(_Args &&... __args) : Object(__args...) {} 35 | }; 36 | } 37 | 38 | template 39 | class alignas(alignof(DummyCachePadded) > S_CACHE_PADDING 40 | ? alignof(DummyCachePadded) 41 | : S_CACHE_PADDING) CachePadded : public Object { 42 | private: 43 | std::uint8_t m_padding[S_CACHE_PADDING - (sizeof(Object) % S_CACHE_PADDING)]; 44 | 45 | public: 46 | template 47 | CachePadded(_Args &&... __args) : Object(__args...) {} 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /src/primitives/harris_kcas.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | An implementation of original K-CAS based on 5 | "A practical multi-word compare-and-swap operation""" 6 | Copyright (C) 2018 Robert Kelly 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | This program is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License for more details. 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | */ 18 | 19 | #include "mem-reclaimer/reclaimer.h" 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace concurrent_data_structures { 27 | 28 | template 29 | class Harris_KCAS { 30 | public: 31 | class KCASDescriptor; 32 | 33 | private: 34 | struct RDCSSDescriptor; 35 | typedef typename MemReclaimer::RecordHandle RecordHandle; 36 | typedef typename MemReclaimer::RecordBase RecordBase; 37 | 38 | static const std::intptr_t S_KCAS_BIT = 0x1; 39 | static const std::intptr_t S_RDCSS_BIT = 0x2; 40 | 41 | // Horrific type to represent a word (pointer or shifted value) or some kind 42 | // of descriptor. 43 | union DescriptorUnion { 44 | 45 | std::intptr_t bits; 46 | RDCSSDescriptor *rdcss_descriptor; 47 | KCASDescriptor *kcas_descriptor; 48 | 49 | DescriptorUnion() noexcept : bits(0) {} 50 | DescriptorUnion(std::intptr_t bits) noexcept : bits(bits) {} 51 | DescriptorUnion(RDCSSDescriptor *desc) noexcept : rdcss_descriptor(desc) {} 52 | DescriptorUnion(KCASDescriptor *desc) noexcept : kcas_descriptor(desc) {} 53 | 54 | static bool is_rdcss(DescriptorUnion desc) { 55 | return (desc.bits & S_RDCSS_BIT) == S_RDCSS_BIT; 56 | } 57 | 58 | static bool is_kcas(DescriptorUnion desc) { 59 | return (desc.bits & S_KCAS_BIT) == S_KCAS_BIT; 60 | } 61 | 62 | static bool is_bits(DescriptorUnion desc) { 63 | return !is_kcas(desc) and !is_rdcss(desc); 64 | } 65 | 66 | static DescriptorUnion mask_bits(DescriptorUnion desc) { 67 | return DescriptorUnion{(desc.bits & ~(S_KCAS_BIT | S_RDCSS_BIT))}; 68 | } 69 | 70 | static DescriptorUnion make_rdcss(DescriptorUnion desc) { 71 | return DescriptorUnion{DescriptorUnion::mask_bits(desc).bits | 72 | S_RDCSS_BIT}; 73 | } 74 | 75 | static DescriptorUnion from_rdcss(RDCSSDescriptor *desc) { 76 | return make_rdcss(DescriptorUnion(desc)); 77 | } 78 | 79 | static DescriptorUnion make_kcas(DescriptorUnion desc) { 80 | return DescriptorUnion{DescriptorUnion::mask_bits(desc).bits | 81 | S_KCAS_BIT}; 82 | } 83 | 84 | static DescriptorUnion from_kcas(KCASDescriptor *desc) { 85 | return make_kcas(DescriptorUnion(desc)); 86 | } 87 | 88 | static DescriptorUnion make_bits(DescriptorUnion desc) { 89 | return DescriptorUnion::mask_bits(desc); 90 | } 91 | }; 92 | 93 | enum class KCASDescriptorStatus { 94 | UNDECIDED, 95 | SUCCEEDED, 96 | FAILED, 97 | }; 98 | 99 | struct RDCSSDescriptor : public RecordBase { 100 | std::atomic *data_location; 101 | DescriptorUnion before, kcas_desc; 102 | std::atomic *status_location; 103 | 104 | RDCSSDescriptor(std::atomic *data_location, 105 | DescriptorUnion before, DescriptorUnion kcas_desc, 106 | std::atomic *status_location) 107 | : data_location(data_location), before(before), kcas_desc(kcas_desc), 108 | status_location(status_location) {} 109 | 110 | // It's implied we are already protecting "desc" with 111 | // some memory relcaiming system. 112 | static DescriptorUnion rdcss(DescriptorUnion input_desc, 113 | RecordHandle &found_rdcss) { 114 | assert(DescriptorUnion::is_rdcss(input_desc)); 115 | RDCSSDescriptor *input_rdcss = 116 | DescriptorUnion::mask_bits(input_desc).rdcss_descriptor; 117 | assert(DescriptorUnion::is_bits(input_rdcss->before.bits)); 118 | while (true) { 119 | DescriptorUnion current = input_rdcss->data_location->load(); 120 | if (DescriptorUnion::is_rdcss(current)) { 121 | if (!found_rdcss.try_protect( 122 | current.rdcss_descriptor, *input_rdcss->data_location, 123 | [](DescriptorUnion desc) { return desc.rdcss_descriptor; })) { 124 | continue; 125 | } 126 | rdcss_complete(current); 127 | continue; 128 | } 129 | if (current.bits != input_rdcss->before.bits) { 130 | assert(!DescriptorUnion::is_rdcss(current)); 131 | return current; 132 | } 133 | 134 | bool success = input_rdcss->data_location->compare_exchange_strong( 135 | current, input_desc); 136 | if (success) { 137 | rdcss_complete(input_desc); 138 | return input_rdcss->before; 139 | } 140 | } 141 | } 142 | 143 | static DescriptorUnion 144 | rdcss_read(const std::atomic *location, 145 | const std::memory_order memory_order, 146 | RecordHandle &found_rdcss) { 147 | while (true) { 148 | DescriptorUnion current = location->load(memory_order); 149 | if (DescriptorUnion::is_rdcss(current)) { 150 | if (!found_rdcss.try_protect( 151 | current.rdcss_descriptor, *location, 152 | [](DescriptorUnion desc) { return desc.rdcss_descriptor; })) { 153 | continue; 154 | } 155 | rdcss_complete(current); 156 | } else { 157 | return current; 158 | } 159 | } 160 | } 161 | 162 | static void rdcss_complete(DescriptorUnion input_desc) { 163 | assert(DescriptorUnion::is_rdcss(input_desc)); 164 | RDCSSDescriptor *input_rdcss_desc = 165 | DescriptorUnion::mask_bits(input_desc).rdcss_descriptor; 166 | KCASDescriptorStatus status = input_rdcss_desc->status_location->load(); 167 | if (status == KCASDescriptorStatus::UNDECIDED) { 168 | /*bool _ = */ input_rdcss_desc->data_location->compare_exchange_strong( 169 | input_desc, input_rdcss_desc->kcas_desc); 170 | } else { 171 | /*bool _ =*/input_rdcss_desc->data_location->compare_exchange_strong( 172 | input_desc, input_rdcss_desc->before); 173 | } 174 | } 175 | }; 176 | 177 | struct EntryPayload { 178 | std::atomic *data_location; 179 | DescriptorUnion before, desired; 180 | }; 181 | 182 | static const std::size_t KCASShift = 2; 183 | MemReclaimer *m_reclaimer; 184 | 185 | bool cas_internal(const std::size_t thread_id, const DescriptorUnion desc, 186 | ReclaimerPin &pin, RecordHandle &found_kcas, 187 | RecordHandle &our_rdcss, RecordHandle &found_rdcss, 188 | bool help = false) { 189 | assert(DescriptorUnion::is_kcas(desc)); 190 | KCASDescriptor *kcas_descriptor = 191 | DescriptorUnion::mask_bits(desc).kcas_descriptor; 192 | // If there is work to do, go ahead! 193 | const std::size_t num_entries = kcas_descriptor->m_num_entries; 194 | KCASDescriptorStatus status = KCASDescriptorStatus::SUCCEEDED; 195 | if (kcas_descriptor->m_status.load() == KCASDescriptorStatus::UNDECIDED) { 196 | for (std::size_t i = help ? 1 : 0; 197 | i < num_entries and status == KCASDescriptorStatus::SUCCEEDED; i++) { 198 | EntryPayload &payload = kcas_descriptor->m_descriptors[i]; 199 | retry: 200 | RDCSSDescriptor *rdcss_desc = 201 | new (m_reclaimer->malloc(sizeof(RDCSSDescriptor))) 202 | RDCSSDescriptor(payload.data_location, payload.before, desc, 203 | &kcas_descriptor->m_status); 204 | DescriptorUnion rdcss = DescriptorUnion::from_rdcss(rdcss_desc); 205 | assert(DescriptorUnion::mask_bits(rdcss).bits == 206 | (std::intptr_t)rdcss_desc); 207 | our_rdcss.set(rdcss_desc); 208 | DescriptorUnion value = RDCSSDescriptor::rdcss(rdcss, found_rdcss); 209 | // Did it work? Is there a K-CAS? 210 | if (DescriptorUnion::is_kcas(value)) { 211 | // Is it our K-CAS? Why? It could be someone helping us. 212 | if (value.bits != desc.bits) { 213 | if (!found_kcas.try_protect(value.kcas_descriptor, 214 | *payload.data_location, 215 | [](DescriptorUnion desc) { 216 | return desc.kcas_descriptor; 217 | })) { 218 | continue; 219 | } 220 | m_reclaimer->free(rdcss_desc); 221 | cas_internal(thread_id, value, pin, found_kcas, our_rdcss, 222 | found_rdcss, true); 223 | goto retry; 224 | } else { 225 | // Someone put our K-CAS there, free our RDCSS desc. 226 | assert(value.bits == desc.bits); 227 | m_reclaimer->free(rdcss_desc); 228 | } 229 | } else if (value.bits != rdcss_desc->before.bits) { 230 | // We didn't win, free our one and exit. 231 | m_reclaimer->free(rdcss_desc); 232 | status = KCASDescriptorStatus::FAILED; 233 | } else { 234 | // We won, retire our descriptor as it could still be viewed. 235 | pin.retire(our_rdcss); 236 | } 237 | } 238 | KCASDescriptorStatus expected_status = KCASDescriptorStatus::UNDECIDED; 239 | kcas_descriptor->m_status.compare_exchange_strong(expected_status, 240 | status); 241 | } 242 | // Phase 2. 243 | bool succeeded = 244 | kcas_descriptor->m_status.load() == KCASDescriptorStatus::SUCCEEDED; 245 | for (std::size_t i = 0; i < num_entries; i++) { 246 | DescriptorUnion new_value = 247 | succeeded ? kcas_descriptor->m_descriptors[i].desired 248 | : kcas_descriptor->m_descriptors[i].before; 249 | DescriptorUnion expected = desc; 250 | kcas_descriptor->m_descriptors[i].data_location->compare_exchange_strong( 251 | expected, new_value); 252 | } 253 | return succeeded; 254 | } 255 | 256 | public: 257 | // Class to wrap around the types being KCAS'd 258 | template class KCASEntry { 259 | private: 260 | union ItemBitsUnion { 261 | Type item; 262 | std::intptr_t raw_bits; 263 | }; 264 | 265 | std::atomic m_entry; 266 | static std::intptr_t from_union(DescriptorUnion desc) { 267 | return (ItemBitsUnion{desc.bits}).item; 268 | } 269 | static DescriptorUnion to_union(const std::intptr_t &inner) { 270 | return DescriptorUnion{(ItemBitsUnion{inner}).raw_bits}; 271 | } 272 | 273 | // union ItemBitsUnion { 274 | // ItemBitsUnion() : raw_bits(0) {} 275 | // Type item; 276 | // std::intptr_t raw_bits; 277 | // }; 278 | 279 | // std::atomic m_entry; 280 | // static Type to_union(std::intptr_t raw_bits) { 281 | // ItemBitsUnion ibu; 282 | // ibu.raw_bits = raw_bits; 283 | // return ibu.item; 284 | // } 285 | // static std::intptr_t from_union(const DescriptorUnion &inner) { 286 | // ItemBitsUnion ibu; 287 | // ibu.item = inner.bits; 288 | // return ibu.raw_bits; 289 | // } 290 | friend class Harris_KCAS; 291 | friend class KCASDescriptor; 292 | }; 293 | 294 | class KCASDescriptor : public RecordBase { 295 | private: 296 | std::size_t state, thread_id; 297 | std::atomic m_status; 298 | std::size_t m_num_entries; 299 | EntryPayload m_descriptors[N]; 300 | 301 | KCASDescriptor() = delete; 302 | KCASDescriptor(const KCASDescriptor &rhs) = delete; 303 | KCASDescriptor &operator=(const KCASDescriptor &rhs) = delete; 304 | KCASDescriptor &operator=(KCASDescriptor &&rhs) = delete; 305 | KCASDescriptor(const std::size_t descriptor_size) 306 | : state(0), m_status(KCASDescriptorStatus::UNDECIDED), 307 | m_num_entries(0) {} 308 | 309 | public: 310 | template 311 | void add_value(KCASEntry *location, const ValType &before, 312 | const ValType &desired) { 313 | DescriptorUnion before_desc = KCASEntry::to_union(before); 314 | before_desc.bits <<= KCASShift; 315 | assert(DescriptorUnion::is_bits(before_desc)); 316 | DescriptorUnion desired_desc = KCASEntry::to_union(desired); 317 | desired_desc.bits <<= KCASShift; 318 | assert(DescriptorUnion::is_bits(desired_desc)); 319 | const std::size_t cur_entry = m_num_entries++; 320 | assert(cur_entry < N); 321 | m_descriptors[cur_entry].before = before_desc; 322 | m_descriptors[cur_entry].desired = desired_desc; 323 | m_descriptors[cur_entry].data_location = &location->m_entry; 324 | } 325 | template 326 | void 327 | add_ptr(const KCASEntry *location, const PtrType &before, 328 | const PtrType &desired, 329 | const std::memory_order success_order = std::memory_order_seq_cst, 330 | const std::memory_order fail_order = std::memory_order_seq_cst) { 331 | const DescriptorUnion before_desc = KCASEntry::to_union(before); 332 | assert(DescriptorUnion::is_bits(before_desc)); 333 | const DescriptorUnion desired_desc = 334 | KCASEntry::to_union(desired); 335 | assert(DescriptorUnion::is_bits(desired_desc)); 336 | const std::size_t cur_entry = m_num_entries++; 337 | assert(cur_entry < N); 338 | m_descriptors[cur_entry].before = before_desc; 339 | m_descriptors[cur_entry].desired = desired_desc; 340 | m_descriptors[cur_entry].data_location = &location->m_entry; 341 | } 342 | friend class Harris_KCAS; 343 | }; 344 | 345 | // 4 ref explanation: 1 for K-CAS descriptor, 1 for any K-CAS descriptor 346 | // we find, 1 for our RDCSS, nd 1 for any RDCSS descriptors 347 | // we find trying to install our own. 348 | Harris_KCAS(const std::size_t threads, MemReclaimer *reclaimer) 349 | : m_reclaimer(reclaimer) {} 350 | 351 | KCASDescriptor *create_descriptor(const std::size_t descriptor_size, 352 | const std::size_t thread_id) { 353 | KCASDescriptor *desc = new (m_reclaimer->malloc(sizeof(KCASDescriptor))) 354 | KCASDescriptor(descriptor_size); 355 | desc->thread_id = thread_id; 356 | return desc; 357 | } 358 | 359 | void free_descriptor(KCASDescriptor *desc) { 360 | desc->state = 2; 361 | m_reclaimer->free(desc); 362 | } 363 | 364 | bool cas(const std::size_t thread_id, ReclaimerPin &pin, 365 | KCASDescriptor *desc, bool record = false) { 366 | std::sort(desc->m_descriptors, desc->m_descriptors + desc->m_num_entries, 367 | [](const EntryPayload &lhs, const EntryPayload &rhs) -> bool { 368 | return lhs.data_location < rhs.data_location; 369 | }); 370 | // 4 references during our K-CAS 371 | RecordHandle our_kcas = pin.get_rec(), found_kcas = pin.get_rec(), 372 | our_rdcss = pin.get_rec(), found_rdcss = pin.get_rec(); 373 | our_kcas.set(desc); 374 | bool res = cas_internal(thread_id, DescriptorUnion::from_kcas(desc), pin, 375 | found_kcas, our_rdcss, found_rdcss); 376 | desc->state = 1; 377 | pin.retire(our_kcas); 378 | return res; 379 | } 380 | 381 | template 382 | ValType 383 | read_value(const std::size_t thread_id, ReclaimerPin &pin, 384 | const KCASEntry *location, 385 | const std::memory_order memory_order = std::memory_order_seq_cst) { 386 | static_assert(!std::is_pointer::value and 387 | !std::is_reference::value, 388 | "Type is not a value."); 389 | RecordHandle found_kcas = pin.get_rec(), our_rdcss = pin.get_rec(), 390 | found_rdcss = pin.get_rec(); 391 | while (true) { 392 | DescriptorUnion desc = RDCSSDescriptor::rdcss_read( 393 | &location->m_entry, memory_order, found_kcas); 394 | // Could still be a K-CAS descriptor. 395 | if (DescriptorUnion::is_kcas(desc)) { 396 | cas_internal(thread_id, desc, pin, found_kcas, our_rdcss, found_rdcss, 397 | true); 398 | continue; 399 | } 400 | assert(DescriptorUnion::is_bits(desc)); 401 | desc.bits >>= KCASShift; 402 | return KCASEntry::from_union(desc); 403 | } 404 | } 405 | 406 | template 407 | PtrType 408 | read_ptr(const std::size_t thread_id, ReclaimerPin &pin, 409 | const KCASEntry *location, 410 | const std::memory_order memory_order = std::memory_order_seq_cst) { 411 | static_assert(std::is_pointer::value, "Type is not a pointer."); 412 | while (true) { 413 | RecordHandle found_kcas = pin.get_rec(), our_rdcss = pin.get_rec(), 414 | found_rdcss = pin.get_rec(); 415 | DescriptorUnion desc = 416 | RDCSSDescriptor::rdcss_read(location, memory_order, found_kcas); 417 | // Could still be a K-CAS descriptor. 418 | if (DescriptorUnion::is_kcas(desc)) { 419 | cas_internal(thread_id, desc, pin, found_kcas, our_rdcss, found_rdcss, 420 | true); 421 | continue; 422 | } 423 | assert(DescriptorUnion::is_bits(desc)); 424 | return KCASEntry::from_union(desc); 425 | } 426 | } 427 | 428 | template 429 | void write_value(const std::size_t thread_id, KCASEntry *location, 430 | const ValType &val, const std::memory_order memory_order = 431 | std::memory_order_seq_cst) { 432 | static_assert(!std::is_pointer::value and 433 | !std::is_reference::value, 434 | "Type is not a value."); 435 | DescriptorUnion desc = KCASEntry::to_union(val); 436 | desc.bits <<= KCASShift; 437 | assert(DescriptorUnion::is_bits(desc)); 438 | location->m_entry.store(desc); 439 | } 440 | 441 | template 442 | void 443 | write_ptr(const std::size_t thread_id, KCASEntry *location, 444 | const PtrType &ptr, 445 | const std::memory_order memory_order = std::memory_order_seq_cst) { 446 | static_assert(std::is_pointer::value, "Type is not a pointer."); 447 | DescriptorUnion desc = KCASEntry::to_union(ptr); 448 | assert(DescriptorUnion::is_bits(desc)); 449 | location->m_entry.store(desc, memory_order); 450 | } 451 | }; 452 | } 453 | -------------------------------------------------------------------------------- /src/primitives/locks.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Simple interface for locks with some implementations. 3 | Copyright (C) 2018 Robert Kelly 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | */ 15 | #include "locks.h" 16 | #include "cache_utils.h" 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace concurrent_data_structures { 24 | 25 | PthreadSpinLock::PthreadSpinLock() { 26 | pthread_spin_init(&m_lock, PTHREAD_PROCESS_PRIVATE); 27 | } 28 | void PthreadSpinLock::lock() { pthread_spin_lock(&m_lock); } 29 | void PthreadSpinLock::unlock() { pthread_spin_unlock(&m_lock); } 30 | 31 | PthreadMutex::PthreadMutex() { pthread_mutex_init(&m_lock, nullptr); } 32 | void PthreadMutex::lock() { pthread_mutex_lock(&m_lock); } 33 | void PthreadMutex::unlock() { pthread_mutex_unlock(&m_lock); } 34 | 35 | TicketLock::TicketLock() : m_entry_ticket(0), m_exit_ticket(0) {} 36 | void TicketLock::lock() { 37 | std::uintptr_t my_entry_ticket = 38 | m_entry_ticket.fetch_add(1, std::memory_order_relaxed); 39 | while (m_exit_ticket.load(std::memory_order_acquire) != my_entry_ticket) { 40 | } 41 | } 42 | void TicketLock::unlock() { 43 | m_exit_ticket.fetch_add(1, std::memory_order_release); 44 | } 45 | 46 | ElidedLock::ElidedLock() { m_lock.store(false, std::memory_order_relaxed); } 47 | void ElidedLock::lock() { 48 | for (std::size_t i = 0; i < MAX_RETRIES; i++) { 49 | unsigned int status = _xbegin(); 50 | if (status == _XBEGIN_STARTED) { 51 | if (!m_lock.load(std::memory_order_relaxed)) { 52 | return; 53 | } else { 54 | _xabort(0xff); 55 | } 56 | } 57 | if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) { 58 | // Wait for lock to be free. 59 | while (m_lock.load(std::memory_order_relaxed)) { 60 | _mm_pause(); 61 | } 62 | } 63 | if (!(status & _XABORT_RETRY) and 64 | !((status & _XABORT_EXPLICIT) and _XABORT_CODE(status) == 0xff)) { 65 | break; 66 | } 67 | } 68 | 69 | while (true) { 70 | bool value = m_lock.load(std::memory_order_relaxed); 71 | if (!value and 72 | m_lock.compare_exchange_weak(value, true, std::memory_order_acquire, 73 | std::memory_order_relaxed)) { 74 | return; 75 | } 76 | _mm_pause(); 77 | } 78 | } 79 | 80 | void ElidedLock::unlock() { 81 | if (!m_lock.load(std::memory_order_relaxed) and _xtest()) { 82 | _xend(); 83 | } else { 84 | m_lock.store(false, std::memory_order_release); 85 | } 86 | } 87 | 88 | ElidedLock TransactionalLock::S_TRANS_LOCK; 89 | 90 | TransactionalLock::TransactionalLock() {} 91 | TransactionalLock::~TransactionalLock() {} 92 | void TransactionalLock::lock() { S_TRANS_LOCK.lock(); } 93 | void TransactionalLock::unlock() { S_TRANS_LOCK.unlock(); } 94 | } 95 | -------------------------------------------------------------------------------- /src/primitives/locks.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | /* 3 | Simple interface for locks with some implementations. 4 | Copyright (C) 2018 Robert Kelly 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | You should have received a copy of the GNU General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | #include "cache_utils.h" 18 | #include 19 | #include 20 | #include 21 | 22 | namespace concurrent_data_structures { 23 | 24 | class alignas(S_CACHE_PADDING) PthreadSpinLock { 25 | private: 26 | pthread_spinlock_t m_lock; 27 | 28 | public: 29 | PthreadSpinLock(); 30 | void lock(); 31 | void unlock(); 32 | }; 33 | 34 | class alignas(S_CACHE_PADDING) PthreadMutex { 35 | 36 | private: 37 | pthread_mutex_t m_lock; 38 | 39 | public: 40 | PthreadMutex(); 41 | void lock(); 42 | void unlock(); 43 | }; 44 | 45 | class alignas(S_CACHE_PADDING) TicketLock { 46 | private: 47 | CachePadded m_entry_ticket, m_exit_ticket; 48 | 49 | public: 50 | TicketLock(); 51 | void lock(); 52 | void unlock(); 53 | }; 54 | 55 | class alignas(S_CACHE_PADDING) ElidedLock { 56 | private: 57 | static const std::size_t MAX_RETRIES = 20; 58 | std::atomic_bool m_lock; 59 | 60 | public: 61 | ElidedLock(); 62 | 63 | void lock(); 64 | void unlock(); 65 | }; 66 | 67 | class TransactionalLock { 68 | private: 69 | static ElidedLock S_TRANS_LOCK; 70 | 71 | public: 72 | TransactionalLock(); 73 | ~TransactionalLock(); 74 | void lock(); 75 | void unlock(); 76 | }; 77 | } 78 | -------------------------------------------------------------------------------- /src/primitives/marked_pointers.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | Marked pointer types for ease of concurrent programming. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | namespace concurrent_data_structures { 22 | 23 | template class MarkedPointer { 24 | private: 25 | static const std::uintptr_t S_HIGHER_MARK = 0xffff000000000000; 26 | static const std::uintptr_t S_LOWER_MARK = 0x0000000000000003; 27 | static const std::uintptr_t S_MARK = S_HIGHER_MARK | S_LOWER_MARK; 28 | 29 | typedef Type *PtrType; 30 | PtrType m_ptr; 31 | 32 | void mark_inplace_bool(const std::uintptr_t index, const bool mark) { 33 | if (mark) { 34 | this->m_ptr = reinterpret_cast( 35 | reinterpret_cast(this->m_ptr) | 36 | (std::uintptr_t(1) << index)); 37 | } else { 38 | this->m_ptr = reinterpret_cast( 39 | reinterpret_cast(this->m_ptr) & 40 | ~(std::uintptr_t(1) << index)); 41 | } 42 | } 43 | 44 | void write_counter_inplace(const std::uintptr_t counter) { 45 | this->m_ptr = reinterpret_cast( 46 | (reinterpret_cast(this->m_ptr) & (~S_HIGHER_MARK)) | 47 | (counter << 48)); 48 | } 49 | 50 | public: 51 | MarkedPointer() noexcept : m_ptr(nullptr) {} 52 | MarkedPointer(const PtrType ptr) noexcept : m_ptr(ptr) {} 53 | MarkedPointer(const PtrType ptr, const std::uint16_t counter) noexcept 54 | : m_ptr(ptr) { 55 | this->write_counter_inplace(counter); 56 | } 57 | 58 | PtrType address() const { 59 | return reinterpret_cast(reinterpret_cast(m_ptr) & 60 | ~S_MARK); 61 | } 62 | 63 | PtrType raw_address() const { return m_ptr; } 64 | 65 | Type &operator*() { return *this->address(); } 66 | 67 | PtrType operator->() { return this->address(); } 68 | 69 | bool operator==(const MarkedPointer &rhs) const { 70 | return m_ptr == rhs.m_ptr; 71 | } 72 | bool operator!=(const MarkedPointer &rhs) const { 73 | return m_ptr != rhs.m_ptr; 74 | } 75 | 76 | const std::uintptr_t lower_mark() const { 77 | return reinterpret_cast(m_ptr) & S_LOWER_MARK; 78 | } 79 | 80 | const std::uint16_t get_counter() const { 81 | return (reinterpret_cast(m_ptr) & S_HIGHER_MARK) >> 48; 82 | } 83 | 84 | const std::uintptr_t full_mark() { 85 | return reinterpret_cast(m_ptr) & S_MARK; 86 | } 87 | 88 | MarkedPointer write_counter(const std::uint16_t counter) const { 89 | const std::uintptr_t counter_ptr = counter; 90 | MarkedPointer copy = this->m_ptr; 91 | copy.write_counter_inplace(counter_ptr); 92 | return copy; 93 | } 94 | 95 | bool is_marked(const std::uintptr_t index) const { 96 | return (this->lower_mark() >> index) & 1; 97 | } 98 | 99 | MarkedPointer mark(const std::uintptr_t index) const { 100 | MarkedPointer copy = this->m_ptr; 101 | copy.mark_inplace_bool(index, true); 102 | return copy; 103 | } 104 | 105 | MarkedPointer unmark(const std::uintptr_t index) const { 106 | MarkedPointer copy = this->m_ptr; 107 | copy.mark_inplace_bool(index, false); 108 | return copy; 109 | } 110 | 111 | MarkedPointer unmark_all() const { 112 | MarkedPointer copy = this->address(); 113 | return copy; 114 | } 115 | 116 | void mark_inplace(const std::uintptr_t index) { 117 | mark_inplace_bool(index, true); 118 | } 119 | 120 | void unmark_inplace(const std::uintptr_t index) { 121 | mark_inplace_bool(index, false); 122 | } 123 | }; 124 | 125 | template class AtomicMarkedPointer { 126 | private: 127 | typedef Type *PtrType; 128 | union AtomicMarkedUnion { 129 | AtomicMarkedUnion(const MarkedPointer &ptr) : m_atomic_rep(ptr) {} 130 | std::atomic> m_atomic_rep; 131 | std::atomic_uintptr_t fetch_rep; 132 | }; 133 | 134 | union MarkedUnion { 135 | MarkedUnion(std::uintptr_t raw_bits) : raw_bits(raw_bits) {} 136 | std::uintptr_t raw_bits; 137 | MarkedPointer marked_pointer; 138 | }; 139 | 140 | AtomicMarkedUnion m_ptr; 141 | 142 | public: 143 | AtomicMarkedPointer() : m_ptr(nullptr) {} 144 | AtomicMarkedPointer(const MarkedPointer ptr) : m_ptr(ptr) {} 145 | AtomicMarkedPointer(const PtrType ptr) : m_ptr(MarkedPointer(ptr)) {} 146 | 147 | MarkedPointer 148 | load(std::memory_order memory_order = std::memory_order_seq_cst) const { 149 | return m_ptr.m_atomic_rep.load(memory_order); 150 | } 151 | 152 | void store(MarkedPointer ptr, 153 | std::memory_order memory_order = std::memory_order_seq_cst) { 154 | m_ptr.m_atomic_rep.store(ptr, memory_order); 155 | } 156 | 157 | bool 158 | compare_exchange_strong(MarkedPointer &expected, 159 | const MarkedPointer &desired, 160 | std::memory_order success = std::memory_order_seq_cst, 161 | std::memory_order fail = std::memory_order_seq_cst) { 162 | return m_ptr.m_atomic_rep.compare_exchange_strong(expected, desired, 163 | success, fail); 164 | } 165 | 166 | bool 167 | compare_exchange_weak(MarkedPointer &expected, 168 | const MarkedPointer &desired, 169 | std::memory_order success = std::memory_order_seq_cst, 170 | std::memory_order fail = std::memory_order_seq_cst) { 171 | return m_ptr.m_atomic_rep.compare_exchange_weak(expected, desired, success, 172 | fail); 173 | } 174 | 175 | MarkedPointer 176 | fetch_mark(const std::uintptr_t index, 177 | std::memory_order memory_order = std::memory_order_seq_cst) { 178 | return fetch_mark_bool(index, true, memory_order); 179 | } 180 | 181 | MarkedPointer 182 | fetch_unmark(const std::uintptr_t index, 183 | std::memory_order memory_order = std::memory_order_seq_cst) { 184 | return fetch_mark_bool(index, false, memory_order); 185 | } 186 | 187 | MarkedPointer 188 | fetch_mark_bool(const std::uintptr_t index, const bool mark, 189 | std::memory_order memory_order = std::memory_order_seq_cst) { 190 | if (mark) { 191 | MarkedUnion after = m_ptr.fetch_rep.fetch_or(1 << index, memory_order); 192 | return after.marked_pointer; 193 | } else { 194 | MarkedUnion after = m_ptr.fetch_rep.fetch_xor(1 << index, memory_order); 195 | return after.marked_pointer; 196 | } 197 | } 198 | }; 199 | } 200 | -------------------------------------------------------------------------------- /src/random/lcg.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | A number of LCG RNGs 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include "pcg_random.h" 19 | #include 20 | #include 21 | 22 | namespace concurrent_data_structures { 23 | 24 | // class LCG { 25 | // private: 26 | // const static std::size_t S_MODULUS_MASK = (std::size_t(1) << 31) - 1; 27 | // // const static std::size_t S_MODULUS_MASK = (std::size_t(1) << 48) - 1; 28 | // // const static std::size_t S_MULTIPLIER = std::size_t(1103515245); 29 | // // const static std::size_t S_MULTIPLIER = std::size_t(25214903917); 30 | // const static std::size_t S_MULTIPLIER = std::size_t(1103515245); 31 | // // const static std::size_t S_INCREMENT = std::size_t(12345); 32 | // // const static std::size_t S_INCREMENT = std::size_t(11); 33 | // const static std::size_t S_INCREMENT = std::size_t(12345); 34 | // std::size_t m_current; 35 | 36 | // public: 37 | // static const std::size_t NUM_BITS = 32; 38 | 39 | // LCG() : m_current(0) {} 40 | // LCG(const std::size_t seed) : m_current(seed & S_MODULUS_MASK) {} 41 | // const std::size_t next_rand() { 42 | // m_current = ((S_MULTIPLIER * m_current) + S_INCREMENT) & S_MODULUS_MASK; 43 | // return m_current >> 16; 44 | // } 45 | //}; 46 | 47 | class PCG { 48 | private: 49 | pcg32 m_random_generator; 50 | std::uniform_int_distribution m_data_distribution; 51 | 52 | public: 53 | static const std::size_t NUM_BITS = 32; 54 | 55 | PCG() 56 | : m_random_generator(pcg_extras::seed_seq_from()), 57 | m_data_distribution(0, std::size_t(1) << NUM_BITS) {} 58 | PCG(const std::size_t seed) 59 | : m_random_generator(seed), 60 | m_data_distribution(0, std::size_t(1) << NUM_BITS) {} 61 | const std::size_t next_rand() { 62 | return m_data_distribution(m_random_generator); 63 | } 64 | }; 65 | 66 | class LCGBSD { 67 | private: 68 | const static std::size_t S_MODULUS_MASK = (std::size_t(1) << 31) - 1; 69 | const static std::size_t S_MULTIPLIER = std::size_t(1103515245); 70 | const static std::size_t S_INCREMENT = std::size_t(12345); 71 | std::size_t m_current; 72 | 73 | public: 74 | static const std::size_t NUM_BITS = 31; 75 | 76 | LCGBSD() : m_current(0) {} 77 | LCGBSD(const std::size_t seed) : m_current(seed & S_MODULUS_MASK) {} 78 | const std::size_t next_rand() { 79 | m_current = ((S_MULTIPLIER * m_current) + S_INCREMENT) & S_MODULUS_MASK; 80 | return m_current; 81 | } 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /src/random/pcg_extras.h: -------------------------------------------------------------------------------- 1 | /* 2 | * PCG Random Number Generation for C++ 3 | * 4 | * Copyright 2014 Melissa O'Neill 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | * For additional information about the PCG random number generation scheme, 19 | * including its license and other licensing options, visit 20 | * 21 | * http://www.pcg-random.org 22 | */ 23 | 24 | /* 25 | * This file provides support code that is useful for random-number generation 26 | * but not specific to the PCG generation scheme, including: 27 | * - 128-bit int support for platforms where it isn't available natively 28 | * - bit twiddling operations 29 | * - I/O of 128-bit and 8-bit integers 30 | * - Handling the evilness of SeedSeq 31 | * - Support for efficiently producing random numbers less than a given 32 | * bound 33 | */ 34 | 35 | #ifndef PCG_EXTRAS_HPP_INCLUDED 36 | #define PCG_EXTRAS_HPP_INCLUDED 1 37 | 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | 51 | #ifdef __GNUC__ 52 | #include 53 | #endif 54 | 55 | /* 56 | * Abstractions for compiler-specific directives 57 | */ 58 | 59 | #ifdef __GNUC__ 60 | #define PCG_NOINLINE __attribute__((noinline)) 61 | #else 62 | #define PCG_NOINLINE 63 | #endif 64 | 65 | /* 66 | * Some members of the PCG library use 128-bit math. When compiling on 64-bit 67 | * platforms, both GCC and Clang provide 128-bit integer types that are ideal 68 | * for the job. 69 | * 70 | * On 32-bit platforms (or with other compilers), we fall back to a C++ 71 | * class that provides 128-bit unsigned integers instead. It may seem 72 | * like we're reinventing the wheel here, because libraries already exist 73 | * that support large integers, but most existing libraries provide a very 74 | * generic multiprecision code, but here we're operating at a fixed size. 75 | * Also, most other libraries are fairly heavyweight. So we use a direct 76 | * implementation. Sadly, it's much slower than hand-coded assembly or 77 | * direct CPU support. 78 | * 79 | */ 80 | #if __SIZEOF_INT128__ 81 | namespace pcg_extras { 82 | typedef __uint128_t pcg128_t; 83 | } 84 | #define PCG_128BIT_CONSTANT(high,low) \ 85 | ((pcg128_t(high) << 64) + low) 86 | #else 87 | #include "pcg_uint128.hpp" 88 | namespace pcg_extras { 89 | typedef pcg_extras::uint_x4 pcg128_t; 90 | } 91 | #define PCG_128BIT_CONSTANT(high,low) \ 92 | pcg128_t(high,low) 93 | #define PCG_EMULATED_128BIT_MATH 1 94 | #endif 95 | 96 | 97 | namespace pcg_extras { 98 | 99 | /* 100 | * We often need to represent a "number of bits". When used normally, these 101 | * numbers are never greater than 128, so an unsigned char is plenty. 102 | * If you're using a nonstandard generator of a larger size, you can set 103 | * PCG_BITCOUNT_T to have it define it as a larger size. (Some compilers 104 | * might produce faster code if you set it to an unsigned int.) 105 | */ 106 | 107 | #ifndef PCG_BITCOUNT_T 108 | typedef uint8_t bitcount_t; 109 | #else 110 | typedef PCG_BITCOUNT_T bitcount_t; 111 | #endif 112 | 113 | /* 114 | * C++ requires us to be able to serialize RNG state by printing or reading 115 | * it from a stream. Because we use 128-bit ints, we also need to be able 116 | * ot print them, so here is code to do so. 117 | * 118 | * This code provides enough functionality to print 128-bit ints in decimal 119 | * and zero-padded in hex. It's not a full-featured implementation. 120 | */ 121 | 122 | template 123 | std::basic_ostream& 124 | operator<<(std::basic_ostream& out, pcg128_t value) 125 | { 126 | auto desired_base = out.flags() & out.basefield; 127 | bool want_hex = desired_base == out.hex; 128 | 129 | if (want_hex) { 130 | uint64_t highpart = uint64_t(value >> 64); 131 | uint64_t lowpart = uint64_t(value); 132 | auto desired_width = out.width(); 133 | if (desired_width > 16) { 134 | out.width(desired_width - 16); 135 | } 136 | if (highpart != 0 || desired_width > 16) 137 | out << highpart; 138 | CharT oldfill; 139 | if (highpart != 0) { 140 | out.width(16); 141 | oldfill = out.fill('0'); 142 | } 143 | auto oldflags = out.setf(decltype(desired_base){}, out.showbase); 144 | out << lowpart; 145 | out.setf(oldflags); 146 | if (highpart != 0) { 147 | out.fill(oldfill); 148 | } 149 | return out; 150 | } 151 | constexpr size_t MAX_CHARS_128BIT = 40; 152 | 153 | char buffer[MAX_CHARS_128BIT]; 154 | char* pos = buffer+sizeof(buffer); 155 | *(--pos) = '\0'; 156 | constexpr auto BASE = pcg128_t(10ULL); 157 | do { 158 | auto div = value / BASE; 159 | auto mod = uint32_t(value - (div * BASE)); 160 | *(--pos) = '0' + mod; 161 | value = div; 162 | } while(value != pcg128_t(0ULL)); 163 | return out << pos; 164 | } 165 | 166 | template 167 | std::basic_istream& 168 | operator>>(std::basic_istream& in, pcg128_t& value) 169 | { 170 | typename std::basic_istream::sentry s(in); 171 | 172 | if (!s) 173 | return in; 174 | 175 | constexpr auto BASE = pcg128_t(10ULL); 176 | pcg128_t current(0ULL); 177 | bool did_nothing = true; 178 | bool overflow = false; 179 | for(;;) { 180 | CharT wide_ch = in.get(); 181 | if (!in.good()) 182 | break; 183 | auto ch = in.narrow(wide_ch, '\0'); 184 | if (ch < '0' || ch > '9') { 185 | in.unget(); 186 | break; 187 | } 188 | did_nothing = false; 189 | pcg128_t digit(uint32_t(ch - '0')); 190 | pcg128_t timesbase = current*BASE; 191 | overflow = overflow || timesbase < current; 192 | current = timesbase + digit; 193 | overflow = overflow || current < digit; 194 | } 195 | 196 | if (did_nothing || overflow) { 197 | in.setstate(std::ios::failbit); 198 | if (overflow) 199 | current = ~pcg128_t(0ULL); 200 | } 201 | 202 | value = current; 203 | 204 | return in; 205 | } 206 | 207 | /* 208 | * Likewise, if people use tiny rngs, we'll be serializing uint8_t. 209 | * If we just used the provided IO operators, they'd read/write chars, 210 | * not ints, so we need to define our own. We *can* redefine this operator 211 | * here because we're in our own namespace. 212 | */ 213 | 214 | template 215 | std::basic_ostream& 216 | operator<<(std::basic_ostream&out, uint8_t value) 217 | { 218 | return out << uint32_t(value); 219 | } 220 | 221 | template 222 | std::basic_istream& 223 | operator>>(std::basic_istream& in, uint8_t target) 224 | { 225 | uint32_t value = 0xdecea5edU; 226 | in >> value; 227 | if (!in && value == 0xdecea5edU) 228 | return in; 229 | if (value > uint8_t(~0)) { 230 | in.setstate(std::ios::failbit); 231 | value = ~0U; 232 | } 233 | target = uint8_t(value); 234 | return in; 235 | } 236 | 237 | /* Unfortunately, the above functions don't get found in preference to the 238 | * built in ones, so we create some more specific overloads that will. 239 | * Ugh. 240 | */ 241 | 242 | inline std::ostream& operator<<(std::ostream& out, uint8_t value) 243 | { 244 | return pcg_extras::operator<< (out, value); 245 | } 246 | 247 | inline std::istream& operator>>(std::istream& in, uint8_t& value) 248 | { 249 | return pcg_extras::operator>> (in, value); 250 | } 251 | 252 | 253 | 254 | /* 255 | * Useful bitwise operations. 256 | */ 257 | 258 | /* 259 | * XorShifts are invertable, but they are someting of a pain to invert. 260 | * This function backs them out. It's used by the whacky "inside out" 261 | * generator defined later. 262 | */ 263 | 264 | template 265 | inline itype unxorshift(itype x, bitcount_t bits, bitcount_t shift) 266 | { 267 | if (2*shift >= bits) { 268 | return x ^ (x >> shift); 269 | } 270 | itype lowmask1 = (itype(1U) << (bits - shift*2)) - 1; 271 | itype highmask1 = ~lowmask1; 272 | itype top1 = x; 273 | itype bottom1 = x & lowmask1; 274 | top1 ^= top1 >> shift; 275 | top1 &= highmask1; 276 | x = top1 | bottom1; 277 | itype lowmask2 = (itype(1U) << (bits - shift)) - 1; 278 | itype bottom2 = x & lowmask2; 279 | bottom2 = unxorshift(bottom2, bits - shift, shift); 280 | bottom2 &= lowmask1; 281 | return top1 | bottom2; 282 | } 283 | 284 | /* 285 | * Rotate left and right. 286 | * 287 | * In ideal world, compilers would spot idiomatic rotate code and convert it 288 | * to a rotate instruction. Of course, opinions vary on what the correct 289 | * idiom is and how to spot it. For clang, sometimes it generates better 290 | * (but still crappy) code if you define PCG_USE_ZEROCHECK_ROTATE_IDIOM. 291 | */ 292 | 293 | template 294 | inline itype rotl(itype value, bitcount_t rot) 295 | { 296 | constexpr bitcount_t bits = sizeof(itype) * 8; 297 | constexpr bitcount_t mask = bits - 1; 298 | #if PCG_USE_ZEROCHECK_ROTATE_IDIOM 299 | return rot ? (value << rot) | (value >> (bits - rot)) : value; 300 | #else 301 | return (value << rot) | (value >> ((- rot) & mask)); 302 | #endif 303 | } 304 | 305 | template 306 | inline itype rotr(itype value, bitcount_t rot) 307 | { 308 | constexpr bitcount_t bits = sizeof(itype) * 8; 309 | constexpr bitcount_t mask = bits - 1; 310 | #if PCG_USE_ZEROCHECK_ROTATE_IDIOM 311 | return rot ? (value >> rot) | (value << (bits - rot)) : value; 312 | #else 313 | return (value >> rot) | (value << ((- rot) & mask)); 314 | #endif 315 | } 316 | 317 | /* Unfortunately, both Clang and GCC sometimes perform poorly when it comes 318 | * to properly recognizing idiomatic rotate code, so for we also provide 319 | * assembler directives (enabled with PCG_USE_INLINE_ASM). Boo, hiss. 320 | * (I hope that these compilers get better so that this code can die.) 321 | * 322 | * These overloads will be preferred over the general template code above. 323 | */ 324 | #if PCG_USE_INLINE_ASM && __GNUC__ && (__x86_64__ || __i386__) 325 | 326 | inline uint8_t rotr(uint8_t value, bitcount_t rot) 327 | { 328 | asm ("rorb %%cl, %0" : "=r" (value) : "0" (value), "c" (rot)); 329 | return value; 330 | } 331 | 332 | inline uint16_t rotr(uint16_t value, bitcount_t rot) 333 | { 334 | asm ("rorw %%cl, %0" : "=r" (value) : "0" (value), "c" (rot)); 335 | return value; 336 | } 337 | 338 | inline uint32_t rotr(uint32_t value, bitcount_t rot) 339 | { 340 | asm ("rorl %%cl, %0" : "=r" (value) : "0" (value), "c" (rot)); 341 | return value; 342 | } 343 | 344 | #if __x86_64__ 345 | inline uint64_t rotr(uint64_t value, bitcount_t rot) 346 | { 347 | asm ("rorq %%cl, %0" : "=r" (value) : "0" (value), "c" (rot)); 348 | return value; 349 | } 350 | #endif // __x86_64__ 351 | 352 | #endif // PCG_USE_INLINE_ASM 353 | 354 | 355 | /* 356 | * The C++ SeedSeq concept (modelled by seed_seq) can fill an array of 357 | * 32-bit integers with seed data, but sometimes we want to produce 358 | * larger or smaller integers. 359 | * 360 | * The following code handles this annoyance. 361 | * 362 | * uneven_copy will copy an array of 32-bit ints to an array of larger or 363 | * smaller ints (actually, the code is general it only needing forward 364 | * iterators). The copy is identical to the one that would be performed if 365 | * we just did memcpy on a standard little-endian machine, but works 366 | * regardless of the endian of the machine (or the weirdness of the ints 367 | * involved). 368 | * 369 | * generate_to initializes an array of integers using a SeedSeq 370 | * object. It is given the size as a static constant at compile time and 371 | * tries to avoid memory allocation. If we're filling in 32-bit constants 372 | * we just do it directly. If we need a separate buffer and it's small, 373 | * we allocate it on the stack. Otherwise, we fall back to heap allocation. 374 | * Ugh. 375 | * 376 | * generate_one produces a single value of some integral type using a 377 | * SeedSeq object. 378 | */ 379 | 380 | /* uneven_copy helper, case where destination ints are less than 32 bit. */ 381 | 382 | template 383 | SrcIter uneven_copy_impl( 384 | SrcIter src_first, DestIter dest_first, DestIter dest_last, 385 | std::true_type) 386 | { 387 | typedef typename std::iterator_traits::value_type src_t; 388 | typedef typename std::iterator_traits::value_type dest_t; 389 | 390 | constexpr bitcount_t SRC_SIZE = sizeof(src_t); 391 | constexpr bitcount_t DEST_SIZE = sizeof(dest_t); 392 | constexpr bitcount_t DEST_BITS = DEST_SIZE * 8; 393 | constexpr bitcount_t SCALE = SRC_SIZE / DEST_SIZE; 394 | 395 | size_t count = 0; 396 | src_t value; 397 | 398 | while (dest_first != dest_last) { 399 | if ((count++ % SCALE) == 0) 400 | value = *src_first++; // Get more bits 401 | else 402 | value >>= DEST_BITS; // Move down bits 403 | 404 | *dest_first++ = dest_t(value); // Truncates, ignores high bits. 405 | } 406 | return src_first; 407 | } 408 | 409 | /* uneven_copy helper, case where destination ints are more than 32 bit. */ 410 | 411 | template 412 | SrcIter uneven_copy_impl( 413 | SrcIter src_first, DestIter dest_first, DestIter dest_last, 414 | std::false_type) 415 | { 416 | typedef typename std::iterator_traits::value_type src_t; 417 | typedef typename std::iterator_traits::value_type dest_t; 418 | 419 | constexpr auto SRC_SIZE = sizeof(src_t); 420 | constexpr auto SRC_BITS = SRC_SIZE * 8; 421 | constexpr auto DEST_SIZE = sizeof(dest_t); 422 | constexpr auto SCALE = (DEST_SIZE+SRC_SIZE-1) / SRC_SIZE; 423 | 424 | while (dest_first != dest_last) { 425 | dest_t value(0UL); 426 | unsigned int shift = 0; 427 | 428 | for (size_t i = 0; i < SCALE; ++i) { 429 | value |= dest_t(*src_first++) << shift; 430 | shift += SRC_BITS; 431 | } 432 | 433 | *dest_first++ = value; 434 | } 435 | return src_first; 436 | } 437 | 438 | /* uneven_copy, call the right code for larger vs. smaller */ 439 | 440 | template 441 | inline SrcIter uneven_copy(SrcIter src_first, 442 | DestIter dest_first, DestIter dest_last) 443 | { 444 | typedef typename std::iterator_traits::value_type src_t; 445 | typedef typename std::iterator_traits::value_type dest_t; 446 | 447 | constexpr bool DEST_IS_SMALLER = sizeof(dest_t) < sizeof(src_t); 448 | 449 | return uneven_copy_impl(src_first, dest_first, dest_last, 450 | std::integral_constant{}); 451 | } 452 | 453 | /* generate_to, fill in a fixed-size array of integral type using a SeedSeq 454 | * (actually works for any random-access iterator) 455 | */ 456 | 457 | template 458 | inline void generate_to_impl(SeedSeq&& generator, DestIter dest, 459 | std::true_type) 460 | { 461 | generator.generate(dest, dest+size); 462 | } 463 | 464 | template 465 | void generate_to_impl(SeedSeq&& generator, DestIter dest, 466 | std::false_type) 467 | { 468 | typedef typename std::iterator_traits::value_type dest_t; 469 | constexpr auto DEST_SIZE = sizeof(dest_t); 470 | constexpr auto GEN_SIZE = sizeof(uint32_t); 471 | 472 | constexpr bool GEN_IS_SMALLER = GEN_SIZE < DEST_SIZE; 473 | constexpr size_t FROM_ELEMS = 474 | GEN_IS_SMALLER 475 | ? size * ((DEST_SIZE+GEN_SIZE-1) / GEN_SIZE) 476 | : (size + (GEN_SIZE / DEST_SIZE) - 1) 477 | / ((GEN_SIZE / DEST_SIZE) + GEN_IS_SMALLER); 478 | // this odd code ^^^^^^^^^^^^^^^^^ is work-around for 479 | // a bug: http://llvm.org/bugs/show_bug.cgi?id=21287 480 | 481 | if (FROM_ELEMS <= 1024) { 482 | uint32_t buffer[FROM_ELEMS]; 483 | generator.generate(buffer, buffer+FROM_ELEMS); 484 | uneven_copy(buffer, dest, dest+size); 485 | } else { 486 | uint32_t* buffer = (uint32_t*) malloc(GEN_SIZE * FROM_ELEMS); 487 | generator.generate(buffer, buffer+FROM_ELEMS); 488 | uneven_copy(buffer, dest, dest+size); 489 | free(buffer); 490 | } 491 | } 492 | 493 | template 494 | inline void generate_to(SeedSeq&& generator, DestIter dest) 495 | { 496 | typedef typename std::iterator_traits::value_type dest_t; 497 | constexpr bool IS_32BIT = sizeof(dest_t) == sizeof(uint32_t); 498 | 499 | generate_to_impl(std::forward(generator), dest, 500 | std::integral_constant{}); 501 | } 502 | 503 | /* generate_one, produce a value of integral type using a SeedSeq 504 | * (optionally, we can have it produce more than one and pick which one 505 | * we want) 506 | */ 507 | 508 | template 509 | inline UInt generate_one(SeedSeq&& generator) 510 | { 511 | UInt result[N]; 512 | generate_to(std::forward(generator), result); 513 | return result[i]; 514 | } 515 | 516 | template 517 | auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound) 518 | -> typename RngType::result_type 519 | { 520 | typedef typename RngType::result_type rtype; 521 | rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound) 522 | % upper_bound; 523 | for (;;) { 524 | rtype r = rng() - RngType::min(); 525 | if (r >= threshold) 526 | return r % upper_bound; 527 | } 528 | } 529 | 530 | template 531 | void shuffle(Iter from, Iter to, RandType&& rng) 532 | { 533 | typedef typename std::iterator_traits::difference_type delta_t; 534 | auto count = to - from; 535 | while (count > 1) { 536 | delta_t chosen(bounded_rand(rng, count)); 537 | --count; 538 | --to; 539 | using std::swap; 540 | swap(*(from+chosen), *to); 541 | } 542 | } 543 | 544 | /* 545 | * Although std::seed_seq is useful, it isn't everything. Often we want to 546 | * initialize a random-number generator some other way, such as from a random 547 | * device. 548 | * 549 | * Technically, it does not meet the requirements of a SeedSequence because 550 | * it lacks some of the rarely-used member functions (some of which would 551 | * be impossible to provide). However the C++ standard is quite specific 552 | * that actual engines only called the generate method, so it ought not to be 553 | * a problem in practice. 554 | */ 555 | 556 | template 557 | class seed_seq_from { 558 | private: 559 | RngType rng_; 560 | 561 | typedef uint_least32_t result_type; 562 | 563 | public: 564 | template 565 | seed_seq_from(Args&&... args) : 566 | rng_(std::forward(args)...) 567 | { 568 | // Nothing (else) to do... 569 | } 570 | 571 | template 572 | void generate(Iter start, Iter finish) 573 | { 574 | for (auto i = start; i != finish; ++i) 575 | *i = result_type(rng_()); 576 | } 577 | 578 | constexpr size_t size() const 579 | { 580 | return (sizeof(typename RngType::result_type) > sizeof(result_type) 581 | && RngType::max() > ~size_t(0UL)) 582 | ? ~size_t(0UL) 583 | : size_t(RngType::max()); 584 | } 585 | }; 586 | 587 | /* 588 | * Sometimes you might want a distinct seed based on when the program 589 | * was compiled. That way, a particular instance of the program will 590 | * behave the same way, but when recompiled it'll produce a different 591 | * value. 592 | */ 593 | 594 | template 595 | struct static_arbitrary_seed { 596 | private: 597 | static constexpr IntType fnv(IntType hash, const char* pos) { 598 | return *pos == '\0' 599 | ? hash 600 | : fnv((hash * IntType(16777619U)) ^ *pos, (pos+1)); 601 | } 602 | 603 | public: 604 | static constexpr IntType value = fnv(IntType(2166136261U ^ sizeof(IntType)), 605 | __DATE__ __TIME__ __FILE__); 606 | }; 607 | 608 | // Sometimes, when debugging or testing, it's handy to be able print the name 609 | // of a (in human-readable form). This code allows the idiom: 610 | // 611 | // cout << printable_typename() 612 | // 613 | // to print out my_foo_type_t (or its concrete type if it is a synonym) 614 | 615 | template 616 | struct printable_typename {}; 617 | 618 | template 619 | std::ostream& operator<<(std::ostream& out, printable_typename) { 620 | const char *implementation_typename = typeid(T).name(); 621 | #ifdef __GNUC__ 622 | int status; 623 | const char* pretty_name = 624 | abi::__cxa_demangle(implementation_typename, NULL, NULL, &status); 625 | if (status == 0) 626 | out << pretty_name; 627 | free((void*) pretty_name); 628 | if (status == 0) 629 | return out; 630 | #endif 631 | out << implementation_typename; 632 | return out; 633 | } 634 | 635 | } // namespace pcg_extras 636 | 637 | #endif // PCG_EXTRAS_HPP_INCLUDED 638 | -------------------------------------------------------------------------------- /src/random/xorshift.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* 4 | A number of XORShift RNGs. 5 | Copyright (C) 2018 Robert Kelly 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | 20 | namespace concurrent_data_structures { 21 | 22 | class XORShift32 { 23 | private: 24 | std::uint32_t m_current; 25 | 26 | public: 27 | static const std::size_t NUM_BITS = 32; 28 | 29 | XORShift32(const std::uint32_t seed) : m_current(seed == 0 ? 1 : seed) {} 30 | const std::size_t next_rand() { 31 | m_current ^= m_current << 13; 32 | m_current ^= m_current >> 17; 33 | m_current ^= m_current << 5; 34 | return m_current; 35 | } 36 | }; 37 | 38 | class XORShift64 { 39 | private: 40 | std::uint64_t m_current; 41 | 42 | public: 43 | static const std::size_t NUM_BITS = 64; 44 | 45 | XORShift64(const std::uint64_t seed) : m_current(seed == 0 ? 1 : seed) {} 46 | const std::size_t next_rand() { 47 | m_current ^= m_current << 13; 48 | m_current ^= m_current >> 7; 49 | m_current ^= m_current << 17; 50 | return m_current; 51 | } 52 | }; 53 | 54 | class XORShift128 { 55 | private: 56 | std::uint32_t m_current[4]; 57 | 58 | public: 59 | static const std::size_t NUM_BITS = 32; 60 | 61 | XORShift128(const std::size_t seed) { 62 | m_current[0] = (seed == 0 ? 1 : seed); 63 | m_current[1] = m_current[0]; 64 | m_current[2] = m_current[0]; 65 | m_current[3] = m_current[0]; 66 | } 67 | const std::uint32_t next_rand() { 68 | uint32_t s = m_current[3], t = m_current[3]; 69 | t ^= t << 11; 70 | t ^= t >> 8; 71 | m_current[3] = m_current[2]; 72 | m_current[2] = m_current[1]; 73 | m_current[1] = m_current[0]; 74 | t ^= s; 75 | t ^= s >> 19; 76 | m_current[0] = t; 77 | return t; 78 | } 79 | }; 80 | } 81 | --------------------------------------------------------------------------------