├── .clang-format ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── eval └── plotting.org ├── example └── example.cpp ├── include ├── bucket.hpp ├── chaining.hpp ├── cobucket.hpp ├── cuckoo_base.hpp ├── cuckoo_deamortized.hpp ├── cuckoo_dysect.hpp ├── cuckoo_independent_2lvl.hpp ├── cuckoo_overlap.hpp ├── cuckoo_simple.hpp ├── displacement_strategies │ ├── dis_bfs0.hpp │ ├── dis_bfs1.hpp │ ├── dis_deterministic_walk.hpp │ ├── dis_random_walk_acyclic.hpp │ ├── dis_random_walk_cyclic.hpp │ ├── dis_random_walk_optimistic.hpp │ ├── dis_trivial.hpp │ └── main_strategies.hpp ├── hasher.hpp ├── iterator_base.hpp ├── prob_base.hpp ├── prob_coalesced.hpp ├── prob_hops.hpp ├── prob_multitable_base.hpp ├── prob_quadratic.hpp ├── prob_robin.hpp └── prob_simple.hpp ├── readme.org ├── script ├── aggregation.sh ├── all_scripts.sh ├── cache_test.sh ├── deletetion.sh ├── growing.sh ├── load_bound.sh └── non_growing.sh ├── source ├── crawl_test.cpp ├── del_test.cpp ├── displ_test.cpp ├── eps_test.cpp ├── in_test.cpp ├── mix_test.cpp ├── mixd_test.cpp ├── mxls_test.cpp ├── probe_new_test.cpp ├── probe_test.cpp ├── reg_test.cpp ├── selection.hpp ├── sing_test.cpp └── time_test.cpp └── utils_replaced ├── commandline.h ├── counting_wait.h ├── default_hash.h ├── hash ├── crc_hash.h ├── murmur2_hash.h ├── murmur3_hash.h └── xx_hash.h ├── test_coordination.h └── thread_basics.h /.clang-format: -------------------------------------------------------------------------------- 1 | utils/.clang-format -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /test 2 | build/* 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "module/xxhash"] 2 | path = module/xxhash 3 | url = https://github.com/Cyan4973/xxHash.git 4 | [submodule "module/malloc_count"] 5 | path = module/malloc_count 6 | url = https://github.com/bingmann/malloc_count.git 7 | [submodule "utils"] 8 | path = utils 9 | url = https://github.com/TooBiased/utils_tm.git 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2018, Tobias Maier 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /example/example.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/hash/murmur2_hash.hpp" 2 | 3 | #include "include/cuckoo_dysect.hpp" 4 | #include 5 | 6 | 7 | int main(int, char**) 8 | { 9 | size_t capacity = 100000; 10 | 11 | using key_type = size_t; 12 | using mapped_type = size_t; 13 | 14 | // Our hash tables can be used very similar to std::unordered_map 15 | std::unordered_map standard_map(capacity); 16 | 17 | // There is just one additional parameter that controls the space efficiency 18 | dysect::cuckoo_dysect dysect_standard(capacity, 1.1); 19 | // This will instantiate a hash table, that will never be larger than: 20 | // 1.1*sizeof(std::pair) + O(1) 21 | 22 | // If you want to have some control on template parameters, following 23 | dysect::cuckoo_dysect< 24 | key_type, mapped_type, 25 | utils_tm::hash_tm::murmur2_hash, // hash function, the result has to 26 | // have 64 significant bits 27 | dysect::cuckoo_config<8, // bucket size 28 | 3, // number of hash functions 29 | 256, // number of subtables 30 | dysect::cuckoo_displacement::bfs // displacement 31 | // algorithm 32 | > > 33 | dysect_special( 34 | capacity, // obvious 35 | 1.05, // memory constraint factor 36 | 10000); // probing depth until an insertion is declared unsuccessful 37 | 38 | 39 | // INSERT 40 | auto [it, b] = dysect_standard.insert(5, 8); 41 | 42 | // b is a bool indicating that the element has been inserted 43 | if (b) 44 | std::cout << "key 5 - inserted - mapped data 8" << std::endl; 45 | else 46 | std::cout << "key 5 - insert unsuccessful (already used?)" << std::endl; 47 | 48 | // it can be used to iterate over the table, or to access the element 49 | if (it == dysect_standard.end()) 50 | std::cout << "key 5 - insert - key not present" 51 | << " (found no free space - probing depth?)" << std::endl; 52 | else 53 | std::cout << "key 5 - insert - mapped data is " << it->second 54 | << " (interesting if it was contained before)" << std::endl; 55 | 56 | 57 | 58 | // FIND 59 | auto it2 = dysect_standard.find(7); 60 | 61 | // the iterator is similar to before 62 | if (it2 == dysect_standard.end()) 63 | std::cout << "key 7 was not found!" << std::endl; 64 | else 65 | std::cout << "key 7 was found contained " << it2->second << std::endl; 66 | 67 | 68 | 69 | // ACCESSOR 70 | dysect_standard[42] = 666; 71 | // this inserts 666 with key 42 or replaces the previous value with that key 72 | } 73 | -------------------------------------------------------------------------------- /include/bucket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/bucket.h 5 | * 6 | * bucket implementation for variants of bucket cuckoo hashing. 7 | * This implementation ensures that all contained elements are stored in the 8 | * beginning of the bucket. This allows some performance optimizations. 9 | * 10 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 11 | * 12 | * Copyright (C) 2017 Tobias Maier 13 | * 14 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 15 | ******************************************************************************/ 16 | 17 | #include 18 | #include 19 | 20 | namespace dysect 21 | { 22 | 23 | template class bucket 24 | { 25 | public: 26 | using key_type = K; 27 | using mapped_type = D; 28 | 29 | private: 30 | using value_intern = std::pair; 31 | 32 | public: 33 | using find_return_type = std::pair; 34 | 35 | bucket() 36 | { 37 | for (size_t i = 0; i < BS; ++i) elements[i] = value_intern(); 38 | } 39 | bucket(const bucket& rhs) = default; 40 | bucket& operator=(const bucket& rhs) = default; 41 | 42 | bool insert(const key_type& k, const mapped_type& d); 43 | bool insert(const value_intern& t); 44 | find_return_type find(const key_type& k); 45 | bool remove(const key_type& k); 46 | find_return_type pop(const key_type& k); 47 | 48 | int probe(const key_type& k); 49 | int displacement(const key_type& k) const; 50 | 51 | bool space(); 52 | value_intern get(const size_t i); 53 | value_intern replace(const size_t i, const value_intern& t); 54 | 55 | 56 | value_intern* insert_ptr(const value_intern& t); 57 | const value_intern* find_ptr(const key_type& k) const; 58 | value_intern* find_ptr(const key_type& k); 59 | std::pair probe_ptr(const key_type& k); 60 | 61 | value_intern elements[BS]; 62 | }; 63 | 64 | 65 | template 66 | inline bool bucket::insert(const key_type& k, const mapped_type& d) 67 | { 68 | for (size_t i = 0; i < BS; ++i) 69 | { 70 | if (elements[i].first) continue; 71 | elements[i].first = k; 72 | elements[i].second = d; 73 | 74 | return true; 75 | } 76 | 77 | return false; 78 | } 79 | 80 | template 81 | inline bool bucket::insert(const value_intern& t) 82 | { 83 | for (size_t i = 0; i < BS; ++i) 84 | { 85 | if (elements[i].first) continue; 86 | 87 | elements[i] = t; 88 | return true; 89 | } 90 | 91 | return false; 92 | } 93 | 94 | template 95 | inline typename bucket::find_return_type 96 | bucket::find(const key_type& k) 97 | { 98 | for (size_t i = 0; i < BS; ++i) 99 | { 100 | if (!elements[i].first) return std::make_pair(false, mapped_type()); 101 | if (elements[i].first == k) 102 | return std::make_pair(true, elements[i].second); 103 | } 104 | return std::make_pair(false, mapped_type()); 105 | } 106 | 107 | template 108 | inline bool bucket::remove(const key_type& k) 109 | { 110 | for (size_t i = 0; i < BS; ++i) 111 | { 112 | if (elements[i].first == k) 113 | { 114 | size_t j = BS - 1; 115 | for (; !elements[j].first; --j) {} 116 | elements[i] = elements[j]; 117 | elements[j] = std::make_pair(key_type(), mapped_type()); 118 | return true; 119 | } 120 | else if (!elements[i].first) 121 | { 122 | break; 123 | } 124 | } 125 | return false; 126 | } 127 | 128 | template 129 | inline typename bucket::find_return_type 130 | bucket::pop(const key_type& k) 131 | { 132 | for (size_t i = 0; i < BS; ++i) 133 | { 134 | if (elements[i].first == k) 135 | { 136 | mapped_type d = elements[i].second; 137 | for (size_t j = i + 1; j < BS; ++j) 138 | { 139 | if (elements[j].first) 140 | { 141 | elements[i] = elements[j]; 142 | i = j; 143 | } 144 | else 145 | break; 146 | } 147 | elements[i] = std::make_pair(key_type(), mapped_type()); 148 | return std::make_pair(true, d); 149 | } 150 | else if (!elements[i].first) 151 | { 152 | break; 153 | } 154 | } 155 | return std::make_pair(false, mapped_type()); 156 | } 157 | 158 | template 159 | inline int bucket::probe(const key_type& k) 160 | { 161 | for (size_t i = 0; i < BS; ++i) 162 | { 163 | if (!elements[i].first) return BS - i; 164 | if (elements[i].first == k) return -1; 165 | } 166 | return 0; 167 | } 168 | 169 | template 170 | inline int bucket::displacement(const key_type& k) const 171 | { 172 | for (size_t i = 0; i < BS; ++i) 173 | { 174 | if (elements[i].first == k) return i; 175 | } 176 | return BS; 177 | } 178 | 179 | template inline bool bucket::space() 180 | { 181 | return !elements[BS - 1].first; 182 | } 183 | 184 | template 185 | inline std::pair bucket::get(size_t i) 186 | { 187 | return elements[i]; 188 | } 189 | 190 | template 191 | inline std::pair 192 | bucket::replace(size_t i, const value_intern& newE) 193 | { 194 | auto temp = elements[i]; 195 | elements[i] = newE; 196 | return temp; 197 | } 198 | 199 | 200 | template 201 | inline std::pair* bucket::insert_ptr(const value_intern& t) 202 | { 203 | for (size_t i = 0; i < BS; ++i) 204 | { 205 | if (elements[i].first) continue; 206 | 207 | elements[i] = t; 208 | return &elements[i]; 209 | } 210 | 211 | return nullptr; 212 | } 213 | 214 | template 215 | inline std::pair* bucket::find_ptr(const key_type& k) 216 | { 217 | for (size_t i = 0; i < BS; ++i) 218 | { 219 | // if (!elements[i].first ) return nullptr; 220 | if (elements[i].first == k) return &elements[i]; 221 | } 222 | return nullptr; 223 | } 224 | 225 | template 226 | inline const std::pair* 227 | bucket::find_ptr(const key_type& k) const 228 | { 229 | for (size_t i = 0; i < BS; ++i) 230 | { 231 | // if (!elements[i].first ) return nullptr; 232 | if (elements[i].first == k) return &elements[i]; 233 | } 234 | return nullptr; 235 | } 236 | 237 | template 238 | inline std::pair*> 239 | bucket::probe_ptr(const key_type& k) 240 | { 241 | for (size_t i = 0; i < BS; ++i) 242 | { 243 | if (!elements[i].first) return std::make_pair(BS - i, &elements[i]); 244 | if (elements[i].first == k) return std::make_pair(-1, &elements[i]); 245 | } 246 | return std::make_pair(0, nullptr); 247 | } 248 | 249 | } // namespace dysect 250 | -------------------------------------------------------------------------------- /include/chaining.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/chaining.hpp 5 | * 6 | * prob_base is similar to cuckoo_base, in that it encapsules 7 | * everything, that all probing based hash tables have in common. 8 | * 9 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 10 | * 11 | * Copyright (C) 2017 Tobias Maier 12 | * 13 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 14 | ******************************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "utils/default_hash.hpp" 22 | #include "utils/fastrange.hpp" 23 | #include "utils/output.hpp" 24 | 25 | #include "bucket.hpp" 26 | #include "iterator_base.hpp" 27 | 28 | namespace otm = utils_tm::out_tm; 29 | 30 | namespace dysect 31 | { 32 | struct triv_config 33 | { 34 | }; 35 | 36 | template class iterator_incr; 37 | 38 | 39 | 40 | template 42 | class chaining 43 | { 44 | private: 45 | using this_type = chaining; 46 | using hash_function_type = HF; 47 | friend iterator_incr; 48 | 49 | public: 50 | using size_type = size_t; 51 | using key_type = K; 52 | using mapped_type = D; 53 | using iterator = iterator_base >; 54 | using const_iterator = iterator_base, true>; 55 | 56 | private: 57 | using value_intern = std::pair; 58 | struct queue_item 59 | { 60 | queue_item* next; 61 | value_intern element; 62 | queue_item(const std::pair& element) 63 | : next(nullptr), element(element) 64 | { 65 | } 66 | }; 67 | 68 | public: 69 | chaining(size_type cap, double alpha, size_type /*steps*/ = 0) 70 | : alpha(alpha), beta((alpha + 1.) / 2.), n(0), 71 | capacity((cap) ? cap * alpha : 2048 * alpha), 72 | thresh((cap) ? cap * beta : 2048 * beta) 73 | { 74 | if (cap) table = std::make_unique(capacity); 75 | std::fill(table.get(), table.get() + capacity, nullptr); 76 | } 77 | 78 | ~chaining() 79 | { 80 | if (!table) return; 81 | for (size_t i = 0; i < capacity; ++i) 82 | { 83 | auto curr = table[i]; 84 | while (curr) 85 | { 86 | auto temp = curr->next; 87 | delete curr; 88 | curr = temp; 89 | } 90 | } 91 | } 92 | 93 | chaining(const chaining&) = delete; 94 | chaining& operator=(const chaining&) = delete; 95 | 96 | chaining(chaining&& rhs) 97 | : alpha(rhs.alpha), beta(rhs.beta), n(rhs.n), capacity(rhs.capacity), 98 | thresh(rhs.thresh), hasher(rhs.hasher), table(std::move(rhs.table)) 99 | { 100 | rhs.table = nullptr; 101 | } 102 | chaining& operator=(chaining&& rhs) 103 | { 104 | if (this == &rhs) return *this; 105 | this->~this_type(); 106 | new (this) this_type(std::move(rhs)); 107 | return *this; 108 | } 109 | 110 | private: 111 | /*** members that should become private at some point *********************/ 112 | double alpha; 113 | double beta; 114 | size_type n; 115 | size_type capacity; 116 | size_type thresh; 117 | hash_function_type hasher; 118 | 119 | std::unique_ptr table; 120 | 121 | public: 122 | // Basic Hash Table Functionality ****************************************** 123 | iterator find(const key_type& k); 124 | const_iterator find(const key_type& k) const; 125 | std::pair insert(const key_type& k, const mapped_type& d); 126 | std::pair insert(const value_intern& t); 127 | size_type erase(const key_type& k); 128 | int displacement(const key_type& k) const; 129 | void grow(); 130 | 131 | // Easy use Accessors for std compliance *********************************** 132 | inline iterator begin(); 133 | inline const_iterator begin() const { return cbegin(); } 134 | inline const_iterator cbegin() const; 135 | inline iterator end() 136 | { 137 | return iterator(nullptr, *this, nullptr, capacity); 138 | } 139 | inline const_iterator end() const { return cend(); } 140 | inline const_iterator cend() const 141 | { 142 | return const_iterator(nullptr, *this, nullptr, capacity); 143 | } 144 | 145 | mapped_type& at(const key_type& k); 146 | const mapped_type& at(const key_type& k) const; 147 | mapped_type& operator[](const key_type& k); 148 | size_type count(const key_type& k) const; 149 | 150 | size_type get_capacity() const { return capacity; } 151 | 152 | private: 153 | void insert_intern(queue_item* item); 154 | 155 | // Easy iterators ********************************************************** 156 | inline iterator make_iterator(queue_item* item, size_type idx) 157 | { 158 | return iterator(&(item->element), *this, item, idx); 159 | } 160 | inline const_iterator make_citerator(queue_item* item, size_type idx) const 161 | { 162 | return const_iterator(&(item->element), *this, item, idx); 163 | } 164 | 165 | // implementation specific functions (static polymorph) ******************** 166 | inline size_type h(key_type k) const 167 | { 168 | return utils_tm::fastrange64(capacity, hasher(k)); 169 | } 170 | 171 | inline void inc_n() 172 | { 173 | if (++n > thresh) grow(); 174 | } 175 | inline void dec_n() { --n; } 176 | 177 | // Private helper function ************************************************* 178 | 179 | public: 180 | inline static void print_init_header(otm::output_type& out) 181 | { 182 | out << otm::width(10) << "f_cap" << otm::width(16) << "overhead_bytes"; 183 | } 184 | inline void print_init_data(otm::output_type& out) 185 | { 186 | long long overhead = 187 | (capacity + n) * sizeof(queue_item*) // pointer 188 | + n * sizeof(std::pair) // elements 189 | - capacity * sizeof(std::pair); 190 | out << otm::width(10) << capacity << otm::width(16) << overhead; 191 | } 192 | }; 193 | 194 | 195 | 196 | // Implementation of main functionality **************************************** 197 | 198 | template 199 | inline typename chaining::iterator 200 | chaining::find(const key_type& k) 201 | { 202 | auto ind = h(k); 203 | 204 | auto curr = table[ind]; 205 | 206 | while (curr) 207 | { 208 | if (curr->element.first == k) return make_iterator(curr, ind); 209 | 210 | curr = curr->next; 211 | } 212 | return end(); 213 | } 214 | 215 | template 216 | inline typename chaining::const_iterator 217 | chaining::find(const key_type& k) const 218 | { 219 | auto ind = h(k); 220 | 221 | auto curr = table[ind]; 222 | 223 | while (curr) 224 | { 225 | if (curr->element.first == k) return make_citerator(curr, ind); 226 | 227 | curr = curr->next; 228 | } 229 | return cend(); 230 | } 231 | 232 | template 233 | inline std::pair::iterator, bool> 234 | chaining::insert(const key_type& k, const mapped_type& d) 235 | { 236 | return insert(std::make_pair(k, d)); 237 | } 238 | 239 | template 240 | inline std::pair::iterator, bool> 241 | chaining::insert(const value_intern& t) 242 | { 243 | auto ind = h(t.first); 244 | 245 | auto curr = table[ind]; 246 | if (!curr) 247 | { 248 | table[ind] = new queue_item(t); 249 | return std::make_pair(make_iterator(curr, ind), true); 250 | } 251 | while (true) 252 | { 253 | if (curr->element.first == t.first) 254 | return std::make_pair(make_iterator(curr, ind), false); 255 | if (!curr->next) 256 | { 257 | curr->next = new queue_item(t); 258 | return std::make_pair(make_iterator(curr, ind), true); 259 | } 260 | curr = curr->next; 261 | } 262 | } 263 | 264 | template 265 | inline typename chaining::size_type 266 | chaining::erase(const key_type& k) 267 | { 268 | auto ind = h(k); 269 | 270 | auto curr = table[ind]; 271 | if (!curr) return 0; 272 | if (curr->element.first == k) 273 | { 274 | table[ind] = curr->next; 275 | return 1; 276 | } 277 | 278 | auto prev = curr; 279 | curr = curr->next; 280 | 281 | while (curr) 282 | { 283 | if (curr->element.first == k) 284 | { 285 | prev->next = curr->next; 286 | return 1; 287 | } 288 | prev = curr; 289 | curr = curr->next; 290 | } 291 | return 0; 292 | } 293 | 294 | template 295 | inline int chaining::displacement(const key_type& k) const 296 | { 297 | auto ind = h(k); 298 | 299 | auto curr = table[ind]; 300 | size_type steps = 0; 301 | 302 | while (curr) 303 | { 304 | if (curr->element.first == k) return steps; 305 | curr = curr->next; 306 | ++steps; 307 | } 308 | return -1; 309 | } 310 | 311 | 312 | template 313 | inline void chaining::grow() 314 | { 315 | this_type ntable(capacity, alpha); 316 | for (size_t i = 0; i < capacity; ++i) 317 | { 318 | while (table[i]) 319 | { 320 | auto item = table[i]; 321 | table[i] = item->next; 322 | ntable.insert_intern(item); 323 | } 324 | } 325 | (*this) = std::move(ntable); 326 | } 327 | 328 | template 329 | inline void chaining::insert_intern(queue_item* item) 330 | { 331 | auto ind = h(item->element.first); 332 | item->next = table[ind]; 333 | table[ind] = item; 334 | } 335 | 336 | // Accessor Implementations **************************************************** 337 | 338 | template 339 | inline typename chaining::mapped_type& 340 | chaining::at(const key_type& k) 341 | { 342 | auto a = find(k); 343 | if (a == end()) 344 | throw std::out_of_range("cannot find key"); 345 | else 346 | return (*a).second; 347 | } 348 | 349 | template 350 | inline const typename chaining::mapped_type& 351 | chaining::at(const key_type& k) const 352 | { 353 | auto a = find(k); 354 | if (a == cend()) 355 | throw std::out_of_range("cannot find key"); 356 | else 357 | return (*a).second; 358 | } 359 | 360 | template 361 | inline typename chaining::mapped_type& 362 | chaining::operator[](const key_type& k) 363 | { 364 | auto t = insert(k, mapped_type()); 365 | return (*t.first).second; 366 | } 367 | 368 | template 369 | inline typename chaining::size_type 370 | chaining::count(const key_type& k) const 371 | { 372 | return (find(k) != cend()) ? 1 : 0; 373 | } 374 | 375 | 376 | template 377 | inline typename chaining::iterator chaining::begin() 378 | { 379 | for (size_t i = 0; i < capacity; ++i) 380 | { 381 | if (table[i]) return make_iterator(table[i], i); 382 | } 383 | return end(); 384 | } 385 | 386 | template 387 | inline typename chaining::const_iterator 388 | chaining::cbegin() const 389 | { 390 | for (size_t i = 0; i < capacity; ++i) 391 | { 392 | if (table[i]) return make_citerator(table[i], i); 393 | } 394 | return cend(); 395 | } 396 | 397 | 398 | // Iterator increment ********************************************************** 399 | 400 | template class iterator_incr 401 | { 402 | public: 403 | using table_type = Chaining; 404 | 405 | private: 406 | using key_type = typename table_type::key_type; 407 | using mapped_type = typename table_type::mapped_type; 408 | using ipointer = std::pair*; 409 | using qitem = typename table_type::queue_item; 410 | 411 | public: 412 | iterator_incr(const table_type& table, qitem* item, size_t index) 413 | : curr_item_ptr(item), table_ptr(&table.table[index]), 414 | end_table_ptr(&table.table[table.capacity - 1]) 415 | { 416 | } 417 | iterator_incr(const iterator_incr&) = default; 418 | iterator_incr& operator=(const iterator_incr&) = default; 419 | 420 | ipointer next(ipointer cur) 421 | { 422 | if (curr_item_ptr->next) 423 | { 424 | curr_item_ptr = curr_item_ptr->next; 425 | return &(curr_item_ptr->element); 426 | } 427 | while (table_ptr < end_table_ptr) 428 | { 429 | ++table_ptr; 430 | if (*table_ptr) 431 | { 432 | curr_item_ptr = *table_ptr; 433 | return &(curr_item_ptr->element); 434 | } 435 | } 436 | return nullptr; 437 | } 438 | 439 | private: 440 | qitem* curr_item_ptr; 441 | qitem** table_ptr; 442 | qitem** end_table_ptr; 443 | }; 444 | 445 | } // namespace dysect 446 | -------------------------------------------------------------------------------- /include/cobucket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/co_bucket.h 5 | * 6 | * co_bucket is the bucket implementation for variants that use overlapping 7 | * buckets. Contrary to bucket, co_bucket does not ensure that all contained 8 | * elements are stored in the beginning of the bucket. Therefore, some 9 | * improvements are not possible. 10 | * 11 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 12 | * 13 | * Copyright (C) 2017 Tobias Maier 14 | * 15 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 16 | ******************************************************************************/ 17 | 18 | #include 19 | #include 20 | 21 | namespace dysect 22 | { 23 | 24 | template class co_bucket 25 | { 26 | private: 27 | using this_type = co_bucket; 28 | 29 | public: 30 | using key_type = K; 31 | using mapped_type = D; 32 | 33 | private: 34 | using value_intern = std::pair; 35 | 36 | public: 37 | using find_return_type = std::pair; 38 | 39 | co_bucket() 40 | { 41 | for (size_t i = 0; i < BS; ++i) 42 | elements[i] = std::make_pair(key_type(), mapped_type()); 43 | } 44 | co_bucket(const co_bucket& rhs) = default; 45 | co_bucket& operator=(const co_bucket& rhs) = default; 46 | 47 | bool insert(const key_type& k, const mapped_type& d); 48 | bool insert(const value_intern& t); 49 | find_return_type find(const key_type& k); 50 | bool remove(const key_type& k); 51 | find_return_type pop(const key_type& k); 52 | 53 | int probe(const key_type& k); 54 | int displacement(const key_type& k) const; 55 | 56 | bool space(); 57 | value_intern get(const size_t i); 58 | value_intern replace(const size_t i, const value_intern& t); 59 | 60 | 61 | value_intern* insert_ptr(const value_intern& t); 62 | const value_intern* find_ptr(const key_type& k) const; 63 | value_intern* find_ptr(const key_type& k); 64 | std::pair probe_ptr(const key_type& k); 65 | 66 | value_intern elements[BS]; 67 | }; 68 | 69 | 70 | template 71 | inline bool co_bucket::insert(const key_type& k, const mapped_type& d) 72 | { 73 | for (size_t i = 0; i < BS; ++i) 74 | { 75 | if (elements[i].first) continue; 76 | elements[i].first = k; 77 | elements[i].second = d; 78 | 79 | return true; 80 | } 81 | 82 | return false; 83 | } 84 | 85 | template 86 | inline bool co_bucket::insert(const value_intern& t) 87 | { 88 | for (size_t i = 0; i < BS; ++i) 89 | { 90 | if (elements[i].first) continue; 91 | elements[i] = t; 92 | return true; 93 | } 94 | 95 | return false; 96 | } 97 | 98 | template 99 | inline typename co_bucket::find_return_type 100 | co_bucket::find(const key_type& k) 101 | { 102 | for (size_t i = 0; i < BS; ++i) 103 | { 104 | if (elements[i].first == k) 105 | return std::make_pair(true, elements[i].second); 106 | } 107 | return std::make_pair(false, mapped_type()); 108 | } 109 | 110 | template 111 | inline bool co_bucket::remove(const key_type& k) 112 | { 113 | for (size_t i = 0; i < BS; ++i) 114 | { 115 | if (elements[i].first == k) 116 | { 117 | elements[i] = std::make_pair(key_type(), mapped_type()); 118 | return true; 119 | } 120 | } 121 | return false; 122 | } 123 | 124 | template 125 | inline typename co_bucket::find_return_type 126 | co_bucket::pop(const key_type& k) 127 | { 128 | for (size_t i = 0; i < BS; ++i) 129 | { 130 | if (elements[i].first == k) 131 | { 132 | mapped_type d = elements[i].second; 133 | elements[i] = std::make_pair(key_type(), mapped_type()); 134 | return std::make_pair(true, d); 135 | } 136 | } 137 | return std::make_pair(false, mapped_type()); 138 | } 139 | 140 | template 141 | inline int co_bucket::probe(const key_type& k) 142 | { 143 | size_t count = 0; 144 | for (size_t i = 0; i < BS; ++i) 145 | { 146 | if (!elements[i].first) ++count; 147 | if (elements[i].first == k) return -1; 148 | } 149 | return count; 150 | } 151 | 152 | template 153 | inline int co_bucket::displacement(const key_type& k) const 154 | { 155 | for (size_t i = 0; i < BS; ++i) 156 | { 157 | if (elements[i].first == k) return i; 158 | } 159 | return BS; 160 | } 161 | 162 | template inline bool co_bucket::space() 163 | { 164 | for (size_t i = 0; i < BS; ++i) 165 | { 166 | if (!elements[i].first) return true; 167 | } 168 | return false; 169 | } 170 | 171 | template 172 | inline std::pair co_bucket::get(size_t i) 173 | { 174 | return elements[i]; 175 | } 176 | 177 | template 178 | inline std::pair 179 | co_bucket::replace(size_t i, const value_intern& newE) 180 | { 181 | auto temp = elements[i]; 182 | elements[i] = newE; 183 | return temp; 184 | } 185 | 186 | 187 | template 188 | inline std::pair* co_bucket::insert_ptr(const value_intern& t) 189 | { 190 | for (size_t i = 0; i < BS; ++i) 191 | { 192 | if (elements[i].first) continue; 193 | 194 | elements[i] = t; 195 | return &elements[i]; 196 | } 197 | 198 | return nullptr; 199 | } 200 | 201 | template 202 | inline std::pair* co_bucket::find_ptr(const key_type& k) 203 | { 204 | for (size_t i = 0; i < BS; ++i) 205 | { 206 | if (elements[i].first == k) return &elements[i]; 207 | } 208 | return nullptr; 209 | } 210 | 211 | template 212 | inline const std::pair* 213 | co_bucket::find_ptr(const key_type& k) const 214 | { 215 | for (size_t i = 0; i < BS; ++i) 216 | { 217 | if (elements[i].first == k) return &elements[i]; 218 | } 219 | return nullptr; 220 | } 221 | 222 | template 223 | inline std::pair*> 224 | co_bucket::probe_ptr(const key_type& k) 225 | { 226 | size_t count = 0; 227 | value_intern* tptr = nullptr; 228 | for (size_t i = 0; i < BS; ++i) 229 | { 230 | if (!elements[i].first) 231 | { 232 | ++count; 233 | tptr = (tptr) ? tptr : &elements[i]; 234 | } 235 | if (elements[i].first == k) return std::make_pair(-1, &elements[i]); 236 | } 237 | return std::make_pair(count, tptr); 238 | } 239 | } // namespace dysect 240 | -------------------------------------------------------------------------------- /include/cuckoo_deamortized.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/cuckoo_deamortized.h 5 | * 6 | * Requirement: 7 | * 8 | * cuckoo_deamortized is a dynamically growing cuckoo table variation 9 | * that uses only one table. The idea is to grow the table in place 10 | * but do it in small steps. Therefore, grown parts of the table 11 | * coexist with ungrown ones. 12 | * 13 | * 0 x 2^k cap = 2^k+x 14 | * +----------+------------------+----------+ - - - - - - - - - - 15 | * | grown | ungrown | new part | overallocated ... 16 | * +----------+------------------+----------+ - - - - - - - - - - 17 | * 18 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 19 | * 20 | * Copyright (C) 2017 Tobias Maier 21 | * 22 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 23 | ******************************************************************************/ 24 | 25 | #include "cuckoo_base.hpp" 26 | #include "utils/default_hash.hpp" 27 | 28 | namespace dysect 29 | { 30 | 31 | template > 33 | class cuckoo_deamortized 34 | : public cuckoo_traits >::base_type 35 | { 36 | private: 37 | using this_type = cuckoo_deamortized; 38 | using base_type = typename cuckoo_traits::base_type; 39 | using bucket_type = typename cuckoo_traits::bucket_type; 40 | using hasher_type = typename cuckoo_traits::hasher_type; 41 | using hashed_type = typename hasher_type::hashed_type; 42 | using ext = typename hasher_type::extractor_type; 43 | 44 | friend base_type; 45 | friend iterator_incr; 46 | 47 | public: 48 | using size_type = typename cuckoo_traits::size_type; 49 | using key_type = typename cuckoo_traits::key_type; 50 | using mapped_type = typename cuckoo_traits::mapped_type; 51 | using iterator = typename base_type::iterator; 52 | using const_iterator = typename base_type::const_iterator; 53 | 54 | private: 55 | using value_intern = std::pair; 56 | 57 | 58 | static constexpr size_type bs = cuckoo_traits::bs; 59 | static constexpr size_type nh = cuckoo_traits::nh; 60 | static constexpr size_type tl = 1; 61 | 62 | static constexpr size_type max_size = 10ull << 30; 63 | static constexpr size_type min_buckets = 4096; 64 | static constexpr size_type grow_step = 32; 65 | 66 | public: 67 | cuckoo_deamortized(size_type cap = 0, double size_constraint = 1.1, 68 | size_type dis_steps = 256, size_type seed = 0) 69 | : base_type(size_constraint, dis_steps, seed) 70 | { 71 | auto temp = reinterpret_cast(operator new(max_size)); 72 | table = std::unique_ptr(temp); 73 | 74 | // SET CAPACITY, BITMASKS, THRESHOLD 75 | size_type tcap = size_type(double(cap) * size_constraint / double(bs)); 76 | size_type tbit = min_buckets << 1; 77 | while (tbit <= tcap) tbit <<= 1; 78 | bitmask_large = tbit - 1; 79 | bitmask_small = bitmask_large >> 1; 80 | 81 | size_type doub = tcap - bitmask_small - 1; 82 | doub /= grow_step; 83 | doub *= grow_step; 84 | 85 | capacity = (doub + bitmask_small + 1) * bs; 86 | bucket_cutoff = doub + bitmask_small + 1; 87 | grow_thresh = size_type(double(capacity + grow_step * bs) / alpha); 88 | 89 | std::fill(table.get(), table.get() + capacity, value_intern()); 90 | } 91 | 92 | cuckoo_deamortized(const cuckoo_deamortized&) = delete; 93 | cuckoo_deamortized& operator=(const cuckoo_deamortized&) = delete; 94 | 95 | cuckoo_deamortized(cuckoo_deamortized&&) = default; 96 | cuckoo_deamortized& operator=(cuckoo_deamortized&&) = default; 97 | 98 | private: 99 | using base_type::alpha; 100 | using base_type::capacity; 101 | using base_type::grow_thresh; 102 | using base_type::hasher; 103 | using base_type::n; 104 | 105 | size_type bucket_cutoff; 106 | size_type bitmask_large; 107 | size_type bitmask_small; 108 | 109 | std::unique_ptr table; 110 | 111 | using base_type::make_citerator; 112 | using base_type::make_iterator; 113 | 114 | public: 115 | using base_type::insert; 116 | 117 | std::pair getTable(size_type i) 118 | { 119 | return (!i) ? std::make_pair(bucket_cutoff, table.get()) 120 | : std::make_pair(0, nullptr); 121 | } 122 | 123 | inline iterator begin() 124 | { 125 | auto temp = make_iterator(&table[0]); 126 | if (!temp->first) temp++; 127 | return temp; 128 | } 129 | 130 | inline const_iterator cbegin() const 131 | { 132 | auto temp = make_citerator(&table[0]); 133 | if (!temp->first) temp++; 134 | return temp; 135 | } 136 | 137 | private: 138 | // Functions for finding buckets ******************************************* 139 | 140 | inline void getBuckets(hashed_type h, bucket_type** mem) const 141 | { 142 | for (size_type i = 0; i < nh; ++i) { mem[i] = getBucket(h, i); } 143 | } 144 | 145 | inline bucket_type* getBucket(hashed_type h, size_type i) const 146 | { 147 | size_type l = ext::loc(h, i) & bitmask_large; 148 | if (l >= bucket_cutoff) l &= bitmask_small; 149 | return reinterpret_cast(&(table[l * bs])); 150 | } 151 | 152 | 153 | 154 | // Size changes (GROWING) ************************************************** 155 | 156 | inline void grow() 157 | { 158 | size_type ncap = capacity + grow_step * bs; 159 | size_type ncutoff = bucket_cutoff + grow_step; 160 | grow_thresh = size_type(double(ncap + grow_step * bs) / alpha); 161 | 162 | std::fill(table.get() + capacity, table.get() + ncap, value_intern()); 163 | 164 | migrate(capacity, ncap); 165 | 166 | capacity = ncap; 167 | bucket_cutoff = ncutoff; 168 | 169 | if (ncutoff >= bitmask_large) 170 | { 171 | bitmask_small = bitmask_large; 172 | bitmask_large = (bitmask_large << 1) + 1; 173 | } 174 | } 175 | 176 | inline void migrate(size_type, size_type) 177 | { 178 | size_type flag = bitmask_large & (~bitmask_small); 179 | size_type ptr0 = (bucket_cutoff & bitmask_small) * bs; 180 | 181 | for (size_type i = 0; i < grow_step; ++i) 182 | { 183 | bucket_type* bucket0_ptr = 184 | reinterpret_cast(table.get() + ptr0 + i * bs); 185 | bucket_type* bucket1_ptr = 186 | reinterpret_cast(table.get() + capacity + i * bs); 187 | 188 | size_type c0 = 0; 189 | size_type c1 = 0; 190 | for (size_type j = 0; j < bs; ++j) 191 | { 192 | auto curr = bucket0_ptr->elements[j]; 193 | bucket0_ptr->elements[j] = value_intern(); 194 | if (!curr.first) break; 195 | 196 | bucket_type* targets[nh]; 197 | hashed_type hash = hasher(curr.first); 198 | getBuckets(hash, targets); 199 | 200 | for (size_type t = 0; t < nh; ++t) 201 | { 202 | if (targets[t] == bucket0_ptr) 203 | { 204 | if (ext::loc(hash, t) & flag) 205 | { 206 | bucket1_ptr->elements[c1++] = curr; 207 | } 208 | else 209 | { 210 | bucket0_ptr->elements[c0++] = curr; 211 | } 212 | break; 213 | } 214 | } 215 | } 216 | } 217 | } 218 | }; 219 | 220 | 221 | 222 | // Traits class defining types ************************************************* 223 | 224 | template 225 | class cuckoo_traits > 226 | { 227 | public: 228 | using specialized_type = cuckoo_deamortized; 229 | using base_type = cuckoo_base; 230 | using config_type = Conf; 231 | 232 | using size_type = size_t; 233 | using key_type = K; 234 | using mapped_type = D; 235 | 236 | static constexpr size_type tl = 1; 237 | static constexpr size_type bs = Conf::bs; 238 | static constexpr size_type nh = Conf::nh; 239 | static constexpr bool fix_errors = Conf::fix_errors; 240 | 241 | using hasher_type = hasher; 242 | using bucket_type = bucket; 243 | }; 244 | 245 | 246 | 247 | // Iterator increment ********************************************************** 248 | 249 | template 250 | class iterator_incr > 251 | { 252 | public: 253 | using table_type = cuckoo_deamortized; 254 | 255 | private: 256 | using size_type = typename table_type::size_type; 257 | using pointer = std::pair*; 258 | static constexpr size_type bs = Conf::bs; 259 | 260 | public: 261 | iterator_incr(const table_type& table_) 262 | : end_ptr(reinterpret_cast(&table_.table[table_.capacity - 1])) 263 | { 264 | } 265 | iterator_incr(const iterator_incr&) = default; 266 | iterator_incr& operator=(const iterator_incr&) = default; 267 | 268 | pointer next(pointer cur) 269 | { 270 | while (cur < end_ptr) 271 | { 272 | if ((++cur)->first) return cur; 273 | } 274 | return nullptr; 275 | } 276 | 277 | private: 278 | pointer end_ptr; 279 | }; 280 | 281 | } // namespace dysect 282 | -------------------------------------------------------------------------------- /include/cuckoo_independent_2lvl.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/cuckoo_independent_2lvl.h 5 | * 6 | * independent2l is a variant of cuckoo hashing, where the base table 7 | * is split similar to DySECT, but all buckets connected to one 8 | * element are placed in the same subtable, therefore, no balancing 9 | * happens between subtables. The tables have to grow in small steps 10 | * (no doubling) in order to hold the given size constraints. During the 11 | * migration, the size constraint is violated. 12 | * 13 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 14 | * 15 | * Copyright (C) 2017 Tobias Maier 16 | * 17 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 18 | ******************************************************************************/ 19 | 20 | #include "cuckoo_base.hpp" 21 | #include "utils/default_hash.hpp" 22 | #include "utils/fastrange.hpp" 23 | #include 24 | 25 | 26 | namespace dysect 27 | { 28 | 29 | template class cuckoo_traits; 30 | 31 | template > 33 | class cuckoo_independent_2lvl 34 | : public cuckoo_traits >::base_type 35 | { 36 | private: 37 | using this_type = cuckoo_independent_2lvl; 38 | using base_type = typename cuckoo_traits::base_type; 39 | using bucket_type = typename cuckoo_traits::bucket_type; 40 | using hasher_type = typename cuckoo_traits::hasher_type; 41 | using hashed_type = typename hasher_type::hashed_type; 42 | using ext = typename hasher_type::extractor_type; 43 | 44 | friend base_type; 45 | friend iterator_incr; 46 | 47 | public: 48 | using size_type = typename base_type::size_type; 49 | using key_type = typename cuckoo_traits::key_type; 50 | using mapped_type = typename cuckoo_traits::mapped_type; 51 | using iterator = typename base_type::iterator; 52 | using const_iterator = typename base_type::const_iterator; 53 | using insert_return_type = typename base_type::insert_return_type; 54 | 55 | private: 56 | using value_intern = std::pair; 57 | 58 | public: 59 | cuckoo_independent_2lvl(size_type cap = 0, double size_constraint = 1.1, 60 | size_type dis_steps = 256, size_type seed = 0) 61 | : base_type(size_constraint, dis_steps, seed), 62 | beta((1.0 + size_constraint) / 2.0) 63 | { 64 | size_type lsize = std::floor(cap * size_constraint / double(tl * bs)); 65 | lsize = std::max(lsize, 256ul); 66 | // double factor = double(lsize)/fac_div; 67 | size_type grow_thresh = double(lsize * bs) / beta; 68 | 69 | for (size_type i = 0; i < tl; ++i) 70 | { 71 | ll_tab[i] = std::make_unique(lsize); 72 | ll_size[i] = lsize; 73 | ll_elem[i] = 0; 74 | ll_thresh[i] = grow_thresh; 75 | // ll_factor[i] = factor; 76 | } 77 | 78 | capacity = bs * tl * lsize; 79 | } 80 | 81 | cuckoo_independent_2lvl(const cuckoo_independent_2lvl&) = delete; 82 | cuckoo_independent_2lvl& operator=(const cuckoo_independent_2lvl&) = delete; 83 | 84 | cuckoo_independent_2lvl(cuckoo_independent_2lvl&& rhs) 85 | : base_type(std::move(rhs)), beta(rhs.beta) 86 | { 87 | for (size_type i = 0; i < tl; ++i) 88 | { 89 | ll_tab[i] = std::move(rhs.ll_tab[i]); 90 | ll_size[i] = rhs.ll_size[i]; 91 | ll_elem[i] = rhs.ll_elem[i]; 92 | ll_thresh[i] = rhs.ll_thresh[i]; 93 | // ll_factor[i] = rhs.ll_factor[i]; 94 | } 95 | } 96 | 97 | cuckoo_independent_2lvl& operator=(cuckoo_independent_2lvl&& rhs) 98 | { 99 | base_type::operator=(std::move(rhs)); 100 | beta = rhs.beta; 101 | 102 | for (size_type i = 0; i < tl; ++i) 103 | { 104 | std::swap(ll_tab[i], rhs.ll_tab[i]); 105 | std::swap(ll_size[i], rhs.ll_size[i]); 106 | std::swap(ll_elem[i], rhs.ll_elem[i]); 107 | std::swap(ll_thresh[i], rhs.ll_thresh[i]); 108 | // std::swap(ll_factor[i], rhs.ll_factor[i]); 109 | } 110 | return *this; 111 | } 112 | 113 | private: 114 | using base_type::alpha; 115 | using base_type::capacity; 116 | using base_type::hasher; 117 | using base_type::n; 118 | 119 | static constexpr size_type bs = cuckoo_traits::bs; 120 | static constexpr size_type tl = cuckoo_traits::tl; 121 | static constexpr size_type nh = cuckoo_traits::nh; 122 | // static constexpr double fac_div = double (1ull << (32 - ct_log(tl))); 123 | static constexpr uint range_shift = 32 - ct_log(tl); 124 | 125 | double beta; 126 | size_type ll_size[tl]; 127 | size_type ll_elem[tl]; 128 | size_type ll_thresh[tl]; 129 | // double ll_factor[tl]; 130 | std::unique_ptr ll_tab[tl]; 131 | 132 | using base_type::make_citerator; 133 | using base_type::make_iterator; 134 | 135 | public: 136 | // Specialized Funcitions (to keep per table counts) *********************** 137 | 138 | inline insert_return_type insert(const key_type k, const mapped_type d) 139 | { 140 | return insert(std::make_pair(k, d)); 141 | } 142 | 143 | inline insert_return_type insert(const std::pair t) 144 | { 145 | auto hash = hasher(t.first); 146 | size_type ttl = ext::tab(hash, 0); 147 | 148 | auto result = base_type::insert(t); 149 | if (result.second) 150 | { 151 | auto currsize = ++ll_elem[ttl]; 152 | if (currsize > ll_thresh[ttl]) grow_tab(ttl); 153 | } 154 | return result; 155 | } 156 | 157 | size_type erase(const key_type k) 158 | { 159 | auto hash = hasher(k); 160 | size_type ttl = ext::tab(hash, 0); 161 | size_type nk = base_type::erase(k); 162 | ll_elem[ttl] -= nk; 163 | return nk; 164 | } 165 | 166 | 167 | 168 | std::pair getTable(size_type i) 169 | { 170 | return (i < tl) ? std::make_pair(ll_size[i], ll_tab[i].get()) 171 | : std::make_pair(0, nullptr); 172 | } 173 | 174 | iterator begin() 175 | { 176 | auto temp = make_iterator(&ll_tab[0][0].elements[0]); 177 | if (!temp->first) temp++; 178 | return temp; 179 | } 180 | 181 | const_iterator cbegin() const 182 | { 183 | auto temp = make_citerator(&ll_tab[0][0].elements[0]); 184 | if (!temp->first) temp++; 185 | return temp; 186 | } 187 | 188 | private: 189 | // Functions for finding buckets ******************************************* 190 | 191 | inline size_type fastrange(size_type cap, size_type h) const 192 | { 193 | return (cap * h) >> range_shift; 194 | } 195 | 196 | inline void get_buckets(hashed_type h, bucket_type** mem) const 197 | { 198 | for (size_type i = 0; i < nh; ++i) mem[i] = get_bucket(h, i); 199 | } 200 | 201 | inline bucket_type* get_bucket(hashed_type h, size_type i) const 202 | { 203 | size_type tab = ext::tab(h, 0); 204 | // return &(ll_tab[tab][ext::loc(h,i)*ll_factor[tab]]); 205 | return &(ll_tab[tab][fastrange(ll_size[tab], ext::loc(h, i))]); 206 | } 207 | 208 | 209 | 210 | // Size changes (GROWING) ************************************************** 211 | void grow() 212 | { 213 | // necessary for base class not implemented since growing is triggered 214 | // individually for each subtable in this specialization 215 | } 216 | 217 | inline void grow_tab(size_type tab) 218 | { 219 | // otm::out() << "growing in table" << tab << std::endl; 220 | // TODO make this prettier 221 | size_type nsize = std::floor(double(ll_elem[tab]) * alpha / double(bs)); 222 | nsize = std::max(nsize, ll_size[tab] + 1); 223 | capacity += (nsize - ll_size[tab]) * bs; 224 | // double nfactor = double(nsize) / fac_div; 225 | size_type nthresh = ll_elem[tab] * beta; 226 | 227 | std::vector grow_buffer; 228 | 229 | auto ntable = std::make_unique(nsize); 230 | migrate(tab, ntable, nsize, grow_buffer); 231 | 232 | ll_tab[tab] = std::move(ntable); 233 | ll_size[tab] = nsize; 234 | // ll_factor[tab] = nfactor; 235 | ll_thresh[tab] = nthresh; 236 | finalize_grow(grow_buffer); 237 | } 238 | 239 | inline void migrate(size_type tab, std::unique_ptr& target, 240 | size_type nsize, std::vector& grow_buffer) 241 | { 242 | size_type csize = ll_size[tab]; 243 | // double cfactor = ll_factor[tab]; 244 | for (size_type i = 0; i < csize; ++i) 245 | { 246 | bucket_type& curr = ll_tab[tab][i]; 247 | 248 | for (size_type j = 0; j < bs; ++j) 249 | { 250 | auto e = curr.elements[j]; 251 | if (!e.first) break; 252 | auto hash = hasher(e.first); 253 | 254 | for (size_type ti = 0; ti < nh; ++ti) 255 | { 256 | if (i == fastrange(csize, ext::loc(hash, ti))) 257 | { 258 | if (!target[fastrange(nsize, ext::loc(hash, ti))] 259 | .insert(e)) 260 | grow_buffer.push_back(e); 261 | break; 262 | } 263 | } 264 | } 265 | } 266 | } 267 | 268 | inline void finalize_grow(std::vector& grow_buffer) 269 | { 270 | for (auto& e : grow_buffer) { base_type::insert(e); } 271 | } 272 | }; 273 | 274 | 275 | 276 | // Traits class defining types ************************************************* 277 | 278 | template 279 | class cuckoo_traits > 280 | { 281 | public: 282 | using specialized_type = cuckoo_independent_2lvl; 283 | using base_type = cuckoo_base; 284 | using config_type = Conf; 285 | 286 | using key_type = K; 287 | using mapped_type = D; 288 | 289 | static constexpr size_t bs = Conf::bs; 290 | static constexpr size_t tl = Conf::tl; 291 | static constexpr size_t nh = Conf::nh; 292 | static_assert(!config_type::fix_errors, 293 | "2lvl independent table does not support fix_errors!"); 294 | static constexpr bool fix_errors = false; 295 | 296 | using hasher_type = hasher; 297 | using bucket_type = bucket; 298 | }; 299 | 300 | 301 | 302 | // Iterator increment ********************************************************** 303 | 304 | template 305 | class iterator_incr > 306 | { 307 | private: 308 | using ipointer = std::pair*; 309 | static constexpr size_t tl = Conf::tl; 310 | static constexpr size_t bs = Conf::bs; 311 | 312 | public: 313 | using table_type = cuckoo_independent_2lvl; 314 | 315 | iterator_incr(const table_type& table_) 316 | : table(table_), end_tab(nullptr), tab(tl + 1) 317 | { 318 | } 319 | iterator_incr(const iterator_incr&) = default; 320 | iterator_incr& operator=(const iterator_incr&) = default; 321 | 322 | ipointer next(ipointer cur) 323 | { 324 | if (tab > tl) initialize_tab(cur); 325 | 326 | auto temp = cur + 1; 327 | if (temp > end_tab) 328 | { 329 | temp = overflow_tab(); 330 | if (!temp) return nullptr; 331 | } 332 | 333 | while (!temp->first) 334 | { 335 | if (++temp > end_tab) return overflow_tab(); 336 | } 337 | return temp; 338 | } 339 | 340 | private: 341 | const table_type& table; 342 | ipointer end_tab; 343 | size_t tab; 344 | 345 | ipointer overflow_tab() 346 | { 347 | if (++tab >= tl) return nullptr; 348 | size_t size = table.ll_size[tab] - 1; 349 | end_tab = reinterpret_cast( 350 | &table.ll_tab[tab][size].elements[bs - 1]); 351 | return reinterpret_cast(&table.ll_tab[tab][0].elements[0]); 352 | } 353 | 354 | void initialize_tab(ipointer ptr) 355 | { 356 | for (size_t i = 0; i < tl; ++i) 357 | { 358 | ipointer tab_b_ptr = 359 | reinterpret_cast(&table.ll_tab[i][0].elements[0]); 360 | ipointer tab_e_ptr = tab_b_ptr + table.ll_size[i] * bs; 361 | 362 | if (tab_b_ptr <= ptr && ptr < tab_e_ptr) 363 | { 364 | tab = i; 365 | end_tab = tab_e_ptr; 366 | return; 367 | } 368 | } 369 | } 370 | }; 371 | 372 | } // namespace dysect 373 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_bfs0.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_bfs0.h 5 | * 6 | * dis_bfs0 implements the bfs displacement strategy. 7 | * 8 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 9 | * 10 | * Copyright (C) 2017 Tobias Maier 11 | * 12 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 13 | ******************************************************************************/ 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace dysect 20 | { 21 | namespace cuckoo_displacement 22 | { 23 | 24 | template class dis_bfs0 25 | { 26 | private: 27 | using key_type = typename Parent::key_type; 28 | using mapped_type = typename Parent::mapped_type; 29 | using value_intern = std::pair; 30 | using parent_type = typename Parent::This_t; 31 | using hashed_type = typename Parent::hashed_type; 32 | using bucket_type = typename Parent::bucket_type; 33 | 34 | using bfs_queue = std::vector >; 35 | 36 | Parent& tab; 37 | const size_t steps; 38 | static constexpr size_t nh = parent_type::nh; 39 | 40 | public: 41 | dis_bfs0(Parent& parent, size_t steps = 256, size_t = 0) 42 | : tab(parent), steps(steps + 1) 43 | { 44 | /* parameter is for symmetry with "rwalk" therefore unused*/ 45 | } 46 | 47 | dis_bfs0(Parent& parent, dis_bfs0&& rhs) : tab(parent), steps(rhs.steps) {} 48 | 49 | inline std::pair 50 | insert(std::pair t, hashed_type hash) 51 | { 52 | bfs_queue bq; 53 | 54 | bucket_type* b[nh]; 55 | tab.get_buckets(hash, b); 56 | 57 | for (size_t i = 0; i < nh; ++i) 58 | { 59 | bq.push_back( 60 | std::tuple(t.first, -1, b[i])); 61 | } 62 | 63 | for (size_t i = 0; i < steps; ++i) 64 | { 65 | if (expand(bq, i)) 66 | { 67 | value_intern* pos = rollBackDisplacements(t, bq); 68 | return std::make_pair((pos) ? bq.size() - nh : -1, pos); 69 | } 70 | } 71 | 72 | return std::make_pair(-1, nullptr); 73 | } 74 | 75 | private: 76 | inline bool expand(bfs_queue& q, size_t index) 77 | { 78 | bucket_type* b = std::get<2>(q[index]); 79 | 80 | for (size_t i = 0; i < tab.bs && q.size() < steps; ++i) 81 | { 82 | key_type k = b->get(i).first; 83 | 84 | auto hash = tab.hasher(k); 85 | 86 | bucket_type* ptr[nh]; 87 | tab.get_buckets(hash, ptr); 88 | for (size_t ti = 0; ti < nh; ++ti) 89 | { 90 | if (ptr[ti] != b) // POTENTIAL BUG!!! continous bucket problem 91 | { 92 | // if (overwatch) std::cout << i << " " << ti << std::endl; 93 | q.emplace_back(k, index, ptr[ti]); 94 | if (ptr[ti]->space()) return true; 95 | } 96 | } 97 | } 98 | return false; 99 | } 100 | 101 | inline value_intern* 102 | rollBackDisplacements(std::pair t, bfs_queue& bq) 103 | { 104 | key_type k1; 105 | int prev1; 106 | bucket_type* b1; 107 | std::tie(k1, prev1, b1) = bq[bq.size() - 1]; 108 | 109 | key_type k2; 110 | int prev2; 111 | bucket_type* b2; 112 | while (prev1 >= 0) 113 | { 114 | std::tie(k2, prev2, b2) = bq[prev1]; 115 | 116 | auto pop = b2->pop(k1); 117 | 118 | b1->insert(k1, pop.second); 119 | 120 | k1 = k2; 121 | prev1 = prev2; 122 | b1 = b2; 123 | } 124 | 125 | return b1->insert_ptr(t); 126 | } 127 | }; 128 | 129 | } // namespace cuckoo_displacement 130 | } // namespace dysect 131 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_bfs1.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_bfs1.h 5 | * 6 | * dis_bfs1 implements the bfs displacement strategy. This variant 7 | * is necessary for the overlapping buckets implementation. 8 | * 9 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 10 | * 11 | * Copyright (C) 2017 Tobias Maier 12 | * 13 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 14 | ******************************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | namespace dysect 21 | { 22 | namespace cuckoo_displacement 23 | { 24 | 25 | template class dis_bfs1 26 | { 27 | private: 28 | using key_type = typename Parent::key_type; 29 | using mapped_type = typename Parent::mapped_type; 30 | using value_intern = std::pair; 31 | using parent_type = typename Parent::this_type; 32 | using hashed_type = typename Parent::hashed_type; 33 | using bucket_type = typename Parent::bucket_type; 34 | 35 | 36 | using bfs_item = std::tuple; 37 | using bfs_queue = std::vector; 38 | 39 | Parent& tab; 40 | const size_t steps; 41 | static constexpr size_t nh = parent_type::nh; 42 | 43 | public: 44 | dis_bfs1(Parent& parent, size_t steps = 256, size_t = 0) 45 | : tab(parent), steps(steps + 1) 46 | { /* parameter is for symmetry with "rwalk" therefore unused*/ 47 | } 48 | 49 | dis_bfs1(Parent& parent, dis_bfs1&& rhs) : tab(parent), steps(rhs.steps) {} 50 | 51 | inline std::pair 52 | insert(value_intern t, hashed_type hash) 53 | { 54 | bfs_queue bq; 55 | bucket_type* b[nh]; 56 | 57 | tab.get_buckets(hash, b); 58 | 59 | value_intern t_copy = t; 60 | 61 | for (size_t i = 0; i < nh; ++i) 62 | { 63 | bq.push_back(bfs_item(&t_copy, -1, b[i])); 64 | } 65 | 66 | for (size_t i = 0; i < steps; ++i) 67 | { 68 | if (expand(bq, i)) 69 | { 70 | value_intern* pos = rollBackDisplacements(bq); 71 | return std::make_pair((pos) ? bq.size() - nh : -1, pos); 72 | } 73 | } 74 | 75 | return std::make_pair(-1, nullptr); 76 | } 77 | 78 | private: 79 | inline bool expand(bfs_queue& q, size_t index) 80 | { 81 | bucket_type* b = std::get<2>(q[index]); 82 | 83 | for (size_t i = 0; i < tab.bs && q.size() < steps; ++i) 84 | { 85 | value_intern* current = &(b->elements[i]); 86 | key_type k = current->first; 87 | 88 | auto hash = tab.hasher(k); 89 | 90 | bucket_type* ptr[nh]; 91 | tab.get_buckets(hash, ptr); 92 | for (size_t ti = 0; ti < nh; ++ti) 93 | { 94 | if (ptr[ti] != b) // POTENTIAL BUG!!! continous bucket problem 95 | { 96 | // if (overwatch) std::cout << i << " " << ti << std::endl; 97 | q.emplace_back(current, index, ptr[ti]); 98 | if (ptr[ti]->space()) return true; 99 | } 100 | } 101 | } 102 | return false; 103 | } 104 | 105 | inline value_intern* rollBackDisplacements(bfs_queue& bq) 106 | { 107 | value_intern* k1; 108 | int prev1; 109 | bucket_type* b1; 110 | std::tie(k1, prev1, b1) = bq[bq.size() - 1]; 111 | 112 | value_intern* t = b1->probe_ptr(key_type()).second; 113 | (*t) = *k1; 114 | 115 | value_intern* k2; 116 | int prev2; 117 | bucket_type* b2; 118 | 119 | value_intern* k0 = nullptr; 120 | while (prev1 >= 0) 121 | { 122 | std::tie(k2, prev2, b2) = bq[prev1]; 123 | (*k1) = *k2; 124 | 125 | k0 = k1; 126 | k1 = k2; 127 | prev1 = prev2; 128 | b1 = b2; 129 | } 130 | 131 | return k0; 132 | } 133 | }; 134 | 135 | } // namespace cuckoo_displacement 136 | } // namespace dysect 137 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_deterministic_walk.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_deterministic_walk.hpp 5 | * 6 | * dis_deterministic_walk implements an experimental displacement technique 7 | *similar to random walk, that explores the search space deterministically. 8 | * 9 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 10 | * 11 | * Copyright (C) 2017 Tobias Maier 12 | * 13 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 14 | ******************************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | template 24 | class dis_deterministic_walk_optimistic 25 | { 26 | private: 27 | using key_type = typename Parent::key_type; 28 | using mapped_type = typename Parent::mapped_type; 29 | using value_intern = std::pair; 30 | 31 | using parent_type = Parent; 32 | using hashed_type = typename Parent::hashed_type; 33 | using bucket_type = typename Parent::bucket_type; 34 | 35 | static constexpr size_t nh = Parent::nh; 36 | 37 | 38 | parent_type& tab; 39 | const size_t steps; 40 | 41 | public: 42 | dis_deterministic_walk_optimistic(parent_type& parent, 43 | size_t steps = 256, 44 | size_t = 0) 45 | : tab(parent), steps(steps) 46 | { 47 | } 48 | 49 | dis_deterministic_walk_optimistic(parent_type& parent, 50 | dis_deterministic_walk_optimistic&& rhs) 51 | : tab(parent), steps(rhs.steps) 52 | { 53 | } 54 | 55 | inline std::pair 56 | insert(value_intern t, hashed_type hash) 57 | { 58 | utils_tm::debug_tm::if_debug( 59 | "deterministic walk is called on a table with bs>1", tab.bs > 1); 60 | 61 | auto tp = t; 62 | bucket_type* tb = tab.get_bucket(hash, 0); // find first bucket 63 | bucket_type* buckets[nh]; 64 | 65 | tp = tb->replace(0, tp); // all buckets are filled thus tp is not dummy 66 | for (size_t i = 0; i < steps; ++i) 67 | { 68 | auto hash = tab.hasher(tp.first); 69 | tab.get_buckets(hash, buckets); // get all tp's buckets 70 | size_t which_bucket = 0; // current bucket of tp + 1 71 | for (size_t j = 0; j < nh - 1; ++j) 72 | // this loop just assumes that tp was in nh-1 if it was nowhere else 73 | { 74 | if (tb == buckets[j]) which_bucket = j + 1; 75 | } 76 | tb = buckets[which_bucket]; 77 | if (tb->space()) 78 | { 79 | tb->insert(tp); 80 | return std::make_pair(i, &(tb->elements[0])); 81 | } 82 | tp = tb->replace(0, tp); 83 | } 84 | return std::make_pair(-1, nullptr); 85 | } 86 | }; 87 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_random_walk_acyclic.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_walk_acyclic.h 5 | * 6 | * dis_random_walk_acyclic 7 | * 8 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 9 | * 10 | * Copyright (C) 2017 Tobias Maier 11 | * 12 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 13 | ******************************************************************************/ 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | namespace dysect 21 | { 22 | namespace cuckoo_displacement 23 | { 24 | 25 | template // think about spezialization if nh = 2 26 | class dis_random_walk_acyclic 27 | { 28 | private: 29 | using key_type = typename Parent::key_type; 30 | using mapped_type = typename Parent::mapped_type; 31 | using value_intern = std::pair; 32 | 33 | using parent_type = Parent; 34 | using hashed_type = typename Parent::hashed_type; 35 | using bucket_type = typename Parent::bucket_type; 36 | 37 | parent_type& tab; 38 | std::mt19937 re; 39 | const size_t steps; 40 | 41 | static constexpr size_t nh = Parent::nh; 42 | 43 | public: 44 | dis_random_walk_acyclic(parent_type& parent, size_t steps = 256, 45 | size_t seed = 30982391937209388ull) 46 | : tab(parent), re(seed), steps(steps) 47 | { 48 | } 49 | 50 | dis_random_walk_acyclic(parent_type& parent, dis_random_walk_acyclic&& rhs) 51 | : tab(parent), re(std::move(rhs.re)), steps(rhs.steps) 52 | { 53 | } 54 | 55 | inline std::pair 56 | insert(std::pair t, hashed_type hash) 57 | { 58 | 59 | std::vector > queue; 60 | std::uniform_int_distribution bin(0, nh - 1); 61 | std::uniform_int_distribution bsd(0, tab.bs - 1); 62 | std::uniform_int_distribution hfd(0, nh - 2); 63 | 64 | auto tp = t; 65 | bucket_type* tb = tab.get_bucket(hash, bin(re)); 66 | 67 | queue.emplace_back(tp, tb); 68 | 69 | size_t i = 0; 70 | for (; !(tb->space()) && i < steps; ++i) 71 | { 72 | auto r = bsd(re); 73 | tp = tb->get(r); 74 | auto hash = tab.hasher(tp.first); 75 | auto tbd = tab.get_bucket(hash, hfd(re)); 76 | if (tbd != tb) 77 | tb = tbd; 78 | else 79 | tb = tab.get_bucket(hash, nh - 1); 80 | 81 | queue.emplace_back(tp, tb); 82 | 83 | // Explicit Cycle Detection (they are popped from the displacement 84 | // queue) 85 | for (size_t j = 0; j < queue.size() - 1; ++j) 86 | { 87 | if (queue[j].second == tb) 88 | { 89 | while (queue.size() > j + 1) queue.pop_back(); 90 | break; 91 | } 92 | } 93 | } 94 | 95 | if (!tb->space()) { return std::make_pair(-1, nullptr); } 96 | 97 | for (size_t i = queue.size() - 1; i > 0; --i) 98 | { 99 | std::tie(tp, tb) = queue[i]; 100 | if (!queue[i - 1].second->remove(tp.first) || 101 | !tb->insert(tp.first, tp.second)) 102 | { 103 | return std::make_pair(-1, nullptr); 104 | } 105 | } 106 | 107 | value_intern* pos = queue[0].second->insert_ptr(t); 108 | 109 | return std::make_pair((pos) ? i : -1, pos); 110 | } 111 | }; 112 | } // namespace cuckoo_displacement 113 | } // namespace dysect 114 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_random_walk_cyclic.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_random_walk_cyclic.h 5 | * 6 | * dis_random_walk_cyclic implements a random walk displacement 7 | * technique, which precomputes the whole displacement path. It does 8 | * not remove possible cycles, instead cycles get handled during the 9 | * actual displacement. 10 | * 11 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 12 | * 13 | * Copyright (C) 2017 Tobias Maier 14 | * 15 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 16 | ******************************************************************************/ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace dysect 24 | { 25 | namespace cuckoo_displacement 26 | { 27 | 28 | template class dis_random_walk_cyclic 29 | { 30 | private: 31 | using key_type = typename Parent::key_type; 32 | using mapped_type = typename Parent::mapped_type; 33 | using value_intern = std::pair; 34 | 35 | using parent_type = Parent; 36 | using hashed_type = typename Parent::hashed_type; 37 | using bucket_type = typename Parent::bucket_type; 38 | 39 | static constexpr size_t nh = Parent::nh; 40 | 41 | parent_type& tab; 42 | std::mt19937 re; 43 | const size_t steps; 44 | 45 | public: 46 | dis_random_walk_cyclic(parent_type& parent, size_t steps = 256, 47 | size_t seed = 30982391937209388ull) 48 | : tab(parent), re(seed), steps(steps) 49 | { 50 | } 51 | 52 | dis_random_walk_cyclic(parent_type& parent, dis_random_walk_cyclic&& rhs) 53 | : tab(parent), re(std::move(rhs.re)), steps(rhs.steps) 54 | { 55 | } 56 | 57 | inline std::pair 58 | insert(value_intern t, hashed_type hash) 59 | { 60 | std::vector > queue; 61 | std::uniform_int_distribution bin(0, nh - 1); 62 | std::uniform_int_distribution bsd(0, tab.bs - 1); 63 | std::uniform_int_distribution hfd(0, nh - 2); 64 | 65 | auto tp = t; 66 | bucket_type* tb = tab.get_bucket(hash, bin(re)); 67 | value_intern* pos = nullptr; 68 | 69 | queue.emplace_back(tp, tb); 70 | for (size_t i = 0; !tb->space() && i < steps; ++i) 71 | { 72 | auto r = bsd(re); 73 | if (tp.first == t.first) pos = &(tb->elements[r]); 74 | tp = tb->replace(r, tp); 75 | 76 | auto hash = tab.hasher(tp.first); 77 | auto tbd = tab.get_bucket(hash, hfd(re)); 78 | if (tbd != tb) 79 | tb = tbd; 80 | else 81 | tb = tab.get_bucket(hash, nh - 1); 82 | 83 | queue.emplace_back(tp, tb); 84 | } 85 | 86 | if (tb->insert(tp)) { return std::make_pair(queue.size() - 1, pos); } 87 | 88 | std::pair ttp; 89 | std::tie(ttp, tb) = queue[queue.size() - 1]; 90 | for (size_t i = queue.size() - 2; i >= 1; --i) 91 | { 92 | std::tie(tp, tb) = queue[i]; 93 | if (!(tb->remove(tp.first))) { std::cout << "f1" << std::endl; } 94 | if (!(tb->insert(ttp.first, ttp.second))) 95 | { 96 | std::cout << "f2" << std::endl; 97 | }; 98 | ttp = tp; 99 | } 100 | 101 | return std::make_pair(-1, nullptr); 102 | } 103 | }; 104 | 105 | } // namespace cuckoo_displacement 106 | } // namespace dysect 107 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_random_walk_optimistic.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_random_walk_optimistic.h 5 | * 6 | * dis_random_walk_optimistic implements a random walk displacement 7 | * technique the displacement path is not stored, therefore, one 8 | * element will get lost during unsuccessful displacements. 9 | * 10 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 11 | * 12 | * Copyright (C) 2017 Tobias Maier 13 | * 14 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 15 | ******************************************************************************/ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | template class dis_random_walk_optimistic 23 | { 24 | private: 25 | using key_type = typename Parent::key_type; 26 | using mapped_type = typename Parent::mapped_type; 27 | using value_intern = std::pair; 28 | 29 | using parent_type = Parent; 30 | using hashed_type = typename Parent::hashed_type; 31 | using bucket_type = typename Parent::bucket_type; 32 | 33 | static constexpr size_t nh = Parent::nh; 34 | 35 | parent_type& tab; 36 | std::mt19937 re; 37 | const size_t steps; 38 | 39 | public: 40 | dis_random_walk_optimistic(parent_type& parent, size_t steps = 256, 41 | size_t seed = 30982391937209388ull) 42 | : tab(parent), re(seed), steps(steps) 43 | { 44 | } 45 | 46 | dis_random_walk_optimistic(parent_type& parent, 47 | dis_random_walk_optimistic&& rhs) 48 | : tab(parent), re(std::move(rhs.re)), steps(rhs.steps) 49 | { 50 | } 51 | 52 | inline std::pair 53 | insert(value_intern t, hashed_type hash) 54 | { 55 | std::uniform_int_distribution bin(0, nh - 1); 56 | std::uniform_int_distribution bsd(0, tab.bs - 1); 57 | // std::uniform_int_distribution hfd(0,nh-2); 58 | 59 | auto tp = t; 60 | bucket_type* tb = tab.get_bucket(hash, bin(re)); 61 | value_intern* pos = nullptr; 62 | 63 | auto r = bsd(re); 64 | tp = tb->replace(r, tp); 65 | pos = &(tb->elements[r]); 66 | 67 | for (size_t i = 0; i < steps; ++i) 68 | { 69 | auto hash = tab.hasher(tp.first); 70 | // auto tbd = tab.get_bucket(hash, hfd(re)); 71 | // if (tbd != tb) tb = tbd; 72 | // else tb = tab.get_bucket(hash, nh-1); 73 | tb = tab.get_bucket(hash, bin(re)); 74 | 75 | if (tb->space()) 76 | { 77 | tb->insert(tp); 78 | return std::make_pair(i, pos); 79 | } 80 | 81 | r = bsd(re); 82 | if (tp.first == t.first) pos = &(tb->elements[r]); 83 | tp = tb->replace(r, tp); 84 | } 85 | 86 | return std::make_pair(-1, nullptr); 87 | } 88 | }; 89 | -------------------------------------------------------------------------------- /include/displacement_strategies/dis_trivial.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/displacement_strategies/dis_trivial.h 5 | * 6 | * dstrat_trivial is the trivialial displacement technique (no displacements 7 | * at all). The hash table will be similar to k-choice bucket hashing. 8 | * 9 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 10 | * 11 | * Copyright (C) 2017 Tobias Maier 12 | * 13 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 14 | ******************************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | namespace dysect 21 | { 22 | namespace cuckoo_displacement 23 | { 24 | 25 | template class dis_trivial 26 | { 27 | private: 28 | using key_type = typename Parent::key_type; 29 | using mapped_type = typename Parent::mapped_type; 30 | using value_intern = std::pair; 31 | 32 | using hashed_type = typename Parent::hashed_type; 33 | 34 | public: 35 | dis_trivial(Parent&, size_t, size_t) {} 36 | dis_trivial(Parent&, dis_trivial&&) {} 37 | 38 | inline std::pair insert(value_intern, hashed_type) 39 | { 40 | return std::make_pair(-1, nullptr); 41 | } 42 | }; 43 | 44 | } // namespace cuckoo_displacement 45 | } // namespace dysect 46 | -------------------------------------------------------------------------------- /include/displacement_strategies/main_strategies.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "dis_bfs1.hpp" 4 | #include "dis_random_walk_optimistic.hpp" 5 | #include "dis_trivial.hpp" 6 | 7 | namespace dysect 8 | { 9 | namespace cuckoo_displacement 10 | { 11 | 12 | template using trivial = dis_trivial; 13 | template using bfs = dis_bfs1; 14 | template using random_walk = dis_random_walk_optimistic; 15 | 16 | } // namespace cuckoo_displacement 17 | } // namespace dysect 18 | -------------------------------------------------------------------------------- /include/hasher.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/hasher.h 5 | * 6 | * the hasher class is used, to evaluate all hash functions for any 7 | * given key. From the result it can extract the correct amount of 8 | * hashed values and split them into appropriate sub parts (subtable 9 | * number + in-table offset). 10 | * 11 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 12 | * 13 | * Copyright (C) 2017 Tobias Maier 14 | * 15 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 16 | ******************************************************************************/ 17 | 18 | #include 19 | #include 20 | 21 | namespace dysect 22 | { 23 | 24 | static constexpr size_t ct_log(size_t k) 25 | { 26 | return (k - 1) ? 1 + ct_log(k >> 1) : 0; 27 | } 28 | 29 | template struct hash_value_splitter; 30 | 31 | template 32 | class hash_value_extractor; 33 | 34 | 35 | 36 | 37 | /* MAIN CLASS *****************************************************************/ 38 | template 40 | class hasher 41 | { 42 | private: 43 | using this_type = hasher; 44 | using hash_function_type = HFct; 45 | static constexpr size_t tab_w = TAB_WIDTH; 46 | static constexpr size_t pair_w = (DPAIR) ? 32 : 64; 47 | static constexpr size_t loc_w = pair_w - tab_w; 48 | static constexpr size_t n_hpair = (LCOMB) ? 2 : NH; 49 | static constexpr size_t n_hval = NH; 50 | static constexpr size_t n_hfct = 51 | (DPAIR) ? (n_hpair >> 1) + (1 & n_hpair) : n_hpair; 52 | static_assert(tab_w < pair_w, 53 | "TAB_WIDTH has to be smaller than PAIR_WIDTH."); 54 | 55 | using splitter_type = hash_value_splitter; 56 | 57 | public: 58 | union hashed_type { 59 | uint64_t hash[n_hfct]; 60 | splitter_type split[n_hfct]; 61 | }; 62 | 63 | static_assert(sizeof(hashed_type) == n_hfct * 8, 64 | "Hashhash_value_splitterter Bigger Than Expected"); 65 | 66 | using extractor_type = 67 | hash_value_extractor; 68 | 69 | /* hasher (itself) ********************************************************/ 70 | private: 71 | hash_function_type fct[n_hfct]; 72 | 73 | public: 74 | hasher() 75 | { 76 | for (size_t i = 0; i < n_hfct; ++i) 77 | { 78 | fct[i] = hash_function_type(2345745572344267838ull + 79 | i * 8768656543548765336ull); 80 | } 81 | } 82 | 83 | hashed_type operator()(Key k) const 84 | { 85 | hashed_type result; 86 | for (size_t i = 0; i < n_hfct; ++i) { result.hash[i] = fct[i](k); } 87 | return result; 88 | } 89 | }; 90 | 91 | 92 | 93 | 94 | /* Hash hash_value_splitterter Specializations 95 | * **********************************************/ 96 | template struct hash_value_splitter 97 | { 98 | uint64_t tab0 : t_w; 99 | uint64_t loc0 : 32 - t_w; 100 | uint64_t tab1 : t_w; 101 | uint64_t loc1 : 32 - t_w; 102 | }; 103 | 104 | template struct hash_value_splitter 105 | { 106 | uint64_t tab : t_w; 107 | uint64_t loc : 64 - t_w; 108 | }; 109 | 110 | template <> struct hash_value_splitter<0, true> 111 | { 112 | static constexpr uint64_t tab0 = 0; 113 | static constexpr uint64_t tab1 = 0; 114 | uint64_t loc0 : 32; 115 | uint64_t loc1 : 32; 116 | }; 117 | 118 | template <> struct hash_value_splitter<0, false> 119 | { 120 | static constexpr uint64_t tab = 0; 121 | uint64_t loc : 64; 122 | }; 123 | 124 | /* Extractor Specializations without LCOMB ************************************/ 125 | template 126 | class hash_value_extractor 127 | { 128 | public: 129 | inline static size_t tab(const Hashed& h, size_t i) 130 | { 131 | return h.split[i].tab; 132 | } 133 | inline static size_t loc(const Hashed& h, size_t i) 134 | { 135 | return h.split[i].loc; 136 | } 137 | }; 138 | 139 | template 140 | class hash_value_extractor 141 | { 142 | public: 143 | inline static size_t tab(const Hashed& h, size_t i) 144 | { 145 | return (i & 1) ? h.split[i >> 1].tab1 : h.split[i >> 1].tab0; 146 | } 147 | inline static size_t loc(const Hashed& h, size_t i) 148 | { 149 | return (i & 1) ? h.split[i >> 1].loc1 : h.split[i >> 1].loc0; 150 | } 151 | }; 152 | 153 | /* Extractor Specializations with LCOMB ***************************************/ 154 | template 155 | class hash_value_extractor 156 | { 157 | private: 158 | static_assert((tab_width != 0) && (tab_width != 64), 159 | "Illegal TAB_WIDTH value 0 or 64."); 160 | static constexpr size_t tab_mask = (1ull << tab_width) - 1; 161 | static constexpr size_t loc_mask = (1ull << (64 - tab_width)) - 1; 162 | 163 | public: 164 | inline static size_t tab(const Hashed& h, size_t i) 165 | { 166 | return (h.split[0].tab + i * (h.split[1].tab | 1)) & tab_mask; 167 | } 168 | inline static size_t loc(const Hashed& h, size_t i) 169 | { 170 | return (h.split[0].loc + i * (h.split[1].loc | 1)) & tab_mask; 171 | } 172 | }; 173 | 174 | template 175 | class hash_value_extractor 176 | { 177 | private: 178 | static constexpr size_t tab_mask = (1ull << tab_width) - 1; 179 | static constexpr size_t loc_mask = (1ull << (32 - tab_width)) - 1; 180 | 181 | public: 182 | inline static size_t tab(const Hashed& h, size_t i) 183 | { 184 | return (h.split[0].tab0 + i * (h.split[0].tab1 | 1)) & tab_mask; 185 | } 186 | inline static size_t loc(const Hashed& h, size_t i) 187 | { 188 | return (h.split[0].loc0 + i * (h.split[0].loc1 | 1)) & loc_mask; 189 | } 190 | }; 191 | 192 | } // namespace dysect 193 | -------------------------------------------------------------------------------- /include/iterator_base.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/iterator_base.h 5 | * 6 | * iterator_base is a basic iterator implementation, that is 7 | * independent from the used table. Appart from the increment 8 | * operator, which uses a table specific increment function. 9 | * 10 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 11 | * 12 | * Copyright (C) 2017 Tobias Maier 13 | * 14 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 15 | ******************************************************************************/ 16 | 17 | #include 18 | 19 | namespace dysect 20 | { 21 | 22 | template class iterator_base 23 | { 24 | private: 25 | using table_type = typename Increment::table_type; 26 | 27 | using key_type = typename table_type::key_type; 28 | using mapped_type = typename table_type::mapped_type; 29 | using value_intern = std::pair; 30 | using value_table = std::pair; 31 | using cval_intern = typename std::conditional::type; 33 | 34 | public: 35 | using difference_type = std::ptrdiff_t; 36 | using value_type = typename std::conditional::type; 38 | using reference = value_type&; 39 | using pointer = value_type*; 40 | using iterator_category = std::forward_iterator_tag; 41 | using increment_type = Increment; 42 | 43 | template 44 | friend void swap(iterator_base& l, iterator_base& r); 45 | template 46 | friend bool 47 | operator==(const iterator_base& l, const iterator_base& r); 48 | template 49 | friend bool 50 | operator!=(const iterator_base& l, const iterator_base& r); 51 | 52 | 53 | // Constructors ************************************************************ 54 | template 55 | iterator_base(cval_intern* pair_, Args&&... args) 56 | : ptr(reinterpret_cast(pair_)), incr(args...) 57 | { 58 | } 59 | 60 | iterator_base(const iterator_base& rhs) : ptr(rhs.ptr), incr(rhs.incr) {} 61 | iterator_base& operator=(const iterator_base& r) 62 | { 63 | ptr = r.ptr; 64 | incr = r.incr; 65 | return *this; 66 | } 67 | 68 | ~iterator_base() = default; 69 | 70 | 71 | // Basic Iterator Functionality 72 | 73 | iterator_base& operator++(int) 74 | { 75 | ptr = incr.next(ptr); 76 | return *this; 77 | } 78 | reference operator*() const { return *ptr; } 79 | pointer operator->() const { return ptr; } 80 | 81 | bool operator==(const iterator_base& rhs) const { return ptr == rhs.ptr; } 82 | bool operator!=(const iterator_base& rhs) const { return ptr != rhs.ptr; } 83 | 84 | private: 85 | pointer ptr; 86 | increment_type incr; 87 | }; 88 | 89 | } // namespace dysect 90 | -------------------------------------------------------------------------------- /include/prob_base.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * include/prob_base.h 5 | * 6 | * prob_base is similar to cuckoo_base, in that it encapsules 7 | * everything, that all probing based hash tables have in common. 8 | * 9 | * Part of Project DySECT - https://github.com/TooBiased/DySECT.git 10 | * 11 | * Copyright (C) 2017 Tobias Maier 12 | * 13 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 14 | ******************************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "utils/fastrange.hpp" 22 | #include "utils/output.hpp" 23 | 24 | #include "bucket.hpp" 25 | #include "iterator_base.hpp" 26 | 27 | namespace otm = utils_tm::out_tm; 28 | 29 | namespace dysect 30 | { 31 | struct triv_config 32 | { 33 | }; 34 | 35 | template class prob_traits; 36 | template class iterator_incr; 37 | 38 | 39 | 40 | 41 | 42 | template class prob_base 43 | { 44 | private: 45 | using this_type = prob_base; 46 | using specialized_type = typename prob_traits::specialized_type; 47 | using hash_function_type = typename prob_traits::hash_function_type; 48 | 49 | friend specialized_type; 50 | friend iterator_incr; 51 | 52 | public: 53 | using size_type = size_t; 54 | using key_type = typename prob_traits::key_type; 55 | using mapped_type = typename prob_traits::mapped_type; 56 | using iterator = iterator_base >; 57 | using const_iterator = iterator_base, true>; 58 | 59 | private: 60 | using value_intern = std::pair; 61 | 62 | public: 63 | prob_base(size_type cap, double alpha) 64 | : alpha(alpha), beta((alpha + 1.) / 2.), n(0), 65 | capacity((cap) ? cap * alpha : 2048 * alpha), 66 | thresh((cap) ? cap * beta : 2048 * beta) 67 | { 68 | if (cap) table = std::make_unique(capacity); 69 | } 70 | 71 | ~prob_base() = default; 72 | 73 | prob_base(const prob_base&) = delete; 74 | prob_base& operator=(const prob_base&) = delete; 75 | 76 | prob_base(prob_base&& rhs) = default; 77 | prob_base& operator=(prob_base&& rhs) = default; 78 | 79 | private: 80 | /*** members that should become private at some point *********************/ 81 | double alpha; 82 | double beta; 83 | size_type n; 84 | size_type capacity; 85 | size_type thresh; 86 | hash_function_type hasher; 87 | 88 | std::unique_ptr table; 89 | 90 | public: 91 | // Basic Hash Table Functionality ****************************************** 92 | iterator find(const key_type& k); 93 | const_iterator find(const key_type& k) const; 94 | std::pair insert(const key_type& k, const mapped_type& d); 95 | std::pair insert(const value_intern& t); 96 | size_type erase(const key_type& k); 97 | int displacement(const key_type& k) const; 98 | 99 | // Easy use Accessors for std compliance *********************************** 100 | inline iterator begin() 101 | { 102 | auto temp = make_iterator(&table[0]); 103 | if (!temp->first) temp++; 104 | return temp; 105 | } 106 | inline const_iterator begin() const 107 | { 108 | return static_cast(this)->cbegin(); 109 | } 110 | inline const_iterator cbegin() const 111 | { 112 | auto temp = make_citerator(&table[0]); 113 | if (!temp->first) temp++; 114 | return temp; 115 | } 116 | inline iterator end() { return make_iterator(nullptr); } 117 | inline const_iterator end() const 118 | { 119 | return static_cast(this)->cend(); 120 | } 121 | inline const_iterator cend() const { return make_citerator(nullptr); } 122 | 123 | mapped_type& at(const key_type& k); 124 | const mapped_type& at(const key_type& k) const; 125 | mapped_type& operator[](const key_type& k); 126 | size_type count(const key_type& k) const; 127 | 128 | size_type get_capacity() const { return capacity; } 129 | 130 | private: 131 | // Easy iterators ********************************************************** 132 | inline iterator make_iterator(value_intern* pair) const 133 | { 134 | return iterator(pair, *this); 135 | } 136 | inline const_iterator make_citerator(value_intern* pair) const 137 | { 138 | return const_iterator(pair, *this); 139 | } 140 | 141 | // implementation specific functions (static polymorph) ******************** 142 | inline size_type h(key_type k) const 143 | { 144 | return static_cast(this)->index(hasher(k)); 145 | } 146 | 147 | inline void inc_n() 148 | { 149 | if (++n > thresh) static_cast(this)->grow(); 150 | } 151 | inline void dec_n() { --n; } 152 | 153 | // Private helper function ************************************************* 154 | void propagate_remove(size_type origin); 155 | 156 | public: 157 | inline static void print_init_header(otm::output_type& out) 158 | { 159 | out << otm::width(10) << "f_cap"; 160 | } 161 | inline void print_init_data(otm::output_type& out) 162 | { 163 | out << otm::width(10) << capacity; 164 | } 165 | }; 166 | 167 | 168 | 169 | // Implementation of main functionality **************************************** 170 | 171 | template 172 | inline typename prob_base::iterator 173 | prob_base::find(const key_type& k) 174 | { 175 | auto ind = h(k); 176 | 177 | for (size_type i = ind;; ++i) 178 | { 179 | size_type ti = static_cast(this)->mod(i); 180 | auto temp = table[ti]; 181 | 182 | if (temp.first == 0) { break; } 183 | else if (temp.first == k) 184 | { 185 | return make_iterator(&table[ti]); 186 | } 187 | } 188 | return end(); 189 | } 190 | 191 | template 192 | inline typename prob_base::const_iterator 193 | prob_base::find(const key_type& k) const 194 | { 195 | auto ind = h(k); 196 | 197 | for (size_type i = ind;; ++i) 198 | { 199 | size_type ti = static_cast(this)->mod(i); 200 | auto temp = table[ti]; 201 | 202 | if (temp.first == 0) { break; } 203 | else if (temp.first == k) 204 | { 205 | return make_citerator(&table[ti]); 206 | } 207 | } 208 | return cend(); 209 | } 210 | 211 | template 212 | inline std::pair::iterator, bool> 213 | prob_base::insert(const key_type& k, const mapped_type& d) 214 | { 215 | return insert(std::make_pair(k, d)); 216 | } 217 | 218 | template 219 | inline std::pair::iterator, bool> 220 | prob_base::insert(const value_intern& t) 221 | { 222 | auto ind = h(t.first); 223 | 224 | for (size_type i = ind;; ++i) 225 | { 226 | size_type ti = static_cast(this)->mod(i); 227 | auto temp = table[ti]; 228 | if (temp.first == t.first) 229 | { 230 | return std::make_pair(make_iterator(&table[ti]), false); 231 | } 232 | if (temp.first == 0) 233 | { 234 | table[ti] = t; 235 | // hcounter.add(i - ind); 236 | static_cast(this)->inc_n(); 237 | return std::make_pair(make_iterator(&table[ti]), true); 238 | } 239 | } 240 | return std::make_pair(end(), false); 241 | } 242 | 243 | template 244 | inline typename prob_base::size_type 245 | prob_base::erase(const key_type& k) 246 | { 247 | auto ind = h(k); 248 | 249 | for (size_type i = ind;; ++i) 250 | { 251 | size_type ti = static_cast(this)->mod(i); 252 | auto temp = table[ti]; 253 | 254 | if (temp.first == 0) { break; } 255 | else if (temp.first == k) 256 | { 257 | dec_n(); 258 | table[ti] = std::make_pair(0, 0); 259 | static_cast(this)->propagate_remove(ti); 260 | return 1; 261 | } 262 | } 263 | return 0; 264 | } 265 | 266 | template 267 | inline int prob_base::displacement(const key_type& k) const 268 | { 269 | auto ind = h(k); 270 | 271 | for (size_type i = ind;; ++i) 272 | { 273 | size_type ti = static_cast(this)->mod(i); 274 | auto temp = table[ti]; 275 | 276 | if (temp.first == 0) { break; } 277 | else if (temp.first == k) 278 | { 279 | return i - ind; 280 | } 281 | } 282 | return -1; 283 | } 284 | 285 | 286 | // Accessor Implementations **************************************************** 287 | 288 | template 289 | inline typename prob_base::mapped_type& 290 | prob_base::at(const key_type& k) 291 | { 292 | auto a = static_cast(this)->find(k); 293 | if (a == end()) 294 | throw std::out_of_range("cannot find key"); 295 | else 296 | return (*a).second; 297 | } 298 | 299 | template 300 | inline const typename prob_base::mapped_type& 301 | prob_base::at(const key_type& k) const 302 | { 303 | auto a = static_cast(this)->find(k); 304 | if (a == cend()) 305 | throw std::out_of_range("cannot find key"); 306 | else 307 | return (*a).second; 308 | } 309 | 310 | template 311 | inline typename prob_base::mapped_type& 312 | prob_base::operator[](const key_type& k) 313 | { 314 | auto t = static_cast(this)->insert(k, mapped_type()); 315 | return (*t.first).second; 316 | } 317 | 318 | template 319 | inline typename prob_base::size_type 320 | prob_base::count(const key_type& k) const 321 | { 322 | return (static_cast(this)->find(k) != cend()) ? 1 323 | : 0; 324 | } 325 | 326 | 327 | 328 | // Private Help Function ******************************************************* 329 | 330 | template 331 | inline void prob_base::propagate_remove(const size_type origin) 332 | { 333 | size_type tempn = n; 334 | n = 0; 335 | for (size_type i = origin + 1;; ++i) 336 | { 337 | size_type ti = static_cast(this)->mod(i); 338 | auto temp = table[ti]; 339 | 340 | if (temp.first == 0) break; 341 | 342 | table[ti] = std::make_pair(0, 0); 343 | insert(temp); 344 | } 345 | n = tempn; 346 | } 347 | 348 | 349 | 350 | // Iterator increment ********************************************************** 351 | 352 | template class iterator_incr > 353 | { 354 | public: 355 | using table_type = prob_base; 356 | 357 | private: 358 | using key_type = typename table_type::key_type; 359 | using mapped_type = typename table_type::mapped_type; 360 | using ipointer = std::pair*; 361 | 362 | public: 363 | iterator_incr(const table_type& table_) 364 | : end_ptr( 365 | reinterpret_cast(&table_.table[table_.capacity - 1])) 366 | { 367 | } 368 | iterator_incr(const iterator_incr&) = default; 369 | iterator_incr& operator=(const iterator_incr&) = default; 370 | 371 | ipointer next(ipointer cur) 372 | { 373 | while (cur < end_ptr) 374 | { 375 | if ((++cur)->first) return cur; 376 | } 377 | return nullptr; 378 | } 379 | 380 | private: 381 | const ipointer end_ptr; 382 | }; 383 | 384 | } // namespace dysect 385 | -------------------------------------------------------------------------------- /include/prob_multitable_base.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils/output.hpp" 4 | 5 | #include "prob_quadratic.hpp" 6 | #include "prob_robin.hpp" 7 | #include "prob_simple.hpp" 8 | 9 | namespace otm = utils_tm::out_tm; 10 | 11 | 12 | // THESE TABLES USE THE LOWERMOST FEW BITS FOR THEIR TABLE ADDRESSING: 13 | // -> SUBTABLES SHOULD USE LINEAR MAPPING (MOSTLY USE UPPER BITS) 14 | 15 | 16 | 17 | namespace dysect 18 | { 19 | 20 | template class multitable_linear 21 | { 22 | private: 23 | using this_type = multitable_linear; 24 | using subtable_type = prob_linear; 25 | using hash_function_type = HF; 26 | 27 | public: 28 | using key_type = typename subtable_type::key_type; 29 | using mapped_type = typename subtable_type::mapped_type; 30 | 31 | using iterator = typename subtable_type::iterator; 32 | using const_iterator = typename subtable_type::const_iterator; 33 | 34 | static constexpr size_t tl = 256; 35 | static constexpr size_t bits = tl - 1; 36 | 37 | private: 38 | hash_function_type hasher; 39 | subtable_type tables[tl]; 40 | 41 | public: 42 | multitable_linear(size_t cap = 0, double size_constraint = 1.1, 43 | size_t /**/ = 0, size_t /**/ = 0) 44 | { 45 | for (size_t i = 0; i < tl; ++i) 46 | { 47 | tables[i] = subtable_type(cap / tl, size_constraint); 48 | } 49 | } 50 | 51 | inline std::pair insert(key_type k, mapped_type d) 52 | { 53 | return insert(std::make_pair(k, d)); 54 | } 55 | 56 | inline std::pair insert(std::pair t) 57 | { 58 | return tables[get_table(t.first)].insert(t); 59 | } 60 | 61 | inline iterator find(key_type k) { return tables[get_table(k)].find(k); } 62 | 63 | inline const_iterator find(key_type k) const 64 | { 65 | return tables[get_table(k)].find(k); 66 | } 67 | 68 | inline size_t erase(key_type k) { return tables[get_table(k)].erase(k); } 69 | 70 | inline int displacement(key_type k) const 71 | { 72 | return tables[get_table(k)].displacement(k); 73 | } 74 | 75 | 76 | inline iterator begin() { return tables[0].begin(); } 77 | inline iterator end() { return tables[0].end(); } 78 | inline const_iterator begin() const { return tables[0].begin(); } 79 | inline const_iterator end() const { return tables[0].end(); } 80 | inline const_iterator cbegin() const { return tables[0].cbegin(); } 81 | inline const_iterator cend() const { return tables[0].cend(); } 82 | 83 | private: 84 | inline size_t get_table(key_type k) const { return hasher(k) & bits; } 85 | 86 | public: 87 | inline static void print_init_header(otm::output_type& out) 88 | { 89 | out << otm::width(10) << "f_cap"; 90 | } 91 | 92 | inline void print_init_data(otm::output_type& out) 93 | { 94 | size_t cap = 0; 95 | for (size_t i = 0; i < tl; ++i) cap += tables[i].get_capacity(); 96 | 97 | out << otm::width(10) << cap; 98 | } 99 | }; 100 | 101 | template class multitable_robin 102 | { 103 | private: 104 | using this_type = multitable_robin; 105 | using subtable_type = prob_robin; 106 | using hash_function_type = HF; 107 | 108 | public: 109 | using key_type = typename subtable_type::key_type; 110 | using mapped_type = typename subtable_type::mapped_type; 111 | 112 | using iterator = typename subtable_type::iterator; 113 | using const_iterator = typename subtable_type::const_iterator; 114 | 115 | static constexpr size_t tl = 256; 116 | static constexpr size_t bits = tl - 1; 117 | 118 | private: 119 | hash_function_type hasher; 120 | subtable_type tables[tl]; 121 | 122 | public: 123 | multitable_robin(size_t cap = 0, double size_constraint = 1.1, 124 | size_t /**/ = 0, size_t /**/ = 0) 125 | { 126 | for (size_t i = 0; i < tl; ++i) 127 | { 128 | tables[i] = subtable_type(cap / tl, size_constraint); 129 | } 130 | } 131 | 132 | inline std::pair insert(key_type k, mapped_type d) 133 | { 134 | return insert(std::make_pair(k, d)); 135 | } 136 | 137 | inline std::pair insert(std::pair t) 138 | { 139 | return tables[get_table(t.first)].insert(t); 140 | } 141 | 142 | inline iterator find(key_type k) { return tables[get_table(k)].find(k); } 143 | 144 | inline iterator find(key_type k) const 145 | { 146 | return tables[get_table(k)].find(k); 147 | } 148 | 149 | inline size_t erase(key_type k) { return tables[get_table(k)].erase(k); } 150 | 151 | inline int displacement(key_type k) const 152 | { 153 | return tables[get_table(k)].displacement(k); 154 | } 155 | 156 | inline iterator begin() { return tables[0].begin(); } 157 | inline iterator end() { return tables[0].end(); } 158 | inline const_iterator begin() const { return tables[0].begin(); } 159 | inline const_iterator end() const { return tables[0].end(); } 160 | inline const_iterator cbegin() const { return tables[0].cbegin(); } 161 | inline const_iterator cend() const { return tables[0].cend(); } 162 | 163 | private: 164 | inline size_t get_table(key_type k) const { return hasher(k) & bits; } 165 | 166 | public: 167 | inline static void print_init_header(otm::output_type& out) 168 | { 169 | out << otm::width(10) << "f_cap"; 170 | } 171 | 172 | inline void print_init_data(otm::output_type& out) 173 | { 174 | size_t cap = 0; 175 | for (size_t i = 0; i < tl; ++i) cap += tables[i].get_capacity(); 176 | 177 | out << otm::width(10) << cap; 178 | } 179 | }; 180 | 181 | 182 | 183 | 184 | template class multitable_quadratic 185 | { 186 | private: 187 | using this_type = multitable_quadratic; 188 | using subtable_type = prob_quadratic; 189 | using hash_function_type = HF; 190 | 191 | public: 192 | using key_type = typename subtable_type::key_type; 193 | using mapped_type = typename subtable_type::mapped_type; 194 | 195 | using iterator = typename subtable_type::iterator; 196 | using const_iterator = typename subtable_type::const_iterator; 197 | 198 | static constexpr size_t tl = 256; 199 | static constexpr size_t bits = tl - 1; 200 | 201 | private: 202 | hash_function_type hasher; 203 | subtable_type tables[tl]; 204 | 205 | public: 206 | multitable_quadratic(size_t cap = 0, double size_constraint = 1.1, 207 | size_t /**/ = 0, size_t /**/ = 0) 208 | { 209 | for (size_t i = 0; i < tl; ++i) 210 | { 211 | tables[i] = subtable_type(cap / tl, size_constraint); 212 | } 213 | } 214 | 215 | inline std::pair insert(key_type k, mapped_type d) 216 | { 217 | return insert(std::make_pair(k, d)); 218 | } 219 | 220 | inline std::pair insert(std::pair t) 221 | { 222 | return tables[get_table(t.first)].insert(t); 223 | } 224 | 225 | inline iterator find(key_type k) { return tables[get_table(k)].find(k); } 226 | 227 | inline iterator find(key_type k) const 228 | { 229 | return tables[get_table(k)].find(k); 230 | } 231 | 232 | inline size_t erase(key_type k) { return tables[get_table(k)].erase(k); } 233 | 234 | inline int displacement(key_type k) const 235 | { 236 | return tables[get_table(k)].displacement(k); 237 | } 238 | 239 | inline iterator begin() { return tables[0].begin(); } 240 | inline iterator end() { return tables[0].end(); } 241 | inline const_iterator begin() const { return tables[0].begin(); } 242 | inline const_iterator end() const { return tables[0].end(); } 243 | inline const_iterator cbegin() const { return tables[0].cbegin(); } 244 | inline const_iterator cend() const { return tables[0].cend(); } 245 | 246 | private: 247 | inline size_t get_table(key_type k) const { return hasher(k) & bits; } 248 | 249 | public: 250 | inline static void print_init_header(otm::output_type& out) 251 | { 252 | out << otm::width(10) << "f_cap"; 253 | } 254 | 255 | inline void print_init_data(otm::output_type& out) 256 | { 257 | size_t cap = 0; 258 | for (size_t i = 0; i < tl; ++i) cap += tables[i].get_capacity(); 259 | 260 | out << otm::width(10) << cap; 261 | } 262 | }; 263 | 264 | 265 | }; // namespace dysect 266 | -------------------------------------------------------------------------------- /readme.org: -------------------------------------------------------------------------------- 1 | * DySECT 2 | 3 | ** Motivation 4 | In many circumstances hash tables can only be space 5 | efficient when they can also adapt to the number of inserted elements. 6 | Otherwise programmers have to correctly estimate the final table size 7 | to create densely filled hash tables. Guessing conservatively will 8 | create sparser tables and guessing optimistically will create slower 9 | tables (too full) or it might even lose elements. 10 | 11 | There has been a lot of research in the area of space efficient hash 12 | tables. But dynamically growing these space efficient tables has not 13 | received the same attention. The conventional wisdom still seems to 14 | be full table migration (into a newly allocated larger table). This 15 | technique violates space constraints even when growing in small steps 16 | because both the source and the target table are allocated at the same 17 | time. 18 | 19 | ** Contents 20 | 21 | *** DySECT 22 | This is our clean and simple data-structure presented at [ESA 2017] 23 | (http://drops.dagstuhl.de/opus/volltexte/2017/7848/pdf/LIPIcs-ESA-2017-58.pdf). 24 | It is based on a number of subtables that can grow independently. 25 | Elements can be moved between subtatbles -- using techniques similar 26 | to cuckoo hashing -- such that the free space generated from growing 27 | one subtable can be used efficiently. 28 | 29 | *** Common Hashing Techniques 30 | We also offer a number implementations of common hashing techniques 31 | (cuckoo hashing, linear probing, robin hood hashing, and hopscotch 32 | hashing). For every one of them we implement a cache efficient 33 | migration technique that is based on the premise that elements are 34 | basically sorted by their hash value (hash value in [0,1) is used as 35 | scaling factor). 36 | 37 | These cannot truely be space efficient, because during a growth step, 38 | both the new and the old table are allocated. But outside of the 39 | migration they are. This necessitates small growing factors and thus 40 | is relatively inefficient. 41 | 42 | *** Inplace migration (through overallocation) 43 | This technique allows us to increase the size of the hash table in 44 | place. Therefore removing the necessity of having both an old and a 45 | new table during the migration. Instead the whole table is reordered 46 | in place. To make this work, we (ab)use the way virtual memory works. 47 | Instead of allocating a piece of memory that has the same size as the 48 | initial size of the hash table, we allocate a large chunk of memory 49 | (maximum final size). This memory will be purely virtual until it is 50 | accessed. Therefore, only the beginning part (where we build the hash 51 | table) is actually mapped to physical memory. Whenever the table is 52 | grown, we access more of the virtual memory, therefore, resizing the 53 | table in place. 54 | 55 | ** Implementation 56 | The goal for our implementation was to stay as close 57 | to possible to the interface established by ~std::unordered_map~. We 58 | are still working on achieving this goal, so the actual interface 59 | might still go through some minor changes. /Take a look at the/ 60 | ~example~ /folder!/ 61 | 62 | *** Operations 63 | Currently there are three "main" differences in the way our interface 64 | is used compared to ~std::unordered_map~. Those differences are the 65 | constructor parameterization (being able to define the memory-factor), 66 | insert operations (insertions can fail), and the bucket interface. 67 | 68 | **** Initialization/Constructors 69 | Usually hash tables are either initialized with a starting capacity or 70 | with a number of elements. Libraries are usually initialized with the 71 | number of expected elements, and scientific implementations are 72 | usually initiated with their table size (that in some cases needs to 73 | be a power of two). 74 | 75 | All of our tables however, are initiated with a number of elements 76 | /n/, and a memory factor /d/ /(larger than 1)/. The table is then 77 | allocated with /d*n/ slots. The tables grow in a greedy fashion, when 78 | more than /n/ elements are inserted. 79 | 80 | **** (Unsuccessful) Insertions 81 | Given the nature of our experiments, it was necessary, to control when 82 | tables grow, and by how much, they are allowed to grow (to control the 83 | minimum fill degree). Because of this, and because of the nature of 84 | cuckoo (and hopscotch) hash tables there is a possibility for 85 | unsuccessful insertions, e.g., when no displacement-path is found 86 | within a cuckoo hash table, that makes space for the new element. 87 | 88 | To find out if an insertion was successful, look at the returned 89 | boolean result or at the returned iterator (and compare it to 90 | ~table.end()~). 91 | 92 | /Note:/ unsuccessful insertions can even lead to segmentation faults, 93 | if there iterator is dereferenced. This can happen inadvertently when 94 | using the ~table[]~ operator on an uninserted element. 95 | 96 | #+BEGIN_SRC c++ 97 | table_type table{100,1.3}; 98 | 99 | // inserting an element 100 | table_type::iterator it = table.begin(); 101 | bool ins; 102 | std::tie(it, ins) = table.insert(10, 42); 103 | 104 | if (ins) 105 | { 106 | // insertion was successful (key 10 was not in the table, now it is) 107 | table[10] = 43; // is safe 108 | } 109 | else 110 | { 111 | // insertion was not successful this could be because either: 112 | if (it != end()) 113 | { 114 | // the key was already in the table 115 | table[10] = 44; // is also safe 116 | } 117 | else 118 | { 119 | // no place was found for the element 120 | table[10] = 45; // tries to dereference table.end() and thus leads 121 | // to a segmentation fault 122 | } 123 | } 124 | #+END_SRC 125 | 126 | **** Bucket Interface 127 | The bucket interface, for accessing all elements hashed to the same 128 | slot of an ~std::unordered_map~ is widely considered to be a 129 | problematic interface. The problem is, that it suggests any kind of 130 | control over the collisions in a hash table, that is not possible when 131 | using a good hash function. Additionally, it is unclear how to model 132 | buckets in other types of hash tables, e.g., a linear probing hash 133 | table (where buckets are overlapping interleaving), much less in a 134 | cuckoo hash table (where buckets have a different purpose/meaning). 135 | 136 | 137 | *** Variants 138 | #+BEGIN_SRC c++ 139 | // our dysect data-structure 140 | dysect::cuckoo_dysect 141 | 142 | // common hashing techniques 143 | dysect::cuckoo_standard 144 | dysect::prob_linear 145 | dysect::prob_robin 146 | dysect::prob_hopscotch 147 | 148 | // in place variants 149 | dysect::cuckoo_dysect_inplace // uses virtual memory trick for subtable migration 150 | dysect::cuckoo_standard_inplace 151 | dysect::prob_linear_inplace 152 | dysect::prob_robin_inplace 153 | dysect::prob_hopscotch_inplace 154 | 155 | // multitable variants of common techniques 156 | dysect::cuckoo_independent_2lvl 157 | dysect::multitable_linear 158 | dysect::multitable_robin 159 | 160 | // experimental stuff 161 | dysect::cuckoo_deamortized 162 | dysect::cuckoo_overlap 163 | dysect::cuckoo_overlap_inplace 164 | dysect::prob_linear_doubling 165 | #+END_SRC 166 | 167 | ** Tests 168 | Try our test files by: 169 | 170 | #+BEGIN_SRC bash 171 | mkdir build 172 | cd build 173 | cmake .. 174 | make 175 | #+END_SRC 176 | 177 | This will create a multitude of folders with different tests, each 178 | built with many of our hashing techniques. Use ccmake, to change 179 | parameters like the hash function, and virtual memory size (for in place 180 | variants). 181 | -------------------------------------------------------------------------------- /script/aggregation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | steps=8192 4 | 5 | binfolder="bin" 6 | outfolder="out" 7 | infile="input" 8 | 9 | for load in 0.8 0.85 0.9 0.925 0.95 0.9625 0.975 0.98 10 | do 11 | for tab in $binfolder/crawl_multi_* 12 | do 13 | for B in 8 14 | do 15 | for H in 3 16 | do 17 | pname="$(basename -- $tab)" 18 | ./$tab -cap 50000 -steps $steps -load $load -tl 256 -bs $B -nh $H -bfs \ 19 | -in $ifile -out ${outfolder}/${pname} 20 | done 21 | done 22 | done 23 | 24 | # for tab in $binfolder/crawl_hop_* 25 | # do 26 | # pname="$(basename -- $tab)" 27 | # ./$tab -cap 50000 -steps $steps -load $load -ns 64 \ 28 | # -in $ifile -out ${outfolder}/${pname} 29 | # done 30 | 31 | for tab in $binfolder/crawl_triv_* 32 | do 33 | pname="$(basename -- $tab)" 34 | ./$tab -cap 50000 -steps $steps -load $load \ 35 | -in $ifile -out ${outfolder}/${pname} 36 | done 37 | done 38 | -------------------------------------------------------------------------------- /script/all_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | build_folder="../build/" 4 | 5 | ln -s /global_data/maier/common_crawl_17-04/CC-MAIN-20160205193905-00000-ip-10-236-182-209.ec2.internal.warc input 6 | 7 | cp ${build_folder}/time/* bin/ 8 | cp ${build_folder}/del/* bin/ 9 | cp ${build_folder}/eps/* bin/ 10 | cp ${build_folder}/crawl/* bin/ 11 | cp ${build_folder}/displ/* bin/ 12 | cp ${build_folder}/sing/* bin/ 13 | 14 | 15 | ./cache_test.sh 16 | ./growing.sh 17 | ./non_growing.sh 18 | #./aggregation.sh 19 | ./deletion.sh 20 | ./load_bound.sh 21 | -------------------------------------------------------------------------------- /script/cache_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | n=7000000 4 | cap=10000000 5 | steps=8192 6 | 7 | binfolder="bin" 8 | outfolder="out" 9 | 10 | for tab in $binfolder/displ_multi_* 11 | do 12 | for B in 8 13 | do 14 | for H in 3 15 | do 16 | pname="$(basename -- $tab)" 17 | ./$tab -n $n -cap $cap -steps $steps -tl 256 -bs $B -nh $H -bfs \ 18 | -out ${outfolder}/${pname} 19 | done 20 | done 21 | done 22 | 23 | for tab in $binfolder/displ_hop_* 24 | do 25 | pname="$(basename -- $tab)" 26 | ./$tab -n $n -cap $cap -steps $steps -ns 64 \ 27 | -out ${outfolder}/${pname} 28 | done 29 | 30 | for tab in $binfolder/displ_triv_* 31 | do 32 | pname="$(basename -- $tab)" 33 | ./$tab -n $n -cap $cap -steps $steps \ 34 | -out ${outfolder}/${pname} 35 | done 36 | -------------------------------------------------------------------------------- /script/deletetion.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mxpre=1500000 4 | mxn=10000000 5 | steps=8192 6 | 7 | binfolder="bin" 8 | outfolder="out" 9 | 10 | for load in 0.8 0.85 0.9 0.925 0.95 0.9625 0.975 0.98 11 | do 12 | for tab in $binfolder/del_multi_* 13 | do 14 | for B in 8 15 | do 16 | for H in 3 17 | do 18 | pname="$(basename -- $tab)" 19 | ./$tab -n $mxn -pre $mxpre -cap 50000 -steps $steps -load $load -tl 256 -bs $B -nh $H -bfs \ 20 | -out ${outfolder}/${pname} 21 | done 22 | done 23 | done 24 | 25 | # for tab in $binfolder/del_hop_* 26 | # do 27 | # ./$tab -n $mxn -pre $mxpre -cap 50000 -steps $steps -load $load -ns 64 \ 28 | # -out ${outfolder}/${pname} 29 | # done 30 | 31 | for tab in $binfolder/del_triv_* 32 | do 33 | pname="$(basename -- $tab)" 34 | ./$tab -n $mxn -pre $mxpre -cap 50000 -steps $steps -load $load \ 35 | -out ${outfolder}/${pname} 36 | done 37 | done 38 | -------------------------------------------------------------------------------- /script/growing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | pre=8000000 4 | n=10000000 5 | steps=8192 6 | 7 | binfolder="bin" 8 | outfolder="out" 9 | 10 | for load in 0.8 0.85 0.9 0.925 0.95 0.9625 0.975 0.98 11 | do 12 | for tab in $binfolder/time_multi_* 13 | do 14 | for B in 12 8 4 15 | do 16 | for H in 4 3 2 17 | do 18 | pname="$(basename -- $tab)" 19 | ./$tab -n $n -pre $pre -cap 50000 -steps $steps -load $load -tl 256 -bs $B -nh $H -bfs \ 20 | -out ${outfolder}/${pname} 21 | ./$tab -n $n -pre $pre -steps $steps -load $load -tl 256 -bs $B -nh $H -bfs \ 22 | -out ${outfolder}/${pname} 23 | done 24 | done 25 | done 26 | 27 | # for tab in $binfolder/time_hop_* 28 | # do 29 | # pname="$(basename -- $tab)" 30 | # ./$tab -n $n -pre $pre -cap 50000 -steps $steps -load $load -ns 64 \ 31 | # -out ${outfolder}/${pname} 32 | # ./$tab -n $n -pre $pre -steps $steps -load $load -ns 64 \ 33 | # -out ${outfolder}/${pname} 34 | # done 35 | 36 | for tab in $binfolder/time_triv_* 37 | do 38 | pname="$(basename -- $tab)" 39 | ./$tab -n $n -pre $pre -cap 50000 -steps $steps -load $load \ 40 | -out ${outfolder}/${pname} 41 | ./$tab -n $n -pre $pre -steps $steps -load $load \ 42 | -out ${outfolder}/${pname} 43 | done 44 | done 45 | -------------------------------------------------------------------------------- /script/load_bound.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | n=1000000 4 | steps=8192 5 | 6 | binfolder="bin" 7 | outfolder="out" 8 | 9 | for tab in $binfolder/sing_multi_* 10 | do 11 | for B in 12 8 4 12 | do 13 | for H in 4 3 2 14 | do 15 | pname="$(basename -- $tab)" 16 | ./$tab -n $n -cap 50000 -steps $steps -tl 256 -bs $B -nh $H -bfs \ 17 | -out ${outfolder}/${pname} & 18 | done 19 | done 20 | done 21 | 22 | for job in `jobs -p` 23 | do 24 | wait $job 25 | done 26 | -------------------------------------------------------------------------------- /script/non_growing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | steps=8192 4 | 5 | binfolder="bin" 6 | outfolder="out" 7 | 8 | for neps in 16777216 20971520 25165824 29360128 9 | do 10 | for tab in $binfolder/eps_multi_* 11 | do 12 | for B in 12 8 4 13 | do 14 | for H in 4 3 2 15 | do 16 | pname="$(basename -- $tab)" 17 | ./$tab -n $neps -steps $steps -tl 256 -bs $B -nh $H -bfs -load 0.8 \ 18 | -out ${outfolder}/${pname} 19 | done 20 | done 21 | done 22 | 23 | # for tab in $binfolder/eps_hop_* 24 | # do 25 | # pname="$(basename -- $tab)" 26 | # ./$tab -n $neps -steps $steps -ns 64 -load 0.8 \ 27 | # -out ${outfolder}/${pname} 28 | # done 29 | 30 | for tab in $binfolder/eps_triv_* 31 | do 32 | pname="$(basename -- $tab)" 33 | ./$tab -n $neps -steps $steps -load 0.8 \ 34 | -out ${outfolder}/${pname} 35 | done 36 | done 37 | -------------------------------------------------------------------------------- /source/crawl_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | #include "utils/command_line_parser.hpp" 3 | #include "utils/default_hash.hpp" 4 | #include "utils/output.hpp" 5 | #include "utils/pin_thread.hpp" 6 | namespace utm = utils_tm; 7 | namespace otm = utils_tm::out_tm; 8 | 9 | #ifdef MALLOC_COUNT 10 | #include "malloc_count.h" 11 | #endif 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | // TWO PROBLEMS!! cutoff words at the end of the buffer 19 | // what happens if read reads only half a buffer 20 | template 21 | struct test_type 22 | { 23 | using table_type = 24 | HASHTYPE; 25 | static constexpr size_t bsize = 2 * 1024 * 1024; 26 | static constexpr size_t hseed = 13358259232739045019ull; 27 | 28 | int operator()( 29 | size_t it, size_t cap, double alpha, size_t steps, std::string inf_name) 30 | { 31 | otm::out() << otm::width(4) << "# it" << otm::width(8) << "alpha"; 32 | table_type::print_init_header(otm::out()); 33 | otm::out() << otm::width(9) << "cap" << otm::width(9) << "time" 34 | << otm::width(10) << "doubles" << otm::width(9) << "individ" 35 | << otm::width(9) << "err" 36 | #ifdef MALLOC_COUNT 37 | << otm::width(7) << "mem" << otm::width(7) << "mx_mem" 38 | #endif 39 | << std::endl; 40 | 41 | 42 | for (size_t i = 0; i < it; ++i) 43 | { 44 | #ifdef MALLOC_COUNT 45 | malloc_count_reset_peak(); 46 | #endif 47 | 48 | table_type table(cap, alpha, steps); 49 | std::ifstream inf(inf_name, std::ifstream::binary); 50 | char buffer[bsize]; 51 | size_t contained = 0; 52 | size_t individual = 0; 53 | size_t max = 0; 54 | size_t errors = 0; 55 | 56 | auto t0 = std::chrono::high_resolution_clock::now(); 57 | while (inf.tellg() >= 0) 58 | { 59 | inf.read(buffer, bsize); 60 | 61 | char* j0 = buffer; 62 | for (char* j = buffer; j < buffer + inf.gcount(); ++j) 63 | { 64 | if (*j == ' ' || *j == '\n') 65 | { 66 | if (j0 < j) 67 | { 68 | size_t key = XXH64(j0, j - j0, hseed); 69 | auto e = table.insert(key, 1); 70 | if (e.second) 71 | ++individual; 72 | else 73 | { 74 | if (e.first != table.end()) 75 | { 76 | size_t a = ++((*e.first).second); 77 | max = (a < max) ? max : a; 78 | ++contained; 79 | } 80 | else 81 | ++errors; 82 | } 83 | } 84 | j0 = j + 1; 85 | } 86 | } 87 | inf.seekg(j0 - buffer - bsize, std::ios_base::cur); 88 | } 89 | auto t1 = std::chrono::high_resolution_clock::now(); 90 | 91 | double ttest = 92 | std::chrono::duration_cast(t1 - t0) 93 | .count() / 94 | 1000.; 95 | 96 | otm::out() << otm::width(4) << i << otm::width(8) << alpha; 97 | table.print_init_data(otm::out()); 98 | otm::out() << otm::width(9) << cap << otm::width(9) << ttest 99 | << otm::width(10) << contained << otm::width(9) 100 | << individual << otm::width(9) << errors 101 | #ifdef MALLOC_COUNT 102 | << otm::width(7) 103 | << double(malloc_count_current()) / double(8 * 2 * n) 104 | << otm::width(7) 105 | << double(malloc_count_peak()) / double(8 * 2 * n) 106 | #endif 107 | << std::endl; 108 | 109 | inf.close(); 110 | } 111 | 112 | return 0; 113 | } 114 | }; 115 | 116 | int main(int argn, char** argc) 117 | { 118 | utm::pin_to_core(0); 119 | utm::command_line_parser c(argn, argc); 120 | 121 | size_t it = c.int_arg("-it", 5); 122 | size_t cap = c.int_arg("-cap", 1000); 123 | size_t steps = c.int_arg("-steps", 512); 124 | 125 | double alpha = c.double_arg("-alpha", 1.1); 126 | double load = c.double_arg("-load", 2.0); 127 | double eps = c.double_arg("-eps", 1.0 - load); 128 | if (eps > 0.) alpha = 1. / (1. - eps); 129 | 130 | if (c.bool_arg("-out") || c.bool_arg("-file")) 131 | { 132 | std::string name = c.str_arg("-out", ""); 133 | name = c.str_arg("-file", name) + ".crawl"; 134 | otm::out().set_file(name); 135 | } 136 | 137 | if (c.bool_arg("-help") || c.bool_arg("-h")) 138 | { 139 | otm::out() << "This test measures access speed under a real world\n" 140 | << "distribution, i.e., word count the file in -in param.\n" 141 | << "\n" 142 | << otm::color::blue << "1. " << otm::color::reset 143 | << "hash each word, and insert the hash into the table\n" 144 | << otm::color::blue << "2. " << otm::color::reset 145 | << "if it was already present, increment its counter" 146 | << std::endl; 147 | return 0; 148 | } 149 | 150 | const std::string inf = c.str_arg("-in", "/home/maier/WorkEnv/tobias.org"); 151 | 152 | return Chooser::execute(c, it, cap, alpha, 153 | steps, inf); 154 | } 155 | -------------------------------------------------------------------------------- /source/del_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | 3 | #include "utils/command_line_parser.hpp" 4 | #include "utils/default_hash.hpp" 5 | #include "utils/output.hpp" 6 | #include "utils/pin_thread.hpp" 7 | namespace utm = utils_tm; 8 | namespace otm = utils_tm::out_tm; 9 | 10 | #ifdef MALLOC_COUNT 11 | #include "malloc_count.h" 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | template 20 | struct test_type 21 | { 22 | using table_type = 23 | HASHTYPE; 24 | 25 | int operator()( 26 | size_t it, size_t n, size_t win, size_t cap, size_t steps, double alpha) 27 | { 28 | otm::out() << otm::width(4) << "# it" << otm::width(8) << "alpha"; 29 | table_type::print_init_header(otm::out()); 30 | otm::out() << otm::width(9) << "cap" << otm::width(9) << "n_win" 31 | << otm::width(9) << "n_mix" << otm::width(8) << "t_win" 32 | << otm::width(8) << "t_mix" << otm::width(8) << "t_eval" 33 | << otm::width(9) << "op_err" << otm::width(9) << "ev_err" 34 | #ifdef MALLOC_COUNT 35 | << otm::width(7) << "mem" 36 | #endif 37 | << std::endl; 38 | 39 | 40 | constexpr size_t range = (1ull << 63) - 1; 41 | 42 | size_t* keys = new size_t[n + win]; 43 | 44 | std::uniform_int_distribution dis(1, range); 45 | std::mt19937_64 re; 46 | 47 | for (size_t i = 0; i < n + win; ++i) { keys[i] = dis(re); } 48 | 49 | for (size_t i = 0; i < it; ++i) 50 | { 51 | auto in_errors = 0; 52 | auto del_errors = 0; 53 | auto fin_errors = 0; 54 | 55 | table_type table(cap, alpha, steps); 56 | 57 | auto t0 = std::chrono::high_resolution_clock::now(); 58 | for (size_t i = 0; i < win; ++i) 59 | { 60 | if (!table.insert(keys[i], i).second) ++in_errors; 61 | } 62 | auto t1 = std::chrono::high_resolution_clock::now(); 63 | for (size_t i = win; i < win + n; ++i) 64 | { 65 | if (!table.insert(keys[i], i).second) ++in_errors; 66 | if (!table.erase(keys[i - win])) ++del_errors; 67 | } 68 | auto t2 = std::chrono::high_resolution_clock::now(); 69 | for (size_t i = 0; i < n; ++i) 70 | { 71 | auto e = table.find(keys[i]); 72 | if (e != table.end()) ++fin_errors; 73 | ; 74 | } 75 | for (size_t i = n; i < win + n; ++i) 76 | { 77 | auto e = table.find(keys[i]); 78 | if (e == table.end()) ++fin_errors; 79 | } 80 | 81 | for (size_t i = n; i < win + n; ++i) 82 | { 83 | if (!table.erase(keys[i])) ++del_errors; 84 | } 85 | 86 | auto t3 = std::chrono::high_resolution_clock::now(); 87 | 88 | double d_win = 89 | std::chrono::duration_cast(t1 - t0) 90 | .count() / 91 | 1000.; 92 | double d_mix = 93 | std::chrono::duration_cast(t2 - t1) 94 | .count() / 95 | 1000.; 96 | double d_eval = 97 | std::chrono::duration_cast(t3 - t2) 98 | .count() / 99 | 1000.; 100 | 101 | otm::out() << otm::width(4) << i << otm::width(8) << alpha; 102 | table.print_init_data(otm::out()); 103 | otm::out() << otm::width(9) << cap << otm::width(9) << win 104 | << otm::width(9) << n << otm::width(8) << d_win 105 | << otm::width(8) << d_mix << otm::width(8) << d_eval 106 | << otm::width(7) << del_errors + in_errors 107 | << otm::width(7) << fin_errors 108 | #ifdef MALLOC_COUNT 109 | << otm::width(7) 110 | << double(malloc_count_current() - 8 * (n + win)) / 111 | double(8 * 2 * win) 112 | #endif 113 | << std::endl; 114 | } 115 | 116 | delete[] keys; 117 | 118 | return 0; 119 | } 120 | }; 121 | 122 | int main(int argn, char** argc) 123 | { 124 | utm::pin_to_core(0); 125 | utm::command_line_parser c(argn, argc); 126 | 127 | size_t it = c.int_arg("-it", 5); 128 | size_t n = c.int_arg("-n", 1000000); 129 | size_t win = c.int_arg("-pre", n / 2); 130 | size_t cap = c.int_arg("-cap", win); 131 | size_t steps = c.int_arg("-steps", 512); 132 | 133 | double alpha = c.double_arg("-alpha", 1.1); 134 | double load = c.double_arg("-load", 2.0); 135 | double eps = c.double_arg("-eps", 1.0 - load); 136 | if (eps > 0.) alpha = 1. / (1. - eps); 137 | 138 | if (c.bool_arg("-out") || c.bool_arg("-file")) 139 | { 140 | std::string name = c.str_arg("-out", ""); 141 | name = c.str_arg("-file", name) + ".del"; 142 | otm::out().set_file(name); 143 | } 144 | 145 | if (c.bool_arg("-help") || c.bool_arg("-h")) 146 | { 147 | otm::out() 148 | << "This test measures access speed of deletions \n" 149 | << "\n" 150 | << otm::color::blue << "1. " << otm::color::reset 151 | << "insert -pre random elements into a table with cap slots\n" 152 | << otm::color::blue << "2. " << otm::color::reset 153 | << "repeat -n insert+erase cycles\n" 154 | << otm::color::blue << "3. " << otm::color::reset 155 | << "evaluate all previous moves" << std::endl; 156 | return 0; 157 | } 158 | 159 | return Chooser::execute(c, it, n, win, cap, 160 | steps, alpha); 161 | } 162 | -------------------------------------------------------------------------------- /source/displ_test.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | 6 | 7 | #include "utils/command_line_parser.hpp" 8 | #include "utils/default_hash.hpp" 9 | #include "utils/output.hpp" 10 | #include "utils/pin_thread.hpp" 11 | namespace utm = utils_tm; 12 | namespace otm = utils_tm::out_tm; 13 | 14 | #include "selection.hpp" 15 | 16 | #ifdef MALLOC_COUNT 17 | #include "malloc_count.h" 18 | #endif 19 | 20 | 21 | template 22 | struct test_type 23 | { 24 | // using table = ProbIndependentBase >; 26 | using table_type = 27 | HASHTYPE; 28 | 29 | int operator()(size_t it, size_t n, size_t cap, size_t mdisp) 30 | { 31 | otm::out() << otm::width(3) << "#it" << otm::width(10) << "n" 32 | << otm::width(10) << "cap" << otm::width(5) << "disp" 33 | << otm::width(9) << "ndisp" << otm::width(10) << "time" 34 | << std::endl; 35 | 36 | constexpr size_t range = std::numeric_limits::max(); 37 | 38 | size_t* keys = new size_t[n]; 39 | std::vector* disps = new std::vector[mdisp + 1]; 40 | 41 | std::uniform_int_distribution dis(1, range); 42 | std::mt19937_64 re; 43 | 44 | for (size_t j = 0; j < it; ++j) 45 | { 46 | for (size_t i = 0; i < n; ++i) { keys[i] = dis(re); } 47 | for (size_t i = 0; i <= mdisp; ++i) { disps[i].clear(); } 48 | 49 | table_type table(cap, 1.0, 1000); 50 | 51 | for (size_t i = 0; i < n; ++i) 52 | { 53 | if (!table.insert(keys[i], i).second) 54 | { 55 | otm::out() << otm::color::red 56 | << "missed insert key: " << otm::color::reset 57 | << keys[i] << std::endl; 58 | } 59 | } 60 | 61 | for (size_t i = 0; i < n; ++i) 62 | { 63 | auto key = keys[i]; 64 | int disp = std::min(table.displacement(key), mdisp); 65 | 66 | if (disp < 0) 67 | { 68 | otm::out() << otm::color::red 69 | << "no displacement key: " << otm::color::reset 70 | << key << std::endl; 71 | } 72 | 73 | disps[disp].push_back(key); 74 | } 75 | 76 | 77 | auto t0 = std::chrono::high_resolution_clock::now(); 78 | for (size_t i = 0; i < n; ++i) 79 | { 80 | if (table.find(keys[i]) == table.end()) 81 | { 82 | otm::out() << otm::color::red 83 | << "unsuccessful query in cache clean" 84 | << otm::color::reset << keys[i] << std::endl; 85 | } 86 | } 87 | auto t1 = std::chrono::high_resolution_clock::now(); 88 | double tfind = 89 | std::chrono::duration_cast(t1 - t0) 90 | .count() / 91 | 1000.; 92 | 93 | for (size_t i = 0; i <= mdisp; ++i) 94 | { 95 | otm::out() << otm::width(3) << j << otm::width(10) << n 96 | << otm::width(10) << cap << otm::width(5) << i 97 | << otm::width(9) << disps[i].size() << std::flush; 98 | 99 | if (!disps[i].size()) 100 | { 101 | otm::out() << otm::width(10) << 0 << otm::width(10) << tfind 102 | << std::endl; 103 | continue; 104 | } 105 | 106 | // clean the cash 107 | for (size_t i = 0; i < n; ++i) 108 | { 109 | if (table.find(keys[i]) == table.end()) 110 | { 111 | otm::out() << otm::color::red 112 | << "unsuccessful query in cache clean" 113 | << otm::color::reset << keys[i] << std::endl; 114 | } 115 | } 116 | 117 | auto t0 = std::chrono::high_resolution_clock::now(); 118 | for (auto& key : disps[i]) 119 | { 120 | if (table.find(key) == table.end()) 121 | otm::out() 122 | << otm::color::red 123 | << "unsuccessful query key: " << otm::color::reset 124 | << key << std::endl; 125 | } 126 | auto t1 = std::chrono::high_resolution_clock::now(); 127 | double tdiff = 128 | std::chrono::duration_cast(t1 - 129 | t0) 130 | .count(); 131 | otm::out() << otm::width(10) << tdiff / disps[i].size() 132 | << otm::width(10) << tfind << std::endl; 133 | } 134 | } 135 | 136 | 137 | delete[] keys; 138 | delete[] disps; 139 | 140 | return 0; 141 | } 142 | }; 143 | 144 | int main(int argn, char** argc) 145 | { 146 | // pin_to_core(0); 147 | utm::command_line_parser c(argn, argc); 148 | 149 | size_t it = c.int_arg("-it", 5); 150 | size_t n = c.int_arg("-n", 700000); 151 | size_t cap = c.int_arg("-cap", 1000000); 152 | size_t mdisp = c.int_arg("-max_disp", 50); 153 | 154 | if (c.bool_arg("-file") || c.bool_arg("-out")) 155 | { 156 | std::string name = c.str_arg("-file", ""); 157 | name = c.str_arg("-out", name) + ".displ"; 158 | otm::out().set_file(name); 159 | } 160 | 161 | if (c.bool_arg("-help") || c.bool_arg("-h")) 162 | { 163 | otm::out() << "This test measures access speed dependant on the \n" 164 | << "displacement of elements. For different hash tables\n" 165 | << "\n" 166 | << otm::color::blue << "1. " << otm::color::reset 167 | << "insert n random elements into a table with cap slots\n" 168 | << otm::color::blue << "2. " << otm::color::reset 169 | << "for each element, measure its displacement\n" 170 | << otm::color::blue << "3. " << otm::color::reset 171 | << "iterate over displacements query all such elements\n" 172 | << " output their number, and the average time per query" 173 | << std::endl; 174 | return 0; 175 | } 176 | 177 | return Chooser::execute(c, it, n, cap, 178 | mdisp); 179 | } 180 | -------------------------------------------------------------------------------- /source/eps_test.cpp: -------------------------------------------------------------------------------- 1 | //#include "include/spacegrow.hpp" 2 | #include "selection.hpp" 3 | 4 | // #include "include/strategies/dstrat_bfs.hpp" 5 | // #include "include/strategies/dstrat_rwalk.hpp" 6 | // #include "include/strategies/dstrat_rwalk_cyclic.hpp" 7 | 8 | #include "utils/command_line_parser.hpp" 9 | #include "utils/default_hash.hpp" 10 | #include "utils/output.hpp" 11 | #include "utils/pin_thread.hpp" 12 | namespace utm = utils_tm; 13 | namespace otm = utils_tm::out_tm; 14 | 15 | #ifdef MALLOC_COUNT 16 | #include "malloc_count.h" 17 | #endif 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | template 25 | struct test_type 26 | { 27 | // using table_type = ProbIndependentBase >; 29 | using table_type = 30 | HASHTYPE; 31 | 32 | constexpr static size_t block_size = 100000; 33 | 34 | void 35 | prepare_find_keys(size_t* fikeys, size_t* inkeys, size_t fis, size_t ins) 36 | { 37 | std::uniform_int_distribution pdis(1, ins); 38 | std::mt19937_64 re(ins * 2098740980990909098ull); 39 | for (size_t i = 0; i < fis; ++i) { fikeys[i] = inkeys[pdis(re)]; } 40 | std::uniform_int_distribution ndis(1, (1ull << 63) - 1); 41 | for (size_t i = fis; i < fis << 1; ++i) { fikeys[i] = ndis(re); } 42 | } 43 | 44 | int operator()( 45 | size_t it, size_t n, size_t steps, double eps, double epstp, size_t win) 46 | { 47 | otm::out() << otm::width(4) << "# it"; 48 | table_type::print_init_header(otm::out()); 49 | otm::out() << otm::width(9) << "n_step" << otm::width(5) << "win" 50 | << otm::width(6) << "load" << otm::width(8) << "t_in" 51 | << otm::width(8) << "t_fip" << otm::width(8) << "t_fin" 52 | << otm::width(8) << "in_err" << otm::width(8) << "fi_err" 53 | #ifdef MALLOC_COUNT 54 | << otm::width(7) << "memory" 55 | #endif 56 | << std::endl; 57 | 58 | constexpr size_t range = (1ull << 63) - 1; 59 | 60 | size_t* inkeys = new size_t[n]; 61 | size_t* fikeys = new size_t[win << 1]; 62 | 63 | std::uniform_int_distribution dis(1, range); 64 | std::mt19937_64 re; 65 | 66 | for (size_t i = 0; i < it; ++i) 67 | { 68 | 69 | // prepare execution randomize each iteration seperately 70 | // to take variance into account 71 | for (size_t i = 0; i < n; ++i) { inkeys[i] = dis(re); } 72 | table_type table(n, 1.0, steps); 73 | 74 | size_t in_errors = 0ull; 75 | size_t fi_errors = 0ull; 76 | 77 | size_t j = 0; 78 | 79 | double tlf = 1. - eps; 80 | size_t nxt_blk = n * tlf; 81 | 82 | prepare_find_keys(fikeys, inkeys, win, nxt_blk); 83 | 84 | while (nxt_blk + win < n) 85 | { 86 | for (; j < nxt_blk; ++j) 87 | { 88 | if (!table.insert(inkeys[j], j).second) ++in_errors; 89 | } 90 | 91 | auto t1 = std::chrono::high_resolution_clock::now(); 92 | for (; j < nxt_blk + win; ++j) 93 | { 94 | if (!table.insert(inkeys[j], j).second) ++in_errors; 95 | } 96 | auto t2 = std::chrono::high_resolution_clock::now(); 97 | for (size_t k = 0; k < win; ++k) 98 | { 99 | auto temp = table.find(fikeys[k]); 100 | if (temp == table.end()) ++fi_errors; 101 | } 102 | auto t3 = std::chrono::high_resolution_clock::now(); 103 | for (size_t k = win; k < win << 1; ++k) 104 | { 105 | auto temp = table.find(fikeys[k]); 106 | if (temp != table.end()) ++fi_errors; 107 | } 108 | auto t4 = std::chrono::high_resolution_clock::now(); 109 | double d_in = 110 | std::chrono::duration_cast(t2 - 111 | t1) 112 | .count() / 113 | double(win); 114 | double d_fip = 115 | std::chrono::duration_cast(t3 - 116 | t2) 117 | .count() / 118 | double(win); 119 | double d_fin = 120 | std::chrono::duration_cast(t4 - 121 | t3) 122 | .count() / 123 | double(win); 124 | 125 | if (in_errors > 100) break; 126 | 127 | otm::out() << otm::width(4) << i; 128 | table.print_init_data(otm::out()); 129 | otm::out() << otm::width(9) << nxt_blk << otm::width(5) << win 130 | << otm::width(6) << tlf << otm::width(8) << d_in 131 | << otm::width(8) << d_fip << otm::width(8) << d_fin 132 | << otm::width(8) << in_errors << otm::width(8) 133 | << fi_errors 134 | #ifdef MALLOC_COUNT 135 | << otm::width(7) 136 | << malloc_count_current() - n * 8 - win * 16 137 | #endif 138 | << std::endl; 139 | 140 | tlf += epstp; 141 | nxt_blk = n * tlf; 142 | 143 | prepare_find_keys(fikeys, inkeys, win, nxt_blk); 144 | } 145 | } 146 | 147 | delete[] inkeys; 148 | 149 | return 0; 150 | } 151 | }; 152 | 153 | int main(int argn, char** argc) 154 | { 155 | utm::pin_to_core(0); 156 | utm::command_line_parser c(argn, argc); 157 | 158 | size_t it = c.int_arg("-it", 5); 159 | size_t n = c.int_arg("-n", 2000000); 160 | size_t steps = c.int_arg("-steps", 512); 161 | 162 | double alpha = c.double_arg("-alpha", 0.1); 163 | double load = c.double_arg("-load", 1.0 - (alpha - 1.0) / alpha); 164 | double eps = c.double_arg("-eps", 1.0 - load); 165 | 166 | const double epstp = c.double_arg("-epsteps", 0.005); 167 | const size_t win = c.int_arg("-win", 1000); 168 | 169 | if (c.bool_arg("-out") || c.bool_arg("-file")) 170 | { 171 | std::string name = c.str_arg("-out", ""); 172 | name = c.str_arg("-file", name) + ".eps"; 173 | otm::out().set_file(name); 174 | } 175 | 176 | if (c.bool_arg("-help") || c.bool_arg("-h")) 177 | { 178 | otm::out() << "Test the performance depending on the fill degree\n" 179 | << "stepwise test after blocks\n" 180 | << "\n" 181 | << otm::color::blue << "1. " << otm::color::reset 182 | << "insert to the next fill degree test window\n" 183 | << otm::color::blue << "2. " << otm::color::reset 184 | << "test insert performance with -win new inserts\n" 185 | << otm::color::blue << "3. " << otm::color::reset 186 | << "query -win random inserted elements\n" 187 | << otm::color::blue << "4. " << otm::color::reset 188 | << "query -win not inserted elements\n" 189 | << otm::color::blue << "repeat" << otm::color::reset 190 | << std::endl; 191 | return 0; 192 | } 193 | 194 | if (eps < 0.) 195 | { 196 | std::cout << "please input eps" << std::endl; 197 | return 8; 198 | } 199 | 200 | return Chooser::execute(c, it, n, steps, eps, 201 | epstp, win); 202 | } 203 | -------------------------------------------------------------------------------- /source/in_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | #include "utils/command_line_parser.hpp" 3 | #include "utils/default_hash.hpp" 4 | #include "utils/output.hpp" 5 | #include "utils/pin_thread.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace utm = utils_tm; 12 | namespace otm = utils_tm::out_tm; 13 | 14 | static constexpr size_t TABLE_BS = 4; 15 | static constexpr size_t TABLE_TL = 1; 16 | 17 | class history_histogram 18 | { 19 | public: 20 | history_histogram(size_t s) : steps(s), hist(new size_t[s]) 21 | { 22 | for (size_t i = 0; i < s; ++i) { hist[i] = 0; } 23 | } 24 | 25 | void add(size_t i) 26 | { 27 | auto ind = (i < steps) ? i : steps - 1; 28 | ++hist[ind]; 29 | } 30 | 31 | void clear() 32 | { 33 | for (size_t i = 0; i < steps; ++i) { hist[i] = 0; } 34 | } 35 | 36 | const size_t steps; 37 | std::unique_ptr hist; 38 | }; 39 | 40 | 41 | template 42 | struct test_type 43 | { 44 | using table_type = 45 | HASHTYPE; 46 | 47 | void print_hist(table_type& table) 48 | { 49 | auto& hcount = table.get_history(); 50 | 51 | otm::out() << otm::width(7) << "# steps" << otm::width(8) << "nFitted" 52 | << std::endl; 53 | 54 | for (size_t i = 0; i < hcount.steps; ++i) 55 | { 56 | otm::out() << otm::width(7) << i << otm::width(8) << hcount.hist[i] 57 | << std::endl; 58 | } 59 | } 60 | 61 | void print_dist(table_type& table) 62 | { 63 | otm::out() << otm::width(5) << "# tab"; 64 | size_t gHist[TABLE_BS + 1]; 65 | for (size_t i = 0; i <= TABLE_BS; ++i) 66 | { 67 | gHist[i] = 0; 68 | otm::out() << otm::width(6) << i; 69 | } 70 | 71 | otm::out() << otm::width(8) << "n" << otm::width(8) << "cap" 72 | << std::endl; 73 | 74 | for (size_t tl = 0; tl < TABLE_TL; ++tl) 75 | { 76 | size_t lHist[TABLE_BS + 1]; 77 | for (size_t i = 0; i <= TABLE_BS; ++i) lHist[i] = 0; 78 | 79 | auto ltab = table.getTable(tl); 80 | 81 | for (size_t j = 0; j < ltab.first; ++j) 82 | { 83 | auto a = ltab.second[j].probe(0); 84 | if (a >= 0) ++lHist[a]; 85 | } 86 | 87 | size_t n = 0; 88 | otm::out() << otm::width(5) << tl; 89 | for (size_t i = 0; i <= TABLE_BS; ++i) 90 | { 91 | otm::out() << otm::width(6) << lHist[i]; 92 | n += lHist[i] * (TABLE_BS - i); 93 | gHist[i] += lHist[i]; 94 | } 95 | otm::out() << otm::width(8) << n << otm::width(8) 96 | << ltab.first * TABLE_BS << std::endl; 97 | } 98 | 99 | size_t n = 0; 100 | otm::out() << otm::width(5) << "#all"; 101 | for (size_t i = 0; i <= TABLE_BS; ++i) 102 | { 103 | otm::out() << otm::width(6) << gHist[i]; 104 | n += gHist[i] * (TABLE_BS - i); 105 | } 106 | otm::out() << otm::width(8) << n << std::endl; 107 | } 108 | 109 | int operator()(size_t n, 110 | size_t pre, 111 | size_t cap, 112 | size_t steps, 113 | double alpha, 114 | const std::string& name) 115 | { 116 | constexpr size_t range = (1ull << 63) - 1; 117 | 118 | size_t* keys = new size_t[n + pre]; 119 | 120 | std::uniform_int_distribution dis(1, range); 121 | std::mt19937_64 re; 122 | 123 | for (size_t i = 0; i < n + pre; ++i) { keys[i] = dis(re); } 124 | 125 | table_type table(cap, alpha, steps); 126 | 127 | for (size_t i = 0; i < pre; ++i) { table.insert(keys[i], i); } 128 | 129 | table.clear_history(); 130 | 131 | for (size_t i = pre; i < pre + n; ++i) { table.insert(keys[i], i); } 132 | 133 | otm::out().set_file(name + ".dist"); 134 | print_dist(table); 135 | otm::out().set_file(name + ".hist"); 136 | print_hist(table); 137 | 138 | delete[] keys; 139 | 140 | return 0; 141 | } 142 | }; 143 | 144 | int main(int argn, char** argc) 145 | { 146 | utm::pin_to_core(0); 147 | utm::command_line_parser c(argn, argc); 148 | 149 | size_t n = c.int_arg("-n", 1000000); 150 | size_t pre = c.int_arg("-pre", 0); 151 | size_t cap = c.int_arg("-cap", 1000000); 152 | size_t steps = c.int_arg("-steps", 512); 153 | 154 | double alpha = c.double_arg("-alpha", 1.1); 155 | double load = c.double_arg("-load", 2.0); 156 | double eps = c.double_arg("-eps", 1.0 - load); 157 | if (eps > 0.) alpha = 1. / (1. - eps); 158 | 159 | std::string name = c.str_arg("-out", "temp"); 160 | name = c.str_arg("-file", name); 161 | 162 | return Chooser::execute(c, n, pre, cap, steps, 163 | alpha, name); 164 | } 165 | -------------------------------------------------------------------------------- /source/mix_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | 3 | #include "utils/command_line_parser.hpp" 4 | #include "utils/default_hash.hpp" 5 | #include "utils/output.hpp" 6 | #include "utils/pin_thread.hpp" 7 | namespace utm = utils_tm; 8 | namespace otm = utils_tm::out_tm; 9 | 10 | #ifdef MALLOC_COUNT 11 | #include "malloc_count.h" 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | template 20 | struct Test 21 | { 22 | // using table_type = ProbIndependentBase >; 24 | using table_type = 25 | HASHTYPE; 26 | 27 | int operator()(size_t it, 28 | size_t n, 29 | size_t pre, 30 | size_t cap, 31 | size_t pattern, 32 | size_t steps, 33 | double alpha) 34 | { 35 | 36 | otm::out() << otm::width(4) << "# it" << otm::width(8) << "alpha"; 37 | table_type::print_init_header(otm::out()); 38 | otm::out() << otm::width(9) << "cap" << otm::width(4) << "pat" 39 | << otm::width(9) << "pre" << otm::width(9) << "n" 40 | << otm::width(8) << "t_pre" << otm::width(8) << "t_mix" 41 | << otm::width(9) << "in_err" 42 | #ifdef MALLOC_COUNT 43 | << otm::width(7) << "memory" 44 | #endif 45 | << std::endl; 46 | 47 | 48 | constexpr size_t range = (1ull << 63) - 1; 49 | 50 | size_t p_rep = n / 10; 51 | size_t* ikeys = new size_t[pre + p_rep * pattern]; 52 | size_t* fkeys = new size_t[p_rep * (10 - pattern)]; 53 | 54 | std::uniform_int_distribution dis(1, range); 55 | std::mt19937_64 re; 56 | 57 | for (size_t i = 0; i < pre; ++i) { ikeys[i] = dis(re); } 58 | size_t pcount = 0; 59 | size_t icount = pre - 1; 60 | size_t fcount = 0; 61 | for (size_t i = pre; i < p_rep * 10 + pre; ++i) 62 | { 63 | if (pcount < pattern) { ikeys[++icount] = dis(re); } 64 | else 65 | { 66 | std::uniform_int_distribution find_dis(0, icount); 67 | size_t k = find_dis(re); 68 | fkeys[fcount++] = ikeys[k]; 69 | } 70 | pcount = (pcount < 9) ? pcount + 1 : 0; 71 | } 72 | 73 | for (size_t i = 0; i < it; ++i) 74 | { 75 | table_type table(cap, alpha, steps); 76 | 77 | auto errors = 0ull; 78 | auto ferrors = 0ull; 79 | 80 | auto t0 = std::chrono::high_resolution_clock::now(); 81 | for (size_t i = 0; i < pre; ++i) 82 | { 83 | if (!table.insert(ikeys[i], i).second) ++errors; 84 | } 85 | 86 | auto t1 = std::chrono::high_resolution_clock::now(); 87 | pcount = 0; 88 | icount = pre; 89 | fcount = 0; 90 | for (size_t i = pre; i < p_rep * 10 + pre; ++i) 91 | { 92 | if (pcount < pattern) 93 | { 94 | if (!table.insert(ikeys[icount++], i).second) ++errors; 95 | } 96 | else 97 | { 98 | auto temp = table.find(fkeys[fcount++]); 99 | if (temp == table.end()) ++ferrors; 100 | } 101 | pcount = (pcount < 9) ? pcount + 1 : 0; 102 | } 103 | 104 | auto t2 = std::chrono::high_resolution_clock::now(); 105 | 106 | double d_pre = 107 | std::chrono::duration_cast(t1 - t0) 108 | .count() / 109 | 1000.; 110 | double d_mix = 111 | std::chrono::duration_cast(t2 - t1) 112 | .count() / 113 | 1000.; 114 | 115 | otm::out() << otm::width(4) << i << otm::width(8) << alpha; 116 | table.print_init_data(otm::out()); 117 | otm::out() << otm::width(9) << cap << otm::width(4) << pattern 118 | << otm::width(9) << pre << otm::width(9) << n 119 | << otm::width(8) << d_pre << otm::width(8) << d_mix 120 | << otm::width(9) << errors 121 | #ifdef MALLOC_COUNT 122 | << otm::width(7) 123 | << double(malloc_count_current() - (n + pre) * 8) / 124 | (0.1 * double(pattern * 8 * 2 * n) + 125 | double(8 * 2 * pre)) 126 | #endif 127 | << std::endl; 128 | 129 | if (ferrors && (!errors)) 130 | otm::out() << "Critical Error" << std::endl; 131 | } 132 | 133 | delete[] ikeys; 134 | delete[] fkeys; 135 | 136 | return 0; 137 | } 138 | }; 139 | 140 | int main(int argn, char** argc) 141 | { 142 | utm::pin_to_core(0); 143 | utm::command_line_parser c(argn, argc); 144 | 145 | size_t it = c.int_arg("-it", 5); 146 | size_t n = c.int_arg("-n", 1000000); 147 | size_t n0 = c.int_arg("-pre", n / 4); 148 | size_t cap = c.int_arg("-cap", n0); 149 | size_t pattern = c.int_arg("-pattern", 5); 150 | size_t steps = c.int_arg("-steps", 512); 151 | 152 | double alpha = c.double_arg("-alpha", 1.1); 153 | double load = c.double_arg("-load", 2.0); 154 | double eps = c.double_arg("-eps", 1.0 - load); 155 | if (eps > 0.) alpha = 1. / (1. - eps); 156 | 157 | if (c.bool_arg("-out") || c.bool_arg("-file")) 158 | { 159 | std::string name = c.str_arg("-out", ""); 160 | name = c.str_arg("-file", name) + ".mix"; 161 | otm::out().set_file(name); 162 | } 163 | 164 | return Chooser::execute(c, it, n, n0, cap, 165 | pattern, steps, alpha); 166 | } 167 | -------------------------------------------------------------------------------- /source/mixd_test.cpp: -------------------------------------------------------------------------------- 1 | //#include "include/spacegrow.hpp" 2 | #include "selection.hpp" 3 | 4 | // #include "include/strategies/dstrat_bfs.hpp" 5 | // #include "include/strategies/dstrat_rwalk.hpp" 6 | // #include "include/strategies/dstrat_rwalk_cyclic.hpp" 7 | 8 | #include "utils/command_line_parser.hpp" 9 | #include "utils/default_hash.hpp" 10 | #include "utils/output.hpp" 11 | #include "utils/pin_thread.hpp" 12 | namespace utm = utils_tm; 13 | namespace otm = utils_tm::out_tm; 14 | 15 | #ifdef MALLOC_COUNT 16 | #include "malloc_count.h" 17 | #endif 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | template 25 | struct Test 26 | { 27 | // using table_type = ProbIndependentBase >; 29 | using table_type = 30 | HASHTYPE; 31 | 32 | 33 | int operator()(size_t it, 34 | size_t n, 35 | size_t pre, 36 | size_t cap, 37 | size_t pattern, 38 | size_t steps, 39 | double alpha) 40 | { 41 | otm::out() << otm::width(4) << "# it" << otm::width(8) << "alpha"; 42 | table_type::print_init_header(otm::out()); 43 | otm::out() << otm::width(9) << "cap" << otm::width(4) << "pat" 44 | << otm::width(9) << "pre" << otm::width(9) << "n" 45 | << otm::width(8) << "t_pre" << otm::width(8) << "t_mix" 46 | << otm::width(9) << "in_err" 47 | #ifdef MALLOC_COUNT 48 | << otm::width(7) << "memory" 49 | #endif 50 | << std::endl; 51 | 52 | constexpr size_t range = (1ull << 63) - 1; 53 | 54 | size_t p_rep = n / 10; 55 | bool* dbool = new bool[pre + p_rep * pattern]; 56 | size_t* ikeys = new size_t[pre + p_rep * pattern]; 57 | size_t* dkeys = new size_t[p_rep * (10 - pattern)]; 58 | 59 | std::fill(dbool, dbool + (pre + p_rep * pattern), false); 60 | 61 | std::uniform_int_distribution dis(1, range); 62 | std::mt19937_64 re; 63 | 64 | for (size_t i = 0; i < pre; ++i) { ikeys[i] = dis(re); } 65 | size_t pcount = 0; 66 | size_t icount = pre - 1; 67 | size_t dcount = 0; 68 | for (size_t i = pre; i < p_rep * 10 + pre; ++i) 69 | { 70 | if (pcount < pattern) { ikeys[++icount] = dis(re); } 71 | else 72 | { 73 | std::uniform_int_distribution find_dis(0, icount); 74 | size_t k = find_dis(re); 75 | while (dbool[k]) k = find_dis(re); 76 | dbool[k] = true; 77 | dkeys[dcount++] = ikeys[k]; 78 | } 79 | pcount = (pcount < 9) ? pcount + 1 : 0; 80 | } 81 | delete[] dbool; 82 | 83 | for (size_t i = 0; i < it; ++i) 84 | { 85 | table_type table(cap, alpha, steps); 86 | 87 | auto errors = 0ull; 88 | auto ferrors = 0ull; 89 | 90 | auto t0 = std::chrono::high_resolution_clock::now(); 91 | for (size_t i = 0; i < pre; ++i) 92 | { 93 | if (!table.insert(ikeys[i], i).second) ++errors; 94 | } 95 | 96 | auto t1 = std::chrono::high_resolution_clock::now(); 97 | pcount = 0; 98 | icount = pre; 99 | dcount = 0; 100 | for (size_t i = pre; i < p_rep * 10 + pre; ++i) 101 | { 102 | if (pcount < pattern) 103 | { 104 | if (!table.insert(ikeys[icount++], i).second) ++errors; 105 | } 106 | else 107 | { 108 | if (!table.erase(dkeys[dcount++])) ++ferrors; 109 | } 110 | pcount = (pcount < 9) ? pcount + 1 : 0; 111 | } 112 | 113 | auto t2 = std::chrono::high_resolution_clock::now(); 114 | 115 | double d_pre = 116 | std::chrono::duration_cast(t1 - t0) 117 | .count() / 118 | 1000.; 119 | double d_mix = 120 | std::chrono::duration_cast(t2 - t1) 121 | .count() / 122 | 1000.; 123 | 124 | otm::out() << otm::width(4) << i << otm::width(8) << alpha; 125 | table.print_init_data(otm::out()); 126 | otm::out() << otm::width(9) << cap << otm::width(4) << pattern 127 | << otm::width(9) << pre << otm::width(9) << n 128 | << otm::width(8) << d_pre << otm::width(8) << d_mix 129 | << otm::width(9) << errors 130 | #ifdef MALLOC_COUNT 131 | << otm::width(7) 132 | << double(malloc_count_current() - (n + pre) * 8) / 133 | (0.1 * double(pattern * 8 * 2 * n) + 134 | double(8 * 2 * pre)) 135 | #endif 136 | << std::endl; 137 | 138 | if (ferrors && (!errors)) 139 | otm::out() << "Critical Error" << std::endl; 140 | } 141 | 142 | delete[] ikeys; 143 | delete[] dkeys; 144 | 145 | return 0; 146 | } 147 | }; 148 | 149 | int main(int argn, char** argc) 150 | { 151 | utm::pin_to_core(0); 152 | utm::command_line_parser c(argn, argc); 153 | 154 | size_t it = c.int_arg("-it", 5); 155 | size_t n = c.int_arg("-n", 1000000); 156 | size_t n0 = c.int_arg("-pre", n / 4); 157 | size_t cap = c.int_arg("-cap", n0); 158 | size_t pattern = c.int_arg("-pattern", 5); 159 | size_t steps = c.int_arg("-steps", 512); 160 | 161 | double alpha = c.double_arg("-alpha", 1.1); 162 | double load = c.double_arg("-load", 2.0); 163 | double eps = c.double_arg("-eps", 1.0 - load); 164 | if (eps > 0.) alpha = 1. / (1. - eps); 165 | 166 | if (c.bool_arg("-out") || c.bool_arg("-file")) 167 | { 168 | std::string name = c.str_arg("-out", ""); 169 | name = c.str_arg("-file", name) + ".mixd"; 170 | otm::out().set_file(name); 171 | } 172 | 173 | return Chooser::execute(c, it, n, n0, cap, 174 | pattern, steps, alpha); 175 | } 176 | -------------------------------------------------------------------------------- /source/mxls_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | #include "utils/command_line_parser.hpp" 3 | #include "utils/default_hash.hpp" 4 | #include "utils/output.hpp" 5 | #include "utils/pin_thread.hpp" 6 | namespace utm = utils_tm; 7 | namespace otm = utils_tm::out_tm; 8 | 9 | #ifdef MALLOC_COUNT 10 | #include "malloc_count.h" 11 | #endif 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | template 19 | struct test_type 20 | { 21 | // using table_type = ProbIndependentBase >; 23 | using table_type = 24 | HASHTYPE; 25 | 26 | int operator()(size_t it, size_t n, size_t steps) 27 | { 28 | otm::out() << otm::width(4) << "# it" << otm::width(9) << "cap" 29 | << otm::width(9) << "n" << otm::width(6) << "steps" 30 | << otm::width(6) << "ngrow"; 31 | table_type::print_init_header(otm::out()); 32 | otm::out() << otm::width(9) << "g_elem" << std::endl; 33 | 34 | constexpr size_t range = (1ull << 63) - 1; 35 | 36 | size_t* keys = new size_t[n]; 37 | 38 | std::uniform_int_distribution dis(1, range); 39 | std::mt19937_64 re; 40 | 41 | for (size_t i = 0; i < it; ++i) 42 | { 43 | for (size_t i = 0; i < n; ++i) { keys[i] = dis(re); } 44 | 45 | table_type table(n, 1.0, steps); 46 | 47 | size_t ngrow = 0; 48 | for (size_t j = 0; j < n; ++j) 49 | { 50 | if (!table.insert(keys[j], j).second) 51 | { 52 | otm::out() << otm::width(4) << i << otm::width(9) << n 53 | << otm::width(9) << n << otm::width(6) << steps 54 | << otm::width(6) << ngrow; 55 | table.print_init_data(otm::out()); 56 | otm::out() << otm::width(9) << j << std::endl; 57 | 58 | ngrow++; 59 | break; 60 | } 61 | } 62 | } 63 | 64 | delete[] keys; 65 | 66 | return 0; 67 | } 68 | }; 69 | 70 | int main(int argn, char** argc) 71 | { 72 | utils_tm::pin_to_core(0); 73 | utils_tm::command_line_parser c(argn, argc); 74 | 75 | size_t it = c.int_arg("-it", 5); 76 | size_t n = c.int_arg("-n", 2000000); 77 | size_t steps = c.int_arg("-steps", 512); 78 | 79 | if (c.bool_arg("-out") || c.bool_arg("-file")) 80 | { 81 | std::string name = c.str_arg("-out", ""); 82 | name = c.str_arg("-file", name) + ".mxls"; 83 | otm::out().set_file(name); 84 | } 85 | 86 | return Chooser::execute(c, it, n, steps); 87 | } 88 | -------------------------------------------------------------------------------- /source/probe_new_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #define EXTENDED 7 | #include "selection.hpp" 8 | #include "utils/command_line_parser.hpp" 9 | #include "utils/default_hash.hpp" 10 | #include "utils/output.hpp" 11 | #include "utils/pin_thread.hpp" 12 | namespace utm = utils_tm; 13 | namespace otm = utils_tm::out_tm; 14 | 15 | class history_raw 16 | { 17 | public: 18 | history_raw(size_t s = 0) { trend.reserve(s); } 19 | 20 | void add(size_t i) { trend.push_back(i); } 21 | void clear() { trend.clear(); } 22 | 23 | std::vector trend; 24 | }; 25 | 26 | template 27 | struct Test 28 | { 29 | // using table_type = ProbIndependentBase >; 31 | using table_type = 32 | HASHTYPE; 33 | 34 | int operator()(size_t it, 35 | size_t n, 36 | size_t steps, 37 | size_t seed, 38 | double lower, 39 | double upper) 40 | { 41 | constexpr size_t range = (1ull << 63) - 1; 42 | 43 | size_t* keys = new size_t[n]; 44 | 45 | for (size_t i = 0; i < it; ++i) 46 | { 47 | std::uniform_int_distribution dis(1, range); 48 | std::mt19937_64 re(seed * i); 49 | 50 | for (size_t j = 0; j < n; ++j) { keys[j] = dis(re); } 51 | 52 | table_type table(n, 1., steps); 53 | 54 | size_t ii = 0; 55 | 56 | auto t0 = std::chrono::high_resolution_clock::now(); 57 | for (; ii < upper * n; ++ii) 58 | { 59 | if (!table.insert(keys[ii], ii).second) break; 60 | } 61 | auto t1 = std::chrono::high_resolution_clock::now(); 62 | 63 | otm::out() << "# seed:" << seed * i // 64 | << " inserted:" << ii << "elements" // 65 | << " into:" << n << "slots" 66 | << " it took:" 67 | << std::chrono::duration_cast( 68 | t1 - t0) 69 | .count() / 70 | 1000. 71 | << "ms" << std::endl; 72 | 73 | otm::out() << "# element_nmbr nmbr_of_slots probed_slots " 74 | "inner_iteration" 75 | << std::endl; 76 | 77 | auto& history = table.get_history(); 78 | for (size_t j = history.trend.size() * lower; j < ii - 1; ++j) 79 | { 80 | otm::out() << j << " " << n << " " << history.trend[j] 81 | << " 0" << std::endl; 82 | } 83 | 84 | for (size_t i = 0; i < it; ++i) 85 | { 86 | auto temptable = table.explicit_copy(n, 1., steps); 87 | 88 | for (size_t j = ii; j < n; ++j) 89 | { 90 | if (!temptable.insert(dis(re), j).second) break; 91 | } 92 | auto& temphistory = temptable.get_history(); 93 | for (size_t j = 0; j < temphistory.trend.size(); ++j) 94 | { 95 | otm::out() 96 | << ii + j << " " << n << " " << temphistory.trend[j] 97 | << " " << i << std::endl; 98 | } 99 | } 100 | } 101 | 102 | delete[] keys; 103 | 104 | return 0; 105 | } 106 | }; 107 | 108 | int main(int argn, char** argc) 109 | { 110 | utm::pin_to_core(0); 111 | utm::command_line_parser c(argn, argc); 112 | 113 | size_t it = c.int_arg("-it", 5); 114 | size_t n = c.int_arg("-n", 2000000); 115 | size_t steps = c.int_arg("-steps", 512); 116 | size_t seed = c.int_arg("-seed", 0xDEADBEEF); 117 | double lower = c.double_arg("-lower", .97); 118 | double upper = c.double_arg("-upper", .998); 119 | 120 | if (c.bool_arg("-out") || c.bool_arg("-file")) 121 | { 122 | std::string name = c.str_arg("-out", ""); 123 | name = c.str_arg("-file", name) + ".probe"; 124 | otm::out().set_file(name); 125 | } 126 | 127 | return Chooser::execute(c, it, n, steps, seed, lower, 128 | upper); 129 | } 130 | -------------------------------------------------------------------------------- /source/probe_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "selection.hpp" 7 | #include "utils/command_line_parser.hpp" 8 | #include "utils/default_hash.hpp" 9 | #include "utils/output.hpp" 10 | #include "utils/pin_thread.hpp" 11 | namespace utm = utils_tm; 12 | namespace otm = utils_tm::out_tm; 13 | 14 | class history_raw 15 | { 16 | public: 17 | history_raw(size_t s) { trend.reserve(s); } 18 | 19 | void add(size_t i) { trend.push_back(i); } 20 | void clear() { trend.clear(); } 21 | 22 | std::vector trend; 23 | }; 24 | 25 | template 26 | struct Test 27 | { 28 | // using table_type = ProbIndependentBase >; 30 | using table_type = 31 | HASHTYPE; 32 | 33 | int operator()(size_t it, size_t n, size_t steps, size_t seed) 34 | { 35 | constexpr size_t range = (1ull << 63) - 1; 36 | 37 | size_t* keys = new size_t[n]; 38 | 39 | for (size_t i = 0; i < it; ++i) 40 | { 41 | std::uniform_int_distribution dis(1, range); 42 | std::mt19937_64 re(seed * i); 43 | 44 | for (size_t j = 0; j < n; ++j) { keys[j] = dis(re); } 45 | 46 | table_type table(n, 1., steps); 47 | 48 | size_t ii = 0; 49 | 50 | auto t0 = std::chrono::high_resolution_clock::now(); 51 | for (; ii < n; ++ii) 52 | { 53 | if (!table.insert(keys[ii], ii).second) break; 54 | } 55 | auto t1 = std::chrono::high_resolution_clock::now(); 56 | 57 | otm::out() << "# seed:" << seed * i // 58 | << " inserted:" << ii << "elements" // 59 | << " into:" << n << "slots" 60 | << " it took:" 61 | << std::chrono::duration_cast( 62 | t1 - t0) 63 | .count() / 64 | 1000. 65 | << "ms" << std::endl; 66 | 67 | otm::out() << "# element_nmbr nmbr_of_slots probed_slots " 68 | "inner_iteration" 69 | << std::endl; 70 | 71 | auto& history = table.get_history(); 72 | for (size_t j = history.trend.size() * 0.97; j < ii - 1; ++j) 73 | { 74 | otm::out() << j << " " << n << " " << history.trend[j] 75 | << " 0" << std::endl; 76 | } 77 | 78 | for (size_t i = 0; i < it; ++i) 79 | { 80 | // pop 1000 elements from the history 81 | auto start = history.trend.size() - 50; 82 | history.trend.resize(start); 83 | for (size_t j = start; j < n; ++j) 84 | { 85 | // TODO this might fail with optimistic rwalk (pretty rare) 86 | if (j < ii - 1) table.erase(keys[j]); 87 | keys[j] = dis(re); 88 | } 89 | for (ii = start; i < n; ++ii) 90 | { 91 | if (!table.insert(keys[ii], ii).second) break; 92 | } 93 | for (size_t j = start; j < ii - 1; ++j) 94 | { 95 | otm::out() << j << " " << n << " " << history.trend[j] 96 | << " " << i << std::endl; 97 | } 98 | } 99 | } 100 | 101 | delete[] keys; 102 | 103 | return 0; 104 | } 105 | }; 106 | 107 | int main(int argn, char** argc) 108 | { 109 | utm::pin_to_core(0); 110 | utm::command_line_parser c(argn, argc); 111 | 112 | size_t it = c.int_arg("-it", 5); 113 | size_t n = c.int_arg("-n", 2000000); 114 | size_t steps = c.int_arg("-steps", 512); 115 | size_t seed = c.int_arg("-seed", 0xDEADBEEF); 116 | 117 | if (c.bool_arg("-out") || c.bool_arg("-file")) 118 | { 119 | std::string name = c.str_arg("-out", ""); 120 | name = c.str_arg("-file", name) + ".probe"; 121 | otm::out().set_file(name); 122 | } 123 | 124 | return Chooser::execute(c, it, n, steps, seed); 125 | } 126 | -------------------------------------------------------------------------------- /source/reg_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | 3 | #include "utils/command_line_parser.hpp" 4 | #include "utils/default_hash.hpp" 5 | #include "utils/output.hpp" 6 | #include "utils/pin_thread.hpp" 7 | namespace utm = utils_tm; 8 | namespace otm = utils_tm::out_tm; 9 | 10 | #ifdef MALLOC_COUNT 11 | #include "malloc_count.h" 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | template 21 | struct test_type 22 | { 23 | using table_type = 24 | HASHTYPE; 25 | 26 | constexpr static size_t block_size = 100000; 27 | 28 | int operator()(size_t it, size_t n, size_t cap, size_t steps, double alpha) 29 | { 30 | otm::out() << otm::width(4) << "# it" << otm::width(6) << "block" 31 | << otm::width(8) << "alpha"; 32 | table_type::print_init_header(out); 33 | otm::out() << otm::width(9) << "cap" << otm::width(9) << "n_full" 34 | << otm::width(8) << "t_in" << otm::width(6) << "in_err" 35 | #ifdef MALLOC_COUNT 36 | << otm::width(7) << "memory" 37 | #endif 38 | << std::endl; 39 | 40 | 41 | constexpr size_t range = (1ull << 63) - 1; 42 | 43 | size_t* keys = new size_t[n]; 44 | 45 | std::uniform_int_distribution dis(1, range); 46 | std::mt19937_64 re; 47 | 48 | for (size_t i = 0; i < n; ++i) { keys[i] = dis(re); } 49 | 50 | for (size_t i = 0; i < it; ++i) 51 | { 52 | table_type table(cap, alpha, steps); 53 | 54 | auto in_errors = 0ull; 55 | 56 | for (size_t j = 0; j < n / block_size; ++j) 57 | { 58 | auto t1 = std::chrono::high_resolution_clock::now(); 59 | for (size_t ti = j * block_size; ti < (j + 1) * block_size; 60 | ++ti) 61 | { 62 | if (!table.insert(keys[ti], ti).second) ++in_errors; 63 | } 64 | auto t2 = std::chrono::high_resolution_clock::now(); 65 | double d_step = 66 | std::chrono::duration_cast(t2 - 67 | t1) 68 | .count() / 69 | 1000.; 70 | 71 | otm::out() << otm::width(4) << i << otm::width(6) << j 72 | << otm::width(8) << alpha; 73 | table.print_init_data(out); 74 | otm::out() << otm::width(9) << cap << otm::width(9) << n 75 | << otm::width(8) << d_step << otm::width(6) 76 | << in_errors 77 | #ifdef MALLOC_COUNT 78 | << otm::width(7) 79 | << double(malloc_count_current() - n * 8) / 80 | double(8 * 2 * block_size) 81 | #endif 82 | << std::endl; 83 | } 84 | } 85 | 86 | delete[] keys; 87 | 88 | return 0; 89 | } 90 | }; 91 | 92 | int main(int argn, char** argc) 93 | { 94 | utils_tm::pin_to_core(0); 95 | utils_tm::command_line_parser c(argn, argc); 96 | 97 | size_t it = c.int_arg("-it", 5); 98 | size_t n = c.int_arg("-n", 2000000); 99 | size_t cap = c.int_arg("-cap", n); 100 | size_t steps = c.int_arg("-steps", 512); 101 | 102 | double alpha = c.double_arg("-alpha", 1.1); 103 | double load = c.double_arg("-load", 2.0); 104 | double eps = c.double_arg("-eps", 1.0 - load); 105 | if (eps > 0.) alpha = 1. / (1. - eps); 106 | 107 | if (c.bool_arg("-out") || c.bool_arg("-file")) 108 | { 109 | std::string name = c.str_arg("-out", ""); 110 | name = c.str_arg("-file", name) + ".reg"; 111 | otm::out().set_file(name); 112 | } 113 | 114 | return Chooser::execute(c, it, n, cap, steps, 115 | alpha); 116 | } 117 | -------------------------------------------------------------------------------- /source/sing_test.cpp: -------------------------------------------------------------------------------- 1 | #include "selection.hpp" 2 | 3 | #include "utils/command_line_parser.hpp" 4 | #include "utils/default_hash.hpp" 5 | #include "utils/output.hpp" 6 | #include "utils/pin_thread.hpp" 7 | namespace utm = utils_tm; 8 | namespace otm = utils_tm::out_tm; 9 | 10 | #ifdef MALLOC_COUNT 11 | #include "malloc_count.h" 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | template 21 | struct test_type 22 | { 23 | using table_type = 24 | HASHTYPE; 25 | 26 | 27 | int operator()(size_t it, size_t n, size_t cap, size_t steps) 28 | { 29 | otm::out() << otm::width(4) << "# it" << otm::width(9) << "cap" 30 | << otm::width(9) << "n" << otm::width(6) << "steps" 31 | << otm::width(6) << "ngrow"; 32 | table_type::print_init_header(otm::out()); 33 | otm::out() << otm::width(9) << "g_elem" << std::endl; 34 | 35 | 36 | constexpr size_t range = (1ull << 63) - 1; 37 | 38 | size_t* keys = new size_t[2 * n]; 39 | 40 | std::uniform_int_distribution dis(1, range); 41 | std::mt19937_64 re; 42 | 43 | for (size_t i = 0; i < it; ++i) 44 | { 45 | for (size_t i = 0; i < n; ++i) { keys[i] = dis(re); } 46 | 47 | table_type table(cap, 1.0, steps); 48 | 49 | size_t ngrow = 0; 50 | for (size_t j = 0; j < n; ++j) 51 | { 52 | while (!table.insert(keys[j], j).second) 53 | { 54 | otm::out() << otm::width(4) << i << otm::width(9) << cap 55 | << otm::width(9) << n << otm::width(6) << steps 56 | << otm::width(6) << ngrow; 57 | table.print_init_data(otm::out()); 58 | otm::out() << otm::width(9) << j << std::endl; 59 | 60 | ngrow++; 61 | table.explicit_grow(); 62 | } 63 | } 64 | } 65 | 66 | delete[] keys; 67 | 68 | return 0; 69 | } 70 | }; 71 | 72 | int main(int argn, char** argc) 73 | { 74 | // pin_to_core(0); 75 | utm::command_line_parser c(argn, argc); 76 | 77 | size_t it = c.int_arg("-it", 5); 78 | size_t n = c.int_arg("-n", 2000000); 79 | size_t cap = c.int_arg("-cap", 50000); 80 | size_t steps = c.int_arg("-steps", 512); 81 | 82 | if (c.bool_arg("-out") || c.bool_arg("-file")) 83 | { 84 | std::string name = c.str_arg("-out", ""); 85 | name = c.str_arg("-file", name) + ".sing"; 86 | otm::out().set_file(name); 87 | } 88 | 89 | return Chooser::execute(c, it, n, cap, 90 | steps); 91 | } 92 | -------------------------------------------------------------------------------- /source/time_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "selection.hpp" 7 | #include "utils/command_line_parser.hpp" 8 | #include "utils/default_hash.hpp" 9 | #include "utils/output.hpp" 10 | #include "utils/pin_thread.hpp" 11 | namespace utm = utils_tm; 12 | namespace otm = utils_tm::out_tm; 13 | 14 | #ifdef MALLOC_COUNT 15 | #include "malloc_count.h" 16 | static constexpr bool malloc_mode = true; 17 | size_t get_malloc() { return malloc_count_current(); } 18 | #else 19 | #include 20 | static constexpr bool malloc_mode = false; 21 | constexpr size_t get_malloc() { return mallinfo2().uordblks(); } 22 | #endif 23 | #ifdef RSS_COUNT 24 | #include 25 | static constexpr bool rss_mode = true; 26 | size_t get_rss() 27 | { 28 | long rss = 0L; 29 | FILE* fp = NULL; 30 | if ((fp = fopen("/proc/self/statm", "r")) == NULL) 31 | return (size_t)0L; /* Can't open? */ 32 | if (fscanf(fp, "%*s%ld", &rss) != 1) 33 | { 34 | fclose(fp); 35 | return (size_t)0L; /* Can't read? */ 36 | } 37 | fclose(fp); 38 | return (size_t)rss; 39 | } 40 | #else 41 | static constexpr bool rss_mode = false; 42 | constexpr size_t get_rss() { return 0; } 43 | #endif 44 | 45 | 46 | template 47 | struct Test 48 | { 49 | // using table_type = ProbIndependentBase >; 51 | using table_type = 52 | HASHTYPE; 53 | 54 | int operator()(size_t it, size_t n, size_t cap, size_t steps, double alpha) 55 | { 56 | otm::out() << otm::width(4) << "# it" << otm::width(8) << "alpha"; 57 | table_type::print_init_header(otm::out()); 58 | otm::out() << otm::width(9) << "cap" << otm::width(9) << "n_full" 59 | << otm::width(10) << "t_in" << otm::width(10) << "t_find+" 60 | << otm::width(10) << "t_find-" << otm::width(9) << "in_err" 61 | << otm::width(9) << "fi_err"; 62 | if constexpr (malloc_mode) otm::out() << otm::width(7) << "memory"; 63 | if constexpr (rss_mode) otm::out() << otm::width(7) << "rss"; 64 | otm::out() << std::endl; 65 | 66 | constexpr size_t range = (1ull << 63) - 1; 67 | 68 | size_t* keys = new size_t[2 * n]; 69 | 70 | std::uniform_int_distribution dis(1, range); 71 | std::mt19937_64 re; 72 | 73 | for (size_t i = 0; i < 2 * n; ++i) { keys[i] = dis(re); } 74 | 75 | for (size_t i = 0; i < it; ++i) 76 | { 77 | size_t start_rss = get_rss(); 78 | 79 | table_type table(cap, alpha, steps); 80 | 81 | auto in_errors = 0ull; 82 | auto fin_errors = 0ull; 83 | 84 | auto t0 = std::chrono::high_resolution_clock::now(); 85 | for (size_t i = 0; i < n && in_errors < 100; ++i) 86 | { 87 | if (!table.insert(keys[i], i).second) ++in_errors; 88 | } 89 | auto t1 = std::chrono::high_resolution_clock::now(); 90 | 91 | [[maybe_unused]] size_t final_rss = get_rss() - start_rss; 92 | 93 | auto t2 = std::chrono::high_resolution_clock::now(); 94 | // const table_type& ctable = table; 95 | for (size_t i = 0; i < n; ++i) 96 | { 97 | auto e = table.find(keys[i]); 98 | if ((e == table.end()) || ((*e).second != i)) 99 | // if (ctable.at(keys[i]) != i) 100 | { 101 | fin_errors++; 102 | } 103 | } 104 | auto t3 = std::chrono::high_resolution_clock::now(); 105 | for (size_t i = n; i < 2 * n; ++i) 106 | { 107 | auto e = table.find(keys[i]); 108 | if ((e != table.end()) && (keys[(*e).second] != keys[i])) 109 | { 110 | fin_errors++; 111 | } 112 | } 113 | auto t4 = std::chrono::high_resolution_clock::now(); 114 | 115 | // double d_in0 = 116 | // std::chrono::duration_cast 117 | // (t1 - t0).count()/1000.; 118 | double d_in = 119 | std::chrono::duration_cast(t1 - t0) 120 | .count() / 121 | 1000.; 122 | double d_fn0 = 123 | std::chrono::duration_cast(t3 - t2) 124 | .count() / 125 | 1000.; 126 | double d_fn1 = 127 | std::chrono::duration_cast(t4 - t3) 128 | .count() / 129 | 1000.; 130 | 131 | otm::out() << otm::width(4) << i << otm::width(8) << alpha; 132 | table.print_init_data(otm::out()); 133 | otm::out() << otm::width(9) << cap << otm::width(9) << n 134 | << otm::width(10) << d_in << otm::width(10) << d_fn0 135 | << otm::width(10) << d_fn1 << otm::width(9) << in_errors 136 | << otm::width(9) << fin_errors; 137 | if constexpr (malloc_mode) 138 | otm::out() << otm::width(7) 139 | << double(get_malloc()) / double(8 * 2 * n) - 1.; 140 | if constexpr (rss_mode) otm::out() << otm::width(7) << final_rss; 141 | otm::out() << std::endl; 142 | } 143 | 144 | delete[] keys; 145 | 146 | return 0; 147 | } 148 | }; 149 | 150 | int main(int argn, char** argc) 151 | { 152 | utm::pin_to_core(0); 153 | utm::command_line_parser c(argn, argc); 154 | 155 | size_t it = c.int_arg("-it", 5); 156 | size_t n = c.int_arg("-n", 2000000); 157 | size_t cap = c.int_arg("-cap", n); 158 | size_t steps = c.int_arg("-steps", 512); 159 | 160 | double alpha = c.double_arg("-alpha", 1.1); 161 | double load = c.double_arg("-load", 2.0); 162 | double eps = c.double_arg("-eps", 1.0 - load); 163 | if (eps > 0.) alpha = 1. / (1. - eps); 164 | 165 | if (c.bool_arg("-out") || c.bool_arg("-file")) 166 | { 167 | std::string name = c.str_arg("-out", ""); 168 | name = c.str_arg("-file", name) + ".time"; 169 | otm::out().set_file(name); 170 | } 171 | 172 | return Chooser::execute(c, it, n, cap, steps, 173 | alpha); 174 | } 175 | -------------------------------------------------------------------------------- /utils_replaced/commandline.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMANDLINE_H 2 | #define COMMANDLINE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /* 11 | template 12 | class Ret 13 | { 14 | public: 15 | Ret() : p(false, T()) { } 16 | Ret(bool b, T t) : p(b , t ) { } 17 | private: 18 | std::pair p; 19 | 20 | public: 21 | operator T() const 22 | { 23 | return p.second; 24 | } 25 | 26 | operator bool() const 27 | { 28 | return p.first; 29 | } 30 | 31 | }; 32 | */ 33 | 34 | class CommandLine 35 | { 36 | public: 37 | CommandLine(int argn, char** argc) 38 | { 39 | std::setlocale(LC_ALL, "en_US.UTF-8"); 40 | for (size_t i = 0; i < size_t(argn); ++i) 41 | { 42 | params.emplace_back(argc[i]); 43 | flags .push_back (ParamCodes::unused); 44 | } 45 | } 46 | 47 | std::string strArg(const std::string& name, const std::string def = "") 48 | { 49 | auto ind = findName(name); 50 | if (ind+1 < params.size()) 51 | { 52 | flags[ind+1] = ParamCodes::used; 53 | return params[ind+1]; 54 | } 55 | else if (ind < params.size()) 56 | { 57 | flags[ind] = ParamCodes::error; 58 | std::cout << "found argument \"" << name << "\" without following integer!" 59 | << std::endl; 60 | } 61 | return def; 62 | } 63 | 64 | int intArg(const std::string& name, int def = 0) 65 | { 66 | auto ind = findName(name); 67 | if (ind+1 < params.size()) 68 | { 69 | flags[ind+1] = ParamCodes::used; 70 | int r = 0; 71 | try 72 | { r = std::stoi(params[ind+1]); } 73 | catch (std::invalid_argument& e) 74 | { 75 | flags[ind+1] = ParamCodes::error; 76 | r = def; 77 | std::cout << "error reading int argument \"" << name 78 | << "\" from console, got \"invalid_argument exception\"" 79 | << std::endl; 80 | } 81 | return r; 82 | } 83 | else if (ind < params.size()) 84 | { 85 | flags[ind] = ParamCodes::error; 86 | std::cout << "found argument \"" << name << "\" without following integer!" 87 | << std::endl; 88 | } 89 | return def; 90 | 91 | } 92 | 93 | double doubleArg(const std::string& name, double def = 0.) 94 | { 95 | std::setlocale(LC_ALL, "en_US.UTF-8"); 96 | auto ind = findName(name); 97 | if (ind+1 < params.size()) 98 | { 99 | flags[ind+1] = ParamCodes::used; 100 | double r = 0; 101 | try 102 | { r = std::stod(params[ind+1]); } 103 | catch (std::invalid_argument& e) 104 | { 105 | flags[ind+1] = ParamCodes::error; 106 | r = def; 107 | std::cout << "error reading double argument \"" << name 108 | << "\" from console, got \"invalid-argument exception\"!" 109 | << std::endl; 110 | } 111 | return r; 112 | } 113 | else if (ind < params.size()) 114 | { 115 | flags[ind] = ParamCodes::error; 116 | std::cout << "found argument \"" << name << "\" without following double!" 117 | << std::endl; 118 | } 119 | return def; 120 | } 121 | 122 | bool boolArg(const std::string& name) 123 | { 124 | return (findName(name)) < params.size(); 125 | } 126 | 127 | bool report() 128 | { 129 | bool un = true; 130 | for (size_t i = 1; i < params.size(); ++i) 131 | { 132 | if (flags[i] != ParamCodes::used) 133 | { 134 | if (flags[i] == ParamCodes::unused) 135 | { 136 | std::cout << "parameter " << i << " = \"" << params[i] 137 | << "\" was unused!" << std::endl; 138 | } 139 | else if ( flags[i] == ParamCodes::error) 140 | { 141 | std::cout << "error reading parameter " << i 142 | << " = \"" << params[i] << "\"" << std::endl; 143 | } 144 | un = false; 145 | } 146 | } 147 | return un; 148 | } 149 | 150 | private: 151 | enum class ParamCodes 152 | { 153 | unused, 154 | used, 155 | error 156 | }; 157 | 158 | std::vector params; 159 | std::vector flags; 160 | 161 | size_t findName(const std::string& name) 162 | { 163 | for (size_t i = 0; i < params.size(); ++i) 164 | { 165 | if (params[i] == name) 166 | { 167 | flags[i] = ParamCodes::used; 168 | return i; 169 | } 170 | } 171 | return params.size(); 172 | } 173 | 174 | }; 175 | 176 | 177 | #endif // COMMANDLINE_H 178 | -------------------------------------------------------------------------------- /utils_replaced/counting_wait.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNTING_WAIT_H 2 | #define COUNTING_WAIT_H 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | static long sys_futex(void *addr1, int op, int val1, struct timespec *timeout, void *addr2, int val3) 16 | { 17 | return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3); 18 | } 19 | 20 | class alignas(64) counting_wait 21 | { 22 | public: 23 | inline counting_wait(int start = 0) : counter(start) 24 | { 25 | if (sizeof(std::atomic_int) != 4) 26 | std::cout << "std::atomic_int has wrong size:" 27 | << sizeof(std::atomic_int)<< std::endl; 28 | } 29 | 30 | inline bool inc_if(int exp) 31 | { 32 | auto temp = exp; 33 | return counter.compare_exchange_strong(temp, exp+1, 34 | std::memory_order_acq_rel, 35 | std::memory_order_acquire); 36 | } 37 | 38 | inline bool wait_if(int exp) 39 | { 40 | //while (counter.load(std::memory_order_acquire) < l_epoch) ; //temporary should soon be removed 41 | auto ecode = sys_futex(&counter, FUTEX_WAIT, exp, NULL, NULL, 0); 42 | return !ecode; 43 | } 44 | 45 | inline uint wake(uint n_threads = 9999) 46 | { 47 | return sys_futex(&counter, FUTEX_WAKE, n_threads, NULL, NULL, 0); 48 | } 49 | 50 | private: 51 | std::atomic_int counter; 52 | }; 53 | 54 | 55 | #endif // COUNTING_WAIT_H 56 | -------------------------------------------------------------------------------- /utils_replaced/default_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /******************************************************************************* 4 | * utils/default_hash.h 5 | * 6 | * In this file we chose an adequat default hash function, if nothing 7 | * else is defined XXHASH is chosen. 8 | * 9 | * If you have any problems with third party codes try defining MURMUR2. 10 | * 11 | * Part of Project growt - https://github.com/TooBiased/growt.git 12 | * 13 | * Copyright (C) 2015-2016 Tobias Maier 14 | * 15 | * All rights reserved. Published under the BSD-2 license in the LICENSE file. 16 | ******************************************************************************/ 17 | 18 | 19 | #if (! (defined(CRC) || \ 20 | defined(MURMUR2) || \ 21 | defined(MURMUR3) || \ 22 | defined(XXHASH)) ) 23 | #define XXHASH 24 | #endif // NO HASH DEFINED 25 | 26 | #ifdef CRC 27 | #include "hash/crc_hash.h" 28 | #define HASHFCT dysect::hash::crc_hash 29 | namespace dysect{ 30 | namespace hash { 31 | 32 | using default_hash = crc_hash; 33 | 34 | } 35 | } 36 | #endif //CRC 37 | 38 | 39 | #ifdef MURMUR2 40 | #include "hash/murmur2_hash.h" 41 | #define HASHFCT dysect::hash::murmur2_hash 42 | namespace dysect{ 43 | namespace hash { 44 | 45 | using default_hash = murmur2_hash; 46 | 47 | } 48 | } 49 | #endif // MURMUR2 50 | 51 | 52 | #ifdef MURMUR3 53 | #include "hash/murmur3_hash.h" 54 | #define HASHFCT dysect::hash::murmur3_hash 55 | namespace dysect{ 56 | namespace hash { 57 | 58 | using default_hash = murmur3_hash; 59 | 60 | } 61 | } 62 | #endif // MURMUR3 63 | 64 | 65 | 66 | #ifdef XXHASH 67 | #include "hash/xx_hash.h" 68 | #define HASHFCT dysect::hash::xx_hash 69 | namespace dysect{ 70 | namespace hash { 71 | 72 | using default_hash = dysect::hash::xx_hash; 73 | 74 | } 75 | } 76 | #endif // XXHASH 77 | -------------------------------------------------------------------------------- /utils_replaced/hash/crc_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | 7 | namespace dysect { 8 | namespace hash { 9 | 10 | struct crc_hash 11 | { 12 | crc_hash(size_t seed = 12923598712359872066ull) 13 | : seed0(seed), seed1(seed*7467732452331123588ull) 14 | { } 15 | 16 | static const size_t significant_digits = 64; 17 | size_t seed0; 18 | size_t seed1; 19 | 20 | inline uint64_t operator()(const uint64_t& k) const 21 | { 22 | return uint64_t( __builtin_ia32_crc32di(k, seed0) 23 | | (__builtin_ia32_crc32di(k, seed1) << 32)); 24 | } 25 | }; 26 | 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /utils_replaced/hash/murmur2_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | 7 | namespace dysect { 8 | namespace hash { 9 | 10 | struct murmur2_hash 11 | { 12 | murmur2_hash(size_t s = 1203989050u) : seed(s) { } 13 | 14 | static constexpr size_t significant_digits = 64; 15 | //const 16 | size_t seed; 17 | 18 | inline uint64_t MurmurHash64A ( const void * key, int len, unsigned int seed ) const 19 | { 20 | const uint64_t m = 0xc6a4a7935bd1e995; 21 | const int r = 47; 22 | 23 | uint64_t h = seed ^ (len * m); 24 | 25 | const uint64_t * data = (const uint64_t *)key; 26 | const uint64_t * end = data + (len/8); 27 | 28 | while(data != end) 29 | { 30 | uint64_t k = *data++; 31 | 32 | k *= m; 33 | k ^= k >> r; 34 | k *= m; 35 | 36 | h ^= k; 37 | h *= m; 38 | } 39 | 40 | const unsigned char * data2 = (const unsigned char*)data; 41 | 42 | switch(len & 7) 43 | { 44 | case 7: h ^= uint64_t(data2[6]) << 48; 45 | case 6: h ^= uint64_t(data2[5]) << 40; 46 | case 5: h ^= uint64_t(data2[4]) << 32; 47 | case 4: h ^= uint64_t(data2[3]) << 24; 48 | case 3: h ^= uint64_t(data2[2]) << 16; 49 | case 2: h ^= uint64_t(data2[1]) << 8; 50 | case 1: h ^= uint64_t(data2[0]); 51 | h *= m; 52 | }; 53 | 54 | h ^= h >> r; 55 | h *= m; 56 | h ^= h >> r; 57 | 58 | return h; 59 | } 60 | 61 | inline uint64_t operator()(const uint64_t k) const 62 | { 63 | auto local = k; 64 | return MurmurHash64A(&local, 8, seed); 65 | } 66 | }; 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /utils_replaced/hash/murmur3_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // We include the cpp to avoid generating another compile unit 4 | #include "MurmurHash3.cpp" 5 | 6 | 7 | namespace dysect { 8 | namespace hash { 9 | 10 | struct murmur3_hash 11 | { 12 | murmur3_hash(size_t s = 1203989050u) : seed(s) { } 13 | 14 | static constexpr size_t significant_digits = 64; 15 | //const 16 | uint seed; 17 | 18 | inline uint64_t operator()(const uint64_t k) const 19 | { 20 | uint64_t local = k; 21 | uint64_t target[2]; 22 | 23 | MurmurHash3_x64_128 (&local, 8, seed, target); 24 | 25 | return target[0]; 26 | } 27 | }; 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /utils_replaced/hash/xx_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // this define ensures, that xxhash is inlined/does not create new compile unit 4 | #define XXH_PRIVATE_API 5 | #include "xxhash.h" 6 | 7 | 8 | namespace dysect { 9 | namespace hash { 10 | 11 | struct xx_hash 12 | { 13 | xx_hash(size_t s = 13358259232739045019ull) : seed(s) { } 14 | 15 | static constexpr size_t significant_digits = 64; 16 | //const 17 | size_t seed; 18 | 19 | inline uint64_t operator()(const uint64_t k) const 20 | { 21 | auto local = k; 22 | return XXH64 (&local, 8, seed); 23 | } 24 | }; 25 | 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /utils_replaced/test_coordination.h: -------------------------------------------------------------------------------- 1 | #ifndef STAGED_TEST 2 | #define STAGED_TEST 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | static std::atomic_size_t level; 11 | static std::atomic_size_t wait_end; 12 | static std::atomic_size_t wait_start; 13 | static std::atomic_size_t n_output; 14 | 15 | static std::chrono::time_point start; 16 | 17 | static thread_local std::chrono::time_point start_own; 18 | 19 | void reset_stages() 20 | { 21 | level.store(0); 22 | wait_end.store(0); 23 | wait_start.store(0); 24 | n_output.store(0); 25 | } 26 | 27 | 28 | // MAINTHREAD SYNCHRONIZATION PRIMITIVES 29 | // TIMED 30 | inline void start_stage_timed(size_t p, size_t lvl) 31 | { 32 | while(wait_start.load(std::memory_order_acquire) < p); 33 | wait_start.store(0, std::memory_order_release); 34 | 35 | start = std::chrono::high_resolution_clock::now(); 36 | level.store(lvl, std::memory_order_release); 37 | } 38 | 39 | inline size_t wait_for_finished_timed(size_t p) 40 | { 41 | while (wait_end.load(std::memory_order_acquire) < p); 42 | wait_end.store(0, std::memory_order_release); 43 | return std::chrono::duration_cast 44 | (std::chrono::high_resolution_clock::now() - start).count(); 45 | 46 | } 47 | 48 | // UNTIMED 49 | inline void start_stage(size_t p, size_t lvl) 50 | { 51 | while(wait_start.load(std::memory_order_acquire) < p); 52 | wait_start.store(0, std::memory_order_release); 53 | 54 | level.store(lvl, std::memory_order_release); 55 | } 56 | 57 | inline size_t wait_for_finished(size_t p) 58 | { 59 | while (wait_end.load(std::memory_order_acquire) < p); 60 | wait_end.store(0, std::memory_order_release); 61 | 62 | return 0ull; 63 | } 64 | 65 | 66 | 67 | // SUBTHREAD SYNCHRONIZATION PRIMITIVES 68 | // TIMED 69 | inline void wait_for_stage_local_timed(size_t lvl) 70 | { 71 | wait_start.fetch_add(1, std::memory_order_acq_rel); 72 | while (level.load(std::memory_order_acquire) < lvl); 73 | start_own = std::chrono::high_resolution_clock::now(); 74 | } 75 | 76 | inline size_t finished_stage_local_timed() 77 | { 78 | wait_end.fetch_add(1, std::memory_order_acq_rel); 79 | return std::chrono::duration_cast 80 | (std::chrono::high_resolution_clock::now() - start_own).count(); 81 | } 82 | 83 | // TIMED GLOBAL (only end because start is taken by mainthread) 84 | inline size_t finished_stage_global_timed() 85 | { 86 | wait_end.fetch_add(1, std::memory_order_acq_rel); 87 | return std::chrono::duration_cast 88 | (std::chrono::high_resolution_clock::now() - start).count(); 89 | } 90 | 91 | // UNTIMED 92 | inline void wait_for_stage(size_t lvl) 93 | { 94 | wait_start.fetch_add(1, std::memory_order_acq_rel); 95 | while (level.load(std::memory_order_acquire) < lvl); 96 | } 97 | 98 | inline size_t finished_stage() 99 | { 100 | wait_end.fetch_add(1, std::memory_order_acq_rel); 101 | return 0; 102 | } 103 | 104 | 105 | 106 | // STARTS P-1 TFunctor THREADS, THEN EXECUTES MFunctor, THEN REJOINS GENERATED THREADS 107 | template 108 | inline int start_threads(MFunctor mf, TFunctor tf, size_t p, Types&& ... param) 109 | { 110 | std::thread* local_thread = new std::thread[p-1]; 111 | 112 | for (size_t i = 0; i < p-1; ++i) 113 | { 114 | local_thread[i] = std::thread(std::forward(tf), p, i+1, 115 | std::ref(std::forward(param))...); 116 | } 117 | 118 | int temp = std::forward(mf)(p, 0, std::forward(param)...); 119 | 120 | // CLEANUP THREADS 121 | for (size_t i = 0; i < p-1; ++i) 122 | { 123 | local_thread[i].join(); 124 | } 125 | 126 | delete[] local_thread; 127 | 128 | return temp; 129 | } 130 | 131 | 132 | 133 | // MAIN THREAD CLASS 134 | template 135 | struct MainThread 136 | { 137 | template 138 | inline static std::pair::type, size_t> 139 | synchronized(Functor f, size_t stage, size_t p, Types&& ... param) 140 | { 141 | start(p, stage); //start_stage_timed(p, stage); 142 | auto temp = std::forward(f)(std::forward(param) ... ); 143 | return std::make_pair(std::move(temp), end(p)); //wait_for_finished_timed(p)); 144 | } 145 | 146 | template 147 | inline static typename std::result_of::type 148 | only_main(Functor f, Types&& ... param) 149 | { 150 | return std::forward(f)(std::forward(param) ... ); 151 | } 152 | 153 | 154 | static const bool is_main = true; 155 | 156 | 157 | template 158 | inline MainThread& operator<<(T&& t)//T&& t) 159 | { 160 | std::cout << std::forward(t); 161 | return *this; 162 | } 163 | 164 | 165 | using omanip_t = std::ostream& (*) (std::ostream &); 166 | inline MainThread& operator<<(omanip_t t) 167 | { 168 | std::cout << t; 169 | return *this; 170 | } 171 | 172 | template 173 | inline static void out(T t, size_t space) 174 | { 175 | std::cout.width(space); 176 | std::cout << t << " " << std::flush; 177 | } 178 | 179 | template 180 | inline static void outAll(size_t id, size_t p, T t, size_t space) 181 | { 182 | auto lvl = n_output.load(); 183 | 184 | while(wait_start.load() < id); 185 | std::cout.width(space); 186 | std::cout << t << " " << std::flush; 187 | wait_start.fetch_add(1); 188 | if (id == p-1) 189 | { 190 | n_output.fetch_add(1); 191 | } 192 | else while(n_output.load() <= lvl); 193 | 194 | while (wait_end.load() < p-1); 195 | wait_end.store(0); 196 | } 197 | }; 198 | 199 | using TimedMainThread = MainThread; 200 | using UnTimedMainThread = MainThread; 201 | 202 | 203 | 204 | // SUB THREAD CLASS 205 | template 206 | struct SubThread 207 | { 208 | template 209 | inline static std::pair::type, size_t> 210 | synchronized(Functor f, size_t stage, size_t /*p unused*/, Types&& ... param) 211 | { 212 | start(stage);//wait_for_stage(stage); 213 | auto temp = std::forward(f)(std::forward(param)...); 214 | //finished_stage(); 215 | return std::make_pair(temp, end()); 216 | } 217 | 218 | template 219 | inline static typename std::result_of::type only_main(Functor f, Types&& ... param) 220 | { 221 | return typename std::result_of::type(); 222 | } 223 | 224 | static const bool is_main = false; 225 | 226 | template 227 | inline SubThread& operator<<(T&& t) 228 | { 229 | return *this; 230 | } 231 | 232 | using omanip_t = std::ostream& (*) (std::ostream &); 233 | inline SubThread& operator<<(omanip_t) 234 | { 235 | return *this; 236 | } 237 | 238 | template 239 | inline static void out(T, size_t) 240 | { 241 | } 242 | 243 | template 244 | inline static void outAll(size_t id, size_t p, T t, size_t space) 245 | { 246 | auto lvl = n_output.load(); 247 | 248 | while(wait_start.load() < id); 249 | std::cout.width(space); 250 | std::cout << t << " " << std::flush; 251 | wait_start.fetch_add(1); 252 | if (id == p-1) 253 | { 254 | wait_start.store(0); 255 | n_output.fetch_add(1); 256 | } 257 | else while(n_output.load() <= lvl); 258 | wait_end.fetch_add(1); 259 | } 260 | }; 261 | 262 | //time is measured relative to the global start 263 | using LocTimedSubThread = SubThread; 264 | using GlobTimedSubThread = SubThread; 265 | using UnTimedSubThread = SubThread; 266 | 267 | 268 | 269 | 270 | 271 | static const size_t block_size = 4096; 272 | 273 | 274 | 275 | //BLOCKWISE EXECUTION IN PARALLEL 276 | template 277 | inline void execute_parallel(std::atomic_size_t& global_counter, size_t e, 278 | Functor f, Types&& ... param) 279 | { 280 | auto c_s = global_counter.fetch_add(block_size); 281 | while (c_s < e) 282 | { 283 | auto c_e = std::min(c_s+block_size, e); 284 | for (size_t i = c_s; i < c_e; ++i) 285 | f(i, std::forward(param)...); 286 | c_s = global_counter.fetch_add(block_size); 287 | } 288 | } 289 | 290 | template 291 | inline void execute_blockwise_parallel(std::atomic_size_t& global_counter, size_t e, 292 | Functor f, Types&& ... param) 293 | { 294 | auto c_s = global_counter.fetch_add(block_size); 295 | while (c_s < e) 296 | { 297 | auto c_e = std::min(c_s+block_size, e); 298 | f(c_s, c_e, std::forward(param)...); 299 | c_s = global_counter.fetch_add(block_size); 300 | } 301 | } 302 | #endif // STAGED_TEST 303 | 304 | -------------------------------------------------------------------------------- /utils_replaced/thread_basics.h: -------------------------------------------------------------------------------- 1 | #ifndef THREAD_BASICS_H 2 | #define THREAD_BASICS_H 3 | 4 | #include 5 | 6 | void pin_to_core(size_t core) 7 | { 8 | cpu_set_t cpuset; 9 | CPU_ZERO(&cpuset); 10 | CPU_SET(core, &cpuset); 11 | pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); 12 | } 13 | 14 | void set_thread_priority(uint pri) 15 | { 16 | struct sched_param param; 17 | param.sched_priority = sched_get_priority_min(pri); 18 | 19 | pthread_setschedparam(pthread_self(), SCHED_RR, ¶m); 20 | } 21 | 22 | #endif // THREAD_BASICS_H 23 | --------------------------------------------------------------------------------