├── .gitignore
├── .gitmodules
├── Makefile
├── README.md
├── test.cc
└── lockfree_hashtable.h


/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | test
3 | *.o
4 | info.plist


--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "HazardPointer"]
2 | 	path = HazardPointer
3 | 	url = https://github.com/bhhbazinga/HazardPointer
4 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | CXX = g++
 2 | CXXFLAGS = -Wall -Wextra -pedantic -std=c++2a -g -o3
 3 | #-fsanitize=address -fsanitize=leak
 4 | #-fsanitize=thread
 5 | EXEC = test
 6 | 
 7 | all: $(EXEC)
 8 | 
 9 | $(EXEC):  test.cc lockfree_hashtable.h HazardPointer/reclaimer.h
10 | 	$(CXX) $(CXXFLAGS) test.cc -o $@ -lpthread 
11 | 
12 | HazardPointer/reclaimer.h:
13 | 	git submodule update --init
14 | 
15 | clean:
16 | 	rm -rf  $(EXEC)
17 | 
18 | .Phony:
19 | 	clean
20 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # LockFreeHashTable
 2 | Lock Free Resizable Hash Table Based On Split-Ordered Lists.
 3 | ## Feature
 4 |   * Thread-safe and Lock-free.
 5 |   * ABA safe.
 6 |   * Support Multi-producer & Multi-consumer.
 7 |   * Use Hazard Pointer to manage memory.
 8 |   * Lock Free LinkedList base on Harris' ListBasedSet, see also [LockFreeLinkedList](https://github.com/bhhbazinga/LockFreeLinkedList)
 9 |   * Resize without waiting.
10 | ## Benchmark
11 |   Magnitude     | Insert      | Find       | Delete     | Insert&Find&Delete|
12 |   :-----------  | :-----------| :----------|:-----------| :-----------------
13 |   10K           | 7.8ms       | 1.6ms      | 2.1ms      | 10.4ms
14 |   100K          | 74.9ms      | 13.4ms     | 18.4ms     | 111.2ms
15 |   1000K         | 961ms       | 117.8ms    | 216.1ms    | 1264.1ms
16 |   
17 | The above data was tested on my 2013 macbook-pro with Intel Core i7 4 cores 2.3 GHz.
18 | 
19 | The data of first three column was obtained by starting 8 threads to insert concurrently, find concurrently, delete concurrently, the data of four column was obtained by starting 2 threads to insert, 2 threads to find, 2 threads to delete concurrently, each looped 10 times to calculate the average time consumption.
20 | See also [test](test.cc).
21 | ## Build
22 | ```
23 | make && ./test
24 | ```
25 | ## API
26 | ```C++
27 | bool Insert(const K& key, const V& value);
28 | bool Insert(const K& key, V&& value);
29 | bool Insert(K&& key, const V& value);
30 | bool Insert(K&& key, V&& value);
31 | bool Find(const K& key, V& value);
32 | bool Delete(const T& data);
33 | size_t size() const;
34 | ```
35 | ## TODO List
36 | - [ ] Shrink Hash Table without waiting.
37 | ## Reference
38 | [1]A Pragmatic Implementation of Non-BlockingLinked-Lists. Timothy L.Harris\
39 | [2]Hazard Pointers: Safe Memory Reclamation for Lock-Free Objects. Maged M. Michael\
40 | [3]Split-Ordered Lists: Lock-Free Extensible Hash Tables. Tel-Aviv University and Sun Microsystems Laboratories, Tel-Aviv, Israel
41 | 


--------------------------------------------------------------------------------
/test.cc:
--------------------------------------------------------------------------------
  1 | #include <chrono>
  2 | #include <iostream>
  3 | #include <memory>
  4 | #include <random>
  5 | #include <thread>
  6 | #include <unordered_map>
  7 | #include <vector>
  8 | 
  9 | #include "lockfree_hashtable.h"
 10 | 
 11 | const int kMaxThreads = std::thread::hardware_concurrency();
 12 | 
 13 | int maxElements;
 14 | LockFreeHashTable<int, int> ht;
 15 | 
 16 | // Insert sucessfully then ++cnt,  delete succesfully then --cnt.
 17 | std::atomic<int> cnt = 0;
 18 | std::atomic<bool> start = false;
 19 | std::unordered_map<int, int*> elements2timespan;
 20 | 
 21 | void onInsert(int divide) {
 22 |   while (!start) {
 23 |     std::this_thread::yield();
 24 |   }
 25 | 
 26 |   int n = maxElements / divide;
 27 |   for (int i = 0; i < n; ++i) {
 28 |     int x = rand() % n;
 29 |     if (ht.Insert(x, x)) {
 30 |       ++cnt;
 31 |     }
 32 |   }
 33 | }
 34 | 
 35 | void onFind(int divide) {
 36 |   while (!start) {
 37 |     std::this_thread::yield();
 38 |   }
 39 | 
 40 |   int n = maxElements / divide;
 41 |   for (int i = 0; i < n; ++i) {
 42 |     int x;
 43 |     ht.Find(rand() % n, x);
 44 |   }
 45 | }
 46 | 
 47 | void onDelete(int divide) {
 48 |   while (!start) {
 49 |     std::this_thread::yield();
 50 |   }
 51 | 
 52 |   int n = maxElements / divide;
 53 |   for (int i = 0; i < n; ++i) {
 54 |     if (ht.Delete(rand() % n)) {
 55 |       --cnt;
 56 |     }
 57 |   }
 58 | }
 59 | 
 60 | void TestConcurrentInsert() {
 61 |   int old_size = ht.size();
 62 |   std::vector<std::thread> threads;
 63 |   for (int i = 0; i < kMaxThreads; ++i) {
 64 |     threads.push_back(std::thread(onInsert, kMaxThreads));
 65 |   }
 66 | 
 67 |   start = true;
 68 |   auto t1_ = std::chrono::steady_clock::now();
 69 |   for (int i = 0; i < kMaxThreads; ++i) {
 70 |     threads[i].join();
 71 |   }
 72 |   auto t2_ = std::chrono::steady_clock::now();
 73 | 
 74 |   assert(cnt + old_size == static_cast<int>(ht.size()));
 75 |   int ms =
 76 |       std::chrono::duration_cast<std::chrono::milliseconds>(t2_ - t1_).count();
 77 |   elements2timespan[maxElements][0] += ms;
 78 |   std::cout << maxElements << " elements insert concurrently, timespan=" << ms
 79 |             << "ms"
 80 |             << "\n";
 81 |   start = false;
 82 | }
 83 | 
 84 | void TestConcurrentFind() {
 85 |   std::vector<std::thread> threads;
 86 |   for (int i = 0; i < kMaxThreads; ++i) {
 87 |     threads.push_back(std::thread(onFind, kMaxThreads));
 88 |   }
 89 | 
 90 |   start = true;
 91 |   auto t1_ = std::chrono::steady_clock::now();
 92 |   for (int i = 0; i < kMaxThreads; ++i) {
 93 |     threads[i].join();
 94 |   }
 95 |   auto t2_ = std::chrono::steady_clock::now();
 96 | 
 97 |   int ms =
 98 |       std::chrono::duration_cast<std::chrono::milliseconds>(t2_ - t1_).count();
 99 |   elements2timespan[maxElements][1] += ms;
100 |   std::cout << maxElements << " elements find concurrently, timespan=" << ms
101 |             << "ms"
102 |             << "\n";
103 | 
104 |   start = false;
105 | }
106 | 
107 | void TestConcurrentDelete() {
108 |   int old_size = ht.size();
109 |   std::vector<std::thread> threads;
110 |   for (int i = 0; i < kMaxThreads; ++i) {
111 |     threads.push_back(std::thread(onDelete, kMaxThreads));
112 |   }
113 | 
114 |   cnt = 0;
115 |   start = true;
116 |   auto t1_ = std::chrono::steady_clock::now();
117 |   for (int i = 0; i < kMaxThreads; ++i) {
118 |     threads[i].join();
119 |   }
120 |   auto t2_ = std::chrono::steady_clock::now();
121 | 
122 |   assert(cnt + old_size == static_cast<int>(ht.size()));
123 |   int ms =
124 |       std::chrono::duration_cast<std::chrono::milliseconds>(t2_ - t1_).count();
125 |   elements2timespan[maxElements][2] += ms;
126 |   std::cout << maxElements << " elements delete concurrently, timespan=" << ms
127 |             << "ms"
128 |             << "\n";
129 | 
130 |   cnt = 0;
131 |   start = false;
132 | }
133 | 
134 | void TestConcurrentInsertAndFindAndDequeue() {
135 |   int old_size = ht.size();
136 | 
137 |   int divide = kMaxThreads / 3;
138 |   std::vector<std::thread> threads;
139 |   for (int i = 0; i < divide; ++i) {
140 |     threads.push_back(std::thread(onInsert, divide));
141 |     threads.push_back(std::thread(onFind, divide));
142 |     threads.push_back(std::thread(onDelete, divide));
143 |   }
144 | 
145 |   cnt = 0;
146 |   start = true;
147 |   auto t1_ = std::chrono::steady_clock::now();
148 |   for (int i = 0; i < threads.size(); ++i) {
149 |     threads[i].join();
150 |   }
151 |   auto t2_ = std::chrono::steady_clock::now();
152 | 
153 |   assert(cnt + old_size == static_cast<int>(ht.size()));
154 |   int ms =
155 |       std::chrono::duration_cast<std::chrono::milliseconds>(t2_ - t1_).count();
156 |   elements2timespan[maxElements][3] += ms;
157 |   std::cout << maxElements
158 |             << " elements insert & find & delete concurrently, timespan=" << ms
159 |             << "ms"
160 |             << "\n";
161 | 
162 |   cnt = 0;
163 |   start = false;
164 | }
165 | 
166 | const int kElements1 = 10000;
167 | const int kElements2 = 100000;
168 | const int kElements3 = 1000000;
169 | 
170 | int main(int argc, char const* argv[]) {
171 |   (void)argc;
172 |   (void)argv;
173 | 
174 |   srand(std::time(0));
175 | 
176 |   std::cout << "Benchmark with " << kMaxThreads << " threads:"
177 |             << "\n";
178 | 
179 |   int elements[] = {kElements1, kElements2, kElements3};
180 |   int timespan1[] = {0, 0, 0, 0};
181 |   int timespan2[] = {0, 0, 0, 0};
182 |   int timespan3[] = {0, 0, 0, 0};
183 | 
184 |   elements2timespan[kElements1] = timespan1;
185 |   elements2timespan[kElements2] = timespan2;
186 |   elements2timespan[kElements3] = timespan3;
187 | 
188 |   for (int i = 0; i < 10; ++i) {
189 |     for (int j = 0; j < 3; ++j) {
190 |       maxElements = elements[j];
191 |       TestConcurrentInsert();
192 |       TestConcurrentFind();
193 |       TestConcurrentDelete();
194 |       TestConcurrentInsertAndFindAndDequeue();
195 |       std::cout << "\n";
196 |     }
197 |   }
198 | 
199 |   for (int i = 0; i < 3; ++i) {
200 |     maxElements = elements[i];
201 |     float avg = static_cast<float>(elements2timespan[maxElements][0]) / 10.0f;
202 |     std::cout << maxElements
203 |               << " elements insert concurrently, average timespan=" << avg
204 |               << "ms"
205 |               << "\n";
206 |     avg = static_cast<float>(elements2timespan[maxElements][1]) / 10.0f;
207 |     std::cout << maxElements
208 |               << " elements find concurrently, average timespan=" << avg << "ms"
209 |               << "\n";
210 |     avg = static_cast<float>(elements2timespan[maxElements][2]) / 10.0f;
211 |     std::cout << maxElements
212 |               << " elements delete concurrently, average timespan=" << avg
213 |               << "ms"
214 |               << "\n";
215 |     avg = static_cast<float>(elements2timespan[maxElements][3]) / 10.0f;
216 |     std::cout << maxElements
217 |               << " elements insert & find & delete concurrently, average timespan="
218 |               << avg << "ms"
219 |               << "\n";
220 |     std::cout << "\n";
221 |   }
222 | 
223 |   return 0;
224 | }


--------------------------------------------------------------------------------
/lockfree_hashtable.h:
--------------------------------------------------------------------------------
  1 | #ifndef LOCKFREE_HASHTABLE_H
  2 | #define LOCKFREE_HASHTABLE_H
  3 | 
  4 | #include <atomic>
  5 | #include <cassert>
  6 | #include <cmath>
  7 | 
  8 | #include "HazardPointer/reclaimer.h"
  9 | 
 10 | // The maximum bucket size equals to kSegmentSize^kMaxLevel, in this case the
 11 | // maximum bucket size is 64^4. If the load factor is 0.5, the maximum number of
 12 | // items that Hash Table contains is 64^4 * 0.5 = 2^23. You can adjust the
 13 | // following two values according to your memory size.
 14 | const int kMaxLevel = 4;
 15 | const int kSegmentSize = 64;
 16 | const size_t kMaxBucketSize = pow(kSegmentSize, kMaxLevel);
 17 | 
 18 | // Hash Table can be stored 2^power_of_2_ * kLoadFactor items.
 19 | const float kLoadFactor = 0.5;
 20 | 
 21 | template <typename K, typename V>
 22 | class TableReclaimer;
 23 | 
 24 | template <typename K, typename V, typename Hash = std::hash<K>>
 25 | class LockFreeHashTable {
 26 |   static_assert(std::is_copy_constructible_v<K>, "K requires copy constructor");
 27 |   static_assert(std::is_copy_constructible_v<V>, "V requires copy constructor");
 28 |   friend TableReclaimer<K, V>;
 29 | 
 30 |   struct Node;
 31 |   struct DummyNode;
 32 |   struct RegularNode;
 33 |   struct Segment;
 34 | 
 35 |   typedef size_t HashKey;
 36 |   typedef size_t BucketIndex;
 37 |   typedef size_t SegmentIndex;
 38 |   typedef std::atomic<DummyNode*> Bucket;
 39 | 
 40 |  public:
 41 |   LockFreeHashTable() : power_of_2_(1), size_(0), hash_func_(Hash()) {
 42 |     // Initialize first bucket
 43 |     int level = 1;
 44 |     Segment* segments = segments_;  // Point to current segment.
 45 |     while (level++ <= kMaxLevel - 2) {
 46 |       Segment* sub_segments = NewSegments(level);
 47 |       segments[0].data.store(sub_segments, std::memory_order_release);
 48 |       segments = sub_segments;
 49 |     }
 50 | 
 51 |     Bucket* buckets = NewBuckets();
 52 |     segments[0].data.store(buckets, std::memory_order_release);
 53 | 
 54 |     DummyNode* head = new DummyNode(0);
 55 |     buckets[0].store(head, std::memory_order_release);
 56 |     head_ = head;
 57 |   }
 58 | 
 59 |   ~LockFreeHashTable() {
 60 |     Node* p = head_;
 61 |     while (p != nullptr) {
 62 |       Node* tmp = p;
 63 |       p = p->next.load(std::memory_order_acquire);
 64 |       tmp->Release();
 65 |     }
 66 |   }
 67 | 
 68 |   LockFreeHashTable(const LockFreeHashTable& other) = delete;
 69 |   LockFreeHashTable(LockFreeHashTable&& other) = delete;
 70 |   LockFreeHashTable& operator=(const LockFreeHashTable& other) = delete;
 71 |   LockFreeHashTable& operator=(LockFreeHashTable&& other) = delete;
 72 | 
 73 |   bool Insert(const K& key, const V& value) {
 74 |     RegularNode* new_node = new RegularNode(key, value, hash_func_);
 75 |     DummyNode* head = GetBucketHeadByHash(new_node->hash);
 76 |     return InsertRegularNode(head, new_node);
 77 |   }
 78 | 
 79 |   bool Insert(K&& key, const V& value) {
 80 |     RegularNode* new_node = new RegularNode(std::move(key), value, hash_func_);
 81 |     DummyNode* head = GetBucketHeadByHash(new_node->hash);
 82 |     return InsertRegularNode(head, new_node);
 83 |   }
 84 | 
 85 |   bool Insert(const K& key, V&& value) {
 86 |     RegularNode* new_node = new RegularNode(key, std::move(value), hash_func_);
 87 |     DummyNode* head = GetBucketHeadByHash(new_node->hash);
 88 |     return InsertRegularNode(head, new_node);
 89 |   }
 90 | 
 91 |   bool Insert(K&& key, V&& value) {
 92 |     RegularNode* new_node =
 93 |         new RegularNode(std::move(key), std::move(value), hash_func_);
 94 |     DummyNode* head = GetBucketHeadByHash(new_node->hash);
 95 |     return InsertRegularNode(head, new_node);
 96 |   }
 97 | 
 98 |   bool Delete(const K& key) {
 99 |     HashKey hash = hash_func_(key);
100 |     DummyNode* head = GetBucketHeadByHash(hash);
101 |     RegularNode delete_node(key, hash_func_);
102 |     return DeleteNode(head, &delete_node);
103 |   }
104 | 
105 |   bool Find(const K& key, V& value) {
106 |     HashKey hash = hash_func_(key);
107 |     DummyNode* head = GetBucketHeadByHash(hash);
108 |     RegularNode find_node(key, hash_func_);
109 |     return FindNode(head, &find_node, value);
110 |   };
111 | 
112 |   size_t size() const { return size_.load(std::memory_order_relaxed); }
113 | 
114 |  private:
115 |   size_t bucket_size() const {
116 |     return 1 << power_of_2_.load(std::memory_order_relaxed);
117 |   }
118 | 
119 |   Segment* NewSegments(int level) {
120 |     Segment* segments = new Segment[kSegmentSize];
121 |     for (int i = 0; i < kSegmentSize; ++i) {
122 |       segments[i].level = level;
123 |       segments[i].data.store(nullptr, std::memory_order_release);
124 |     }
125 |     return segments;
126 |   }
127 | 
128 |   Bucket* NewBuckets() {
129 |     Bucket* buckets = new Bucket[kSegmentSize];
130 |     for (int i = 0; i < kSegmentSize; ++i) {
131 |       buckets[i].store(nullptr, std::memory_order_release);
132 |     }
133 |     return buckets;
134 |   }
135 | 
136 |   // Initialize bucket recursively.
137 |   DummyNode* InitializeBucket(BucketIndex bucket_index);
138 | 
139 |   // When the table size is 2^i , a logical table bucket b contains items whose
140 |   // keys k maintain k mod 2^i = b. When the size becomes 2^i+1, the items of
141 |   // this bucket are split into two buckets: some remain in the bucket b, and
142 |   // others, for which k mod 2^(i+1) == b + 2^i.
143 |   BucketIndex GetBucketParent(BucketIndex bucket_index) const {
144 |     //__builtin_clzl: Get number of leading zero bits.
145 |     // Unset the MSB(most significant bit) of bucket_index;
146 |     return (~(0x8000000000000000 >> (__builtin_clzl(bucket_index))) &
147 |             bucket_index);
148 |   };
149 | 
150 |   // Get the head node of bucket, if bucket not exist then return nullptr or
151 |   // return head.
152 |   DummyNode* GetBucketHeadByIndex(BucketIndex bucket_index);
153 | 
154 |   // Get the head node of bucket, if bucket not exist then initialize it and
155 |   // return head.
156 |   DummyNode* GetBucketHeadByHash(HashKey hash) {
157 |     BucketIndex bucket_index = (hash & (bucket_size() - 1));
158 |     DummyNode* head = GetBucketHeadByIndex(bucket_index);
159 |     if (nullptr == head) {
160 |       head = InitializeBucket(bucket_index);
161 |     }
162 |     return head;
163 |   }
164 | 
165 |   // Harris' OrderedListBasedset with Michael's hazard pointer to manage memory,
166 |   // See also https://github.com/bhhbazinga/LockFreeLinkedList.
167 |   bool InsertRegularNode(DummyNode* head, RegularNode* new_node);
168 |   bool InsertDummyNode(DummyNode* head, DummyNode* new_node,
169 |                        DummyNode** real_head);
170 |   bool DeleteNode(DummyNode* head, Node* delete_node);
171 |   bool FindNode(DummyNode* head, RegularNode* find_node, V& value) {
172 |     Node* prev;
173 |     Node* cur;
174 |     HazardPointer prev_hp, cur_hp;
175 |     bool found = SearchNode(head, find_node, &prev, &cur, prev_hp, cur_hp);
176 |     auto& reclaimer = TableReclaimer<K, V>::GetInstance();
177 |     if (found) {
178 |       V* value_ptr;
179 |       V* temp;
180 |       do {
181 |         // When find and insert concurrently value may be deleted,
182 |         // see InsertRegularNode, so value must be marked as hazard.
183 |         value_ptr = static_cast<RegularNode*>(cur)->value.load(
184 |             std::memory_order_consume);
185 |         temp = value_ptr;
186 |         value_ptr = static_cast<RegularNode*>(cur)->value.load(
187 |             std::memory_order_consume);
188 |       } while (temp != value_ptr);
189 |       reclaimer.ReclaimNoHazardPointer();
190 |       value = *value_ptr;
191 |     }
192 |     return found;
193 |   }
194 | 
195 |   // Traverse list begin with head until encounter nullptr or the first node
196 |   // which is greater than or equals to the given search_node.
197 |   bool SearchNode(DummyNode* head, Node* search_node, Node** prev_ptr,
198 |                   Node** cur_ptr, HazardPointer& prev_hp,
199 |                   HazardPointer& cur_hp);
200 | 
201 |   // Compare two nodes according to their reverse_hash and the key.
202 |   bool Less(Node* node1, Node* node2) const {
203 |     if (node1->reverse_hash != node2->reverse_hash) {
204 |       return node1->reverse_hash < node2->reverse_hash;
205 |     }
206 | 
207 |     if (node1->IsDummy() || node2->IsDummy()) {
208 |       // When initialize bucket concurrently, that could happen.
209 |       return false;
210 |     }
211 | 
212 |     return static_cast<RegularNode*>(node1)->key <
213 |            static_cast<RegularNode*>(node2)->key;
214 |   }
215 | 
216 |   bool GreaterOrEquals(Node* node1, Node* node2) const {
217 |     return !(Less(node1, node2));
218 |   }
219 | 
220 |   bool Equals(Node* node1, Node* node2) const {
221 |     return !Less(node1, node2) && !Less(node2, node1);
222 |   }
223 | 
224 |   bool is_marked_reference(Node* next) const {
225 |     return (reinterpret_cast<unsigned long>(next) & 0x1) == 0x1;
226 |   }
227 | 
228 |   Node* get_marked_reference(Node* next) const {
229 |     return reinterpret_cast<Node*>(reinterpret_cast<unsigned long>(next) | 0x1);
230 |   }
231 | 
232 |   Node* get_unmarked_reference(Node* next) const {
233 |     return reinterpret_cast<Node*>(reinterpret_cast<unsigned long>(next) &
234 |                                    ~0x1);
235 |   }
236 | 
237 |   static void OnDeleteNode(void* ptr) { delete static_cast<Node*>(ptr); }
238 | 
239 |   struct Node {
240 |     Node(HashKey hash_, bool dummy)
241 |         : hash(hash_),
242 |           reverse_hash(dummy ? DummyKey(hash) : RegularKey(hash)),
243 |           next(nullptr) {}
244 | 
245 |     virtual void Release() = 0;
246 | 
247 |     virtual ~Node() {}
248 | 
249 |     HashKey Reverse(HashKey hash) const {
250 |       return reverse8bits_[hash & 0xff] << 56 |
251 |              reverse8bits_[(hash >> 8) & 0xff] << 48 |
252 |              reverse8bits_[(hash >> 16) & 0xff] << 40 |
253 |              reverse8bits_[(hash >> 24) & 0xff] << 32 |
254 |              reverse8bits_[(hash >> 32) & 0xff] << 24 |
255 |              reverse8bits_[(hash >> 40) & 0xff] << 16 |
256 |              reverse8bits_[(hash >> 48) & 0xff] << 8 |
257 |              reverse8bits_[(hash >> 56) & 0xff];
258 |     }
259 |     HashKey RegularKey(HashKey hash) const {
260 |       return Reverse(hash | 0x8000000000000000);
261 |     }
262 |     HashKey DummyKey(HashKey hash) const { return Reverse(hash); }
263 | 
264 |     virtual bool IsDummy() const { return (reverse_hash & 0x1) == 0; }
265 |     Node* get_next() const { return next.load(std::memory_order_acquire); }
266 | 
267 |     const HashKey hash;
268 |     const HashKey reverse_hash;
269 |     std::atomic<Node*> next;
270 |   };
271 | 
272 |   // Head node of bucket
273 |   struct DummyNode : Node {
274 |     DummyNode(BucketIndex bucket_index) : Node(bucket_index, true) {}
275 |     ~DummyNode() override {}
276 | 
277 |     void Release() override { delete this; }
278 | 
279 |     bool IsDummy() const override { return true; }
280 |   };
281 | 
282 |   struct RegularNode : Node {
283 |     RegularNode(const K& key_, const V& value_, const Hash& hash_func)
284 |         : Node(hash_func(key_), false), key(key_), value(new V(value_)) {}
285 |     RegularNode(const K& key_, V&& value_, const Hash& hash_func)
286 |         : Node(hash_func(key_), false),
287 |           key(key_),
288 |           value(new V(std::move(value_))) {}
289 |     RegularNode(K&& key_, const V& value_, const Hash& hash_func)
290 |         : Node(hash_func(key_), false),
291 |           key(std::move(key_)),
292 |           value(new V(value_)) {}
293 |     RegularNode(K&& key_, V&& value_, const Hash& hash_func)
294 |         : Node(hash_func(key_), false),
295 |           key(std::move(key_)),
296 |           value(new V(std::move(value_))) {}
297 | 
298 |     RegularNode(const K& key_, const Hash& hash_func)
299 |         : Node(hash_func(key_), false), key(key_), value(nullptr) {}
300 | 
301 |     ~RegularNode() override {
302 |       V* ptr = value.load(std::memory_order_consume);
303 |       if (ptr != nullptr)
304 |         delete ptr;  // If update a node, value of this node is nullptr.
305 |     }
306 | 
307 |     void Release() override { delete this; }
308 | 
309 |     bool IsDummy() const override { return false; }
310 | 
311 |     const K key;
312 |     std::atomic<V*> value;
313 |   };
314 | 
315 |   struct Segment {
316 |     Segment() : level(1), data(nullptr) {}
317 |     explicit Segment(int level_) : level(level_), data(nullptr) {}
318 | 
319 |     Bucket* get_sub_buckets() const {
320 |       return static_cast<Bucket*>(data.load(std::memory_order_consume));
321 |     }
322 | 
323 |     Segment* get_sub_segments() const {
324 |       return static_cast<Segment*>(data.load(std::memory_order_consume));
325 |     }
326 | 
327 |     ~Segment() {
328 |       void* ptr = data.load(std::memory_order_consume);
329 |       if (nullptr == ptr) return;
330 | 
331 |       if (level == kMaxLevel - 1) {
332 |         Bucket* buckets = static_cast<Bucket*>(ptr);
333 |         delete[] buckets;
334 |       } else {
335 |         Segment* sub_segments = static_cast<Segment*>(ptr);
336 |         delete[] sub_segments;
337 |       }
338 |     }
339 | 
340 |     int level;                // Level of segment.
341 |     std::atomic<void*> data;  // If level == kMaxLevel then data point to
342 |                               // buckets else data point to segments.
343 |   };
344 | 
345 |   std::atomic<size_t> power_of_2_;   // Bucket size == 2^power_of_2_.
346 |   std::atomic<size_t> size_;         // Item size.
347 |   Hash hash_func_;                   // Hash function.
348 |   Segment segments_[kSegmentSize];   // Top level sengments.
349 |   static size_t reverse8bits_[256];  // Lookup table for reverse bits quickly.
350 |   DummyNode* head_;                  // Head of linkedlist.
351 |   static Reclaimer::HazardPointerList global_hp_list_;
352 | };
353 | 
354 | template <typename K, typename V, typename Hash>
355 | Reclaimer::HazardPointerList LockFreeHashTable<K, V, Hash>::global_hp_list_;
356 | 
357 | template <typename K, typename V>
358 | class TableReclaimer : public Reclaimer {
359 |   friend LockFreeHashTable<K, V>;
360 | 
361 |  private:
362 |   TableReclaimer(HazardPointerList& hp_list) : Reclaimer(hp_list) {}
363 |   ~TableReclaimer() override = default;
364 | 
365 |   static TableReclaimer<K, V>& GetInstance() {
366 |     thread_local static TableReclaimer reclaimer(
367 |         LockFreeHashTable<K, V>::global_hp_list_);
368 |     return reclaimer;
369 |   }
370 | };
371 | 
372 | // Fast reverse bits using Lookup Table.
373 | #define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
374 | #define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
375 | #define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
376 | // Lookup Table that store the reverse of each 8bit number.
377 | template <typename K, typename V, typename Hash>
378 | size_t LockFreeHashTable<K, V, Hash>::reverse8bits_[256] = {R6(0), R6(2), R6(1),
379 |                                                             R6(3)};
380 | 
381 | template <typename K, typename V, typename Hash>
382 | typename LockFreeHashTable<K, V, Hash>::DummyNode*
383 | LockFreeHashTable<K, V, Hash>::InitializeBucket(BucketIndex bucket_index) {
384 |   BucketIndex parent_index = GetBucketParent(bucket_index);
385 |   DummyNode* parent_head = GetBucketHeadByIndex(parent_index);
386 |   if (nullptr == parent_head) {
387 |     parent_head = InitializeBucket(parent_index);
388 |   }
389 | 
390 |   int level = 1;
391 |   Segment* segments = segments_;  // Point to current segment.
392 |   while (level++ <= kMaxLevel - 2) {
393 |     Segment& cur_segment =
394 |         segments[(bucket_index / static_cast<SegmentIndex>(pow(
395 |                                      kSegmentSize, kMaxLevel - level + 1))) %
396 |                  kSegmentSize];
397 |     Segment* sub_segments = cur_segment.get_sub_segments();
398 |     if (nullptr == sub_segments) {
399 |       // Try allocate segments.
400 |       sub_segments = NewSegments(level);
401 |       void* expected = nullptr;
402 |       if (!cur_segment.data.compare_exchange_strong(
403 |               expected, sub_segments, std::memory_order_release)) {
404 |         delete[] sub_segments;
405 |         sub_segments = static_cast<Segment*>(expected);
406 |       }
407 |     }
408 |     segments = sub_segments;
409 |   }
410 | 
411 |   Segment& cur_segment = segments[(bucket_index / kSegmentSize) % kSegmentSize];
412 |   Bucket* buckets = cur_segment.get_sub_buckets();
413 |   if (nullptr == buckets) {
414 |     // Try allocate buckets.
415 |     void* expected = nullptr;
416 |     buckets = NewBuckets();
417 |     if (!cur_segment.data.compare_exchange_strong(expected, buckets,
418 |                                                   std::memory_order_release)) {
419 |       delete[] buckets;
420 |       buckets = static_cast<Bucket*>(expected);
421 |     }
422 |   }
423 | 
424 |   Bucket& bucket = buckets[bucket_index % kSegmentSize];
425 |   DummyNode* head = bucket.load(std::memory_order_consume);
426 |   if (nullptr == head) {
427 |     // Try allocate dummy head.
428 |     head = new DummyNode(bucket_index);
429 |     DummyNode* real_head;  // If insert failed, real_head is the head of bucket.
430 |     if (InsertDummyNode(parent_head, head, &real_head)) {
431 |       // Dummy head must be inserted into the list before storing into bucket.
432 |       bucket.store(head, std::memory_order_release);
433 |     } else {
434 |       delete head;
435 |       head = real_head;
436 |     }
437 |   }
438 |   return head;
439 | }
440 | 
441 | template <typename K, typename V, typename Hash>
442 | typename LockFreeHashTable<K, V, Hash>::DummyNode*
443 | LockFreeHashTable<K, V, Hash>::GetBucketHeadByIndex(BucketIndex bucket_index) {
444 |   int level = 1;
445 |   const Segment* segments = segments_;
446 |   while (level++ <= kMaxLevel - 2) {
447 |     segments =
448 |         segments[(bucket_index / static_cast<SegmentIndex>(pow(
449 |                                      kSegmentSize, kMaxLevel - level + 1))) %
450 |                  kSegmentSize]
451 |             .get_sub_segments();
452 |     if (nullptr == segments) return nullptr;
453 |   }
454 | 
455 |   Bucket* buckets =
456 |       segments[(bucket_index / kSegmentSize) % kSegmentSize].get_sub_buckets();
457 |   if (nullptr == buckets) return nullptr;
458 | 
459 |   Bucket& bucket = buckets[bucket_index % kSegmentSize];
460 |   return bucket.load(std::memory_order_consume);
461 | }
462 | 
463 | template <typename K, typename V, typename Hash>
464 | bool LockFreeHashTable<K, V, Hash>::InsertDummyNode(DummyNode* parent_head,
465 |                                                     DummyNode* new_head,
466 |                                                     DummyNode** real_head) {
467 |   Node* prev;
468 |   Node* cur;
469 |   HazardPointer prev_hp, cur_hp;
470 |   do {
471 |     if (SearchNode(parent_head, new_head, &prev, &cur, prev_hp, cur_hp)) {
472 |       // The head of bucket already insert into list.
473 |       *real_head = static_cast<DummyNode*>(cur);
474 |       return false;
475 |     }
476 |     new_head->next.store(cur, std::memory_order_release);
477 |   } while (!prev->next.compare_exchange_weak(
478 |       cur, new_head, std::memory_order_release, std::memory_order_relaxed));
479 |   return true;
480 | }
481 | 
482 | // Insert regular node into hash table, if its key is already exists in
483 | // hash table then update it and return false else return true.
484 | template <typename K, typename V, typename Hash>
485 | bool LockFreeHashTable<K, V, Hash>::InsertRegularNode(DummyNode* head,
486 |                                                       RegularNode* new_node) {
487 |   Node* prev;
488 |   Node* cur;
489 |   HazardPointer prev_hp, cur_hp;
490 |   auto& reclaimer = TableReclaimer<K, V>::GetInstance();
491 |   do {
492 |     if (SearchNode(head, new_node, &prev, &cur, prev_hp, cur_hp)) {
493 |       V* new_value = new_node->value.load(std::memory_order_consume);
494 |       V* old_value = static_cast<RegularNode*>(cur)->value.exchange(
495 |           new_value, std::memory_order_release);
496 |       reclaimer.ReclaimLater(old_value,
497 |                              [](void* ptr) { delete static_cast<V*>(ptr); });
498 |       new_node->value.store(nullptr, std::memory_order_release);
499 |       delete new_node;
500 |       return false;
501 |     }
502 |     new_node->next.store(cur, std::memory_order_release);
503 |   } while (!prev->next.compare_exchange_weak(
504 |       cur, new_node, std::memory_order_release, std::memory_order_relaxed));
505 | 
506 |   size_t size = size_.fetch_add(1, std::memory_order_relaxed) + 1;
507 |   size_t power = power_of_2_.load(std::memory_order_relaxed);
508 |   if ((1 << power) * kLoadFactor < size) {
509 |     if (power_of_2_.compare_exchange_strong(power, power + 1,
510 |                                             std::memory_order_release)) {
511 |       assert(bucket_size() <=
512 |              kMaxBucketSize);  // Out of memory or you can change the kMaxLevel
513 |                                // and kSegmentSize.
514 |     }
515 |   }
516 |   return true;
517 | }
518 | 
519 | template <typename K, typename V, typename Hash>
520 | bool LockFreeHashTable<K, V, Hash>::SearchNode(DummyNode* head,
521 |                                                Node* search_node,
522 |                                                Node** prev_ptr, Node** cur_ptr,
523 |                                                HazardPointer& prev_hp,
524 |                                                HazardPointer& cur_hp) {
525 |   auto& reclaimer = TableReclaimer<K, V>::GetInstance();
526 | try_again:
527 |   Node* prev = head;
528 |   Node* cur = prev->get_next();
529 |   Node* next;
530 |   while (true) {
531 |     cur_hp.UnMark();
532 |     cur_hp = HazardPointer(&reclaimer, cur);
533 |     // Make sure prev is the predecessor of cur,
534 |     // so that cur is properly marked as hazard.
535 |     if (prev->get_next() != cur) goto try_again;
536 | 
537 |     if (nullptr == cur) {
538 |       *prev_ptr = prev;
539 |       *cur_ptr = cur;
540 |       return false;
541 |     }
542 | 
543 |     next = cur->get_next();
544 |     if (is_marked_reference(next)) {
545 |       if (!prev->next.compare_exchange_strong(cur,
546 |                                               get_unmarked_reference(next)))
547 |         goto try_again;
548 | 
549 |       reclaimer.ReclaimLater(cur, LockFreeHashTable<K, V, Hash>::OnDeleteNode);
550 |       reclaimer.ReclaimNoHazardPointer();
551 |       size_.fetch_sub(1, std::memory_order_relaxed);
552 |       cur = get_unmarked_reference(next);
553 |     } else {
554 |       if (prev->get_next() != cur) goto try_again;
555 | 
556 |       // Can not get copy_cur after above invocation,
557 |       // because prev may not be the predecessor of cur at this point.
558 |       if (GreaterOrEquals(cur, search_node)) {
559 |         *prev_ptr = prev;
560 |         *cur_ptr = cur;
561 |         return Equals(cur, search_node);
562 |       }
563 | 
564 |       // Swap cur_hp and prev_hp.
565 |       HazardPointer tmp = std::move(cur_hp);
566 |       cur_hp = std::move(prev_hp);
567 |       prev_hp = std::move(tmp);
568 | 
569 |       prev = cur;
570 |       cur = next;
571 |     }
572 |   };
573 | 
574 |   assert(false);
575 |   return false;
576 | }
577 | 
578 | template <typename K, typename V, typename Hash>
579 | bool LockFreeHashTable<K, V, Hash>::DeleteNode(DummyNode* head,
580 |                                                Node* delete_node) {
581 |   Node* prev;
582 |   Node* cur;
583 |   Node* next;
584 |   HazardPointer prev_hp, cur_hp;
585 |   do {
586 |     do {
587 |       if (!SearchNode(head, delete_node, &prev, &cur, prev_hp, cur_hp)) {
588 |         return false;
589 |       }
590 |       next = cur->get_next();
591 |     } while (is_marked_reference(next));
592 |     // Logically delete cur by marking cur->next.
593 |   } while (!cur->next.compare_exchange_weak(next, get_marked_reference(next),
594 |                                             std::memory_order_release,
595 |                                             std::memory_order_relaxed));
596 | 
597 |   if (prev->next.compare_exchange_strong(cur, next,
598 |                                          std::memory_order_release)) {
599 |     size_.fetch_sub(1, std::memory_order_relaxed);
600 |     auto& reclaimer = TableReclaimer<K, V>::GetInstance();
601 |     reclaimer.ReclaimLater(cur, LockFreeHashTable<K, V, Hash>::OnDeleteNode);
602 |     reclaimer.ReclaimNoHazardPointer();
603 |   } else {
604 |     prev_hp.UnMark();
605 |     cur_hp.UnMark();
606 |     SearchNode(head, delete_node, &prev, &cur, prev_hp, cur_hp);
607 |   }
608 | 
609 |   return true;
610 | }
611 | #endif  // LOCKFREE_HASHTABLE_H
612 | 


--------------------------------------------------------------------------------