├── .clang-format
├── .gitignore
├── ART
    ├── Key.h
    ├── LeafArray.cpp
    ├── LeafArray.h
    ├── N.cpp
    ├── N.h
    ├── N16.cpp
    ├── N16.h
    ├── N256.cpp
    ├── N256.h
    ├── N4.cpp
    ├── N4.h
    ├── N48.cpp
    ├── N48.h
    ├── Tree.cpp
    └── Tree.h
├── CMakeLists.txt
├── LICENSE
├── README.md
├── benchmark
    ├── benchmarks.h
    ├── config.h
    ├── coordinator.h
    ├── generator.cpp
    ├── generator.h
    ├── microbench.h
    └── timer.h
├── fast_fair
    ├── fast_fair.h
    ├── fast_fair_acma.h
    ├── fast_fair_alloc.h
    └── pmdk_gc.h
├── format
├── lf-skiplist
    ├── atomic_ops_if.h
    ├── lf-skiplist-alloc.cpp
    ├── lf-skiplist-alloc.h
    ├── skiplist-acma.cpp
    ├── skiplist-acma.h
    ├── skiplist.cpp
    ├── skiplist.h
    └── sl_gc.h
├── nvm_mgr
    ├── Epoch.cpp
    ├── Epoch.h
    ├── EpochGuard.h
    ├── nvm_mgr.cpp
    ├── nvm_mgr.h
    ├── pmalloc_wrap.h
    ├── threadinfo.cpp
    ├── threadinfo.h
    └── util.h
├── perf
    ├── acmma.cpp
    ├── art_simple_bench.cpp
    ├── main.cpp
    └── malloc_diff.cpp
├── script
    ├── loop.sh
    └── test.sh
└── test
    ├── test.cpp
    ├── test_basic.cpp
    ├── test_correctness.cpp
    ├── test_epoch.cpp
    ├── test_leafarray.cpp
    ├── test_nvm_mgr.cpp
    ├── test_recovery.cpp
    └── test_skiplist.cpp


/.clang-format:
--------------------------------------------------------------------------------
1 | ---
2 | Language: Cpp
3 | BasedOnStyle: LLVM
4 | IndentWidth: 4
5 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | build/
 2 | .vscode/
 3 | cmake-build-debug-teaker-18/
 4 | cmake-build-debug-teaker-22/
 5 | .idea/
 6 | result/
 7 | script/
 8 | .DS_Store
 9 | /cmake-build-debug/
10 | /cmake-build-debug-remote/
11 | /dot/
12 | 


--------------------------------------------------------------------------------
/ART/Key.h:
--------------------------------------------------------------------------------
 1 | #ifndef ART_KEY_H
 2 | #define ART_KEY_H
 3 | 
 4 | #include <assert.h>
 5 | #include <cstring>
 6 | #include <memory>
 7 | #include <stdint.h>
 8 | 
 9 | namespace PART_ns {
10 | 
11 | struct Key {
12 |     uint64_t value;
13 |     size_t key_len;
14 |     size_t val_len;
15 |     uint64_t key;
16 |     uint8_t *fkey;
17 | 
18 |     Key() {}
19 | 
20 |     Key(uint64_t key_, size_t key_len_, uint64_t value_) {
21 |         value = value_;
22 |         key_len = key_len_;
23 |         val_len = sizeof(uint64_t);
24 |         key = key_;
25 |         fkey = (uint8_t *)&key;
26 |     }
27 | 
28 |     void Init(uint64_t key_, size_t key_len_, uint64_t value_) {
29 |         value = value_;
30 |         key_len = key_len_;
31 |         val_len = sizeof(uint64_t);
32 |         key = key_;
33 |         fkey = (uint8_t *)&key;
34 |     }
35 | 
36 |     void Init(char *key_, size_t key_len_, char *value_, size_t val_len_) {
37 |         val_len = val_len_;
38 |         value = (uint64_t)value_;
39 |         key_len = key_len_;
40 |         fkey = (uint8_t *)key_;
41 |     }
42 | 
43 |     inline Key *make_leaf(char *key, size_t key_len, uint64_t value);
44 | 
45 |     inline Key *make_leaf(uint64_t key, size_t key_len, uint64_t value);
46 | 
47 |     inline size_t getKeyLen() const;
48 | 
49 |     inline uint16_t getFingerPrint() const;
50 | } __attribute__((aligned(64)));
51 | 
52 | inline Key *Key::make_leaf(char *key, size_t key_len, uint64_t value) {
53 |     void *aligned_alloc;
54 |     posix_memalign(&aligned_alloc, 64, sizeof(Key) + key_len);
55 |     Key *k = reinterpret_cast<Key *>(aligned_alloc);
56 | 
57 |     k->value = value;
58 |     k->key_len = key_len;
59 |     memcpy(k->fkey, key, key_len);
60 | 
61 |     return k;
62 | }
63 | 
64 | inline Key *Key::make_leaf(uint64_t key, size_t key_len, uint64_t value) {
65 |     void *aligned_alloc;
66 |     posix_memalign(&aligned_alloc, 64, sizeof(Key) + key_len);
67 |     Key *k = reinterpret_cast<Key *>(aligned_alloc);
68 | 
69 |     k->value = value;
70 |     k->key_len = key_len;
71 |     reinterpret_cast<uint64_t *>(&k->fkey[0])[0] = __builtin_bswap64(key);
72 | 
73 |     return k;
74 | }
75 | 
76 | inline size_t Key::getKeyLen() const {
77 |     return key_len;
78 | }
79 | 
80 | inline uint16_t Key::getFingerPrint() const {
81 |     uint16_t re = 0;
82 |     for (int i = 0; i < key_len; i++) {
83 |         re = re * 131 + this->fkey[i];
84 |     }
85 |     return re;
86 | }
87 | } // namespace PART_ns
88 | 
89 | #endif // ART_KEY_H


--------------------------------------------------------------------------------
/ART/LeafArray.cpp:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by 谢威宇 on 2021/4/3.
  3 | //
  4 | 
  5 | #include "LeafArray.h"
  6 | #include "EpochGuard.h"
  7 | #include "threadinfo.h"
  8 | 
  9 | using namespace NVMMgr_ns;
 10 | namespace PART_ns {
 11 | size_t PART_ns::LeafArray::getRightmostSetBit() const {
 12 |     auto b = bitmap.load();
 13 |     auto pos = b._Find_first();
 14 |     assert(pos < LeafArrayLength);
 15 |     return pos;
 16 | }
 17 | void PART_ns::LeafArray::setBit(size_t bit_pos, bool to) {
 18 |     auto b = bitmap.load();
 19 |     b[bit_pos] = to;
 20 |     bitmap.store(b);
 21 | }
 22 | uint16_t PART_ns::LeafArray::getFingerPrint(size_t pos) const {
 23 |     auto x = reinterpret_cast<uint64_t>(leaf[pos].load());
 24 |     uint16_t re = x >> FingerPrintShift;
 25 |     return re;
 26 | }
 27 | Leaf *LeafArray::lookup(const Key *k) const {
 28 |     uint16_t finger_print = k->getFingerPrint();
 29 | 
 30 |     auto b = bitmap.load();
 31 | 
 32 | #ifdef FIND_FIRST
 33 |     auto i = b[0] ? 0 : 1;
 34 |     while (i < LeafArrayLength) {
 35 |         auto fingerprint_ptr = this->leaf[i].load();
 36 |         if (fingerprint_ptr != 0) {
 37 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
 38 |             auto ptr = reinterpret_cast<Leaf *>(
 39 |                 fingerprint_ptr ^
 40 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
 41 |             if (finger_print == thisfp && ptr->checkKey(k)) {
 42 |                 return ptr;
 43 |             }
 44 |         }
 45 |         i = b._Find_next(i);
 46 |     }
 47 | #else
 48 |     for (int i = 0; i < LeafArrayLength; i++) {
 49 |         if (b[i] == false)
 50 |             continue;
 51 |         auto fingerprint_ptr = this->leaf[i].load();
 52 |         if (fingerprint_ptr != 0) {
 53 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
 54 |             auto ptr = reinterpret_cast<Leaf *>(
 55 |                 fingerprint_ptr ^
 56 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
 57 |             if (finger_print == thisfp && ptr->checkKey(k)) {
 58 |                 return ptr;
 59 |             }
 60 |         }
 61 |     }
 62 | #endif
 63 | 
 64 |     return nullptr;
 65 | }
 66 | bool LeafArray::insert(Leaf *l, bool flush) {
 67 |     auto b = bitmap.load();
 68 |     b.flip();
 69 |     auto pos = b._Find_first();
 70 |     if (pos < LeafArrayLength) {
 71 |         b.flip();
 72 |         b[pos] = true;
 73 |         bitmap.store(b);
 74 |         auto s =
 75 |             (static_cast<uintptr_t>(l->getFingerPrint()) << FingerPrintShift) |
 76 |             (reinterpret_cast<uintptr_t>(l));
 77 |         leaf[pos].store(s);
 78 |         if (flush)
 79 |             flush_data((void *)&leaf[pos], sizeof(std::atomic<uintptr_t>));
 80 | 
 81 |         return true;
 82 |     } else {
 83 |         return false;
 84 |     }
 85 | }
 86 | bool LeafArray::remove(const Key *k) {
 87 |     uint16_t finger_print = k->getFingerPrint();
 88 |     auto b = bitmap.load();
 89 |     auto i = b[0] ? 0 : 1;
 90 |     while (i < LeafArrayLength) {
 91 |         auto fingerprint_ptr = this->leaf[i].load();
 92 |         if (fingerprint_ptr != 0) {
 93 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
 94 |             auto ptr = reinterpret_cast<Leaf *>(
 95 |                 fingerprint_ptr ^
 96 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
 97 |             if (finger_print == thisfp && ptr->checkKey(k)) {
 98 |                 leaf[i].store(0);
 99 |                 flush_data(&leaf[i], sizeof(std::atomic<uintptr_t>));
100 |                 EpochGuard::DeleteNode(ptr);
101 |                 b[i] = false;
102 |                 bitmap.store(b);
103 |                 return true;
104 |             }
105 |         }
106 |         i = b._Find_next(i);
107 |     }
108 |     return false;
109 | }
110 | void LeafArray::reload() {
111 |     auto b = bitmap.load();
112 |     for (int i = 0; i < LeafArrayLength; i++) {
113 |         if (leaf[i].load() != 0) {
114 |             b[i] = true;
115 |         } else {
116 |             b[i] = false;
117 |         }
118 |     }
119 |     bitmap.store(b);
120 | }
121 | void LeafArray::graphviz_debug(std::ofstream &f) {
122 |     char buf[1000] = {};
123 |     sprintf(buf + strlen(buf), "node%lx [label=\"",
124 |             reinterpret_cast<uintptr_t>(this));
125 |     sprintf(buf + strlen(buf), "LeafArray\n");
126 |     sprintf(buf + strlen(buf), "count: %zu\n", bitmap.load().count());
127 |     sprintf(buf + strlen(buf), "\"]\n");
128 | 
129 |     auto b = bitmap.load();
130 |     auto i = b[0] ? 0 : 1;
131 |     while (i < LeafArrayLength) {
132 |         auto fingerprint_ptr = this->leaf[i].load();
133 |         if (fingerprint_ptr != 0) {
134 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
135 |             auto ptr = reinterpret_cast<Leaf *>(
136 |                 fingerprint_ptr ^
137 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
138 | 
139 |             sprintf(buf + strlen(buf), "node%lx -- node%lx \n",
140 |                     reinterpret_cast<uintptr_t>(this),
141 |                     reinterpret_cast<uintptr_t>(ptr));
142 |         }
143 |         i = b._Find_next(i);
144 |     }
145 | 
146 |     f << buf;
147 | 
148 |     b = bitmap.load();
149 |     i = b[0] ? 0 : 1;
150 |     while (i < LeafArrayLength) {
151 |         auto fingerprint_ptr = this->leaf[i].load();
152 |         if (fingerprint_ptr != 0) {
153 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
154 |             auto ptr = reinterpret_cast<Leaf *>(
155 |                 fingerprint_ptr ^
156 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
157 | 
158 |             ptr->graphviz_debug(f);
159 |         }
160 |         i = b._Find_next(i);
161 |     }
162 | }
163 | 
164 | void LeafArray::splitAndUnlock(N *parentNode, uint8_t parentKey,
165 |                                bool &need_restart) {
166 | 
167 |     parentNode->writeLockOrRestart(need_restart);
168 | 
169 |     if (need_restart) {
170 |         this->writeUnlock();
171 |         return;
172 |     }
173 | 
174 |     auto b = bitmap.load();
175 |     auto leaf_count = b.count();
176 |     std::vector<char *> keys;
177 |     //    char **keys = new char *[leaf_count];
178 |     std::vector<int> lens;
179 |     //    int *lens = new int[leaf_count];
180 | 
181 |     auto i = b[0] ? 0 : 1;
182 |     while (i < LeafArrayLength) {
183 |         auto fingerprint_ptr = this->leaf[i].load();
184 |         if (fingerprint_ptr != 0) {
185 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
186 |             auto ptr = reinterpret_cast<Leaf *>(
187 |                 fingerprint_ptr ^
188 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
189 |             keys.push_back(ptr->GetKey());
190 |             lens.push_back(ptr->key_len);
191 |         }
192 |         i = b._Find_next(i);
193 |     }
194 |     //    printf("spliting\n");
195 | 
196 |     std::vector<char> common_prefix;
197 |     int level = 0;
198 |     level = parentNode->getLevel() + 1;
199 |     // assume keys are not substring of another key
200 | 
201 |     // todo: get common prefix can be optimized by binary search
202 |     while (true) {
203 |         bool out = false;
204 |         for (i = 0; i < leaf_count; i++) {
205 |             if (level < lens[i]) {
206 |                 if (i == 0) {
207 |                     common_prefix.push_back(keys[i][level]);
208 |                 } else {
209 |                     if (keys[i][level] != common_prefix.back()) {
210 | 
211 |                         common_prefix.pop_back();
212 | 
213 |                         out = true;
214 |                         break;
215 |                     }
216 |                 }
217 |             } else {
218 |                 // assume keys are not substring of another key
219 |                 assert(0);
220 |             }
221 |         }
222 |         if (out)
223 |             break;
224 |         level++;
225 |     }
226 |     std::map<char, LeafArray *> split_array;
227 |     for (i = 0; i < leaf_count; i++) {
228 |         if (split_array.count(keys[i][level]) == 0) {
229 |             split_array[keys[i][level]] =
230 |                 new (alloc_new_node_from_type(NTypes::LeafArray))
231 |                     LeafArray(level);
232 |         }
233 |         split_array.at(keys[i][level])->insert(getLeafAt(i), false);
234 |     }
235 | 
236 |     N *n;
237 |     uint8_t *prefix_start = reinterpret_cast<uint8_t *>(common_prefix.data());
238 |     auto prefix_len = common_prefix.size();
239 |     auto leaf_array_count = split_array.size();
240 |     if (leaf_array_count <= 4) {
241 |         n = new (alloc_new_node_from_type(NTypes::N4))
242 |             N4(level, prefix_start, prefix_len);
243 |     } else if (leaf_array_count > 4 && leaf_array_count <= 16) {
244 |         n = new (alloc_new_node_from_type(NTypes::N16))
245 |             N16(level, prefix_start, prefix_len);
246 |     } else if (leaf_array_count > 16 && leaf_array_count <= 48) {
247 |         n = new (alloc_new_node_from_type(NTypes::N48))
248 |             N48(level, prefix_start, prefix_len);
249 |     } else if (leaf_array_count > 48 && leaf_array_count <= 256) {
250 |         n = new (alloc_new_node_from_type(NTypes::N256))
251 |             N256(level, prefix_start, prefix_len);
252 |     } else {
253 |         assert(0);
254 |     }
255 |     for (const auto &p : split_array) {
256 |         unchecked_insert(n, p.first, setLeafArray(p.second), true);
257 |         flush_data(p.second, sizeof(LeafArray));
258 |     }
259 | 
260 |     N::change(parentNode, parentKey, n);
261 |     parentNode->writeUnlock();
262 | 
263 |     this->writeUnlockObsolete();
264 |     EpochGuard::DeleteNode(this);
265 | }
266 | Leaf *LeafArray::getLeafAt(size_t pos) const {
267 |     auto t = reinterpret_cast<uintptr_t>(this->leaf[pos].load());
268 |     t = (t << 16) >> 16;
269 |     return reinterpret_cast<Leaf *>(t);
270 | }
271 | uint32_t LeafArray::getCount() const { return bitmap.load().count(); }
272 | bool LeafArray::isFull() const { return getCount() == LeafArrayLength; }
273 | std::vector<Leaf *> LeafArray::getSortedLeaf(const Key *start, const Key *end,
274 |                                              int start_level,
275 |                                              bool compare_start,
276 |                                              bool compare_end) {
277 |     std::vector<Leaf *> leaves;
278 |     auto b = bitmap.load();
279 |     auto i = b[0] ? 0 : 1;
280 | 
281 |     while (i < LeafArrayLength) {
282 |         auto ptr = getLeafAt(i);
283 |         i = b._Find_next(i);
284 |         // start <= ptr < end
285 |         if (compare_start) {
286 |             auto lt_start = leaf_key_lt(ptr, start, start_level);
287 |             if (lt_start == true) {
288 |                 continue;
289 |             }
290 |         }
291 |         if (compare_end) {
292 |             auto lt_end = leaf_key_lt(ptr, end, start_level);
293 |             if (lt_end == false) {
294 |                 continue;
295 |             }
296 |         }
297 |         leaves.push_back(ptr);
298 |     }
299 | #ifdef SORT_LEAVES
300 |     std::sort(leaves.begin(), leaves.end(),
301 |               [start_level](Leaf *a, Leaf *b) -> bool {
302 |                   leaf_lt(a, b, start_level);
303 |               });
304 | #endif
305 |     return leaves;
306 | }
307 | bool LeafArray::update(const Key *k, Leaf *l) {
308 |     uint16_t finger_print = k->getFingerPrint();
309 |     auto b = bitmap.load();
310 | 
311 | #ifdef FIND_FIRST
312 |     auto i = b[0] ? 0 : 1;
313 |     while (i < LeafArrayLength) {
314 |         auto fingerprint_ptr = this->leaf[i].load();
315 |         if (fingerprint_ptr != 0) {
316 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
317 |             auto ptr = reinterpret_cast<Leaf *>(
318 |                 fingerprint_ptr ^
319 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
320 |             if (finger_print == thisfp && ptr->checkKey(k)) {
321 |                 auto news = fingerPrintLeaf(finger_print, l);
322 |                 leaf[i].store(news);
323 |                 flush_data(&leaf[i], sizeof(std::atomic<uintptr_t>));
324 |                 return true;
325 |             }
326 |         }
327 |         i = b._Find_next(i);
328 |     }
329 | #else
330 |     for (int i = 0; i < LeafArrayLength; i++) {
331 |         if (b[i] == false)
332 |             continue;
333 |         auto fingerprint_ptr = this->leaf[i].load();
334 |         if (fingerprint_ptr != 0) {
335 |             uint16_t thisfp = fingerprint_ptr >> FingerPrintShift;
336 |             auto ptr = reinterpret_cast<Leaf *>(
337 |                 fingerprint_ptr ^
338 |                 (static_cast<uintptr_t>(thisfp) << FingerPrintShift));
339 |             if (finger_print == thisfp && ptr->checkKey(k)) {
340 |                 auto news = fingerPrintLeaf(finger_print, l);
341 |                 leaf[i].store(news);
342 |                 flush_data(&leaf[i], sizeof(std::atomic<uintptr_t>));
343 |                 return true;
344 |             }
345 |         }
346 |     }
347 | #endif
348 |     return false;
349 | }
350 | uintptr_t LeafArray::fingerPrintLeaf(uint16_t fingerPrint, Leaf *l) {
351 |     uintptr_t mask = (1LL << FingerPrintShift) - 1;
352 |     auto f = uintptr_t(fingerPrint);
353 |     return (reinterpret_cast<uintptr_t>(l) & mask) | (f << FingerPrintShift);
354 | }
355 | N *LeafArray::getAnyChild() const {
356 |     auto b = bitmap.load();
357 |     auto i = b._Find_first();
358 |     if (i == LeafArrayLength) {
359 |         return nullptr;
360 |     } else {
361 |         return N::setLeaf(getLeafAt(i));
362 |     }
363 | }
364 | 
365 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/LeafArray.h:
--------------------------------------------------------------------------------
 1 | //
 2 | // Created by 谢威宇 on 2021/4/3.
 3 | //
 4 | 
 5 | #ifndef P_ART_LEAFARRAY_H
 6 | #define P_ART_LEAFARRAY_H
 7 | #include "N.h"
 8 | #include <atomic>
 9 | #include <bitset>
10 | #include <functional>
11 | 
12 | namespace PART_ns {
13 | 
14 | const size_t LeafArrayLength = 64;
15 | const size_t FingerPrintShift = 48;
16 | 
17 | class LeafArray : public N {
18 |   public:
19 |     std::atomic<uintptr_t> leaf[LeafArrayLength];
20 |     std::atomic<std::bitset<LeafArrayLength>>
21 |         bitmap; // 0 means used slot; 1 means empty slot
22 | 
23 |   public:
24 |     LeafArray(uint32_t level = -1) : N(NTypes::LeafArray, level, {}, 0) {
25 |         bitmap.store(std::bitset<LeafArrayLength>{}.reset());
26 |         memset(leaf, 0, sizeof(leaf));
27 |     }
28 | 
29 |     virtual ~LeafArray() {}
30 | 
31 |     size_t getRightmostSetBit() const;
32 | 
33 |     void setBit(size_t bit_pos, bool to = true);
34 | 
35 |     uint16_t getFingerPrint(size_t pos) const;
36 | 
37 |     Leaf *getLeafAt(size_t pos) const;
38 | 
39 |     N *getAnyChild() const;
40 | 
41 |     static uintptr_t fingerPrintLeaf(uint16_t fingerPrint, Leaf *l);
42 | 
43 |     Leaf *lookup(const Key *k) const;
44 | 
45 |     bool update(const Key *k, Leaf *l);
46 | 
47 |     bool insert(Leaf *l, bool flush);
48 | 
49 |     bool remove(const Key *k);
50 | 
51 |     void reload();
52 | 
53 |     uint32_t getCount() const;
54 | 
55 |     bool isFull() const;
56 | 
57 |     void splitAndUnlock(N *parentNode, uint8_t parentKey, bool &need_restart);
58 | 
59 |     std::vector<Leaf *> getSortedLeaf(const Key *start, const Key *end,
60 |                                       int start_level, bool compare_start,
61 |                                       bool compare_end);
62 | 
63 |     void graphviz_debug(std::ofstream &f);
64 | 
65 | } __attribute__((aligned(64)));
66 | } // namespace PART_ns
67 | #endif // P_ART_LEAFARRAY_H
68 | 


--------------------------------------------------------------------------------
/ART/N.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "Epoch.h"
  4 | #include "Key.h"
  5 | #include "util.h"
  6 | #include <atomic>
  7 | #include <fstream>
  8 | #include <set>
  9 | #include <stdint.h>
 10 | #include <string.h>
 11 | 
 12 | namespace PART_ns {
 13 | 
 14 | int gethelpcount();
 15 | 
 16 | enum class NTypes : uint8_t {
 17 |     N4 = 1,
 18 |     N16 = 2,
 19 |     N48 = 3,
 20 |     N256 = 4,
 21 |     Leaf = 5,
 22 |     LeafArray = 6
 23 | };
 24 | 
 25 | class LeafArray;
 26 | 
 27 | class BaseNode {
 28 |   public:
 29 |     NTypes type;
 30 |     BaseNode(NTypes type_) : type(type_) {}
 31 |     virtual ~BaseNode() {}
 32 | };
 33 | 
 34 | class Leaf : public BaseNode {
 35 |   public:
 36 |     size_t key_len;
 37 |     size_t val_len;
 38 | //    uint64_t key;
 39 | // variable key
 40 | #ifdef KEY_INLINE
 41 |     char kv[0]; // append key and value
 42 | #else
 43 |     uint8_t *fkey;
 44 |     char *value;
 45 | #endif
 46 | 
 47 |   public:
 48 |     Leaf(const Key *k);
 49 |     Leaf(uint8_t *key_, size_t key_len_, char *value_,
 50 |          size_t val_len_); // used for update
 51 |     // use for test
 52 |     Leaf() : BaseNode(NTypes::Leaf) {}
 53 | 
 54 |     virtual ~Leaf() {}
 55 | 
 56 |     bool checkKey(const Key *k) const {
 57 | #ifdef KEY_INLINE
 58 |         if (key_len == k->getKeyLen() && memcmp(kv, k->fkey, key_len) == 0)
 59 |             return true;
 60 |         return false;
 61 | #else
 62 |         if (key_len == k->getKeyLen() && memcmp(fkey, k->fkey, key_len) == 0)
 63 |             return true;
 64 |         return false;
 65 | #endif
 66 |     }
 67 |     size_t getKeyLen() const { return key_len; }
 68 |     char *GetKey() {
 69 | #ifdef KEY_INLINE
 70 |         return kv;
 71 | #else
 72 |         return (char *)fkey;
 73 | #endif
 74 |     }
 75 |     char *GetValue() {
 76 | #ifdef KEY_INLINE
 77 |         return kv + key_len;
 78 | #else
 79 |         return value;
 80 | #endif
 81 |     }
 82 | 
 83 |     uint16_t getFingerPrint();
 84 | 
 85 |     void graphviz_debug(std::ofstream &f);
 86 | 
 87 | } __attribute__((aligned(64)));
 88 | 
 89 | static constexpr uint32_t maxStoredPrefixLength = 4;
 90 | struct Prefix {
 91 |     uint32_t prefixCount = 0;
 92 |     uint8_t prefix[maxStoredPrefixLength];
 93 | };
 94 | static_assert(sizeof(Prefix) == 8, "Prefix should be 64 bit long");
 95 | 
 96 | class N : public BaseNode {
 97 |   protected:
 98 |     N(NTypes type, uint32_t level, const uint8_t *prefix, uint32_t prefixLength)
 99 |         : BaseNode(type), level(level) {
100 |         type_version_lock_obsolete = new std::atomic<uint64_t>;
101 |         type_version_lock_obsolete->store(0b100);
102 |         recovery_latch.store(0, std::memory_order_seq_cst);
103 |         setType(type);
104 |         setPrefix(prefix, prefixLength, false);
105 |     }
106 | 
107 |     N(NTypes type, uint32_t level, const Prefix &prefi)
108 |         : BaseNode(type), prefix(prefi), level(level) {
109 |         type_version_lock_obsolete = new std::atomic<uint64_t>;
110 |         type_version_lock_obsolete->store(0b100);
111 |         recovery_latch.store(0, std::memory_order_seq_cst);
112 |         setType(type);
113 |     }
114 | 
115 |     N(const N &) = delete;
116 | 
117 |     N(N &&) = delete;
118 | 
119 |     virtual ~N() {}
120 | 
121 |     // 3b type 59b version 1b lock 1b obsolete
122 |     // obsolete means this node has been deleted
123 |     std::atomic<uint64_t> *type_version_lock_obsolete;
124 | 
125 |     alignas(64) std::atomic<Prefix> prefix;
126 |     const uint32_t level;
127 |     uint16_t count = 0;
128 |     uint16_t compactCount = 0;
129 | 
130 |     uint64_t generation_version = 0;
131 |     std::atomic<uint64_t> recovery_latch;
132 | 
133 |     static const uint64_t dirty_bit = ((uint64_t)1 << 60);
134 | 
135 |     void setType(NTypes type);
136 | 
137 |     static uint64_t convertTypeToVersion(NTypes type);
138 | 
139 |   public:
140 |     static inline N *setDirty(N *val) {
141 |         return (N *)((uint64_t)val | dirty_bit);
142 |     }
143 |     static inline N *clearDirty(N *val) {
144 |         return (N *)((uint64_t)val & (~dirty_bit));
145 |     }
146 |     static inline bool isDirty(N *val) { return (uint64_t)val & dirty_bit; }
147 | 
148 |     static void helpFlush(std::atomic<N *> *n);
149 | 
150 |     void set_generation();
151 | 
152 |     uint64_t get_generation();
153 | 
154 |     void check_generation();
155 | 
156 |     NTypes getType() const;
157 | 
158 |     uint32_t getLevel() const;
159 | 
160 |     static uint32_t getCount(N *node);
161 | 
162 |     void setCount(uint16_t count_, uint16_t compactCount_);
163 | 
164 |     bool isLocked(uint64_t version) const;
165 | 
166 |     void writeLockOrRestart(bool &needRestart);
167 | 
168 |     void lockVersionOrRestart(uint64_t &version, bool &needRestart);
169 | 
170 |     void writeUnlock();
171 | 
172 |     uint64_t getVersion() const;
173 | 
174 |     /**
175 |      * returns true if node hasn't been changed in between
176 |      */
177 |     bool checkOrRestart(uint64_t startRead) const;
178 |     bool readVersionOrRestart(uint64_t startRead) const;
179 | 
180 |     static bool isObsolete(uint64_t version);
181 | 
182 |     /**
183 |      * can only be called when node is locked
184 |      */
185 |     void writeUnlockObsolete() { type_version_lock_obsolete->fetch_add(0b11); }
186 | 
187 |     static N *getChild(const uint8_t k, N *node);
188 | 
189 |     static void insertAndUnlock(N *node, N *parentNode, uint8_t keyParent,
190 |                                 uint8_t key, N *val, bool &needRestart);
191 | 
192 |     static void change(N *node, uint8_t key, N *val);
193 | 
194 |     static void removeAndUnlock(N *node, uint8_t key, N *parentNode,
195 |                                 uint8_t keyParent, bool &needRestart);
196 | 
197 |     Prefix getPrefi() const;
198 | 
199 |     void setPrefix(const uint8_t *prefix, uint32_t length, bool flush);
200 | 
201 |     void addPrefixBefore(N *node, uint8_t key);
202 | 
203 |     static Leaf *getLeaf(const N *n);
204 | 
205 |     static bool isLeaf(const N *n);
206 | 
207 |     static N *setLeaf(const Leaf *k);
208 | 
209 |     static LeafArray *getLeafArray(const N *n);
210 | 
211 |     static bool isLeafArray(const N *n);
212 | 
213 |     static N *setLeafArray(const LeafArray *la);
214 | 
215 |     static N *getAnyChild(N *n);
216 | 
217 |     static Leaf *getAnyChildTid(const N *n);
218 | 
219 |     static void deleteChildren(N *node);
220 | 
221 |     static void deleteNode(N *node);
222 | 
223 |     static std::tuple<N *, uint8_t> getSecondChild(N *node, const uint8_t k);
224 | 
225 |     template <typename curN, typename biggerN>
226 |     static void tryInsertOrGrowAndUnlock(curN *n, N *parentNode,
227 |                                          uint8_t keyParent, uint8_t key, N *val,
228 |                                          NTypes type, bool &needRestart);
229 | 
230 |     template <typename curN>
231 |     static void compactAndInsertAndUnlock(curN *n, N *parentNode,
232 |                                           uint8_t keyParent, uint8_t key,
233 |                                           N *val, NTypes type,
234 |                                           bool &needRestart);
235 | 
236 |     template <typename curN, typename smallerN>
237 |     static void removeAndShrink(curN *n, N *parentNode, uint8_t keyParent,
238 |                                 uint8_t key, NTypes type, bool &needRestart);
239 | 
240 |     static void getChildren(N *node, uint8_t start, uint8_t end,
241 |                             std::tuple<uint8_t, N *> children[],
242 |                             uint32_t &childrenCount);
243 | 
244 |     static void rebuild_node(N *node,
245 |                              std::vector<std::pair<uint64_t, size_t>> &rs,
246 |                              uint64_t start_addr, uint64_t end_addr,
247 |                              int thread_id);
248 | 
249 |     static void graphviz_debug(std::ofstream &f, N *node);
250 | 
251 |     // do insert without checking anything
252 |     static void unchecked_insert(N *node, uint8_t key_byte, N *child,
253 |                                  bool flush);
254 | 
255 |     static bool key_keylen_lt(char *a, const int alen, char *b, const int blen,
256 |                               const int compare_level);
257 | 
258 |     static bool leaf_lt(Leaf *a, Leaf *b, int compare_level);
259 |     static bool leaf_key_lt(Leaf *a, const Key *b, const int compare_level);
260 |     static bool key_leaf_lt(const Key *a, Leaf *b, const int compare_level);
261 |     static bool key_key_lt(const Key *a, const Key *b);
262 | 
263 |     static const int ZentryKeyShift = 48;
264 |     static uint8_t getZentryKey(uintptr_t zentry);
265 |     static N *getZentryPtr(uintptr_t zentry);
266 |     static std::pair<uint8_t, N *> getZentryKeyPtr(uintptr_t zentry);
267 |     static uintptr_t makeZentry(uint8_t key,N* node);
268 | 
269 | } __attribute__((aligned(64)));
270 | 
271 | } // namespace PART_ns
272 | 


--------------------------------------------------------------------------------
/ART/N16.cpp:
--------------------------------------------------------------------------------
  1 | #include "N16.h"
  2 | #include "LeafArray.h"
  3 | #include "N.h"
  4 | #include <algorithm>
  5 | #include <assert.h>
  6 | #include <emmintrin.h> // x86 SSE intrinsics
  7 | 
  8 | namespace PART_ns {
  9 | 
 10 | bool N16::insert(uint8_t key, N *n, bool flush) {
 11 |     if (compactCount == 16) {
 12 |         return false;
 13 |     }
 14 | 
 15 | #ifdef ZENTRY
 16 |     zens[compactCount].store(makeZentry(flipSign(key), n));
 17 |     if (flush)
 18 |         flush_data(&zens[compactCount], sizeof(std::atomic<uintptr_t>));
 19 | #else
 20 |     keys[compactCount].store(flipSign(key), std::memory_order_seq_cst);
 21 |     if (flush)
 22 |         flush_data((void *)&keys[compactCount], sizeof(std::atomic<uint8_t>));
 23 | 
 24 |     children[compactCount].store(n, std::memory_order_seq_cst);
 25 |     if (flush)
 26 |         flush_data((void *)&children[compactCount], sizeof(std::atomic<N *>));
 27 | #endif
 28 |     compactCount++;
 29 |     count++;
 30 |     return true;
 31 | }
 32 | 
 33 | void N16::change(uint8_t key, N *val) {
 34 |     auto childPos = getChildPos(key);
 35 |     assert(childPos != -1);
 36 | #ifdef ZENTRY
 37 |     zens[childPos].store(makeZentry(flipSign(key), val));
 38 |     flush_data(&zens[childPos], sizeof(std::atomic<uintptr_t>));
 39 | #else
 40 |     children[childPos].store(val, std::memory_order_seq_cst);
 41 |     flush_data(&children[childPos], sizeof(std::atomic<N *>));
 42 | #endif
 43 | }
 44 | 
 45 | int N16::getChildPos(const uint8_t k) {
 46 | #ifdef ZENTRY
 47 |     uint8_t keys[16] = {};
 48 |     for (int i = 0; i < 16; i++) {
 49 |         keys[i] = getZentryKey(zens[i].load());
 50 |     }
 51 |     __m128i cmp = _mm_cmpeq_epi8(
 52 |         _mm_set1_epi8(flipSign(k)),
 53 |         _mm_loadu_si128(reinterpret_cast<const __m128i *>(keys)));
 54 |     unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << compactCount) - 1);
 55 |     while (bitfield) {
 56 |         uint8_t pos = ctz(bitfield);
 57 |         if (getZentryPtr(zens[pos]) != nullptr) {
 58 |             return pos;
 59 |         }
 60 |         bitfield = bitfield ^ (1 << pos);
 61 |     }
 62 |     return -1;
 63 | #else
 64 |     __m128i cmp = _mm_cmpeq_epi8(
 65 |         _mm_set1_epi8(flipSign(k)),
 66 |         _mm_loadu_si128(reinterpret_cast<const __m128i *>(keys)));
 67 |     unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << compactCount) - 1);
 68 |     while (bitfield) {
 69 |         uint8_t pos = ctz(bitfield);
 70 |         if (children[pos].load() != nullptr) {
 71 |             return pos;
 72 |         }
 73 |         bitfield = bitfield ^ (1 << pos);
 74 |     }
 75 |     return -1;
 76 | #endif
 77 | }
 78 | 
 79 | N *N16::getChild(const uint8_t k) {
 80 | #ifdef ZENTRY
 81 |     uint8_t keys[16] = {};
 82 |     for (int i = 0; i < 16; i++) {
 83 |         keys[i] = getZentryKey(zens[i].load());
 84 |     }
 85 |     __m128i cmp = _mm_cmpeq_epi8(
 86 |         _mm_set1_epi8(flipSign(k)),
 87 |         _mm_loadu_si128(reinterpret_cast<const __m128i *>(keys)));
 88 |     unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << 16) - 1);
 89 |     while (bitfield) {
 90 |         uint8_t pos = ctz(bitfield);
 91 |         auto p = getZentryKeyPtr(zens[pos]);
 92 | 
 93 |         if (p.second != nullptr && p.first == flipSign(k)) {
 94 |             return p.second;
 95 |         }
 96 |         bitfield = bitfield ^ (1 << pos);
 97 |     }
 98 |     return nullptr;
 99 | #else
100 |     __m128i cmp = _mm_cmpeq_epi8(
101 |         _mm_set1_epi8(flipSign(k)),
102 |         _mm_loadu_si128(reinterpret_cast<const __m128i *>(keys)));
103 |     unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << 16) - 1);
104 |     while (bitfield) {
105 |         uint8_t pos = ctz(bitfield);
106 | 
107 |         N *child = children[pos].load();
108 |         if (child != nullptr && keys[pos].load() == flipSign(k)) {
109 |             return child;
110 |         }
111 |         bitfield = bitfield ^ (1 << pos);
112 |     }
113 |     return nullptr;
114 | #endif
115 | }
116 | 
117 | bool N16::remove(uint8_t k, bool force, bool flush) {
118 |     if (count <= 3 && !force) {
119 |         return false;
120 |     }
121 |     auto leafPlace = getChildPos(k);
122 |     assert(leafPlace != -1);
123 | #ifdef ZENTRY
124 |     zens[leafPlace].store(0);
125 |     flush_data(&zens[leafPlace], sizeof(std::atomic<uintptr_t>));
126 | #else
127 |     children[leafPlace].store(nullptr, std::memory_order_seq_cst);
128 |     flush_data((void *)&children[leafPlace], sizeof(std::atomic<N *>));
129 | #endif
130 |     count--;
131 |     assert(getChild(k) == nullptr);
132 |     return true;
133 | }
134 | 
135 | N *N16::getAnyChild() const {
136 |     N *anyChild = nullptr;
137 |     for (int i = 0; i < 16; ++i) {
138 | #ifdef ZENTRY
139 |         auto child = getZentryPtr(zens[i].load());
140 | #else
141 |         N *child = children[i].load();
142 | #endif
143 |         if (child != nullptr) {
144 |             if (N::isLeaf(child)) {
145 |                 return child;
146 |             }
147 |             anyChild = child;
148 |         }
149 |     }
150 |     return anyChild;
151 | }
152 | 
153 | void N16::deleteChildren() {
154 |     for (std::size_t i = 0; i < compactCount; ++i) {
155 | #ifdef ZENTRY
156 |         N *child = N::clearDirty(getZentryPtr(zens[i].load()));
157 | #else
158 |         N *child = N::clearDirty(children[i].load());
159 | #endif
160 |         if (child != nullptr) {
161 |             N::deleteChildren(child);
162 |             N::deleteNode(child);
163 |         }
164 |     }
165 | }
166 | 
167 | void N16::getChildren(uint8_t start, uint8_t end,
168 |                       std::tuple<uint8_t, N *> children[],
169 |                       uint32_t &childrenCount) {
170 |     childrenCount = 0;
171 |     for (int i = 0; i < compactCount; ++i) {
172 | #ifdef ZENTRY
173 |         auto p = getZentryKeyPtr(zens[i].load());
174 |         p.first = flipSign(p.first);
175 |         if (p.first >= start && p.first <= end) {
176 |             if (p.second != nullptr) {
177 |                 children[childrenCount] = std::make_tuple(p.first, p.second);
178 |                 childrenCount++;
179 |             }
180 |         }
181 | #else
182 |         uint8_t key = flipSign(this->keys[i].load());
183 |         if (key >= start && key <= end) {
184 |             N *child = this->children[i].load();
185 |             if (child != nullptr) {
186 |                 children[childrenCount] = std::make_tuple(key, child);
187 |                 childrenCount++;
188 |             }
189 |         }
190 | #endif
191 |     }
192 |     std::sort(children, children + childrenCount,
193 |               [](auto &first, auto &second) {
194 |                   return std::get<0>(first) < std::get<0>(second);
195 |               });
196 | }
197 | 
198 | uint32_t N16::getCount() const {
199 |     uint32_t cnt = 0;
200 |     for (uint32_t i = 0; i < compactCount && cnt < 3; i++) {
201 | #ifdef ZENTRY
202 |         auto child = getZentryPtr(zens[i].load());
203 | #else
204 |         N *child = children[i].load();
205 | #endif
206 |         if (child != nullptr)
207 |             ++cnt;
208 |     }
209 |     return cnt;
210 | }
211 | void N16::graphviz_debug(std::ofstream &f) {
212 |     char buf[10000] = {};
213 |     sprintf(buf + strlen(buf), "node%lx [label=\"",
214 |             reinterpret_cast<uintptr_t>(this));
215 |     sprintf(buf + strlen(buf), "N16 %d\n", level);
216 |     auto pre = this->getPrefi();
217 |     sprintf(buf + strlen(buf), "Prefix Len: %d\n", pre.prefixCount);
218 |     sprintf(buf + strlen(buf), "Prefix: ");
219 |     for (int i = 0; i < std::min(pre.prefixCount, maxStoredPrefixLength); i++) {
220 |         sprintf(buf + strlen(buf), "%c ", pre.prefix[i]);
221 |     }
222 |     sprintf(buf + strlen(buf), "\n");
223 |     sprintf(buf + strlen(buf), "count: %d\n", count);
224 |     sprintf(buf + strlen(buf), "compact: %d\n", compactCount);
225 |     sprintf(buf + strlen(buf), "\"]\n");
226 | 
227 |     for (int i = 0; i < compactCount; i++) {
228 | #ifdef ZENTRY
229 |         auto pp = getZentryKeyPtr(zens[i].load());
230 |         auto p = pp.second;
231 |         auto x = pp.first;
232 | #else
233 |         auto p = children[i].load();
234 |         auto x = keys[i].load();
235 | #endif
236 |         if (p != nullptr) {
237 |             x = flipSign(x);
238 |             auto addr = reinterpret_cast<uintptr_t>(p);
239 |             if (isLeaf(p)) {
240 |                 addr = reinterpret_cast<uintptr_t>(getLeaf(p));
241 |             }
242 |             sprintf(buf + strlen(buf), "node%lx -- node%lx [label=\"%c\"]\n",
243 |                     reinterpret_cast<uintptr_t>(this), addr, x);
244 |         }
245 |     }
246 |     f << buf;
247 | 
248 |     for (int i = 0; i < compactCount; i++) {
249 | #ifdef ZENTRY
250 |         auto p = getZentryPtr(zens[i].load());
251 | #else
252 |         auto p = children[i].load();
253 | #endif
254 |         if (p != nullptr) {
255 |             if (isLeaf(p)) {
256 | #ifdef LEAF_ARRAY
257 |                 auto la = getLeafArray(p);
258 |                 la->graphviz_debug(f);
259 | #else
260 |                 auto l = getLeaf(p);
261 |                 l->graphviz_debug(f);
262 | #endif
263 |             } else {
264 |                 N::graphviz_debug(f, p);
265 |             }
266 |         }
267 |     }
268 | }
269 | 
270 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N16.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "N.h"
  4 | 
  5 | namespace PART_ns {
  6 | 
  7 | class N16 : public N {
  8 |   public:
  9 | #ifdef ZENTRY
 10 |     std::atomic<uintptr_t> zens[16];
 11 | #else
 12 |     std::atomic<uint8_t> keys[16];
 13 |     std::atomic<N *> children[16];
 14 | #endif
 15 |     static uint8_t flipSign(uint8_t keyByte) {
 16 |         // Flip the sign bit, enables signed SSE comparison of unsigned values,
 17 |         // used by Node16
 18 |         return keyByte ^ 128;
 19 |     }
 20 | 
 21 |     static inline unsigned ctz(uint16_t x) {
 22 |         // Count trailing zeros, only defined for x>0
 23 | #ifdef __GNUC__
 24 |         return __builtin_ctz(x);
 25 | #else
 26 |         // Adapted from Hacker's Delight
 27 |         unsigned n = 1;
 28 |         if ((x & 0xFF) == 0) {
 29 |             n += 8;
 30 |             x = x >> 8;
 31 |         }
 32 |         if ((x & 0x0F) == 0) {
 33 |             n += 4;
 34 |             x = x >> 4;
 35 |         }
 36 |         if ((x & 0x03) == 0) {
 37 |             n += 2;
 38 |             x = x >> 2;
 39 |         }
 40 |         return n - (x & 1);
 41 | #endif
 42 |     }
 43 | 
 44 |     int getChildPos(const uint8_t k);
 45 | 
 46 |   public:
 47 |     N16(uint32_t level, const uint8_t *prefix, uint32_t prefixLength)
 48 |         : N(NTypes::N16, level, prefix, prefixLength) {
 49 | 
 50 | #ifdef ZENTRY
 51 |         memset(zens, 0, sizeof(zens));
 52 | #else
 53 |         memset(keys, 0, sizeof(keys));
 54 |         memset(children, 0, sizeof(children));
 55 | #endif
 56 |     }
 57 | 
 58 |     N16(uint32_t level, const Prefix &prefi) : N(NTypes::N16, level, prefi) {
 59 | #ifdef ZENTRY
 60 |         memset(zens, 0, sizeof(zens));
 61 | #else
 62 |         memset(keys, 0, sizeof(keys));
 63 |         memset(children, 0, sizeof(children));
 64 | #endif
 65 |     }
 66 | 
 67 |     virtual ~N16() {}
 68 | 
 69 |     bool insert(uint8_t key, N *n, bool flush);
 70 | 
 71 |     template <class NODE> void copyTo(NODE *n) const {
 72 |         for (unsigned i = 0; i < compactCount; i++) {
 73 | 
 74 | #ifdef ZENTRY
 75 |             auto z = zens[i].load();
 76 |             N *child = getZentryPtr(z);
 77 |             if (child != nullptr) {
 78 |                 // not flush
 79 |                 n->insert(flipSign(getZentryKey(z)), child, false);
 80 |             }
 81 | #else
 82 |             N *child = children[i].load();
 83 |             if (child != nullptr) {
 84 |                 // not flush
 85 |                 n->insert(flipSign(keys[i].load()), child, false);
 86 |             }
 87 | #endif
 88 |         }
 89 |     }
 90 | 
 91 |     void change(uint8_t key, N *val);
 92 | 
 93 |     N *getChild(const uint8_t k);
 94 | 
 95 |     bool remove(uint8_t k, bool force, bool flush);
 96 | 
 97 |     N *getAnyChild() const;
 98 | 
 99 |     void deleteChildren();
100 | 
101 |     void getChildren(uint8_t start, uint8_t end,
102 |                      std::tuple<uint8_t, N *> children[],
103 |                      uint32_t &childrenCount);
104 | 
105 |     uint32_t getCount() const;
106 |     void graphviz_debug(std::ofstream &f);
107 | } __attribute__((aligned(64)));
108 | 
109 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N256.cpp:
--------------------------------------------------------------------------------
  1 | #include "N256.h"
  2 | #include "LeafArray.h"
  3 | #include "N.h"
  4 | #include <algorithm>
  5 | #include <assert.h>
  6 | 
  7 | namespace PART_ns {
  8 | 
  9 | void N256::deleteChildren() {
 10 |     for (uint64_t i = 0; i < 256; ++i) {
 11 |         N *child = N::clearDirty(children[i].load());
 12 |         if (child != nullptr) {
 13 |             N::deleteChildren(child);
 14 |             N::deleteNode(child);
 15 |         }
 16 |     }
 17 | }
 18 | 
 19 | bool N256::insert(uint8_t key, N *val, bool flush) {
 20 |     if (flush) {
 21 |         uint64_t oldp = (1ull << 56) | ((uint64_t)key << 48);
 22 |     }
 23 | 
 24 |     children[key].store(val, std::memory_order_seq_cst);
 25 |     if (flush) {
 26 |         flush_data((void *)&children[key], sizeof(std::atomic<N *>));
 27 |     }
 28 | 
 29 |     count++;
 30 |     return true;
 31 | }
 32 | 
 33 | void N256::change(uint8_t key, N *n) {
 34 | 
 35 |     children[key].store(n, std::memory_order_seq_cst);
 36 |     flush_data((void *)&children[key], sizeof(std::atomic<N *>));
 37 | }
 38 | 
 39 | N *N256::getChild(const uint8_t k) {
 40 |     N *child = children[k].load();
 41 |     return child;
 42 | }
 43 | 
 44 | bool N256::remove(uint8_t k, bool force, bool flush) {
 45 |     if (count <= 37 && !force) {
 46 |         return false;
 47 |     }
 48 | 
 49 |     children[k].store(nullptr, std::memory_order_seq_cst);
 50 |     flush_data((void *)&children[k], sizeof(std::atomic<N *>));
 51 |     count--;
 52 |     return true;
 53 | }
 54 | 
 55 | N *N256::getAnyChild() const {
 56 |     N *anyChild = nullptr;
 57 |     for (uint64_t i = 0; i < 256; ++i) {
 58 |         N *child = children[i].load();
 59 | 
 60 |         if (child != nullptr) {
 61 |             if (N::isLeaf(child)) {
 62 |                 return child;
 63 |             }
 64 |             anyChild = child;
 65 |         }
 66 |     }
 67 |     return anyChild;
 68 | }
 69 | 
 70 | void N256::getChildren(uint8_t start, uint8_t end,
 71 |                        std::tuple<uint8_t, N *> children[],
 72 |                        uint32_t &childrenCount) {
 73 |     childrenCount = 0;
 74 |     for (unsigned i = start; i <= end; i++) {
 75 |         N *child = this->children[i].load();
 76 | 
 77 |         if (child != nullptr) {
 78 |             children[childrenCount] = std::make_tuple(i, child);
 79 |             childrenCount++;
 80 |         }
 81 |     }
 82 | }
 83 | 
 84 | uint32_t N256::getCount() const {
 85 |     uint32_t cnt = 0;
 86 |     for (uint32_t i = 0; i < 256 && cnt < 3; i++) {
 87 |         N *child = children[i].load();
 88 |         if (child != nullptr)
 89 |             cnt++;
 90 |     }
 91 |     return cnt;
 92 | }
 93 | void N256::graphviz_debug(std::ofstream &f) {
 94 |     char buf[10000] = {};
 95 |     sprintf(buf + strlen(buf), "node%lx [label=\"",
 96 |             reinterpret_cast<uintptr_t>(this));
 97 |     sprintf(buf + strlen(buf), "N256 %d\n",level);
 98 |     auto pre = this->getPrefi();
 99 |     sprintf(buf + strlen(buf), "Prefix Len: %d\n", pre.prefixCount);
100 |     sprintf(buf + strlen(buf), "Prefix: ");
101 |     for (int i = 0; i < std::min(pre.prefixCount, maxStoredPrefixLength); i++) {
102 |         sprintf(buf + strlen(buf), "%c ", pre.prefix[i]);
103 |     }
104 |     sprintf(buf + strlen(buf), "\n");
105 |     sprintf(buf + strlen(buf), "count: %d\n", count);
106 |     sprintf(buf + strlen(buf), "compact: %d\n", compactCount);
107 |     sprintf(buf + strlen(buf), "\"]\n");
108 | 
109 |     for (auto i = 0; i < 256; i++) {
110 |         auto p = children[i].load();
111 |         if (p != nullptr) {
112 |             auto x = i;
113 |             auto addr = reinterpret_cast<uintptr_t>(p);
114 |             if (isLeaf(p)) {
115 |                 addr = reinterpret_cast<uintptr_t>(getLeaf(p));
116 |             }
117 |             sprintf(buf + strlen(buf), "node%lx -- node%lx [label=\"%c\"]\n",
118 |                     reinterpret_cast<uintptr_t>(this), addr, x);
119 |         }
120 |     }
121 |     f << buf;
122 | 
123 |     for (auto &i : children) {
124 |         auto p = i.load();
125 |         if (p != nullptr) {
126 |             if (isLeaf(p)) {
127 | #ifdef LEAF_ARRAY
128 |                 auto la = getLeafArray(p);
129 |                 la->graphviz_debug(f);
130 | #else
131 |                 auto l = getLeaf(p);
132 |                 l->graphviz_debug(f);
133 | #endif
134 |             } else {
135 |                 N::graphviz_debug(f, p);
136 |             }
137 |         }
138 |     }
139 | }
140 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N256.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "N.h"
 4 | 
 5 | namespace PART_ns {
 6 | 
 7 | class N256 : public N {
 8 |   public:
 9 |     std::atomic<N *> children[256];
10 | 
11 |   public:
12 |     N256(uint32_t level, const uint8_t *prefix, uint32_t prefixLength)
13 |         : N(NTypes::N256, level, prefix, prefixLength) {
14 |         memset(children, '\0', sizeof(children));
15 |     }
16 | 
17 |     N256(uint32_t level, const Prefix &prefi) : N(NTypes::N256, level, prefi) {
18 |         memset(children, '\0', sizeof(children));
19 |     }
20 | 
21 |     virtual ~N256() {}
22 | 
23 |     bool insert(uint8_t key, N *val, bool flush);
24 | 
25 |     template <class NODE> void copyTo(NODE *n) const {
26 |         for (int i = 0; i < 256; ++i) {
27 |             N *child = children[i].load();
28 |             if (child != nullptr) {
29 |                 // not flush
30 |                 n->insert(i, child, false);
31 |             }
32 |         }
33 |     }
34 | 
35 |     void change(uint8_t key, N *n);
36 | 
37 |     N *getChild(const uint8_t k);
38 | 
39 |     bool remove(uint8_t k, bool force, bool flush);
40 | 
41 |     N *getAnyChild() const;
42 | 
43 |     void deleteChildren();
44 | 
45 |     void getChildren(uint8_t start, uint8_t end,
46 |                      std::tuple<uint8_t, N *> children[],
47 |                      uint32_t &childrenCount);
48 | 
49 |     uint32_t getCount() const;
50 | 
51 |     void graphviz_debug(std::ofstream &f);
52 | } __attribute__((aligned(64)));
53 | 
54 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N4.cpp:
--------------------------------------------------------------------------------
  1 | #include "N4.h"
  2 | #include "LeafArray.h"
  3 | #include "N.h"
  4 | #include <algorithm>
  5 | #include <assert.h>
  6 | 
  7 | namespace PART_ns {
  8 | 
  9 | void N4::deleteChildren() {
 10 |     for (uint32_t i = 0; i < compactCount; ++i) {
 11 | #ifdef ZENTRY
 12 |         N *child = N::clearDirty(getZentryPtr(zens[i]));
 13 | #else
 14 |         N *child = N::clearDirty(children[i].load());
 15 | #endif
 16 |         if (child != nullptr) {
 17 |             N::deleteChildren(child);
 18 |             N::deleteNode(child);
 19 |         }
 20 |     }
 21 | }
 22 | 
 23 | bool N4::insert(uint8_t key, N *n, bool flush) {
 24 |     if (compactCount == 4) {
 25 |         return false;
 26 |     }
 27 | #ifdef ZENTRY
 28 |     zens[compactCount].store(makeZentry(key, n));
 29 |     if (flush)
 30 |         flush_data(&zens[compactCount], sizeof(std::atomic<uintptr_t>));
 31 | 
 32 | #else
 33 |     keys[compactCount].store(key, std::memory_order_seq_cst);
 34 |     if (flush)
 35 |         flush_data((void *)&keys[compactCount], sizeof(std::atomic<uint8_t>));
 36 | 
 37 |     children[compactCount].store(n, std::memory_order_seq_cst);
 38 |     if (flush) {
 39 |         flush_data((void *)&children[compactCount], sizeof(std::atomic<N *>));
 40 |     }
 41 | #endif
 42 |     compactCount++;
 43 |     count++;
 44 |     return true;
 45 | }
 46 | 
 47 | void N4::change(uint8_t key, N *val) {
 48 |     for (uint32_t i = 0; i < compactCount; ++i) {
 49 | #ifdef ZENTRY
 50 |         auto p = getZentryKeyPtr(zens[i].load());
 51 |         if (p.second != nullptr && p.first == key) {
 52 |             zens[i].store(makeZentry(key, val));
 53 |             flush_data((void *)&zens[i], sizeof(std::atomic<uintptr_t>));
 54 |             return;
 55 |         }
 56 | #else
 57 |         N *child = children[i].load();
 58 |         if (child != nullptr && keys[i].load() == key) {
 59 |             children[i].store(val, std::memory_order_seq_cst);
 60 |             flush_data((void *)&children[i], sizeof(std::atomic<N *>));
 61 |             return;
 62 |         }
 63 | #endif
 64 |     }
 65 | }
 66 | 
 67 | N *N4::getChild(const uint8_t k) {
 68 |     for (uint32_t i = 0; i < 4; ++i) {
 69 | #ifdef ZENTRY
 70 |         auto p = getZentryKeyPtr(zens[i].load());
 71 |         if (p.second != nullptr && p.first == k) {
 72 |             return p.second;
 73 |         }
 74 | #else
 75 |         N *child = children[i].load();
 76 |         if (child != nullptr && keys[i].load() == k) {
 77 |             return child;
 78 |         }
 79 | #endif
 80 |     }
 81 |     return nullptr;
 82 | }
 83 | 
 84 | bool N4::remove(uint8_t k, bool force, bool flush) {
 85 |     for (uint32_t i = 0; i < compactCount; ++i) {
 86 | #ifdef ZENTRY
 87 |         auto p = getZentryKeyPtr(zens[i].load());
 88 |         if (p.second != nullptr && p.first == k) {
 89 |             zens[i].store(0);
 90 |             flush_data(&zens[i], sizeof(std::atomic<uintptr_t>));
 91 |             count--;
 92 |             return true;
 93 |         }
 94 | #else
 95 |         if (children[i] != nullptr && keys[i].load() == k) {
 96 |             children[i].store(nullptr, std::memory_order_seq_cst);
 97 |             flush_data((void *)&children[i], sizeof(std::atomic<N *>));
 98 |             count--;
 99 |             return true;
100 |         }
101 | #endif
102 |     }
103 |     return false;
104 | }
105 | 
106 | N *N4::getAnyChild() const {
107 |     N *anyChild = nullptr;
108 |     for (uint32_t i = 0; i < 4; ++i) {
109 | #ifdef ZENTRY
110 |         N *child = getZentryPtr(zens[i].load());
111 |         if (child != nullptr) {
112 |             if (N::isLeaf(child)) {
113 |                 return child;
114 |             }
115 |             anyChild = child;
116 |         }
117 | #else
118 |         N *child = children[i].load();
119 |         if (child != nullptr) {
120 |             if (N::isLeaf(child)) {
121 |                 return child;
122 |             }
123 |             anyChild = child;
124 |         }
125 | #endif
126 |     }
127 |     return anyChild;
128 | }
129 | 
130 | // in the critical section
131 | std::tuple<N *, uint8_t> N4::getSecondChild(const uint8_t key) const {
132 |     for (uint32_t i = 0; i < compactCount; ++i) {
133 | #ifdef ZENTRY
134 |         auto p = getZentryKeyPtr(zens[i].load());
135 |         if (p.second != nullptr) {
136 |             if (p.first != key) {
137 |                 return std::make_tuple(p.second, p.first);
138 |             }
139 |         }
140 | #else
141 |         N *child = children[i].load();
142 |         if (child != nullptr) {
143 |             uint8_t k = keys[i].load();
144 |             if (k != key) {
145 |                 return std::make_tuple(child, k);
146 |             }
147 |         }
148 | #endif
149 |     }
150 |     return std::make_tuple(nullptr, 0);
151 | }
152 | 
153 | // must read persisted child
154 | void N4::getChildren(uint8_t start, uint8_t end,
155 |                      std::tuple<uint8_t, N *> children[],
156 |                      uint32_t &childrenCount) {
157 |     childrenCount = 0;
158 |     for (uint32_t i = 0; i < 4; ++i) {
159 | #ifdef ZENTRY
160 |         auto p = getZentryKeyPtr(zens[i]);
161 |         if (p.first >= start && p.first <= end) {
162 |             if (p.second != nullptr) {
163 |                 children[childrenCount] = std::make_tuple(p.first, p.second);
164 |                 childrenCount++;
165 |             }
166 |         }
167 | #else
168 |         uint8_t key = this->keys[i].load();
169 |         if (key >= start && key <= end) {
170 |             N *child = this->children[i].load();
171 |             if (child != nullptr) {
172 |                 children[childrenCount] = std::make_tuple(key, child);
173 |                 childrenCount++;
174 |             }
175 |         }
176 | #endif
177 |     }
178 |     std::sort(children, children + childrenCount,
179 |               [](auto &first, auto &second) {
180 |                   return std::get<0>(first) < std::get<0>(second);
181 |               });
182 | }
183 | 
184 | uint32_t N4::getCount() const {
185 |     uint32_t cnt = 0;
186 |     for (uint32_t i = 0; i < compactCount && cnt < 3; i++) {
187 | #ifdef ZENTRY
188 |         N *child = getZentryPtr(zens[i].load());
189 | #else
190 |         N *child = children[i].load();
191 | #endif
192 |         if (child != nullptr)
193 |             cnt++;
194 |     }
195 |     return cnt;
196 | }
197 | void N4::graphviz_debug(std::ofstream &f) {
198 |     char buf[1000] = {};
199 |     sprintf(buf + strlen(buf), "node%lx [label=\"",
200 |             reinterpret_cast<uintptr_t>(this));
201 |     sprintf(buf + strlen(buf), "N4 %d\n", level);
202 |     auto pre = this->getPrefi();
203 |     sprintf(buf + strlen(buf), "Prefix Len: %d\n", pre.prefixCount);
204 |     sprintf(buf + strlen(buf), "Prefix: ");
205 |     for (int i = 0; i < std::min(pre.prefixCount, maxStoredPrefixLength); i++) {
206 |         sprintf(buf + strlen(buf), "%c ", pre.prefix[i]);
207 |     }
208 |     sprintf(buf + strlen(buf), "\n");
209 |     sprintf(buf + strlen(buf), "count: %d\n", count);
210 |     sprintf(buf + strlen(buf), "compact: %d\n", compactCount);
211 |     sprintf(buf + strlen(buf), "\"]\n");
212 | 
213 |     for (int i = 0; i < compactCount; i++) {
214 | #ifdef ZENTRY
215 |         auto pp = getZentryKeyPtr(zens[i].load());
216 |         auto p = pp.second;
217 |         auto x = pp.first;
218 | #else
219 |         auto p = children[i].load();
220 |         auto x = keys[i].load();
221 | #endif
222 |         if (p != nullptr) {
223 | 
224 |             auto addr = reinterpret_cast<uintptr_t>(p);
225 |             if (isLeaf(p)) {
226 |                 addr = reinterpret_cast<uintptr_t>(getLeaf(p));
227 |             }
228 |             sprintf(buf + strlen(buf), "node%lx -- node%lx [label=\"%c\"]\n",
229 |                     reinterpret_cast<uintptr_t>(this), addr, x);
230 |         }
231 |     }
232 |     f << buf;
233 | 
234 |     for (int i = 0; i < compactCount; i++) {
235 | #ifdef ZENTRY
236 |         auto p = getZentryPtr(zens[i].load());
237 | #else
238 |         auto p = children[i].load();
239 | #endif
240 |         if (p != nullptr) {
241 |             if (isLeaf(p)) {
242 | #ifdef LEAF_ARRAY
243 |                 auto la = getLeafArray(p);
244 |                 la->graphviz_debug(f);
245 | #else
246 |                 auto l = getLeaf(p);
247 |                 l->graphviz_debug(f);
248 | #endif
249 |             } else {
250 |                 N::graphviz_debug(f, p);
251 |             }
252 |         }
253 |     }
254 | }
255 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N4.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "N.h"
 4 | 
 5 | namespace PART_ns {
 6 | 
 7 | class N4 : public N {
 8 |   public:
 9 | #ifdef ZENTRY
10 |     std::atomic<uintptr_t> zens[4];
11 | #else
12 |     std::atomic<uint8_t> keys[4];
13 |     std::atomic<N *> children[4];
14 | #endif
15 |   public:
16 |     N4(uint32_t level, const uint8_t *prefix, uint32_t prefixLength)
17 |         : N(NTypes::N4, level, prefix, prefixLength) {
18 | #ifdef ZENTRY
19 |         memset(zens, 0, sizeof(zens));
20 | #else
21 |         memset(keys, 0, sizeof(keys));
22 |         memset(children, 0, sizeof(children));
23 | #endif
24 |     }
25 | 
26 |     N4(uint32_t level, const Prefix &prefi) : N(NTypes::N4, level, prefi) {
27 | #ifdef ZENTRY
28 |         memset(zens, 0, sizeof(zens));
29 | #else
30 |         memset(keys, 0, sizeof(keys));
31 |         memset(children, 0, sizeof(children));
32 | #endif
33 |     }
34 | 
35 |     virtual ~N4() {}
36 | 
37 |     bool insert(uint8_t key, N *n, bool flush);
38 | 
39 |     template <class NODE> void copyTo(NODE *n) const {
40 |         for (uint32_t i = 0; i < compactCount; ++i) {
41 | #ifdef ZENTRY
42 |             auto z = zens[i].load();
43 |             N *child = getZentryPtr(z);
44 |             if (child != nullptr) {
45 |                 // not flush
46 |                 n->insert(getZentryKey(z), child, false);
47 |             }
48 | #else
49 |             N *child = children[i].load();
50 |             if (child != nullptr) {
51 |                 // not flush
52 |                 n->insert(keys[i].load(), child, false);
53 |             }
54 | #endif
55 |         }
56 |     }
57 | 
58 |     void change(uint8_t key, N *val);
59 | 
60 |     N *getChild(const uint8_t k);
61 | 
62 |     bool remove(uint8_t k, bool force, bool flush);
63 | 
64 |     N *getAnyChild() const;
65 | 
66 |     std::tuple<N *, uint8_t> getSecondChild(const uint8_t key) const;
67 | 
68 |     void deleteChildren();
69 | 
70 |     void getChildren(uint8_t start, uint8_t end,
71 |                      std::tuple<uint8_t, N *> children[],
72 |                      uint32_t &childrenCount);
73 | 
74 |     uint32_t getCount() const;
75 | 
76 |     void graphviz_debug(std::ofstream &f);
77 | } __attribute__((aligned(64)));
78 | 
79 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N48.cpp:
--------------------------------------------------------------------------------
  1 | #include "N48.h"
  2 | #include "LeafArray.h"
  3 | #include "N.h"
  4 | #include <algorithm>
  5 | #include <assert.h>
  6 | 
  7 | namespace PART_ns {
  8 | 
  9 | bool N48::insert(uint8_t key, N *n, bool flush) {
 10 |     if (compactCount == 48) {
 11 |         return false;
 12 |     }
 13 | 
 14 | #ifdef ZENTRY
 15 |     childIndex[key].store(compactCount, std::memory_order_seq_cst);
 16 | 
 17 |     zens[compactCount].store(makeZentry(key, n));
 18 |     if (flush) {
 19 |         flush_data(&zens[compactCount], sizeof(std::atomic<uintptr_t>));
 20 |     }
 21 | #else
 22 |     childIndex[key].store(compactCount, std::memory_order_seq_cst);
 23 |     if (flush)
 24 |         flush_data((void *)&childIndex[key], sizeof(std::atomic<uint8_t>));
 25 | 
 26 |     children[compactCount].store(n, std::memory_order_seq_cst);
 27 |     if (flush) {
 28 |         flush_data((void *)&children[compactCount], sizeof(std::atomic<N *>));
 29 |     }
 30 | #endif
 31 | 
 32 |     compactCount++;
 33 |     count++;
 34 |     return true;
 35 | }
 36 | 
 37 | void N48::change(uint8_t key, N *val) {
 38 |     uint8_t index = childIndex[key].load();
 39 |     assert(index != emptyMarker);
 40 | #ifdef ZENTRY
 41 |     zens[index].store(makeZentry(key, val));
 42 |     flush_data(&zens[index], sizeof(std::atomic<uintptr_t>));
 43 | #else
 44 |     children[index].store(val, std::memory_order_seq_cst);
 45 |     flush_data((void *)&children[index], sizeof(std::atomic<N *>));
 46 | #endif
 47 | }
 48 | 
 49 | N *N48::getChild(const uint8_t k) {
 50 |     uint8_t index = childIndex[k].load();
 51 |     if (index == emptyMarker) {
 52 |         return nullptr;
 53 |     } else {
 54 | #ifdef ZENTRY
 55 |         auto child = getZentryPtr(zens[index].load());
 56 | #else
 57 |         N *child = children[index].load();
 58 | #endif
 59 |         return child;
 60 |     }
 61 | }
 62 | 
 63 | bool N48::remove(uint8_t k, bool force, bool flush) {
 64 |     if (count <= 12 && !force) {
 65 |         return false;
 66 |     }
 67 |     uint8_t index = childIndex[k].load();
 68 |     assert(index != emptyMarker);
 69 | #ifdef ZENTRY
 70 |     zens[index].store(0);
 71 |     flush_data(&zens[index], sizeof(std::atomic<uintptr_t>));
 72 | #else
 73 |     children[index].store(nullptr, std::memory_order_seq_cst);
 74 |     flush_data((void *)&children[index], sizeof(std::atomic<N *>));
 75 | #endif
 76 |     count--;
 77 |     assert(getChild(k) == nullptr);
 78 |     return true;
 79 | }
 80 | 
 81 | N *N48::getAnyChild() const {
 82 |     N *anyChild = nullptr;
 83 |     for (unsigned i = 0; i < 48; i++) {
 84 | #ifdef ZENTRY
 85 |         auto child = getZentryPtr(zens[i].load());
 86 | #else
 87 |         N *child = children[i].load();
 88 | #endif
 89 |         if (child != nullptr) {
 90 |             if (N::isLeaf(child)) {
 91 |                 return child;
 92 |             }
 93 |             anyChild = child;
 94 |         }
 95 |     }
 96 |     return anyChild;
 97 | }
 98 | 
 99 | void N48::deleteChildren() {
100 |     for (unsigned i = 0; i < 256; i++) {
101 |         uint8_t index = childIndex[i].load();
102 | #ifdef ZENTRY
103 |         auto child = getZentryPtr(zens[index].load());
104 |         if (index != emptyMarker && child != nullptr) {
105 |             N *child = N::clearDirty(child);
106 |             N::deleteChildren(child);
107 |             N::deleteNode(child);
108 |         }
109 | #else
110 |         if (index != emptyMarker && children[index].load() != nullptr) {
111 |             N *child = N::clearDirty(children[index].load());
112 |             N::deleteChildren(child);
113 |             N::deleteNode(child);
114 |         }
115 | #endif
116 |     }
117 | }
118 | 
119 | void N48::getChildren(uint8_t start, uint8_t end,
120 |                       std::tuple<uint8_t, N *> children[],
121 |                       uint32_t &childrenCount) {
122 |     childrenCount = 0;
123 |     for (unsigned i = start; i <= end; i++) {
124 |         uint8_t index = this->childIndex[i].load();
125 | #ifdef ZENTRY
126 |         auto child = getZentryPtr(zens[index].load());
127 |         if (index != emptyMarker && child != nullptr) {
128 |             if (child != nullptr) {
129 |                 children[childrenCount] = std::make_tuple(i, child);
130 |                 childrenCount++;
131 |             }
132 |         }
133 | #else
134 |         if (index != emptyMarker && this->children[index] != nullptr) {
135 |             N *child = this->children[index].load();
136 | 
137 |             if (child != nullptr) {
138 |                 children[childrenCount] = std::make_tuple(i, child);
139 |                 childrenCount++;
140 |             }
141 |         }
142 | #endif
143 |     }
144 | }
145 | 
146 | uint32_t N48::getCount() const {
147 |     uint32_t cnt = 0;
148 |     for (uint32_t i = 0; i < 256 && cnt < 3; i++) {
149 |         uint8_t index = childIndex[i].load();
150 | #ifdef ZENTRY
151 |         if (index != emptyMarker && getZentryPtr(zens[index].load()) != nullptr)
152 |             cnt++;
153 | #else
154 |         if (index != emptyMarker && children[index].load() != nullptr)
155 |             cnt++;
156 | #endif
157 |     }
158 |     return cnt;
159 | }
160 | void N48::graphviz_debug(std::ofstream &f) {
161 |     char buf[10000] = {};
162 |     sprintf(buf + strlen(buf), "node%lx [label=\"",
163 |             reinterpret_cast<uintptr_t>(this));
164 |     sprintf(buf + strlen(buf), "N48 %d\n", level);
165 |     auto pre = this->getPrefi();
166 |     sprintf(buf + strlen(buf), "Prefix Len: %d\n", pre.prefixCount);
167 |     sprintf(buf + strlen(buf), "Prefix: ");
168 |     for (int i = 0; i < std::min(pre.prefixCount, maxStoredPrefixLength); i++) {
169 |         sprintf(buf + strlen(buf), "%c ", pre.prefix[i]);
170 |     }
171 |     sprintf(buf + strlen(buf), "\n");
172 |     sprintf(buf + strlen(buf), "count: %d\n", count);
173 |     sprintf(buf + strlen(buf), "compact: %d\n", compactCount);
174 |     sprintf(buf + strlen(buf), "\"]\n");
175 | 
176 |     for (auto &i : childIndex) {
177 |         auto ci = i.load();
178 |         if (ci != emptyMarker) {
179 | #ifdef ZENTRY
180 |             auto pp = getZentryKeyPtr(zens[ci].load());
181 |             auto p = pp.second;
182 |             auto x = pp.first;
183 | #else
184 |             auto p = children[i].load();
185 |             auto x = ci;
186 | #endif
187 |             if (p != nullptr) {
188 |                 auto addr = reinterpret_cast<uintptr_t>(p);
189 |                 if (isLeaf(p)) {
190 |                     addr = reinterpret_cast<uintptr_t>(getLeaf(p));
191 |                 }
192 |                 sprintf(buf + strlen(buf),
193 |                         "node%lx -- node%lx [label=\"%c\"]\n",
194 |                         reinterpret_cast<uintptr_t>(this), addr, x);
195 |             }
196 |         }
197 |     }
198 |     f << buf;
199 | 
200 |     for (auto &i : childIndex) {
201 |         auto ci = i.load();
202 |         if (ci != emptyMarker) {
203 | #ifdef ZENTRY
204 |             auto p = getZentryPtr(zens[ci].load());
205 | #else
206 |             auto p = children[ci].load();
207 | #endif
208 |             if (p != nullptr) {
209 |                 if (isLeaf(p)) {
210 | #ifdef LEAF_ARRAY
211 |                     auto la = getLeafArray(p);
212 |                     la->graphviz_debug(f);
213 | #else
214 |                     auto l = getLeaf(p);
215 |                     l->graphviz_debug(f);
216 | #endif
217 |                 } else {
218 |                     N::graphviz_debug(f, p);
219 |                 }
220 |             }
221 |         }
222 |     }
223 | }
224 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/N48.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "N.h"
 4 | 
 5 | namespace PART_ns {
 6 | 
 7 | class N48 : public N {
 8 |   public:
 9 |     std::atomic<uint8_t> childIndex[256];
10 | #ifdef ZENTRY
11 |     std::atomic<uintptr_t> zens[48];
12 | #else
13 |     std::atomic<N *> children[48];
14 | #endif
15 |   public:
16 |     static const uint8_t emptyMarker = 48;
17 | 
18 |     N48(uint32_t level, const uint8_t *prefix, uint32_t prefixLength)
19 |         : N(NTypes::N48, level, prefix, prefixLength) {
20 |         memset(childIndex, emptyMarker, sizeof(childIndex));
21 | #ifdef ZENTRY
22 |         memset(zens, 0, sizeof(zens));
23 | #else
24 |         memset(children, 0, sizeof(children));
25 | #endif
26 |     }
27 | 
28 |     N48(uint32_t level, const Prefix &prefi) : N(NTypes::N48, level, prefi) {
29 |         memset(childIndex, emptyMarker, sizeof(childIndex));
30 | #ifdef ZENTRY
31 |         memset(zens, 0, sizeof(zens));
32 | #else
33 |         memset(children, 0, sizeof(children));
34 | #endif
35 |     }
36 | 
37 |     virtual ~N48() {}
38 | 
39 |     bool insert(uint8_t key, N *n, bool flush);
40 | 
41 |     template <class NODE> void copyTo(NODE *n) const {
42 |         for (unsigned i = 0; i < 256; i++) {
43 |             uint8_t index = childIndex[i].load();
44 | #ifdef ZENTRY
45 |             auto child = getZentryPtr(zens[index].load());
46 |             if (index != emptyMarker && child != nullptr) {
47 |                 // not flush
48 |                 n->insert(i, child, false);
49 |             }
50 | #else
51 |             if (index != emptyMarker && children[index].load() != nullptr) {
52 |                 // not flush
53 |                 n->insert(i, children[index].load(), false);
54 |             }
55 | #endif
56 |         }
57 |     }
58 | 
59 |     void change(uint8_t key, N *val);
60 | 
61 |     N *getChild(const uint8_t k);
62 | 
63 |     bool remove(uint8_t k, bool force, bool flush);
64 | 
65 |     N *getAnyChild() const;
66 | 
67 |     void deleteChildren();
68 | 
69 |     void getChildren(uint8_t start, uint8_t end,
70 |                      std::tuple<uint8_t, N *> children[],
71 |                      uint32_t &childrenCount);
72 | 
73 |     uint32_t getCount() const;
74 | 
75 |     void graphviz_debug(std::ofstream &f);
76 | } __attribute__((aligned(64)));
77 | 
78 | } // namespace PART_ns


--------------------------------------------------------------------------------
/ART/Tree.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "N.h"
 3 | #include "N16.h"
 4 | #include "N256.h"
 5 | #include "N4.h"
 6 | #include "N48.h"
 7 | #include "LeafArray.h"
 8 | #include <libpmemobj.h>
 9 | #include <set>
10 | 
11 | namespace PART_ns {
12 | 
13 | class Tree {
14 |   public:
15 |   private:
16 |     N *root;
17 | 
18 |     bool checkKey(const Key *ret, const Key *k) const;
19 | 
20 |   public:
21 |     enum class CheckPrefixResult : uint8_t { Match, NoMatch, OptimisticMatch };
22 | 
23 |     enum class CheckPrefixPessimisticResult : uint8_t {
24 |         Match,
25 |         NoMatch,
26 |         SkippedLevel
27 |     };
28 | 
29 |     enum class PCCompareResults : uint8_t {
30 |         Smaller,
31 |         Equal,
32 |         Bigger,
33 |         SkippedLevel
34 |     };
35 |     enum class PCEqualsResults : uint8_t {
36 |         BothMatch,
37 |         Contained,
38 |         NoMatch,
39 |         SkippedLevel
40 |     };
41 |     enum class OperationResults : uint8_t {
42 |         Success,
43 |         NotFound, // remove
44 |         Existed,  // insert
45 |         UnSuccess
46 |     };
47 |     static CheckPrefixResult checkPrefix(N *n, const Key *k, uint32_t &level);
48 | 
49 |     static CheckPrefixPessimisticResult
50 |     checkPrefixPessimistic(N *n, const Key *k, uint32_t &level,
51 |                            uint8_t &nonMatchingKey, Prefix &nonMatchingPrefix);
52 | 
53 |     static PCCompareResults checkPrefixCompare(const N *n, const Key *k,
54 |                                                uint32_t &level);
55 | 
56 |     static PCEqualsResults checkPrefixEquals(const N *n, uint32_t &level,
57 |                                              const Key *start, const Key *end);
58 | 
59 |   public:
60 |     Tree();
61 | 
62 |     Tree(const Tree &) = delete;
63 | 
64 |     ~Tree();
65 | 
66 |     void rebuild(std::vector<std::pair<uint64_t, size_t>> &rs,
67 |                  uint64_t start_addr, uint64_t end_addr, int thread_id);
68 | 
69 |     Leaf *lookup(const Key *k) const;
70 | 
71 |     OperationResults update(const Key *k) const;
72 | 
73 |     bool lookupRange(const Key *start, const Key *end, const Key *continueKey,
74 |                      Leaf *result[], std::size_t resultLen,
75 |                      std::size_t &resultCount) const;
76 | 
77 |     OperationResults insert(const Key *k);
78 | 
79 |     OperationResults remove(const Key *k);
80 | 
81 |     Leaf *allocLeaf(const Key *k) const;
82 | 
83 |     void graphviz_debug();
84 | } __attribute__((aligned(64)));
85 | 
86 | #ifdef ARTPMDK
87 | void *allocate_size(size_t size);
88 | 
89 | #endif
90 | 
91 | #ifdef COUNT_ALLOC
92 | double getalloctime();
93 | #endif
94 | 
95 | #ifdef CHECK_COUNT
96 | int get_count();
97 | #endif
98 | } // namespace PART_ns
99 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8)
 2 | project(P-ART)
 3 | 
 4 | if (NOT CMAKE_BUILD_TYPE)
 5 |     message(STATUS "No build type selected, default to Release")
 6 |     set(CMAKE_BUILD_TYPE Release)
 7 | else ()
 8 |     message(STATUS "Build type is set to ${CMAKE_BUILD_TYPE}")
 9 | endif ()
10 | 
11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -O0 -g -march=native -mavx -mavx2")
12 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -O0 -g -march=native -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -mavx -mavx2")
13 | 
14 | ## Instruction options for Cache line flush
15 | #add_definitions(-DCLFLUSH)
16 | #add_definitions(-DCLFLUSH_OPT)
17 | add_definitions(-DCLWB)
18 | add_definitions(-DVARIABLE_LENGTH)
19 | add_definitions(-DUSE_PMDK)
20 | add_definitions(-DACMA) # for fastfair and skiplist with dcmm
21 | add_definitions(-DFF_GC) # ff_gc
22 | add_definitions(-DRECLAIM_MEMORY)
23 | add_definitions(-DKEY_INLINE)
24 | #add_definitions(-DARTPMDK) # for DLART with PMDK
25 | #add_definitions(-DCOUNT_ALLOC)
26 | #add_definitions(-DLOG_FREE)
27 | #add_definitions(-DCHECK_COUNT)
28 | add_definitions(-DINSTANT_RESTART)
29 | 
30 | add_definitions(-DLEAF_ARRAY)
31 | add_definitions(-DFIND_FIRST)
32 | #add_definitions(-DSORT_LEAVES)
33 | 
34 | #add_definitions(-DZENTRY)
35 | 
36 | #malloc
37 | #add_definitions(-DPMALLOC)
38 | #add_definitions(-DTXPMALLOC)
39 | add_definitions(-DTRANSACTIONAL)
40 | 
41 | link_libraries(pthread atomic boost_system boost_thread gtest)
42 | link_libraries(pmemobj)
43 | find_library(TbbLib tbb)
44 | 
45 | include_directories(./ART
46 |         ./benchmark
47 |         ./nvm_mgr
48 |         ./test
49 |         ./fast_fair
50 |         ./lf-skiplist)
51 | 
52 | aux_source_directory(ART DIR_ART_SRCS)
53 | aux_source_directory(benchmark DIR_BENCH_SRCS)
54 | aux_source_directory(fast_fair DIR_FF_SRCS)
55 | aux_source_directory(lf-skiplist DIR_SL_SRCS)
56 | aux_source_directory(nvm_mgr   DIR_NVMMGR_SRCS)
57 | aux_source_directory(perf      DIR_PERF_SRCS)
58 | aux_source_directory(test      DIR_TEST_SRC)
59 | 
60 | add_library(Indexes ${DIR_ART_SRCS} ${DIR_BENCH_SRCS} ${DIR_FF_SRCS} ${DIR_NVMMGR_SRCS} ${DIR_SL_SRCS} )
61 | target_link_libraries(Indexes ${TbbLib})
62 | 
63 | 
64 | #set(P_ART_BENCH perf/art_simple_bench.cpp)
65 | #add_executable(art_simple_bench ${P_ART_BENCH})
66 | #target_link_libraries(art_simple_bench Indexes)
67 | 
68 | set(BENCHMARK perf/main.cpp)
69 | add_executable(benchmark ${BENCHMARK})
70 | target_link_libraries(benchmark Indexes)
71 | 
72 | set(MALLOC_DIFF perf/malloc_diff.cpp)
73 | add_executable(malloc_diff ${MALLOC_DIFF})
74 | target_link_libraries(malloc_diff Indexes)
75 | 
76 | add_executable(unittest ${DIR_TEST_SRC})
77 | target_link_libraries(unittest Indexes gtest)
78 | 
79 | #add_executable(gen_mail script/gen_mail.cpp)
80 | #add_executable(extract_data script/extract_data.cpp)
81 | 
82 | add_executable(acmma perf/acmma.cpp)
83 | target_link_libraries(acmma Indexes)


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright 2014-2015 Florian Scheibner
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |     http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 
15 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## ROART: Range Optimized Adaptive Radix Tree
 2 | 
 3 | ## Build & Run
 4 | 
 5 | #### Build
 6 | 
 7 | ```
 8 | $ mkdir build
 9 | $ cd build
10 | $ cmake ..
11 | $ make -j
12 | ```
13 | 
14 | #### Run
15 | 
16 | ```
17 | $ ./benchmark
18 | ```


--------------------------------------------------------------------------------
/benchmark/benchmarks.h:
--------------------------------------------------------------------------------
 1 | #ifndef _BENCHMARKS_H_
 2 | #define _BENCHMARKS_H_
 3 | 
 4 | #include "config.h"
 5 | #include "microbench.h"
 6 | 
 7 | static Benchmark *getBenchmark(Config conf) {
 8 |     switch (conf.benchmark) {
 9 |     case READ_ONLY:
10 |         return new ReadOnlyBench(conf);
11 |     case INSERT_ONLY:
12 |         return new InsertOnlyBench(conf);
13 |     case UPDATE_ONLY:
14 |         return new UpdateOnlyBench(conf);
15 |     case DELETE_ONLY:
16 |         return new DeleteOnlyBench(conf);
17 |     case MIXED_BENCH:
18 |         return new MixedBench(conf);
19 |     case YCSB_A:
20 |         return new YSCBA(conf);
21 |     case YCSB_B:
22 |         return new YSCBB(conf);
23 |     case YCSB_C:
24 |         return new YSCBC(conf);
25 |     case YCSB_D:
26 |         return new YSCBD(conf);
27 |     case YCSB_E:
28 |         return new YSCBE(conf);
29 |     case SCAN_BENCH:
30 |         return new ScanBench(conf);
31 |     case RECOVERY_BENCH:
32 |         return new UpdateOnlyBench(conf);
33 |     default:
34 |         printf("none support benchmark %d\n", conf.benchmark);
35 |         exit(0);
36 |     }
37 |     return NULL;
38 | }
39 | #endif
40 | 


--------------------------------------------------------------------------------
/benchmark/config.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cassert>
  4 | #include <getopt.h>
  5 | #include <iostream>
  6 | #include <unistd.h>
  7 | 
  8 | const int max_thread_num = 36;
  9 | 
 10 | inline const char *nvm_dir = "/mnt/pmem0/jzc/";
 11 | 
 12 | enum IndexType { PART, FAST_FAIR, SKIPLIST, _IndexTypeNumber };
 13 | 
 14 | enum KeyType { Integer, String, _KeyTypeNumber };
 15 | 
 16 | enum DataDistrubute { RANDOM, ZIPFIAN, _DataDistrbuteNumber };
 17 | 
 18 | enum BenchMarkType {
 19 |     READ_ONLY,
 20 |     INSERT_ONLY,
 21 |     UPDATE_ONLY,
 22 |     DELETE_ONLY,
 23 |     MIXED_BENCH,
 24 | 
 25 |     YCSB_A,
 26 |     YCSB_B,
 27 |     YCSB_C,
 28 |     YCSB_D,
 29 |     YCSB_E,
 30 | 
 31 |     SCAN_BENCH,
 32 |     RECOVERY_BENCH,
 33 |     _BenchMarkType
 34 | };
 35 | 
 36 | struct Config {
 37 |     IndexType type;
 38 |     BenchMarkType benchmark;
 39 |     KeyType key_type;
 40 | 
 41 |     int num_threads;
 42 |     unsigned long long init_keys;
 43 |     int time;
 44 |     int val_length;
 45 |     bool share_memory;
 46 |     float duration;
 47 | 
 48 |     int email;
 49 | 
 50 |     std::string filename;
 51 |     DataDistrubute workload;
 52 |     int read_ratio; // for read-upadte benchmark, (read_ratio)%.
 53 | 
 54 |     float skewness;
 55 |     int scan_length;
 56 |     int throughput;
 57 |     bool latency_test;
 58 | 
 59 |     bool instant_restart;
 60 | 
 61 |     void report() {
 62 |         printf("--- Config ---\n");
 63 |         printf(
 64 |             "type:\t %d\nbenchmark:\t %d\nthreads:\t %d\ninit_keys:\t %lld\n",
 65 |             type, benchmark, num_threads, init_keys);
 66 |         printf("--------------\n");
 67 |     }
 68 | };
 69 | 
 70 | static struct option opts[] = {
 71 |     {"help", no_argument, NULL, 'h'},
 72 |     {"type", required_argument, NULL, 't'},
 73 |     {"num_threads", required_argument, NULL, 'n'},
 74 |     {"keys", required_argument, NULL, 'k'},
 75 |     {"share_memory", no_argument, NULL, 's'},
 76 |     {"duration", required_argument, NULL, 'd'},
 77 |     {"benchmark", required_argument, NULL, 'b'},
 78 |     {"filename", required_argument, NULL, 'f'},
 79 |     {"workload", required_argument, NULL, 'w'},
 80 |     {"skewness", required_argument, NULL, 'S'},
 81 |     {"scan_length", required_argument, NULL, 'l'},
 82 |     {"read_ratio", required_argument, NULL, 'r'},
 83 | };
 84 | 
 85 | static void usage_exit(FILE *out) {
 86 |     fprintf(
 87 |         out,
 88 |         "Command line options : nstore <options> \n"
 89 |         "   -h --help              : Print help message \n"
 90 |         "   -t --type              : Index type : 0 (PART) 1 (FAST_FAIR) 2 "
 91 |         "(SKIPLIST) \n"
 92 |         "   -K --key_type          : Key type : 0 (Integer) 1 (String) \n"
 93 |         "   -n --num_threads       : Number of workers \n"
 94 |         "   -k --keys              : Number of key-value pairs at begin\n"
 95 |         "   -L --value_length      : Length of string value\n"
 96 |         "   -e --email             : Email List key: 0(rand) 1(email key)\n"
 97 |         "   -s --non_share_memory  : Use different index instances among "
 98 |         "different workers\n"
 99 |         "   -d --duration          : Execution time\n"
100 |         "   -b --benchmark         : Benchmark type, 0-%d\n"
101 |         "   -w --workload          : type of workload: 0 (RANDOM) 1 (ZIPFIAN)\n"
102 |         "   -S --skewed            : skewness: 0-1 (default 0.99)\n"
103 |         "   -l --scan_length       : scan_length: int (default 100)\n"
104 |         "   -r --read_ratio        : read ratio: int (default 50)\n",
105 |         _BenchMarkType - 1);
106 |     exit(EXIT_FAILURE);
107 | }
108 | 
109 | static void parse_arguments(int argc, char *argv[], Config &state) {
110 |     // Default Values
111 |     state.type = PART;
112 |     state.num_threads = 4;
113 |     state.key_type = String;
114 |     state.email = 0;
115 |     state.init_keys = 20000000;
116 |     state.time = 5;
117 |     state.val_length = 8;
118 |     state.share_memory = true;
119 |     state.duration = 1;
120 |     state.benchmark = SCAN_BENCH;
121 |     state.workload = RANDOM;
122 |     state.skewness = 0.99;
123 |     state.scan_length = 100;
124 |     state.read_ratio = 50;
125 |     state.throughput = 10000000;
126 |     state.latency_test = false;
127 |     state.instant_restart = false;
128 | 
129 |     // Parse args
130 |     while (1) {
131 |         int idx = 0;
132 |         int c = getopt_long(argc, argv, "f:t:K:n:k:L:sd:b:w:S:l:r:T:e:i", opts,
133 |                             &idx);
134 | 
135 |         if (c == -1)
136 |             break;
137 | 
138 |         switch (c) {
139 |         case 'b':
140 |             state.benchmark = (BenchMarkType)atoi(optarg);
141 |             break;
142 |         case 'd':
143 |             state.duration = atof(optarg);
144 |             break;
145 |         case 't':
146 |             state.type = (IndexType)atoi(optarg);
147 |             break;
148 |         case 'K':
149 |             state.key_type = (KeyType)atoi(optarg);
150 |             break;
151 |         case 'e':
152 |             state.email = atoi(optarg);
153 |             break;
154 |         case 'n':
155 |             state.num_threads = atoi(optarg);
156 |             break;
157 |         case 'k':
158 |             state.init_keys = (1llu << atoi(optarg));
159 |             break;
160 |         case 'L':
161 |             state.val_length = atoi(optarg);
162 |             break;
163 |         case 's':
164 |             state.share_memory = false;
165 |             break;
166 |         case 'f':
167 |             state.filename = std::string(optarg);
168 |             break;
169 |         case 'w':
170 |             state.workload = (DataDistrubute)atoi(optarg);
171 |             break;
172 |         case 'S':
173 |             state.skewness = atof(optarg);
174 |             break;
175 |         case 'l':
176 |             state.scan_length = atoi(optarg);
177 |             break;
178 |         case 'r':
179 |             state.read_ratio = atoi(optarg);
180 |             break;
181 |         case 'T':
182 |             state.throughput = atoi(optarg);
183 |             state.latency_test = true;
184 |             break;
185 |         case 'h':
186 |             usage_exit(stdout);
187 |             break;
188 |         case 'i':
189 |             state.instant_restart = true;
190 |             break;
191 |         default:
192 |             fprintf(stderr, "\nUnknown option: -%c-\n", c);
193 |             usage_exit(stderr);
194 |         }
195 |     }
196 |     if (state.instant_restart == true) {
197 |         std::cout << "----------test instant restart----------\n";
198 |     }
199 |     if (state.key_type == String) {
200 |         std::cout << "value length: " << state.val_length << "\n";
201 |     }
202 |     if (state.workload == ZIPFIAN)
203 |         std::cout << "zipfian skewness " << state.skewness << "\n";
204 |     std::cout << "read ratio: " << state.read_ratio << "\n";
205 |     if (state.email == 1) {
206 |         std::cout << "email key\n";
207 |     } else {
208 |         std::cout << "rand key\n";
209 |     }
210 |     // state.report();
211 | }
212 | 


--------------------------------------------------------------------------------
/benchmark/generator.cpp:
--------------------------------------------------------------------------------
  1 | #include "generator.h"
  2 | #include "util.h"
  3 | 
  4 | std::mutex ZipfWrapper::gen_mtx;
  5 | std::map<std::string, WorkloadFile *> ZipfWrapper::wf_map;
  6 | 
  7 | std::mutex dataset_mtx;
  8 | 
  9 | WorkloadFile::WorkloadFile(std::string filename) {
 10 |     std::ifstream fin;
 11 |     fin.open(filename, std::ios::in | std::ios::binary);
 12 |     fin.seekg(0, std::ios::end);
 13 |     int size = fin.tellg();
 14 | 
 15 |     bufsize = size / sizeof(int);
 16 |     buffer = new int[bufsize];
 17 | 
 18 |     fin.seekg(0);
 19 |     fin.read((char *)buffer, sizeof(int) * bufsize);
 20 |     fin.close();
 21 | }
 22 | 
 23 | static const uint64_t kFNVPrime64 = 1099511628211;
 24 | unsigned int hashfunc(uint32_t val) {
 25 |     uint32_t hash = 123;
 26 |     int i;
 27 |     for (i = 0; i < sizeof(uint32_t); i++) {
 28 |         uint64_t octet = val & 0x00ff;
 29 |         val = val >> 8;
 30 | 
 31 |         hash = hash ^ octet;
 32 |         hash = hash * kFNVPrime64;
 33 |     }
 34 |     return hash;
 35 | }
 36 | 
 37 | int ZipfGenerator::randomInt() {
 38 |     double d = rdm.randomDouble();
 39 | 
 40 |     int low = 0, high = size;
 41 |     while (low < high - 1) {
 42 |         int mid = (low + high) / 2;
 43 |         if (zipfs[mid] <= d && zipfs[mid + 1] > d) {
 44 |             low = mid;
 45 |             break;
 46 |         } else if (zipfs[mid] > d) {
 47 |             high = mid;
 48 |         } else {
 49 |             low = mid;
 50 |         }
 51 |     }
 52 |     return hashfunc(low) % size;
 53 | }
 54 | 
 55 | void ZipfGenerator::init(double s, int inital) {
 56 |     zipfs = new double[inital];
 57 |     double sum = 0.0;
 58 |     for (int i = 1; i < inital + 1; i++) {
 59 |         zipfs[i - 1] = 1.0 / (float)pow((double)i, s);
 60 |         sum += zipfs[i - 1];
 61 |     }
 62 |     zipfs[0] = 1.0 / sum;
 63 |     for (int i = 1; i < inital; i++) {
 64 |         zipfs[i] = zipfs[i] / sum + zipfs[i - 1];
 65 |     }
 66 | }
 67 | 
 68 | ZipfGenerator::ZipfGenerator(double s, int inital) : size(inital) {
 69 |     init(s, inital);
 70 | }
 71 | 
 72 | ZipfWrapper::ZipfWrapper(double s, int inital) {
 73 |     cursor = random();
 74 |     std::string filename = get_file_name(s);
 75 |     gen_mtx.lock();
 76 |     if (wf_map.find(filename) == wf_map.end()) {
 77 |         if (access(filename.c_str(), 0)) {
 78 |             std::cout << filename << " not exists, generate it now\n";
 79 |             ZipfGenerator zipf(s, inital);
 80 |             std::ofstream myfile;
 81 |             myfile.open(filename, std::ios::out | std::ios::binary);
 82 |             for (unsigned long long i = 0; i < inital * 16; i++) {
 83 |                 int d = zipf.randomInt();
 84 |                 myfile.write((char *)&d, sizeof(int));
 85 |             }
 86 |             myfile.close();
 87 |         }
 88 | 
 89 |         wf_map[filename] = new WorkloadFile(filename);
 90 |     }
 91 |     wf = wf_map[filename];
 92 |     gen_mtx.unlock();
 93 | }
 94 | 
 95 | DataSet::DataSet(int size, int key_length, int email)
 96 |     : data_size(size), key_len(key_length), emailkey(email) {
 97 |     if (emailkey == 0) { // rand string key
 98 |         std::string fn_str = get_file_name_str(key_len);
 99 |         dataset_mtx.lock();
100 |         if (access(fn_str.c_str(), 0)) {
101 |             std::cout << fn_str << " not exist, generate it now\n";
102 |             RandomGenerator rdm(key_len);
103 |             std::ofstream myfile;
104 |             myfile.open(fn_str, std::ios::out);
105 |             for (unsigned long long i = 0; i < data_size; i++) {
106 |                 std::string s = rdm.RandomStr();
107 |                 myfile << s << "\n";
108 |             }
109 |             myfile.close();
110 |         }
111 | 
112 |         std::cout << "start to load data\n";
113 |         std::ifstream fstr;
114 |         wl_str = new std::string[data_size];
115 |         fstr.open(fn_str, std::ios::in);
116 |         for (int i = 0; i < data_size; i++) {
117 |             fstr >> wl_str[i];
118 |         }
119 |         fstr.close();
120 |         dataset_mtx.unlock();
121 |         std::cout << "load random string key successfully\n";
122 |     } else { // email key
123 |         std::string email_key_file = "/tmp/email_key";
124 |         dataset_mtx.lock();
125 |         std::ifstream fstr;
126 |         wl_str = new std::string[data_size];
127 |         fstr.open(email_key_file, std::ios::in);
128 |         for (int i = 0; i < data_size; i++) {
129 |             fstr >> wl_str[i];
130 |         }
131 |         fstr.close();
132 |         dataset_mtx.unlock();
133 |         std::cout << "load string data successfully\n";
134 |     }
135 | }
136 | 


--------------------------------------------------------------------------------
/benchmark/generator.h:
--------------------------------------------------------------------------------
  1 | #ifndef GENERATOR_H
  2 | #define GENERATOR_H
  3 | 
  4 | #include <atomic>
  5 | #include <cstdio>
  6 | #include <fstream>
  7 | #include <iostream>
  8 | #include <map>
  9 | #include <mutex>
 10 | #include <unistd.h>
 11 | 
 12 | #include <fcntl.h>
 13 | #include <sys/stat.h>
 14 | #include <sys/types.h>
 15 | 
 16 | #include "config.h"
 17 | #include <bits/stdc++.h>
 18 | 
 19 | class WorkloadGenerator {
 20 |   public:
 21 |     virtual long long Next() = 0;
 22 | };
 23 | 
 24 | /*
 25 |  * Fast random number generator, using eran48/nrand48; All the functions work by
 26 |  * generating a sequence of 48-bit integers, X[i], accourding to the liner
 27 |  * congruential formula: Xn+1 = (aXn + c) mod m;  where n >= 0 If you want to
 28 |  * generate the same sequence again, you can call "reset" function.
 29 |  */
 30 | class RandomGenerator : public WorkloadGenerator {
 31 |     unsigned short seed[3];
 32 |     unsigned short seed2[3];
 33 |     unsigned short inital[3];
 34 |     unsigned short inital2[3];
 35 |     int key_len;
 36 | 
 37 |   public:
 38 |     RandomGenerator(int key_len_ = 8) : key_len(key_len_) {
 39 |         for (int i = 0; i < 3; i++) {
 40 |             inital[i] = seed[i] = rand();
 41 |             inital2[i] = seed2[i] = rand();
 42 |         }
 43 |     }
 44 |     int randomInt() { return nrand48(seed) ^ nrand48(seed2); }
 45 | 
 46 |     std::string RandomStr() {
 47 |         int len = randomInt() % 10 + 5; // len 8-16 // 5-15
 48 |         //        int len = randomInt() % 125  + 4;
 49 |         std::string res = "";
 50 |         for (int i = 0; i < len; i++) {
 51 |                         char c = randomInt() % 10 + '0'; // 0-9
 52 | //            char c = randomInt() % 94 + 33;
 53 |             //            char c = randomInt() % 127 + 1;
 54 |             res += c;
 55 |         }
 56 |         return res;
 57 |     }
 58 | 
 59 |     double randomDouble() { return erand48(seed) * erand48(seed2); }
 60 |     void setSeed(unsigned short newseed[3], unsigned short newseed2[3]) {
 61 |         memcpy(seed, newseed, sizeof(unsigned short) * 3);
 62 |         memcpy(inital, newseed, sizeof(unsigned short) * 3);
 63 |         memcpy(seed2, newseed2, sizeof(unsigned short) * 3);
 64 |         memcpy(inital2, newseed, sizeof(unsigned short) * 3);
 65 |     }
 66 |     void reset() {
 67 |         memcpy(seed, inital, sizeof(unsigned short) * 3);
 68 |         memcpy(seed2, inital2, sizeof(unsigned short) * 3);
 69 |     }
 70 |     long long Next() { return randomInt(); }
 71 | } __attribute__((aligned(64)));
 72 | 
 73 | class WorkloadFile {
 74 |     int bufsize;
 75 |     int *buffer;
 76 | 
 77 |   public:
 78 |     WorkloadFile(std::string filename);
 79 |     int get(uint32_t c) { return buffer[c % bufsize]; }
 80 | };
 81 | 
 82 | class ZipfGenerator {
 83 |     double *zipfs;
 84 |     RandomGenerator rdm;
 85 |     int size;
 86 | 
 87 |     void init(double s, int inital);
 88 | 
 89 |   public:
 90 |     ZipfGenerator(double s, int inital = (1 << 20));
 91 |     ~ZipfGenerator() { delete zipfs; }
 92 |     int randomInt();
 93 | } __attribute__((aligned(64)));
 94 | 
 95 | class ZipfWrapper : public WorkloadGenerator {
 96 |     static std::mutex gen_mtx;
 97 |     static std::map<std::string, WorkloadFile *> wf_map;
 98 | 
 99 |     uint32_t cursor;
100 |     WorkloadFile *wf;
101 | 
102 |     static std::string get_file_name(double s) {
103 |         std::stringstream ss;
104 |         ss << (int)((s + 0.001) * 100);
105 |         return "/tmp/" + ss.str() + "zipfian_data";
106 |     }
107 | 
108 |   public:
109 |     ZipfWrapper(double s, int inital = (1 << 20));
110 |     long long Next() { return wf->get(cursor++); }
111 | };
112 | 
113 | class DataSet {
114 |   public:
115 |     std::string *wl_str;
116 |     int data_size;
117 |     int key_len;
118 |     int emailkey;
119 | 
120 |     static std::string get_file_name_str(int len) {
121 |         std::stringstream ss;
122 |         ss << len;
123 |         return "/tmp/random_str_data" + ss.str();
124 |     }
125 | 
126 |     DataSet(int size, int key_length, int email);
127 | };
128 | 
129 | #endif // GENERATOR_H


--------------------------------------------------------------------------------
/benchmark/microbench.h:
--------------------------------------------------------------------------------
  1 | #ifndef _MICRO_BEHCN_H
  2 | #define _MICRO_BEHCN_H
  3 | 
  4 | #include "config.h"
  5 | #include "generator.h"
  6 | #include "util.h"
  7 | #include <assert.h>
  8 | #include <utility>
  9 | 
 10 | enum OperationType {
 11 |     INSERT,
 12 |     REMOVE,
 13 |     UPDATE,
 14 |     GET,
 15 |     SCAN,
 16 |     MIXED,
 17 |     _OpreationTypeNumber
 18 | };
 19 | 
 20 | template <typename T> inline void swap(T &a, T &b) {
 21 |     T tmp = a;
 22 |     a = b;
 23 |     b = tmp;
 24 | }
 25 | 
 26 | long long *random_shuffle(int s) {
 27 |     long long *x = new long long[s];
 28 |     for (int i = 0; i < s; i++) {
 29 |         x[i] = i;
 30 |     }
 31 |     for (int i = 0; i < s; i++) {
 32 |         swap(x[i], x[random() % s]);
 33 |     }
 34 |     return x;
 35 | }
 36 | 
 37 | class Benchmark {
 38 |   public:
 39 |     WorkloadGenerator *workload;
 40 |     long long key_range;
 41 |     long long init_key;
 42 |     long long *x;
 43 |     Config _conf;
 44 |     DataSet *dataset;
 45 |     RandomGenerator rdm;
 46 | 
 47 |     Benchmark(Config &conf) : init_key(0), _conf(conf) {
 48 |         dataset = new DataSet(conf.init_keys, conf.val_length, conf.email);
 49 |         if (conf.workload == RANDOM) {
 50 |             workload = new RandomGenerator();
 51 |         } else if (conf.workload == ZIPFIAN) {
 52 |             workload = new ZipfWrapper(conf.skewness, conf.init_keys);
 53 |         }
 54 | 
 55 |         x = NULL;
 56 |     }
 57 | 
 58 |     virtual ~Benchmark() {
 59 |         if (x != NULL)
 60 |             delete[] x;
 61 |     }
 62 | 
 63 |     virtual std::pair<OperationType, long long> nextIntOperation() {
 64 |         return std::make_pair(INSERT, workload->Next());
 65 |     }
 66 | 
 67 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
 68 |         long long next = workload->Next();
 69 |         return std::make_pair(INSERT, dataset->wl_str[next % _conf.init_keys]);
 70 |     }
 71 | 
 72 |     virtual long long nextInitIntKey() {
 73 |         if (x == NULL)
 74 |             x = random_shuffle(_conf.init_keys);
 75 |         return x[init_key++ % _conf.init_keys];
 76 |         // return init_key++ % _conf.init_keys;
 77 |     }
 78 | 
 79 |     virtual std::string nextInitStrKey() {
 80 |         return dataset->wl_str[(init_key++) % _conf.init_keys];
 81 |     }
 82 | } __attribute__((aligned(64)));
 83 | 
 84 | class ReadOnlyBench : public Benchmark {
 85 |   public:
 86 |     ReadOnlyBench(Config &conf) : Benchmark(conf) {}
 87 | 
 88 |     std::pair<OperationType, long long> nextIntOperation() {
 89 |         long long d = workload->Next() % _conf.init_keys;
 90 |         return std::make_pair(GET, d);
 91 |     }
 92 | 
 93 |     std::pair<OperationType, std::string> nextStrOperation() {
 94 |         long long next = workload->Next() % _conf.init_keys;
 95 |         return std::make_pair(GET, dataset->wl_str[next]);
 96 |     }
 97 | } __attribute__((aligned(64)));
 98 | 
 99 | class InsertOnlyBench : public Benchmark {
100 |     RandomGenerator rdm;
101 | 
102 |   public:
103 |     InsertOnlyBench(Config &conf) : Benchmark(conf) {}
104 | 
105 |     std::pair<OperationType, long long> nextIntOperation() {
106 |         long long d = workload->Next() % _conf.init_keys;
107 | #ifdef INSERT_DUP
108 |         long long x = 1;
109 | #else
110 |         long long x = rdm.randomInt() % 128;
111 | #endif
112 | 
113 |         return std::make_pair(INSERT, d * x);
114 |     }
115 | 
116 |     std::pair<OperationType, std::string> nextStrOperation() {
117 |         long long next = workload->Next() % _conf.init_keys;
118 |         std::string s = dataset->wl_str[next];
119 |         char c = rdm.randomInt() % 94 + 33;
120 |         char d = rdm.randomInt() % 94 + 33;
121 |         s = d + s;
122 |         s = c + s;
123 |         return std::make_pair(INSERT, s);
124 |     }
125 | 
126 | #ifndef INSERT_DUP
127 | 
128 |     long long nextInitIntKey() { return 128 * Benchmark::nextInitIntKey(); }
129 | 
130 | #endif
131 | } __attribute__((aligned(64)));
132 | 
133 | class UpdateOnlyBench : public Benchmark {
134 |   public:
135 |     UpdateOnlyBench(Config &conf) : Benchmark(conf) {}
136 | 
137 |     OperationType nextOp() { return UPDATE; }
138 | 
139 |     std::pair<OperationType, long long> nextIntOperation() {
140 |         long long d = workload->Next() % _conf.init_keys;
141 |         return std::make_pair(UPDATE, d);
142 |     }
143 | 
144 |     std::pair<OperationType, std::string> nextStrOperation() {
145 |         long long next = workload->Next() % _conf.init_keys;
146 |         return std::make_pair(UPDATE, dataset->wl_str[next]);
147 |     }
148 | } __attribute__((aligned(64)));
149 | 
150 | class DeleteOnlyBench : public Benchmark {
151 |   public:
152 |     DeleteOnlyBench(Config &conf) : Benchmark(conf) {}
153 | 
154 |     std::pair<OperationType, long long> nextIntOperation() {
155 |         long long d = workload->Next() % _conf.init_keys;
156 |         return std::make_pair(REMOVE, d);
157 |     }
158 | 
159 |     std::pair<OperationType, std::string> nextStrOperation() {
160 |         long long next = workload->Next() % _conf.init_keys;
161 |         return std::make_pair(REMOVE, dataset->wl_str[next]);
162 |     }
163 | } __attribute__((aligned(64)));
164 | 
165 | class MixedBench : public Benchmark {
166 |     int round;
167 |     long long key;
168 |     std::string skey;
169 | 
170 |   public:
171 |     MixedBench(Config &conf) : Benchmark(conf) {}
172 | 
173 |     std::pair<OperationType, long long> nextIntOperation() {
174 |         std::pair<OperationType, long long> result;
175 |         long long _key = workload->Next() % _conf.init_keys;
176 |         switch (round) {
177 |         case 0:
178 |             key = workload->Next() % _conf.init_keys;
179 |             result = std::make_pair(REMOVE, key);
180 |             break;
181 |         case 1:
182 |             result = std::make_pair(INSERT, key);
183 |             break;
184 |         case 2:
185 |             result = std::make_pair(UPDATE, _key);
186 |             break;
187 |         case 3:
188 |             result = std::make_pair(GET, _key);
189 |             break;
190 |         default:
191 |             assert(0);
192 |         }
193 |         round++;
194 |         round %= 4;
195 |         return result;
196 |     }
197 | 
198 |     std::pair<OperationType, std::string> nextStrOperation() {
199 |         std::pair<OperationType, std::string> result;
200 |         long long next = workload->Next() % _conf.init_keys;
201 |         std::string _key = dataset->wl_str[next];
202 |         switch (round) {
203 |         case 0:
204 |             next = workload->Next() % _conf.init_keys;
205 |             skey = dataset->wl_str[next];
206 |             result = std::make_pair(REMOVE, skey);
207 |             break;
208 |         case 1:
209 |             result = std::make_pair(INSERT, skey);
210 |             break;
211 |         case 2:
212 |             result = std::make_pair(UPDATE, _key);
213 |             break;
214 |         case 3:
215 |             result = std::make_pair(GET, _key);
216 |             break;
217 |         default:
218 |             assert(0);
219 |         }
220 |         round++;
221 |         round %= 4;
222 |         return result;
223 |     }
224 | } __attribute__((aligned(64)));
225 | 
226 | class ScanBench : public Benchmark {
227 |   public:
228 |     ScanBench(Config &conf) : Benchmark(conf) {}
229 | 
230 |     std::pair<OperationType, long long> nextIntOperation() {
231 |         long long d = workload->Next() % _conf.init_keys;
232 |         return std::make_pair(SCAN, d);
233 |     }
234 | 
235 |     std::pair<OperationType, std::string> nextStrOperation() {
236 |         long long next = workload->Next() % _conf.init_keys;
237 |         std::string s = dataset->wl_str[next];
238 |         return std::make_pair(SCAN, s);
239 |     }
240 | } __attribute__((aligned(64)));
241 | 
242 | class YSCBA : public Benchmark {
243 |   public:
244 |     //	readRate = 0.5;
245 |     //	writeRate = 0.5;
246 |     int read_ratio = 50;
247 | 
248 |     RandomGenerator rdm;
249 | 
250 |     YSCBA(Config &conf) : Benchmark(conf) {}
251 | 
252 |     virtual std::pair<OperationType, long long> nextIntOperation() {
253 |         int k = rdm.randomInt() % 100;
254 |         if (k > read_ratio) {
255 |             long long x = rdm.randomInt() % 128;
256 |             return std::make_pair(INSERT,
257 |                                   x * (workload->Next() % _conf.init_keys));
258 |         } else {
259 |             return std::make_pair(GET, workload->Next() % _conf.init_keys);
260 |         }
261 |     }
262 | 
263 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
264 |         int k = rdm.randomInt() % 100;
265 |         long long next = workload->Next() % _conf.init_keys;
266 |         std::string s = dataset->wl_str[next];
267 |         if (k > read_ratio) {
268 |             char c = rdm.randomInt() % 94 + 33;
269 |             char d = rdm.randomInt() % 94 + 33;
270 |             s = d + s;
271 |             s = c + s;
272 |             return std::make_pair(INSERT, s);
273 |         } else {
274 |             return std::make_pair(GET, s);
275 |         }
276 |     }
277 | } __attribute__((aligned(64)));
278 | 
279 | class YSCBB : public Benchmark {
280 |   public:
281 |     //	readRate = 0.95;
282 |     //	writeRate = 0.05;
283 |     int read_ratio = 95;
284 |     RandomGenerator rdm;
285 | 
286 |     YSCBB(Config &conf) : Benchmark(conf) {}
287 | 
288 |     virtual std::pair<OperationType, long long> nextIntOperation() {
289 |         int k = rdm.randomInt() % 100;
290 |         if (k < read_ratio) {
291 |             return std::make_pair(GET, workload->Next() % _conf.init_keys);
292 |         } else {
293 |             long long x = rdm.randomInt() % 128;
294 |             return std::make_pair(INSERT,
295 |                                   x * (workload->Next() % _conf.init_keys));
296 |         }
297 |     }
298 | 
299 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
300 |         int k = rdm.randomInt() % 100;
301 |         long long next = workload->Next() % _conf.init_keys;
302 |         std::string s = dataset->wl_str[next];
303 |         if (k > read_ratio) {
304 |             char c = rdm.randomInt() % 94 + 33;
305 |             char d = rdm.randomInt() % 94 + 33;
306 |             s = d + s;
307 |             s = c + s;
308 |             return std::make_pair(INSERT, s);
309 |         } else {
310 |             return std::make_pair(GET, s);
311 |         }
312 |     }
313 | } __attribute__((aligned(64)));
314 | 
315 | // 50% read, 50% update
316 | class YSCBC : public Benchmark {
317 |   public:
318 |     //	readRate = 0.5;
319 |     //	writeRate = 0.5;
320 |     int read_ratio = 50;
321 | 
322 |     RandomGenerator rdm;
323 | 
324 |     YSCBC(Config &conf) : Benchmark(conf) {}
325 | 
326 |     virtual std::pair<OperationType, long long> nextIntOperation() {
327 |         int k = rdm.randomInt() % 100;
328 |         if (k > read_ratio) {
329 |             return std::make_pair(UPDATE, (workload->Next() % _conf.init_keys));
330 |         } else {
331 |             return std::make_pair(GET, workload->Next() % _conf.init_keys);
332 |         }
333 |     }
334 | 
335 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
336 |         int k = rdm.randomInt() % 100;
337 |         long long next = workload->Next() % _conf.init_keys;
338 |         std::string s = dataset->wl_str[next];
339 |         if (k > read_ratio) {
340 |             return std::make_pair(UPDATE, s);
341 |         } else {
342 |             return std::make_pair(GET, s);
343 |         }
344 |     }
345 | } __attribute__((aligned(64)));
346 | 
347 | // workload D, 95 read 5 insert
348 | class YSCBD : public Benchmark {
349 |   public:
350 |     int read_ratio = 95;
351 | 
352 |     RandomGenerator rdm;
353 | 
354 |     YSCBD(Config &conf) : Benchmark(conf) {}
355 | 
356 |     virtual std::pair<OperationType, long long> nextIntOperation() {
357 |         int k = rdm.randomInt() % 100;
358 |         if (k < read_ratio) {
359 |             return std::make_pair(GET, workload->Next() % _conf.init_keys);
360 |         } else {
361 |             int d = rdm.randomInt() % 128;
362 |             return std::make_pair(INSERT,
363 |                                   (workload->Next() % _conf.init_keys) * d);
364 |         }
365 |     }
366 | 
367 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
368 |         int k = rdm.randomInt() % 100;
369 |         long long next = workload->Next() % _conf.init_keys;
370 |         std::string s = dataset->wl_str[next];
371 |         if (k > read_ratio) {
372 |             char c = rdm.randomInt() % 94 + 33;
373 |             char d = rdm.randomInt() % 94 + 33;
374 |             s = d + s;
375 |             s = c + s;
376 |             return std::make_pair(INSERT, s);
377 |         } else {
378 |             return std::make_pair(GET, s);
379 |         }
380 |     }
381 | } __attribute__((aligned(64)));
382 | 
383 | // scan/insert
384 | class YSCBE : public Benchmark {
385 |   public:
386 |     int scan_ratio = 95;
387 |     RandomGenerator rdm;
388 | 
389 |     YSCBE(Config &conf) : Benchmark(conf) {}
390 | 
391 |     virtual std::pair<OperationType, long long> nextIntOperation() {
392 |         int k = rdm.randomInt() % 100;
393 |         if (k < scan_ratio) {
394 |             return std::make_pair(SCAN, workload->Next() % _conf.init_keys);
395 |         } else {
396 |             int d = rdm.randomInt() % 128;
397 |             return std::make_pair(INSERT,
398 |                                   (workload->Next() % _conf.init_keys) * d);
399 |         }
400 |     }
401 | 
402 |     virtual std::pair<OperationType, std::string> nextStrOperation() {
403 |         int k = rdm.randomInt() % 100;
404 |         long long next = workload->Next() % _conf.init_keys;
405 |         std::string s = dataset->wl_str[next];
406 |         if (k < scan_ratio) {
407 |             return std::make_pair(SCAN, s);
408 |         } else {
409 |             char c = rdm.randomInt() % 94 + 33;
410 |             char d = rdm.randomInt() % 94 + 33;
411 |             s = d + s;
412 |             s = c + s;
413 |             return std::make_pair(INSERT, s);
414 |         }
415 |     }
416 | } __attribute__((aligned(64)));
417 | 
418 | #endif
419 | 


--------------------------------------------------------------------------------
/benchmark/timer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "util.h"
 4 | #include <cstdlib>
 5 | #include <sys/time.h>
 6 | 
 7 | class timer {
 8 |   public:
 9 |     timer() {
10 |         total.tv_sec = total.tv_usec = 0;
11 |         diff.tv_sec = diff.tv_usec = 0;
12 |     }
13 | 
14 |     double duration() {
15 |         double duration;
16 | 
17 |         duration = (total.tv_sec) * 1000000.0; // sec to us
18 |         duration += (total.tv_usec);           // us
19 | 
20 |         return duration * 1000.0; // ns
21 |     }
22 | 
23 |     void start() { gettimeofday(&t1, NULL); }
24 | 
25 |     void end() {
26 |         gettimeofday(&t2, NULL);
27 |         timersub(&t2, &t1, &diff);
28 |         timeradd(&diff, &total, &total);
29 |     }
30 | 
31 |     void reset() {
32 |         total.tv_sec = total.tv_usec = 0;
33 |         diff.tv_sec = diff.tv_usec = 0;
34 |     }
35 | 
36 |     timeval t1, t2, diff;
37 |     timeval total;
38 | };
39 | 
40 | class cpuCycleTimer {
41 |   public:
42 |     long long t1, t2, total;
43 |     int count = 0;
44 | 
45 |     cpuCycleTimer() { reset(); }
46 |     void start() { t1 = rdtsc(); }
47 |     void end() {
48 |         t2 = rdtsc();
49 |         total += t2 - t1;
50 |         count++;
51 |     }
52 |     void reset() { count = t1 = t2 = total = 0; }
53 |     double duration() { // ns
54 |         return total / CPU_FREQUENCY;
55 |     }
56 |     double average() { return (double)total / CPU_FREQUENCY / count; }
57 |     long long Countnum() { return count; }
58 | };
59 | 


--------------------------------------------------------------------------------
/fast_fair/pmdk_gc.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <iostream>
  4 | #include <libpmemobj.h>
  5 | #include <thread>
  6 | 
  7 | namespace fastfair {
  8 | 
  9 | static const int GC_NODE_COUNT_THREADHOLD = 1024;
 10 | 
 11 | static const int GC_INTERVAL = 50;
 12 | 
 13 | POBJ_LAYOUT_BEGIN(fast_fair);
 14 | POBJ_LAYOUT_TOID(fast_fair, struct key_item);
 15 | POBJ_LAYOUT_TOID(fast_fair, char);
 16 | POBJ_LAYOUT_END(fast_fair);
 17 | 
 18 | typedef struct key_item {
 19 |     size_t key_len;
 20 |     char key[];
 21 | } key_item;
 22 | 
 23 | class GarbageNode {
 24 |   public:
 25 |     uint64_t delete_epoch;
 26 |     void *node_p;
 27 |     GarbageNode *next_p;
 28 | 
 29 |     GarbageNode(uint64_t p_delete_epoch, void *p_node_p)
 30 |         : delete_epoch{p_delete_epoch}, node_p{p_node_p}, next_p{nullptr} {}
 31 | 
 32 |     GarbageNode() : delete_epoch{0UL}, node_p{nullptr}, next_p{nullptr} {}
 33 | } __attribute__((aligned(64)));
 34 | 
 35 | class GCMetaData {
 36 |   public:
 37 |     uint64_t last_active_epoch;
 38 | 
 39 |     GarbageNode header;
 40 |     GarbageNode *last_p;
 41 | 
 42 |     int node_count;
 43 | 
 44 |     GCMetaData() {
 45 |         last_active_epoch = static_cast<uint64_t>(-1);
 46 |         last_p = &header;
 47 |         node_count = 0;
 48 |     }
 49 | 
 50 |     ~GCMetaData() {}
 51 | } __attribute__((aligned(64)));
 52 | 
 53 | class Epoch_Mgr {
 54 |   public:
 55 |     std::thread *thread_p;
 56 |     bool exit_flag;
 57 |     uint64_t epoch;
 58 | 
 59 |   public:
 60 |     Epoch_Mgr() : exit_flag(false), epoch(0) {}
 61 | 
 62 |     ~Epoch_Mgr() {
 63 |         exit_flag = true;
 64 |         std::cout << "[EPOCH]\tepoch mgr exit\n";
 65 |     }
 66 | 
 67 |     inline void IncreaseEpoch() { epoch++; }
 68 | 
 69 |     uint64_t GetGlobalEpoch() { return epoch; }
 70 | 
 71 |     void ThreadFunc() {
 72 |         std::cout << "[EPOCH]\tglobal epoch thread start\n";
 73 |         while (exit_flag == false) {
 74 |             IncreaseEpoch();
 75 |             std::chrono::milliseconds duration(GC_INTERVAL);
 76 |             std::this_thread::sleep_for(duration);
 77 |         }
 78 |         std::cout << "[EPOCH]\tglobal epoch thread exit\n";
 79 |         return;
 80 |     }
 81 | 
 82 |     void StartThread() {
 83 |         thread_p = new std::thread{[this]() { this->ThreadFunc(); }};
 84 |         return;
 85 |     }
 86 | 
 87 | } __attribute__((aligned(64)));
 88 | 
 89 | class threadinfo {
 90 |   public:
 91 |     GCMetaData *md;
 92 |     Epoch_Mgr *mgr;
 93 | 
 94 |     threadinfo *next;
 95 |     threadinfo *head;
 96 | 
 97 |     PMEMobjpool *pool;
 98 | 
 99 |     int id;
100 | 
101 |     threadinfo(Epoch_Mgr *mgr_) : mgr(mgr_) {}
102 | 
103 |     ~threadinfo() {}
104 | 
105 |     void JoinEpoch() { md->last_active_epoch = mgr->GetGlobalEpoch(); }
106 | 
107 |     void LeaveEpoch() { md->last_active_epoch = static_cast<uint64_t>(-1); }
108 | 
109 |     void AddGarbageNode(void *node_p) {
110 |         TX_BEGIN(pool) {
111 |             pmemobj_tx_add_range_direct(&md->last_p->next_p, sizeof(uint64_t));
112 |             pmemobj_tx_add_range_direct(&md->last_p, sizeof(uint64_t));
113 |             pmemobj_tx_add_range_direct(&md->node_count, sizeof(int));
114 | 
115 |             PMEMoid p =
116 |                 pmemobj_tx_alloc(sizeof(GarbageNode), TOID_TYPE_NUM(char));
117 |             GarbageNode *gn = new (pmemobj_direct(p))
118 |                 GarbageNode(mgr->GetGlobalEpoch(), node_p);
119 |             md->last_p->next_p = gn;
120 |             md->last_p = gn;
121 |             md->node_count++;
122 |         }
123 |         TX_END
124 | 
125 |         if (md->node_count > GC_NODE_COUNT_THREADHOLD) {
126 |             // Use current thread's gc id to perform GC
127 |             PerformGC();
128 |         }
129 | 
130 |         return;
131 |     }
132 | 
133 |     uint64_t SummarizeGCEpoch(threadinfo *head_) {
134 |         threadinfo *tti = head->next;
135 |         uint64_t min_epoch = static_cast<uint64_t>(-1);
136 |         while (tti != nullptr) {
137 |             min_epoch = std::min(min_epoch, tti->md->last_active_epoch);
138 |             tti = tti->next;
139 |         }
140 |         return min_epoch;
141 |     }
142 | 
143 |     void PerformGC() {
144 |         // First of all get the minimum epoch of all active threads
145 |         // This is the upper bound for deleted epoch in garbage node
146 |         uint64_t min_epoch = SummarizeGCEpoch(head);
147 | 
148 |         // This is the pointer we use to perform GC
149 |         // Note that we only fetch the metadata using the current thread-local
150 |         // id
151 |         GarbageNode *header_p = &(md->header);
152 |         GarbageNode *first_p = header_p->next_p;
153 | 
154 |         // Then traverse the linked list
155 |         // Only reclaim memory when the deleted epoch < min epoch
156 |         while (first_p != nullptr && first_p->delete_epoch < min_epoch) {
157 |             // First unlink the current node from the linked list
158 |             // This could set it to nullptr
159 |             TX_BEGIN(pool) {
160 |                 pmemobj_tx_add_range_direct(&header_p->next_p,
161 |                                             sizeof(uint64_t));
162 |                 pmemobj_tx_add_range_direct(&md->node_count, sizeof(int));
163 |                 header_p->next_p = first_p->next_p;
164 | 
165 |                 PMEMoid ptr = pmemobj_oid(first_p->node_p);
166 |                 pmemobj_tx_free(ptr);
167 | 
168 |                 md->node_count--;
169 |             }
170 |             TX_END
171 | 
172 |             // Then free memory
173 |             //            FreeEpochNode(first_p->node_p);
174 | 
175 |             //            delete first_p;
176 |             first_p = header_p->next_p;
177 |         }
178 | 
179 |         // If we have freed all nodes in the linked list we should
180 |         // reset last_p to the header
181 |         if (first_p == nullptr) {
182 |             md->last_p = header_p;
183 |         }
184 |         return;
185 |     }
186 | };
187 | } // namespace fastfair
188 | 


--------------------------------------------------------------------------------
/format:
--------------------------------------------------------------------------------
1 | find ART benchmark fast_fair lf-skiplist nvm_mgr test perf -type f -name '*.cpp' -o -name '*.h' | xargs clang-format -style=file -i
2 | 


--------------------------------------------------------------------------------
/lf-skiplist/atomic_ops_if.h:
--------------------------------------------------------------------------------
  1 | /* this file was taken from ascylib (https://github.com/LPD-EPFL/ASCYLIB) */
  2 | 
  3 | #ifndef _ATOMIC_OPS_IF_H_INCLUDED_
  4 | #define _ATOMIC_OPS_IF_H_INCLUDED_
  5 | 
  6 | #include <inttypes.h>
  7 | 
  8 | #ifdef __sparc__
  9 | /*
 10 |  *  sparc code
 11 |  */
 12 | 
 13 | #include <atomic.h>
 14 | 
 15 | // test-and-set uint8_t
 16 | static inline uint8_t tas_uint8(volatile uint8_t *addr) {
 17 |     uint8_t oldval;
 18 |     __asm__ __volatile__("ldstub %1,%0"
 19 |                          : "=r"(oldval), "=m"(*addr)
 20 |                          : "m"(*addr)
 21 |                          : "memory");
 22 |     return oldval;
 23 | }
 24 | 
 25 | // Compare-and-swap
 26 | #define CAS_PTR(a, b, c) atomic_cas_ptr(a, b, c)
 27 | #define CAS_U8(a, b, c) atomic_cas_8(a, b, c)
 28 | #define CAS_U16(a, b, c) atomic_cas_16(a, b, c)
 29 | #define CAS_U32(a, b, c) atomic_cas_32(a, b, c)
 30 | #define CAS_U64(a, b, c) atomic_cas_64(a, b, c)
 31 | // Swap
 32 | #define SWAP_PTR(a, b) atomic_swap_ptr(a, b)
 33 | #define SWAP_U8(a, b) atomic_swap_8(a, b)
 34 | #define SWAP_U16(a, b) atomic_swap_16(a, b)
 35 | #define SWAP_U32(a, b) atomic_swap_32(a, b)
 36 | #define SWAP_U64(a, b) atomic_swap_64(a, b)
 37 | // Fetch-and-increment
 38 | #define FAI_U8(a) (atomic_inc_8_nv(a) - 1)
 39 | #define FAI_U16(a) (atomic_inc_16_nv(a) - 1)
 40 | #define FAI_U32(a) (atomic_inc_32_nv(a) - 1)
 41 | #define FAI_U64(a) (atomic_inc_64_nv(a) - 1)
 42 | // Fetch-and-decrement
 43 | #define FAD_U8(a) (atomic_dec_8_nv(a, ) + 1)
 44 | #define FAD_U16(a) (atomic_dec_16_nv(a) + 1)
 45 | #define FAD_U32(a) (atomic_dec_32_nv(a) + 1)
 46 | #define FAD_U64(a) (atomic_dec_64_nv(a) + 1)
 47 | // Increment-and-fetch
 48 | #define IAF_U8(a) atomic_inc_8_nv(a)
 49 | #define IAF_U16(a) atomic_inc_16_nv(a)
 50 | #define IAF_U32(a) atomic_inc_32_nv(a)
 51 | #define IAF_U64(a) atomic_inc_64_nv(a)
 52 | // Decrement-and-fetch
 53 | #define DAF_U8(a) atomic_dec_8_nv(a)
 54 | #define DAF_U16(a) atomic_dec_16_nv(a)
 55 | #define DAF_U32(a) atomic_dec_32_nv(a)
 56 | #define DAF_U64(a) atomic_dec_64_nv(a)
 57 | // Test-and-set
 58 | #define TAS_U8(a) tas_uint8(a)
 59 | // Memory barrier
 60 | #define MEM_BARRIER                                                            \
 61 |     asm volatile("membar #LoadLoad | #LoadStore | #StoreLoad | #StoreStore");
 62 | // end of sparc code
 63 | 
 64 | #elif defined(__tile__)
 65 | /*
 66 |  *  Tilera code
 67 |  */
 68 | #include <arch/atomic.h>
 69 | #include <arch/cycle.h>
 70 | // atomic operations interface
 71 | // Compare-and-swap
 72 | #define CAS_PTR(a, b, c) arch_atomic_val_compare_and_exchange(a, b, c)
 73 | #define CAS_U8(a, b, c) arch_atomic_val_compare_and_exchange(a, b, c)
 74 | #define CAS_U16(a, b, c) arch_atomic_val_compare_and_exchange(a, b, c)
 75 | #define CAS_U32(a, b, c) arch_atomic_val_compare_and_exchange(a, b, c)
 76 | #define CAS_U64(a, b, c) arch_atomic_val_compare_and_exchange(a, b, c)
 77 | // Swap
 78 | #define SWAP_PTR(a, b) arch_atomic_exchange(a, b)
 79 | #define SWAP_U8(a, b) arch_atomic_exchange(a, b)
 80 | #define SWAP_U16(a, b) arch_atomic_exchange(a, b)
 81 | #define SWAP_U32(a, b) arch_atomic_exchange(a, b)
 82 | #define SWAP_U64(a, b) arch_atomic_exchange(a, b)
 83 | // Fetch-and-increment
 84 | #define FAI_U8(a) arch_atomic_increment(a)
 85 | #define FAI_U16(a) arch_atomic_increment(a)
 86 | #define FAI_U32(a) arch_atomic_increment(a)
 87 | #define FAI_U64(a) arch_atomic_increment(a)
 88 | // Fetch-and-decrement
 89 | #define FAD_U8(a) arch_atomic_decrement(a)
 90 | #define FAD_U16(a) arch_atomic_decrement(a)
 91 | #define FAD_U32(a) arch_atomic_decrement(a)
 92 | #define FAD_U64(a) arch_atomic_decrement(a)
 93 | // Increment-and-fetch
 94 | #define IAF_U8(a) (arch_atomic_increment(a) + 1)
 95 | #define IAF_U16(a) (arch_atomic_increment(a) + 1)
 96 | #define IAF_U32(a) (arch_atomic_increment(a) + 1)
 97 | #define IAF_U64(a) (arch_atomic_increment(a) + 1)
 98 | // Decrement-and-fetch
 99 | #define DAF_U8(a) (arch_atomic_decrement(a) - 1)
100 | #define DAF_U16(a) (arch_atomic_decrement(a) - 1)
101 | #define DAF_U32(a) (arch_atomic_decrement(a) - 1)
102 | #define DAF_U64(a) (arch_atomic_decrement(a) - 1)
103 | // Test-and-set
104 | #define TAS_U8(a) arch_atomic_val_compare_and_exchange(a, 0, 0xff)
105 | // Memory barrier
106 | #define MEM_BARRIER arch_atomic_full_barrier()
107 | #define LOAD_BARRIER arch_atomic_read_barrier()
108 | #define STORE_BARRIER arch_atomic_write_barrier()
109 | 
110 | static inline void AO_nop_full(void) { MEM_BARRIER; }
111 | 
112 | #define AO_store_full(addr, val) arch_atomic_write(addr, val)
113 | #define AO_load_full(addr) arch_atomic_access_once((*addr))
114 | // Relax CPU
115 | // define PAUSE cycle_relax()
116 | 
117 | // end of tilera code
118 | #else
119 | /*
120 |  *  x86 code
121 |  */
122 | 
123 | #if defined(__SSE__)
124 | #include <xmmintrin.h>
125 | #endif
126 | 
127 | // test-and-set uint8_t
128 | static inline uint8_t tas_uint8(volatile uint8_t *addr) {
129 |     uint8_t oldval;
130 |     __asm__ __volatile__("xchgb %0,%1"
131 |                          : "=q"(oldval), "=m"(*addr)
132 |                          : "0"((unsigned char)0xff), "m"(*addr)
133 |                          : "memory");
134 |     return (uint8_t)oldval;
135 | }
136 | 
137 | // atomic operations interface
138 | // Compare-and-swap
139 | #define CAS_PTR(a, b, c) __sync_val_compare_and_swap(a, b, c)
140 | #define CAS_U8(a, b, c) __sync_val_compare_and_swap(a, b, c)
141 | #define CAS_U16(a, b, c) __sync_val_compare_and_swap(a, b, c)
142 | #define CAS_U32(a, b, c) __sync_val_compare_and_swap(a, b, c)
143 | #define CAS_U64(a, b, c) __sync_val_compare_and_swap(a, b, c)
144 | // Swap
145 | #define SWAP_PTR(a, b) swap_pointer(a, b)
146 | #define SWAP_U8(a, b) swap_uint8(a, b)
147 | #define SWAP_U16(a, b) swap_uint16(a, b)
148 | #define SWAP_U32(a, b) swap_uint32(a, b)
149 | #define SWAP_U64(a, b) swap_uint64(a, b)
150 | // Fetch-and-increment
151 | #define FAI_U8(a) __sync_fetch_and_add(a, 1)
152 | #define FAI_U16(a) __sync_fetch_and_add(a, 1)
153 | #define FAI_U32(a) __sync_fetch_and_add(a, 1)
154 | #define FAI_U64(a) __sync_fetch_and_add(a, 1)
155 | // Fetch-and-decrement
156 | #define FAD_U8(a) __sync_fetch_and_sub(a, 1)
157 | #define FAD_U16(a) __sync_fetch_and_sub(a, 1)
158 | #define FAD_U32(a) __sync_fetch_and_sub(a, 1)
159 | #define FAD_U64(a) __sync_fetch_and_sub(a, 1)
160 | // Increment-and-fetch
161 | #define IAF_U8(a) __sync_add_and_fetch(a, 1)
162 | #define IAF_U16(a) __sync_add_and_fetch(a, 1)
163 | #define IAF_U32(a) __sync_add_and_fetch(a, 1)
164 | #define IAF_U64(a) __sync_add_and_fetch(a, 1)
165 | // Decrement-and-fetch
166 | #define DAF_U8(a) __sync_sub_and_fetch(a, 1)
167 | #define DAF_U16(a) __sync_sub_and_fetch(a, 1)
168 | #define DAF_U32(a) __sync_sub_and_fetch(a, 1)
169 | #define DAF_U64(a) __sync_sub_and_fetch(a, 1)
170 | // Test-and-set
171 | #define TAS_U8(a) tas_uint8(a)
172 | // Memory barrier
173 | /* #define MEM_BARRIER __sync_synchronize() */
174 | #define MEM_BARRIER // nop on the opteron for these benchmarks
175 | // Relax CPU
176 | //#define PAUSE _mm_pause()
177 | 
178 | /*End of x86 code*/
179 | #endif
180 | 
181 | /* start --generic code */
182 | 
183 | #define CAS_U64_bool(addr, old, new) (old == CAS_U64(addr, old, new))
184 | 
185 | /* static inline uint8_t */
186 | /* CAS_U64_bool(volatile AO_t* addr, AO_t old, AO_t new) */
187 | /* { */
188 | /*   return (old == CAS_U64(addr, old, new)); */
189 | /* } */
190 | 
191 | /* #define ATOMIC_CAS_MB(a, e, v)          (AO_compare_and_swap_full((volatile
192 |  * AO_t *)(a), (AO_t)(e), (AO_t)(v))) */
193 | /* #define ATOMIC_FETCH_AND_INC_FULL(a)    (AO_fetch_and_add1_full((volatile
194 |  * AO_t *)(a))) */
195 | 
196 | #define ATOMIC_CAS_MB(a, e, v)                                                 \
197 |     CAS_U64_bool((volatile AO_t *)(a), (AO_t)(e), (AO_t)(v))
198 | #define ATOMIC_FETCH_AND_INC_FULL(a) FAI_U32(a)
199 | 
200 | /* end -- generic code */
201 | 
202 | #endif
203 | 


--------------------------------------------------------------------------------
/lf-skiplist/lf-skiplist-alloc.h:
--------------------------------------------------------------------------------
  1 | //#ifndef _LF_SKIPLIST_H_
  2 | //#define _LF_SKIPLIST_H_
  3 | //
  4 | ///*
  5 | //        lock-free skip-list algorithm
  6 | //*/
  7 | //#include "atomic_ops_if.h"
  8 | //#include "sl_gc.h"
  9 | //#include "util.h"
 10 | //#include <cstdint>
 11 | //#include <libpmemobj.h>
 12 | //#include <stdio.h>
 13 | //#include <stdlib.h>
 14 | //
 15 | // namespace skiplist {
 16 | //
 17 | //#define CACHE_LINES_PER_NV_NODE \
 18 | //    3 // TODO does nv-jemalloc need to be aware of this?
 19 | //
 20 | // const int max_level = 20; // one cache-line node; use 13 for two cache-line
 21 | //                          // nodes
 22 | //
 23 | // typedef uintptr_t UINT_PTR;
 24 | // typedef uint32_t UINT32;
 25 | // typedef void *PVOID;
 26 | //
 27 | //#define MIN_KEY 1
 28 | //#define MAX_KEY 2
 29 | //
 30 | //#define NODE_PADDING 1
 31 | //
 32 | //#define CACHE_SIZE 64
 33 | //#define PMEM_CACHE_ALIGNED ALIGNED(CACHE_SIZE)
 34 | //
 35 | // static inline UINT_PTR mark_ptr_cache(UINT_PTR p) {
 36 | //    return (p | (UINT_PTR)0x04);
 37 | //}
 38 | //
 39 | // static inline UINT_PTR unmark_ptr_cache(UINT_PTR p) {
 40 | //    return (p & ~(UINT_PTR)0x04);
 41 | //}
 42 | //
 43 | // static inline int is_marked_ptr_cache(UINT_PTR p) {
 44 | //    return (int)(p & (UINT_PTR)0x04);
 45 | //}
 46 | //
 47 | // inline void flush_and_try_unflag(PVOID *target) {
 48 | //    // return;
 49 | //    PVOID value = *target;
 50 | //    if (is_marked_ptr_cache((UINT_PTR)value)) {
 51 | //        flush_data(target, sizeof(PVOID));
 52 | //        CAS_PTR((volatile PVOID *)target, value,
 53 | //                (PVOID)unmark_ptr_cache((UINT_PTR)value));
 54 | //    }
 55 | //}
 56 | //
 57 | //// links a node and persists it
 58 | //// marks the link while it is doing the persist
 59 | // inline PVOID link_and_persist(PVOID *target, PVOID oldvalue, PVOID value) {
 60 | //    // return CAS_PTR(target,oldvalue, value);
 61 | //    PVOID res;
 62 | //    res = CAS_PTR(target, (PVOID)oldvalue,
 63 | //                  (PVOID)mark_ptr_cache((UINT_PTR)value));
 64 | //
 65 | //    // if cas successful, we updated the link, but it still needs flushing
 66 | //    if (res != oldvalue) {
 67 | //        return res; // nothing gets fluhed
 68 | //    }
 69 | //    flush_data(target, sizeof(PVOID));
 70 | //    CAS_PTR((volatile PVOID *)target, (PVOID)mark_ptr_cache((UINT_PTR)value),
 71 | //            (PVOID)value);
 72 | //    return res;
 73 | //}
 74 | //
 75 | // typedef char *skey_t;
 76 | // typedef char *svalue_t;
 77 | //
 78 | // struct node_t {
 79 | //    skey_t key;
 80 | //    svalue_t value;
 81 | //    int key_len;
 82 | //    int val_len;
 83 | //
 84 | //    uint8_t max_min_flag;
 85 | //
 86 | //    node_t *next[max_level +
 87 | //                 2]; // in our allocator, we will be working with chunks of
 88 | //                 the
 89 | //                     // same size, so every node should have the same size
 90 | //    uint32_t toplevel;
 91 | //    uint8_t node_flags;
 92 | //};
 93 | //
 94 | // typedef node_t *skiplist_t;
 95 | //
 96 | // const int skiplist_node_size = sizeof(node_t);
 97 | //
 98 | // svalue_t skiplist_find(skiplist_t *sl, skey_t key);
 99 | //
100 | // int skiplist_insert(skiplist_t *sl, skey_t key, svalue_t val);
101 | //
102 | // svalue_t skiplist_remove(skiplist_t *sl, skey_t key);
103 | //
104 | // void skiplist_update(skiplist_t *sl, skey_t key, svalue_t val);
105 | //
106 | // void skiplist_scan(skiplist_t *sl, skey_t min, svalue_t *buf, int num, int
107 | // &off,
108 | //                   char *scan_value);
109 | //
110 | // skiplist_t *new_skiplist();
111 | //
112 | // void init_pmem();
113 | // void *allocate(size_t size);
114 | // void register_thread();
115 | //
116 | //} // namespace skiplist
117 | //
118 | //#endif
119 | 


--------------------------------------------------------------------------------
/lf-skiplist/skiplist-acma.h:
--------------------------------------------------------------------------------
  1 | #ifndef _SKIPLIST_H_
  2 | #define _SKIPLIST_H_
  3 | 
  4 | /*
  5 |         lock-free skip-list algorithm
  6 | */
  7 | #include "atomic_ops_if.h"
  8 | #include "EpochGuard.h"
  9 | #include "nvm_mgr.h"
 10 | #include "threadinfo.h"
 11 | #include "util.h"
 12 | #include <cstdint>
 13 | #include <libpmemobj.h>
 14 | #include <stdio.h>
 15 | #include <stdlib.h>
 16 | 
 17 |  namespace skiplist {
 18 | 
 19 | #define CACHE_LINES_PER_NV_NODE \
 20 |     3 // TODO does nv-jemalloc need to be aware of this?
 21 | 
 22 |     const int max_level = 20; // one cache-line node; use 13 for two
 23 | //    cache-line
 24 | // nodes
 25 | 
 26 |     typedef uintptr_t UINT_PTR;
 27 |     typedef uint32_t UINT32;
 28 |     typedef void *PVOID;
 29 | 
 30 | #define MIN_KEY 1
 31 | #define MAX_KEY 2
 32 | 
 33 | #define NODE_PADDING 1
 34 | 
 35 | #define CACHE_SIZE 64
 36 | #define PMEM_CACHE_ALIGNED ALIGNED(CACHE_SIZE)
 37 | 
 38 |     static inline UINT_PTR mark_ptr_cache(UINT_PTR p) {
 39 |         return (p | (UINT_PTR)0x04);
 40 |     }
 41 | 
 42 |     static inline UINT_PTR unmark_ptr_cache(UINT_PTR p) {
 43 |         return (p & ~(UINT_PTR)0x04);
 44 |     }
 45 | 
 46 |     static inline int is_marked_ptr_cache(UINT_PTR p) {
 47 |         return (int)(p & (UINT_PTR)0x04);
 48 |     }
 49 | 
 50 |     inline void flush_and_try_unflag(PVOID *target) {
 51 |         // return;
 52 |         PVOID value = *target;
 53 |         if (is_marked_ptr_cache((UINT_PTR)value)) {
 54 |             flush_data(target, sizeof(PVOID));
 55 |             CAS_PTR((volatile PVOID *)target, value,
 56 |                     (PVOID)unmark_ptr_cache((UINT_PTR)value));
 57 |         }
 58 |     }
 59 | 
 60 | // links a node and persists it
 61 | // marks the link while it is doing the persist
 62 |     inline PVOID link_and_persist(PVOID *target, PVOID oldvalue, PVOID value)
 63 |     {
 64 |         // return CAS_PTR(target,oldvalue, value);
 65 |         PVOID res;
 66 |         res = CAS_PTR(target, (PVOID)oldvalue,
 67 |                       (PVOID)mark_ptr_cache((UINT_PTR)value));
 68 | 
 69 |         // if cas successful, we updated the link, but it still needs flushing
 70 |         if (res != oldvalue) {
 71 |             return res; // nothing gets fluhed
 72 |         }
 73 |         flush_data(target, sizeof(PVOID));
 74 |         CAS_PTR((volatile PVOID *)target,
 75 |         (PVOID)mark_ptr_cache((UINT_PTR)value),
 76 |                 (PVOID)value);
 77 |         return res;
 78 |     }
 79 | 
 80 | #ifdef VARIABLE_LENGTH
 81 |     typedef char *skey_t;
 82 |     typedef char *svalue_t;
 83 | #else
 84 |     typedef uint64_t skey_t;
 85 |  typedef uint64_t svalue_t;
 86 | #endif
 87 | 
 88 |     struct node_t {
 89 |         skey_t key;
 90 |         svalue_t value;
 91 |         int key_len;
 92 |         int val_len;
 93 | 
 94 |         uint8_t max_min_flag;
 95 | 
 96 |         node_t *next[max_level +
 97 |                      2]; // in our allocator, we will be working with chunk of the
 98 |         // same size, so every node should have the same size
 99 |         uint32_t toplevel;
100 |         uint8_t node_flags;
101 |     };
102 | 
103 |     typedef node_t *skiplist_t;
104 | 
105 |     const int skiplist_node_size = sizeof(node_t);
106 | 
107 |     svalue_t skiplist_find(skiplist_t *sl, skey_t key);
108 | 
109 |     int skiplist_insert(skiplist_t *sl, skey_t key, svalue_t val);
110 | 
111 |     svalue_t skiplist_remove(skiplist_t *sl, skey_t key);
112 | 
113 |     void skiplist_update(skiplist_t *sl, skey_t key, svalue_t val);
114 | 
115 |     void skiplist_scan(skiplist_t *sl, skey_t min, svalue_t *buf, int num, int
116 |     &off,
117 |                        char *scan_value);
118 | 
119 |     skiplist_t *new_skiplist();
120 | } // namespace skiplist
121 | 
122 | #endif
123 | 


--------------------------------------------------------------------------------
/lf-skiplist/skiplist.h:
--------------------------------------------------------------------------------
  1 | //#ifndef _SKIPLIST_H_
  2 | //#define _SKIPLIST_H_
  3 | //
  4 | ///*
  5 | //        lock-free skip-list algorithm
  6 | //*/
  7 | //#include "atomic_ops_if.h"
  8 | //#include "sl_gc.h"
  9 | //#include "util.h"
 10 | //#include <cstdint>
 11 | //#include <libpmemobj.h>
 12 | //#include <stdio.h>
 13 | //#include <stdlib.h>
 14 | //
 15 | //namespace skiplist {
 16 | //
 17 | //#define CACHE_LINES_PER_NV_NODE                                                \
 18 | //    3 // TODO does nv-jemalloc need to be aware of this?
 19 | //
 20 | //const int max_level = 20; // one cache-line node; use 13 for two cache-line
 21 | //// nodes
 22 | //
 23 | //typedef uintptr_t UINT_PTR;
 24 | //typedef uint32_t UINT32;
 25 | //typedef void *PVOID;
 26 | //
 27 | //#define MIN_KEY 1
 28 | //#define MAX_KEY 2
 29 | //
 30 | //#define NODE_PADDING 1
 31 | //
 32 | //#define CACHE_SIZE 64
 33 | //#define PMEM_CACHE_ALIGNED ALIGNED(CACHE_SIZE)
 34 | //
 35 | //static inline UINT_PTR mark_ptr_cache(UINT_PTR p) {
 36 | //    return (p | (UINT_PTR)0x04);
 37 | //}
 38 | //
 39 | //static inline UINT_PTR unmark_ptr_cache(UINT_PTR p) {
 40 | //    return (p & ~(UINT_PTR)0x04);
 41 | //}
 42 | //
 43 | //static inline int is_marked_ptr_cache(UINT_PTR p) {
 44 | //    return (int)(p & (UINT_PTR)0x04);
 45 | //}
 46 | //
 47 | //inline void flush_and_try_unflag(PVOID *target) {
 48 | //    // return;
 49 | //    PVOID value = *target;
 50 | //    if (is_marked_ptr_cache((UINT_PTR)value)) {
 51 | //        flush_data(target, sizeof(PVOID));
 52 | //        CAS_PTR((volatile PVOID *)target, value,
 53 | //                (PVOID)unmark_ptr_cache((UINT_PTR)value));
 54 | //    }
 55 | //}
 56 | //
 57 | //// links a node and persists it
 58 | //// marks the link while it is doing the persist
 59 | //inline PVOID link_and_persist(PVOID *target, PVOID oldvalue, PVOID value) {
 60 | //    // return CAS_PTR(target,oldvalue, value);
 61 | //    PVOID res;
 62 | //    res = CAS_PTR(target, (PVOID)oldvalue,
 63 | //                  (PVOID)mark_ptr_cache((UINT_PTR)value));
 64 | //
 65 | //    // if cas successful, we updated the link, but it still needs flushing
 66 | //    if (res != oldvalue) {
 67 | //        return res; // nothing gets fluhed
 68 | //    }
 69 | //    flush_data(target, sizeof(PVOID));
 70 | //    CAS_PTR((volatile PVOID *)target, (PVOID)mark_ptr_cache((UINT_PTR)value),
 71 | //            (PVOID)value);
 72 | //    return res;
 73 | //}
 74 | //
 75 | //#ifdef VARIABLE_LENGTH
 76 | //typedef char *skey_t;
 77 | //typedef char *svalue_t;
 78 | //#else
 79 | //typedef uint64_t skey_t;
 80 | //typedef uint64_t svalue_t;
 81 | //#endif
 82 | //
 83 | //struct node_t {
 84 | //    skey_t key;
 85 | //    svalue_t value;
 86 | //    int key_len;
 87 | //    int val_len;
 88 | //
 89 | //    uint8_t max_min_flag;
 90 | //
 91 | //    node_t *next[max_level +
 92 | //                 2]; // in our allocator, we will be working with chunks of the
 93 | //    // same size, so every node should have the same size
 94 | //    uint32_t toplevel;
 95 | //    uint8_t node_flags;
 96 | //};
 97 | //
 98 | //typedef node_t *skiplist_t;
 99 | //
100 | //const int skiplist_node_size = sizeof(node_t);
101 | //
102 | //svalue_t skiplist_find(skiplist_t *sl, skey_t key);
103 | //
104 | //int skiplist_insert(skiplist_t *sl, skey_t key, svalue_t val);
105 | //
106 | //svalue_t skiplist_remove(skiplist_t *sl, skey_t key);
107 | //
108 | //void skiplist_update(skiplist_t *sl, skey_t key, svalue_t val);
109 | //
110 | //void skiplist_scan(skiplist_t *sl, skey_t min, svalue_t *buf, int num, int &off,
111 | //                   char *scan_value);
112 | //
113 | //skiplist_t *new_skiplist();
114 | //
115 | //void init_pmem();
116 | //void *allocate(size_t size);
117 | //void register_thread();
118 | //
119 | //} // namespace skiplist
120 | //
121 | //#endif
122 | 


--------------------------------------------------------------------------------
/lf-skiplist/sl_gc.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <iostream>
  4 | #include <libpmemobj.h>
  5 | #include <thread>
  6 | 
  7 | namespace skiplist {
  8 | 
  9 | static const int GC_NODE_COUNT_THREADHOLD = 1024;
 10 | 
 11 | static const int GC_INTERVAL = 50;
 12 | 
 13 | POBJ_LAYOUT_BEGIN(skiplist);
 14 | POBJ_LAYOUT_TOID(skiplist, struct key_item);
 15 | POBJ_LAYOUT_TOID(skiplist, char);
 16 | POBJ_LAYOUT_END(skiplist);
 17 | 
 18 | typedef struct key_item {
 19 |     size_t key_len;
 20 |     char key[];
 21 | } key_item;
 22 | 
 23 | class GarbageNode {
 24 |   public:
 25 |     uint64_t delete_epoch;
 26 |     void *node_p;
 27 |     GarbageNode *next_p;
 28 | 
 29 |     GarbageNode(uint64_t p_delete_epoch, void *p_node_p)
 30 |         : delete_epoch{p_delete_epoch}, node_p{p_node_p}, next_p{nullptr} {}
 31 | 
 32 |     GarbageNode() : delete_epoch{0UL}, node_p{nullptr}, next_p{nullptr} {}
 33 | } __attribute__((aligned(64)));
 34 | 
 35 | class GCMetaData {
 36 |   public:
 37 |     uint64_t last_active_epoch;
 38 | 
 39 |     GarbageNode header;
 40 |     GarbageNode *last_p;
 41 | 
 42 |     int node_count;
 43 | 
 44 |     GCMetaData() {
 45 |         last_active_epoch = static_cast<uint64_t>(-1);
 46 |         last_p = &header;
 47 |         node_count = 0;
 48 |     }
 49 | 
 50 |     ~GCMetaData() {}
 51 | } __attribute__((aligned(64)));
 52 | 
 53 | class Epoch_Mgr {
 54 |   public:
 55 |     std::thread *thread_p;
 56 |     bool exit_flag;
 57 |     uint64_t epoch;
 58 | 
 59 |   public:
 60 |     Epoch_Mgr() : exit_flag(false), epoch(0) {}
 61 | 
 62 |     ~Epoch_Mgr() {
 63 |         exit_flag = true;
 64 |         std::cout << "[EPOCH]\tepoch mgr exit\n";
 65 |     }
 66 | 
 67 |     inline void IncreaseEpoch() { epoch++; }
 68 | 
 69 |     uint64_t GetGlobalEpoch() { return epoch; }
 70 | 
 71 |     void ThreadFunc() {
 72 |         std::cout << "[EPOCH]\tglobal epoch thread start\n";
 73 |         while (exit_flag == false) {
 74 |             IncreaseEpoch();
 75 |             std::chrono::milliseconds duration(GC_INTERVAL);
 76 |             std::this_thread::sleep_for(duration);
 77 |         }
 78 |         std::cout << "[EPOCH]\tglobal epoch thread exit\n";
 79 |         return;
 80 |     }
 81 | 
 82 |     void StartThread() {
 83 |         thread_p = new std::thread{[this]() { this->ThreadFunc(); }};
 84 |         return;
 85 |     }
 86 | 
 87 | } __attribute__((aligned(64)));
 88 | 
 89 | class threadinfo {
 90 |   public:
 91 |     GCMetaData *md;
 92 |     Epoch_Mgr *mgr;
 93 | 
 94 |     threadinfo *next;
 95 |     threadinfo *head;
 96 | 
 97 |     PMEMobjpool *pool;
 98 | 
 99 |     int id;
100 | 
101 |     threadinfo(Epoch_Mgr *mgr_) : mgr(mgr_) {}
102 | 
103 |     ~threadinfo() {}
104 | 
105 |     void JoinEpoch() { md->last_active_epoch = mgr->GetGlobalEpoch(); }
106 | 
107 |     void LeaveEpoch() { md->last_active_epoch = static_cast<uint64_t>(-1); }
108 | 
109 |     void AddGarbageNode(void *node_p) {
110 |         TX_BEGIN(pool) {
111 |             pmemobj_tx_add_range_direct(&md->last_p->next_p, sizeof(uint64_t));
112 |             pmemobj_tx_add_range_direct(&md->last_p, sizeof(uint64_t));
113 |             pmemobj_tx_add_range_direct(&md->node_count, sizeof(int));
114 | 
115 |             PMEMoid p =
116 |                 pmemobj_tx_alloc(sizeof(GarbageNode), TOID_TYPE_NUM(char));
117 |             GarbageNode *gn = new (pmemobj_direct(p))
118 |                 GarbageNode(mgr->GetGlobalEpoch(), node_p);
119 |             md->last_p->next_p = gn;
120 |             md->last_p = gn;
121 |             md->node_count++;
122 |         }
123 |         TX_END
124 | 
125 |         if (md->node_count > GC_NODE_COUNT_THREADHOLD) {
126 |             // Use current thread's gc id to perform GC
127 |             PerformGC();
128 |         }
129 | 
130 |         return;
131 |     }
132 | 
133 |     uint64_t SummarizeGCEpoch(threadinfo *head_) {
134 |         threadinfo *tti = head->next;
135 |         uint64_t min_epoch = static_cast<uint64_t>(-1);
136 |         while (tti != nullptr) {
137 |             min_epoch = std::min(min_epoch, tti->md->last_active_epoch);
138 |             tti = tti->next;
139 |         }
140 |         return min_epoch;
141 |     }
142 | 
143 |     void PerformGC() {
144 |         // First of all get the minimum epoch of all active threads
145 |         // This is the upper bound for deleted epoch in garbage node
146 |         uint64_t min_epoch = SummarizeGCEpoch(head);
147 | 
148 |         // This is the pointer we use to perform GC
149 |         // Note that we only fetch the metadata using the current thread-local
150 |         // id
151 |         GarbageNode *header_p = &(md->header);
152 |         GarbageNode *first_p = header_p->next_p;
153 | 
154 |         // Then traverse the linked list
155 |         // Only reclaim memory when the deleted epoch < min epoch
156 |         while (first_p != nullptr && first_p->delete_epoch < min_epoch) {
157 |             // First unlink the current node from the linked list
158 |             // This could set it to nullptr
159 |             TX_BEGIN(pool) {
160 |                 pmemobj_tx_add_range_direct(&header_p->next_p,
161 |                                             sizeof(uint64_t));
162 |                 pmemobj_tx_add_range_direct(&md->node_count, sizeof(int));
163 |                 header_p->next_p = first_p->next_p;
164 | 
165 |                 PMEMoid ptr = pmemobj_oid(first_p->node_p);
166 |                 pmemobj_tx_free(ptr);
167 | 
168 |                 md->node_count--;
169 |             }
170 |             TX_END
171 | 
172 |             // Then free memory
173 |             //            FreeEpochNode(first_p->node_p);
174 | 
175 |             //            delete first_p;
176 |             first_p = header_p->next_p;
177 |         }
178 | 
179 |         // If we have freed all nodes in the linked list we should
180 |         // reset last_p to the header
181 |         if (first_p == nullptr) {
182 |             md->last_p = header_p;
183 |         }
184 |         return;
185 |     }
186 | };
187 | } // namespace skiplist
188 | 


--------------------------------------------------------------------------------
/nvm_mgr/Epoch.cpp:
--------------------------------------------------------------------------------
1 | #include "Epoch.h"
2 | 
3 | namespace NVMMgr_ns {
4 | int epoch = 0;
5 | bool exit_flag = false;
6 | } // namespace NVMMgr_ns


--------------------------------------------------------------------------------
/nvm_mgr/Epoch.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <chrono>
  4 | #include <iostream>
  5 | #include <thread>
  6 | 
  7 | namespace NVMMgr_ns {
  8 | 
  9 | static const int GC_NODE_COUNT_THREADHOLD = 1024;
 10 | 
 11 | static const int GC_INTERVAL = 50;
 12 | 
 13 | extern int epoch;
 14 | extern bool exit_flag;
 15 | 
 16 | class GarbageNode {
 17 |   public:
 18 |     // The epoch that this node is unlinked
 19 |     // This do not have to be exact - just make sure it is no earlier than the
 20 |     // actual epoch it is unlinked from the data structure
 21 |     uint64_t delete_epoch;
 22 |     void *node_p;
 23 |     GarbageNode *next_p;
 24 | 
 25 |     GarbageNode(uint64_t p_delete_epoch, void *p_node_p)
 26 |         : delete_epoch{p_delete_epoch}, node_p{p_node_p}, next_p{nullptr} {}
 27 | 
 28 |     GarbageNode() : delete_epoch{0UL}, node_p{nullptr}, next_p{nullptr} {}
 29 | } __attribute__((aligned(64)));
 30 | 
 31 | class GCMetaData {
 32 |   public:
 33 |     // This is the last active epoch counter; all garbages before this counter
 34 |     // are guaranteed to be not being used by this thread
 35 |     // So if we take a global minimum of this value, that minimum could be
 36 |     // be used as the global epoch value to decide whether a garbage node could
 37 |     // be recycled
 38 |     uint64_t last_active_epoch;
 39 | 
 40 |     GarbageNode header;
 41 | 
 42 |     // This points to the last node in the garbage node linked list
 43 |     // We always append new nodes to this pointer, and thus inside one
 44 |     // node's context these garbage nodes are always sorted, from low
 45 |     // epoch to high epoch. This facilitates memory reclaimation since we
 46 |     // just start from the lowest epoch garbage and traverse the linked list
 47 |     // until we see an epoch >= GC epoch
 48 |     GarbageNode *last_p;
 49 | 
 50 |     // The number of nodes inside this GC context
 51 |     // We use this as a threshold to trigger GC
 52 |     int node_count;
 53 | 
 54 |     GCMetaData() {
 55 |         last_active_epoch = static_cast<uint64_t>(-1);
 56 |         last_p = &header;
 57 |         node_count = 0;
 58 |     }
 59 |     ~GCMetaData() {}
 60 | } __attribute__((aligned(64)));
 61 | 
 62 | class Epoch_Mgr {
 63 |   public:
 64 |     std::thread *thread_p;
 65 | 
 66 |   public:
 67 |     Epoch_Mgr() { exit_flag = false; }
 68 | 
 69 |     ~Epoch_Mgr() {
 70 |         exit_flag = true;
 71 |         std::cout << "[EPOCH]\tepoch mgr exit\n";
 72 |     }
 73 | 
 74 |     inline void IncreaseEpoch() { epoch++; }
 75 | 
 76 |     static uint64_t GetGlobalEpoch() { return epoch; }
 77 | 
 78 |     /*
 79 |      * ThreadFunc() - The cleaner thread executes this every GC_INTERVAL ms
 80 |      *
 81 |      * This function exits when exit flag is set to true
 82 |      */
 83 |     void ThreadFunc() {
 84 |         // While the parent is still running
 85 |         // We do not worry about race condition here
 86 |         // since even if we missed one we could always
 87 |         // hit the correct value on next try
 88 |         std::cout << "[EPOCH]\tglobal epoch thread start\n";
 89 |         while (exit_flag == false) {
 90 |             // printf("Start new epoch cycle\n");
 91 |             IncreaseEpoch();
 92 | 
 93 |             // Sleep for 50 ms
 94 |             std::chrono::milliseconds duration(GC_INTERVAL);
 95 |             std::this_thread::sleep_for(duration);
 96 |         }
 97 |         std::cout << "[EPOCH]\tglobal epoch thread exit\n";
 98 |         return;
 99 |     }
100 | 
101 |     /*
102 |      * StartThread() - Start cleaner thread for garbage collection
103 |      *
104 |      * NOTE: This is not called in the constructor, and needs to be
105 |      * called manually
106 |      */
107 |     void StartThread() {
108 |         thread_p = new std::thread{[this]() { this->ThreadFunc(); }};
109 | 
110 |         return;
111 |     }
112 | } __attribute__((aligned(64)));
113 | 
114 | } // namespace NVMMgr_ns
115 | 


--------------------------------------------------------------------------------
/nvm_mgr/EpochGuard.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "threadinfo.h"
 4 | 
 5 | namespace NVMMgr_ns {
 6 | class EpochGuard {
 7 |   public:
 8 |     EpochGuard() { JoinNewEpoch(); }
 9 |     ~EpochGuard() { LeaveThisEpoch(); }
10 |     static void DeleteNode(void *node) { MarkNodeGarbage(node); }
11 | };
12 | } // namespace NVMMgr_ns


--------------------------------------------------------------------------------
/nvm_mgr/nvm_mgr.cpp:
--------------------------------------------------------------------------------
  1 | #include "nvm_mgr.h"
  2 | #include "Tree.h"
  3 | #include "threadinfo.h"
  4 | #include <cassert>
  5 | #include <iostream>
  6 | #include <math.h>
  7 | #include <mutex>
  8 | #include <stdio.h>
  9 | #include <vector>
 10 | 
 11 | namespace NVMMgr_ns {
 12 | 
 13 | // global nvm manager
 14 | NVMMgr *nvm_mgr = NULL;
 15 | std::mutex _mtx;
 16 | 
 17 | int create_file(const char *file_name, uint64_t file_size) {
 18 |     std::ofstream fout(file_name);
 19 |     if (fout) {
 20 |         fout.close();
 21 |         int result = truncate(file_name, file_size);
 22 |         if (result != 0) {
 23 |             printf("[NVM MGR]\ttruncate new file failed\n");
 24 |             exit(1);
 25 |         }
 26 |     } else {
 27 |         printf("[NVM MGR]\tcreate new file failed\n");
 28 |         exit(1);
 29 |     }
 30 | 
 31 |     return 0;
 32 | }
 33 | 
 34 | NVMMgr::NVMMgr() {
 35 |     // access 的返回结果， 0: 存在， 1: 不存在
 36 |     int initial = access(get_filename(), F_OK);
 37 |     first_created = false;
 38 | 
 39 |     if (initial) {
 40 |         int result = create_file(get_filename(), filesize);
 41 |         if (result != 0) {
 42 |             printf("[NVM MGR]\tcreate file failed when initalizing\n");
 43 |             exit(1);
 44 |         }
 45 |         first_created = true;
 46 |         printf("[NVM MGR]\tcreate file success.\n");
 47 |     }
 48 | 
 49 |     // open file
 50 |     fd = open(get_filename(), O_RDWR);
 51 |     if (fd < 0) {
 52 |         printf("[NVM MGR]\tfailed to open nvm file\n");
 53 |         exit(-1);
 54 |     }
 55 |     if (ftruncate(fd, filesize) < 0) {
 56 |         printf("[NVM MGR]\tfailed to truncate file\n");
 57 |         exit(-1);
 58 |     }
 59 |     printf("[NVM MGR]\topen file %s success.\n", get_filename());
 60 | 
 61 |     // mmap
 62 |     void *addr = mmap((void *)start_addr, filesize, PROT_READ | PROT_WRITE,
 63 |                       MAP_SHARED, fd, 0);
 64 | 
 65 |     if (addr != (void *)start_addr) {
 66 |         printf("[NVM MGR]\tmmap failed %p \n", addr);
 67 |         exit(0);
 68 |     }
 69 |     printf("[NVM MGR]\tmmap successfully\n");
 70 | 
 71 |     // initialize meta data
 72 |     meta_data = static_cast<Head *>(addr);
 73 |     if (initial) {
 74 |         // set status of head and set zero for bitmap
 75 |         // persist it
 76 |         memset((void *)meta_data, 0, PGSIZE);
 77 | 
 78 |         meta_data->status = magic_number;
 79 |         meta_data->threads = 0;
 80 |         meta_data->free_bit_offset = 0;
 81 |         meta_data->generation_version = 0;
 82 | 
 83 |         flush_data((void *)meta_data, PGSIZE);
 84 |         printf("[NVM MGR]\tinitialize nvm file's head\n");
 85 |     } else {
 86 |         meta_data->generation_version++;
 87 |         flush_data((void *)&meta_data->generation_version, sizeof(uint64_t));
 88 |         printf("nvm mgr restart, the free offset is %lld, generation version "
 89 |                "is %lld\n",
 90 |                meta_data->free_bit_offset, meta_data->generation_version);
 91 |     }
 92 | }
 93 | 
 94 | NVMMgr::~NVMMgr() {
 95 |     // normally exit
 96 |     printf("[NVM MGR]\tnormally exits, NVM reset..\n");
 97 |     //        Head *head = (Head *) start_addr;
 98 |     //        flush_data((void *) head, sizeof(Head));
 99 |     munmap((void *)start_addr, filesize);
100 |     close(fd);
101 | }
102 | 
103 | //    void NVMMgr::recover_done() {
104 | //        Head *head = (Head *) start_addr;
105 | //        head->threads = 0;
106 | //        flush_data((void *) head, sizeof(Head));
107 | //    }
108 | 
109 | // bool NVMMgr::reload_free_blocks() {
110 | //    assert(free_page_list.empty());
111 | //
112 | //    while (true) {
113 | //        if (free_bit_offset >= (filesize / PGSIZE) - (max_threads + 1)) {
114 | //            return false;
115 | //        }
116 | //
117 | //        uint8_t value = meta_data->bitmap[free_bit_offset];
118 | //
119 | //        // not free
120 | //        if (value != 0) {
121 | //            free_bit_offset++;
122 | //            continue;
123 | //        } else if (value == 0) { // free
124 | //            for (int i = 0; i < 8; i++) {
125 | //                if (free_bit_offset >=
126 | //                    (filesize / PGSIZE) - (max_threads + 1)) {
127 | //                    break;
128 | //                }
129 | //                if (meta_data->bitmap[free_bit_offset] != 0) {
130 | //                    free_bit_offset++;
131 | //                    continue;
132 | //                }
133 | //
134 | //                free_page_list.push_back(free_bit_offset);
135 | //                free_bit_offset++;
136 | //            }
137 | //        }
138 | //        break;
139 | //    }
140 | ////    std::cout << "[NVM MGR]\treload free blocks, now free_page_list size is
141 | ///" /              << free_page_list.size() << "\n";
142 | //    return true;
143 | //}
144 | 
145 | void *NVMMgr::alloc_thread_info() {
146 |     // not thread safe
147 |     size_t index = meta_data->threads++;
148 |     flush_data((void *)&(meta_data->threads), sizeof(int));
149 |     return (void *)(thread_local_start + index * PGSIZE);
150 | }
151 | 
152 | void *NVMMgr::get_thread_info(int tid) {
153 |     return (void *)(thread_local_start + tid * PGSIZE);
154 | }
155 | 
156 | void *NVMMgr::alloc_block(int tid) {
157 |     std::lock_guard<std::mutex> lock(_mtx);
158 | 
159 |     uint64_t id = meta_data->free_bit_offset;
160 |     meta_data->free_bit_offset++;
161 |     meta_data->bitmap[id] = tid;
162 |     flush_data((void *)&(meta_data->bitmap[id]), sizeof(uint8_t));
163 |     flush_data((void *)&(meta_data->free_bit_offset), sizeof(uint64_t));
164 | 
165 |     void *addr = (void *)(data_block_start + id * PGSIZE);
166 | 
167 |     //    printf("[NVM MGR]\talloc a new block %d, type is %d\n", id, type);
168 |     //    std::cout<<"alloc a new block "<< meta_data->free_bit_offset<<"\n";
169 |     //    std::cout<<"meta data addr "<< meta_data<<"\n";
170 |     //    std::cout<<"mgr addr" <<this<<"\n";
171 | 
172 |     return addr;
173 | }
174 | 
175 | // mutiple threads to recovery free list for
176 | // threads using recovery_set
177 | void NVMMgr::recovery_free_memory(PART_ns::Tree *art, int forward_thread) {
178 |     int owner = 0;
179 |     for (int i = 0; i < meta_data->free_bit_offset; i++) {
180 |         meta_data->bitmap[i] = (owner++) % forward_thread;
181 |     }
182 |     std::cout << "finish set owner, all " << meta_data->free_bit_offset
183 |               << " pages\n";
184 | 
185 |     const size_t power_two[10] = {8,   16,  32,   64,   128,
186 |                                   256, 512, 1024, 2048, 4096};
187 |     const int thread_num = 36;
188 |     std::thread *tid[thread_num];
189 |     int per_thread_block = meta_data->free_bit_offset / thread_num;
190 |     if (meta_data->free_bit_offset % thread_num != 0)
191 |         per_thread_block++;
192 | 
193 |     std::cout << "every thread needs to recover " << per_thread_block
194 |               << " pages\n";
195 | 
196 |     for (int i = 0; i < thread_num; i++) {
197 |         tid[i] = new std::thread(
198 |             [&](int id) {
199 |                 // [start, end]
200 |                 uint64_t start = id * per_thread_block;
201 |                 uint64_t end = (id + 1) * per_thread_block;
202 |                 //                std::cout << "start " << start
203 |                 //                          << " end " << end<<"\n";
204 |                 uint64_t start_addr = data_block_start + start * PGSIZE;
205 |                 uint64_t end_addr = std::min(
206 |                     data_block_start + end * PGSIZE,
207 |                     data_block_start + meta_data->free_bit_offset * PGSIZE);
208 |                 //                std::set<std::pair<uint64_t, size_t>>
209 |                 //                    recovery_set; // used for memory recovery
210 |                 std::vector<std::pair<uint64_t, size_t>> recovery_set;
211 | 
212 |                 art->rebuild(recovery_set, start_addr, end_addr, id);
213 | #ifdef RECLAIM_MEMORY
214 | 
215 |                 std::sort(recovery_set.begin(), recovery_set.end());
216 |                 std::cout << "start to reclaim\n";
217 | 
218 | //                for (int i = 0; i < recovery_set.size(); i++) {
219 | //                    uint64_t this_addr = recovery_set[i].first;
220 | //                    uint64_t this_size = recovery_set[i].second;
221 | //                    for (int id = 0; id < 10; id++) {
222 | //                        if (this_size <= power_two[id]) {
223 | //                            this_size = power_two[id];
224 | //                            break;
225 | //                        }
226 | //                    }
227 | //
228 | //                    int j = start_addr / PGSIZE;
229 | //                    int tid =
230 | //                        meta_data
231 | //                            ->bitmap[j]; // this block belong to which thread
232 | //                    thread_info *the_ti = (thread_info *)get_thread_info(tid);
233 | //
234 | //                    the_ti->free_list->insert_into_freelist(
235 | //                        start_addr, this_addr - start_addr);
236 | //                    start_addr = this_addr + this_size;
237 | //                }
238 | //
239 | //                int j = start_addr / PGSIZE;
240 | //                int tid =
241 | //                    meta_data->bitmap[j]; // this block belong to which thread
242 | //                thread_info *the_ti = (thread_info *)get_thread_info(tid);
243 | //
244 | //                if (end_addr >= start_addr) {
245 | //                    the_ti->free_list->insert_into_freelist(
246 | //                        start_addr, end_addr - start_addr);
247 | //                }
248 | #endif
249 |             },
250 |             i);
251 |     }
252 |     for (int i = 0; i < thread_num; i++) {
253 |         tid[i]->join();
254 |     }
255 | }
256 | 
257 | /*
258 |  * interface to call methods of nvm_mgr
259 |  */
260 | NVMMgr *get_nvm_mgr() {
261 |     std::lock_guard<std::mutex> lock(_mtx);
262 | 
263 |     if (nvm_mgr == NULL) {
264 |         printf("[NVM MGR]\tnvm manager is not initilized.\n");
265 |         assert(0);
266 |     }
267 |     return nvm_mgr;
268 | }
269 | 
270 | bool init_nvm_mgr() {
271 |     std::lock_guard<std::mutex> lock(_mtx);
272 | 
273 |     if (nvm_mgr) {
274 |         printf("[NVM MGR]\tnvm manager has already been initilized.\n");
275 |         return false;
276 |     }
277 |     nvm_mgr = new NVMMgr();
278 |     return true;
279 | }
280 | 
281 | void close_nvm_mgr() {
282 |     std::lock_guard<std::mutex> lock(_mtx);
283 |     std::cout << "[NVM MGR]\tclose nvm mgr\n";
284 |     if (nvm_mgr != NULL) {
285 |         delete nvm_mgr;
286 |         nvm_mgr = NULL;
287 |     }
288 | }
289 | } // namespace NVMMgr_ns
290 | 


--------------------------------------------------------------------------------
/nvm_mgr/nvm_mgr.h:
--------------------------------------------------------------------------------
  1 | #ifndef nvm_mgr_h
  2 | #define nvm_mgr_h
  3 | 
  4 | #include "Tree.h"
  5 | #include "util.h"
  6 | #include <fcntl.h>
  7 | #include <list>
  8 | #include <set>
  9 | #include <stdio.h>
 10 | #include <stdlib.h>
 11 | #include <string>
 12 | #include <sys/mman.h>
 13 | #include <sys/stat.h>
 14 | 
 15 | namespace NVMMgr_ns {
 16 | 
 17 | class NVMMgr {
 18 |     /*
 19 |      *
 20 |      * A simple structure to manage the NVM file
 21 |      *
 22 |      *   / 256K  /  128 * 256K  /     ...     /
 23 |      *  / head  / thread local / data blocks /
 24 |      *
 25 |      *  head:
 26 |      *       Avaiable to the NVM manager, including root and bitmap to indicate
 27 |      *       how many blocks have been allocated.
 28 |      *
 29 |      *  thread local:
 30 |      *       Each thread can own a thread local persistent memory area. After
 31 |      * system crashes, NVM manager gives these thread local persistent memory to
 32 |      * the application. The application may use thread local variables to help
 33 |      * recovery. For example, in RNTree, a leaf node is logged in the thread
 34 |      * local area before it is splitted. The recovery procedure can use the
 35 |      * thread local log to guarantee the crash consistency of the leaf.
 36 |      *
 37 |      *       The function "recover_done" should be invoked after the recovery,
 38 |      * otherwise these thread local persistent memories are leaked. The maximum
 39 |      * number of thread local blocks can be allocated is hard coded in this
 40 |      * file.
 41 |      *
 42 |      *  data:
 43 |      *       True persistent memory allocated for applications. For simplicity,
 44 |      * we do not recyle memory.
 45 |      *
 46 |      */
 47 |   public:
 48 |     static const int magic_number = 12345;
 49 |     static const int max_threads = 64;
 50 | 
 51 |     static const int PGSIZE = 256 * 1024;                     // 256K
 52 |     static const long long filesize = 1024LL * 1024 * PGSIZE; // 256GB
 53 | 
 54 |     static const size_t start_addr = 0x50000000;
 55 |     static const size_t thread_local_start = start_addr + PGSIZE;
 56 |     static const size_t data_block_start =
 57 |         thread_local_start + PGSIZE * max_threads;
 58 | 
 59 |     static const char *get_filename() {
 60 |         static const std::string filename = std::string(nvm_dir) + "part.data";
 61 |         return filename.c_str();
 62 |     }
 63 | 
 64 |     struct Head {
 65 |         // TODO: some other meta data for recovery
 66 |         char root[4096]; // for root
 67 |         uint64_t generation_version;
 68 |         uint64_t free_bit_offset;
 69 |         int status;        // if equal to magic_number, it is reopen
 70 |         int threads;       // threads number
 71 |         uint8_t bitmap[0]; // show every page type
 72 |         // 0: free, 1: N4, 2: N16, 3: N48, 4: N256, 5: Leaf
 73 |     };
 74 | 
 75 |   public:
 76 |     NVMMgr();
 77 | 
 78 |     ~NVMMgr();
 79 | 
 80 |     //    bool reload_free_blocks();
 81 | 
 82 |     void *alloc_tree_root() { return (void *)meta_data; }
 83 | 
 84 |     void *alloc_thread_info();
 85 | 
 86 |     void *get_thread_info(int tid);
 87 | 
 88 |     void *alloc_block(int tid);
 89 | 
 90 |     void recovery_free_memory(PART_ns::Tree *art, int forward_thread);
 91 | 
 92 |     uint64_t get_generation_version() { return meta_data->generation_version; }
 93 | 
 94 |     // volatile metadata and rebuild when recovery
 95 |     int fd;
 96 |     bool first_created;
 97 | 
 98 |     // persist it as the head of nvm region
 99 |     Head *meta_data;
100 | } __attribute__((aligned(64)));
101 | 
102 | NVMMgr *get_nvm_mgr();
103 | 
104 | // true: first initialize
105 | // false: have been initialized
106 | bool init_nvm_mgr();
107 | 
108 | void close_nvm_mgr();
109 | } // namespace NVMMgr_ns
110 | 
111 | #endif // nvm_mgr_h
112 | 


--------------------------------------------------------------------------------
/nvm_mgr/pmalloc_wrap.h:
--------------------------------------------------------------------------------
 1 | #ifndef pmalloc_wrap_h
 2 | #define pmalloc_wrap_h
 3 | 
 4 | #include "nvm_mgr.h"
 5 | #include <assert.h>
 6 | #include <iostream>
 7 | #include <list>
 8 | #include <mutex>
 9 | #include <numa.h>
10 | #include <stdio.h>
11 | #include <stdlib.h>
12 | 
13 | #define USE_NVM_MALLOC
14 | 
15 | namespace NVMMgr_ns {
16 | 
17 | class PMBlockAllocator {
18 |     int alignment = 64;
19 |     NVMMgr *mgr;
20 | 
21 |   public:
22 |     PMBlockAllocator(NVMMgr *mgr_ = NULL) { mgr = mgr_; }
23 |     ~PMBlockAllocator() {}
24 | 
25 |     void *alloc_block(int tid) {
26 |         if (mgr == NULL) {
27 |             mgr = get_nvm_mgr();
28 |             if (mgr == NULL) {
29 |                 std::cout << "[PMBLOCK]\tneed to call init_nvm_mgr() first\n";
30 |                 assert(0);
31 |             }
32 |         }
33 | #ifdef USE_NVM_MALLOC
34 |         // mgr is thread safe
35 |         return mgr->alloc_block(tid);
36 | #else
37 |         return aligned_alloc(alignment, NVMMgr::PGSIZE);
38 | #endif // USE_NVM_MALLOC
39 |     }
40 | 
41 |     void free_block(void *block) {
42 |         // TODO: free block
43 |     }
44 | } __attribute__((aligned(64)));
45 | 
46 | } // namespace NVMMgr_ns
47 | #endif
48 | 


--------------------------------------------------------------------------------
/nvm_mgr/threadinfo.cpp:
--------------------------------------------------------------------------------
  1 | #include "threadinfo.h"
  2 | #include "nvm_mgr.h"
  3 | #include "pmalloc_wrap.h"
  4 | #include "timer.h"
  5 | #include <assert.h>
  6 | #include <iostream>
  7 | #include <list>
  8 | #include <mutex>
  9 | 
 10 | namespace NVMMgr_ns {
 11 | 
 12 | // global block allocator
 13 | PMBlockAllocator *pmblock = nullptr;
 14 | 
 15 | // global threadinfo lock to protect alloc thread info
 16 | std::mutex ti_lock;
 17 | 
 18 | // global threadinfo list hread
 19 | thread_info *ti_list_head = nullptr;
 20 | 
 21 | // thread local info
 22 | __thread thread_info *ti = nullptr;
 23 | 
 24 | // global thread id
 25 | int tid = 0;
 26 | 
 27 | // global Epoch_Mgr
 28 | Epoch_Mgr *epoch_mgr = nullptr;
 29 | 
 30 | #ifdef COUNT_ALLOC
 31 | __thread cpuCycleTimer *dcmm_time = nullptr;
 32 | double getdcmmalloctime() { return dcmm_time->duration(); }
 33 | #endif
 34 | 
 35 | #ifdef INSTANT_RESTART
 36 | uint64_t thread_result[40][8];
 37 | void init() { memset(thread_result, 0, sizeof(thread_result)); }
 38 | void increase(int id) { thread_result[id][0]++; }
 39 | uint64_t total(int thread_num) {
 40 |     uint64_t ans = 0;
 41 |     for (int i = 0; i < thread_num; i++) {
 42 |         ans += thread_result[i][0];
 43 |     }
 44 |     return ans;
 45 | }
 46 | 
 47 | __thread uint64_t thread_generation = 0;
 48 | uint64_t get_threadlocal_generation() { return thread_generation; }
 49 | #endif
 50 | 
 51 | size_t get_node_size(PART_ns::NTypes type) {
 52 |     switch (type) {
 53 |     case PART_ns::NTypes::N4:
 54 |         return sizeof(PART_ns::N4);
 55 |     case PART_ns::NTypes::N16:
 56 |         return sizeof(PART_ns::N16);
 57 |     case PART_ns::NTypes::N48:
 58 |         return sizeof(PART_ns::N48);
 59 |     case PART_ns::NTypes::N256:
 60 |         return sizeof(PART_ns::N256);
 61 |     case PART_ns::NTypes::Leaf:
 62 |         return sizeof(PART_ns::Leaf);
 63 |     case PART_ns::NTypes::LeafArray:
 64 |         return sizeof(PART_ns::LeafArray);
 65 |     default:
 66 |         std::cout << "[ALLOC NODE]\twrong type\n";
 67 |         assert(0);
 68 |     }
 69 | }
 70 | 
 71 | size_t size_align(size_t s, int align) {
 72 |     return ((s + align - 1) / align) * align;
 73 | }
 74 | 
 75 | size_t convert_power_two(size_t s) {
 76 |     return ti->free_list->get_power_two_size(s);
 77 | }
 78 | 
 79 | /**************************PMFreeList interface*****************************/
 80 | 
 81 | // PMFreeList::PMFreeList(PMBlockAllocator *pmb_) : pmb(pmb_) {
 82 | //    free_node_list.clear();
 83 | //}
 84 | 
 85 | // void *PMFreeList::alloc_node(PART_ns::NTypes type) {
 86 | //
 87 | //    if (free_node_list.empty()) {
 88 | //        size_t node_size = size_align(get_node_size(type), 64);
 89 | //        //        std::cout << "[ALLOC NODE]\tnode type " << (int)type << ",
 90 | //        //        node size "
 91 | //        //                  << node_size << "\n";
 92 | //        void *addr = pmb->alloc_block((int)type);
 93 | //
 94 | //        for (int i = 0; i + node_size <= NVMMgr::PGSIZE; i += node_size) {
 95 | //            free_node_list.push_back((uint64_t)addr + i);
 96 | //        }
 97 | //    }
 98 | //    uint64_t pos = free_node_list.front();
 99 | //    free_node_list.pop_front();
100 | //
101 | //    return (void *)pos;
102 | //}
103 | //
104 | // void PMFreeList::free_node(void *addr) {
105 | //    free_node_list.push_back((uint64_t)addr);
106 | //}
107 | 
108 | /*************************buddy_allocator interface**************************/
109 | 
110 | void buddy_allocator::insert_into_freelist(uint64_t addr, size_t size) {
111 |     uint64_t curr_addr = addr;
112 |     size_t curr_size = size;
113 |     while (curr_size) {
114 |         //        std::cout<<"size is "<<curr_size <<"\n";
115 |         for (int curr_id = free_list_number - 1; curr_id >= 0; curr_id--) {
116 |             if (curr_addr % power_two[curr_id] == 0 &&
117 |                 curr_size >= power_two[curr_id]) {
118 |                 free_list[curr_id].push(curr_addr);
119 |                 curr_addr += power_two[curr_id];
120 |                 curr_size -= power_two[curr_id];
121 |                 break;
122 |             }
123 |         }
124 |     }
125 |     assert(curr_size == 0);
126 | }
127 | 
128 | uint64_t buddy_allocator::get_addr(int id) {
129 |     uint64_t addr;
130 |     if (id == free_list_number - 1) {
131 |         if (!free_list[id].try_pop(addr)) {
132 |             // empty, allocate block from nvm_mgr
133 |             thread_info *ti = (thread_info *)get_threadinfo();
134 |             addr = (uint64_t)pmb->alloc_block(ti->id);
135 |             for (int i = power_two[id]; i < NVMMgr::PGSIZE;
136 |                  i += power_two[id]) {
137 |                 free_list[id].push(addr + (uint64_t)i);
138 |             }
139 |         }
140 |         return addr;
141 |     }
142 | 
143 |     // pop successfully
144 |     if (free_list[id].try_pop(addr)) {
145 |         return addr;
146 |     } else { // empty
147 |         addr = get_addr(id + 1);
148 |         // get a bigger page and splitAndUnlock half into free_list
149 |         free_list[id].push(addr + power_two[id]);
150 |         return addr;
151 |     }
152 | }
153 | 
154 | // alloc size smaller than 4k
155 | void *buddy_allocator::alloc_node(size_t size) {
156 |     int id;
157 |     for (int i = 0; i < free_list_number; i++) {
158 |         if (power_two[i] >= size) {
159 |             id = i;
160 |             break;
161 |         }
162 |     }
163 |     return (void *)get_addr(id);
164 | }
165 | 
166 | size_t buddy_allocator::get_power_two_size(size_t s) {
167 |     int id = free_list_number;
168 |     for (int i = 0; i < free_list_number; i++) {
169 |         if (power_two[i] >= s) {
170 |             id = i;
171 |             break;
172 |         }
173 |     }
174 |     assert(id < free_list_number);
175 |     return power_two[id];
176 | }
177 | 
178 | /*************************thread_info interface**************************/
179 | 
180 | thread_info::thread_info() {
181 |     //    node4_free_list = new PMFreeList(pmblock);
182 |     //    node16_free_list = new PMFreeList(pmblock);
183 |     //    node48_free_list = new PMFreeList(pmblock);
184 |     //    node256_free_list = new PMFreeList(pmblock);
185 |     //    leaf_free_list = new PMFreeList(pmblock);
186 | 
187 |     free_list = new buddy_allocator(pmblock);
188 | 
189 |     md = new GCMetaData();
190 |     _lock = 0;
191 |     id = tid++;
192 | }
193 | 
194 | thread_info::~thread_info() {
195 |     //    delete node4_free_list;
196 |     //    delete node16_free_list;
197 |     //    delete node48_free_list;
198 |     //    delete node256_free_list;
199 |     //    delete leaf_free_list;
200 | 
201 |     delete free_list;
202 |     delete md;
203 | }
204 | 
205 | void thread_info::AddGarbageNode(void *node_p) {
206 |     GarbageNode *garbage_node_p =
207 |         new GarbageNode(Epoch_Mgr::GetGlobalEpoch(), node_p);
208 |     assert(garbage_node_p != nullptr);
209 | 
210 |     // Link this new node to the end of the linked list
211 |     // and then update last_p
212 |     md->last_p->next_p = garbage_node_p;
213 |     md->last_p = garbage_node_p;
214 |     //    PART_ns::BaseNode *n = (PART_ns::BaseNode *)node_p;
215 |     //    std::cout << "[TEST]\tgarbage node type " << (int)(n->type) << "\n";
216 |     // Update the counter
217 |     md->node_count++;
218 | 
219 |     // It is possible that we could not free enough number of nodes to
220 |     // make it less than this threshold
221 |     // So it is important to let the epoch counter be constantly increased
222 |     // to guarantee progress
223 |     if (md->node_count > GC_NODE_COUNT_THREADHOLD) {
224 |         // Use current thread's gc id to perform GC
225 |         PerformGC();
226 |     }
227 | 
228 |     return;
229 | }
230 | 
231 | void thread_info::PerformGC() {
232 |     // First of all get the minimum epoch of all active threads
233 |     // This is the upper bound for deleted epoch in garbage node
234 |     uint64_t min_epoch = SummarizeGCEpoch();
235 | 
236 |     // This is the pointer we use to perform GC
237 |     // Note that we only fetch the metadata using the current thread-local id
238 |     GarbageNode *header_p = &(md->header);
239 |     GarbageNode *first_p = header_p->next_p;
240 | 
241 |     // Then traverse the linked list
242 |     // Only reclaim memory when the deleted epoch < min epoch
243 |     while (first_p != nullptr && first_p->delete_epoch < min_epoch) {
244 |         // First unlink the current node from the linked list
245 |         // This could set it to nullptr
246 |         header_p->next_p = first_p->next_p;
247 | 
248 |         // Then free memory
249 |         FreeEpochNode(first_p->node_p);
250 | 
251 |         delete first_p;
252 |         assert(md->node_count != 0UL);
253 |         md->node_count--;
254 | 
255 |         first_p = header_p->next_p;
256 |     }
257 | 
258 |     // If we have freed all nodes in the linked list we should
259 |     // reset last_p to the header
260 |     if (first_p == nullptr) {
261 |         md->last_p = header_p;
262 |     }
263 | 
264 |     return;
265 | }
266 | 
267 | void thread_info::FreeEpochNode(void *node_p) {
268 |     PART_ns::BaseNode *n = reinterpret_cast<PART_ns::BaseNode *>(node_p);
269 | 
270 |     if (n->type == PART_ns::NTypes::Leaf) {
271 |         // reclaim leaf key
272 |         PART_ns::Leaf *leaf = (PART_ns::Leaf *)n;
273 | #ifdef KEY_INLINE
274 |         free_node_from_size((uint64_t)n, sizeof(PART_ns::Leaf) + leaf->key_len +
275 |                                              leaf->val_len);
276 | #else
277 |         free_node_from_size((uint64_t)(leaf->fkey), leaf->key_len);
278 |         free_node_from_size((uint64_t)(leaf->value), leaf->val_len);
279 |         free_node_from_type((uint64_t)n, n->type);
280 | #endif
281 |     } else {
282 |         // reclaim the node
283 |         free_node_from_type((uint64_t)n, n->type);
284 |     }
285 | }
286 | 
287 | void *alloc_new_node_from_type(PART_ns::NTypes type) {
288 | #ifdef COUNT_ALLOC
289 |     if (dcmm_time == nullptr)
290 |         dcmm_time = new cpuCycleTimer();
291 |     dcmm_time->start();
292 | #endif
293 | 
294 |     size_t node_size = size_align(get_node_size(type), 64);
295 |     void *addr = ti->free_list->alloc_node(node_size);
296 | 
297 | #ifdef COUNT_ALLOC
298 |     dcmm_time->end();
299 | #endif
300 |     return addr;
301 | }
302 | 
303 | void *alloc_new_node_from_size(size_t size) {
304 | #ifdef COUNT_ALLOC
305 |     if (dcmm_time == nullptr)
306 |         dcmm_time = new cpuCycleTimer();
307 |     dcmm_time->start();
308 | #endif
309 | 
310 |     void *addr = ti->free_list->alloc_node(size);
311 | #ifdef COUNT_ALLOC
312 |     dcmm_time->end();
313 | #endif
314 |     return addr;
315 | }
316 | 
317 | void free_node_from_type(uint64_t addr, PART_ns::NTypes type) {
318 |     size_t node_size = size_align(get_node_size(type), 64);
319 |     node_size = ti->free_list->get_power_two_size(node_size);
320 |     ti->free_list->insert_into_freelist(addr, node_size);
321 | }
322 | 
323 | void free_node_from_size(uint64_t addr, size_t size) {
324 |     size_t node_size = ti->free_list->get_power_two_size(size);
325 |     ti->free_list->insert_into_freelist(addr, node_size);
326 | }
327 | 
328 | void register_threadinfo() {
329 | #ifdef INSTANT_RESTART
330 |     NVMMgr *mgr = get_nvm_mgr();
331 |     thread_generation = mgr->get_generation_version();
332 | #endif
333 |     std::lock_guard<std::mutex> lock_guard(ti_lock);
334 | 
335 |     if (pmblock == nullptr) {
336 |         pmblock = new PMBlockAllocator(get_nvm_mgr());
337 |         std::cout << "[THREAD]\tfirst new pmblock\n";
338 |         //        std::cout<<"PPPPP meta data addr "<<
339 |         //        get_nvm_mgr()->meta_data<<"\n";
340 |     }
341 |     if (epoch_mgr == nullptr) {
342 |         epoch_mgr = new Epoch_Mgr();
343 | 
344 |         // need to call function to create a new thread to increase epoch
345 |         epoch_mgr->StartThread();
346 |         std::cout << "[THREAD]\tfirst new epoch_mgr and add global epoch\n";
347 |     }
348 |     if (ti == nullptr) {
349 |         if (tid == NVMMgr::max_threads) {
350 |             std::cout << "[THREAD]\tno available threadinfo to allocate\n";
351 |             assert(0);
352 |         }
353 |         NVMMgr *mgr = get_nvm_mgr();
354 |         //        std::cout<<"in thread get mgr meta data addr
355 |         //        "<<mgr->meta_data<<"\n";
356 | 
357 |         ti = new (mgr->alloc_thread_info()) thread_info();
358 |         ti->next = ti_list_head;
359 |         ti_list_head = ti;
360 | 
361 |         // persist thread info
362 |         flush_data((void *)ti, 128);
363 |         std::cout << "[THREAD]\talloc thread info " << ti->id << "\n";
364 |     }
365 | }
366 | 
367 | void unregister_threadinfo() {
368 |     std::lock_guard<std::mutex> lock_guard(ti_lock);
369 |     thread_info *cti = ti_list_head;
370 |     if (cti == ti) {
371 |         ti_list_head = cti->next;
372 |     } else {
373 |         thread_info *next = cti->next;
374 |         while (true) {
375 |             assert(next);
376 |             if (next == ti) {
377 |                 cti->next = next->next;
378 |                 break;
379 |             }
380 |             cti = next;
381 |             next = next->next;
382 |         }
383 |     }
384 |     std::cout << "[THREAD]\tunregister thread\n";
385 |     //    delete ti;
386 |     ti = nullptr;
387 |     if (ti_list_head == nullptr) {
388 |         // reset all, only use for gtest
389 |         delete epoch_mgr;
390 |         epoch_mgr = nullptr;
391 |         delete pmblock;
392 |         pmblock = nullptr;
393 |         tid = 0;
394 |     }
395 | }
396 | 
397 | void *get_threadinfo() { return (void *)ti; }
398 | 
399 | void JoinNewEpoch() { ti->JoinEpoch(); }
400 | 
401 | void LeaveThisEpoch() { ti->LeaveEpoch(); }
402 | 
403 | void MarkNodeGarbage(void *node) { ti->AddGarbageNode(node); }
404 | 
405 | uint64_t SummarizeGCEpoch() {
406 |     assert(ti_list_head);
407 | 
408 |     // Use the first metadata's epoch as min and update it on the fly
409 |     thread_info *tmp = ti_list_head;
410 |     uint64_t min_epoch = tmp->md->last_active_epoch;
411 | 
412 |     // This might not be executed if there is only one thread
413 |     while (tmp->next) {
414 |         tmp = tmp->next;
415 |         min_epoch = std::min(min_epoch, tmp->md->last_active_epoch);
416 |     }
417 | 
418 |     return min_epoch;
419 | }
420 | 
421 | } // namespace NVMMgr_ns
422 | 


--------------------------------------------------------------------------------
/nvm_mgr/threadinfo.h:
--------------------------------------------------------------------------------
  1 | #ifndef thread_info_h
  2 | #define thread_info_h
  3 | 
  4 | #include "Epoch.h"
  5 | #include "N.h"
  6 | #include "N16.h"
  7 | #include "N256.h"
  8 | #include "N4.h"
  9 | #include "N48.h"
 10 | #include "LeafArray.h"
 11 | #include "pmalloc_wrap.h"
 12 | #include "tbb/concurrent_queue.h"
 13 | #include <list>
 14 | 
 15 | namespace NVMMgr_ns {
 16 | /*
 17 |  * Persistent leaf management
 18 |  */
 19 | 
 20 | /*
 21 |  * fixed length allocator
 22 |  */
 23 | // class PMFreeList {
 24 | //    std::list<uint64_t> free_node_list;
 25 | //    PMBlockAllocator *pmb;
 26 | //
 27 | //  public:
 28 | //    PMFreeList(PMBlockAllocator *pmb_);
 29 | //    ~PMFreeList() {}
 30 | //
 31 | //    void *alloc_node(PART_ns::NTypes nt);
 32 | //    void free_node(void *addr);
 33 | //
 34 | //    int get_freelist_size() { return free_node_list.size(); }
 35 | //} __attribute__((aligned(64)));
 36 | 
 37 | const static int free_list_number = 10; // from 8byte to 4K
 38 | // maintain free memory like buddy system in linux
 39 | class buddy_allocator {
 40 |   private:
 41 |     const size_t power_two[free_list_number] = {8,   16,  32,   64,   128,
 42 |                                                 256, 512, 1024, 2048, 4096};
 43 |     tbb::concurrent_queue<uint64_t> free_list[free_list_number];
 44 |     PMBlockAllocator *pmb;
 45 | 
 46 |   public:
 47 |     buddy_allocator(PMBlockAllocator *pmb_) : pmb(pmb_) {
 48 |         for (int i = 0; i < free_list_number; i++)
 49 |             free_list[i].clear();
 50 |     }
 51 |     ~buddy_allocator() {}
 52 |     void insert_into_freelist(uint64_t addr, size_t size);
 53 |     void *alloc_node(size_t size);
 54 |     uint64_t get_addr(int id);
 55 |     size_t get_power_two_size(size_t s);
 56 |     int get_freelist_size(int id) { return free_list[id].unsafe_size(); }
 57 | };
 58 | 
 59 | class thread_info {
 60 |   public:
 61 |     int id;
 62 |     volatile int _lock;
 63 |     thread_info *next;
 64 | 
 65 |     //    // fixed length free list
 66 |     //    PMFreeList *node4_free_list;
 67 |     //    PMFreeList *node16_free_list;
 68 |     //    PMFreeList *node48_free_list;
 69 |     //    PMFreeList *node256_free_list;
 70 |     //    PMFreeList *leaf_free_list;
 71 | 
 72 |     // variable length free list
 73 |     buddy_allocator *free_list;
 74 | 
 75 |     // epoch based GC metadata
 76 |     GCMetaData *md;
 77 | 
 78 |     char static_log[4032]; // 整个 thread_info的长度为 4096，所以剩下的内存
 79 |                            // 4096-64 = 4032 都可以用来做 static log。
 80 |   public:
 81 |     // interface
 82 |     thread_info();
 83 |     ~thread_info();
 84 | 
 85 |     void *get_static_log() { return (void *)static_log; }
 86 |     int get_thread_id() { return id; }
 87 |     inline void JoinEpoch() {
 88 |         md->last_active_epoch = Epoch_Mgr::GetGlobalEpoch();
 89 |     }
 90 | 
 91 |     inline void LeaveEpoch() {
 92 |         // This will make ie never be counted as active for GC
 93 |         md->last_active_epoch = static_cast<uint64_t>(-1);
 94 |     }
 95 | 
 96 |     /*
 97 |      * AddGarbageNode() - Adds a garbage node into the thread-local GC context
 98 |      *
 99 |      * Since the thread local GC context is only accessed by this thread, this
100 |      * process does not require any atomicity
101 |      *
102 |      * This is always called by the thread owning thread local data, so we
103 |      * do not have to worry about thread identity issues
104 |      */
105 | 
106 |     void AddGarbageNode(void *node_p);
107 | 
108 |     /*
109 |      * PerformGC() - This function performs GC on the current thread's garbage
110 |      *               chain using the call back function
111 |      *
112 |      * Note that this function only collects for the current thread. Therefore
113 |      * this function does not have to be atomic since its
114 |      *
115 |      * Note that this function should be used with thread_id, since it will
116 |      * also be called inside the destructor - so we could not rely on
117 |      * GetCurrentGCMetaData()
118 |      */
119 |     void PerformGC();
120 |     void FreeEpochNode(void *node_p);
121 | 
122 | } __attribute__((aligned(64)));
123 | 
124 | #ifdef COUNT_ALLOC
125 | double getdcmmalloctime();
126 | #endif
127 | 
128 | #ifdef INSTANT_RESTART
129 | void init();
130 | void increase(int id);
131 | uint64_t total(int thread_num);
132 | uint64_t get_threadlocal_generation();
133 | #endif
134 | 
135 | void register_threadinfo();
136 | void unregister_threadinfo();
137 | 
138 | void *get_threadinfo();
139 | 
140 | void JoinNewEpoch();
141 | void LeaveThisEpoch();
142 | void MarkNodeGarbage(void *node);
143 | uint64_t SummarizeGCEpoch();
144 | 
145 | size_t size_align(size_t s, int align);
146 | size_t get_node_size(PART_ns::NTypes type);
147 | size_t convert_power_two(size_t s);
148 | 
149 | void *alloc_new_node_from_type(PART_ns::NTypes type);
150 | void *alloc_new_node_from_size(size_t size);
151 | void free_node_from_type(uint64_t addr, PART_ns::NTypes type);
152 | void free_node_from_size(uint64_t addr, size_t size);
153 | 
154 | } // namespace NVMMgr_ns
155 | #endif
156 | 


--------------------------------------------------------------------------------
/nvm_mgr/util.h:
--------------------------------------------------------------------------------
 1 | #ifndef UTIL_H
 2 | #define UTIL_H
 3 | 
 4 | #include "generator.h"
 5 | #include <stdio.h>
 6 | #include <stdlib.h>
 7 | 
 8 | 
 9 | 
10 | #define MOR __ATOMIC_SEQ_CST
11 | #define ATM_GET(var) (var)
12 | #define ATM_LOAD(var, val) __atomic_load(&(var), &(val), MOR)
13 | #define ATM_STORE(var, val) __atomic_store(&(var), &(val), MOR)
14 | #define ATM_CAS(var, exp, val)                                                 \
15 |     __atomic_compare_exchange(&(var), &(exp), &(val), false, MOR, MOR)
16 | #define ATM_CAS_PTR(var, exp, val)                                             \
17 |     __atomic_compare_exchange_n((var), &(exp), (val), false, MOR, MOR)
18 | #define ATM_FETCH_ADD(var, val) __atomic_fetch_add(&(var), (val), MOR)
19 | #define ATM_FETCH_SUB(var, val) __atomic_fetch_sub(&(var), (val), MOR)
20 | 
21 | /**
22 |  * CPU cycles
23 |  */
24 | 
25 | static __always_inline uint64_t rdtsc() {
26 |     unsigned int lo, hi;
27 |     __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi));
28 |     return ((uint64_t)hi << 32) | lo;
29 | }
30 | 
31 | #define CPU_FREQUENCY 2.2 // 2.2 GHZ
32 | 
33 | #define PM_FENCE()                                                             \
34 |     do {                                                                       \
35 |         if (getenv("NVM_LATENCY")) {                                           \
36 |             long long t1 = rdtsc();                                            \
37 |             long long duration = CPU_FREQUENCY * atoi(getenv("NVM_LATENCY"));  \
38 |             while (rdtsc() - t1 < duration) {                                  \
39 |                 asm("pause");                                                  \
40 |             }                                                                  \
41 |         }                                                                      \
42 |     } while (0)
43 | 
44 | #define asm_clwb(addr)                                                         \
45 |     asm volatile(".byte 0x66; xsaveopt %0" : "+m"(*(volatile char *)addr));
46 | 
47 | #define asm_clflush(addr)                                                      \
48 |     ({ __asm__ __volatile__("clflush %0" : : "m"(*addr)); })
49 | 
50 | // static inline void asm_mfence(void)
51 | #define asm_mfence()                                                           \
52 |     ({                                                                         \
53 |         PM_FENCE();                                                            \
54 |         __asm__ __volatile__("mfence");                                        \
55 |     })
56 | 
57 | // static inline void asm_sfence(void)
58 | #define asm_sfence()                                                           \
59 |     ({                                                                         \
60 |         PM_FENCE();                                                            \
61 |         __asm__ __volatile__("sfence");                                        \
62 |     })
63 | 
64 | #define CACHE_ALIGN 64
65 | 
66 | static void flush_data(void *addr, size_t len) {
67 |     char *end = (char *)(addr) + len;
68 |     char *ptr = (char *)((unsigned long)addr & ~(CACHE_ALIGN - 1));
69 |     for (; ptr < end; ptr += CACHE_ALIGN)
70 |         asm_clwb(ptr);
71 |     asm_mfence();
72 | }
73 | 
74 | // prefetch instruction
75 | //
76 | inline void prefetch(const void *ptr) {
77 | #ifdef NOPREFETCH
78 |     (void)ptr;
79 | #else
80 |     typedef struct {
81 |         char x[CACHE_ALIGN];
82 |     } cacheline_t;
83 |     asm volatile("prefetcht0 %0" : : "m"(*(const cacheline_t *)ptr));
84 | #endif
85 | }
86 | 
87 | /**
88 |  * @brief find first zero bit in a word
89 |  * @details
90 |  *
91 |  * @param long the word to find
92 |  * @return the location of the first zero bit
93 |  */
94 | 
95 | static __always_inline unsigned long ffz(unsigned long word) {
96 |     asm("rep; bsf %1,%0" : "=r"(word) : "r"(~word));
97 |     return word;
98 | }
99 | #endif


--------------------------------------------------------------------------------
/perf/acmma.cpp:
--------------------------------------------------------------------------------
  1 | #include "nvm_mgr.h"
  2 | #include "threadinfo.h"
  3 | #include <boost/thread/barrier.hpp>
  4 | #include <chrono>
  5 | #include <iostream>
  6 | #include <libpmemobj.h>
  7 | #include <stdlib.h>
  8 | #include <string.h>
  9 | #include <unistd.h>
 10 | 
 11 | using namespace std;
 12 | 
 13 | void usage() {
 14 |     cout << "lack of parameters, please input ./acmma [t] [num]\n"
 15 |          << "[t] is the type of test\n"
 16 |          << "0: pmemobj_alloc, 1: dcmm， 2: nvm_malloc\n"
 17 |          << "[num] is the number of worker threads\n";
 18 | }
 19 | 
 20 | POBJ_LAYOUT_BEGIN(allocator);
 21 | POBJ_LAYOUT_ROOT(allocator, struct my_root);
 22 | POBJ_LAYOUT_TOID(allocator, char);
 23 | POBJ_LAYOUT_END(allocator);
 24 | 
 25 | struct my_root {
 26 |     PMEMoid ptr;
 27 | };
 28 | 
 29 | const int max_addr = 1000000;
 30 | uint64_t addr[max_addr + 5];
 31 | 
 32 | const int thread_to_core[36] = {
 33 |         0,  4,  8,  12, 16, 20, 24, 28, 32,
 34 |         36, 40, 44, 48, 52, 56, 60, 64, 68, // numa node 0
 35 |         1,  5,  9,  13, 17, 21, 25, 29, 33,
 36 |         37, 41, 45, 49, 53, 57, 61, 65, 69 // numa node 1
 37 | };
 38 | 
 39 | int stick_this_thread_to_core(int core_id) {
 40 |     int num_cores = sysconf(_SC_NPROCESSORS_ONLN);
 41 |     if (core_id < 0 || core_id >= num_cores)
 42 |         return EINVAL;
 43 | 
 44 |     cpu_set_t cpuset;
 45 |     CPU_ZERO(&cpuset);
 46 |     CPU_SET(thread_to_core[core_id], &cpuset);
 47 | 
 48 |     pthread_t current_thread = pthread_self();
 49 |     return pthread_setaffinity_np(current_thread, sizeof(cpu_set_t),
 50 |                                   &cpuset);
 51 | }
 52 | 
 53 | 
 54 | int main(int argc, char **argv) {
 55 |     if (argc != 3) {
 56 |         usage();
 57 |         return 0;
 58 |     }
 59 |     int type = atoi(argv[1]);
 60 |     int threads_num = atoi(argv[2]);
 61 | 
 62 |     system("rm -rf /mnt/pmem0/matianmao/acmma.data");
 63 |     system("rm -rf /mnt/pmem0/matianmao/part.data");
 64 | 
 65 |     const char *pool_name = "/mnt/pmem0/matianmao/acmma.data";
 66 |     const char *layout_name = "acmma";
 67 |     size_t pool_size = 64ull * 1024 * 1024 * 1024;
 68 | 
 69 |     PMEMobjpool *tmp_pool;
 70 |     if (access(pool_name, 0)) {
 71 |         tmp_pool = pmemobj_create(pool_name, layout_name, pool_size, 0666);
 72 |     } else {
 73 |         tmp_pool = pmemobj_open(pool_name, layout_name);
 74 |     }
 75 | 
 76 |     std::thread *tid[threads_num];
 77 | 
 78 |     int *done = new int;
 79 |     *done = 0;
 80 |     boost::barrier *bar = new boost::barrier(threads_num + 1);
 81 |     uint64_t result[threads_num];
 82 | 
 83 |     int nsize = 64;
 84 |     std::string name = "";
 85 |     if (type == 2) {
 86 |         // TODO: nvm_malloc
 87 |         name = "nvm_malloc";
 88 |     } else if (type == 1) {
 89 |         // acmma
 90 |         name = "DCMM";
 91 |         std::cout << "dcmm\n";
 92 |         NVMMgr_ns::init_nvm_mgr();
 93 |         NVMMgr_ns::NVMMgr *mgr = NVMMgr_ns::get_nvm_mgr();
 94 |         for (int i = 0; i < threads_num; i++) {
 95 |             result[i] = 0;
 96 |             tid[i] = new std::thread(
 97 |                 [&](int id) {
 98 |                     stick_this_thread_to_core(id);
 99 |                     NVMMgr_ns::register_threadinfo();
100 | //                    int s = 0;
101 |                     int tx = 0;
102 |                     bar->wait();
103 |                     while ((*done) == 0) {
104 |                         addr[result[id] % max_addr] =
105 |                             (uint64_t)NVMMgr_ns::alloc_new_node_from_size(
106 |                                 nsize);
107 |                         tx++;
108 | //                        s++;
109 | //                        if (s >= 10)
110 | //                            s = 0;
111 |                     }
112 |                     result[id] = tx;
113 |                 },
114 |                 i);
115 |         }
116 |     } else if (type == 0) {
117 |         name = "PMDK";
118 |         std::cout << "pmalloc\n";
119 |         for (int i = 0; i < threads_num; i++) {
120 |             result[i] = 0;
121 |             tid[i] = new std::thread(
122 |                 [&](int id) {
123 |                     stick_this_thread_to_core(id);
124 | //                    int s = 0;
125 |                     int tx = 0;
126 |                     bar->wait();
127 |                     PMEMoid ptr;
128 |                     while ((*done) == 0) {
129 |                         //                    std::cout<<"alloc\n";
130 |                         pmemobj_zalloc(tmp_pool, &ptr, nsize,
131 |                                        TOID_TYPE_NUM(char));
132 |                         void *addr = (void *)pmemobj_direct(ptr);
133 |                         tx++;
134 | //                        s++;
135 | //                        if (s >= 10)
136 | //                            s = 0;
137 |                     }
138 |                     result[id] = tx;
139 |                 },
140 |                 i);
141 |         }
142 |     }
143 | 
144 |     bar->wait();
145 |     std::this_thread::sleep_for(std::chrono::seconds(1));
146 |     *done = 1;
147 |     uint64_t res = 0;
148 |     for (int i = 0; i < threads_num; i++) {
149 |         tid[i]->join();
150 |         res += result[i];
151 |     }
152 | 
153 |     std::cout << name<< ", threads: "<< threads_num <<", throughput " << res/1000000.0 << " Mtps\n";
154 |     return 0;
155 | }


--------------------------------------------------------------------------------
/perf/art_simple_bench.cpp:
--------------------------------------------------------------------------------
 1 | #include "tbb/tbb.h"
 2 | #include <chrono>
 3 | #include <iostream>
 4 | #include <random>
 5 | #include <thread>
 6 | 
 7 | using namespace std;
 8 | 
 9 | #include "Tree.h"
10 | 
11 | using namespace PART_ns;
12 | 
13 | inline void clear_data() { system("rm -rf /mnt/pmem0/matianmao/part.data"); }
14 | void run(char **argv) {
15 |     clear_data();
16 |     std::cout << "Simple Example of P-ART" << std::endl;
17 | 
18 |     uint64_t n = std::atoll(argv[1]);
19 |     uint64_t *keys = new uint64_t[n];
20 |     std::vector<Key *> Keys;
21 | 
22 |     Keys.reserve(n);
23 |     Tree *tree = new Tree();
24 |     // Generate keys
25 |     for (uint64_t i = 0; i < n; i++) {
26 |         keys[i] = i;
27 |         // Keys[i] = Keys[i]->make_leaf(i, sizeof(uint64_t), i);
28 |         Keys[i] = new Key(keys[i], sizeof(uint64_t), keys[i]);
29 |         tree->insert(Keys[i]);
30 |     }
31 | 
32 |     const int num_thread = atoi(argv[2]);
33 |     tbb::task_scheduler_init init(num_thread);
34 |     // std::thread *tid[num_thread];
35 |     // int every_thread_num = n / num_thread + 1;
36 | 
37 |     // simple benchmark, not precise
38 |     printf("operation,n,ops/s\n");
39 |     {
40 |         // Build tree
41 |         auto starttime = std::chrono::system_clock::now();
42 |         tbb::parallel_for(
43 |             tbb::blocked_range<uint64_t>(0, n),
44 |             [&](const tbb::blocked_range<uint64_t> &range) {
45 |                 for (uint64_t i = range.begin(); i != range.end(); i++) {
46 |                     Keys[i]->Init(2 * keys[i], sizeof(uint64_t), keys[i]);
47 |                     tree->insert(Keys[i]);
48 |                 }
49 |             });
50 | 
51 |         auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
52 |             std::chrono::system_clock::now() - starttime);
53 |         printf("Throughput: insert,%ld,%f ops/us\n", n,
54 |                (n * 1.0) / duration.count());
55 |         printf("Elapsed time: insert,%ld,%f sec\n", n,
56 |                duration.count() / 1000000.0);
57 |     }
58 | 
59 |     {
60 |         // Lookup
61 |         auto starttime = std::chrono::system_clock::now();
62 |         tbb::parallel_for(tbb::blocked_range<uint64_t>(0, n),
63 |                           [&](const tbb::blocked_range<uint64_t> &range) {
64 |                               for (uint64_t i = range.begin(); i != range.end();
65 |                                    i++) {
66 |                                   //   Keys[i] = Keys[i]->make_leaf(i,
67 |                                   //   sizeof(i), i);
68 |                                   // Keys[i]->Init(keys[i], sizeof(uint64_t),
69 |                                   // keys[i]);
70 |                                   tree->lookup(Keys[i]);
71 |                               }
72 |                           });
73 |         auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
74 |             std::chrono::system_clock::now() - starttime);
75 |         printf("Throughput: lookup,%ld,%f ops/us\n", n,
76 |                (n * 1.0) / duration.count());
77 |         printf("Elapsed time: lookup,%ld,%f sec\n", n,
78 |                duration.count() / 1000000.0);
79 |     }
80 | 
81 |     delete[] keys;
82 | }
83 | 
84 | int main(int argc, char **argv) {
85 |     if (argc != 3) {
86 |         printf(
87 |             "usage: %s [n] [nthreads]\nn: number of keys (integer)\nnthreads: "
88 |             "number of threads (integer)\n",
89 |             argv[0]);
90 |         return 1;
91 |     }
92 | 
93 |     run(argv);
94 |     return 0;
95 | }


--------------------------------------------------------------------------------
/perf/main.cpp:
--------------------------------------------------------------------------------
 1 | #include "config.h"
 2 | #include "coordinator.h"
 3 | #include <fstream>
 4 | using namespace std;
 5 | 
 6 | inline void clear_data() {
 7 |     system((std::string("rm -rf ") + nvm_dir + "part.data").c_str());
 8 |     system((std::string("rm -rf ") + nvm_dir + "fast_fair.data").c_str());
 9 |     system((std::string("rm -rf ") + nvm_dir + "skiplist.data").c_str());
10 | }
11 | 
12 | int main(int argc, char **argv) {
13 |     clear_data();
14 |     Config conf;
15 |     parse_arguments(argc, argv, conf);
16 | 
17 |     Coordinator<long long, long long, 64> coordinator(conf);
18 |     coordinator.run();
19 | }
20 | 


--------------------------------------------------------------------------------
/perf/malloc_diff.cpp:
--------------------------------------------------------------------------------
  1 | #include "util.h"
  2 | #include <chrono>
  3 | #include <iostream>
  4 | #include <libpmemobj.h>
  5 | #include <stdlib.h>
  6 | #include <string.h>
  7 | #include <unistd.h>
  8 | 
  9 | using namespace std;
 10 | 
 11 | POBJ_LAYOUT_BEGIN(allocator);
 12 | POBJ_LAYOUT_ROOT(allocator, struct my_root);
 13 | POBJ_LAYOUT_TOID(allocator, char);
 14 | POBJ_LAYOUT_END(allocator);
 15 | 
 16 | struct my_root {
 17 |     PMEMoid ptr;
 18 | };
 19 | uint64_t addr[1000005];
 20 | 
 21 | // test malloc, libvmmalloc, pmemobj_alloc, pmdk transactional allocator
 22 | void usage() {
 23 |     cout << "lack of parameters, please input ./malloc_diff [t] [size]\n"
 24 |          << "[t] is the type of test\n"
 25 |          << "0: new/libvmmalloc, 1: pmemobj_alloc, 2: tx+pmemobj_alloc, 3: "
 26 |             "pmdk transactional allocator\n"
 27 |          << "[size] is the size of allocated block\n";
 28 | }
 29 | 
 30 | int main(int argc, char **argv) {
 31 |     cout << "[ALLOC]test different allocate\n";
 32 | 
 33 |     if (argc < 3) {
 34 |         usage();
 35 |     }
 36 |     int type = atoi(argv[1]);
 37 |     size_t nsize = atoi(argv[2]);
 38 |     int test_iter = 1000000;
 39 | 
 40 |     system("rm -rf /mnt/pmem0/matianmao/test_alloc.data");
 41 |     const char *pool_name = "/mnt/pmem0/matianmao/test_alloc.data";
 42 |     const char *layout_name = "pmdk-alloc";
 43 |     size_t pool_size = 1024 * 1024 * 256;
 44 | 
 45 |     PMEMobjpool *tmp_pool;
 46 |     if (access(pool_name, 0)) {
 47 |         tmp_pool = pmemobj_create(pool_name, layout_name, pool_size, 0666);
 48 |     } else {
 49 |         tmp_pool = pmemobj_open(pool_name, layout_name);
 50 |     }
 51 | 
 52 |     char value[nsize + 5];
 53 |     memset(value, 'a', nsize);
 54 |     auto starttime = chrono::system_clock::now();
 55 |     if (type == 0) {
 56 |         cout << "test new/libvmmalloc\n";
 57 |         for (int i = 0; i < test_iter; i++) {
 58 |             addr[i] = (uint64_t)malloc(nsize);
 59 |             //            memcpy((char *)addr[i], value, nsize);
 60 |             //            flush_data((void*)addr[i], nsize);
 61 |         }
 62 |     } else {
 63 |         if (type == 1) {
 64 |             cout << "test pmemobj_alloc\n";
 65 |             TOID(struct my_root) root = POBJ_ROOT(tmp_pool, struct my_root);
 66 |             for (int i = 0; i < test_iter; i++) {
 67 |                 int ret =
 68 |                     pmemobj_zalloc(tmp_pool, &D_RW(root)->ptr,
 69 |                                    sizeof(char) * nsize, TOID_TYPE_NUM(char));
 70 |                 if (ret) {
 71 |                     cout << "pmemobj_zalloc error\n";
 72 |                     return 1;
 73 |                 }
 74 |                 char *ad = (char *)pmemobj_direct(D_RW(root)->ptr);
 75 |                 memcpy(ad, value, nsize);
 76 |                 flush_data(ad, nsize);
 77 |             }
 78 |         } else if (type == 2) {
 79 |             cout << "test tx+pmemobj_alloc\n";
 80 |             PMEMoid ptr;
 81 |             for (int i = 0; i < test_iter; i++) {
 82 |                 TX_BEGIN(tmp_pool) {
 83 |                     ptr = pmemobj_tx_zalloc(sizeof(char) * nsize,
 84 |                                             TOID_TYPE_NUM(char));
 85 |                 }
 86 |                 TX_END
 87 |                 char *ad = (char *)pmemobj_direct(ptr);
 88 |                 memcpy(ad, value, nsize);
 89 |                 flush_data(ad, nsize);
 90 |             }
 91 |         } else {
 92 |             cout << "test pmdk transactional allocator\n";
 93 |             TOID(struct my_root) root = POBJ_ROOT(tmp_pool, struct my_root);
 94 |             for (int i = 0; i < 1000000; i++) {
 95 |                 TX_BEGIN(tmp_pool) {
 96 |                     TX_ADD(root);
 97 |                     D_RW(root)->ptr = pmemobj_tx_zalloc(sizeof(char) * nsize,
 98 |                                                         TOID_TYPE_NUM(char));
 99 |                 }
100 |                 TX_END
101 |                 char *ad = (char *)pmemobj_direct(D_RW(root)->ptr);
102 |                 memcpy(ad, value, nsize);
103 |                 flush_data(ad, nsize);
104 |             }
105 |         }
106 |     }
107 |     auto end = chrono::system_clock::now();
108 |     auto duration =
109 |         chrono::duration_cast<std::chrono::microseconds>(end - starttime);
110 | 
111 |     cout << "type is: " << type << " size is: " << nsize << "duration time "
112 |          << duration.count() << "us\n";
113 |     return 0;
114 | }


--------------------------------------------------------------------------------
/script/loop.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | for((i=0;i<10000;i++))
4 | do
5 |     echo $i
6 |     ../build/benchmark -t 1 -K 1 -b 3 -n 24 >> a.log
7 | done


--------------------------------------------------------------------------------
/script/test.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | #LD_PRELOAD="/usr/local/lib/libvmmalloc.so.1" ../build/benchmark -t ${type} -K 1 -b ${bench} -n ${tnum}
 4 | 
 5 | echo "test microbench"
 6 | bash test_microbench.sh
 7 | 
 8 | echo "test skewness"
 9 | bash test_skew.sh
10 | 
11 | echo "test rr scalability"
12 | bash test_scalability.sh
13 | 
14 | echo "test ycsb"
15 | bash test_ycsb.sh


--------------------------------------------------------------------------------
/test/test.cpp:
--------------------------------------------------------------------------------
1 | #include <gtest/gtest.h>
2 | 
3 | int main(int argc, char **argv) {
4 |     testing::InitGoogleTest(&argc, argv);
5 |     // Runs all tests using Google Test.
6 |     return RUN_ALL_TESTS();
7 | }


--------------------------------------------------------------------------------
/test/test_basic.cpp:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by 谢威宇 on 2021/4/1.
  3 | //
  4 | #include "N.h"
  5 | #include "Tree.h"
  6 | 
  7 | #include <atomic>
  8 | #include <gtest/gtest.h>
  9 | #include <iostream>
 10 | #include <string>
 11 | inline void clear_data() {
 12 |     system((std::string("rm -rf ") + nvm_dir + "part.data").c_str());
 13 | }
 14 | 
 15 | TEST(BasicTest, test_z) {
 16 |     std::cout << static_cast<int>('\0') << std::endl;
 17 |     std::cout << sizeof(size_t) << std::endl;
 18 |     std::cout << "Size of Leaf: " << sizeof(PART_ns::Leaf) << std::endl;
 19 |     std::cout << "Size of N: " << sizeof(PART_ns::N) << std::endl;
 20 |     std::cout << "Size of N4: " << sizeof(PART_ns::N4) << std::endl;
 21 |     std::cout << "Size of N16: " << sizeof(PART_ns::N16) << std::endl;
 22 |     std::cout << "Size of N48: " << sizeof(PART_ns::N48) << std::endl;
 23 |     std::cout << "Size of N256: " << sizeof(PART_ns::N256) << std::endl;
 24 |     std::cout << "Size of std::atomic<uint64_t>: "
 25 |               << sizeof(std::atomic<uint64_t>) << std::endl;
 26 |     std::cout << "Size of std::atomic<std::bitset<64>: "
 27 |               << sizeof(std::atomic<std::bitset<64>>) << std::endl;
 28 |     std::cout
 29 |         << "Size of "
 30 |            "std::pair<std::atomic<uint16_t>,std::atomic<std::bitset<48>>>: "
 31 |         << sizeof(
 32 |                std::pair<std::atomic<uint16_t>, std::atomic<std::bitset<48>>>)
 33 |         << std::endl;
 34 | 
 35 |     std::cout << "Int value of nullptr: "
 36 |               << reinterpret_cast<uintptr_t>(nullptr) << std::endl;
 37 | 
 38 |     int a = 432;
 39 |     auto pa = &a;
 40 |     std::cout << "The address of a(pa): " << std::hex
 41 |               << reinterpret_cast<uintptr_t>(pa)
 42 |               << ", deref it and get: " << std::dec << *pa << std::endl;
 43 |     pa = reinterpret_cast<int *>(reinterpret_cast<uintptr_t>(pa) ^
 44 |                                  (0xffffLL << 48));
 45 |     std::cout << "The modified address of a(pa): " << std::hex
 46 |               << reinterpret_cast<uintptr_t>(pa) << std::endl;
 47 | }
 48 | 
 49 | TEST(TreeTest, graph_viz) {
 50 |     clear_data();
 51 |     auto art = new PART_ns::Tree();
 52 |     auto *k = new PART_ns::Key();
 53 |     std::vector<std::string> keys = {
 54 |         "1121",
 55 |         "1131",
 56 |         "1132",
 57 |         "1142",
 58 |         "1141",
 59 |         "1122",
 60 |         "1123",
 61 |         "1124",
 62 |         "121",
 63 |     };
 64 |     for (auto &s : keys) {
 65 |         k->Init(const_cast<char *>(s.c_str()), s.length(), "123", 3);
 66 |         art->insert(k);
 67 |     }
 68 | 
 69 |     art->graphviz_debug();
 70 | 
 71 |     for (auto &s : keys) {
 72 |         k->Init(const_cast<char *>(s.c_str()), s.length(), "123", 3);
 73 |         auto re = art->lookup(k);
 74 |         std::cout << std::string(re->GetKey()) << std::endl;
 75 |     }
 76 |     std::cout << "finish3 "<<PART_ns::LeafArrayLength << std::endl;
 77 | }
 78 | 
 79 | TEST(TreeTest, test_insert_and_lookup) {
 80 |     clear_data();
 81 |     auto art = new PART_ns::Tree();
 82 |     auto *k = new PART_ns::Key();
 83 |     std::vector<std::string> keys = {"111", "123", "211"};
 84 | 
 85 |     for (auto &s : keys) {
 86 |         k->Init(const_cast<char *>(s.c_str()), s.length(),
 87 |                 const_cast<char *>(s.c_str()), s.length());
 88 |         std::cout << s << std::endl;
 89 | 
 90 |         art->insert(k);
 91 |     }
 92 |     std::cout << "insertok" << std::endl;
 93 |     art->graphviz_debug();
 94 |     for (auto &s : keys) {
 95 |         k->Init(const_cast<char *>(s.c_str()), s.length(),
 96 |                 const_cast<char *>(s.c_str()), s.length());
 97 |         auto l = art->lookup(k);
 98 | 
 99 |         std::cout << std::string(l->GetKey()) << std::endl;
100 |         ASSERT_TRUE(l->checkKey(k));
101 |     }
102 | 
103 |     std::cout << "finish2 " << std::endl;
104 | }
105 | 


--------------------------------------------------------------------------------
/test/test_epoch.cpp:
--------------------------------------------------------------------------------
  1 | #include <gtest/gtest.h>
  2 | #include <iostream>
  3 | #include <stdio.h>
  4 | #include <stdlib.h>
  5 | #include <thread>
  6 | #include <vector>
  7 | 
  8 | #include "N.h"
  9 | #include "Tree.h"
 10 | #include "nvm_mgr.h"
 11 | #include "pmalloc_wrap.h"
 12 | #include "threadinfo.h"
 13 | 
 14 | using namespace NVMMgr_ns;
 15 | 
 16 | 
 17 | 
 18 | inline void clear_data() { system("rm -rf /mnt/pmem0/matianmao/part.data"); }
 19 | 
 20 | 
 21 | 
 22 | TEST(TestEpoch, Epoch_Mgr) {
 23 |     clear_data();
 24 |     std::cout << "[TEST]\ttest epoch_mgr\n";
 25 |     uint64_t e = Epoch_Mgr::GetGlobalEpoch();
 26 |     Epoch_Mgr *epoch_mgr = new Epoch_Mgr();
 27 |     ASSERT_EQ(Epoch_Mgr::GetGlobalEpoch(), e);
 28 |     epoch_mgr->IncreaseEpoch();
 29 |     epoch_mgr->IncreaseEpoch();
 30 |     epoch_mgr->IncreaseEpoch();
 31 |     ASSERT_EQ(Epoch_Mgr::GetGlobalEpoch(), e + 3);
 32 | 
 33 |     epoch_mgr->StartThread();
 34 |     e = Epoch_Mgr::GetGlobalEpoch();
 35 | 
 36 |     //    for (int i = 0; i < 2; i++) {
 37 |     //        uint64_t curr_e = Epoch_Mgr::GetGlobalEpoch();
 38 |     //        ASSERT_EQ(e, curr_e) << "not equal, e: " << e << ", curr_e: " <<
 39 |     //        curr_e;
 40 |     //
 41 |     //        std::chrono::milliseconds duration1(GC_INTERVAL);
 42 |     //        std::this_thread::sleep_for(duration1);
 43 |     //        e++;
 44 |     //    }
 45 |     //    std::cout << "[TEST]\ttest epoch increase successfully\n";
 46 | 
 47 |     delete epoch_mgr;
 48 |     std::chrono::milliseconds duration2(GC_INTERVAL);
 49 |     std::this_thread::sleep_for(duration2);
 50 | 
 51 |     e = Epoch_Mgr::GetGlobalEpoch();
 52 |     std::cout << "[TEST]\tdelete epoch_mgr and now epoch is " << e << "\n";
 53 | 
 54 |     std::chrono::milliseconds duration3(GC_INTERVAL * 10);
 55 |     std::this_thread::sleep_for(duration3);
 56 |     ASSERT_EQ(e, Epoch_Mgr::GetGlobalEpoch());
 57 |     std::cout << "[TEST]\ttest exit global epoch thread successfully\n";
 58 | 
 59 |     init_nvm_mgr();
 60 |     register_threadinfo();
 61 |     std::chrono::milliseconds duration4(GC_INTERVAL * 5);
 62 |     std::this_thread::sleep_for(duration4);
 63 |     std::cout << "[TEST]\trestart epoch mgr again, now epoch is "
 64 |               << Epoch_Mgr::GetGlobalEpoch() << "\n";
 65 |     ASSERT_GT(Epoch_Mgr::GetGlobalEpoch(), e);
 66 | 
 67 |     unregister_threadinfo();
 68 |     std::chrono::milliseconds duration1(GC_INTERVAL);
 69 |     std::this_thread::sleep_for(duration1);
 70 |     close_nvm_mgr();
 71 | }
 72 | 
 73 | TEST(TestEpoch, epoch_based_gc) {
 74 |     clear_data();
 75 |     std::cout << "[TEST]\ttest epoch_based_gc\n";
 76 | 
 77 |     init_nvm_mgr();
 78 |     register_threadinfo();
 79 | 
 80 |     // initialize different types node
 81 |     uint8_t *prefix = new uint8_t[4];
 82 |     uint32_t level = 0;
 83 |     uint32_t prefixlen = 4;
 84 |     memcpy(prefix, "abc", 3);
 85 | 
 86 |     PART_ns::N4 *n4 = new (alloc_new_node_from_type(PART_ns::NTypes::N4))
 87 |         PART_ns::N4(level, prefix, prefixlen);
 88 |     PART_ns::N16 *n16 = new (alloc_new_node_from_type(PART_ns::NTypes::N16))
 89 |         PART_ns::N16(level, prefix, prefixlen);
 90 |     PART_ns::N48 *n48 = new (alloc_new_node_from_type(PART_ns::NTypes::N48))
 91 |         PART_ns::N48(level, prefix, prefixlen);
 92 |     PART_ns::N256 *n256 = new (alloc_new_node_from_type(PART_ns::NTypes::N256))
 93 |         PART_ns::N256(level, prefix, prefixlen);
 94 |     PART_ns::Leaf *leaf =
 95 |         new (alloc_new_node_from_type(PART_ns::NTypes::Leaf)) PART_ns::Leaf();
 96 | 
 97 |     //    PART_ns::BaseNode *ll = (PART_ns::BaseNode *)leaf;
 98 |     //
 99 |     //    PART_ns::BaseNode *aa = (PART_ns::BaseNode *)((void *)leaf);
100 |     //
101 |     //    PART_ns::BaseNode *bb = reinterpret_cast<PART_ns::BaseNode *>(leaf);
102 |     //
103 |     //    printf("virtual different type %d %d %d %d\n", (int)(leaf->type),
104 |     //           (int)(ll->type), (int)(aa->type), (int)(bb->type));
105 | 
106 |     std::cout << "[TEST]\talloc different nodes\n";
107 | 
108 |     NVMMgr *mgr = get_nvm_mgr();
109 |     thread_info *ti = (thread_info *)get_threadinfo();
110 |     ASSERT_EQ(mgr->meta_data->free_bit_offset, 1);
111 |     printf("[TEST]\taddr: %x %x %x %x %x\n", n4, n16, n48, n256, leaf);
112 |     int num[free_list_number];
113 |     for (int i = 0; i < free_list_number; i++) {
114 |         num[i] = ti->free_list->get_freelist_size(i);
115 |         printf("[TEST]\t2^%d byte freelist size is %d\n", i, num[i]);
116 |     }
117 | 
118 |     std::cout << "[TEST]\tcheck every node's address successfully\n";
119 | 
120 |     MarkNodeGarbage(n4); // delete at an active epoch
121 |     MarkNodeGarbage(n16);
122 |     MarkNodeGarbage(n48);
123 |     MarkNodeGarbage(n256);
124 |     MarkNodeGarbage(leaf);
125 | 
126 |     // GC for different types nodes
127 |     ti->PerformGC();
128 |     for (int i = 0; i < free_list_number; i++) {
129 |         num[i] = ti->free_list->get_freelist_size(i);
130 |         printf("[TEST]\t2^%d byte freelist size is %d\n", i, num[i]);
131 |     }
132 |     std::cout << "[TEST]\tperform GC finish\n";
133 | 
134 |     unregister_threadinfo();
135 |     std::chrono::milliseconds duration1(GC_INTERVAL);
136 |     std::this_thread::sleep_for(duration1);
137 |     close_nvm_mgr();
138 | }
139 | 


--------------------------------------------------------------------------------
/test/test_leafarray.cpp:
--------------------------------------------------------------------------------
 1 | //
 2 | // Created by 谢威宇 on 2021/4/1.
 3 | //
 4 | #include "N.h"
 5 | #include "Tree.h"
 6 | 
 7 | #include <atomic>
 8 | #include <gtest/gtest.h>
 9 | #include <iostream>
10 | #include <string>
11 | inline void clear_data() {
12 |     system((std::string("rm -rf ") + nvm_dir + "part.data").c_str());
13 | }
14 | 
15 | TEST(LeafArrayTest, bitmap_test) {
16 |     PART_ns::LeafArray *n = new PART_ns::LeafArray(1);
17 |     for (int i = 0; i < PART_ns::LeafArrayLength; i++) {
18 |         auto x = n->getRightmostSetBit();
19 |         //        std::cout << x << std::endl;
20 |         ASSERT_EQ(i, x);
21 |         n->setBit(x, false);
22 |     }
23 |     n->leaf[3].store(0);
24 |     //    std::cout << n->getFingerPrint(3) << std::endl;
25 |     ASSERT_EQ(n->getFingerPrint(3), (1 << 16) - 1);
26 | 
27 |     std::bitset<64> b = 11;
28 |     //    std::cout << b << std::endl;
29 |     auto i = b[0] ? 0 : 1;
30 |     while (i < 64) {
31 |         //        std::cout << i << std::endl;
32 |         i = b._Find_next(i);
33 |     }
34 | }
35 | 
36 | TEST(LeafArrayTest, finger_print_test) { PART_ns::Key k(); }


--------------------------------------------------------------------------------
/test/test_nvm_mgr.cpp:
--------------------------------------------------------------------------------
  1 | #include <gtest/gtest.h>
  2 | #include <iostream>
  3 | #include <stdio.h>
  4 | #include <stdlib.h>
  5 | #include <thread>
  6 | #include <vector>
  7 | 
  8 | #include "N.h"
  9 | #include "nvm_mgr.h"
 10 | #include "pmalloc_wrap.h"
 11 | #include "threadinfo.h"
 12 | 
 13 | using namespace NVMMgr_ns;
 14 | 
 15 | inline void clear_data() { system("rm -rf /mnt/pmem0/matianmao/part.data"); }
 16 | 
 17 | TEST(TestNVMMgr, nvm_mgr) {
 18 |     std::cout << "[TEST]\tstart to test nvm_mgr\n";
 19 |     clear_data();
 20 | 
 21 |     NVMMgr *mgr = new NVMMgr();
 22 | 
 23 |     ASSERT_EQ(mgr->meta_data->free_bit_offset, 0);
 24 | 
 25 |     std::cout << "[TEST]\talloc block successfully\n";
 26 | 
 27 |     uint64_t start = NVMMgr::thread_local_start;
 28 |     int thread_num = 4;
 29 |     for (int i = 0; i < thread_num; i++) {
 30 |         void *addr = mgr->alloc_thread_info();
 31 |         ASSERT_EQ((size_t)addr, start + i * NVMMgr::PGSIZE);
 32 |     }
 33 | 
 34 |     std::cout << "[TEST]\talloc thread info successfully\n";
 35 | 
 36 |     // check the meta_data
 37 |     NVMMgr::Head *head =
 38 |         reinterpret_cast<NVMMgr::Head *>(mgr->alloc_tree_root());
 39 |     ASSERT_EQ(head->threads, thread_num);
 40 |     std::cout << "[TEST]\tcheck meta data successfully\n";
 41 | 
 42 |     delete mgr;
 43 | }
 44 | 
 45 | TEST(TestNVMMgr, PMBlockAllocator) {
 46 |     std::cout << "[TEST]\tstart to test PMBlockAllocator\n";
 47 |     clear_data();
 48 | 
 49 |     init_nvm_mgr();
 50 |     PMBlockAllocator *pmb = new PMBlockAllocator(get_nvm_mgr());
 51 |     register_threadinfo();
 52 | 
 53 |     int bl_num = 10;
 54 |     int tid = 2;
 55 |     uint64_t start = NVMMgr::data_block_start;
 56 |     NVMMgr *mgr = get_nvm_mgr();
 57 |     for (int i = 0; i < bl_num; i++) {
 58 |         // alloc some blocks using interface of pmblockalloctor
 59 |         void *addr = pmb->alloc_block(tid);
 60 |         ASSERT_EQ((uint64_t)addr, start + i * NVMMgr::PGSIZE);
 61 |         ASSERT_EQ(mgr->meta_data->free_bit_offset, i + 1);
 62 |         ASSERT_EQ(mgr->meta_data->bitmap[i], tid);
 63 |     }
 64 |     std::cout << "[TEST]\tpmb alloc block successfully\n";
 65 | 
 66 |     unregister_threadinfo();
 67 |     delete pmb;
 68 |     close_nvm_mgr();
 69 | }
 70 | 
 71 | // TEST(TestNVMMgr, PMFreeList) {
 72 | //    std::cout << "[TEST]\tstart to test PMFreeList\n";
 73 | //    clear_data();
 74 | //
 75 | //    init_nvm_mgr();
 76 | //    PMBlockAllocator *pmb = new PMBlockAllocator(get_nvm_mgr());
 77 | //    PMFreeList *pf = new PMFreeList(pmb);
 78 | //
 79 | //    register_threadinfo();
 80 | //
 81 | //    int node_num = 10;
 82 | //    PART_ns::NTypes type = PART_ns::NTypes::N4;
 83 | //    size_t node_size = sizeof(PART_ns::N4);
 84 | //    uint64_t start = NVMMgr::data_block_start;
 85 | //    for (int i = 0; i < node_num; i++) {
 86 | //        // alloc some node from a block using interface of pmfreelist
 87 | //        void *addr = pf->alloc_node(type);
 88 | //        ASSERT_EQ((uint64_t)addr, start + i * node_size);
 89 | //    }
 90 | //    std::cout << "[TEST]\tpf alloc node successfully\n";
 91 | //    std::cout << "[TEST]\tnode size is " << node_size << ", freelist size is "
 92 | //              << pf->get_freelist_size() << "\n";
 93 | //    ASSERT_EQ(pf->get_freelist_size(), NVMMgr::PGSIZE / node_size - node_num);
 94 | //
 95 | //    unregister_threadinfo();
 96 | //    delete pf;
 97 | //    delete pmb;
 98 | //    close_nvm_mgr();
 99 | //}
100 | 
101 | TEST(TestNVMMgr, buddy_allocator) {
102 |     std::cout << "[TEST]\tstart to test buddy allocator\n";
103 |     clear_data();
104 | 
105 |     init_nvm_mgr();
106 |     NVMMgr *mgr = get_nvm_mgr();
107 |     PMBlockAllocator *pmb = new PMBlockAllocator(get_nvm_mgr());
108 |     buddy_allocator *ba = new buddy_allocator(pmb);
109 |     register_threadinfo();
110 |     thread_info *ti = (thread_info *)get_threadinfo();
111 | 
112 |     void *addr = ba->alloc_node(4096);
113 |     uint64_t start = NVMMgr::data_block_start;
114 |     ASSERT_EQ(ba->get_freelist_size(free_list_number - 1),
115 |               (NVMMgr::PGSIZE / 4096) - 1);
116 |     ASSERT_EQ((uint64_t)addr, start);
117 |     ASSERT_EQ(mgr->meta_data->free_bit_offset, 1);
118 |     ASSERT_EQ(mgr->meta_data->bitmap[0], ti->id);
119 | 
120 |     std::cout << "[TEST]\tstart to test reclaim\n";
121 |     ba->insert_into_freelist(0, 64 + 32);
122 |     ASSERT_EQ(ba->get_freelist_size(2), 1);
123 |     ASSERT_EQ(ba->get_freelist_size(3), 1);
124 |     ba->insert_into_freelist(0, 1024 + 256 + 128 + 32 + 8);
125 |     ASSERT_EQ(ba->get_freelist_size(7), 1);
126 |     ASSERT_EQ(ba->get_freelist_size(5), 1);
127 |     ASSERT_EQ(ba->get_freelist_size(4), 1);
128 |     ASSERT_EQ(ba->get_freelist_size(2), 2);
129 |     ASSERT_EQ(ba->get_freelist_size(0), 1);
130 |     ba->insert_into_freelist(0, 256 + 256 + 64 + 16);
131 |     ASSERT_EQ(ba->get_freelist_size(6), 1);
132 |     ASSERT_EQ(ba->get_freelist_size(3), 2);
133 |     ASSERT_EQ(ba->get_freelist_size(1), 1);
134 |     std::cout << "[TEST]\ttest reclaim successfully\n";
135 | 
136 |     unregister_threadinfo();
137 |     delete ba;
138 |     delete pmb;
139 |     close_nvm_mgr();
140 | }
141 | 
142 | TEST(TestNVMMgr, thread_info) {
143 |     std::cout << "[TEST]\tstart to test thread_info\n";
144 |     clear_data();
145 | 
146 |     // initialize a global nvm_mgr
147 |     init_nvm_mgr();
148 | 
149 |     // initialize a thread and global pmblockallocator
150 |     register_threadinfo();
151 |     NVMMgr *mgr = get_nvm_mgr();
152 |     thread_info *ti = (thread_info *)get_threadinfo();
153 | 
154 |     //    ASSERT_EQ(get_thread_id(), 0);
155 |     void *n4 = alloc_new_node_from_type(PART_ns::NTypes::N4);
156 |     void *n16 = alloc_new_node_from_type(PART_ns::NTypes::N16);
157 |     void *n48 = alloc_new_node_from_type(PART_ns::NTypes::N48);
158 |     void *n256 = alloc_new_node_from_type(PART_ns::NTypes::N256);
159 |     void *leaf = alloc_new_node_from_type(PART_ns::NTypes::Leaf);
160 | 
161 |     std::cout << "[TEST]\talloc different nodes\n";
162 | 
163 |     uint64_t start = NVMMgr::data_block_start;
164 | 
165 |     for (int i = 0; i < free_list_number; i++) {
166 |         std::cout << ti->free_list->get_freelist_size(i) << "\n";
167 |     }
168 | 
169 |     ASSERT_EQ(ti->free_list->get_freelist_size(free_list_number - 1),
170 |               (NVMMgr::PGSIZE / 4096) - 2);
171 |     ASSERT_EQ((uint64_t)n4, start);
172 |     //    std::cout<<"n4 addr "<<(uint64_t)n4<<"\n";
173 |     //    std::cout<<"meta data addr "<< (uint64_t)(mgr->meta_data)<<"\n";
174 |     //    std::cout<<"mgr addr" <<(uint64_t)mgr<<"\n";
175 |     ASSERT_EQ(mgr->meta_data->free_bit_offset, 1);
176 |     ASSERT_EQ(mgr->meta_data->bitmap[0], ti->id);
177 | 
178 |     int currnum = 0;
179 |     for (int i = 0; i < free_list_number; i++) {
180 |         currnum += ti->free_list->get_freelist_size(i);
181 |     }
182 | 
183 |     std::cout << "[TEST]\tcheck every node's address successfully\n";
184 | 
185 |     free_node_from_type((uint64_t)n4, PART_ns::NTypes::N4);
186 |     free_node_from_type((uint64_t)n16, PART_ns::NTypes::N16);
187 |     free_node_from_type((uint64_t)n48, PART_ns::NTypes::N48);
188 |     free_node_from_type((uint64_t)n256, PART_ns::NTypes::N256);
189 |     free_node_from_type((uint64_t)leaf, PART_ns::NTypes::Leaf);
190 | 
191 |     std::cout << "[TEST]\tfree nodes successfully\n";
192 | 
193 |     int nownum = 0;
194 |     for (int i = 0; i < free_list_number; i++) {
195 |         nownum += ti->free_list->get_freelist_size(i);
196 |     }
197 |     ASSERT_EQ(nownum, currnum + 5);
198 | 
199 |     std::cout << "[TEST]\tfreelist's size correct\n";
200 | 
201 |     unregister_threadinfo();
202 |     close_nvm_mgr();
203 | }


--------------------------------------------------------------------------------
/test/test_recovery.cpp:
--------------------------------------------------------------------------------
 1 | //#include <gtest/gtest.h>
 2 | //#include <iostream>
 3 | //#include <stdio.h>
 4 | //#include <stdlib.h>
 5 | //#include <thread>
 6 | //#include <vector>
 7 | //
 8 | //#include "N.h"
 9 | //#include "Tree.h"
10 | //#include "nvm_mgr.h"
11 | //#include "pmalloc_wrap.h"
12 | //#include "threadinfo.h"
13 | //
14 | // using namespace NVMMgr_ns;
15 | // inline void clear_data() { system("rm -rf /mnt/pmem0/matianmao/part.data"); }
16 | //
17 | // TEST(TestRecovery, recovery1) {
18 | //    clear_data();
19 | //    std::cout << "[TEST]\tstart to test recovery1\n";
20 | //
21 | //    init_nvm_mgr();
22 | //    register_threadinfo();
23 | //    NVMMgr *mgr = get_nvm_mgr();
24 | //    thread_info *ti = (thread_info *)get_threadinfo();
25 | //
26 | //    int block_num = 3;
27 | //    mgr->meta_data->free_bit_offset = block_num;
28 | //    for (int i = 0; i < block_num; i++) {
29 | //        mgr->meta_data->bitmap[i] = ti->id;
30 | //    }
31 | //    mgr->recovery_free_memory();
32 | //    std::cout << "[TEST]\treclaim memory successfully\n";
33 | //
34 | //    std::cout << ti->free_list->get_freelist_size(free_list_number - 1) <<
35 | //    "\n"; ASSERT_EQ(ti->free_list->get_freelist_size(free_list_number - 1),
36 | //              NVMMgr::PGSIZE * block_num / 4096);
37 | //
38 | //    unregister_threadinfo();
39 | //    close_nvm_mgr();
40 | //}
41 | //
42 | // TEST(TestRecovery, recovery2) {
43 | //    clear_data();
44 | //    std::cout << "[TEST]\tstart to test recovery1\n";
45 | //
46 | //    init_nvm_mgr();
47 | //    register_threadinfo();
48 | //    NVMMgr *mgr = get_nvm_mgr();
49 | //    thread_info *ti = (thread_info *)get_threadinfo();
50 | //
51 | //    int block_num = 1;
52 | //    mgr->meta_data->free_bit_offset = block_num;
53 | //    for (int i = 0; i < block_num; i++) {
54 | //        mgr->meta_data->bitmap[i] = ti->id;
55 | //    }
56 | //    uint64_t start = NVMMgr::data_block_start;
57 | //    mgr->recovery_set.insert(std::make_pair(start, 4096));
58 | //    mgr->recovery_set.insert(std::make_pair(start + 4096, 8));
59 | //    mgr->recovery_set.insert(std::make_pair(start + 4096 + 16, 8));
60 | //    mgr->recovery_set.insert(std::make_pair(start + 4096 + 64, 16));
61 | //
62 | //    mgr->recovery_free_memory();
63 | //    int num[free_list_number] = {2, 1, 2, 0, 1, 1, 1, 1, 1, 62};
64 | //    for (int i = 0; i < free_list_number; i++) {
65 | //        ASSERT_EQ(ti->free_list->get_freelist_size(i), num[i]);
66 | //        std::cout << ti->free_list->get_freelist_size(i) << "\n";
67 | //    }
68 | //
69 | //    unregister_threadinfo();
70 | //    close_nvm_mgr();
71 | //}
72 | 


--------------------------------------------------------------------------------
/test/test_skiplist.cpp:
--------------------------------------------------------------------------------
  1 | //#include "lf-skiplist-alloc.h"
  2 | //#include <gtest/gtest.h>
  3 | //#include <thread>
  4 | //
  5 | // using namespace skiplist;
  6 | //
  7 | // inline void clear_data() {
  8 | //    system("rm -rf /mnt/pmem0/matianmao/skiplist.data");
  9 | //}
 10 | //
 11 | // TEST(TestSkipList, skiplist) {
 12 | //    std::cout << "start test skiplist\n";
 13 | //    clear_data();
 14 | //    init_pmem();
 15 | //    skiplist_t *sl = new_skiplist();
 16 | //    std::cout << "skiplist create\n";
 17 | //
 18 | //    const int test_num = 1000000;
 19 | //    const int threads_num = 4;
 20 | //    const int num_per_thread = test_num / threads_num;
 21 | //
 22 | //    for (int i = 0; i < test_num; i++) {
 23 | //        std::string key = std::to_string(i);
 24 | //
 25 | //        const char *k = key.c_str();
 26 | //        ASSERT_EQ(strlen(k), key.size());
 27 | //
 28 | //        skiplist_insert(sl, (char *)key.c_str(), (char *)key.c_str());
 29 | //        char *value = skiplist_find(sl, (char *)key.c_str());
 30 | //        ASSERT_TRUE(value);
 31 | //        ASSERT_EQ(strlen(value), key.size());
 32 | //        ASSERT_EQ(memcmp(value, (char *)key.c_str(), strlen(value)), 0);
 33 | //        //        std::cout<<"insert "<<i<<"\n";
 34 | //    }
 35 | //
 36 | //    std::cout << "init insert finish\n";
 37 | //
 38 | //    std::thread *tid[threads_num];
 39 | //    for (int i = 0; i < threads_num; i++) {
 40 | //        tid[i] = new std::thread(
 41 | //            [&](int id) {
 42 | //                register_thread();
 43 | //
 44 | //                // find
 45 | //                for (int j = 0; j < num_per_thread; j++) {
 46 | //                    std::string key = std::to_string(j * threads_num + id);
 47 | //
 48 | //                    char *value = skiplist_find(sl, (char *)key.c_str());
 49 | //                    ASSERT_TRUE(value);
 50 | //                    ASSERT_EQ(strlen(value), key.size());
 51 | //                    //                std::cout<<strlen(value)<<"
 52 | //                    //                "<<value<<"\n";
 53 | //                    ASSERT_EQ(memcmp(value, (char *)key.c_str(),
 54 | //                    strlen(value)),
 55 | //                              0);
 56 | //
 57 | //                    char *delete_val = skiplist_remove(sl, (char
 58 | //                    *)key.c_str()); ASSERT_TRUE(delete_val);
 59 | //                    ASSERT_EQ(strlen(delete_val), strlen(value));
 60 | //                    ASSERT_EQ(memcmp(delete_val, value, strlen(value)), 0);
 61 | //
 62 | //                    value = skiplist_find(sl, (char *)key.c_str());
 63 | //                    ASSERT_FALSE(value);
 64 | //                }
 65 | //
 66 | //                std::cout << "thread " << id << " read delete read finish\n";
 67 | //
 68 | //                for (int j = 0; j < num_per_thread; j++) {
 69 | //                    std::string key = std::to_string(j * threads_num + id);
 70 | //                    skiplist_insert(sl, (char *)key.c_str(),
 71 | //                                    (char *)key.c_str());
 72 | //                    char *value = skiplist_find(sl, (char *)key.c_str());
 73 | //                    ASSERT_TRUE(value);
 74 | //                    ASSERT_EQ(strlen(value), key.size());
 75 | //                    ASSERT_EQ(memcmp(value, (char *)key.c_str(),
 76 | //                    strlen(value)),
 77 | //                              0);
 78 | //
 79 | //                    std::string newval = "0" + key + "0";
 80 | //                    skiplist_update(sl, (char *)key.c_str(),
 81 | //                                    (char *)newval.c_str());
 82 | //                    char *newvalue = skiplist_find(sl, (char *)key.c_str());
 83 | //                    ASSERT_TRUE(newvalue);
 84 | //                    ASSERT_EQ(strlen(newvalue), newval.size());
 85 | //                    ASSERT_EQ(strlen(newvalue), strlen(value) + 2);
 86 | //                    ASSERT_EQ(memcmp(newvalue + 1, value, strlen(value)), 0);
 87 | //                    std::string s1(newvalue);
 88 | //                    std::string s2(value);
 89 | //                    s2 = "0" + s2 + "0";
 90 | //                    ASSERT_EQ(s1, s2);
 91 | //                }
 92 | //
 93 | //                std::cout << "thread " << id
 94 | //                          << " insert read update read check\n";
 95 | //            },
 96 | //            i);
 97 | //    }
 98 | //
 99 | //    for (int i = 0; i < threads_num; i++) {
100 | //        tid[i]->join();
101 | //    }
102 | //}


--------------------------------------------------------------------------------