├── README.md ├── LICENSE ├── include ├── MurmurHash2.h ├── ThreadPool.hpp ├── bloom.hpp ├── util.h ├── bloom.h ├── ART.hpp ├── btreeolc.hpp └── art_tree.hpp ├── CMakeLists.txt ├── misc └── ARTOLC │ ├── N256.cpp │ ├── index_key.h │ ├── Tree.h │ ├── Epoche.h │ ├── N48.cpp │ ├── N4.cpp │ ├── N16.cpp │ ├── Key.h │ ├── Epoche.cpp │ ├── ARTOLC.hpp │ ├── N.h │ ├── N.cpp │ └── Tree.cpp ├── src ├── art_idx.cpp ├── bloom.c ├── util.cpp └── MurmurHash2.cpp └── test ├── concur_dptree_test.cxx └── dptree_test.cxx /README.md: -------------------------------------------------------------------------------- 1 | # DPTree-code 2 | Source code of the VLDB 2020 paper: [DPTree: Differential Indexing for Persistent Memory](http://www.vldb.org/pvldb/vol13/p421-zhou.pdf) 3 | 4 | # Building 5 | 6 | DPTree depends on a few libraries: 7 | * tcmalloc 8 | * stx-btree 9 | * Intel's TBB library 10 | 11 | To install the dependencies on Ubuntu 18.04, run following commands: 12 | 13 | ```bash 14 | sudo apt install stx-btree-dev 15 | sudo apt install libgoogle-perftools-dev 16 | sudo apt install libbtbb-dev 17 | ``` 18 | Once the dependencies are installed, you can compile the tests and benchmarks using: 19 | ```bash 20 | mkdir build 21 | cd build 22 | cmake .. 23 | make -j 10 24 | ``` 25 | This produces `dptree` and `concur_dptree` executables for benchmarks under the build directory. 26 | Checkout the source code of `test/dptree.cxx` or `test/concur_dptree.cxx` on how to use them. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 zxjcarrot 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /include/MurmurHash2.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash2 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH2_H_ 6 | #define _MURMURHASH2_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | uint32_t MurmurHash2 ( const void * key, int len, uint32_t seed ); 30 | uint64_t MurmurHash64A ( const void * key, int len, uint64_t seed ); 31 | uint64_t MurmurHash64B ( const void * key, int len, uint64_t seed ); 32 | uint32_t MurmurHash2A ( const void * key, int len, uint32_t seed ); 33 | uint32_t MurmurHashNeutral2 ( const void * key, int len, uint32_t seed ); 34 | uint32_t MurmurHashAligned2 ( const void * key, int len, uint32_t seed ); 35 | 36 | //----------------------------------------------------------------------------- 37 | 38 | #endif // _MURMURHASH2_H_ 39 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8) 2 | project(DPTree) 3 | 4 | set(CMAKE_VERBOSE_MAKEFILE ON) 5 | 6 | macro(use_cxx11) 7 | if (CMAKE_VERSION VERSION_LESS "3.1") 8 | if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 9 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++11") 10 | endif () 11 | else () 12 | set (CMAKE_CXX_STANDARD 11) 13 | endif () 14 | endmacro(use_cxx11) 15 | 16 | #set(CMAKE_CXX_STANDARD 11) 17 | use_cxx11() 18 | include_directories(include) 19 | include_directories(misc) 20 | 21 | set (SIMD_FLAGS "-g3 -msse -msse2 -mavx -mrtm") 22 | execute_process(COMMAND bash "-c" "cat /proc/cpuinfo | grep avx2" 23 | OUTPUT_VARIABLE AVX2_ENABLED) 24 | if (AVX2_ENABLED STREQUAL "") 25 | message(STATUS "avx2 instruction set is required" ) 26 | else () 27 | set (SIMD_FLAGS "${SIMD_FLAGS} -mavx2") 28 | endif() 29 | execute_process(COMMAND bash "-c" "cat /proc/cpuinfo | grep avx512" 30 | OUTPUT_VARIABLE AVX512_ENABLED) 31 | if (AVX512_ENABLED STREQUAL "") 32 | message(STATUS "avx512 is not enabled on this machine") 33 | else() 34 | set (SIMD_FLAGS "${SIMD_FLAGS} -mavx512f -DHAS_AVX512") 35 | endif() 36 | 37 | #-DCOUNT_CLFLUSH -DCOUNT_CLFLUSH -DNVM_DRAM_MODE -DNVM_DRAM_MODE -DMEASURE_LEAF_SEARCH -DNVM_DRAM_MODE 38 | set (FLAGS " -DCOUNT_CLFLUSH -fpermissive -ftree-vectorize") 39 | set (FLAGS "${FLAGS} ") 40 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS} ${SIMD_FLAGS}") 41 | 42 | #set(CMAKE_CXX_FLAGS_MYREL "-O3 -g") 43 | 44 | set (CMAKE_BUILD_TYPE "RelWithDebInfo") 45 | 46 | 47 | add_executable (concur_dptree test/concur_dptree_test.cxx 48 | src/util.cpp 49 | src/ART.cpp 50 | misc/ARTOLC/Epoche.cpp 51 | misc/ARTOLC/Tree.cpp 52 | src/art_idx.cpp 53 | src/MurmurHash2.cpp 54 | src/bloom.c) 55 | 56 | add_executable (dptree test/dptree_test.cxx 57 | src/util.cpp 58 | src/ART.cpp 59 | misc/ARTOLC/Epoche.cpp 60 | misc/ARTOLC/Tree.cpp 61 | src/art_idx.cpp 62 | src/MurmurHash2.cpp 63 | src/bloom.c) 64 | 65 | 66 | target_link_libraries (concur_dptree pthread tbb tcmalloc_minimal) 67 | target_link_libraries (dptree pthread tbb tcmalloc_minimal) 68 | -------------------------------------------------------------------------------- /misc/ARTOLC/N256.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_OLC { 6 | 7 | bool N256::isFull() const { 8 | return false; 9 | } 10 | 11 | bool N256::isUnderfull() const { 12 | return count == 37; 13 | } 14 | 15 | void N256::deleteChildren() { 16 | for (uint64_t i = 0; i < 256; ++i) { 17 | if (children[i] != nullptr) { 18 | N::deleteChildren(children[i]); 19 | N::deleteNode(children[i]); 20 | } 21 | } 22 | } 23 | 24 | void N256::insert(uint8_t key, N *val) { 25 | children[key] = val; 26 | count++; 27 | } 28 | 29 | template 30 | void N256::copyTo(NODE *n) const { 31 | for (int i = 0; i < 256; ++i) { 32 | if (children[i] != nullptr) { 33 | n->insert(i, children[i]); 34 | } 35 | } 36 | } 37 | 38 | bool N256::change(uint8_t key, N *n) { 39 | children[key] = n; 40 | return true; 41 | } 42 | 43 | N *N256::getChild(const uint8_t k) const { 44 | return children[k]; 45 | } 46 | 47 | void N256::remove(uint8_t k) { 48 | children[k] = nullptr; 49 | count--; 50 | } 51 | 52 | N *N256::getAnyChild() const { 53 | N *anyChild = nullptr; 54 | for (uint64_t i = 0; i < 256; ++i) { 55 | if (children[i] != nullptr) { 56 | if (N::isLeaf(children[i])) { 57 | return children[i]; 58 | } else { 59 | anyChild = children[i]; 60 | } 61 | } 62 | } 63 | return anyChild; 64 | } 65 | 66 | uint64_t N256::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 67 | uint32_t &childrenCount) const { 68 | restart: 69 | bool needRestart = false; 70 | uint64_t v; 71 | v = readLockOrRestart(needRestart); 72 | if (needRestart) goto restart; 73 | childrenCount = 0; 74 | for (unsigned i = start; i <= end; i++) { 75 | if (this->children[i] != nullptr) { 76 | children[childrenCount] = std::make_tuple(i, this->children[i]); 77 | childrenCount++; 78 | } 79 | } 80 | readUnlockOrRestart(v, needRestart); 81 | if (needRestart) goto restart; 82 | return v; 83 | } 84 | } -------------------------------------------------------------------------------- /misc/ARTOLC/index_key.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _INDEX_KEY_H 3 | #define _INDEX_KEY_H 4 | 5 | #include 6 | #include 7 | 8 | template 9 | class GenericKey { 10 | public: 11 | char data[keySize]; 12 | public: 13 | inline void setFromString(std::string key) { 14 | memset(data, 0, keySize); 15 | if(key.size() >= keySize) { 16 | memcpy(data, key.c_str(), keySize - 1); 17 | data[keySize - 1] = '\0'; 18 | } else { 19 | strcpy(data, key.c_str()); 20 | } 21 | 22 | return; 23 | } 24 | 25 | // Constructor - Fills it with 0x00 26 | // This is for the skiplist to initialize an empty node 27 | GenericKey(int) { memset(data, 0x00, keySize); } 28 | GenericKey() { memset(data, 0x00, keySize); } 29 | // Copy constructor 30 | GenericKey(const GenericKey &other) { memcpy(data, other.data, keySize); } 31 | inline GenericKey &operator=(const GenericKey &other) { 32 | memcpy(data, other.data, keySize); 33 | return *this; 34 | } 35 | 36 | inline bool operator<(const GenericKey &other) { return strcmp(data, other.data) < 0; } 37 | inline bool operator>(const GenericKey &other) { return strcmp(data, other.data) > 0; } 38 | inline bool operator==(const GenericKey &other) { return strcmp(data, other.data) == 0; } 39 | // Derived operators 40 | inline bool operator!=(const GenericKey &other) { return !(*this == other); } 41 | inline bool operator<=(const GenericKey &other) { return !(*this > other); } 42 | inline bool operator>=(const GenericKey &other) { return !(*this < other); } 43 | }; 44 | 45 | template 46 | class GenericComparator { 47 | public: 48 | GenericComparator() {} 49 | 50 | inline bool operator()(const GenericKey &lhs, const GenericKey &rhs) const { 51 | int diff = strcmp(lhs.data, rhs.data); 52 | return diff < 0; 53 | } 54 | }; 55 | 56 | template 57 | class GenericEqualityChecker { 58 | public: 59 | GenericEqualityChecker() {} 60 | 61 | inline bool operator()(const GenericKey &lhs, const GenericKey &rhs) const { 62 | int diff = strcmp(lhs.data, rhs.data); 63 | return diff == 0; 64 | } 65 | }; 66 | 67 | template 68 | class GenericHasher { 69 | public: 70 | GenericHasher() {} 71 | 72 | inline size_t operator()(const GenericKey &lhs) const { 73 | (void)lhs; 74 | return 0UL; 75 | } 76 | }; 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /misc/ARTOLC/Tree.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by florian on 18.11.15. 3 | // 4 | 5 | #ifndef ART_OPTIMISTICLOCK_COUPLING_N_H 6 | #define ART_OPTIMISTICLOCK_COUPLING_N_H 7 | #include "N.h" 8 | 9 | using namespace ART; 10 | 11 | namespace ART_OLC { 12 | static void noop() {} 13 | class Tree { 14 | public: 15 | using LoadKeyFunction = void (*)(TID tid, Key &key); 16 | 17 | private: 18 | N *const root; 19 | 20 | TID checkKey(const TID tid, const Key &k) const; 21 | 22 | LoadKeyFunction loadKey; 23 | 24 | Epoche epoche{256}; 25 | 26 | public: 27 | enum class CheckPrefixResult : uint8_t { 28 | Match, 29 | NoMatch, 30 | OptimisticMatch 31 | }; 32 | 33 | enum class CheckPrefixPessimisticResult : uint8_t { 34 | Match, 35 | NoMatch, 36 | }; 37 | 38 | enum class PCCompareResults : uint8_t { 39 | Smaller, 40 | Equal, 41 | Bigger, 42 | }; 43 | enum class PCEqualsResults : uint8_t { 44 | BothMatch, 45 | Contained, 46 | NoMatch 47 | }; 48 | static CheckPrefixResult checkPrefix(N* n, const Key &k, uint32_t &level); 49 | 50 | static CheckPrefixPessimisticResult checkPrefixPessimistic(N *n, const Key &k, uint32_t &level, 51 | uint8_t &nonMatchingKey, 52 | Prefix &nonMatchingPrefix, 53 | LoadKeyFunction loadKey, bool &needRestart); 54 | 55 | static PCCompareResults checkPrefixCompare(const N* n, const Key &k, uint8_t fillKey, uint32_t &level, LoadKeyFunction loadKey, bool &needRestart); 56 | 57 | static PCEqualsResults checkPrefixEquals(const N* n, uint32_t &level, const Key &start, const Key &end, LoadKeyFunction loadKey, bool &needRestart); 58 | 59 | public: 60 | 61 | Tree(LoadKeyFunction loadKey); 62 | 63 | Tree(const Tree &) = delete; 64 | 65 | Tree(Tree &&t) : root(t.root), loadKey(t.loadKey) { } 66 | 67 | ~Tree(); 68 | 69 | ThreadInfo getThreadInfo(); 70 | 71 | TID lookup(const Key &k, ThreadInfo &threadEpocheInfo) const; 72 | 73 | bool lookupRange(const Key &start, const Key &end, Key &continueKey, TID result[], std::size_t resultLen, 74 | std::size_t &resultCount, ThreadInfo &threadEpocheInfo) const; 75 | 76 | void insert(const Key &k, TID tid, ThreadInfo &epocheInfo, std::function insert_func=noop); 77 | 78 | void remove(const Key &k, TID tid, ThreadInfo &epocheInfo); 79 | }; 80 | } 81 | #endif //ART_OPTIMISTICLOCK_COUPLING_N_H 82 | -------------------------------------------------------------------------------- /include/ThreadPool.hpp: -------------------------------------------------------------------------------- 1 | #ifndef THREAD_POOL_H 2 | #define THREAD_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | class ThreadPool { 15 | public: 16 | ThreadPool(size_t); 17 | template 18 | auto enqueue(F&& f, Args&&... args) 19 | -> std::future::type>; 20 | ~ThreadPool(); 21 | private: 22 | // need to keep track of threads so we can join them 23 | std::vector< std::thread > workers; 24 | // the task queue 25 | std::queue< std::function > tasks; 26 | 27 | // synchronization 28 | std::mutex queue_mutex; 29 | std::condition_variable condition; 30 | bool stop; 31 | }; 32 | 33 | // the constructor just launches some amount of workers 34 | inline ThreadPool::ThreadPool(size_t threads) 35 | : stop(false) 36 | { 37 | for(size_t i = 0;i task; 44 | 45 | { 46 | std::unique_lock lock(this->queue_mutex); 47 | this->condition.wait(lock, 48 | [this]{ return this->stop || !this->tasks.empty(); }); 49 | if(this->stop && this->tasks.empty()) 50 | return; 51 | task = std::move(this->tasks.front()); 52 | this->tasks.pop(); 53 | } 54 | 55 | task(); 56 | } 57 | } 58 | ); 59 | } 60 | 61 | // add new work item to the pool 62 | template 63 | auto ThreadPool::enqueue(F&& f, Args&&... args) 64 | -> std::future::type> 65 | { 66 | using return_type = typename std::result_of::type; 67 | 68 | auto task = std::make_shared< std::packaged_task >( 69 | std::bind(std::forward(f), std::forward(args)...) 70 | ); 71 | 72 | std::future res = task->get_future(); 73 | { 74 | std::unique_lock lock(queue_mutex); 75 | 76 | // don't allow enqueueing after stopping the pool 77 | if(stop) 78 | throw std::runtime_error("enqueue on stopped ThreadPool"); 79 | 80 | tasks.emplace([task](){ (*task)(); }); 81 | } 82 | condition.notify_one(); 83 | return res; 84 | } 85 | 86 | // the destructor joins all threads 87 | inline ThreadPool::~ThreadPool() 88 | { 89 | { 90 | std::unique_lock lock(queue_mutex); 91 | stop = true; 92 | } 93 | condition.notify_all(); 94 | for(std::thread &worker: workers) 95 | worker.join(); 96 | } 97 | 98 | #endif -------------------------------------------------------------------------------- /misc/ARTOLC/Epoche.h: -------------------------------------------------------------------------------- 1 | #ifndef ART_EPOCHE_H 2 | #define ART_EPOCHE_H 3 | 4 | #include 5 | #include 6 | #include "tbb/enumerable_thread_specific.h" 7 | #include "tbb/combinable.h" 8 | 9 | namespace ART { 10 | 11 | struct LabelDelete { 12 | std::array nodes; 13 | uint64_t epoche; 14 | std::size_t nodesCount; 15 | LabelDelete *next; 16 | }; 17 | 18 | class DeletionList { 19 | LabelDelete *headDeletionList = nullptr; 20 | LabelDelete *freeLabelDeletes = nullptr; 21 | std::size_t deletitionListCount = 0; 22 | 23 | public: 24 | std::atomic localEpoche; 25 | size_t thresholdCounter{0}; 26 | 27 | ~DeletionList(); 28 | LabelDelete *head(); 29 | 30 | void add(void *n, uint64_t globalEpoch); 31 | 32 | void remove(LabelDelete *label, LabelDelete *prev); 33 | 34 | std::size_t size(); 35 | 36 | std::uint64_t deleted = 0; 37 | std::uint64_t added = 0; 38 | }; 39 | 40 | class Epoche; 41 | class EpocheGuard; 42 | 43 | class ThreadInfo { 44 | friend class Epoche; 45 | friend class EpocheGuard; 46 | Epoche &epoche; 47 | DeletionList &deletionList; 48 | 49 | 50 | DeletionList & getDeletionList() const; 51 | public: 52 | 53 | ThreadInfo(Epoche &epoche); 54 | 55 | ThreadInfo(const ThreadInfo &ti) : epoche(ti.epoche), deletionList(ti.deletionList) { 56 | } 57 | 58 | ~ThreadInfo(); 59 | 60 | Epoche & getEpoche() const; 61 | }; 62 | 63 | class Epoche { 64 | friend class ThreadInfo; 65 | std::atomic currentEpoche{0}; 66 | 67 | tbb::enumerable_thread_specific deletionLists; 68 | 69 | size_t startGCThreshhold; 70 | 71 | 72 | public: 73 | Epoche(size_t startGCThreshhold) : startGCThreshhold(startGCThreshhold) { } 74 | 75 | ~Epoche(); 76 | 77 | void enterEpoche(ThreadInfo &epocheInfo); 78 | 79 | void markNodeForDeletion(void *n, ThreadInfo &epocheInfo); 80 | 81 | void exitEpocheAndCleanup(ThreadInfo &info); 82 | 83 | void showDeleteRatio(); 84 | 85 | }; 86 | 87 | class EpocheGuard { 88 | ThreadInfo &threadEpocheInfo; 89 | public: 90 | 91 | EpocheGuard(ThreadInfo &threadEpocheInfo) : threadEpocheInfo(threadEpocheInfo) { 92 | threadEpocheInfo.getEpoche().enterEpoche(threadEpocheInfo); 93 | } 94 | 95 | ~EpocheGuard() { 96 | threadEpocheInfo.getEpoche().exitEpocheAndCleanup(threadEpocheInfo); 97 | } 98 | }; 99 | 100 | class EpocheGuardReadonly { 101 | public: 102 | 103 | EpocheGuardReadonly(ThreadInfo &threadEpocheInfo) { 104 | threadEpocheInfo.getEpoche().enterEpoche(threadEpocheInfo); 105 | } 106 | 107 | ~EpocheGuardReadonly() { 108 | } 109 | }; 110 | 111 | inline ThreadInfo::~ThreadInfo() { 112 | deletionList.localEpoche.store(std::numeric_limits::max()); 113 | } 114 | } 115 | 116 | #endif //ART_EPOCHE_H 117 | -------------------------------------------------------------------------------- /misc/ARTOLC/N48.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_OLC { 6 | 7 | bool N48::isFull() const { 8 | return count == 48; 9 | } 10 | 11 | bool N48::isUnderfull() const { 12 | return count == 12; 13 | } 14 | 15 | void N48::insert(uint8_t key, N *n) { 16 | unsigned pos = count; 17 | if (children[pos]) { 18 | for (pos = 0; children[pos] != nullptr; pos++); 19 | } 20 | children[pos] = n; 21 | childIndex[key] = (uint8_t) pos; 22 | count++; 23 | } 24 | 25 | template 26 | void N48::copyTo(NODE *n) const { 27 | for (unsigned i = 0; i < 256; i++) { 28 | if (childIndex[i] != emptyMarker) { 29 | n->insert(i, children[childIndex[i]]); 30 | } 31 | } 32 | } 33 | 34 | bool N48::change(uint8_t key, N *val) { 35 | children[childIndex[key]] = val; 36 | return true; 37 | } 38 | 39 | N *N48::getChild(const uint8_t k) const { 40 | if (childIndex[k] == emptyMarker) { 41 | return nullptr; 42 | } else { 43 | return children[childIndex[k]]; 44 | } 45 | } 46 | 47 | N *N48::getMaxChild() const { 48 | for (int i = 255; i >= 0; --i) { 49 | if (childIndex[i] != emptyMarker) { 50 | return children[childIndex[i]]; 51 | } 52 | } 53 | } 54 | 55 | void N48::remove(uint8_t k) { 56 | assert(childIndex[k] != emptyMarker); 57 | children[childIndex[k]] = nullptr; 58 | childIndex[k] = emptyMarker; 59 | count--; 60 | assert(getChild(k) == nullptr); 61 | } 62 | 63 | N *N48::getAnyChild() const { 64 | N *anyChild = nullptr; 65 | for (unsigned i = 0; i < 256; i++) { 66 | if (childIndex[i] != emptyMarker) { 67 | if (N::isLeaf(children[childIndex[i]])) { 68 | return children[childIndex[i]]; 69 | } else { 70 | anyChild = children[childIndex[i]]; 71 | }; 72 | } 73 | } 74 | return anyChild; 75 | } 76 | 77 | void N48::deleteChildren() { 78 | for (unsigned i = 0; i < 256; i++) { 79 | if (childIndex[i] != emptyMarker) { 80 | N::deleteChildren(children[childIndex[i]]); 81 | N::deleteNode(children[childIndex[i]]); 82 | } 83 | } 84 | } 85 | 86 | uint64_t N48::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 87 | uint32_t &childrenCount) const { 88 | restart: 89 | bool needRestart = false; 90 | uint64_t v; 91 | v = readLockOrRestart(needRestart); 92 | if (needRestart) goto restart; 93 | childrenCount = 0; 94 | for (unsigned i = start; i <= end; i++) { 95 | if (this->childIndex[i] != emptyMarker) { 96 | children[childrenCount] = std::make_tuple(i, this->children[this->childIndex[i]]); 97 | childrenCount++; 98 | } 99 | } 100 | readUnlockOrRestart(v, needRestart); 101 | if (needRestart) goto restart; 102 | return v; 103 | } 104 | } -------------------------------------------------------------------------------- /misc/ARTOLC/N4.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_OLC { 6 | 7 | void N4::deleteChildren() { 8 | for (uint32_t i = 0; i < count; ++i) { 9 | N::deleteChildren(children[i]); 10 | N::deleteNode(children[i]); 11 | } 12 | } 13 | 14 | bool N4::isFull() const { 15 | return count == 4; 16 | } 17 | 18 | bool N4::isUnderfull() const { 19 | return false; 20 | } 21 | 22 | void N4::insert(uint8_t key, N *n) { 23 | unsigned pos; 24 | for (pos = 0; (pos < count) && (keys[pos] < key); pos++); 25 | memmove(keys + pos + 1, keys + pos, count - pos); 26 | memmove(children + pos + 1, children + pos, (count - pos) * sizeof(N*)); 27 | keys[pos] = key; 28 | children[pos] = n; 29 | count++; 30 | } 31 | 32 | template 33 | void N4::copyTo(NODE *n) const { 34 | for (uint32_t i = 0; i < count; ++i) { 35 | n->insert(keys[i], children[i]); 36 | } 37 | } 38 | 39 | bool N4::change(uint8_t key, N *val) { 40 | for (uint32_t i = 0; i < count; ++i) { 41 | if (keys[i] == key) { 42 | children[i] = val; 43 | return true; 44 | } 45 | } 46 | assert(false); 47 | __builtin_unreachable(); 48 | } 49 | 50 | N *N4::getChild(const uint8_t k) const { 51 | for (uint32_t i = 0; i < count; ++i) { 52 | if (keys[i] == k) { 53 | return children[i]; 54 | } 55 | } 56 | return nullptr; 57 | } 58 | 59 | void N4::remove(uint8_t k) { 60 | for (uint32_t i = 0; i < count; ++i) { 61 | if (keys[i] == k) { 62 | memmove(keys + i, keys + i + 1, count - i - 1); 63 | memmove(children + i, children + i + 1, (count - i - 1) * sizeof(N *)); 64 | count--; 65 | return; 66 | } 67 | } 68 | } 69 | 70 | N *N4::getAnyChild() const { 71 | N *anyChild = nullptr; 72 | for (uint32_t i = 0; i < count; ++i) { 73 | if (N::isLeaf(children[i])) { 74 | return children[i]; 75 | } else { 76 | anyChild = children[i]; 77 | } 78 | } 79 | return anyChild; 80 | } 81 | 82 | std::tuple N4::getSecondChild(const uint8_t key) const { 83 | for (uint32_t i = 0; i < count; ++i) { 84 | if (keys[i] != key) { 85 | return std::make_tuple(children[i], keys[i]); 86 | } 87 | } 88 | return std::make_tuple(nullptr, 0); 89 | } 90 | 91 | uint64_t N4::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 92 | uint32_t &childrenCount) const { 93 | restart: 94 | bool needRestart = false; 95 | uint64_t v; 96 | v = readLockOrRestart(needRestart); 97 | if (needRestart) goto restart; 98 | childrenCount = 0; 99 | for (uint32_t i = 0; i < count; ++i) { 100 | if (this->keys[i] >= start && this->keys[i] <= end) { 101 | children[childrenCount] = std::make_tuple(this->keys[i], this->children[i]); 102 | childrenCount++; 103 | } 104 | } 105 | readUnlockOrRestart(v, needRestart); 106 | if (needRestart) goto restart; 107 | return v; 108 | } 109 | } -------------------------------------------------------------------------------- /src/art_idx.cpp: -------------------------------------------------------------------------------- 1 | #include "art_idx.hpp" 2 | 3 | namespace ART_IDX { 4 | static thread_local std::vector ancestors; 5 | Node *art_tree::lowerBound(Node *node, uint8_t key[], unsigned keyLength, unsigned depth, unsigned maxKeyLength, bool &pref) 6 | { 7 | // Find the node with a matching key, optimistic version 8 | ancestors.clear(); 9 | 10 | while (node != NULL) 11 | { 12 | ancestors.push_back(node); 13 | if (isLeaf(node)) 14 | { 15 | return node; 16 | } 17 | 18 | int nodePrefixLen = node->prefixLength; 19 | if (node->prefixLength) 20 | { 21 | unsigned pos = 0; 22 | if (node->prefixLength <= maxPrefixLength) 23 | { 24 | for (; pos < node->prefixLength && key[depth + pos] == node->prefix[pos]; pos++) 25 | ; 26 | if (pos < node->prefixLength) 27 | { 28 | if (node->prefix[pos] < key[depth + pos]) 29 | { 30 | goto backtrack; 31 | } 32 | else 33 | { // node->prefix[pos] > key[depth + pos] 34 | return minimum(node); 35 | } 36 | } 37 | } 38 | else 39 | { 40 | for (; pos < maxPrefixLength && key[depth + pos] == node->prefix[pos]; pos++) 41 | ; 42 | if (pos >= maxPrefixLength) 43 | { 44 | uint8_t leafKey[maxKeyLength]; 45 | auto minLeaf = minimum(node); 46 | loadLowerBoundKey(getLeafValue(minLeaf), leafKey); 47 | for (; pos < node->prefixLength && key[depth + pos] == leafKey[depth + pos]; ++pos) 48 | ; 49 | if (pos < node->prefixLength) 50 | { 51 | if (leafKey[depth + pos] < key[depth + pos]) 52 | { 53 | goto backtrack; 54 | } 55 | else 56 | { // leafKey[depth + pos] > key[depth + pos] 57 | pref = true; 58 | return minLeaf; 59 | } 60 | } 61 | } 62 | } 63 | depth += node->prefixLength; 64 | } 65 | 66 | bool equal = false; 67 | node = findChildLowerBound(node, key[depth], equal); 68 | if (node == nullptr) 69 | { 70 | depth -= nodePrefixLen; 71 | goto backtrack; 72 | } 73 | if (equal == false) 74 | { 75 | return minimum(node); 76 | } 77 | depth++; 78 | } 79 | 80 | backtrack: 81 | if (ancestors.empty() == false) 82 | { 83 | ancestors.pop_back(); 84 | } 85 | while (ancestors.empty() == false) 86 | { 87 | auto node = ancestors.back(); 88 | --depth; 89 | if (key[depth] < 255) 90 | { 91 | bool equal = false; 92 | auto n = findChildLowerBound(node, key[depth] + 1, equal); 93 | if (n) 94 | { 95 | return minimum(n); 96 | } 97 | } 98 | depth -= node->prefixLength; 99 | ancestors.pop_back(); 100 | } 101 | assert(depth == 0); 102 | return NULL; 103 | } 104 | } -------------------------------------------------------------------------------- /misc/ARTOLC/N16.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | #include // x86 SSE intrinsics 5 | 6 | namespace ART_OLC { 7 | 8 | bool N16::isFull() const { 9 | return count == 16; 10 | } 11 | 12 | bool N16::isUnderfull() const { 13 | return count == 3; 14 | } 15 | 16 | void N16::insert(uint8_t key, N *n) { 17 | uint8_t keyByteFlipped = flipSign(key); 18 | __m128i cmp = _mm_cmplt_epi8(_mm_set1_epi8(keyByteFlipped), _mm_loadu_si128(reinterpret_cast<__m128i *>(keys))); 19 | uint16_t bitfield = _mm_movemask_epi8(cmp) & (0xFFFF >> (16 - count)); 20 | unsigned pos = bitfield ? ctz(bitfield) : count; 21 | memmove(keys + pos + 1, keys + pos, count - pos); 22 | memmove(children + pos + 1, children + pos, (count - pos) * sizeof(uintptr_t)); 23 | keys[pos] = keyByteFlipped; 24 | children[pos] = n; 25 | count++; 26 | } 27 | 28 | template 29 | void N16::copyTo(NODE *n) const { 30 | for (unsigned i = 0; i < count; i++) { 31 | n->insert(flipSign(keys[i]), children[i]); 32 | } 33 | } 34 | 35 | bool N16::change(uint8_t key, N *val) { 36 | N **childPos = const_cast(getChildPos(key)); 37 | assert(childPos != nullptr); 38 | *childPos = val; 39 | return true; 40 | } 41 | 42 | N *const *N16::getChildPos(const uint8_t k) const { 43 | __m128i cmp = _mm_cmpeq_epi8(_mm_set1_epi8(flipSign(k)), 44 | _mm_loadu_si128(reinterpret_cast(keys))); 45 | unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << count) - 1); 46 | if (bitfield) { 47 | return &children[ctz(bitfield)]; 48 | } else { 49 | return nullptr; 50 | } 51 | } 52 | 53 | N *N16::getChild(const uint8_t k) const { 54 | N *const *childPos = getChildPos(k); 55 | if (childPos == nullptr) { 56 | return nullptr; 57 | } else { 58 | return *childPos; 59 | } 60 | } 61 | 62 | void N16::remove(uint8_t k) { 63 | N *const *leafPlace = getChildPos(k); 64 | assert(leafPlace != nullptr); 65 | std::size_t pos = leafPlace - children; 66 | memmove(keys + pos, keys + pos + 1, count - pos - 1); 67 | memmove(children + pos, children + pos + 1, (count - pos - 1) * sizeof(N *)); 68 | count--; 69 | assert(getChild(k) == nullptr); 70 | } 71 | 72 | N *N16::getAnyChild() const { 73 | for (int i = 0; i < count; ++i) { 74 | if (N::isLeaf(children[i])) { 75 | return children[i]; 76 | } 77 | } 78 | return children[0]; 79 | } 80 | 81 | void N16::deleteChildren() { 82 | for (std::size_t i = 0; i < count; ++i) { 83 | N::deleteChildren(children[i]); 84 | N::deleteNode(children[i]); 85 | } 86 | } 87 | 88 | uint64_t N16::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 89 | uint32_t &childrenCount) const { 90 | restart: 91 | bool needRestart = false; 92 | uint64_t v; 93 | v = readLockOrRestart(needRestart); 94 | if (needRestart) goto restart; 95 | childrenCount = 0; 96 | auto startPos = getChildPos(start); 97 | auto endPos = getChildPos(end); 98 | if (startPos == nullptr) { 99 | startPos = this->children; 100 | } 101 | if (endPos == nullptr) { 102 | endPos = this->children + (count - 1); 103 | } 104 | for (auto p = startPos; p <= endPos; ++p) { 105 | children[childrenCount] = std::make_tuple(flipSign(keys[p - this->children]), *p); 106 | childrenCount++; 107 | } 108 | readUnlockOrRestart(v, needRestart); 109 | if (needRestart) goto restart; 110 | return v; 111 | } 112 | } -------------------------------------------------------------------------------- /misc/ARTOLC/Key.h: -------------------------------------------------------------------------------- 1 | #ifndef ART_KEY_H 2 | #define ART_KEY_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using KeyLen = uint32_t; 10 | 11 | class Key { 12 | public: 13 | 14 | static constexpr uint32_t stackLen = 128; 15 | uint32_t len = 0; 16 | 17 | uint8_t *data = nullptr; 18 | 19 | uint8_t stackKey[stackLen]; 20 | 21 | Key(uint64_t k) { setInt(k); } 22 | 23 | void setInt(uint64_t k) { data = stackKey; len = 8; *reinterpret_cast(stackKey) = __builtin_bswap64(k); } 24 | 25 | uint64_t getInt() { return __builtin_bswap64(*reinterpret_cast(stackKey)); } 26 | 27 | Key() {} 28 | 29 | ~Key(); 30 | 31 | //Key(const Key &key); 32 | 33 | Key(Key &&key); 34 | 35 | void set(const char bytes[], const std::size_t length); 36 | 37 | void operator=(const char key[]); 38 | //Key& operator=(const Key &key); 39 | bool operator==(const Key &k) const { 40 | if (k.getKeyLen() != getKeyLen()) { 41 | return false; 42 | } 43 | return std::memcmp(&k[0], data, getKeyLen()) == 0; 44 | } 45 | 46 | uint8_t &operator[](std::size_t i); 47 | 48 | const uint8_t &operator[](std::size_t i) const; 49 | 50 | KeyLen getKeyLen() const; 51 | 52 | void setKeyLen(KeyLen len); 53 | 54 | }; 55 | 56 | 57 | inline uint8_t &Key::operator[](std::size_t i) { 58 | assert(i < len); 59 | return data[i]; 60 | } 61 | 62 | inline const uint8_t &Key::operator[](std::size_t i) const { 63 | assert(i < len); 64 | return data[i]; 65 | } 66 | 67 | inline KeyLen Key::getKeyLen() const { return len; } 68 | 69 | inline Key::~Key() { 70 | if (len > stackLen) { 71 | delete[] data; 72 | data = nullptr; 73 | } 74 | } 75 | 76 | inline Key::Key(Key &&key) { 77 | len = key.len; 78 | if (len > stackLen) { 79 | data = key.data; 80 | key.data = nullptr; 81 | } else { 82 | memcpy(stackKey, key.stackKey, key.len); 83 | data = stackKey; 84 | } 85 | } 86 | // 87 | //inline Key::Key(const Key &key) { 88 | // len = key.len; 89 | // if (len > stackLen) { 90 | // data = new uint8_t[len]; 91 | // memcpy(data, key.data, len); 92 | // } else { 93 | // memcpy(stackKey, key.stackKey, key.len); 94 | // data = stackKey; 95 | // } 96 | //} 97 | 98 | inline void Key::set(const char bytes[], const std::size_t length) { 99 | if (len > stackLen) { 100 | delete[] data; 101 | } 102 | if (length <= stackLen) { 103 | memcpy(stackKey, bytes, length); 104 | data = stackKey; 105 | } else { 106 | data = new uint8_t[length]; 107 | memcpy(data, bytes, length); 108 | } 109 | len = length; 110 | } 111 | // 112 | //inline Key& Key::operator=(const Key &key) { 113 | // if (this == &key) 114 | // return *this; 115 | // if (len > stackLen) { 116 | // delete[] data; 117 | // } 118 | // 119 | // data = nullptr; 120 | // len = key.len; 121 | // 122 | // if (len > stackLen) { 123 | // data = new uint8_t[len]; 124 | // memcpy(data, key.data, len); 125 | // } else { 126 | // memcpy(stackKey, key.stackKey, key.len); 127 | // data = stackKey; 128 | // } 129 | // return *this; 130 | //} 131 | 132 | 133 | inline void Key::operator=(const char key[]) { 134 | if (len > stackLen) { 135 | delete[] data; 136 | } 137 | len = strlen(key); 138 | if (len <= stackLen) { 139 | memcpy(stackKey, key, len); 140 | data = stackKey; 141 | } else { 142 | data = new uint8_t[len]; 143 | memcpy(data, key, len); 144 | } 145 | } 146 | 147 | inline void Key::setKeyLen(KeyLen newLen) { 148 | if (len == newLen) return; 149 | if (len > stackLen) { 150 | delete[] data; 151 | } 152 | len = newLen; 153 | if (len > stackLen) { 154 | data = new uint8_t[len]; 155 | } else { 156 | data = stackKey; 157 | } 158 | } 159 | 160 | #endif // ART_KEY_H 161 | -------------------------------------------------------------------------------- /test/concur_dptree_test.cxx: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | 17 | int parallel_merge_worker_num = 16; 18 | 19 | float range_sizes [] = {0.0001, 0.001, 0.01}; 20 | const int repeatn = 50000000; 21 | uint64_t cacheline[8]; 22 | 23 | uint64_t strip_upsert_value(uint64_t v) { 24 | return v >> 1; 25 | } 26 | 27 | uint64_t make_upsert_value(uint64_t v) { 28 | return (v << 1); 29 | } 30 | 31 | 32 | void concur_dptree_test(int nworkers, int n) { 33 | 34 | dptree::concur_dptree index; 35 | auto insert_func = [&](int start, int end) { 36 | for (int i = start; i < end; ++i) { 37 | index.insert(keys[i], keys[i]); 38 | } 39 | }; 40 | 41 | measure([&]() -> unsigned { 42 | std::vector workers; 43 | int range_size = n / nworkers; 44 | int start = 0; 45 | int end = range_size; 46 | for (;start < n;) { 47 | workers.emplace_back(std::thread(insert_func, start, end)); 48 | start = end; 49 | end = std::min(end + range_size, (int)n); 50 | } 51 | std::for_each(workers.begin(), workers.end(), [](std::thread & t) { t.join(); }); 52 | // printf("# merges: %d\n", index.get_merges()); 53 | // printf("merge time: %f secs\n", index.get_merge_time()); 54 | printf("real merge time: %f secs\n", index.get_real_merge_time()); 55 | printf("merge wait time: %f secs\n", index.get_merge_wait_time()); 56 | printf("merge work time: %f secs\n", index.get_merge_work_time()); 57 | printf("inner nodes build time: %f secs\n", index.get_inner_node_build_time()); 58 | printf("base tree flush time: %f secs\n", index.get_flushtime()); 59 | 60 | return n; 61 | }, "concur-dptree-insert"); 62 | while (index.is_merging()); 63 | 64 | auto lookup_func = [&](int start, int end, std::atomic & count) { 65 | int repeat = 1; 66 | int c = 0; 67 | repeat = repeatn / (end - start) / 2; 68 | if (repeat < 1) 69 | repeat = 1; 70 | for (uint64_t r = 0; r < repeat; ++r) { 71 | for (size_t i = start; i < end; i++) { 72 | uint64_t key = lookupKeys[i]; 73 | uint64_t value = 0; 74 | bool res = index.lookup(key, value); 75 | assert(res); 76 | assert(key == strip_upsert_value(value)); 77 | c += 1; 78 | } 79 | } 80 | count += c; 81 | }; 82 | 83 | measure([&]() -> unsigned { 84 | std::vector workers; 85 | int range_size = n / nworkers; 86 | int start = 0; 87 | int end = range_size; 88 | std::atomic count(0); 89 | for (;start < n && workers.size() < nworkers;) { 90 | workers.emplace_back(std::thread(lookup_func, start, end, std::ref(count))); 91 | start = end; 92 | end = std::min(end + range_size, (int)n); 93 | } 94 | std::for_each(workers.begin(), workers.end(), [](std::thread & t) { t.join(); }); 95 | printf("probes: %lu\n", index.get_probes()); 96 | printf("avg probes per lookup: %f\n", index.get_probes() / (count.load() + 0.1)); 97 | return count.load(); 98 | }, "concur-dptree-search"); 99 | 100 | } 101 | 102 | 103 | int main(int argc, char const *argv[]) { 104 | int n = 10000000; 105 | bool sparseKey = false; 106 | int nworkers = 1; 107 | if (argc > 1) 108 | n = atoi(argv[1]); 109 | if (argc > 2) 110 | nworkers = atoi(argv[2]); 111 | if (argc > 3) 112 | parallel_merge_worker_num = atoi(argv[3]); 113 | else 114 | parallel_merge_worker_num = nworkers; 115 | if (argc > 4) 116 | sparseKey = atoi(argv[4]); 117 | if (argc > 5) 118 | write_latency_in_ns = atoi(argv[5]); 119 | printf("nworkers: %d, parallel_merge_worker_num %d\n", nworkers, parallel_merge_worker_num); 120 | prepareKeys(n, "", sparseKey, false); 121 | concur_dptree_test(nworkers, n); 122 | return 0; 123 | } -------------------------------------------------------------------------------- /include/bloom.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "MurmurHash2.h" 3 | #include "bloom.h" 4 | #include 5 | #include 6 | #include 7 | 8 | template 9 | class bloom1 10 | { 11 | public: 12 | using key_type = Key; 13 | typedef uint64_t word_t; 14 | static constexpr int bits_per_byte = 8; 15 | static constexpr int bytes_per_word = 8; 16 | static constexpr int bits_per_word = bits_per_byte * bytes_per_word; 17 | static constexpr int member_bit_width = 6; 18 | static constexpr int member_bit_mask = (1 << member_bit_width) - 1; 19 | static constexpr int word_hash_bits = 24; 20 | static constexpr int word_hash_bits_mask = (1ULL << word_hash_bits) - 1; 21 | static constexpr int k = 3; 22 | static constexpr int member_hash_bits = std::ceil(k * member_bit_width); 23 | static constexpr uint64_t hash_seed = 0xdeadbeef12345678ULL; 24 | bloom1(int entries, double fp_rate) 25 | { 26 | this->entries = entries; 27 | error = fp_rate; 28 | 29 | double num = log(error); 30 | double denom = 0.480453013918201; // ln(2)^2 31 | double bpe = -(num / denom); 32 | 33 | double dentries = (double)entries; 34 | bits_m = (int)(dentries * bpe); 35 | 36 | if (bits_m % bits_per_byte) 37 | { 38 | bytes_n = (bits_m / bits_per_byte) + 1; 39 | } 40 | else 41 | { 42 | bytes_n = bits_m / bits_per_byte; 43 | } 44 | 45 | if (bytes_n % bytes_per_word) 46 | { 47 | words_l = (bytes_n / bytes_per_word) + 1; 48 | } 49 | else 50 | { 51 | words_l = bytes_n / bytes_per_word; 52 | } 53 | hash_bits = word_hash_bits + member_hash_bits; 54 | // assert(hash_bits <= bits_per_word); 55 | words = (word_t*)calloc(words_l, sizeof(word_t)); 56 | } 57 | 58 | ~bloom1() { free(words); } 59 | 60 | uint64_t make_word_mask(uint64_t h, int &word_off) 61 | { 62 | uint64_t old_h = h; 63 | uint64_t word_mask = 0; 64 | word_off = (h & word_hash_bits_mask) % words_l; 65 | h >>= word_hash_bits; 66 | int bits_left = member_hash_bits; 67 | int h_bits = bits_per_word - word_hash_bits; 68 | while (bits_left) 69 | { 70 | if (h_bits < member_bit_width) 71 | { 72 | h = MurmurHash64A(&old_h, sizeof(old_h), hash_seed); 73 | old_h = h; 74 | h_bits = bits_per_word; 75 | } 76 | int member_off = h & member_bit_mask; 77 | word_mask |= 1ULL << member_off; 78 | h >>= member_bit_width; 79 | h_bits -= member_bit_width; 80 | bits_left -= member_bit_width; 81 | } 82 | return word_mask; 83 | } 84 | 85 | void insert_unsafe(const key_type &key) 86 | { 87 | uint64_t h = MurmurHash64A(&key, sizeof(key), hash_seed); 88 | int word_off = -1; 89 | uint64_t word_mask = make_word_mask(h, word_off); 90 | words[word_off] |= word_mask; 91 | } 92 | 93 | void insert(const key_type &key) 94 | { 95 | uint64_t h = MurmurHash64A(&key, sizeof(key), hash_seed); 96 | int word_off = -1; 97 | uint64_t word_mask = make_word_mask(h, word_off); 98 | __sync_fetch_and_or(&words[word_off], word_mask); 99 | } 100 | 101 | bool check(const key_type &key) 102 | { 103 | uint64_t h = MurmurHash64A(&key, sizeof(key), hash_seed); 104 | int word_off = -1; 105 | uint64_t word_mask = make_word_mask(h, word_off); 106 | return (words[word_off] & word_mask) == word_mask; 107 | } 108 | 109 | int entries; 110 | int bits_m; 111 | int bytes_n; 112 | int words_l; 113 | int hash_bits; 114 | word_t *words; 115 | double error; 116 | }; 117 | 118 | template 119 | class bloom_opt 120 | { 121 | public: 122 | using key_type = Key; 123 | bloom_opt(int entries, double fp_rate) 124 | { 125 | this->entries = entries; 126 | bloom_init(&bloom, entries, fp_rate); 127 | this->bits_m = bloom.bits; 128 | this->bytes_n = this->bits_m / 8; 129 | this->words_l = this->bytes_n / 8; 130 | this->k = bloom.hashes; 131 | } 132 | 133 | ~bloom_opt() { bloom_free(&bloom); } 134 | 135 | void insert_unsafe(const key_type &key) 136 | { 137 | bloom_add_nonatomic(&bloom, &key, sizeof(key)); 138 | } 139 | 140 | void insert(const key_type &key) { bloom_add(&bloom, &key, sizeof(key)); } 141 | 142 | bool check(const key_type &key) 143 | { 144 | return bloom_check(&bloom, &key, sizeof(key)); 145 | } 146 | struct bloom bloom; 147 | int entries; 148 | int bits_m; 149 | int bytes_n; 150 | int words_l; 151 | int k; 152 | }; -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef USE_PAPI 12 | #include 13 | #endif 14 | #ifdef NVM_DRAM_MODE 15 | #include "pmalloc.h" 16 | #endif 17 | 18 | #define END_PADDING_SIZE(x) ((cacheline_size - ((x) % cacheline_size)) % cacheline_size) 19 | #define ALIGN(addr, alignment) ((char *)((unsigned long)(addr) & ~((alignment) - 1))) 20 | #define CACHELINE_ALIGN(addr) ALIGN(addr, 64) 21 | extern std::vector keys; 22 | extern std::vector insert_more_keys; 23 | extern std::vector lookupKeys; 24 | extern std::vector sortedKeys; 25 | extern std::vector notExistKeys; 26 | extern unsigned long write_latency_in_ns; 27 | extern unsigned long CPU_FREQ_MHZ; 28 | extern unsigned long long cycles_total; 29 | extern int parallel_merge_worker_num; 30 | static constexpr int cacheline_size = 64; 31 | unsigned long long cycles_now(); 32 | unsigned ctz(unsigned); 33 | void clear_cache(); 34 | double secs_now(void); 35 | void print_flush_stat(); 36 | void percentile_report(const std::string & name, std::vector & nums); 37 | double measure(std::function f, const std::string & bench_name, unsigned iteration = 1, bool print_mem = true); 38 | void prepareKeys(int n, const std::string & keyFile, bool sparseKey, bool sorted = true); 39 | void prepareInsertMoreKeys(int n, const std::string &keyFile, bool sparseKey, bool sorted); 40 | void cpu_pause(); 41 | void mfence(); 42 | void sfence(); 43 | void clflush(volatile void *p); 44 | void clflush_then_sfence(volatile void *p); 45 | void clflush_len(volatile void *data, int len); 46 | void clflush_len_no_fence(volatile void *data, int len); 47 | void prefetch(char *ptr, size_t len); 48 | int nvm_dram_alloc(void **ptr, size_t align, size_t size); 49 | void nvm_dram_free(void * ptr, size_t size); 50 | 51 | int wbinvd(); 52 | #ifndef CACHE_LINE_SIZE 53 | #define CACHE_LINE_SIZE 64 // 64 byte cache line on x86 and x86-64 54 | #endif 55 | 56 | #define INT64S_PER_CACHELINE (CACHE_LINE_SIZE / sizeof(int64_t)) 57 | #define INT64S_PER_CACHELINE_SCALE 4 58 | 59 | static thread_local bool cpuIdInitialized; 60 | static thread_local uint64_t cpuId; 61 | template 62 | class DistributedCounter2 { 63 | public: 64 | static_assert(buckets == 0 || (buckets & (buckets - 1)) == 0, "buckets must be a multiple of 2"); 65 | 66 | DistributedCounter2(int initVal = 0) { 67 | countArrayPtr = malloc(buckets * INT64S_PER_CACHELINE * sizeof(int64_t) + CACHE_LINE_SIZE - 1); 68 | memset(countArrayPtr, 0, buckets * INT64S_PER_CACHELINE * sizeof(int64_t) + CACHE_LINE_SIZE - 1); 69 | countArray = (int64_t *)(((size_t)countArrayPtr + CACHE_LINE_SIZE - 1) & ~(CACHE_LINE_SIZE - 1)); 70 | increment(initVal); 71 | } 72 | 73 | ~DistributedCounter2() { 74 | free(countArrayPtr); 75 | } 76 | inline void increment(int v = 1) { 77 | __atomic_add_fetch(&countArray[arrayIndex() * INT64S_PER_CACHELINE], v, __ATOMIC_RELAXED); 78 | } 79 | 80 | inline void decrement(int v = 1) { 81 | __atomic_sub_fetch(&countArray[arrayIndex() * INT64S_PER_CACHELINE], v, __ATOMIC_RELAXED); 82 | } 83 | 84 | int64_t get() { 85 | int64_t val = 0; 86 | for (int i = 0; i < totalINT64S; i += INT64S_PER_CACHELINE) { 87 | val += __atomic_load_n(&countArray[i], __ATOMIC_RELAXED); 88 | } 89 | return val; 90 | } 91 | 92 | int64_t get_unsafe() { 93 | int64_t val = 0; 94 | for (int i = 0; i < totalINT64S; i += INT64S_PER_CACHELINE) { 95 | val += countArray[i]; 96 | } 97 | return val; 98 | } 99 | private: 100 | static constexpr int totalINT64S = buckets * INT64S_PER_CACHELINE; 101 | inline uint64_t getCPUId() { 102 | if (cpuIdInitialized == false) { 103 | cpuId = (uint64_t)std::hash{}(std::this_thread::get_id()); 104 | cpuIdInitialized = true; 105 | //printf("cpuid %lu, arrayIndex %d, pointer %p\n", cpuId, arrayIndex(), &countArray[arrayIndex() * INT64S_PER_CACHELINE]); 106 | } 107 | return cpuId; 108 | } 109 | 110 | inline int arrayIndex() { 111 | return getCPUId() & (buckets - 1); 112 | } 113 | 114 | int64_t * countArray; 115 | void * countArrayPtr; 116 | }; 117 | 118 | 119 | class elided_spin_lock { 120 | private: 121 | int lockvar; 122 | char padding[END_PADDING_SIZE(sizeof(lockvar))]; 123 | public: 124 | elided_spin_lock(): lockvar(0) {} 125 | 126 | void lock() { 127 | /* Acquire lock with lock elision */ 128 | while (__atomic_exchange_n(&lockvar, 1, __ATOMIC_ACQUIRE|__ATOMIC_HLE_ACQUIRE)) 129 | _mm_pause(); /* Abort failed transaction */ 130 | } 131 | 132 | void unlock() { 133 | __atomic_store_n(&lockvar, 0, __ATOMIC_RELEASE|__ATOMIC_HLE_RELEASE); 134 | } 135 | }; -------------------------------------------------------------------------------- /include/bloom.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012-2017, Jyri J. Virkki 3 | * All rights reserved. 4 | * 5 | * This file is under BSD license. See LICENSE file. 6 | */ 7 | 8 | #ifndef _BLOOM_H 9 | #define _BLOOM_H 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | 16 | /** *************************************************************************** 17 | * Structure to keep track of one bloom filter. Caller needs to 18 | * allocate this and pass it to the functions below. First call for 19 | * every struct must be to bloom_init(). 20 | * 21 | */ 22 | struct bloom 23 | { 24 | // These fields are part of the public interface of this structure. 25 | // Client code may read these values if desired. Client code MUST NOT 26 | // modify any of these. 27 | int entries; 28 | double error; 29 | int bits; 30 | int bytes; 31 | int hashes; 32 | 33 | // Fields below are private to the implementation. These may go away or 34 | // change incompatibly at any moment. Client code MUST NOT access or rely 35 | // on these. 36 | double bpe; 37 | unsigned char * bf; 38 | int ready; 39 | }; 40 | 41 | 42 | /** *************************************************************************** 43 | * Initialize the bloom filter for use. 44 | * 45 | * The filter is initialized with a bit field and number of hash functions 46 | * according to the computations from the wikipedia entry: 47 | * http://en.wikipedia.org/wiki/Bloom_filter 48 | * 49 | * Optimal number of bits is: 50 | * bits = (entries * ln(error)) / ln(2)^2 51 | * 52 | * Optimal number of hash functions is: 53 | * hashes = bpe * ln(2) 54 | * 55 | * Parameters: 56 | * ----------- 57 | * bloom - Pointer to an allocated struct bloom (see above). 58 | * entries - The expected number of entries which will be inserted. 59 | * Must be at least 1000 (in practice, likely much larger). 60 | * error - Probability of collision (as long as entries are not 61 | * exceeded). 62 | * 63 | * Return: 64 | * ------- 65 | * 0 - on success 66 | * 1 - on failure 67 | * 68 | */ 69 | int bloom_init(struct bloom * bloom, int entries, double error); 70 | 71 | 72 | /** *************************************************************************** 73 | * Deprecated, use bloom_init() 74 | * 75 | */ 76 | int bloom_init_size(struct bloom * bloom, int entries, double error, 77 | unsigned int cache_size); 78 | 79 | 80 | /** *************************************************************************** 81 | * Check if the given element is in the bloom filter. Remember this may 82 | * return false positive if a collision occured. 83 | * 84 | * Parameters: 85 | * ----------- 86 | * bloom - Pointer to an allocated struct bloom (see above). 87 | * buffer - Pointer to buffer containing element to check. 88 | * len - Size of 'buffer'. 89 | * 90 | * Return: 91 | * ------- 92 | * 0 - element is not present 93 | * 1 - element is present (or false positive due to collision) 94 | * -1 - bloom not initialized 95 | * 96 | */ 97 | int bloom_check(struct bloom * bloom, const void * buffer, int len); 98 | 99 | 100 | /** *************************************************************************** 101 | * Add the given element to the bloom filter. 102 | * The return code indicates if the element (or a collision) was already in, 103 | * so for the common check+add use case, no need to call check separately. 104 | * 105 | * Parameters: 106 | * ----------- 107 | * bloom - Pointer to an allocated struct bloom (see above). 108 | * buffer - Pointer to buffer containing element to add. 109 | * len - Size of 'buffer'. 110 | * 111 | * Return: 112 | * ------- 113 | * 0 - element was not present and was added 114 | * 1 - element (or a collision) had already been added previously 115 | * -1 - bloom not initialized 116 | * 117 | */ 118 | void bloom_add(struct bloom * bloom, const void * buffer, int len); 119 | 120 | 121 | /** *************************************************************************** 122 | * Print (to stdout) info about this bloom filter. Debugging aid. 123 | * 124 | */ 125 | void bloom_print(struct bloom * bloom); 126 | 127 | 128 | /** *************************************************************************** 129 | * Deallocate internal storage. 130 | * 131 | * Upon return, the bloom struct is no longer usable. You may call bloom_init 132 | * again on the same struct to reinitialize it again. 133 | * 134 | * Parameters: 135 | * ----------- 136 | * bloom - Pointer to an allocated struct bloom (see above). 137 | * 138 | * Return: none 139 | * 140 | */ 141 | void bloom_free(struct bloom * bloom); 142 | 143 | 144 | void bloom_add_nonatomic(struct bloom * bloom, const void * buffer, int len); 145 | 146 | /** *************************************************************************** 147 | * Returns version string compiled into library. 148 | * 149 | * Return: version string 150 | * 151 | */ 152 | const char * bloom_version(); 153 | 154 | #ifdef __cplusplus 155 | } 156 | #endif 157 | 158 | #endif 159 | -------------------------------------------------------------------------------- /misc/ARTOLC/Epoche.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by florian on 22.10.15. 3 | // 4 | #ifndef EPOCHE_CPP 5 | #define EPOCHE_CPP 6 | 7 | #include 8 | #include 9 | #include "Epoche.h" 10 | using namespace ART; 11 | 12 | 13 | inline DeletionList::~DeletionList() { 14 | assert(deletitionListCount == 0 && headDeletionList == nullptr); 15 | LabelDelete *cur = nullptr, *next = freeLabelDeletes; 16 | while (next != nullptr) { 17 | cur = next; 18 | next = cur->next; 19 | delete cur; 20 | } 21 | freeLabelDeletes = nullptr; 22 | } 23 | 24 | inline std::size_t DeletionList::size() { 25 | return deletitionListCount; 26 | } 27 | 28 | inline void DeletionList::remove(LabelDelete *label, LabelDelete *prev) { 29 | if (prev == nullptr) { 30 | headDeletionList = label->next; 31 | } else { 32 | prev->next = label->next; 33 | } 34 | deletitionListCount -= label->nodesCount; 35 | 36 | label->next = freeLabelDeletes; 37 | freeLabelDeletes = label; 38 | deleted += label->nodesCount; 39 | } 40 | 41 | inline void DeletionList::add(void *n, uint64_t globalEpoch) { 42 | deletitionListCount++; 43 | LabelDelete *label; 44 | if (headDeletionList != nullptr && headDeletionList->nodesCount < headDeletionList->nodes.size()) { 45 | label = headDeletionList; 46 | } else { 47 | if (freeLabelDeletes != nullptr) { 48 | label = freeLabelDeletes; 49 | freeLabelDeletes = freeLabelDeletes->next; 50 | } else { 51 | label = new LabelDelete(); 52 | } 53 | label->nodesCount = 0; 54 | label->next = headDeletionList; 55 | headDeletionList = label; 56 | } 57 | label->nodes[label->nodesCount] = n; 58 | label->nodesCount++; 59 | label->epoche = globalEpoch; 60 | 61 | added++; 62 | } 63 | 64 | inline LabelDelete *DeletionList::head() { 65 | return headDeletionList; 66 | } 67 | 68 | inline void Epoche::enterEpoche(ThreadInfo &epocheInfo) { 69 | unsigned long curEpoche = currentEpoche.load(std::memory_order_relaxed); 70 | epocheInfo.getDeletionList().localEpoche.store(curEpoche, std::memory_order_release); 71 | } 72 | 73 | inline void Epoche::markNodeForDeletion(void *n, ThreadInfo &epocheInfo) { 74 | epocheInfo.getDeletionList().add(n, currentEpoche.load()); 75 | epocheInfo.getDeletionList().thresholdCounter++; 76 | } 77 | 78 | inline void Epoche::exitEpocheAndCleanup(ThreadInfo &epocheInfo) { 79 | DeletionList &deletionList = epocheInfo.getDeletionList(); 80 | if ((deletionList.thresholdCounter & (64 - 1)) == 1) { 81 | currentEpoche++; 82 | } 83 | if (deletionList.thresholdCounter > startGCThreshhold) { 84 | if (deletionList.size() == 0) { 85 | deletionList.thresholdCounter = 0; 86 | return; 87 | } 88 | deletionList.localEpoche.store(std::numeric_limits::max()); 89 | 90 | uint64_t oldestEpoche = std::numeric_limits::max(); 91 | for (auto &epoche : deletionLists) { 92 | auto e = epoche.localEpoche.load(); 93 | if (e < oldestEpoche) { 94 | oldestEpoche = e; 95 | } 96 | } 97 | 98 | LabelDelete *cur = deletionList.head(), *next, *prev = nullptr; 99 | while (cur != nullptr) { 100 | next = cur->next; 101 | 102 | if (cur->epoche < oldestEpoche) { 103 | for (std::size_t i = 0; i < cur->nodesCount; ++i) { 104 | operator delete(cur->nodes[i]); 105 | } 106 | deletionList.remove(cur, prev); 107 | } else { 108 | prev = cur; 109 | } 110 | cur = next; 111 | } 112 | deletionList.thresholdCounter = 0; 113 | } 114 | } 115 | 116 | inline Epoche::~Epoche() { 117 | uint64_t oldestEpoche = std::numeric_limits::max(); 118 | for (auto &epoche : deletionLists) { 119 | auto e = epoche.localEpoche.load(); 120 | if (e < oldestEpoche) { 121 | oldestEpoche = e; 122 | } 123 | } 124 | for (auto &d : deletionLists) { 125 | LabelDelete *cur = d.head(), *next, *prev = nullptr; 126 | while (cur != nullptr) { 127 | next = cur->next; 128 | 129 | assert(cur->epoche < oldestEpoche); 130 | for (std::size_t i = 0; i < cur->nodesCount; ++i) { 131 | operator delete(cur->nodes[i]); 132 | } 133 | d.remove(cur, prev); 134 | cur = next; 135 | } 136 | } 137 | } 138 | 139 | inline void Epoche::showDeleteRatio() { 140 | for (auto &d : deletionLists) { 141 | std::cout << "deleted " << d.deleted << " of " << d.added << std::endl; 142 | } 143 | } 144 | 145 | inline ThreadInfo::ThreadInfo(Epoche &epoche) 146 | : epoche(epoche), deletionList(epoche.deletionLists.local()) { } 147 | 148 | inline DeletionList &ThreadInfo::getDeletionList() const { 149 | return deletionList; 150 | } 151 | 152 | inline Epoche &ThreadInfo::getEpoche() const { 153 | return epoche; 154 | } 155 | 156 | #endif //EPOCHE_CPP -------------------------------------------------------------------------------- /misc/ARTOLC/ARTOLC.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Tree.h" 3 | #include "index_key.h" 4 | 5 | template 6 | class ArtOLCIndex 7 | { 8 | public: 9 | ~ArtOLCIndex() { delete idx; } 10 | 11 | 12 | static void setKey(Key &k, uint64_t key) { k.setInt(key); } 13 | static void setKey(Key &k, GenericKey<31> key) { k.set(key.data, 31); } 14 | 15 | template 16 | bool insert(KeyType key, uint64_t value, F f) 17 | { 18 | auto t = idx->getThreadInfo(); 19 | Key k; 20 | setKey(k, key); 21 | idx->insert(k, value, t, f); 22 | return true; 23 | } 24 | 25 | bool find(KeyType key, uint64_t & v) 26 | { 27 | auto t = idx->getThreadInfo(); 28 | Key k; 29 | setKey(k, key); 30 | uint64_t result = idx->lookup(k, t); 31 | v = result; 32 | return true; 33 | } 34 | template 35 | bool upsert(KeyType key, uint64_t value, F f) 36 | { 37 | auto t = idx->getThreadInfo(); 38 | Key k; 39 | setKey(k, key); 40 | idx->insert(k, value, t, f); 41 | return true; 42 | } 43 | 44 | bool scan(const KeyType & start_key, int range, KeyType & continue_key, TID results[], size_t & result_count) 45 | { 46 | auto t = idx->getThreadInfo(); 47 | Key startKey; 48 | startKey.setInt(start_key); 49 | result_count = 0; 50 | Key continueKey; 51 | bool has_more = idx->lookupRange(startKey, maxKey, continueKey, results, range, result_count, 52 | t); 53 | continue_key = continueKey.getInt(); 54 | return has_more; 55 | } 56 | 57 | 58 | ArtOLCIndex(uint64_t kt) 59 | { 60 | if (sizeof(KeyType) == 8) 61 | { 62 | idx = new ART_OLC::Tree([](TID tid, Key &key) { 63 | key.setInt(*reinterpret_cast(tid)); 64 | }); 65 | maxKey.setInt(~0ull); 66 | } 67 | else 68 | { 69 | idx = new ART_OLC::Tree([](TID tid, Key &key) { 70 | key.set(reinterpret_cast(tid), 31); 71 | }); 72 | uint8_t m[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 73 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 74 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 75 | 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; 76 | maxKey.set((char *)m, 31); 77 | } 78 | } 79 | 80 | 81 | struct iterator { 82 | static const uint64_t scan_unit_size = 64; 83 | ArtOLCIndex *artindex; 84 | int count; 85 | int pos; 86 | bool nomore; 87 | KeyType continue_key; 88 | KeyType start_key; 89 | KeyType cur_key; 90 | uint64_t values[scan_unit_size]; 91 | 92 | explicit iterator(ArtOLCIndex *artindex=nullptr) 93 | : artindex(artindex), count(0), pos(0), nomore(false), start_key(0), cur_key(std::numeric_limits::max()), continue_key(0){} 94 | 95 | explicit iterator(ArtOLCIndex *artindex, const KeyType & startKey) 96 | : artindex(artindex), count(0), pos(0), nomore(false), start_key(startKey), cur_key(std::numeric_limits::max()), continue_key(0){ 97 | while (nomore == false && count == 0) 98 | fill(); 99 | if (is_end()) { 100 | cur_key = std::numeric_limits::max(); 101 | } else { 102 | cur_key = *(KeyType*)values[pos]; 103 | } 104 | } 105 | 106 | void fill() { 107 | if (nomore == false) { 108 | start_key = continue_key; 109 | size_t result_count = 0; 110 | nomore = !artindex->scan(start_key, scan_unit_size, continue_key, values, result_count); 111 | count = result_count; 112 | } else { 113 | count = 0; 114 | } 115 | pos = 0; 116 | } 117 | 118 | int next_node(KeyType&last_key) { 119 | last_key = this->last_key(); 120 | int ret = count; 121 | fill(); 122 | return ret; 123 | } 124 | 125 | bool is_end() { return count == 0 && nomore == true; } 126 | 127 | iterator &operator++() { 128 | if (++pos == count) { 129 | fill(); 130 | } 131 | if (is_end()) { 132 | cur_key = std::numeric_limits::max(); 133 | } else { 134 | cur_key = *(KeyType*)values[pos]; 135 | } 136 | return *this; 137 | } 138 | 139 | bool operator==(const iterator &rhs) { 140 | return cur_key == rhs.cur_key; 141 | } 142 | 143 | bool operator!=(const iterator &rhs) { 144 | return !operator==(rhs); 145 | } 146 | 147 | KeyType last_key() { return *(KeyType *) values[count - 1]; } 148 | 149 | KeyType key() { return *(KeyType *) values[pos]; } 150 | 151 | uint64_t value() { return values[pos]; } 152 | }; 153 | 154 | iterator begin() { 155 | return iterator(this, std::numeric_limits::min()); 156 | } 157 | iterator end() { 158 | return iterator(this); 159 | } 160 | 161 | iterator lookup_range(const KeyType & start_key) { 162 | return iterator(this, start_key); 163 | } 164 | private: 165 | Key maxKey; 166 | ART_OLC::Tree *idx; 167 | }; 168 | -------------------------------------------------------------------------------- /test/dptree_test.cxx: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | using namespace std; 15 | 16 | 17 | float range_sizes[] = {0.0001, 0.001, 0.01}; 18 | const int repeatn = 50000000; 19 | uint64_t cacheline[8]; 20 | unsigned lookup_test_iteration = 1; 21 | 22 | 23 | uint64_t make_upsert_value(uint64_t v) { return (v << 1); } 24 | 25 | uint64_t strip_upsert_value(uint64_t v) { return v >> 1; } 26 | 27 | void dpftree_cvhftree_test() { 28 | dtree::dpftree index; 29 | int repeat = 1; 30 | int c = 0; 31 | measure( 32 | [&]() -> unsigned { 33 | size_t i = 0; 34 | while (i < keys.size()) { 35 | index.insert(keys[i], keys[i]); 36 | i++; 37 | } 38 | printf("# merges: %d\n", index.get_merges()); 39 | printf("merge time: %f secs\n", index.get_merge_time()); 40 | printf("real merge time: %f secs\n", index.get_real_merge_time()); 41 | printf("inner nodes build time: %f secs\n", 42 | index.get_inner_node_build_time()); 43 | printf("buffer tree size: %d\n", (int) index.get_buffer_tree_size()); 44 | printf("buffer tree node count: %d\n", (int) index.get_buffer_tree_node_count()); 45 | printf("base tree leaf count: %d\n", (int) index.get_base_tree_leaf_count()); 46 | printf("wal entry count: %d\n", (int) index.get_wal_entry_count()); 47 | printf("bloom entries: %d\n", index.get_bloom_entries()); 48 | printf("bloom bytes: %d\n", index.get_bloom_bytes()); 49 | printf("bloom hashes: %d\n", index.get_bloom_hashes()); 50 | printf("bloom words: %d\n", index.get_bloom_words()); 51 | return keys.size(); 52 | }, 53 | "dptree-cvhtree-insert", 1, true); 54 | clear_cache(); 55 | while (index.merging()); 56 | 57 | double avg_per_search = measure( 58 | [&]() -> unsigned { 59 | repeat = repeatn / lookupKeys.size() / 2; 60 | if (repeat < 1) 61 | repeat = 1; 62 | auto static_lookup_count_save = index.static_lookup_count; 63 | auto dyn_lookup_count_save = index.dyn_lookup_count; 64 | 65 | for (uint64_t r = 0; r < repeat; ++r) { 66 | for (size_t i = 0; i < lookupKeys.size(); i++) { 67 | uint64_t key = lookupKeys[i]; 68 | uint64_t value = 0; 69 | bool res = index.lookup(key, value); 70 | //probes_vec.push_back(index.get_probes() - probes_save); 71 | //assert(res); 72 | assert(key == strip_upsert_value(value)); 73 | } 74 | } 75 | auto static_lookup_count = 76 | index.static_lookup_count - static_lookup_count_save; 77 | auto dynamic_lookup_count = 78 | index.dyn_lookup_count - dyn_lookup_count_save; 79 | auto probes = index.get_probes(); 80 | printf("dynamic lookup count: %d\n", dynamic_lookup_count); 81 | printf("static lookup count: %d\n", static_lookup_count); 82 | printf("avg probes %f\n", probes / (repeat * lookupKeys.size() + 0.0)); 83 | return repeat * lookupKeys.size(); 84 | }, 85 | "dpftree-cvhftree-search", lookup_test_iteration); 86 | index.clear_probes(); 87 | 88 | uint64_t s = 0; 89 | uint64_t rc = 0; 90 | for (int i = 0; i < sizeof(range_sizes) / sizeof(float); ++i) { 91 | clear_cache(); 92 | int range_size = range_sizes[i] * keys.size(); 93 | printf("range size: %d\n", range_size); 94 | measure( 95 | [&]() -> unsigned { 96 | repeat = 5; 97 | for (int r = 0; r < repeat; ++r) { 98 | for (size_t i = 0; i < sortedKeys.size();) { 99 | uint64_t startPos = rand() % keys.size(); 100 | uint64_t endPos = 101 | std::min(startPos + range_size, (uint64_t) keys.size() - 1); 102 | uint64_t keyStart = sortedKeys[startPos]; 103 | uint64_t keyEnd = sortedKeys[endPos]; 104 | std::vector res; 105 | res.reserve(endPos - startPos); 106 | index.lookup_range(keyStart, keyEnd, res); 107 | assert(res.size() == endPos - startPos); 108 | for (int j = 0; j < res.size(); ++j) { 109 | if (strip_upsert_value(res[j]) != sortedKeys[startPos + j]) { 110 | assert(false); 111 | } 112 | s += res[j]; 113 | ++rc; 114 | } 115 | i += endPos - startPos; 116 | } 117 | } 118 | return repeat * sortedKeys.size(); 119 | }, 120 | "dpftree-cvhftree-scan-" + std::to_string(range_sizes[i])); 121 | } 122 | printf("%lu %lu\n", s, rc); 123 | } 124 | 125 | int main(int argc, char const *argv[]) 126 | { 127 | int n = 10000000; 128 | bool sparseKey = false; 129 | string keyFile = "zipf-keys-s1.0.csv"; 130 | if (argc > 1) 131 | n = atoi(argv[1]); 132 | if (argc > 2) 133 | keyFile = argv[2]; 134 | if (argc > 3) 135 | sparseKey = atoi(argv[3]); 136 | if (argc > 4) 137 | write_latency_in_ns = atoi(argv[4]); 138 | printf("started sparseKey %d\n", sparseKey); 139 | prepareKeys(n, keyFile, sparseKey, false); 140 | dpftree_cvhftree_test(); 141 | return 0; 142 | } -------------------------------------------------------------------------------- /src/bloom.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012-2017, Jyri J. Virkki 3 | * All rights reserved. 4 | * 5 | * This file is under BSD license. See LICENSE file. 6 | */ 7 | 8 | /* 9 | * Refer to bloom.h for documentation on the public interfaces. 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #define MAKESTRING(n) STRING(n) 24 | #define STRING(n) #n 25 | 26 | #include "../include/bloom.h" 27 | 28 | //----------------------------------------------------------------------------- 29 | // MurmurHash2, by Austin Appleby 30 | 31 | // Note - This code makes a few assumptions about how your machine behaves - 32 | 33 | // 1. We can read a 4-byte value from any address without crashing 34 | // 2. sizeof(int) == 4 35 | 36 | // And it has a few limitations - 37 | 38 | // 1. It will not work incrementally. 39 | // 2. It will not produce the same results on little-endian and big-endian 40 | // machines. 41 | 42 | unsigned int murmurhash2(const void *key, int len, const unsigned int seed) 43 | { 44 | // 'm' and 'r' are mixing constants generated offline. 45 | // They're not really 'magic', they just happen to work well. 46 | 47 | const unsigned int m = 0x5bd1e995; 48 | const int r = 24; 49 | 50 | // Initialize the hash to a 'random' value 51 | 52 | unsigned int h = seed ^ len; 53 | 54 | // Mix 4 bytes at a time into the hash 55 | 56 | const unsigned char *data = (const unsigned char *)key; 57 | 58 | while (len >= 4) 59 | { 60 | unsigned int k = *(unsigned int *)data; 61 | 62 | k *= m; 63 | k ^= k >> r; 64 | k *= m; 65 | 66 | h *= m; 67 | h ^= k; 68 | 69 | data += 4; 70 | len -= 4; 71 | } 72 | 73 | // Handle the last few bytes of the input array 74 | 75 | switch (len) 76 | { 77 | case 3: 78 | h ^= data[2] << 16; 79 | case 2: 80 | h ^= data[1] << 8; 81 | case 1: 82 | h ^= data[0]; 83 | h *= m; 84 | }; 85 | 86 | // Do a few final mixes of the hash to ensure the last few 87 | // bytes are well-incorporated. 88 | 89 | h ^= h >> 13; 90 | h *= m; 91 | h ^= h >> 15; 92 | 93 | return h; 94 | } 95 | 96 | inline static int test_bit_set_bit_nonatomic(unsigned char *buf, unsigned int x, 97 | int set_bit) 98 | { 99 | unsigned char c; 100 | unsigned int byte = x >> 3; 101 | unsigned int mask = 1 << (x % 8); 102 | c = buf[byte]; 103 | if (c & mask) 104 | { 105 | return 1; 106 | } 107 | else if (set_bit) 108 | { 109 | buf[byte] = c | mask; 110 | } 111 | return 0; 112 | } 113 | 114 | 115 | inline static int set_bit(unsigned char *buf, unsigned int x) 116 | { 117 | unsigned char c; 118 | unsigned int byte = x >> 3; 119 | unsigned char mask = 1 << (x % 8); 120 | __sync_fetch_and_or(&buf[byte], mask); 121 | unsigned char new_val = c | mask; 122 | return 0; 123 | } 124 | 125 | inline static int test_bit_set_bit(unsigned char *buf, unsigned int x, 126 | int set_bit) 127 | { 128 | unsigned char c; 129 | unsigned int byte = x >> 3; 130 | unsigned char mask = 1 << (x % 8); 131 | c = buf[byte]; 132 | if (c & mask) 133 | { 134 | return 1; 135 | } 136 | else if (set_bit) 137 | { 138 | __sync_fetch_and_or(&buf[byte], mask); 139 | unsigned char new_val = c | mask; 140 | } 141 | return 0; 142 | } 143 | 144 | static int bloom_check_add(struct bloom *bloom, const void *buffer, int len, 145 | int add) 146 | { 147 | if (bloom->ready == 0) 148 | { 149 | printf("bloom at %p not initialized!\n", (void *)bloom); 150 | return -1; 151 | } 152 | 153 | int hits = 0; 154 | register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); 155 | register unsigned int b = murmurhash2(buffer, len, a); 156 | register unsigned int x; 157 | register unsigned int i; 158 | 159 | for (i = 0; i < bloom->hashes; i++) 160 | { 161 | x = (a + i * b) % bloom->bits; 162 | if (test_bit_set_bit(bloom->bf, x, add)) 163 | { 164 | hits++; 165 | } 166 | } 167 | 168 | if (hits == bloom->hashes) 169 | { 170 | return 1; // 1 == element already in (or collision) 171 | } 172 | 173 | return 0; 174 | } 175 | 176 | 177 | static void bloom_add_(struct bloom *bloom, const void *buffer, int len) 178 | { 179 | if (bloom->ready == 0) 180 | { 181 | printf("bloom at %p not initialized!\n", (void *)bloom); 182 | } 183 | 184 | int hits = 0; 185 | register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); 186 | register unsigned int b = murmurhash2(buffer, len, a); 187 | register unsigned int x; 188 | register unsigned int i; 189 | 190 | for (i = 0; i < bloom->hashes; i++) 191 | { 192 | x = (a + i * b) % bloom->bits; 193 | set_bit(bloom->bf, x); 194 | } 195 | 196 | } 197 | 198 | int bloom_init_size(struct bloom *bloom, int entries, double error, 199 | unsigned int cache_size) 200 | { 201 | return bloom_init(bloom, entries, error); 202 | } 203 | 204 | int bloom_init(struct bloom *bloom, int entries, double error) 205 | { 206 | bloom->ready = 0; 207 | 208 | if (entries < 1000 || error == 0) 209 | { 210 | return 1; 211 | } 212 | 213 | bloom->entries = entries; 214 | bloom->error = error; 215 | 216 | double num = log(bloom->error); 217 | double denom = 0.480453013918201; // ln(2)^2 218 | bloom->bpe = -(num / denom); 219 | 220 | double dentries = (double)entries; 221 | bloom->bits = (int)(dentries * bloom->bpe); 222 | 223 | if (bloom->bits % 8) 224 | { 225 | bloom->bytes = (bloom->bits / 8) + 1; 226 | } 227 | else 228 | { 229 | bloom->bytes = bloom->bits / 8; 230 | } 231 | 232 | bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2) 233 | 234 | bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char)); 235 | if (bloom->bf == NULL) 236 | { 237 | return 1; 238 | } 239 | 240 | bloom->ready = 1; 241 | return 0; 242 | } 243 | 244 | int bloom_check(struct bloom *bloom, const void *buffer, int len) 245 | { 246 | return bloom_check_add(bloom, buffer, len, 0); 247 | } 248 | 249 | void bloom_add(struct bloom *bloom, const void *buffer, int len) 250 | { 251 | bloom_add_(bloom, buffer, len); 252 | } 253 | 254 | void bloom_add_nonatomic(struct bloom *bloom, const void *buffer, int len) 255 | { 256 | bloom_add_(bloom, buffer, len); 257 | } 258 | 259 | void bloom_print(struct bloom *bloom) 260 | { 261 | printf("bloom at %p\n", (void *)bloom); 262 | printf(" ->entries = %d\n", bloom->entries); 263 | printf(" ->error = %f\n", bloom->error); 264 | printf(" ->bits = %d\n", bloom->bits); 265 | printf(" ->bits per elem = %f\n", bloom->bpe); 266 | printf(" ->bytes = %d\n", bloom->bytes); 267 | printf(" ->hash functions = %d\n", bloom->hashes); 268 | } 269 | 270 | void bloom_free(struct bloom *bloom) 271 | { 272 | if (bloom->ready) 273 | { 274 | free(bloom->bf); 275 | } 276 | bloom->ready = 0; 277 | } 278 | 279 | const char *bloom_version() { return MAKESTRING(BLOOM_VERSION); } 280 | -------------------------------------------------------------------------------- /include/ART.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Adaptive Radix Tree 3 | Viktor Leis, 2012 4 | leis@in.tum.de 5 | 6 | Modified by Huanchen Zhang, 2016 7 | Modified by Xinjing Zhou, 2018 8 | */ 9 | 10 | #include // malloc, free 11 | #include // memset, memcpy 12 | #include // integer types 13 | #include // x86 SSE intrinsics 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace std; 23 | 24 | // Constants for the node types 25 | static const int8_t NodeType4=0; 26 | static const int8_t NodeType16=1; 27 | static const int8_t NodeType48=2; 28 | static const int8_t NodeType256=3; 29 | 30 | // The maximum prefix length for compressed paths stored in the 31 | // header, if the path is longer it is loaded from the database on 32 | // demand 33 | static const unsigned maxPrefixLength=9; 34 | 35 | static const unsigned NodeDItemTHold=227; 36 | 37 | // Shared header of all inner nodes 38 | struct Node { 39 | // length of the compressed path (prefix) 40 | uint32_t prefixLength; 41 | // number of non-null children 42 | uint16_t count; 43 | // node type 44 | int8_t type; 45 | // compressed path (prefix) 46 | uint8_t prefix[maxPrefixLength]; 47 | 48 | Node(int8_t type) : prefixLength(0),count(0),type(type) {} 49 | }; 50 | 51 | // Node with up to 4 children 52 | struct Node4 : Node { 53 | uint8_t key[4]; 54 | Node* child[4]; 55 | 56 | Node4() : Node(NodeType4) { 57 | memset(key,0,sizeof(key)); 58 | memset(child,0,sizeof(child)); 59 | } 60 | }; 61 | 62 | // Node with up to 16 children 63 | struct Node16 : Node { 64 | uint8_t key[16]; 65 | Node* child[16]; 66 | 67 | Node16() : Node(NodeType16) { 68 | memset(key,0,sizeof(key)); 69 | memset(child,0,sizeof(child)); 70 | } 71 | }; 72 | 73 | static const uint8_t emptyMarker=48; 74 | 75 | // Node with up to 48 children 76 | struct Node48 : Node { 77 | uint8_t childIndex[256]; 78 | Node* child[48]; 79 | 80 | Node48() : Node(NodeType48) { 81 | memset(childIndex,emptyMarker,sizeof(childIndex)); 82 | memset(child,0,sizeof(child)); 83 | } 84 | }; 85 | 86 | // Node with up to 256 children 87 | struct Node256 : Node { 88 | Node* child[256]; 89 | 90 | Node256() : Node(NodeType256) { 91 | memset(child,0,sizeof(child)); 92 | } 93 | }; 94 | 95 | typedef struct { 96 | Node* node; 97 | uint16_t cursor; 98 | } NodeCursor; 99 | 100 | class ART; 101 | class ARTIter; 102 | 103 | 104 | class ART { 105 | 106 | public: 107 | static inline Node* makeLeaf(uintptr_t tid){ 108 | // Create a pseudo-leaf 109 | return reinterpret_cast((tid<<1)|1); 110 | } 111 | 112 | static inline uintptr_t getLeafValue(Node* node){ 113 | // The the value stored in the pseudo-leaf 114 | return reinterpret_cast(node)>>1; 115 | } 116 | 117 | static inline bool isLeaf(Node* node) { 118 | // Is the node a leaf? 119 | return reinterpret_cast(node)&1; 120 | } 121 | 122 | static inline uint8_t flipSign(uint8_t keyByte) { 123 | // Flip the sign bit, enables signed SSE comparison of unsigned values, used by Node16 124 | return keyByte^128; 125 | } 126 | 127 | //inline void loadKey(uintptr_t tid,uint8_t key[]); 128 | static inline unsigned ctz(uint16_t x) { 129 | // Count trailing zeros, only defined for x>0 130 | #ifdef __GNUC__ 131 | return __builtin_ctz(x); 132 | #else 133 | // Adapted from Hacker's Delight 134 | unsigned n=1; 135 | if ((x&0xFF)==0) {n+=8; x=x>>8;} 136 | if ((x&0x0F)==0) {n+=4; x=x>>4;} 137 | if ((x&0x03)==0) {n+=2; x=x>>2;} 138 | return n-(x&1); 139 | #endif 140 | } 141 | 142 | //**************************************************************** 143 | 144 | inline Node** findChild(Node* n,uint8_t keyByte); 145 | inline Node* minimum(Node* node); 146 | inline bool leafMatches(Node* leaf,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 147 | inline unsigned prefixMismatch(Node* node,uint8_t key[],unsigned depth,unsigned maxKeyLength); 148 | 149 | Node** lookupRef(Node** nodeRef, Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 150 | inline Node* lookup(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 151 | inline Node* lookupPessimistic(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 152 | 153 | //**************************************************************** 154 | 155 | inline int CompareToPrefix(Node* node,uint8_t key[],unsigned depth,unsigned maxKeyLength); 156 | inline Node* findChild_recordPath(Node* n, uint8_t keyByte, ARTIter* iter); 157 | inline Node* lower_bound(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength, ARTIter *iter); 158 | 159 | //**************************************************************** 160 | 161 | inline unsigned min(unsigned a,unsigned b); 162 | inline void copyPrefix(Node* src,Node* dst); 163 | 164 | void upsert(Node* node,Node** nodeRef,uint8_t key[],unsigned depth,uintptr_t value,unsigned maxKeyLength, std::function insertMakeLeaf); 165 | inline void insert(Node* node,Node** nodeRef,uint8_t key[],unsigned depth,uintptr_t value,unsigned maxKeyLength); 166 | 167 | inline void insertNode4(Node4* node,Node** nodeRef,uint8_t keyByte,Node* child); 168 | inline void insertNode16(Node16* node,Node** nodeRef,uint8_t keyByte,Node* child); 169 | inline void insertNode48(Node48* node,Node** nodeRef,uint8_t keyByte,Node* child); 170 | inline void insertNode256(Node256* node,Node** nodeRef,uint8_t keyByte,Node* child); 171 | 172 | //**************************************************************** 173 | 174 | inline void erase(Node* node,Node** nodeRef,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 175 | 176 | inline void eraseNode4(Node4* node,Node** nodeRef,Node** leafPlace); 177 | inline void eraseNode16(Node16* node,Node** nodeRef,Node** leafPlace); 178 | inline void eraseNode48(Node48* node,Node** nodeRef,uint8_t keyByte); 179 | inline void eraseNode256(Node256* node,Node** nodeRef,uint8_t keyByte); 180 | 181 | ART(); 182 | ART(std::function loadKey, std::function upsertFunc); 183 | 184 | void load(vector &keys, vector &values, unsigned maxKeyLength); 185 | void load(vector &keys, vector &values); 186 | void insert(uint8_t key[], uintptr_t value, unsigned maxKeyLength); 187 | void upsert(uint8_t key[], uintptr_t value, unsigned maxKeyLength, std::function insertMakeLeaf); 188 | uint64_t lookup(uint8_t key[], unsigned keyLength, unsigned maxKeyLength); 189 | uint64_t lookup(uint64_t key64); 190 | Node** lookupRef(uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength); 191 | 192 | bool lower_bound(uint8_t key[], unsigned keyLength, unsigned maxKeyLength, ARTIter* iter); 193 | bool lower_bound(uint64_t key64, ARTIter* iter); 194 | void erase(uint8_t key[], unsigned keyLength, unsigned maxKeyLength); 195 | uint64_t getMemory(); 196 | 197 | friend class ARTIter; 198 | 199 | private: 200 | Node* root; 201 | unsigned key_length; 202 | 203 | //stats 204 | uint64_t memory; 205 | uint64_t num_items; 206 | uint64_t node4_count; 207 | uint64_t node16_count; 208 | uint64_t node48_count; 209 | uint64_t node256_count; 210 | 211 | // This address is used to communicate that search failed 212 | Node* nullNode; 213 | std::function loadKey; 214 | std::function upsertFunc; 215 | }; 216 | 217 | class ARTIter { 218 | public: 219 | inline Node* minimum_recordPath(Node* node); 220 | inline Node* nextSlot(); 221 | inline Node* currentLeaf(); 222 | inline Node* nextLeaf(); 223 | 224 | ARTIter(); 225 | ARTIter(ART* idx); 226 | 227 | uint64_t value(); 228 | uint64_t key() { return key64; } 229 | bool operator ++ (int); 230 | 231 | friend class ART; 232 | 233 | private: 234 | ART* index; 235 | std::vector node_stack; 236 | uint64_t val; 237 | uint64_t key64; 238 | }; 239 | 240 | -------------------------------------------------------------------------------- /misc/ARTOLC/N.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by florian on 05.08.15. 3 | // 4 | 5 | #ifndef ART_OPTIMISTIC_LOCK_COUPLING_N_H 6 | #define ART_OPTIMISTIC_LOCK_COUPLING_N_H 7 | //#define ART_NOREADLOCK 8 | //#define ART_NOWRITELOCK 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "Key.h" 14 | #include "Epoche.h" 15 | 16 | using TID = uint64_t; 17 | 18 | using namespace ART; 19 | namespace ART_OLC { 20 | /* 21 | * SynchronizedTree 22 | * LockCouplingTree 23 | * LockCheckFreeReadTree 24 | * UnsynchronizedTree 25 | */ 26 | 27 | enum class NTypes : uint8_t { 28 | N4 = 0, 29 | N16 = 1, 30 | N48 = 2, 31 | N256 = 3 32 | }; 33 | 34 | static constexpr uint32_t maxStoredPrefixLength = 11; 35 | 36 | using Prefix = uint8_t[maxStoredPrefixLength]; 37 | 38 | class N { 39 | protected: 40 | N(NTypes type, const uint8_t *prefix, uint32_t prefixLength) { 41 | setType(type); 42 | setPrefix(prefix, prefixLength); 43 | } 44 | 45 | N(const N &) = delete; 46 | 47 | N(N &&) = delete; 48 | 49 | //2b type 60b version 1b lock 1b obsolete 50 | std::atomic typeVersionLockObsolete{0b100}; 51 | // version 1, unlocked, not obsolete 52 | uint32_t prefixCount = 0; 53 | 54 | uint8_t count = 0; 55 | Prefix prefix; 56 | 57 | 58 | void setType(NTypes type); 59 | 60 | static uint64_t convertTypeToVersion(NTypes type); 61 | 62 | public: 63 | 64 | NTypes getType() const; 65 | 66 | uint32_t getCount() const; 67 | 68 | bool isLocked(uint64_t version) const; 69 | 70 | void writeLockOrRestart(bool &needRestart); 71 | 72 | void upgradeToWriteLockOrRestart(uint64_t &version, bool &needRestart); 73 | 74 | void writeUnlock(); 75 | 76 | uint64_t readLockOrRestart(bool &needRestart) const; 77 | 78 | /** 79 | * returns true if node hasn't been changed in between 80 | */ 81 | void checkOrRestart(uint64_t startRead, bool &needRestart) const; 82 | void readUnlockOrRestart(uint64_t startRead, bool &needRestart) const; 83 | 84 | static bool isObsolete(uint64_t version); 85 | 86 | /** 87 | * can only be called when node is locked 88 | */ 89 | void writeUnlockObsolete() { 90 | typeVersionLockObsolete.fetch_add(0b11); 91 | } 92 | 93 | static N *getChild(const uint8_t k, const N *node); 94 | 95 | static void insertAndUnlock(N *node, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, N *val, bool &needRestart, 96 | ThreadInfo &threadInfo, std::function insert_func); 97 | 98 | static bool change(N *node, uint8_t key, N *val); 99 | 100 | static void removeAndUnlock(N *node, uint64_t v, uint8_t key, N *parentNode, uint64_t parentVersion, uint8_t keyParent, bool &needRestart, ThreadInfo &threadInfo); 101 | 102 | bool hasPrefix() const; 103 | 104 | const uint8_t *getPrefix() const; 105 | 106 | void setPrefix(const uint8_t *prefix, uint32_t length); 107 | 108 | void addPrefixBefore(N *node, uint8_t key); 109 | 110 | uint32_t getPrefixLength() const; 111 | 112 | static TID getLeaf(const N *n); 113 | 114 | static bool isLeaf(const N *n); 115 | 116 | static N *setLeaf(TID tid); 117 | 118 | static N *getAnyChild(const N *n); 119 | 120 | static TID getAnyChildTid(const N *n, bool &needRestart); 121 | 122 | static void deleteChildren(N *node); 123 | 124 | static void deleteNode(N *node); 125 | 126 | static std::tuple getSecondChild(N *node, const uint8_t k); 127 | 128 | template 129 | static void insertGrow(curN *n, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, N *val, bool &needRestart, ThreadInfo &threadInfo, std::function insert_func); 130 | 131 | template 132 | static void removeAndShrink(curN *n, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, bool &needRestart, ThreadInfo &threadInfo); 133 | 134 | static uint64_t getChildren(const N *node, uint8_t start, uint8_t end, std::tuple children[], 135 | uint32_t &childrenCount); 136 | }; 137 | 138 | class N4 : public N { 139 | public: 140 | uint8_t keys[4]; 141 | N *children[4] = {nullptr, nullptr, nullptr, nullptr}; 142 | 143 | public: 144 | N4(const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N4, prefix, 145 | prefixLength) { } 146 | 147 | void insert(uint8_t key, N *n); 148 | 149 | template 150 | void copyTo(NODE *n) const; 151 | 152 | bool change(uint8_t key, N *val); 153 | 154 | N *getChild(const uint8_t k) const; 155 | 156 | void remove(uint8_t k); 157 | 158 | N *getAnyChild() const; 159 | 160 | bool isFull() const; 161 | 162 | bool isUnderfull() const; 163 | 164 | std::tuple getSecondChild(const uint8_t key) const; 165 | 166 | void deleteChildren(); 167 | 168 | uint64_t getChildren(uint8_t start, uint8_t end, std::tuple *&children, 169 | uint32_t &childrenCount) const; 170 | }; 171 | 172 | class N16 : public N { 173 | public: 174 | uint8_t keys[16]; 175 | N *children[16]; 176 | 177 | static uint8_t flipSign(uint8_t keyByte) { 178 | // Flip the sign bit, enables signed SSE comparison of unsigned values, used by Node16 179 | return keyByte ^ 128; 180 | } 181 | 182 | static inline unsigned ctz(uint16_t x) { 183 | // Count trailing zeros, only defined for x>0 184 | #ifdef __GNUC__ 185 | return __builtin_ctz(x); 186 | #else 187 | // Adapted from Hacker's Delight 188 | unsigned n=1; 189 | if ((x&0xFF)==0) {n+=8; x=x>>8;} 190 | if ((x&0x0F)==0) {n+=4; x=x>>4;} 191 | if ((x&0x03)==0) {n+=2; x=x>>2;} 192 | return n-(x&1); 193 | #endif 194 | } 195 | 196 | N *const *getChildPos(const uint8_t k) const; 197 | 198 | public: 199 | N16(const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N16, prefix, 200 | prefixLength) { 201 | memset(keys, 0, sizeof(keys)); 202 | memset(children, 0, sizeof(children)); 203 | } 204 | 205 | void insert(uint8_t key, N *n); 206 | 207 | template 208 | void copyTo(NODE *n) const; 209 | 210 | bool change(uint8_t key, N *val); 211 | 212 | N *getChild(const uint8_t k) const; 213 | 214 | void remove(uint8_t k); 215 | 216 | N *getAnyChild() const; 217 | 218 | bool isFull() const; 219 | 220 | bool isUnderfull() const; 221 | 222 | void deleteChildren(); 223 | 224 | uint64_t getChildren(uint8_t start, uint8_t end, std::tuple *&children, 225 | uint32_t &childrenCount) const; 226 | }; 227 | 228 | class N48 : public N { 229 | uint8_t childIndex[256]; 230 | N *children[48]; 231 | public: 232 | static const uint8_t emptyMarker = 48; 233 | 234 | N48(const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N48, prefix, 235 | prefixLength) { 236 | memset(childIndex, emptyMarker, sizeof(childIndex)); 237 | memset(children, 0, sizeof(children)); 238 | } 239 | 240 | void insert(uint8_t key, N *n); 241 | 242 | template 243 | void copyTo(NODE *n) const; 244 | 245 | bool change(uint8_t key, N *val); 246 | 247 | N *getChild(const uint8_t k) const; 248 | 249 | N *getMaxChild() const; 250 | void remove(uint8_t k); 251 | 252 | N *getAnyChild() const; 253 | 254 | bool isFull() const; 255 | 256 | bool isUnderfull() const; 257 | 258 | void deleteChildren(); 259 | 260 | uint64_t getChildren(uint8_t start, uint8_t end, std::tuple *&children, 261 | uint32_t &childrenCount) const; 262 | }; 263 | 264 | class N256 : public N { 265 | N *children[256]; 266 | 267 | public: 268 | N256(const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N256, prefix, 269 | prefixLength) { 270 | memset(children, '\0', sizeof(children)); 271 | } 272 | 273 | void insert(uint8_t key, N *val); 274 | 275 | template 276 | void copyTo(NODE *n) const; 277 | 278 | bool change(uint8_t key, N *n); 279 | 280 | N *getChild(const uint8_t k) const; 281 | 282 | void remove(uint8_t k); 283 | 284 | N *getAnyChild() const; 285 | 286 | bool isFull() const; 287 | 288 | bool isUnderfull() const; 289 | 290 | void deleteChildren(); 291 | 292 | uint64_t getChildren(uint8_t start, uint8_t end, std::tuple *&children, 293 | uint32_t &childrenCount) const; 294 | }; 295 | } 296 | #endif //ART_OPTIMISTIC_LOCK_COUPLING_N_H 297 | -------------------------------------------------------------------------------- /src/util.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "util.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #ifdef _MSC_VER 9 | #include 10 | #else 11 | #include 12 | #endif 13 | using namespace std; 14 | 15 | unsigned long write_latency_in_ns; 16 | unsigned long CPU_FREQ_MHZ = 2200; 17 | unsigned long long cycles_total = 0; 18 | atomic clflush_count(0); 19 | atomic sfence_count(0); 20 | vector keys; 21 | vector lookupKeys; 22 | vector sortedKeys; 23 | vector notExistKeys; 24 | 25 | void prepareKeys(int n, const string &keyFile, bool sparseKey, bool sorted) 26 | { 27 | uint64_t end; 28 | unordered_set keySet; 29 | size_t totalKeyCount = n; 30 | keys.reserve(totalKeyCount); 31 | lookupKeys.reserve(totalKeyCount); 32 | std::random_device rd; 33 | 34 | std::mt19937_64 e2(0); 35 | 36 | std::uniform_int_distribution dist(0, std::llround(std::pow(2, 63))); 37 | 38 | if (sparseKey == false) 39 | { 40 | end = n; 41 | for (uint64_t i = 0; i < end; ++i) 42 | { 43 | keys.push_back(i); 44 | } 45 | } 46 | else 47 | { 48 | keySet.reserve(totalKeyCount); 49 | end = std::numeric_limits::max(); 50 | while (keySet.size() != totalKeyCount) 51 | { 52 | uint64_t key = dist(e2) % end; 53 | keySet.insert(key); 54 | } 55 | auto tset = keySet; 56 | for (int i = 0; i < n && keySet.empty() == false; ++i) 57 | { 58 | uint64_t key = *keySet.begin(); 59 | keySet.erase(key); 60 | keys.push_back(key); 61 | } 62 | for (int i = 0; i < totalKeyCount; ++i) { 63 | uint64_t key = dist(e2) % end; 64 | if (tset.find(key) != tset.end()) { 65 | --i; 66 | continue; 67 | } 68 | notExistKeys.push_back(key); 69 | } 70 | } 71 | 72 | if (sorted) 73 | { 74 | sort(keys.begin(), keys.end()); 75 | } 76 | lookupKeys = keys; 77 | std::random_shuffle(lookupKeys.begin(), lookupKeys.end()); 78 | sortedKeys = keys; 79 | std::sort(sortedKeys.begin(), sortedKeys.end()); 80 | //print_memory_usage(); 81 | printf("Keys: %d, Lookup Keys: %d, NotExist Keys: %d\n", (int)keys.size(), (int)lookupKeys.size(), (int) notExistKeys.size()); 82 | } 83 | 84 | inline unsigned ctz(unsigned x) { 85 | // Count trailing zeros, only defined for x>0 86 | #ifdef __GNUC__ 87 | return __builtin_ctz(x); 88 | #else 89 | // Adapted from Hacker's Delight 90 | unsigned n=1; 91 | if ((x&0xFF)==0) {n+=8; x=x>>8;} 92 | if ((x&0x0F)==0) {n+=4; x=x>>4;} 93 | if ((x&0x03)==0) {n+=2; x=x>>2;} 94 | return n-(x&1); 95 | #endif 96 | } 97 | 98 | unsigned long long cycles_now() { 99 | _mm_lfence(); 100 | unsigned long long v = __rdtsc(); 101 | _mm_lfence(); 102 | return v; 103 | } 104 | 105 | double secs_now(void) 106 | { 107 | struct timeval now_tv; 108 | gettimeofday(&now_tv, NULL); 109 | return ((double)now_tv.tv_sec) + ((double)now_tv.tv_usec) / 1000000.0; 110 | } 111 | 112 | void clear_cache() 113 | { 114 | // Remove cache 115 | int size = 256 * 1024 * 1024; 116 | char *garbage = new char[size]; 117 | for (int i = 0; i < size; ++i) 118 | garbage[i] = i; 119 | for (int i = 100; i < size; ++i) 120 | garbage[i] += garbage[i - 100]; 121 | delete[] garbage; 122 | } 123 | 124 | void print_flush_stat() { 125 | printf("clfluh count: %lld\n", clflush_count.load()); 126 | } 127 | #include 128 | double measure(std::function f, const string &bench_name, unsigned iteration, bool print_mem) 129 | { 130 | #ifdef USE_PAPI 131 | const int nevents = 4; 132 | int papi_events[nevents] = {PAPI_L3_TCM, PAPI_TOT_CYC, PAPI_TOT_INS, PAPI_L3_TCA /*, PAPI_TLB_DM*/}, ret; 133 | 134 | long long values[nevents]; 135 | if ((ret = PAPI_start_counters(papi_events, nevents)) != PAPI_OK) 136 | { 137 | fprintf(stderr, "PAPI failed to start counters: %s\n", PAPI_strerror(ret)); 138 | exit(1); 139 | } 140 | #endif 141 | 142 | 143 | 144 | clflush_count = 0; 145 | sfence_count = 0; 146 | char tcmalloc_stats_buf[1024]; 147 | print_mem = 0; 148 | if (print_mem) { 149 | MallocExtension::instance()->GetStats(tcmalloc_stats_buf, 150 | sizeof(tcmalloc_stats_buf)); 151 | } 152 | if (print_mem) 153 | printf("%s\n", tcmalloc_stats_buf); 154 | 155 | double start = secs_now(); 156 | unsigned c = 0; 157 | for (int i = 0; i < iteration; ++i) { 158 | c += f() + 1; 159 | } 160 | double end = secs_now(); 161 | if (print_mem) { 162 | MallocExtension::instance()->GetStats(tcmalloc_stats_buf, 163 | sizeof(tcmalloc_stats_buf)); 164 | } 165 | if (print_mem) 166 | printf("%s\n", tcmalloc_stats_buf); 167 | double avg = (end - start) * 1000000000L / c; 168 | printf("Benchmark %s, Elapsed %f secs, %f ns/op, %f mops, %u ops in total, %lld flushes, %f flushes/op, %f sfences/op\n", 169 | bench_name.c_str(), (end - start), avg, 170 | c / 1000000.0 / (end - start), c, clflush_count.load(), clflush_count.load() / (c + 0.0), sfence_count.load() / (c + 0.0)); 171 | #ifdef USE_PAPI 172 | if ((ret = PAPI_stop_counters(values, nevents)) != PAPI_OK) 173 | { 174 | fprintf(stderr, "PAPI failed to stop counters: %s\n", PAPI_strerror(ret)); 175 | exit(1); 176 | } 177 | 178 | printf("Level 3 cache misses: %lld\n", values[0]); 179 | printf("Total cycles: %lld\n", values[1]); 180 | printf("Instructions completed: %lld\n", values[2]); 181 | printf("Level 3 accesses: %lld\n", values[3]); 182 | //printf("Data translation lookaside buffer misses: %lld\n", values[4]); 183 | //printf("Cycles stalled on any resource: %lld\n", values[3]); 184 | #endif 185 | printf("\n"); 186 | return avg; 187 | } 188 | 189 | void cpu_pause() 190 | { 191 | _mm_pause(); 192 | } 193 | 194 | inline unsigned long read_tsc(void) 195 | { 196 | unsigned long var; 197 | unsigned int hi, lo; 198 | 199 | asm volatile("rdtsc" 200 | : "=a"(lo), "=d"(hi)); 201 | var = ((unsigned long long int)hi << 32) | lo; 202 | 203 | return var; 204 | } 205 | void mfence() 206 | { 207 | asm volatile("mfence" :: 208 | : "memory"); 209 | } 210 | 211 | void sfence() { 212 | asm volatile("sfence" :: 213 | : "memory"); 214 | } 215 | 216 | void nontemporal_store_256(void *mem_addr, void *c) 217 | { 218 | __m256i x = _mm256_load_si256((__m256i const *)c); 219 | unsigned long etsc = read_tsc() + 220 | (unsigned long)(write_latency_in_ns * CPU_FREQ_MHZ / 1000); 221 | _mm256_stream_si256((__m256i *)mem_addr, x); 222 | while (read_tsc() < etsc) 223 | cpu_pause(); 224 | asm volatile("sfence" :: 225 | : "memory"); 226 | #ifdef COUNT_CLFLUSH 227 | clflush_count.fetch_add(1); 228 | sfence_count.fetch_add(1); 229 | #endif 230 | } 231 | 232 | void prefetch(char *ptr, size_t len) 233 | { 234 | if (ptr == nullptr) 235 | return; 236 | for (char *p = ptr; p < ptr + len; p += cacheline_size) 237 | { 238 | __builtin_prefetch(p); 239 | } 240 | } 241 | 242 | #ifdef HAS_AVX512 243 | void nontemporal_store_512_fenced(void *mem_addr, void *c) 244 | { 245 | auto t = _mm512_load_si512((const __m512i *)c); 246 | unsigned long etsc = read_tsc() + 247 | (unsigned long)(write_latency_in_ns * CPU_FREQ_MHZ / 1000); 248 | _mm512_stream_si512((__m512i *)mem_addr, t); 249 | while (read_tsc() < etsc) 250 | cpu_pause(); 251 | asm volatile("mfence" :: 252 | : "memory"); 253 | #ifdef COUNT_CLFLUSH 254 | clflush_count.fetch_add(1); 255 | #endif 256 | } 257 | 258 | void nontemporal_store_512(void *mem_addr, void *c) 259 | { 260 | auto t = _mm512_load_si512((const __m512i *)c); 261 | unsigned long etsc = read_tsc() + 262 | (unsigned long)(write_latency_in_ns * CPU_FREQ_MHZ / 1000); 263 | _mm512_stream_si512((__m512i *)mem_addr, t); 264 | while (read_tsc() < etsc) 265 | cpu_pause(); 266 | #ifdef COUNT_CLFLUSH 267 | clflush_count.fetch_add(1); 268 | #endif 269 | } 270 | 271 | #endif 272 | 273 | void clflush_then_sfence(volatile void *p) 274 | { 275 | volatile char *ptr = CACHELINE_ALIGN(p); 276 | #ifdef COUNT_CLFLUSH 277 | clflush_count.fetch_add(1); 278 | sfence_count.fetch_add(1); 279 | #endif 280 | asm volatile("clwb %0" 281 | : "+m"(*ptr)); 282 | sfence(); 283 | } 284 | 285 | void clflush(volatile void *p) 286 | { 287 | volatile char *ptr = CACHELINE_ALIGN(p); 288 | #ifdef COUNT_CLFLUSH 289 | clflush_count.fetch_add(1); 290 | #endif 291 | asm volatile("clwb (%0)" ::"r"(ptr)); 292 | } 293 | 294 | void clflush_len_no_fence(volatile void *data, int len, std::function stat_updater) { 295 | volatile char *ptr = CACHELINE_ALIGN(data); 296 | for (; ptr < (char *)data + len; ptr += cacheline_size) 297 | { 298 | #ifdef COUNT_CLFLUSH 299 | clflush_count.fetch_add(1); 300 | stat_updater(); 301 | #endif 302 | asm volatile("clwb %0" 303 | : "+m"(*(volatile char *)ptr)); 304 | } 305 | } 306 | 307 | 308 | void clflush_len_no_fence(volatile void *data, int len) { 309 | volatile char *ptr = CACHELINE_ALIGN(data); 310 | for (; ptr < (char *)data + len; ptr += cacheline_size) 311 | { 312 | #ifdef COUNT_CLFLUSH 313 | clflush_count.fetch_add(1); 314 | #endif 315 | asm volatile("clwb %0" 316 | : "+m"(*(volatile char *)ptr)); 317 | } 318 | } 319 | 320 | void clflush_len(volatile void *data, int len) 321 | { 322 | volatile char *ptr = CACHELINE_ALIGN(data); 323 | for (; ptr < (char *)data + len; ptr += cacheline_size) 324 | { 325 | #ifdef COUNT_CLFLUSH 326 | clflush_count.fetch_add(1); 327 | #endif 328 | asm volatile("clwb %0" 329 | : "+m"(*(volatile char *)ptr)); 330 | } 331 | #ifdef COUNT_CLFLUSH 332 | sfence(); 333 | sfence_count.fetch_add(1); 334 | #endif 335 | } 336 | 337 | int nvm_dram_alloc(void **ptr, size_t align, size_t size) 338 | { 339 | assert(size < 1073741824UL); 340 | int ret = posix_memalign(ptr, align, size); 341 | return ret; 342 | } 343 | 344 | void nvm_dram_free(void *ptr, size_t size) 345 | { 346 | free(ptr); 347 | } 348 | 349 | int nvm_dram_alloc_cacheline_aligned(void **p, size_t size) 350 | { 351 | assert(size < 1073741824UL); 352 | assert(size % cacheline_size == 0); 353 | int ret = posix_memalign(p, cacheline_size, size); 354 | return ret; 355 | } 356 | 357 | void nvm_dram_free_cacheline_aligned(void *ptr) 358 | { 359 | assert(((uintptr_t)ptr & (cacheline_size - 1)) == 0); 360 | free(ptr); 361 | } 362 | 363 | void percentile_report(const std::string & name, std::vector & nums) { 364 | std::sort(nums.begin(), nums.end()); 365 | auto sum = accumulate(nums.begin(), nums.end(), 0LL); 366 | printf("name %s, total %lld, avg %f, 50p %d, 70p %d, 80p %d, 90p %d, 95p " 367 | "%d, 99p %d, 99.9p %d\n", 368 | name.c_str(),sum, 369 | sum / (nums.size() + 0.1), 370 | nums[nums.size() * 0.5], 371 | nums[nums.size() * 0.7], 372 | nums[nums.size() * 0.8], 373 | nums[nums.size() * 0.9], 374 | nums[nums.size() * 0.95], 375 | nums[nums.size() * 0.99], 376 | nums[nums.size() * 0.999]); 377 | } 378 | 379 | int wbinvd() { 380 | return system("cat /proc/wbinvd"); 381 | } 382 | -------------------------------------------------------------------------------- /src/MurmurHash2.cpp: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash2 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | // Note - This code makes a few assumptions about how your machine behaves - 6 | 7 | // 1. We can read a 4-byte value from any address without crashing 8 | // 2. sizeof(int) == 4 9 | 10 | // And it has a few limitations - 11 | 12 | // 1. It will not work incrementally. 13 | // 2. It will not produce the same results on little-endian and big-endian 14 | // machines. 15 | #include 16 | #include "../include/MurmurHash2.h" 17 | 18 | //----------------------------------------------------------------------------- 19 | // Platform-specific functions and macros 20 | 21 | // Microsoft Visual Studio 22 | 23 | #if defined(_MSC_VER) 24 | 25 | #define BIG_CONSTANT(x) (x) 26 | 27 | // Other compilers 28 | 29 | #else // defined(_MSC_VER) 30 | 31 | #define BIG_CONSTANT(x) (x##LLU) 32 | 33 | #endif // !defined(_MSC_VER) 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | uint32_t MurmurHash2(const void *key, int len, uint32_t seed) 38 | { 39 | // 'm' and 'r' are mixing constants generated offline. 40 | // They're not really 'magic', they just happen to work well. 41 | 42 | const uint32_t m = 0x5bd1e995; 43 | const int r = 24; 44 | 45 | // Initialize the hash to a 'random' value 46 | 47 | uint32_t h = seed ^ len; 48 | 49 | // Mix 4 bytes at a time into the hash 50 | 51 | const unsigned char *data = (const unsigned char *)key; 52 | 53 | while (len >= 4) 54 | { 55 | uint32_t k = *(uint32_t *)data; 56 | 57 | k *= m; 58 | k ^= k >> r; 59 | k *= m; 60 | 61 | h *= m; 62 | h ^= k; 63 | 64 | data += 4; 65 | len -= 4; 66 | } 67 | 68 | // Handle the last few bytes of the input array 69 | 70 | switch (len) 71 | { 72 | case 3: 73 | h ^= data[2] << 16; 74 | case 2: 75 | h ^= data[1] << 8; 76 | case 1: 77 | h ^= data[0]; 78 | h *= m; 79 | }; 80 | 81 | // Do a few final mixes of the hash to ensure the last few 82 | // bytes are well-incorporated. 83 | 84 | h ^= h >> 13; 85 | h *= m; 86 | h ^= h >> 15; 87 | 88 | return h; 89 | } 90 | 91 | //----------------------------------------------------------------------------- 92 | // MurmurHash2, 64-bit versions, by Austin Appleby 93 | 94 | // The same caveats as 32-bit MurmurHash2 apply here - beware of alignment 95 | // and endian-ness issues if used across multiple platforms. 96 | 97 | // 64-bit hash for 64-bit platforms 98 | 99 | uint64_t MurmurHash64A(const void *key, int len, uint64_t seed) 100 | { 101 | const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); 102 | const int r = 47; 103 | 104 | uint64_t h = seed ^ (len * m); 105 | 106 | const uint64_t *data = (const uint64_t *)key; 107 | const uint64_t *end = data + (len / 8); 108 | 109 | while (data != end) 110 | { 111 | uint64_t k = *data++; 112 | 113 | k *= m; 114 | k ^= k >> r; 115 | k *= m; 116 | 117 | h ^= k; 118 | h *= m; 119 | } 120 | 121 | const unsigned char *data2 = (const unsigned char *)data; 122 | 123 | switch (len & 7) 124 | { 125 | case 7: 126 | h ^= uint64_t(data2[6]) << 48; 127 | case 6: 128 | h ^= uint64_t(data2[5]) << 40; 129 | case 5: 130 | h ^= uint64_t(data2[4]) << 32; 131 | case 4: 132 | h ^= uint64_t(data2[3]) << 24; 133 | case 3: 134 | h ^= uint64_t(data2[2]) << 16; 135 | case 2: 136 | h ^= uint64_t(data2[1]) << 8; 137 | case 1: 138 | h ^= uint64_t(data2[0]); 139 | h *= m; 140 | }; 141 | 142 | h ^= h >> r; 143 | h *= m; 144 | h ^= h >> r; 145 | 146 | return h; 147 | } 148 | 149 | // 64-bit hash for 32-bit platforms 150 | 151 | uint64_t MurmurHash64B(const void *key, int len, uint64_t seed) 152 | { 153 | const uint32_t m = 0x5bd1e995; 154 | const int r = 24; 155 | 156 | uint32_t h1 = uint32_t(seed) ^ len; 157 | uint32_t h2 = uint32_t(seed >> 32); 158 | 159 | const uint32_t *data = (const uint32_t *)key; 160 | 161 | while (len >= 8) 162 | { 163 | uint32_t k1 = *data++; 164 | k1 *= m; 165 | k1 ^= k1 >> r; 166 | k1 *= m; 167 | h1 *= m; 168 | h1 ^= k1; 169 | len -= 4; 170 | 171 | uint32_t k2 = *data++; 172 | k2 *= m; 173 | k2 ^= k2 >> r; 174 | k2 *= m; 175 | h2 *= m; 176 | h2 ^= k2; 177 | len -= 4; 178 | } 179 | 180 | if (len >= 4) 181 | { 182 | uint32_t k1 = *data++; 183 | k1 *= m; 184 | k1 ^= k1 >> r; 185 | k1 *= m; 186 | h1 *= m; 187 | h1 ^= k1; 188 | len -= 4; 189 | } 190 | 191 | switch (len) 192 | { 193 | case 3: 194 | h2 ^= ((unsigned char *)data)[2] << 16; 195 | case 2: 196 | h2 ^= ((unsigned char *)data)[1] << 8; 197 | case 1: 198 | h2 ^= ((unsigned char *)data)[0]; 199 | h2 *= m; 200 | }; 201 | 202 | h1 ^= h2 >> 18; 203 | h1 *= m; 204 | h2 ^= h1 >> 22; 205 | h2 *= m; 206 | h1 ^= h2 >> 17; 207 | h1 *= m; 208 | h2 ^= h1 >> 19; 209 | h2 *= m; 210 | 211 | uint64_t h = h1; 212 | 213 | h = (h << 32) | h2; 214 | 215 | return h; 216 | } 217 | 218 | //----------------------------------------------------------------------------- 219 | // MurmurHash2A, by Austin Appleby 220 | 221 | // This is a variant of MurmurHash2 modified to use the Merkle-Damgard 222 | // construction. Bulk speed should be identical to Murmur2, small-key speed 223 | // will be 10%-20% slower due to the added overhead at the end of the hash. 224 | 225 | // This variant fixes a minor issue where null keys were more likely to 226 | // collide with each other than expected, and also makes the function 227 | // more amenable to incremental implementations. 228 | 229 | #define mmix(h, k) \ 230 | { \ 231 | k *= m; \ 232 | k ^= k >> r; \ 233 | k *= m; \ 234 | h *= m; \ 235 | h ^= k; \ 236 | } 237 | 238 | uint32_t MurmurHash2A(const void *key, int len, uint32_t seed) 239 | { 240 | const uint32_t m = 0x5bd1e995; 241 | const int r = 24; 242 | uint32_t l = len; 243 | 244 | const unsigned char *data = (const unsigned char *)key; 245 | 246 | uint32_t h = seed; 247 | 248 | while (len >= 4) 249 | { 250 | uint32_t k = *(uint32_t *)data; 251 | 252 | mmix(h, k); 253 | 254 | data += 4; 255 | len -= 4; 256 | } 257 | 258 | uint32_t t = 0; 259 | 260 | switch (len) 261 | { 262 | case 3: 263 | t ^= data[2] << 16; 264 | case 2: 265 | t ^= data[1] << 8; 266 | case 1: 267 | t ^= data[0]; 268 | }; 269 | 270 | mmix(h, t); 271 | mmix(h, l); 272 | 273 | h ^= h >> 13; 274 | h *= m; 275 | h ^= h >> 15; 276 | 277 | return h; 278 | } 279 | 280 | //----------------------------------------------------------------------------- 281 | // CMurmurHash2A, by Austin Appleby 282 | 283 | // This is a sample implementation of MurmurHash2A designed to work 284 | // incrementally. 285 | 286 | // Usage - 287 | 288 | // CMurmurHash2A hasher 289 | // hasher.Begin(seed); 290 | // hasher.Add(data1,size1); 291 | // hasher.Add(data2,size2); 292 | // ... 293 | // hasher.Add(dataN,sizeN); 294 | // uint32_t hash = hasher.End() 295 | 296 | class CMurmurHash2A 297 | { 298 | public: 299 | void Begin(uint32_t seed = 0) 300 | { 301 | m_hash = seed; 302 | m_tail = 0; 303 | m_count = 0; 304 | m_size = 0; 305 | } 306 | 307 | void Add(const unsigned char *data, int len) 308 | { 309 | m_size += len; 310 | 311 | MixTail(data, len); 312 | 313 | while (len >= 4) 314 | { 315 | uint32_t k = *(uint32_t *)data; 316 | 317 | mmix(m_hash, k); 318 | 319 | data += 4; 320 | len -= 4; 321 | } 322 | 323 | MixTail(data, len); 324 | } 325 | 326 | uint32_t End(void) 327 | { 328 | mmix(m_hash, m_tail); 329 | mmix(m_hash, m_size); 330 | 331 | m_hash ^= m_hash >> 13; 332 | m_hash *= m; 333 | m_hash ^= m_hash >> 15; 334 | 335 | return m_hash; 336 | } 337 | 338 | private: 339 | static const uint32_t m = 0x5bd1e995; 340 | static const int r = 24; 341 | 342 | void MixTail(const unsigned char *&data, int &len) 343 | { 344 | while (len && ((len < 4) || m_count)) 345 | { 346 | m_tail |= (*data++) << (m_count * 8); 347 | 348 | m_count++; 349 | len--; 350 | 351 | if (m_count == 4) 352 | { 353 | mmix(m_hash, m_tail); 354 | m_tail = 0; 355 | m_count = 0; 356 | } 357 | } 358 | } 359 | 360 | uint32_t m_hash; 361 | uint32_t m_tail; 362 | uint32_t m_count; 363 | uint32_t m_size; 364 | }; 365 | 366 | //----------------------------------------------------------------------------- 367 | // MurmurHashNeutral2, by Austin Appleby 368 | 369 | // Same as MurmurHash2, but endian- and alignment-neutral. 370 | // Half the speed though, alas. 371 | 372 | uint32_t MurmurHashNeutral2(const void *key, int len, uint32_t seed) 373 | { 374 | const uint32_t m = 0x5bd1e995; 375 | const int r = 24; 376 | 377 | uint32_t h = seed ^ len; 378 | 379 | const unsigned char *data = (const unsigned char *)key; 380 | 381 | while (len >= 4) 382 | { 383 | uint32_t k; 384 | 385 | k = data[0]; 386 | k |= data[1] << 8; 387 | k |= data[2] << 16; 388 | k |= data[3] << 24; 389 | 390 | k *= m; 391 | k ^= k >> r; 392 | k *= m; 393 | 394 | h *= m; 395 | h ^= k; 396 | 397 | data += 4; 398 | len -= 4; 399 | } 400 | 401 | switch (len) 402 | { 403 | case 3: 404 | h ^= data[2] << 16; 405 | case 2: 406 | h ^= data[1] << 8; 407 | case 1: 408 | h ^= data[0]; 409 | h *= m; 410 | }; 411 | 412 | h ^= h >> 13; 413 | h *= m; 414 | h ^= h >> 15; 415 | 416 | return h; 417 | } 418 | 419 | //----------------------------------------------------------------------------- 420 | // MurmurHashAligned2, by Austin Appleby 421 | 422 | // Same algorithm as MurmurHash2, but only does aligned reads - should be safer 423 | // on certain platforms. 424 | 425 | // Performance will be lower than MurmurHash2 426 | 427 | #define MIX(h, k, m) \ 428 | { \ 429 | k *= m; \ 430 | k ^= k >> r; \ 431 | k *= m; \ 432 | h *= m; \ 433 | h ^= k; \ 434 | } 435 | 436 | uint32_t MurmurHashAligned2(const void *key, int len, uint32_t seed) 437 | { 438 | const uint32_t m = 0x5bd1e995; 439 | const int r = 24; 440 | 441 | const unsigned char *data = (const unsigned char *)key; 442 | 443 | uint32_t h = seed ^ len; 444 | 445 | int align = (uint64_t)data & 3; 446 | 447 | if (align && (len >= 4)) 448 | { 449 | // Pre-load the temp registers 450 | 451 | uint32_t t = 0, d = 0; 452 | 453 | switch (align) 454 | { 455 | case 1: 456 | t |= data[2] << 16; 457 | case 2: 458 | t |= data[1] << 8; 459 | case 3: 460 | t |= data[0]; 461 | } 462 | 463 | t <<= (8 * align); 464 | 465 | data += 4 - align; 466 | len -= 4 - align; 467 | 468 | int sl = 8 * (4 - align); 469 | int sr = 8 * align; 470 | 471 | // Mix 472 | 473 | while (len >= 4) 474 | { 475 | d = *(uint32_t *)data; 476 | t = (t >> sr) | (d << sl); 477 | 478 | uint32_t k = t; 479 | 480 | MIX(h, k, m); 481 | 482 | t = d; 483 | 484 | data += 4; 485 | len -= 4; 486 | } 487 | 488 | // Handle leftover data in temp registers 489 | 490 | d = 0; 491 | 492 | if (len >= align) 493 | { 494 | switch (align) 495 | { 496 | case 3: 497 | d |= data[2] << 16; 498 | case 2: 499 | d |= data[1] << 8; 500 | case 1: 501 | d |= data[0]; 502 | } 503 | 504 | uint32_t k = (t >> sr) | (d << sl); 505 | MIX(h, k, m); 506 | 507 | data += align; 508 | len -= align; 509 | 510 | //---------- 511 | // Handle tail bytes 512 | 513 | switch (len) 514 | { 515 | case 3: 516 | h ^= data[2] << 16; 517 | case 2: 518 | h ^= data[1] << 8; 519 | case 1: 520 | h ^= data[0]; 521 | h *= m; 522 | }; 523 | } 524 | else 525 | { 526 | switch (len) 527 | { 528 | case 3: 529 | d |= data[2] << 16; 530 | case 2: 531 | d |= data[1] << 8; 532 | case 1: 533 | d |= data[0]; 534 | case 0: 535 | h ^= (t >> sr) | (d << sl); 536 | h *= m; 537 | } 538 | } 539 | 540 | h ^= h >> 13; 541 | h *= m; 542 | h ^= h >> 15; 543 | 544 | return h; 545 | } 546 | else 547 | { 548 | while (len >= 4) 549 | { 550 | uint32_t k = *(uint32_t *)data; 551 | 552 | MIX(h, k, m); 553 | 554 | data += 4; 555 | len -= 4; 556 | } 557 | 558 | //---------- 559 | // Handle tail bytes 560 | 561 | switch (len) 562 | { 563 | case 3: 564 | h ^= data[2] << 16; 565 | case 2: 566 | h ^= data[1] << 8; 567 | case 1: 568 | h ^= data[0]; 569 | h *= m; 570 | }; 571 | 572 | h ^= h >> 13; 573 | h *= m; 574 | h ^= h >> 15; 575 | 576 | return h; 577 | } 578 | } 579 | 580 | //----------------------------------------------------------------------------- 581 | -------------------------------------------------------------------------------- /misc/ARTOLC/N.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "N.h" 5 | #include "N4.cpp" 6 | #include "N16.cpp" 7 | #include "N48.cpp" 8 | #include "N256.cpp" 9 | 10 | namespace ART_OLC { 11 | 12 | void N::setType(NTypes type) { 13 | typeVersionLockObsolete.fetch_add(convertTypeToVersion(type)); 14 | } 15 | 16 | uint64_t N::convertTypeToVersion(NTypes type) { 17 | return (static_cast(type) << 62); 18 | } 19 | 20 | NTypes N::getType() const { 21 | return static_cast(typeVersionLockObsolete.load(std::memory_order_relaxed) >> 62); 22 | } 23 | 24 | void N::writeLockOrRestart(bool &needRestart) { 25 | 26 | uint64_t version; 27 | version = readLockOrRestart(needRestart); 28 | if (needRestart) return; 29 | 30 | upgradeToWriteLockOrRestart(version, needRestart); 31 | if (needRestart) return; 32 | } 33 | 34 | void N::upgradeToWriteLockOrRestart(uint64_t &version, bool &needRestart) { 35 | if (typeVersionLockObsolete.compare_exchange_strong(version, version + 0b10)) { 36 | version = version + 0b10; 37 | } else { 38 | needRestart = true; 39 | } 40 | } 41 | 42 | void N::writeUnlock() { 43 | typeVersionLockObsolete.fetch_add(0b10); 44 | } 45 | 46 | N *N::getAnyChild(const N *node) { 47 | switch (node->getType()) { 48 | case NTypes::N4: { 49 | auto n = static_cast(node); 50 | return n->getAnyChild(); 51 | } 52 | case NTypes::N16: { 53 | auto n = static_cast(node); 54 | return n->getAnyChild(); 55 | } 56 | case NTypes::N48: { 57 | auto n = static_cast(node); 58 | return n->getAnyChild(); 59 | } 60 | case NTypes::N256: { 61 | auto n = static_cast(node); 62 | return n->getAnyChild(); 63 | } 64 | } 65 | assert(false); 66 | __builtin_unreachable(); 67 | } 68 | 69 | bool N::change(N *node, uint8_t key, N *val) { 70 | switch (node->getType()) { 71 | case NTypes::N4: { 72 | auto n = static_cast(node); 73 | return n->change(key, val); 74 | } 75 | case NTypes::N16: { 76 | auto n = static_cast(node); 77 | return n->change(key, val); 78 | } 79 | case NTypes::N48: { 80 | auto n = static_cast(node); 81 | return n->change(key, val); 82 | } 83 | case NTypes::N256: { 84 | auto n = static_cast(node); 85 | return n->change(key, val); 86 | } 87 | } 88 | assert(false); 89 | __builtin_unreachable(); 90 | } 91 | 92 | template 93 | void N::insertGrow(curN *n, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, N *val, bool &needRestart, ThreadInfo &threadInfo, std::function insert_func) { 94 | if (!n->isFull()) { 95 | if (parentNode != nullptr) { 96 | parentNode->readUnlockOrRestart(parentVersion, needRestart); 97 | if (needRestart) return; 98 | } 99 | n->upgradeToWriteLockOrRestart(v, needRestart); 100 | if (needRestart) return; 101 | n->insert(key, val); 102 | insert_func(); 103 | n->writeUnlock(); 104 | return; 105 | } 106 | 107 | parentNode->upgradeToWriteLockOrRestart(parentVersion, needRestart); 108 | if (needRestart) return; 109 | 110 | n->upgradeToWriteLockOrRestart(v, needRestart); 111 | if (needRestart) { 112 | parentNode->writeUnlock(); 113 | return; 114 | } 115 | 116 | auto nBig = new biggerN(n->getPrefix(), n->getPrefixLength()); 117 | n->copyTo(nBig); 118 | nBig->insert(key, val); 119 | 120 | N::change(parentNode, keyParent, nBig); 121 | insert_func(); 122 | n->writeUnlockObsolete(); 123 | threadInfo.getEpoche().markNodeForDeletion(n, threadInfo); 124 | parentNode->writeUnlock(); 125 | } 126 | 127 | void N::insertAndUnlock(N *node, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, N *val, bool &needRestart, ThreadInfo &threadInfo, std::function insert_func) { 128 | switch (node->getType()) { 129 | case NTypes::N4: { 130 | auto n = static_cast(node); 131 | insertGrow(n, v, parentNode, parentVersion, keyParent, key, val, needRestart, threadInfo, insert_func); 132 | break; 133 | } 134 | case NTypes::N16: { 135 | auto n = static_cast(node); 136 | insertGrow(n, v, parentNode, parentVersion, keyParent, key, val, needRestart, threadInfo, insert_func); 137 | break; 138 | } 139 | case NTypes::N48: { 140 | auto n = static_cast(node); 141 | insertGrow(n, v, parentNode, parentVersion, keyParent, key, val, needRestart, threadInfo, insert_func); 142 | break; 143 | } 144 | case NTypes::N256: { 145 | auto n = static_cast(node); 146 | insertGrow(n, v, parentNode, parentVersion, keyParent, key, val, needRestart, threadInfo, insert_func); 147 | break; 148 | } 149 | } 150 | } 151 | 152 | inline N *N::getChild(const uint8_t k, const N *node) { 153 | switch (node->getType()) { 154 | case NTypes::N4: { 155 | auto n = static_cast(node); 156 | return n->getChild(k); 157 | } 158 | case NTypes::N16: { 159 | auto n = static_cast(node); 160 | return n->getChild(k); 161 | } 162 | case NTypes::N48: { 163 | auto n = static_cast(node); 164 | return n->getChild(k); 165 | } 166 | case NTypes::N256: { 167 | auto n = static_cast(node); 168 | return n->getChild(k); 169 | } 170 | } 171 | assert(false); 172 | __builtin_unreachable(); 173 | } 174 | 175 | void N::deleteChildren(N *node) { 176 | if (N::isLeaf(node)) { 177 | return; 178 | } 179 | switch (node->getType()) { 180 | case NTypes::N4: { 181 | auto n = static_cast(node); 182 | n->deleteChildren(); 183 | return; 184 | } 185 | case NTypes::N16: { 186 | auto n = static_cast(node); 187 | n->deleteChildren(); 188 | return; 189 | } 190 | case NTypes::N48: { 191 | auto n = static_cast(node); 192 | n->deleteChildren(); 193 | return; 194 | } 195 | case NTypes::N256: { 196 | auto n = static_cast(node); 197 | n->deleteChildren(); 198 | return; 199 | } 200 | } 201 | assert(false); 202 | __builtin_unreachable(); 203 | } 204 | 205 | template 206 | void N::removeAndShrink(curN *n, uint64_t v, N *parentNode, uint64_t parentVersion, uint8_t keyParent, uint8_t key, bool &needRestart, ThreadInfo &threadInfo) { 207 | if (!n->isUnderfull() || parentNode == nullptr) { 208 | if (parentNode != nullptr) { 209 | parentNode->readUnlockOrRestart(parentVersion, needRestart); 210 | if (needRestart) return; 211 | } 212 | n->upgradeToWriteLockOrRestart(v, needRestart); 213 | if (needRestart) return; 214 | 215 | n->remove(key); 216 | n->writeUnlock(); 217 | return; 218 | } 219 | parentNode->upgradeToWriteLockOrRestart(parentVersion, needRestart); 220 | if (needRestart) return; 221 | 222 | n->upgradeToWriteLockOrRestart(v, needRestart); 223 | if (needRestart) { 224 | parentNode->writeUnlock(); 225 | return; 226 | } 227 | 228 | auto nSmall = new smallerN(n->getPrefix(), n->getPrefixLength()); 229 | 230 | n->copyTo(nSmall); 231 | nSmall->remove(key); 232 | N::change(parentNode, keyParent, nSmall); 233 | 234 | n->writeUnlockObsolete(); 235 | threadInfo.getEpoche().markNodeForDeletion(n, threadInfo); 236 | parentNode->writeUnlock(); 237 | } 238 | 239 | void N::removeAndUnlock(N *node, uint64_t v, uint8_t key, N *parentNode, uint64_t parentVersion, uint8_t keyParent, bool &needRestart, ThreadInfo &threadInfo) { 240 | switch (node->getType()) { 241 | case NTypes::N4: { 242 | auto n = static_cast(node); 243 | removeAndShrink(n, v, parentNode, parentVersion, keyParent, key, needRestart, threadInfo); 244 | break; 245 | } 246 | case NTypes::N16: { 247 | auto n = static_cast(node); 248 | removeAndShrink(n, v, parentNode, parentVersion, keyParent, key, needRestart, threadInfo); 249 | break; 250 | } 251 | case NTypes::N48: { 252 | auto n = static_cast(node); 253 | removeAndShrink(n, v, parentNode, parentVersion, keyParent, key, needRestart, threadInfo); 254 | break; 255 | } 256 | case NTypes::N256: { 257 | auto n = static_cast(node); 258 | removeAndShrink(n, v, parentNode, parentVersion, keyParent, key, needRestart, threadInfo); 259 | break; 260 | } 261 | } 262 | } 263 | 264 | bool N::isLocked(uint64_t version) const { 265 | return ((version & 0b10) == 0b10); 266 | } 267 | 268 | uint64_t N::readLockOrRestart(bool &needRestart) const { 269 | uint64_t version; 270 | version = typeVersionLockObsolete.load(); 271 | /* do { 272 | version = typeVersionLockObsolete.load(); 273 | } while (isLocked(version));*/ 274 | if (isLocked(version) || isObsolete(version)) { 275 | needRestart = true; 276 | } 277 | return version; 278 | //uint64_t version; 279 | //while (isLocked(version)) _mm_pause(); 280 | //return version; 281 | } 282 | 283 | bool N::isObsolete(uint64_t version) { 284 | return (version & 1) == 1; 285 | } 286 | 287 | void N::checkOrRestart(uint64_t startRead, bool &needRestart) const { 288 | readUnlockOrRestart(startRead, needRestart); 289 | } 290 | 291 | void N::readUnlockOrRestart(uint64_t startRead, bool &needRestart) const { 292 | needRestart = (startRead != typeVersionLockObsolete.load()); 293 | } 294 | 295 | uint32_t N::getPrefixLength() const { 296 | return prefixCount; 297 | } 298 | 299 | bool N::hasPrefix() const { 300 | return prefixCount > 0; 301 | } 302 | 303 | uint32_t N::getCount() const { 304 | return count; 305 | } 306 | 307 | const uint8_t *N::getPrefix() const { 308 | return prefix; 309 | } 310 | 311 | void N::setPrefix(const uint8_t *prefix, uint32_t length) { 312 | if (length > 0) { 313 | memcpy(this->prefix, prefix, std::min(length, maxStoredPrefixLength)); 314 | prefixCount = length; 315 | } else { 316 | prefixCount = 0; 317 | } 318 | } 319 | 320 | void N::addPrefixBefore(N *node, uint8_t key) { 321 | uint32_t prefixCopyCount = std::min(maxStoredPrefixLength, node->getPrefixLength() + 1); 322 | memmove(this->prefix + prefixCopyCount, this->prefix, 323 | std::min(this->getPrefixLength(), maxStoredPrefixLength - prefixCopyCount)); 324 | memcpy(this->prefix, node->prefix, std::min(prefixCopyCount, node->getPrefixLength())); 325 | if (node->getPrefixLength() < maxStoredPrefixLength) { 326 | this->prefix[prefixCopyCount - 1] = key; 327 | } 328 | this->prefixCount += node->getPrefixLength() + 1; 329 | } 330 | 331 | 332 | bool N::isLeaf(const N *n) { 333 | return (reinterpret_cast(n) & (static_cast(1) << 63)) == (static_cast(1) << 63); 334 | } 335 | 336 | N *N::setLeaf(TID tid) { 337 | return reinterpret_cast(tid | (static_cast(1) << 63)); 338 | } 339 | 340 | TID N::getLeaf(const N *n) { 341 | return (reinterpret_cast(n) & ((static_cast(1) << 63) - 1)); 342 | } 343 | 344 | std::tuple N::getSecondChild(N *node, const uint8_t key) { 345 | switch (node->getType()) { 346 | case NTypes::N4: { 347 | auto n = static_cast(node); 348 | return n->getSecondChild(key); 349 | } 350 | default: { 351 | assert(false); 352 | __builtin_unreachable(); 353 | } 354 | } 355 | } 356 | 357 | void N::deleteNode(N *node) { 358 | if (N::isLeaf(node)) { 359 | return; 360 | } 361 | switch (node->getType()) { 362 | case NTypes::N4: { 363 | auto n = static_cast(node); 364 | delete n; 365 | return; 366 | } 367 | case NTypes::N16: { 368 | auto n = static_cast(node); 369 | delete n; 370 | return; 371 | } 372 | case NTypes::N48: { 373 | auto n = static_cast(node); 374 | delete n; 375 | return; 376 | } 377 | case NTypes::N256: { 378 | auto n = static_cast(node); 379 | delete n; 380 | return; 381 | } 382 | } 383 | delete node; 384 | } 385 | 386 | 387 | TID N::getAnyChildTid(const N *n, bool &needRestart) { 388 | const N *nextNode = n; 389 | 390 | while (true) { 391 | const N *node = nextNode; 392 | auto v = node->readLockOrRestart(needRestart); 393 | if (needRestart) return 0; 394 | 395 | nextNode = getAnyChild(node); 396 | node->readUnlockOrRestart(v, needRestart); 397 | if (needRestart) return 0; 398 | 399 | assert(nextNode != nullptr); 400 | if (isLeaf(nextNode)) { 401 | return getLeaf(nextNode); 402 | } 403 | } 404 | } 405 | 406 | uint64_t N::getChildren(const N *node, uint8_t start, uint8_t end, std::tuple children[], 407 | uint32_t &childrenCount) { 408 | switch (node->getType()) { 409 | case NTypes::N4: { 410 | auto n = static_cast(node); 411 | return n->getChildren(start, end, children, childrenCount); 412 | } 413 | case NTypes::N16: { 414 | auto n = static_cast(node); 415 | return n->getChildren(start, end, children, childrenCount); 416 | } 417 | case NTypes::N48: { 418 | auto n = static_cast(node); 419 | return n->getChildren(start, end, children, childrenCount); 420 | } 421 | case NTypes::N256: { 422 | auto n = static_cast(node); 423 | return n->getChildren(start, end, children, childrenCount); 424 | } 425 | } 426 | assert(false); 427 | __builtin_unreachable(); 428 | } 429 | } -------------------------------------------------------------------------------- /include/btreeolc.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * BTreeOLC_child_layout.h - This file contains a modified version that 3 | * uses the key-value pair layout 4 | * 5 | * We use this to test whether child node layout will affect performance 6 | */ 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | // std::pair 15 | #include 16 | #include 17 | 18 | namespace btreeolc 19 | { 20 | 21 | void noop() {} 22 | enum class PageType : uint8_t 23 | { 24 | BTreeInner = 1, 25 | BTreeLeaf = 2 26 | }; 27 | 28 | static void prefetch(char *ptr, size_t len) { 29 | if (ptr == nullptr) 30 | return; 31 | for (char *p = ptr; p < ptr + len; p += 64) 32 | { 33 | __builtin_prefetch(p); 34 | } 35 | } 36 | 37 | static const uint64_t pageSize = 512; 38 | static const uint64_t leafPageSize = 512; 39 | 40 | struct OptLock 41 | { 42 | std::atomic typeVersionLockObsolete{0b100}; 43 | 44 | bool isLocked(uint64_t version) { return ((version & 0b10) == 0b10); } 45 | 46 | uint64_t readLockOrRestart(bool &needRestart) 47 | { 48 | uint64_t version; 49 | version = typeVersionLockObsolete.load(std::memory_order_relaxed); 50 | if (isLocked(version)) 51 | { 52 | needRestart = true; 53 | _mm_pause(); 54 | } 55 | return version; 56 | } 57 | 58 | void writeLockOrRestart(bool &needRestart) 59 | { 60 | uint64_t version; 61 | version = readLockOrRestart(needRestart); 62 | if (needRestart) 63 | return; 64 | 65 | upgradeToWriteLockOrRestart(version, needRestart); 66 | if (needRestart) 67 | return; 68 | } 69 | 70 | void upgradeToWriteLockOrRestart(uint64_t &version, bool &needRestart) 71 | { 72 | if (typeVersionLockObsolete.compare_exchange_strong(version, 73 | version + 0b10)) 74 | { 75 | version = version + 0b10; 76 | } 77 | else 78 | { 79 | //_mm_pause(); 80 | needRestart = true; 81 | } 82 | } 83 | 84 | void downgradeToReadLock(uint64_t & version) { 85 | version = typeVersionLockObsolete.fetch_add(0b10); 86 | version += 0b10; 87 | } 88 | 89 | void writeUnlock() { typeVersionLockObsolete.fetch_add(0b10); } 90 | 91 | bool isObsolete(uint64_t version) { return (version & 1) == 1; } 92 | 93 | void checkOrRestart(uint64_t startRead, bool &needRestart) const 94 | { 95 | readUnlockOrRestart(startRead, needRestart); 96 | } 97 | 98 | void readUnlockOrRestart(uint64_t startRead, bool &needRestart) const 99 | { 100 | needRestart = (startRead != typeVersionLockObsolete.load()); 101 | } 102 | 103 | void writeUnlockObsolete() { typeVersionLockObsolete.fetch_add(0b11); } 104 | }; 105 | 106 | struct NodeBase : public OptLock 107 | { 108 | PageType type; 109 | uint16_t count; 110 | }; 111 | 112 | struct BTreeLeafBase : public NodeBase 113 | { 114 | static const PageType typeMarker = PageType::BTreeLeaf; 115 | }; 116 | 117 | template 118 | struct BTreeLeaf : public BTreeLeafBase 119 | { 120 | // This is the element type of the leaf node 121 | using KeyValueType = std::pair; 122 | static const uint64_t maxEntries = 123 | (leafPageSize - sizeof(NodeBase)) / (sizeof(KeyValueType)); 124 | 125 | // This is the array that we perform search on 126 | KeyValueType data[maxEntries]; 127 | 128 | struct BTreeLeaf *next; 129 | BTreeLeaf() 130 | { 131 | count = 0; 132 | type = typeMarker; 133 | next = nullptr; 134 | } 135 | 136 | bool isFull() { return count == maxEntries; }; 137 | 138 | unsigned lowerBound(Key k) 139 | { 140 | unsigned lower = 0; 141 | unsigned upper = count; 142 | while (lower < upper && data[lower].first < k) 143 | ++lower; 144 | // do 145 | // { 146 | // unsigned mid = ((upper - lower) / 2) + lower; 147 | // // This is the key at the pivot position 148 | // const Key &middle_key = data[mid].first; 149 | 150 | // if (k < middle_key) 151 | // { 152 | // upper = mid; 153 | // } 154 | // else if (k > middle_key) 155 | // { 156 | // lower = mid + 1; 157 | // } 158 | // else 159 | // { 160 | // return mid; 161 | // } 162 | // } while (lower < upper); 163 | return lower; 164 | } 165 | 166 | void update(Key k, Payload p, unsigned pos) { 167 | assert(data[pos].first == k); 168 | data[pos].second = p; 169 | } 170 | 171 | Key max_key() { 172 | assert(count); 173 | return data[count - 1].first; 174 | } 175 | 176 | void insert_at(Key k, Payload p, unsigned pos) { 177 | assert(data[pos].first == k); 178 | if (count) { 179 | memmove(data + pos + 1, data + pos, sizeof(KeyValueType) * (count - pos)); 180 | // memmove(payloads+pos+1,payloads+pos,sizeof(Payload)*(count-pos)); 181 | data[pos].first = k; 182 | data[pos].second = p; 183 | } else { 184 | data[0].first = k; 185 | data[0].second = p; 186 | } 187 | ++count; 188 | } 189 | 190 | bool insert(Key k, Payload p) 191 | { 192 | assert(count < maxEntries); 193 | if (count) 194 | { 195 | unsigned pos = lowerBound(k); 196 | if ((pos < count) && (data[pos].first == k)) 197 | { 198 | // Upsert 199 | data[pos].second = p; 200 | return true; 201 | } 202 | memmove(data + pos + 1, data + pos, sizeof(KeyValueType) * (count - pos)); 203 | // memmove(payloads+pos+1,payloads+pos,sizeof(Payload)*(count-pos)); 204 | data[pos].first = k; 205 | data[pos].second = p; 206 | } 207 | else 208 | { 209 | data[0].first = k; 210 | data[0].second = p; 211 | } 212 | count++; 213 | return false; 214 | } 215 | 216 | BTreeLeaf *split(Key &sep) 217 | { 218 | BTreeLeaf *newLeaf = new BTreeLeaf(); 219 | newLeaf->count = count - (count / 2); 220 | count = count - newLeaf->count; 221 | newLeaf->next = next; 222 | next = newLeaf; 223 | memcpy(newLeaf->data, data + count, sizeof(KeyValueType) * newLeaf->count); 224 | // memcpy(newLeaf->payloads, payloads+count, 225 | // sizeof(Payload)*newLeaf->count); 226 | sep = data[count - 1].first; 227 | return newLeaf; 228 | } 229 | }; 230 | 231 | struct BTreeInnerBase : public NodeBase 232 | { 233 | static const PageType typeMarker = PageType::BTreeInner; 234 | }; 235 | 236 | 237 | template 238 | struct BTreeInner : public BTreeInnerBase 239 | { 240 | static const uint64_t maxEntries = 241 | (pageSize - sizeof(NodeBase)) / (sizeof(Key) + sizeof(NodeBase *)); 242 | Key keys[maxEntries]; 243 | NodeBase *children[maxEntries]; 244 | 245 | BTreeInner() 246 | { 247 | count = 0; 248 | type = typeMarker; 249 | } 250 | 251 | bool isFull() { return count == (maxEntries - 1); }; 252 | 253 | unsigned lowerBoundBF(Key k) 254 | { 255 | auto base = keys; 256 | unsigned n = count; 257 | while (n > 1) 258 | { 259 | const unsigned half = n / 2; 260 | base = (base[half] < k) ? (base + half) : base; 261 | n -= half; 262 | } 263 | return (*base < k) + base - keys; 264 | } 265 | 266 | unsigned lowerBound(Key k) 267 | { 268 | unsigned lower = 0; 269 | unsigned upper = count; 270 | while (lower < upper && keys[lower] < k) 271 | ++lower; 272 | return lower; 273 | } 274 | 275 | BTreeInner *split(Key &sep) 276 | { 277 | BTreeInner *newInner = new BTreeInner(); 278 | newInner->count = count - (count / 2); 279 | count = count - newInner->count - 1; 280 | sep = keys[count]; 281 | memcpy(newInner->keys, keys + count + 1, 282 | sizeof(Key) * (newInner->count + 1)); 283 | memcpy(newInner->children, children + count + 1, 284 | sizeof(NodeBase *) * (newInner->count + 1)); 285 | return newInner; 286 | } 287 | 288 | Key max_key() { 289 | assert(count); 290 | return keys[count - 1]; 291 | } 292 | 293 | void insert(Key k, NodeBase *child) 294 | { 295 | assert(count < maxEntries - 1); 296 | unsigned pos = lowerBound(k); 297 | memmove(keys + pos + 1, keys + pos, sizeof(Key) * (count - pos + 1)); 298 | memmove(children + pos + 1, children + pos, 299 | sizeof(NodeBase *) * (count - pos + 1)); 300 | keys[pos] = k; 301 | children[pos] = child; 302 | std::swap(children[pos], children[pos + 1]); 303 | count++; 304 | } 305 | }; 306 | 307 | template 308 | struct BTree 309 | { 310 | std::atomic root; 311 | 312 | BTree() { root = new BTreeLeaf(); } 313 | 314 | void makeRoot(Key k, NodeBase *leftChild, NodeBase *rightChild) 315 | { 316 | auto inner = new BTreeInner(); 317 | inner->count = 1; 318 | inner->keys[0] = k; 319 | inner->children[0] = leftChild; 320 | inner->children[1] = rightChild; 321 | root = inner; 322 | } 323 | 324 | void destroy(NodeBase* node) { 325 | if (node == nullptr) return; 326 | if (node->type == PageType::BTreeInner) { 327 | auto inner = static_cast *>(node); 328 | for (int i = 0; i < inner->count; ++i) { 329 | destroy(inner->children[i]); 330 | } 331 | delete inner; 332 | } else { 333 | auto leaf = static_cast *>(node); 334 | delete leaf; 335 | } 336 | } 337 | ~BTree() { 338 | destroy(root.load()); 339 | } 340 | 341 | void yield(int count) 342 | { 343 | if (count > 3) 344 | sched_yield(); 345 | else 346 | _mm_pause(); 347 | } 348 | 349 | 350 | 351 | void insert(Key k, Value v, std::function insert_func=noop) 352 | { 353 | int restartCount = 0; 354 | restart: 355 | if (restartCount++) 356 | yield(restartCount); 357 | bool needRestart = false; 358 | 359 | 360 | // Current node 361 | NodeBase *node = root; 362 | uint64_t versionNode = node->readLockOrRestart(needRestart); 363 | if (needRestart || (node != root)) 364 | goto restart; 365 | 366 | // Parent of current node 367 | BTreeInner *parent = nullptr; 368 | uint64_t versionParent; 369 | 370 | while (node->type == PageType::BTreeInner) 371 | { 372 | auto inner = static_cast *>(node); 373 | 374 | // Split eagerly if full 375 | if (inner->isFull()) 376 | { 377 | // Lock 378 | if (parent) 379 | { 380 | parent->upgradeToWriteLockOrRestart(versionParent, needRestart); 381 | if (needRestart) 382 | goto restart; 383 | } 384 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 385 | if (needRestart) 386 | { 387 | if (parent) 388 | parent->writeUnlock(); 389 | goto restart; 390 | } 391 | if (!parent && (node != root)) 392 | { // there's a new parent 393 | node->writeUnlock(); 394 | goto restart; 395 | } 396 | // Split 397 | Key sep; 398 | BTreeInner *newInner = inner->split(sep); 399 | if (parent) 400 | parent->insert(sep, newInner); 401 | else 402 | makeRoot(sep, inner, newInner); 403 | 404 | if (parent) { 405 | parent->downgradeToReadLock(versionParent); 406 | } 407 | 408 | if (k > sep) { 409 | inner->writeUnlock(); 410 | inner = newInner; 411 | versionNode = newInner->readLockOrRestart(needRestart); 412 | if (needRestart) 413 | goto restart; 414 | } else { 415 | node->downgradeToReadLock(versionNode); 416 | } 417 | } 418 | 419 | if (parent) 420 | { 421 | parent->readUnlockOrRestart(versionParent, needRestart); 422 | if (needRestart) 423 | goto restart; 424 | } 425 | 426 | parent = inner; 427 | versionParent = versionNode; 428 | 429 | node = inner->children[inner->lowerBound(k)]; 430 | inner->checkOrRestart(versionNode, needRestart); 431 | if (needRestart) 432 | goto restart; 433 | prefetch((char*)node, pageSize); 434 | versionNode = node->readLockOrRestart(needRestart); 435 | if (needRestart) 436 | goto restart; 437 | } 438 | 439 | auto leaf = static_cast *>(node); 440 | 441 | // Split leaf if full 442 | if (leaf->count == leaf->maxEntries) 443 | { 444 | // Lock 445 | if (parent) 446 | { 447 | parent->upgradeToWriteLockOrRestart(versionParent, needRestart); 448 | if (needRestart) 449 | goto restart; 450 | } 451 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 452 | if (needRestart) 453 | { 454 | if (parent) 455 | parent->writeUnlock(); 456 | goto restart; 457 | } 458 | if (!parent && (node != root)) 459 | { // there's a new parent 460 | node->writeUnlock(); 461 | goto restart; 462 | } 463 | // Split 464 | Key sep; 465 | BTreeLeaf *newLeaf = leaf->split(sep); 466 | if (k > sep) { 467 | newLeaf->insert(k, v); 468 | } else { 469 | leaf->insert(k, v); 470 | } 471 | insert_func(); 472 | 473 | if (parent) 474 | parent->insert(sep, newLeaf); 475 | else 476 | makeRoot(sep, leaf, newLeaf); 477 | // Unlock and restart 478 | node->writeUnlock(); 479 | if (parent) 480 | parent->writeUnlock(); 481 | return; // success 482 | } 483 | else 484 | { 485 | // only lock leaf node 486 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 487 | if (needRestart) 488 | goto restart; 489 | if (parent) 490 | { 491 | parent->readUnlockOrRestart(versionParent, needRestart); 492 | if (needRestart) 493 | { 494 | node->writeUnlock(); 495 | goto restart; 496 | } 497 | } 498 | leaf->insert(k, v); 499 | insert_func(); 500 | node->writeUnlock(); 501 | return; // success 502 | } 503 | } 504 | 505 | void upsert(Key k, Value v, std::function should_insert_func, std::function insert_func = noop) 506 | { 507 | int restartCount = 0; 508 | restart: 509 | if (restartCount++) 510 | yield(restartCount); 511 | bool needRestart = false; 512 | 513 | // Current node 514 | NodeBase *node = root; 515 | uint64_t versionNode = node->readLockOrRestart(needRestart); 516 | if (needRestart || (node != root)) 517 | goto restart; 518 | 519 | // Parent of current node 520 | BTreeInner *parent = nullptr; 521 | uint64_t versionParent; 522 | 523 | while (node->type == PageType::BTreeInner) 524 | { 525 | auto inner = static_cast *>(node); 526 | 527 | // Split eagerly if full 528 | if (inner->isFull()) 529 | { 530 | // Lock 531 | if (parent) 532 | { 533 | parent->upgradeToWriteLockOrRestart(versionParent, needRestart); 534 | if (needRestart) 535 | goto restart; 536 | } 537 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 538 | if (needRestart) 539 | { 540 | if (parent) 541 | parent->writeUnlock(); 542 | goto restart; 543 | } 544 | if (!parent && (node != root)) 545 | { // there's a new parent 546 | node->writeUnlock(); 547 | goto restart; 548 | } 549 | // Split 550 | Key sep; 551 | BTreeInner *newInner = inner->split(sep); 552 | if (parent) 553 | parent->insert(sep, newInner); 554 | else 555 | makeRoot(sep, inner, newInner); 556 | // Unlock and restart 557 | node->writeUnlock(); 558 | if (parent) 559 | parent->writeUnlock(); 560 | goto restart; 561 | } 562 | 563 | if (parent) 564 | { 565 | parent->readUnlockOrRestart(versionParent, needRestart); 566 | if (needRestart) 567 | goto restart; 568 | } 569 | 570 | parent = inner; 571 | versionParent = versionNode; 572 | 573 | node = inner->children[inner->lowerBound(k)]; 574 | inner->checkOrRestart(versionNode, needRestart); 575 | if (needRestart) 576 | goto restart; 577 | versionNode = node->readLockOrRestart(needRestart); 578 | if (needRestart) 579 | goto restart; 580 | } 581 | 582 | auto leaf = static_cast *>(node); 583 | 584 | // Split leaf if full 585 | if (leaf->count == leaf->maxEntries) 586 | { 587 | // Lock 588 | if (parent) 589 | { 590 | parent->upgradeToWriteLockOrRestart(versionParent, needRestart); 591 | if (needRestart) 592 | goto restart; 593 | } 594 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 595 | if (needRestart) 596 | { 597 | if (parent) 598 | parent->writeUnlock(); 599 | goto restart; 600 | } 601 | if (!parent && (node != root)) 602 | { // there's a new parent 603 | node->writeUnlock(); 604 | goto restart; 605 | } 606 | // Split 607 | Key sep; 608 | BTreeLeaf *newLeaf = leaf->split(sep); 609 | if (parent) 610 | parent->insert(sep, newLeaf); 611 | else 612 | makeRoot(sep, leaf, newLeaf); 613 | // Unlock and restart 614 | node->writeUnlock(); 615 | if (parent) 616 | parent->writeUnlock(); 617 | goto restart; 618 | } 619 | else 620 | { 621 | // only lock leaf node 622 | node->upgradeToWriteLockOrRestart(versionNode, needRestart); 623 | if (needRestart) 624 | goto restart; 625 | if (parent) 626 | { 627 | parent->readUnlockOrRestart(versionParent, needRestart); 628 | if (needRestart) 629 | { 630 | node->writeUnlock(); 631 | goto restart; 632 | } 633 | } 634 | unsigned pos = leaf->lowerBound(k); 635 | bool found = leaf->count == 0 ? false : leaf->data[pos].first == k; 636 | if (found) { 637 | leaf->update(k, v, pos); 638 | insert_func(); 639 | } else { 640 | bool insert = should_insert_func(k); 641 | if (insert) { 642 | leaf->insert_at(k, v, pos); 643 | insert_func(); 644 | } 645 | } 646 | node->writeUnlock(); 647 | return; // success 648 | } 649 | } 650 | 651 | bool lookup(Key k, Value &result) 652 | { 653 | int restartCount = 0; 654 | restart: 655 | if (restartCount++) 656 | yield(restartCount); 657 | bool needRestart = false; 658 | 659 | NodeBase *node = root; 660 | uint64_t versionNode = node->readLockOrRestart(needRestart); 661 | if (needRestart || (node != root)) 662 | goto restart; 663 | 664 | // Parent of current node 665 | BTreeInner *parent = nullptr; 666 | uint64_t versionParent; 667 | 668 | while (node->type == PageType::BTreeInner) 669 | { 670 | auto inner = static_cast *>(node); 671 | 672 | if (parent) 673 | { 674 | parent->readUnlockOrRestart(versionParent, needRestart); 675 | if (needRestart) 676 | goto restart; 677 | } 678 | 679 | parent = inner; 680 | versionParent = versionNode; 681 | 682 | node = inner->children[inner->lowerBound(k)]; 683 | prefetch((char*)node, pageSize); 684 | inner->checkOrRestart(versionNode, needRestart); 685 | if (needRestart) 686 | goto restart; 687 | versionNode = node->readLockOrRestart(needRestart); 688 | if (needRestart) 689 | goto restart; 690 | } 691 | 692 | BTreeLeaf *leaf = static_cast *>(node); 693 | unsigned pos = leaf->lowerBound(k); 694 | bool success = false; 695 | if ((pos < leaf->count) && (leaf->data[pos].first == k)) 696 | { 697 | success = true; 698 | result = leaf->data[pos].second; 699 | } 700 | if (parent) 701 | { 702 | parent->readUnlockOrRestart(versionParent, needRestart); 703 | if (needRestart) 704 | goto restart; 705 | } 706 | node->readUnlockOrRestart(versionNode, needRestart); 707 | if (needRestart) 708 | goto restart; 709 | 710 | return success; 711 | } 712 | 713 | uint64_t scan(Key k, int range, typename BTreeLeaf::KeyValueType *output) 714 | { 715 | int restartCount = 0; 716 | restart: 717 | if (restartCount++) 718 | yield(restartCount); 719 | bool needRestart = false; 720 | 721 | NodeBase *node = root; 722 | uint64_t versionNode = node->readLockOrRestart(needRestart); 723 | if (needRestart || (node != root)) 724 | goto restart; 725 | 726 | // Parent of current node 727 | BTreeInner *parent = nullptr; 728 | uint64_t versionParent; 729 | 730 | while (node->type == PageType::BTreeInner) 731 | { 732 | auto inner = static_cast *>(node); 733 | 734 | if (parent) 735 | { 736 | parent->readUnlockOrRestart(versionParent, needRestart); 737 | if (needRestart) 738 | goto restart; 739 | } 740 | 741 | parent = inner; 742 | versionParent = versionNode; 743 | 744 | node = inner->children[inner->lowerBound(k)]; 745 | inner->checkOrRestart(versionNode, needRestart); 746 | if (needRestart) 747 | goto restart; 748 | versionNode = node->readLockOrRestart(needRestart); 749 | if (needRestart) 750 | goto restart; 751 | } 752 | 753 | BTreeLeaf *leaf = static_cast *>(node); 754 | unsigned pos = leaf->lowerBound(k); 755 | int count = 0; 756 | for (unsigned i = pos; i < leaf->count; i++) 757 | { 758 | if (count == range) 759 | break; 760 | output[count++] = leaf->data[i]; 761 | } 762 | 763 | if (parent) 764 | { 765 | parent->readUnlockOrRestart(versionParent, needRestart); 766 | if (needRestart) 767 | goto restart; 768 | } 769 | node->readUnlockOrRestart(versionNode, needRestart); 770 | if (needRestart) 771 | goto restart; 772 | 773 | return count; 774 | } 775 | 776 | struct range_iterator 777 | { 778 | BTree *btree; 779 | int count; 780 | int pos; 781 | static const uint64_t scan_unit_size = BTreeLeaf::maxEntries; 782 | typename BTreeLeaf::KeyValueType kvs[scan_unit_size]; 783 | 784 | explicit range_iterator(BTree *btree, const Key &startKey) : btree(btree), count(0), pos(0) 785 | { 786 | fill(startKey); 787 | } 788 | 789 | void fill(const Key &nextKey) 790 | { 791 | count = btree->scan(nextKey, scan_unit_size, kvs); 792 | pos = 0; 793 | } 794 | 795 | bool is_end() { return count == 0; } 796 | range_iterator &operator++() 797 | { 798 | if (++pos == count) 799 | { 800 | fill(kvs[pos - 1].first + 1); 801 | } 802 | return *this; 803 | } 804 | bool operator==(const range_iterator &rhs) = delete; 805 | bool operator!=(const range_iterator &rhs) = delete; 806 | Key key() { return kvs[pos].first; } 807 | Value value() { return kvs[pos].second; } 808 | }; 809 | 810 | range_iterator lookup_range(const Key &startKey) 811 | { 812 | return range_iterator(this, startKey); 813 | } 814 | range_iterator end() {} 815 | 816 | struct unsafe_iterator 817 | { 818 | BTreeLeaf *node; 819 | int pos; 820 | explicit unsafe_iterator() : node(nullptr), pos(-1) {} 821 | explicit unsafe_iterator(BTreeLeaf *node, int pos = 0) : node(node), pos(pos) {} 822 | 823 | unsafe_iterator &operator++() 824 | { 825 | if (++pos == node->count) 826 | { 827 | node = node->next; 828 | pos = 0; 829 | } 830 | return *this; 831 | } 832 | bool operator==(const unsafe_iterator &rhs) { return node == rhs.node && pos == rhs.pos; } 833 | bool operator!=(const unsafe_iterator &rhs) { return !(*this == rhs); } 834 | int next_node(Key & last_key) { 835 | int ret = node->count - pos; 836 | last_key = node->data[node->count - 1].first; 837 | node = node->next; 838 | pos = 0; 839 | return ret; 840 | } 841 | Key key() { return node->data[pos].first; } 842 | Value value() { return node->data[pos].second; } 843 | }; 844 | 845 | unsafe_iterator begin_unsafe() 846 | { 847 | NodeBase *node = root; 848 | while (node->type == PageType::BTreeInner) 849 | { 850 | auto inner = static_cast *>(node); 851 | node = inner->children[0]; 852 | } 853 | assert(node->type == PageType::BTreeLeaf); 854 | return unsafe_iterator((BTreeLeaf*)node, 0); 855 | } 856 | 857 | unsafe_iterator end_unsafe() 858 | { 859 | return unsafe_iterator(nullptr, 0); 860 | } 861 | }; 862 | 863 | } // namespace btreeolc -------------------------------------------------------------------------------- /misc/ARTOLC/Tree.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "Tree.h" 4 | #include "N.cpp" 5 | #include "Epoche.cpp" 6 | #include "Key.h" 7 | 8 | 9 | namespace ART_OLC { 10 | 11 | Tree::Tree(LoadKeyFunction loadKey) : root(new N256( nullptr, 0)), loadKey(loadKey) { 12 | } 13 | 14 | Tree::~Tree() { 15 | N::deleteChildren(root); 16 | N::deleteNode(root); 17 | } 18 | 19 | ThreadInfo Tree::getThreadInfo() { 20 | return ThreadInfo(this->epoche); 21 | } 22 | 23 | 24 | void yield(int count) { 25 | if (count>3) 26 | sched_yield(); 27 | else 28 | _mm_pause(); 29 | } 30 | 31 | TID Tree::lookup(const Key &k, ThreadInfo &threadEpocheInfo) const { 32 | EpocheGuardReadonly epocheGuard(threadEpocheInfo); 33 | int restartCount = 0; 34 | restart: 35 | if (restartCount++) 36 | yield(restartCount); 37 | bool needRestart = false; 38 | 39 | N *node; 40 | N *parentNode = nullptr; 41 | uint64_t v; 42 | uint32_t level = 0; 43 | bool optimisticPrefixMatch = false; 44 | 45 | node = root; 46 | v = node->readLockOrRestart(needRestart); 47 | if (needRestart) goto restart; 48 | while (true) { 49 | switch (checkPrefix(node, k, level)) { // increases level 50 | case CheckPrefixResult::NoMatch: 51 | node->readUnlockOrRestart(v, needRestart); 52 | if (needRestart) goto restart; 53 | return 0; 54 | case CheckPrefixResult::OptimisticMatch: 55 | optimisticPrefixMatch = true; 56 | // fallthrough 57 | case CheckPrefixResult::Match: 58 | if (k.getKeyLen() <= level) { 59 | return 0; 60 | } 61 | parentNode = node; 62 | node = N::getChild(k[level], parentNode); 63 | parentNode->checkOrRestart(v,needRestart); 64 | if (needRestart) goto restart; 65 | 66 | if (node == nullptr) { 67 | return 0; 68 | } 69 | if (N::isLeaf(node)) { 70 | parentNode->readUnlockOrRestart(v, needRestart); 71 | if (needRestart) goto restart; 72 | 73 | TID tid = N::getLeaf(node); 74 | if (level < k.getKeyLen() - 1 || optimisticPrefixMatch) { 75 | return checkKey(tid, k); 76 | } 77 | return tid; 78 | } 79 | level++; 80 | } 81 | uint64_t nv = node->readLockOrRestart(needRestart); 82 | if (needRestart) goto restart; 83 | 84 | parentNode->readUnlockOrRestart(v, needRestart); 85 | if (needRestart) goto restart; 86 | v = nv; 87 | } 88 | } 89 | 90 | bool Tree::lookupRange(const Key &start, const Key &end, Key &continueKey, TID result[], 91 | std::size_t resultSize, std::size_t &resultsFound, ThreadInfo &threadEpocheInfo) const { 92 | for (uint32_t i = 0; i < std::min(start.getKeyLen(), end.getKeyLen()); ++i) { 93 | if (start[i] > end[i]) { 94 | resultsFound = 0; 95 | return false; 96 | } else if (start[i] < end[i]) { 97 | break; 98 | } 99 | } 100 | EpocheGuard epocheGuard(threadEpocheInfo); 101 | TID toContinue = 0; 102 | std::function copy = [&result, &resultSize, &resultsFound, &toContinue, ©](const N *node) { 103 | if (N::isLeaf(node)) { 104 | if (resultsFound == resultSize) { 105 | toContinue = N::getLeaf(node); 106 | return; 107 | } 108 | result[resultsFound] = N::getLeaf(node); 109 | resultsFound++; 110 | } else { 111 | std::tuple children[256]; 112 | uint32_t childrenCount = 0; 113 | N::getChildren(node, 0u, 255u, children, childrenCount); 114 | for (uint32_t i = 0; i < childrenCount; ++i) { 115 | const N *n = std::get<1>(children[i]); 116 | copy(n); 117 | if (toContinue != 0) { 118 | break; 119 | } 120 | } 121 | } 122 | }; 123 | std::function findStart = [©, &start, &findStart, &toContinue, this]( 124 | N *node, uint8_t nodeK, uint32_t level, const N *parentNode, uint64_t vp) { 125 | if (N::isLeaf(node)) { 126 | copy(node); 127 | return; 128 | } 129 | uint64_t v; 130 | PCCompareResults prefixResult; 131 | 132 | { 133 | readAgain: 134 | bool needRestart = false; 135 | v = node->readLockOrRestart(needRestart); 136 | if (needRestart) goto readAgain; 137 | 138 | prefixResult = checkPrefixCompare(node, start, 0, level, loadKey, needRestart); 139 | if (needRestart) goto readAgain; 140 | 141 | parentNode->readUnlockOrRestart(vp, needRestart); 142 | if (needRestart) { 143 | readParentAgain: 144 | needRestart = false; 145 | vp = parentNode->readLockOrRestart(needRestart); 146 | if (needRestart) goto readParentAgain; 147 | 148 | node = N::getChild(nodeK, parentNode); 149 | 150 | parentNode->readUnlockOrRestart(vp, needRestart); 151 | if (needRestart) goto readParentAgain; 152 | 153 | if (node == nullptr) { 154 | return; 155 | } 156 | if (N::isLeaf(node)) { 157 | copy(node); 158 | return; 159 | } 160 | goto readAgain; 161 | } 162 | node->readUnlockOrRestart(v, needRestart); 163 | if (needRestart) goto readAgain; 164 | } 165 | 166 | switch (prefixResult) { 167 | case PCCompareResults::Bigger: 168 | copy(node); 169 | break; 170 | case PCCompareResults::Equal: { 171 | uint8_t startLevel = (start.getKeyLen() > level) ? start[level] : 0; 172 | std::tuple children[256]; 173 | uint32_t childrenCount = 0; 174 | v = N::getChildren(node, startLevel, 255, children, childrenCount); 175 | for (uint32_t i = 0; i < childrenCount; ++i) { 176 | const uint8_t k = std::get<0>(children[i]); 177 | N *n = std::get<1>(children[i]); 178 | if (k == startLevel) { 179 | findStart(n, k, level + 1, node, v); 180 | } else if (k > startLevel) { 181 | copy(n); 182 | } 183 | if (toContinue != 0) { 184 | break; 185 | } 186 | } 187 | break; 188 | } 189 | case PCCompareResults::Smaller: 190 | break; 191 | } 192 | }; 193 | std::function findEnd = [©, &end, &toContinue, &findEnd, this]( 194 | N *node, uint8_t nodeK, uint32_t level, const N *parentNode, uint64_t vp) { 195 | if (N::isLeaf(node)) { 196 | return; 197 | } 198 | uint64_t v; 199 | PCCompareResults prefixResult; 200 | { 201 | readAgain: 202 | bool needRestart = false; 203 | v = node->readLockOrRestart(needRestart); 204 | if (needRestart) goto readAgain; 205 | 206 | prefixResult = checkPrefixCompare(node, end, 255, level, loadKey, needRestart); 207 | if (needRestart) goto readAgain; 208 | 209 | parentNode->readUnlockOrRestart(vp, needRestart); 210 | if (needRestart) { 211 | readParentAgain: 212 | vp = parentNode->readLockOrRestart(needRestart); 213 | if (needRestart) goto readParentAgain; 214 | 215 | node = N::getChild(nodeK, parentNode); 216 | 217 | parentNode->readUnlockOrRestart(vp, needRestart); 218 | if (needRestart) goto readParentAgain; 219 | 220 | if (node == nullptr) { 221 | return; 222 | } 223 | if (N::isLeaf(node)) { 224 | return; 225 | } 226 | goto readAgain; 227 | } 228 | node->readUnlockOrRestart(v, needRestart); 229 | if (needRestart) goto readAgain; 230 | } 231 | switch (prefixResult) { 232 | case PCCompareResults::Smaller: 233 | copy(node); 234 | break; 235 | case PCCompareResults::Equal: { 236 | uint8_t endLevel = (end.getKeyLen() > level) ? end[level] : 255; 237 | std::tuple children[256]; 238 | uint32_t childrenCount = 0; 239 | v = N::getChildren(node, 0, endLevel, children, childrenCount); 240 | for (uint32_t i = 0; i < childrenCount; ++i) { 241 | const uint8_t k = std::get<0>(children[i]); 242 | N *n = std::get<1>(children[i]); 243 | if (k == endLevel) { 244 | findEnd(n, k, level + 1, node, v); 245 | } else if (k < endLevel) { 246 | copy(n); 247 | } 248 | if (toContinue != 0) { 249 | break; 250 | } 251 | } 252 | break; 253 | } 254 | case PCCompareResults::Bigger: 255 | break; 256 | } 257 | }; 258 | 259 | 260 | int restartCount = 0; 261 | restart: 262 | if (restartCount++) 263 | yield(restartCount); 264 | bool needRestart = false; 265 | 266 | resultsFound = 0; 267 | 268 | uint32_t level = 0; 269 | N *node = nullptr; 270 | N *nextNode = root; 271 | N *parentNode; 272 | uint64_t v = 0; 273 | uint64_t vp; 274 | 275 | while (true) { 276 | parentNode = node; 277 | vp = v; 278 | node = nextNode; 279 | PCEqualsResults prefixResult; 280 | v = node->readLockOrRestart(needRestart); 281 | if (needRestart) goto restart; 282 | prefixResult = checkPrefixEquals(node, level, start, end, loadKey, needRestart); 283 | if (needRestart) goto restart; 284 | if (parentNode != nullptr) { 285 | parentNode->readUnlockOrRestart(vp, needRestart); 286 | if (needRestart) goto restart; 287 | } 288 | node->readUnlockOrRestart(v, needRestart); 289 | if (needRestart) goto restart; 290 | 291 | switch (prefixResult) { 292 | case PCEqualsResults::NoMatch: { 293 | return false; 294 | } 295 | case PCEqualsResults::Contained: { 296 | copy(node); 297 | break; 298 | } 299 | case PCEqualsResults::BothMatch: { 300 | uint8_t startLevel = (start.getKeyLen() > level) ? start[level] : 0; 301 | uint8_t endLevel = (end.getKeyLen() > level) ? end[level] : 255; 302 | if (startLevel != endLevel) { 303 | std::tuple children[256]; 304 | uint32_t childrenCount = 0; 305 | v = N::getChildren(node, startLevel, endLevel, children, childrenCount); 306 | for (uint32_t i = 0; i < childrenCount; ++i) { 307 | const uint8_t k = std::get<0>(children[i]); 308 | N *n = std::get<1>(children[i]); 309 | if (k == startLevel) { 310 | findStart(n, k, level + 1, node, v); 311 | } else if (k > startLevel && k < endLevel) { 312 | copy(n); 313 | } else if (k == endLevel) { 314 | findEnd(n, k, level + 1, node, v); 315 | } 316 | if (toContinue) { 317 | break; 318 | } 319 | } 320 | } else { 321 | nextNode = N::getChild(startLevel, node); 322 | node->readUnlockOrRestart(v, needRestart); 323 | if (needRestart) goto restart; 324 | level++; 325 | continue; 326 | } 327 | break; 328 | } 329 | } 330 | break; 331 | } 332 | if (toContinue != 0) { 333 | loadKey(toContinue, continueKey); 334 | return true; 335 | } else { 336 | return false; 337 | } 338 | } 339 | 340 | 341 | TID Tree::checkKey(const TID tid, const Key &k) const { 342 | Key kt; 343 | this->loadKey(tid, kt); 344 | if (k == kt) { 345 | return tid; 346 | } 347 | return 0; 348 | } 349 | void Tree::insert(const Key &k, TID tid, ThreadInfo &epocheInfo, std::function insert_func) { 350 | EpocheGuard epocheGuard(epocheInfo); 351 | int restartCount = 0; 352 | restart: 353 | if (restartCount++) 354 | yield(restartCount); 355 | bool needRestart = false; 356 | 357 | N *node = nullptr; 358 | N *nextNode = root; 359 | N *parentNode = nullptr; 360 | uint8_t parentKey, nodeKey = 0; 361 | uint64_t parentVersion = 0; 362 | uint32_t level = 0; 363 | 364 | while (true) { 365 | parentNode = node; 366 | parentKey = nodeKey; 367 | node = nextNode; 368 | auto v = node->readLockOrRestart(needRestart); 369 | if (needRestart) goto restart; 370 | 371 | uint32_t nextLevel = level; 372 | 373 | uint8_t nonMatchingKey; 374 | Prefix remainingPrefix; 375 | auto res = checkPrefixPessimistic(node, k, nextLevel, nonMatchingKey, remainingPrefix, 376 | this->loadKey, needRestart); // increases level 377 | if (needRestart) goto restart; 378 | switch (res) { 379 | case CheckPrefixPessimisticResult::NoMatch: { 380 | parentNode->upgradeToWriteLockOrRestart(parentVersion, needRestart); 381 | if (needRestart) goto restart; 382 | 383 | node->upgradeToWriteLockOrRestart(v, needRestart); 384 | if (needRestart) { 385 | parentNode->writeUnlock(); 386 | goto restart; 387 | } 388 | // 1) Create new node which will be parent of node, Set common prefix, level to this node 389 | auto newNode = new N4(node->getPrefix(), nextLevel - level); 390 | 391 | // 2) add node and (tid, *k) as children 392 | newNode->insert(k[nextLevel], N::setLeaf(tid)); 393 | newNode->insert(nonMatchingKey, node); 394 | 395 | // 3) upgradeToWriteLockOrRestart, update parentNode to point to the new node, unlock 396 | N::change(parentNode, parentKey, newNode); 397 | parentNode->writeUnlock(); 398 | 399 | // 4) update prefix of node, unlock 400 | node->setPrefix(remainingPrefix, 401 | node->getPrefixLength() - ((nextLevel - level) + 1)); 402 | 403 | insert_func(); 404 | node->writeUnlock(); 405 | return; 406 | } 407 | case CheckPrefixPessimisticResult::Match: 408 | break; 409 | } 410 | level = nextLevel; 411 | nodeKey = k[level]; 412 | nextNode = N::getChild(nodeKey, node); 413 | node->checkOrRestart(v,needRestart); 414 | if (needRestart) goto restart; 415 | 416 | if (nextNode == nullptr) { 417 | N::insertAndUnlock(node, v, parentNode, parentVersion, parentKey, nodeKey, N::setLeaf(tid), needRestart, epocheInfo, insert_func); 418 | if (needRestart) goto restart; 419 | return; 420 | } 421 | 422 | if (parentNode != nullptr) { 423 | parentNode->readUnlockOrRestart(parentVersion, needRestart); 424 | if (needRestart) goto restart; 425 | } 426 | 427 | if (N::isLeaf(nextNode)) { 428 | node->upgradeToWriteLockOrRestart(v, needRestart); 429 | if (needRestart) goto restart; 430 | 431 | Key key; 432 | loadKey(N::getLeaf(nextNode), key); 433 | 434 | if (key == k) { 435 | // upsert 436 | N::change(node, k[level], N::setLeaf(tid)); 437 | insert_func(); 438 | node->writeUnlock(); 439 | return; 440 | } 441 | 442 | level++; 443 | uint32_t prefixLength = 0; 444 | while (key[level + prefixLength] == k[level + prefixLength]) { 445 | prefixLength++; 446 | } 447 | 448 | auto n4 = new N4(&k[level], prefixLength); 449 | n4->insert(k[level + prefixLength], N::setLeaf(tid)); 450 | n4->insert(key[level + prefixLength], nextNode); 451 | N::change(node, k[level - 1], n4); 452 | insert_func(); 453 | node->writeUnlock(); 454 | return; 455 | } 456 | level++; 457 | parentVersion = v; 458 | } 459 | } 460 | 461 | void Tree::remove(const Key &k, TID tid, ThreadInfo &threadInfo) { 462 | EpocheGuard epocheGuard(threadInfo); 463 | int restartCount = 0; 464 | restart: 465 | if (restartCount++) 466 | yield(restartCount); 467 | bool needRestart = false; 468 | 469 | N *node = nullptr; 470 | N *nextNode = root; 471 | N *parentNode = nullptr; 472 | uint8_t parentKey, nodeKey = 0; 473 | uint64_t parentVersion = 0; 474 | uint32_t level = 0; 475 | 476 | while (true) { 477 | parentNode = node; 478 | parentKey = nodeKey; 479 | node = nextNode; 480 | auto v = node->readLockOrRestart(needRestart); 481 | if (needRestart) goto restart; 482 | 483 | switch (checkPrefix(node, k, level)) { // increases level 484 | case CheckPrefixResult::NoMatch: 485 | node->readUnlockOrRestart(v, needRestart); 486 | if (needRestart) goto restart; 487 | return; 488 | case CheckPrefixResult::OptimisticMatch: 489 | // fallthrough 490 | case CheckPrefixResult::Match: { 491 | nodeKey = k[level]; 492 | nextNode = N::getChild(nodeKey, node); 493 | 494 | node->checkOrRestart(v, needRestart); 495 | if (needRestart) goto restart; 496 | 497 | if (nextNode == nullptr) { 498 | node->readUnlockOrRestart(v, needRestart); 499 | if (needRestart) goto restart; 500 | return; 501 | } 502 | if (N::isLeaf(nextNode)) { 503 | if (N::getLeaf(nextNode) != tid) { 504 | return; 505 | } 506 | assert(parentNode == nullptr || node->getCount() != 1); 507 | if (node->getCount() == 2 && parentNode != nullptr) { 508 | parentNode->upgradeToWriteLockOrRestart(parentVersion, needRestart); 509 | if (needRestart) goto restart; 510 | 511 | node->upgradeToWriteLockOrRestart(v, needRestart); 512 | if (needRestart) { 513 | parentNode->writeUnlock(); 514 | goto restart; 515 | } 516 | // 1. check remaining entries 517 | N *secondNodeN; 518 | uint8_t secondNodeK; 519 | std::tie(secondNodeN, secondNodeK) = N::getSecondChild(node, nodeKey); 520 | if (N::isLeaf(secondNodeN)) { 521 | //N::remove(node, k[level]); not necessary 522 | N::change(parentNode, parentKey, secondNodeN); 523 | 524 | parentNode->writeUnlock(); 525 | node->writeUnlockObsolete(); 526 | this->epoche.markNodeForDeletion(node, threadInfo); 527 | } else { 528 | secondNodeN->writeLockOrRestart(needRestart); 529 | if (needRestart) { 530 | node->writeUnlock(); 531 | parentNode->writeUnlock(); 532 | goto restart; 533 | } 534 | 535 | //N::remove(node, k[level]); not necessary 536 | N::change(parentNode, parentKey, secondNodeN); 537 | parentNode->writeUnlock(); 538 | 539 | secondNodeN->addPrefixBefore(node, secondNodeK); 540 | secondNodeN->writeUnlock(); 541 | 542 | node->writeUnlockObsolete(); 543 | this->epoche.markNodeForDeletion(node, threadInfo); 544 | } 545 | } else { 546 | N::removeAndUnlock(node, v, k[level], parentNode, parentVersion, parentKey, needRestart, threadInfo); 547 | if (needRestart) goto restart; 548 | } 549 | return; 550 | } 551 | level++; 552 | parentVersion = v; 553 | } 554 | } 555 | } 556 | } 557 | 558 | inline typename Tree::CheckPrefixResult Tree::checkPrefix(N *n, const Key &k, uint32_t &level) { 559 | if (n->hasPrefix()) { 560 | if (k.getKeyLen() <= level + n->getPrefixLength()) { 561 | return CheckPrefixResult::NoMatch; 562 | } 563 | for (uint32_t i = 0; i < std::min(n->getPrefixLength(), maxStoredPrefixLength); ++i) { 564 | if (n->getPrefix()[i] != k[level]) { 565 | return CheckPrefixResult::NoMatch; 566 | } 567 | ++level; 568 | } 569 | if (n->getPrefixLength() > maxStoredPrefixLength) { 570 | level = level + (n->getPrefixLength() - maxStoredPrefixLength); 571 | return CheckPrefixResult::OptimisticMatch; 572 | } 573 | } 574 | return CheckPrefixResult::Match; 575 | } 576 | 577 | typename Tree::CheckPrefixPessimisticResult Tree::checkPrefixPessimistic(N *n, const Key &k, uint32_t &level, 578 | uint8_t &nonMatchingKey, 579 | Prefix &nonMatchingPrefix, 580 | LoadKeyFunction loadKey, bool &needRestart) { 581 | if (n->hasPrefix()) { 582 | uint32_t prevLevel = level; 583 | Key kt; 584 | for (uint32_t i = 0; i < n->getPrefixLength(); ++i) { 585 | if (i == maxStoredPrefixLength) { 586 | auto anyTID = N::getAnyChildTid(n, needRestart); 587 | if (needRestart) return CheckPrefixPessimisticResult::Match; 588 | loadKey(anyTID, kt); 589 | } 590 | uint8_t curKey = i >= maxStoredPrefixLength ? kt[level] : n->getPrefix()[i]; 591 | if (curKey != k[level]) { 592 | nonMatchingKey = curKey; 593 | if (n->getPrefixLength() > maxStoredPrefixLength) { 594 | if (i < maxStoredPrefixLength) { 595 | auto anyTID = N::getAnyChildTid(n, needRestart); 596 | if (needRestart) return CheckPrefixPessimisticResult::Match; 597 | loadKey(anyTID, kt); 598 | } 599 | memcpy(nonMatchingPrefix, &kt[0] + level + 1, std::min((n->getPrefixLength() - (level - prevLevel) - 1), 600 | maxStoredPrefixLength)); 601 | } else { 602 | memcpy(nonMatchingPrefix, n->getPrefix() + i + 1, n->getPrefixLength() - i - 1); 603 | } 604 | return CheckPrefixPessimisticResult::NoMatch; 605 | } 606 | ++level; 607 | } 608 | } 609 | return CheckPrefixPessimisticResult::Match; 610 | } 611 | 612 | typename Tree::PCCompareResults Tree::checkPrefixCompare(const N *n, const Key &k, uint8_t fillKey, uint32_t &level, 613 | LoadKeyFunction loadKey, bool &needRestart) { 614 | if (n->hasPrefix()) { 615 | Key kt; 616 | for (uint32_t i = 0; i < n->getPrefixLength(); ++i) { 617 | if (i == maxStoredPrefixLength) { 618 | auto anyTID = N::getAnyChildTid(n, needRestart); 619 | if (needRestart) return PCCompareResults::Equal; 620 | loadKey(anyTID, kt); 621 | } 622 | uint8_t kLevel = (k.getKeyLen() > level) ? k[level] : fillKey; 623 | 624 | uint8_t curKey = i >= maxStoredPrefixLength ? kt[level] : n->getPrefix()[i]; 625 | if (curKey < kLevel) { 626 | return PCCompareResults::Smaller; 627 | } else if (curKey > kLevel) { 628 | return PCCompareResults::Bigger; 629 | } 630 | ++level; 631 | } 632 | } 633 | return PCCompareResults::Equal; 634 | } 635 | 636 | typename Tree::PCEqualsResults Tree::checkPrefixEquals(const N *n, uint32_t &level, const Key &start, const Key &end, 637 | LoadKeyFunction loadKey, bool &needRestart) { 638 | if (n->hasPrefix()) { 639 | Key kt; 640 | for (uint32_t i = 0; i < n->getPrefixLength(); ++i) { 641 | if (i == maxStoredPrefixLength) { 642 | auto anyTID = N::getAnyChildTid(n, needRestart); 643 | if (needRestart) return PCEqualsResults::BothMatch; 644 | loadKey(anyTID, kt); 645 | } 646 | uint8_t startLevel = (start.getKeyLen() > level) ? start[level] : 0; 647 | uint8_t endLevel = (end.getKeyLen() > level) ? end[level] : 255; 648 | 649 | uint8_t curKey = i >= maxStoredPrefixLength ? kt[level] : n->getPrefix()[i]; 650 | if (curKey > startLevel && curKey < endLevel) { 651 | return PCEqualsResults::Contained; 652 | } else if (curKey < startLevel || curKey > endLevel) { 653 | return PCEqualsResults::NoMatch; 654 | } 655 | ++level; 656 | } 657 | } 658 | return PCEqualsResults::BothMatch; 659 | } 660 | } 661 | -------------------------------------------------------------------------------- /include/art_tree.hpp: -------------------------------------------------------------------------------- 1 | #include // malloc, free 2 | #include // memset, memcpy 3 | #include // integer types 4 | #include // x86 SSE intrinsics 5 | #include 6 | #include 7 | #include // gettime 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace Art { 16 | // Constants for the node types 17 | static const int8_t NodeType4=0; 18 | static const int8_t NodeType16=1; 19 | static const int8_t NodeType48=2; 20 | static const int8_t NodeType256=3; 21 | 22 | // The maximum prefix length for compressed paths stored in the 23 | // header, if the path is longer it is loaded from the database on 24 | // demand 25 | static const unsigned maxPrefixLength=9; 26 | 27 | // Shared header of all inner nodes 28 | struct Node { 29 | // length of the compressed path (prefix) 30 | uint32_t prefixLength; 31 | // number of non-null children 32 | uint16_t count; 33 | // node type 34 | int8_t type; 35 | // compressed path (prefix) 36 | uint8_t prefix[maxPrefixLength]; 37 | 38 | Node(int8_t type) : prefixLength(0),count(0),type(type) {} 39 | }; 40 | 41 | // Node with up to 4 children 42 | struct Node4 : Node { 43 | uint8_t key[4]; 44 | Node* child[4]; 45 | 46 | Node4() : Node(NodeType4) { 47 | memset(key,0,sizeof(key)); 48 | memset(child,0,sizeof(child)); 49 | } 50 | }; 51 | 52 | // Node with up to 16 children 53 | struct Node16 : Node { 54 | uint8_t key[16]; 55 | Node* child[16]; 56 | 57 | Node16() : Node(NodeType16) { 58 | memset(key,0,sizeof(key)); 59 | memset(child,0,sizeof(child)); 60 | } 61 | }; 62 | 63 | static const uint8_t emptyMarker=48; 64 | 65 | // Node with up to 48 children 66 | struct Node48 : Node { 67 | uint8_t childIndex[256]; 68 | Node* child[48]; 69 | 70 | Node48() : Node(NodeType48) { 71 | memset(childIndex,emptyMarker,sizeof(childIndex)); 72 | memset(child,0,sizeof(child)); 73 | } 74 | }; 75 | 76 | // Node with up to 256 children 77 | struct Node256 : Node { 78 | Node* child[256]; 79 | 80 | Node256() : Node(NodeType256) { 81 | memset(child,0,sizeof(child)); 82 | } 83 | }; 84 | 85 | inline Node* makeLeaf(uintptr_t tid) { 86 | // Create a pseudo-leaf 87 | return reinterpret_cast((tid<<1)|1); 88 | } 89 | 90 | inline uintptr_t getLeafValue(Node* node) { 91 | // The the value stored in the pseudo-leaf 92 | return reinterpret_cast(node)>>1; 93 | } 94 | 95 | inline bool isLeaf(Node* node) { 96 | // Is the node a leaf? 97 | return reinterpret_cast(node)&1; 98 | } 99 | 100 | uint8_t flipSign(uint8_t keyByte) { 101 | // Flip the sign bit, enables signed SSE comparison of unsigned values, used by Node16 102 | return keyByte^128; 103 | } 104 | 105 | std::function loadKey = [](uintptr_t tid,uint8_t key[]) { 106 | // Store the key of the tuple into the key vector 107 | // Implementation is database specific 108 | reinterpret_cast(key)[0]=__builtin_bswap64(tid); 109 | }; 110 | 111 | std::function loadLowerBoundKey; 112 | 113 | // This address is used to communicate that search failed 114 | Node* nullNode=NULL; 115 | 116 | static inline unsigned ctz(uint16_t x) { 117 | // Count trailing zeros, only defined for x>0 118 | #ifdef __GNUC__ 119 | return __builtin_ctz(x); 120 | #else 121 | // Adapted from Hacker's Delight 122 | unsigned n=1; 123 | if ((x&0xFF)==0) {n+=8; x=x>>8;} 124 | if ((x&0x0F)==0) {n+=4; x=x>>4;} 125 | if ((x&0x03)==0) {n+=2; x=x>>2;} 126 | return n-(x&1); 127 | #endif 128 | } 129 | 130 | Node* findChildLowerBound(Node* n,uint8_t keyByte, bool & equal) { 131 | // Find the next child for the keyByte 132 | switch (n->type) { 133 | case NodeType4: { 134 | Node4* node=static_cast(n); 135 | unsigned i = 0; 136 | for (; i < node->count && node->key[i] < keyByte; i++); 137 | equal = i < node->count && node->key[i] == keyByte; 138 | return i < node->count ? node->child[i] : nullptr; 139 | } 140 | case NodeType16: { 141 | Node16* node=static_cast(n); 142 | __m128i cmp=_mm_cmpgt_epi8(_mm_set1_epi8(flipSign(keyByte)),_mm_loadu_si128(reinterpret_cast<__m128i*>(node->key))); 143 | unsigned bitfield=(~_mm_movemask_epi8(cmp))&((1<count)-1); 144 | if (bitfield) { 145 | int idx = ctz(bitfield); 146 | equal = node->key[idx] == flipSign(keyByte); 147 | return node->child[idx]; 148 | } else { 149 | return nullptr; 150 | } 151 | } 152 | case NodeType48: { 153 | Node48* node = static_cast(n); 154 | equal = node->childIndex[keyByte] != emptyMarker; 155 | for (uint16_t b = keyByte; b <= 255; ++b) 156 | if (node->childIndex[b] != emptyMarker) 157 | return node->child[node->childIndex[b]]; 158 | return nullptr; 159 | } 160 | case NodeType256: { 161 | Node256* node = static_cast(n); 162 | equal = node->child[keyByte]; 163 | for (uint16_t b = keyByte; b <=255; ++b) 164 | if (node->child[b]) 165 | return node->child[b]; 166 | return nullptr; 167 | } 168 | } 169 | throw; // Unreachable 170 | } 171 | Node** findChild(Node* n,uint8_t keyByte) { 172 | // Find the next child for the keyByte 173 | switch (n->type) { 174 | case NodeType4: { 175 | Node4* node=static_cast(n); 176 | for (unsigned i=0;icount;i++) 177 | if (node->key[i]==keyByte) 178 | return &node->child[i]; 179 | return &nullNode; 180 | } 181 | case NodeType16: { 182 | Node16* node=static_cast(n); 183 | __m128i cmp=_mm_cmpeq_epi8(_mm_set1_epi8(flipSign(keyByte)),_mm_loadu_si128(reinterpret_cast<__m128i*>(node->key))); 184 | unsigned bitfield=_mm_movemask_epi8(cmp)&((1<count)-1); 185 | if (bitfield) 186 | return &node->child[ctz(bitfield)]; 187 | else 188 | return &nullNode; 189 | } 190 | case NodeType48: { 191 | Node48* node=static_cast(n); 192 | if (node->childIndex[keyByte]!=emptyMarker) 193 | return &node->child[node->childIndex[keyByte]]; else 194 | return &nullNode; 195 | } 196 | case NodeType256: { 197 | Node256* node=static_cast(n); 198 | return &(node->child[keyByte]); 199 | } 200 | } 201 | throw; // Unreachable 202 | } 203 | 204 | Node* minimum(Node* node) { 205 | // Find the leaf with smallest key 206 | if (!node) 207 | return NULL; 208 | 209 | if (isLeaf(node)) 210 | return node; 211 | 212 | switch (node->type) { 213 | case NodeType4: { 214 | Node4* n=static_cast(node); 215 | return minimum(n->child[0]); 216 | } 217 | case NodeType16: { 218 | Node16* n=static_cast(node); 219 | return minimum(n->child[0]); 220 | } 221 | case NodeType48: { 222 | Node48* n=static_cast(node); 223 | unsigned pos=0; 224 | while (n->childIndex[pos]==emptyMarker) 225 | pos++; 226 | return minimum(n->child[n->childIndex[pos]]); 227 | } 228 | case NodeType256: { 229 | Node256* n=static_cast(node); 230 | unsigned pos=0; 231 | while (!n->child[pos]) 232 | pos++; 233 | return minimum(n->child[pos]); 234 | } 235 | } 236 | throw; // Unreachable 237 | } 238 | 239 | Node* maximum(Node* node) { 240 | // Find the leaf with largest key 241 | if (!node) 242 | return NULL; 243 | 244 | if (isLeaf(node)) 245 | return node; 246 | 247 | switch (node->type) { 248 | case NodeType4: { 249 | Node4* n=static_cast(node); 250 | return maximum(n->child[n->count-1]); 251 | } 252 | case NodeType16: { 253 | Node16* n=static_cast(node); 254 | return maximum(n->child[n->count-1]); 255 | } 256 | case NodeType48: { 257 | Node48* n=static_cast(node); 258 | unsigned pos=255; 259 | while (n->childIndex[pos]==emptyMarker) 260 | pos--; 261 | return maximum(n->child[n->childIndex[pos]]); 262 | } 263 | case NodeType256: { 264 | Node256* n=static_cast(node); 265 | unsigned pos=255; 266 | while (!n->child[pos]) 267 | pos--; 268 | return maximum(n->child[pos]); 269 | } 270 | } 271 | throw; // Unreachable 272 | } 273 | 274 | bool leafMatches(Node* leaf,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength) { 275 | // Check if the key of the leaf is equal to the searched key 276 | if (depth!=keyLength) { 277 | uint8_t leafKey[maxKeyLength]; 278 | loadKey(getLeafValue(leaf),leafKey); 279 | for (unsigned i=depth;iprefixLength>maxPrefixLength) { 290 | for (pos=0;posprefix[pos]) 292 | return pos; 293 | uint8_t minKey[maxKeyLength]; 294 | loadKey(getLeafValue(minimum(node)),minKey); 295 | for (;posprefixLength;pos++) 296 | if (key[depth+pos]!=minKey[depth+pos]) 297 | return pos; 298 | } else { 299 | for (pos=0;posprefixLength;pos++) 300 | if (key[depth+pos]!=node->prefix[pos]) 301 | return pos; 302 | } 303 | return pos; 304 | } 305 | 306 | 307 | Node** lookupLeafPtr(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength) { 308 | // Find the node with a matching key, optimistic version 309 | 310 | bool skippedPrefix=false; // Did we optimistically skip some prefix without checking it? 311 | Node**leafPtr = &node; 312 | while (node!=NULL) { 313 | if (isLeaf(node)) { 314 | break; 315 | // if (depth!=keyLength) { 316 | // // Check leaf 317 | // uint8_t leafKey[maxKeyLength]; 318 | // loadKey(getLeafValue(node),leafKey); 319 | // for (unsigned i=(skippedPrefix?0:depth);iprefixLength) { 326 | if (node->prefixLengthprefixLength;pos++) 328 | if (key[depth+pos]!=node->prefix[pos]) 329 | return NULL; 330 | } else 331 | skippedPrefix=true; 332 | depth+=node->prefixLength; 333 | } 334 | 335 | leafPtr = findChild(node,key[depth]); 336 | node = *leafPtr; 337 | depth++; 338 | } 339 | 340 | return leafPtr; 341 | } 342 | 343 | Node* lookup(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength) { 344 | // Find the node with a matching key, optimistic version 345 | 346 | bool skippedPrefix=false; // Did we optimistically skip some prefix without checking it? 347 | 348 | while (node!=NULL) { 349 | if (isLeaf(node)) { 350 | if (!skippedPrefix&&depth==keyLength) // No check required 351 | return node; 352 | 353 | return node; 354 | } 355 | 356 | if (node->prefixLength) { 357 | if (node->prefixLengthprefixLength;pos++) 359 | if (key[depth+pos]!=node->prefix[pos]) 360 | return NULL; 361 | } else 362 | skippedPrefix=true; 363 | depth+=node->prefixLength; 364 | } 365 | 366 | node=*findChild(node,key[depth]); 367 | depth++; 368 | } 369 | 370 | return NULL; 371 | } 372 | 373 | Node * findNextChild(Node* node, uint8_t keyByte) { 374 | 375 | return NULL; 376 | } 377 | 378 | thread_local std::vector ancestors; 379 | // return the first leaf whose key is no less than the search key 380 | Node* lowerBound(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength, bool & pref) { 381 | // Find the node with a matching key, optimistic version 382 | ancestors.clear(); 383 | while (node != NULL) { 384 | ancestors.push_back(node); 385 | if (isLeaf(node)) { 386 | return node; 387 | } 388 | 389 | int nodePrefixLen = node->prefixLength; 390 | if (node->prefixLength) { 391 | unsigned pos=0; 392 | if (node->prefixLength <= maxPrefixLength) { 393 | for (;pos < node->prefixLength && key[depth+pos] == node->prefix[pos]; pos++); 394 | if (pos < node->prefixLength) { 395 | if (node->prefix[pos] < key[depth + pos]) { 396 | goto backtrack; 397 | } else { // node->prefix[pos] > key[depth + pos] 398 | return minimum(node); 399 | } 400 | } 401 | } else { 402 | for (;pos < maxPrefixLength && key[depth+pos] == node->prefix[pos]; pos++); 403 | if (pos >= maxPrefixLength) { 404 | uint8_t leafKey[maxKeyLength]; 405 | auto minLeaf = minimum(node); 406 | loadLowerBoundKey(getLeafValue(minLeaf), leafKey); 407 | for (;pos < node->prefixLength && key[depth+pos] == leafKey[depth + pos]; ++pos); 408 | if (pos < node->prefixLength) { 409 | if (leafKey[depth + pos] < key[depth + pos]) { 410 | goto backtrack; 411 | } else { // leafKey[depth + pos] > key[depth + pos] 412 | pref = true; 413 | return minLeaf; 414 | } 415 | } 416 | } 417 | } 418 | depth += node->prefixLength; 419 | } 420 | 421 | bool equal = false; 422 | node = findChildLowerBound(node, key[depth], equal); 423 | if (node == nullptr) { 424 | depth -= nodePrefixLen; 425 | goto backtrack; 426 | } 427 | if (equal == false) { 428 | return minimum(node); 429 | } 430 | depth++; 431 | } 432 | 433 | backtrack: 434 | if (ancestors.empty() == false) { 435 | ancestors.pop_back(); 436 | } 437 | while (ancestors.empty() == false) { 438 | auto node = ancestors.back(); 439 | --depth; 440 | if (key[depth] < 255) { 441 | bool equal = false; 442 | auto n = findChildLowerBound(node, key[depth] + 1 , equal); 443 | if (n) { 444 | return minimum(n); 445 | } 446 | } 447 | depth -= node->prefixLength; 448 | ancestors.pop_back(); 449 | } 450 | assert(depth == 0); 451 | return NULL; 452 | } 453 | 454 | Node* lookupPessimistic(Node* node,uint8_t key[],unsigned keyLength,unsigned depth,unsigned maxKeyLength) { 455 | // Find the node with a matching key, alternative pessimistic version 456 | 457 | while (node!=NULL) { 458 | if (isLeaf(node)) { 459 | if (leafMatches(node,key,keyLength,depth,maxKeyLength)) 460 | return node; 461 | return NULL; 462 | } 463 | 464 | if (prefixMismatch(node,key,depth,maxKeyLength)!=node->prefixLength) 465 | return NULL; else 466 | depth+=node->prefixLength; 467 | 468 | node=*findChild(node,key[depth]); 469 | depth++; 470 | } 471 | 472 | return NULL; 473 | } 474 | 475 | // Forward references 476 | void insertNode4(Node4* node,Node** nodeRef,uint8_t keyByte,Node* child); 477 | void insertNode16(Node16* node,Node** nodeRef,uint8_t keyByte,Node* child); 478 | void insertNode48(Node48* node,Node** nodeRef,uint8_t keyByte,Node* child); 479 | void insertNode256(Node256* node,Node** nodeRef,uint8_t keyByte,Node* child); 480 | 481 | unsigned min(unsigned a,unsigned b) { 482 | // Helper function 483 | return (aprefixLength=src->prefixLength; 489 | memcpy(dst->prefix,src->prefix,min(src->prefixLength,maxPrefixLength)); 490 | } 491 | 492 | std::function upsertFunc; 493 | 494 | void upsert(Node* node,Node** nodeRef,uint8_t key[],unsigned depth,uintptr_t value,unsigned maxKeyLength, std::function insertMakeLeaf) { 495 | // Insert the leaf value into the tree 496 | 497 | if (node==NULL) { 498 | *nodeRef=insertMakeLeaf(value); 499 | return; 500 | } 501 | 502 | if (isLeaf(node)) { 503 | // Replace leaf with Node4 and store both leaves in it 504 | uint8_t existingKey[maxKeyLength]; 505 | loadKey(getLeafValue(node),existingKey); 506 | unsigned newPrefixLength=0; 507 | while (depth+newPrefixLength < maxKeyLength && existingKey[depth+newPrefixLength]==key[depth+newPrefixLength]) 508 | newPrefixLength++; 509 | if (depth+newPrefixLength == maxKeyLength) { 510 | upsertFunc(nodeRef, value); 511 | return; 512 | } 513 | Node4* newNode=new Node4(); 514 | newNode->prefixLength=newPrefixLength; 515 | memcpy(newNode->prefix,key+depth,min(newPrefixLength,maxPrefixLength)); 516 | *nodeRef=newNode; 517 | 518 | insertNode4(newNode,nodeRef,existingKey[depth+newPrefixLength],node); 519 | insertNode4(newNode,nodeRef,key[depth+newPrefixLength], insertMakeLeaf(value)); 520 | return; 521 | } 522 | 523 | // Handle prefix of inner node 524 | if (node->prefixLength) { 525 | unsigned mismatchPos=prefixMismatch(node,key,depth,maxKeyLength); 526 | if (mismatchPos!=node->prefixLength) { 527 | // Prefix differs, create new node 528 | Node4* newNode=new Node4(); 529 | *nodeRef=newNode; 530 | newNode->prefixLength=mismatchPos; 531 | memcpy(newNode->prefix,node->prefix,min(mismatchPos,maxPrefixLength)); 532 | // Break up prefix 533 | if (node->prefixLengthprefix[mismatchPos],node); 535 | node->prefixLength-=(mismatchPos+1); 536 | memmove(node->prefix,node->prefix+mismatchPos+1,min(node->prefixLength,maxPrefixLength)); 537 | } else { 538 | node->prefixLength-=(mismatchPos+1); 539 | uint8_t minKey[maxKeyLength]; 540 | loadKey(getLeafValue(minimum(node)),minKey); 541 | insertNode4(newNode,nodeRef,minKey[depth+mismatchPos],node); 542 | memmove(node->prefix,minKey+depth+mismatchPos+1,min(node->prefixLength,maxPrefixLength)); 543 | } 544 | insertNode4(newNode,nodeRef,key[depth+mismatchPos],insertMakeLeaf(value)); 545 | return; 546 | } 547 | depth+=node->prefixLength; 548 | } 549 | 550 | // Recurse 551 | Node** child=findChild(node,key[depth]); 552 | if (*child) { 553 | upsert(*child,child,key,depth+1,value,maxKeyLength,insertMakeLeaf); 554 | return; 555 | } 556 | 557 | // Insert leaf into inner node 558 | Node* newNode=insertMakeLeaf(value); 559 | switch (node->type) { 560 | case NodeType4: insertNode4(static_cast(node),nodeRef,key[depth],newNode); break; 561 | case NodeType16: insertNode16(static_cast(node),nodeRef,key[depth],newNode); break; 562 | case NodeType48: insertNode48(static_cast(node),nodeRef,key[depth],newNode); break; 563 | case NodeType256: insertNode256(static_cast(node),nodeRef,key[depth],newNode); break; 564 | } 565 | } 566 | void insert(Node* node,Node** nodeRef,uint8_t key[],unsigned depth,uintptr_t value,unsigned maxKeyLength) { 567 | // Insert the leaf value into the tree 568 | 569 | if (node==NULL) { 570 | *nodeRef=makeLeaf(value); 571 | return; 572 | } 573 | 574 | if (isLeaf(node)) { 575 | // Replace leaf with Node4 and store both leaves in it 576 | uint8_t existingKey[maxKeyLength]; 577 | loadKey(getLeafValue(node),existingKey); 578 | unsigned newPrefixLength=0; 579 | while (existingKey[depth+newPrefixLength]==key[depth+newPrefixLength]) 580 | newPrefixLength++; 581 | 582 | Node4* newNode=new Node4(); 583 | newNode->prefixLength=newPrefixLength; 584 | memcpy(newNode->prefix,key+depth,min(newPrefixLength,maxPrefixLength)); 585 | *nodeRef=newNode; 586 | 587 | insertNode4(newNode,nodeRef,existingKey[depth+newPrefixLength],node); 588 | insertNode4(newNode,nodeRef,key[depth+newPrefixLength],makeLeaf(value)); 589 | return; 590 | } 591 | 592 | // Handle prefix of inner node 593 | if (node->prefixLength) { 594 | unsigned mismatchPos=prefixMismatch(node,key,depth,maxKeyLength); 595 | if (mismatchPos!=node->prefixLength) { 596 | // Prefix differs, create new node 597 | Node4* newNode=new Node4(); 598 | *nodeRef=newNode; 599 | newNode->prefixLength=mismatchPos; 600 | memcpy(newNode->prefix,node->prefix,min(mismatchPos,maxPrefixLength)); 601 | // Break up prefix 602 | if (node->prefixLengthprefix[mismatchPos],node); 604 | node->prefixLength-=(mismatchPos+1); 605 | memmove(node->prefix,node->prefix+mismatchPos+1,min(node->prefixLength,maxPrefixLength)); 606 | } else { 607 | node->prefixLength-=(mismatchPos+1); 608 | uint8_t minKey[maxKeyLength]; 609 | loadKey(getLeafValue(minimum(node)),minKey); 610 | insertNode4(newNode,nodeRef,minKey[depth+mismatchPos],node); 611 | memmove(node->prefix,minKey+depth+mismatchPos+1,min(node->prefixLength,maxPrefixLength)); 612 | } 613 | insertNode4(newNode,nodeRef,key[depth+mismatchPos],makeLeaf(value)); 614 | return; 615 | } 616 | depth+=node->prefixLength; 617 | } 618 | 619 | // Recurse 620 | Node** child=findChild(node,key[depth]); 621 | if (*child) { 622 | insert(*child,child,key,depth+1,value,maxKeyLength); 623 | return; 624 | } 625 | 626 | // Insert leaf into inner node 627 | Node* newNode=makeLeaf(value); 628 | switch (node->type) { 629 | case NodeType4: insertNode4(static_cast(node),nodeRef,key[depth],newNode); break; 630 | case NodeType16: insertNode16(static_cast(node),nodeRef,key[depth],newNode); break; 631 | case NodeType48: insertNode48(static_cast(node),nodeRef,key[depth],newNode); break; 632 | case NodeType256: insertNode256(static_cast(node),nodeRef,key[depth],newNode); break; 633 | } 634 | } 635 | 636 | void insertNode4(Node4* node,Node** nodeRef,uint8_t keyByte,Node* child) { 637 | // Insert leaf into inner node 638 | if (node->count<4) { 639 | // Insert element 640 | unsigned pos; 641 | for (pos=0;(poscount)&&(node->key[pos]key+pos+1,node->key+pos,node->count-pos); 643 | memmove(node->child+pos+1,node->child+pos,(node->count-pos)*sizeof(uintptr_t)); 644 | node->key[pos]=keyByte; 645 | node->child[pos]=child; 646 | node->count++; 647 | } else { 648 | // Grow to Node16 649 | Node16* newNode=new Node16(); 650 | *nodeRef=newNode; 651 | newNode->count=4; 652 | copyPrefix(node,newNode); 653 | for (unsigned i=0;i<4;i++) 654 | newNode->key[i]=flipSign(node->key[i]); 655 | memcpy(newNode->child,node->child,node->count*sizeof(uintptr_t)); 656 | delete node; 657 | return insertNode16(newNode,nodeRef,keyByte,child); 658 | } 659 | } 660 | 661 | void insertNode16(Node16* node,Node** nodeRef,uint8_t keyByte,Node* child) { 662 | // Insert leaf into inner node 663 | if (node->count<16) { 664 | // Insert element 665 | uint8_t keyByteFlipped=flipSign(keyByte); 666 | __m128i cmp=_mm_cmplt_epi8(_mm_set1_epi8(keyByteFlipped),_mm_loadu_si128(reinterpret_cast<__m128i*>(node->key))); 667 | uint16_t bitfield=_mm_movemask_epi8(cmp)&(0xFFFF>>(16-node->count)); 668 | unsigned pos=bitfield?ctz(bitfield):node->count; 669 | memmove(node->key+pos+1,node->key+pos,node->count-pos); 670 | memmove(node->child+pos+1,node->child+pos,(node->count-pos)*sizeof(uintptr_t)); 671 | node->key[pos]=keyByteFlipped; 672 | node->child[pos]=child; 673 | node->count++; 674 | } else { 675 | // Grow to Node48 676 | Node48* newNode=new Node48(); 677 | *nodeRef=newNode; 678 | memcpy(newNode->child,node->child,node->count*sizeof(uintptr_t)); 679 | for (unsigned i=0;icount;i++) 680 | newNode->childIndex[flipSign(node->key[i])]=i; 681 | copyPrefix(node,newNode); 682 | newNode->count=node->count; 683 | delete node; 684 | return insertNode48(newNode,nodeRef,keyByte,child); 685 | } 686 | } 687 | 688 | void insertNode48(Node48* node,Node** nodeRef,uint8_t keyByte,Node* child) { 689 | // Insert leaf into inner node 690 | if (node->count<48) { 691 | // Insert element 692 | unsigned pos=node->count; 693 | if (node->child[pos]) 694 | for (pos=0;node->child[pos]!=NULL;pos++); 695 | node->child[pos]=child; 696 | node->childIndex[keyByte]=pos; 697 | node->count++; 698 | } else { 699 | // Grow to Node256 700 | Node256* newNode=new Node256(); 701 | for (unsigned i=0;i<256;i++) 702 | if (node->childIndex[i]!=48) 703 | newNode->child[i]=node->child[node->childIndex[i]]; 704 | newNode->count=node->count; 705 | copyPrefix(node,newNode); 706 | *nodeRef=newNode; 707 | delete node; 708 | return insertNode256(newNode,nodeRef,keyByte,child); 709 | } 710 | } 711 | 712 | void insertNode256(Node256* node,Node** nodeRef,uint8_t keyByte,Node* child) { 713 | // Insert leaf into inner node 714 | node->count++; 715 | node->child[keyByte]=child; 716 | } 717 | 718 | 719 | void destroyNode4(Node4* node); 720 | void destroyNode16(Node16* node); 721 | void destroyNode48(Node48* node); 722 | void destroyNode256(Node256* node); 723 | void destroyNode(Node* node) { 724 | if (node == nullptr || isLeaf(node)) return; 725 | switch (node->type) { 726 | case NodeType4: destroyNode4(static_cast(node)); break; 727 | case NodeType16: destroyNode16(static_cast(node)); break; 728 | case NodeType48: destroyNode48(static_cast(node)); break; 729 | case NodeType256: destroyNode256(static_cast(node)); break; 730 | } 731 | } 732 | 733 | void destroyNode4(Node4* node) { 734 | for (int i = 0; i < node->count; ++i) { 735 | destroyNode(node->child[i]); 736 | } 737 | delete node; 738 | } 739 | 740 | void destroyNode16(Node16* node) { 741 | for (int i = 0; i < node->count; ++i) { 742 | destroyNode(node->child[i]); 743 | } 744 | delete node; 745 | } 746 | 747 | void destroyNode48(Node48* node) { 748 | for (int i = 0; i < 256; ++i) { 749 | if (node->childIndex[i] != emptyMarker) 750 | destroyNode(node->child[node->childIndex[i]]); 751 | } 752 | delete node; 753 | } 754 | 755 | void destroyNode256(Node256* node) { 756 | for (int i = 0; i < 256; ++i) { 757 | if (node->child[i] != nullptr) 758 | destroyNode(node->child[i]); 759 | } 760 | delete node; 761 | } 762 | 763 | template 766 | Node* bulkLoad(std::vector> & kvs, int lo, int hi, int depth, KeyByteExtractFunc ExtractKeyByte, KeyLenFunc KeyLen) { 767 | if (hi - lo + 1 == 1) return makeLeaf(kvs[lo].second); 768 | int childCnt; 769 | int singlePart; 770 | std::vector> parts(256, std::make_pair(-1, -1)); 771 | auto partition = [&](){ 772 | parts = std::vector>(256, std::make_pair(-1, -1)); 773 | childCnt = 0; 774 | singlePart = -1; 775 | for (int i = lo; i <= hi; ++i) { 776 | auto & kv = kvs[i]; 777 | uint8_t byte = ExtractKeyByte(kv.first, depth); 778 | if (parts[byte].first == -1) { 779 | parts[byte].first = parts[byte].second = i; 780 | childCnt++; 781 | singlePart = byte; 782 | } else { 783 | parts[byte].second = i; 784 | } 785 | } 786 | }; 787 | 788 | partition(); 789 | 790 | Node nmeta(0); 791 | 792 | if (childCnt == 1) { 793 | // only one partition, compute the longest common prefix 794 | int si = parts[singlePart].first, ei = parts[singlePart].second; 795 | bool diff = false; 796 | int pos = 0; 797 | while (diff == false) { 798 | uint8_t byte = ExtractKeyByte(kvs[si].first, depth + pos); 799 | for (int i = si + 1; i <= ei && diff == false; ++i) { 800 | if (depth + pos >= KeyLen(kvs[i].first) || ExtractKeyByte(kvs[i].first, depth + pos) != byte) { 801 | diff = true; 802 | } 803 | } 804 | if (diff == false) { 805 | if (pos < maxPrefixLength) { 806 | nmeta.prefix[pos] = byte; 807 | } 808 | ++pos; 809 | ++nmeta.prefixLength; 810 | } 811 | } 812 | if (nmeta.prefixLength > 0) { 813 | depth += nmeta.prefixLength; 814 | // repartition 815 | partition(); 816 | } 817 | } 818 | Node* n = nullptr; 819 | if (childCnt <= 4) { 820 | n = new Node4; 821 | } else if (childCnt <= 16) { 822 | n = new Node16; 823 | } else if (childCnt <= 48) { 824 | n = new Node48; 825 | } else { 826 | n = new Node256; 827 | } 828 | n->prefixLength = nmeta.prefixLength; 829 | if (n->prefixLength) { 830 | memcpy(n->prefix, nmeta.prefix, n->prefixLength); 831 | } 832 | 833 | for (int i = 0; i < 256; ++i) { 834 | uint8_t byte = i; 835 | if (parts[byte].first == -1) continue; 836 | int partitionSize = parts[byte].second - parts[byte].first + 1; 837 | Node* child = bulkLoad(kvs, parts[byte].first, parts[byte].second, depth + 1, ExtractKeyByte, KeyLen); 838 | switch(n->type) { 839 | case NodeType4: { 840 | Node4* n4 = (Node4*)n; 841 | n4->key[n4->count] = byte; 842 | n4->child[n4->count] = child; 843 | ++n4->count; 844 | } 845 | break; 846 | case NodeType16: { 847 | Node16* n16 = (Node16*)n; 848 | n16->key[n16->count] = flipSign(byte); 849 | n16->child[n16->count] = child; 850 | ++n16->count; 851 | } 852 | break; 853 | case NodeType48: { 854 | Node48* n48 = (Node48*)n; 855 | int pos = n48->count; 856 | n48->child[pos]=child; 857 | n48->childIndex[byte]=pos; 858 | ++n48->count; 859 | } 860 | break; 861 | case NodeType256: { 862 | Node256* n256 = (Node256*)n; 863 | n256->child[byte] = child; 864 | ++n256->count; 865 | } 866 | break; 867 | } 868 | } 869 | return n; 870 | } 871 | 872 | } --------------------------------------------------------------------------------