├── CMakeLists.txt ├── LICENSE ├── README.md ├── example ├── CMakeLists.txt └── example.cpp ├── include ├── arch.h ├── common.h ├── numa-config.h ├── ordo_clock.h ├── pactree.h ├── pmem.h └── pptr.h ├── lib ├── CMakeLists.txt └── PDL-ART │ ├── CMakeLists.txt │ ├── Epoche.cpp │ ├── Epoche.h │ ├── Key.h │ ├── LICENSE │ ├── N.cpp │ ├── N.h │ ├── N16.cpp │ ├── N256.cpp │ ├── N4.cpp │ ├── N48.cpp │ ├── Tree.cpp │ └── Tree.h ├── src ├── CMakeLists.txt ├── Combiner.h ├── Oplog.cpp ├── Oplog.h ├── SearchLayer.h ├── VersionedLock.h ├── WorkerThread.cpp ├── WorkerThread.h ├── bitset.h ├── linkedList.cpp ├── linkedList.h ├── listNode.cpp ├── listNode.h ├── main.cpp ├── pactree.cpp ├── pactreeImpl.h └── threadData.h └── tools ├── cpu-topology.py └── get-numa-config.sh /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | 4 | set(CMAKE_BUILD_TYPE Release) 5 | 6 | project(pactree) 7 | 8 | project(pactree) 9 | 10 | execute_process(COMMAND cat /proc/cpuinfo COMMAND grep clwb OUTPUT_VARIABLE CLWB) 11 | if(CLWB) 12 | message(STATUS "Use clwb") 13 | else() 14 | add_definitions(-DNO_CLWB) 15 | message(STATUS "Use clflush instaed of clwb") 16 | endif() 17 | 18 | set(CMAKE_CXX_STANDARD 14) 19 | set(CMAKE_CXX_FLAGS "-pthread -Wall -Wextra -march=native -mavx512bw") 20 | set(CMAKE_CXX_FLAGS_DEBUG "-g") 21 | set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g") 22 | 23 | include_directories(${CMAKE_SOURCE_DIR}/include) 24 | enable_testing() 25 | 26 | add_subdirectory(lib/PDL-ART) 27 | add_subdirectory(src) 28 | add_subdirectory(example) 29 | #add_subdirectory(tests) 30 | #add_subdirectory(benchmarks) 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019-2021 Virginia Tech 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PACTree 2 | 3 | This repository contains the artifact for the SOSP'21 paper: 4 | 5 | *"PACTree: A High Performance Persistent Range Index Using PAC Guidelines" 6 | Wook-Hee Kim, R. Madhava Krishnan, Xinwei Fu, Sanidhya Kashyap, and Changwoo Min 7 | In Proceedings of the 28th ACM Symposium on Operating Systems Principles (ACM SOSP 2021)* 8 | 9 | 10 | ## Directory structure 11 | ```{.sh} 12 | pactree 13 | ├── include # public headers of pactree 14 | ├── lib # PDL-ART 15 | ├── src # pactree source code 16 | ├── tools # misc build tools 17 | ``` 18 | ## System configuration 19 | - Fedora 29 or higher // Ubuntu 18.04 or higher 20 | - x86-64 CPU supporting AVX512 instructions 21 | ``` 22 | # You can check using the following command: 23 | $ cat /proc/cpuinfo | egrep -ho 'avx[^ ]*' | sort -u 24 | ``` 25 | - gcc 7.0 or higher 26 | - at least 30 GB for each partition of NVM or SSD 27 | 28 | 29 | ## Dependency 30 | ### Install packages 31 | - Install latest version of CMake 32 | ``` 33 | 1. Download latest version from https://cmake.org/download/ 34 | $ tar -xvf cmake-your_version.tar.gz 35 | $ cd cmake-your-version 36 | $ ./bootstrap 37 | $ make 38 | $ sudo make install 39 | ``` 40 | - Install other packages 41 | ``` 42 | #Ubuntu 43 | $ sudo apt-get install g++ libtbb-dev libjemalloc-dev libnuma-dev libpmem-dev libpmemobj-dev python zlib1g-dev libboost-dev 44 | 45 | #Felodra 46 | $ sudo yum install cmake make gcc-c++ tbb-devel jemalloc-devel numactl-devel libpmem-devel libpmemobj-devel python3 zlib-devel boost-devel 47 | ``` 48 | 49 | ## Environment setting for actual NVM 50 | ### Mount dax file systems 51 | ``` 52 | $ sudo mkfs.ext4 -b 4096 -E stride=512 -F /dev/pmem0 53 | $ sudo mount -o dax /dev/pmem0 /mnt/pmem0 54 | 55 | $ sudo mkfs.ext4 -b 4096 -E stride=512 -F /dev/pmem1 56 | $ sudo mount -o dax /dev/pmem0 /mnt/pmem1 57 | ``` 58 | 59 | ## Environment setting for non-NVM 60 | You can use SSD instead of acutal NVM. 61 | ``` 62 | # vi ~/.bashrc and add following: 63 | export PMEM_IS_PMEM_FORCE=1 64 | # This is for eliminating msync overhead. 65 | ``` 66 | ``` 67 | source ~/.bashrc 68 | ``` 69 | ``` 70 | $ sudo mkdir /mnt/pmem0 71 | $ sudo chmod 777 /mnt/pmem0 72 | 73 | $ sudo mkdir /mnt/pmem1 74 | $ sudo chmod 777 /mnt/pmem1 75 | ``` 76 | 77 | ## How to compile 78 | ``` 79 | $ bash ./tools/get-numa-config.sh 80 | $ mkdir build 81 | $ cd build 82 | $ cmake .. 83 | $ make 84 | ``` 85 | You can find PDL-ART library(libpdlart.a) at build/lib/PDL-ART. 86 | You can find PACTree library(libpactree.a) at build/src/ . 87 | 88 | ## PACTree Configuration 89 | ``` 90 | $ vi include/common.h 91 | #define STRINGKEY // Set String key type 92 | #define KEYLENGTH 32 // Set String key length (current : 32) 93 | #define MULTIPOOL // Use multiple NUMA Persistent Memory Pool 94 | ``` 95 | ## Example 96 | If you want to use PACTree for your project, please follow below: 97 | ``` 98 | 1. Make your project directory 99 | 2. Add your project directory to the CMakeLists.txt 100 | 3. Write CMakeLists.txt for your project to your project directory 101 | ``` 102 | Please refer ./CMakeLists.txt, example/example.cpp, example/CMakeLists.txt. 103 | You can find the example at build/example/. 104 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SRCS *.cpp) 2 | MESSAGE($SRCS) 3 | ADD_EXECUTABLE(pactree-example example.cpp) 4 | 5 | target_include_directories(pactree PUBLIC 6 | ${CMAKE_CURRENT_SOURCE_DIR} 7 | ) 8 | TARGET_LINK_LIBRARIES( 9 | pactree-example 10 | pactree 11 | pdlart 12 | tbb 13 | numa 14 | pmem 15 | pmemobj 16 | ) 17 | 18 | 19 | -------------------------------------------------------------------------------- /example/example.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "pactree.h" 5 | #include 6 | 7 | #define NUMDATA 100 8 | 9 | int main(int argc, char **argv){ 10 | pactree *pt = new pactree(1); 11 | pt->registerThread(); 12 | 13 | for(int i = 1; i < NUMDATA; i++) { 14 | pt->insert(i,i); 15 | } 16 | 17 | for(int i = 1; i < NUMDATA; i++) { 18 | if(i!=pt->lookup(i)){ 19 | printf("error"); 20 | exit(1); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /include/arch.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _RLU_ARCH_H 5 | #define _RLU_ARCH_H 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | #ifndef __KERNEL__ 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #ifndef likely 21 | #define likely(x) __builtin_expect((unsigned long)(x), 1) 22 | #endif 23 | #ifndef unlikely 24 | #define unlikely(x) __builtin_expect((unsigned long)(x), 0) 25 | #endif 26 | #else /* __KERNEL__ */ 27 | #include 28 | #include 29 | #include 30 | #include 31 | #endif /* __KERNEL__ */ 32 | 33 | #ifndef __read_mostly 34 | #define __read_mostly __attribute__((__section__(".data..read_mostly"))) 35 | #endif 36 | 37 | /* 38 | * machine-, architecture-specific information 39 | */ 40 | #define ____ptr_aligned __attribute__((aligned(sizeof(void *)))) 41 | 42 | #ifndef __KERNEL__ 43 | #define PAGE_SIZE 4096 44 | #define L1_CACHE_BYTES 64 45 | #define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES))) 46 | #endif /* __KERNEL__ */ 47 | 48 | #define CACHE_LINE_PREFETCH_UNIT (2) 49 | #define CACHE_DEFAULT_PADDING \ 50 | ((CACHE_LINE_PREFETCH_UNIT * L1_CACHE_BYTES) / sizeof(long)) 51 | 52 | #define ____cacheline_aligned2 \ 53 | __attribute__((aligned(CACHE_LINE_PREFETCH_UNIT * L1_CACHE_BYTES))) 54 | 55 | #define ____page_aligned __attribute__((aligned(PAGE_SIZE))) 56 | 57 | #ifndef __packed 58 | #define __packed __attribute__((packed)) 59 | #endif 60 | 61 | /*#ifndef static_assert 62 | #define static_assert(e) (sizeof(struct { int : (-!(e)); })) 63 | #define static_assert_msg(e, msg) static_assert(e) 64 | #endif*/ 65 | 66 | #ifndef __KERNEL__ 67 | static inline void __attribute__((__always_inline__)) smp_mb(void) 68 | { 69 | __asm__ __volatile__("mfence" ::: "memory"); 70 | } 71 | 72 | static inline void __attribute__((__always_inline__)) smp_rmb(void) 73 | { 74 | __asm__ __volatile__("lfence" ::: "memory"); 75 | } 76 | 77 | static inline void __attribute__((__always_inline__)) smp_wmb(void) 78 | { 79 | __asm__ __volatile__("sfence" ::: "memory"); 80 | } 81 | 82 | static inline void __attribute__((__always_inline__)) barrier(void) 83 | { 84 | __asm__ __volatile__("" ::: "memory"); 85 | } 86 | #endif 87 | 88 | static inline void __attribute__((__always_inline__)) smp_wmb_tso(void) 89 | { 90 | barrier(); 91 | } 92 | 93 | #define smp_cas(__ptr, __old_val, __new_val) \ 94 | __sync_bool_compare_and_swap(__ptr, __old_val, __new_val) 95 | 96 | #define smp_cas_v(__ptr, __old_val, __new_val, __fetched_val) \ 97 | ({ \ 98 | (__fetched_val) = __sync_val_compare_and_swap( \ 99 | __ptr, __old_val, __new_val); \ 100 | (__fetched_val) == (__old_val); \ 101 | }) 102 | 103 | #define smp_cas16b(__ptr, __old_val1, __old_val2, __new_val1, __new_val2) \ 104 | ({ \ 105 | char result; \ 106 | __asm__ __volatile__("lock; cmpxchg16b %0; setz %1" \ 107 | : "=m"(*(__ptr)), "=a"(result) \ 108 | : "m"(*(__ptr)), "d"(__old_val2), \ 109 | "a"(__old_val1), "c"(__new_val2), \ 110 | "b"(__new_val1) \ 111 | : "memory"); \ 112 | (int)result; \ 113 | }) 114 | 115 | #define smp_swap(__ptr, __val) __sync_lock_test_and_set(__ptr, __val) 116 | 117 | #define smp_atomic_load(__ptr) \ 118 | ({ __sync_val_compare_and_swap(__ptr, __ptr, __ptr); }) 119 | 120 | #define smp_atomic_store(__ptr, __val) (void)smp_swap(__ptr, __val) 121 | 122 | #define smp_faa(__ptr, __val) __sync_fetch_and_add(__ptr, __val) 123 | 124 | #define smp_fas(__ptr, __val) __sync_fetch_and_sub(__ptr, __val) 125 | 126 | #define cpu_relax() asm volatile("pause\n" : : : "memory") 127 | 128 | static inline uint64_t __attribute__((__always_inline__)) read_tsc(void) 129 | { 130 | uint32_t a, d; 131 | __asm __volatile("rdtsc" : "=a"(a), "=d"(d)); 132 | return ((uint64_t)a) | (((uint64_t)d) << 32); 133 | } 134 | 135 | static inline unsigned long read_coreid_rdtscp(int *chip, int *core) { 136 | unsigned long a,d,c; 137 | __asm__ volatile("rdtscp" : "=a" (a), "=d" (d), "=c" (c)); 138 | *chip = (c & 0xFFF000)>>12; 139 | *core = c & 0xFFF; 140 | return ((unsigned long)a) | (((unsigned long)d) << 32);; 141 | } 142 | 143 | 144 | 145 | static inline uint64_t __attribute__((__always_inline__)) read_tscp(void) 146 | { 147 | uint32_t a, d; 148 | __asm __volatile("rdtscp" : "=a"(a), "=d"(d)); 149 | return ((uint64_t)a) | (((uint64_t)d) << 32); 150 | } 151 | 152 | #ifndef __KERNEL__ 153 | static inline void cpuid(int i, unsigned int *a, unsigned int *b, 154 | unsigned int *c, unsigned int *d) 155 | { 156 | /* https://bit.ly/2uGziVO */ 157 | __asm __volatile("cpuid" 158 | : "=a"(*a), "=b"(*b), "=c"(*c), "=d"(*d) 159 | : "a"(i), "c"(0)); 160 | } 161 | #endif 162 | 163 | static inline unsigned int max_cpu_freq(void) 164 | { 165 | /* https://bit.ly/2EbkRZp */ 166 | unsigned int regs[4]; 167 | cpuid(0x16, ®s[0], ®s[1], ®s[2], ®s[3]); 168 | return regs[1]; 169 | } 170 | 171 | 172 | #define cache_prefetchr_high(__ptr) __builtin_prefetch((void *)__ptr, 0, 3) 173 | 174 | #define cache_prefetchr_mid(__ptr) __builtin_prefetch((void *)__ptr, 0, 2) 175 | 176 | #define cache_prefetchr_low(__ptr) __builtin_prefetch((void *)__ptr, 0, 0) 177 | 178 | #define cache_prefetchw_high(__ptr) __builtin_prefetch((void *)__ptr, 1, 3) 179 | 180 | #define cache_prefetchw_mid(__ptr) __builtin_prefetch((void *)__ptr, 1, 2) 181 | 182 | #define cache_prefetchw_low(__ptr) __builtin_prefetch((void *)__ptr, 1, 0) 183 | #ifdef NO_CLWB 184 | static inline void clwb(volatile void *p) 185 | { 186 | asm volatile("clflush (%0)" ::"r"(p)); 187 | } 188 | #else 189 | static inline void clwb(volatile void *p) 190 | { 191 | asm volatile(".byte 0x66; xsaveopt %0" : "+m" (p)); 192 | } 193 | #endif 194 | 195 | #ifdef __cplusplus 196 | } 197 | #endif 198 | 199 | #endif /* _RLU_ARCH_H */ 200 | -------------------------------------------------------------------------------- /include/common.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef PACTREE_COMMON_H 5 | #define PACTREE_COMMON_H 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #define MULTIPOOL 13 | #define MAX_NUMA 1 14 | //#define STRINGKEY 15 | #define WORKER_THREAD_PER_NUMA 1 16 | #define KEYLENGTH 32 17 | //#define SYNC 18 | 19 | //#define PACTREE_ENABLE_STATS 20 | 21 | 22 | template 23 | class StringKey { 24 | private: 25 | char data[keySize]; 26 | size_t keyLength = 0; 27 | public: 28 | StringKey() { memset(data, 0x00, keySize);} 29 | StringKey(const StringKey &other) { 30 | memcpy(data, other.data, keySize); 31 | } 32 | StringKey(const char bytes[]) {set(bytes, strlen(bytes));} 33 | StringKey(int k) { 34 | setFromString(std::to_string(k)); 35 | } 36 | inline StringKey &operator=(const StringKey &other) { 37 | memcpy(data, other.data, keySize); 38 | // keyLength = other.keyLength; 39 | return *this; 40 | } 41 | inline bool operator<(const StringKey &other) { return strcmp(data, other.data) < 0;} 42 | inline bool operator>(const StringKey &other) { return strcmp(data, other.data) > 0;} 43 | inline bool operator==(const StringKey &other) { return strcmp(data, other.data) == 0;} 44 | inline bool operator!=(const StringKey &other) { return !(*this == other);} 45 | inline bool operator<=(const StringKey &other) { return !(*this > other);} 46 | inline bool operator>=(const StringKey &other) {return !(*this < other);} 47 | 48 | size_t size() const { 49 | if (keyLength) 50 | return keyLength; 51 | else 52 | return strlen(data); 53 | } 54 | 55 | inline void setFromString(std::string key) { 56 | memset(data, 0, keySize); 57 | if (key.size() >= keySize) { 58 | memcpy(data, key.c_str(), keySize - 1); 59 | data[keySize - 1] = '\0'; 60 | keyLength = keySize; 61 | } else { 62 | strcpy(data, key.c_str()); 63 | keyLength = key.size(); 64 | } 65 | return; 66 | } 67 | 68 | inline void set(const char bytes[], const std::size_t length) { 69 | assert(length <= keySize-1); 70 | memcpy(data, bytes, length); 71 | // keyLength = length; 72 | data[length] = '\0'; 73 | } 74 | const char* getData() const { return data;} 75 | //friend ostream & operator << (ostream &out, const StringKey &k); 76 | }; 77 | 78 | #ifdef STRINGKEY 79 | typedef StringKey Key_t; 80 | #else 81 | typedef uint64_t Key_t; 82 | #endif 83 | typedef uint64_t Val_t; 84 | 85 | class OpStruct { 86 | public: 87 | enum Operation {dummy, insert, remove, done}; 88 | Operation op; // 4 89 | uint16_t poolId; //2 90 | uint8_t hash; //1 91 | Key_t key; // 8 92 | void* oldNodePtr; // old node_ptr 8 93 | PMEMoid newNodeOid; // new node_ptr 16 94 | Key_t newKey; // new key ; 8 95 | Val_t newVal;// new value ; 8 96 | uint64_t ts; // 8 97 | bool operator< (const OpStruct& ops) const { 98 | return (ts < ops.ts); 99 | } 100 | }; 101 | 102 | #endif 103 | -------------------------------------------------------------------------------- /include/numa-config.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | enum { 5 | NUM_SOCKET = 2, 6 | NUM_PHYSICAL_CPU_PER_SOCKET = 16, 7 | SMT_LEVEL = 2, 8 | }; 9 | 10 | const int OS_CPU_ID[NUM_SOCKET][NUM_PHYSICAL_CPU_PER_SOCKET][SMT_LEVEL] = { 11 | { /* socket id: 0 */ 12 | { /* physical cpu id: 0 */ 13 | 0, 32, }, 14 | { /* physical cpu id: 1 */ 15 | 1, 33, }, 16 | { /* physical cpu id: 2 */ 17 | 2, 34, }, 18 | { /* physical cpu id: 3 */ 19 | 3, 35, }, 20 | { /* physical cpu id: 4 */ 21 | 4, 36, }, 22 | { /* physical cpu id: 5 */ 23 | 5, 37, }, 24 | { /* physical cpu id: 6 */ 25 | 6, 38, }, 26 | { /* physical cpu id: 7 */ 27 | 7, 39, }, 28 | { /* physical cpu id: 8 */ 29 | 8, 40, }, 30 | { /* physical cpu id: 9 */ 31 | 9, 41, }, 32 | { /* physical cpu id: 10 */ 33 | 10, 42, }, 34 | { /* physical cpu id: 11 */ 35 | 11, 43, }, 36 | { /* physical cpu id: 12 */ 37 | 12, 44, }, 38 | { /* physical cpu id: 13 */ 39 | 13, 45, }, 40 | { /* physical cpu id: 14 */ 41 | 14, 46, }, 42 | { /* physical cpu id: 15 */ 43 | 15, 47, }, 44 | }, 45 | { /* socket id: 1 */ 46 | { /* physical cpu id: 0 */ 47 | 16, 48, }, 48 | { /* physical cpu id: 1 */ 49 | 17, 49, }, 50 | { /* physical cpu id: 2 */ 51 | 18, 50, }, 52 | { /* physical cpu id: 3 */ 53 | 19, 51, }, 54 | { /* physical cpu id: 4 */ 55 | 20, 52, }, 56 | { /* physical cpu id: 5 */ 57 | 21, 53, }, 58 | { /* physical cpu id: 6 */ 59 | 22, 54, }, 60 | { /* physical cpu id: 7 */ 61 | 23, 55, }, 62 | { /* physical cpu id: 8 */ 63 | 24, 56, }, 64 | { /* physical cpu id: 9 */ 65 | 25, 57, }, 66 | { /* physical cpu id: 10 */ 67 | 26, 58, }, 68 | { /* physical cpu id: 11 */ 69 | 27, 59, }, 70 | { /* physical cpu id: 12 */ 71 | 28, 60, }, 72 | { /* physical cpu id: 13 */ 73 | 29, 61, }, 74 | { /* physical cpu id: 14 */ 75 | 30, 62, }, 76 | { /* physical cpu id: 15 */ 77 | 31, 63, }, 78 | }, 79 | }; 80 | -------------------------------------------------------------------------------- /include/ordo_clock.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _ORDO_CLOCK_H 5 | #define _ORDO_CLOCK_H 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | #include "arch.h" 12 | 13 | #define ORDO_LESS_THAN (-1) 14 | #define ORDO_GREATER_THAN (1) 15 | #define ORDO_UNCERTAIN (0) 16 | 17 | /* ORDO boundary of our evaluation set up 18 | * 224-core machine: 1214 clock cycles 19 | * 120-core machine: 650 clock cycles */ 20 | //#define __ORDO_BOUNDARY (1214) 21 | #define __ORDO_BOUNDARY (0) 22 | 23 | #ifdef ORDO_CONFIGURABLE_BOUNDARY 24 | /* Since clock difference is a read-mostly variable that is never 25 | * updated once it is initialized. Thus, to completely prevent 26 | * the false sharing of its cacheline, we put padding around it. */ 27 | static unsigned long _g_ordo_array[2 * CACHE_DEFAULT_PADDING] __read_mostly; 28 | #define g_ordo_boundary (_g_ordo_array[CACHE_DEFAULT_PADDING]) 29 | #else 30 | #define g_ordo_boundary __ORDO_BOUNDARY 31 | #endif /* ORDO_CONFIGURABLE_BOUNDARY */ 32 | 33 | static inline void ordo_clock_init(void) 34 | { 35 | #ifdef ORDO_CONFIGURABLE_BOUNDARY 36 | g_ordo_boundary = __ORDO_BOUNDARY; 37 | #endif 38 | } 39 | 40 | static inline unsigned long ordo_boundary(void) 41 | { 42 | return g_ordo_boundary; 43 | } 44 | 45 | static inline unsigned long ordo_get_clock(void) 46 | { 47 | /* rdtscp() is a serializing variant, which is not 48 | * reordered in an instruction pipeline. */ 49 | return read_tscp(); 50 | } 51 | 52 | static inline unsigned long ordo_get_clock_relaxed(void) 53 | { 54 | /* rdtsc() is not a serializing instruction so 55 | * rdtsc could be reordered in an instruction pipeline. 56 | * If rdtsc() can be barrier-ed with other instructions, 57 | * such as memory fences, it is okay to use rdtsc() 58 | * without worrying such reordering. */ 59 | return read_tsc(); 60 | } 61 | 62 | static inline int ordo_lt_clock(unsigned long t1, unsigned long t2) 63 | { 64 | return (t1 + g_ordo_boundary) < t2; 65 | } 66 | 67 | static inline int ordo_gt_clock(unsigned long t1, unsigned long t2) 68 | { 69 | return t1 > (t2 + g_ordo_boundary); 70 | } 71 | 72 | static inline int ordo_cmp_clock(unsigned long t1, unsigned long t2) 73 | { 74 | if (ordo_lt_clock(t1, t2)) 75 | return ORDO_LESS_THAN; 76 | if (ordo_gt_clock(t1, t2)) 77 | return ORDO_GREATER_THAN; 78 | return ORDO_UNCERTAIN; 79 | } 80 | 81 | static inline unsigned long ordo_new_clock(unsigned long t) 82 | { 83 | unsigned long new_clk; 84 | 85 | while (!ordo_gt_clock(new_clk = ordo_get_clock(), t)) { 86 | smp_mb(); 87 | } 88 | return new_clk; 89 | } 90 | 91 | #ifdef __cplusplus 92 | } 93 | #endif 94 | 95 | #endif /* _ORDO_CLOCK_H */ 96 | -------------------------------------------------------------------------------- /include/pactree.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef pactreeAPI_H 5 | #define pactreeAPI_H 6 | #include "pactreeImpl.h" 7 | #include "common.h" 8 | class pactree{ 9 | private: 10 | pactreeImpl *pt; 11 | public: 12 | pactree(int numa) { 13 | pt = initPT(numa); 14 | //hl = new HydraListImpl(numa); 15 | } 16 | ~pactree() { 17 | //delete hl; 18 | } 19 | bool insert(Key_t key, Val_t val) { 20 | return pt->insert(key, val); 21 | } 22 | bool update(Key_t key, Val_t val) { 23 | return pt->update(key, val); 24 | } 25 | Val_t lookup(Key_t key) { 26 | return pt->lookup(key); 27 | } 28 | Val_t remove(Key_t key) { 29 | return pt->remove(key); 30 | } 31 | uint64_t scan(Key_t startKey, int range, std::vector &result) { 32 | return pt->scan(startKey, range, result); 33 | } 34 | void registerThread() { 35 | pt->registerThread(); 36 | } 37 | void unregisterThread() { 38 | pt->unregisterThread(); 39 | } 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /include/pmem.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include 5 | #include 6 | #include 7 | #include "arch.h" 8 | 9 | #define MASK 0x8000FFFFFFFFFFFF 10 | #define MASK_DIRTY 0xDFFFFFFFFFFFFFFF //DIRTY_BIT 11 | #define MASK_POOL 0x7FFFFFFFFFFFFFFF 12 | 13 | //#define MEM_AMOUNT 14 | typedef struct root_obj{ 15 | PMEMoid ptr[2]; 16 | // PMEMoid ptr2; 17 | }root_obj; 18 | void printMemAmount(); 19 | void addMemAmount(unsigned long amt); 20 | void zeroMemAmount(); 21 | 22 | class PMem { 23 | private: 24 | static void *baseAddresses[6]; // dram 25 | static void* logVaddr[2]; 26 | // static PMEMoid logSpace[4]; 27 | public: 28 | static void *getBaseOf(int poolId) { 29 | return baseAddresses[poolId]; 30 | } 31 | static bool alloc(int poolId, size_t size, void **p) { 32 | // allocate a size memory from pool_id 33 | // and store/persist address to *p 34 | // return true on succeed 35 | PMEMobjpool *pop = (PMEMobjpool *)baseAddresses[poolId]; 36 | PMEMoid oid; 37 | int ret = pmemobj_alloc(pop, &oid, size, 0, NULL, NULL); 38 | if(ret){ 39 | std::cerr<<"alloc error"< (((unsigned long)poolId) << 48 | oid.off); 44 | return true; 45 | } 46 | 47 | static bool alloc(int poolId, size_t size, void **p, PMEMoid *oid) { 48 | // allocate a size memory from pool_id 49 | // and store/persist address to *p 50 | // return true on succeed 51 | PMEMobjpool *pop = (PMEMobjpool *)baseAddresses[poolId]; 52 | int ret = pmemobj_alloc(pop, oid, size, 0, NULL, NULL); 53 | if(ret){ 54 | std::cerr<<"alloc error"< (((unsigned long)poolId) << 48 | oid->off); 59 | return true; 60 | } 61 | static void free(void *pptr) { 62 | // p -> pool_id and offset 63 | // then perform free 64 | int poolId = (((unsigned long)pptr)&MASK_POOL) >> 48; 65 | void *rawPtr = (void *)(((unsigned long)pptr)& MASK + (unsigned long)baseAddresses[poolId]); 66 | PMEMoid ptr = pmemobj_oid(rawPtr); 67 | pmemobj_free(&ptr); 68 | } 69 | 70 | static void freeVaddr(void *vaddr) { 71 | // p -> pool_id and offset 72 | // then perform free 73 | PMEMoid ptr = pmemobj_oid(vaddr); 74 | pmemobj_free(&ptr); 75 | } 76 | 77 | // case 1 78 | static bool bind(int poolId, const char *nvm_path, size_t size, void **root_p, int *is_created) { 79 | // open nvm_path with PMDK api and associate 80 | PMEMobjpool *pop; 81 | const char* layout = "phydra"; 82 | if (access(nvm_path, F_OK) != 0) { 83 | pop = pmemobj_create(nvm_path, layout, size, 0666); 84 | if(pop == nullptr){ 85 | std::cerr<<"bind create error "<<"path : "<(pop); 89 | *is_created = 1; 90 | } 91 | else{ 92 | pop = pmemobj_open(nvm_path,layout); 93 | if(pop == nullptr){ 94 | std::cerr<<"bind open error"<(pop); 99 | } 100 | PMEMoid g_root = pmemobj_root(pop, sizeof(root_obj)); 101 | *root_p=(root_obj*)pmemobj_direct(g_root); 102 | zeroMemAmount(); 103 | return true; 104 | 105 | } 106 | static bool unbind(int poolId) { 107 | PMEMobjpool *pop = reinterpret_cast(baseAddresses[poolId]); 108 | pmemobj_close(pop); 109 | return true; 110 | } 111 | 112 | static bool bindLog(int poolId, const char *nvm_path, size_t size) { 113 | PMEMobjpool *pop; 114 | int ret; 115 | 116 | if (access(nvm_path, F_OK) != 0) { 117 | pop = pmemobj_create(nvm_path, POBJ_LAYOUT_NAME(nv), size, 0666); 118 | if(pop == nullptr){ 119 | std::cerr<<"bind create error"<(pop); 123 | PMEMoid g_root = pmemobj_root(pop, sizeof(PMEMoid)); 124 | // PMEMoid g_root = pmemobj_root(pop, 64UL*1024UL*1024UL*1024UL); 125 | int ret = pmemobj_alloc(pop, &g_root, 512UL*1024UL*1024UL, 0, NULL, NULL); 126 | if(ret){ 127 | std::cerr<<"!!! alloc error"<(pop); 142 | logVaddr[poolId] = pmemobj_direct(g_root); 143 | } 144 | return true; 145 | } 146 | 147 | static bool unbindLog(int poolId) { 148 | PMEMobjpool *pop = reinterpret_cast(baseAddresses[poolId*3+2]); 149 | pmemobj_close(pop); 150 | return true; 151 | } 152 | 153 | static void* getOpLog(int i){ 154 | unsigned long vaddr = (unsigned long)logVaddr[0]; 155 | //printf("vaddr :%p %p\n",vaddr, (void *)(vaddr+(64*i))); 156 | 157 | return (void*)(vaddr + (64*i)); 158 | } 159 | 160 | }; 161 | 162 | static inline void flushToNVM(char *data, size_t sz){ 163 | volatile char *ptr = (char *)((unsigned long)data & ~(L1_CACHE_BYTES- 1)); 164 | for (; ptr < data + sz; ptr += L1_CACHE_BYTES) { 165 | clwb(ptr); 166 | } 167 | } 168 | 169 | -------------------------------------------------------------------------------- /include/pptr.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "pmem.h" 5 | 6 | 7 | template 8 | class pptr { 9 | public: 10 | //TEST 11 | pptr() noexcept : rawPtr{} {} 12 | pptr(int poolId, unsigned long offset){ 13 | rawPtr = ((unsigned long)poolId) << 48 | offset; 14 | 15 | } 16 | T *operator->() { 17 | int poolId = (rawPtr&MASK_POOL) >> 48; 18 | void *baseAddr = PMem::getBaseOf(poolId); 19 | unsigned long offset = rawPtr & MASK; 20 | return (T *)((unsigned long)baseAddr + offset); 21 | } 22 | 23 | T *getVaddr() { 24 | unsigned long offset = rawPtr & MASK; 25 | if(offset == 0){ 26 | return nullptr; 27 | } 28 | 29 | int poolId = (rawPtr&MASK_POOL) >> 48; 30 | void *baseAddr = PMem::getBaseOf(poolId); 31 | 32 | return (T *)((unsigned long)baseAddr + offset); 33 | } 34 | 35 | unsigned long getRawPtr() { 36 | return rawPtr; 37 | } 38 | 39 | unsigned long setRawPtr(void *p) { 40 | rawPtr = (unsigned long)p; 41 | } 42 | 43 | inline void markDirty(){ 44 | rawPtr= ((1UL << 61) | rawPtr); 45 | } 46 | 47 | bool isDirty(){ 48 | return (((1UL << 61) & rawPtr)==(1UL<<61)); 49 | } 50 | 51 | inline void markClean(){ 52 | rawPtr= (rawPtr&MASK_DIRTY); 53 | } 54 | 55 | private: 56 | unsigned long rawPtr; // 16b + 48 b // nvm 57 | }; 58 | 59 | -------------------------------------------------------------------------------- /lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #add_subdirectory(ARTROWEX) 2 | add_subdirectory(PDL-ART) 3 | -------------------------------------------------------------------------------- /lib/PDL-ART/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_FLAGS_RELEASE "-O3") 2 | file(GLOB artsrc "*.cpp" "*.h") 3 | 4 | add_library(pdlart STATIC ${artsrc}) 5 | 6 | target_include_directories(pdlart PUBLIC 7 | ${CMAKE_CURRENT_SOURCE_DIR} 8 | ) 9 | 10 | INSTALL(TARGETS pdlart 11 | ARCHIVE DESTINATION ${CMAKE_SOURCE_DIR} 12 | ) 13 | 14 | -------------------------------------------------------------------------------- /lib/PDL-ART/Epoche.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by florian on 22.10.15. 3 | // 4 | #ifndef EPOCHE_CPP 5 | #define EPOCHE_CPP 6 | 7 | #include 8 | #include 9 | #include 10 | #include "Epoche.h" 11 | 12 | using namespace ART; 13 | 14 | 15 | inline DeletionList::~DeletionList() { 16 | assert(deletitionListCount == 0 && headDeletionList == nullptr); 17 | LabelDelete *cur = nullptr, *next = freeLabelDeletes; 18 | while (next != nullptr) { 19 | cur = next; 20 | next = cur->next; 21 | delete cur; 22 | } 23 | freeLabelDeletes = nullptr; 24 | } 25 | 26 | inline std::size_t DeletionList::size() { 27 | return deletitionListCount; 28 | } 29 | 30 | inline void DeletionList::remove(LabelDelete *label, LabelDelete *prev) { 31 | if (prev == nullptr) { 32 | headDeletionList = label->next; 33 | } else { 34 | prev->next = label->next; 35 | } 36 | deletitionListCount -= label->nodesCount; 37 | 38 | label->next = freeLabelDeletes; 39 | freeLabelDeletes = label; 40 | deleted += label->nodesCount; 41 | } 42 | 43 | inline void DeletionList::add(void *n, uint64_t globalEpoch) { 44 | deletitionListCount++; 45 | LabelDelete *label; 46 | if (headDeletionList != nullptr && headDeletionList->nodesCount < headDeletionList->nodes.size()) { 47 | label = headDeletionList; 48 | } else { 49 | if (freeLabelDeletes != nullptr) { 50 | label = freeLabelDeletes; 51 | freeLabelDeletes = freeLabelDeletes->next; 52 | } else { 53 | label = new LabelDelete(); 54 | } 55 | label->nodesCount = 0; 56 | label->next = headDeletionList; 57 | headDeletionList = label; 58 | } 59 | label->nodes[label->nodesCount] = n; 60 | label->nodesCount++; 61 | label->epoche = globalEpoch; 62 | 63 | added++; 64 | } 65 | 66 | inline LabelDelete *DeletionList::head() { 67 | return headDeletionList; 68 | } 69 | 70 | inline void Epoche::enterEpoche(ThreadInfo &epocheInfo) { 71 | unsigned long curEpoche = currentEpoche.load(std::memory_order_relaxed); 72 | epocheInfo.getDeletionList().localEpoche.store(curEpoche, std::memory_order_release); 73 | } 74 | 75 | inline void Epoche::markNodeForDeletion(void *n, ThreadInfo &epocheInfo) { 76 | epocheInfo.getDeletionList().add(n, currentEpoche.load()); 77 | epocheInfo.getDeletionList().thresholdCounter++; 78 | } 79 | 80 | inline void Epoche::exitEpocheAndCleanup(ThreadInfo &epocheInfo) { 81 | DeletionList &deletionList = epocheInfo.getDeletionList(); 82 | if ((deletionList.thresholdCounter & (64 - 1)) == 1) { 83 | currentEpoche++; 84 | } 85 | if (deletionList.thresholdCounter > startGCThreshhold) { 86 | if (deletionList.size() == 0) { 87 | deletionList.thresholdCounter = 0; 88 | return; 89 | } 90 | deletionList.localEpoche.store(std::numeric_limits::max()); 91 | 92 | uint64_t oldestEpoche = std::numeric_limits::max(); 93 | for (auto &epoche : deletionLists) { 94 | auto e = epoche.localEpoche.load(); 95 | if (e < oldestEpoche) { 96 | oldestEpoche = e; 97 | } 98 | } 99 | 100 | LabelDelete *cur = deletionList.head(), *next, *prev = nullptr; 101 | while (cur != nullptr) { 102 | next = cur->next; 103 | 104 | if (cur->epoche < oldestEpoche) { 105 | for (std::size_t i = 0; i < cur->nodesCount; ++i) { 106 | //operator delete(cur->nodes[i]); 107 | //PMem::freeVaddr(cur->nodes[i]); 108 | PMEMoid ptr = pmemobj_oid(cur->nodes[i]); 109 | pmemobj_free(&ptr); 110 | } 111 | deletionList.remove(cur, prev); 112 | } else { 113 | prev = cur; 114 | } 115 | cur = next; 116 | } 117 | deletionList.thresholdCounter = 0; 118 | } 119 | } 120 | 121 | inline Epoche::~Epoche() { 122 | uint64_t oldestEpoche = std::numeric_limits::max(); 123 | for (auto &epoche : deletionLists) { 124 | auto e = epoche.localEpoche.load(); 125 | if (e < oldestEpoche) { 126 | oldestEpoche = e; 127 | } 128 | } 129 | for (auto &d : deletionLists) { 130 | LabelDelete *cur = d.head(), *next, *prev = nullptr; 131 | while (cur != nullptr) { 132 | next = cur->next; 133 | 134 | assert(cur->epoche < oldestEpoche); 135 | for (std::size_t i = 0; i < cur->nodesCount; ++i) { 136 | //operator delete(cur->nodes[i]); 137 | //PMem::freeVaddr(cur->nodes[i]); 138 | PMEMoid ptr = pmemobj_oid(cur->nodes[i]); 139 | pmemobj_free(&ptr); 140 | } 141 | d.remove(cur, prev); 142 | cur = next; 143 | } 144 | } 145 | } 146 | 147 | inline void Epoche::showDeleteRatio() { 148 | for (auto &d : deletionLists) { 149 | std::cout << "deleted " << d.deleted << " of " << d.added << std::endl; 150 | } 151 | } 152 | 153 | inline ThreadInfo::ThreadInfo(Epoche &epoche) 154 | : epoche(epoche), deletionList(epoche.deletionLists.local()) { } 155 | 156 | inline DeletionList &ThreadInfo::getDeletionList() const { 157 | return deletionList; 158 | } 159 | 160 | inline Epoche &ThreadInfo::getEpoche() const { 161 | return epoche; 162 | } 163 | 164 | #endif //EPOCHE_CPP 165 | -------------------------------------------------------------------------------- /lib/PDL-ART/Epoche.h: -------------------------------------------------------------------------------- 1 | #ifndef ART_EPOCHE_H 2 | #define ART_EPOCHE_H 3 | 4 | #include 5 | #include 6 | #include "tbb/enumerable_thread_specific.h" 7 | #include "tbb/combinable.h" 8 | 9 | namespace ART { 10 | 11 | struct LabelDelete { 12 | std::array nodes; 13 | uint64_t epoche; 14 | std::size_t nodesCount; 15 | LabelDelete *next; 16 | }; 17 | 18 | class DeletionList { 19 | LabelDelete *headDeletionList = nullptr; 20 | LabelDelete *freeLabelDeletes = nullptr; 21 | std::size_t deletitionListCount = 0; 22 | 23 | public: 24 | std::atomic localEpoche; 25 | size_t thresholdCounter{0}; 26 | 27 | ~DeletionList(); 28 | 29 | LabelDelete *head(); 30 | 31 | void add(void *n, uint64_t globalEpoch); 32 | 33 | void remove(LabelDelete *label, LabelDelete *prev); 34 | 35 | std::size_t size(); 36 | 37 | std::uint64_t deleted = 0; 38 | std::uint64_t added = 0; 39 | }; 40 | 41 | class Epoche; 42 | class EpocheGuard; 43 | 44 | class ThreadInfo { 45 | friend class Epoche; 46 | friend class EpocheGuard; 47 | Epoche &epoche; 48 | DeletionList &deletionList; 49 | 50 | DeletionList & getDeletionList() const; 51 | public: 52 | 53 | ThreadInfo(Epoche &epoche); 54 | 55 | ThreadInfo(const ThreadInfo &ti) : epoche(ti.epoche), deletionList(ti.deletionList) { 56 | } 57 | 58 | ~ThreadInfo(); 59 | 60 | Epoche & getEpoche() const; 61 | }; 62 | 63 | class Epoche { 64 | friend class ThreadInfo; 65 | std::atomic currentEpoche{0}; 66 | 67 | tbb::enumerable_thread_specific deletionLists; 68 | 69 | size_t startGCThreshhold; 70 | 71 | 72 | public: 73 | Epoche(size_t startGCThreshhold) : startGCThreshhold(startGCThreshhold) { } 74 | 75 | ~Epoche(); 76 | 77 | void enterEpoche(ThreadInfo &epocheInfo); 78 | 79 | void markNodeForDeletion(void *n, ThreadInfo &epocheInfo); 80 | 81 | void exitEpocheAndCleanup(ThreadInfo &info); 82 | 83 | void showDeleteRatio(); 84 | 85 | }; 86 | 87 | class EpocheGuard { 88 | ThreadInfo &threadEpocheInfo; 89 | public: 90 | 91 | EpocheGuard(ThreadInfo &threadEpocheInfo) : threadEpocheInfo(threadEpocheInfo) { 92 | threadEpocheInfo.getEpoche().enterEpoche(threadEpocheInfo); 93 | } 94 | 95 | ~EpocheGuard() { 96 | threadEpocheInfo.getEpoche().exitEpocheAndCleanup(threadEpocheInfo); 97 | } 98 | }; 99 | 100 | class EpocheGuardReadonly { 101 | public: 102 | 103 | EpocheGuardReadonly(ThreadInfo &threadEpocheInfo) { 104 | threadEpocheInfo.getEpoche().enterEpoche(threadEpocheInfo); 105 | } 106 | 107 | ~EpocheGuardReadonly() { 108 | } 109 | }; 110 | 111 | inline ThreadInfo::~ThreadInfo() { 112 | deletionList.localEpoche.store(std::numeric_limits::max()); 113 | } 114 | } 115 | 116 | #endif //ART_EPOCHE_H 117 | -------------------------------------------------------------------------------- /lib/PDL-ART/Key.h: -------------------------------------------------------------------------------- 1 | #ifndef ART_KEY_H 2 | #define ART_KEY_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using KeyLen = uint32_t; 10 | 11 | class Key { 12 | public: 13 | 14 | static constexpr uint32_t stackLen = 128; 15 | uint32_t len = 0; 16 | 17 | uint8_t *data; 18 | 19 | uint8_t stackKey[stackLen]; 20 | 21 | Key(uint64_t k) { setInt(k); } 22 | 23 | void setInt(uint64_t k) { 24 | data = stackKey; 25 | len = 8; 26 | *reinterpret_cast(stackKey) = __builtin_bswap64(k); 27 | } 28 | 29 | Key() {} 30 | 31 | ~Key(); 32 | 33 | Key(const Key &key) = delete; 34 | 35 | Key(Key &&key); 36 | 37 | void set(const char bytes[], const std::size_t length); 38 | 39 | void operator=(const char key[]); 40 | 41 | bool operator==(const Key &k) const { 42 | if (k.getKeyLen() != getKeyLen()) { 43 | return false; 44 | } 45 | return std::memcmp(&k[0], data, getKeyLen()) == 0; 46 | } 47 | 48 | bool operator<(const Key &k) const { 49 | for (uint32_t i = 0; i < std::min(k.getKeyLen(), getKeyLen()); ++i) { 50 | if (data[i] > k[i]) 51 | return false; 52 | else if (data[i] < k[i]) 53 | return true; 54 | } 55 | } 56 | 57 | uint8_t &operator[](std::size_t i); 58 | 59 | const uint8_t &operator[](std::size_t i) const; 60 | 61 | KeyLen getKeyLen() const; 62 | 63 | void setKeyLen(KeyLen len); 64 | 65 | }; 66 | 67 | 68 | inline uint8_t &Key::operator[](std::size_t i) { 69 | assert(i < len); 70 | return data[i]; 71 | } 72 | 73 | inline const uint8_t &Key::operator[](std::size_t i) const { 74 | assert(i < len); 75 | return data[i]; 76 | } 77 | 78 | inline KeyLen Key::getKeyLen() const { return len; } 79 | 80 | inline Key::~Key() { 81 | if (len > stackLen) { 82 | delete[] data; 83 | data = nullptr; 84 | } 85 | } 86 | 87 | inline Key::Key(Key &&key) { 88 | len = key.len; 89 | if (len > stackLen) { 90 | data = key.data; 91 | key.data = nullptr; 92 | } else { 93 | memcpy(stackKey, key.stackKey, key.len); 94 | data = stackKey; 95 | } 96 | } 97 | 98 | inline void Key::set(const char bytes[], const std::size_t length) { 99 | if (len > stackLen) { 100 | delete[] data; 101 | } 102 | if (length <= stackLen) { 103 | memcpy(stackKey, bytes, length); 104 | data = stackKey; 105 | } else { 106 | data = new uint8_t[length]; 107 | memcpy(data, bytes, length); 108 | } 109 | len = length; 110 | } 111 | 112 | inline void Key::operator=(const char key[]) { 113 | if (len > stackLen) { 114 | delete[] data; 115 | } 116 | len = strlen(key); 117 | if (len <= stackLen) { 118 | memcpy(stackKey, key, len); 119 | data = stackKey; 120 | } else { 121 | data = new uint8_t[len]; 122 | memcpy(data, key, len); 123 | } 124 | } 125 | 126 | inline void Key::setKeyLen(KeyLen newLen) { 127 | if (len == newLen) return; 128 | if (len > stackLen) { 129 | delete[] data; 130 | } 131 | len = newLen; 132 | if (len > stackLen) { 133 | data = new uint8_t[len]; 134 | } else { 135 | data = stackKey; 136 | } 137 | } 138 | 139 | #endif // ART_KEY_H 140 | -------------------------------------------------------------------------------- /lib/PDL-ART/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019-2021 Virginia Tech 2 | Copyright 2014-2015 Florian Scheibner 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | -------------------------------------------------------------------------------- /lib/PDL-ART/N.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "N.h" 7 | #include "N4.cpp" 8 | #include "N16.cpp" 9 | #include "N48.cpp" 10 | #include "N256.cpp" 11 | 12 | namespace ART_ROWEX { 13 | void N::setType(NTypes type) { 14 | typeVersionLockObsolete.fetch_add(convertTypeToVersion(type)); 15 | // smp_faa(&(pLock.lock),convertTypeToVersion(type)); 16 | } 17 | 18 | uint64_t N::convertTypeToVersion(NTypes type) { 19 | return (static_cast(type) << 62); 20 | } 21 | 22 | NTypes N::getType() const { 23 | // return static_cast((pLock.lock) >> 62); 24 | return static_cast(typeVersionLockObsolete.load(std::memory_order_relaxed) >> 62); 25 | } 26 | 27 | uint32_t N::getLevel() const { 28 | return level; 29 | } 30 | 31 | void N::setLevel(uint32_t lev){ 32 | level=lev; 33 | } 34 | 35 | void N::writeLockOrRestart(bool &needRestart,uint32_t genId) { 36 | uint64_t version = typeVersionLockObsolete.load(std::memory_order_relaxed); 37 | uint64_t new_ver=0; 38 | uint64_t tmp=0; 39 | uint32_t lockGenId; 40 | uint32_t verLock; 41 | 42 | NTypes type = static_cast(version >> 62); 43 | bool isLeaf = (reinterpret_cast(version) & (static_cast(1) << 63)) 44 | == (static_cast(1) << 63); 45 | 46 | tmp+=(uint64_t)((uint64_t)type<<62); 47 | tmp+=(uint64_t)(1<<63); 48 | 49 | lockGenId = (version - tmp)>>32; 50 | verLock = (uint32_t)(version-tmp); 51 | new_ver += tmp; 52 | new_ver += ((uint64_t)genId<<32); 53 | new_ver += 0b10; 54 | 55 | if(lockGenId != genId){ 56 | if(!typeVersionLockObsolete.compare_exchange_weak(version, new_ver)){ 57 | needRestart = true; 58 | return; 59 | } 60 | } 61 | else{ 62 | do{ 63 | version = typeVersionLockObsolete.load(); 64 | while (isLocked(version)) { 65 | _mm_pause(); 66 | version = typeVersionLockObsolete.load(); 67 | } 68 | if (isObsolete(version)) { 69 | needRestart = true; 70 | return; 71 | } 72 | } while (!typeVersionLockObsolete.compare_exchange_weak(version, version + 0b10)); 73 | } 74 | #if 0 75 | do { 76 | version = typeVersionLockObsolete.load(); 77 | while (isLocked(version)) { 78 | _mm_pause(); 79 | version = typeVersionLockObsolete.load(); 80 | } 81 | if (isObsolete(version)) { 82 | needRestart = true; 83 | return; 84 | } 85 | } while (!typeVersionLockObsolete.compare_exchange_weak(version, version + 0b10)); 86 | #endif 87 | } 88 | 89 | void N::lockVersionOrRestart(uint64_t &version, bool &needRestart, uint32_t genId) { 90 | uint64_t new_ver=0; 91 | uint64_t tmp=0; 92 | uint32_t lockGenId; 93 | uint32_t verLock; 94 | 95 | NTypes type = static_cast(version >> 62); 96 | bool isLeaf = (reinterpret_cast(version) & (static_cast(1) << 63)) 97 | == (static_cast(1) << 63); 98 | 99 | tmp+=(uint64_t)((uint64_t)type<<62); 100 | tmp+=(uint64_t)(1<<63); 101 | 102 | lockGenId = (version - tmp)>>32; 103 | verLock = (uint32_t)(version-tmp); 104 | new_ver += tmp; 105 | new_ver += ((uint64_t)genId<<32); 106 | new_ver += 0b10; 107 | 108 | 109 | 110 | 111 | if (isLocked(version) || isObsolete(version)) { 112 | needRestart = true; 113 | return; 114 | } 115 | if(lockGenId != genId){ 116 | if(!typeVersionLockObsolete.compare_exchange_weak(version, new_ver)){ 117 | needRestart = true; 118 | return; 119 | } 120 | } 121 | else{ 122 | if (typeVersionLockObsolete.compare_exchange_strong(version, version + 0b10)) { 123 | version = version + 0b10; 124 | } 125 | else { 126 | needRestart = true; 127 | } 128 | } 129 | 130 | /* if (typeVersionLockObsolete.compare_exchange_strong(version, version + 0b10)) { 131 | version = version + 0b10; 132 | } else { 133 | needRestart = true; 134 | }*/ 135 | } 136 | 137 | void N::writeUnlock() { 138 | typeVersionLockObsolete.fetch_add(0b10); 139 | } 140 | 141 | N *N::getAnyChild(const N *node) { 142 | switch (node->getType()) { 143 | case NTypes::N4: { 144 | auto n = static_cast(node); 145 | return n->getAnyChild(); 146 | } 147 | case NTypes::N16: { 148 | auto n = static_cast(node); 149 | return n->getAnyChild(); 150 | } 151 | case NTypes::N48: { 152 | auto n = static_cast(node); 153 | return n->getAnyChild(); 154 | } 155 | case NTypes::N256: { 156 | auto n = static_cast(node); 157 | return n->getAnyChild(); 158 | } 159 | } 160 | assert(false); 161 | __builtin_unreachable(); 162 | } 163 | 164 | N *N::getAnyChildReverse(const N *node) { 165 | switch (node->getType()) { 166 | case NTypes::N4: { 167 | auto n = static_cast(node); 168 | return n->getAnyChildReverse(); 169 | } 170 | case NTypes::N16: { 171 | auto n = static_cast(node); 172 | return n->getAnyChildReverse(); 173 | } 174 | case NTypes::N48: { 175 | auto n = static_cast(node); 176 | return n->getAnyChildReverse(); 177 | } 178 | case NTypes::N256: { 179 | auto n = static_cast(node); 180 | return n->getAnyChildReverse(); 181 | } 182 | } 183 | assert(false); 184 | __builtin_unreachable(); 185 | } 186 | 187 | void N::change(N *node, uint8_t key, pptr val) { 188 | switch (node->getType()) { 189 | case NTypes::N4: { 190 | auto n = static_cast(node); 191 | n->change(key, val); 192 | return; 193 | } 194 | case NTypes::N16: { 195 | auto n = static_cast(node); 196 | n->change(key, val); 197 | return; 198 | } 199 | case NTypes::N48: { 200 | auto n = static_cast(node); 201 | n->change(key, val); 202 | return; 203 | } 204 | case NTypes::N256: { 205 | auto n = static_cast(node); 206 | n->change(key, val); 207 | return; 208 | } 209 | } 210 | assert(false); 211 | __builtin_unreachable(); 212 | } 213 | 214 | template 215 | void N::insertGrow(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, ThreadInfo &threadInfo, bool &needRestart, OpStruct *oplog,uint32_t genId) { 216 | if (n->insert(key, val)) { 217 | n->writeUnlock(); 218 | return; 219 | } 220 | 221 | #ifdef MULTIPOOL 222 | int chip,core; 223 | read_coreid_rdtscp(&chip,&core); 224 | uint16_t poolId = (uint16_t)(3*chip); 225 | #else 226 | uint16_t poolId = 0; 227 | #endif 228 | //FIX ME 229 | /* pptr ologPtr; 230 | PMEMoid oid; 231 | PMem::alloc(poolId,sizeof(OpStruct),(void **)&ologPtr, &oid); 232 | OpStruct *olog = ologPtr.getVaddr();*/ 233 | pptr nBigPtr; 234 | unsigned long poolAddr = (unsigned long)pmemobj_pool_by_ptr(parentNode); 235 | uint16_t id = -1; 236 | unsigned long off= (unsigned long)parentNode - poolAddr; 237 | for(int i=0; i<2; i++){ 238 | if((unsigned long)PMem::getBaseOf(i*3) == poolAddr){ 239 | id = i; 240 | break; 241 | } 242 | } 243 | pptr parentPtr(id,off); 244 | oplog->op = OpStruct::insert; 245 | oplog->oldNodePtr = (void*)parentPtr.getRawPtr();; 246 | 247 | PMem::alloc(poolId,sizeof(biggerN),(void **)&nBigPtr, &(oplog->newNodeOid)); 248 | biggerN *nBig = (biggerN *)new(nBigPtr.getVaddr()) biggerN(n->getLevel(),n->getPrefi()); 249 | 250 | n->copyTo(nBig); 251 | nBig->insert(key, val); 252 | flushToNVM((char*)nBig, sizeof(biggerN)); 253 | 254 | parentNode->writeLockOrRestart(needRestart,genId); 255 | if (needRestart) { 256 | PMem::free((void *)nBigPtr.getRawPtr()); 257 | n->writeUnlock(); 258 | return; 259 | } 260 | 261 | N::change(parentNode, keyParent, nBigPtr); 262 | oplog->op = OpStruct::done; 263 | parentNode->writeUnlock(); 264 | 265 | n->writeUnlockObsolete(); 266 | threadInfo.getEpoche().markNodeForDeletion(n, threadInfo); 267 | } 268 | 269 | template 270 | void N::insertCompact(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId) { 271 | 272 | #ifdef MULTIPOOL 273 | int chip,core; 274 | read_coreid_rdtscp(&chip,&core); 275 | uint16_t poolId = (uint16_t)(3*chip); 276 | #else 277 | uint16_t poolId = 0; 278 | #endif 279 | pptr nNewPtr; 280 | /* pptr ologPtr; 281 | PMEMoid oid; 282 | PMem::alloc(poolId,sizeof(OpStruct),(void **)&ologPtr, &oid); 283 | OpStruct *olog = ologPtr.getVaddr();*/ 284 | 285 | unsigned long poolAddr = (unsigned long)pmemobj_pool_by_ptr(parentNode); 286 | uint16_t id = -1; 287 | unsigned long off= (unsigned long)parentNode - poolAddr; 288 | for(int i=0; i<2; i++){ 289 | if((unsigned long)PMem::getBaseOf(i*3) == poolAddr){ 290 | id = i; 291 | break; 292 | } 293 | } 294 | pptr parentPtr(id,off); 295 | oplog->op = OpStruct::insert; 296 | oplog->oldNodePtr = (void*)parentPtr.getRawPtr();; 297 | 298 | PMem::alloc(poolId,sizeof(curN),(void **)&nNewPtr,&(oplog->newNodeOid)); 299 | curN *nNew = (curN *)new(nNewPtr.getVaddr()) curN(n->getLevel(),n->getPrefi()); 300 | 301 | n->copyTo(nNew); 302 | nNew->insert(key, val); 303 | flushToNVM((char*)nNew,sizeof(curN)); 304 | 305 | parentNode->writeLockOrRestart(needRestart,genId); 306 | if (needRestart) { 307 | PMem::free((void *)nNewPtr.getRawPtr()); 308 | n->writeUnlock(); 309 | return; 310 | } 311 | 312 | N::change(parentNode, keyParent, nNewPtr); 313 | oplog->op = OpStruct::done; 314 | parentNode->writeUnlock(); 315 | 316 | n->writeUnlockObsolete(); 317 | threadInfo.getEpoche().markNodeForDeletion(n, threadInfo); 318 | } 319 | 320 | void N::insertAndUnlock(N *node, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId) { 321 | switch (node->getType()) { 322 | case NTypes::N4: { 323 | auto n = static_cast(node); 324 | uint32_t cValues = n->countValues; 325 | uint16_t nCompactCount = (uint16_t)(cValues>>16); 326 | uint16_t nCount = (uint16_t)cValues; 327 | if (nCompactCount == 4 && nCount <= 3) { 328 | //if (n->compactCount == 4 && n->count <= 3) { 329 | insertCompact(n, parentNode, keyParent, key, val, threadInfo, needRestart, oplog, genId); 330 | break; 331 | } 332 | insertGrow(n, parentNode, keyParent, key, val, threadInfo, needRestart, oplog, genId); 333 | break; 334 | } 335 | case NTypes::N16: { 336 | auto n = static_cast(node); 337 | uint32_t cValues = n->countValues; 338 | uint16_t nCompactCount = (uint16_t)cValues>>16; 339 | uint16_t nCount = (uint16_t)cValues; 340 | if (nCompactCount == 16 && nCount <= 14) { 341 | //if (n->compactCount == 16 && n->count <= 14) { 342 | insertCompact(n, parentNode, keyParent, key, val, threadInfo, needRestart, oplog, genId); 343 | break; 344 | } 345 | insertGrow(n, parentNode, keyParent, key, val, threadInfo, needRestart, oplog, genId); 346 | break; 347 | } 348 | case NTypes::N48: { 349 | auto n = static_cast(node); 350 | uint32_t cValues = n->countValues; 351 | uint16_t nCompactCount = (uint16_t)cValues>>16; 352 | uint16_t nCount = (uint16_t)cValues; 353 | if (nCompactCount == 48 && nCount != 48) { 354 | //if (n->compactCount == 48 && n->count != 48) { 355 | insertCompact(n, parentNode, keyParent, key, val, threadInfo, needRestart, oplog, genId); 356 | break; 357 | } 358 | insertGrow(n, parentNode, keyParent, key, val, threadInfo, needRestart,oplog, genId); 359 | break; 360 | } 361 | case NTypes::N256: { 362 | auto n = static_cast(node); 363 | n->insert(key, val); 364 | node->writeUnlock(); 365 | break; 366 | } 367 | } 368 | } 369 | 370 | pptr N::getChildPptr(const uint8_t k, N *node){ 371 | switch (node->getType()) { 372 | case NTypes::N4: { 373 | auto n = static_cast(node); 374 | return n->getChildPptr(k); 375 | } 376 | case NTypes::N16: { 377 | auto n = static_cast(node); 378 | return n->getChildPptr(k); 379 | } 380 | case NTypes::N48: { 381 | auto n = static_cast(node); 382 | return n->getChildPptr(k); 383 | } 384 | case NTypes::N256: { 385 | auto n = static_cast(node); 386 | return n->getChildPptr(k); 387 | } 388 | } 389 | assert(false); 390 | __builtin_unreachable(); 391 | } 392 | 393 | N *N::getChild(const uint8_t k, N *node){ 394 | switch (node->getType()) { 395 | case NTypes::N4: { 396 | auto n = static_cast(node); 397 | return n->getChild(k); 398 | } 399 | case NTypes::N16: { 400 | auto n = static_cast(node); 401 | return n->getChild(k); 402 | } 403 | case NTypes::N48: { 404 | auto n = static_cast(node); 405 | return n->getChild(k); 406 | } 407 | case NTypes::N256: { 408 | auto n = static_cast(node); 409 | return n->getChild(k); 410 | } 411 | } 412 | assert(false); 413 | __builtin_unreachable(); 414 | } 415 | 416 | void N::deleteChildren(N *node) { 417 | if (N::isLeaf(node)) { 418 | return; 419 | } 420 | switch (node->getType()) { 421 | case NTypes::N4: { 422 | auto n = static_cast(node); 423 | n->deleteChildren(); 424 | return; 425 | } 426 | case NTypes::N16: { 427 | auto n = static_cast(node); 428 | n->deleteChildren(); 429 | return; 430 | } 431 | case NTypes::N48: { 432 | auto n = static_cast(node); 433 | n->deleteChildren(); 434 | return; 435 | } 436 | case NTypes::N256: { 437 | auto n = static_cast(node); 438 | n->deleteChildren(); 439 | return; 440 | } 441 | } 442 | assert(false); 443 | __builtin_unreachable(); 444 | } 445 | 446 | template 447 | void N::removeAndShrink(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId) { 448 | if (n->remove(key, parentNode == nullptr)) { 449 | n->writeUnlock(); 450 | return; 451 | } 452 | 453 | //auto nSmall = new smallerN(n->getLevel(), n->getPrefi()); 454 | 455 | #ifdef MULTIPOOL 456 | int chip,core; 457 | read_coreid_rdtscp(&chip,&core); 458 | uint16_t poolId = (uint16_t)(3*chip); 459 | #else 460 | uint16_t poolId = 0; 461 | #endif 462 | 463 | /* pptr nNewPtr; 464 | pptr ologPtr; 465 | PMEMoid oid; 466 | PMem::alloc(poolId,sizeof(OpStruct),(void **)&ologPtr, &oid); 467 | OpStruct *olog = ologPtr.getVaddr();*/ 468 | pptr nSmallPtr; 469 | unsigned long poolAddr = (unsigned long)pmemobj_pool_by_ptr(parentNode); 470 | uint16_t id = -1; 471 | unsigned long off= (unsigned long)parentNode - poolAddr; 472 | for(int i=0; i<2; i++){ 473 | if((unsigned long)PMem::getBaseOf(i*3) == poolAddr){ 474 | id = i; 475 | break; 476 | } 477 | } 478 | pptr parentPtr(id,off); 479 | 480 | //pptr nSmallPtr; 481 | oplog->op = OpStruct::insert; 482 | oplog->oldNodePtr = (void*)parentPtr.getRawPtr();; 483 | PMem::alloc(poolId,sizeof(smallerN),(void **)&nSmallPtr,&(oplog->newNodeOid)); 484 | smallerN *nSmall= (smallerN *)new(nSmallPtr.getVaddr()) smallerN(n->getLevel(),n->getPrefi()); 485 | 486 | /* nSmall->eetType(nodeType); 487 | nSmall->setLevel(n->getLevel()); 488 | Prefix p = n->getPrefi(); 489 | nSmall->setPrefix(p.prefix, p.prefixCount,false);*/ 490 | 491 | parentNode->writeLockOrRestart(needRestart,genId); 492 | if (needRestart) { 493 | //freeNVM(nSmall); 494 | n->writeUnlock(); 495 | return; 496 | } 497 | 498 | n->remove(key, true); 499 | n->copyTo(nSmall); 500 | flushToNVM((char*)nSmall,sizeof(smallerN)); 501 | N::change(parentNode, keyParent, nSmallPtr); 502 | oplog->op = OpStruct::done; 503 | 504 | parentNode->writeUnlock(); 505 | n->writeUnlockObsolete(); 506 | threadInfo.getEpoche().markNodeForDeletion(n, threadInfo); 507 | } 508 | 509 | void N::removeAndUnlock(N *node, uint8_t key, N *parentNode, uint8_t keyParent, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId) { 510 | switch (node->getType()) { 511 | case NTypes::N4: { 512 | auto n = static_cast(node); 513 | n->remove(key, false); 514 | n->writeUnlock(); 515 | break; 516 | } 517 | case NTypes::N16: { 518 | auto n = static_cast(node); 519 | 520 | NTypes nodeType = NTypes::N4; 521 | removeAndShrink(n, parentNode, keyParent, key, threadInfo, needRestart, oplog, genId); 522 | break; 523 | } 524 | case NTypes::N48: { 525 | NTypes nodeType = NTypes::N16; 526 | auto n = static_cast(node); 527 | removeAndShrink(n, parentNode, keyParent, key, threadInfo, needRestart, oplog, genId); 528 | break; 529 | } 530 | case NTypes::N256: { 531 | NTypes nodeType = NTypes::N48; 532 | auto n = static_cast(node); 533 | removeAndShrink(n, parentNode, keyParent, key, threadInfo, needRestart, oplog, genId); 534 | break; 535 | } 536 | } 537 | } 538 | 539 | bool N::isLocked(uint64_t version) const { 540 | return ((version & 0b10) == 0b10); 541 | } 542 | 543 | uint64_t N::getVersion() const { 544 | return typeVersionLockObsolete.load(); 545 | } 546 | 547 | bool N::isObsolete(uint64_t version) { 548 | return (version & 1) == 1; 549 | } 550 | 551 | bool N::checkOrRestart(uint64_t startRead) const { 552 | return readUnlockOrRestart(startRead); 553 | } 554 | 555 | bool N::readUnlockOrRestart(uint64_t startRead) const { 556 | return startRead == typeVersionLockObsolete.load(); 557 | } 558 | 559 | uint32_t N::getCount() const { 560 | uint32_t cValues = countValues; 561 | uint16_t c= (uint16_t)cValues; 562 | return c; 563 | } 564 | 565 | Prefix N::getPrefi() const { 566 | return prefix.load(); 567 | } 568 | 569 | void N::setPrefix(const uint8_t *prefix, uint32_t length, bool flush) { 570 | if (length > 0) { 571 | Prefix p; 572 | memcpy(p.prefix, prefix, std::min(length, maxStoredPrefixLength)); 573 | p.prefixCount = length; 574 | this->prefix.store(p, std::memory_order_release); 575 | } else { 576 | Prefix p; 577 | p.prefixCount = 0; 578 | this->prefix.store(p, std::memory_order_release); 579 | } 580 | if(flush) 581 | flushToNVM((char*)&(this->prefix),sizeof(Prefix)); 582 | } 583 | 584 | void N::addPrefixBefore(N* node, uint8_t key) { 585 | Prefix p = this->getPrefi(); 586 | Prefix nodeP = node->getPrefi(); 587 | uint32_t prefixCopyCount = std::min(maxStoredPrefixLength, nodeP.prefixCount + 1); 588 | memmove(p.prefix + prefixCopyCount, p.prefix, std::min(p.prefixCount, maxStoredPrefixLength - prefixCopyCount)); 589 | memcpy(p.prefix, nodeP.prefix, std::min(prefixCopyCount, nodeP.prefixCount)); 590 | if (nodeP.prefixCount < maxStoredPrefixLength) { 591 | p.prefix[prefixCopyCount - 1] = key; 592 | } 593 | p.prefixCount += nodeP.prefixCount + 1; 594 | this->prefix.store(p, std::memory_order_release); 595 | flushToNVM((char*)&(this->prefix),sizeof(Prefix)); 596 | } 597 | 598 | bool N::isLeaf(const N *n) { 599 | return (reinterpret_cast(n) & (static_cast(1) << 63)) == (static_cast(1) << 63); 600 | } 601 | 602 | N *N::setLeaf(TID tid) { 603 | return reinterpret_cast(tid | (static_cast(1) << 63)); 604 | } 605 | 606 | TID N::getLeaf(const N *n) { 607 | return (reinterpret_cast(n) & ((static_cast(1) << 63) - 1)); 608 | } 609 | 610 | std::tuple N::getSecondChild(N *node, const uint8_t key) { 611 | switch (node->getType()) { 612 | case NTypes::N4: { 613 | auto n = static_cast(node); 614 | return n->getSecondChild(key); 615 | } 616 | default: { 617 | assert(false); 618 | __builtin_unreachable(); 619 | } 620 | } 621 | } 622 | 623 | void N::deleteNode(N *node) { 624 | if (N::isLeaf(node)) { 625 | return; 626 | } 627 | switch (node->getType()) { 628 | case NTypes::N4: { 629 | auto n = static_cast(node); 630 | PMem::freeVaddr((void *)n); 631 | return; 632 | } 633 | case NTypes::N16: { 634 | auto n = static_cast(node); 635 | PMem::freeVaddr((void *)n); 636 | return; 637 | } 638 | case NTypes::N48: { 639 | auto n = static_cast(node); 640 | PMem::freeVaddr((void *)n); 641 | return; 642 | } 643 | case NTypes::N256: { 644 | auto n = static_cast(node); 645 | PMem::freeVaddr((void *)n); 646 | return; 647 | } 648 | } 649 | PMem::freeVaddr(node); 650 | } 651 | 652 | TID N::getAnyChildTid(const N *n) { 653 | const N *nextNode = n; 654 | 655 | while (true) { 656 | const N *node = nextNode; 657 | nextNode = getAnyChild(node); 658 | 659 | assert(nextNode != nullptr); 660 | if (isLeaf(nextNode)) { 661 | return getLeaf(nextNode); 662 | } 663 | } 664 | } 665 | 666 | TID N::getAnyChildTidReverse(const N *n) { 667 | const N *nextNode = n; 668 | 669 | while (true) { 670 | const N *node = nextNode; 671 | nextNode = getAnyChildReverse(node); 672 | 673 | assert(nextNode != nullptr); 674 | if (isLeaf(nextNode)) { 675 | return getLeaf(nextNode); 676 | } 677 | } 678 | } 679 | 680 | void N::getChildren(const N *node, uint8_t start, uint8_t end, std::tuple children[], 681 | uint32_t &childrenCount) { 682 | switch (node->getType()) { 683 | case NTypes::N4: { 684 | auto n = static_cast(node); 685 | n->getChildren(start, end, children, childrenCount); 686 | return; 687 | } 688 | case NTypes::N16: { 689 | auto n = static_cast(node); 690 | n->getChildren(start, end, children, childrenCount); 691 | return; 692 | } 693 | case NTypes::N48: { 694 | auto n = static_cast(node); 695 | n->getChildren(start, end, children, childrenCount); 696 | return; 697 | } 698 | case NTypes::N256: { 699 | auto n = static_cast(node); 700 | n->getChildren(start, end, children, childrenCount); 701 | return; 702 | } 703 | } 704 | } 705 | N *N::getSmallestChild(const N *node, uint8_t start) { 706 | switch (node->getType()) { 707 | case NTypes::N4: { 708 | auto n = static_cast(node); 709 | return n->getSmallestChild(start); 710 | } 711 | case NTypes::N16: { 712 | auto n = static_cast(node); 713 | return n->getSmallestChild(start); 714 | } 715 | case NTypes::N48: { 716 | auto n = static_cast(node); 717 | return n->getSmallestChild(start); 718 | } 719 | case NTypes::N256: { 720 | auto n = static_cast(node); 721 | return n->getSmallestChild(start); 722 | } 723 | } 724 | assert(false); 725 | __builtin_unreachable(); 726 | } 727 | N *N::getLargestChild(const N *node, uint8_t start) { 728 | switch (node->getType()) { 729 | case NTypes::N4: { 730 | auto n = static_cast(node); 731 | return n->getLargestChild(start); 732 | } 733 | case NTypes::N16: { 734 | auto n = static_cast(node); 735 | return n->getLargestChild(start); 736 | } 737 | case NTypes::N48: { 738 | auto n = static_cast(node); 739 | return n->getLargestChild(start); 740 | } 741 | case NTypes::N256: { 742 | auto n = static_cast(node); 743 | return n->getLargestChild(start); 744 | } 745 | } 746 | assert(false); 747 | __builtin_unreachable(); 748 | } 749 | } 750 | -------------------------------------------------------------------------------- /lib/PDL-ART/N.h: -------------------------------------------------------------------------------- 1 | 2 | // Created by florian on 05.08.15. 3 | // 4 | 5 | #ifndef ART_ROWEX_N_H 6 | #define ART_ROWEX_N_H 7 | //#define ART_NOREADLOCK 8 | //#define ART_NOWRITELOCK 9 | #include 10 | #include 11 | #include 12 | #include "Key.h" 13 | #include "Epoche.h" 14 | #include "common.h" 15 | #include "pptr.h" 16 | #include "ordo_clock.h" 17 | 18 | 19 | using TID = uint64_t; 20 | 21 | 22 | using namespace ART; 23 | namespace ART_ROWEX { 24 | /* 25 | * SynchronizedTree 26 | * LockCouplingTree 27 | * LockCheckFreeReadTree 28 | * UnsynchronizedTree 29 | */ 30 | 31 | enum class NTypes : uint8_t { 32 | N4 = 0, 33 | N16 = 1, 34 | N48 = 2, 35 | N256 = 3 36 | }; 37 | 38 | static constexpr uint32_t maxStoredPrefixLength = 4; 39 | struct Prefix { 40 | uint32_t prefixCount = 0; 41 | uint8_t prefix[maxStoredPrefixLength]; 42 | }; 43 | static_assert(sizeof(Prefix) == 8, "Prefix should be 64 bit long"); 44 | 45 | typedef struct p16BLock{ 46 | uint64_t genId; 47 | uint64_t lock; 48 | }p16BLock; 49 | 50 | class N { 51 | protected: 52 | N(NTypes type, uint32_t level, const uint8_t *prefix, uint32_t prefixLength) : level(level) { 53 | setPrefix(prefix, prefixLength,false); 54 | setType(type); 55 | } 56 | 57 | N(NTypes type, uint32_t level, const Prefix &prefi) : prefix(prefi), level(level) { 58 | setType(type); 59 | } 60 | 61 | N(const N &) = delete; 62 | 63 | N(N &&) = delete; 64 | 65 | //2b type 60b version 1b lock 1b obsolete 66 | //2b type 28b gen id 32b version 1b lock 1b obsolete 67 | std::atomic typeVersionLockObsolete{0b100}; 68 | //genID? 69 | 70 | // version 1, unlocked, not obsolete 71 | std::atomic prefix; 72 | uint32_t level; 73 | uint32_t countValues=0; //count 2B, compactCount 2B 74 | // uint16_t count = 0; 75 | // uint16_t compactCount = 0; 76 | 77 | 78 | 79 | 80 | static uint64_t convertTypeToVersion(NTypes type); 81 | 82 | public: 83 | void setType(NTypes type); 84 | // static idxPtr allocAndWriteMemLog(OpStruct *oplog, int size, mappingTable *mtable); 85 | 86 | NTypes getType() const; 87 | 88 | uint32_t getLevel() const; 89 | 90 | void setLevel(uint32_t level); 91 | 92 | uint32_t getCount() const; 93 | 94 | bool isLocked(uint64_t version) const; 95 | 96 | void writeLockOrRestart(bool &needRestart, uint32_t genId); 97 | 98 | void lockVersionOrRestart(uint64_t &version, bool &needRestart, uint32_t genId); 99 | 100 | void writeUnlock(); 101 | 102 | uint64_t getVersion() const; 103 | 104 | /** 105 | * returns true if node hasn't been changed in between 106 | */ 107 | bool checkOrRestart(uint64_t startRead) const; 108 | bool readUnlockOrRestart(uint64_t startRead) const; 109 | 110 | static bool isObsolete(uint64_t version); 111 | 112 | /** 113 | * can only be called when node is locked 114 | */ 115 | void writeUnlockObsolete() { 116 | // smp_faa(&(pLock.lock),0b11); 117 | typeVersionLockObsolete.fetch_add(0b11); 118 | } 119 | 120 | static N *getChild(const uint8_t k, N *node); 121 | static pptr getChildPptr(const uint8_t k, N *node); 122 | 123 | static void insertAndUnlock(N *node, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, 124 | ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId); 125 | 126 | static void change(N *node, uint8_t key, pptr val); 127 | 128 | static void removeAndUnlock(N *node, uint8_t key, N *parentNode, uint8_t keyParent, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId); 129 | 130 | Prefix getPrefi() const; 131 | 132 | void setPrefix(const uint8_t *prefix, uint32_t length, bool flush); 133 | 134 | void addPrefixBefore(N *node, uint8_t key); 135 | 136 | static TID getLeaf(const N *n); 137 | 138 | static bool isLeaf(const N *n); 139 | 140 | static N *setLeaf(TID tid); 141 | 142 | static N *getAnyChild(const N *n); 143 | static N *getAnyChildReverse(const N *n); 144 | 145 | static TID getAnyChildTid(const N *n); 146 | static TID getAnyChildTidReverse(const N *n); 147 | 148 | static void deleteChildren(N *node); 149 | 150 | static void deleteNode(N *node); 151 | 152 | static std::tuple getSecondChild(N *node, const uint8_t k); 153 | 154 | template 155 | static void insertGrow(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId); 156 | 157 | template 158 | static void insertCompact(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, pptr val, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId); 159 | 160 | template 161 | static void removeAndShrink(curN *n, N *parentNode, uint8_t keyParent, uint8_t key, ThreadInfo &threadInfo, bool &needRestart, OpStruct* oplog, uint32_t genId); 162 | 163 | static void getChildren(const N *node, uint8_t start, uint8_t end, std::tuple children[], 164 | uint32_t &childrenCount); 165 | static N *getSmallestChild(const N* node, uint8_t start); 166 | static N *getLargestChild(const N* node, uint8_t end); 167 | }; 168 | 169 | 170 | class N4 : public N { 171 | public: 172 | 173 | std::atomic keys[4]; 174 | std::atomic> children[4]; 175 | 176 | public: 177 | void *operator new(size_t size, void *location){ 178 | return location; 179 | } 180 | 181 | 182 | 183 | N4(uint32_t level, const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N4, level, prefix, prefixLength) { 184 | memset(keys, 0, sizeof(keys)); 185 | memset(children, 0, sizeof(children)); 186 | } 187 | 188 | N4(uint32_t level, const Prefix &prefi) : N(NTypes::N4, level, prefi) { 189 | memset(keys, 0, sizeof(keys)); 190 | memset(children, 0, sizeof(children)); 191 | } 192 | 193 | bool insert(uint8_t key, pptr n); 194 | bool insert(uint8_t key, pptr n, bool flush); 195 | 196 | template 197 | void copyTo(NODE *n) const; 198 | 199 | void change(uint8_t key, pptr val); 200 | 201 | N *getChild(const uint8_t k) const; 202 | pptr getChildPptr(const uint8_t k) const; 203 | 204 | bool remove(uint8_t k, bool force); 205 | 206 | N *getAnyChild() const; 207 | N *getAnyChildReverse() const; 208 | 209 | std::tuple getSecondChild(const uint8_t key) const; 210 | 211 | void deleteChildren(); 212 | 213 | void getChildren(uint8_t start, uint8_t end, std::tuple *&children, 214 | uint32_t &childrenCount) const; 215 | N *getSmallestChild(uint8_t start) const; 216 | N *getLargestChild(uint8_t end) const; 217 | }; 218 | 219 | class N16 : public N { 220 | public: 221 | void *operator new(size_t size, void *location){ 222 | return location; 223 | } 224 | 225 | std::atomic keys[16]; 226 | std::atomic> children[16]; 227 | 228 | static uint8_t flipSign(uint8_t keyByte) { 229 | // Flip the sign bit, enables signed SSE comparison of unsigned values, used by Node16 230 | return keyByte ^ 128; 231 | } 232 | 233 | static inline unsigned ctz(uint16_t x) { 234 | // Count trailing zeros, only defined for x>0 235 | #ifdef __GNUC__ 236 | return __builtin_ctz(x); 237 | #else 238 | // Adapted from Hacker's Delight 239 | unsigned n=1; 240 | if ((x&0xFF)==0) {n+=8; x=x>>8;} 241 | if ((x&0x0F)==0) {n+=4; x=x>>4;} 242 | if ((x&0x03)==0) {n+=2; x=x>>2;} 243 | return n-(x&1); 244 | #endif 245 | } 246 | 247 | std::atomic> *getChildPos(const uint8_t k); 248 | 249 | public: 250 | N16(uint32_t level, const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N16, level, prefix, 251 | prefixLength) { 252 | memset(keys, 0, sizeof(keys)); 253 | memset(children, 0, sizeof(children)); 254 | } 255 | 256 | N16(uint32_t level, const Prefix &prefi) : N(NTypes::N16, level, prefi) { 257 | memset(keys, 0, sizeof(keys)); 258 | memset(children, 0, sizeof(children)); 259 | } 260 | 261 | bool insert(uint8_t key, pptr n); 262 | bool insert(uint8_t key, pptr n, bool flush); 263 | 264 | template 265 | void copyTo(NODE *n) const; 266 | 267 | void change(uint8_t key, pptr val); 268 | 269 | N *getChild(const uint8_t k) const; 270 | pptr getChildPptr(const uint8_t k) const; 271 | 272 | bool remove(uint8_t k, bool force); 273 | 274 | N *getAnyChild() const; 275 | N *getAnyChildReverse() const; 276 | 277 | void deleteChildren(); 278 | 279 | void getChildren(uint8_t start, uint8_t end, std::tuple *&children, 280 | uint32_t &childrenCount) const; 281 | N *getSmallestChild(uint8_t start) const; 282 | N *getLargestChild(uint8_t end) const; 283 | }; 284 | 285 | class N48 : public N { 286 | std::atomic childIndex[256]; 287 | std::atomic> children[48]; 288 | //std::atomic children[48]; 289 | public: 290 | void *operator new(size_t size, void *location){ 291 | return location; 292 | } 293 | 294 | static const uint8_t emptyMarker = 48; 295 | 296 | N48(uint32_t level, const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N48, level, prefix, 297 | prefixLength) { 298 | memset(childIndex, emptyMarker, sizeof(childIndex)); 299 | memset(children, 0, sizeof(children)); 300 | } 301 | 302 | N48(uint32_t level, const Prefix &prefi) : N(NTypes::N48, level, prefi) { 303 | memset(childIndex, emptyMarker, sizeof(childIndex)); 304 | memset(children, 0, sizeof(children)); 305 | } 306 | 307 | bool insert(uint8_t key, pptr n); 308 | bool insert(uint8_t key, pptr n, bool flush); 309 | 310 | template 311 | void copyTo(NODE *n) const; 312 | 313 | void change(uint8_t key, pptr val); 314 | 315 | N *getChild(const uint8_t k) const; 316 | pptr getChildPptr(const uint8_t k) const; 317 | 318 | bool remove(uint8_t k, bool force); 319 | 320 | N *getAnyChild() const; 321 | N *getAnyChildReverse() const; 322 | 323 | void deleteChildren(); 324 | 325 | void getChildren(uint8_t start, uint8_t end, std::tuple *&children, 326 | uint32_t &childrenCount) const; 327 | N *getSmallestChild(uint8_t start) const; 328 | N *getLargestChild(uint8_t end) const; 329 | }; 330 | 331 | class N256 : public N { 332 | std::atomic> children[256]; 333 | 334 | public: 335 | void *operator new(size_t size, void *location){ 336 | return location; 337 | } 338 | 339 | N256(uint32_t level, const uint8_t *prefix, uint32_t prefixLength) : N(NTypes::N256, level, prefix, 340 | prefixLength) { 341 | memset(children, '\0', sizeof(children)); 342 | } 343 | 344 | N256(uint32_t level, const Prefix &prefi) : N(NTypes::N256, level, prefi) { 345 | memset(children, '\0', sizeof(children)); 346 | } 347 | 348 | bool insert(uint8_t key, pptr val); 349 | bool insert(uint8_t key, pptr val, bool flush); 350 | 351 | template 352 | void copyTo(NODE *n) const; 353 | 354 | void change(uint8_t key, pptr n); 355 | 356 | N *getChild(const uint8_t k) const; 357 | pptr getChildPptr(const uint8_t k) const; 358 | 359 | bool remove(uint8_t k, bool force); 360 | 361 | N *getAnyChild() const; 362 | N *getAnyChildReverse() const; 363 | 364 | void deleteChildren(); 365 | 366 | void getChildren(uint8_t start, uint8_t end, std::tuple *&children, 367 | uint32_t &childrenCount) const; 368 | N *getSmallestChild(uint8_t start) const; 369 | N *getLargestChild(uint8_t end) const; 370 | }; 371 | } 372 | #endif //ART_ROWEX_N_H 373 | -------------------------------------------------------------------------------- /lib/PDL-ART/N16.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | #include // x86 SSE intrinsics 5 | 6 | namespace ART_ROWEX { 7 | 8 | bool N16::insert(uint8_t key, pptr n) { 9 | return insert(key,n,true); 10 | } 11 | bool N16::insert(uint8_t key, pptr n, bool flush) { 12 | uint16_t compactCount = (uint16_t)(countValues>>16); 13 | if (compactCount == 16) { 14 | return false; 15 | } 16 | 17 | n.markDirty(); //DL 18 | children[compactCount].store(n, std::memory_order_release); 19 | //TODO 20 | if(flush){ 21 | flushToNVM((char*)&children[compactCount],sizeof(pptr)); 22 | smp_wmb(); 23 | } 24 | 25 | keys[compactCount].store(flipSign(key), std::memory_order_release); 26 | uint32_t increaseCountValues = (1<<16)+1; 27 | countValues+=increaseCountValues; // visible point 28 | //compactCount++; 29 | //count++; 30 | if(flush){ 31 | flushToNVM((char*)this,L1_CACHE_BYTES); 32 | smp_wmb(); 33 | } 34 | n.markClean(); //DL 35 | children[compactCount].store(n, std::memory_order_release); 36 | return true; 37 | } 38 | 39 | template 40 | void N16::copyTo(NODE *n) const { 41 | uint16_t compactCount = (uint16_t)(countValues>>16); 42 | for (unsigned i = 0; i < compactCount; i++) { 43 | pptr child = children[i].load(); 44 | N *rawChild = child.getVaddr(); 45 | if (rawChild != nullptr) { 46 | n->insert(flipSign(keys[i]), child,false); 47 | } 48 | } 49 | } 50 | 51 | void N16::change(uint8_t key, pptr val) { 52 | auto childPos = getChildPos(key); 53 | assert(childPos != nullptr); 54 | val.markDirty(); //DL 55 | childPos->store(val, std::memory_order_release); 56 | flushToNVM((char*)childPos,sizeof(pptr)); 57 | smp_wmb(); 58 | val.markClean(); //DL 59 | //DL 60 | childPos->store(val, std::memory_order_release); 61 | } 62 | 63 | std::atomic> *N16::getChildPos(const uint8_t k) { 64 | uint16_t compactCount = (uint16_t)(countValues>>16); 65 | __m128i cmp = _mm_cmpeq_epi8(_mm_set1_epi8(flipSign(k)), 66 | _mm_loadu_si128(reinterpret_cast(keys))); 67 | unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << compactCount) - 1); 68 | while (bitfield) { 69 | uint8_t pos = ctz(bitfield); 70 | pptr child = children[pos].load(); 71 | N *rawChild = child.getVaddr(); 72 | 73 | if (rawChild != nullptr) { 74 | return &children[pos]; 75 | } 76 | bitfield = bitfield ^ (1 << pos); 77 | } 78 | return nullptr; 79 | } 80 | 81 | pptr N16::getChildPptr(const uint8_t k) const { 82 | __m128i cmp = _mm_cmpeq_epi8(_mm_set1_epi8(flipSign(k)), 83 | _mm_loadu_si128(reinterpret_cast(keys))); 84 | unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << 16) - 1); 85 | while (bitfield) { 86 | uint8_t pos = ctz(bitfield); 87 | pptr child = children[pos].load(); 88 | while(child.isDirty()){ //DL 89 | child = children[pos].load(); 90 | } 91 | N *rawChild = child.getVaddr(); 92 | if (rawChild != nullptr && keys[pos].load() == flipSign(k)) { 93 | return child; 94 | } 95 | bitfield = bitfield ^ (1 << pos); 96 | } 97 | 98 | pptr nullPtr(0,0); 99 | return nullPtr; 100 | } 101 | 102 | N *N16::getChild(const uint8_t k) const { 103 | __m128i cmp = _mm_cmpeq_epi8(_mm_set1_epi8(flipSign(k)), 104 | _mm_loadu_si128(reinterpret_cast(keys))); 105 | unsigned bitfield = _mm_movemask_epi8(cmp) & ((1 << 16) - 1); 106 | while (bitfield) { 107 | uint8_t pos = ctz(bitfield); 108 | pptr child = children[pos].load(); 109 | while(child.isDirty()){ //DL 110 | child = children[pos].load(); 111 | } 112 | N *rawChild = child.getVaddr(); 113 | if (rawChild != nullptr && keys[pos].load() == flipSign(k)) { 114 | return rawChild; 115 | } 116 | bitfield = bitfield ^ (1 << pos); 117 | } 118 | return nullptr; 119 | } 120 | 121 | bool N16::remove(uint8_t k, bool force) { 122 | uint16_t count = (uint16_t)countValues; 123 | if (count == 3 && !force) { 124 | return false; 125 | } 126 | auto leafPlace = getChildPos(k); 127 | assert(leafPlace != nullptr); 128 | pptr nullPtr(0,0); 129 | nullPtr.markDirty(); 130 | leafPlace->store(nullPtr, std::memory_order_release); 131 | flushToNVM((char*)leafPlace,sizeof(pptr)); 132 | smp_wmb(); 133 | // count--; 134 | uint32_t decreaseCountValues = (1<<16); 135 | countValues-=decreaseCountValues; // visible point 136 | flushToNVM((char*)this,L1_CACHE_BYTES); 137 | smp_wmb(); 138 | assert(getChild(k) == nullptr); 139 | nullPtr.markClean(); 140 | leafPlace->store(nullPtr, std::memory_order_release); 141 | return true; 142 | } 143 | 144 | N *N16::getAnyChild() const { 145 | N *anyChild = nullptr; 146 | for (int i = 0; i < 16; ++i) { 147 | pptr child = children[i].load(); 148 | while(child.isDirty()){ //DL 149 | child = children[i].load(); 150 | } 151 | N *rawChild = child.getVaddr(); 152 | if (rawChild != nullptr) { 153 | if (N::isLeaf(rawChild)) { 154 | return rawChild; 155 | } 156 | anyChild = rawChild; 157 | } 158 | } 159 | return anyChild; 160 | } 161 | 162 | N *N16::getAnyChildReverse() const { 163 | N *anyChild = nullptr; 164 | for (int i = 15; i >= 0; --i) { 165 | pptr child = children[i].load(); 166 | while(child.isDirty()){ //DL 167 | child = children[i].load(); 168 | } 169 | N *rawChild = child.getVaddr(); 170 | if (rawChild != nullptr) { 171 | if (N::isLeaf(rawChild)) { 172 | return rawChild; 173 | } 174 | anyChild = rawChild; 175 | } 176 | } 177 | return anyChild; 178 | } 179 | 180 | void N16::deleteChildren() { 181 | uint16_t compactCount = (uint16_t)(countValues>>16); 182 | for (std::size_t i = 0; i < compactCount; ++i) { 183 | pptr child = children[i].load(); 184 | N *rawChild = child.getVaddr(); 185 | if (rawChild != nullptr) { 186 | N::deleteChildren(rawChild); 187 | N::deleteNode(rawChild); 188 | } 189 | } 190 | } 191 | 192 | void N16::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 193 | uint32_t &childrenCount) const { 194 | childrenCount = 0; 195 | uint16_t compactCount = (uint16_t)(countValues>>16); 196 | for (int i = 0; i < compactCount; ++i) { 197 | uint8_t key = flipSign(this->keys[i]); 198 | if (key >= start && key <= end) { 199 | pptr child = this->children[i].load(); 200 | while(child.isDirty()){ //DL 201 | child = this->children[i].load(); 202 | } 203 | N *rawChild = child.getVaddr(); 204 | if (rawChild != nullptr) { 205 | children[childrenCount] = std::make_tuple(key, rawChild); 206 | childrenCount++; 207 | } 208 | } 209 | } 210 | std::sort(children, children + childrenCount, [](auto &first, auto &second) { 211 | return std::get<0>(first) < std::get<0>(second); 212 | }); 213 | } 214 | N *N16::getSmallestChild(uint8_t start) const { 215 | N *smallestChild = nullptr; 216 | uint8_t minKey = 255; 217 | uint16_t compactCount = (uint16_t)(countValues>>16); 218 | for (int i = 0; i < compactCount; ++i) { 219 | uint8_t key = flipSign(this->keys[i]); 220 | if (key >= start && key <= minKey) { 221 | pptr child = this->children[i].load(); 222 | while(child.isDirty()){ //DL 223 | child = this->children[i].load(); 224 | } 225 | N *rawChild = child.getVaddr(); 226 | if (rawChild != nullptr) { 227 | minKey = key; 228 | smallestChild = rawChild; 229 | } 230 | } 231 | } 232 | return smallestChild; 233 | } 234 | N *N16::getLargestChild(uint8_t end) const { 235 | N *largestChild = nullptr; 236 | uint8_t maxKey = 0; 237 | uint16_t compactCount = (uint16_t)(countValues>>16); 238 | for (int i = 0; i < compactCount; ++i) { 239 | uint8_t key = flipSign(this->keys[i]); 240 | if (key <= end && key >= maxKey) { 241 | pptr child = this->children[i].load(); 242 | while(child.isDirty()){ //DL 243 | child = children[i].load(); 244 | } 245 | N *rawChild = child.getVaddr(); 246 | if (rawChild != nullptr) { 247 | maxKey = key; 248 | largestChild = rawChild; 249 | } 250 | } 251 | } 252 | return largestChild; 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /lib/PDL-ART/N256.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_ROWEX { 6 | 7 | void N256::deleteChildren() { 8 | for (uint64_t i = 0; i < 256; ++i) { 9 | pptr child = children[i].load(); 10 | N *rawChild = child.getVaddr(); 11 | if (rawChild != nullptr) { 12 | N::deleteChildren(rawChild); 13 | N::deleteNode(rawChild); 14 | } 15 | } 16 | } 17 | 18 | bool N256::insert(uint8_t key, pptr val) { 19 | return insert(key,val,true); 20 | } 21 | bool N256::insert(uint8_t key, pptr val,bool flush) { 22 | //uint16_t count = (uint16_t)countValues; 23 | val.markDirty(); //DL 24 | children[key].store(val, std::memory_order_release); 25 | if(flush){ 26 | flushToNVM((char*)&children[key],sizeof(pptr)); 27 | smp_wmb(); 28 | } 29 | //count++; 30 | uint32_t increaseCountValues = (1<<16); 31 | countValues+=increaseCountValues; // visible point 32 | if(flush){ 33 | flushToNVM((char*)this,L1_CACHE_BYTES); 34 | smp_wmb(); 35 | } 36 | val.markClean(); //DL 37 | children[key].store(val, std::memory_order_release); 38 | return true; 39 | } 40 | 41 | template 42 | void N256::copyTo(NODE *n) const { 43 | for (int i = 0; i < 256; ++i) { 44 | pptr child = children[i].load(); 45 | N *rawChild = child.getVaddr(); 46 | if (rawChild != nullptr) { 47 | n->insert(i, child,false); 48 | } 49 | } 50 | } 51 | 52 | void N256::change(uint8_t key, pptr n) { 53 | n.markDirty(); //DL 54 | children[key].store(n, std::memory_order_release); 55 | flushToNVM((char*)&children[key],sizeof(pptr)); 56 | smp_wmb(); 57 | n.markClean(); //DL 58 | children[key].store(n, std::memory_order_release); 59 | } 60 | 61 | pptr N256::getChildPptr(const uint8_t k) const { 62 | pptr child = children[k].load(); 63 | while(child.isDirty()){ //DL 64 | child = children[k].load(); 65 | } 66 | return child; 67 | } 68 | 69 | N *N256::getChild(const uint8_t k) const { 70 | pptr child = children[k].load(); 71 | while(child.isDirty()){ //DL 72 | child = children[k].load(); 73 | } 74 | N *rawChild = child.getVaddr(); 75 | return rawChild; 76 | } 77 | 78 | bool N256::remove(uint8_t k, bool force) { 79 | uint16_t count = (uint16_t)countValues; 80 | if (count == 37 && !force) { 81 | return false; 82 | } 83 | pptr nullPtr(0,0); 84 | nullPtr.markDirty(); //DL 85 | children[k].store(nullPtr, std::memory_order_release); 86 | flushToNVM((char*)&children[k],sizeof(pptr)); 87 | smp_wmb(); 88 | //count--; 89 | uint32_t decreaseCountValues = (1<<16); 90 | countValues-=decreaseCountValues; // visible point 91 | flushToNVM((char*)this,L1_CACHE_BYTES); 92 | smp_wmb(); 93 | nullPtr.markClean(); //DL 94 | children[k].store(nullPtr, std::memory_order_release); 95 | return true; 96 | } 97 | 98 | N *N256::getAnyChild() const { 99 | N *anyChild = nullptr; 100 | for (uint64_t i = 0; i < 256; ++i) { 101 | pptr child = children[i].load(); 102 | N *rawChild = child.getVaddr(); 103 | if (rawChild != nullptr) { 104 | if (N::isLeaf(rawChild)) { 105 | return rawChild; 106 | } else { 107 | anyChild = rawChild; 108 | } 109 | } 110 | } 111 | return anyChild; 112 | } 113 | 114 | N *N256::getAnyChildReverse() const { 115 | N *anyChild = nullptr; 116 | for (int i = 255; i >= 0; --i) { 117 | pptr child = children[i].load(); 118 | N *rawChild = child.getVaddr(); 119 | if (rawChild != nullptr) { 120 | if (N::isLeaf(rawChild)) { 121 | return rawChild; 122 | } else { 123 | anyChild = rawChild; 124 | } 125 | } 126 | } 127 | return anyChild; 128 | } 129 | 130 | void N256::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 131 | uint32_t &childrenCount) const { 132 | childrenCount = 0; 133 | for (unsigned i = start; i <= end; i++) { 134 | pptr child = this->children[i].load(); 135 | N *rawChild = child.getVaddr(); 136 | if (rawChild != nullptr) { 137 | children[childrenCount] = std::make_tuple(i, rawChild); 138 | childrenCount++; 139 | } 140 | } 141 | } 142 | 143 | N *N256::getSmallestChild(uint8_t start) const { 144 | N *smallestChild = nullptr; 145 | for (int i = start; i < 256; ++i) { 146 | pptr child = this->children[i].load(); 147 | N *rawChild = child.getVaddr(); 148 | if (rawChild != nullptr) { 149 | return rawChild; 150 | } 151 | } 152 | return smallestChild; 153 | } 154 | N *N256::getLargestChild(uint8_t end) const { 155 | N *largestChild = nullptr; 156 | for (int i = end; i >= 0; --i) { 157 | pptr child = this->children[i].load(); 158 | while(child.isDirty()){ //DL 159 | child = children[i].load(); 160 | } 161 | N *rawChild = child.getVaddr(); 162 | if (rawChild != nullptr) { 163 | return rawChild; 164 | } 165 | } 166 | return largestChild; 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /lib/PDL-ART/N4.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_ROWEX { 6 | 7 | 8 | void N4::deleteChildren() { 9 | uint16_t compactCount = (uint16_t)(countValues>>16); 10 | for (uint32_t i = 0; i < compactCount; ++i) { 11 | pptr child = children[i].load(); 12 | N* rawChild = child.getVaddr(); 13 | if (rawChild != nullptr) { 14 | N::deleteChildren(rawChild); 15 | N::deleteNode(rawChild); 16 | } 17 | } 18 | } 19 | 20 | bool N4::insert(uint8_t key, pptr n) { 21 | return insert(key,n,true); 22 | } 23 | 24 | bool N4::insert(uint8_t key, pptr n, bool flush) { 25 | uint16_t compactCount = (uint16_t)(countValues>>16); 26 | if (compactCount == 4) { 27 | return false; 28 | } 29 | keys[compactCount].store(key, std::memory_order_release); 30 | n.markDirty(); 31 | children[compactCount].store(n, std::memory_order_release); //visible point for insert function 32 | // idxPtr childPtr = children[compactCount].load(); 33 | uint32_t increaseCountValues = (1<<16)+1; 34 | countValues+=increaseCountValues; // visible point 35 | if(flush){ 36 | flushToNVM((char*)this, sizeof(N4)); 37 | smp_wmb(); 38 | } 39 | n.markClean(); 40 | children[compactCount].store(n, std::memory_order_release); //visible point for insert function 41 | 42 | //compactCount++; 43 | //count++; 44 | return true; 45 | } 46 | 47 | template 48 | void N4::copyTo(NODE *n) const { 49 | uint16_t compactCount = (uint16_t)(countValues>>16); 50 | for (uint32_t i = 0; i < compactCount; ++i) { 51 | pptr child = children[i].load(); 52 | N *rawChild = child.getVaddr(); 53 | if (rawChild != nullptr) { 54 | n->insert(keys[i].load(), child, false); //no flush 55 | } 56 | } 57 | } 58 | 59 | void N4::change(uint8_t key, pptr val) { 60 | uint16_t compactCount = (uint16_t)(countValues>>16); 61 | for (uint32_t i = 0; i < compactCount; ++i) { 62 | pptr child = children[i].load(); 63 | N *rawChild = child.getVaddr(); 64 | if (rawChild != nullptr && keys[i].load() == key) { 65 | //DL 66 | val.markDirty(); 67 | children[i].store(val, std::memory_order_release); 68 | flushToNVM((char*)&children[i],sizeof(pptr)); 69 | smp_wmb(); 70 | //DL 71 | val.markClean(); 72 | //DL 73 | children[i].store(val, std::memory_order_release); 74 | return; 75 | } 76 | } 77 | assert(false); 78 | __builtin_unreachable(); 79 | } 80 | pptr N4::getChildPptr(const uint8_t k) const { 81 | for (uint32_t i = 0; i < 4; ++i) { 82 | pptr child = children[i].load(); 83 | while(child.isDirty()){ //DL 84 | child = children[i].load(); 85 | } 86 | N *rawChild = child.getVaddr(); 87 | if (rawChild != nullptr && keys[i].load() == k) { 88 | return child; 89 | } 90 | } 91 | pptr nullPtr(0,0); 92 | return nullPtr; 93 | } 94 | 95 | N *N4::getChild(const uint8_t k) const { 96 | for (uint32_t i = 0; i < 4; ++i) { 97 | pptr child = children[i].load(); 98 | while(child.isDirty()){ //DL 99 | child = children[i].load(); 100 | } 101 | N *rawChild = child.getVaddr(); 102 | if (rawChild != nullptr && keys[i].load() == k) { 103 | return rawChild; 104 | } 105 | } 106 | return nullptr; 107 | } 108 | 109 | bool N4::remove(uint8_t k, bool /*force*/) { 110 | uint16_t compactCount = (uint16_t)(countValues>>16); 111 | for (uint32_t i = 0; i < compactCount; ++i) { 112 | pptr child = children[i].load(); 113 | N *rawChild = child.getVaddr(); 114 | if (rawChild != nullptr && keys[i].load() == k) { 115 | uint32_t decreaseCountValues = (1<<16); 116 | countValues-=decreaseCountValues; // visible point 117 | pptr nullPtr(0,0); 118 | nullPtr.markDirty(); 119 | children[i].store(nullPtr, std::memory_order_release); 120 | flushToNVM((char*)&children[i],sizeof(pptr)); 121 | smp_wmb(); 122 | nullPtr.markClean(); 123 | children[i].store(nullPtr, std::memory_order_release); 124 | return true; 125 | } 126 | } 127 | assert(false); 128 | __builtin_unreachable(); 129 | } 130 | 131 | N *N4::getAnyChild() const { 132 | N *anyChild = nullptr; 133 | for (uint32_t i = 0; i < 4; ++i) { 134 | pptr child = children[i].load(); 135 | while(child.isDirty()){ //DL 136 | child = children[i].load(); 137 | } 138 | N *rawChild = child.getVaddr(); 139 | if (rawChild != nullptr) { 140 | if (N::isLeaf(rawChild)) { 141 | return rawChild; 142 | } 143 | anyChild = rawChild; 144 | } 145 | } 146 | return anyChild; 147 | } 148 | 149 | N *N4::getAnyChildReverse() const { 150 | N *anyChild = nullptr; 151 | for (int i = 3; i >= 0; --i) { 152 | pptr child = children[i].load(); 153 | while(child.isDirty()){ //DL 154 | child = children[i].load(); 155 | } 156 | N *rawChild = child.getVaddr(); 157 | if (rawChild != nullptr) { 158 | if (N::isLeaf(rawChild)) { 159 | return rawChild; 160 | } 161 | anyChild = rawChild; 162 | } 163 | } 164 | return anyChild; 165 | } 166 | 167 | std::tuple N4::getSecondChild(const uint8_t key) const { 168 | uint16_t compactCount = (uint16_t)(countValues>>16); 169 | for (uint32_t i = 0; i < compactCount; ++i) { 170 | pptr child = children[i].load(); 171 | while(child.isDirty()){ //DL 172 | child = children[i].load(); 173 | } 174 | N *rawChild = child.getVaddr(); 175 | if (rawChild != nullptr) { 176 | uint8_t k = keys[i].load(); 177 | if (k != key){ 178 | return std::make_tuple(rawChild, k); 179 | } 180 | } 181 | } 182 | return std::make_tuple(nullptr, 0); 183 | } 184 | 185 | void N4::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 186 | uint32_t &childrenCount) const { 187 | childrenCount = 0; 188 | for (uint32_t i = 0; i < 4; ++i) { 189 | uint8_t key = this->keys[i].load(); 190 | if (key >= start && key <= end) { 191 | pptr child = this->children[i].load(); 192 | while(child.isDirty()){ //DL 193 | child = this->children[i].load(); 194 | } 195 | N *rawChild = child.getVaddr(); 196 | if (rawChild != nullptr) { 197 | children[childrenCount] = std::make_tuple(key, rawChild); 198 | childrenCount++; 199 | } 200 | } 201 | } 202 | std::sort(children, children + childrenCount, [](auto &first, auto &second) { 203 | return std::get<0>(first) < std::get<0>(second); 204 | }); 205 | } 206 | N *N4::getSmallestChild(uint8_t start) const { 207 | N *smallestChild = nullptr; 208 | uint8_t minKey = 255; 209 | for (uint32_t i = 0; i < 4; ++i) { 210 | uint8_t key = this->keys[i].load(); 211 | if (key >= start && key <= minKey) { 212 | pptr child = children[i].load(); 213 | while(child.isDirty()){ //DL 214 | child = children[i].load(); 215 | } 216 | N *rawChild = child.getVaddr(); 217 | if (rawChild != nullptr) { 218 | minKey = key; 219 | smallestChild = rawChild; 220 | } 221 | } 222 | } 223 | return smallestChild; 224 | } 225 | N *N4::getLargestChild(uint8_t end) const { 226 | N *largestChild = nullptr; 227 | uint8_t maxKey = 0; 228 | for (uint32_t i = 0; i < 4; ++i) { 229 | uint8_t key = this->keys[i].load(); 230 | if (key <= end && key >= maxKey) { 231 | pptr child = children[i].load(); 232 | while(child.isDirty()){ //DL 233 | child = children[i].load(); 234 | } 235 | N *rawChild = child.getVaddr(); 236 | if (rawChild != nullptr) { 237 | maxKey = key; 238 | largestChild = rawChild; 239 | } 240 | } 241 | } 242 | return largestChild; 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /lib/PDL-ART/N48.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "N.h" 4 | 5 | namespace ART_ROWEX { 6 | 7 | bool N48::insert(uint8_t key, pptr n) { 8 | return insert(key,n,true); 9 | } 10 | bool N48::insert(uint8_t key, pptr n, bool flush) { 11 | uint16_t compactCount = (uint16_t)(countValues>>16); 12 | if (compactCount == 48) { 13 | return false; 14 | } 15 | n.markDirty(); //DL 16 | children[compactCount].store(n, std::memory_order_release); 17 | if(flush) flushToNVM((char*)&children[compactCount],sizeof(pptr)); 18 | childIndex[key].store(compactCount, std::memory_order_release); 19 | if(flush){ 20 | flushToNVM((char*)&childIndex[key],sizeof(uint8_t)); 21 | smp_wmb(); 22 | } 23 | uint32_t increaseCountValues = (1<<16)+1; 24 | countValues+=increaseCountValues; // visible point 25 | if(flush){ 26 | flushToNVM((char*)this,L1_CACHE_BYTES); 27 | smp_wmb(); 28 | } 29 | n.markClean(); //DL 30 | children[compactCount].store(n, std::memory_order_release); 31 | //compactCount++; 32 | //count++; 33 | return true; 34 | } 35 | 36 | template 37 | void N48::copyTo(NODE *n) const { 38 | for (unsigned i = 0; i < 256; i++) { 39 | uint8_t index = childIndex[i].load(); 40 | if (index != emptyMarker) { 41 | n->insert(i, children[index],false); 42 | } 43 | } 44 | } 45 | 46 | void N48::change(uint8_t key, pptr val) { 47 | uint8_t index = childIndex[key].load(); 48 | assert(index != emptyMarker); 49 | val.markDirty(); //DL 50 | children[index].store(val, std::memory_order_release); 51 | flushToNVM((char*)&children[index],sizeof(pptr)); 52 | smp_wmb(); 53 | val.markClean(); //DL 54 | children[index].store(val, std::memory_order_release); 55 | return; 56 | } 57 | 58 | pptr N48::getChildPptr(const uint8_t k) const { 59 | uint8_t index = childIndex[k].load(); 60 | if (index == emptyMarker) { 61 | pptr nullPtr(0,0); 62 | return nullPtr; 63 | } else { 64 | pptr child = children[index].load(); 65 | while(child.isDirty()){ //DL 66 | child = children[index].load(); 67 | } 68 | return child; 69 | //return children[index].load(); 70 | } 71 | } 72 | 73 | 74 | 75 | N *N48::getChild(const uint8_t k) const { 76 | uint8_t index = childIndex[k].load(); 77 | if (index == emptyMarker) { 78 | return nullptr; 79 | } else { 80 | pptr child = children[index].load(); 81 | while(child.isDirty()){ //DL 82 | child = children[index].load(); 83 | } 84 | N *rawChild = child.getVaddr(); 85 | return rawChild; 86 | //return children[index].load(); 87 | } 88 | } 89 | 90 | bool N48::remove(uint8_t k, bool force) { 91 | uint16_t count = (uint16_t)countValues; 92 | if (count == 12 && !force) { 93 | return false; 94 | } 95 | assert(childIndex[k] != emptyMarker); 96 | pptr nullPtr(0,0); 97 | nullPtr.markDirty(); 98 | children[childIndex[k]].store(nullPtr, std::memory_order_release); 99 | flushToNVM((char*)&children[childIndex[k]],sizeof(pptr)); 100 | childIndex[k].store(emptyMarker, std::memory_order_release); 101 | flushToNVM((char*)&childIndex[k],sizeof(uint8_t)); 102 | smp_wmb(); 103 | //count--; 104 | uint32_t decreaseCountValues = 1<<16; 105 | countValues-=decreaseCountValues; // visible point 106 | flushToNVM((char*)this,L1_CACHE_BYTES); 107 | smp_wmb(); 108 | assert(getChild(k) == nullptr); 109 | nullPtr.markClean(); 110 | children[childIndex[k]].store(nullPtr, std::memory_order_release); 111 | return true; 112 | } 113 | 114 | N *N48::getAnyChild() const { 115 | N *anyChild = nullptr; 116 | for (unsigned i = 0; i < 48; i++) { 117 | pptr child = children[i].load(); 118 | N *rawChild = child.getVaddr(); 119 | if (rawChild != nullptr) { 120 | if (N::isLeaf(rawChild)) { 121 | return rawChild; 122 | } 123 | anyChild = rawChild; 124 | } 125 | } 126 | return anyChild; 127 | } 128 | 129 | N *N48::getAnyChildReverse() const { 130 | N *anyChild = nullptr; 131 | for (int i = 47; i >= 0; i--) { 132 | pptr child = children[i].load(); 133 | while(child.isDirty()){ //DL 134 | child = children[i].load(); 135 | } 136 | N *rawChild = child.getVaddr(); 137 | if (rawChild != nullptr) { 138 | if (N::isLeaf(rawChild)) { 139 | return rawChild; 140 | } 141 | anyChild = rawChild; 142 | } 143 | } 144 | return anyChild; 145 | } 146 | 147 | void N48::deleteChildren() { 148 | for (unsigned i = 0; i < 256; i++) { 149 | if (childIndex[i] != emptyMarker) { 150 | pptr child = children[i].load(); 151 | N *rawChild = child.getVaddr(); 152 | N::deleteChildren(rawChild); 153 | N::deleteNode(rawChild); 154 | } 155 | } 156 | } 157 | 158 | void N48::getChildren(uint8_t start, uint8_t end, std::tuple *&children, 159 | uint32_t &childrenCount) const { 160 | childrenCount = 0; 161 | for (unsigned i = start; i <= end; i++) { 162 | uint8_t index = this->childIndex[i].load(); 163 | if (index != emptyMarker) { 164 | pptr child = this->children[index].load(); 165 | while(child.isDirty()){ //DL 166 | child = this->children[index].load(); 167 | } 168 | N *rawChild = child.getVaddr(); 169 | if (rawChild != nullptr) { 170 | children[childrenCount] = std::make_tuple(i, rawChild); 171 | childrenCount++; 172 | } 173 | } 174 | } 175 | } 176 | 177 | N *N48::getSmallestChild(uint8_t start) const { 178 | N *smallestChild = nullptr; 179 | for (int i = start; i < 256; ++i) { 180 | uint8_t index = this->childIndex[i].load(); 181 | if (index != emptyMarker) { 182 | pptr child = children[index].load(); 183 | while(child.isDirty()){ //DL 184 | child = children[index].load(); 185 | } 186 | N *rawChild = child.getVaddr(); 187 | if (rawChild != nullptr) { 188 | return rawChild; 189 | } 190 | } 191 | } 192 | return smallestChild; 193 | } 194 | N *N48::getLargestChild(uint8_t end) const { 195 | N *largestChild = nullptr; 196 | for (int i = end; i >= 0; --i) { 197 | uint8_t index = this->childIndex[i].load(); 198 | if (index != emptyMarker) { 199 | pptr child = children[index].load(); 200 | while(child.isDirty()){ //DL 201 | child = children[index].load(); 202 | } 203 | N *rawChild = child.getVaddr(); 204 | if (rawChild != nullptr) { 205 | return rawChild; 206 | } 207 | } 208 | } 209 | return largestChild; 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /lib/PDL-ART/Tree.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by florian on 18.11.15. 3 | // 4 | 5 | #ifndef ART_ROWEX_TREE_H 6 | #define ART_ROWEX_TREE_H 7 | #include "N.h" 8 | #include "Key.h" 9 | #include "Epoche.h" 10 | 11 | 12 | using namespace ART; 13 | 14 | namespace ART_ROWEX { 15 | 16 | class Tree { 17 | public: 18 | using LoadKeyFunction = void (*)(TID tid, Key &key); 19 | uint64_t genId; 20 | 21 | private: 22 | pptr root; 23 | //N *const root; 24 | 25 | TID checkKey(const TID tid, const Key &k) const; 26 | 27 | LoadKeyFunction loadKey; 28 | 29 | Epoche epoche{256}; 30 | OpStruct oplogs[10000]; 31 | //OpStruct oplogs[10000000]; 32 | 33 | // OpStruct oplogs[100000]; 34 | // std::atomic oplogsCount{0}; 35 | 36 | uint64_t oplogsCount; 37 | 38 | // static mappingTable mtable; 39 | public: 40 | /* void *operator new(size_t size){ 41 | return allocNVM(size); 42 | } 43 | 44 | void operator delete(void *ptr){ 45 | return freeNVM(ptr); 46 | }*/ 47 | 48 | 49 | 50 | enum class CheckPrefixResult : uint8_t { 51 | Match, 52 | NoMatch, 53 | OptimisticMatch 54 | }; 55 | 56 | enum class CheckPrefixPessimisticResult : uint8_t { 57 | Match, 58 | NoMatch, 59 | SkippedLevel 60 | }; 61 | 62 | enum class PCCompareResults : uint8_t { 63 | Smaller, 64 | Equal, 65 | Bigger, 66 | SkippedLevel 67 | }; 68 | enum class PCEqualsResults : uint8_t { 69 | PartialMatch, 70 | BothMatch, 71 | Contained, 72 | NoMatch, 73 | SkippedLevel 74 | }; 75 | static CheckPrefixResult checkPrefix(N* n, const Key &k, uint32_t &level); 76 | 77 | static CheckPrefixPessimisticResult checkPrefixPessimistic(N *n, const Key &k, uint32_t &level, 78 | uint8_t &nonMatchingKey, 79 | Prefix &nonMatchingPrefix, 80 | LoadKeyFunction loadKey); 81 | 82 | static PCCompareResults checkPrefixCompare(const N* n, const Key &k, uint32_t &level, LoadKeyFunction loadKey); 83 | 84 | static PCEqualsResults checkPrefixEquals(const N* n, uint32_t &level, const Key &start, const Key &end, LoadKeyFunction loadKey); 85 | 86 | public: 87 | 88 | void recover(); 89 | Tree(LoadKeyFunction loadKey); 90 | 91 | Tree(const Tree &) = delete; 92 | 93 | Tree(Tree &&t) : root(t.root), loadKey(t.loadKey) { } 94 | 95 | ~Tree(); 96 | 97 | ThreadInfo getThreadInfo() ; 98 | 99 | TID lookup(const Key &k, ThreadInfo &threadEpocheInfo) const; 100 | 101 | TID lookupNext(const Key &k, ThreadInfo &threadEpocheInfo) const; 102 | 103 | void insert(const Key &k, TID tid, ThreadInfo &epocheInfo); 104 | void remove(const Key &k, TID tid, ThreadInfo &epocheInfo); 105 | 106 | #ifdef SYNC 107 | TID lookupNextwithLock(const Key &start, void** savenode, ThreadInfo &threadEpocheInfo) const; 108 | bool nodeUnlock(void* savenode, ThreadInfo &threadEpocheInfo) const; 109 | bool insert(const Key &k, TID tid, ThreadInfo &epocheInfo, void *locked_node); 110 | #endif 111 | 112 | 113 | 114 | bool isEmpty(); 115 | }; 116 | 117 | } 118 | #endif //ART_ROWEX_TREE_H 119 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file (GLOB pactree_SRCS "*.h" "*.cpp") 2 | add_library(pactree STATIC 3 | ${pactree_SRCS} 4 | ) 5 | target_include_directories(pactree PUBLIC 6 | ${CMAKE_CURRENT_SOURCE_DIR} 7 | ) 8 | add_executable (main main.cpp) 9 | target_link_libraries (main pactree numa jemalloc pdlart pmemobj pmem) 10 | INSTALL(TARGETS pactree 11 | ARCHIVE DESTINATION ${CMAKE_SOURCE_DIR} 12 | ) 13 | -------------------------------------------------------------------------------- /src/Combiner.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef pactree_COMBINER_H 5 | #define pactree_COMBINER_H 6 | #include "Oplog.h" 7 | #include "WorkerThread.h" 8 | #include 9 | #include 10 | 11 | 12 | class CombinerThread { 13 | private: 14 | std::queue*>> logQueue; 15 | unsigned long doneCountCombiner; 16 | 17 | public: 18 | CombinerThread () { doneCountCombiner = 0; } 19 | std::vector* combineLogs() { 20 | std::atomic_fetch_add(&curQ, 1ul); 21 | int qnum = static_cast((curQ - 1) % 2); 22 | auto mergedLog = new std::vector; 23 | for (auto &i : g_perThreadLog) { 24 | Oplog &log = *i; 25 | log.lock(qnum); 26 | auto op_ = log.getQ(qnum); 27 | if(!op_->empty()) 28 | mergedLog->insert(std::end(*mergedLog), std::begin(*op_), std::end(*op_)); 29 | log.resetQ(qnum); 30 | log.unlock(qnum); 31 | } 32 | std::sort(mergedLog->begin(), mergedLog->end()); 33 | combinerSplits +=mergedLog->size(); 34 | if (!mergedLog->empty()) { 35 | logQueue.push(make_pair(doneCountCombiner, mergedLog)); 36 | doneCountCombiner++; 37 | return mergedLog; 38 | } else { 39 | delete mergedLog; 40 | return nullptr; 41 | } 42 | } 43 | 44 | void broadcastMergedLog(std::vector* mergedLog, int activeNuma) { 45 | for(auto i = 0; i < activeNuma * WORKER_THREAD_PER_NUMA; i++) 46 | g_workQueue[i].push(mergedLog); 47 | } 48 | 49 | uint64_t freeMergedLogs(int activeNuma, bool force) { 50 | if (!force && logQueue.size() < 100) return 0; 51 | 52 | unsigned long minDoneCountWt = ULONG_MAX; 53 | for (auto i = 0; i < activeNuma * WORKER_THREAD_PER_NUMA; i++) { 54 | unsigned long logDoneCount = g_WorkerThreadInst[i]->getLogDoneCount(); 55 | if (logDoneCount < minDoneCountWt) 56 | minDoneCountWt = logDoneCount; 57 | } 58 | 59 | while (!logQueue.empty() && logQueue.front().first < minDoneCountWt) { 60 | auto mergedLog = logQueue.front().second; 61 | for (auto opsPtr : *mergedLog){ 62 | // PMEMoid ptr = pmemobj_oid(opsPtr); 63 | // pmemobj_free(&ptr); 64 | // ptr = pmemobj_oid(mergedLog); 65 | // pmemobj_free(&ptr); 66 | } 67 | // delete opsPtr; 68 | 69 | delete mergedLog; 70 | logQueue.pop(); 71 | } 72 | return minDoneCountWt; 73 | } 74 | 75 | bool mergedLogsToBeFreed() { return logQueue.empty(); } 76 | }; 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /src/Oplog.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "Oplog.h" 5 | #include "ordo_clock.h" 6 | #include "pptr.h" 7 | #include 8 | 9 | std::set g_perThreadLog; 10 | boost::lockfree::spsc_queue*, boost::lockfree::capacity<10000>> g_workQueue[MAX_NUMA * WORKER_THREAD_PER_NUMA]; 11 | thread_local Oplog* Oplog::perThreadLog; 12 | std::atomic numSplits; 13 | int combinerSplits = 0; 14 | std::atomic curQ; 15 | Oplog::Oplog() { 16 | } 17 | 18 | void Oplog::enq(OpStruct *ops) { 19 | #ifdef SYNC 20 | unsigned long qnum = 0; 21 | #else 22 | unsigned long qnum = curQ % 2; 23 | while (!qLock[qnum].try_lock()) { 24 | std::atomic_thread_fence(std::memory_order_acq_rel); 25 | qnum = curQ % 2; 26 | } 27 | #endif 28 | op_[qnum].push_back(ops); 29 | #ifndef SYNC 30 | qLock[qnum].unlock(); 31 | #endif 32 | //std::atomic_fetch_add(&numSplits, 1); 33 | } 34 | 35 | 36 | void Oplog::enq(OpStruct::Operation op, Key_t key, uint8_t hash, void* newNodePtr) { 37 | OpStruct* ops = new OpStruct; 38 | ops->op = op; 39 | ops->key = key; 40 | ops->hash = static_cast(hash % WORKER_THREAD_PER_NUMA); 41 | //ops->newNodePtr = newNodePtr; 42 | ops->ts = ordo_get_clock(); 43 | #ifdef SYNC 44 | unsigned long qnum = 0; 45 | #else 46 | unsigned long qnum = curQ % 2; 47 | while (!qLock[qnum].try_lock()) { 48 | std::atomic_thread_fence(std::memory_order_acq_rel); 49 | qnum = curQ % 2; 50 | } 51 | #endif 52 | op_[qnum].push_back(ops); 53 | #ifndef SYNC 54 | qLock[qnum].unlock(); 55 | #endif 56 | //std::atomic_fetch_add(&numSplits, 1); 57 | } 58 | 59 | void Oplog::resetQ(int qnum) { 60 | op_[qnum].clear(); 61 | } 62 | 63 | 64 | std::vector* Oplog::getQ(int qnum) { 65 | return &op_[qnum]; 66 | } 67 | 68 | 69 | Oplog* Oplog::getOpLog() { 70 | Oplog* perThreadLog = Oplog::getPerThreadInstance(); 71 | if(!g_perThreadLog.count(perThreadLog)) { 72 | perThreadLog = new Oplog; 73 | g_perThreadLog.insert(perThreadLog); 74 | setPerThreadInstance(perThreadLog); 75 | } 76 | return perThreadLog; 77 | } 78 | void Oplog::enqPerThreadLog(OpStruct::Operation op, Key_t key, uint8_t hash, void* newNodePtr) { 79 | Oplog* perThreadLog = getOpLog(); 80 | perThreadLog->enq(op, key, hash, newNodePtr); 81 | } 82 | 83 | void Oplog::enqPerThreadLog(OpStruct* ops){ 84 | Oplog* perThreadLog = getOpLog(); 85 | perThreadLog->enq(ops); 86 | } 87 | 88 | void Oplog::writeOpLog(OpStruct *oplog, OpStruct::Operation op, Key_t key, void* oldNodeRawPtr, uint16_t poolId, Key_t newKey, Val_t newVal){ 89 | oplog->op= op; 90 | oplog->key = key; 91 | oplog->oldNodePtr = oldNodeRawPtr; //should be persistent ptr 92 | oplog->poolId = poolId; 93 | oplog->newKey = newKey; 94 | oplog->newVal= newVal; 95 | } 96 | 97 | -------------------------------------------------------------------------------- /src/Oplog.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _OPLOG_H_ 5 | #define _OPLOG_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "common.h" 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | class Oplog { 18 | private: 19 | std::mutex qLock[2]; 20 | std::vector oplog1; 21 | std::vector oplog2; 22 | std::vector> op_{oplog1, oplog2}; 23 | static thread_local Oplog* perThreadLog; 24 | public: 25 | Oplog(); 26 | void resetQ(int Qnum); 27 | std::vector *getQ(int qnum); 28 | static Oplog* getPerThreadInstance(){return perThreadLog;} 29 | static void setPerThreadInstance(Oplog* ptr){perThreadLog = ptr;} 30 | static Oplog* getOpLog(); 31 | static void enqPerThreadLog(OpStruct::Operation op, Key_t key, uint8_t hash, void* listNodePtr); 32 | static void enqPerThreadLog(OpStruct* ops); 33 | void enq(OpStruct::Operation op, Key_t key, uint8_t hash, void* listNodePtr); 34 | void enq(OpStruct *ops) ; 35 | void lock(int qnum) {qLock[qnum].lock();} 36 | void unlock(int qnum) {qLock[qnum].unlock();} 37 | static void writeOpLog(OpStruct *oplog, OpStruct::Operation op, Key_t key, void* oldNodeRawPtr, uint16_t poolId, Key_t newKey, Val_t newVal); 38 | }; 39 | extern std::set g_perThreadLog; 40 | extern boost::lockfree::spsc_queue*, boost::lockfree::capacity<10000>> g_workQueue[MAX_NUMA * WORKER_THREAD_PER_NUMA]; 41 | extern std::atomic numSplits; 42 | extern int combinerSplits; 43 | extern std::atomic curQ; 44 | #endif 45 | -------------------------------------------------------------------------------- /src/SearchLayer.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "common.h" 5 | #include "numa.h" 6 | #include "../lib/PDL-ART/Tree.h" 7 | 8 | class PDLARTIndex { 9 | private: 10 | Key minKey; 11 | Key_t curMin; 12 | pptr idxPtr; 13 | ART_ROWEX::Tree *idx; 14 | ART_ROWEX::Tree *dummy_idx; 15 | uint32_t numInserts = 0; 16 | int numa; 17 | public: 18 | PDLARTIndex(char* path, size_t size) { //for test.. 19 | if (typeid(Key_t) == typeid(uint64_t)) { 20 | int is_created; 21 | root_obj *sl_root; 22 | PMem::bind(0,path,size,(void **)&sl_root,&is_created); 23 | PMem::alloc(0,sizeof(ART_ROWEX::Tree),(void **)&idxPtr); 24 | 25 | idx = new(idxPtr.getVaddr()) ART_ROWEX::Tree([] (TID tid,Key &key){ 26 | key.setInt(*reinterpret_cast(tid)); 27 | }); 28 | // 29 | minKey.setInt(0); 30 | curMin = ULLONG_MAX; 31 | dummy_idx = new ART_ROWEX::Tree([] (TID tid,Key &key){ 32 | key.setInt(*reinterpret_cast(tid)); 33 | }); 34 | } 35 | else{ 36 | #ifdef STRINGKEY 37 | printf("here for string\n"); 38 | int is_created; 39 | root_obj *sl_root; 40 | PMem::bind(0,path,size,(void **)&sl_root,&is_created); 41 | PMem::alloc(0,sizeof(ART_ROWEX::Tree),(void **)&idxPtr); 42 | 43 | idx = new(idxPtr.getVaddr()) ART_ROWEX::Tree([] (TID tid,Key &key){ 44 | key.set(reinterpret_cast(tid), KEYLENGTH); 45 | }); 46 | std::string maxString= "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"; 47 | curMin.setFromString(maxString); 48 | #endif 49 | } 50 | } 51 | PDLARTIndex() { 52 | if (typeid(Key_t) == typeid(uint64_t)) { 53 | PMem::alloc(0,sizeof(ART_ROWEX::Tree),(void **)&idxPtr); 54 | idx = new(idxPtr.getVaddr()) ART_ROWEX::Tree([] (TID tid,Key &key){ 55 | key.setInt(*reinterpret_cast(tid)); 56 | }); 57 | dummy_idx = new ART_ROWEX::Tree([] (TID tid,Key &key){ 58 | key.setInt(*reinterpret_cast(tid)); 59 | }); 60 | } 61 | else{ 62 | printf("here for string\n"); 63 | PMem::alloc(0,sizeof(ART_ROWEX::Tree),(void **)&idxPtr); 64 | 65 | idx = new(idxPtr.getVaddr()) ART_ROWEX::Tree([] (TID tid,Key &key){ 66 | key.set(reinterpret_cast(tid), KEYLENGTH); 67 | }); 68 | 69 | } 70 | } 71 | 72 | ~PDLARTIndex() { 73 | //delete idx; 74 | } 75 | void init(){ 76 | pptr dummyidxPtr; 77 | idx = (ART_ROWEX::Tree *)(idxPtr.getVaddr()); 78 | idx->genId++; 79 | dummy_idx = new ART_ROWEX::Tree([] (TID tid,Key &key){ 80 | key.setInt(*reinterpret_cast(tid)); 81 | }); 82 | 83 | } 84 | void setNuma(int numa){this->numa=numa;} 85 | void setKey(Key& k, uint64_t key) {k.setInt(key);} 86 | void setKey(Key& k, StringKey key) {k.set(key.getData(), KEYLENGTH);} 87 | bool insert(Key_t key, void *ptr) { 88 | auto t = dummy_idx->getThreadInfo(); 89 | Key k; 90 | setKey(k, key); 91 | idx->insert(k, (unsigned long)ptr, t); 92 | if (key < curMin) 93 | curMin = key; 94 | numInserts++; 95 | return true; 96 | } 97 | bool remove(Key_t key, void *ptr) { 98 | auto t = dummy_idx->getThreadInfo(); 99 | Key k; 100 | setKey(k, key); 101 | idx->remove(k, (unsigned long)ptr, t); 102 | numInserts--; 103 | return true; 104 | } 105 | #ifdef SYNC 106 | bool insert(Key_t key, void* ptr, void *locked_node) { 107 | bool ret; 108 | auto t = idx->getThreadInfo(); 109 | Key k; 110 | setKey(k, key); 111 | ret = idx->insert(k, reinterpret_cast(ptr), t,locked_node); 112 | numInserts++; 113 | if (key < curMin) curMin = key; 114 | return ret; 115 | } 116 | 117 | void* lookupwithLock(Key_t key, void** node) { 118 | if (key <= curMin) 119 | return nullptr; 120 | auto t = idx->getThreadInfo(); 121 | Key endKey; 122 | setKey(endKey, key); 123 | 124 | auto result = idx->lookupNextwithLock(endKey, node, t); 125 | return reinterpret_cast(result); 126 | } 127 | 128 | bool nodeUnlock(void *node){ 129 | auto t = idx->getThreadInfo(); 130 | 131 | auto result = idx->nodeUnlock(node, t); 132 | return result; 133 | } 134 | #endif 135 | 136 | 137 | //Gets the value of the key if present or the value of key just less than/greater than key 138 | void* lookup(Key_t key) { 139 | if (key <= curMin) 140 | return nullptr; 141 | auto t = dummy_idx->getThreadInfo(); 142 | Key endKey; 143 | setKey(endKey, key); 144 | 145 | auto result = idx->lookupNext(endKey, t); 146 | return reinterpret_cast(result); 147 | } 148 | void* lookup2(Key_t key) { 149 | if (key <= curMin){ 150 | return nullptr; 151 | } 152 | auto t = dummy_idx->getThreadInfo(); 153 | Key endKey; 154 | setKey(endKey, key); 155 | 156 | auto result = idx->lookup(endKey, t); 157 | return reinterpret_cast(result); 158 | } 159 | 160 | 161 | // Art segfaults if range operation is done when there are less than 2 keys 162 | bool isEmpty() { 163 | return (numInserts < 2); 164 | } 165 | uint32_t size() {return numInserts;} 166 | 167 | }; 168 | //typedef SortedArray SearchLayer; 169 | typedef PDLARTIndex SearchLayer; 170 | -------------------------------------------------------------------------------- /src/VersionedLock.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _VERSIONEDLOCK_H 5 | #define _VERSIONEDLOCK_H 6 | #include 7 | #include 8 | #include "arch.h" 9 | 10 | #define PAUSE asm volatile("pause\n": : :"memory"); 11 | typedef unsigned long version_t; 12 | 13 | class VersionedLock { 14 | private: 15 | std::atomic version; 16 | 17 | public: 18 | VersionedLock() : version(2) {} 19 | 20 | version_t read_lock(uint32_t genId) { 21 | version_t ver = version.load(std::memory_order_acquire); 22 | uint32_t lockGenId; 23 | uint32_t verLock; 24 | 25 | lockGenId = ver >> 32; 26 | verLock = (int32_t)ver; 27 | 28 | if(genId != lockGenId){ 29 | // versionLock = 2; 30 | version_t new_ver = (genId << 32) + 2; 31 | if(version.compare_exchange_weak(ver, new_ver)){ 32 | return new_ver; 33 | } 34 | return 0; 35 | } 36 | else{ 37 | if ((ver & 1) != 0) { 38 | return 0; 39 | } 40 | return ver; 41 | } 42 | } 43 | 44 | version_t write_lock(uint32_t genId) { 45 | version_t ver = version.load(std::memory_order_acquire); 46 | uint32_t lockGenId; 47 | uint32_t verLock; 48 | 49 | lockGenId = ver >> 32; 50 | verLock = (int32_t)ver; 51 | 52 | if(genId != lockGenId){ 53 | // versionLock = 2; 54 | version_t new_ver = (genId << 32) + 3; 55 | if(version.compare_exchange_weak(ver, new_ver)){ 56 | return new_ver; 57 | } 58 | return 0; 59 | 60 | } 61 | else{ 62 | if ((ver & 1) == 0 && version.compare_exchange_weak(ver, ver+1)) { 63 | return ver; 64 | } 65 | return 0; 66 | } 67 | } 68 | 69 | bool read_unlock(version_t old_version) { 70 | std::atomic_thread_fence(std::memory_order_acquire); 71 | version_t new_version = version.load(std::memory_order_acquire); 72 | return new_version == old_version; 73 | } 74 | 75 | void write_unlock() { 76 | version.fetch_add(1, std::memory_order_release); 77 | return; 78 | } 79 | }; 80 | #endif //_VERSIONEDLOCK_H 81 | -------------------------------------------------------------------------------- /src/WorkerThread.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "WorkerThread.h" 5 | #include "Oplog.h" 6 | #include "Combiner.h" 7 | #include "pactree.h" 8 | #include 9 | #include 10 | 11 | WorkerThread::WorkerThread(int id, int activeNuma) { 12 | this->workerThreadId = id; 13 | this->activeNuma = activeNuma; 14 | this->workQueue = &g_workQueue[workerThreadId]; 15 | this->logDoneCount = 0; 16 | this->opcount = 0; 17 | if (id == 0) freeQueue = new std::queue>; 18 | } 19 | 20 | bool WorkerThread::applyOperation() { 21 | std::vector* oplog = workQueue->front(); 22 | int numaNode = workerThreadId % activeNuma; 23 | SearchLayer* sl = g_perNumaSlPtr[numaNode]; 24 | uint8_t hash = static_cast(workerThreadId / activeNuma); 25 | bool ret = false; 26 | for (auto opsPtr : *oplog) { 27 | OpStruct &ops = *opsPtr; 28 | if (ops.hash != hash) continue; 29 | opcount++; 30 | if (ops.op == OpStruct::insert){ 31 | void *newNodePtr= reinterpret_cast (((unsigned long)(ops.poolId)) << 48 | (ops.newNodeOid.off)); 32 | sl->insert(ops.key, newNodePtr); 33 | opsPtr->op = OpStruct::done; 34 | } 35 | else if (ops.op == OpStruct::remove) { 36 | sl->remove(ops.key, ops.oldNodePtr); 37 | opsPtr->op = OpStruct::done; 38 | flushToNVM((char*)&ops,sizeof(OpStruct)); 39 | if (workerThreadId == 0) { 40 | std::pair removePair; 41 | removePair.first = logDoneCount; 42 | removePair.second = ops.oldNodePtr; 43 | freeQueue->push(removePair); 44 | ret = true; 45 | } 46 | } 47 | else{ 48 | if(ops.op == OpStruct::done){ 49 | printf("done? %p\n",opsPtr); 50 | } 51 | exit(1); 52 | } 53 | flushToNVM((char*)opsPtr,sizeof(OpStruct)); 54 | smp_wmb(); 55 | } 56 | workQueue->pop(); 57 | logDoneCount++; 58 | return ret; 59 | } 60 | 61 | bool WorkerThread::isWorkQueueEmpty() { 62 | return !workQueue->read_available(); 63 | } 64 | 65 | void WorkerThread::freeListNodes(uint64_t removeCount) { 66 | assert(workerThreadId == 0 && freeQueue != NULL); 67 | if (freeQueue->empty()) return; 68 | while (!freeQueue->empty()) { 69 | std::pair removePair = freeQueue->front(); 70 | if (removePair.first < removeCount) { 71 | PMEMoid ptr = pmemobj_oid(removePair.second); 72 | pmemobj_free(&ptr); 73 | freeQueue->pop(); 74 | } 75 | else break; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/WorkerThread.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef PACTREE_WORKERTHREAD_H 5 | #define PACTREE_WORKERTHREAD_H 6 | #include "common.h" 7 | #include "Oplog.h" 8 | #include 9 | 10 | class WorkerThread { 11 | private: 12 | boost::lockfree::spsc_queue*, boost::lockfree::capacity<10000>> *workQueue; 13 | int workerThreadId; 14 | int activeNuma; 15 | unsigned long logDoneCount; 16 | std::queue> *freeQueue; 17 | public: 18 | unsigned long opcount; 19 | WorkerThread(int id, int activeNuma); 20 | bool applyOperation(); 21 | bool isWorkQueueEmpty(); 22 | unsigned long getLogDoneCount() {return logDoneCount;} 23 | void freeListNodes(uint64_t removeCount); 24 | }; 25 | extern std::vector g_WorkerThreadInst; 26 | 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/bitset.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef pactree_BITSET_H 5 | #define pactree_BITSET_H 6 | 7 | #include 8 | #include 9 | 10 | namespace hydra { 11 | class bitset { 12 | private: 13 | uint64_t bits[2]; 14 | bool testBit(int pos, uint64_t &bits) { 15 | return (bits & (1UL << (pos))) != 0; 16 | } 17 | void setBit(int pos, uint64_t &bits) { 18 | bits |= 1UL << pos; 19 | } 20 | void resetBit(int pos, uint64_t &bits) { 21 | bits &= ~(1UL << pos); 22 | } 23 | public: 24 | void clear() { 25 | bits[0] = 0; 26 | bits[1] = 0; 27 | } 28 | void set(int index) { 29 | //assert(index < 128 && index >= 0); 30 | setBit(index, bits[index/64]); 31 | } 32 | void reset(int index) { 33 | //assert(index < 128 && index >= 0); 34 | resetBit(index, bits[index/64]); 35 | } 36 | bool test(int index) { 37 | //assert(index < 128 && index >= 0); 38 | return testBit(index, bits[index/64]); 39 | } 40 | bool operator[] (int index) { 41 | return test(index); 42 | } 43 | uint64_t to_ulong() { 44 | return bits[0]; 45 | } 46 | uint64_t to_ulong(int index) { 47 | return bits[index]; 48 | } 49 | 50 | }; 51 | }; 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/linkedList.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include 5 | #include 6 | #include 7 | #include "linkedList.h" 8 | #include "SearchLayer.h" 9 | //std::atomic numSplits; 10 | std::atomic dists[5]; 11 | extern std::mutex mtx[112]; 12 | 13 | void printDists(){ 14 | uint64_t sum =0 ; 15 | for(int i=0; i<5; i++){ 16 | printf("%lu \n",dists[i].load()); 17 | sum+=dists[i].load(); 18 | } 19 | for(int i=0; i<5; i++){ 20 | printf("%lf \%\n",(double)(dists[i].load())/(double)sum*100.0); 21 | 22 | } 23 | } 24 | 25 | std::atomic pmemAmount; 26 | 27 | void zeroMemAmount(){ 28 | pmemAmount.store(0); 29 | } 30 | 31 | void addMemAmount(unsigned long amt){ 32 | pmemAmount+=amt; 33 | } 34 | void printMemAmount(){ 35 | printf("amount :%lu\n",pmemAmount.load()); 36 | } 37 | 38 | ListNode* LinkedList::initialize() { 39 | genId =0; 40 | OpStruct* oplog = (OpStruct *)PMem::getOpLog(0); 41 | 42 | PMem::alloc(1,sizeof(ListNode),(void **)&headPtr,&(oplog->newNodeOid)); 43 | ListNode* head = (ListNode*)new(headPtr.getVaddr()) ListNode(); 44 | flushToNVM((char*)&headPtr,sizeof(pptr)); 45 | smp_wmb(); 46 | 47 | OpStruct *oplog2 = (OpStruct *)PMem::getOpLog(1); 48 | 49 | pptr tailPtr; 50 | PMem::alloc(1,sizeof(ListNode),(void **)&tailPtr, &(oplog->newNodeOid)); 51 | ListNode* tail= (ListNode*)new(tailPtr.getVaddr()) ListNode(); 52 | 53 | oplog->op=OpStruct::done; 54 | oplog2->op=OpStruct::done; 55 | 56 | pptrnullPtr(0,0); 57 | 58 | head->setNext(tailPtr); 59 | head->setPrev(nullPtr); 60 | head->setCur(headPtr); 61 | #ifdef STRINGKEY 62 | std::string minString= "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; 63 | std::string maxString= "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"; 64 | Key_t max; 65 | max.setFromString(maxString); 66 | Key_t min; 67 | min.setFromString(minString); 68 | head->setMin(min); 69 | head->insert(min,0,0); 70 | tail->insert(max, 0,0); 71 | tail->setMin(max); 72 | head->setMax(max); 73 | #else 74 | head->insert(0,0,0); 75 | head->setMin(0); 76 | head->setMax(ULLONG_MAX); 77 | tail->insert(ULLONG_MAX, 0,0); 78 | tail->setMin(ULLONG_MAX); 79 | #endif 80 | 81 | // tail->setNumEntries(MAX_ENTRIES); //This prevents merge of tail 82 | tail->setFullBitmap(); 83 | tail->setNext(nullPtr); 84 | tail->setPrev(headPtr); 85 | tail->setCur(tailPtr); 86 | 87 | 88 | flushToNVM((char*)head,sizeof(ListNode)); 89 | flushToNVM((char*)tail,sizeof(ListNode)); 90 | smp_wmb(); 91 | 92 | for(int i=0; i<5;i++){ 93 | dists[i].store(0); 94 | } 95 | 96 | return head; 97 | } 98 | #ifdef SYNC 99 | bool LinkedList::insert(Key_t key, Val_t value, ListNode* head,void **locked_parent_node) { 100 | int retryCount = 0; 101 | restart: 102 | ListNode* cur = head; 103 | while (1) { 104 | if (cur->getMin() > key) { 105 | cur = cur->getPrev(); 106 | continue; 107 | } 108 | if (!cur->checkRange(key)) { 109 | cur = cur->getNext(); 110 | continue; 111 | } 112 | break; 113 | } 114 | if (!cur->writeLock(genId)){ 115 | goto restart; 116 | } 117 | if (cur->getDeleted()) { 118 | cur->writeUnlock(); 119 | goto restart; 120 | } 121 | if (!cur->checkRange(key)) { 122 | cur->writeUnlock(); 123 | goto restart; 124 | } 125 | bool ret = cur->insert(key, value,locked_parent_node); 126 | return ret; 127 | } 128 | 129 | #endif 130 | 131 | 132 | 133 | bool LinkedList::insert(Key_t key, Val_t value, ListNode* head, int threadId) { 134 | restart: 135 | ListNode* cur = head; 136 | 137 | int dist = 0; 138 | while (1) { 139 | if (cur->getMin() > key) { 140 | #ifdef DIST 141 | dist++; 142 | #endif 143 | cur = cur->getPrev(); 144 | continue; 145 | } 146 | if (!cur->checkRange(key)) { 147 | #ifdef DIST 148 | dist++; 149 | #endif 150 | cur = cur->getNext(); 151 | continue; 152 | } 153 | break; 154 | } 155 | 156 | if (!cur->writeLock(genId)) 157 | goto restart; 158 | if (cur->getDeleted()) { 159 | cur->writeUnlock(); 160 | goto restart; 161 | } 162 | if (!cur->checkRange(key)) { 163 | cur->writeUnlock(); 164 | goto restart; 165 | } 166 | #ifdef DIST 167 | if(dist==0) dists[0]++; 168 | else if(dist<2) dists[1]++; 169 | else if(dist<5) dists[2]++; 170 | else if(dist<50) dists[3]++; 171 | else dists[4]++; 172 | #endif 173 | 174 | bool ret = cur->insert(key, value,threadId); 175 | cur->writeUnlock(); 176 | return ret; 177 | } 178 | 179 | bool LinkedList::update(Key_t key, Val_t value, ListNode* head) { 180 | restart: 181 | ListNode* cur = head; 182 | 183 | while (1) { 184 | if (cur->getMin() > key) { 185 | cur = cur->getPrev(); 186 | continue; 187 | } 188 | if (!cur->checkRange(key)) { 189 | cur = cur->getNext(); 190 | continue; 191 | } 192 | break; 193 | } 194 | 195 | if (!cur->writeLock(genId)) 196 | goto restart; 197 | if (cur->getDeleted()) { 198 | cur->writeUnlock(); 199 | goto restart; 200 | } 201 | if (!cur->checkRange(key)) { 202 | cur->writeUnlock(); 203 | goto restart; 204 | } 205 | bool ret = cur->update(key, value); 206 | cur->writeUnlock(); 207 | return ret; 208 | } 209 | 210 | bool LinkedList::remove(Key_t key, ListNode *head) { 211 | restart: 212 | ListNode* cur = head; 213 | 214 | while (1) { 215 | if (cur->getMin() > key) { 216 | cur = cur->getPrev(); 217 | continue; 218 | } 219 | if (!cur->checkRange(key)) { 220 | cur = cur->getNext(); 221 | continue; 222 | } 223 | break; 224 | } 225 | if (!cur->writeLock(genId)) { 226 | if (head->getPrev() != nullptr) head = head->getPrev(); 227 | goto restart; 228 | } 229 | if (cur->getDeleted()) { 230 | if (head->getPrev() != nullptr) head = head->getPrev(); 231 | cur->writeUnlock(); 232 | goto restart; 233 | } 234 | if (!cur->checkRange(key)) { 235 | if (head->getPrev() != nullptr) head = head->getPrev(); 236 | cur->writeUnlock(); 237 | goto restart; 238 | } 239 | 240 | bool ret = cur->remove(key,genId); 241 | cur->writeUnlock(); 242 | return ret; 243 | } 244 | 245 | bool LinkedList::probe(Key_t key, ListNode *head) { 246 | restart: 247 | ListNode* cur = head; 248 | 249 | while (1) { 250 | if (cur->getMin() > key) { 251 | cur = cur->getPrev(); 252 | continue; 253 | } 254 | if (!cur->checkRange(key)) { 255 | cur = cur->getNext(); 256 | continue; 257 | } 258 | break; 259 | } 260 | 261 | version_t readVersion = cur->readLock(genId); 262 | //Concurrent Update 263 | if (!readVersion) 264 | goto restart; 265 | if (cur->getDeleted()){ 266 | goto restart; 267 | } 268 | if (!cur->checkRange(key)) { 269 | goto restart; 270 | } 271 | bool ret = false; 272 | ret = cur->probe(key); 273 | if (!cur->readUnlock(readVersion)) 274 | goto restart; 275 | return ret; 276 | } 277 | 278 | bool LinkedList::lookup(Key_t key, Val_t &value, ListNode *head) { 279 | restart: 280 | ListNode* cur = head; 281 | int count = 0; 282 | 283 | while (1) { 284 | if (cur->getMin() > key) { 285 | cur = cur->getPrev(); 286 | continue; 287 | } 288 | if (!cur->checkRange(key)) { 289 | cur = cur->getNext(); 290 | continue; 291 | } 292 | break; 293 | } 294 | version_t readVersion = cur->readLock(genId); 295 | //Concurrent Update 296 | if (!readVersion) 297 | goto restart; 298 | if (cur->getDeleted()){ 299 | goto restart; 300 | } 301 | if (!cur->checkRange(key)) { 302 | goto restart; 303 | } 304 | bool ret = false; 305 | ret = cur->lookup(key, value); 306 | if (!cur->readUnlock(readVersion)) 307 | goto restart; 308 | return ret; 309 | } 310 | 311 | void LinkedList::print(ListNode *head) { 312 | ListNode* cur = head; 313 | while (cur->getNext() != nullptr) { 314 | cur->print(); 315 | cur = cur->getNext(); 316 | } 317 | std::cout << "\n"; 318 | return; 319 | } 320 | 321 | uint32_t LinkedList::size(ListNode *head) { 322 | ListNode* cur = head; 323 | int count = 0; 324 | while (cur->getNext() != nullptr) { 325 | count++; 326 | cur = cur->getNext(); 327 | } 328 | return count; 329 | } 330 | 331 | ListNode *LinkedList::getHead() { 332 | ListNode* head = (ListNode*) headPtr.getVaddr(); 333 | return head; 334 | } 335 | 336 | uint64_t LinkedList::scan(Key_t startKey, int range, std::vector &rangeVector, ListNode *head) { 337 | restart: 338 | ListNode* cur = head; 339 | rangeVector.clear(); 340 | // Find the start Node 341 | while (1) { 342 | if (cur->getMin() > startKey) { 343 | cur = cur->getPrev(); 344 | continue; 345 | } 346 | if (!cur->checkRange(startKey)) { 347 | cur = cur->getNext(); 348 | continue; 349 | } 350 | break; 351 | } 352 | bool end = false; 353 | assert(rangeVector.size() == 0); 354 | std::vector resultBuffer; 355 | while (rangeVector.size() < range && !end) { 356 | version_t readVersion = cur->readLock(genId); 357 | //Concurrent Update 358 | if (!readVersion) 359 | continue; 360 | /*resultBuffer.clear();*/ 361 | if (cur->getDeleted()) 362 | goto restart; 363 | end = cur->scan(startKey, range, rangeVector, readVersion,genId); 364 | if(!cur->readUnlock(readVersion)){ 365 | continue; 366 | } 367 | cur = cur->getNext(); 368 | } 369 | return rangeVector.size(); 370 | } 371 | 372 | bool LinkedList::Recovery(void* p_sl) { 373 | genId+=1; 374 | SearchLayer *sl = (SearchLayer *)p_sl; 375 | int i=0; 376 | for(i=0; i<1000*112; i++){ 377 | OpStruct *oplog = (OpStruct*)PMem::getOpLog(i); 378 | if((oplog->op == OpStruct::insert)){ 379 | pptr node; 380 | pptr next_node; 381 | node.setRawPtr(oplog->oldNodePtr); 382 | next_node = node->getNextPtr(); 383 | if((next_node.getVaddr()!=pmemobj_direct(oplog->newNodeOid))){ 384 | printf("case 1\n"); 385 | // not connected 386 | pmemobj_free(&oplog->newNodeOid); 387 | next_node = node->recoverSplit(oplog); 388 | } 389 | else{ 390 | node->recoverNode(oplog->key); 391 | if(sl->isEmpty()){ 392 | sl->insert(oplog->key,oplog->oldNodePtr); 393 | } 394 | else if(sl->lookup(oplog->key)!=oplog->oldNodePtr){ 395 | sl->insert(oplog->key,oplog->oldNodePtr); 396 | } 397 | } 398 | ListNode* nnextNode = next_node->getNext(); 399 | nnextNode->setPrev(next_node); 400 | oplog->op=OpStruct::done; 401 | } 402 | else if((oplog->op == OpStruct::remove)){ 403 | pptr node; // deleting node 404 | pptr prev_node; 405 | node.setRawPtr(oplog->oldNodePtr); 406 | prev_node = node->getPrevPtr(); 407 | prev_node->recoverMergingNode(node.getVaddr()); 408 | if(sl->isEmpty()){ 409 | ; 410 | } 411 | else if(sl->lookup(oplog->key)==(void*)node.getRawPtr()){ 412 | sl->remove(oplog->key,(void*)node.getRawPtr()); 413 | } 414 | pmemobj_free(&oplog->newNodeOid); 415 | } 416 | } 417 | } 418 | -------------------------------------------------------------------------------- /src/linkedList.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _LINKEDLIST_H 5 | #define _LINKEDLIST_H 6 | #include "listNode.h" 7 | //#define DIST 8 | 9 | enum Operation {lt, gt}; 10 | 11 | class LinkedList { 12 | private: 13 | pptr headPtr; 14 | OpStruct oplogs[100000]; // 500 *200 15 | int nunOpLogs[112]; 16 | uint64_t genId; 17 | 18 | public: 19 | ListNode* initialize(); 20 | #ifdef SYNC 21 | bool insert(Key_t key, Val_t value, ListNode* head,void** locked_parent_node); 22 | #endif 23 | bool insert(Key_t key, Val_t value, ListNode* head, int threadId); 24 | bool update(Key_t key, Val_t value, ListNode* head); 25 | bool remove(Key_t key, ListNode* head); 26 | bool probe(Key_t key, ListNode* head); 27 | bool lookup(Key_t key, Val_t &value, ListNode* head); 28 | uint64_t scan(Key_t startKey, int range, std::vector &rangeVector, ListNode *head); 29 | void print(ListNode *head); 30 | uint32_t size(ListNode* head); 31 | ListNode* getHead(); 32 | bool Recovery(void* sl); 33 | }; 34 | void printDists(); 35 | 36 | #endif //_LINKEDLIST_H 37 | -------------------------------------------------------------------------------- /src/listNode.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include 5 | #include 6 | #include "Oplog.h" 7 | #include "listNode.h" 8 | #include "threadData.h" 9 | #include "pactree.h" 10 | 11 | thread_local int logCnt=0; 12 | thread_local int coreId=-1; 13 | extern thread_local ThreadData* curThreadData; 14 | std::mutex mtx[112]; 15 | 16 | ListNode :: ListNode(){ 17 | deleted = false; 18 | bitMap.clear(); 19 | lastScanVersion = 0; 20 | } 21 | 22 | void ListNode::setCur(pptr ptr) { 23 | this->curPtr = ptr; 24 | } 25 | 26 | void ListNode::setNext(pptr ptr) { 27 | this->nextPtr = ptr; 28 | } 29 | 30 | void ListNode::setPrev(pptr ptr) { 31 | this->prevPtr = ptr; 32 | } 33 | 34 | void ListNode::setMin(Key_t key) { 35 | this->min = key; 36 | } 37 | 38 | void ListNode::setMax(Key_t key) { 39 | this->max = key; 40 | } 41 | 42 | void ListNode::setFullBitmap() { 43 | for(int i=0; ideleted = deleted; 50 | } 51 | 52 | ListNode *ListNode::getNext() { 53 | ListNode *next = nextPtr.getVaddr(); 54 | return next; 55 | } 56 | 57 | pptr ListNode::getNextPtr() { 58 | return nextPtr; 59 | } 60 | 61 | pptr ListNode::getCurPtr() { 62 | return curPtr; 63 | } 64 | 65 | ListNode *ListNode::getPrev() { 66 | ListNode *prev= prevPtr.getVaddr(); 67 | return prev; 68 | } 69 | 70 | pptr ListNode::getPrevPtr() { 71 | return prevPtr; 72 | } 73 | 74 | version_t ListNode::readLock(uint64_t genId) { 75 | return verLock.read_lock(genId); 76 | } 77 | 78 | version_t ListNode::writeLock(uint64_t genId) { 79 | return verLock.write_lock(genId); 80 | } 81 | 82 | bool ListNode::readUnlock(version_t oldVersion) { 83 | return verLock.read_unlock(oldVersion); 84 | } 85 | 86 | void ListNode::writeUnlock() { 87 | return verLock.write_unlock(); 88 | } 89 | 90 | int ListNode :: getNumEntries() 91 | { 92 | int numEntries = 0; 93 | for(int i=0; i* ListNode :: getKeyArray() 102 | { 103 | return this->keyArray; 104 | } 105 | 106 | std::pair* ListNode :: getValueArray() 107 | { 108 | return this->keyArray; 109 | } 110 | 111 | Key_t ListNode :: getMin() 112 | { 113 | return this->min; 114 | } 115 | 116 | Key_t ListNode::getMax() { 117 | return this->max; 118 | } 119 | 120 | bool ListNode::getDeleted() { 121 | return deleted; 122 | } 123 | 124 | bool compare2 (std::pair &x, std::pair &y) { 125 | return x.first < y.first; 126 | } 127 | bool compare(std::pair &i, std::pair &j) { 128 | return i.first < j.first; 129 | } 130 | 131 | pptr ListNode :: split(Key_t key, Val_t val, uint8_t keyHash, int threadId, void **olog) 132 | { 133 | //Find median 134 | std::pair copyArray[MAX_ENTRIES]; 135 | std::copy(std::begin(keyArray), std::end(keyArray), std::begin(copyArray)); 136 | std::sort(std::begin(copyArray), std::end(copyArray), compare); 137 | Key_t median = copyArray[MAX_ENTRIES/2].first; 138 | int newNumEntries = 0; 139 | Key_t newMin = median; 140 | 141 | int chip, core; 142 | read_coreid_rdtscp(&chip,&core); 143 | if(core!=coreId) 144 | coreId=core; 145 | 146 | mtx[core].lock(); 147 | int numLogsPerThread = 1000; 148 | int logIdx= numLogsPerThread *(core) + logCnt; 149 | logCnt++; 150 | if(logCnt==numLogsPerThread){ 151 | logCnt=0; 152 | } 153 | OpStruct* oplog; 154 | 155 | #ifdef MULTIPOOL 156 | uint16_t poolId = (uint16_t)(3*chip+1); 157 | #else 158 | uint16_t poolId = 1; 159 | #endif 160 | /*pptr oplogPtr; 161 | PMEMoid oid; 162 | PMem::alloc(poolId,sizeof(OpStruct),(void **)&(oplogPtr), &oid); 163 | oplog=(OpStruct *) oplogPtr.getVaddr();*/ 164 | 165 | oplog = (OpStruct *)PMem::getOpLog(logIdx); 166 | // 1) Add Oplog and set the infomration for current(overflown) node. 167 | Oplog::writeOpLog(oplog, OpStruct::insert, newMin, (void*)curPtr.getRawPtr(), poolId, key, val); 168 | flushToNVM((char*)oplog,sizeof(OpStruct)); 169 | smp_wmb(); 170 | // 2) Allocate new data node and store persistent pointer to the oplog. 171 | pptr newNodePtr; 172 | PMem::alloc(poolId,sizeof(ListNode),(void **)&(newNodePtr),&(oplog->newNodeOid)); 173 | if(newNodePtr.getVaddr()==nullptr){ 174 | exit(1); 175 | } 176 | unsigned long nPtr = newNodePtr.getRawPtr(); 177 | 178 | ListNode* newNode = (ListNode*)new(newNodePtr.getVaddr()) ListNode(); 179 | 180 | // 3) Perform Split Operation 181 | // 3-1) Update New node 182 | int removeIndex = -1; 183 | for(int i = 0; i < MAX_ENTRIES; i++) { 184 | if(keyArray[i].first >= median) { 185 | newNode->insertAtIndex(keyArray[i], newNumEntries, fingerPrint[i], false); 186 | newNumEntries++; 187 | } 188 | } 189 | if (key >= newMin) 190 | newNode->insertAtIndex(std::make_pair(key, val), newNumEntries, keyHash, false); 191 | newNode->setMin(newMin); 192 | newNode->setNext(nextPtr); 193 | newNode->setCur(newNodePtr); 194 | newNode->setDeleted(false); 195 | 196 | newNode->setPrev(curPtr); 197 | newNode->setMax(getMax()); 198 | setMax(newMin); 199 | flushToNVM((char*)newNode,sizeof(ListNode)); 200 | smp_wmb(); 201 | 202 | // 3-2) Setting next pointer to the New node 203 | nextPtr = newNodePtr; 204 | 205 | // 3-3) Update overflown node 206 | // 3-4) Make copy of the bitmap and update the bitmap value. 207 | hydra::bitset curBitMap = *(this->getBitMap()); 208 | int numEntries = getNumEntries(); 209 | for(int i = 0; i < MAX_ENTRIES; i++) { 210 | if(keyArray[i].first >= median) { 211 | curBitMap.reset(i); 212 | numEntries--; 213 | if (removeIndex == -1) removeIndex = i; 214 | } 215 | } 216 | 217 | // 3-4) copy updated bitmap to overflown node. 218 | // What if the size of node is bigger than 64 entries? 219 | bitMap = curBitMap; 220 | flushToNVM((char*)this,L1_CACHE_BYTES); 221 | smp_wmb(); 222 | if (key < newMin) { 223 | if (removeIndex != -1) 224 | insertAtIndex(std::make_pair(key, val), removeIndex, keyHash, true); 225 | else 226 | insertAtIndex(std::make_pair(key, val), numEntries, keyHash, true); 227 | } 228 | 229 | ListNode* nextNode = newNodePtr->getNext(); 230 | nextNode->setPrev(newNodePtr); 231 | Oplog::enqPerThreadLog(oplog); 232 | *olog=oplog; 233 | mtx[core].unlock(); 234 | 235 | return newNodePtr; 236 | } 237 | 238 | 239 | pptr ListNode :: split(Key_t key, Val_t val, uint8_t keyHash, int threadId) 240 | { 241 | //Find median 242 | std::pair copyArray[MAX_ENTRIES]; 243 | std::copy(std::begin(keyArray), std::end(keyArray), std::begin(copyArray)); 244 | std::sort(std::begin(copyArray), std::end(copyArray), compare); 245 | Key_t median = copyArray[MAX_ENTRIES/2].first; 246 | int newNumEntries = 0; 247 | Key_t newMin = median; 248 | 249 | int chip, core; 250 | read_coreid_rdtscp(&chip,&core); 251 | if(core!=coreId) 252 | coreId=core; 253 | 254 | mtx[core].lock(); 255 | int numLogsPerThread = 1000; 256 | int logIdx= numLogsPerThread *(core) + logCnt; 257 | logCnt++; 258 | if(logCnt==numLogsPerThread){ 259 | logCnt=0; 260 | } 261 | OpStruct* oplog; 262 | 263 | #ifdef MULTIPOOL 264 | uint16_t poolId = (uint16_t)(3*chip+1); 265 | #else 266 | uint16_t poolId = 1; 267 | #endif 268 | /* pptr oplogPtr; 269 | PMEMoid oid; 270 | PMem::alloc(poolId,sizeof(OpStruct),(void **)&(oplogPtr), &oid); 271 | oplog=(OpStruct *) oplogPtr.getVaddr();*/ 272 | oplog = (OpStruct *)PMem::getOpLog(logIdx); 273 | 274 | // 1) Add Oplog and set the infomration for current(overflown) node. 275 | Oplog::writeOpLog(oplog, OpStruct::insert, newMin, (void*)curPtr.getRawPtr(), poolId, key, val); 276 | flushToNVM((char*)oplog,sizeof(OpStruct)); 277 | //smp_wmb(); 278 | //exit(1); //case 1 279 | 280 | // 2) Allocate new data node and store persistent pointer to the oplog. 281 | pptr newNodePtr; 282 | PMem::alloc(poolId,sizeof(ListNode),(void **)&(newNodePtr),&(oplog->newNodeOid)); 283 | if(newNodePtr.getVaddr()==nullptr){ 284 | exit(1); 285 | } 286 | unsigned long nPtr = newNodePtr.getRawPtr(); 287 | 288 | ListNode* newNode = (ListNode*)new(newNodePtr.getVaddr()) ListNode(); 289 | 290 | // 3) Perform Split Operation 291 | // 3-1) Update New node 292 | int removeIndex = -1; 293 | for(int i = 0; i < MAX_ENTRIES; i++) { 294 | if(keyArray[i].first >= median) { 295 | newNode->insertAtIndex(keyArray[i], newNumEntries, fingerPrint[i], false); 296 | newNumEntries++; 297 | } 298 | } 299 | if (key >= newMin) 300 | newNode->insertAtIndex(std::make_pair(key, val), newNumEntries, keyHash, false); 301 | newNode->setMin(newMin); 302 | newNode->setNext(nextPtr); 303 | newNode->setCur(newNodePtr); 304 | newNode->setDeleted(false); 305 | 306 | newNode->setPrev(curPtr); 307 | newNode->setMax(getMax()); 308 | setMax(newMin); 309 | flushToNVM((char*)newNode,sizeof(ListNode)); 310 | smp_wmb(); 311 | 312 | // 3-2) Setting next pointer to the New node 313 | nextPtr = newNodePtr; 314 | //exit(1); //case 2 315 | 316 | // 3-3) Update overflown node 317 | // 3-4) Make copy of the bitmap and update the bitmap value. 318 | hydra::bitset curBitMap = *(this->getBitMap()); 319 | int numEntries = getNumEntries(); 320 | for(int i = 0; i < MAX_ENTRIES; i++) { 321 | if(keyArray[i].first >= median) { 322 | curBitMap.reset(i); 323 | numEntries--; 324 | if (removeIndex == -1) removeIndex = i; 325 | } 326 | } 327 | 328 | // 3-4) copy updated bitmap to overflown node. 329 | // What if the size of node is bigger than 64 entries? 330 | bitMap = curBitMap; 331 | flushToNVM((char*)this,L1_CACHE_BYTES); 332 | smp_wmb(); 333 | if (key < newMin) { 334 | if (removeIndex != -1) 335 | insertAtIndex(std::make_pair(key, val), removeIndex, keyHash, true); 336 | else 337 | insertAtIndex(std::make_pair(key, val), numEntries, keyHash, true); 338 | } 339 | 340 | ListNode* nextNode = newNodePtr->getNext(); 341 | nextNode->setPrev(newNodePtr); 342 | Oplog::enqPerThreadLog(oplog); 343 | 344 | 345 | mtx[core].unlock(); 346 | //exit(1); //case 3 347 | return newNodePtr; 348 | } 349 | 350 | ListNode* ListNode :: mergeWithNext(uint64_t genId) 351 | { 352 | 353 | //exit(1); 354 | ListNode* deleteNode = nextPtr.getVaddr(); 355 | if (!deleteNode->writeLock(genId)) return nullptr; 356 | 357 | 358 | // 1) Add Oplog and set the infomration for current(overflown) node. 359 | 360 | int numLogsPerThread = 1000; 361 | 362 | int chip, core; 363 | read_coreid_rdtscp(&chip,&core); 364 | if(core!=coreId) 365 | coreId=core; 366 | 367 | mtx[core].lock(); 368 | int logIdx= numLogsPerThread *(core) + logCnt; 369 | logCnt++; 370 | OpStruct* oplog = (OpStruct *)PMem::getOpLog(logIdx); 371 | 372 | #ifdef MULTIPOOL 373 | uint16_t poolId = (uint16_t)(3*chip+1); 374 | #else 375 | uint16_t poolId = 1; 376 | #endif 377 | 378 | Oplog::writeOpLog(oplog, OpStruct::remove, getMin(),(void*)getNextPtr().getRawPtr(), poolId, -1, -1); 379 | oplog->newNodeOid = pmemobj_oid(deleteNode); 380 | flushToNVM((char*)oplog,sizeof(OpStruct)); 381 | smp_wmb(); 382 | // printf("case 4\n"); 383 | // exit(0); //case 4 384 | 385 | int cur = 0; 386 | for (int i = 0; i < MAX_ENTRIES; i++) { 387 | if (deleteNode->getBitMap()->test(i)) { 388 | for (int j = cur; j < MAX_ENTRIES; j++) { 389 | if (!bitMap[j]) { 390 | uint8_t keyHash = deleteNode->getFingerPrintArray()[i]; 391 | std::pair &kv = deleteNode->getKeyArray()[i]; 392 | insertAtIndex(kv, j, keyHash, false); 393 | cur = j + 1; 394 | break; 395 | } 396 | } 397 | } 398 | } 399 | flushToNVM((char*)this, sizeof(ListNode)); 400 | smp_wmb(); 401 | ListNode *next = deleteNode->getNext(); 402 | setMax(deleteNode->getMax()); 403 | deleteNode->setDeleted(true); 404 | this->setNext(deleteNode->getNextPtr()); 405 | flushToNVM((char*)this, L1_CACHE_BYTES); 406 | smp_wmb(); 407 | next->setPrev(curPtr); 408 | flushToNVM((char*)next, L1_CACHE_BYTES); 409 | smp_wmb(); 410 | mtx[core].unlock(); 411 | //printf("case 5\n"); 412 | //exit(0); //case 4 413 | 414 | return deleteNode; 415 | } 416 | 417 | ListNode* ListNode :: mergeWithPrev(uint64_t genId) 418 | { 419 | ListNode* deleteNode = this; 420 | ListNode* mergeNode = prevPtr.getVaddr(); 421 | if (!mergeNode->writeLock(genId)) return nullptr; 422 | 423 | // 1) Add Oplog and set the infomration for current(overflown) node. 424 | 425 | int numLogsPerThread = 1000; 426 | 427 | int chip, core; 428 | read_coreid_rdtscp(&chip,&core); 429 | if(core!=coreId) 430 | coreId=core; 431 | 432 | mtx[core].lock(); 433 | int logIdx= numLogsPerThread *(core) + logCnt; 434 | logCnt++; 435 | OpStruct* oplog = (OpStruct *)PMem::getOpLog(logIdx); 436 | 437 | #ifdef MULTIPOOL 438 | uint16_t poolId = (uint16_t)(3*chip+1); 439 | #else 440 | uint16_t poolId = 1; 441 | #endif 442 | 443 | Oplog::writeOpLog(oplog, OpStruct::remove, mergeNode->getMin(),(void*)mergeNode->getNextPtr().getRawPtr(), poolId, -1, -1); 444 | oplog->newNodeOid = pmemobj_oid(deleteNode); 445 | flushToNVM((char*)oplog,sizeof(OpStruct)); 446 | smp_wmb(); 447 | //printf("case 6\n"); 448 | //exit(1); //case 6 449 | 450 | 451 | 452 | int cur = 0; 453 | for (int i = 0; i < MAX_ENTRIES; i++) { 454 | if (deleteNode->getBitMap()->test(i)) { 455 | for (int j = cur; j < MAX_ENTRIES; j++) { 456 | if (!mergeNode->getBitMap()->test(j)) { 457 | uint8_t keyHash = deleteNode->getFingerPrintArray()[i]; 458 | Key_t key = deleteNode->getKeyArray()[i].first; 459 | Val_t val = deleteNode->getValueArray()[i].second; 460 | mergeNode->insertAtIndex(std::make_pair(key, val), j, keyHash, false); 461 | cur = j + 1; 462 | break; 463 | } 464 | } 465 | } 466 | } 467 | flushToNVM((char*)mergeNode, sizeof(ListNode)); 468 | smp_wmb(); 469 | // printf("case 7\n"); 470 | //exit(1); //case 7 471 | 472 | ListNode *next = deleteNode->getNext(); 473 | mergeNode->setMax(deleteNode->getMax()); 474 | mergeNode->setNext(deleteNode->getNextPtr()); 475 | flushToNVM((char*)mergeNode, L1_CACHE_BYTES); 476 | smp_wmb(); 477 | // printf("case 8\n"); 478 | //exit(1); //case 8 479 | 480 | next->setPrev(prevPtr); 481 | flushToNVM((char*)next, L1_CACHE_BYTES); 482 | smp_wmb(); 483 | 484 | //printf("case 9\n"); 485 | //exit(1); //case 9 486 | mergeNode->writeUnlock(); 487 | mtx[core].unlock(); 488 | return deleteNode; 489 | } 490 | 491 | bool ListNode :: insertAtIndex(std::pair key, int index, uint8_t keyHash, bool flush) 492 | { 493 | keyArray[index] = key; 494 | if(flush){ 495 | flushToNVM((char*)&keyArray[index],sizeof(keyArray[index])); 496 | } 497 | 498 | fingerPrint[index] = keyHash; 499 | if(flush){ 500 | flushToNVM((char*)&fingerPrint[index],sizeof(uint8_t)); 501 | smp_wmb(); 502 | } 503 | 504 | bitMap.set(index); 505 | if(flush){ 506 | flushToNVM((char*)this, L1_CACHE_BYTES); 507 | smp_wmb(); 508 | } 509 | return true; 510 | } 511 | 512 | bool ListNode :: updateAtIndex(Val_t value, int index) 513 | { 514 | keyArray[index].second = value; 515 | flushToNVM((char*)&keyArray[index],sizeof(keyArray[index])); 516 | smp_wmb(); 517 | return true; 518 | } 519 | 520 | bool ListNode :: removeFromIndex(int index) 521 | { 522 | bitMap.reset(index); 523 | flushToNVM((char*)this, L1_CACHE_BYTES); 524 | smp_wmb(); 525 | return true; 526 | } 527 | 528 | uint8_t ListNode :: getKeyInsertIndex(Key_t key) 529 | { 530 | for (uint8_t i = 0; i < MAX_ENTRIES; i++) { 531 | if (!bitMap[i]) return i; 532 | } 533 | } 534 | 535 | 536 | #if MAX_ENTRIES==64 537 | 538 | int ListNode:: getKeyIndex(Key_t key, uint8_t keyHash) { 539 | __m512i v1 = _mm512_loadu_si512((__m512i*)fingerPrint); 540 | __m512i v2 = _mm512_set1_epi8((int8_t)keyHash); 541 | uint64_t mask = _mm512_cmpeq_epi8_mask(v1, v2); 542 | uint64_t bitMapMask = bitMap.to_ulong(); 543 | uint64_t posToCheck_64 = (mask) & bitMapMask; 544 | uint32_t posToCheck = (uint32_t) posToCheck_64; 545 | while(posToCheck) { 546 | int pos; 547 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 548 | if (keyArray[pos].first == key) 549 | return pos; 550 | posToCheck = posToCheck & (~(1 << pos)); 551 | } 552 | posToCheck = (uint32_t) (posToCheck_64 >> 32); 553 | while(posToCheck) { 554 | int pos; 555 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 556 | if (keyArray[pos + 32].first == key) 557 | return pos + 32; 558 | posToCheck = posToCheck & (~(1 << pos)); 559 | } 560 | return -1; 561 | 562 | } 563 | 564 | int ListNode:: getFreeIndex(Key_t key, uint8_t keyHash) { 565 | int freeIndex; 566 | int numEntries = getNumEntries(); 567 | if (numEntries != 0 && getKeyIndex(key, keyHash) != -1) return -1; 568 | 569 | uint64_t bitMapMask = bitMap.to_ulong(); 570 | if (!(~bitMapMask)) return -2; 571 | 572 | uint64_t freeIndexMask = ~(bitMapMask); 573 | if ((uint32_t)freeIndexMask) 574 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 575 | else { 576 | freeIndexMask = freeIndexMask >> 32; 577 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 578 | freeIndex += 32; 579 | } 580 | return freeIndex; 581 | 582 | } 583 | #elif MAX_ENTRIES==128 584 | 585 | int ListNode:: getKeyIndex(Key_t key, uint8_t keyHash) { 586 | __m512i v1 = _mm512_loadu_si512((__m512i*)fingerPrint); 587 | __m512i v2 = _mm512_set1_epi8((int8_t)keyHash); 588 | uint64_t mask = _mm512_cmpeq_epi8_mask(v1, v2); 589 | uint64_t bitMapMask = bitMap.to_ulong(0); 590 | uint64_t posToCheck_64 = (mask) & bitMapMask; 591 | uint32_t posToCheck = (uint32_t) posToCheck_64; 592 | while(posToCheck) { 593 | int pos; 594 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 595 | if (keyArray[pos].first == key) 596 | return pos; 597 | posToCheck = posToCheck & (~(1 << pos)); 598 | } 599 | posToCheck = (uint32_t) (posToCheck_64 >> 32); 600 | while(posToCheck) { 601 | int pos; 602 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 603 | if (keyArray[pos + 32].first == key) 604 | return pos + 32; 605 | posToCheck = posToCheck & (~(1 << pos)); 606 | } 607 | v1 = _mm512_loadu_si512((__m512i*)&fingerPrint[64]); 608 | mask = _mm512_cmpeq_epi8_mask(v1, v2); 609 | bitMapMask = bitMap.to_ulong(1); 610 | posToCheck_64 = mask & bitMapMask; 611 | posToCheck = (uint32_t) posToCheck_64; 612 | while(posToCheck) { 613 | int pos; 614 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 615 | if (keyArray[pos + 64].first == key) 616 | return pos + 64; 617 | posToCheck = posToCheck & (~(1 << pos)); 618 | } 619 | posToCheck = (uint32_t) (posToCheck_64 >> 32); 620 | while(posToCheck) { 621 | int pos; 622 | asm("bsrl %1, %0" : "=r"(pos) : "r"((uint32_t)posToCheck)); 623 | if (keyArray[pos + 96].first == key) 624 | return pos + 96; 625 | posToCheck = posToCheck & (~(1 << pos)); 626 | } 627 | return -1; 628 | 629 | } 630 | 631 | int ListNode:: getFreeIndex(Key_t key, uint8_t keyHash) { 632 | int freeIndex; 633 | int numEntries = getNumEntries(); 634 | if (numEntries != 0 && getKeyIndex(key, keyHash) != -1) return -1; 635 | 636 | uint64_t bitMapMask_lower = bitMap.to_ulong(0); 637 | uint64_t bitMapMask_upper = bitMap.to_ulong(1); 638 | if ((~(bitMapMask_lower) == 0x0) && (~(bitMapMask_upper) == 0x0)) return -2; 639 | else if (~(bitMapMask_lower) != 0x0) { 640 | 641 | uint64_t freeIndexMask = ~(bitMapMask_lower); 642 | if ((uint32_t)freeIndexMask) 643 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 644 | else { 645 | freeIndexMask = freeIndexMask >> 32; 646 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 647 | freeIndex += 32; 648 | } 649 | 650 | return freeIndex; 651 | } else { 652 | uint64_t freeIndexMask = ~(bitMapMask_upper); 653 | if ((uint32_t)freeIndexMask) { 654 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 655 | freeIndex += 64; 656 | } 657 | else { 658 | freeIndexMask = freeIndexMask >> 32; 659 | asm("bsf %1, %0" : "=r"(freeIndex) : "r"((uint32_t)freeIndexMask)); 660 | freeIndex += 96; 661 | } 662 | 663 | return freeIndex; 664 | } 665 | 666 | } 667 | 668 | #else 669 | int ListNode:: getKeyIndex(Key_t key, uint8_t keyHash) { 670 | int count = 0; 671 | int numEntries = getNumEntries(); 672 | for (uint8_t i = 0; i < MAX_ENTRIES; i++) { 673 | if (bitMap[i] && keyHash == fingerPrint[i] && keyArray[i].first == key) 674 | return (int) i; 675 | if (bitMap[i]) count++; 676 | if (count == numEntries) break; 677 | } 678 | return -1; 679 | 680 | } 681 | 682 | int ListNode:: getFreeIndex(Key_t key, uint8_t keyHash) { 683 | int freeIndex = -2; 684 | int count = 0; 685 | int numEntries = getNumEntries(); 686 | bool keyCheckNeeded = true; 687 | for (uint8_t i = 0; i < MAX_ENTRIES; i++) { 688 | if (keyCheckNeeded && bitMap[i] && keyHash == fingerPrint[i] && keyArray[i].first == key) 689 | return -1; 690 | if (freeIndex == -2 && !bitMap[i]) freeIndex = i; 691 | if (bitMap[i]) { 692 | count++; 693 | if (count == numEntries) 694 | keyCheckNeeded = false; 695 | } 696 | } 697 | return freeIndex; 698 | 699 | } 700 | #endif 701 | 702 | bool ListNode::insert(Key_t key, Val_t value,int threadId) { 703 | uint8_t keyHash = getKeyFingerPrint(key); 704 | int index = getFreeIndex(key, keyHash); 705 | if (index == -1) return false; // Key exitst 706 | if (index == -2) { //No free index 707 | pptr newNodePtr=split(key, value, keyHash,threadId); 708 | ListNode* newNode =newNodePtr.getVaddr(); 709 | ListNode* nextNode = newNode->getNext(); 710 | nextNode->setPrev(newNodePtr); 711 | return true; 712 | } 713 | if (!insertAtIndex(std::make_pair(key, value), (uint8_t)index, keyHash, true)) 714 | return false; 715 | return true; 716 | } 717 | #ifdef SYNC 718 | int ListNode::insert(Key_t key, Val_t value, void **locked_parent_node) { 719 | uint8_t keyHash = getKeyFingerPrint(key); 720 | int index = getFreeIndex(key, keyHash); 721 | if (index == -1) return false; // Key exitst 722 | if (index == -2) { //No free index 723 | SearchLayer* sl = g_perNumaSlPtr[0]; 724 | OpStruct *oplog; 725 | pptr newNodePtr=split(key, value, keyHash,0,(void**)&oplog); 726 | ListNode* newNode =newNodePtr.getVaddr(); 727 | ListNode* nextNode = newNode->getNext(); 728 | nextNode->setPrev(newNodePtr); 729 | if(*locked_parent_node!=nullptr){ 730 | sl->nodeUnlock(*locked_parent_node); 731 | *locked_parent_node = nullptr; 732 | } 733 | 734 | void *nNode= reinterpret_cast (((unsigned long)(oplog->poolId)) << 48 | (oplog->newNodeOid.off)); 735 | sl->insert(oplog->key,(void *)nNode); 736 | writeUnlock(); 737 | return true; 738 | } 739 | if (!insertAtIndex(std::make_pair(key, value), (uint8_t)index, keyHash, true)){ 740 | writeUnlock(); 741 | return false; 742 | } 743 | writeUnlock(); 744 | return true; 745 | } 746 | #endif 747 | 748 | bool ListNode::update(Key_t key, Val_t value) { 749 | uint8_t keyHash = getKeyFingerPrint(key); 750 | int index = getKeyIndex(key, keyHash); 751 | if (index == -1) return false; // Key Does not exit 752 | if (!updateAtIndex(value, (uint8_t)index)) 753 | return false; 754 | return true; 755 | } 756 | 757 | bool ListNode :: remove(Key_t key, uint64_t genId) 758 | { 759 | uint8_t keyHash = getKeyFingerPrint(key); 760 | int index = getKeyIndex(key, keyHash); 761 | int numEntries = getNumEntries(); 762 | ListNode *prev = prevPtr.getVaddr(); 763 | ListNode *next= nextPtr.getVaddr(); 764 | if (index == -1) 765 | return false; 766 | if (!removeFromIndex(index)) 767 | return false; 768 | if (numEntries + next->getNumEntries() < MAX_ENTRIES/2) { 769 | ListNode* delNode = mergeWithNext(genId); 770 | return true; 771 | } 772 | if (prev != NULL && numEntries + prev->getNumEntries() < MAX_ENTRIES/2) { 773 | ListNode* delNode = mergeWithPrev(genId); 774 | } 775 | return true; 776 | } 777 | 778 | bool ListNode :: checkRange(Key_t key) 779 | { 780 | return min <= key && key < max; 781 | } 782 | bool ListNode :: checkRangeLookup(Key_t key) 783 | { 784 | int numEntries = getNumEntries(); 785 | int ne = numEntries; 786 | return min <= key && key <= keyArray[ne-1].first; 787 | } 788 | 789 | bool ListNode::probe(Key_t key) { 790 | int keyHash = getKeyFingerPrint(key); 791 | int index = getKeyIndex(key, keyHash); 792 | if (index == -1) 793 | return false; 794 | else 795 | return true; 796 | } 797 | 798 | bool ListNode::lookup(Key_t key, Val_t &value) { 799 | int keyHash = getKeyFingerPrint(key); 800 | int index = getKeyIndex(key, keyHash); 801 | if (index == -1){ 802 | return false; 803 | } 804 | else { 805 | value = keyArray[index].second; 806 | return true; 807 | } 808 | } 809 | 810 | void ListNode::print() { 811 | int numEntries = getNumEntries(); 812 | printf("numEntries:%d min:%s max :%s\n",numEntries, getMin(),getMax()); 813 | #ifdef STRINGKEY 814 | for(int i = 0; i < MAX_ENTRIES; i++){ 815 | if(bitMap[i]) 816 | printf("%s, ",keyArray[i].first); 817 | } 818 | #else 819 | for(int i = 0; i < MAX_ENTRIES; i++){ 820 | if(bitMap[i]) 821 | printf("%lu, ",keyArray[i].first); 822 | } 823 | #endif 824 | std::cout << "::"< &rangeVector, uint64_t writeVersion, uint64_t genId) { 828 | restart: 829 | ListNode* next = nextPtr.getVaddr(); 830 | if (next == nullptr) 831 | return true; 832 | 833 | int todo = static_cast(range - rangeVector.size()); 834 | int numEntries = getNumEntries(); 835 | if (todo < 1) assert(0 && "ListNode:: scan: todo < 1"); 836 | if (writeVersion > lastScanVersion) { 837 | if(!generatePermuter(writeVersion, genId)){ 838 | goto restart; 839 | } 840 | } 841 | uint8_t startIndex = 0; 842 | if (startKey > min) startIndex = permuterLowerBound(startKey); 843 | for (uint8_t i = startIndex; i < numEntries && todo > 0; i++) { 844 | rangeVector.push_back(keyArray[permuter[i]].second); 845 | todo--; 846 | } 847 | return rangeVector.size() == range; 848 | } 849 | 850 | 851 | uint8_t ListNode::getKeyFingerPrint(Key_t key) { 852 | #ifdef STRINGKEY 853 | uint32_t length = key.size(); 854 | uint32_t hash = length; 855 | const char* str = key.getData(); 856 | 857 | for (uint32_t i = 0; i < length; ++str, ++i) { 858 | hash = ((hash << 5) ^ (hash >> 27)) ^ (*str); 859 | } 860 | return (uint8_t) (hash & 0xFF); 861 | #else 862 | key = (~key) + (key << 18); // key = (key << 18) - key - 1; 863 | key = key ^ (key >> 31); 864 | key = (key + (key << 2)) + (key << 4); 865 | key = key ^ (key >> 11); 866 | key = key + (key << 6); 867 | key = key ^ (key >> 22); 868 | return (uint8_t) (key); 869 | #endif 870 | } 871 | 872 | hydra::bitset *ListNode::getBitMap() { 873 | return &bitMap; 874 | } 875 | 876 | uint8_t *ListNode::getFingerPrintArray() { 877 | return fingerPrint; 878 | } 879 | 880 | bool ListNode::generatePermuter(uint64_t writeVersion,uint64_t genId) { 881 | if(pLock.write_lock(genId)==0){ 882 | return false; 883 | } 884 | if (writeVersion == lastScanVersion) { 885 | pLock.write_unlock(); 886 | return true; 887 | } 888 | std::vector> copyArray; 889 | for (uint8_t i = 0; i < MAX_ENTRIES; i++) { 890 | if (bitMap[i]) copyArray.push_back(std::make_pair(keyArray[i].first, i)); 891 | } 892 | #ifdef STRINGKEY 893 | std::sort(copyArray.begin(), copyArray.end(), compare2); 894 | #else 895 | std::sort(copyArray.begin(), copyArray.end()); 896 | #endif 897 | for (uint8_t i = 0; i < copyArray.size(); i++) { 898 | permuter[i] = copyArray[i].second; 899 | } 900 | flushToNVM((char*)permuter,L1_CACHE_BYTES); 901 | smp_wmb(); 902 | lastScanVersion = writeVersion; 903 | flushToNVM((char*)&lastScanVersion,sizeof(uint64_t)); 904 | smp_wmb(); 905 | pLock.write_unlock(); 906 | return true; 907 | } 908 | 909 | int ListNode :: permuterLowerBound(Key_t key) 910 | { 911 | int lower = 0; 912 | int numEntries = getNumEntries(); 913 | int upper = numEntries; 914 | do { 915 | int mid = ((upper-lower)/2) + lower; 916 | int actualMid = permuter[mid]; 917 | if (key < keyArray[actualMid].first) { 918 | upper = mid; 919 | } else if (key > keyArray[actualMid].first) { 920 | lower = mid + 1; 921 | } else { 922 | return mid; 923 | } 924 | } while (lower < upper); 925 | return (uint8_t) lower; 926 | } 927 | pptr ListNode::recoverSplit(OpStruct *olog){ 928 | uint8_t keyHash = getKeyFingerPrint(olog->newKey); 929 | pptr newNodePtr=split(olog->newKey, olog->newVal, keyHash,0); 930 | return newNodePtr; 931 | } 932 | 933 | 934 | void ListNode::recoverNode(Key_t min_key){ 935 | // delete duplicate entries 936 | hydra::bitset curBitMap = *(this->getBitMap()); 937 | for(int i=0; igetBitMap()); 949 | bool exist = false; 950 | int idx = -1; 951 | for(int i=0; igetKeyArray()[i].first==keyArray[j].first){ 957 | exist=true; 958 | break; 959 | } 960 | } 961 | } 962 | else if(idx==-1){ 963 | idx = i; 964 | } 965 | if(!exist){ 966 | Key_t key= deleted_node->getKeyArray()[i].first; 967 | Val_t val = deleted_node->getKeyArray()[i].second; 968 | uint8_t keyHash = getKeyFingerPrint(key); 969 | insertAtIndex(std::make_pair(key, val), (uint8_t)idx, keyHash, true); 970 | } 971 | } 972 | } 973 | 974 | -------------------------------------------------------------------------------- /src/listNode.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef _LISTNODE_H 5 | #define _LISTNODE_H 6 | 7 | #include 8 | #include "bitset.h" 9 | #include "VersionedLock.h" 10 | #include "common.h" 11 | #include "arch.h" 12 | #include "ordo_clock.h" 13 | #include "../lib/PDL-ART/Tree.h" 14 | #include 15 | #include 16 | 17 | //#define MAX_ENTRIES 32 18 | #define MAX_ENTRIES 64 19 | //#define MAX_ENTRIES 128 20 | class ListNode 21 | { 22 | private: 23 | Key_t min; // 8B 24 | Key_t max; //8B 25 | // uint8_t numEntries; //1B 26 | pptr curPtr; 27 | pptr nextPtr; 28 | pptr prevPtr; 29 | bool deleted; // 1B 30 | hydra::bitset bitMap; //8B 31 | uint8_t fingerPrint[MAX_ENTRIES]; //64B 32 | VersionedLock verLock; //8B 33 | 34 | std::pair keyArray[MAX_ENTRIES]; // 16*64 = 1024B 35 | 36 | uint64_t lastScanVersion; // 8B 37 | VersionedLock pLock; // ? 38 | //std::mutex pLock; // ? 39 | uint8_t permuter[MAX_ENTRIES]; //64B 40 | 41 | pptr split(Key_t key, Val_t val, uint8_t keyHash, int threadId); 42 | pptr split(Key_t key, Val_t val, uint8_t keyHash, int threadId, void **olog); 43 | ListNode* mergeWithNext(uint64_t genId); 44 | ListNode* mergeWithPrev(uint64_t genId); 45 | uint8_t getKeyInsertIndex(Key_t key); 46 | int getKeyIndex(Key_t key, uint8_t keyHash); 47 | int getFreeIndex(Key_t key, uint8_t keyHash); 48 | bool insertAtIndex(std::pair key, int index, uint8_t keyHash, bool flush); 49 | bool updateAtIndex(Val_t value, int index); 50 | bool removeFromIndex(int index); 51 | int lowerBound(Key_t key); 52 | uint8_t getKeyFingerPrint(Key_t key); 53 | bool generatePermuter(uint64_t writeVersion,uint64_t genId); 54 | int permuterLowerBound(Key_t key); 55 | 56 | public: 57 | ListNode(); 58 | #ifdef SYNC 59 | int insert(Key_t key, Val_t value, void **locked_parent_node); 60 | #endif 61 | bool insert(Key_t key, Val_t value, int threadId); 62 | bool update(Key_t key, Val_t value); 63 | bool remove(Key_t key, uint64_t genId); 64 | bool probe(Key_t key); //return True if key exists 65 | bool lookup(Key_t key, Val_t &value); 66 | bool scan(Key_t startKey, int range, std::vector &rangeVector, uint64_t writeVersion, uint64_t genId); 67 | void print(); 68 | bool checkRange(Key_t key); 69 | bool checkRangeLookup(Key_t key); 70 | version_t readLock(uint64_t genId); 71 | version_t writeLock(uint64_t genId); 72 | bool readUnlock(version_t); 73 | void writeUnlock(); 74 | void setCur(pptr next); 75 | void setNext(pptr next); 76 | void setPrev(pptr prev); 77 | void setMin(Key_t key); 78 | void setMax(Key_t key); 79 | // void setNumEntries(int numEntries); 80 | void setDeleted(bool deleted); 81 | void setFullBitmap() ; 82 | ListNode* getNext(); 83 | pptr getNextPtr(); 84 | ListNode* getPrev(); 85 | pptr getPrevPtr(); 86 | pptr getCurPtr(); 87 | int getNumEntries(); 88 | Key_t getMin(); 89 | Key_t getMax(); 90 | std::pair* getKeyArray(); 91 | std::pair* getValueArray(); 92 | hydra::bitset *getBitMap(); 93 | uint8_t *getFingerPrintArray(); 94 | bool getDeleted(); 95 | void recoverNode(Key_t); 96 | pptr recoverSplit(OpStruct* olog); 97 | void recoverMergingNode(ListNode *deleted_node); 98 | 99 | }; 100 | #endif 101 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include "pactree.h" 5 | 6 | int main() { 7 | return 0; 8 | } 9 | -------------------------------------------------------------------------------- /src/pactree.cpp: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #include 5 | #include 6 | #include 7 | #include "numa.h" 8 | #include "pactreeImpl.h" 9 | #include "numa-config.h" 10 | #include "Combiner.h" 11 | #include "WorkerThread.h" 12 | #include 13 | 14 | 15 | #ifdef PACTREE_ENABLE_STATS 16 | #define acc_sl_time(x) (curThreadData->sltime+=x) 17 | #define acc_dl_time(x) (curThreadData->dltime+=x) 18 | #define hydralist_reset_timers() \ 19 | total_sl_time = 0; \ 20 | total_dl_time = 0; 21 | #define hydralist_start_timer() \ 22 | { \ 23 | unsigned long __b_e_g_i_n__; \ 24 | __b_e_g_i_n__ = read_tsc(); 25 | #define hydralist_stop_timer(tick) \ 26 | (tick) += read_tsc() - __b_e_g_i_n__; \ 27 | } 28 | #else 29 | #define acc_sl_time(x) 30 | #define acc_dl_time(x) 31 | #define hydralist_start_timer() 32 | #define hydralist_stop_timer(tick) 33 | #define hydralist_reset_timers() 34 | #endif 35 | 36 | std::vector g_WorkerThreadInst(MAX_NUMA * WORKER_THREAD_PER_NUMA); 37 | std::vector g_perNumaSlPtr(MAX_NUMA); 38 | std::set g_threadDataSet; 39 | std::atomic g_globalStop; 40 | std::atomic g_combinerStop; 41 | thread_local int pactreeImpl::threadNumaNode = -1; 42 | thread_local ThreadData* curThreadData = NULL; 43 | int pactreeImpl::totalNumaActive = 0; 44 | volatile bool threadInitialized[MAX_NUMA]; 45 | volatile bool slReady[MAX_NUMA]; 46 | std::mutex g_threadDataLock; 47 | uint64_t g_removeCount; 48 | volatile std::atomic g_removeDetected; 49 | 50 | void *PMem::baseAddresses[] = {}; 51 | void *PMem::logVaddr[] = {}; 52 | 53 | void workerThreadExec(int threadId, int activeNuma, root_obj *root) { 54 | WorkerThread wt(threadId, activeNuma); 55 | g_WorkerThreadInst[threadId] = &wt; 56 | if (threadId < activeNuma) { 57 | while (threadInitialized[threadId] == 0); 58 | slReady[threadId] = false; 59 | // assert(numa_node_of_cpu(sched_getcpu()) == threadId); 60 | g_perNumaSlPtr[threadId] = pactreeImpl::createSearchLayer(root,threadId); 61 | g_perNumaSlPtr[threadId]->setNuma(threadId); 62 | slReady[threadId] = true; 63 | } 64 | int count = 0; 65 | uint64_t lastRemoveCount = 0; 66 | while(!g_combinerStop) { 67 | usleep(200); 68 | while(!wt.isWorkQueueEmpty()) { 69 | count++; 70 | wt.applyOperation(); 71 | } 72 | if (threadId == 0 && lastRemoveCount != g_removeCount) { 73 | // wt.freeListNodes(g_removeCount); 74 | } 75 | } 76 | while(!wt.isWorkQueueEmpty()) { 77 | count++; 78 | if (wt.applyOperation() && !g_removeDetected) { 79 | g_removeDetected.store(true); 80 | } 81 | } 82 | //If worker threads are stopping that means there are no more 83 | //user threads 84 | if (threadId == 0) { 85 | wt.freeListNodes(ULLONG_MAX); 86 | } 87 | g_WorkerThreadInst[threadId] = NULL; 88 | 89 | printf("Worker thread: %d Ops: %d\n", threadId, wt.opcount); 90 | } 91 | 92 | uint64_t gracePeriodInit(std::vector &threadsToWait) { 93 | uint64_t curTime = ordo_get_clock(); 94 | for (auto td : g_threadDataSet) { 95 | if (td->getFinish()) { 96 | g_threadDataLock.lock(); 97 | g_threadDataSet.erase(td); 98 | g_threadDataLock.unlock(); 99 | free(td); 100 | continue; 101 | } 102 | if (td->getRunCnt() % 2) { 103 | if (ordo_gt_clock(td->getLocalClock(), curTime)) continue; 104 | else threadsToWait.push_back(td); 105 | } 106 | } 107 | return curTime; 108 | 109 | } 110 | void waitForThreads(std::vector &threadsToWait, uint64_t gpStartTime) { 111 | for (int i = 0; i < threadsToWait.size(); i++) { 112 | if (threadsToWait[i] == NULL) continue; 113 | ThreadData* td = threadsToWait[i]; 114 | if (td->getRunCnt() % 2 == 0) continue; 115 | if (ordo_gt_clock(td->getLocalClock(), gpStartTime)) continue; 116 | while (td->getRunCnt() % 2) { 117 | usleep(1); 118 | std::atomic_thread_fence(std::memory_order::memory_order_acq_rel); 119 | } 120 | } 121 | } 122 | void broadcastDoneCount(uint64_t removeCount) { 123 | g_removeCount = removeCount; 124 | } 125 | void combinerThreadExec(int activeNuma) { 126 | CombinerThread ct; 127 | int count = 0; 128 | while(!g_globalStop) { 129 | std::vector *mergedLog = ct.combineLogs(); 130 | if (mergedLog != nullptr){ 131 | count++; 132 | ct.broadcastMergedLog(mergedLog, activeNuma); 133 | } 134 | uint64_t doneCountWt = ct.freeMergedLogs(activeNuma, false); 135 | std::vector threadsToWait; 136 | if (g_removeDetected && doneCountWt != 0) { 137 | uint64_t gpStartTime = gracePeriodInit(threadsToWait); 138 | waitForThreads(threadsToWait, gpStartTime); 139 | broadcastDoneCount(doneCountWt); 140 | g_removeDetected.store(false); 141 | } else 142 | usleep(100); 143 | } 144 | //TODO fix this 145 | int i = 20; 146 | while(i--){ 147 | usleep(200); 148 | std::vector *mergedLog = ct.combineLogs(); 149 | if (mergedLog != nullptr){ 150 | count++; 151 | ct.broadcastMergedLog(mergedLog, activeNuma); 152 | } 153 | } 154 | while (!ct.mergedLogsToBeFreed()) 155 | ct.freeMergedLogs(activeNuma, true); 156 | g_combinerStop = true; 157 | printf("Combiner thread: Ops: %d\n", count); 158 | } 159 | 160 | void pinThread(std::thread *t, int numaId) { 161 | cpu_set_t cpuSet; 162 | CPU_ZERO(&cpuSet); 163 | for (int i = 0; i < NUM_SOCKET; i++) { 164 | for (int j = 0; j < SMT_LEVEL; j++) { 165 | CPU_SET(OS_CPU_ID[numaId][i][j], &cpuSet); 166 | } 167 | } 168 | int rc = pthread_setaffinity_np(t->native_handle(), sizeof(cpu_set_t), &cpuSet); 169 | assert(rc == 0); 170 | } 171 | int cnt=0; 172 | void pactreeImpl::createWorkerThread(int numNuma, root_obj *root) { 173 | for(int i = 0; i < numNuma * WORKER_THREAD_PER_NUMA; i++) { 174 | threadInitialized[i % numNuma] = false; 175 | std::thread* wt = new std::thread(workerThreadExec, i, numNuma,root); 176 | usleep(1); 177 | wtArray->push_back(wt); 178 | pinThread(wt, i % numNuma); 179 | threadInitialized[i % numNuma] = true; 180 | } 181 | 182 | } 183 | 184 | void pactreeImpl::createCombinerThread() { 185 | combinerThead = new std::thread(combinerThreadExec, totalNumaActive); 186 | } 187 | 188 | pactreeImpl *initPT(int numa){ 189 | const char* path = "/mnt/pmem0/dl"; 190 | size_t sz = 1UL*1024UL*1024UL*1024UL; //10GB 191 | int isCreated = 0; 192 | int isCreated2 = 0; 193 | root_obj* root = nullptr; 194 | root_obj* sl_root = nullptr; 195 | 196 | const char *sl_path = "/mnt/pmem0/sl"; 197 | size_t sl_size = 1UL*1024UL*1024UL*1024UL; 198 | 199 | PMem::bind(0,sl_path,sl_size,(void **)&sl_root,&isCreated); 200 | if (isCreated == 0) { 201 | printf("Reading Search layer from an existing pactree.\n"); 202 | 203 | } 204 | const char* log_path = "/mnt/pmem0/log"; 205 | PMem::bindLog(0,log_path,sz); 206 | 207 | PMem::bind(1,path,sz,(void **)&root,&isCreated2); 208 | 209 | #ifdef MULTIPOOL 210 | const char* path2 = "/mnt/pmem1/dl"; 211 | const char* sl_path2 = "/mnt/pmem1/sl"; 212 | const char* log_path2 = "/mnt/pmem1/log"; 213 | root_obj* root2 = nullptr; 214 | root_obj* sl_root2 = nullptr; 215 | PMem::bind(3,sl_path2,sl_size,(void **)&sl_root2,&isCreated); 216 | PMem::bind(4,path2,sz,(void **)&root2,&isCreated); 217 | PMem::bindLog(1,log_path2,sz); 218 | #endif 219 | if (isCreated2 == 0) { 220 | pactreeImpl *pt = (pactreeImpl*) pmemobj_direct(root->ptr[0]); 221 | pt->init(numa,sl_root); 222 | return pt; 223 | } 224 | PMEMobjpool *pop = (PMEMobjpool *)PMem::getBaseOf(1); 225 | 226 | int ret = pmemobj_alloc(pop, &(root->ptr[0]), sizeof(pactreeImpl), 0, NULL, NULL); 227 | void *rootVaddr = pmemobj_direct(root->ptr[0]); 228 | pactreeImpl *pt= (pactreeImpl *)new(rootVaddr) pactreeImpl(numa,sl_root); 229 | flushToNVM((char *)root, sizeof(root_obj)); 230 | smp_wmb(); 231 | return pt; 232 | } 233 | 234 | void pactreeImpl::init(int numNuma, root_obj* root) { 235 | int id = 0; 236 | 237 | assert(numNuma <= NUM_SOCKET); 238 | totalNumaActive = numNuma; 239 | wtArray= new std::vector(totalNumaActive); 240 | g_WorkerThreadInst.clear(); 241 | g_perNumaSlPtr.clear(); 242 | g_threadDataSet.clear(); 243 | 244 | g_perNumaSlPtr.resize(totalNumaActive); 245 | g_globalStop = false; 246 | g_combinerStop = false; 247 | createWorkerThread(numNuma,root); 248 | createCombinerThread(); 249 | for(int i = 0; i < totalNumaActive; i++) { 250 | while(slReady[i] == false); 251 | } 252 | recover(); 253 | hydralist_reset_timers(); 254 | } 255 | 256 | pactreeImpl::pactreeImpl(int numNuma, root_obj* root) { 257 | 258 | assert(numNuma <= NUM_SOCKET); 259 | totalNumaActive = numNuma; 260 | wtArray= new std::vector(totalNumaActive); 261 | g_perNumaSlPtr.resize(totalNumaActive); 262 | dl.initialize(); 263 | // need to read from PM 264 | 265 | g_globalStop = false; 266 | g_combinerStop = false; 267 | #ifdef SYNC 268 | totalNumaActive = 1; 269 | g_perNumaSlPtr.resize(totalNumaActive); 270 | g_perNumaSlPtr[0] = pactreeImpl::createSearchLayer(root,0); 271 | #else 272 | createWorkerThread(numNuma,root); 273 | createCombinerThread(); 274 | for(int i = 0; i < totalNumaActive; i++) { 275 | while(slReady[i] == false); 276 | } 277 | #endif 278 | hydralist_reset_timers(); 279 | } 280 | 281 | pactreeImpl::~pactreeImpl() { 282 | g_globalStop = true; 283 | for (auto &t : *wtArray) 284 | t->join(); 285 | combinerThead->join(); 286 | 287 | std::cout << "Total splits: " << numSplits << std::endl; 288 | std::cout << "Combiner splits: " << combinerSplits << std::endl; 289 | #ifdef PACTREE_ENABLE_STATS 290 | std::cout << "total_dl_time: " << total_dl_time/numThreads/1000 << std::endl; 291 | std::cout << "total_sl_time: " << total_sl_time/numThreads/1000 << std::endl; 292 | #endif 293 | } 294 | 295 | ListNode *pactreeImpl::getJumpNode(Key_t &key) { 296 | int numaNode = getThreadNuma(); 297 | SearchLayer& sl = *g_perNumaSlPtr[numaNode]; 298 | if (sl.isEmpty()) { 299 | return dl.getHead(); 300 | } 301 | ListNode *jumpNode = reinterpret_cast(sl.lookup(key)); 302 | // idxPtr2 jumpNode = reinterpret_cast(sl.lookup(key)); 303 | if (jumpNode == nullptr) jumpNode = dl.getHead(); 304 | return jumpNode; 305 | } 306 | 307 | #ifdef SYNC 308 | 309 | // lock interface 310 | ListNode* pactreeImpl::getJumpNodewithLock(Key_t &key, void** node) { 311 | SearchLayer& sl = *g_perNumaSlPtr[0]; 312 | if (sl.isEmpty()) return dl.getHead(); 313 | auto * jumpNode = reinterpret_cast(sl.lookupwithLock(key, node)); 314 | if (jumpNode == nullptr){ 315 | jumpNode = dl.getHead(); 316 | } 317 | return jumpNode; 318 | } 319 | 320 | // unlock interface 321 | bool pactreeImpl::JumpNodewithUnLock(void* node) { 322 | SearchLayer& sl = *g_perNumaSlPtr[0]; 323 | 324 | return sl.nodeUnlock(node); 325 | } 326 | #endif 327 | 328 | 329 | 330 | bool pactreeImpl::insert(Key_t &key, Val_t val) { 331 | uint64_t clock = ordo_get_clock(); 332 | curThreadData->read_lock(clock); 333 | uint64_t ticks = 0; 334 | 335 | int threadId = curThreadData->getThreadId (); 336 | bool ret; 337 | hydralist_start_timer(); 338 | ListNode *jumpNode; 339 | #ifdef SYNC 340 | void *art_node=nullptr; 341 | jumpNode = getJumpNodewithLock(key, &art_node); 342 | #else 343 | jumpNode = getJumpNode(key); 344 | #endif 345 | 346 | hydralist_stop_timer(ticks); 347 | hydralist_start_timer(); 348 | acc_sl_time(ticks); 349 | #ifndef SYNC 350 | ret = dl.insert(key, val, jumpNode,threadId); 351 | #else 352 | ret = dl.insert(key, val, jumpNode,&art_node); 353 | if(art_node!=nullptr){ 354 | JumpNodewithUnLock(art_node) ; 355 | } 356 | #endif 357 | 358 | 359 | hydralist_stop_timer(ticks); 360 | acc_dl_time(ticks); 361 | curThreadData->read_unlock(); 362 | 363 | return ret; 364 | } 365 | 366 | bool pactreeImpl::update(Key_t &key, Val_t val) { 367 | uint64_t clock = ordo_get_clock(); 368 | curThreadData->read_lock(clock); 369 | 370 | ListNode *jumpNode = getJumpNode(key); 371 | 372 | bool ret = dl.update(key, val, jumpNode); 373 | curThreadData->read_unlock(); 374 | return ret; 375 | } 376 | 377 | Val_t pactreeImpl::lookup(Key_t &key) { 378 | 379 | uint64_t clock = ordo_get_clock(); 380 | curThreadData->read_lock(clock); 381 | Val_t val; 382 | uint64_t ticks; 383 | //hydralist_start_timer(); 384 | 385 | ListNode *jumpNode = getJumpNode(key); 386 | //hydralist_stop_timer(ticks); 387 | //acc_sl_time(ticks); 388 | 389 | //hydralist_start_timer(); 390 | // 391 | dl.lookup(key, val, jumpNode); 392 | //hydralist_stop_timer(ticks); 393 | //acc_dl_time(ticks); 394 | curThreadData->read_unlock(); 395 | return val; 396 | } 397 | 398 | bool pactreeImpl::remove(Key_t &key) { 399 | uint64_t clock = ordo_get_clock(); 400 | curThreadData->read_lock(clock); 401 | 402 | ListNode *jumpNode = getJumpNode(key); 403 | 404 | bool ret = dl.remove(key, jumpNode); 405 | curThreadData->read_unlock(); 406 | return ret; 407 | } 408 | 409 | SearchLayer *pactreeImpl::createSearchLayer(root_obj *root, int threadId) { 410 | if(pmemobj_direct(root->ptr[threadId])==nullptr){ 411 | pptr sPtr; 412 | PMem::alloc(0,sizeof(SearchLayer),(void **)&sPtr,&(root->ptr[threadId])); 413 | SearchLayer *s = new(sPtr.getVaddr()) SearchLayer(); 414 | return s; 415 | } 416 | else{ 417 | SearchLayer *s = (SearchLayer *)pmemobj_direct(root->ptr[threadId]); 418 | s->init(); 419 | return s; 420 | } 421 | 422 | // Read from Root Object or Allocate new one. 423 | } 424 | 425 | unsigned long tacc_rdtscp(int *chip, int *core) { 426 | unsigned long a,d,c; 427 | __asm__ volatile("rdtscp" : "=a" (a), "=d" (d), "=c" (c)); 428 | *chip = (c & 0xFFF000)>>12; 429 | *core = c & 0xFFF; 430 | return ((unsigned long)a) | (((unsigned long)d) << 32);; 431 | } 432 | 433 | int pactreeImpl::getThreadNuma() { 434 | int chip; 435 | int core; 436 | if(threadNumaNode == -1) { 437 | tacc_rdtscp(&chip, &core); 438 | threadNumaNode = chip; 439 | if(threadNumaNode > 8) 440 | assert(0); 441 | } 442 | if(totalNumaActive <= threadNumaNode) 443 | return 0; 444 | else 445 | return threadNumaNode; 446 | } 447 | 448 | uint64_t pactreeImpl::scan(Key_t &startKey, int range, std::vector &result) { 449 | ListNode *jumpNode = getJumpNode(startKey); 450 | 451 | return dl.scan(startKey, range, result, jumpNode); 452 | } 453 | 454 | void pactreeImpl::registerThread() { 455 | int threadId = numThreads.fetch_add(1); 456 | auto td = new ThreadData(threadId); 457 | g_threadDataLock.lock(); 458 | g_threadDataSet.insert(td); 459 | g_threadDataLock.unlock(); 460 | curThreadData = td; 461 | std::atomic_thread_fence(std::memory_order_acq_rel); 462 | } 463 | 464 | void pactreeImpl::unregisterThread() { 465 | if (curThreadData == NULL) return; 466 | int threadId = curThreadData->getThreadId(); 467 | curThreadData->setfinish(); 468 | #ifdef PACTREE_ENABLE_STATS 469 | total_dl_time.fetch_add(curThreadData->dltime); 470 | total_sl_time.fetch_add(curThreadData->sltime); 471 | #endif 472 | } 473 | 474 | void pactreeImpl::recover() { 475 | dl.Recovery(g_perNumaSlPtr[0]); 476 | } 477 | -------------------------------------------------------------------------------- /src/pactreeImpl.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef pactree_H 5 | #define pactree_H 6 | 7 | #include 8 | #include 9 | #include 10 | #include "common.h" 11 | #include "linkedList.h" 12 | #include 13 | #include 14 | #include "SearchLayer.h" 15 | #include "threadData.h" 16 | 17 | extern std::vector g_perNumaSlPtr; 18 | extern std::set g_threadDataSet; 19 | typedef LinkedList DataLayer; 20 | class pactreeImpl { 21 | private: 22 | // SearchLayer sl; 23 | DataLayer dl; 24 | std::vector *wtArray; // CurrentOne but there should be numa number of threads 25 | std::thread* combinerThead; 26 | static thread_local int threadNumaNode; 27 | void createWorkerThread(int numNuma,root_obj *root); 28 | void createCombinerThread(); 29 | ListNode *getJumpNode(Key_t &key); 30 | static int totalNumaActive; 31 | std::atomic numThreads; 32 | 33 | public: 34 | pptr sl; 35 | explicit pactreeImpl(int numNuma,root_obj *root); 36 | ~pactreeImpl(); 37 | bool insert(Key_t &key, Val_t val); 38 | bool update(Key_t &key, Val_t val); 39 | bool remove(Key_t &key); 40 | void registerThread(); 41 | void unregisterThread(); 42 | Val_t lookup(Key_t &key); 43 | void recover(); 44 | #ifdef SYNC 45 | ListNode* getJumpNodewithLock(Key_t &key, void** node); 46 | bool JumpNodewithUnLock(void* node); 47 | #endif 48 | uint64_t scan(Key_t &startKey, int range, std::vector &result); 49 | static SearchLayer* createSearchLayer(root_obj *root, int threadId); 50 | static int getThreadNuma(); 51 | void init(int numNuma, root_obj* root) ; 52 | 53 | #ifdef PACTREE_ENABLE_STATS 54 | std::atomic total_sl_time; 55 | std::atomic total_dl_time; 56 | #endif 57 | }; 58 | 59 | 60 | pactreeImpl *initPT(int numa); 61 | #endif //pactree_H 62 | -------------------------------------------------------------------------------- /src/threadData.h: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | #ifndef pactree_THREADS_H 5 | #define pactree_THREADS_H 6 | #include "common.h" 7 | #include 8 | 9 | class ThreadData { 10 | private: 11 | int threadId; 12 | uint64_t localClock; 13 | bool finish; 14 | volatile std::atomic runCnt; 15 | public: 16 | ThreadData(int threadId) { 17 | this->threadId = threadId; 18 | this->finish = false; 19 | this->runCnt = 0; 20 | #ifdef PACTREE_ENABLE_STATS 21 | sltime = 0; 22 | dltime = 0; 23 | #endif 24 | } 25 | #ifdef PACTREE_ENABLE_STATS 26 | uint64_t sltime; 27 | uint64_t dltime; 28 | #endif 29 | void setThreadId (int threadId) {this->threadId = threadId;} 30 | int getThreadId () {return this->threadId;} 31 | void setfinish() {this->finish = true;} 32 | bool getFinish() {return this->finish;} 33 | void setLocalClock (uint64_t clock) {this->localClock = clock;} 34 | uint64_t getLocalClock() {return this->localClock;} 35 | void incrementRunCntAtomic() { runCnt.fetch_add(1);}; 36 | void incrementRunCnt() { runCnt++;}; 37 | uint64_t getRunCnt() {return this->runCnt;} 38 | void read_lock(uint64_t clock) { 39 | this->setLocalClock(clock); 40 | this->incrementRunCntAtomic(); 41 | } 42 | void read_unlock() { 43 | this->incrementRunCnt(); 44 | } 45 | 46 | 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /tools/cpu-topology.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import sys 8 | import argparse 9 | 10 | ''' 11 | /proc/cpuinfo 12 | processor : => kernel cpu id 13 | core id : => pcpu id 14 | physical id : => socket 15 | cpu cores : => number of cores in a socket 16 | ''' 17 | 18 | def get_num_socket(cpuinfo): 19 | socket_id_set = set() 20 | for cpu in cpuinfo: 21 | socket_id_set.add(int(cpu["physical id"])) 22 | return len(socket_id_set) 23 | 24 | def build_cpu_topology(cpuinfo_file, lang): 25 | cpuinfo = [dict(map(str.strip, line.split(":", 1)) 26 | for line in block.splitlines()) 27 | for block in open(cpuinfo_file).read().split("\n\n") 28 | if len(block.strip())] 29 | topology_map = {} 30 | socket_id_set = set() 31 | physical_cpu_id_set = set() 32 | for cpu in cpuinfo: 33 | socket_id = cpu["physical id"] 34 | socket_id_set.add( int(socket_id)) 35 | physical_cpu_id = cpu["core id"] 36 | physical_cpu_id_set.add( int(physical_cpu_id)) 37 | os_cpu_id = cpu["processor"] 38 | key = ":".join([socket_id, physical_cpu_id]) 39 | topology_map.setdefault(key, []).append(os_cpu_id) 40 | if lang == "c": 41 | print_cpu_topology_in_c(cpuinfo, topology_map, socket_id_set, 42 | physical_cpu_id_set) 43 | elif lang == "python": 44 | print_cpu_topology_in_python(cpuinfo, topology_map, socket_id_set, 45 | physical_cpu_id_set) 46 | else: 47 | print("Unknow language: %s\n" % lang) 48 | 49 | def print_cpu_topology_in_c(cpuinfo, topology_map, socket_id_set, physical_cpu_id_set): 50 | num_socket = get_num_socket(cpuinfo) 51 | num_physical_cpu_per_socket = int(cpuinfo[0]["cpu cores"]) 52 | smt_level = int(len(cpuinfo) / (num_socket * num_physical_cpu_per_socket)) 53 | 54 | print("enum {") 55 | print(" NUM_SOCKET = %s," % num_socket) 56 | print(" NUM_PHYSICAL_CPU_PER_SOCKET = %s," 57 | % num_physical_cpu_per_socket) 58 | print(" SMT_LEVEL = %s," % smt_level) 59 | print("};") 60 | print("") 61 | 62 | print("const int OS_CPU_ID[%s][%s][%s] = {" 63 | % ("NUM_SOCKET", "NUM_PHYSICAL_CPU_PER_SOCKET", "SMT_LEVEL")) 64 | for socket_id in sorted(socket_id_set): 65 | print(" { /* socket id: %s */" % socket_id) 66 | for physical_cpu_id in sorted(physical_cpu_id_set): 67 | key = ":".join([str(socket_id), str(physical_cpu_id)]) 68 | print(" { /* physical cpu id: %s */" % physical_cpu_id) 69 | print(" ", end = "") 70 | for (smt_id, os_cpu_id) in enumerate(topology_map[key]): 71 | print("%s, " % os_cpu_id, end = "") 72 | print(" },") # physical cpu id 73 | print(" },") # socket id 74 | print("};") 75 | 76 | def print_cpu_topology_in_python(cpuinfo, topology_map, socket_id_set, physical_cpu_id_set): 77 | num_socket = get_num_socket(cpuinfo) 78 | num_physical_cpu_per_socket = int(cpuinfo[0]["cpu cores"]) 79 | smt_level = int(len(cpuinfo) / (num_socket * num_physical_cpu_per_socket)) 80 | 81 | print("#!/usr/bin/env python3") 82 | print("") 83 | print("NUM_SOCKET = %s" % num_socket) 84 | print("NUM_PHYSICAL_CPU_PER_SOCKET = %s" % num_physical_cpu_per_socket) 85 | print("SMT_LEVEL = %s" % smt_level) 86 | print("") 87 | print("OS_CPU_ID = {} # socket_id, physical_cpu_id, smt_id") 88 | for socket_id in sorted(socket_id_set): 89 | for physical_cpu_id in sorted(physical_cpu_id_set): 90 | key = ":".join([str(socket_id), str(physical_cpu_id)]) 91 | for (smt_id, os_cpu_id) in enumerate(topology_map[key]): 92 | print("OS_CPU_ID[%s,%s,%s] = %s" % 93 | (socket_id, physical_cpu_id, smt_id, os_cpu_id)) 94 | print("") 95 | 96 | def parse_option(): 97 | """Parse command line opetion""" 98 | # arg parser 99 | parser = argparse.ArgumentParser() 100 | parser.add_argument('--cpuinfo', '-c', nargs='?', default="/proc/cpuinfo", 101 | help="path names of cpuinfo") 102 | parser.add_argument('--lang', '-l', nargs='?', choices=["c", "python"], default="c", 103 | help="output language") 104 | # parse args 105 | args = parser.parse_args() 106 | cpuinfo = args.cpuinfo 107 | lang = args.lang 108 | 109 | # return args 110 | return (cpuinfo, lang) 111 | 112 | if __name__ == "__main__": 113 | (cpuinfo, lang) = parse_option() 114 | build_cpu_topology(cpuinfo, lang) 115 | 116 | -------------------------------------------------------------------------------- /tools/get-numa-config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 2019-2021 Virginia Tech 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | python3 tools/cpu-topology.py > ./include/numa-config.h 7 | --------------------------------------------------------------------------------