├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── TODO.md ├── TellStoreConfig.cmake.in ├── client ├── CMakeLists.txt ├── ClientManager.cpp ├── ClientSocket.cpp ├── ScanMemory.cpp └── Table.cpp ├── cmake ├── FindJemalloc.cmake ├── FindSparsehash.cmake └── FindTBB.cmake ├── common ├── CMakeLists.txt ├── GenericTuple.cpp ├── MessageTypes.cpp └── Record.cpp ├── config.h.in ├── deltamain ├── CMakeLists.txt ├── DeltaMainRewriteStore.hpp ├── InsertHash.cpp ├── InsertHash.hpp ├── Record.cpp ├── Record.hpp ├── Table.cpp ├── Table.hpp ├── colstore │ ├── ColumnMapContext.cpp │ ├── ColumnMapContext.hpp │ ├── ColumnMapPage.cpp │ ├── ColumnMapPage.hpp │ ├── ColumnMapRecord.cpp │ ├── ColumnMapRecord.hpp │ ├── ColumnMapScanProcessor.cpp │ ├── ColumnMapScanProcessor.hpp │ ├── LLVMColumnMapAggregation.cpp │ ├── LLVMColumnMapAggregation.hpp │ ├── LLVMColumnMapMaterialize.cpp │ ├── LLVMColumnMapMaterialize.hpp │ ├── LLVMColumnMapProjection.cpp │ ├── LLVMColumnMapProjection.hpp │ ├── LLVMColumnMapScan.cpp │ ├── LLVMColumnMapScan.hpp │ ├── LLVMColumnMapUtils.cpp │ └── LLVMColumnMapUtils.hpp └── rowstore │ ├── RowStoreContext.hpp │ ├── RowStorePage.cpp │ ├── RowStorePage.hpp │ ├── RowStoreRecord.cpp │ ├── RowStoreRecord.hpp │ ├── RowStoreScanProcessor.cpp │ └── RowStoreScanProcessor.hpp ├── logstructured ├── CMakeLists.txt ├── ChainedVersionRecord.hpp ├── GcScanProcessor.cpp ├── GcScanProcessor.hpp ├── HashScanProcessor.cpp ├── HashScanProcessor.hpp ├── LogstructuredMemoryStore.hpp ├── Table.cpp ├── Table.hpp ├── VersionRecordIterator.cpp └── VersionRecordIterator.hpp ├── server ├── CMakeLists.txt ├── ServerConfig.hpp ├── ServerScanQuery.cpp ├── ServerScanQuery.hpp ├── ServerSocket.cpp ├── ServerSocket.hpp ├── Storage.hpp └── main.cpp ├── tellstore ├── AbstractTuple.hpp ├── ClientConfig.hpp ├── ClientManager.hpp ├── ClientSocket.hpp ├── ErrorCode.hpp ├── GenericTuple.hpp ├── MessageTypes.hpp ├── Record.hpp ├── ScanMemory.hpp ├── StdTypes.hpp ├── Table.hpp ├── TransactionRunner.hpp └── TransactionType.hpp ├── tests ├── CMakeLists.txt ├── DummyCommitManager.cpp ├── DummyCommitManager.hpp ├── client │ └── TestClient.cpp ├── deltamain │ └── testInsertHash.cpp ├── logstructured │ └── testTable.cpp ├── main.cpp ├── simpleTests.cpp ├── testCommitManager.cpp ├── testCuckooMap.cpp ├── testLog.cpp └── testOpenAddressingHash.cpp └── util ├── CMakeLists.txt ├── CuckooHash.cpp ├── CuckooHash.hpp ├── LLVMBuilder.cpp ├── LLVMBuilder.hpp ├── LLVMJIT.cpp ├── LLVMJIT.hpp ├── LLVMRowAggregation.cpp ├── LLVMRowAggregation.hpp ├── LLVMRowProjection.cpp ├── LLVMRowProjection.hpp ├── LLVMRowScan.cpp ├── LLVMRowScan.hpp ├── LLVMScan.cpp ├── LLVMScan.hpp ├── Log.cpp ├── Log.hpp ├── OpenAddressingHash.cpp ├── OpenAddressingHash.hpp ├── PageManager.cpp ├── PageManager.hpp ├── Scan.hpp ├── ScanQuery.cpp ├── ScanQuery.hpp ├── StorageConfig.hpp ├── TableManager.hpp ├── UnsafeAtomic.hpp ├── VersionManager.hpp └── functional.hpp /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.swp 3 | *.pyc 4 | compile_commands.json 5 | .clang_complete 6 | build 7 | CMakeLists.txt.user 8 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "externals/gtest"] 2 | path = externals/gtest 3 | url = https://github.com/google/googletest.git 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.12) 2 | project(TellStore) 3 | 4 | set(PAGE_SIZE "0x200000" CACHE STRING "The pagesize to use in bytes") 5 | set(TOTAL_MEMORY "0x80000000" CACHE STRING "The pagesize to use in bytes") 6 | set(MAX_QUERY_SHARING "1024" CACHE STRING "The maximal number of queries a scan query accepts") 7 | set(HASHMAP_CAPACITY "0x800000" CACHE STRING "Number of elements to allocate for the hashmap") 8 | 9 | # Set default install paths 10 | set(BIN_INSTALL_DIR bin CACHE PATH "Installation directory for binaries") 11 | set(CMAKE_INSTALL_DIR cmake CACHE PATH "Installation directory for CMake files") 12 | set(INCLUDE_INSTALL_DIR include CACHE PATH "Installation directory for header files") 13 | set(LIB_INSTALL_DIR lib CACHE PATH "Installation directory for libraries") 14 | 15 | # Set the TellStore directory 16 | set(TellStore_DIR ${CMAKE_CURRENT_BINARY_DIR} CACHE PATH "Path to the TellStore binaries and configuration") 17 | 18 | # Set CMake modules path 19 | list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) 20 | 21 | # Set compile options 22 | # The cx16 flag is required for GCC to enable 128 bit atomics 23 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -mcx16") 24 | 25 | # Find dependencies 26 | find_package(Boost REQUIRED) 27 | find_package(Crossbow COMPONENTS Allocator InfinIO Logger REQUIRED) 28 | find_package(CommitManager REQUIRED) 29 | find_package(Sparsehash REQUIRED) 30 | find_package(TBB REQUIRED) 31 | find_package(Jemalloc REQUIRED) 32 | find_package(LLVM 3.7 EXACT REQUIRED) 33 | 34 | # Create configuration file 35 | configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h) 36 | 37 | add_subdirectory(common) 38 | add_subdirectory(util) 39 | add_subdirectory(deltamain) 40 | add_subdirectory(logstructured) 41 | add_subdirectory(server) 42 | add_subdirectory(client) 43 | 44 | ################### 45 | # GTEST 46 | ################### 47 | add_subdirectory(externals/gtest) 48 | 49 | enable_testing() 50 | add_subdirectory(tests) 51 | 52 | # Create cmake config file 53 | configure_file(TellStoreConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/TellStoreConfig.cmake @ONLY) 54 | install(FILES ${CMAKE_CURRENT_BINARY_DIR}/TellStoreConfig.cmake DESTINATION ${CMAKE_INSTALL_DIR}) 55 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | ### General 2 | 3 | - [ ] Take PageManager out of the epoch mechanism 4 | - [ ] Add DropTable command 5 | - [x] Fix alignment in serialized records 6 | - [ ] Do not crash on shutdown 7 | - [ ] Cache SnapshotDescriptor in server 8 | - [ ] Move SnapshotDescriptor code from CommitManager to TellStore (do not link against CommitManager) 9 | - [ ] Profile and improve scan query evaluation 10 | - [x] Materialize tuple directly into InfinIO buffer 11 | 12 | ### Delta-Main Rewrite 13 | 14 | - [ ] Fill nearly empty clean pages with inserts during garbage collection 15 | - [ ] Truncate update log only up to the point where all entries are sealed 16 | - [ ] ColumnMap: Only write into update page when table has variable sized fields 17 | - [x] ColumnMap: Grow insert hash table on demand 18 | 19 | ### Log-Structured Memory 20 | 21 | - [ ] Elements must be written to the log in version-chain order for later replication 22 | - [ ] Write Revert logs into version chain 23 | -------------------------------------------------------------------------------- /TellStoreConfig.cmake.in: -------------------------------------------------------------------------------- 1 | set(_comp_file "${CMAKE_CURRENT_LIST_DIR}/TellStoreClientTargets.cmake") 2 | if(TARGET tellstore-client) 3 | set(TellStore_FOUND True) 4 | elseif(EXISTS ${_comp_file}) 5 | include(${_comp_file}) 6 | set(TellStore_FOUND True) 7 | else() 8 | set(TellStore_FOUND False) 9 | endif() 10 | -------------------------------------------------------------------------------- /client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore client library 3 | ################### 4 | set(CLIENT_SRCS 5 | ClientManager.cpp 6 | ClientSocket.cpp 7 | ScanMemory.cpp 8 | Table.cpp 9 | ) 10 | 11 | set(CLIENT_PUBLIC_HDR 12 | ClientConfig.hpp 13 | ClientManager.hpp 14 | ClientSocket.hpp 15 | ScanMemory.hpp 16 | Table.hpp 17 | TransactionRunner.hpp 18 | TransactionType.hpp 19 | StdTypes.hpp 20 | ) 21 | 22 | # Transform public header list to use absolute paths 23 | foreach(hdr ${CLIENT_PUBLIC_HDR}) 24 | list(APPEND CLIENT_PUBLIC_HDR_ABSOLUTE ${PROJECT_SOURCE_DIR}/tellstore/${hdr}) 25 | endforeach() 26 | 27 | # Retrieve the public header list for the common library (client has to install those headers) 28 | get_target_property(COMMON_PUBLIC_HDR_ABSOLUTE tellstore-common PUBLIC_HEADER) 29 | 30 | # Add TellStore client library 31 | add_library(tellstore-client ${CLIENT_SRCS} ${CLIENT_PUBLIC_HDR_ABSOLUTE}) 32 | set_target_properties(tellstore-client PROPERTIES PUBLIC_HEADER 33 | "${COMMON_PUBLIC_HDR_ABSOLUTE};${CLIENT_PUBLIC_HDR_ABSOLUTE}") 34 | target_include_directories(tellstore-client PUBLIC $/${INCLUDE_INSTALL_DIR}> 35 | $) 36 | 37 | # Workaround for link failure with GCC 5 (GCC Bug 65913) 38 | if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") 39 | target_link_libraries(tellstore-client PUBLIC atomic) 40 | endif() 41 | 42 | # Link against TellStore common 43 | target_link_libraries(tellstore-client PRIVATE tellstore-common) 44 | 45 | # Link against CommitManager 46 | target_link_libraries(tellstore-client PUBLIC commitmanager-client) 47 | 48 | # Link against Boost 49 | target_include_directories(tellstore-client PUBLIC ${Boost_INCLUDE_DIRS}) 50 | 51 | # Link against Crossbow 52 | target_include_directories(tellstore-client PRIVATE ${Crossbow_INCLUDE_DIRS}) 53 | target_link_libraries(tellstore-client PUBLIC crossbow_infinio crossbow_logger) 54 | 55 | # Install the library 56 | install(TARGETS tellstore-client EXPORT TellStoreClientTargets 57 | ARCHIVE DESTINATION ${LIB_INSTALL_DIR} 58 | LIBRARY DESTINATION ${LIB_INSTALL_DIR} 59 | PUBLIC_HEADER DESTINATION ${INCLUDE_INSTALL_DIR}/tellstore) 60 | 61 | # Install the cmake target 62 | install(EXPORT TellStoreClientTargets DESTINATION ${CMAKE_INSTALL_DIR}) 63 | -------------------------------------------------------------------------------- /client/ScanMemory.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | namespace tell { 31 | namespace store { 32 | 33 | ScanMemoryManager::ScanMemoryManager(crossbow::infinio::InfinibandService& service, size_t chunkCount, 34 | size_t chunkLength) 35 | : mChunkLength(chunkLength), 36 | mChunkStack(chunkCount, nullptr) { 37 | if (chunkLength % 8 != 0) { 38 | throw std::runtime_error("Length of chunks must be divisible by 8"); 39 | } 40 | mRegion = crossbow::infinio::AllocatedMemoryRegion(service.allocateMemoryRegion(chunkCount * mChunkLength, 41 | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE)); 42 | auto data = mRegion.address(); 43 | for (decltype(chunkCount) i = 0; i < chunkCount; ++i) { 44 | if (!mChunkStack.push(reinterpret_cast(data))) { 45 | throw std::runtime_error("Unable to push scan chunk on stack"); 46 | } 47 | data += mChunkLength; 48 | } 49 | } 50 | 51 | ScanMemory ScanMemoryManager::acquire() { 52 | void* data; 53 | if (!mChunkStack.pop(data)) { 54 | return ScanMemory(); 55 | } 56 | 57 | return ScanMemory(this, data, mChunkLength, mRegion.rkey()); 58 | } 59 | 60 | void ScanMemoryManager::release(void* data) { 61 | LOG_ASSERT((reinterpret_cast(data) >= mRegion.address()) 62 | && (reinterpret_cast(data) < mRegion.address() + mRegion.length()), 63 | "Data pointing not into the chunk") 64 | LOG_ASSERT((reinterpret_cast(data) - mRegion.address()) % mChunkLength == 0u, 65 | "Data pointing not at the beginning of a chunk") 66 | while (!mChunkStack.push(data)); 67 | } 68 | 69 | ScanMemory::~ScanMemory() { 70 | if (mManager != nullptr) { 71 | mManager->release(mData); 72 | } 73 | } 74 | 75 | ScanMemory& ScanMemory::operator=(ScanMemory&& other) { 76 | if (mManager != nullptr) { 77 | mManager->release(mData); 78 | } 79 | mManager = other.mManager; 80 | other.mManager = nullptr; 81 | 82 | mData = other.mData; 83 | other.mData = nullptr; 84 | 85 | mLength = other.mLength; 86 | other.mLength = 0u; 87 | 88 | mKey = other.mKey; 89 | other.mKey = 0u; 90 | 91 | return *this; 92 | } 93 | 94 | } // namespace store 95 | } // namespace tell 96 | -------------------------------------------------------------------------------- /client/Table.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace tell { 29 | namespace store { 30 | 31 | void* Tuple::operator new(size_t size, uint32_t dataLen) { 32 | LOG_ASSERT(size == sizeof(Tuple), "Requested size does not match Tuple size"); 33 | return ::malloc(size + dataLen); 34 | } 35 | 36 | void Tuple::operator delete(void* ptr) { 37 | ::free(ptr); 38 | } 39 | 40 | std::unique_ptr Tuple::deserialize(crossbow::buffer_reader& reader) { 41 | auto version = reader.read(); 42 | auto isNewest = reader.read(); 43 | reader.align(sizeof(uint32_t)); 44 | auto size = reader.read(); 45 | 46 | std::unique_ptr tuple(new (size) Tuple(version, isNewest, size)); 47 | if (tuple && size > 0) { 48 | memcpy(const_cast(tuple->data()), reader.read(size), size); 49 | } 50 | return tuple; 51 | } 52 | 53 | GenericTuple Table::toGenericTuple(const char* data) const { 54 | GenericTuple tuple; 55 | 56 | for (decltype(mRecord.fieldCount()) id = 0; id < mRecord.fieldCount(); ++id) { 57 | auto& fieldMeta = mRecord.getFieldMeta(id); 58 | 59 | bool isNull = false; 60 | FieldType type; 61 | auto field = mRecord.data(data, id, isNull, &type); 62 | if (isNull) { 63 | continue; 64 | } 65 | 66 | boost::any value; 67 | switch (type) { 68 | 69 | case FieldType::SMALLINT: { 70 | value = *reinterpret_cast(field); 71 | } break; 72 | 73 | case FieldType::INT: { 74 | value = *reinterpret_cast(field); 75 | } break; 76 | 77 | case FieldType::BIGINT: { 78 | value = *reinterpret_cast(field); 79 | } break; 80 | 81 | case FieldType::FLOAT: { 82 | value = *reinterpret_cast(field); 83 | } break; 84 | 85 | case FieldType::DOUBLE: { 86 | value = *reinterpret_cast(field); 87 | } break; 88 | 89 | case FieldType::TEXT: 90 | case FieldType::BLOB: { 91 | auto offsetData = reinterpret_cast(field); 92 | auto offset = offsetData[0]; 93 | auto length = offsetData[1] - offset; 94 | value = crossbow::string(data + offset, length); 95 | } break; 96 | 97 | default: { 98 | throw std::logic_error("Invalid field type"); 99 | } break; 100 | } 101 | 102 | tuple.emplace(fieldMeta.field.name(), std::move(value)); 103 | } 104 | 105 | return tuple; 106 | } 107 | 108 | } // namespace store 109 | } // namespace tell 110 | -------------------------------------------------------------------------------- /cmake/FindJemalloc.cmake: -------------------------------------------------------------------------------- 1 | # - Find Jemalloc 2 | 3 | # Look for the jemalloc header 4 | find_path(Jemalloc_INCLUDE_DIR 5 | NAMES jemalloc/jemalloc.h 6 | HINTS ${Jemalloc_ROOT} ENV Jemalloc_ROOT 7 | PATH_SUFFIXES include) 8 | 9 | # Look for the jemalloc library 10 | find_library(Jemalloc_LIBRARY 11 | NAMES jemalloc 12 | HINTS ${Jemalloc_ROOT} ENV Jemalloc_ROOT 13 | PATH_SUFFIXES lib) 14 | 15 | set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY}) 16 | set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR}) 17 | 18 | mark_as_advanced(Jemalloc_INCLUDE_DIR Jemalloc_LIBRARY) 19 | 20 | include(FindPackageHandleStandardArgs) 21 | find_package_handle_standard_args(Jemalloc 22 | FOUND_VAR Jemalloc_FOUND 23 | REQUIRED_VARS Jemalloc_LIBRARY Jemalloc_INCLUDE_DIR) 24 | -------------------------------------------------------------------------------- /cmake/FindSparsehash.cmake: -------------------------------------------------------------------------------- 1 | # - Find Sparsehash 2 | 3 | # Look for the sparsehash header 4 | find_path(Sparsehash_INCLUDE_DIR 5 | NAMES sparsehash/sparsetable 6 | HINTS ${Sparsehash_ROOT} ENV Sparsehash_ROOT 7 | PATH_SUFFIXES include) 8 | 9 | set(Sparsehash_INCLUDE_DIRS ${Sparsehash_INCLUDE_DIR}) 10 | 11 | mark_as_advanced(Sparsehash_INCLUDE_DIR) 12 | 13 | include(FindPackageHandleStandardArgs) 14 | find_package_handle_standard_args(Sparsehash 15 | FOUND_VAR Sparsehash_FOUND 16 | REQUIRED_VARS Sparsehash_INCLUDE_DIR) 17 | -------------------------------------------------------------------------------- /cmake/FindTBB.cmake: -------------------------------------------------------------------------------- 1 | find_path(TBB_INCLUDE_DIR tbb/atomic.h) 2 | find_library(TBB_LIBRARY NAMES tbb libtbb) 3 | 4 | set(TBB_LIBRARIES ${TBB_LIBRARY}) 5 | set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) 6 | 7 | include(FindPackageHandleStandardArgs) 8 | find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARY TBB_INCLUDE_DIR) 9 | mark_as_advanced(TBB_INCLUDE_DIR TBB_LIBRARY) 10 | -------------------------------------------------------------------------------- /common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore common 3 | ################### 4 | set(COMMON_SRCS 5 | GenericTuple.cpp 6 | MessageTypes.cpp 7 | Record.cpp 8 | ) 9 | 10 | set(COMMON_PUBLIC_HDR 11 | AbstractTuple.hpp 12 | ErrorCode.hpp 13 | GenericTuple.hpp 14 | MessageTypes.hpp 15 | Record.hpp 16 | ) 17 | 18 | # Transform public header list to use absolute paths 19 | foreach(hdr ${COMMON_PUBLIC_HDR}) 20 | list(APPEND COMMON_PUBLIC_HDR_ABSOLUTE ${PROJECT_SOURCE_DIR}/tellstore/${hdr}) 21 | endforeach() 22 | 23 | # Add TellStore common library 24 | add_library(tellstore-common STATIC ${COMMON_SRCS} ${COMMON_PUBLIC_HDR_ABSOLUTE}) 25 | target_include_directories(tellstore-common PUBLIC ${PROJECT_SOURCE_DIR}) 26 | set_target_properties(tellstore-common PROPERTIES PUBLIC_HEADER "${COMMON_PUBLIC_HDR_ABSOLUTE}") 27 | 28 | # Link against Crossbow 29 | target_include_directories(tellstore-common PUBLIC ${Crossbow_INCLUDE_DIRS}) 30 | target_link_libraries(tellstore-common PUBLIC crossbow_logger) 31 | -------------------------------------------------------------------------------- /common/GenericTuple.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include 25 | 26 | #include 27 | 28 | namespace tell { 29 | namespace store { 30 | 31 | AbstractTuple::~AbstractTuple() = default; 32 | 33 | GenericTupleSerializer::GenericTupleSerializer(const Record& record, GenericTuple tuple) 34 | : mRecord(record) 35 | , mTuple(std::move(tuple)) 36 | , mSize(mRecord.sizeOfTuple(mTuple)) 37 | {} 38 | 39 | GenericTupleSerializer::~GenericTupleSerializer() = default; 40 | 41 | size_t GenericTupleSerializer::size() const { 42 | return mSize; 43 | } 44 | 45 | void GenericTupleSerializer::serialize(char* dest) const { 46 | mRecord.create(dest, mTuple, mSize); 47 | } 48 | 49 | } // namespace store 50 | } // namespace tell 51 | -------------------------------------------------------------------------------- /common/MessageTypes.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include 25 | 26 | namespace tell { 27 | namespace store { 28 | 29 | const crossbow::string& handshakeString() { 30 | static crossbow::string str("TELLSTORE"); 31 | return str; 32 | } 33 | 34 | } // namespace store 35 | } // namespace tell 36 | -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | // Only activate log scan 6 | #define USE_LOG_SCAN 7 | 8 | namespace tell { 9 | namespace store { 10 | 11 | constexpr size_t TELL_PAGE_SIZE = @PAGE_SIZE@; 12 | constexpr size_t TOTAL_MEMORY = @TOTAL_MEMORY@; 13 | constexpr size_t HASHMAP_CAPACITY = @HASHMAP_CAPACITY@; 14 | constexpr size_t MAX_QUERY_SHARING = @MAX_QUERY_SHARING@; 15 | 16 | } // namespace store 17 | } // namespace tell 18 | -------------------------------------------------------------------------------- /deltamain/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore Delta Main implementation 3 | ################### 4 | set(DELTAMAIN_SRCS 5 | InsertHash.cpp 6 | Record.cpp 7 | Table.cpp 8 | colstore/ColumnMapContext.cpp 9 | colstore/ColumnMapPage.cpp 10 | colstore/ColumnMapRecord.cpp 11 | colstore/ColumnMapScanProcessor.cpp 12 | colstore/LLVMColumnMapAggregation.cpp 13 | colstore/LLVMColumnMapMaterialize.cpp 14 | colstore/LLVMColumnMapProjection.cpp 15 | colstore/LLVMColumnMapScan.cpp 16 | colstore/LLVMColumnMapUtils.cpp 17 | rowstore/RowStorePage.cpp 18 | rowstore/RowStoreRecord.cpp 19 | rowstore/RowStoreScanProcessor.cpp 20 | ) 21 | 22 | set(DELTAMAIN_PRIVATE_HDR 23 | DeltaMainRewriteStore.hpp 24 | InsertHash.hpp 25 | Record.hpp 26 | Table.hpp 27 | colstore/ColumnMapContext.hpp 28 | colstore/ColumnMapPage.hpp 29 | colstore/ColumnMapRecord.hpp 30 | colstore/ColumnMapScanProcessor.hpp 31 | colstore/LLVMColumnMapAggregation.hpp 32 | colstore/LLVMColumnMapMaterialize.hpp 33 | colstore/LLVMColumnMapProjection.hpp 34 | colstore/LLVMColumnMapUtils.hpp 35 | colstore/LLVMColumnMapScan.hpp 36 | rowstore/RowStoreContext.hpp 37 | rowstore/RowStorePage.hpp 38 | rowstore/RowStoreRecord.hpp 39 | rowstore/RowStoreScanProcessor.hpp 40 | ) 41 | 42 | # Add TellStore delta main library 43 | add_library(tellstore-deltamain STATIC ${DELTAMAIN_SRCS} ${DELTAMAIN_PRIVATE_HDR}) 44 | target_include_directories(tellstore-deltamain PUBLIC ${PROJECT_SOURCE_DIR}) 45 | 46 | # Link against TellStore util 47 | target_link_libraries(tellstore-deltamain PUBLIC tellstore-util) 48 | -------------------------------------------------------------------------------- /deltamain/DeltaMainRewriteStore.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "Table.hpp" 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include 34 | #include 35 | 36 | namespace tell { 37 | namespace commitmanager { 38 | class SnapshotDescriptor; 39 | } // namespace commitmanager 40 | 41 | namespace store { 42 | 43 | class ScanQuery; 44 | 45 | template 46 | struct DeltaMainRewriteStore : crossbow::non_copyable, crossbow::non_movable { 47 | using Table = deltamain::Table; 48 | using GC = deltamain::GarbageCollector; 49 | 50 | static const char* implementationName() { 51 | return Context::implementationName(); 52 | } 53 | 54 | DeltaMainRewriteStore(const StorageConfig& config) 55 | : mPageManager(PageManager::construct(config.totalMemory)) 56 | , tableManager(*mPageManager, config, gc, mVersionManager) 57 | { 58 | } 59 | 60 | 61 | DeltaMainRewriteStore(const StorageConfig& config, size_t totalMem) 62 | : mPageManager(PageManager::construct(totalMem)) 63 | , tableManager(*mPageManager, config, gc, mVersionManager) 64 | { 65 | } 66 | 67 | bool createTable(const crossbow::string &name, 68 | const Schema& schema, 69 | uint64_t& idx) 70 | { 71 | return tableManager.createTable(name, schema, idx, tableManager.config().hashMapCapacity); 72 | } 73 | 74 | std::vector getTables() const 75 | { 76 | return tableManager.getTables(); 77 | } 78 | 79 | const Table* getTable(uint64_t id) const 80 | { 81 | return tableManager.getTable(id); 82 | } 83 | 84 | const Table* getTable(const crossbow::string& name, uint64_t& id) const 85 | { 86 | return tableManager.getTable(name, id); 87 | } 88 | 89 | template 90 | int get(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot, Fun fun) 91 | { 92 | return tableManager.get(tableId, key, snapshot, std::move(fun)); 93 | } 94 | 95 | int update(uint64_t tableId, uint64_t key, size_t size, const char* data, 96 | const commitmanager::SnapshotDescriptor& snapshot) 97 | { 98 | return tableManager.update(tableId, key, size, data, snapshot); 99 | } 100 | 101 | int insert(uint64_t tableId, uint64_t key, size_t size, const char* data, 102 | const commitmanager::SnapshotDescriptor& snapshot) 103 | { 104 | return tableManager.insert(tableId, key, size, data, snapshot); 105 | } 106 | 107 | int remove(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot) 108 | { 109 | return tableManager.remove(tableId, key, snapshot); 110 | } 111 | 112 | int revert(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot) 113 | { 114 | return tableManager.revert(tableId, key, snapshot); 115 | } 116 | 117 | int scan(uint64_t tableId, ScanQuery* query) 118 | { 119 | return tableManager.scan(tableId, query); 120 | } 121 | 122 | /** 123 | * We use this method mostly for test purposes. But 124 | * it might be handy in the future as well. If possible, 125 | * this should be implemented in an efficient way. 126 | */ 127 | void forceGC() 128 | { 129 | tableManager.forceGC(); 130 | } 131 | 132 | private: 133 | PageManager::Ptr mPageManager; 134 | GC gc; 135 | VersionManager mVersionManager; 136 | TableManager tableManager; 137 | }; 138 | 139 | using DeltaMainRewriteRowStore = DeltaMainRewriteStore; 140 | using DeltaMainRewriteColumnStore = DeltaMainRewriteStore; 141 | 142 | } // namespace store 143 | } // namespace tell 144 | -------------------------------------------------------------------------------- /deltamain/Record.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "Record.hpp" 25 | 26 | #include 27 | 28 | #include 29 | 30 | namespace tell { 31 | namespace store { 32 | namespace deltamain { 33 | 34 | template 35 | void InsertRecordImpl::collect(uint64_t minVersion, uint64_t highestVersion, std::vector& elements) const { 36 | // Check if the oldest element is the oldest readable element 37 | if (highestVersion <= minVersion) { 38 | return; 39 | } 40 | 41 | auto logEntry = LogEntry::entryFromData(reinterpret_cast(mEntry)); 42 | elements.emplace_back(mEntry->version, mEntry->data(), logEntry->size() - sizeof(InsertLogEntry)); 43 | } 44 | 45 | int InsertRecord::canUpdate(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, 46 | RecordType type) const { 47 | // Check if the element was already overwritten by an element in the update log 48 | if (mEntry->version >= highestVersion) { 49 | return (type == RecordType::DELETE ? 0 : error::invalid_write); 50 | } 51 | 52 | if (!snapshot.inReadSet(mEntry->version)) { 53 | return error::not_in_snapshot; 54 | } 55 | 56 | return (type == RecordType::DATA ? 0 : error::invalid_write); 57 | } 58 | 59 | int InsertRecord::canRevert(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, bool& needsRevert) const { 60 | // The element only needs a revert if it was not overwritten by the update log and has the same version 61 | needsRevert = !(mEntry->version >= highestVersion || mEntry->version != snapshot.version()); 62 | return 0; 63 | } 64 | 65 | UpdateRecordIterator::UpdateRecordIterator(const UpdateLogEntry* record, uint64_t baseVersion) 66 | : mCurrent(record), 67 | mBaseVersion(baseVersion), 68 | mLowestVersion(std::numeric_limits::max()) { 69 | if (!mCurrent) { 70 | return; 71 | } 72 | LOG_ASSERT(mCurrent->version >= mBaseVersion, "Version of element in Update Log must never be lower than " 73 | "base version"); 74 | 75 | // Update lowest version 76 | mLowestVersion = mCurrent->version; 77 | 78 | // Forward version chain if the element was a revert 79 | auto logEntry = LogEntry::entryFromData(reinterpret_cast(mCurrent)); 80 | if (logEntry->type() == crossbow::to_underlying(RecordType::REVERT)) { 81 | next(); 82 | } 83 | } 84 | 85 | void UpdateRecordIterator::next() { 86 | auto entry = mCurrent; 87 | while (true) { 88 | if (entry->version == mBaseVersion) { 89 | mCurrent = nullptr; 90 | return; 91 | } 92 | 93 | // Forward version chain until we see an element with smaller version number or we reached the end 94 | // This assumes that the version chain is sorted by version number which it is not: But the only case where 95 | // a newer version number can follow an older version number is when the newer version was reverted so it 96 | // can be ignored anyway. 97 | do { 98 | entry = reinterpret_cast(entry->previous.load()); 99 | if (!entry || entry->version < mBaseVersion) { 100 | mCurrent = nullptr; 101 | return; 102 | } 103 | } while (entry->version >= mLowestVersion); 104 | 105 | // Update lowest version 106 | mLowestVersion = entry->version; 107 | 108 | // Forward version chain if the element was a revert 109 | auto logEntry = LogEntry::entryFromData(reinterpret_cast(entry)); 110 | if (logEntry->type() != crossbow::to_underlying(RecordType::REVERT)) { 111 | break; 112 | } 113 | } 114 | mCurrent = entry; 115 | } 116 | 117 | template class InsertRecordImpl; 118 | template class InsertRecordImpl; 119 | 120 | } // namespace deltamain 121 | } // namespace store 122 | } // namespace tell 123 | -------------------------------------------------------------------------------- /deltamain/colstore/ColumnMapContext.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "ColumnMapContext.hpp" 25 | 26 | #include "LLVMColumnMapMaterialize.hpp" 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | namespace tell { 43 | namespace store { 44 | namespace deltamain { 45 | 46 | ColumnMapContext::ColumnMapContext(const PageManager& pageManager, const Record& record) 47 | : mRecord(record), 48 | mPageData(reinterpret_cast(pageManager.data())), 49 | mHeaderSize(mRecord.headerSize()), 50 | mFixedSize(0u) { 51 | mFixedMetaData.reserve(mRecord.fixedSizeFieldCount()); 52 | for (decltype(mRecord.fixedSizeFieldCount()) i = 0; i < mRecord.fixedSizeFieldCount(); ++i) { 53 | auto& field = mRecord.getFieldMeta(i).field; 54 | auto fieldLength = field.staticSize(); 55 | 56 | mFixedMetaData.emplace_back(mFixedSize, fieldLength); 57 | mFixedSize += fieldLength; 58 | } 59 | 60 | mStaticSize = sizeof(ColumnMapMainEntry) + sizeof(uint32_t) + mHeaderSize + mFixedSize 61 | + mRecord.varSizeFieldCount() * sizeof(ColumnMapHeapEntry); 62 | mStaticCapacity = MAX_DATA_SIZE / mStaticSize; 63 | 64 | // Build and compile Materialize function via LLVM 65 | prepareMaterializeFunction(); 66 | } 67 | 68 | void ColumnMapContext::prepareMaterializeFunction() { 69 | #ifndef NDEBUG 70 | LOG_INFO("Generating LLVM materialize function"); 71 | auto startTime = std::chrono::steady_clock::now(); 72 | #endif 73 | 74 | // Create LLVM context and module 75 | llvm::LLVMContext context; 76 | llvm::Module module("Materialize", context); 77 | module.setDataLayout(mLLVMJit.getTargetMachine()->createDataLayout()); 78 | module.setTargetTriple(mLLVMJit.getTargetMachine()->getTargetTriple().getTriple()); 79 | 80 | LLVMColumnMapMaterializeBuilder::createFunction(*this, module, mLLVMJit.getTargetMachine()); 81 | 82 | #ifndef NDEBUG 83 | LOG_INFO("Dumping LLVM Code before optimizations"); 84 | module.dump(); 85 | #endif 86 | 87 | // Setup optimizations 88 | llvm::legacy::PassManager modulePass; 89 | #ifndef NDEBUG 90 | modulePass.add(llvm::createVerifierPass()); 91 | #endif 92 | modulePass.add(llvm::createTargetTransformInfoWrapperPass(mLLVMJit.getTargetMachine()->getTargetIRAnalysis())); 93 | 94 | llvm::legacy::FunctionPassManager functionPass(&module); 95 | #ifndef NDEBUG 96 | functionPass.add(llvm::createVerifierPass()); 97 | #endif 98 | functionPass.add(llvm::createTargetTransformInfoWrapperPass(mLLVMJit.getTargetMachine()->getTargetIRAnalysis())); 99 | 100 | llvm::PassManagerBuilder optimizationBuilder; 101 | optimizationBuilder.OptLevel = 2; 102 | optimizationBuilder.LoadCombine = true; 103 | optimizationBuilder.populateFunctionPassManager(functionPass); 104 | optimizationBuilder.populateModulePassManager(modulePass); 105 | optimizationBuilder.populateLTOPassManager(modulePass); 106 | 107 | functionPass.doInitialization(); 108 | for (auto& fun : module) { 109 | functionPass.run(fun); 110 | } 111 | functionPass.doFinalization(); 112 | 113 | modulePass.run(module); 114 | 115 | #ifndef NDEBUG 116 | LOG_INFO("Dumping LLVM Code after optimizations"); 117 | module.dump(); 118 | #endif 119 | 120 | // Compile the module 121 | mLLVMJit.addModule(&module); 122 | 123 | // Get function pointer for materialization function 124 | mMaterializeFun = mLLVMJit.findFunction( 125 | LLVMColumnMapMaterializeBuilder::FUNCTION_NAME); 126 | 127 | #ifndef NDEBUG 128 | auto endTime = std::chrono::steady_clock::now(); 129 | LOG_INFO("Generating LLVM materialize function took %1%ns", (endTime - startTime).count()); 130 | #endif 131 | } 132 | 133 | } // namespace deltamain 134 | } // namespace store 135 | } // namespace tell 136 | -------------------------------------------------------------------------------- /deltamain/colstore/ColumnMapContext.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "ColumnMapPage.hpp" 27 | #include "ColumnMapScanProcessor.hpp" 28 | #include "LLVMColumnMapMaterialize.hpp" 29 | 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | #include 36 | 37 | namespace tell { 38 | namespace store { 39 | 40 | class PageManager; 41 | class Record; 42 | 43 | namespace deltamain { 44 | 45 | class ColumnMapRecord; 46 | class ConstColumnMapRecord; 47 | 48 | /** 49 | * @brief ColumnMap specific metadata for every fixed size field in a record 50 | */ 51 | struct FixedColumnMetaData { 52 | FixedColumnMetaData(uint32_t _offset, uint32_t _length) 53 | : offset(_offset), 54 | length(_length) { 55 | } 56 | 57 | uint32_t offset; 58 | uint32_t length; 59 | }; 60 | 61 | /** 62 | * @brief Context containing approach-specific information for the column map implementation 63 | */ 64 | class ColumnMapContext { 65 | public: 66 | using Scan = ColumnMapScan; 67 | using Page = ColumnMapMainPage; 68 | using PageModifier = ColumnMapPageModifier; 69 | 70 | using MainRecord = ColumnMapRecord; 71 | using ConstMainRecord = ConstColumnMapRecord; 72 | 73 | /** 74 | * @brief Worst case padding overhead on a page 75 | * 76 | * If the number of elements is uneven a 4-byte padding has to be added after the size array so that the following 77 | * record data is 8-byte aligned. A padding of up to 7 bytes needs to be added after the null bytevector so that the 78 | * first fixed size field is 8-byte aligned. In addition a maximum padding of 6 byte has to be inserted to ensure 79 | * that the ColumnMapHeapEntry array is 8-byte aligned after the last fixed size field. 80 | */ 81 | static constexpr uint32_t PAGE_PADDING_OVERHEAD = 17u; 82 | 83 | /** 84 | * @brief Maximum number of bytes that fit into a column map page 85 | * 86 | * If the record contains any variable sized field an additional sentry ColumnMapHeapEntry must be allocated. 87 | */ 88 | static constexpr uint32_t MAX_DATA_SIZE = TELL_PAGE_SIZE 89 | - (sizeof(ColumnMapMainPage) + sizeof(ColumnMapHeapEntry) + PAGE_PADDING_OVERHEAD); 90 | 91 | static const char* implementationName() { 92 | return "Delta-Main Rewrite (Column Map)"; 93 | } 94 | 95 | /** 96 | * @brief Calculate the index of the given entry on the page 97 | */ 98 | static uint32_t pageIndex(const ColumnMapMainPage* page, const ColumnMapMainEntry* entry) { 99 | return static_cast(entry - page->entryData()); 100 | } 101 | 102 | ColumnMapContext(const PageManager& pageManager, const Record& record); 103 | 104 | const Record& record() const { 105 | return mRecord; 106 | } 107 | 108 | /** 109 | * @brief Size of an element in the column map (excluding the variable heap) 110 | */ 111 | uint32_t staticSize() const { 112 | return mStaticSize; 113 | } 114 | 115 | /** 116 | * @brief Maximum number of elements fitting in one page (without the variable size heap) 117 | */ 118 | uint32_t staticCapacity() const { 119 | return mStaticCapacity; 120 | } 121 | 122 | /** 123 | * @brief Size of the header segment of a record 124 | */ 125 | uint32_t headerSize() const { 126 | return mHeaderSize; 127 | } 128 | 129 | /** 130 | * @brief Size of the fixed size value segment of a record 131 | */ 132 | uint32_t fixedSize() const { 133 | return mFixedSize; 134 | } 135 | 136 | /** 137 | * @brief ColumnMap specific metadata for all static sized fields of a record 138 | */ 139 | const std::vector& fixedMetaData() const { 140 | return mFixedMetaData; 141 | } 142 | 143 | /** 144 | * @brief The page the given element is located on 145 | */ 146 | const ColumnMapMainPage* pageFromEntry(const ColumnMapMainEntry* entry) const { 147 | return reinterpret_cast(reinterpret_cast(entry) 148 | - ((reinterpret_cast(entry) - mPageData) % TELL_PAGE_SIZE)); 149 | } 150 | 151 | /** 152 | * @brief Materialize the tuple from the page into the destination buffer 153 | */ 154 | void materialize(const ColumnMapMainPage* page, uint32_t idx, char* dest) const { 155 | mMaterializeFun(reinterpret_cast(page), idx, dest); 156 | } 157 | 158 | /** 159 | * @brief Function pointer for materializing a tuple from the page 160 | */ 161 | LLVMColumnMapMaterializeBuilder::Signature materializeFunction() const { 162 | return mMaterializeFun; 163 | } 164 | 165 | /** 166 | * @brief The complete size a record will fill in a page 167 | */ 168 | uint32_t calculateFillSize(const char* data) const { 169 | return mStaticSize + mRecord.heapSize(data); 170 | } 171 | 172 | private: 173 | void prepareMaterializeFunction(); 174 | 175 | const Record& mRecord; 176 | 177 | /// Pointer to the start of the page manager data region 178 | uintptr_t mPageData; 179 | 180 | /// \copydoc ColumnMapContext::staticSize() const 181 | uint32_t mStaticSize; 182 | 183 | /// \copydoc ColumnMapContext::staticCapacity() const 184 | uint32_t mStaticCapacity; 185 | 186 | /// \copydoc ColumnMapContext::headerSize() const 187 | uint32_t mHeaderSize; 188 | 189 | /// \copydoc ColumnMapContext::fixedSize() const 190 | uint32_t mFixedSize; 191 | 192 | /// \copydoc ColumnMapContext::fixedMetaData() const 193 | std::vector mFixedMetaData; 194 | 195 | /// \copydoc ColumnMapContext::materializeFunction() const 196 | LLVMColumnMapMaterializeBuilder::Signature mMaterializeFun; 197 | 198 | LLVMJIT mLLVMJit; 199 | }; 200 | 201 | } // namespace deltamain 202 | } // namespace store 203 | } // namespace tell 204 | -------------------------------------------------------------------------------- /deltamain/colstore/ColumnMapRecord.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "ColumnMapRecord.hpp" 25 | 26 | namespace tell { 27 | namespace store { 28 | namespace deltamain { 29 | 30 | int ColumnMapRecord::canUpdate(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, 31 | RecordType type) const { 32 | auto page = mContext.pageFromEntry(mEntry); 33 | auto entries = page->entryData(); 34 | for (auto i = ColumnMapContext::pageIndex(page, mEntry); i < page->count && entries[i].key == mEntry->key; ++i) { 35 | // Skip elements already overwritten by an element in the update log 36 | if (entries[i].version >= highestVersion) { 37 | continue; 38 | } 39 | 40 | if (!snapshot.inReadSet(entries[i].version)) { 41 | return error::not_in_snapshot; 42 | } 43 | 44 | auto recordSizes = page->sizeData(); 45 | auto actualType = (recordSizes[i] == 0 ? RecordType::DELETE : RecordType::DATA); 46 | return (type == actualType ? 0 : error::invalid_write); 47 | } 48 | 49 | // All elements were overwritten by the update log, behave as if no element was written 50 | return (type == RecordType::DELETE ? 0 : error::invalid_write); 51 | } 52 | 53 | int ColumnMapRecord::canRevert(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, 54 | bool& needsRevert) const { 55 | auto page = mContext.pageFromEntry(mEntry); 56 | auto entries = page->entryData(); 57 | for (auto i = ColumnMapContext::pageIndex(page, mEntry); i < page->count && entries[i].key == mEntry->key; ++i) { 58 | // Skip elements already overwritten by an element in the update log 59 | if (entries[i].version >= highestVersion) { 60 | continue; 61 | } 62 | 63 | if (entries[i].version < snapshot.version()) { 64 | needsRevert = false; 65 | return 0; 66 | } 67 | 68 | // Check if version history has element 69 | if (entries[i].version > snapshot.version()) { 70 | needsRevert = false; 71 | for (; i < page->count && entries[i].key == mEntry->key; ++i) { 72 | if (entries[i].version < snapshot.version()) { 73 | return 0; 74 | } 75 | if (entries[i].version == snapshot.version()) { 76 | return error::not_in_snapshot; 77 | } 78 | } 79 | return 0; 80 | } 81 | 82 | needsRevert = true; 83 | return 0; 84 | } 85 | 86 | needsRevert = false; 87 | return 0; 88 | } 89 | 90 | template class ColumnMapRecordImpl; 91 | template class ColumnMapRecordImpl; 92 | 93 | } // namespace deltamain 94 | } // namespace store 95 | } // namespace tell 96 | -------------------------------------------------------------------------------- /deltamain/colstore/ColumnMapRecord.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "ColumnMapContext.hpp" 27 | #include "ColumnMapPage.hpp" 28 | 29 | #include 30 | 31 | #include 32 | 33 | #include 34 | 35 | #include 36 | 37 | #include 38 | #include 39 | 40 | namespace tell { 41 | namespace store { 42 | namespace deltamain { 43 | 44 | template 45 | class ColumnMapRecordImpl { 46 | public: 47 | ColumnMapRecordImpl(T entry, const ColumnMapContext& context) 48 | : mEntry(entry), 49 | mNewest(mEntry->newest.load()), 50 | mContext(context) { 51 | } 52 | 53 | uint64_t key() const { 54 | return mEntry->key; 55 | } 56 | 57 | bool valid() const { 58 | return (mNewest & crossbow::to_underlying(NewestPointerTag::INVALID)) == 0; 59 | } 60 | 61 | T value() const { 62 | return mEntry; 63 | } 64 | 65 | uintptr_t newest() const { 66 | return mNewest; 67 | } 68 | 69 | uint64_t baseVersion() const { 70 | return mEntry->version; 71 | } 72 | 73 | template 74 | int get(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, Fun fun, bool isNewest) const; 75 | 76 | protected: 77 | T mEntry; 78 | uintptr_t mNewest; 79 | const ColumnMapContext& mContext; 80 | }; 81 | 82 | template 83 | template 84 | int ColumnMapRecordImpl::get(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, Fun fun, 85 | bool isNewest) const { 86 | auto page = mContext.pageFromEntry(mEntry); 87 | auto entries = page->entryData(); 88 | for (auto i = ColumnMapContext::pageIndex(page, mEntry); i < page->count && entries[i].key == mEntry->key; ++i) { 89 | // Skip elements already overwritten by an element in the update log 90 | if (entries[i].version >= highestVersion) { 91 | continue; 92 | } 93 | 94 | // Check if the element is readable by the snapshot 95 | if (!snapshot.inReadSet(entries[i].version)) { 96 | isNewest = false; 97 | continue; 98 | } 99 | 100 | auto recordSizes = page->sizeData(); 101 | if (recordSizes[i] == 0) { 102 | return (isNewest ? error::not_found : error::not_in_snapshot); 103 | } 104 | 105 | auto dest = fun(recordSizes[i], entries[i].version, isNewest); 106 | mContext.materialize(page, i, dest); 107 | return 0; 108 | } 109 | 110 | return (isNewest ? error::not_found : error::not_in_snapshot); 111 | } 112 | 113 | extern template class ColumnMapRecordImpl; 114 | 115 | class ConstColumnMapRecord : public ColumnMapRecordImpl { 116 | using Base = ColumnMapRecordImpl; 117 | public: 118 | using Base::Base; 119 | 120 | ConstColumnMapRecord(const void* record, const ColumnMapContext& context) 121 | : Base(reinterpret_cast(record), context) { 122 | } 123 | }; 124 | 125 | extern template class ColumnMapRecordImpl; 126 | 127 | class ColumnMapRecord : public ColumnMapRecordImpl { 128 | using Base = ColumnMapRecordImpl; 129 | public: 130 | using Base::Base; 131 | 132 | ColumnMapRecord(void* record, const ColumnMapContext& context) 133 | : Base(reinterpret_cast(record), context) { 134 | } 135 | 136 | bool tryUpdate(uintptr_t value) { 137 | return mEntry->newest.compare_exchange_strong(mNewest, value); 138 | } 139 | 140 | bool tryInvalidate() { 141 | return mEntry->newest.compare_exchange_strong(mNewest, crossbow::to_underlying(NewestPointerTag::INVALID)); 142 | } 143 | 144 | int canUpdate(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, RecordType type) const; 145 | 146 | int canRevert(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, bool& needsRevert) const; 147 | }; 148 | 149 | } // namespace deltamain 150 | } // namespace store 151 | } // namespace tell 152 | -------------------------------------------------------------------------------- /deltamain/colstore/ColumnMapScanProcessor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "LLVMColumnMapAggregation.hpp" 27 | #include "LLVMColumnMapProjection.hpp" 28 | #include "LLVMColumnMapScan.hpp" 29 | 30 | #include 31 | #include 32 | 33 | #include 34 | 35 | #include 36 | #include 37 | 38 | namespace tell { 39 | namespace store { 40 | 41 | class Record; 42 | 43 | namespace deltamain { 44 | 45 | class ColumnMapContext; 46 | struct ColumnMapMainEntry; 47 | struct ColumnMapMainPage; 48 | struct InsertLogEntry; 49 | struct UpdateLogEntry; 50 | 51 | template 52 | class Table; 53 | 54 | class ColumnMapScanProcessor; 55 | 56 | class ColumnMapScan : public LLVMRowScanBase { 57 | public: 58 | using ScanProcessor = ColumnMapScanProcessor; 59 | 60 | using ColumnScanFun = LLVMColumnMapScanBuilder::Signature; 61 | 62 | using ColumnProjectionFun = LLVMColumnMapProjectionBuilder::Signature; 63 | 64 | using ColumnAggregationFun = LLVMColumnMapAggregationBuilder::Signature; 65 | 66 | ColumnMapScan(Table* table, std::vector queries); 67 | 68 | void prepareQuery(); 69 | 70 | void prepareMaterialization(); 71 | 72 | std::vector> startScan(size_t numThreads); 73 | 74 | private: 75 | Table* mTable; 76 | 77 | ColumnScanFun mColumnScanFun; 78 | 79 | std::vector mColumnMaterializeFuns; 80 | }; 81 | 82 | class ColumnMapScanProcessor : public LLVMRowScanProcessorBase { 83 | public: 84 | using LogIterator = Log::ConstLogIterator; 85 | using PageList = std::vector; 86 | 87 | ColumnMapScanProcessor(const ColumnMapContext& context, const Record& record, 88 | const std::vector& queries, const PageList& pages, size_t pageIdx, size_t pageEndIdx, 89 | const LogIterator& logIter, const LogIterator& logEnd, ColumnMapScan::ColumnScanFun columnScanFun, 90 | const std::vector& columnMaterializeFuns, ColumnMapScan::RowScanFun rowScanFun, 91 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts); 92 | 93 | void process(); 94 | 95 | private: 96 | void processMainPage(const ColumnMapMainPage* page, uint64_t startIdx, uint64_t endIdx); 97 | 98 | void evaluateMainQueries(const ColumnMapMainPage* page, uint64_t startIdx, uint64_t endIdx); 99 | 100 | uint64_t processUpdateRecord(const UpdateLogEntry* ptr, uint64_t baseVersion, uint64_t& validTo); 101 | 102 | const ColumnMapContext& mContext; 103 | 104 | ColumnMapScan::ColumnScanFun mColumnScanFun; 105 | 106 | std::vector mColumnMaterializeFuns; 107 | 108 | const PageList& pages; 109 | size_t pageIdx; 110 | size_t pageEndIdx; 111 | LogIterator logIter; 112 | LogIterator logEnd; 113 | 114 | std::vector mKeyData; 115 | std::vector mValidFromData; 116 | std::vector mValidToData; 117 | }; 118 | 119 | } // namespace deltamain 120 | } // namespace store 121 | } // namespace tell 122 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapAggregation.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace tell { 34 | namespace store { 35 | 36 | class ScanQuery; 37 | 38 | namespace deltamain { 39 | 40 | class ColumnMapContext; 41 | 42 | /** 43 | * @brief Helper class creating the column map aggregation function 44 | */ 45 | class LLVMColumnMapAggregationBuilder : private FunctionBuilder { 46 | public: 47 | using Signature = uint32_t (*) ( 48 | const char* /* page */, 49 | uint64_t /* startIdx */, 50 | uint64_t /* endIdx */, 51 | const char* /* result */, 52 | char* /* dest */); 53 | 54 | static void createFunction(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target, 55 | const std::string& name, ScanQuery* query) { 56 | LLVMColumnMapAggregationBuilder builder(context, module, target, name); 57 | builder.build(query); 58 | } 59 | 60 | private: 61 | static constexpr size_t page = 0; 62 | static constexpr size_t startIdx = 1; 63 | static constexpr size_t endIdx = 2; 64 | static constexpr size_t result = 3; 65 | static constexpr size_t dest = 4; 66 | 67 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 68 | return llvm::Type::getInt32Ty(context); 69 | } 70 | 71 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 72 | return { 73 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "page" }, 74 | { llvm::Type::getInt64Ty(context), "startIdx" }, 75 | { llvm::Type::getInt64Ty(context), "endIdx" }, 76 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "result" }, 77 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "dest" } 78 | }; 79 | } 80 | 81 | LLVMColumnMapAggregationBuilder(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target, 82 | const std::string& name); 83 | 84 | void build(ScanQuery* query); 85 | 86 | const ColumnMapContext& mContext; 87 | 88 | llvm::StructType* mMainPageStructTy; 89 | 90 | uint64_t mRegisterWidth; 91 | }; 92 | 93 | } // namespace deltamain 94 | } // namespace store 95 | } // namespace tell 96 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapMaterialize.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace tell { 34 | namespace store { 35 | namespace deltamain { 36 | 37 | class ColumnMapContext; 38 | 39 | /** 40 | * @brief Helper class creating the column map materialize function 41 | */ 42 | class LLVMColumnMapMaterializeBuilder : private FunctionBuilder { 43 | public: 44 | using Signature = uint32_t (*) ( 45 | const char* /* page */, 46 | uint32_t /* idx */, 47 | char* /* dest */); 48 | 49 | static const std::string FUNCTION_NAME; 50 | 51 | static void createFunction(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target) { 52 | LLVMColumnMapMaterializeBuilder builder(context, module, target); 53 | builder.build(); 54 | } 55 | 56 | private: 57 | static constexpr size_t page = 0; 58 | static constexpr size_t idx = 1; 59 | static constexpr size_t dest = 2; 60 | 61 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 62 | return llvm::Type::getInt32Ty(context); 63 | } 64 | 65 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 66 | return { 67 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "page" }, 68 | { llvm::Type::getInt32Ty(context), "idx" }, 69 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "dest" } 70 | }; 71 | } 72 | 73 | LLVMColumnMapMaterializeBuilder(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target); 74 | 75 | void build(); 76 | 77 | const ColumnMapContext& mContext; 78 | 79 | llvm::StructType* mMainPageStructTy; 80 | llvm::StructType* mHeapEntryStructTy; 81 | }; 82 | 83 | } // namespace deltamain 84 | } // namespace store 85 | } // namespace tell 86 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapProjection.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace tell { 34 | namespace store { 35 | 36 | class ScanQuery; 37 | 38 | namespace deltamain { 39 | 40 | class ColumnMapContext; 41 | 42 | /** 43 | * @brief Helper class creating the column map projection function 44 | */ 45 | class LLVMColumnMapProjectionBuilder : private FunctionBuilder { 46 | public: 47 | using Signature = uint32_t (*) ( 48 | const char* /* page */, 49 | uint32_t /* idx */, 50 | char* /* dest */); 51 | 52 | static void createFunction(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target, 53 | const std::string& name, ScanQuery* query) { 54 | LLVMColumnMapProjectionBuilder builder(context, module, target, name); 55 | builder.build(query); 56 | } 57 | 58 | private: 59 | static constexpr size_t page = 0; 60 | static constexpr size_t idx = 1; 61 | static constexpr size_t dest = 2; 62 | 63 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 64 | return llvm::Type::getInt32Ty(context); 65 | } 66 | 67 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 68 | return { 69 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "page" }, 70 | { llvm::Type::getInt32Ty(context), "idx" }, 71 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "dest" } 72 | }; 73 | } 74 | 75 | LLVMColumnMapProjectionBuilder(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target, 76 | const std::string& name); 77 | 78 | void build(ScanQuery* query); 79 | 80 | const ColumnMapContext& mContext; 81 | 82 | llvm::StructType* mMainPageStructTy; 83 | llvm::StructType* mHeapEntryStructTy; 84 | }; 85 | 86 | } // namespace deltamain 87 | } // namespace store 88 | } // namespace tell 89 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapScan.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | namespace tell { 35 | namespace store { 36 | namespace deltamain { 37 | 38 | class ColumnMapContext; 39 | 40 | /** 41 | * @brief Helper class creating the column map scan function 42 | */ 43 | class LLVMColumnMapScanBuilder : private FunctionBuilder { 44 | public: 45 | using Signature = void (*) ( 46 | const uint64_t* /* keyData */, 47 | const uint64_t* /* validFromData */, 48 | const uint64_t* /* validToData */, 49 | const char* /* page */, 50 | uint64_t /* startIdx */, 51 | uint64_t /* endIdx */, 52 | char* /* resultData */); 53 | 54 | static const std::string FUNCTION_NAME; 55 | 56 | static void createFunction(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target, 57 | const ScanAST& scanAst) { 58 | LLVMColumnMapScanBuilder builder(context, module, target); 59 | builder.buildScan(scanAst); 60 | } 61 | 62 | private: 63 | static constexpr size_t keyData = 0; 64 | static constexpr size_t validFromData = 1; 65 | static constexpr size_t validToData = 2; 66 | static constexpr size_t page = 3; 67 | static constexpr size_t startIdx = 4; 68 | static constexpr size_t endIdx = 5; 69 | static constexpr size_t resultData = 6; 70 | 71 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 72 | return llvm::Type::getVoidTy(context); 73 | } 74 | 75 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 76 | return { 77 | { llvm::Type::getInt64Ty(context)->getPointerTo(), "keyData" }, 78 | { llvm::Type::getInt64Ty(context)->getPointerTo(), "validFromData" }, 79 | { llvm::Type::getInt64Ty(context)->getPointerTo(), "validToData" }, 80 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "page" }, 81 | { llvm::Type::getInt64Ty(context), "startIdx" }, 82 | { llvm::Type::getInt64Ty(context), "endIdx" }, 83 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "resultData" } 84 | }; 85 | } 86 | 87 | LLVMColumnMapScanBuilder(const ColumnMapContext& context, llvm::Module& module, llvm::TargetMachine* target); 88 | 89 | void buildScan(const ScanAST& scanAst); 90 | 91 | void buildFixedField(const ScanAST& scanAst, const FieldAST& fieldAst); 92 | 93 | std::tuple buildFixedFieldEvaluation(llvm::Value* srcData, 94 | llvm::Value* nullData, llvm::Value* start, uint64_t vectorSize, std::vector& conjunctsGenerated, 95 | const ScanAST& scanAst, const FieldAST& fieldAst, const llvm::Twine& name); 96 | 97 | void buildVariableField(const ScanAST& scanAst, const FieldAST& fieldAst); 98 | 99 | void buildQuery(bool needsKey, const std::vector& queries); 100 | 101 | std::tuple buildQueryEvaluation(llvm::Value* start, 102 | llvm::Value* validFromStart, llvm::Value* validToStart, llvm::Value* keyStart, uint64_t vectorSize, 103 | const std::vector& queries, const llvm::Twine& name); 104 | 105 | void buildResult(const std::vector& queries); 106 | 107 | void buildConjunctMerge(uint64_t vectorSize, uint32_t src, uint32_t dest); 108 | 109 | llvm::Value* buildConjunctMerge(llvm::Value* lhsStart, llvm::Value* lhsEnd, llvm::Value* rhsStart, 110 | uint64_t vectorSize, const llvm::Twine& name); 111 | 112 | const ColumnMapContext& mContext; 113 | 114 | llvm::StructType* mMainPageStructTy; 115 | llvm::StructType* mHeapEntryStructTy; 116 | 117 | llvm::Value* mMainPage; 118 | llvm::Value* mCount; 119 | llvm::Value* mHeaderData; 120 | llvm::Value* mFixedData; 121 | llvm::Value* mVariableData; 122 | 123 | uint64_t mRegisterWidth; 124 | 125 | std::vector mVectorConjunctsGenerated; 126 | std::vector mScalarConjunctsGenerated; 127 | }; 128 | 129 | } // namespace deltamain 130 | } // namespace store 131 | } // namespace tell 132 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapUtils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "LLVMColumnMapUtils.hpp" 25 | 26 | #include "ColumnMapPage.hpp" 27 | 28 | namespace tell { 29 | namespace store { 30 | namespace deltamain { 31 | 32 | llvm::StructType* getColumnMapMainPageTy(llvm::LLVMContext& context) { 33 | static_assert(sizeof(ColumnMapMainPage) == 16, "Size of ColumnMapHeapEntry must be 16"); 34 | static_assert(offsetof(ColumnMapMainPage, count) == 0, "Offset of count must be 0"); 35 | static_assert(offsetof(ColumnMapMainPage, headerOffset) == 4, "Offset of headerOffset must be 4"); 36 | static_assert(offsetof(ColumnMapMainPage, fixedOffset) == 8, "Offset of fixedOffset must be 8"); 37 | static_assert(offsetof(ColumnMapMainPage, variableOffset) == 12, "Offset of variableOffset must be 12"); 38 | return llvm::StructType::get(context, { 39 | llvm::Type::getInt32Ty(context), // count 40 | llvm::Type::getInt32Ty(context), // headerOffset 41 | llvm::Type::getInt32Ty(context), // fixedOffset 42 | llvm::Type::getInt32Ty(context) // variableOffset 43 | }); 44 | } 45 | 46 | llvm::StructType* getColumnMapHeapEntriesTy(llvm::LLVMContext& context) { 47 | static_assert(sizeof(ColumnMapHeapEntry) == 8, "Size of ColumnMapHeapEntry must be 8"); 48 | static_assert(offsetof(ColumnMapHeapEntry, offset) == 0, "Offset of offset must be 0"); 49 | static_assert(offsetof(ColumnMapHeapEntry, prefix) == 4, "Offset of prefix must be 4"); 50 | return llvm::StructType::get(context, { 51 | llvm::Type::getInt32Ty(context), // offset 52 | llvm::Type::getInt32Ty(context) // prefix 53 | }); 54 | } 55 | 56 | } // namespace deltamain 57 | } // namespace store 58 | } // namespace tell 59 | -------------------------------------------------------------------------------- /deltamain/colstore/LLVMColumnMapUtils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | 29 | namespace tell { 30 | namespace store { 31 | namespace deltamain { 32 | 33 | llvm::StructType* getColumnMapMainPageTy(llvm::LLVMContext& context); 34 | 35 | llvm::StructType* getColumnMapHeapEntriesTy(llvm::LLVMContext& context); 36 | 37 | } // namespace deltamain 38 | } // namespace store 39 | } // namespace tell 40 | -------------------------------------------------------------------------------- /deltamain/rowstore/RowStoreContext.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "RowStorePage.hpp" 27 | #include "RowStoreRecord.hpp" 28 | #include "RowStoreScanProcessor.hpp" 29 | 30 | namespace tell { 31 | namespace store { 32 | 33 | class PageManager; 34 | class Record; 35 | 36 | namespace deltamain { 37 | 38 | class RowStoreContext { 39 | public: 40 | using Scan = RowStoreScan; 41 | using Page = RowStoreMainPage; 42 | using PageModifier = RowStorePageModifier; 43 | 44 | using MainRecord = RowStoreRecord; 45 | using ConstMainRecord = ConstRowStoreRecord; 46 | 47 | static const char* implementationName() { 48 | return "Delta-Main Rewrite (Row Store)"; 49 | } 50 | 51 | RowStoreContext(const PageManager& /* pageManager */, const Record& /* record */) { 52 | } 53 | }; 54 | 55 | } // namespace deltamain 56 | } // namespace store 57 | } // namespace tell 58 | -------------------------------------------------------------------------------- /deltamain/rowstore/RowStorePage.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "RowStoreRecord.hpp" 27 | 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | namespace tell { 37 | namespace store { 38 | 39 | class Modifier; 40 | class PageManager; 41 | 42 | namespace deltamain { 43 | 44 | class alignas(8) RowStoreMainPage { 45 | public: 46 | template 47 | class IteratorImpl { 48 | public: 49 | static constexpr bool is_const_iterator = std::is_const::value; 50 | using reference = typename std::conditional::type; 51 | using pointer = typename std::conditional::type; 52 | 53 | IteratorImpl(uintptr_t current) 54 | : mCurrent(current) { 55 | } 56 | 57 | IteratorImpl& operator++() { 58 | auto entry = reinterpret_cast(mCurrent); 59 | auto offsets = entry->offsetData(); 60 | mCurrent += offsets[entry->versionCount]; 61 | return *this; 62 | } 63 | 64 | IteratorImpl operator++(int) { 65 | IteratorImpl result(*this); 66 | operator++(); 67 | return result; 68 | } 69 | 70 | bool operator==(const IteratorImpl& rhs) const { 71 | return (mCurrent == rhs.mCurrent); 72 | } 73 | 74 | bool operator!=(const IteratorImpl& rhs) const { 75 | return !operator==(rhs); 76 | } 77 | 78 | reference operator*() const { 79 | return *operator->(); 80 | } 81 | 82 | pointer operator->() const { 83 | return reinterpret_cast(mCurrent); 84 | } 85 | 86 | private: 87 | /// Current record this iterator is pointing to 88 | uintptr_t mCurrent; 89 | }; 90 | 91 | using Iterator = IteratorImpl; 92 | using ConstIterator = IteratorImpl; 93 | 94 | RowStoreMainPage() 95 | : mOffset(0u) { 96 | } 97 | 98 | ConstIterator cbegin() const { 99 | return ConstIterator(reinterpret_cast(data())); 100 | } 101 | 102 | Iterator begin() { 103 | return Iterator(reinterpret_cast(data())); 104 | } 105 | 106 | ConstIterator begin() const { 107 | return cbegin(); 108 | } 109 | 110 | ConstIterator cend() const { 111 | return ConstIterator(reinterpret_cast(data()) + mOffset); 112 | } 113 | 114 | Iterator end() { 115 | return Iterator(reinterpret_cast(data()) + mOffset); 116 | } 117 | 118 | ConstIterator end() const { 119 | return cend(); 120 | } 121 | 122 | bool needsCleaning(uint64_t minVersion) const; 123 | 124 | RowStoreMainEntry* append(uint64_t key, const std::vector& elements); 125 | 126 | RowStoreMainEntry* append(const RowStoreMainEntry* record); 127 | 128 | private: 129 | const char* data() const { 130 | return reinterpret_cast(this) + sizeof(RowStoreMainPage); 131 | } 132 | 133 | char* data() { 134 | return const_cast(const_cast(this)->data()); 135 | } 136 | 137 | uint64_t mOffset; 138 | }; 139 | 140 | class RowStorePageModifier { 141 | public: 142 | RowStorePageModifier(const RowStoreContext& /* context */, PageManager& pageManager, Modifier& mainTableModifier, 143 | uint64_t minVersion) 144 | : mPageManager(pageManager), 145 | mMainTableModifier(mainTableModifier), 146 | mMinVersion(minVersion), 147 | mFillPage(nullptr) { 148 | } 149 | 150 | bool clean(RowStoreMainPage* page); 151 | 152 | bool append(InsertRecord& oldRecord); 153 | 154 | std::vector done() { 155 | return std::move(mPageList); 156 | } 157 | 158 | private: 159 | template 160 | bool collectElements(Rec& rec); 161 | 162 | template 163 | void recycleEntry(Rec& oldRecord, RowStoreMainEntry* newRecord, bool replace); 164 | 165 | template 166 | RowStoreMainEntry* internalAppend(Fun fun); 167 | 168 | PageManager& mPageManager; 169 | 170 | Modifier& mMainTableModifier; 171 | 172 | uint64_t mMinVersion; 173 | 174 | std::vector mPageList; 175 | 176 | RowStoreMainPage* mFillPage; 177 | 178 | std::vector mElements; 179 | }; 180 | 181 | } // namespace deltamain 182 | } // namespace store 183 | } // namespace tell 184 | -------------------------------------------------------------------------------- /deltamain/rowstore/RowStoreRecord.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "RowStoreRecord.hpp" 25 | 26 | #include 27 | 28 | namespace tell { 29 | namespace store { 30 | namespace deltamain { 31 | 32 | RowStoreMainEntry* RowStoreMainEntry::serialize(void* ptr, uint64_t key, const std::vector& elements) { 33 | auto entry = new (ptr) RowStoreMainEntry(key, elements.size()); 34 | 35 | auto versions = entry->versionData(); 36 | auto offsets = entry->offsetData(); 37 | offsets[0] = serializedHeaderSize(elements.size()); 38 | for (decltype(elements.size()) i = 0; i < elements.size(); ++i) { 39 | auto& element = elements[i]; 40 | versions[i] = element.version; 41 | offsets[i + 1] = offsets[i] + element.size; 42 | memcpy(reinterpret_cast(entry) + offsets[i], element.data, element.size); 43 | } 44 | return entry; 45 | } 46 | 47 | RowStoreMainEntry* RowStoreMainEntry::serialize(void* ptr, const RowStoreMainEntry* oldEntry, uint32_t oldSize) { 48 | LOG_ASSERT(oldSize == oldEntry->offsetData()[oldEntry->versionCount], "Calculated size does not match"); 49 | 50 | // Can not copy the complete record at once as the next field can be modified in the meantime 51 | // First create a new header and then copy all immutable data fields. 52 | auto offsets = oldEntry->offsetData(); 53 | auto entry = new (ptr) RowStoreMainEntry(oldEntry->key, oldEntry->versionCount); 54 | memcpy(entry->data(), oldEntry->data(), offsets[oldEntry->versionCount] - sizeof(RowStoreMainEntry)); 55 | return entry; 56 | } 57 | 58 | template 59 | bool RowStoreRecordImpl::needsCleaning(uint64_t minVersion) const { 60 | // In case the record has pending updates it needs to be cleaned 61 | if (mNewest != 0u) { 62 | return true; 63 | } 64 | // If only one version is in the record it does not need cleaning 65 | if (mEntry->versionCount == 1) { 66 | return false; 67 | } 68 | // The record needs cleaning if the last version can be purged 69 | auto versions = mEntry->versionData(); 70 | return (versions[mEntry->versionCount - 1] < minVersion); 71 | } 72 | 73 | template 74 | void RowStoreRecordImpl::collect(uint64_t minVersion, uint64_t highestVersion, 75 | std::vector& elements) const { 76 | // Check if the oldest element is the oldest readable element 77 | if (highestVersion <= minVersion) { 78 | return; 79 | } 80 | 81 | auto versions = mEntry->versionData(); 82 | auto offsets = mEntry->offsetData(); 83 | 84 | // Skip elements already overwritten by an element in the update log 85 | typename std::remove_constversionCount)>::type i = 0; 86 | for (; i < mEntry->versionCount && versions[i] >= highestVersion; ++i) { 87 | } 88 | 89 | // Append all valid elements newer than the lowest active version (if they exist) 90 | for (; i < mEntry->versionCount && versions[i] > minVersion; ++i) { 91 | elements.emplace_back(versions[i], reinterpret_cast(mEntry) + offsets[i], 92 | offsets[i + 1] - offsets[i]); 93 | } 94 | 95 | // Append the newest element that is older or equal the lowest active version (if it exists) 96 | if (i < mEntry->versionCount) { 97 | elements.emplace_back(versions[i], reinterpret_cast(mEntry) + offsets[i], 98 | offsets[i + 1] - offsets[i]); 99 | } 100 | } 101 | 102 | int RowStoreRecord::canUpdate(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, 103 | RecordType type) const { 104 | auto versions = mEntry->versionData(); 105 | 106 | // Skip elements already overwritten by an element in the update log 107 | typename std::remove_constversionCount)>::type i = 0; 108 | for (; i < mEntry->versionCount && versions[i] >= highestVersion; ++i) { 109 | } 110 | 111 | // All elements were overwritten by the update log, behave as if no element was written 112 | if (i >= mEntry->versionCount) { 113 | return (type == RecordType::DELETE ? 0 : error::invalid_write); 114 | } 115 | if (!snapshot.inReadSet(versions[i])) { 116 | return error::not_in_snapshot; 117 | } 118 | 119 | auto offsets = mEntry->offsetData(); 120 | auto size = offsets[i + 1] - offsets[i]; 121 | auto actualType = (size == 0 ? RecordType::DELETE : RecordType::DATA); 122 | return (type == actualType ? 0 : error::invalid_write); 123 | } 124 | 125 | int RowStoreRecord::canRevert(uint64_t highestVersion, const commitmanager::SnapshotDescriptor& snapshot, 126 | bool& needsRevert) const { 127 | auto versions = mEntry->versionData(); 128 | 129 | // Skip elements already overwritten by an element in the update log 130 | typename std::remove_constversionCount)>::type i = 0; 131 | for (; i < mEntry->versionCount && versions[i] >= highestVersion; ++i) { 132 | } 133 | 134 | if (i >= mEntry->versionCount) { 135 | needsRevert = false; 136 | return 0; 137 | } 138 | 139 | if (versions[i] < snapshot.version()) { 140 | needsRevert = false; 141 | return 0; 142 | } 143 | // Check if version history has element 144 | if (versions[i] > snapshot.version()) { 145 | needsRevert = false; 146 | for (; i < mEntry->versionCount; ++i) { 147 | if (versions[i] < snapshot.version()) { 148 | return 0; 149 | } 150 | if (versions[i] == snapshot.version()) { 151 | return error::not_in_snapshot; 152 | } 153 | } 154 | return 0; 155 | } 156 | 157 | needsRevert = true; 158 | return 0; 159 | } 160 | 161 | template class RowStoreRecordImpl; 162 | template class RowStoreRecordImpl; 163 | 164 | } // namespace deltamain 165 | } // namespace store 166 | } // namespace tell 167 | -------------------------------------------------------------------------------- /deltamain/rowstore/RowStoreScanProcessor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "RowStoreScanProcessor.hpp" 25 | 26 | #include "RowStorePage.hpp" 27 | #include "RowStoreRecord.hpp" 28 | 29 | #include 30 | #include 31 | 32 | namespace tell { 33 | namespace store { 34 | namespace deltamain { 35 | 36 | RowStoreScan::RowStoreScan(Table* table, std::vector queries) 37 | : LLVMRowScanBase(table->record(), std::move(queries)), 38 | mTable(table) { 39 | } 40 | 41 | std::vector> RowStoreScan::startScan(size_t numThreads) { 42 | return mTable->startScan(numThreads, mQueries, mRowScanFun, mRowMaterializeFuns, mScanAst.numConjunct); 43 | } 44 | 45 | RowStoreScanProcessor::RowStoreScanProcessor(const RowStoreContext& /* context */, const Record& record, 46 | const std::vector& queries, const PageList& pages, size_t pageIdx, size_t pageEndIdx, 47 | const LogIterator& logIter, const LogIterator& logEnd, RowStoreScan::RowScanFun rowScanFun, 48 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts) 49 | : LLVMRowScanProcessorBase(record, queries, rowScanFun, rowMaterializeFuns, numConjuncts), 50 | pages(pages), 51 | pageIdx(pageIdx), 52 | pageEndIdx(pageEndIdx), 53 | logIter(logIter), 54 | logEnd(logEnd) { 55 | } 56 | 57 | void RowStoreScanProcessor::process() { 58 | for (auto i = pageIdx; i < pageEndIdx; ++i) { 59 | for (auto& ptr : *pages[i]) { 60 | processMainRecord(&ptr); 61 | } 62 | } 63 | for (auto insIter = logIter; insIter != logEnd; ++insIter) { 64 | if (!insIter->sealed()) { 65 | continue; 66 | } 67 | processInsertRecord(reinterpret_cast(insIter->data())); 68 | } 69 | } 70 | 71 | void RowStoreScanProcessor::processMainRecord(const RowStoreMainEntry* ptr) { 72 | ConstRowStoreRecord record(ptr); 73 | 74 | auto versions = ptr->versionData(); 75 | auto validTo = std::numeric_limits::max(); 76 | 77 | typename std::remove_constversionCount)>::type i = 0; 78 | if (record.newest() != 0u) { 79 | if (!record.valid()) { 80 | return; 81 | } 82 | 83 | if (auto main = newestMainRecord(record.newest())) { 84 | return processMainRecord(reinterpret_cast(main)); 85 | } 86 | 87 | auto lowestVersion = processUpdateRecord(reinterpret_cast(record.newest()), 88 | record.baseVersion(), validTo); 89 | 90 | // Skip elements already overwritten by an element in the update log 91 | for (; i < ptr->versionCount && versions[i] >= lowestVersion; ++i) { 92 | } 93 | } 94 | 95 | auto offsets = ptr->offsetData(); 96 | for (; i < ptr->versionCount; ++i) { 97 | auto sz = offsets[i + 1] - offsets[i]; 98 | 99 | // Check if the entry marks a deletion: Skip element 100 | if (sz == 0) { 101 | validTo = versions[i]; 102 | continue; 103 | } 104 | 105 | auto data = reinterpret_cast(ptr) + offsets[i]; 106 | processRowRecord(ptr->key, versions[i], validTo, data, sz); 107 | validTo = versions[i]; 108 | } 109 | } 110 | 111 | void RowStoreScanProcessor::processInsertRecord(const InsertLogEntry* ptr) { 112 | ConstInsertRecord record(ptr); 113 | 114 | auto validTo = std::numeric_limits::max(); 115 | 116 | if (record.newest() != 0u) { 117 | if (!record.valid()) { 118 | return; 119 | } 120 | 121 | if (auto main = newestMainRecord(record.newest())) { 122 | return processMainRecord(reinterpret_cast(main)); 123 | } 124 | 125 | auto lowestVersion = processUpdateRecord(reinterpret_cast(record.newest()), 126 | record.baseVersion(), validTo); 127 | 128 | if (ptr->version >= lowestVersion) { 129 | return; 130 | } 131 | } 132 | 133 | auto entry = LogEntry::entryFromData(reinterpret_cast(ptr)); 134 | processRowRecord(ptr->key, ptr->version, validTo, ptr->data(), entry->size() - sizeof(InsertLogEntry)); 135 | } 136 | 137 | uint64_t RowStoreScanProcessor::processUpdateRecord(const UpdateLogEntry* ptr, uint64_t baseVersion, 138 | uint64_t& validTo) { 139 | UpdateRecordIterator updateIter(ptr, baseVersion); 140 | for (; !updateIter.done(); updateIter.next()) { 141 | auto entry = LogEntry::entryFromData(reinterpret_cast(updateIter.value())); 142 | 143 | // Check if the entry marks a deletion: Skip element 144 | if (entry->type() == crossbow::to_underlying(RecordType::DELETE)) { 145 | validTo = updateIter->version; 146 | continue; 147 | } 148 | 149 | processRowRecord(updateIter->key, updateIter->version, validTo, updateIter->data(), 150 | entry->size() - sizeof(UpdateLogEntry)); 151 | validTo = updateIter->version; 152 | } 153 | return updateIter.lowestVersion(); 154 | } 155 | 156 | } // namespace deltamain 157 | } // namespace store 158 | } // namespace tell 159 | -------------------------------------------------------------------------------- /deltamain/rowstore/RowStoreScanProcessor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | 35 | namespace tell { 36 | namespace store { 37 | 38 | class Record; 39 | 40 | namespace deltamain { 41 | 42 | struct InsertLogEntry; 43 | class RowStoreContext; 44 | struct RowStoreMainEntry; 45 | class RowStoreMainPage; 46 | struct UpdateLogEntry; 47 | 48 | template 49 | class Table; 50 | 51 | class RowStoreScanProcessor; 52 | 53 | class RowStoreScan : public LLVMRowScanBase { 54 | public: 55 | using ScanProcessor = RowStoreScanProcessor; 56 | 57 | RowStoreScan(Table* table, std::vector queries); 58 | 59 | using LLVMRowScanBase::prepareQuery; 60 | 61 | using LLVMRowScanBase::prepareMaterialization; 62 | 63 | std::vector> startScan(size_t numThreads); 64 | 65 | private: 66 | Table* mTable; 67 | }; 68 | 69 | class RowStoreScanProcessor : public LLVMRowScanProcessorBase { 70 | public: 71 | using LogIterator = Log::ConstLogIterator; 72 | using PageList = std::vector; 73 | 74 | RowStoreScanProcessor(const RowStoreContext& context, const Record& record, const std::vector& queries, 75 | const PageList& pages, size_t pageIdx, size_t pageEndIdx, const LogIterator& logIter, 76 | const LogIterator& logEnd, RowStoreScan::RowScanFun rowScanFun, 77 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts); 78 | 79 | void process(); 80 | 81 | private: 82 | void processMainRecord(const RowStoreMainEntry* ptr); 83 | 84 | void processInsertRecord(const InsertLogEntry* ptr); 85 | 86 | uint64_t processUpdateRecord(const UpdateLogEntry* ptr, uint64_t baseVersion, uint64_t& validTo); 87 | 88 | const PageList& pages; 89 | size_t pageIdx; 90 | size_t pageEndIdx; 91 | LogIterator logIter; 92 | LogIterator logEnd; 93 | }; 94 | 95 | } // namespace deltamain 96 | } // namespace store 97 | } // namespace tell 98 | -------------------------------------------------------------------------------- /logstructured/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore Logstructured implementation 3 | ################### 4 | set(LOGSTRUCTURED_SRCS 5 | GcScanProcessor.cpp 6 | HashScanProcessor.cpp 7 | Table.cpp 8 | VersionRecordIterator.cpp 9 | ) 10 | 11 | set(LOGSTRUCTURED_PRIVATE_HDR 12 | ChainedVersionRecord.hpp 13 | GcScanProcessor.hpp 14 | HashScanProcessor.hpp 15 | LogstructuredMemoryStore.hpp 16 | Table.hpp 17 | VersionRecordIterator.hpp 18 | ) 19 | 20 | # Add TellStore logstructured library 21 | add_library(tellstore-logstructured STATIC ${LOGSTRUCTURED_SRCS} ${LOGSTRUCTURED_PRIVATE_HDR}) 22 | target_include_directories(tellstore-logstructured PUBLIC ${PROJECT_SOURCE_DIR}) 23 | 24 | # Link against TellStore util 25 | target_link_libraries(tellstore-logstructured PUBLIC tellstore-util) 26 | -------------------------------------------------------------------------------- /logstructured/GcScanProcessor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | #include 34 | #include 35 | 36 | namespace tell { 37 | namespace store { 38 | 39 | struct LogstructuredMemoryStore; 40 | 41 | namespace logstructured { 42 | 43 | class ChainedVersionRecord; 44 | class GcScanGarbageCollector; 45 | class GcScanProcessor; 46 | class Table; 47 | 48 | class GcScan : public LLVMRowScanBase { 49 | public: 50 | using ScanProcessor = GcScanProcessor; 51 | using GarbageCollector = GcScanGarbageCollector; 52 | 53 | GcScan(Table* table, std::vector queries); 54 | 55 | using LLVMRowScanBase::prepareQuery; 56 | 57 | using LLVMRowScanBase::prepareMaterialization; 58 | 59 | /** 60 | * @brief Start a full scan of this table 61 | * 62 | * @param numThreads Number of threads to use for the scan 63 | * @return A scan processor for each thread 64 | */ 65 | std::vector> startScan(size_t numThreads); 66 | 67 | private: 68 | Table* mTable; 69 | }; 70 | 71 | /** 72 | * @brief Scan processor for the Log-Structured Memory approach that performs Garbage Collection as part of its scan 73 | */ 74 | class GcScanProcessor : public LLVMRowScanProcessorBase { 75 | public: 76 | using LogImpl = Log; 77 | using PageIterator = LogImpl::PageIterator; 78 | 79 | GcScanProcessor(Table& table, const std::vector& queries, const PageIterator& begin, 80 | const PageIterator& end, uint64_t minVersion, GcScan::RowScanFun rowScanFun, 81 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts); 82 | 83 | /** 84 | * @brief Scans over all entries in the log 85 | * 86 | * Processes all valid entries with the associated scan queries. 87 | * 88 | * Performs garbage collection while scanning over a page. 89 | */ 90 | void process(); 91 | 92 | private: 93 | /** 94 | * @brief Advance the entry iterator to the next entry, advancing to the next page if necessary 95 | * 96 | * The processor must not be at the end when calling this function. 97 | * 98 | * @return True if the iterator points to a valid element, false if it reached the end 99 | */ 100 | bool advanceEntry(); 101 | 102 | /** 103 | * @brief Advance the page iterator to the next page containing a valid entry 104 | * 105 | * The processor must be at the end of a page but not at the end of the log when calling this function. 106 | * 107 | * @return True if the iterator points to a valid element, false if it reached the end 108 | */ 109 | bool advancePage(); 110 | 111 | /** 112 | * @brief Recycle the given element 113 | * 114 | * Copies the element to a recycling page and replaces the old element in the version list with the new element. 115 | * 116 | * @param oldElement The element to recycle 117 | * @param size Size of the old element 118 | * @param type Type of the old element 119 | */ 120 | void recycleEntry(ChainedVersionRecord* oldElement, uint32_t size, uint32_t type); 121 | 122 | /** 123 | * @brief Replaces the given old element with the given new element in the version list of the record 124 | * 125 | * The two elements must belong to the same record. 126 | * 127 | * @param oldElement Old element to remove from the version list 128 | * @param newElement New element to replace the old element with 129 | * @return Whether the replacement was successful 130 | */ 131 | bool replaceElement(ChainedVersionRecord* oldElement, ChainedVersionRecord* newElement); 132 | 133 | Table& mTable; 134 | uint64_t mMinVersion; 135 | 136 | LogImpl::PageIterator mPagePrev; 137 | LogImpl::PageIterator mPageIt; 138 | LogImpl::PageIterator mPageEnd; 139 | 140 | LogPage::EntryIterator mEntryIt; 141 | LogPage::EntryIterator mEntryEnd; 142 | 143 | LogPage* mRecyclingHead; 144 | LogPage* mRecyclingTail; 145 | 146 | /// Amount of garbage in the current page 147 | uint32_t mGarbage; 148 | 149 | /// Whether all entries in the current page were sealed 150 | bool mSealed; 151 | 152 | /// Whether the current page is being recycled 153 | /// Initialized to false to prevent the first page from being garbage collected 154 | bool mRecycle; 155 | }; 156 | 157 | /** 158 | * @brief Garbage collector for the Log-Structured Memory approach that performs Garbage Collection as part of its scan 159 | */ 160 | class GcScanGarbageCollector { 161 | public: 162 | GcScanGarbageCollector(LogstructuredMemoryStore& storage) 163 | : mStorage(storage) { 164 | } 165 | 166 | void run(const std::vector& tables, uint64_t minVersion); 167 | 168 | private: 169 | LogstructuredMemoryStore& mStorage; 170 | }; 171 | 172 | } // namespace logstructured 173 | } // namespace store 174 | } // namespace tell 175 | -------------------------------------------------------------------------------- /logstructured/HashScanProcessor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "HashScanProcessor.hpp" 25 | 26 | #include "ChainedVersionRecord.hpp" 27 | #include "LogstructuredMemoryStore.hpp" 28 | #include "Table.hpp" 29 | #include "VersionRecordIterator.hpp" 30 | 31 | #include 32 | 33 | #include 34 | 35 | namespace tell { 36 | namespace store { 37 | namespace logstructured { 38 | 39 | HashScan::HashScan(Table* table, std::vector queries) 40 | : LLVMRowScanBase(table->record(), std::move(queries)), 41 | mTable(table) { 42 | } 43 | 44 | std::vector> HashScan::startScan(size_t numThreads) { 45 | if (numThreads == 0) { 46 | return {}; 47 | } 48 | 49 | std::vector> result; 50 | result.reserve(numThreads); 51 | 52 | auto version = mTable->minVersion(); 53 | auto capacity = mTable->mHashMap.capacity(); 54 | 55 | auto step = capacity / numThreads; 56 | auto mod = capacity % numThreads; 57 | for (decltype(numThreads) i = 0; i < numThreads; ++i) { 58 | auto start = i * step + std::min(i, mod); 59 | auto end = start + step + (i < mod ? 1 : 0); 60 | 61 | result.emplace_back(new HashScanProcessor(*mTable, mQueries, start, end, version, mRowScanFun, 62 | mRowMaterializeFuns, mScanAst.numConjunct)); 63 | } 64 | 65 | return result; 66 | } 67 | 68 | HashScanProcessor::HashScanProcessor(Table& table, const std::vector& queries, size_t start, size_t end, 69 | uint64_t minVersion, HashScan::RowScanFun rowScanFun, 70 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts) 71 | : LLVMRowScanProcessorBase(table.record(), queries, rowScanFun, rowMaterializeFuns, numConjuncts), 72 | mTable(table), 73 | mMinVersion(minVersion), 74 | mStart(start), 75 | mEnd(end) { 76 | } 77 | 78 | void HashScanProcessor::process() { 79 | mTable.mHashMap.forEach(mStart, mEnd, [this] (uint64_t tableId, uint64_t key, void* ptr) { 80 | if (tableId != mTable.tableId()) { 81 | return; 82 | } 83 | 84 | auto lastVersion = ChainedVersionRecord::ACTIVE_VERSION; 85 | for (VersionRecordIterator recIter(mTable, reinterpret_cast(ptr)); !recIter.done(); 86 | recIter.next()) { 87 | auto record = recIter.value(); 88 | 89 | // Skip element if it is not yet sealed 90 | auto entry = LogEntry::entryFromData(reinterpret_cast(record)); 91 | if (BOOST_UNLIKELY(!entry->sealed())) { 92 | continue; 93 | } 94 | 95 | // The record iterator might reset itself to the beginning of the version chain if iterator consistency can 96 | // not be guaranteed. Keep track of the lowest version we have read to prevent scanning tuple more than 97 | // once. 98 | if (record->validFrom() >= lastVersion) { 99 | continue; 100 | } 101 | lastVersion = record->validFrom(); 102 | 103 | // Skip the element if it is not a data entry (i.e. deletion) 104 | if (crossbow::from_underlying(entry->type()) != VersionRecordType::DATA) { 105 | continue; 106 | } 107 | 108 | auto recordLength = entry->size() - sizeof(ChainedVersionRecord); 109 | processRowRecord(key, lastVersion, recIter.validTo(), record->data(), recordLength); 110 | 111 | // Check if the iterator reached the element with minimum version. The remaining older elements have to be 112 | // superseeded by newer elements in any currently valid Snapshot Descriptor. 113 | if (lastVersion <= mMinVersion) { 114 | break; 115 | } 116 | } 117 | }); 118 | } 119 | 120 | void HashScanGarbageCollector::run(const std::vector& /* tables */, uint64_t /* minVersion */) { 121 | // TODO Implement 122 | } 123 | 124 | } // namespace logstructured 125 | } // namespace store 126 | } // namespace tell 127 | -------------------------------------------------------------------------------- /logstructured/HashScanProcessor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | #include 33 | #include 34 | 35 | namespace tell { 36 | namespace store { 37 | 38 | struct LogstructuredMemoryStore; 39 | 40 | namespace logstructured { 41 | 42 | class HashScanGarbageCollector; 43 | class HashScanProcessor; 44 | class Table; 45 | 46 | class HashScan : public LLVMRowScanBase { 47 | public: 48 | using ScanProcessor = HashScanProcessor; 49 | using GarbageCollector = HashScanGarbageCollector; 50 | 51 | HashScan(Table* table, std::vector queries); 52 | 53 | using LLVMRowScanBase::prepareQuery; 54 | 55 | using LLVMRowScanBase::prepareMaterialization; 56 | 57 | std::vector> startScan(size_t numThreads); 58 | 59 | private: 60 | Table* mTable; 61 | }; 62 | 63 | /** 64 | * @brief Scan processor for the Log-Structured Memory approach scanning over the hash table 65 | */ 66 | class HashScanProcessor : public LLVMRowScanProcessorBase { 67 | public: 68 | HashScanProcessor(Table& table, const std::vector& queries, size_t start, size_t end, 69 | uint64_t minVersion, HashScan::RowScanFun rowScanFun, 70 | const std::vector& rowMaterializeFuns, uint32_t numConjuncts); 71 | 72 | /** 73 | * @brief Scans over all entries in the hash table 74 | * 75 | * Processes all valid entries with the associated scan queries. 76 | */ 77 | void process(); 78 | 79 | private: 80 | Table& mTable; 81 | uint64_t mMinVersion; 82 | 83 | size_t mStart; 84 | size_t mEnd; 85 | }; 86 | 87 | /** 88 | * @brief Garbage collector for the Log-Structured Memory approach scanning over the hash table 89 | */ 90 | class HashScanGarbageCollector { 91 | public: 92 | HashScanGarbageCollector(LogstructuredMemoryStore&) 93 | {} 94 | 95 | void run(const std::vector& tables, uint64_t minVersion); 96 | }; 97 | 98 | } // namespace logstructured 99 | } // namespace store 100 | } // namespace tell 101 | -------------------------------------------------------------------------------- /logstructured/LogstructuredMemoryStore.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "Table.hpp" 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include 34 | #include 35 | 36 | #include 37 | 38 | namespace tell { 39 | namespace commitmanager { 40 | class SnapshotDescriptor; 41 | } // namespace commitmanager 42 | 43 | namespace store { 44 | 45 | class ScanQuery; 46 | 47 | /** 48 | * @brief A Storage implementation using a Log-Structured Memory approach as its data store 49 | */ 50 | struct LogstructuredMemoryStore : crossbow::non_copyable, crossbow::non_movable { 51 | public: 52 | using Table = logstructured::Table; 53 | using GC = Table::GarbageCollector; 54 | 55 | static const char* implementationName() { 56 | return "Log-Structured Memory"; 57 | } 58 | 59 | LogstructuredMemoryStore(const StorageConfig& config) 60 | : mPageManager(PageManager::construct(config.totalMemory)), 61 | mGc(*this), 62 | mTableManager(*mPageManager, config, mGc, mVersionManager), 63 | mHashMap(config.hashMapCapacity) { 64 | } 65 | 66 | bool createTable(const crossbow::string& name, const Schema& schema, uint64_t& idx) { 67 | return mTableManager.createTable(name, schema, idx, mVersionManager, mHashMap); 68 | } 69 | 70 | std::vector getTables() const { 71 | return mTableManager.getTables(); 72 | } 73 | 74 | const Table* getTable(uint64_t id) const { 75 | return mTableManager.getTable(id); 76 | } 77 | 78 | const Table* getTable(const crossbow::string& name, uint64_t& id) const { 79 | return mTableManager.getTable(name, id); 80 | } 81 | 82 | template 83 | int get(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot, Fun fun) { 84 | return mTableManager.get(tableId, key, snapshot, std::move(fun)); 85 | } 86 | 87 | int update(uint64_t tableId, uint64_t key, size_t size, const char* data, 88 | const commitmanager::SnapshotDescriptor& snapshot) { 89 | return mTableManager.update(tableId, key, size, data, snapshot); 90 | } 91 | 92 | int insert(uint64_t tableId, uint64_t key, size_t size, const char* data, 93 | const commitmanager::SnapshotDescriptor& snapshot) { 94 | return mTableManager.insert(tableId, key, size, data, snapshot); 95 | } 96 | 97 | int remove(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot) { 98 | return mTableManager.remove(tableId, key, snapshot); 99 | } 100 | 101 | int revert(uint64_t tableId, uint64_t key, const commitmanager::SnapshotDescriptor& snapshot) { 102 | return mTableManager.revert(tableId, key, snapshot); 103 | } 104 | 105 | int scan(uint64_t tableId, ScanQuery* query) { 106 | return mTableManager.scan(tableId, query); 107 | } 108 | 109 | /** 110 | * We use this method mostly for test purposes. But 111 | * it might be handy in the future as well. If possible, 112 | * this should be implemented in an efficient way. 113 | */ 114 | void forceGC() { 115 | mTableManager.forceGC(); 116 | } 117 | 118 | private: 119 | PageManager::Ptr mPageManager; 120 | GC mGc; 121 | VersionManager mVersionManager; 122 | TableManager mTableManager; 123 | 124 | Table::HashTable mHashMap; 125 | }; 126 | 127 | } // namespace store 128 | } // namespace tell 129 | -------------------------------------------------------------------------------- /server/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore server 3 | ################### 4 | set(SERVER_SRCS 5 | ServerScanQuery.cpp 6 | ServerSocket.cpp 7 | ) 8 | 9 | set(SERVER_PRIVATE_HDR 10 | ServerConfig.hpp 11 | ServerScanQuery.hpp 12 | ServerSocket.hpp 13 | Storage.hpp 14 | ) 15 | 16 | macro(add_tellstored _name _implementation) 17 | # Add TellStore server executable 18 | add_executable(tellstored-${_name} main.cpp ${SERVER_SRCS} ${SERVER_PRIVATE_HDR}) 19 | target_compile_definitions(tellstored-${_name} PRIVATE ${ARGN}) 20 | 21 | # Link against TellStore library 22 | target_link_libraries(tellstored-${_name} PRIVATE tellstore-${_implementation} tellstore-common) 23 | 24 | # Link against Boost 25 | target_include_directories(tellstored-${_name} PRIVATE ${Boost_INCLUDE_DIRS}) 26 | 27 | # Link against Crossbow 28 | target_include_directories(tellstored-${_name} PRIVATE ${Crossbow_INCLUDE_DIRS}) 29 | target_link_libraries(tellstored-${_name} PRIVATE crossbow_allocator crossbow_infinio crossbow_logger) 30 | 31 | # Link against Jemalloc 32 | target_include_directories(tellstored-${_name} PRIVATE ${Jemalloc_INCLUDE_DIRS}) 33 | target_link_libraries(tellstored-${_name} PRIVATE ${Jemalloc_LIBRARIES}) 34 | 35 | # Install the binary 36 | install(TARGETS tellstored-${_name} 37 | RUNTIME DESTINATION ${BIN_INSTALL_DIR}) 38 | endmacro() 39 | 40 | add_tellstored(logstructured logstructured USE_LOGSTRUCTURED_MEMORY) 41 | add_tellstored(rowstore deltamain USE_DELTA_MAIN_REWRITE USE_ROW_STORE) 42 | add_tellstored(columnmap deltamain USE_DELTA_MAIN_REWRITE USE_COLUMN_MAP) 43 | -------------------------------------------------------------------------------- /server/ServerConfig.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | 28 | namespace tell { 29 | namespace store { 30 | 31 | /** 32 | * @brief The ServerConfig struct containing configuration parameters for the TellStore server 33 | */ 34 | struct ServerConfig { 35 | /// Port to listen for incoming client connections 36 | uint16_t port = 7241; 37 | 38 | /// Number of network threads to process requests on 39 | int numNetworkThreads = 2; 40 | 41 | /// Size of the buffers used in scans 42 | uint32_t scanBufferLength = 0x100000; 43 | 44 | /// Maximum number of buffers used in scans 45 | uint32_t scanBufferCount = 256; 46 | 47 | /// Maximum number of scan buffers that are in flight on a socket at the same time 48 | /// This limit might be exceeded by a small factor 49 | uint64_t maxInflightScanBuffer = 16; 50 | 51 | /// Maximum number of messages per batch 52 | size_t maxBatchSize = 16; 53 | }; 54 | 55 | } // namespace store 56 | } // namespace tell 57 | -------------------------------------------------------------------------------- /server/Storage.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | #if defined USE_DELTA_MAIN_REWRITE 28 | #include 29 | #elif defined USE_LOGSTRUCTURED_MEMORY 30 | #include 31 | #else 32 | #error "Unknown implementation" 33 | #endif 34 | 35 | namespace tell { 36 | namespace store { 37 | 38 | #if defined USE_DELTA_MAIN_REWRITE 39 | #if defined USE_ROW_STORE 40 | using Storage = DeltaMainRewriteRowStore; 41 | #elif defined USE_COLUMN_MAP 42 | using Storage = DeltaMainRewriteColumnStore; 43 | #else 44 | #error "Unknown implementation" 45 | #endif 46 | 47 | #elif defined USE_LOGSTRUCTURED_MEMORY 48 | using Storage = LogstructuredMemoryStore; 49 | 50 | #else 51 | #error "Unknown implementation" 52 | #endif 53 | 54 | } // namespace store 55 | } // namespace tell 56 | -------------------------------------------------------------------------------- /server/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include "ServerConfig.hpp" 24 | #include "ServerSocket.hpp" 25 | #include "Storage.hpp" 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | int main(int argc, const char** argv) { 37 | tell::store::StorageConfig storageConfig; 38 | tell::store::ServerConfig serverConfig; 39 | bool help = false; 40 | crossbow::string logLevel("DEBUG"); 41 | 42 | auto opts = crossbow::program_options::create_options(argv[0], 43 | crossbow::program_options::value<'h'>("help", &help), 44 | crossbow::program_options::value<'l'>("log-level", &logLevel), 45 | crossbow::program_options::value<'p'>("port", &serverConfig.port), 46 | crossbow::program_options::value<'m'>("memory", &storageConfig.totalMemory), 47 | crossbow::program_options::value<'c'>("capacity", &storageConfig.hashMapCapacity), 48 | crossbow::program_options::value<-1>("network-threads", &serverConfig.numNetworkThreads, 49 | crossbow::program_options::tag::ignore_short{}), 50 | crossbow::program_options::value<-2>("scan-threads", &storageConfig.numScanThreads, 51 | crossbow::program_options::tag::ignore_short{}), 52 | crossbow::program_options::value<-3>("gc-interval", &storageConfig.gcInterval, 53 | crossbow::program_options::tag::ignore_short{})); 54 | 55 | try { 56 | crossbow::program_options::parse(opts, argc, argv); 57 | } catch (crossbow::program_options::argument_not_found e) { 58 | std::cerr << e.what() << std::endl << std::endl; 59 | crossbow::program_options::print_help(std::cout, opts); 60 | return 1; 61 | } 62 | 63 | if (help) { 64 | crossbow::program_options::print_help(std::cout, opts); 65 | return 0; 66 | } 67 | 68 | crossbow::infinio::InfinibandLimits infinibandLimits; 69 | infinibandLimits.receiveBufferCount = 1024; 70 | infinibandLimits.sendBufferCount = 256; 71 | infinibandLimits.bufferLength = 128 * 1024; 72 | infinibandLimits.sendQueueLength = 128; 73 | infinibandLimits.completionQueueLength = 2048; 74 | 75 | crossbow::logger::logger->config.level = crossbow::logger::logLevelFromString(logLevel); 76 | 77 | LOG_INFO("Starting TellStore server"); 78 | LOG_INFO("--- Backend: %1%", tell::store::Storage::implementationName()); 79 | LOG_INFO("--- Port: %1%", serverConfig.port); 80 | LOG_INFO("--- Network Threads: %1%", serverConfig.numNetworkThreads); 81 | LOG_INFO("--- GC Interval: %1%s", storageConfig.gcInterval); 82 | LOG_INFO("--- Total Memory: %1%GB", double(storageConfig.totalMemory) / double(1024 * 1024 * 1024)); 83 | LOG_INFO("--- Scan Threads: %1%", storageConfig.numScanThreads); 84 | LOG_INFO("--- Hash Map Capacity: %1%", storageConfig.hashMapCapacity); 85 | 86 | // Initialize allocator 87 | crossbow::allocator::init(); 88 | 89 | LOG_INFO("Initialize storage"); 90 | tell::store::Storage storage(storageConfig); 91 | 92 | LOG_INFO("Initialize network server"); 93 | crossbow::infinio::InfinibandService service(infinibandLimits); 94 | tell::store::ServerManager server(service, storage, serverConfig); 95 | LOG_INFO("Storage ready"); 96 | service.run(); 97 | 98 | LOG_INFO("Exiting TellStore server"); 99 | return 0; 100 | } 101 | -------------------------------------------------------------------------------- /tellstore/AbstractTuple.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | #include 25 | 26 | namespace tell { 27 | namespace store { 28 | 29 | class AbstractTuple { 30 | public: 31 | virtual ~AbstractTuple(); 32 | virtual size_t size() const = 0; 33 | virtual void serialize(char* dest) const = 0; 34 | }; 35 | 36 | } // namespace store 37 | } // namespace tell 38 | 39 | -------------------------------------------------------------------------------- /tellstore/ClientConfig.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | namespace tell { 33 | namespace store { 34 | 35 | struct ClientConfig { 36 | static crossbow::infinio::Endpoint parseCommitManager(const crossbow::string& host) { 37 | return crossbow::infinio::Endpoint(crossbow::infinio::Endpoint::ipv4(), host); 38 | } 39 | 40 | static inline std::vector parseTellStore(const crossbow::string& host); 41 | 42 | ClientConfig() 43 | : maxPendingResponses(48ull), 44 | maxBatchSize(16ull), 45 | numNetworkThreads(2ull) { 46 | infinibandConfig.receiveBufferCount = 256; 47 | infinibandConfig.sendBufferCount = 256; 48 | infinibandConfig.bufferLength = 128 * 1024; 49 | infinibandConfig.sendQueueLength = 128; 50 | infinibandConfig.completionQueueLength = 512; 51 | } 52 | 53 | /// Configuration for the Infiniband devices 54 | crossbow::infinio::InfinibandLimits infinibandConfig; 55 | 56 | /// Address of the CommitManager to connect to 57 | crossbow::infinio::Endpoint commitManager; 58 | 59 | /// Address of the TellStore to connect to 60 | std::vector tellStore; 61 | 62 | /// Maximum number of concurrent pending network requests (per connection) 63 | size_t maxPendingResponses; 64 | 65 | /// Maximum number of messages per batch 66 | size_t maxBatchSize; 67 | 68 | /// Number of network threads to process transactions on 69 | size_t numNetworkThreads; 70 | }; 71 | 72 | std::vector ClientConfig::parseTellStore(const crossbow::string& host) { 73 | if (host.empty()) { 74 | return {}; 75 | } 76 | 77 | std::vector result; 78 | size_t i = 0; 79 | while (true) { 80 | auto pos = host.find(';', i); 81 | result.emplace_back(crossbow::infinio::Endpoint::ipv4(), host.substr(i, pos)); 82 | if (pos == crossbow::string::npos) { 83 | break; 84 | } 85 | i = pos + 1; 86 | } 87 | return result; 88 | } 89 | 90 | } // namespace store 91 | } // namespace tell 92 | -------------------------------------------------------------------------------- /tellstore/ErrorCode.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace tell { 30 | namespace store { 31 | namespace error { 32 | 33 | /** 34 | * @brief TellStore errors triggered while executing an operation 35 | */ 36 | enum errors { 37 | /// Server received an unknown request type. 38 | unkown_request = 1, 39 | 40 | /// Client received an unkown response type. 41 | unkown_response, 42 | 43 | /// Table does not exist. 44 | invalid_table, 45 | 46 | /// The tuple was invalid. 47 | invalid_tuple, 48 | 49 | /// Snapshot sent to the server was invalid. 50 | invalid_snapshot, 51 | 52 | /// ID sent to the server was invalid. 53 | invalid_scan, 54 | 55 | /// Operation failed due to server overload. 56 | server_overlad, 57 | 58 | /// Out of memory. 59 | out_of_memory, 60 | 61 | /// Tuple not found. 62 | not_found, 63 | 64 | /// Most recent tuple is not in read set of the snapshot. 65 | not_in_snapshot, 66 | 67 | /// Write operation unable to complete. 68 | invalid_write, 69 | }; 70 | 71 | /** 72 | * @brief Category for TellStore errors 73 | */ 74 | class error_category : public std::error_category { 75 | public: 76 | const char* name() const noexcept { 77 | return "tell.store"; 78 | } 79 | 80 | std::string message(int value) const { 81 | switch (value) { 82 | case error::unkown_request: 83 | return "Server received an unknown request type"; 84 | 85 | case error::unkown_response: 86 | return "Client received an unkown response type"; 87 | 88 | case error::invalid_table: 89 | return "Table was invalid"; 90 | 91 | case error::invalid_tuple: 92 | return "Tuple was invalid"; 93 | 94 | case error::invalid_snapshot: 95 | return "Snapshot was invalid"; 96 | 97 | case error::invalid_scan: 98 | return "Scan ID was invalid"; 99 | 100 | case error::server_overlad: 101 | return "Operation failed due to server overload"; 102 | 103 | case out_of_memory: 104 | return "Out of memory"; 105 | 106 | case not_found: 107 | return "Tuple not found"; 108 | 109 | case not_in_snapshot: 110 | return "Most recent tuple is not in read set of the snapshot"; 111 | 112 | case invalid_write: 113 | return "Write operation unable to complete"; 114 | 115 | default: 116 | return "tell.store.server error"; 117 | } 118 | } 119 | }; 120 | 121 | inline const std::error_category& get_error_category() { 122 | static error_category instance; 123 | return instance; 124 | } 125 | 126 | inline std::error_code make_error_code(error::errors e) { 127 | return std::error_code(static_cast(e), get_error_category()); 128 | } 129 | 130 | } // namespace error 131 | } // namespace store 132 | } // namespace tell 133 | 134 | namespace std { 135 | 136 | template<> 137 | struct is_error_code_enum : public std::true_type { 138 | }; 139 | 140 | } // namespace std 141 | -------------------------------------------------------------------------------- /tellstore/GenericTuple.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | 34 | namespace tell { 35 | namespace store { 36 | 37 | class Record; 38 | 39 | using GenericTuple = std::unordered_map; 40 | 41 | class GenericTupleSerializer : public AbstractTuple { 42 | const Record& mRecord; 43 | GenericTuple mTuple; 44 | size_t mSize; 45 | public: 46 | GenericTupleSerializer(const Record& record, GenericTuple tuple); 47 | 48 | virtual ~GenericTupleSerializer(); 49 | 50 | virtual size_t size() const override; 51 | 52 | virtual void serialize(char* dest) const override; 53 | }; 54 | 55 | } // namespace store 56 | } // namespace tell 57 | -------------------------------------------------------------------------------- /tellstore/MessageTypes.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | namespace tell { 31 | namespace store { 32 | 33 | /** 34 | * @brief Unique string sent as first argument in the connection handshake 35 | */ 36 | const crossbow::string& handshakeString(); 37 | 38 | /** 39 | * @brief The possible messages types of a request 40 | */ 41 | enum class RequestType : uint32_t { 42 | CREATE_TABLE = 0x1u, 43 | GET_TABLES, 44 | GET_TABLE, 45 | GET, 46 | UPDATE, 47 | INSERT, 48 | REMOVE, 49 | REVERT, 50 | SCAN, 51 | SCAN_PROGRESS, 52 | COMMIT, 53 | }; 54 | 55 | /** 56 | * @brief The possible messages types of a response 57 | */ 58 | enum class ResponseType : uint32_t { 59 | CREATE_TABLE = 0x01u, 60 | GET_TABLES, 61 | GET_TABLE, 62 | GET, 63 | MODIFICATION, 64 | SCAN, 65 | COMMIT, 66 | }; 67 | 68 | } // namespace store 69 | } // namespace tell 70 | -------------------------------------------------------------------------------- /tellstore/ScanMemory.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | namespace crossbow { 33 | namespace infinio { 34 | class InfinibandService; 35 | } // namespace infinio 36 | } // namespace crossbow 37 | 38 | namespace tell { 39 | namespace store { 40 | 41 | class ScanMemory; 42 | 43 | /** 44 | * @brief Buffer pool managing memory used for to store scan tuples 45 | */ 46 | class ScanMemoryManager : crossbow::non_copyable, crossbow::non_movable { 47 | public: 48 | ScanMemoryManager(crossbow::infinio::InfinibandService& service, size_t chunkCount, size_t chunkLength); 49 | 50 | /** 51 | * @brief Acquire a buffer from the scan memory pool 52 | */ 53 | ScanMemory acquire(); 54 | 55 | private: 56 | friend class ScanMemory; 57 | 58 | /** 59 | * @brief Release the data pointer to the chunk pool 60 | */ 61 | void release(void* data); 62 | 63 | size_t mChunkLength; 64 | 65 | crossbow::infinio::AllocatedMemoryRegion mRegion; 66 | 67 | crossbow::fixed_size_stack mChunkStack; 68 | }; 69 | 70 | /** 71 | * @brief Buffer to store scan tuples 72 | */ 73 | class ScanMemory : crossbow::non_copyable { 74 | public: 75 | ScanMemory() 76 | : mManager(nullptr), 77 | mData(nullptr), 78 | mLength(0u), 79 | mKey(0u) { 80 | } 81 | 82 | ScanMemory(ScanMemoryManager* manager, void* data, size_t length, uint32_t key) 83 | : mManager(manager), 84 | mData(data), 85 | mLength(length), 86 | mKey(key) { 87 | } 88 | 89 | ~ScanMemory(); 90 | 91 | ScanMemory(ScanMemory&& other) 92 | : mManager(other.mManager), 93 | mData(other.mData), 94 | mLength(other.mLength), 95 | mKey(other.mKey) { 96 | other.mManager = nullptr; 97 | other.mData = nullptr; 98 | other.mLength = 0u; 99 | other.mKey = 0u; 100 | } 101 | 102 | ScanMemory& operator=(ScanMemory&& other); 103 | 104 | bool valid() const { 105 | return (mManager != nullptr); 106 | } 107 | 108 | const void* data() const { 109 | return mData; 110 | } 111 | 112 | size_t length() const { 113 | return mLength; 114 | } 115 | 116 | uint32_t key() const { 117 | return mKey; 118 | } 119 | 120 | private: 121 | ScanMemoryManager* mManager; 122 | 123 | void* mData; 124 | size_t mLength; 125 | uint32_t mKey; 126 | }; 127 | 128 | } // namespace store 129 | } // namespace tell 130 | -------------------------------------------------------------------------------- /tellstore/StdTypes.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | #include 25 | 26 | namespace tell { 27 | namespace store { 28 | 29 | enum class TableType : uint8_t { 30 | UNKNOWN, 31 | TRANSACTIONAL, 32 | NON_TRANSACTIONAL, 33 | }; 34 | 35 | enum class FieldType 36 | : uint16_t { 37 | NOTYPE = 0, 38 | NULLTYPE = 1, 39 | SMALLINT = 2, 40 | INT, 41 | BIGINT, 42 | FLOAT, 43 | DOUBLE, 44 | TEXT, // this is used for CHAR and VARCHAR as well 45 | BLOB 46 | }; 47 | 48 | enum class PredicateType : uint8_t { 49 | EQUAL = 1, 50 | NOT_EQUAL, 51 | LESS, 52 | LESS_EQUAL, 53 | GREATER, 54 | GREATER_EQUAL, 55 | PREFIX_LIKE, 56 | PREFIX_NOT_LIKE, 57 | POSTFIX_LIKE, 58 | POSTFIX_NOT_LIKE, 59 | IS_NULL, 60 | IS_NOT_NULL 61 | }; 62 | 63 | enum class AggregationType : uint8_t { 64 | MIN = 1, 65 | MAX, 66 | SUM, 67 | CNT, 68 | }; 69 | 70 | enum ScanQueryType : uint8_t { 71 | FULL = 0x1u, 72 | PROJECTION, 73 | AGGREGATION, 74 | }; 75 | 76 | } // namespace store 77 | } // namespace tell 78 | -------------------------------------------------------------------------------- /tellstore/Table.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | namespace crossbow { 37 | class buffer_reader; 38 | } // namespace crossbow 39 | 40 | namespace tell { 41 | namespace store { 42 | 43 | class Tuple final : crossbow::non_copyable, crossbow::non_movable { 44 | public: // Construction 45 | void* operator new(size_t size, uint32_t dataLen); 46 | 47 | void* operator new[](size_t size) = delete; 48 | 49 | void operator delete(void* ptr); 50 | 51 | void operator delete[](void* ptr) = delete; 52 | 53 | public: // Serialization 54 | static std::unique_ptr deserialize(crossbow::buffer_reader& reader); 55 | 56 | public: 57 | const char* data() const { 58 | return reinterpret_cast(this) + sizeof(Tuple); 59 | } 60 | 61 | uint64_t version() const { 62 | return mVersion; 63 | } 64 | 65 | bool isNewest() const { 66 | return mIsNewest; 67 | } 68 | 69 | size_t size() const { 70 | return mSize; 71 | } 72 | 73 | private: 74 | Tuple(uint64_t version, bool isNewest, uint32_t size) 75 | : mVersion(version), 76 | mIsNewest(isNewest), 77 | mSize(size) { 78 | } 79 | 80 | uint64_t mVersion; 81 | bool mIsNewest; 82 | uint32_t mSize; 83 | }; 84 | 85 | class Table { 86 | public: 87 | Table() 88 | : mTableId(0x0u) { 89 | } 90 | 91 | Table(uint64_t tableId, Schema schema) 92 | : mTableId(tableId), 93 | mRecord(std::move(schema)) { 94 | } 95 | 96 | Table(uint64_t tableId, crossbow::string tableName, Schema schema) 97 | : mTableId(tableId), 98 | mTableName(std::move(tableName)), 99 | mRecord(std::move(schema)) { 100 | } 101 | 102 | uint64_t tableId() const { 103 | return mTableId; 104 | } 105 | 106 | const crossbow::string& tableName() const { 107 | return mTableName; 108 | } 109 | 110 | const Record& record() const { 111 | return mRecord; 112 | } 113 | 114 | TableType tableType() const { 115 | return mRecord.schema().type(); 116 | } 117 | 118 | template 119 | T field(const crossbow::string& name, const char* data) const; 120 | 121 | GenericTuple toGenericTuple(const char* data) const; 122 | 123 | private: 124 | uint64_t mTableId; 125 | crossbow::string mTableName; 126 | Record mRecord; 127 | }; 128 | 129 | template 130 | T Table::field(const crossbow::string& name, const char* data) const { 131 | Record::id_t id; 132 | if (!mRecord.idOf(name, id)) { 133 | throw std::logic_error("Field not found"); 134 | } 135 | 136 | bool isNull = false; 137 | FieldType type; 138 | auto field = mRecord.data(data, id, isNull, &type); 139 | if (isNull) { 140 | throw std::logic_error("Field is null"); 141 | } 142 | 143 | boost::any value; 144 | switch (type) { 145 | 146 | case FieldType::SMALLINT: { 147 | value = *reinterpret_cast(field); 148 | } break; 149 | 150 | case FieldType::INT: { 151 | value = *reinterpret_cast(field); 152 | } break; 153 | 154 | case FieldType::BIGINT: { 155 | value = *reinterpret_cast(field); 156 | } break; 157 | 158 | case FieldType::FLOAT: { 159 | value = *reinterpret_cast(field); 160 | } break; 161 | 162 | case FieldType::DOUBLE: { 163 | value = *reinterpret_cast(field); 164 | } break; 165 | 166 | case FieldType::TEXT: 167 | case FieldType::BLOB: { 168 | auto offsetData = reinterpret_cast(field); 169 | auto offset = offsetData[0]; 170 | auto length = offsetData[1] - offset; 171 | value = crossbow::string(data + offset, length); 172 | } break; 173 | 174 | default: { 175 | throw std::logic_error("Invalid field type"); 176 | } break; 177 | } 178 | 179 | return boost::any_cast(value); 180 | } 181 | 182 | } // namespace store 183 | } // namespace tell 184 | -------------------------------------------------------------------------------- /tellstore/TransactionType.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | #include 25 | 26 | namespace tell { 27 | namespace store { 28 | 29 | /** 30 | * @brief Type of a transaction 31 | * 32 | * - A READ_WRITE transaction is a default transaction, allowed 33 | * to execute read as well as write operations. 34 | * - A READ_ONLY transaction will not be allowed to issue any write 35 | * requests. This allows Tell to do several optimizations. The most 36 | * important is, that other transactions might have its version 37 | * number in their read sets (since there won't be any tuples with 38 | * that version, they don't have to exclude this version from their 39 | * read set). 40 | * - An ANALYTICAL transaction is a special case of a READ_WRITE 41 | * transaction. Its read set will only consist of one base version. 42 | * Therefore it won't see the newest data. On the other hand, this 43 | * will allow for faster scans (and they won't violate any transacional 44 | * guarantees). 45 | */ 46 | enum class TransactionType : uint8_t { 47 | READ_WRITE, 48 | READ_ONLY, 49 | ANALYTICAL, 50 | }; 51 | 52 | } // namespace store 53 | } // namespace tell 54 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # Unit tests 3 | ################### 4 | set(TEST_SRCS 5 | DummyCommitManager.cpp 6 | DummyCommitManager.hpp 7 | testCuckooMap.cpp 8 | testCommitManager.cpp 9 | testLog.cpp 10 | testOpenAddressingHash.cpp 11 | simpleTests.cpp 12 | deltamain/testInsertHash.cpp 13 | logstructured/testTable.cpp 14 | ) 15 | 16 | set(TEST_PRIVATE_HDR 17 | DummyCommitManager.hpp 18 | ) 19 | 20 | # Add test executable 21 | add_executable(tests main.cpp ${TEST_SRCS} ${TEST_PRIVATE_HDR}) 22 | target_include_directories(tests PRIVATE ${PROJECT_BINARY_DIR}) 23 | target_link_libraries(tests PRIVATE tellstore-deltamain tellstore-logstructured) 24 | 25 | # Link test against GTest 26 | target_include_directories(tests PRIVATE ${gtest_SOURCE_DIR}/include) 27 | target_link_libraries(tests PRIVATE gtest gtest_main) 28 | 29 | # Link against Crossbow 30 | target_include_directories(tests PRIVATE ${Crossbow_INCLUDE_DIRS}) 31 | target_link_libraries(tests PRIVATE crossbow_allocator) 32 | 33 | # Link against Jemalloc 34 | target_include_directories(tests PRIVATE ${Jemalloc_INCLUDE_DIRS}) 35 | target_link_libraries(tests PRIVATE ${Jemalloc_LIBRARIES}) 36 | 37 | add_test(tests tests) 38 | 39 | ################### 40 | # TellStore client test 41 | ################### 42 | set(CLIENT_TEST_SRCS 43 | client/TestClient.cpp 44 | ) 45 | 46 | # Add TellStore test client executable 47 | add_executable(tellstore-test ${CLIENT_TEST_SRCS}) 48 | 49 | # Link against TellStore client 50 | target_link_libraries(tellstore-test PRIVATE tellstore-client) 51 | 52 | # Link against Crossbow 53 | target_include_directories(tellstore-test PRIVATE ${Crossbow_INCLUDE_DIRS}) 54 | target_link_libraries(tellstore-test PRIVATE crossbow_infinio crossbow_logger) 55 | 56 | # Link against Jemalloc 57 | target_include_directories(tellstore-test PRIVATE ${Jemalloc_INCLUDE_DIRS}) 58 | target_link_libraries(tellstore-test PRIVATE ${Jemalloc_LIBRARIES}) 59 | -------------------------------------------------------------------------------- /tests/DummyCommitManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include "DummyCommitManager.hpp" 24 | 25 | #include 26 | 27 | namespace tell { 28 | namespace store { 29 | 30 | Transaction DummyCommitManager::startTx(bool readonly /* = false */) { 31 | Lock _(mMutex); 32 | if (!mManager.startTransaction(readonly)) { 33 | throw std::logic_error("Error starting transaction"); 34 | } 35 | return {*this, mManager.createSnapshot()}; 36 | } 37 | 38 | void DummyCommitManager::commitTx(uint64_t version) { 39 | Lock _(mMutex); 40 | if (!mManager.commitTransaction(version)) { 41 | throw std::logic_error("Error committing transaction"); 42 | } 43 | } 44 | 45 | Transaction::~Transaction() { 46 | if (!mCommitted) 47 | mManager->abortTx(mDescriptor->version()); 48 | } 49 | 50 | Transaction::Transaction(Transaction&& other) 51 | : mManager(other.mManager), 52 | mDescriptor(std::move(other.mDescriptor)), 53 | mCommitted(other.mCommitted) { 54 | other.mCommitted = true; 55 | } 56 | 57 | Transaction& Transaction::operator=(Transaction&& other) { 58 | // First we need to destroy ourself 59 | if (!mCommitted) 60 | mManager->abortTx(mDescriptor->version()); 61 | mManager = other.mManager; 62 | mDescriptor = std::move(other.mDescriptor); 63 | mCommitted = other.mCommitted; 64 | other.mCommitted = true; 65 | return *this; 66 | } 67 | 68 | void Transaction::commit() { 69 | if (mCommitted) 70 | return; 71 | mManager->commitTx(mDescriptor->version()); 72 | mCommitted = true; 73 | } 74 | 75 | void Transaction::abort() { 76 | if (mCommitted) 77 | return; 78 | mManager->abortTx(mDescriptor->version()); 79 | mCommitted = true; 80 | } 81 | 82 | } // namespace store 83 | } // namespace tell 84 | -------------------------------------------------------------------------------- /tests/DummyCommitManager.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | namespace tell { 33 | namespace store { 34 | 35 | class Transaction; 36 | 37 | class DummyCommitManager { 38 | public: 39 | Transaction startTx(bool readonly = false); 40 | 41 | private: 42 | friend class Transaction; 43 | 44 | void abortTx(uint64_t version) { 45 | commitTx(version); 46 | } 47 | 48 | void commitTx(uint64_t version); 49 | 50 | commitmanager::CommitManager mManager; 51 | 52 | mutable std::mutex mMutex; 53 | using Lock = std::lock_guard; 54 | }; 55 | 56 | using CommitManager = DummyCommitManager; 57 | 58 | /** 59 | * @brief Simple snapshot holder 60 | * 61 | * The idea behind this object is, to make sure that a user never forgets to commit or abort a transaction. Objects of 62 | * this class will simply hold a snapshot descriptor and will commit a transaction when the object gets destroyed (if 63 | * the user did not do so manually). 64 | */ 65 | class Transaction : crossbow::non_copyable { 66 | DummyCommitManager* mManager; 67 | std::unique_ptr mDescriptor; 68 | bool mCommitted = false; 69 | public: 70 | Transaction(DummyCommitManager& manager, std::unique_ptr snapshot) 71 | : mManager(&manager), 72 | mDescriptor(std::move(snapshot)) { 73 | } 74 | 75 | ~Transaction(); 76 | 77 | Transaction(Transaction&& other); 78 | Transaction& operator=(Transaction&& other); 79 | 80 | const commitmanager::SnapshotDescriptor& operator*() const { 81 | return *operator->(); 82 | } 83 | 84 | const commitmanager::SnapshotDescriptor* operator->() const { 85 | return mDescriptor.get(); 86 | } 87 | 88 | public: 89 | operator const commitmanager::SnapshotDescriptor&() const { 90 | return *mDescriptor; 91 | } 92 | 93 | const commitmanager::SnapshotDescriptor& descriptor() const { 94 | return *mDescriptor; 95 | } 96 | 97 | void commit(); 98 | 99 | void abort(); 100 | }; 101 | 102 | } // namespace store 103 | } // namespace tell 104 | -------------------------------------------------------------------------------- /tests/deltamain/testInsertHash.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include 25 | 26 | #include 27 | 28 | using namespace tell::store; 29 | using namespace tell::store::deltamain; 30 | 31 | namespace { 32 | 33 | class InsertTableTest : public ::testing::Test { 34 | protected: 35 | InsertTableTest() 36 | : mTable(128), 37 | mElement1(0x1u), mElement2(0x2u), mElement3(0x3u) { 38 | } 39 | 40 | InsertTable mTable; 41 | 42 | uint64_t mElement1; 43 | uint64_t mElement2; 44 | uint64_t mElement3; 45 | }; 46 | 47 | /** 48 | * @class InsertTable 49 | * @test Check if a simple get after insert returns the element 50 | */ 51 | TEST_F(InsertTableTest, insertAndGet) { 52 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 53 | 54 | EXPECT_EQ(&mElement1, mTable.get(10u)); 55 | } 56 | 57 | /** 58 | * @class InsertTable 59 | * @test Check if multiple get and inserts return the correct elements 60 | */ 61 | TEST_F(InsertTableTest, insertAndGetMultiple) { 62 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 63 | EXPECT_TRUE(mTable.insert(11u, &mElement2)); 64 | EXPECT_TRUE(mTable.insert(138u, &mElement3)); 65 | 66 | EXPECT_EQ(&mElement1, mTable.get(10u)); 67 | EXPECT_EQ(&mElement2, mTable.get(11u)); 68 | EXPECT_EQ(&mElement3, mTable.get(138u)); 69 | } 70 | 71 | /** 72 | * @class InsertTable 73 | * @test Check if inserting a duplicate fails 74 | */ 75 | TEST_F(InsertTableTest, insertDuplicate) { 76 | void* actualData = nullptr; 77 | EXPECT_TRUE(mTable.insert(10u, &mElement1, &actualData)); 78 | EXPECT_EQ(nullptr, actualData); 79 | 80 | EXPECT_FALSE(mTable.insert(10u, &mElement2, &actualData)); 81 | EXPECT_EQ(&mElement1, actualData); 82 | 83 | EXPECT_EQ(&mElement1, mTable.get(10u)); 84 | } 85 | 86 | /** 87 | * @class InsertTable 88 | * @test Check if removing an element and inserting another one works correctly 89 | */ 90 | TEST_F(InsertTableTest, removeAndInsert) { 91 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 92 | EXPECT_EQ(&mElement1, mTable.get(10u)); 93 | 94 | EXPECT_TRUE(mTable.remove(10u, &mElement1)); 95 | EXPECT_EQ(nullptr, mTable.get(10u)); 96 | 97 | EXPECT_TRUE(mTable.insert(10u, &mElement2)); 98 | EXPECT_EQ(&mElement2, mTable.get(10u)); 99 | } 100 | 101 | /** 102 | * @class InsertTable 103 | * @test Check if removing a changed element is prevented 104 | */ 105 | TEST_F(InsertTableTest, removeChanged) { 106 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 107 | 108 | void* actualData = nullptr; 109 | EXPECT_FALSE(mTable.remove(10u, &mElement2, &actualData)); 110 | EXPECT_EQ(&mElement1, actualData); 111 | EXPECT_EQ(&mElement1, mTable.get(10u)); 112 | } 113 | 114 | /** 115 | * @class InsertTable 116 | * @test Check if updating an element works 117 | */ 118 | TEST_F(InsertTableTest, update) { 119 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 120 | 121 | EXPECT_TRUE(mTable.update(10u, &mElement1, &mElement2)); 122 | EXPECT_EQ(&mElement2, mTable.get(10u)); 123 | } 124 | 125 | /** 126 | * @class InsertTable 127 | * @test Check if updating a changed element is prevented 128 | */ 129 | TEST_F(InsertTableTest, updateChanged) { 130 | EXPECT_TRUE(mTable.insert(10u, &mElement1)); 131 | 132 | void* actualData = nullptr; 133 | EXPECT_FALSE(mTable.update(10u, &mElement3, &mElement2, &actualData)); 134 | EXPECT_EQ(&mElement1, actualData); 135 | EXPECT_EQ(&mElement1, mTable.get(10u)); 136 | } 137 | 138 | /** 139 | * @class DynamicInsertTable 140 | * @test Check if resizing works correctly 141 | */ 142 | TEST(DynamicInsertTableTest, resize) { 143 | DynamicInsertTable table(4u); 144 | uint64_t element1 = 0x1u; 145 | uint64_t element2 = 0x2u; 146 | uint64_t element3 = 0x3u; 147 | uint64_t element4 = 0x4u; 148 | uint64_t element5 = 0x5u; 149 | 150 | EXPECT_TRUE(table.insert(10u, &element1)); 151 | EXPECT_TRUE(table.insert(11u, &element2)); 152 | EXPECT_TRUE(table.insert(138u, &element3)); 153 | EXPECT_TRUE(table.insert(139u, &element4)); 154 | EXPECT_TRUE(table.insert(140u, &element5)); 155 | 156 | EXPECT_FALSE(table.insert(10u, &element2)); 157 | EXPECT_FALSE(table.insert(139u, &element2)); 158 | 159 | EXPECT_EQ(&element1, table.get(10u)); 160 | EXPECT_EQ(&element2, table.get(11u)); 161 | EXPECT_EQ(&element3, table.get(138u)); 162 | EXPECT_EQ(&element4, table.get(139u)); 163 | EXPECT_EQ(&element5, table.get(140u)); 164 | } 165 | 166 | } 167 | -------------------------------------------------------------------------------- /tests/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include 24 | 25 | #include 26 | 27 | int main(int argc, char** argv) { 28 | crossbow::allocator::init(); 29 | ::testing::InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | -------------------------------------------------------------------------------- /tests/testCommitManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include "DummyCommitManager.hpp" 24 | #include 25 | 26 | using namespace tell; 27 | using namespace tell::store; 28 | 29 | namespace { 30 | 31 | TEST(commit_manager_test, simple) { 32 | CommitManager commitManager; 33 | auto oldDescriptor = commitManager.startTx(); 34 | auto newDescriptor = commitManager.startTx(); 35 | auto newV = newDescriptor->version(); 36 | EXPECT_FALSE(oldDescriptor->inReadSet(newDescriptor->version())); 37 | EXPECT_FALSE(newDescriptor->inReadSet(oldDescriptor->version())); 38 | newDescriptor.commit(); 39 | newDescriptor = commitManager.startTx(); 40 | EXPECT_FALSE(oldDescriptor->inReadSet(newDescriptor->version())); 41 | EXPECT_TRUE(newDescriptor->inReadSet(newV)); 42 | EXPECT_FALSE(newDescriptor->inReadSet(oldDescriptor->version())); 43 | oldDescriptor.commit(); 44 | newDescriptor.commit(); 45 | } 46 | 47 | TEST(commit_manager_test, many_transactions) { 48 | CommitManager commitManager; 49 | std::vector descriptors; 50 | descriptors.reserve(1024); 51 | // start 1024 transactions 52 | std::vector versions; 53 | versions.reserve(1024); 54 | for (uint64_t i = 0; i < 1024; ++i) { 55 | descriptors.emplace_back(commitManager.startTx()); 56 | auto version = descriptors[i]->version(); 57 | EXPECT_EQ(i + 1, version) << "Expected version " << i + 1 << " does not match actual version " << version; 58 | versions.push_back(version); 59 | } 60 | // committing every second transaction 61 | for (uint64_t i = 0; i < 1024; i += 2) { 62 | descriptors[i].commit(); 63 | } 64 | auto snapshot = commitManager.startTx(); 65 | // now every second version should be in the read set 66 | for (uint64_t i = 0; i < 1024; ++i) { 67 | EXPECT_EQ(i % 2 == 0, snapshot->inReadSet(versions[i])) << i << "th transaction (" << versions[i] << 68 | ") should " << (i % 2 == 0 ? "" : "not ") << "be in the read set"; 69 | } 70 | // Commit the other transactions 71 | for (uint64_t i = 1; i < 1024; i += 2) { 72 | descriptors[i].commit(); 73 | } 74 | auto allCommitted = commitManager.startTx(); 75 | for (uint64_t i = 0; i < 1024; ++i) { 76 | EXPECT_EQ(i % 2 == 0, snapshot->inReadSet(versions[i])); 77 | EXPECT_TRUE(allCommitted->inReadSet(versions[i])); 78 | } 79 | EXPECT_EQ(1024u, allCommitted->baseVersion()); 80 | snapshot.commit(); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /tests/testCuckooMap.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | using namespace tell::store; 30 | 31 | class CuckooTest : public ::testing::Test { 32 | protected: 33 | CuckooTest() 34 | : pageManager(PageManager::construct(20 * TELL_PAGE_SIZE)), 35 | table(crossbow::allocator::construct(*pageManager)) { 36 | } 37 | 38 | virtual ~CuckooTest() { 39 | table->destroy(); 40 | crossbow::allocator::destroy_now(table); 41 | } 42 | 43 | PageManager::Ptr pageManager; 44 | 45 | CuckooTable* table; 46 | }; 47 | 48 | TEST_F(CuckooTest, GetOnEmpty) { 49 | CuckooTable& currTable = *table; 50 | // we check for 1000 random values, that get fails 51 | constexpr int num_tests = 1000; 52 | std::random_device rnd; 53 | std::uniform_int_distribution dist; 54 | for (int i = 0; i < num_tests; ++i) { 55 | auto val = dist(rnd); 56 | ASSERT_EQ(currTable.get(val), nullptr) << "Value " << val << " must not exist in table"; 57 | } 58 | } 59 | 60 | TEST_F(CuckooTest, SimpleInsert) { 61 | crossbow::allocator alloc; 62 | CuckooTable& currTable = *table; 63 | uint64_t key = 1937; 64 | std::unique_ptr value(new int(8713)); 65 | auto modifier = currTable.modifier(); 66 | ASSERT_TRUE(modifier.insert(key, value.get(), false)) << "Insertion of inexistent value not succeeded"; 67 | auto oldTable = table; 68 | table = modifier.done(); 69 | ASSERT_NE(table, nullptr) << "Modifier done returned nullptr"; 70 | ASSERT_NE(table, oldTable) << "After modification, the table must change"; 71 | crossbow::allocator::destroy_now(oldTable); 72 | CuckooTable& nTable = *table; 73 | int* ptr = reinterpret_cast(nTable.get(key)); 74 | ASSERT_EQ(ptr, value.get()) << "Table get returned wrong value"; 75 | ASSERT_EQ(*ptr, 8713) << "Value changed"; 76 | } 77 | 78 | class CuckooTestFilled : public CuckooTest { 79 | protected: 80 | constexpr static size_t numEntries = 1024; 81 | std::set entries; 82 | int value; 83 | CuckooTestFilled() : value(1) { 84 | // instantiate with a seed, to make sure, that we always get 85 | // the same numbers 86 | std::mt19937 rnd(1); 87 | std::uniform_int_distribution dist; 88 | for (size_t i = 0u; i < numEntries; ++i) { 89 | auto res = entries.insert(dist(rnd)); 90 | if (!res.second) --i; 91 | } 92 | } 93 | virtual ~CuckooTestFilled() {} 94 | virtual void SetUp() { 95 | crossbow::allocator alloc; 96 | auto m = table->modifier(); 97 | for (auto e : entries) { 98 | m.insert(e, &value, false); 99 | } 100 | auto old = table; 101 | table = m.done(); 102 | crossbow::allocator::destroy_now(old); 103 | } 104 | }; 105 | 106 | TEST_F(CuckooTestFilled, AllExist) { 107 | CuckooTable& t = *table; 108 | for (auto e : entries) { 109 | auto ptr = t.get(e); 110 | ASSERT_NE(ptr, nullptr); 111 | ASSERT_EQ(*reinterpret_cast(ptr), value); 112 | } 113 | } 114 | 115 | TEST_F(CuckooTestFilled, DoesNotReplace) { 116 | std::unique_ptr value(new int(8713)); 117 | crossbow::allocator alloc; 118 | Modifier m = table->modifier(); 119 | for (auto e : entries) { 120 | ASSERT_FALSE(m.insert(e, value.get(), false)) << "Replaced value for " << e; 121 | } 122 | } 123 | 124 | TEST_F(CuckooTestFilled, TestResize) { 125 | crossbow::allocator alloc; 126 | int oVal = 2; 127 | Modifier m = table->modifier(); 128 | size_t oldCapacity = m.capacity(); 129 | decltype(entries) newEntries; 130 | std::mt19937 rnd(10); 131 | std::uniform_int_distribution dist; 132 | auto toAdd = m.capacity() - m.size(); 133 | for (decltype(toAdd) i = 0; i <= toAdd; ++i) { 134 | auto nVal = dist(rnd); 135 | if (entries.find(nVal) != entries.end()) { 136 | --i; 137 | continue; 138 | } 139 | ASSERT_TRUE(m.insert(nVal, &oVal, false)); 140 | } 141 | ASSERT_NE(oldCapacity, m.capacity()); 142 | auto oldTable = table; 143 | table = m.done(); 144 | crossbow::allocator::destroy_now(oldTable); 145 | auto& t = *table; 146 | for (auto e : entries) { 147 | auto ptr = reinterpret_cast(t.get(e)); 148 | ASSERT_EQ(ptr, &value); 149 | ASSERT_EQ(*ptr, value); 150 | } 151 | for (auto e : newEntries) { 152 | auto ptr = reinterpret_cast(t.get(e)); 153 | ASSERT_EQ(ptr, &oVal); 154 | ASSERT_EQ(*ptr, oVal); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /tests/testOpenAddressingHash.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #include 24 | 25 | #include 26 | 27 | using namespace tell::store; 28 | 29 | namespace { 30 | 31 | class OpenAddressingTableTest : public ::testing::Test { 32 | protected: 33 | OpenAddressingTableTest() 34 | : mTable(1024), 35 | mElement1(0x1u), mElement2(0x2u), mElement3(0x3u) { 36 | } 37 | 38 | OpenAddressingTable mTable; 39 | 40 | uint64_t mElement1; 41 | uint64_t mElement2; 42 | uint64_t mElement3; 43 | }; 44 | 45 | /** 46 | * @class OpenAddressingTable 47 | * @test Check if a simple get after insert returns the element 48 | */ 49 | TEST_F(OpenAddressingTableTest, insertAndGet) { 50 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 51 | 52 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 53 | } 54 | 55 | /** 56 | * @class OpenAddressingTable 57 | * @test Check if multiple get and inserts return the correct elements 58 | */ 59 | TEST_F(OpenAddressingTableTest, insertAndGetMultiple) { 60 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 61 | EXPECT_TRUE(mTable.insert(10u, 12u, &mElement2)); 62 | EXPECT_TRUE(mTable.insert(11u, 11u, &mElement3)); 63 | 64 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 65 | EXPECT_EQ(&mElement2, mTable.get(10u, 12u)); 66 | EXPECT_EQ(&mElement3, mTable.get(11u, 11u)); 67 | } 68 | 69 | /** 70 | * @class OpenAddressingTable 71 | * @test Check if inserting a duplicate fails 72 | */ 73 | TEST_F(OpenAddressingTableTest, insertDuplicate) { 74 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 75 | EXPECT_FALSE(mTable.insert(10u, 11u, &mElement2)); 76 | 77 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 78 | } 79 | 80 | /** 81 | * @class OpenAddressingTable 82 | * @test Check if erasing an element works correctly 83 | */ 84 | TEST_F(OpenAddressingTableTest, erase) { 85 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 86 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 87 | 88 | EXPECT_TRUE(mTable.erase(10u, 11u, &mElement1)); 89 | EXPECT_EQ(nullptr, mTable.get(10u, 11u)); 90 | } 91 | 92 | /** 93 | * @class OpenAddressingTable 94 | * @test Check if erasing a non existing element works 95 | */ 96 | TEST_F(OpenAddressingTableTest, eraseNonExisting) { 97 | EXPECT_EQ(nullptr, mTable.get(10u, 11u)); 98 | 99 | EXPECT_TRUE(mTable.erase(10u, 11u, &mElement1)); 100 | EXPECT_EQ(nullptr, mTable.get(10u, 11u)); 101 | } 102 | 103 | /** 104 | * @class OpenAddressingTable 105 | * @test Check if erasing a changed element is prevented 106 | */ 107 | TEST_F(OpenAddressingTableTest, eraseChanged) { 108 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 109 | 110 | EXPECT_FALSE(mTable.erase(10u, 11u, &mElement2)); 111 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 112 | } 113 | 114 | /** 115 | * @class OpenAddressingTable 116 | * @test Check if updating an element works 117 | */ 118 | TEST_F(OpenAddressingTableTest, update) { 119 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 120 | 121 | EXPECT_TRUE(mTable.update(10u, 11u, &mElement1, &mElement2)); 122 | EXPECT_EQ(&mElement2, mTable.get(10u, 11u)); 123 | } 124 | 125 | /** 126 | * @class OpenAddressingTable 127 | * @test Check if updating a changed element is prevented 128 | */ 129 | TEST_F(OpenAddressingTableTest, updateChanged) { 130 | EXPECT_TRUE(mTable.insert(10u, 11u, &mElement1)); 131 | 132 | EXPECT_FALSE(mTable.update(10u, 11u, &mElement3, &mElement2)); 133 | EXPECT_EQ(&mElement1, mTable.get(10u, 11u)); 134 | } 135 | 136 | } 137 | -------------------------------------------------------------------------------- /util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################### 2 | # TellStore Util library 3 | ################### 4 | set(UTIL_SRCS 5 | CuckooHash.cpp 6 | LLVMBuilder.cpp 7 | LLVMJIT.cpp 8 | LLVMRowAggregation.cpp 9 | LLVMRowProjection.cpp 10 | LLVMRowScan.cpp 11 | LLVMScan.cpp 12 | Log.cpp 13 | OpenAddressingHash.cpp 14 | PageManager.cpp 15 | ScanQuery.cpp 16 | ) 17 | 18 | set(UTIL_PRIVATE_HDR 19 | CuckooHash.hpp 20 | functional.hpp 21 | LLVMBuilder.hpp 22 | LLVMJIT.hpp 23 | LLVMRowAggregation.hpp 24 | LLVMRowProjection.hpp 25 | LLVMRowScan.hpp 26 | LLVMScan.hpp 27 | Log.hpp 28 | OpenAddressingHash.hpp 29 | PageManager.hpp 30 | Scan.hpp 31 | ScanQuery.hpp 32 | StorageConfig.hpp 33 | TableManager.hpp 34 | UnsafeAtomic.hpp 35 | VersionManager.hpp 36 | ) 37 | 38 | set(USED_LLVM_LIBRARIES 39 | LLVMX86Disassembler 40 | LLVMX86AsmParser 41 | LLVMX86CodeGen 42 | LLVMSelectionDAG 43 | LLVMAsmPrinter 44 | LLVMCodeGen 45 | LLVMScalarOpts 46 | LLVMProfileData 47 | LLVMInstCombine 48 | LLVMTransformUtils 49 | LLVMipa 50 | LLVMAnalysis 51 | LLVMX86Desc 52 | LLVMMCDisassembler 53 | LLVMX86Info 54 | LLVMX86AsmPrinter 55 | LLVMX86Utils 56 | LLVMMCJIT 57 | LLVMTarget 58 | LLVMExecutionEngine 59 | LLVMRuntimeDyld 60 | LLVMObject 61 | LLVMMCParser 62 | LLVMBitReader 63 | LLVMMC 64 | LLVMCore 65 | LLVMSupport 66 | LLVMVectorize 67 | LLVMPasses 68 | ) 69 | 70 | # Add TellStore util library 71 | add_library(tellstore-util STATIC ${UTIL_SRCS} ${UTIL_PRIVATE_HDR} ${PROJECT_BINARY_DIR}/config.h) 72 | 73 | target_include_directories(tellstore-util PUBLIC ${PROJECT_SOURCE_DIR}) 74 | target_include_directories(tellstore-util PUBLIC ${PROJECT_BINARY_DIR}) 75 | 76 | # Workaround for link failure with GCC 5 (GCC Bug 65913) 77 | if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") 78 | target_link_libraries(tellstore-util PUBLIC atomic) 79 | endif() 80 | 81 | # Link against TellStore common library 82 | target_link_libraries(tellstore-util PUBLIC tellstore-common) 83 | 84 | # Link against CommitManager library 85 | target_link_libraries(tellstore-util PUBLIC commitmanager-client) 86 | 87 | # Link against Boost 88 | target_include_directories(tellstore-util PUBLIC ${Boost_INCLUDE_DIRS}) 89 | 90 | # Link against Crossbow 91 | target_include_directories(tellstore-util PUBLIC ${Crossbow_INCLUDE_DIRS}) 92 | target_link_libraries(tellstore-util PUBLIC crossbow_allocator crossbow_logger) 93 | 94 | # Link against TBB 95 | target_include_directories(tellstore-util PUBLIC ${TBB_INCLUDE_DIRS}) 96 | target_link_libraries(tellstore-util PUBLIC ${TBB_LIBRARIES}) 97 | 98 | # Link against LLVM 99 | target_link_libraries(tellstore-util PUBLIC ${USED_LLVM_LIBRARIES}) 100 | target_include_directories(tellstore-util PUBLIC ${LLVM_INCLUDE_DIRS}) 101 | target_compile_definitions(tellstore-util PUBLIC ${LLVM_DEFINITIONS}) 102 | -------------------------------------------------------------------------------- /util/CuckooHash.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "PageManager.hpp" 31 | #include "functional.hpp" 32 | 33 | #include 34 | 35 | namespace tell { 36 | namespace store { 37 | 38 | class Modifier; 39 | 40 | /** 41 | * This is a simple hash table that uses 42 | * Cuckoo hashing for efficient key lookup. 43 | * 44 | * This hash table allows for concurrent 45 | * lookup, but writing has to be done single 46 | * threaded. In this COW is used, to modify 47 | * the table. As soon as the modification 48 | * is done, a CAS operation is used to swap 49 | * the new with the old version. 50 | */ 51 | class CuckooTable { 52 | private: 53 | using EntryT = std::pair; 54 | 55 | static constexpr size_t ENTRIES_PER_PAGE = TELL_PAGE_SIZE / sizeof(EntryT); 56 | static_assert(isPowerOf2(ENTRIES_PER_PAGE), "Entries per page needs to be a power of two"); 57 | 58 | PageManager& mPageManager; 59 | std::vector mPages; 60 | cuckoo_hash_function hash1; 61 | cuckoo_hash_function hash2; 62 | cuckoo_hash_function hash3; 63 | size_t mSize; 64 | public: 65 | CuckooTable(PageManager& pageManager); 66 | ~CuckooTable(); 67 | 68 | /** 69 | * This method should be only used on 70 | * Shutdown: it will destroy the table, 71 | * therefore the caller has to know, that 72 | * it does not get used anymore. 73 | * 74 | * Usually the CuckooHashMap works as follow: 75 | * It is read-only and the Modifier does a COW 76 | * and will delete unused data. If the user wants 77 | * to delete the table itself, she has to 78 | * do it with this method - deleting an instance 79 | * of CuckooTable does not delete any data. 80 | */ 81 | void destroy(); 82 | 83 | friend class Modifier; 84 | friend class crossbow::allocator; 85 | 86 | private: 87 | CuckooTable(PageManager& pageManager, 88 | std::vector&& pages, 89 | cuckoo_hash_function hash1, 90 | cuckoo_hash_function hash2, 91 | cuckoo_hash_function hash3, 92 | size_t size); 93 | 94 | public: 95 | const void* get(uint64_t key) const; 96 | 97 | void* get(uint64_t key) { 98 | return const_cast(const_cast(this)->get(key)); 99 | } 100 | 101 | Modifier modifier(); 102 | 103 | size_t capacity() const; 104 | 105 | private: // helper functions 106 | const EntryT& at(unsigned h, size_t idx) const; 107 | }; 108 | 109 | class Modifier { 110 | friend class CuckooTable; 111 | friend class crossbow::allocator; 112 | 113 | using EntryT = typename CuckooTable::EntryT; 114 | static constexpr size_t ENTRIES_PER_PAGE = CuckooTable::ENTRIES_PER_PAGE; 115 | private: 116 | CuckooTable& mTable; 117 | std::vector pageWasModified; 118 | mutable std::vector mPages; 119 | cuckoo_hash_function hash1; 120 | cuckoo_hash_function hash2; 121 | cuckoo_hash_function hash3; 122 | size_t mSize; 123 | std::vector mToDelete; 124 | private: 125 | Modifier(CuckooTable& table) 126 | : mTable(table), 127 | pageWasModified(table.mPages.size(), false), 128 | mPages(table.mPages), 129 | hash1(table.hash1), 130 | hash2(table.hash2), 131 | hash3(table.hash3), 132 | mSize(table.mSize) 133 | { 134 | } 135 | 136 | public: 137 | ~Modifier(); 138 | CuckooTable* done() const; 139 | 140 | /** 141 | * inserts or replaces a value. Will return true iff the key 142 | * did exist before in the hash table 143 | */ 144 | bool insert(uint64_t key, void* value, bool replace = false); 145 | 146 | const void* get(uint64_t key) const; 147 | 148 | void* get(uint64_t key) { 149 | return const_cast(const_cast(this)->get(key)); 150 | } 151 | 152 | bool remove(uint64_t key); 153 | 154 | EntryT& at(unsigned h, size_t idx, size_t& pageIdx); 155 | const EntryT& at(unsigned h, size_t idx, size_t& pageIdx) const; 156 | 157 | size_t capacity() const; 158 | size_t size() const; 159 | 160 | bool cow(unsigned h, size_t idx); 161 | 162 | void rehash(); 163 | }; 164 | 165 | } // namespace store 166 | } // namespace tell 167 | -------------------------------------------------------------------------------- /util/LLVMJIT.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "LLVMJIT.hpp" 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | #include 44 | #include 45 | 46 | namespace tell { 47 | namespace store { 48 | 49 | LLVMCompiler llvmCompiler; 50 | 51 | LLVMCompilerT::LLVMCompilerT() 52 | : mTarget(nullptr) { 53 | std::array args = {{ 54 | "tellstore", 55 | "--disable-lsr" 56 | }}; 57 | llvm::cl::ParseCommandLineOptions(args.size(), args.data()); 58 | 59 | llvm::InitializeNativeTarget(); 60 | llvm::InitializeNativeTargetAsmPrinter(); 61 | llvm::InitializeNativeTargetAsmParser(); 62 | 63 | // Get the architecture name of the host 64 | mProcessTriple = llvm::sys::getProcessTriple(); 65 | mHostCPUName = llvm::sys::getHostCPUName(); 66 | 67 | // Setup all CPU features available on the host 68 | llvm::SubtargetFeatures features; 69 | llvm::StringMap hostFeatures; 70 | if (llvm::sys::getHostCPUFeatures(hostFeatures)) { 71 | for (auto& feature : hostFeatures) { 72 | features.AddFeature(feature.first(), feature.second); 73 | } 74 | } 75 | mFeatures = features.getString(); 76 | 77 | // Lookup the LLVM target for the host 78 | std::string error; 79 | mTarget = llvm::TargetRegistry::lookupTarget(mProcessTriple, error); 80 | if (!mTarget) { 81 | throw std::runtime_error(error); 82 | } 83 | 84 | // Load the standard library 85 | llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr); 86 | } 87 | 88 | std::unique_ptr LLVMCompilerT::createTargetMachine() { 89 | return std::unique_ptr(mTarget->createTargetMachine(mProcessTriple, mHostCPUName, mFeatures, 90 | mOptions, llvm::Reloc::Default, llvm::CodeModel::JITDefault, llvm::CodeGenOpt::Aggressive)); 91 | } 92 | 93 | LLVMJIT::LLVMJIT() 94 | : mTargetMachine(llvmCompiler->createTargetMachine()), 95 | mDataLayout(mTargetMachine->createDataLayout()), 96 | mCompileLayer(mObjectLayer, llvm::orc::SimpleCompiler(*mTargetMachine)) { 97 | } 98 | 99 | LLVMJIT::~LLVMJIT() = default; 100 | 101 | LLVMJIT::ModuleHandle LLVMJIT::addModule(llvm::Module* module) { 102 | auto resolver = llvm::orc::createLambdaResolver([this] (const std::string& name) { 103 | if (auto sym = mCompileLayer.findSymbol(name, true)) { 104 | return llvm::RuntimeDyld::SymbolInfo(sym.getAddress(), sym.getFlags()); 105 | } 106 | if (auto symAddr = llvm::RTDyldMemoryManager::getSymbolAddressInProcess(name)) { 107 | return llvm::RuntimeDyld::SymbolInfo(symAddr, llvm::JITSymbolFlags::Exported); 108 | } 109 | return llvm::RuntimeDyld::SymbolInfo(nullptr); 110 | }, [] (const std::string& /* name */) { 111 | return nullptr; 112 | }); 113 | auto handle = mCompileLayer.addModuleSet(std::vector{{module}}, 114 | llvm::make_unique(), std::move(resolver)); 115 | mObjectLayer.emitAndFinalize(handle); 116 | return handle; 117 | } 118 | 119 | } // namespace store 120 | } // namespace tell 121 | -------------------------------------------------------------------------------- /util/LLVMJIT.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | * 23 | * -------------------------------------------------------------------- 24 | * 25 | * This file was copied and slightly adapted from the KaleidoscopeJIT.h 26 | * unter the following licencse: 27 | * 28 | * ===----- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope ----*- C++ -*-=== 29 | * 30 | * The LLVM Compiler Infrastructure 31 | * 32 | * This file is distributed under the University of Illinois Open Source 33 | * License. See http://www.llvm.org for details. 34 | * 35 | */ 36 | 37 | #pragma once 38 | 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | #include 46 | #include 47 | 48 | #include 49 | 50 | namespace llvm { 51 | class Module; 52 | class TargetMachine; 53 | } // namespace llvm 54 | 55 | namespace tell { 56 | namespace store { 57 | 58 | /** 59 | * @brief Singleton for initializing and configuring the LLVM infrastructure 60 | * 61 | * Initializes the native target and sets up the TargetMachine with the features available on the host the application 62 | * is running. 63 | */ 64 | class LLVMCompilerT { 65 | public: 66 | LLVMCompilerT(); 67 | 68 | std::unique_ptr createTargetMachine(); 69 | 70 | private: 71 | std::string mProcessTriple; 72 | 73 | std::string mHostCPUName; 74 | 75 | std::string mFeatures; 76 | 77 | llvm::TargetOptions mOptions; 78 | 79 | const llvm::Target* mTarget; 80 | }; 81 | 82 | using LLVMCompiler = crossbow::singleton; 83 | 84 | extern LLVMCompiler llvmCompiler; 85 | 86 | /** 87 | * @brief JIT based on LLVM 88 | */ 89 | class LLVMJIT : crossbow::non_copyable, crossbow::non_movable { 90 | public: 91 | using ObjectLayer = llvm::orc::ObjectLinkingLayer<>; 92 | using CompileLayer = llvm::orc::IRCompileLayer; 93 | using ModuleHandle = CompileLayer::ModuleSetHandleT; 94 | 95 | LLVMJIT(); 96 | 97 | ~LLVMJIT(); 98 | 99 | llvm::TargetMachine* getTargetMachine() { 100 | return mTargetMachine.get(); 101 | } 102 | 103 | /** 104 | * @brief Compile the given module through the JIT 105 | * 106 | * The compiled functions can then be retrieved through the LLVMJIT::findSymbol function. Afterwards the module can 107 | * be destroyed. 108 | */ 109 | ModuleHandle addModule(llvm::Module* module); 110 | 111 | void removeModule(ModuleHandle handle) { 112 | mCompileLayer.removeModuleSet(handle); 113 | } 114 | 115 | llvm::orc::JITSymbol findSymbol(const std::string& name) { 116 | return mCompileLayer.findSymbol(mangle(name), true); 117 | } 118 | 119 | template 120 | Func findFunction(const std::string& name) { 121 | auto func = mCompileLayer.findSymbol(mangle(name), true); 122 | if (!func) { 123 | throw std::runtime_error("Unknown symbol"); 124 | } 125 | return reinterpret_cast(func.getAddress()); 126 | } 127 | 128 | private: 129 | std::string mangle(const std::string& name) { 130 | std::string mangledName; 131 | { 132 | llvm::raw_string_ostream mangledNameStream(mangledName); 133 | llvm::Mangler::getNameWithPrefix(mangledNameStream, name, mDataLayout); 134 | } 135 | return mangledName; 136 | } 137 | 138 | std::unique_ptr mTargetMachine; 139 | llvm::DataLayout mDataLayout; 140 | ObjectLayer mObjectLayer; 141 | CompileLayer mCompileLayer; 142 | }; 143 | 144 | } // namespace store 145 | } // namespace tell 146 | -------------------------------------------------------------------------------- /util/LLVMRowAggregation.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace tell { 34 | namespace store { 35 | 36 | class Record; 37 | class ScanQuery; 38 | 39 | /** 40 | * @brief Helper class creating the row aggregation function 41 | */ 42 | class LLVMRowAggregationBuilder : private FunctionBuilder { 43 | public: 44 | using Signature = uint32_t (*) ( 45 | const char* /* src */, 46 | uint32_t /* size */, 47 | char* /* dest */); 48 | 49 | static void createFunction(const Record& record, llvm::Module& module, llvm::TargetMachine* target, 50 | const std::string& name, ScanQuery* query) { 51 | LLVMRowAggregationBuilder builder(record, module, target, name); 52 | builder.build(query); 53 | } 54 | 55 | private: 56 | static constexpr size_t src = 0; 57 | static constexpr size_t size = 1; 58 | static constexpr size_t dest = 2; 59 | 60 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 61 | return llvm::Type::getInt32Ty(context); 62 | } 63 | 64 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 65 | return { 66 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "src" }, 67 | { llvm::Type::getInt32Ty(context), "size" }, 68 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "dest" } 69 | }; 70 | } 71 | 72 | LLVMRowAggregationBuilder(const Record& record, llvm::Module& module, llvm::TargetMachine* target, 73 | const std::string& name); 74 | 75 | void build(ScanQuery* query); 76 | 77 | const Record& mRecord; 78 | }; 79 | 80 | } // namespace store 81 | } // namespace tell 82 | -------------------------------------------------------------------------------- /util/LLVMRowProjection.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace tell { 34 | namespace store { 35 | 36 | class Record; 37 | class ScanQuery; 38 | 39 | /** 40 | * @brief Helper class creating the row projection function 41 | */ 42 | class LLVMRowProjectionBuilder : private FunctionBuilder { 43 | public: 44 | using Signature = uint32_t (*) ( 45 | const char* /* src */, 46 | uint32_t /* size */, 47 | char* /* dest */); 48 | 49 | static const std::string FUNCTION_NAME; 50 | 51 | static void createFunction(const Record& record, llvm::Module& module, llvm::TargetMachine* target, 52 | const std::string& name, ScanQuery* query) { 53 | LLVMRowProjectionBuilder builder(record, module, target, name); 54 | builder.build(query); 55 | } 56 | 57 | private: 58 | static constexpr size_t src = 0; 59 | static constexpr size_t size = 1; 60 | static constexpr size_t dest = 2; 61 | 62 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 63 | return llvm::Type::getInt32Ty(context); 64 | } 65 | 66 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 67 | return { 68 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "src" }, 69 | { llvm::Type::getInt32Ty(context), "size" }, 70 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "dest" } 71 | }; 72 | } 73 | 74 | LLVMRowProjectionBuilder(const Record& record, llvm::Module& module, llvm::TargetMachine* target, 75 | const std::string& name); 76 | 77 | void build(ScanQuery* query); 78 | 79 | const Record& mRecord; 80 | }; 81 | 82 | } // namespace store 83 | } // namespace tell 84 | -------------------------------------------------------------------------------- /util/LLVMRowScan.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | namespace tell { 35 | namespace store { 36 | 37 | struct ScanAST; 38 | 39 | /** 40 | * @brief Helper class creating the row store scan function 41 | */ 42 | class LLVMRowScanBuilder : private FunctionBuilder { 43 | public: 44 | static const std::string FUNCTION_NAME; 45 | 46 | static void createFunction(llvm::Module& module, llvm::TargetMachine* target, const ScanAST& scanAst) { 47 | LLVMRowScanBuilder builder(module, target); 48 | builder.buildScan(scanAst); 49 | } 50 | 51 | private: 52 | static constexpr size_t key = 0; 53 | static constexpr size_t validFrom = 1; 54 | static constexpr size_t validTo = 2; 55 | static constexpr size_t recordData = 3; 56 | static constexpr size_t resultData = 4; 57 | 58 | static llvm::Type* buildReturnTy(llvm::LLVMContext& context) { 59 | return llvm::Type::getVoidTy(context); 60 | } 61 | 62 | static std::vector> buildParamTy(llvm::LLVMContext& context) { 63 | return { 64 | { llvm::Type::getInt64Ty(context), "key" }, 65 | { llvm::Type::getInt64Ty(context), "validFrom" }, 66 | { llvm::Type::getInt64Ty(context), "validTo" }, 67 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "recordData" }, 68 | { llvm::Type::getInt8Ty(context)->getPointerTo(), "resultData" } 69 | }; 70 | } 71 | 72 | LLVMRowScanBuilder(llvm::Module& module, llvm::TargetMachine* target); 73 | 74 | void buildScan(const ScanAST& scanAst); 75 | 76 | std::vector mConjunctsGenerated; 77 | }; 78 | 79 | } //namespace store 80 | } //namespace tell 81 | -------------------------------------------------------------------------------- /util/PageManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | 24 | #include "PageManager.hpp" 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | #include 33 | 34 | namespace tell { 35 | namespace store { 36 | 37 | PageManager::PageManager(size_t size) 38 | : mData(mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, 0, 0)), 39 | mSize(size), 40 | mPages(size / TELL_PAGE_SIZE, nullptr) 41 | { 42 | if (mData == MAP_FAILED) { 43 | throw std::bad_alloc(); 44 | } 45 | LOG_ASSERT(mSize % TELL_PAGE_SIZE == 0, "Size must divide the page size"); 46 | auto numPages = mSize / TELL_PAGE_SIZE; 47 | memset(mData, 0, mSize); 48 | auto data = reinterpret_cast(mData); 49 | data += mSize - TELL_PAGE_SIZE; // data does now point to the last page 50 | for (decltype(numPages) i = 0ul; i < numPages; ++i) { 51 | __attribute__((unused)) auto res = mPages.push(data); 52 | LOG_ASSERT(res, "Pusing page did not succeed"); 53 | data -= TELL_PAGE_SIZE; 54 | } 55 | LOG_ASSERT(mPages.size() == numPages, "Not all pages were added to the stack"); 56 | } 57 | 58 | PageManager::~PageManager() { 59 | // TODO Fix this behavior 60 | // Wait for all pages to be released 61 | // This is required as the epoch might delete the PageManager while a previous epoch is being deleted (with a 62 | // reference to this page manager). 63 | while (mPages.capacity() != mPages.size()); 64 | munmap(mData, mSize); 65 | } 66 | 67 | void* PageManager::alloc() { 68 | void* page; 69 | auto success = mPages.pop(page); 70 | LOG_ASSERT(!success || (page != nullptr), "Successful pop must not return null pages"); 71 | LOG_ASSERT(!success || (page >= mData && page < reinterpret_cast(mData) + mSize), "Page points out of bound"); 72 | LOG_ASSERT(!success || (reinterpret_cast(page) - reinterpret_cast(mData)) % TELL_PAGE_SIZE == 0, 73 | "Pointer points not to beginning of page"); 74 | if (!success) { 75 | return nullptr; 76 | } 77 | memset(page, 0, TELL_PAGE_SIZE); 78 | return page; 79 | } 80 | 81 | void PageManager::free(void* page) { 82 | LOG_ASSERT(page != nullptr, "Page must not be null"); 83 | LOG_ASSERT(page >= mData && page < reinterpret_cast(mData) + mSize, "Page points out of bound"); 84 | LOG_ASSERT((reinterpret_cast(page) - reinterpret_cast(mData)) % TELL_PAGE_SIZE == 0, 85 | "Pointer points not to beginning of page"); 86 | freeEmpty(page); 87 | } 88 | 89 | void PageManager::freeEmpty(void* page) { 90 | while (!mPages.push(page)); 91 | } 92 | 93 | } // namespace store 94 | } // namespace tell 95 | -------------------------------------------------------------------------------- /util/PageManager.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | #include 32 | #include 33 | 34 | namespace tell { 35 | namespace store { 36 | 37 | class PageManager; 38 | 39 | struct PageManagerDeleter { 40 | void operator()(PageManager* pageManager) { 41 | crossbow::allocator::destroy_in_order(pageManager); 42 | } 43 | }; 44 | 45 | /** 46 | * This class purpose is to store all pages 47 | * allocated. It keeps an internal list of 48 | * free pages. All page allocations need to 49 | * be made through this class. 50 | */ 51 | class PageManager: crossbow::non_copyable, crossbow::non_movable { 52 | private: 53 | void* mData; 54 | size_t mSize; 55 | crossbow::fixed_size_stack mPages; 56 | public: 57 | using Ptr = std::unique_ptr; 58 | 59 | /** 60 | * @brief Constructs a new page manager pointer 61 | * 62 | * The resulting page manager is allocated and destroyed within the epoch. 63 | */ 64 | static PageManager::Ptr construct(size_t size) { 65 | return PageManager::Ptr(crossbow::allocator::construct(size)); 66 | } 67 | 68 | /** 69 | * This class must not instantiated more than once! 70 | * 71 | * The constructor will allocate #size number 72 | * of bytes. At the moment, growing and shrinking is 73 | * not supported. This might be implemented later. 74 | * 75 | * \pre {#size has to be a multiplication of #PAGE_SIZE} 76 | */ 77 | PageManager(size_t size); 78 | 79 | ~PageManager(); 80 | 81 | const void* data() const { 82 | return mData; 83 | } 84 | 85 | void* data() { 86 | return const_cast(const_cast(this)->data()); 87 | } 88 | 89 | size_t size() const { 90 | return mSize; 91 | } 92 | 93 | /** 94 | * Allocates a new page. It is safe to call this method 95 | * concurrently. It will return nullptr, if there is no 96 | * space left. 97 | */ 98 | void* alloc(); 99 | 100 | /** 101 | * Returns the given page back to the pool 102 | */ 103 | void free(void* page); 104 | 105 | /** 106 | * Returns the given (already zeroed) page back to the pool 107 | */ 108 | void freeEmpty(void* page); 109 | }; 110 | 111 | } // namespace store 112 | } // namespace tell 113 | -------------------------------------------------------------------------------- /util/StorageConfig.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | 28 | namespace tell { 29 | namespace store { 30 | struct StorageConfig { 31 | uint16_t gcInterval = 60; 32 | size_t totalMemory = TOTAL_MEMORY; 33 | size_t numScanThreads = 2; 34 | size_t hashMapCapacity = HASHMAP_CAPACITY; 35 | }; 36 | } // namespace store 37 | } // namespace tell 38 | -------------------------------------------------------------------------------- /util/UnsafeAtomic.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace tell { 30 | namespace store { 31 | 32 | /** 33 | * @brief Unsafe 128 bit atomic 34 | * 35 | * Allows for atomic 128 bit compare and swap but loads and stores are implemented as two separate 64 bit operations. 36 | */ 37 | template 38 | class alignas(16) UnsafeAtomic { 39 | public: 40 | static_assert(sizeof(T) == 16, "Only 16 byte types are supported"); 41 | static_assert(alignof(T) == 16, "Only 16 byte aligned types are supported"); 42 | 43 | #if defined(__GNUC__) && __GNUC__ < 5 44 | #else 45 | static_assert(std::is_trivially_copyable::value, "Only trivially copyable types are supported"); 46 | #endif 47 | 48 | T unsafe_load() const noexcept { 49 | T result; 50 | __atomic_load(reinterpret_cast(&mElement), reinterpret_cast(&result), 51 | std::memory_order_seq_cst); 52 | __atomic_load(reinterpret_cast(&mElement) + 1, reinterpret_cast(&result) + 1, 53 | std::memory_order_seq_cst); 54 | return result; 55 | } 56 | 57 | void unsafe_store(T element) noexcept { 58 | __atomic_store(reinterpret_cast(&mElement), reinterpret_cast(&element), 59 | std::memory_order_seq_cst); 60 | __atomic_store(reinterpret_cast(&mElement) + 1, reinterpret_cast(&element) + 1, 61 | std::memory_order_seq_cst); 62 | } 63 | 64 | bool compare_exchange_strong(T& expected, T desired) noexcept { 65 | return __atomic_compare_exchange(&mElement, &expected, &desired, false, std::memory_order_seq_cst, 66 | std::memory_order_seq_cst); 67 | } 68 | 69 | private: 70 | T mElement; 71 | }; 72 | 73 | } // namespace store 74 | } // namespace tell 75 | -------------------------------------------------------------------------------- /util/VersionManager.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | namespace tell { 33 | namespace store { 34 | 35 | class VersionManager : crossbow::non_copyable, crossbow::non_movable { 36 | public: 37 | VersionManager() 38 | : mLowestActiveVersion(0x1u) { 39 | } 40 | 41 | uint64_t lowestActiveVersion() const { 42 | return mLowestActiveVersion.load(); 43 | } 44 | 45 | void addSnapshot(const commitmanager::SnapshotDescriptor& snapshot) { 46 | auto lowestActiveVersion = mLowestActiveVersion.load(); 47 | while (lowestActiveVersion < snapshot.lowestActiveVersion()) { 48 | if (!mLowestActiveVersion.compare_exchange_strong(lowestActiveVersion, snapshot.lowestActiveVersion())) { 49 | continue; 50 | } 51 | return; 52 | } 53 | } 54 | 55 | private: 56 | std::atomic mLowestActiveVersion; 57 | }; 58 | 59 | } // namespace store 60 | } // namespace tell 61 | -------------------------------------------------------------------------------- /util/functional.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2015 ETH Zurich Systems Group (http://www.systems.ethz.ch/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Markus Pilman 18 | * Simon Loesing 19 | * Thomas Etter 20 | * Kevin Bocksrocker 21 | * Lucas Braun 22 | */ 23 | #pragma once 24 | #include 25 | 26 | #include "PageManager.hpp" 27 | 28 | #include 29 | #include 30 | 31 | namespace tell { 32 | namespace store { 33 | 34 | template 35 | constexpr T log2Of(T x, T tmp = 0) { 36 | return x == 1 ? tmp : log2Of(x/2, tmp+1); 37 | } 38 | 39 | constexpr bool isPowerOf2(size_t x) { 40 | return x == 1 || (x % 2 == 0 && isPowerOf2(x/2)); 41 | } 42 | 43 | /** 44 | * This is a very simple general hash function. 45 | */ 46 | class cuckoo_hash_function { 47 | uint64_t mSalt; 48 | uint64_t w_M; // m==64 w_M = w - M 49 | public: 50 | cuckoo_hash_function(size_t tableSize) 51 | : mSalt(0) { 52 | std::random_device rd; 53 | std::uniform_int_distribution dist; 54 | while (mSalt == 0) { 55 | mSalt = dist(rd); 56 | } 57 | uint64_t M = log2Of(uint64_t(tableSize)); 58 | w_M = 64 - M; 59 | } 60 | 61 | size_t operator()(uint64_t k) const { 62 | return (mSalt*k) >> (w_M); 63 | } 64 | }; 65 | 66 | } // namespace store 67 | } // namespace tell 68 | --------------------------------------------------------------------------------