├── .gitmodules ├── README-LinkBench.md ├── .gitignore ├── .clang-format ├── test ├── main.cpp ├── allocator.cpp ├── block_manager.cpp ├── utils.cpp ├── bloom_filter.cpp ├── futex.cpp ├── graph.cpp ├── blocks.cpp └── transaction.cpp ├── core ├── livegraph.hpp ├── types.hpp ├── utils.hpp ├── allocator.hpp ├── futex.hpp ├── graph.hpp ├── bloom_filter.hpp ├── edge_iterator.hpp ├── block_manager.hpp ├── transaction.hpp ├── commit_manager.hpp └── blocks.hpp ├── CMakeLists.txt ├── bind ├── livegraph.hpp └── livegraph.cpp ├── README.md ├── src ├── graph.cpp └── transaction.cpp ├── LICENSE └── cmake └── FindTBB.cmake /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/doctest"] 2 | path = deps/doctest 3 | url = https://github.com/onqtam/doctest.git 4 | -------------------------------------------------------------------------------- /README-LinkBench.md: -------------------------------------------------------------------------------- 1 | # Reproduce LinkBench Results of LiveGraph 2 | 3 | Please refer to https://github.com/thu-pacman/LiveGraph-Binary/blob/master/LinkBench/README.md 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeLists.txt.user 2 | CMakeCache.txt 3 | CMakeFiles 4 | CMakeScripts 5 | Testing 6 | Makefile 7 | cmake_install.cmake 8 | install_manifest.txt 9 | compile_commands.json 10 | CTestTestfile.cmake 11 | _deps 12 | build 13 | .ycm_extra_conf.py 14 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | Language: Cpp 2 | BasedOnStyle: LLVM 3 | IndentWidth: 4 4 | AccessModifierOffset: -4 5 | BreakBeforeBraces: Allman 6 | BinPackParameters: false 7 | ColumnLimit: 120 8 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 9 | NamespaceIndentation: All 10 | -------------------------------------------------------------------------------- /test/main.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN 17 | #include 18 | -------------------------------------------------------------------------------- /core/livegraph.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include "edge_iterator.hpp" 19 | #include "graph.hpp" 20 | #include "transaction.hpp" 21 | -------------------------------------------------------------------------------- /core/types.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | namespace livegraph 22 | { 23 | using label_t = uint16_t; 24 | using vertex_t = uint64_t; 25 | using order_t = uint8_t; 26 | using timestamp_t = int64_t; 27 | 28 | } // namespace livegraph 29 | -------------------------------------------------------------------------------- /test/allocator.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | #include "core/allocator.hpp" 21 | 22 | using namespace livegraph; 23 | 24 | TEST_CASE("testing the SparseArrayAllocator") 25 | { 26 | SparseArrayAllocator allocator; 27 | auto allocator_for_int = std::allocator_traits::rebind_alloc(allocator); 28 | auto int_data = allocator_for_int.allocate(1000); 29 | allocator_for_int.deallocate(int_data, 1000); 30 | } 31 | -------------------------------------------------------------------------------- /test/block_manager.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "blocks.cpp" 19 | #include "core/block_manager.hpp" 20 | #include "core/utils.hpp" 21 | 22 | using namespace livegraph; 23 | 24 | TEST_CASE("testing the BlockManager") 25 | { 26 | BlockManager manager(""); 27 | size_t size = sizeof(EdgeLabelBlockHeader); 28 | order_t order = size_to_order(size); 29 | CHECK(size == 32); 30 | CHECK(order == 5); 31 | auto pointer = manager.alloc(order); 32 | 33 | EdgeLabelBlockHeader *block = manager.convert(pointer); 34 | block->set_order(order); 35 | manager.free(pointer, order); 36 | 37 | CHECK(manager.convert(manager.NULLPOINTER) == nullptr); 38 | } 39 | -------------------------------------------------------------------------------- /test/utils.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "core/utils.hpp" 19 | 20 | using namespace livegraph; 21 | 22 | TEST_CASE("testing the size_to_order") 23 | { 24 | CHECK(size_to_order(0) == 0); 25 | CHECK(size_to_order(1) == 0); 26 | CHECK(size_to_order(2) == 1); 27 | CHECK(size_to_order(3) == 2); 28 | CHECK(size_to_order(4) == 2); 29 | CHECK(size_to_order(5) == 3); 30 | CHECK(size_to_order(6) == 3); 31 | CHECK(size_to_order(7) == 3); 32 | CHECK(size_to_order(8) == 3); 33 | } 34 | 35 | TEST_CASE("testing cmp_timestamp") 36 | { 37 | timestamp_t a = -10, b = 10; 38 | CHECK(cmp_timestamp(&a, 10) > 0); 39 | CHECK(cmp_timestamp(&a, 0) > 0); 40 | CHECK(cmp_timestamp(&b, 0) > 0); 41 | CHECK(cmp_timestamp(&b, 10) == 0); 42 | CHECK(cmp_timestamp(&b, 15) < 0); 43 | CHECK(cmp_timestamp(&a, 10, 10) == 0); 44 | CHECK(cmp_timestamp(&a, 10, 123) > 0); 45 | CHECK(cmp_timestamp(&a, 0, 123) > 0); 46 | CHECK(cmp_timestamp(&b, 0, 123) > 0); 47 | CHECK(cmp_timestamp(&b, 10, 123) == 0); 48 | CHECK(cmp_timestamp(&b, 15, 123) < 0); 49 | } 50 | -------------------------------------------------------------------------------- /core/utils.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | 20 | #include "types.hpp" 21 | 22 | namespace livegraph 23 | { 24 | inline void compiler_fence() { asm volatile("" ::: "memory"); } 25 | 26 | inline order_t size_to_order(size_t size) 27 | { 28 | order_t order = (order_t)((size & (size - 1)) != 0); 29 | while (size > 1) 30 | { 31 | order += 1; 32 | size >>= 1; 33 | } 34 | return order; 35 | } 36 | 37 | inline int cmp_timestamp(const timestamp_t *xp, timestamp_t y) // y > 0 38 | { 39 | timestamp_t x = *xp; 40 | if (x < 0) 41 | return 1; 42 | if (x < y) 43 | return -1; 44 | if (x == y) 45 | return 0; 46 | return 1; 47 | } 48 | 49 | inline int cmp_timestamp(const timestamp_t *xp, timestamp_t y, 50 | timestamp_t local_txn_id) // y > 0 51 | { 52 | timestamp_t x = *xp; 53 | if (-x == local_txn_id) 54 | return 0; 55 | if (x < 0) 56 | return 1; 57 | if (x < y) 58 | return -1; 59 | if (x == y) 60 | return 0; 61 | return 1; 62 | } 63 | 64 | } // namespace livegraph 65 | -------------------------------------------------------------------------------- /core/allocator.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | namespace livegraph 25 | { 26 | 27 | template struct SparseArrayAllocator 28 | { 29 | using value_type = T; 30 | 31 | SparseArrayAllocator() = default; 32 | template constexpr SparseArrayAllocator(const SparseArrayAllocator &) noexcept {} 33 | T *allocate(size_t n) 34 | { 35 | size_t size = n * sizeof(T); 36 | if (n > std::numeric_limits::max() / sizeof(T)) 37 | throw std::bad_alloc(); 38 | auto data = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); 39 | if (data == MAP_FAILED) 40 | throw std::bad_alloc(); 41 | return static_cast(data); 42 | } 43 | 44 | void deallocate(T *data, size_t n) noexcept 45 | { 46 | 47 | size_t size = n * sizeof(T); 48 | munmap(data, size); 49 | } 50 | 51 | template bool operator==(const SparseArrayAllocator &) { return true; } 52 | template bool operator!=(const SparseArrayAllocator &) { return false; } 53 | }; 54 | 55 | } // namespace livegraph 56 | -------------------------------------------------------------------------------- /test/bloom_filter.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "core/bloom_filter.hpp" 23 | 24 | using namespace livegraph; 25 | 26 | TEST_CASE("testing the BloomFilter") 27 | { 28 | SUBCASE("empty BloomFilter") 29 | { 30 | BloomFilter empty_filter; 31 | CHECK(!empty_filter.valid()); 32 | } 33 | 34 | SUBCASE("insert and find") 35 | { 36 | const size_t log_size = 15; 37 | const size_t size = 1ul << log_size; 38 | const size_t range = 1ul << log_size; 39 | const size_t num_checks = 1ul << (log_size + 3); 40 | const size_t log_bytes_per_elements = 2; 41 | std::unordered_set ref; 42 | std::mt19937_64 random; 43 | 44 | void *buf = aligned_alloc(4096, size * (1ul << log_bytes_per_elements)); 45 | BloomFilter filter(log_size + log_bytes_per_elements, buf); 46 | CHECK(filter.valid()); 47 | 48 | for (size_t i = 0; i < size; i++) 49 | { 50 | auto data = random() % range; 51 | ref.emplace(data); 52 | filter.insert(data); 53 | } 54 | 55 | size_t num_false_positive = 0; 56 | size_t num_negetive = 0; 57 | for (size_t i = 0; i < num_checks; i++) 58 | { 59 | auto data = random() % range; 60 | if (ref.find(data) != ref.end()) 61 | { 62 | CHECK(filter.find(data)); 63 | } 64 | else 65 | { 66 | num_negetive++; 67 | if (filter.find(data)) 68 | num_false_positive++; 69 | } 70 | } 71 | 72 | CHECK(num_false_positive * 10 < num_negetive); // should less than 0.1 73 | 74 | free(buf); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(LiveGraph VERSION 0.0.1 LANGUAGES CXX) 3 | set(CMAKE_CXX_STANDARD 17) 4 | set(THREADS_PREFER_PTHREAD_FLAG ON) 5 | 6 | find_package(Threads REQUIRED) 7 | link_libraries(Threads::Threads) 8 | 9 | find_package(OpenMP) 10 | if (OPENMP_FOUND) 11 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 12 | set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 13 | endif() 14 | 15 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) 16 | find_package(TBB REQUIRED) 17 | include_directories(${TBB_INCLUDE_DIRS}) 18 | link_directories(${TBB_LIBRARY_DIRS}) 19 | link_libraries(${TBB_LIBRARIES}) 20 | 21 | include(CheckCXXCompilerFlag) 22 | check_cxx_compiler_flag("-march=native" COMPILER_SUPPORTS_MARCH_NATIVE) 23 | if(COMPILER_SUPPORTS_MARCH_NATIVE) 24 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") 25 | else() 26 | message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no march native support.") 27 | endif() 28 | 29 | set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") 30 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") 31 | if(NOT CMAKE_BUILD_TYPE) 32 | set(CMAKE_BUILD_TYPE "Release" CACHE STRING 33 | "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) 34 | endif(NOT CMAKE_BUILD_TYPE) 35 | 36 | include_directories(${PROJECT_SOURCE_DIR}) 37 | 38 | add_library(corelib STATIC 39 | src/graph.cpp 40 | src/transaction.cpp) 41 | set_property(TARGET corelib PROPERTY POSITION_INDEPENDENT_CODE ON) 42 | 43 | add_library(livegraph SHARED bind/livegraph.cpp src/graph.cpp src/transaction.cpp) 44 | target_link_libraries(livegraph corelib) 45 | install(TARGETS livegraph DESTINATION lib) 46 | install(FILES bind/livegraph.hpp DESTINATION include) 47 | 48 | set(PROJECT_DEPS_DIR ${PROJECT_SOURCE_DIR}/deps) 49 | 50 | option(BUILD_TESTING "Build the testing tree." ON) 51 | if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING) 52 | add_subdirectory(${PROJECT_DEPS_DIR}/doctest EXCLUDE_FROM_ALL) 53 | add_executable(tests 54 | test/main.cpp 55 | test/allocator.cpp 56 | test/blocks.cpp 57 | test/block_manager.cpp 58 | test/bloom_filter.cpp 59 | test/futex.cpp 60 | test/graph.cpp 61 | test/transaction.cpp 62 | test/utils.cpp 63 | bind/livegraph.cpp) 64 | target_link_libraries(tests corelib doctest::doctest) 65 | enable_testing() 66 | include(${doctest_SOURCE_DIR}/scripts/cmake/doctest.cmake) 67 | doctest_discover_tests(tests) 68 | endif() 69 | -------------------------------------------------------------------------------- /test/futex.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "core/futex.hpp" 26 | 27 | using namespace livegraph; 28 | 29 | TEST_CASE("testing the Futex") 30 | { 31 | SUBCASE("lock and unlock") 32 | { 33 | const size_t size = 1ul << 15; 34 | std::mt19937_64 random; 35 | std::vector data(size); 36 | uint64_t sum = 0, min = UINT64_MAX; 37 | uint64_t lock_sum = 0, lock_min = UINT64_MAX; 38 | Futex mutex; 39 | 40 | mutex.lock(); 41 | mutex.unlock(); 42 | mutex.lock(); 43 | mutex.clear(); 44 | 45 | for (size_t i = 0; i < size; i++) 46 | { 47 | data[i] = random(); 48 | sum += data[i]; 49 | if (min > data[i]) 50 | min = data[i]; 51 | } 52 | 53 | #pragma omp parallel for 54 | for (size_t i = 0; i < size; i++) 55 | { 56 | std::lock_guard lock(mutex); 57 | lock_sum += data[i]; 58 | if (lock_min > data[i]) 59 | lock_min = data[i]; 60 | } 61 | 62 | CHECK(sum == lock_sum); 63 | CHECK(min == lock_min); 64 | } 65 | 66 | SUBCASE("try_lock_for") 67 | { 68 | Futex futex; 69 | std::thread thread1([&]() { 70 | futex.lock(); 71 | std::this_thread::sleep_for(std::chrono::seconds(2)); 72 | futex.unlock(); 73 | }); 74 | std::thread thread2([&]() { 75 | std::this_thread::sleep_for(std::chrono::seconds(1)); 76 | CHECK(!futex.try_lock_for(std::chrono::milliseconds(1))); 77 | CHECK(futex.try_lock_for(std::chrono::seconds(2))); 78 | futex.unlock(); 79 | }); 80 | thread1.join(); 81 | thread2.join(); 82 | futex.lock(); 83 | futex.unlock(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /core/futex.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace livegraph 27 | { 28 | class Futex 29 | { 30 | public: 31 | void lock() 32 | { 33 | while (true) 34 | { 35 | __sync_fetch_and_add(&num_using, 1); 36 | if (__sync_bool_compare_and_swap(&futexp, 0, 1)) 37 | break; 38 | int ret = futex(&futexp, FUTEX_WAIT, 1, nullptr, nullptr, 0); 39 | __sync_fetch_and_sub(&num_using, 1); 40 | if (ret == -1 && errno != EAGAIN) 41 | throw std::runtime_error("Futex wait error."); 42 | } 43 | } 44 | 45 | template bool try_lock_for(const std::chrono::duration &timeout_duration) 46 | { 47 | const struct timespec timeout = {.tv_sec = timeout_duration / std::chrono::seconds(1), 48 | .tv_nsec = (timeout_duration % std::chrono::seconds(1)) / 49 | std::chrono::nanoseconds(1)}; 50 | while (true) 51 | { 52 | __sync_fetch_and_add(&num_using, 1); 53 | if (__sync_bool_compare_and_swap(&futexp, 0, 1)) 54 | break; 55 | int ret = futex(&futexp, FUTEX_WAIT, 1, &timeout, nullptr, 0); 56 | __sync_fetch_and_sub(&num_using, 1); 57 | if (ret == -1 && errno != EAGAIN && errno != ETIMEDOUT) 58 | throw std::runtime_error("Futex wait error."); 59 | if (ret != 0 && errno == ETIMEDOUT) 60 | return false; 61 | } 62 | return true; 63 | } 64 | 65 | void unlock() 66 | { 67 | if (__sync_bool_compare_and_swap(&futexp, 1, 0)) 68 | { 69 | if (__sync_sub_and_fetch(&num_using, 1) == 0) 70 | return; 71 | int ret = futex(&futexp, FUTEX_WAKE, 1, nullptr, nullptr, 0); 72 | if (ret == -1) 73 | throw std::runtime_error("Futex wake error."); 74 | } 75 | } 76 | 77 | void clear() 78 | { 79 | futexp = 0; 80 | num_using = 0; 81 | } 82 | 83 | Futex() : futexp(0), num_using(0) {} 84 | 85 | private: 86 | int futexp; 87 | int num_using; 88 | inline static int 89 | futex(int *uaddr, int futex_op, int val, const struct timespec *timeout, int *uaddr2, int val3) 90 | { 91 | return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); 92 | } 93 | }; 94 | } // namespace livegraph 95 | -------------------------------------------------------------------------------- /bind/livegraph.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace livegraph 23 | { 24 | class Graph; 25 | class Transaction; 26 | class EdgeIterator; 27 | } // namespace livegraph 28 | 29 | namespace lg 30 | { 31 | using label_t = uint16_t; 32 | using vertex_t = uint64_t; 33 | using order_t = uint8_t; 34 | using timestamp_t = int64_t; 35 | 36 | class EdgeIterator; 37 | class Transaction; 38 | 39 | class Graph 40 | { 41 | public: 42 | Graph(std::string block_path = "", 43 | std::string wal_path = "", 44 | size_t max_block_size = 1ul << 40, 45 | vertex_t max_vertex_id = 1ul << 40); 46 | ~Graph(); 47 | 48 | vertex_t get_max_vertex_id() const; 49 | 50 | timestamp_t compact(timestamp_t read_epoch_id = NO_TRANSACTION); 51 | 52 | Transaction begin_transaction(); 53 | Transaction begin_read_only_transaction(); 54 | Transaction begin_batch_loader(); 55 | 56 | private: 57 | const std::unique_ptr graph; 58 | constexpr static timestamp_t NO_TRANSACTION = -1; 59 | }; 60 | 61 | class Transaction 62 | { 63 | public: 64 | class RollbackExcept : public std::runtime_error 65 | { 66 | public: 67 | RollbackExcept(const std::string &what_arg) : std::runtime_error(what_arg) {} 68 | RollbackExcept(const char *what_arg) : std::runtime_error(what_arg) {} 69 | }; 70 | 71 | Transaction(std::unique_ptr _txn); 72 | ~Transaction(); 73 | 74 | timestamp_t get_read_epoch_id() const; 75 | 76 | vertex_t new_vertex(bool use_recycled_vertex = false); 77 | void put_vertex(vertex_t vertex_id, std::string_view data); 78 | bool del_vertex(vertex_t vertex_id, bool recycle = false); 79 | 80 | void put_edge(vertex_t src, label_t label, vertex_t dst, std::string_view edge_data, bool force_insert = false); 81 | bool del_edge(vertex_t src, label_t label, vertex_t dst); 82 | 83 | std::string_view get_vertex(vertex_t vertex_id); 84 | std::string_view get_edge(vertex_t src, label_t label, vertex_t dst); 85 | EdgeIterator get_edges(vertex_t src, label_t label, bool reverse = false); 86 | 87 | timestamp_t commit(bool wait_visable = true); 88 | void abort(); 89 | 90 | private: 91 | const std::unique_ptr txn; 92 | }; 93 | 94 | class EdgeIterator 95 | { 96 | public: 97 | EdgeIterator(std::unique_ptr _iter); 98 | ~EdgeIterator(); 99 | 100 | bool valid() const; 101 | void next(); 102 | vertex_t dst_id() const; 103 | std::string_view edge_data() const; 104 | 105 | private: 106 | const std::unique_ptr iter; 107 | }; 108 | 109 | } // namespace lg 110 | -------------------------------------------------------------------------------- /bind/livegraph.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "livegraph.hpp" 17 | #include "core/livegraph.hpp" 18 | 19 | using namespace lg; 20 | namespace impl = livegraph; 21 | 22 | Graph::Graph(std::string block_path, std::string wal_path, size_t max_block_size, vertex_t max_vertex_id) 23 | : graph(std::make_unique(block_path, wal_path, max_block_size, max_vertex_id)) 24 | { 25 | } 26 | 27 | Graph::~Graph() = default; 28 | 29 | vertex_t Graph::get_max_vertex_id() const { return graph->get_max_vertex_id(); } 30 | 31 | timestamp_t Graph::compact(timestamp_t read_epoch_id) { return graph->compact(read_epoch_id); } 32 | 33 | Transaction Graph::begin_transaction() { return std::make_unique(graph->begin_transaction()); } 34 | 35 | Transaction Graph::begin_read_only_transaction() 36 | { 37 | return std::make_unique(graph->begin_read_only_transaction()); 38 | } 39 | 40 | Transaction Graph::begin_batch_loader() { return std::make_unique(graph->begin_batch_loader()); } 41 | 42 | Transaction::Transaction(std::unique_ptr _txn) : txn(std::move(_txn)) {} 43 | 44 | Transaction::~Transaction() = default; 45 | 46 | timestamp_t Transaction::get_read_epoch_id() const { return txn->get_read_epoch_id(); } 47 | 48 | vertex_t Transaction::new_vertex(bool use_recycled_vertex) { return txn->new_vertex(use_recycled_vertex); } 49 | 50 | void Transaction::put_vertex(vertex_t vertex_id, std::string_view data) 51 | { 52 | try 53 | { 54 | txn->put_vertex(vertex_id, data); 55 | } 56 | catch (impl::Transaction::RollbackExcept e) 57 | { 58 | throw RollbackExcept(e.what()); 59 | } 60 | } 61 | 62 | bool Transaction::del_vertex(vertex_t vertex_id, bool recycle) 63 | { 64 | try 65 | { 66 | return txn->del_vertex(vertex_id, recycle); 67 | } 68 | catch (impl::Transaction::RollbackExcept e) 69 | { 70 | throw RollbackExcept(e.what()); 71 | } 72 | } 73 | 74 | void Transaction::put_edge(vertex_t src, label_t label, vertex_t dst, std::string_view edge_data, bool force_insert) 75 | { 76 | try 77 | { 78 | txn->put_edge(src, label, dst, edge_data, force_insert); 79 | } 80 | catch (impl::Transaction::RollbackExcept e) 81 | { 82 | throw RollbackExcept(e.what()); 83 | } 84 | } 85 | 86 | bool Transaction::del_edge(vertex_t src, label_t label, vertex_t dst) 87 | { 88 | try 89 | { 90 | return txn->del_edge(src, label, dst); 91 | } 92 | catch (impl::Transaction::RollbackExcept e) 93 | { 94 | throw RollbackExcept(e.what()); 95 | } 96 | } 97 | 98 | std::string_view Transaction::get_vertex(vertex_t vertex_id) { return txn->get_vertex(vertex_id); } 99 | 100 | std::string_view Transaction::get_edge(vertex_t src, label_t label, vertex_t dst) 101 | { 102 | return txn->get_edge(src, label, dst); 103 | } 104 | 105 | EdgeIterator Transaction::get_edges(vertex_t src, label_t label, bool reverse) 106 | { 107 | return std::make_unique(txn->get_edges(src, label, reverse)); 108 | } 109 | 110 | timestamp_t Transaction::commit(bool wait_visable) { return txn->commit(wait_visable); } 111 | 112 | void Transaction::abort() { txn->abort(); } 113 | 114 | EdgeIterator::EdgeIterator(std::unique_ptr _iter) : iter(std::move(_iter)) {} 115 | 116 | EdgeIterator::~EdgeIterator() = default; 117 | 118 | bool EdgeIterator::valid() const { return iter->valid(); } 119 | 120 | void EdgeIterator::next() { iter->next(); } 121 | 122 | vertex_t EdgeIterator::dst_id() const { return iter->dst_id(); } 123 | 124 | std::string_view EdgeIterator::edge_data() const { return iter->edge_data(); } 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LiveGraph 2 | 3 | ## Description 4 | This is the open-source implementation for LiveGraph: 5 | 6 | LiveGraph: a transactional graph storage system with purely sequential adjacency list scans. 7 | Xiaowei Zhu, Guanyu Feng, Marco Serafini, Xiaosong Ma, Jiping Yu, Lei Xie, Ashraf Aboulnaga, and Wenguang Chen. 8 | Proc. VLDB Endow. 13, 7 (March 2020), 1020–1034. DOI:https://doi.org/10.14778/3384345.3384351 9 | 10 | ## Dependency 11 | - [CMake](https://gitlab.kitware.com/cmake/cmake) 12 | - [TBB](https://github.com/oneapi-src/oneTBB) 13 | - OpenMP and C++17 14 | - AVX-2 15 | 16 | ## API 17 | 18 | ### class `livegraph::Graph` 19 | 20 | Members | 21 | --------------------------------| 22 | `public inline `[`Graph`](#d5/d1f/classlivegraph_1_1Graph_1a9eaef12fb2758edf6fba991e694fc78b)`(std::string block_path,std::string wal_path,size_t _max_block_size,vertex_t _max_vertex_id)` | 23 | `public inline vertex_t `[`get_max_vertex_id`](#d5/d1f/classlivegraph_1_1Graph_1ae32a89731e2bb72261e46256994f50f6)`() const` | 24 | `public timestamp_t `[`compact`](#d5/d1f/classlivegraph_1_1Graph_1a0317d09d47305d76012beeaecbb14110)`(timestamp_t read_epoch_id)` | 25 | `public `[`Transaction`](#de/d80/classlivegraph_1_1Transaction)` `[`begin_transaction`](#d5/d1f/classlivegraph_1_1Graph_1a0dc3e75cc1a569c887ed3644aa58c25d)`()` | 26 | `public `[`Transaction`](#de/d80/classlivegraph_1_1Transaction)` `[`begin_read_only_transaction`](#d5/d1f/classlivegraph_1_1Graph_1a859327efb8164edb84b4cbad088ed129)`()` | 27 | `public `[`Transaction`](#de/d80/classlivegraph_1_1Transaction)` `[`begin_batch_loader`](#d5/d1f/classlivegraph_1_1Graph_1a16e393872779448ceec388242e7840b9)`()` | 28 | 29 | ### class `livegraph::Transaction` 30 | 31 | Members | 32 | --------------------------------| 33 | `public inline timestamp_t `[`get_read_epoch_id`](#de/d80/classlivegraph_1_1Transaction_1a4cfbb234332801135186637581c703bb)`() const` | 34 | `public vertex_t `[`new_vertex`](#de/d80/classlivegraph_1_1Transaction_1ab56133b8ed4004eab383dc4d4a1ea52c)`(bool use_recycled_vertex)` | 35 | `public void `[`put_vertex`](#de/d80/classlivegraph_1_1Transaction_1adae09fe945b64a309973657b55317138)`(vertex_t vertex_id,std::string_view data)` | 36 | `public bool `[`del_vertex`](#de/d80/classlivegraph_1_1Transaction_1a83b4e9b86c13890c8d7019778149a545)`(vertex_t vertex_id,bool recycle)` | 37 | `public void `[`put_edge`](#de/d80/classlivegraph_1_1Transaction_1aecf88f276f3768a5b0d01253b64ae21b)`(vertex_t src,label_t label,vertex_t dst,std::string_view edge_data,bool force_insert)` | 38 | `public bool `[`del_edge`](#de/d80/classlivegraph_1_1Transaction_1a5b22fa1a3b0a1e33874bfbd99755c597)`(vertex_t src,label_t label,vertex_t dst)` | 39 | `public std::string_view `[`get_vertex`](#de/d80/classlivegraph_1_1Transaction_1a1cd41d8828d4ad5096ded9f340f0ae24)`(vertex_t vertex_id)` | 40 | `public std::string_view `[`get_edge`](#de/d80/classlivegraph_1_1Transaction_1ad445ab1a95bfb81829ef3a9c1e221573)`(vertex_t src,label_t label,vertex_t dst)` | 41 | `public `[`EdgeIterator`](#d4/d30/classlivegraph_1_1EdgeIterator)` `[`get_edges`](#de/d80/classlivegraph_1_1Transaction_1aaec4015431a6afd9c710b3fca427555d)`(vertex_t src,label_t label,bool reverse)` | 42 | `public timestamp_t `[`commit`](#de/d80/classlivegraph_1_1Transaction_1a6a3b303f8a44d7757d3b9cf8c79f23a0)`(bool wait_visable)` | 43 | `public void `[`abort`](#de/d80/classlivegraph_1_1Transaction_1a11db8852227f0bf3bf103483032711f5)`()` | 44 | 45 | ### class `livegraph::EdgeIterator` 46 | 47 | Members | 48 | --------------------------------| 49 | `public inline bool `[`valid`](#d4/d30/classlivegraph_1_1EdgeIterator_1a016d207a852844281c3e4d33afb9bde5)`() const` | 50 | `public inline void `[`next`](#d4/d30/classlivegraph_1_1EdgeIterator_1aa3af5e0a36848a3da43791744808a534)`()` | 51 | `public inline vertex_t `[`dst_id`](#d4/d30/classlivegraph_1_1EdgeIterator_1a3a545a34b93fcb7a10da270d62195e05)`() const` | 52 | `public inline std::string_view `[`edge_data`](#d4/d30/classlivegraph_1_1EdgeIterator_1aaa5ea498742f1f244d52ef596f652f57)`() const` | 53 | 54 | ### class `livegraph::Transaction::RollbackExcept` 55 | 56 | ```cpp 57 | class livegraph::Transaction::RollbackExcept : public runtime_error 58 | ``` 59 | 60 | Members | 61 | --------------------------------| 62 | `public inline `[`RollbackExcept`](#df/d6f/classlivegraph_1_1Transaction_1_1RollbackExcept_1a5495fa81b025448e10b3f754b2065bd6)`(const std::string & what_arg)` | 63 | `public inline `[`RollbackExcept`](#df/d6f/classlivegraph_1_1Transaction_1_1RollbackExcept_1a470f073f1e4a421ee91b17aa218f296d)`(const char * what_arg)` | 64 | -------------------------------------------------------------------------------- /core/graph.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "allocator.hpp" 27 | #include "block_manager.hpp" 28 | #include "commit_manager.hpp" 29 | #include "futex.hpp" 30 | 31 | namespace livegraph 32 | { 33 | class EdgeIterator; 34 | class Transaction; 35 | 36 | class Graph 37 | { 38 | public: 39 | Graph(std::string block_path = "", 40 | std::string wal_path = "", 41 | size_t _max_block_size = 1ul << 40, 42 | vertex_t _max_vertex_id = 1ul << 40) 43 | : mutex(), 44 | epoch_id(0), 45 | transaction_id(0), 46 | vertex_id(0), 47 | read_epoch_table(NO_TRANSACTION), 48 | compact_table(), 49 | recycled_vertex_ids(), 50 | max_vertex_id(_max_vertex_id), 51 | array_allocator(), 52 | block_manager(block_path, _max_block_size), 53 | commit_manager(wal_path, epoch_id) 54 | { 55 | auto futex_allocater = 56 | std::allocator_traits::rebind_alloc(array_allocator); 57 | vertex_futexes = futex_allocater.allocate(max_vertex_id); 58 | 59 | auto pointer_allocater = 60 | std::allocator_traits::rebind_alloc(array_allocator); 61 | vertex_ptrs = pointer_allocater.allocate(max_vertex_id); 62 | edge_label_ptrs = pointer_allocater.allocate(max_vertex_id); 63 | } 64 | 65 | Graph(const Graph &) = delete; 66 | 67 | Graph(Graph &&) = delete; 68 | 69 | ~Graph() noexcept 70 | { 71 | auto futex_allocater = 72 | std::allocator_traits::rebind_alloc(array_allocator); 73 | futex_allocater.deallocate(vertex_futexes, max_vertex_id); 74 | 75 | auto pointer_allocater = 76 | std::allocator_traits::rebind_alloc(array_allocator); 77 | pointer_allocater.deallocate(vertex_ptrs, max_vertex_id); 78 | pointer_allocater.deallocate(edge_label_ptrs, max_vertex_id); 79 | } 80 | 81 | vertex_t get_max_vertex_id() const { return vertex_id; } 82 | 83 | timestamp_t compact(timestamp_t read_epoch_id = NO_TRANSACTION); 84 | 85 | Transaction begin_transaction(); 86 | Transaction begin_read_only_transaction(); 87 | Transaction begin_batch_loader(); 88 | 89 | private: 90 | using cacheline_padding_t = char[64]; 91 | 92 | cacheline_padding_t padding0; 93 | std::mutex mutex; 94 | cacheline_padding_t padding1; 95 | std::atomic epoch_id; 96 | cacheline_padding_t padding2; 97 | std::atomic transaction_id; 98 | cacheline_padding_t padding3; 99 | std::atomic vertex_id; 100 | cacheline_padding_t padding4; 101 | 102 | tbb::enumerable_thread_specific read_epoch_table; 103 | tbb::enumerable_thread_specific> compact_table; 104 | 105 | tbb::concurrent_queue recycled_vertex_ids; 106 | 107 | const vertex_t max_vertex_id; 108 | 109 | SparseArrayAllocator array_allocator; 110 | BlockManager block_manager; 111 | CommitManager commit_manager; 112 | 113 | Futex *vertex_futexes; 114 | uintptr_t *vertex_ptrs; 115 | uintptr_t *edge_label_ptrs; 116 | 117 | constexpr static size_t COMPACTION_CYCLE = 1ul << 20; 118 | constexpr static timestamp_t ROLLBACK_TOMBSTONE = INT64_MAX; 119 | constexpr static timestamp_t NO_TRANSACTION = -1; 120 | constexpr static timestamp_t RO_TRANSACTION = ROLLBACK_TOMBSTONE - 1; 121 | constexpr static vertex_t VERTEX_TOMBSTONE = UINT64_MAX; 122 | constexpr static auto TIMEOUT = std::chrono::milliseconds(1); 123 | constexpr static size_t COMPACT_EDGE_BLOCK_THRESHOLD = 5; // at least compact 20% edges 124 | 125 | friend class EdgeIterator; 126 | friend class Transaction; 127 | }; 128 | } // namespace livegraph 129 | -------------------------------------------------------------------------------- /core/bloom_filter.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | // Copied from Apache Impala (incubating), usable under the terms in the Apache 19 | // License, Version 2.0. 20 | 21 | // This is a block Bloom filter (from Putze et al.'s "Cache-, Hash- and 22 | // Space-EfficientBloom Filters") with some twists: 23 | // 24 | // 1. Each block is a split Bloom filter - see Section 2.1 of Broder and 25 | // Mitzenmacher's "Network Applications of Bloom Filters: A Survey". 26 | // 27 | // 2. The number of bits set per insert() is contant in order to take advantage 28 | // of SIMD instructions. 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | namespace livegraph 37 | { 38 | 39 | class BloomFilter 40 | { 41 | public: 42 | BloomFilter() : log_num_buckets(0), directory(nullptr), directory_mask(0) {} 43 | 44 | BloomFilter(size_t log_size, void *buffer) 45 | : log_num_buckets(std::max(1ul, log_size - LOG_BUCKET_BYTE_SIZE)), 46 | directory((bucket_t *)buffer), 47 | directory_mask((1ul << log_num_buckets) - 1) 48 | { 49 | } 50 | 51 | BloomFilter(const BloomFilter &) = default; 52 | BloomFilter(BloomFilter &&) = default; 53 | BloomFilter &operator=(const BloomFilter &) = default; 54 | BloomFilter &operator=(BloomFilter &&a) = default; 55 | ~BloomFilter() = default; 56 | 57 | bool valid() const { return log_num_buckets != 0; } 58 | 59 | void clear() { memset(directory, 0, (1ul << log_num_buckets) * sizeof(bucket_t)); } 60 | 61 | size_t size() const 62 | { 63 | if (valid()) 64 | return 1ul << (log_num_buckets + LOG_BUCKET_BYTE_SIZE); 65 | else 66 | return 0; 67 | } 68 | 69 | [[gnu::always_inline]] inline static uint64_t get_hash(uint64_t key) 70 | { 71 | const uint64_t SEED[4] = {0x818c3f78ull, 0x672f4a3aull, 0xabd04d69ull, 0x12b51f95ull}; 72 | const unsigned __int128 m = *reinterpret_cast(&SEED[0]); 73 | const unsigned __int128 a = *reinterpret_cast(&SEED[2]); 74 | return (a + m * key) >> 64; 75 | } 76 | 77 | [[gnu::always_inline]] inline static __m256i get_mask(const uint32_t hash) 78 | { 79 | const __m256i ones = _mm256_set1_epi32(1); 80 | // Odd contants for hashing: 81 | const __m256i rehash = _mm256_setr_epi32(0x47b6137bU, 0x44974d91U, 0x8824ad5bU, 0xa2b7289dU, 0x705495c7U, 82 | 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U); 83 | // Load hash into a YMM register, repeated eight times 84 | __m256i hash_data = _mm256_set1_epi32(hash); 85 | // Multiply-shift hashing ala Dietzfelbinger et al.: multiply 'hash' 86 | // by eight different odd constants, then keep the 5 most 87 | // significant bits from each product. 88 | hash_data = _mm256_mullo_epi32(rehash, hash_data); 89 | hash_data = _mm256_srli_epi32(hash_data, 27); 90 | // Use these 5 bits to shift a single bit to a location in each 91 | // 32-bit lane 92 | return _mm256_sllv_epi32(ones, hash_data); 93 | } 94 | 95 | [[gnu::always_inline]] inline void insert(size_t id) 96 | { 97 | const uint64_t hash = get_hash(id); 98 | const uint32_t bucket_id = hash & directory_mask; 99 | const __m256i mask = get_mask(hash >> log_num_buckets); 100 | __m256i *const bucket = &reinterpret_cast<__m256i *>(directory)[bucket_id]; 101 | _mm256_store_si256(bucket, _mm256_or_si256(*bucket, mask)); 102 | } 103 | 104 | [[gnu::always_inline]] inline bool find(size_t id) const 105 | { 106 | const uint64_t hash = get_hash(id); 107 | const uint32_t bucket_idx = hash & directory_mask; 108 | const __m256i mask = get_mask(hash >> log_num_buckets); 109 | const __m256i bucket = reinterpret_cast<__m256i *>(directory)[bucket_idx]; 110 | // We should return true if 'bucket' has a one wherever 'mask' does. 111 | // _mm256_testc_si256 takes the negation of its first argument and 112 | // ands that with its second argument. In our case, the result is 113 | // zero everywhere iff there is a one in 'bucket' wherever 'mask' is 114 | // one. testc returns 1 if the result is 0 everywhere and returns 0 115 | // otherwise. 116 | return _mm256_testc_si256(bucket, mask); 117 | } 118 | 119 | private: 120 | using bucket_t = uint32_t[8]; 121 | static constexpr size_t LOG_BUCKET_BYTE_SIZE = 5; 122 | size_t log_num_buckets; 123 | bucket_t *directory; 124 | uint32_t directory_mask; 125 | }; 126 | } // namespace livegraph 127 | -------------------------------------------------------------------------------- /core/edge_iterator.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include "blocks.hpp" 19 | #include "graph.hpp" 20 | #include "utils.hpp" 21 | 22 | namespace livegraph 23 | { 24 | class EdgeIterator 25 | { 26 | public: 27 | EdgeIterator(EdgeEntry *_entries, 28 | char *_data, 29 | size_t _num_entries, 30 | size_t _data_length, 31 | timestamp_t _read_epoch_id, 32 | timestamp_t _local_txn_id, 33 | bool _reverse) 34 | : entries(_entries), 35 | data(_data), 36 | num_entries(_num_entries), 37 | data_length(_data_length), 38 | read_epoch_id(_read_epoch_id), 39 | local_txn_id(_local_txn_id), 40 | reverse(_reverse) 41 | { 42 | if (!reverse) 43 | { 44 | entries_cursor = entries - num_entries; // at the begining 45 | data_cursor = data + data_length; // at the end 46 | } 47 | else 48 | { 49 | entries_cursor = entries; // at the end 50 | data_cursor = data; // at the begining 51 | } 52 | 53 | if (!reverse) 54 | { 55 | while (valid()) 56 | { 57 | if (cmp_timestamp(entries_cursor->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 0 && 58 | cmp_timestamp(entries_cursor->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 0) 59 | { 60 | break; 61 | } 62 | data_cursor -= entries_cursor->get_length(); 63 | entries_cursor++; 64 | } 65 | } 66 | else 67 | { 68 | while (valid()) 69 | { 70 | if (cmp_timestamp((entries_cursor - 1)->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 71 | 0 && 72 | cmp_timestamp((entries_cursor - 1)->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 73 | 0) 74 | { 75 | break; 76 | } 77 | data_cursor += (entries_cursor - 1)->get_length(); 78 | entries_cursor--; 79 | } 80 | } 81 | } 82 | 83 | EdgeIterator(const EdgeIterator &) = default; 84 | 85 | EdgeIterator(EdgeIterator &&) = default; 86 | 87 | bool valid() const 88 | { 89 | if (!reverse) 90 | return !(entries_cursor == entries); 91 | else 92 | return !(entries_cursor == entries - num_entries); 93 | } 94 | 95 | void next() 96 | { 97 | if (!reverse) 98 | { 99 | while (valid()) 100 | { 101 | data_cursor -= entries_cursor->get_length(); 102 | entries_cursor++; 103 | if (cmp_timestamp(entries_cursor->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 0 && 104 | cmp_timestamp(entries_cursor->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 0) 105 | { 106 | break; 107 | } 108 | } 109 | } 110 | else 111 | { 112 | while (valid()) 113 | { 114 | data_cursor += (entries_cursor - 1)->get_length(); 115 | entries_cursor--; 116 | if (cmp_timestamp((entries_cursor - 1)->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 117 | 0 && 118 | cmp_timestamp((entries_cursor - 1)->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 119 | 0) 120 | { 121 | break; 122 | } 123 | } 124 | } 125 | } 126 | 127 | vertex_t dst_id() const 128 | { 129 | if (!valid()) 130 | return Graph::VERTEX_TOMBSTONE; 131 | if (!reverse) 132 | return entries_cursor->get_dst(); 133 | else 134 | return (entries_cursor - 1)->get_dst(); 135 | } 136 | 137 | std::string_view edge_data() const 138 | { 139 | if (!valid()) 140 | return std::string_view(); 141 | if (!reverse) 142 | return std::string_view(data_cursor - entries_cursor->get_length(), entries_cursor->get_length()); 143 | else 144 | return std::string_view(data_cursor, (entries_cursor - 1)->get_length()); 145 | } 146 | 147 | private: 148 | EdgeEntry *entries; 149 | char *data; 150 | size_t num_entries; 151 | size_t data_length; 152 | timestamp_t read_epoch_id; 153 | timestamp_t local_txn_id; 154 | bool reverse; 155 | EdgeEntry *entries_cursor; 156 | char *data_cursor; 157 | }; 158 | } // namespace livegraph 159 | -------------------------------------------------------------------------------- /core/block_manager.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #include "types.hpp" 31 | 32 | namespace livegraph 33 | { 34 | class BlockManager 35 | { 36 | public: 37 | constexpr static uintptr_t NULLPOINTER = 0; // UINTPTR_MAX; 38 | 39 | BlockManager(std::string path, size_t _capacity = 1ul << 40) 40 | : capacity(_capacity), 41 | mutex(), 42 | free_blocks(std::vector>(LARGE_BLOCK_THRESHOLD, std::vector())), 43 | large_free_blocks(MAX_ORDER, std::vector()) 44 | { 45 | if (path.empty()) 46 | { 47 | fd = EMPTY_FD; 48 | data = 49 | mmap(nullptr, capacity, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); 50 | if (data == MAP_FAILED) 51 | throw std::runtime_error("mmap block error."); 52 | } 53 | else 54 | { 55 | fd = open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0640); 56 | if (fd == EMPTY_FD) 57 | throw std::runtime_error("open block file error."); 58 | if (ftruncate(fd, FILE_TRUNC_SIZE) != 0) 59 | throw std::runtime_error("ftruncate block file error."); 60 | data = mmap(nullptr, capacity, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 61 | if (data == MAP_FAILED) 62 | throw std::runtime_error("mmap block error."); 63 | } 64 | 65 | if (madvise(data, capacity, MADV_RANDOM) != 0) 66 | throw std::runtime_error("madvise block error."); 67 | 68 | file_size = FILE_TRUNC_SIZE; 69 | used_size = 0; 70 | 71 | null_holder = alloc(LARGE_BLOCK_THRESHOLD); 72 | } 73 | 74 | ~BlockManager() 75 | { 76 | free(null_holder, LARGE_BLOCK_THRESHOLD); 77 | msync(data, capacity, MS_SYNC); 78 | munmap(data, capacity); 79 | if (fd != EMPTY_FD) 80 | close(fd); 81 | } 82 | 83 | uintptr_t alloc(order_t order) 84 | { 85 | uintptr_t pointer = NULLPOINTER; 86 | if (order < LARGE_BLOCK_THRESHOLD) 87 | { 88 | pointer = pop(free_blocks.local(), order); 89 | } 90 | else 91 | { 92 | std::lock_guard lock(mutex); 93 | pointer = pop(large_free_blocks, order); 94 | } 95 | 96 | if (pointer == NULLPOINTER) 97 | { 98 | size_t block_size = 1ul << order; 99 | pointer = used_size.fetch_add(block_size); 100 | 101 | if (pointer + block_size >= file_size) 102 | { 103 | auto new_file_size = ((pointer + block_size) / FILE_TRUNC_SIZE + 1) * FILE_TRUNC_SIZE; 104 | std::lock_guard lock(mutex); 105 | if (new_file_size >= file_size) 106 | { 107 | if (fd != EMPTY_FD) 108 | { 109 | if (ftruncate(fd, new_file_size) != 0) 110 | throw std::runtime_error("ftruncate block file error."); 111 | } 112 | file_size = new_file_size; 113 | } 114 | } 115 | } 116 | 117 | return pointer; 118 | } 119 | 120 | void free(uintptr_t block, order_t order) 121 | { 122 | if (order < LARGE_BLOCK_THRESHOLD) 123 | { 124 | push(free_blocks.local(), order, block); 125 | } 126 | else 127 | { 128 | std::lock_guard lock(mutex); 129 | push(large_free_blocks, order, block); 130 | } 131 | } 132 | 133 | template inline T *convert(uintptr_t block) 134 | { 135 | if (__builtin_expect((block == NULLPOINTER), 0)) 136 | return nullptr; 137 | return reinterpret_cast(reinterpret_cast(data) + block); 138 | } 139 | 140 | private: 141 | const size_t capacity; 142 | int fd; 143 | void *data; 144 | std::mutex mutex; 145 | tbb::enumerable_thread_specific>> free_blocks; 146 | std::vector> large_free_blocks; 147 | std::atomic used_size, file_size; 148 | uintptr_t null_holder; 149 | 150 | uintptr_t pop(std::vector> &free_block, order_t order) 151 | { 152 | uintptr_t pointer = NULLPOINTER; 153 | if (free_block[order].size()) 154 | { 155 | pointer = free_block[order].back(); 156 | free_block[order].pop_back(); 157 | } 158 | return pointer; 159 | } 160 | 161 | void push(std::vector> &free_block, order_t order, uintptr_t pointer) 162 | { 163 | free_block[order].push_back(pointer); 164 | } 165 | 166 | constexpr static int EMPTY_FD = -1; 167 | constexpr static order_t MAX_ORDER = 64; 168 | constexpr static order_t LARGE_BLOCK_THRESHOLD = 20; 169 | constexpr static size_t FILE_TRUNC_SIZE = 1ul << 30; // 1GB 170 | }; 171 | 172 | class BlockManagerLibc 173 | { 174 | public: 175 | constexpr static uintptr_t NULLPOINTER = UINTPTR_MAX; 176 | 177 | uintptr_t alloc(order_t order) 178 | { 179 | auto p = aligned_alloc(1ul << order, 1ul << order); 180 | if (!p) 181 | throw std::runtime_error("Failed to alloc block"); 182 | return reinterpret_cast(p); 183 | } 184 | 185 | void free(uintptr_t block, order_t order) { ::free(reinterpret_cast(block)); } 186 | 187 | template T *convert(uintptr_t block) 188 | { 189 | if (block == NULLPOINTER) 190 | return nullptr; 191 | return reinterpret_cast(block); 192 | } 193 | }; 194 | } // namespace livegraph 195 | -------------------------------------------------------------------------------- /src/graph.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "core/graph.hpp" 17 | #include "core/transaction.hpp" 18 | 19 | using namespace livegraph; 20 | 21 | Transaction Graph::begin_transaction() 22 | { 23 | auto local_txn_id = transaction_id.fetch_add(1, std::memory_order_relaxed) + 1; // txn_id begin from 1 24 | auto read_epoch_id = epoch_id.load(std::memory_order_acquire); 25 | read_epoch_table.local() = read_epoch_id; 26 | if (local_txn_id % COMPACTION_CYCLE == 0) 27 | compact(local_txn_id); 28 | return Transaction(*this, local_txn_id, read_epoch_id, false, true); 29 | } 30 | 31 | Transaction Graph::begin_read_only_transaction() 32 | { 33 | auto read_epoch_id = epoch_id.load(std::memory_order_acquire); 34 | read_epoch_table.local() = read_epoch_id; 35 | return Transaction(*this, RO_TRANSACTION, read_epoch_id, false, false); 36 | } 37 | 38 | Transaction Graph::begin_batch_loader() 39 | { 40 | auto read_epoch_id = epoch_id.load(std::memory_order_acquire); 41 | read_epoch_table.local() = read_epoch_id; 42 | return Transaction(*this, RO_TRANSACTION, read_epoch_id, true, false); 43 | } 44 | 45 | timestamp_t Graph::compact(timestamp_t read_epoch_id) 46 | { 47 | if (read_epoch_id == NO_TRANSACTION) 48 | read_epoch_id = epoch_id.load(); 49 | for (auto id : read_epoch_table) 50 | { 51 | if (id != NO_TRANSACTION && id < read_epoch_id) 52 | read_epoch_id = id; 53 | } 54 | 55 | size_t recycled_block_size = 0; 56 | std::unordered_set new_compact_table; 57 | 58 | for (vertex_t vid : compact_table.local()) 59 | { 60 | if (!vertex_futexes[vid].try_lock_for(TIMEOUT)) 61 | { 62 | new_compact_table.emplace(vid); 63 | continue; 64 | } 65 | 66 | bool need_future_compact = false; 67 | 68 | auto compact_n2o_blocks = [&](uintptr_t pointer) { 69 | auto block = block_manager.convert(pointer); 70 | while (block) 71 | { 72 | // The first block before the minimal epoch 73 | // Blocks before that are garbage 74 | if (cmp_timestamp(block->get_creation_time_pointer(), read_epoch_id) < 0) 75 | { 76 | std::vector> pointers_to_recycle; 77 | 78 | auto garbage_pointer = block->get_prev_pointer(); 79 | auto garbage_block = block_manager.convert(garbage_pointer); 80 | while (garbage_block) 81 | { 82 | pointers_to_recycle.emplace_back(garbage_pointer, garbage_block->get_order()); 83 | garbage_pointer = garbage_block->get_prev_pointer(); 84 | garbage_block = block_manager.convert(garbage_pointer); 85 | } 86 | 87 | block->set_prev_pointer(block_manager.NULLPOINTER); 88 | for (auto [pointer, order] : pointers_to_recycle) 89 | { 90 | recycled_block_size += 1ul << order; 91 | block_manager.free(pointer, order); 92 | } 93 | 94 | break; 95 | } 96 | 97 | pointer = block->get_prev_pointer(); 98 | block = block_manager.convert(pointer); 99 | // The next block is garbage with larger epoch 100 | if (block) 101 | need_future_compact = true; 102 | } 103 | }; 104 | 105 | // Compact VertexBlock 106 | compact_n2o_blocks(vertex_ptrs[vid]); 107 | 108 | // Compact EdgeLabelBlock 109 | compact_n2o_blocks(edge_label_ptrs[vid]); 110 | 111 | // Compact EdgeBlock 112 | { 113 | auto edge_label_pointer = edge_label_ptrs[vid]; 114 | auto edge_label_block = block_manager.convert(edge_label_pointer); 115 | if (edge_label_block) 116 | { 117 | for (size_t i = 0; i < edge_label_block->get_num_entries(); i++) 118 | { 119 | auto &label_entry = edge_label_block->get_entries()[i]; 120 | auto pointer = label_entry.get_pointer(); 121 | auto edge_block = block_manager.convert(pointer); 122 | if (!edge_block) 123 | continue; 124 | compact_n2o_blocks(pointer); 125 | 126 | size_t new_num_entries = 0; 127 | size_t new_data_length = 0; 128 | 129 | // Scan deleted edges 130 | auto entries = edge_block->get_entries(); 131 | auto data = edge_block->get_data(); 132 | auto num_entries = edge_block->get_num_entries(); 133 | for (size_t i = 0; i < num_entries; i++) 134 | { 135 | entries--; 136 | if (cmp_timestamp(entries->get_deletion_time_pointer(), read_epoch_id) > 0) 137 | { 138 | new_num_entries++; 139 | new_data_length += entries->get_length(); 140 | } 141 | } 142 | entries = edge_block->get_entries(); // Reset cursor 143 | 144 | if (new_num_entries == num_entries) 145 | continue; 146 | 147 | // Copy a new edge block 148 | need_future_compact = true; 149 | 150 | auto size = sizeof(EdgeBlockHeader) + new_num_entries * sizeof(EdgeEntry) + new_data_length; 151 | auto order = size_to_order(size); 152 | 153 | if (order > edge_block->BLOOM_FILTER_PORTION && 154 | size + (1ul << (order - edge_block->BLOOM_FILTER_PORTION)) >= 155 | (1ul << edge_block->BLOOM_FILTER_THRESHOLD)) 156 | { 157 | size += 1ul << (order - edge_block->BLOOM_FILTER_PORTION); 158 | } 159 | order = size_to_order(size); 160 | 161 | auto new_pointer = block_manager.alloc(order); 162 | 163 | auto new_edge_block = block_manager.convert(new_pointer); 164 | new_edge_block->fill(order, vid, read_epoch_id, pointer, edge_block->get_committed_time()); 165 | 166 | auto bloom_filter = new_edge_block->get_bloom_filter(); 167 | for (size_t i = 0; i < num_entries; i++) 168 | { 169 | entries--; 170 | if (cmp_timestamp(entries->get_deletion_time_pointer(), read_epoch_id) > 0) 171 | new_edge_block->append(*entries, data, bloom_filter); 172 | data += entries->get_length(); 173 | } 174 | 175 | label_entry.set_pointer(new_pointer); 176 | 177 | // printf("Compact %lu edges, %lu data\n", 178 | // num_entries-new_num_entries, 179 | // edge_block->get_data_length()-new_data_length); 180 | } 181 | } 182 | } 183 | 184 | if (need_future_compact) 185 | new_compact_table.emplace(vid); 186 | vertex_futexes[vid].unlock(); 187 | } 188 | 189 | compact_table.local().swap(new_compact_table); 190 | 191 | // printf("Compact %lu bytes blocks\n", recycled_block_size); 192 | 193 | return read_epoch_id; 194 | } 195 | -------------------------------------------------------------------------------- /core/transaction.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "blocks.hpp" 26 | #include "graph.hpp" 27 | #include "utils.hpp" 28 | 29 | namespace livegraph 30 | { 31 | class Transaction 32 | { 33 | enum class OPType 34 | { 35 | NewVertex, 36 | PutVertex, 37 | DelVertex, 38 | PutEdge, 39 | DelEdge, 40 | }; 41 | 42 | public: 43 | class RollbackExcept : public std::runtime_error 44 | { 45 | public: 46 | RollbackExcept(const std::string &what_arg) : std::runtime_error(what_arg) {} 47 | RollbackExcept(const char *what_arg) : std::runtime_error(what_arg) {} 48 | }; 49 | 50 | Transaction( 51 | Graph &_graph, timestamp_t _local_txn_id, timestamp_t _read_epoch_id, bool _batch_update, bool _trace_cache) 52 | : graph(_graph), 53 | local_txn_id(_local_txn_id), 54 | read_epoch_id(_read_epoch_id), 55 | batch_update(_batch_update), 56 | trace_cache(_trace_cache), 57 | write_epoch_id(batch_update ? read_epoch_id : -local_txn_id), 58 | valid(true), 59 | wal(), 60 | vertex_ptr_cache(), 61 | edge_ptr_cache(), 62 | block_cache(), 63 | edge_block_num_entries_data_length_cache(), 64 | new_vertex_cache(), 65 | recycled_vertex_cache(), 66 | acquired_locks(), 67 | timestamps_to_update() 68 | { 69 | wal_append((uint64_t)0); // number of operations 70 | wal_append(read_epoch_id); 71 | wal_append(local_txn_id); 72 | } 73 | 74 | Transaction(const Transaction &) = delete; 75 | 76 | Transaction(Transaction &&txn) 77 | : graph(txn.graph), 78 | local_txn_id(std::move(txn.local_txn_id)), 79 | read_epoch_id(std::move(txn.read_epoch_id)), 80 | batch_update(std::move(txn.batch_update)), 81 | trace_cache(std::move(txn.trace_cache)), 82 | write_epoch_id(std::move(txn.write_epoch_id)), 83 | valid(std::move(txn.valid)), 84 | wal(std::move(txn.wal)), 85 | vertex_ptr_cache(std::move(txn.vertex_ptr_cache)), 86 | edge_ptr_cache(std::move(txn.edge_ptr_cache)), 87 | block_cache(std::move(txn.block_cache)), 88 | edge_block_num_entries_data_length_cache(std::move(txn.edge_block_num_entries_data_length_cache)), 89 | new_vertex_cache(std::move(txn.new_vertex_cache)), 90 | recycled_vertex_cache(std::move(txn.recycled_vertex_cache)), 91 | acquired_locks(std::move(txn.acquired_locks)), 92 | timestamps_to_update(std::move(txn.timestamps_to_update)) 93 | { 94 | txn.valid = false; 95 | } 96 | 97 | timestamp_t get_read_epoch_id() const { return read_epoch_id; } 98 | 99 | vertex_t new_vertex(bool use_recycled_vertex = false); 100 | void put_vertex(vertex_t vertex_id, std::string_view data); 101 | bool del_vertex(vertex_t vertex_id, bool recycle = false); 102 | 103 | void put_edge(vertex_t src, label_t label, vertex_t dst, std::string_view edge_data, bool force_insert = false); 104 | bool del_edge(vertex_t src, label_t label, vertex_t dst); 105 | 106 | std::string_view get_vertex(vertex_t vertex_id); 107 | std::string_view get_edge(vertex_t src, label_t label, vertex_t dst); 108 | EdgeIterator get_edges(vertex_t src, label_t label, bool reverse = false); 109 | 110 | timestamp_t commit(bool wait_visable = true); 111 | void abort(); 112 | 113 | ~Transaction() 114 | { 115 | if (valid) 116 | abort(); 117 | } 118 | 119 | private: 120 | Graph &graph; 121 | const timestamp_t local_txn_id; 122 | const timestamp_t read_epoch_id; 123 | const bool batch_update; 124 | const bool trace_cache; 125 | const timestamp_t write_epoch_id; 126 | bool valid; 127 | std::string wal; 128 | 129 | std::unordered_map vertex_ptr_cache; 130 | std::map, uintptr_t> edge_ptr_cache; 131 | std::vector> block_cache; 132 | std::unordered_map> edge_block_num_entries_data_length_cache; 133 | std::vector new_vertex_cache; 134 | std::deque recycled_vertex_cache; 135 | 136 | std::unordered_set acquired_locks; 137 | std::vector> timestamps_to_update; 138 | 139 | template >> inline void wal_append(T data) 140 | { 141 | wal.append(reinterpret_cast(&data), sizeof(T)); 142 | } 143 | 144 | inline void wal_append(std::string_view data) 145 | { 146 | wal_append(data.size()); 147 | wal.append(data); 148 | } 149 | 150 | inline uint64_t &wal_num_ops() { return *reinterpret_cast(wal.data()); } 151 | 152 | void check_writable() 153 | { 154 | if (!batch_update && !trace_cache) 155 | throw std::invalid_argument("The transaction is read-only without cache."); 156 | } 157 | 158 | void check_valid() 159 | { 160 | if (!valid) 161 | throw std::invalid_argument("The transaction is committed or aborted."); 162 | } 163 | 164 | void check_vertex_id(vertex_t vertex_id) 165 | { 166 | if (vertex_id >= graph.vertex_id.load(std::memory_order_relaxed)) 167 | throw std::invalid_argument("The vertex id is invalid."); 168 | } 169 | 170 | void ensure_vertex_lock(vertex_t vertex_id) 171 | { 172 | auto iter = acquired_locks.find(vertex_id); 173 | if (iter != acquired_locks.end()) 174 | return; 175 | if (!graph.vertex_futexes[vertex_id].try_lock_for(Graph::TIMEOUT)) 176 | throw RollbackExcept("Deadlock on Vertex: " + std::to_string(vertex_id) + "."); 177 | acquired_locks.emplace_hint(iter, vertex_id); 178 | } 179 | 180 | void ensure_no_confict(vertex_t vertex_id) 181 | { 182 | auto header = graph.block_manager.convert(graph.vertex_ptrs[vertex_id]); 183 | if (header && cmp_timestamp(header->get_creation_time_pointer(), read_epoch_id, local_txn_id) > 0) 184 | throw RollbackExcept("Write-write confict on: " + std::to_string(vertex_id) + "."); 185 | } 186 | 187 | void clean() 188 | { 189 | for (const auto &vertex_id : acquired_locks) 190 | { 191 | graph.vertex_futexes[vertex_id].unlock(); 192 | } 193 | valid = false; 194 | graph.read_epoch_table.local() = Graph::NO_TRANSACTION; 195 | } 196 | 197 | std::pair get_num_entries_data_length_cache(EdgeBlockHeader *edge_block) const 198 | { 199 | if (batch_update || !trace_cache) 200 | return edge_block->get_num_entries_data_length_atomic(); 201 | auto iter = edge_block_num_entries_data_length_cache.find(edge_block); 202 | if (iter == edge_block_num_entries_data_length_cache.end()) 203 | return edge_block->get_num_entries_data_length_atomic(); 204 | else 205 | return iter->second; 206 | } 207 | 208 | void set_num_entries_data_length_cache(EdgeBlockHeader *edge_block, size_t num_entries, size_t data_length) 209 | { 210 | if (batch_update) 211 | edge_block->set_num_entries_data_length_atomic(num_entries, data_length); 212 | else 213 | edge_block_num_entries_data_length_cache[edge_block] = {num_entries, data_length}; 214 | } 215 | 216 | std::pair 217 | find_edge(vertex_t dst, EdgeBlockHeader *edge_block, size_t num_entries, size_t data_length); 218 | 219 | uintptr_t locate_edge_block(vertex_t src, label_t label); 220 | 221 | void update_edge_label_block(vertex_t src, label_t label, uintptr_t edge_block_pointer); 222 | 223 | void ensure_no_confict(vertex_t src, label_t label); 224 | }; 225 | } // namespace livegraph 226 | -------------------------------------------------------------------------------- /core/commit_manager.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "types.hpp" 28 | 29 | namespace livegraph 30 | { 31 | 32 | class CommitManager 33 | { 34 | public: 35 | CommitManager(std::string path, std::atomic &_global_epoch_id) 36 | : fd(EMPTY_FD), 37 | seq_front{0, 0}, 38 | seq_rear{0, 0}, 39 | mutex(), 40 | client_mutex(), 41 | cv_server(), 42 | cv_client(), 43 | global_client_mutex(0), 44 | used_size(0), 45 | file_size(0), 46 | global_epoch_id(_global_epoch_id), 47 | writing_epoch_id(global_epoch_id), 48 | unfinished_epoch_id(), 49 | queue(), 50 | closed(false), 51 | server_thread([&] { server_loop(); }) 52 | { 53 | if (!path.empty()) 54 | { 55 | fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0640); 56 | if (fd == EMPTY_FD) 57 | throw std::runtime_error("open wal file error."); 58 | if (ftruncate(fd, FILE_TRUNC_SIZE) != 0) 59 | throw std::runtime_error("ftruncate wal file error."); 60 | } 61 | file_size = FILE_TRUNC_SIZE; 62 | } 63 | 64 | ~CommitManager() 65 | { 66 | closed.store(true); 67 | cv_server.notify_one(); 68 | server_thread.join(); 69 | if (fd != EMPTY_FD) 70 | close(fd); 71 | } 72 | 73 | std::pair *> register_commit(std::string_view wal) 74 | { 75 | while (true) 76 | { 77 | auto local_client_mutex = global_client_mutex.load(); 78 | std::unique_lock lock(mutex[local_client_mutex]); 79 | if (local_client_mutex != global_client_mutex.load()) 80 | continue; 81 | 82 | timestamp_t local_commit_epoch_id; 83 | std::atomic *local_num_unfinished; 84 | 85 | queue[local_client_mutex].emplace(wal, &local_commit_epoch_id, &local_num_unfinished); 86 | auto my_seq = seq_rear[local_client_mutex]++; 87 | 88 | lock.unlock(); 89 | cv_server.notify_one(); 90 | 91 | std::unique_lock client_lock(client_mutex[local_client_mutex]); 92 | if (seq_front[local_client_mutex] <= my_seq) 93 | { 94 | cv_client[local_client_mutex].wait(client_lock, 95 | [&]() { return seq_front[local_client_mutex] > my_seq; }); 96 | } 97 | 98 | return {local_commit_epoch_id, local_num_unfinished}; 99 | } 100 | } 101 | 102 | void finish_commit(timestamp_t local_commit_epoch_id, std::atomic *local_num_unfinished, bool wait) 103 | { 104 | local_num_unfinished->fetch_sub(1); 105 | while (wait && global_epoch_id < local_commit_epoch_id) 106 | { 107 | cv_server.notify_one(); 108 | std::this_thread::yield(); 109 | } 110 | } 111 | 112 | private: 113 | int fd; 114 | size_t seq_front[2]; //(server) increment after fsync() is finished 115 | size_t seq_rear[2]; // (client) increment after push() is finished 116 | std::mutex mutex[2]; // (server/clients) serialize queue operations 117 | std::mutex client_mutex[2]; // (clients) wait for fsync() to finish 118 | std::condition_variable cv_server; // (server) wait when the queue is empty 119 | std::condition_variable cv_client[2]; // (clients) wait for fsync() to finish 120 | std::atomic global_client_mutex; 121 | size_t used_size; 122 | size_t file_size; 123 | std::atomic &global_epoch_id; 124 | timestamp_t writing_epoch_id; 125 | std::queue>> unfinished_epoch_id; 126 | std::queue **>> 127 | queue[2]; // wal, epoch_id, unfinished 128 | std::atomic closed; 129 | std::thread server_thread; 130 | 131 | constexpr static size_t FILE_TRUNC_SIZE = 1ul << 30; // 1GB 132 | constexpr static int EMPTY_FD = -1; 133 | constexpr static auto SERVER_SPIN_INTERVAL = std::chrono::microseconds(100); 134 | 135 | void check_unfinished_epoch_id() 136 | { 137 | while (!unfinished_epoch_id.empty()) 138 | { 139 | auto &[current_epoch_id, num_unfinished] = unfinished_epoch_id.front(); 140 | if (num_unfinished.load() == 0) 141 | { 142 | global_epoch_id = current_epoch_id; 143 | unfinished_epoch_id.pop(); 144 | } 145 | else 146 | { 147 | break; 148 | } 149 | } 150 | } 151 | 152 | void server_loop() 153 | { 154 | while (true) 155 | { 156 | check_unfinished_epoch_id(); 157 | int local_client_mutex = global_client_mutex.load(); 158 | std::unique_lock lock(mutex[local_client_mutex]); 159 | auto &local_queue = queue[local_client_mutex]; 160 | while (local_queue.empty() && !closed.load()) 161 | { 162 | cv_server.wait_for(lock, SERVER_SPIN_INTERVAL, [&]() { 163 | check_unfinished_epoch_id(); 164 | return !local_queue.empty() || closed.load(); 165 | }); 166 | check_unfinished_epoch_id(); 167 | cv_client[local_client_mutex ^ 1].notify_all(); 168 | } 169 | std::unique_lock client_lock(client_mutex[local_client_mutex]); 170 | 171 | global_client_mutex ^= 1; 172 | 173 | size_t num_txns = local_queue.size(); 174 | 175 | if (!num_txns) 176 | break; 177 | 178 | ++writing_epoch_id; 179 | 180 | unfinished_epoch_id.emplace(writing_epoch_id, 0); 181 | 182 | auto &num_unfinished = unfinished_epoch_id.back().second; 183 | 184 | std::string group_wal; 185 | group_wal.append(reinterpret_cast(&writing_epoch_id), sizeof(writing_epoch_id)); 186 | group_wal.append(reinterpret_cast(&num_txns), sizeof(num_txns)); 187 | 188 | for (size_t i = 0; i < num_txns; i++) 189 | { 190 | auto &[wal, ret_epoch_id, ret_num] = local_queue.front(); 191 | 192 | group_wal.append(wal); 193 | *ret_epoch_id = writing_epoch_id; 194 | *ret_num = &num_unfinished; 195 | ++num_unfinished; 196 | local_queue.pop(); 197 | } 198 | 199 | auto expected_size = used_size + group_wal.size(); 200 | if (expected_size > file_size) 201 | { 202 | size_t new_file_size = (expected_size / FILE_TRUNC_SIZE + 1) * FILE_TRUNC_SIZE; 203 | if (fd != EMPTY_FD) 204 | { 205 | if (ftruncate(fd, new_file_size) != 0) 206 | throw std::runtime_error("ftruncate wal file error."); 207 | } 208 | file_size = new_file_size; 209 | } 210 | 211 | used_size += group_wal.size(); 212 | 213 | if (fd != EMPTY_FD) 214 | { 215 | if ((size_t)write(fd, group_wal.c_str(), group_wal.size()) != group_wal.size()) 216 | std::runtime_error("write wal file error."); 217 | } 218 | 219 | if (fd != EMPTY_FD) 220 | { 221 | if (fdatasync(fd) != 0) 222 | std::runtime_error("fdatasync wal file error."); 223 | } 224 | 225 | ++num_unfinished; 226 | 227 | lock.unlock(); 228 | seq_front[local_client_mutex] += num_txns; 229 | cv_client[local_client_mutex].notify_all(); 230 | client_lock.unlock(); 231 | 232 | --num_unfinished; 233 | } 234 | } 235 | }; 236 | 237 | } // namespace livegraph 238 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /test/graph.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "bind/livegraph.hpp" 27 | #include "core/livegraph.hpp" 28 | 29 | TYPE_TO_STRING(livegraph::Graph); 30 | TYPE_TO_STRING(lg::Graph); 31 | 32 | TEST_CASE_TEMPLATE("testing the Graph", GraphType, livegraph::Graph, lg::Graph) 33 | { 34 | using namespace livegraph; 35 | const vertex_t max_vertices = 256; 36 | const label_t max_label = 16; 37 | const label_t max_init_label = 4; 38 | const size_t num_edges_per_vertex = max_vertices; 39 | const size_t max_data_length = 64; 40 | const size_t num_transactions = 1024; 41 | const size_t ops_per_txn = 4; 42 | std::vector> vertices(max_vertices); 43 | std::vector, std::string>> edges(max_vertices); 44 | std::vector mutexs(max_vertices); 45 | 46 | { 47 | GraphType graph("./block.mmap"); 48 | 49 | #pragma omp parallel 50 | { 51 | std::mt19937 rand(omp_get_thread_num()); 52 | auto gendata = [&]() { 53 | auto length = rand() % max_data_length + 1; 54 | std::string data; 55 | for (size_t i = 0; i < length; i++) 56 | { 57 | data += 'a' + rand() % 26; 58 | } 59 | return data; 60 | }; 61 | 62 | auto check_vertex = [&](auto &txn, vertex_t src) { 63 | auto read_epoch_id = txn.get_read_epoch_id(); 64 | std::lock_guard lock(mutexs[src]); 65 | { 66 | auto iter = --vertices[src].upper_bound(read_epoch_id); 67 | CHECK(iter->first <= read_epoch_id); 68 | CHECK(iter->second == txn.get_vertex(src)); 69 | } 70 | for (label_t label = 0; label < max_label; label++) 71 | { 72 | auto edge_iter = txn.get_edges(src, label); 73 | size_t num_edges = 0, ref_num_edges = 0; 74 | while (edge_iter.valid()) 75 | { 76 | num_edges++; 77 | auto iter = --edges[src].upper_bound(std::make_tuple(label, edge_iter.dst_id(), read_epoch_id)); 78 | auto [ref_label, ref_dst, ref_timestamp] = iter->first; 79 | CHECK(ref_label == label); 80 | CHECK(ref_dst == edge_iter.dst_id()); 81 | CHECK(ref_timestamp <= read_epoch_id); 82 | CHECK(iter->second == edge_iter.edge_data()); 83 | edge_iter.next(); 84 | } 85 | 86 | auto iter = edges[src].lower_bound(std::make_tuple(label, 0, 0)); 87 | while (iter != edges[src].end()) 88 | { 89 | auto [ref_label, ref_dst, ref_timestamp] = iter->first; 90 | if (ref_label != label) 91 | break; 92 | auto next_iter = iter; 93 | ++next_iter; 94 | auto [next_label, next_dst, next_timestamp] = next_iter->first; 95 | if (next_iter == edges[src].end() || next_label != label || next_dst != ref_dst || 96 | next_timestamp > read_epoch_id) 97 | { 98 | if (ref_timestamp <= read_epoch_id && !iter->second.empty()) 99 | ++ref_num_edges; 100 | } 101 | ++iter; 102 | } 103 | 104 | CHECK(num_edges == ref_num_edges); 105 | } 106 | }; 107 | auto check_graph = [&](auto &txn) { 108 | for (size_t src = 0; src < max_vertices; src++) 109 | { 110 | check_vertex(txn, src); 111 | } 112 | }; 113 | 114 | { 115 | auto txn = graph.begin_batch_loader(); 116 | #pragma omp for 117 | for (size_t i = 0; i < max_vertices; i++) 118 | { 119 | auto src = txn.new_vertex(); 120 | auto data = gendata(); 121 | txn.put_vertex(src, data); 122 | vertices[src][0] = data; 123 | } 124 | #pragma omp for 125 | for (size_t i = 0; i < max_vertices; i++) 126 | { 127 | auto src = i; 128 | for (size_t j = 0; j < num_edges_per_vertex; j++) 129 | { 130 | auto label = rand() % max_init_label; 131 | auto dst = rand() % max_vertices; 132 | auto data = gendata(); 133 | txn.put_edge(src, label, dst, data); 134 | edges[src][std::make_tuple(label, dst, 0)] = data; 135 | } 136 | } 137 | #pragma omp for 138 | for (size_t i = 0; i < max_vertices; i++) 139 | { 140 | check_vertex(txn, i); 141 | } 142 | } 143 | 144 | for (size_t i = 0; i < num_transactions; i++) 145 | { 146 | auto type = rand() % 1000; 147 | if (type < 1) 148 | { 149 | auto txn = graph.begin_read_only_transaction(); 150 | check_graph(txn); 151 | MESSAGE("Successfully checked the graph at " << txn.get_read_epoch_id()); 152 | } 153 | else if (type < 2) 154 | { 155 | auto read_epoch_id = graph.compact(); 156 | MESSAGE("Successfully compacted the graph at " << read_epoch_id); 157 | } 158 | else if (type < 100) 159 | { 160 | auto txn = graph.begin_read_only_transaction(); 161 | auto src = rand() % max_vertices; 162 | check_vertex(txn, src); 163 | } 164 | else 165 | { 166 | while (true) 167 | { 168 | std::map> lock; 169 | std::map local_vertices; 170 | std::map, std::string> local_edges; 171 | auto txn = graph.begin_transaction(); 172 | try 173 | { 174 | for (size_t j = 0; j < ops_per_txn; j++) 175 | { 176 | auto src = rand() % max_vertices; 177 | auto type = rand() % 100; 178 | if (type < 1) 179 | { 180 | txn.abort(); 181 | throw typename decltype(txn)::RollbackExcept("Abort."); 182 | } 183 | if (type < 10) 184 | { 185 | txn.del_vertex(src); 186 | CHECK(txn.get_vertex(src) == ""); 187 | local_vertices[src] = ""; 188 | } 189 | else if (type < 20) 190 | { 191 | auto label = (type < 30) ? (rand() % max_label) : (rand() % max_init_label); 192 | auto dst = rand() % max_vertices; 193 | txn.del_edge(src, label, dst); 194 | CHECK(txn.get_edge(src, label, dst) == ""); 195 | local_edges[std::make_tuple(src, label, dst)] = ""; 196 | } 197 | else if (type < 60) 198 | { 199 | auto data = gendata(); 200 | txn.put_vertex(src, data); 201 | CHECK(txn.get_vertex(src) == data); 202 | local_vertices[src] = data; 203 | } 204 | else 205 | { 206 | auto label = (type < 70) ? (rand() % max_label) : (rand() % max_init_label); 207 | auto dst = rand() % max_vertices; 208 | auto data = gendata(); 209 | txn.put_edge(src, label, dst, data); 210 | CHECK(txn.get_edge(src, label, dst) == data); 211 | local_edges[std::make_tuple(src, label, dst)] = data; 212 | } 213 | if (lock.find(src) == lock.end()) 214 | lock.emplace(src, mutexs[src]); 215 | } 216 | 217 | for (auto [src, data] : local_vertices) 218 | { 219 | CHECK(txn.get_vertex(src) == data); 220 | } 221 | for (auto [edge, data] : local_edges) 222 | { 223 | auto [src, label, dst] = edge; 224 | CHECK(txn.get_edge(src, label, dst) == data); 225 | } 226 | 227 | auto write_epoch_id = txn.commit(); 228 | 229 | auto read_txn = graph.begin_read_only_transaction(); 230 | for (auto [src, data] : local_vertices) 231 | { 232 | vertices[src][write_epoch_id] = data; 233 | CHECK(read_txn.get_vertex(src) == data); 234 | } 235 | for (auto [edge, data] : local_edges) 236 | { 237 | auto [src, label, dst] = edge; 238 | edges[src][std::make_tuple(label, dst, write_epoch_id)] = data; 239 | if (read_txn.get_edge(src, label, dst) != data) 240 | { 241 | read_txn.get_edge(src, label, dst); 242 | } 243 | CHECK(read_txn.get_edge(src, label, dst) == data); 244 | } 245 | } 246 | catch (typename decltype(txn)::RollbackExcept &e) 247 | { 248 | continue; 249 | } 250 | break; 251 | } 252 | } 253 | } 254 | } 255 | } 256 | 257 | CHECK(std::remove("./block.mmap") == 0); 258 | } 259 | -------------------------------------------------------------------------------- /core/blocks.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | #include "bloom_filter.hpp" 22 | #include "types.hpp" 23 | #include "utils.hpp" 24 | 25 | namespace livegraph 26 | { 27 | class BlockHeader 28 | { 29 | public: 30 | enum class Type : uint8_t 31 | { 32 | FREE, 33 | VERTEX, 34 | EDGE, 35 | EDGE_LABEL, 36 | SPECIAL 37 | }; 38 | 39 | order_t get_order() const { return order; } 40 | 41 | void set_order(order_t order) { this->order = order; } 42 | 43 | size_t get_block_size() const { return 1ul << order; } 44 | 45 | Type get_type() const { return type; } 46 | 47 | void set_type(Type type) { this->type = type; } 48 | 49 | void fill(order_t order, Type type) 50 | { 51 | set_order(order); 52 | set_type(type); 53 | } 54 | 55 | private: 56 | order_t order; 57 | Type type; 58 | }; 59 | 60 | class N2OBlockHeader : public BlockHeader 61 | { 62 | public: 63 | vertex_t get_vertex_id() const { return ((vertex_t)vid_high << 32) + (vertex_t)vid_low; } 64 | 65 | void set_vertex_id(vertex_t vid) 66 | { 67 | assert(vid <= ((vertex_t)UINT16_MAX << 32) + UINT32_MAX); 68 | vid_high = (vid >> 32) & UINT16_MAX; 69 | vid_low = vid & UINT32_MAX; 70 | } 71 | 72 | timestamp_t get_creation_time() const { return creation_time; } 73 | 74 | timestamp_t *get_creation_time_pointer() { return &creation_time; } 75 | 76 | void set_creation_time(timestamp_t creation_time) { this->creation_time = creation_time; } 77 | 78 | uintptr_t get_prev_pointer() const { return prev_pointer; } 79 | 80 | void set_prev_pointer(uintptr_t prev_pointer) { this->prev_pointer = prev_pointer; } 81 | 82 | void fill(order_t order, Type type, vertex_t vid, timestamp_t creation_time, uintptr_t prev_pointer) 83 | { 84 | BlockHeader::fill(order, type); 85 | set_vertex_id(vid); 86 | set_creation_time(creation_time); 87 | set_prev_pointer(prev_pointer); 88 | } 89 | 90 | private: 91 | uint16_t vid_high; 92 | uint32_t vid_low; 93 | timestamp_t creation_time; 94 | uintptr_t prev_pointer; 95 | }; 96 | 97 | class VertexBlockHeader : public N2OBlockHeader 98 | { 99 | public: 100 | size_t get_length() const { return length; } 101 | 102 | void set_length(size_t length) { this->length = length; } 103 | 104 | const char *get_data() const { return data; } 105 | 106 | char *get_data() { return data; } 107 | 108 | void clear() { set_length(0); } 109 | 110 | bool set_data(const char *data, size_t length) 111 | { 112 | if (sizeof(*this) + length > get_block_size()) 113 | return false; 114 | for (size_t i = 0; i < length; i++) 115 | get_data()[i] = data[i]; 116 | set_length(length); 117 | return true; 118 | } 119 | 120 | constexpr static size_t TOMBSTONE = UINT64_MAX; 121 | 122 | void fill(order_t order, 123 | vertex_t vid, 124 | timestamp_t creation_time, 125 | uintptr_t prev_pointer, 126 | const char *data, 127 | size_t length) 128 | { 129 | N2OBlockHeader::fill(order, Type::VERTEX, vid, creation_time, prev_pointer); 130 | if (length == TOMBSTONE) 131 | set_length(TOMBSTONE); 132 | else 133 | set_data(data, length); 134 | } 135 | 136 | private: 137 | size_t length; 138 | char data[0]; 139 | }; 140 | 141 | class EdgeLabelEntry 142 | { 143 | public: 144 | label_t get_label() const { return label; } 145 | 146 | void set_label(label_t label) { this->label = label; } 147 | 148 | uintptr_t get_pointer() const { return pointer; } 149 | 150 | void set_pointer(uintptr_t pointer) { this->pointer = pointer; } 151 | 152 | private: 153 | label_t label; 154 | uintptr_t pointer; 155 | }; 156 | 157 | class EdgeLabelBlockHeader : public N2OBlockHeader 158 | { 159 | public: 160 | size_t get_num_entries() const { return num_entries; } 161 | 162 | void set_num_entries(size_t num_entries) { this->num_entries = num_entries; } 163 | 164 | const EdgeLabelEntry *get_entries() const { return entries; } 165 | 166 | EdgeLabelEntry *get_entries() { return entries; } 167 | 168 | void clear() { set_num_entries(0); } 169 | bool append(EdgeLabelEntry entry) 170 | { 171 | auto num = get_num_entries(); 172 | if (sizeof(*this) + (num + 1) * sizeof(entry) > get_block_size()) 173 | return false; 174 | get_entries()[num] = entry; 175 | compiler_fence(); 176 | set_num_entries(num + 1); 177 | return true; 178 | } 179 | 180 | void fill(order_t order, vertex_t vid, timestamp_t creation_time, uintptr_t prev_pointer) 181 | { 182 | N2OBlockHeader::fill(order, Type::EDGE_LABEL, vid, creation_time, prev_pointer); 183 | clear(); 184 | } 185 | 186 | private: 187 | size_t num_entries; 188 | EdgeLabelEntry entries[0]; 189 | }; 190 | 191 | class EdgeEntry 192 | { 193 | public: 194 | vertex_t get_dst() const { return ((vertex_t)dst_high << 32) + (vertex_t)dst_low; } 195 | 196 | void set_dst(vertex_t dst) 197 | { 198 | assert(dst <= ((vertex_t)UINT16_MAX << 32) + UINT32_MAX); 199 | dst_high = (dst >> 32) & UINT16_MAX; 200 | dst_low = dst & UINT32_MAX; 201 | } 202 | 203 | timestamp_t get_creation_time() const { return creation_time; } 204 | 205 | timestamp_t *get_creation_time_pointer() { return &creation_time; } 206 | 207 | void set_creation_time(timestamp_t creation_time) { this->creation_time = creation_time; } 208 | 209 | timestamp_t get_deletion_time() const { return deletion_time; } 210 | 211 | timestamp_t *get_deletion_time_pointer() { return &deletion_time; } 212 | 213 | void set_deletion_time(timestamp_t deletion_time) { this->deletion_time = deletion_time; } 214 | 215 | uint16_t get_length() const { return length; } 216 | 217 | void set_length(uint16_t length) { this->length = length; } 218 | 219 | private: 220 | uint16_t length; 221 | uint16_t dst_high; 222 | uint32_t dst_low; 223 | timestamp_t creation_time; 224 | timestamp_t deletion_time; 225 | }; 226 | 227 | class EdgeBlockHeader : public N2OBlockHeader 228 | { 229 | public: 230 | timestamp_t get_committed_time() const { return committed_time; } 231 | 232 | timestamp_t *get_committed_time_pointer() { return &committed_time; } 233 | 234 | void set_committed_time(timestamp_t committed_time) { this->committed_time = committed_time; } 235 | 236 | size_t get_data_length() const { return tail.data.data_length; } 237 | 238 | void set_data_length(size_t data_length) { this->tail.data.data_length = data_length; } 239 | 240 | const char *get_data() const { return data; } 241 | 242 | char *get_data() { return data; } 243 | 244 | size_t get_num_entries() const { return tail.data.num_entries; } 245 | 246 | void set_num_entries(size_t num_entries) { this->tail.data.num_entries = num_entries; } 247 | 248 | const EdgeEntry *get_entries() const 249 | { 250 | size_t block_size = get_block_size(); 251 | if (get_order() >= BLOOM_FILTER_THRESHOLD) 252 | block_size -= block_size >> BLOOM_FILTER_PORTION; 253 | return (EdgeEntry *)((uint8_t *)this + block_size); 254 | } 255 | 256 | EdgeEntry *get_entries() 257 | { 258 | size_t block_size = get_block_size(); 259 | if (get_order() >= BLOOM_FILTER_THRESHOLD) 260 | block_size -= block_size >> BLOOM_FILTER_PORTION; 261 | return (EdgeEntry *)((uint8_t *)this + block_size); 262 | } 263 | 264 | const BloomFilter get_bloom_filter() const 265 | { 266 | if (get_order() < BLOOM_FILTER_THRESHOLD) 267 | return BloomFilter(); 268 | size_t block_size = get_block_size(); 269 | size_t bloom_filter_size = block_size >> BLOOM_FILTER_PORTION; 270 | return BloomFilter(bloom_filter_size, ((uint8_t *)this) + block_size - bloom_filter_size); 271 | } 272 | 273 | BloomFilter get_bloom_filter() 274 | { 275 | if (get_order() < BLOOM_FILTER_THRESHOLD) 276 | return BloomFilter(); 277 | size_t block_size = get_block_size(); 278 | size_t bloom_filter_size = block_size >> BLOOM_FILTER_PORTION; 279 | return BloomFilter(get_order() - BLOOM_FILTER_PORTION, ((uint8_t *)this) + block_size - bloom_filter_size); 280 | } 281 | 282 | void clear() 283 | { 284 | set_num_entries(0); 285 | set_data_length(0); 286 | auto filter = get_bloom_filter(); 287 | if (filter.valid()) 288 | filter.clear(); 289 | } 290 | 291 | bool has_space(EdgeEntry entry, size_t num_entries, size_t data_length) const 292 | { 293 | size_t block_size = get_block_size(); 294 | size_t bloom_filter_size; 295 | if (get_order() < BLOOM_FILTER_THRESHOLD) 296 | bloom_filter_size = 0; 297 | else 298 | bloom_filter_size = block_size >> BLOOM_FILTER_PORTION; 299 | if (sizeof(*this) + (num_entries + 1) * sizeof(entry) + data_length + entry.get_length() + 300 | bloom_filter_size > 301 | get_block_size()) 302 | return false; 303 | else 304 | return true; 305 | } 306 | 307 | EdgeEntry *append(EdgeEntry entry, const char *data, BloomFilter &filter) 308 | { 309 | auto num = get_num_entries(); 310 | auto length = get_data_length(); 311 | if (!has_space(entry, num, length)) 312 | return nullptr; 313 | *(get_entries() - num - 1) = entry; 314 | for (size_t i = 0; i < entry.get_length(); i++) 315 | (get_data() + length)[i] = data[i]; 316 | compiler_fence(); 317 | set_num_entries(num + 1); 318 | set_data_length(length + entry.get_length()); 319 | if (filter.valid()) 320 | filter.insert(entry.get_dst()); 321 | return get_entries() - num - 1; 322 | } 323 | 324 | EdgeEntry *append(EdgeEntry entry, const char *data) 325 | { 326 | auto filter = get_bloom_filter(); 327 | return append(entry, data, filter); 328 | } 329 | 330 | EdgeEntry *append_without_update_size(EdgeEntry entry, const char *data, size_t num, size_t length) 331 | { 332 | auto filter = get_bloom_filter(); 333 | if (!has_space(entry, num, length)) 334 | return nullptr; 335 | *(get_entries() - num - 1) = entry; 336 | for (size_t i = 0; i < entry.get_length(); i++) 337 | (get_data() + length)[i] = data[i]; 338 | if (filter.valid()) 339 | filter.insert(entry.get_dst()); 340 | return get_entries() - num - 1; 341 | } 342 | 343 | void set_num_entries_data_length_atomic(size_t num_entries, size_t data_length) 344 | { 345 | Int128Union new_val; 346 | new_val.data.num_entries = num_entries; 347 | new_val.data.data_length = data_length; 348 | // _mm_store_si128(&tail.m128i, new_val.m128i); 349 | // should be inlined as "lock cmpxchg16b" 350 | while (!__sync_bool_compare_and_swap(&tail.int128, tail.int128, new_val.int128)) 351 | _mm_pause(); 352 | } 353 | 354 | std::pair get_num_entries_data_length_atomic() 355 | { 356 | // Int128Union new_val, cur_val; 357 | // should be inlined as "lock cmpxchg16b" 358 | // cur_val.int128 = __sync_val_compare_and_swap(&tail.int128, new_val.int128, tail.int128); 359 | Int128Union cur_val; 360 | cur_val.m128i = _mm_load_si128(&tail.m128i); 361 | return std::make_pair(cur_val.data.num_entries, cur_val.data.data_length); 362 | } 363 | 364 | void 365 | fill(order_t order, vertex_t vid, timestamp_t creation_time, uintptr_t prev_pointer, timestamp_t committed_time) 366 | { 367 | N2OBlockHeader::fill(order, Type::EDGE, vid, creation_time, prev_pointer); 368 | set_committed_time(committed_time); 369 | clear(); 370 | } 371 | 372 | constexpr static order_t BLOOM_FILTER_THRESHOLD = 10; 373 | constexpr static order_t BLOOM_FILTER_PORTION = 4; 374 | 375 | private: 376 | timestamp_t committed_time; 377 | union alignas(16) Int128Union { 378 | struct 379 | { 380 | size_t num_entries; 381 | size_t data_length; 382 | } data; 383 | __int128 int128; 384 | __m128i m128i; 385 | } tail; 386 | char data[0]; 387 | }; 388 | 389 | static_assert(sizeof(BlockHeader) == 2); 390 | static_assert(sizeof(N2OBlockHeader) == 24); 391 | static_assert(sizeof(VertexBlockHeader) == 32); 392 | static_assert(sizeof(EdgeLabelEntry) == 16); 393 | static_assert(sizeof(EdgeLabelBlockHeader) == 32); 394 | static_assert(sizeof(EdgeEntry) == 24); 395 | static_assert(sizeof(EdgeBlockHeader) == 48); 396 | } // namespace livegraph 397 | -------------------------------------------------------------------------------- /cmake/FindTBB.cmake: -------------------------------------------------------------------------------- 1 | # - Find ThreadingBuildingBlocks include dirs and libraries 2 | # Use this module by invoking find_package with the form: 3 | # find_package(TBB 4 | # [REQUIRED] # Fail with error if TBB is not found 5 | # ) # 6 | # Once done, this will define 7 | # 8 | # TBB_FOUND - system has TBB 9 | # TBB_INCLUDE_DIRS - the TBB include directories 10 | # TBB_LIBRARIES - TBB libraries to be lined, doesn't include malloc or 11 | # malloc proxy 12 | # TBB::tbb - imported target for the TBB library 13 | # 14 | # TBB_VERSION_MAJOR - Major Product Version Number 15 | # TBB_VERSION_MINOR - Minor Product Version Number 16 | # TBB_INTERFACE_VERSION - Engineering Focused Version Number 17 | # TBB_COMPATIBLE_INTERFACE_VERSION - The oldest major interface version 18 | # still supported. This uses the engineering 19 | # focused interface version numbers. 20 | # 21 | # TBB_MALLOC_FOUND - system has TBB malloc library 22 | # TBB_MALLOC_INCLUDE_DIRS - the TBB malloc include directories 23 | # TBB_MALLOC_LIBRARIES - The TBB malloc libraries to be lined 24 | # TBB::malloc - imported target for the TBB malloc library 25 | # 26 | # TBB_MALLOC_PROXY_FOUND - system has TBB malloc proxy library 27 | # TBB_MALLOC_PROXY_INCLUDE_DIRS = the TBB malloc proxy include directories 28 | # TBB_MALLOC_PROXY_LIBRARIES - The TBB malloc proxy libraries to be lined 29 | # TBB::malloc_proxy - imported target for the TBB malloc proxy library 30 | # 31 | # 32 | # This module reads hints about search locations from variables: 33 | # ENV TBB_ARCH_PLATFORM - for eg. set it to "mic" for Xeon Phi builds 34 | # ENV TBB_ROOT or just TBB_ROOT - root directory of tbb installation 35 | # ENV TBB_BUILD_PREFIX - specifies the build prefix for user built tbb 36 | # libraries. Should be specified with ENV TBB_ROOT 37 | # and optionally... 38 | # ENV TBB_BUILD_DIR - if build directory is different than ${TBB_ROOT}/build 39 | # 40 | # 41 | # Modified by Robert Maynard from the original OGRE source 42 | # 43 | #------------------------------------------------------------------- 44 | # This file is part of the CMake build system for OGRE 45 | # (Object-oriented Graphics Rendering Engine) 46 | # For the latest info, see http://www.ogre3d.org/ 47 | # 48 | # The contents of this file are placed in the public domain. Feel 49 | # free to make use of it in any way you like. 50 | #------------------------------------------------------------------- 51 | # 52 | #============================================================================= 53 | # Copyright 2010-2012 Kitware, Inc. 54 | # Copyright 2012 Rolf Eike Beer 55 | # 56 | # Distributed under the OSI-approved BSD License (the "License"); 57 | # see accompanying file Copyright.txt for details. 58 | # 59 | # This software is distributed WITHOUT ANY WARRANTY; without even the 60 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 61 | # See the License for more information. 62 | #============================================================================= 63 | # (To distribute this file outside of CMake, substitute the full 64 | # License text for the above reference.) 65 | 66 | 67 | #============================================================================= 68 | # FindTBB helper functions and macros 69 | # 70 | 71 | #==================================================== 72 | # Fix the library path in case it is a linker script 73 | #==================================================== 74 | function(tbb_extract_real_library library real_library) 75 | if(NOT UNIX OR NOT EXISTS ${library}) 76 | set(${real_library} "${library}" PARENT_SCOPE) 77 | return() 78 | endif() 79 | 80 | #Read in the first 4 bytes and see if they are the ELF magic number 81 | set(_elf_magic "7f454c46") 82 | file(READ ${library} _hex_data OFFSET 0 LIMIT 4 HEX) 83 | if(_hex_data STREQUAL _elf_magic) 84 | #we have opened a elf binary so this is what 85 | #we should link to 86 | set(${real_library} "${library}" PARENT_SCOPE) 87 | return() 88 | endif() 89 | 90 | file(READ ${library} _data OFFSET 0 LIMIT 1024) 91 | if("${_data}" MATCHES "INPUT \\(([^(]+)\\)") 92 | #extract out the .so name from REGEX MATCH command 93 | set(_proper_so_name "${CMAKE_MATCH_1}") 94 | 95 | #construct path to the real .so which is presumed to be in the same directory 96 | #as the input file 97 | get_filename_component(_so_dir "${library}" DIRECTORY) 98 | set(${real_library} "${_so_dir}/${_proper_so_name}" PARENT_SCOPE) 99 | else() 100 | #unable to determine what this library is so just hope everything works 101 | #and pass it unmodified. 102 | set(${real_library} "${library}" PARENT_SCOPE) 103 | endif() 104 | endfunction() 105 | 106 | #=============================================== 107 | # Do the final processing for the package find. 108 | #=============================================== 109 | macro(findpkg_finish PREFIX TARGET_NAME) 110 | if (${PREFIX}_INCLUDE_DIR AND ${PREFIX}_LIBRARY) 111 | set(${PREFIX}_FOUND TRUE) 112 | set (${PREFIX}_INCLUDE_DIRS ${${PREFIX}_INCLUDE_DIR}) 113 | set (${PREFIX}_LIBRARIES ${${PREFIX}_LIBRARY}) 114 | else () 115 | if (${PREFIX}_FIND_REQUIRED AND NOT ${PREFIX}_FIND_QUIETLY) 116 | message(FATAL_ERROR "Required library ${PREFIX} not found.") 117 | endif () 118 | endif () 119 | 120 | if (NOT TARGET "TBB::${TARGET_NAME}") 121 | if (${PREFIX}_LIBRARY_RELEASE) 122 | tbb_extract_real_library(${${PREFIX}_LIBRARY_RELEASE} real_release) 123 | endif () 124 | if (${PREFIX}_LIBRARY_DEBUG) 125 | tbb_extract_real_library(${${PREFIX}_LIBRARY_DEBUG} real_debug) 126 | endif () 127 | add_library(TBB::${TARGET_NAME} UNKNOWN IMPORTED) 128 | set_target_properties(TBB::${TARGET_NAME} PROPERTIES 129 | INTERFACE_INCLUDE_DIRECTORIES "${${PREFIX}_INCLUDE_DIR}") 130 | if (${PREFIX}_LIBRARY_DEBUG AND ${PREFIX}_LIBRARY_RELEASE) 131 | set_target_properties(TBB::${TARGET_NAME} PROPERTIES 132 | IMPORTED_LOCATION "${real_release}" 133 | IMPORTED_LOCATION_DEBUG "${real_debug}" 134 | IMPORTED_LOCATION_RELEASE "${real_release}") 135 | elseif (${PREFIX}_LIBRARY_RELEASE) 136 | set_target_properties(TBB::${TARGET_NAME} PROPERTIES 137 | IMPORTED_LOCATION "${real_release}") 138 | elseif (${PREFIX}_LIBRARY_DEBUG) 139 | set_target_properties(TBB::${TARGET_NAME} PROPERTIES 140 | IMPORTED_LOCATION "${real_debug}") 141 | endif () 142 | endif () 143 | 144 | #mark the following variables as internal variables 145 | mark_as_advanced(${PREFIX}_INCLUDE_DIR 146 | ${PREFIX}_LIBRARY 147 | ${PREFIX}_LIBRARY_DEBUG 148 | ${PREFIX}_LIBRARY_RELEASE) 149 | endmacro() 150 | 151 | #=============================================== 152 | # Generate debug names from given release names 153 | #=============================================== 154 | macro(get_debug_names PREFIX) 155 | foreach(i ${${PREFIX}}) 156 | set(${PREFIX}_DEBUG ${${PREFIX}_DEBUG} ${i}d ${i}D ${i}_d ${i}_D ${i}_debug ${i}) 157 | endforeach() 158 | endmacro() 159 | 160 | #=============================================== 161 | # See if we have env vars to help us find tbb 162 | #=============================================== 163 | macro(getenv_path VAR) 164 | set(ENV_${VAR} $ENV{${VAR}}) 165 | # replace won't work if var is blank 166 | if (ENV_${VAR}) 167 | string( REGEX REPLACE "\\\\" "/" ENV_${VAR} ${ENV_${VAR}} ) 168 | endif () 169 | endmacro() 170 | 171 | #=============================================== 172 | # Couple a set of release AND debug libraries 173 | #=============================================== 174 | macro(make_library_set PREFIX) 175 | if (${PREFIX}_RELEASE AND ${PREFIX}_DEBUG) 176 | set(${PREFIX} optimized ${${PREFIX}_RELEASE} debug ${${PREFIX}_DEBUG}) 177 | elseif (${PREFIX}_RELEASE) 178 | set(${PREFIX} ${${PREFIX}_RELEASE}) 179 | elseif (${PREFIX}_DEBUG) 180 | set(${PREFIX} ${${PREFIX}_DEBUG}) 181 | endif () 182 | endmacro() 183 | 184 | 185 | #============================================================================= 186 | # Now to actually find TBB 187 | # 188 | 189 | # Get path, convert backslashes as ${ENV_${var}} 190 | getenv_path(TBB_ROOT) 191 | 192 | # initialize search paths 193 | set(TBB_PREFIX_PATH ${TBB_ROOT} ${ENV_TBB_ROOT}) 194 | set(TBB_INC_SEARCH_PATH "") 195 | set(TBB_LIB_SEARCH_PATH "") 196 | 197 | 198 | # If user built from sources 199 | set(TBB_BUILD_PREFIX $ENV{TBB_BUILD_PREFIX}) 200 | if (TBB_BUILD_PREFIX AND ENV_TBB_ROOT) 201 | getenv_path(TBB_BUILD_DIR) 202 | if (NOT ENV_TBB_BUILD_DIR) 203 | set(ENV_TBB_BUILD_DIR ${ENV_TBB_ROOT}/build) 204 | endif () 205 | 206 | # include directory under ${ENV_TBB_ROOT}/include 207 | list(APPEND TBB_LIB_SEARCH_PATH 208 | ${ENV_TBB_BUILD_DIR}/${TBB_BUILD_PREFIX}_release 209 | ${ENV_TBB_BUILD_DIR}/${TBB_BUILD_PREFIX}_debug) 210 | endif () 211 | 212 | 213 | # For Windows, let's assume that the user might be using the precompiled 214 | # TBB packages from the main website. These use a rather awkward directory 215 | # structure (at least for automatically finding the right files) depending 216 | # on platform and compiler, but we'll do our best to accommodate it. 217 | # Not adding the same effort for the precompiled linux builds, though. Those 218 | # have different versions for CC compiler versions and linux kernels which 219 | # will never adequately match the user's setup, so there is no feasible way 220 | # to detect the "best" version to use. The user will have to manually 221 | # select the right files. (Chances are the distributions are shipping their 222 | # custom version of tbb, anyway, so the problem is probably nonexistent.) 223 | if (WIN32 AND MSVC) 224 | set(COMPILER_PREFIX "vc7.1") 225 | if (MSVC_VERSION EQUAL 1400) 226 | set(COMPILER_PREFIX "vc8") 227 | elseif(MSVC_VERSION EQUAL 1500) 228 | set(COMPILER_PREFIX "vc9") 229 | elseif(MSVC_VERSION EQUAL 1600) 230 | set(COMPILER_PREFIX "vc10") 231 | elseif(MSVC_VERSION EQUAL 1700) 232 | set(COMPILER_PREFIX "vc11") 233 | elseif(MSVC_VERSION EQUAL 1800) 234 | set(COMPILER_PREFIX "vc12") 235 | elseif(MSVC_VERSION EQUAL 1900) 236 | set(COMPILER_PREFIX "vc14") 237 | endif () 238 | 239 | # for each prefix path, add ia32/64\${COMPILER_PREFIX}\lib to the lib search path 240 | foreach (dir IN LISTS TBB_PREFIX_PATH) 241 | if (CMAKE_CL_64) 242 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/ia64/${COMPILER_PREFIX}/lib) 243 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/ia64/${COMPILER_PREFIX}) 244 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/intel64/${COMPILER_PREFIX}/lib) 245 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/intel64/${COMPILER_PREFIX}) 246 | else () 247 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/ia32/${COMPILER_PREFIX}/lib) 248 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/ia32/${COMPILER_PREFIX}) 249 | endif () 250 | endforeach () 251 | endif () 252 | 253 | # For OS X binary distribution, choose libc++ based libraries for Mavericks (10.9) 254 | # and above and AppleClang 255 | if (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND 256 | NOT CMAKE_SYSTEM_VERSION VERSION_LESS 13.0) 257 | set (USE_LIBCXX OFF) 258 | cmake_policy(GET CMP0025 POLICY_VAR) 259 | 260 | if (POLICY_VAR STREQUAL "NEW") 261 | if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") 262 | set (USE_LIBCXX ON) 263 | endif () 264 | else () 265 | if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 266 | set (USE_LIBCXX ON) 267 | endif () 268 | endif () 269 | 270 | if (USE_LIBCXX) 271 | foreach (dir IN LISTS TBB_PREFIX_PATH) 272 | list (APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/libc++ ${dir}/libc++/lib) 273 | endforeach () 274 | endif () 275 | endif () 276 | 277 | # check compiler ABI 278 | if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 279 | set(COMPILER_PREFIX) 280 | if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.7) 281 | list(APPEND COMPILER_PREFIX "gcc4.7") 282 | endif() 283 | if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.4) 284 | list(APPEND COMPILER_PREFIX "gcc4.4") 285 | endif() 286 | list(APPEND COMPILER_PREFIX "gcc4.1") 287 | elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") 288 | set(COMPILER_PREFIX) 289 | if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6) 290 | list(APPEND COMPILER_PREFIX "gcc4.7") 291 | endif() 292 | list(APPEND COMPILER_PREFIX "gcc4.4") 293 | else() # Assume compatibility with 4.4 for other compilers 294 | list(APPEND COMPILER_PREFIX "gcc4.4") 295 | endif () 296 | 297 | # if platform architecture is explicitly specified 298 | set(TBB_ARCH_PLATFORM $ENV{TBB_ARCH_PLATFORM}) 299 | if (TBB_ARCH_PLATFORM) 300 | foreach (dir IN LISTS TBB_PREFIX_PATH) 301 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/${TBB_ARCH_PLATFORM}/lib) 302 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/${TBB_ARCH_PLATFORM}) 303 | endforeach () 304 | endif () 305 | 306 | foreach (dir IN LISTS TBB_PREFIX_PATH) 307 | foreach (prefix IN LISTS COMPILER_PREFIX) 308 | if (CMAKE_SIZEOF_VOID_P EQUAL 8) 309 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/intel64) 310 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/intel64/${prefix}) 311 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/intel64/lib) 312 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/intel64/${prefix}/lib) 313 | else () 314 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/ia32) 315 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib/ia32/${prefix}) 316 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/ia32/lib) 317 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/ia32/${prefix}/lib) 318 | endif () 319 | endforeach() 320 | endforeach () 321 | 322 | # add general search paths 323 | foreach (dir IN LISTS TBB_PREFIX_PATH) 324 | list(APPEND TBB_LIB_SEARCH_PATH ${dir}/lib ${dir}/Lib ${dir}/lib/tbb 325 | ${dir}/Libs) 326 | list(APPEND TBB_INC_SEARCH_PATH ${dir}/include ${dir}/Include 327 | ${dir}/include/tbb) 328 | endforeach () 329 | 330 | set(TBB_LIBRARY_NAMES tbb) 331 | get_debug_names(TBB_LIBRARY_NAMES) 332 | 333 | 334 | find_path(TBB_INCLUDE_DIR 335 | NAMES tbb/tbb.h 336 | PATHS ${TBB_INC_SEARCH_PATH}) 337 | 338 | find_library(TBB_LIBRARY_RELEASE 339 | NAMES ${TBB_LIBRARY_NAMES} 340 | PATHS ${TBB_LIB_SEARCH_PATH}) 341 | find_library(TBB_LIBRARY_DEBUG 342 | NAMES ${TBB_LIBRARY_NAMES_DEBUG} 343 | PATHS ${TBB_LIB_SEARCH_PATH}) 344 | make_library_set(TBB_LIBRARY) 345 | 346 | findpkg_finish(TBB tbb) 347 | 348 | #if we haven't found TBB no point on going any further 349 | if (NOT TBB_FOUND) 350 | return() 351 | endif () 352 | 353 | #============================================================================= 354 | # Look for TBB's malloc package 355 | set(TBB_MALLOC_LIBRARY_NAMES tbbmalloc) 356 | get_debug_names(TBB_MALLOC_LIBRARY_NAMES) 357 | 358 | find_path(TBB_MALLOC_INCLUDE_DIR 359 | NAMES tbb/tbb.h 360 | PATHS ${TBB_INC_SEARCH_PATH}) 361 | 362 | find_library(TBB_MALLOC_LIBRARY_RELEASE 363 | NAMES ${TBB_MALLOC_LIBRARY_NAMES} 364 | PATHS ${TBB_LIB_SEARCH_PATH}) 365 | find_library(TBB_MALLOC_LIBRARY_DEBUG 366 | NAMES ${TBB_MALLOC_LIBRARY_NAMES_DEBUG} 367 | PATHS ${TBB_LIB_SEARCH_PATH}) 368 | make_library_set(TBB_MALLOC_LIBRARY) 369 | 370 | findpkg_finish(TBB_MALLOC tbbmalloc) 371 | 372 | #============================================================================= 373 | # Look for TBB's malloc proxy package 374 | set(TBB_MALLOC_PROXY_LIBRARY_NAMES tbbmalloc_proxy) 375 | get_debug_names(TBB_MALLOC_PROXY_LIBRARY_NAMES) 376 | 377 | find_path(TBB_MALLOC_PROXY_INCLUDE_DIR 378 | NAMES tbb/tbbmalloc_proxy.h 379 | PATHS ${TBB_INC_SEARCH_PATH}) 380 | 381 | find_library(TBB_MALLOC_PROXY_LIBRARY_RELEASE 382 | NAMES ${TBB_MALLOC_PROXY_LIBRARY_NAMES} 383 | PATHS ${TBB_LIB_SEARCH_PATH}) 384 | find_library(TBB_MALLOC_PROXY_LIBRARY_DEBUG 385 | NAMES ${TBB_MALLOC_PROXY_LIBRARY_NAMES_DEBUG} 386 | PATHS ${TBB_LIB_SEARCH_PATH}) 387 | make_library_set(TBB_MALLOC_PROXY_LIBRARY) 388 | 389 | findpkg_finish(TBB_MALLOC_PROXY tbbmalloc_proxy) 390 | 391 | 392 | #============================================================================= 393 | #parse all the version numbers from tbb 394 | if(NOT TBB_VERSION) 395 | 396 | #only read the start of the file 397 | file(STRINGS 398 | "${TBB_INCLUDE_DIR}/tbb/tbb_stddef.h" 399 | TBB_VERSION_CONTENTS 400 | REGEX "VERSION") 401 | 402 | string(REGEX REPLACE 403 | ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" 404 | TBB_VERSION_MAJOR "${TBB_VERSION_CONTENTS}") 405 | 406 | string(REGEX REPLACE 407 | ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" 408 | TBB_VERSION_MINOR "${TBB_VERSION_CONTENTS}") 409 | 410 | string(REGEX REPLACE 411 | ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" 412 | TBB_INTERFACE_VERSION "${TBB_VERSION_CONTENTS}") 413 | 414 | string(REGEX REPLACE 415 | ".*#define TBB_COMPATIBLE_INTERFACE_VERSION ([0-9]+).*" "\\1" 416 | TBB_COMPATIBLE_INTERFACE_VERSION "${TBB_VERSION_CONTENTS}") 417 | 418 | endif() 419 | -------------------------------------------------------------------------------- /test/blocks.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | #include "core/blocks.hpp" 21 | 22 | using namespace livegraph; 23 | 24 | TEST_CASE("testing the BlockHeader") 25 | { 26 | BlockHeader header; 27 | 28 | header.set_order(5); 29 | header.set_type(BlockHeader::Type::SPECIAL); 30 | CHECK(header.get_type() == BlockHeader::Type::SPECIAL); 31 | CHECK(header.get_order() == 5); 32 | CHECK(header.get_block_size() == 1ul << 5); 33 | } 34 | 35 | TEST_CASE("testing the N2OBlockHeader") 36 | { 37 | N2OBlockHeader header; 38 | 39 | header.set_order(5); 40 | header.set_type(N2OBlockHeader::Type::SPECIAL); 41 | 42 | header.set_vertex_id(233); 43 | CHECK(header.get_vertex_id() == 233); 44 | header.set_vertex_id(0xffffffff); 45 | CHECK(header.get_vertex_id() == 0xffffffff); 46 | header.set_vertex_id(0x1234abcdeful); 47 | CHECK(header.get_vertex_id() == 0x1234abcdef); 48 | 49 | header.set_creation_time(0xaabbccddeefful); 50 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 51 | auto creation_time_pointer = header.get_creation_time_pointer(); 52 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 53 | *creation_time_pointer = 0xffeeddccbbaal; 54 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 55 | 56 | header.set_prev_pointer(0x1122334455667788ul); 57 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 58 | 59 | CHECK(header.get_type() == N2OBlockHeader::Type::SPECIAL); 60 | CHECK(header.get_order() == 5); 61 | CHECK(header.get_block_size() == 1ul << 5); 62 | CHECK(header.get_vertex_id() == 0x1234abcdef); 63 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 64 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 65 | } 66 | 67 | TEST_CASE("testing the VertexBlockHeader") 68 | { 69 | const order_t log_size = 6; // size = 64 bytes, left 24 bytes 70 | auto buf = malloc(1ul << log_size); 71 | VertexBlockHeader &header = *(VertexBlockHeader *)buf; 72 | 73 | header.set_order(log_size); 74 | header.set_type(VertexBlockHeader::Type::VERTEX); 75 | 76 | header.set_vertex_id(233); 77 | CHECK(header.get_vertex_id() == 233); 78 | header.set_vertex_id(0xffffffff); 79 | CHECK(header.get_vertex_id() == 0xffffffff); 80 | header.set_vertex_id(0x1234abcdeful); 81 | CHECK(header.get_vertex_id() == 0x1234abcdef); 82 | 83 | header.set_creation_time(0xaabbccddeefful); 84 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 85 | auto creation_time_pointer = header.get_creation_time_pointer(); 86 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 87 | *creation_time_pointer = 0xffeeddccbbaal; 88 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 89 | 90 | header.set_prev_pointer(0x1122334455667788ul); 91 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 92 | 93 | const char *str = "Hello World!"; 94 | CHECK(header.set_data(str, strlen(str))); 95 | CHECK(header.get_length() == strlen(str)); 96 | CHECK(std::string(header.get_data(), header.get_length()) == str); 97 | CHECK(!header.set_data(str, (1ul << log_size) - sizeof(header) + 1)); 98 | CHECK(std::string(header.get_data(), header.get_length()) == str); 99 | 100 | CHECK(header.get_type() == VertexBlockHeader::Type::VERTEX); 101 | CHECK(header.get_order() == log_size); 102 | CHECK(header.get_block_size() == 1ul << log_size); 103 | CHECK(header.get_vertex_id() == 0x1234abcdef); 104 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 105 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 106 | CHECK(header.get_length() == strlen(str)); 107 | CHECK(std::string(header.get_data(), header.get_length()) == str); 108 | 109 | free(buf); 110 | } 111 | 112 | TEST_CASE("testing the EdgeLabelEntry") 113 | { 114 | EdgeLabelEntry entry; 115 | 116 | entry.set_label(1234); 117 | entry.set_pointer(0x123123321321ul); 118 | CHECK(entry.get_label() == 1234); 119 | CHECK(entry.get_pointer() == 0x123123321321ul); 120 | } 121 | 122 | TEST_CASE("testing the EdgeLabelBlockHeader") 123 | { 124 | const order_t log_size = 6; // size = 64 bytes, left 32 bytes (2 entry) 125 | auto buf = malloc(1ul << log_size); 126 | EdgeLabelBlockHeader &header = *(EdgeLabelBlockHeader *)buf; 127 | 128 | header.set_order(log_size); 129 | header.set_type(EdgeLabelBlockHeader::Type::EDGE_LABEL); 130 | 131 | header.set_vertex_id(233); 132 | CHECK(header.get_vertex_id() == 233); 133 | header.set_vertex_id(0xffffffff); 134 | CHECK(header.get_vertex_id() == 0xffffffff); 135 | header.set_vertex_id(0x1234abcdeful); 136 | CHECK(header.get_vertex_id() == 0x1234abcdef); 137 | 138 | header.set_creation_time(0xaabbccddeefful); 139 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 140 | auto creation_time_pointer = header.get_creation_time_pointer(); 141 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 142 | *creation_time_pointer = 0xffeeddccbbaal; 143 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 144 | 145 | header.set_prev_pointer(0x1122334455667788ul); 146 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 147 | 148 | header.clear(); 149 | EdgeLabelEntry entry; 150 | entry.set_label(1111); 151 | entry.set_pointer(0x1111111111111111ul); 152 | CHECK(header.append(entry)); 153 | entry.set_label(2222); 154 | entry.set_pointer(0x2222222222222222ul); 155 | CHECK(header.append(entry)); 156 | entry.set_label(3333); 157 | entry.set_pointer(0x3333333333333333ul); 158 | CHECK(!header.append(entry)); 159 | CHECK(header.get_num_entries() == 2); 160 | CHECK(header.get_entries()[0].get_label() == 1111); 161 | CHECK(header.get_entries()[0].get_pointer() == 0x1111111111111111ul); 162 | CHECK(header.get_entries()[1].get_label() == 2222); 163 | CHECK(header.get_entries()[1].get_pointer() == 0x2222222222222222ul); 164 | 165 | CHECK(header.get_type() == EdgeLabelBlockHeader::Type::EDGE_LABEL); 166 | CHECK(header.get_order() == log_size); 167 | CHECK(header.get_block_size() == 1ul << log_size); 168 | CHECK(header.get_vertex_id() == 0x1234abcdef); 169 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 170 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 171 | 172 | free(buf); 173 | } 174 | 175 | TEST_CASE("testing the EdgeEntry") 176 | { 177 | EdgeEntry entry; 178 | entry.set_dst(233); 179 | CHECK(entry.get_dst() == 233); 180 | entry.set_dst(0xffffffff); 181 | CHECK(entry.get_dst() == 0xffffffff); 182 | entry.set_dst(0x1234abcdeful); 183 | CHECK(entry.get_dst() == 0x1234abcdef); 184 | 185 | entry.set_creation_time(0xaabbccddeefful); 186 | CHECK(entry.get_creation_time() == 0xaabbccddeeffl); 187 | auto creation_time_pointer = entry.get_creation_time_pointer(); 188 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 189 | *creation_time_pointer = 0xffeeddccbbaal; 190 | CHECK(entry.get_creation_time() == 0xffeeddccbbaal); 191 | 192 | entry.set_deletion_time(0xabcdefabcdeful); 193 | CHECK(entry.get_deletion_time() == 0xabcdefabcdeful); 194 | auto deletion_time_pointer = entry.get_deletion_time_pointer(); 195 | CHECK(*deletion_time_pointer == 0xabcdefabcdeful); 196 | *deletion_time_pointer = 0x1234432112344321l; 197 | CHECK(entry.get_deletion_time() == 0x1234432112344321l); 198 | 199 | entry.set_length(0x1122ul); 200 | CHECK(entry.get_length() == 0x1122ul); 201 | 202 | CHECK(entry.get_dst() == 0x1234abcdef); 203 | CHECK(entry.get_creation_time() == 0xffeeddccbbaal); 204 | CHECK(entry.get_deletion_time() == 0x1234432112344321l); 205 | CHECK(entry.get_length() == 0x1122ul); 206 | } 207 | 208 | TEST_CASE("testing the EdgeLabelBlockHeader") 209 | { 210 | const order_t log_size = 6; // size = 64 bytes, left 32 bytes 211 | auto buf = malloc(1ul << log_size); 212 | EdgeLabelBlockHeader &header = *(EdgeLabelBlockHeader *)buf; 213 | 214 | header.set_order(log_size); 215 | header.set_type(EdgeLabelBlockHeader::Type::EDGE_LABEL); 216 | 217 | header.set_vertex_id(233); 218 | CHECK(header.get_vertex_id() == 233); 219 | header.set_vertex_id(0xffffffff); 220 | CHECK(header.get_vertex_id() == 0xffffffff); 221 | header.set_vertex_id(0x1234abcdeful); 222 | CHECK(header.get_vertex_id() == 0x1234abcdef); 223 | 224 | header.set_creation_time(0xaabbccddeefful); 225 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 226 | auto creation_time_pointer = header.get_creation_time_pointer(); 227 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 228 | *creation_time_pointer = 0xffeeddccbbaal; 229 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 230 | 231 | header.set_prev_pointer(0x1122334455667788ul); 232 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 233 | 234 | header.clear(); 235 | EdgeLabelEntry entry; 236 | entry.set_label(1111); 237 | entry.set_pointer(0x1111111111111111ul); 238 | CHECK(header.append(entry)); 239 | entry.set_label(2222); 240 | entry.set_pointer(0x2222222222222222ul); 241 | CHECK(header.append(entry)); 242 | entry.set_label(3333); 243 | entry.set_pointer(0x3333333333333333ul); 244 | CHECK(!header.append(entry)); 245 | CHECK(header.get_num_entries() == 2); 246 | CHECK(header.get_entries()[0].get_label() == 1111); 247 | CHECK(header.get_entries()[0].get_pointer() == 0x1111111111111111ul); 248 | CHECK(header.get_entries()[1].get_label() == 2222); 249 | CHECK(header.get_entries()[1].get_pointer() == 0x2222222222222222ul); 250 | 251 | CHECK(header.get_type() == EdgeLabelBlockHeader::Type::EDGE_LABEL); 252 | CHECK(header.get_order() == log_size); 253 | CHECK(header.get_block_size() == 1ul << log_size); 254 | CHECK(header.get_vertex_id() == 0x1234abcdef); 255 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 256 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 257 | 258 | free(buf); 259 | } 260 | 261 | TEST_CASE("testing the EdgeEntry") 262 | { 263 | EdgeEntry entry; 264 | entry.set_dst(233); 265 | CHECK(entry.get_dst() == 233); 266 | entry.set_dst(0xffffffff); 267 | CHECK(entry.get_dst() == 0xffffffff); 268 | entry.set_dst(0x1234abcdeful); 269 | CHECK(entry.get_dst() == 0x1234abcdef); 270 | 271 | entry.set_creation_time(0xaabbccddeefful); 272 | CHECK(entry.get_creation_time() == 0xaabbccddeeffl); 273 | auto creation_time_pointer = entry.get_creation_time_pointer(); 274 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 275 | *creation_time_pointer = 0xffeeddccbbaal; 276 | CHECK(entry.get_creation_time() == 0xffeeddccbbaal); 277 | 278 | entry.set_deletion_time(0xabcdefabcdeful); 279 | CHECK(entry.get_deletion_time() == 0xabcdefabcdeful); 280 | auto deletion_time_pointer = entry.get_deletion_time_pointer(); 281 | CHECK(*deletion_time_pointer == 0xabcdefabcdeful); 282 | *deletion_time_pointer = 0x1234432112344321l; 283 | CHECK(entry.get_deletion_time() == 0x1234432112344321l); 284 | 285 | entry.set_length(0x1122ul); 286 | CHECK(entry.get_length() == 0x1122ul); 287 | 288 | CHECK(entry.get_dst() == 0x1234abcdef); 289 | CHECK(entry.get_creation_time() == 0xffeeddccbbaal); 290 | CHECK(entry.get_deletion_time() == 0x1234432112344321l); 291 | CHECK(entry.get_length() == 0x1122ul); 292 | } 293 | 294 | TEST_CASE("testing the EdgeBlockHeader") 295 | { 296 | SUBCASE("EdgeBlockHeader without BloomFilter") 297 | { 298 | const order_t log_size = 7; // size = 128 bytes, left 80 bytes, 3 edges (2 bytes data) 299 | auto buf = malloc(1ul << log_size); 300 | EdgeBlockHeader &header = *(EdgeBlockHeader *)buf; 301 | 302 | header.set_order(log_size); 303 | header.set_type(EdgeLabelBlockHeader::Type::EDGE); 304 | 305 | CHECK(!header.get_bloom_filter().valid()); 306 | 307 | header.set_vertex_id(233); 308 | CHECK(header.get_vertex_id() == 233); 309 | header.set_vertex_id(0xffffffff); 310 | CHECK(header.get_vertex_id() == 0xffffffff); 311 | header.set_vertex_id(0x1234abcdeful); 312 | CHECK(header.get_vertex_id() == 0x1234abcdef); 313 | 314 | header.set_creation_time(0xaabbccddeefful); 315 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 316 | auto creation_time_pointer = header.get_creation_time_pointer(); 317 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 318 | *creation_time_pointer = 0xffeeddccbbaal; 319 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 320 | 321 | header.set_prev_pointer(0x1122334455667788ul); 322 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 323 | 324 | header.set_committed_time(0xabcdefabcdeful); 325 | CHECK(header.get_committed_time() == 0xabcdefabcdeful); 326 | auto committed_time_pointer = header.get_committed_time_pointer(); 327 | CHECK(*committed_time_pointer == 0xabcdefabcdeful); 328 | *committed_time_pointer = 0x1234432112344321l; 329 | CHECK(header.get_committed_time() == 0x1234432112344321l); 330 | 331 | header.clear(); 332 | for (int i = 0; i < 4; i++) 333 | { 334 | std::string data = std::to_string(i) + "."; 335 | EdgeEntry entry; 336 | entry.set_creation_time(i); 337 | entry.set_deletion_time(-i); 338 | entry.set_dst(i * 1234); 339 | entry.set_length(data.size()); 340 | CHECK((bool)header.append(entry, data.c_str()) == (i < 3)); 341 | } 342 | CHECK(header.get_num_entries() == 3); 343 | CHECK(header.get_data_length() == 6); 344 | 345 | auto data = header.get_data(); 346 | auto entries = header.get_entries(); 347 | for (int i = 0; i < 3; i++) 348 | { 349 | std::string ref = std::to_string(i) + "."; 350 | entries--; 351 | CHECK(entries->get_creation_time() == i); 352 | CHECK(entries->get_deletion_time() == -i); 353 | CHECK(entries->get_dst() == i * 1234); 354 | CHECK(entries->get_length() == ref.size()); 355 | CHECK(std::string(data, entries->get_length()) == ref); 356 | data += entries->get_length(); 357 | } 358 | 359 | CHECK(header.get_type() == EdgeLabelBlockHeader::Type::EDGE); 360 | CHECK(header.get_order() == log_size); 361 | CHECK(header.get_block_size() == 1ul << log_size); 362 | CHECK(header.get_vertex_id() == 0x1234abcdef); 363 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 364 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 365 | CHECK(header.get_committed_time() == 0x1234432112344321l); 366 | 367 | free(buf); 368 | } 369 | 370 | SUBCASE("EdgeBlockHeader with BloomFilter") 371 | { 372 | const order_t log_size = 10; // size = 1024 bytes, left 912 bytes, 31 edges (5 bytes data) 373 | auto buf = aligned_alloc(32, 1ul << log_size); 374 | EdgeBlockHeader &header = *(EdgeBlockHeader *)buf; 375 | 376 | header.set_order(log_size); 377 | header.set_type(EdgeLabelBlockHeader::Type::EDGE); 378 | 379 | CHECK(header.get_bloom_filter().valid()); 380 | 381 | header.set_vertex_id(233); 382 | CHECK(header.get_vertex_id() == 233); 383 | header.set_vertex_id(0xffffffff); 384 | CHECK(header.get_vertex_id() == 0xffffffff); 385 | header.set_vertex_id(0x1234abcdeful); 386 | CHECK(header.get_vertex_id() == 0x1234abcdef); 387 | 388 | header.set_creation_time(0xaabbccddeefful); 389 | CHECK(header.get_creation_time() == 0xaabbccddeeffl); 390 | auto creation_time_pointer = header.get_creation_time_pointer(); 391 | CHECK(*creation_time_pointer == 0xaabbccddeeffl); 392 | *creation_time_pointer = 0xffeeddccbbaal; 393 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 394 | 395 | header.set_prev_pointer(0x1122334455667788ul); 396 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 397 | 398 | header.set_committed_time(0xabcdefabcdeful); 399 | CHECK(header.get_committed_time() == 0xabcdefabcdeful); 400 | auto committed_time_pointer = header.get_committed_time_pointer(); 401 | CHECK(*committed_time_pointer == 0xabcdefabcdeful); 402 | *committed_time_pointer = 0x1234432112344321l; 403 | CHECK(header.get_committed_time() == 0x1234432112344321l); 404 | 405 | auto filter = header.get_bloom_filter(); 406 | header.clear(); 407 | for (int i = 0; i < 32; i++) 408 | { 409 | std::string data = std::to_string(i); 410 | while (data.size() < 5) 411 | data += "."; 412 | EdgeEntry entry; 413 | entry.set_creation_time(i); 414 | entry.set_deletion_time(-i); 415 | entry.set_dst(i * 1234); 416 | entry.set_length(data.size()); 417 | if (i % 2) 418 | { 419 | CHECK((bool)header.append(entry, data.c_str(), filter) == (i < 31)); 420 | } 421 | else 422 | { 423 | CHECK((bool)header.append(entry, data.c_str()) == (i < 31)); 424 | } 425 | } 426 | CHECK(header.get_num_entries() == 31); 427 | CHECK(header.get_data_length() == 5 * 31); 428 | 429 | auto data = header.get_data(); 430 | auto entries = header.get_entries(); 431 | for (int i = 0; i < 31; i++) 432 | { 433 | std::string ref = std::to_string(i); 434 | while (ref.size() < 5) 435 | ref += "."; 436 | entries--; 437 | CHECK(entries->get_creation_time() == i); 438 | CHECK(entries->get_deletion_time() == -i); 439 | CHECK(entries->get_dst() == i * 1234); 440 | CHECK(entries->get_length() == ref.size()); 441 | CHECK(std::string(data, entries->get_length()) == ref); 442 | CHECK(header.get_bloom_filter().find(entries->get_dst())); 443 | data += entries->get_length(); 444 | } 445 | 446 | header.set_num_entries_data_length_atomic(123, 321); 447 | CHECK(header.get_num_entries() == 123); 448 | CHECK(header.get_data_length() == 321); 449 | CHECK(header.get_num_entries_data_length_atomic() == std::pair{123, 321}); 450 | 451 | CHECK(header.get_type() == EdgeLabelBlockHeader::Type::EDGE); 452 | CHECK(header.get_order() == log_size); 453 | CHECK(header.get_block_size() == 1ul << log_size); 454 | CHECK(header.get_vertex_id() == 0x1234abcdef); 455 | CHECK(header.get_creation_time() == 0xffeeddccbbaal); 456 | CHECK(header.get_prev_pointer() == 0x1122334455667788ul); 457 | CHECK(header.get_committed_time() == 0x1234432112344321l); 458 | 459 | free(buf); 460 | } 461 | } 462 | -------------------------------------------------------------------------------- /test/transaction.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | #include "core/livegraph.hpp" 22 | 23 | using namespace livegraph; 24 | 25 | TEST_CASE("testing the Transaction: new_vertex") 26 | { 27 | Graph graph; 28 | 29 | { 30 | auto txn = graph.begin_transaction(); 31 | CHECK(txn.new_vertex() == 0); 32 | CHECK(txn.new_vertex() == 1); 33 | CHECK(txn.new_vertex() == 2); 34 | } 35 | { 36 | auto txn = graph.begin_transaction(); 37 | CHECK(txn.new_vertex() == 3); 38 | CHECK(txn.new_vertex() == 4); 39 | CHECK(txn.new_vertex() == 5); 40 | } 41 | { 42 | auto txn = graph.begin_transaction(); 43 | CHECK(txn.new_vertex(true) == 0); 44 | CHECK(txn.new_vertex(true) == 1); 45 | CHECK(txn.new_vertex(true) == 2); 46 | txn.abort(); 47 | } 48 | { 49 | auto txn = graph.begin_transaction(); 50 | CHECK(txn.new_vertex(true) == 3); 51 | CHECK(txn.new_vertex(true) == 4); 52 | CHECK(txn.new_vertex(true) == 5); 53 | txn.commit(); 54 | } 55 | { 56 | auto txn = graph.begin_transaction(); 57 | CHECK(txn.new_vertex(true) == 0); 58 | CHECK(txn.new_vertex(true) == 1); 59 | CHECK(txn.new_vertex(true) == 2); 60 | CHECK(txn.new_vertex(true) == 6); 61 | CHECK(txn.new_vertex(true) == 7); 62 | CHECK(txn.new_vertex(true) == 8); 63 | } 64 | { 65 | auto txn1 = graph.begin_transaction(); 66 | auto txn2 = graph.begin_transaction(); 67 | CHECK(txn1.new_vertex(true) == 0); 68 | CHECK(txn2.new_vertex(true) == 1); 69 | CHECK(txn1.new_vertex(true) == 2); 70 | CHECK(txn2.new_vertex(true) == 6); 71 | CHECK(txn1.new_vertex(true) == 7); 72 | CHECK(txn2.new_vertex(true) == 8); 73 | txn1.commit(); 74 | } 75 | { 76 | auto txn = graph.begin_batch_loader(); 77 | CHECK(txn.new_vertex(true) == 1); 78 | CHECK(txn.new_vertex(true) == 6); 79 | CHECK(txn.new_vertex(true) == 8); 80 | CHECK(txn.new_vertex(true) == 9); 81 | CHECK(txn.new_vertex() == 10); 82 | } 83 | { 84 | auto txn = graph.begin_read_only_transaction(); 85 | CHECK_THROWS_AS(txn.new_vertex(), std::invalid_argument); 86 | } 87 | { 88 | auto txn = graph.begin_transaction(); 89 | CHECK(txn.new_vertex(true) == 11); 90 | } 91 | } 92 | 93 | TEST_CASE("testing the Transaction: put/get/del_vertex") 94 | { 95 | Graph graph; 96 | 97 | { 98 | auto txn = graph.begin_transaction(); 99 | CHECK(txn.new_vertex() == 0); 100 | CHECK(txn.new_vertex() == 1); 101 | CHECK(txn.new_vertex() == 2); 102 | txn.commit(); 103 | } 104 | { 105 | auto txn = graph.begin_transaction(); 106 | CHECK(txn.get_vertex(0) == ""); 107 | CHECK(txn.get_vertex(1) == ""); 108 | CHECK(txn.get_vertex(2) == ""); 109 | } 110 | { 111 | auto txn1 = graph.begin_transaction(); 112 | txn1.put_vertex(0, "aaaa"); 113 | txn1.put_vertex(1, "bbbb"); 114 | txn1.put_vertex(2, "cccc"); 115 | CHECK(txn1.get_vertex(0) == "aaaa"); 116 | CHECK(txn1.get_vertex(1) == "bbbb"); 117 | CHECK(txn1.get_vertex(2) == "cccc"); 118 | auto txn2 = graph.begin_transaction(); 119 | CHECK(txn2.get_vertex(0) == ""); 120 | CHECK(txn2.get_vertex(1) == ""); 121 | CHECK(txn2.get_vertex(2) == ""); 122 | } 123 | { 124 | auto txn = graph.begin_transaction(); 125 | CHECK(txn.get_vertex(0) == ""); 126 | CHECK(txn.get_vertex(1) == ""); 127 | CHECK(txn.get_vertex(2) == ""); 128 | } 129 | { 130 | auto txn = graph.begin_transaction(); 131 | txn.put_vertex(0, "aaaa"); 132 | txn.put_vertex(1, "bbbb"); 133 | txn.put_vertex(2, "cccc"); 134 | CHECK(txn.get_vertex(0) == "aaaa"); 135 | CHECK(txn.get_vertex(1) == "bbbb"); 136 | CHECK(txn.get_vertex(2) == "cccc"); 137 | auto txn2 = graph.begin_transaction(); 138 | CHECK(txn2.get_vertex(0) == ""); 139 | CHECK(txn2.get_vertex(1) == ""); 140 | CHECK(txn2.get_vertex(2) == ""); 141 | txn.commit(); 142 | } 143 | { 144 | auto txn = graph.begin_transaction(); 145 | CHECK(txn.get_vertex(0) == "aaaa"); 146 | CHECK(txn.get_vertex(1) == "bbbb"); 147 | CHECK(txn.get_vertex(2) == "cccc"); 148 | } 149 | 150 | { 151 | auto txn_b = graph.begin_batch_loader(); 152 | txn_b.put_vertex(0, "AAAA"); 153 | txn_b.put_vertex(1, "BBBB"); 154 | txn_b.put_vertex(2, "CCCC"); 155 | } 156 | { 157 | auto txn = graph.begin_transaction(); 158 | CHECK(txn.get_vertex(0) == "AAAA"); 159 | CHECK(txn.get_vertex(1) == "BBBB"); 160 | CHECK(txn.get_vertex(2) == "CCCC"); 161 | } 162 | 163 | { 164 | auto txn = graph.begin_transaction(); 165 | CHECK(txn.del_vertex(0)); 166 | CHECK(txn.get_vertex(0) == ""); 167 | CHECK(!txn.del_vertex(0)); 168 | txn.put_vertex(0, "aaa"); 169 | CHECK(txn.get_vertex(0) == "aaa"); 170 | CHECK(txn.del_vertex(1)); 171 | CHECK(txn.get_vertex(1) == ""); 172 | CHECK(txn.del_vertex(2)); 173 | CHECK(txn.get_vertex(2) == ""); 174 | txn.put_vertex(0, "aaaa"); 175 | txn.put_vertex(1, "bbbb"); 176 | txn.put_vertex(2, "cccc"); 177 | CHECK(txn.get_vertex(0) == "aaaa"); 178 | CHECK(txn.get_vertex(1) == "bbbb"); 179 | CHECK(txn.get_vertex(2) == "cccc"); 180 | auto txn2 = graph.begin_transaction(); 181 | CHECK(txn2.get_vertex(0) == "AAAA"); 182 | CHECK(txn2.get_vertex(1) == "BBBB"); 183 | CHECK(txn2.get_vertex(2) == "CCCC"); 184 | txn.commit(); 185 | CHECK(txn2.get_vertex(0) == "AAAA"); 186 | CHECK(txn2.get_vertex(1) == "BBBB"); 187 | CHECK(txn2.get_vertex(2) == "CCCC"); 188 | } 189 | { 190 | auto txn = graph.begin_transaction(); 191 | CHECK(txn.get_vertex(0) == "aaaa"); 192 | CHECK(txn.get_vertex(1) == "bbbb"); 193 | CHECK(txn.get_vertex(2) == "cccc"); 194 | } 195 | 196 | { 197 | auto txn1 = graph.begin_transaction(); 198 | CHECK(txn1.del_vertex(0, true)); 199 | auto txn2 = graph.begin_transaction(); 200 | CHECK(txn2.new_vertex(true) == 3); 201 | txn1.commit(); 202 | CHECK(txn2.new_vertex(true) == 0); 203 | CHECK(!txn2.del_vertex(3)); 204 | CHECK(txn2.new_vertex(true) == 4); 205 | CHECK(!txn2.del_vertex(3, true)); 206 | txn2.commit(); 207 | auto txn3 = graph.begin_transaction(); 208 | CHECK(txn3.new_vertex(true) == 3); 209 | CHECK(!txn3.del_vertex(3, true)); 210 | CHECK(txn3.new_vertex(true) == 3); 211 | txn3.abort(); 212 | auto txn4 = graph.begin_transaction(); 213 | CHECK(txn4.new_vertex(true) == 3); 214 | } 215 | 216 | { // Deadlock 217 | auto txn1 = graph.begin_transaction(); 218 | auto txn2 = graph.begin_transaction(); 219 | txn1.put_vertex(0, "AAAA"); 220 | CHECK_THROWS_AS(txn2.put_vertex(0, "aaaa"), Transaction::RollbackExcept); 221 | txn2.del_vertex(1); 222 | CHECK_THROWS_AS(txn1.del_vertex(1), Transaction::RollbackExcept); 223 | } 224 | 225 | { // Write-write conflict 226 | auto txn1 = graph.begin_transaction(); 227 | auto txn2 = graph.begin_transaction(); 228 | txn1.put_vertex(0, "AAAA"); 229 | txn1.del_vertex(1); 230 | txn1.commit(); 231 | CHECK_THROWS_AS(txn2.put_vertex(0, "aaaa"), Transaction::RollbackExcept); 232 | CHECK_THROWS_AS(txn2.del_vertex(1), Transaction::RollbackExcept); 233 | } 234 | { 235 | auto txn = graph.begin_read_only_transaction(); 236 | CHECK_THROWS_AS(txn.put_vertex(0, "aaaa"), std::invalid_argument); 237 | CHECK_THROWS_AS(txn.del_vertex(0, "aaaa"), std::invalid_argument); 238 | CHECK(txn.get_vertex(0) == "AAAA"); 239 | } 240 | { 241 | auto txn = graph.begin_transaction(); 242 | CHECK_THROWS_AS(txn.put_vertex(10, ""), std::invalid_argument); 243 | CHECK_THROWS_AS(txn.del_vertex(10), std::invalid_argument); 244 | txn.abort(); 245 | CHECK_THROWS_AS(txn.put_vertex(0, ""), std::invalid_argument); 246 | CHECK_THROWS_AS(txn.get_vertex(0), std::invalid_argument); 247 | CHECK_THROWS_AS(txn.del_vertex(0), std::invalid_argument); 248 | } 249 | } 250 | 251 | TEST_CASE("testing the Transaction: put/get/del_edge") 252 | { 253 | Graph graph; 254 | 255 | const vertex_t vertices = 128, src_vertices = 4; 256 | const label_t labels = 16; 257 | 258 | auto loop = [&](auto f) { 259 | for (label_t label = 0; label < labels; label++) 260 | for (vertex_t i = 0; i < src_vertices; i++) 261 | for (vertex_t j = 0; j < vertices; j++) 262 | f(i, label, j); 263 | }; 264 | 265 | { 266 | auto txn = graph.begin_transaction(); 267 | for (vertex_t i = 0; i < vertices; i++) 268 | CHECK(txn.new_vertex() == i); 269 | txn.commit(); 270 | } 271 | { 272 | auto txn1 = graph.begin_transaction(); 273 | loop([&](vertex_t i, label_t label, vertex_t j) { 274 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 275 | txn1.put_edge(i, label, j, data); 276 | }); 277 | loop([&](vertex_t i, label_t label, vertex_t j) { 278 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 279 | CHECK(txn1.get_edge(i, label, j) == data); 280 | }); 281 | for (label_t label = 0; label < labels; label++) 282 | { 283 | for (vertex_t i = 0; i < src_vertices; i++) 284 | { 285 | auto riter = txn1.get_edges(i, label, true); 286 | for (vertex_t j = 0; j < vertices; j++) 287 | { 288 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 289 | CHECK(riter.dst_id() == j); 290 | CHECK(riter.edge_data() == data); 291 | riter.next(); 292 | } 293 | CHECK(!riter.valid()); 294 | CHECK(riter.dst_id() == (vertex_t)-1); 295 | CHECK(riter.edge_data() == ""); 296 | auto iter = txn1.get_edges(i, label); 297 | for (vertex_t j = 0; j < vertices; j++) 298 | { 299 | std::string data = 300 | std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(vertices - j - 1); 301 | CHECK(iter.dst_id() == vertices - j - 1); 302 | CHECK(iter.edge_data() == data); 303 | iter.next(); 304 | } 305 | CHECK(!iter.valid()); 306 | CHECK(iter.dst_id() == (vertex_t)-1); 307 | CHECK(iter.edge_data() == ""); 308 | } 309 | } 310 | auto txn2 = graph.begin_transaction(); 311 | loop([&](vertex_t i, label_t label, vertex_t j) { 312 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 313 | CHECK(txn2.get_edge(i, label, j) == ""); 314 | }); 315 | } 316 | { 317 | auto txn = graph.begin_transaction(); 318 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn.get_edge(i, label, j) == ""); }); 319 | for (label_t label = 0; label < labels; label++) 320 | { 321 | for (vertex_t i = 0; i < src_vertices; i++) 322 | { 323 | auto riter = txn.get_edges(i, label, true); 324 | CHECK(!riter.valid()); 325 | auto iter = txn.get_edges(i, label); 326 | CHECK(!iter.valid()); 327 | } 328 | } 329 | } 330 | { 331 | auto txn1 = graph.begin_transaction(); 332 | loop([&](vertex_t i, label_t label, vertex_t j) { 333 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 334 | txn1.put_edge(i, label, j, data); 335 | }); 336 | loop([&](vertex_t i, label_t label, vertex_t j) { 337 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 338 | CHECK(txn1.get_edge(i, label, j) == data); 339 | }); 340 | auto txn2 = graph.begin_transaction(); 341 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn2.get_edge(i, label, j) == ""); }); 342 | txn1.commit(); 343 | } 344 | { 345 | auto txn = graph.begin_transaction(); 346 | loop([&](vertex_t i, label_t label, vertex_t j) { 347 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 348 | CHECK(txn.get_edge(i, label, j) == data); 349 | }); 350 | for (label_t label = 0; label < labels; label++) 351 | { 352 | for (vertex_t i = 0; i < src_vertices; i++) 353 | { 354 | auto riter = txn.get_edges(i, label, true); 355 | for (vertex_t j = 0; j < vertices; j++) 356 | { 357 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 358 | CHECK(riter.dst_id() == j); 359 | CHECK(riter.edge_data() == data); 360 | riter.next(); 361 | } 362 | CHECK(!riter.valid()); 363 | CHECK(riter.dst_id() == (vertex_t)-1); 364 | CHECK(riter.edge_data() == ""); 365 | auto iter = txn.get_edges(i, label); 366 | for (vertex_t j = 0; j < vertices; j++) 367 | { 368 | std::string data = 369 | std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(vertices - j - 1); 370 | CHECK(iter.dst_id() == vertices - j - 1); 371 | CHECK(iter.edge_data() == data); 372 | iter.next(); 373 | } 374 | CHECK(!iter.valid()); 375 | CHECK(iter.dst_id() == (vertex_t)-1); 376 | CHECK(iter.edge_data() == ""); 377 | } 378 | } 379 | } 380 | 381 | { 382 | auto txn = graph.begin_batch_loader(); 383 | loop([&](vertex_t i, label_t label, vertex_t j) { 384 | std::string data = std::to_string(i) + ".." + std::to_string(label) + "->" + std::to_string(j); 385 | txn.put_edge(i, label, j, data); 386 | }); 387 | } 388 | { 389 | auto txn = graph.begin_transaction(); 390 | loop([&](vertex_t i, label_t label, vertex_t j) { 391 | std::string data = std::to_string(i) + ".." + std::to_string(label) + "->" + std::to_string(j); 392 | CHECK(txn.get_edge(i, label, j) == data); 393 | }); 394 | } 395 | 396 | { 397 | auto txn = graph.begin_transaction(); 398 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn.del_edge(i, label, j)); }); 399 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn.get_edge(i, label, j) == ""); }); 400 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(!txn.del_edge(i, label, j)); }); 401 | loop([&](vertex_t i, label_t label, vertex_t j) { 402 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 403 | txn.put_edge(i, label, j, data); 404 | }); 405 | loop([&](vertex_t i, label_t label, vertex_t j) { 406 | std::string data = std::to_string(i) + "." + std::to_string(label) + "->" + std::to_string(j); 407 | CHECK(txn.get_edge(i, label, j) == data); 408 | }); 409 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn.del_edge(i, label, j)); }); 410 | loop([&](vertex_t i, label_t label, vertex_t j) { CHECK(txn.get_edge(i, label, j) == ""); }); 411 | loop([&](vertex_t i, label_t label, vertex_t j) { 412 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 413 | txn.put_edge(i, label, j, data); 414 | }); 415 | loop([&](vertex_t i, label_t label, vertex_t j) { 416 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 417 | CHECK(txn.get_edge(i, label, j) == data); 418 | }); 419 | auto txn2 = graph.begin_transaction(); 420 | loop([&](vertex_t i, label_t label, vertex_t j) { 421 | std::string data = std::to_string(i) + ".." + std::to_string(label) + "->" + std::to_string(j); 422 | CHECK(txn2.get_edge(i, label, j) == data); 423 | }); 424 | txn.commit(); 425 | loop([&](vertex_t i, label_t label, vertex_t j) { 426 | std::string data = std::to_string(i) + ".." + std::to_string(label) + "->" + std::to_string(j); 427 | CHECK(txn2.get_edge(i, label, j) == data); 428 | }); 429 | } 430 | { 431 | auto txn = graph.begin_transaction(); 432 | loop([&](vertex_t i, label_t label, vertex_t j) { 433 | std::string data = std::to_string(i) + "-" + std::to_string(label) + "->" + std::to_string(j); 434 | CHECK(txn.get_edge(i, label, j) == data); 435 | }); 436 | } 437 | 438 | { // Deadlock 439 | auto txn1 = graph.begin_transaction(); 440 | auto txn2 = graph.begin_transaction(); 441 | txn1.put_edge(0, 0, 1, "AAAA"); 442 | CHECK_THROWS_AS(txn2.put_edge(0, 1, 2, "aaaa"), Transaction::RollbackExcept); 443 | txn2.del_edge(1, 0, 3); 444 | CHECK_THROWS_AS(txn1.del_edge(1, 1, 4), Transaction::RollbackExcept); 445 | } 446 | 447 | { // Write-write conflict 448 | auto txn1 = graph.begin_transaction(); 449 | auto txn2 = graph.begin_transaction(); 450 | txn1.put_edge(0, 0, 1, "AAAA"); 451 | txn1.del_edge(1, 0, 3); 452 | txn1.commit(); 453 | txn2.put_edge(0, 1, 2, "aaaa"); 454 | CHECK_THROWS_AS(txn2.put_edge(0, 0, 3, "aaaa"), Transaction::RollbackExcept); 455 | CHECK(txn2.del_edge(1, 1, 4)); 456 | CHECK_THROWS_AS(txn2.del_edge(1, 0, 4), Transaction::RollbackExcept); 457 | } 458 | { 459 | auto txn = graph.begin_read_only_transaction(); 460 | CHECK_THROWS_AS(txn.put_edge(0, 0, 1, "aaaa"), std::invalid_argument); 461 | CHECK_THROWS_AS(txn.del_edge(0, 0, 1), std::invalid_argument); 462 | CHECK(txn.get_edge(0, 0, 1) == "AAAA"); 463 | } 464 | { 465 | auto txn = graph.begin_transaction(); 466 | CHECK_THROWS_AS(txn.put_edge(0, 0, vertices, "aaaa"), std::invalid_argument); 467 | CHECK_THROWS_AS(txn.put_edge(vertices, 0, 0, "aaaa"), std::invalid_argument); 468 | CHECK_THROWS_AS(txn.del_edge(0, 0, vertices), std::invalid_argument); 469 | CHECK_THROWS_AS(txn.del_edge(vertices, 0, 0), std::invalid_argument); 470 | txn.abort(); 471 | CHECK_THROWS_AS(txn.put_edge(0, 0, 1, "aaaa"), std::invalid_argument); 472 | CHECK_THROWS_AS(txn.del_edge(0, 0, 1), std::invalid_argument); 473 | CHECK_THROWS_AS(txn.get_edge(0, 0, 1), std::invalid_argument); 474 | } 475 | } 476 | -------------------------------------------------------------------------------- /src/transaction.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "core/transaction.hpp" 17 | #include "core/edge_iterator.hpp" 18 | #include "core/graph.hpp" 19 | 20 | using namespace livegraph; 21 | 22 | vertex_t Transaction::new_vertex(bool use_recycled_vertex) 23 | { 24 | check_valid(); 25 | check_writable(); 26 | 27 | vertex_t vertex_id; 28 | if (!batch_update && recycled_vertex_cache.size()) 29 | { 30 | vertex_id = recycled_vertex_cache.front(); 31 | recycled_vertex_cache.pop_front(); 32 | } 33 | else if (!use_recycled_vertex || (!graph.recycled_vertex_ids.try_pop(vertex_id))) 34 | { 35 | vertex_id = graph.vertex_id.fetch_add(1, std::memory_order_relaxed); 36 | } 37 | graph.vertex_futexes[vertex_id].clear(); 38 | graph.vertex_ptrs[vertex_id] = graph.block_manager.NULLPOINTER; 39 | graph.edge_label_ptrs[vertex_id] = graph.block_manager.NULLPOINTER; 40 | 41 | if (!batch_update) 42 | { 43 | new_vertex_cache.emplace_back(vertex_id); 44 | ++wal_num_ops(); 45 | wal_append(OPType::NewVertex); 46 | wal_append(vertex_id); 47 | } 48 | 49 | return vertex_id; 50 | } 51 | 52 | void Transaction::put_vertex(vertex_t vertex_id, std::string_view data) 53 | { 54 | check_valid(); 55 | check_writable(); 56 | check_vertex_id(vertex_id); 57 | 58 | uintptr_t prev_pointer; 59 | if (batch_update) 60 | { 61 | graph.vertex_futexes[vertex_id].lock(); 62 | prev_pointer = graph.vertex_ptrs[vertex_id]; 63 | } 64 | else 65 | { 66 | ensure_vertex_lock(vertex_id); 67 | auto cache_iter = vertex_ptr_cache.find(vertex_id); 68 | if (cache_iter != vertex_ptr_cache.end()) 69 | prev_pointer = cache_iter->second; 70 | else 71 | { 72 | ensure_no_confict(vertex_id); 73 | prev_pointer = graph.vertex_ptrs[vertex_id]; 74 | } 75 | } 76 | 77 | auto size = sizeof(VertexBlockHeader) + data.size(); 78 | auto order = size_to_order(size); 79 | auto pointer = graph.block_manager.alloc(order); 80 | 81 | auto vertex_block = graph.block_manager.convert(pointer); 82 | vertex_block->fill(order, vertex_id, write_epoch_id, prev_pointer, data.data(), data.size()); 83 | 84 | graph.compact_table.local().emplace(vertex_id); 85 | 86 | if (batch_update) 87 | { 88 | graph.vertex_ptrs[vertex_id] = pointer; 89 | graph.vertex_futexes[vertex_id].unlock(); 90 | } 91 | else 92 | { 93 | block_cache.emplace_back(pointer, order); 94 | timestamps_to_update.emplace_back(vertex_block->get_creation_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 95 | vertex_ptr_cache[vertex_id] = pointer; 96 | 97 | ++wal_num_ops(); 98 | wal_append(OPType::PutVertex); 99 | wal_append(vertex_id); 100 | wal_append(data); 101 | } 102 | } 103 | 104 | bool Transaction::del_vertex(vertex_t vertex_id, bool recycle) 105 | { 106 | check_valid(); 107 | check_writable(); 108 | check_vertex_id(vertex_id); 109 | 110 | uintptr_t prev_pointer; 111 | if (batch_update) 112 | { 113 | graph.vertex_futexes[vertex_id].lock(); 114 | prev_pointer = graph.vertex_ptrs[vertex_id]; 115 | } 116 | else 117 | { 118 | ensure_vertex_lock(vertex_id); 119 | auto cache_iter = vertex_ptr_cache.find(vertex_id); 120 | if (cache_iter != vertex_ptr_cache.end()) 121 | prev_pointer = cache_iter->second; 122 | else 123 | { 124 | ensure_no_confict(vertex_id); 125 | prev_pointer = graph.vertex_ptrs[vertex_id]; 126 | } 127 | } 128 | 129 | bool ret = false; 130 | auto prev_vertex_block = graph.block_manager.convert(prev_pointer); 131 | if (prev_vertex_block && prev_vertex_block->get_length() != prev_vertex_block->TOMBSTONE) 132 | { 133 | ret = true; 134 | auto size = sizeof(VertexBlockHeader); 135 | auto order = size_to_order(size); 136 | auto pointer = graph.block_manager.alloc(order); 137 | 138 | auto vertex_block = graph.block_manager.convert(pointer); 139 | vertex_block->fill(order, vertex_id, write_epoch_id, prev_pointer, nullptr, vertex_block->TOMBSTONE); 140 | 141 | graph.compact_table.local().emplace(vertex_id); 142 | 143 | if (!batch_update) 144 | { 145 | block_cache.emplace_back(pointer, order); 146 | timestamps_to_update.emplace_back(vertex_block->get_creation_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 147 | vertex_ptr_cache[vertex_id] = pointer; 148 | } 149 | } 150 | 151 | if (batch_update) 152 | { 153 | if (recycle) 154 | graph.recycled_vertex_ids.push(vertex_id); 155 | graph.vertex_futexes[vertex_id].unlock(); 156 | } 157 | else 158 | { 159 | ++wal_num_ops(); 160 | wal_append(OPType::DelVertex); 161 | wal_append(vertex_id); 162 | wal_append(recycle); 163 | 164 | if (recycle) 165 | recycled_vertex_cache.emplace_back(vertex_id); 166 | } 167 | 168 | return ret; 169 | } 170 | 171 | std::string_view Transaction::get_vertex(vertex_t vertex_id) 172 | { 173 | check_valid(); 174 | 175 | if (vertex_id >= graph.vertex_id.load(std::memory_order_relaxed)) 176 | return std::string_view(); 177 | 178 | uintptr_t pointer; 179 | if (batch_update || !trace_cache) 180 | { 181 | pointer = graph.vertex_ptrs[vertex_id]; 182 | } 183 | else 184 | { 185 | auto cache_iter = vertex_ptr_cache.find(vertex_id); 186 | if (cache_iter != vertex_ptr_cache.end()) 187 | pointer = cache_iter->second; 188 | else 189 | pointer = graph.vertex_ptrs[vertex_id]; 190 | } 191 | 192 | auto vertex_block = graph.block_manager.convert(pointer); 193 | while (vertex_block) 194 | { 195 | if (cmp_timestamp(vertex_block->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 0) 196 | break; 197 | pointer = vertex_block->get_prev_pointer(); 198 | vertex_block = graph.block_manager.convert(pointer); 199 | } 200 | 201 | // if (!(batch_update || !trace_cache)) 202 | //{ 203 | // vertex_ptr_cache[vertex_id] = pointer; 204 | //} 205 | 206 | if (!vertex_block || vertex_block->get_length() == vertex_block->TOMBSTONE) 207 | return std::string_view(); 208 | 209 | return std::string_view(vertex_block->get_data(), vertex_block->get_length()); 210 | } 211 | 212 | std::pair 213 | Transaction::find_edge(vertex_t dst, EdgeBlockHeader *edge_block, size_t num_entries, size_t data_length) 214 | { 215 | if (!edge_block) 216 | return {nullptr, nullptr}; 217 | 218 | auto bloom_filter = edge_block->get_bloom_filter(); 219 | if (bloom_filter.valid() && !bloom_filter.find(dst)) 220 | return {nullptr, nullptr}; 221 | 222 | auto entries = edge_block->get_entries() - num_entries; 223 | auto data = edge_block->get_data() + data_length; 224 | for (size_t i = 0; i < num_entries; i++) 225 | { 226 | data -= entries->get_length(); 227 | if (entries->get_dst() == dst && 228 | cmp_timestamp(entries->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 0 && 229 | cmp_timestamp(entries->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 0) 230 | { 231 | return {entries, data}; 232 | } 233 | entries++; 234 | } 235 | 236 | return {nullptr, nullptr}; 237 | } 238 | 239 | uintptr_t Transaction::locate_edge_block(vertex_t src, label_t label) 240 | { 241 | auto pointer = graph.edge_label_ptrs[src]; 242 | if (pointer == graph.block_manager.NULLPOINTER) 243 | return pointer; 244 | auto edge_label_block = graph.block_manager.convert(pointer); 245 | for (size_t i = 0; i < edge_label_block->get_num_entries(); i++) 246 | { 247 | auto label_entry = edge_label_block->get_entries()[i]; 248 | if (label_entry.get_label() == label) 249 | { 250 | auto pointer = label_entry.get_pointer(); 251 | while (pointer != graph.block_manager.NULLPOINTER) 252 | { 253 | auto edge_block = graph.block_manager.convert(pointer); 254 | if (cmp_timestamp(edge_block->get_creation_time_pointer(), read_epoch_id, local_txn_id) <= 0) 255 | break; 256 | pointer = edge_block->get_prev_pointer(); 257 | } 258 | return pointer; 259 | } 260 | } 261 | return graph.block_manager.NULLPOINTER; 262 | } 263 | 264 | void Transaction::ensure_no_confict(vertex_t src, label_t label) 265 | { 266 | auto pointer = graph.edge_label_ptrs[src]; 267 | if (pointer == graph.block_manager.NULLPOINTER) 268 | return; 269 | auto edge_label_block = graph.block_manager.convert(pointer); 270 | for (size_t i = 0; i < edge_label_block->get_num_entries(); i++) 271 | { 272 | auto label_entry = edge_label_block->get_entries()[i]; 273 | if (label_entry.get_label() == label) 274 | { 275 | auto pointer = label_entry.get_pointer(); 276 | if (pointer != graph.block_manager.NULLPOINTER) 277 | { 278 | auto header = graph.block_manager.convert(pointer); 279 | if (header && cmp_timestamp(header->get_committed_time_pointer(), read_epoch_id, local_txn_id) > 0) 280 | throw RollbackExcept("Write-write confict on: " + std::to_string(src) + ": " + 281 | std::to_string(label) + "."); 282 | } 283 | return; 284 | } 285 | } 286 | } 287 | 288 | void Transaction::update_edge_label_block(vertex_t src, label_t label, uintptr_t edge_block_pointer) 289 | { 290 | auto pointer = graph.edge_label_ptrs[src]; 291 | auto edge_label_block = graph.block_manager.convert(pointer); 292 | if (edge_label_block) 293 | { 294 | for (size_t i = 0; i < edge_label_block->get_num_entries(); i++) 295 | { 296 | auto &label_entry = edge_label_block->get_entries()[i]; 297 | if (label_entry.get_label() == label) 298 | { 299 | label_entry.set_pointer(edge_block_pointer); 300 | return; 301 | } 302 | } 303 | } 304 | 305 | EdgeLabelEntry label_entry; 306 | label_entry.set_label(label); 307 | label_entry.set_pointer(edge_block_pointer); 308 | 309 | if (!edge_label_block || !edge_label_block->append(label_entry)) 310 | { 311 | auto num_entries = edge_label_block ? edge_label_block->get_num_entries() : 0; 312 | auto size = sizeof(EdgeLabelBlockHeader) + (1 + num_entries) * sizeof(EdgeLabelEntry); 313 | auto order = size_to_order(size); 314 | 315 | auto new_pointer = graph.block_manager.alloc(order); 316 | 317 | auto new_edge_label_block = graph.block_manager.convert(new_pointer); 318 | new_edge_label_block->fill(order, src, write_epoch_id, pointer); 319 | 320 | if (!batch_update) 321 | { 322 | block_cache.emplace_back(new_pointer, order); 323 | timestamps_to_update.emplace_back(new_edge_label_block->get_creation_time_pointer(), 324 | Graph::ROLLBACK_TOMBSTONE); 325 | } 326 | 327 | for (size_t i = 0; i < num_entries; i++) 328 | { 329 | auto old_label_entry = edge_label_block->get_entries()[i]; 330 | new_edge_label_block->append(old_label_entry); 331 | } 332 | 333 | new_edge_label_block->append(label_entry); 334 | 335 | graph.edge_label_ptrs[src] = new_pointer; 336 | } 337 | } 338 | 339 | void Transaction::put_edge(vertex_t src, label_t label, vertex_t dst, std::string_view edge_data, bool force_insert) 340 | { 341 | check_valid(); 342 | check_writable(); 343 | check_vertex_id(src); 344 | check_vertex_id(dst); 345 | 346 | uintptr_t pointer; 347 | if (batch_update) 348 | { 349 | graph.vertex_futexes[src].lock(); 350 | pointer = locate_edge_block(src, label); 351 | } 352 | else 353 | { 354 | ensure_vertex_lock(src); 355 | auto cache_iter = edge_ptr_cache.find(std::make_pair(src, label)); 356 | if (cache_iter != edge_ptr_cache.end()) 357 | { 358 | pointer = cache_iter->second; 359 | } 360 | else 361 | { 362 | ensure_no_confict(src, label); 363 | pointer = locate_edge_block(src, label); 364 | edge_ptr_cache.emplace_hint(cache_iter, std::make_pair(src, label), pointer); 365 | } 366 | } 367 | 368 | EdgeEntry entry; 369 | entry.set_length(edge_data.size()); 370 | entry.set_dst(dst); 371 | entry.set_creation_time(write_epoch_id); 372 | entry.set_deletion_time(Graph::ROLLBACK_TOMBSTONE); 373 | 374 | auto edge_block = graph.block_manager.convert(pointer); 375 | 376 | auto [num_entries, data_length] = 377 | edge_block ? get_num_entries_data_length_cache(edge_block) : std::pair{0, 0}; 378 | 379 | if (!edge_block || !edge_block->has_space(entry, num_entries, data_length)) 380 | { 381 | auto size = sizeof(EdgeBlockHeader) + (1 + num_entries) * sizeof(EdgeEntry) + data_length + entry.get_length(); 382 | auto order = size_to_order(size); 383 | 384 | if (order > edge_block->BLOOM_FILTER_PORTION && 385 | size + (1ul << (order - edge_block->BLOOM_FILTER_PORTION)) >= (1ul << edge_block->BLOOM_FILTER_THRESHOLD)) 386 | { 387 | size += 1ul << (order - edge_block->BLOOM_FILTER_PORTION); 388 | } 389 | order = size_to_order(size); 390 | 391 | auto new_pointer = graph.block_manager.alloc(order); 392 | 393 | auto new_edge_block = graph.block_manager.convert(new_pointer); 394 | new_edge_block->fill(order, src, write_epoch_id, pointer, write_epoch_id); 395 | 396 | if (!batch_update) 397 | { 398 | block_cache.emplace_back(new_pointer, order); 399 | timestamps_to_update.emplace_back(new_edge_block->get_creation_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 400 | // timestamps_to_update.emplace_back(new_edge_block->get_committed_time_pointer(), 401 | // Graph::ROLLBACK_TOMBSTONE); update when commit 402 | } 403 | 404 | if (edge_block) 405 | { 406 | auto entries = edge_block->get_entries(); 407 | auto data = edge_block->get_data(); 408 | 409 | auto bloom_filter = new_edge_block->get_bloom_filter(); 410 | for (size_t i = 0; i < num_entries; i++) 411 | { 412 | entries--; 413 | // skip deleted edges 414 | if (cmp_timestamp(entries->get_deletion_time_pointer(), read_epoch_id, local_txn_id) > 0) 415 | { 416 | auto edge = new_edge_block->append(*entries, data, bloom_filter); // direct update size 417 | if (!batch_update && edge->get_creation_time() == -local_txn_id) 418 | timestamps_to_update.emplace_back(edge->get_creation_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 419 | } 420 | data += entries->get_length(); 421 | } 422 | } 423 | 424 | if (batch_update) 425 | update_edge_label_block(src, label, new_pointer); 426 | 427 | pointer = new_pointer; 428 | edge_block = new_edge_block; 429 | std::tie(num_entries, data_length) = new_edge_block->get_num_entries_data_length_atomic(); 430 | } 431 | 432 | if (!force_insert) 433 | { 434 | auto prev_edge = find_edge(dst, edge_block, num_entries, data_length); 435 | 436 | if (prev_edge.first) 437 | { 438 | prev_edge.first->set_deletion_time(write_epoch_id); 439 | if (!batch_update) 440 | timestamps_to_update.emplace_back(prev_edge.first->get_deletion_time_pointer(), 441 | Graph::ROLLBACK_TOMBSTONE); 442 | } 443 | } 444 | 445 | auto edge = edge_block->append_without_update_size(entry, edge_data.data(), num_entries, data_length); 446 | set_num_entries_data_length_cache(edge_block, num_entries + 1, data_length + entry.get_length()); 447 | if (!batch_update) 448 | timestamps_to_update.emplace_back(edge->get_creation_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 449 | 450 | graph.compact_table.local().emplace(src); 451 | 452 | if (batch_update) 453 | { 454 | graph.vertex_futexes[src].unlock(); 455 | } 456 | else 457 | { 458 | edge_ptr_cache[std::make_pair(src, label)] = pointer; 459 | ++wal_num_ops(); 460 | wal_append(OPType::PutEdge); 461 | wal_append(src); 462 | wal_append(label); 463 | wal_append(dst); 464 | wal_append(force_insert); 465 | wal_append(edge_data); 466 | } 467 | } 468 | 469 | bool Transaction::del_edge(vertex_t src, label_t label, vertex_t dst) 470 | { 471 | check_valid(); 472 | check_writable(); 473 | check_vertex_id(src); 474 | check_vertex_id(dst); 475 | 476 | uintptr_t pointer; 477 | if (batch_update) 478 | { 479 | graph.vertex_futexes[src].lock(); 480 | pointer = locate_edge_block(src, label); 481 | } 482 | else 483 | { 484 | ensure_vertex_lock(src); 485 | auto cache_iter = edge_ptr_cache.find(std::make_pair(src, label)); 486 | if (cache_iter != edge_ptr_cache.end()) 487 | { 488 | pointer = cache_iter->second; 489 | } 490 | else 491 | { 492 | ensure_no_confict(src, label); 493 | pointer = locate_edge_block(src, label); 494 | edge_ptr_cache.emplace_hint(cache_iter, std::make_pair(src, label), pointer); 495 | } 496 | } 497 | 498 | auto edge_block = graph.block_manager.convert(pointer); 499 | 500 | if (!edge_block) 501 | return false; 502 | 503 | auto [num_entries, data_length] = get_num_entries_data_length_cache(edge_block); 504 | auto edge = find_edge(dst, edge_block, num_entries, data_length); 505 | 506 | if (edge.first) 507 | { 508 | edge.first->set_deletion_time(write_epoch_id); 509 | if (!batch_update) 510 | timestamps_to_update.emplace_back(edge.first->get_deletion_time_pointer(), Graph::ROLLBACK_TOMBSTONE); 511 | } 512 | 513 | graph.compact_table.local().emplace(src); 514 | 515 | if (batch_update) 516 | { 517 | graph.vertex_futexes[src].unlock(); 518 | } 519 | else 520 | { 521 | edge_ptr_cache[std::make_pair(src, label)] = pointer; 522 | // make sure commit will change committed_time 523 | set_num_entries_data_length_cache(edge_block, num_entries, data_length); 524 | 525 | ++wal_num_ops(); 526 | wal_append(OPType::DelEdge); 527 | wal_append(src); 528 | wal_append(label); 529 | wal_append(dst); 530 | } 531 | 532 | if (edge.first != nullptr) 533 | return true; 534 | else 535 | return false; 536 | } 537 | 538 | std::string_view Transaction::get_edge(vertex_t src, label_t label, vertex_t dst) 539 | { 540 | check_valid(); 541 | 542 | if (src >= graph.vertex_id.load(std::memory_order_relaxed)) 543 | return std::string_view(); 544 | 545 | uintptr_t pointer; 546 | if (batch_update || !trace_cache) 547 | { 548 | pointer = locate_edge_block(src, label); 549 | } 550 | else 551 | { 552 | auto cache_iter = edge_ptr_cache.find(std::make_pair(src, label)); 553 | if (cache_iter != edge_ptr_cache.end()) 554 | { 555 | pointer = cache_iter->second; 556 | } 557 | else 558 | { 559 | pointer = locate_edge_block(src, label); 560 | // edge_ptr_cache.emplace_hint(cache_iter, std::make_pair(src, label), pointer); 561 | } 562 | } 563 | 564 | auto edge_block = graph.block_manager.convert(pointer); 565 | 566 | if (!edge_block) 567 | return std::string_view(); 568 | 569 | auto [num_entries, data_length] = get_num_entries_data_length_cache(edge_block); 570 | auto edge = find_edge(dst, edge_block, num_entries, data_length); 571 | 572 | if (edge.first) 573 | return std::string_view(edge.second, edge.first->get_length()); 574 | else 575 | return std::string_view(); 576 | } 577 | 578 | void Transaction::abort() 579 | { 580 | check_valid(); 581 | 582 | for (const auto &p : timestamps_to_update) 583 | { 584 | *p.first = p.second; 585 | } 586 | 587 | for (const auto &vid : new_vertex_cache) 588 | { 589 | graph.recycled_vertex_ids.push(vid); 590 | } 591 | 592 | for (const auto &p : block_cache) 593 | { 594 | graph.block_manager.free(p.first, p.second); 595 | } 596 | 597 | clean(); 598 | } 599 | 600 | EdgeIterator Transaction::get_edges(vertex_t src, label_t label, bool reverse) 601 | { 602 | check_valid(); 603 | 604 | if (src >= graph.vertex_id.load(std::memory_order_relaxed)) 605 | return EdgeIterator(nullptr, nullptr, 0, 0, read_epoch_id, local_txn_id, reverse); 606 | 607 | uintptr_t pointer; 608 | if (batch_update || !trace_cache) 609 | { 610 | pointer = locate_edge_block(src, label); 611 | } 612 | else 613 | { 614 | auto cache_iter = edge_ptr_cache.find(std::make_pair(src, label)); 615 | if (cache_iter != edge_ptr_cache.end()) 616 | { 617 | pointer = cache_iter->second; 618 | } 619 | else 620 | { 621 | pointer = locate_edge_block(src, label); 622 | // edge_ptr_cache.emplace_hint(cache_iter, std::make_pair(src, label), pointer); 623 | } 624 | } 625 | 626 | auto edge_block = graph.block_manager.convert(pointer); 627 | 628 | if (!edge_block) 629 | return EdgeIterator(nullptr, nullptr, 0, 0, read_epoch_id, local_txn_id, reverse); 630 | 631 | auto [num_entries, data_length] = get_num_entries_data_length_cache(edge_block); 632 | 633 | return EdgeIterator(edge_block->get_entries(), edge_block->get_data(), num_entries, data_length, read_epoch_id, 634 | local_txn_id, reverse); 635 | } 636 | 637 | timestamp_t Transaction::commit(bool wait_visable) 638 | { 639 | check_valid(); 640 | check_writable(); 641 | 642 | if (batch_update) 643 | return read_epoch_id; 644 | 645 | auto [commit_epoch_id, num_unfinished] = graph.commit_manager.register_commit(wal); 646 | 647 | for (const auto &p : vertex_ptr_cache) 648 | { 649 | auto vertex_id = p.first; 650 | auto pointer = p.second; 651 | if (graph.vertex_ptrs[vertex_id] != pointer) 652 | graph.vertex_ptrs[vertex_id] = pointer; 653 | } 654 | 655 | for (const auto &vid : recycled_vertex_cache) 656 | { 657 | graph.recycled_vertex_ids.push(vid); 658 | } 659 | 660 | for (const auto &p : edge_block_num_entries_data_length_cache) 661 | { 662 | p.first->set_num_entries_data_length_atomic(p.second.first, p.second.second); 663 | timestamps_to_update.emplace_back(p.first->get_committed_time_pointer(), p.first->get_committed_time()); 664 | p.first->set_committed_time(write_epoch_id); 665 | } 666 | 667 | for (const auto &p : edge_ptr_cache) 668 | { 669 | auto prev_pointer = locate_edge_block(p.first.first, p.first.second); 670 | if (p.second != prev_pointer) 671 | { 672 | update_edge_label_block(p.first.first, p.first.second, p.second); 673 | } 674 | } 675 | 676 | for (const auto &p : timestamps_to_update) 677 | { 678 | *p.first = commit_epoch_id; 679 | } 680 | 681 | clean(); 682 | 683 | graph.commit_manager.finish_commit(commit_epoch_id, num_unfinished, wait_visable); 684 | 685 | return commit_epoch_id; 686 | } 687 | --------------------------------------------------------------------------------