├── .clang-format ├── .gitignore ├── CMakeLists.txt ├── README.md ├── bandwidth_and_latency ├── CMakeLists.txt ├── benchmark_tpch_q1.py ├── benchmark_tpch_q14.py ├── dram.h ├── dram_bandwidth.cc ├── gbench_latencies.cc ├── lehmer64.h ├── splitmix64.h ├── ssd.h └── ssd_bandwidth.fio ├── benchmarks ├── clickhouse.md ├── dram_bandwidth_results.csv ├── duckdb.md ├── notebook.R ├── run.py ├── tpch_q1.csv ├── tpch_q14.csv └── umbra.md ├── cppcoro ├── .clang-format ├── CMakeLists.txt ├── LICENSE.txt ├── README.md ├── include │ └── cppcoro │ │ ├── allocator.hpp │ │ ├── awaitable_traits.hpp │ │ ├── broken_promise.hpp │ │ ├── coroutine.hpp │ │ ├── detail │ │ ├── any.hpp │ │ ├── get_awaiter.hpp │ │ ├── is_awaiter.hpp │ │ ├── lightweight_manual_reset_event.hpp │ │ ├── remove_rvalue_reference.hpp │ │ ├── sync_wait_task.hpp │ │ ├── unwrap_reference.hpp │ │ ├── void_value.hpp │ │ ├── when_all_counter.hpp │ │ ├── when_all_ready_awaitable.hpp │ │ └── when_all_task.hpp │ │ ├── is_awaitable.hpp │ │ ├── sync_wait.hpp │ │ ├── task.hpp │ │ └── when_all_ready.hpp └── lib │ └── lightweight_manual_reset_event.cpp ├── queries ├── CMakeLists.txt ├── tpch_q1.cc └── tpch_q14.cc └── storage ├── CMakeLists.txt └── src └── storage ├── file.cc ├── file.h ├── find_pattern.h ├── io_uring.h ├── load_data.cc ├── schema.h ├── swip.h ├── types.cc └── types.h /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | build 3 | data -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | project(async) 4 | 5 | set(CMAKE_CXX_STANDARD 20) 6 | add_compile_options(-Wall -Wextra -Werror -march=native -fcoroutines) 7 | 8 | # add_library(uring STATIC IMPORTED) 9 | # set_target_properties(uring PROPERTIES 10 | # IMPORTED_LOCATION "/PATH_TO_LIBURING/liburing/src/liburing.a" 11 | # INTERFACE_INCLUDE_DIRECTORIES "/PATH_TO_LIBURING/liburing/src/include" 12 | # ) 13 | 14 | if(NOT DEFINED ASYNCHRONOUS_IO_PAGE_SIZE_POWER) 15 | set(ASYNCHRONOUS_IO_PAGE_SIZE_POWER 16) 16 | endif() 17 | 18 | add_compile_definitions(ASYNCHRONOUS_IO_PAGE_SIZE_POWER=${ASYNCHRONOUS_IO_PAGE_SIZE_POWER}) 19 | message("ASYNCHRONOUS_IO_PAGE_SIZE_POWER = ${ASYNCHRONOUS_IO_PAGE_SIZE_POWER}") 20 | 21 | find_package(Threads REQUIRED) 22 | 23 | # add_compile_definitions(BENCH) 24 | # add_library(gbench STATIC IMPORTED) 25 | # set_target_properties(gbench PROPERTIES 26 | # IMPORTED_LOCATION "/PATH_TO_GBENCH/build/src/libbenchmark.a" 27 | # INTERFACE_INCLUDE_DIRECTORIES "/PATH_TO_GBENCH/include" 28 | # ) 29 | # add_subdirectory(bandwidth_and_latency) 30 | 31 | add_subdirectory(cppcoro) 32 | add_subdirectory(queries) 33 | add_subdirectory(storage) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What Are You Waiting For? Use Coroutines for Asynchronous I/O to Hide I/O Latencies and Maximize the Read Bandwidth! 2 | 3 | This repository contains micro-benchmarks to examine the benefits of asynchronous I/O for query processing using C++-Coroutines and `io_uring`. 4 | 5 | ## Build 6 | 7 | We depend on [liburing](https://github.com/axboe/liburing): 8 | 9 | ``` 10 | git clone https://github.com/axboe/liburing.git 11 | cd liburing 12 | ./configure 13 | make 14 | ``` 15 | 16 | Now, you can either make `liburing` available system-wide: 17 | 18 | ``` 19 | sudo make install 20 | ``` 21 | 22 | Or, you can uncomment and adapt the following lines in the project's `CMakeLists.txt`: 23 | 24 | ``` 25 | add_library(uring STATIC IMPORTED) 26 | set_target_properties(uring PROPERTIES 27 | IMPORTED_LOCATION "/PATH_TO_LIBURING/liburing/src/liburing.a" 28 | INTERFACE_INCLUDE_DIRECTORIES "/PATH_TO_LIBURING/liburing/src/include" 29 | ) 30 | ``` 31 | 32 | We only support building with GCC 11.3.0 or newer. 33 | You can specify the size of a database page by setting `-DASYNCHRONOUS_IO_PAGE_SIZE_POWER` (default: 16). 34 | An `ASYNCHRONOUS_IO_PAGE_SIZE_POWER` of 16 means 2^16 bytes per page. 35 | `ASYNCHRONOUS_IO_PAGE_SIZE_POWER` must be in the range [12, 22]. 36 | 37 | ``` 38 | mkdir build 39 | cd build 40 | cmake -G "Ninja" -DCMAKE_C_COMPILER=$(which gcc) -DCMAKE_CXX_COMPILER=$(which g++) -DCMAKE_BUILD_TYPE=Release -DASYNCHRONOUS_IO_PAGE_SIZE_POWER=19 .. 41 | ninja 42 | ``` 43 | 44 | ## Load Data 45 | 46 | Before you can load the data into our custom format, you need to generate it. 47 | Note, that you can specify the scale factor after the `-s` (default: 1). 48 | Scale factor of 1 is 1 GB of data. 49 | Adapt the path below to control in which directory the files are created. 50 | 51 | ``` 52 | git clone https://github.com/electrum/tpch-dbgen.git 53 | cd tpch-dbgen 54 | make 55 | DSS_PATH=/PATH/TO/DIR ./dbgen -s 1 56 | ``` 57 | 58 | Here is the help message of the `load_data` executable: 59 | 60 | ``` 61 | ./build/storage/load_data --help 62 | Usage: ./build/storage/load_data lineitemQ1 lineitem.tbl lineitemQ1.dat | lineitemQ14 lineitem.tbl lineitemQ14.dat | part part.tbl part.dat 63 | ``` 64 | 65 | To actually load the data, execute the following commands: 66 | 67 | ``` 68 | ./build/storage/load_data lineitemQ1 data/lineitem.tbl data/lineitemQ1.dat 69 | ./build/storage/load_data lineitemQ14 data/lineitem.tbl data/lineitemQ14.dat 70 | ./build/storage/load_data part data/part.tbl data/part.dat 71 | ``` 72 | 73 | ## Query 1 74 | 75 | ### Usage 76 | 77 | ``` 78 | ./build/queries/tpch_q1 --help 79 | Usage: ./build/queries/tpch_q1 lineitem.dat num_threads num_entries_per_ring num_tuples_per_morsel do_work do_random_io print_result print_header 80 | ``` 81 | 82 | ### Example 83 | 84 | ``` 85 | ./build/queries/tpch_q1 data/lineitemQ1.dat 128 128 1000 true true true true 86 | ``` 87 | 88 | ## Query 14 89 | 90 | ### Usage 91 | 92 | ``` 93 | ./build/queries/tpch_q14 --help 94 | Usage: ./build/queries/tpch_q14 lineitem.dat part.dat num_threads num_entries_per_ring num_tuples_per_coroutine print_result print_header 95 | ``` 96 | 97 | ### Example 98 | 99 | ``` 100 | ./build/queries/tpch_q14 data/lineitemQ14.dat data/part.dat 64 32 1000 true true 101 | ``` 102 | -------------------------------------------------------------------------------- /bandwidth_and_latency/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(gbench_latencies gbench_latencies.cc) 2 | target_link_libraries(gbench_latencies gbench) 3 | 4 | add_executable(dram_bandwidth dram_bandwidth.cc) 5 | target_link_libraries(dram_bandwidth Threads::Threads) -------------------------------------------------------------------------------- /bandwidth_and_latency/benchmark_tpch_q1.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import itertools 4 | 5 | path_to_build_directory = '/home/merzljak/async/build-bench' 6 | path_to_source_directory = '/home/merzljak/async' 7 | path_to_tpch_directory = '/raid0/data/tpch/sf100' 8 | path_to_data_directory = '/raid0/merzljak/data/sf100' 9 | path_to_cxx_compiler = '/usr/bin/g++' 10 | path_to_output = '/home/merzljak/async/benchmark_results/tpch_q1.csv' 11 | numactl = ['numactl', '--membind=0', '--cpubind=0'] 12 | 13 | page_size_power_list = [16, 17, 18] 14 | num_threads_list = [1, 2, 4, 8, 16, 32, 64, 128] 15 | num_entries_per_ring_list = [2, 4, 8, 16, 32, 64, 128, 256, 512] 16 | num_tuples_per_morsel_list = [500, 1_000, 5_000, 10_000, 100_000] 17 | 18 | output = open(path_to_output, 'w') 19 | print_header = "true" 20 | 21 | for page_size_power in page_size_power_list: 22 | # Configure the project 23 | print(f'Configure the project with page size power of {page_size_power}') 24 | subprocess.run(['cmake', '-S', path_to_source_directory, '-B', path_to_build_directory, '-DCMAKE_BUILD_TYPE=Release', 25 | f'-DASYNCHRONOUS_IO_PAGE_SIZE_POWER={page_size_power}', f'-DCMAKE_CXX_COMPILER={path_to_cxx_compiler}'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 26 | # Build the project 27 | print(f'Build the project with page size power of {page_size_power}') 28 | subprocess.run( 29 | ['cmake', '--build', path_to_build_directory, '--clean-first'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 30 | # Load the data 31 | print(f'Load the data with page size power of {page_size_power}') 32 | lineitem_dat = os.path.join(path_to_data_directory, 'lineitem.dat') 33 | subprocess.run(numactl + [os.path.join(path_to_build_directory, 'storage', 'load_data'), 34 | 'lineitemQ1', os.path.join(path_to_tpch_directory, 'lineitem.tbl'), lineitem_dat], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 35 | 36 | # Execute the query using all possible configurations 37 | for num_threads, num_entries_per_ring, num_tuples_per_morsel in itertools.product(num_threads_list, num_entries_per_ring_list, num_tuples_per_morsel_list): 38 | print('Run benchmark with the configuration {}'.format({('num_threads', num_threads), 39 | ('num_entries_per_ring', num_entries_per_ring), ('num_tuples_per_morsel', num_tuples_per_morsel)})) 40 | subprocess.run(numactl + [os.path.join(path_to_build_directory, 'queries', 'tpch_q1'), 41 | lineitem_dat, str(num_threads), str(num_entries_per_ring), str(num_tuples_per_morsel), "true", "false", "false", print_header], check=True, stdout=output, stderr=subprocess.PIPE) 42 | print_header = "false" 43 | -------------------------------------------------------------------------------- /bandwidth_and_latency/benchmark_tpch_q14.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import itertools 4 | 5 | path_to_build_directory = '/home/merzljak/async/build-bench' 6 | path_to_source_directory = '/home/merzljak/async' 7 | path_to_tpch_directory = '/raid0/merzljak/tpch/sf10' 8 | path_to_data_directory = '/raid0/merzljak/data/sf10' 9 | path_to_cxx_compiler = '/usr/bin/g++' 10 | path_to_output = '/home/merzljak/async/benchmark_results/tpch_q14.csv' 11 | numactl = ['numactl', '--membind=0', '--cpubind=0'] 12 | 13 | page_size_power_list = [12] 14 | num_threads_list = [1, 2, 4, 8, 16, 32, 64, 128] 15 | num_entries_per_ring_list = [8, 16, 32, 64, 128, 256, 512] 16 | num_tuples_per_morsel_list = [500, 1_000] 17 | 18 | output = open(path_to_output, 'w') 19 | print_header = "true" 20 | 21 | for page_size_power in page_size_power_list: 22 | # Configure the project 23 | print(f'Configure the project with page size power of {page_size_power}') 24 | subprocess.run(['cmake', '-S', path_to_source_directory, '-B', path_to_build_directory, '-DCMAKE_BUILD_TYPE=Release', 25 | f'-DASYNCHRONOUS_IO_PAGE_SIZE_POWER={page_size_power}', f'-DCMAKE_CXX_COMPILER={path_to_cxx_compiler}'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 26 | # Build the project 27 | print(f'Build the project with page size power of {page_size_power}') 28 | subprocess.run( 29 | ['cmake', '--build', path_to_build_directory, '--clean-first'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 30 | # Load the data 31 | print(f'Load the data with page size power of {page_size_power}') 32 | lineitem_dat = os.path.join(path_to_data_directory, 'lineitem.dat') 33 | part_dat = os.path.join(path_to_data_directory, 'part.dat') 34 | subprocess.run(numactl + [os.path.join(path_to_build_directory, 'storage', 'load_data'), 35 | 'lineitemQ14', os.path.join(path_to_tpch_directory, 'lineitem.tbl'), lineitem_dat], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 36 | subprocess.run(numactl + [os.path.join(path_to_build_directory, 'storage', 'load_data'), 37 | 'part', os.path.join(path_to_tpch_directory, 'part.tbl'), part_dat], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 38 | 39 | # Execute the query using all possible configurations 40 | for num_threads, num_entries_per_ring, num_tuples_per_morsel in itertools.product(num_threads_list, num_entries_per_ring_list, num_tuples_per_morsel_list): 41 | print('Run benchmark with the configuration {}'.format({('num_threads', num_threads), 42 | ('num_entries_per_ring', num_entries_per_ring), ('num_tuples_per_morsel', num_tuples_per_morsel)})) 43 | subprocess.run(numactl + [os.path.join(path_to_build_directory, 'queries', 'tpch_q14'), 44 | lineitem_dat, part_dat, str(num_threads), str(num_entries_per_ring), str(num_tuples_per_morsel), "false", print_header], check=True, stdout=output, stderr=subprocess.PIPE) 45 | print_header = "false" 46 | -------------------------------------------------------------------------------- /bandwidth_and_latency/dram.h: -------------------------------------------------------------------------------- 1 | #ifndef BANDWIDTH_AND_LATENCY_DRAM_H_ 2 | #define BANDWIDTH_AND_LATENCY_DRAM_H_ 3 | 4 | #include 5 | 6 | namespace dram { 7 | 8 | constexpr size_t kSizeOfCacheLine = 64ull; 9 | 10 | struct alignas(kSizeOfCacheLine) CacheLine { 11 | CacheLine* next; 12 | const std::array payload{1, 1, 1, 1, 1, 1, 1}; 13 | }; 14 | 15 | static_assert(sizeof(CacheLine) == kSizeOfCacheLine && 16 | alignof(CacheLine) == kSizeOfCacheLine); 17 | 18 | constexpr size_t kNumCacheLines1GiB = (1ull << 30) / kSizeOfCacheLine; 19 | constexpr size_t kNumCacheLines2GiB = (1ull << 31) / kSizeOfCacheLine; 20 | constexpr size_t kNumCacheLines4GiB = (1ull << 32) / kSizeOfCacheLine; 21 | constexpr size_t kNumCacheLines8GiB = (1ull << 33) / kSizeOfCacheLine; 22 | constexpr size_t kNumCacheLines16GiB = (1ull << 34) / kSizeOfCacheLine; 23 | constexpr size_t kNumCacheLines32GiB = (1ull << 35) / kSizeOfCacheLine; 24 | constexpr size_t kNumCacheLines64GiB = (1ull << 36) / kSizeOfCacheLine; 25 | constexpr size_t kNumCacheLines128GiB = (1ull << 37) / kSizeOfCacheLine; 26 | 27 | } // namespace dram 28 | 29 | #endif -------------------------------------------------------------------------------- /bandwidth_and_latency/dram_bandwidth.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "dram.h" 12 | #include "lehmer64.h" 13 | 14 | using namespace dram; 15 | 16 | namespace { 17 | 18 | constexpr size_t kNumCacheLines = kNumCacheLines128GiB; 19 | static_assert(std::has_single_bit(kNumCacheLines), 20 | "kNumCacheLines must be a power of 2"); 21 | 22 | // Return the bandwidth in GB/s 23 | double DoReads(std::span cache_lines, size_t num_threads, 24 | size_t max_num_iterations_per_thread, bool do_random_io) { 25 | std::chrono::steady_clock::time_point start_time_point; 26 | std::chrono::steady_clock::time_point stop_time_point; 27 | 28 | // The barrier is used twice. The first time, the completion function sets the 29 | // start_time_point. The second time, it sets the stop_time_point. 30 | std::barrier barrier(num_threads, 31 | [current = &start_time_point, 32 | next = &stop_time_point]() mutable noexcept { 33 | *current = std::chrono::steady_clock::now(); 34 | current = next; 35 | }); 36 | 37 | // After a thread has performed max_num_iterations_per_thread read operations, 38 | // it sets finished to true. All threads will stop immediately and write the 39 | // number of read operations they performed into iterations_per_thread. 40 | std::atomic finished{false}; 41 | std::vector iterations_per_thread(num_threads, 0ull); 42 | 43 | std::vector threads; 44 | threads.reserve(num_threads); 45 | 46 | for (size_t thread_idx = 0; thread_idx != num_threads; ++thread_idx) { 47 | threads.emplace_back([&barrier, cache_lines, &iterations_per_thread, 48 | thread_idx, max_num_iterations_per_thread, &finished, 49 | do_random_io, num_threads]() { 50 | if (do_random_io) { 51 | // To simulate random I/O, we use a fast random number generator: 52 | // https://lemire.me/blog/2019/03/19/the-fastest-conventional-random-number-generator-that-can-pass-big-crush/. 53 | // Each lehmer state gets a unique seed not used by any other thread. 54 | lehmer64_state_t lehmer_state1; 55 | lehmer64_seed(&lehmer_state1, 7 * thread_idx); 56 | lehmer64_state_t lehmer_state2; 57 | lehmer64_seed(&lehmer_state2, 7 * thread_idx + 1); 58 | lehmer64_state_t lehmer_state3; 59 | lehmer64_seed(&lehmer_state3, 7 * thread_idx + 2); 60 | 61 | // Start the benchmark 62 | barrier.arrive_and_wait(); 63 | 64 | size_t num_iterations1 = 0; 65 | size_t num_iterations2 = 0; 66 | size_t num_iterations3 = 0; 67 | for (size_t i = 0; i < max_num_iterations_per_thread && !finished; 68 | i += 3) { 69 | // cache_lines[i].payload.front() is 1 70 | num_iterations1 += 71 | cache_lines[lehmer64(&lehmer_state1) % kNumCacheLines] 72 | .payload.front(); 73 | num_iterations2 += 74 | cache_lines[lehmer64(&lehmer_state2) % kNumCacheLines] 75 | .payload.front(); 76 | num_iterations3 += 77 | cache_lines[lehmer64(&lehmer_state3) % kNumCacheLines] 78 | .payload.front(); 79 | } 80 | 81 | // Signal the other threads to stop 82 | finished = true; 83 | // Finish the benchmark 84 | barrier.arrive_and_wait(); 85 | 86 | iterations_per_thread[thread_idx] = 87 | num_iterations1 + num_iterations2 + num_iterations3; 88 | } else { 89 | // Let the threads start at different locations to minimize the 90 | // effectiveness of CPU caches 91 | size_t cache_lines_per_thread = 92 | (kNumCacheLines + num_threads - 1) / num_threads; 93 | auto begin_idx = thread_idx * cache_lines_per_thread; 94 | 95 | // Start the benchmark 96 | barrier.arrive_and_wait(); 97 | 98 | size_t num_iterations = 0; 99 | for (size_t i = begin_idx, 100 | end = max_num_iterations_per_thread + begin_idx; 101 | i != end && !finished; ++i) { 102 | // compute the next index 103 | size_t index = i % kNumCacheLines; 104 | // cache_lines[i].payload.front() is 1 105 | num_iterations += cache_lines[index].payload.front(); 106 | } 107 | 108 | // Signal the other threads to stop 109 | finished = true; 110 | // Finish the benchmark 111 | barrier.arrive_and_wait(); 112 | 113 | iterations_per_thread[thread_idx] = num_iterations; 114 | } 115 | }); 116 | } 117 | 118 | for (auto &t : threads) { 119 | t.join(); 120 | } 121 | 122 | auto total_iterations = std::accumulate(iterations_per_thread.begin(), 123 | iterations_per_thread.end(), 0ull); 124 | 125 | std::chrono::nanoseconds nanos{stop_time_point - start_time_point}; 126 | return total_iterations * sizeof(CacheLine) / double(nanos.count()); 127 | } 128 | 129 | void PrintCSVHeader() { std::cout << "access_pattern,num_threads,bandwidth\n"; } 130 | 131 | void PrintResult(const char *access_pattern, size_t num_threads, 132 | double bandwidth) { 133 | std::cout << access_pattern << "," << num_threads << "," << bandwidth << "\n"; 134 | } 135 | } // namespace 136 | 137 | int main() { 138 | PrintCSVHeader(); 139 | 140 | std::vector cache_lines(kNumCacheLines); 141 | 142 | // Sequential reads 143 | for (size_t num_threads = 1; num_threads <= 128; ++num_threads) { 144 | // If kNumCacheLines == kNumCacheLines128GiB, the threads read 1TiB in total 145 | PrintResult("sequential", num_threads, 146 | DoReads(cache_lines, num_threads, 147 | kNumCacheLines / num_threads * 8, false)); 148 | } 149 | 150 | // Random reads 151 | for (size_t num_threads = 1; num_threads <= 128; ++num_threads) { 152 | // If kNumCacheLines == kNumCacheLines128GiB, the threads read 1TiB in total 153 | PrintResult("random", num_threads, 154 | DoReads(cache_lines, num_threads, 155 | kNumCacheLines / num_threads * 8, true)); 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /bandwidth_and_latency/gbench_latencies.cc: -------------------------------------------------------------------------------- 1 | #ifdef BENCH 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "benchmark/benchmark.h" 10 | #include "dram.h" 11 | #include "lehmer64.h" 12 | #include "ssd.h" 13 | 14 | using namespace dram; 15 | using namespace ssd; 16 | 17 | // Create a 512 GiB large file with: `dd if=/dev/zero of=file.dat bs=1GiB 18 | // count=512` 19 | const char *kFileName = "/raid0/merzljak/io/file.dat"; 20 | 21 | // Write output to /dev/null to prevent certain compiler optimizations 22 | std::ofstream null{"/dev/null"}; 23 | 24 | static void BM_DRAMRandReadLatency(benchmark::State &state) { 25 | constexpr size_t kNumCacheLines = kNumCacheLines128GiB; 26 | 27 | std::vector data(kNumCacheLines); 28 | 29 | // Generate random sequence of indexes 30 | std::vector indexes(kNumCacheLines); 31 | std::iota(indexes.begin(), indexes.end(), 0ull); 32 | // Shuffle indexes 33 | std::random_device rd; 34 | std::mt19937 g(rd()); 35 | g.seed(42); 36 | std::shuffle(indexes.begin(), indexes.end(), g); 37 | 38 | // Initialize data 39 | for (size_t i = 0, end = kNumCacheLines - 1; i != end; ++i) { 40 | data[indexes[i]].next = &data[indexes[i + 1]]; 41 | } 42 | data[indexes.back()].next = &data[indexes.front()]; 43 | 44 | // Perform one (dependent) random read after another 45 | CacheLine *current = &data.front(); 46 | for (auto _ : state) { 47 | current = current->next; 48 | } 49 | null << current; 50 | } 51 | BENCHMARK(BM_DRAMRandReadLatency); 52 | 53 | static void Expect(bool predicate, const char *what = "Expect failed") { 54 | if (!predicate) { 55 | throw std::runtime_error{what}; 56 | } 57 | } 58 | 59 | static void SSDBench(benchmark::State &state, ssize_t page_size, 60 | bool do_random_io) { 61 | File file{kFileName}; 62 | size_t num_pages = file.file_size / page_size; 63 | auto entries = InitializeEntries(num_pages, page_size, do_random_io); 64 | 65 | // The buffer must be aligned since we perform direct I/O 66 | void *buffer = std::aligned_alloc(page_size, page_size); 67 | 68 | size_t i = 0; 69 | 70 | for (auto _ : state) { 71 | Expect(pread(file.fd, buffer, page_size, entries[i % num_pages].offset) == 72 | page_size); 73 | ++i; 74 | } 75 | 76 | std::free(buffer); 77 | } 78 | 79 | static void BM_SSDSeqReadLatency4KiB(benchmark::State &state) { 80 | ssize_t page_size = kPageSize4KiB; 81 | bool do_random_io = false; 82 | SSDBench(state, page_size, do_random_io); 83 | } 84 | BENCHMARK(BM_SSDSeqReadLatency4KiB); 85 | 86 | static void BM_SSDSeqReadLatency64KiB(benchmark::State &state) { 87 | ssize_t page_size = kPageSize64KiB; 88 | bool do_random_io = false; 89 | SSDBench(state, page_size, do_random_io); 90 | } 91 | BENCHMARK(BM_SSDSeqReadLatency64KiB); 92 | 93 | static void BM_SSDSeqReadLatency512KiB(benchmark::State &state) { 94 | ssize_t page_size = kPageSize512KiB; 95 | bool do_random_io = false; 96 | SSDBench(state, page_size, do_random_io); 97 | } 98 | BENCHMARK(BM_SSDSeqReadLatency512KiB); 99 | 100 | static void BM_SSDRandReadLatency4KiB(benchmark::State &state) { 101 | ssize_t page_size = kPageSize4KiB; 102 | bool do_random_io = true; 103 | SSDBench(state, page_size, do_random_io); 104 | } 105 | BENCHMARK(BM_SSDRandReadLatency4KiB); 106 | 107 | static void BM_SSDRandReadLatency64KiB(benchmark::State &state) { 108 | ssize_t page_size = kPageSize64KiB; 109 | bool do_random_io = true; 110 | SSDBench(state, page_size, do_random_io); 111 | } 112 | BENCHMARK(BM_SSDRandReadLatency64KiB); 113 | 114 | static void BM_SSDRandReadLatency512KiB(benchmark::State &state) { 115 | ssize_t page_size = kPageSize512KiB; 116 | bool do_random_io = true; 117 | SSDBench(state, page_size, do_random_io); 118 | } 119 | BENCHMARK(BM_SSDRandReadLatency512KiB); 120 | 121 | static void BM_Lehmer(benchmark::State &state) { 122 | constexpr uint64_t kPowerOfTwo = 1ull << 24; 123 | lehmer64_state_t lehmer_state; 124 | lehmer64_seed(&lehmer_state, 42); 125 | 126 | uint64_t sum = 0; 127 | for (auto _ : state) { 128 | sum += (lehmer64(&lehmer_state) % kPowerOfTwo); 129 | } 130 | null << sum; 131 | } 132 | BENCHMARK(BM_Lehmer); 133 | 134 | static void BM_Lehmer3(benchmark::State &state) { 135 | constexpr uint64_t kPowerOfTwo = 1ull << 24; 136 | lehmer64_state_t lehmer_state1; 137 | lehmer64_seed(&lehmer_state1, 1); 138 | lehmer64_state_t lehmer_state2; 139 | lehmer64_seed(&lehmer_state2, 3); 140 | lehmer64_state_t lehmer_state3; 141 | lehmer64_seed(&lehmer_state3, 5); 142 | 143 | uint64_t sum1 = 0; 144 | uint64_t sum2 = 0; 145 | uint64_t sum3 = 0; 146 | for (auto _ : state) { 147 | sum1 += (lehmer64(&lehmer_state1) % kPowerOfTwo); 148 | sum2 += (lehmer64(&lehmer_state2) % kPowerOfTwo); 149 | sum3 += (lehmer64(&lehmer_state3) % kPowerOfTwo); 150 | } 151 | null << sum1 + sum2 + sum3; 152 | } 153 | BENCHMARK(BM_Lehmer3); 154 | 155 | BENCHMARK_MAIN(); 156 | #endif -------------------------------------------------------------------------------- /bandwidth_and_latency/lehmer64.h: -------------------------------------------------------------------------------- 1 | // Code adapted from 2 | // https://github.com/lemire/testingRNG/blob/master/source/lehmer64.h 3 | 4 | #ifndef LEHMER64_H 5 | #define LEHMER64_H 6 | 7 | #include "splitmix64.h" 8 | 9 | using lehmer64_state_t = __uint128_t; 10 | 11 | /** 12 | * D. H. Lehmer, Mathematical methods in large-scale computing units. 13 | * Proceedings of a Second Symposium on Large Scale Digital Calculating 14 | * Machinery; 15 | * Annals of the Computation Laboratory, Harvard Univ. 26 (1951), pp. 141-146. 16 | * 17 | * P L'Ecuyer, Tables of linear congruential generators of different sizes and 18 | * good lattice structure. Mathematics of Computation of the American 19 | * Mathematical 20 | * Society 68.225 (1999): 249-260. 21 | */ 22 | 23 | static inline void lehmer64_seed(lehmer64_state_t* state, uint64_t seed) { 24 | *state = (((__uint128_t)splitmix64_stateless(seed)) << 64) + 25 | splitmix64_stateless(seed + 1); 26 | } 27 | 28 | static inline uint64_t lehmer64(lehmer64_state_t* state) { 29 | *state *= UINT64_C(0xda942042e4dd58b5); 30 | return *state >> 64; 31 | } 32 | 33 | #endif -------------------------------------------------------------------------------- /bandwidth_and_latency/splitmix64.h: -------------------------------------------------------------------------------- 1 | // Code adapted from 2 | // https://github.com/lemire/testingRNG/blob/master/source/splitmix64.h 3 | 4 | #ifndef SPLITMIX64_H 5 | #define SPLITMIX64_H 6 | 7 | /* Modified by D. Lemire, August 2017 */ 8 | /*** 9 | Fast Splittable Pseudorandom Number Generators 10 | Steele Jr, Guy L., Doug Lea, and Christine H. Flood. "Fast splittable 11 | pseudorandom number generators." 12 | ACM SIGPLAN Notices 49.10 (2014): 453-472. 13 | ***/ 14 | 15 | /* Written in 2015 by Sebastiano Vigna (vigna@acm.org) 16 | To the extent possible under law, the author has dedicated all copyright 17 | and related and neighboring rights to this software to the public domain 18 | worldwide. This software is distributed without any warranty. 19 | See . */ 20 | 21 | #include 22 | 23 | // original documentation by Vigna: 24 | /* This is a fixed-increment version of Java 8's SplittableRandom generator 25 | See http://dx.doi.org/10.1145/2714064.2660195 and 26 | http://docs.oracle.com/javase/8/docs/api/java/util/SplittableRandom.html 27 | It is a very fast generator passing BigCrush, and it can be useful if 28 | for some reason you absolutely want 64 bits of state; otherwise, we 29 | rather suggest to use a xoroshiro128+ (for moderately parallel 30 | computations) or xorshift1024* (for massively parallel computations) 31 | generator. */ 32 | 33 | // same as splitmix64, but does not change the state, designed by D. Lemire 34 | static inline uint64_t splitmix64_stateless(uint64_t index) { 35 | uint64_t z = (index + UINT64_C(0x9E3779B97F4A7C15)); 36 | z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9); 37 | z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB); 38 | return z ^ (z >> 31); 39 | } 40 | 41 | #endif // SPLITMIX64_H -------------------------------------------------------------------------------- /bandwidth_and_latency/ssd.h: -------------------------------------------------------------------------------- 1 | #ifndef BANDWIDTH_AND_LATENCY_SSD_H_ 2 | #define BANDWIDTH_AND_LATENCY_SSD_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace ssd { 17 | 18 | constexpr size_t kPageSize4KiB = 1ull << 12; 19 | constexpr size_t kPageSize64KiB = 1ull << 16; 20 | constexpr size_t kPageSize512KiB = 1ull << 19; 21 | 22 | struct File { 23 | File(const char *path_name) { 24 | int flags = O_RDONLY | O_NOATIME | O_DIRECT; 25 | fd = open(path_name, flags); 26 | if (fd == -1) { 27 | throw std::system_error{errno, std::generic_category()}; 28 | } 29 | file_size = lseek(fd, 0, SEEK_END); 30 | } 31 | 32 | ~File() { close(fd); } 33 | 34 | int fd; 35 | off_t file_size; 36 | }; 37 | 38 | struct FileOffsetEntry { 39 | off_t offset; 40 | }; 41 | 42 | inline std::vector InitializeEntries(size_t num_pages, 43 | size_t page_size, 44 | bool do_random_io) { 45 | std::vector entries(num_pages); 46 | 47 | for (size_t i = 0; i != num_pages; ++i) { 48 | entries[i].offset = i * page_size; 49 | } 50 | 51 | if (do_random_io) { 52 | std::random_device rd; 53 | std::mt19937 g(rd()); 54 | g.seed(42); 55 | 56 | std::shuffle(entries.begin(), entries.end(), g); 57 | } 58 | 59 | return entries; 60 | } 61 | 62 | } // namespace ssd 63 | 64 | #endif -------------------------------------------------------------------------------- /bandwidth_and_latency/ssd_bandwidth.fio: -------------------------------------------------------------------------------- 1 | [global] 2 | kb_base=1000 # Inputs comply with IEC 80000-13 and the International System of Units (SI) 3 | runtime=30s # Tell fio to terminate processing after the specified period of time 4 | time_based # If set, fio will run for the duration of the runtime specified even if the file(s) are completely read or written 5 | directory=/raid0/merzljak/fio/ 6 | filename=test.dat # We want to share files between jobs 7 | size=256GiB # The size of the file 8 | direct=1 # Use direct I/O 9 | blocksize=256KiB # The block size used for I/O units 10 | ioengine=io_uring # Fast Linux native asynchronous I/O 11 | iodepth=128 # Number of I/O units to keep in flight against the file. 12 | thread # fio will create jobs by using POSIX Threads’ function pthread_create(3) to create threads 13 | group_reporting # To see the final report per-group instead of per-job 14 | 15 | [seq-read] 16 | readwrite=read # Sequential reads 17 | numjobs=4 # Create 4 clones of this job 18 | stonewall 19 | 20 | [rand-read] 21 | readwrite=randread # Random reads 22 | numjobs=4 # Create 4 clones of this job 23 | stonewall -------------------------------------------------------------------------------- /benchmarks/clickhouse.md: -------------------------------------------------------------------------------- 1 | # ClickHouse 2 | 3 | ## Build ClickHouse 4 | 5 | ```sh 6 | git clone https://github.com/ClickHouse/ClickHouse.git 7 | cd ClickHouse/ 8 | git submodule update --init --recursive 9 | mkdir build 10 | cd build 11 | export CC=clang CXX=clang++ 12 | cmake -D CMAKE_BUILD_TYPE=Release .. 13 | ninja clickhouse-server clickhouse-client 14 | ``` 15 | 16 | ## Run ClickHouse Server 17 | 18 | ```sh 19 | cd programs/server/ 20 | numactl --membind=0 --cpubind=0 ../../build/programs/clickhouse server 21 | ``` 22 | 23 | ## Run ClickHouse Client 24 | 25 | ```sh 26 | ./build/programs/clickhouse client --multiline --multiquery 27 | ``` 28 | 29 | ## Create schema 30 | 31 | ``` 32 | CREATE TABLE part 33 | ( 34 | p_partkey Int32 NOT NULL, 35 | p_name String NOT NULL, 36 | p_mfgr FixedString(25) NOT NULL, 37 | p_brand FixedString(10) NOT NULL, 38 | p_type String NOT NULL, 39 | p_size Int32 NOT NULL, 40 | p_container FixedString(10) NOT NULL, 41 | p_retailprice Decimal(12,2) NOT NULL, 42 | p_comment String NOT NULL 43 | ) ENGINE = Memory; 44 | 45 | CREATE TABLE supplier 46 | ( 47 | s_suppkey Int32 NOT NULL, 48 | s_name FixedString(25) NOT NULL, 49 | s_address String NOT NULL, 50 | s_nationkey Int32 NOT NULL, 51 | s_phone FixedString(15) NOT NULL, 52 | s_acctbal Decimal(12,2) NOT NULL, 53 | s_comment String NOT NULL 54 | ) ENGINE = Memory; 55 | 56 | CREATE TABLE partsupp 57 | ( 58 | ps_partkey Int32 NOT NULL, 59 | ps_suppkey Int32 NOT NULL, 60 | ps_availqty Int32 NOT NULL, 61 | ps_supplycost Decimal(12,2) NOT NULL, 62 | ps_comment String NOT NULL 63 | ) ENGINE = Memory; 64 | 65 | CREATE TABLE customer 66 | ( 67 | c_custkey Int32 NOT NULL, 68 | c_name String NOT NULL, 69 | c_address String NOT NULL, 70 | c_nationkey Int32 NOT NULL, 71 | c_phone FixedString(15) NOT NULL, 72 | c_acctbal Decimal(12,2) NOT NULL, 73 | c_mktsegment FixedString(10) NOT NULL, 74 | c_comment String NOT NULL 75 | ) ENGINE = Memory; 76 | 77 | CREATE TABLE orders 78 | ( 79 | o_orderkey Int32 NOT NULL, 80 | o_custkey Int32 NOT NULL, 81 | o_orderstatus FixedString(1) NOT NULL, 82 | o_totalprice Decimal(12,2) NOT NULL, 83 | o_orderdate Date NOT NULL, 84 | o_orderpriority FixedString(15) NOT NULL, 85 | o_clerk FixedString(15) NOT NULL, 86 | o_shippriority Int32 NOT NULL, 87 | o_comment String NOT NULL 88 | ) ENGINE = Memory; 89 | 90 | CREATE TABLE lineitem 91 | ( 92 | l_orderkey Int32 NOT NULL, 93 | l_partkey Int32 NOT NULL, 94 | l_suppkey Int32 NOT NULL, 95 | l_linenumber Int32 NOT NULL, 96 | l_quantity Decimal(12,2) NOT NULL, 97 | l_extendedprice Decimal(12,2) NOT NULL, 98 | l_discount Decimal(12,2) NOT NULL, 99 | l_tax Decimal(12,2) NOT NULL, 100 | l_returnflag FixedString(1) NOT NULL, 101 | l_linestatus FixedString(1) NOT NULL, 102 | l_shipdate Date NOT NULL, 103 | l_commitdate Date NOT NULL, 104 | l_receiptdate Date NOT NULL, 105 | l_shipinstruct FixedString(25) NOT NULL, 106 | l_shipmode FixedString(10) NOT NULL, 107 | l_comment String NOT NULL 108 | ) ENGINE = Memory; 109 | 110 | CREATE TABLE nation 111 | ( 112 | n_nationkey Int32 NOT NULL, 113 | n_name FixedString(25) NOT NULL, 114 | n_regionkey Int32 NOT NULL, 115 | n_comment String NOT NULL 116 | ) ENGINE = Memory; 117 | 118 | CREATE TABLE region 119 | ( 120 | r_regionkey Int32 NOT NULL, 121 | r_name FixedString(25) NOT NULL, 122 | r_comment String NOT NULL 123 | ) ENGINE = Memory; 124 | 125 | ``` 126 | 127 | ## Load data 128 | 129 | ``` 130 | SET max_memory_usage = 0; 131 | SET max_threads = 64; 132 | SET format_csv_delimiter = '|'; 133 | 134 | INSERT INTO part FROM INFILE '/raid0/data/tpch/sf30/part.tbl' FORMAT CSV; 135 | INSERT INTO supplier FROM INFILE '/raid0/data/tpch/sf30/supplier.tbl' FORMAT CSV; 136 | INSERT INTO partsupp FROM INFILE '/raid0/data/tpch/sf30/partsupp.tbl' FORMAT CSV; 137 | INSERT INTO customer FROM INFILE '/raid0/data/tpch/sf30/customer.tbl' FORMAT CSV; 138 | INSERT INTO orders FROM INFILE '/raid0/data/tpch/sf30/orders.tbl' FORMAT CSV; 139 | INSERT INTO lineitem FROM INFILE '/raid0/data/tpch/sf30/lineitem.tbl' FORMAT CSV; 140 | INSERT INTO nation FROM INFILE '/raid0/data/tpch/sf30/nation.tbl' FORMAT CSV; 141 | INSERT INTO region FROM INFILE '/raid0/data/tpch/sf30/region.tbl' FORMAT CSV; 142 | ``` 143 | 144 | ## Queries 145 | 146 | ### Query 1 147 | 148 | ``` 149 | select 150 | l_returnflag, 151 | l_linestatus, 152 | sum(l_quantity) as sum_qty, 153 | sum(l_extendedprice) as sum_base_price, 154 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 155 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 156 | avg(l_quantity) as avg_qty, 157 | avg(l_extendedprice) as avg_price, 158 | avg(l_discount) as avg_disc, 159 | count(*) as count_order 160 | from 161 | lineitem 162 | where 163 | l_shipdate <= date '1998-12-01' - interval '90' day 164 | group by 165 | l_returnflag, 166 | l_linestatus 167 | order by 168 | l_returnflag, 169 | l_linestatus; 170 | ``` 171 | 172 | ### Query 2 173 | 174 | ``` 175 | select 176 | s_acctbal, 177 | s_name, 178 | n_name, 179 | p_partkey, 180 | p_mfgr, 181 | s_address, 182 | s_phone, 183 | s_comment 184 | from 185 | part, 186 | supplier, 187 | partsupp, 188 | nation, 189 | region, 190 | ( 191 | select 192 | min(ps_supplycost) min_ps, ps_partkey pr_key 193 | from 194 | partsupp, 195 | supplier, 196 | nation, 197 | region 198 | where 199 | s_suppkey = ps_suppkey 200 | and s_nationkey = n_nationkey 201 | and n_regionkey = r_regionkey 202 | and r_name = 'EUROPE' 203 | group by ps_partkey 204 | ) precomp 205 | where 206 | p_partkey = ps_partkey 207 | and s_suppkey = ps_suppkey 208 | and p_size = 15 209 | and p_type like '%BRASS' 210 | and s_nationkey = n_nationkey 211 | and n_regionkey = r_regionkey 212 | and r_name = 'EUROPE' 213 | and ps_supplycost = min_ps 214 | and p_partkey = pr_key 215 | order by 216 | s_acctbal desc, 217 | n_name, 218 | s_name, 219 | p_partkey 220 | limit 221 | 100; 222 | ``` 223 | 224 | ``` 225 | 226 | SELECT min(ps_supplycost) AS min_ps, ps_partkey AS pr_key 227 | FROM partsupp INNER ALL JOIN supplier ON ps_suppkey = s_suppkey INNER ALL JOIN nation ON s_nationkey = n_nationkey INNER ALL JOIN region ON n_regionkey = r_regionkey 228 | WHERE r_name = 'EUROPE' 229 | GROUP BY ps_partkey 230 | ``` 231 | 232 | ### Query 14 233 | 234 | ``` 235 | EXPLAIN SYNTAX SELECT 236 | 100.00 * sum(case 237 | when p_type like 'PROMO%' 238 | then l_extendedprice * (1 - l_discount) 239 | else 0 240 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 241 | FROM part INNER ALL JOIN lineitem ON p_partkey = l_partkey 242 | WHERE 243 | l_shipdate >= date '1995-09-01' 244 | and l_shipdate < date '1995-09-01' + interval '1' month; 245 | ``` -------------------------------------------------------------------------------- /benchmarks/dram_bandwidth_results.csv: -------------------------------------------------------------------------------- 1 | access_pattern,num_threads,bandwidth 2 | Sequential,1,24.4757 3 | Sequential,2,45.8663 4 | Sequential,3,66.3951 5 | Sequential,4,84.1155 6 | Sequential,5,99.7546 7 | Sequential,6,112.019 8 | Sequential,7,122.037 9 | Sequential,8,127.625 10 | Sequential,9,128.55 11 | Sequential,10,131.485 12 | Sequential,11,132.847 13 | Sequential,12,135.184 14 | Sequential,13,136.683 15 | Sequential,14,142.995 16 | Sequential,15,141.899 17 | Sequential,16,138.569 18 | Sequential,17,141.65 19 | Sequential,18,143.57 20 | Sequential,19,144.115 21 | Sequential,20,142.953 22 | Sequential,21,143.066 23 | Sequential,22,145.178 24 | Sequential,23,142.906 25 | Sequential,24,143.298 26 | Sequential,25,150.318 27 | Sequential,26,142.862 28 | Sequential,27,148.608 29 | Sequential,28,142.43 30 | Sequential,29,146.654 31 | Sequential,30,141.045 32 | Sequential,31,143.581 33 | Sequential,32,146.864 34 | Sequential,33,144.227 35 | Sequential,34,146.238 36 | Sequential,35,150.444 37 | Sequential,36,144.141 38 | Sequential,37,147.327 39 | Sequential,38,145.727 40 | Sequential,39,145.762 41 | Sequential,40,146.509 42 | Sequential,41,146.641 43 | Sequential,42,146.644 44 | Sequential,43,147.046 45 | Sequential,44,146.29 46 | Sequential,45,146.967 47 | Sequential,46,146.97 48 | Sequential,47,147.575 49 | Sequential,48,146.581 50 | Sequential,49,146.897 51 | Sequential,50,146.232 52 | Sequential,51,152.14 53 | Sequential,52,146.866 54 | Sequential,53,147.413 55 | Sequential,54,147.44 56 | Sequential,55,148.088 57 | Sequential,56,142.448 58 | Sequential,57,147.896 59 | Sequential,58,147.519 60 | Sequential,59,148.339 61 | Sequential,60,147.719 62 | Sequential,61,147.576 63 | Sequential,62,147.428 64 | Sequential,63,147.99 65 | Sequential,64,148.445 66 | Sequential,65,148.63 67 | Sequential,66,148.621 68 | Sequential,67,148.558 69 | Sequential,68,147.76 70 | Sequential,69,149.103 71 | Sequential,70,148.372 72 | Sequential,71,147.602 73 | Sequential,72,148.904 74 | Sequential,73,148.949 75 | Sequential,74,149.098 76 | Sequential,75,147.098 77 | Sequential,76,149.376 78 | Sequential,77,149.821 79 | Sequential,78,150.496 80 | Sequential,79,149.594 81 | Sequential,80,149.845 82 | Sequential,81,149.568 83 | Sequential,82,149.619 84 | Sequential,83,148.932 85 | Sequential,84,149.281 86 | Sequential,85,148.821 87 | Sequential,86,149.239 88 | Sequential,87,150.763 89 | Sequential,88,149.894 90 | Sequential,89,148.066 91 | Sequential,90,149.881 92 | Sequential,91,150.184 93 | Sequential,92,149.004 94 | Sequential,93,149.869 95 | Sequential,94,150.081 96 | Sequential,95,150.431 97 | Sequential,96,150.289 98 | Sequential,97,148.958 99 | Sequential,98,149.897 100 | Sequential,99,150.459 101 | Sequential,100,150.495 102 | Sequential,101,150.755 103 | Sequential,102,150.541 104 | Sequential,103,150.896 105 | Sequential,104,149.236 106 | Sequential,105,150.419 107 | Sequential,106,150.821 108 | Sequential,107,150.986 109 | Sequential,108,150.73 110 | Sequential,109,150.935 111 | Sequential,110,149.258 112 | Sequential,111,150.814 113 | Sequential,112,150.394 114 | Sequential,113,150.779 115 | Sequential,114,151.1 116 | Sequential,115,150.242 117 | Sequential,116,150.655 118 | Sequential,117,151.096 119 | Sequential,118,151.003 120 | Sequential,119,151.308 121 | Sequential,120,151.005 122 | Sequential,121,150.21 123 | Sequential,122,151.154 124 | Sequential,123,150.406 125 | Sequential,124,150.912 126 | Sequential,125,151.459 127 | Sequential,126,151.825 128 | Sequential,127,151.784 129 | Sequential,128,151.762 130 | Random,1,2.79037 131 | Random,2,4.64803 132 | Random,3,7.10846 133 | Random,4,8.94636 134 | Random,5,11.5075 135 | Random,6,13.6972 136 | Random,7,16.1832 137 | Random,8,19.0428 138 | Random,9,21.2576 139 | Random,10,22.1526 140 | Random,11,24.2305 141 | Random,12,26.2835 142 | Random,13,28.3851 143 | Random,14,30.455 144 | Random,15,32.1501 145 | Random,16,33.9413 146 | Random,17,35.8726 147 | Random,18,37.7734 148 | Random,19,40.0165 149 | Random,20,40.9785 150 | Random,21,43.3697 151 | Random,22,44.4926 152 | Random,23,45.7991 153 | Random,24,47.0566 154 | Random,25,47.996 155 | Random,26,48.6389 156 | Random,27,48.9198 157 | Random,28,49.6905 158 | Random,29,50.4635 159 | Random,30,49.8918 160 | Random,31,49.8096 161 | Random,32,51.1304 162 | Random,33,51.1021 163 | Random,34,51.9087 164 | Random,35,52.696 165 | Random,36,54.0661 166 | Random,37,52.4711 167 | Random,38,52.4682 168 | Random,39,51.9207 169 | Random,40,52.6476 170 | Random,41,52.8563 171 | Random,42,52.1282 172 | Random,43,53.7692 173 | Random,44,52.1848 174 | Random,45,52.1609 175 | Random,46,54.7341 176 | Random,47,52.1729 177 | Random,48,52.5673 178 | Random,49,54.5921 179 | Random,50,52.3235 180 | Random,51,58.4019 181 | Random,52,61.3183 182 | Random,53,61.322 183 | Random,54,61.5341 184 | Random,55,61.2547 185 | Random,56,61.3956 186 | Random,57,62.1031 187 | Random,58,62.0367 188 | Random,59,61.3212 189 | Random,60,60.5085 190 | Random,61,60.6428 191 | Random,62,60.8124 192 | Random,63,60.7341 193 | Random,64,62.5005 194 | Random,65,61.8691 195 | Random,66,60.1009 196 | Random,67,60.3387 197 | Random,68,60.5261 198 | Random,69,60.5575 199 | Random,70,59.4994 200 | Random,71,59.7664 201 | Random,72,73.6529 202 | Random,73,62.2183 203 | Random,74,59.5615 204 | Random,75,59.3149 205 | Random,76,58.9872 206 | Random,77,62.8114 207 | Random,78,59.5815 208 | Random,79,61.6374 209 | Random,80,59.8587 210 | Random,81,61.3118 211 | Random,82,63.3934 212 | Random,83,60.5603 213 | Random,84,60.235 214 | Random,85,58.8784 215 | Random,86,59.9088 216 | Random,87,60.0921 217 | Random,88,60.3789 218 | Random,89,78.0627 219 | Random,90,59.192 220 | Random,91,59.7059 221 | Random,92,60.2354 222 | Random,93,66.3726 223 | Random,94,58.0351 224 | Random,95,59.386 225 | Random,96,58.2486 226 | Random,97,65.8506 227 | Random,98,58.595 228 | Random,99,58.4387 229 | Random,100,74.7763 230 | Random,101,59.0462 231 | Random,102,59.2876 232 | Random,103,58.5477 233 | Random,104,70.3271 234 | Random,105,61.6389 235 | Random,106,57.8688 236 | Random,107,58.612 237 | Random,108,58.4587 238 | Random,109,58.7332 239 | Random,110,60.2815 240 | Random,111,58.348 241 | Random,112,62.5105 242 | Random,113,75.543 243 | Random,114,57.9621 244 | Random,115,57.6823 245 | Random,116,73.8017 246 | Random,117,58.4714 247 | Random,118,57.8147 248 | Random,119,58.1099 249 | Random,120,59.0503 250 | Random,121,62.6081 251 | Random,122,62.6885 252 | Random,123,57.8498 253 | Random,124,57.792 254 | Random,125,65.4816 255 | Random,126,57.225 256 | Random,127,71.5299 257 | Random,128,62.0446 258 | -------------------------------------------------------------------------------- /benchmarks/duckdb.md: -------------------------------------------------------------------------------- 1 | # DuckDB 2 | 3 | ## Build 4 | 5 | ``` 6 | git clone https://github.com/duckdb/duckdb.git 7 | cd duckdb/ 8 | mkdir build 9 | cd build 10 | export CC=clang CXX=clang++ 11 | cmake -G "Ninja" -D CMAKE_BUILD_TYPE=Release .. 12 | ninja 13 | ``` 14 | 15 | ## Start CLI 16 | 17 | ``` 18 | numactl --membind=0 --cpubind=0 ./duckdb 19 | PRAGMA threads=64; 20 | .timer ON 21 | ``` 22 | 23 | ## Create schema 24 | 25 | ``` 26 | create table part ( 27 | p_partkey integer not null, 28 | p_name varchar(55) not null, 29 | p_mfgr char(25) not null, 30 | p_brand char(10) not null, 31 | p_type varchar(25) not null, 32 | p_size integer not null, 33 | p_container char(10) not null, 34 | p_retailprice decimal(12,2) not null, 35 | p_comment varchar(23) not null, 36 | primary key (p_partkey) 37 | ); 38 | 39 | create table supplier ( 40 | s_suppkey integer not null, 41 | s_name char(25) not null, 42 | s_address varchar(40) not null, 43 | s_nationkey integer not null, 44 | s_phone char(15) not null, 45 | s_acctbal decimal(12,2) not null, 46 | s_comment varchar(101) not null, 47 | primary key (s_suppkey) 48 | ); 49 | 50 | create table partsupp ( 51 | ps_partkey integer not null, 52 | ps_suppkey integer not null, 53 | ps_availqty integer not null, 54 | ps_supplycost decimal(12,2) not null, 55 | ps_comment varchar(199) not null, 56 | primary key (ps_partkey,ps_suppkey) 57 | ); 58 | 59 | create table customer ( 60 | c_custkey integer not null, 61 | c_name varchar(25) not null, 62 | c_address varchar(40) not null, 63 | c_nationkey integer not null, 64 | c_phone char(15) not null, 65 | c_acctbal decimal(12,2) not null, 66 | c_mktsegment char(10) not null, 67 | c_comment varchar(117) not null, 68 | primary key (c_custkey) 69 | ); 70 | 71 | create table orders ( 72 | o_orderkey integer not null, 73 | o_custkey integer not null, 74 | o_orderstatus char(1) not null, 75 | o_totalprice decimal(12,2) not null, 76 | o_orderdate date not null, 77 | o_orderpriority char(15) not null, 78 | o_clerk char(15) not null, 79 | o_shippriority integer not null, 80 | o_comment varchar(79) not null, 81 | primary key (o_orderkey) 82 | ); 83 | 84 | create table lineitem ( 85 | l_orderkey integer not null, 86 | l_partkey integer not null, 87 | l_suppkey integer not null, 88 | l_linenumber integer not null, 89 | l_quantity decimal(12,2) not null, 90 | l_extendedprice decimal(12,2) not null, 91 | l_discount decimal(12,2) not null, 92 | l_tax decimal(12,2) not null, 93 | l_returnflag char(1) not null, 94 | l_linestatus char(1) not null, 95 | l_shipdate date not null, 96 | l_commitdate date not null, 97 | l_receiptdate date not null, 98 | l_shipinstruct char(25) not null, 99 | l_shipmode char(10) not null, 100 | l_comment varchar(44) not null, 101 | primary key (l_orderkey,l_linenumber) 102 | ); 103 | 104 | create table nation ( 105 | n_nationkey integer not null, 106 | n_name char(25) not null, 107 | n_regionkey integer not null, 108 | n_comment varchar(152) not null, 109 | primary key (n_nationkey) 110 | ); 111 | 112 | create table region ( 113 | r_regionkey integer not null, 114 | r_name char(25) not null, 115 | r_comment varchar(152) not null, 116 | primary key (r_regionkey) 117 | ); 118 | ``` 119 | 120 | ## Load data 121 | 122 | ``` 123 | COPY part from '/raid0/data/tpch/sf30/part.tbl' ( DELIMITER '|' ); 124 | COPY supplier from '/raid0/data/tpch/sf30/supplier.tbl' ( DELIMITER '|' ); 125 | COPY partsupp from '/raid0/data/tpch/sf30/partsupp.tbl' ( DELIMITER '|' ); 126 | COPY customer from '/raid0/data/tpch/sf30/customer.tbl' ( DELIMITER '|' ); 127 | COPY orders from '/raid0/data/tpch/sf30/orders.tbl' ( DELIMITER '|' ); 128 | COPY lineitem from '/raid0/data/tpch/sf30/lineitem.tbl' ( DELIMITER '|' ); 129 | COPY nation from '/raid0/data/tpch/sf30/nation.tbl' ( DELIMITER '|' ); 130 | COPY region from '/raid0/data/tpch/sf30/region.tbl' ( DELIMITER '|' ); 131 | ``` 132 | 133 | ## Queries 134 | 135 | ### Query 1 136 | 137 | ``` 138 | select 139 | l_returnflag, 140 | l_linestatus, 141 | sum(l_quantity) as sum_qty, 142 | sum(l_extendedprice) as sum_base_price, 143 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 144 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 145 | avg(l_quantity) as avg_qty, 146 | avg(l_extendedprice) as avg_price, 147 | avg(l_discount) as avg_disc, 148 | count(*) as count_order 149 | from 150 | lineitem 151 | where 152 | l_shipdate <= date '1998-12-01' - interval '90' day 153 | group by 154 | l_returnflag, 155 | l_linestatus 156 | order by 157 | l_returnflag, 158 | l_linestatus; 159 | ``` 160 | 161 | ### Query 2 162 | 163 | ``` 164 | select 165 | s_acctbal, 166 | s_name, 167 | n_name, 168 | p_partkey, 169 | p_mfgr, 170 | s_address, 171 | s_phone, 172 | s_comment 173 | from 174 | part, 175 | supplier, 176 | partsupp, 177 | nation, 178 | region 179 | where 180 | p_partkey = ps_partkey 181 | and s_suppkey = ps_suppkey 182 | and p_size = 15 183 | and p_type like '%BRASS' 184 | and s_nationkey = n_nationkey 185 | and n_regionkey = r_regionkey 186 | and r_name = 'EUROPE' 187 | and ps_supplycost = ( 188 | select 189 | min(ps_supplycost) 190 | from 191 | partsupp, 192 | supplier, 193 | nation, 194 | region 195 | where 196 | p_partkey = ps_partkey 197 | and s_suppkey = ps_suppkey 198 | and s_nationkey = n_nationkey 199 | and n_regionkey = r_regionkey 200 | and r_name = 'EUROPE' 201 | ) 202 | order by 203 | s_acctbal desc, 204 | n_name, 205 | s_name, 206 | p_partkey 207 | limit 208 | 100; 209 | ``` -------------------------------------------------------------------------------- /benchmarks/notebook.R: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ## TPC-H Q1 4 | 5 | ```{r} 6 | library(tidyverse) 7 | results <- read_csv("/Users/leonard/Desktop/tpch_q1.csv") 8 | results %>% 9 | filter(page_size_power == 16) %>% 10 | mutate(percent_cached = factor((num_cached_pages * 100) %/% num_total_pages)) %>% 11 | select(num_threads, 12 | percent_cached, 13 | num_entries_per_ring, 14 | throughput) %>% 15 | group_by(num_threads, percent_cached, num_entries_per_ring) %>% 16 | summarise(throughput = max(throughput), 17 | .groups = 'drop') %>% 18 | mutate( 19 | num_threads = recode( 20 | factor(num_threads), 21 | "1" = "1 Thread", 22 | "2" = "2 Threads", 23 | "4" = "4 Threads", 24 | "8" = "8 Threads", 25 | "16" = "16 Threads", 26 | "32" = "32 Threads", 27 | "64" = "64 Threads", 28 | "128" = "128 Threads" 29 | ) 30 | ) %>% 31 | ggplot() + 32 | geom_col( 33 | mapping = aes( 34 | x = percent_cached, 35 | y = throughput, 36 | fill = factor(num_entries_per_ring) 37 | ), 38 | position = "dodge" 39 | ) + 40 | facet_grid(cols = vars(num_threads)) + 41 | labs(x = "Fraction of pages cached in DRAM (in %)", 42 | y = "Bandwidth of processing TPC-H Q1 (in GB/s)", 43 | fill = "Kind of I/O (I/O depth per thread)") + 44 | scale_fill_discrete( 45 | limits = c("0", "2", "4", "8", "16", "32", "64", "128", "256", "512"), 46 | labels = c( 47 | "Sync. (1)", 48 | "Async. (2)", 49 | "Async. (4)", 50 | "Async. (8)", 51 | "Async. (16)", 52 | "Async. (32)", 53 | "Async. (64)", 54 | "Async. (128)", 55 | "Async. (256)", 56 | "Async. (512)" 57 | ) 58 | ) + 59 | scale_y_continuous(breaks = scales::pretty_breaks(n = 30)) + 60 | theme(legend.position = "bottom") 61 | ``` 62 | 63 | ## TPC-H Q14 64 | 65 | ```{r} 66 | library(tidyverse) 67 | results <- read_csv("/Users/leonard/Desktop/tpch_q14.csv") 68 | results %>% 69 | filter(num_cached_references < num_total_references) %>% 70 | mutate(percent_cached = factor((num_cached_references * 100) %/% num_total_references)) %>% 71 | mutate(lookups_per_second = num_total_references / (time / 1000)) %>% 72 | select(num_threads, 73 | percent_cached, 74 | num_entries_per_ring, 75 | lookups_per_second) %>% 76 | group_by(num_threads, percent_cached, num_entries_per_ring) %>% 77 | summarise(lookups_per_second = max(lookups_per_second), 78 | .groups = 'drop') %>% 79 | mutate( 80 | num_threads = recode( 81 | factor(num_threads), 82 | "1" = "1 Thread", 83 | "2" = "2 Threads", 84 | "4" = "4 Threads", 85 | "8" = "8 Threads", 86 | "16" = "16 Threads", 87 | "32" = "32 Threads", 88 | "64" = "64 Threads", 89 | "128" = "128 Threads" 90 | ) 91 | ) %>% 92 | ggplot() + 93 | geom_col( 94 | mapping = aes( 95 | x = percent_cached, 96 | y = lookups_per_second, 97 | fill = factor(num_entries_per_ring) 98 | ), 99 | position = "dodge" 100 | ) + 101 | facet_grid(cols = vars(num_threads)) + 102 | labs(x = "Fraction of index accesses that are a cache hit (in %)", 103 | y = "Number of lookups per second", 104 | fill = "Kind of I/O (I/O depth per thread)") + 105 | scale_fill_discrete( 106 | limits = c("0", "8", "16", "32", "64", "128", "256", "512"), 107 | labels = c( 108 | "Sync. (1)", 109 | "Async. (8)", 110 | "Async. (16)", 111 | "Async. (32)", 112 | "Async. (64)", 113 | "Async. (128)", 114 | "Async. (256)", 115 | "Async. (512)" 116 | ) 117 | ) + 118 | scale_y_continuous(breaks = scales::pretty_breaks(n = 40)) + 119 | theme(legend.position = "bottom") 120 | ``` 121 | 122 | # Get higher throughput with fewer threads 123 | 124 | ## TPC-H Q1 125 | 126 | ```{r} 127 | library(tidyverse) 128 | results <- read_csv("/Users/leonard/Desktop/tpch_q1.csv") 129 | results %>% 130 | filter(page_size_power == 16 & 131 | num_entries_per_ring %in% c(0, 4, 64, 128)) %>% 132 | mutate(percent_cached = factor((num_cached_pages * 100) %/% num_total_pages)) %>% 133 | filter(percent_cached == 60) %>% 134 | mutate(throughput_per_thread = throughput / num_threads) %>% 135 | select(num_threads, 136 | num_entries_per_ring, 137 | throughput_per_thread) %>% 138 | group_by(num_threads, num_entries_per_ring) %>% 139 | summarise(throughput_per_thread = max(throughput_per_thread), 140 | .groups = 'drop') %>% 141 | ggplot() + 142 | geom_line(mapping = aes( 143 | x = factor(num_threads), 144 | y = throughput_per_thread, 145 | color = factor(num_entries_per_ring), 146 | group = factor(num_entries_per_ring) 147 | )) + 148 | labs( 149 | title = "Bandwidth per thread of processing TPC-H Q1", 150 | subtitle = "Page size of 64 KiB, 60% cached", 151 | x = "Number of threads", 152 | y = "Bandwidth per thread (in GB/s)", 153 | color = "Kind of I/O (I/O depth per thread)" 154 | ) + 155 | scale_color_discrete( 156 | limits = c("0", "4", "64", "128"), 157 | labels = c( 158 | "Synchronous (1)", 159 | "Asynchronous (4)", 160 | "Asynchronous (64)", 161 | "Asynchronous (128)" 162 | ) 163 | ) + 164 | scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) 165 | ``` 166 | 167 | ## TPC-H Q14 168 | 169 | ```{r} 170 | library(tidyverse) 171 | results <- read_csv("/Users/leonard/Desktop/tpch_q14.csv") 172 | results %>% 173 | filter(num_entries_per_ring %in% c(0, 8, 32, 64, 256, 512)) %>% 174 | mutate(percent_cached = factor((num_cached_references * 100) %/% num_total_references)) %>% 175 | filter(percent_cached %in% c(60)) %>% 176 | mutate(lookups_per_second = num_total_references / (time / 1000)) %>% 177 | mutate(lookups_per_second_per_thread = lookups_per_second / num_threads) %>% 178 | select(num_threads, 179 | num_entries_per_ring, 180 | lookups_per_second_per_thread) %>% 181 | group_by(num_threads, num_entries_per_ring) %>% 182 | summarise( 183 | lookups_per_second_per_thread = max(lookups_per_second_per_thread), 184 | .groups = 'drop' 185 | ) %>% 186 | ggplot() + 187 | geom_line( 188 | mapping = aes( 189 | x = factor(num_threads), 190 | y = lookups_per_second_per_thread, 191 | color = factor(num_entries_per_ring), 192 | group = factor(num_entries_per_ring) 193 | ) 194 | ) + 195 | labs( 196 | title = "Lookups per second per thread of processing TPC-H Q14", 197 | subtitle = "Page size of 4 KiB, 60% cached", 198 | x = "Number of threads", 199 | y = "Number of lookups per second per thread", 200 | color = "Kind of I/O (I/O depth per thread)" 201 | ) + 202 | scale_color_discrete( 203 | limits = c("0", "8", "32", "64", "256", "512"), 204 | labels = c( 205 | "Sync. (1)", 206 | "Async. (8)", 207 | "Async. (32)", 208 | "Async. (64)", 209 | "Async. (256)", 210 | "Async. (512)" 211 | ) 212 | ) + 213 | scale_y_continuous(breaks = scales::pretty_breaks(n = 15)) 214 | ``` 215 | 216 | # A lot of data must be cached for sequential I/O to catch up with random I/O 217 | 218 | ```{r} 219 | library(tidyverse) 220 | results <- read_csv("/Users/leonard/Desktop/tpch_q1.csv") %>% 221 | filter(page_size_power == 16 & 222 | num_threads == 8 & 223 | num_entries_per_ring %in% c(0, 128)) 224 | 225 | y_intercept <- results %>% 226 | filter(num_cached_pages == 0 & 227 | num_entries_per_ring == 128) %>% 228 | group_by(num_entries_per_ring) %>% 229 | summarise(throughput = max(throughput), 230 | .groups = 'drop') %>% 231 | pull(throughput) 232 | 233 | results %>% 234 | mutate(percent_cached = factor((num_cached_pages * 100) %/% num_total_pages)) %>% 235 | select(percent_cached, 236 | num_entries_per_ring, 237 | throughput) %>% 238 | group_by(percent_cached, num_entries_per_ring) %>% 239 | summarise(throughput = max(throughput), 240 | .groups = 'drop') %>% 241 | ggplot() + 242 | geom_line(mapping = aes( 243 | x = percent_cached, 244 | y = throughput, 245 | color = factor(num_entries_per_ring), 246 | group = factor(num_entries_per_ring) 247 | )) + 248 | geom_hline(yintercept = y_intercept) + 249 | labs( 250 | title = "Bandwidth of processing TPC-H Q1", 251 | subtitle = "Page size of 64 KiB, 8 threads", 252 | x = "Fraction of pages cached in main memory (in %)", 253 | y = "Bandwidth (in GB/s)", 254 | color = "Kind of I/O (I/O depth per thread)" 255 | ) + 256 | scale_color_discrete( 257 | limits = c("0", "128"), 258 | labels = c("Synchronous (1)", 259 | "Asynchronous (128)") 260 | ) + 261 | scale_y_continuous(breaks = scales::pretty_breaks(n = 20)) 262 | ``` 263 | 264 | # You need fewer threads to get max ssd bandwidth than dram bandwidth 265 | 266 | ```{r} 267 | library(tidyverse) 268 | results <- read_csv("/Users/leonard/Desktop/tpch_q1.csv") %>% 269 | filter(page_size_power == 16 & 270 | num_threads == 32 & 271 | num_entries_per_ring %in% c(0, 64)) 272 | 273 | y_intercept <- results %>% 274 | filter(num_cached_pages == 0 & 275 | num_entries_per_ring == 64) %>% 276 | group_by(num_entries_per_ring) %>% 277 | summarise(throughput = max(throughput), 278 | .groups = 'drop') %>% 279 | pull(throughput) 280 | 281 | results %>% 282 | mutate(percent_cached = factor((num_cached_pages * 100) %/% num_total_pages)) %>% 283 | select(percent_cached, 284 | num_entries_per_ring, 285 | throughput) %>% 286 | group_by(percent_cached, num_entries_per_ring) %>% 287 | summarise(throughput = max(throughput), 288 | .groups = 'drop') %>% 289 | ggplot() + 290 | geom_line(mapping = aes( 291 | x = percent_cached, 292 | y = throughput, 293 | color = factor(num_entries_per_ring), 294 | group = factor(num_entries_per_ring) 295 | )) + 296 | geom_hline(yintercept = y_intercept) + 297 | labs( 298 | title = "Bandwidth of processing TPC-H Q1", 299 | subtitle = "Page size of 64 KiB, 32 threads", 300 | x = "Fraction of pages cached in main memory (in %)", 301 | y = "Bandwidth (in GB/s)", 302 | color = "Kind of I/O (I/O depth per thread)" 303 | ) + 304 | scale_color_discrete( 305 | limits = c("0", "64"), 306 | labels = c("Synchronous (1)", 307 | "Asynchronous (64)") 308 | ) + 309 | scale_y_continuous(breaks = scales::pretty_breaks(n = 30)) 310 | ``` -------------------------------------------------------------------------------- /benchmarks/run.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import itertools 4 | 5 | path_to_build_directory = "/home/merzljak/async/build2" 6 | path_to_source_directory = "/home/merzljak/async" 7 | path_to_tpch_directory = "/nvmeSpace/merzljak/sf20" 8 | path_to_data_directory = "/nvmeSpace/merzljak/data/sf20" 9 | path_to_cxx_compiler = "/usr/bin/g++-11" 10 | path_to_query1_out = "/home/merzljak/async/benchmark/query1_out.csv" 11 | path_to_query14_out = "/home/merzljak/async/benchmark/query14_out.csv" 12 | 13 | page_size_powers_q1 = [12, 14, 16, 18, 20, 22] 14 | page_size_powers_q14 = [12, 13, 14, 15, 16] 15 | num_threads = [2, 4, 8, 12, 16, 20] 16 | num_entries_per_ring = [2, 4, 8, 16, 32, 64] 17 | do_work = ['true', 'false'] 18 | do_random_io = ['true', 'false'] 19 | num_tuples_per_coroutine = [1, 5, 25, 125, 625, 3125, 15625] 20 | 21 | query1_out = open(path_to_query1_out, "w") 22 | query14_out = open(path_to_query14_out, "w") 23 | print_header = "true" 24 | 25 | for page_size_power in page_size_powers_q1: 26 | subprocess.run(["cmake", "-S", path_to_source_directory, "-B", path_to_build_directory, "-DCMAKE_BUILD_TYPE=Release", 27 | "-DASYNCHRONOUS_IO_PAGE_SIZE_POWER={}".format(page_size_power), "-DCMAKE_CXX_COMPILER={}".format(path_to_cxx_compiler)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 28 | subprocess.run( 29 | ["cmake", "--build", path_to_build_directory, "--clean-first"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 30 | lineitemq1 = os.path.join(path_to_data_directory, "lineitemQ1.dat") 31 | part = os.path.join(path_to_data_directory, "part.dat") 32 | subprocess.run([os.path.join(path_to_build_directory, "executables", "load_data"), 33 | "lineitemQ1", os.path.join(path_to_tpch_directory, "lineitem.tbl"), lineitemq1], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 34 | 35 | print_header = "true" 36 | for threads, entries_per_ring, work, random_io in itertools.product(num_threads, num_entries_per_ring, do_work, do_random_io): 37 | subprocess.run([os.path.join(path_to_build_directory, "executables", "tpch_q1"), 38 | lineitemq1, str(threads), str(entries_per_ring), work, random_io, "false", print_header], check=True, stdout=query1_out, stderr=subprocess.PIPE) 39 | print_header = "false" 40 | 41 | for page_size_power in page_size_powers_q14: 42 | subprocess.run(["cmake", "-S", path_to_source_directory, "-B", path_to_build_directory, "-DCMAKE_BUILD_TYPE=Release", 43 | "-DASYNCHRONOUS_IO_PAGE_SIZE_POWER={}".format(page_size_power), "-DCMAKE_CXX_COMPILER={}".format(path_to_cxx_compiler)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 44 | subprocess.run( 45 | ["cmake", "--build", path_to_build_directory, "--clean-first"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 46 | lineitemq14 = os.path.join(path_to_data_directory, "lineitemQ14.dat") 47 | part = os.path.join(path_to_data_directory, "part.dat") 48 | subprocess.run([os.path.join(path_to_build_directory, "executables", "load_data"), 49 | "lineitemQ14", os.path.join(path_to_tpch_directory, "lineitem.tbl"), lineitemq14], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 50 | subprocess.run([os.path.join(path_to_build_directory, "executables", "load_data"), 51 | "part", os.path.join(path_to_tpch_directory, "part.tbl"), part], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 52 | 53 | print_header = "true" 54 | for threads, entries_per_ring, tuples_per_coroutine in itertools.product(num_threads, num_entries_per_ring, num_tuples_per_coroutine): 55 | subprocess.run([os.path.join(path_to_build_directory, "executables", "tpch_q14"), 56 | lineitemq14, part, str(threads), str(entries_per_ring), str(tuples_per_coroutine), "false", print_header], check=True, stdout=query14_out, stderr=subprocess.PIPE) 57 | print_header = "false" 58 | -------------------------------------------------------------------------------- /benchmarks/umbra.md: -------------------------------------------------------------------------------- 1 | # Umbra 2 | 3 | ## Build 4 | 5 | ``` 6 | git clone https://gitlab.db.in.tum.de/umbra/umbra-students.git 7 | make -j 128 8 | ``` 9 | 10 | ## Start CLI 11 | 12 | ``` 13 | export PARALLEL=64 14 | numactl --membind=0 --cpubind=0 ./bin/sql 15 | ``` 16 | 17 | ## Create schema 18 | 19 | ``` 20 | create table part ( 21 | p_partkey integer not null, 22 | p_name varchar(55) not null, 23 | p_mfgr char(25) not null, 24 | p_brand char(10) not null, 25 | p_type varchar(25) not null, 26 | p_size integer not null, 27 | p_container char(10) not null, 28 | p_retailprice decimal(12,2) not null, 29 | p_comment varchar(23) not null 30 | ); 31 | 32 | create table supplier ( 33 | s_suppkey integer not null, 34 | s_name char(25) not null, 35 | s_address varchar(40) not null, 36 | s_nationkey integer not null, 37 | s_phone char(15) not null, 38 | s_acctbal decimal(12,2) not null, 39 | s_comment varchar(101) not null 40 | ); 41 | 42 | create table partsupp ( 43 | ps_partkey integer not null, 44 | ps_suppkey integer not null, 45 | ps_availqty integer not null, 46 | ps_supplycost decimal(12,2) not null, 47 | ps_comment varchar(199) not null 48 | ); 49 | 50 | create table customer ( 51 | c_custkey integer not null, 52 | c_name varchar(25) not null, 53 | c_address varchar(40) not null, 54 | c_nationkey integer not null, 55 | c_phone char(15) not null, 56 | c_acctbal decimal(12,2) not null, 57 | c_mktsegment char(10) not null, 58 | c_comment varchar(117) not null 59 | ); 60 | 61 | create table orders ( 62 | o_orderkey integer not null, 63 | o_custkey integer not null, 64 | o_orderstatus char(1) not null, 65 | o_totalprice decimal(12,2) not null, 66 | o_orderdate date not null, 67 | o_orderpriority char(15) not null, 68 | o_clerk char(15) not null, 69 | o_shippriority integer not null, 70 | o_comment varchar(79) not null 71 | ); 72 | 73 | create table lineitem ( 74 | l_orderkey integer not null, 75 | l_partkey integer not null, 76 | l_suppkey integer not null, 77 | l_linenumber integer not null, 78 | l_quantity decimal(12,2) not null, 79 | l_extendedprice decimal(12,2) not null, 80 | l_discount decimal(12,2) not null, 81 | l_tax decimal(12,2) not null, 82 | l_returnflag char(1) not null, 83 | l_linestatus char(1) not null, 84 | l_shipdate date not null, 85 | l_commitdate date not null, 86 | l_receiptdate date not null, 87 | l_shipinstruct char(25) not null, 88 | l_shipmode char(10) not null, 89 | l_comment varchar(44) not null 90 | ); 91 | 92 | create table nation ( 93 | n_nationkey integer not null, 94 | n_name char(25) not null, 95 | n_regionkey integer not null, 96 | n_comment varchar(152) not null 97 | ); 98 | 99 | create table region ( 100 | r_regionkey integer not null, 101 | r_name char(25) not null, 102 | r_comment varchar(152) not null 103 | ); 104 | ``` 105 | 106 | ## Load data 107 | 108 | ``` 109 | copy part from '/raid0/data/tpch/sf30/part.tbl' delimiter '|'; 110 | copy supplier from '/raid0/data/tpch/sf30/supplier.tbl' delimiter '|'; 111 | copy partsupp from '/raid0/data/tpch/sf30/partsupp.tbl' delimiter '|'; 112 | copy customer from '/raid0/data/tpch/sf30/customer.tbl' delimiter '|'; 113 | copy orders from '/raid0/data/tpch/sf30/orders.tbl' delimiter '|'; 114 | copy lineitem from '/raid0/data/tpch/sf30/lineitem.tbl' delimiter '|'; 115 | copy nation from '/raid0/data/tpch/sf30/nation.tbl' delimiter '|'; 116 | copy region from '/raid0/data/tpch/sf30/region.tbl' delimiter '|'; 117 | ``` 118 | 119 | ## Queries 120 | 121 | ### Query 1 122 | 123 | ``` 124 | select 125 | l_returnflag, 126 | l_linestatus, 127 | sum(l_quantity) as sum_qty, 128 | sum(l_extendedprice) as sum_base_price, 129 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 130 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 131 | avg(l_quantity) as avg_qty, 132 | avg(l_extendedprice) as avg_price, 133 | avg(l_discount) as avg_disc, 134 | count(*) as count_order 135 | from 136 | lineitem 137 | where 138 | l_shipdate <= date '1998-12-01' - interval '90' day 139 | group by 140 | l_returnflag, 141 | l_linestatus 142 | order by 143 | l_returnflag, 144 | l_linestatus; 145 | ``` 146 | 147 | ### Query 2 148 | 149 | ``` 150 | select 151 | s_acctbal, 152 | s_name, 153 | n_name, 154 | p_partkey, 155 | p_mfgr, 156 | s_address, 157 | s_phone, 158 | s_comment 159 | from 160 | part, 161 | supplier, 162 | partsupp, 163 | nation, 164 | region 165 | where 166 | p_partkey = ps_partkey 167 | and s_suppkey = ps_suppkey 168 | and p_size = 15 169 | and p_type like '%BRASS' 170 | and s_nationkey = n_nationkey 171 | and n_regionkey = r_regionkey 172 | and r_name = 'EUROPE' 173 | and ps_supplycost = ( 174 | select 175 | min(ps_supplycost) 176 | from 177 | partsupp, 178 | supplier, 179 | nation, 180 | region 181 | where 182 | p_partkey = ps_partkey 183 | and s_suppkey = ps_suppkey 184 | and s_nationkey = n_nationkey 185 | and n_regionkey = r_regionkey 186 | and r_name = 'EUROPE' 187 | ) 188 | order by 189 | s_acctbal desc, 190 | n_name, 191 | s_name, 192 | p_partkey 193 | limit 194 | 100; 195 | ``` -------------------------------------------------------------------------------- /cppcoro/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: LLVM 3 | --- 4 | Language: Cpp 5 | Standard: Cpp11 6 | ColumnLimit: 100 7 | TabWidth: 4 8 | IndentWidth: 4 9 | UseTab: ForContinuationAndIndentation 10 | AccessModifierOffset: -4 11 | AlignAfterOpenBracket: AlwaysBreak 12 | AlignConsecutiveAssignments: false 13 | AlignConsecutiveDeclarations: false 14 | AlignEscapedNewlines: Left 15 | AlignOperands: false 16 | AlignTrailingComments: true 17 | AllowAllParametersOfDeclarationOnNextLine: true 18 | AllowShortBlocksOnASingleLine: false 19 | AllowShortCaseLabelsOnASingleLine: false 20 | AllowShortIfStatementsOnASingleLine: false 21 | AllowShortFunctionsOnASingleLine: InlineOnly 22 | AllowShortLoopsOnASingleLine: false 23 | AlwaysBreakAfterReturnType: None 24 | AlwaysBreakTemplateDeclarations: true 25 | BinPackArguments: false 26 | BinPackParameters: false 27 | BreakBeforeBinaryOperators: None 28 | BreakBeforeBraces: Custom 29 | BraceWrapping: { 30 | AfterClass: true, 31 | AfterControlStatement: true, 32 | AfterEnum: true, 33 | AfterFunction: true, 34 | AfterNamespace: true, 35 | AfterStruct: true, 36 | AfterUnion: true, 37 | BeforeCatch: true, 38 | BeforeElse: true, 39 | IndentBraces: false, 40 | #SplitEmptyFunctionBody: false 41 | } 42 | BreakBeforeInheritanceComma: true 43 | BreakBeforeTernaryOperators: true 44 | BreakConstructorInitializers: BeforeComma 45 | ContinuationIndentWidth: 4 46 | Cpp11BracedListStyle: false 47 | IncludeCategories: 48 | - Regex: '^$' 49 | Priority: 1 50 | - Regex: '^ 4 | #include 5 | #include 6 | #include 7 | 8 | class FixedAllocator 9 | { 10 | public: 11 | FixedAllocator(uint32_t allocation_size, uint16_t num_blocks) noexcept 12 | : allocation_size(allocation_size) 13 | , num_blocks(num_blocks) 14 | { 15 | } 16 | 17 | void* allocate() 18 | { 19 | if (free_list.empty()) 20 | { 21 | auto size = allocation_size * num_blocks; 22 | raw_data.emplace_back(std::make_unique(size)); 23 | free_list.reserve(num_blocks * raw_data.size()); 24 | uint16_t i = 0; 25 | for (unsigned char* p = raw_data.back().get(); i != num_blocks; 26 | p += allocation_size, ++i) 27 | { 28 | free_list.push_back(p); 29 | } 30 | } 31 | 32 | unsigned char* result = free_list.back(); 33 | free_list.pop_back(); 34 | return result; 35 | } 36 | 37 | void deallocate(void* p) noexcept { free_list.push_back(static_cast(p)); } 38 | 39 | uint32_t getAllocationSize() const noexcept { return allocation_size; } 40 | 41 | private: 42 | std::vector> raw_data; 43 | std::vector free_list; 44 | uint32_t allocation_size; 45 | uint16_t num_blocks; 46 | }; 47 | 48 | class Allocator 49 | { 50 | public: 51 | explicit Allocator(uint16_t num_blocks) 52 | : num_blocks(num_blocks) 53 | { 54 | } 55 | 56 | void* allocate(uint32_t allocation_size) 57 | { 58 | for (auto& alloc : fixed_allocators) 59 | { 60 | if (alloc.getAllocationSize() == allocation_size) 61 | { 62 | return alloc.allocate(); 63 | } 64 | } 65 | 66 | return fixed_allocators.emplace_back(allocation_size, num_blocks).allocate(); 67 | } 68 | 69 | void deallocate(void* p, uint32_t allocation_size) noexcept 70 | { 71 | for (auto& alloc : fixed_allocators) 72 | { 73 | if (alloc.getAllocationSize() == allocation_size) 74 | { 75 | return alloc.deallocate(p); 76 | } 77 | } 78 | } 79 | 80 | private: 81 | std::vector fixed_allocators; 82 | uint16_t num_blocks; 83 | }; 84 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/awaitable_traits.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_AWAITABLE_TRAITS_HPP_INCLUDED 6 | #define CPPCORO_AWAITABLE_TRAITS_HPP_INCLUDED 7 | 8 | #include 9 | 10 | #include 11 | 12 | namespace cppcoro 13 | { 14 | template 15 | struct awaitable_traits 16 | { 17 | }; 18 | 19 | template 20 | struct awaitable_traits< 21 | T, 22 | std::void_t()))>> 23 | { 24 | using awaiter_t = decltype(cppcoro::detail::get_awaiter(std::declval())); 25 | 26 | using await_result_t = decltype(std::declval().await_resume()); 27 | }; 28 | } // namespace cppcoro 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/broken_promise.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_BROKEN_PROMISE_HPP_INCLUDED 6 | #define CPPCORO_BROKEN_PROMISE_HPP_INCLUDED 7 | 8 | #include 9 | 10 | namespace cppcoro 11 | { 12 | /// \brief 13 | /// Exception thrown when you attempt to retrieve the result of 14 | /// a task that has been detached from its promise/coroutine. 15 | class broken_promise : public std::logic_error 16 | { 17 | public: 18 | broken_promise() 19 | : std::logic_error("broken promise") 20 | { 21 | } 22 | }; 23 | } // namespace cppcoro 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/coroutine.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CPPCORO_COROUTINE_HPP_INCLUDED 2 | #define CPPCORO_COROUTINE_HPP_INCLUDED 3 | 4 | #include 5 | 6 | namespace cppcoro 7 | { 8 | using std::coroutine_handle; 9 | using std::noop_coroutine; 10 | using std::suspend_always; 11 | using std::suspend_never; 12 | } // namespace cppcoro 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/any.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_ANY_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_ANY_HPP_INCLUDED 7 | 8 | namespace cppcoro 9 | { 10 | namespace detail 11 | { 12 | // Helper type that can be cast-to from any type. 13 | struct any 14 | { 15 | template 16 | any(T&&) noexcept 17 | { 18 | } 19 | }; 20 | } // namespace detail 21 | } // namespace cppcoro 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/get_awaiter.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_GET_AWAITER_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_GET_AWAITER_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | 11 | namespace cppcoro 12 | { 13 | namespace detail 14 | { 15 | template 16 | auto get_awaiter_impl(T&& value, int) noexcept( 17 | noexcept(static_cast(value).operator co_await())) 18 | -> decltype(static_cast(value).operator co_await()) 19 | { 20 | return static_cast(value).operator co_await(); 21 | } 22 | 23 | template 24 | auto get_awaiter_impl(T&& value, long) noexcept( 25 | noexcept(operator co_await(static_cast(value)))) 26 | -> decltype(operator co_await(static_cast(value))) 27 | { 28 | return operator co_await(static_cast(value)); 29 | } 30 | 31 | template::value, int> = 0> 32 | T&& get_awaiter_impl(T&& value, cppcoro::detail::any) noexcept 33 | { 34 | return static_cast(value); 35 | } 36 | 37 | template 38 | auto get_awaiter(T&& value) noexcept( 39 | noexcept(detail::get_awaiter_impl(static_cast(value), 123))) 40 | -> decltype(detail::get_awaiter_impl(static_cast(value), 123)) 41 | { 42 | return detail::get_awaiter_impl(static_cast(value), 123); 43 | } 44 | } // namespace detail 45 | } // namespace cppcoro 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/is_awaiter.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_IS_AWAITER_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_IS_AWAITER_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | 11 | namespace cppcoro 12 | { 13 | namespace detail 14 | { 15 | template 16 | struct is_coroutine_handle : std::false_type 17 | { 18 | }; 19 | 20 | template 21 | struct is_coroutine_handle> : std::true_type 22 | { 23 | }; 24 | 25 | // NOTE: We're accepting a return value of coroutine_handle

here 26 | // which is an extension supported by Clang which is not yet part of 27 | // the C++ coroutines TS. 28 | template 29 | struct is_valid_await_suspend_return_value 30 | : std::disjunction, std::is_same, is_coroutine_handle> 31 | { 32 | }; 33 | 34 | template> 35 | struct is_awaiter : std::false_type 36 | { 37 | }; 38 | 39 | // NOTE: We're testing whether await_suspend() will be callable using an 40 | // arbitrary coroutine_handle here by checking if it supports being passed 41 | // a coroutine_handle. This may result in a false-result for some 42 | // types which are only awaitable within a certain context. 43 | template 44 | struct is_awaiter< 45 | T, 46 | std::void_t< 47 | decltype(std::declval().await_ready()), 48 | decltype(std::declval().await_suspend( 49 | std::declval>())), 50 | decltype(std::declval().await_resume())>> 51 | : std::conjunction< 52 | std::is_constructible().await_ready())>, 53 | detail::is_valid_await_suspend_return_value< 54 | decltype(std::declval().await_suspend( 55 | std::declval>()))>> 56 | { 57 | }; 58 | } // namespace detail 59 | } // namespace cppcoro 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/lightweight_manual_reset_event.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_LIGHTWEIGHT_MANUAL_RESET_EVENT_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_LIGHTWEIGHT_MANUAL_RESET_EVENT_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | 11 | namespace cppcoro 12 | { 13 | namespace detail 14 | { 15 | class lightweight_manual_reset_event 16 | { 17 | public: 18 | lightweight_manual_reset_event(bool initiallySet = false); 19 | 20 | ~lightweight_manual_reset_event(); 21 | 22 | void set() noexcept; 23 | 24 | void reset() noexcept; 25 | 26 | void wait() noexcept; 27 | 28 | private: 29 | std::atomic m_value; 30 | }; 31 | } // namespace detail 32 | } // namespace cppcoro 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/remove_rvalue_reference.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_REMOVE_RVALUE_REFERENCE_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_REMOVE_RVALUE_REFERENCE_HPP_INCLUDED 7 | 8 | namespace cppcoro 9 | { 10 | namespace detail 11 | { 12 | template 13 | struct remove_rvalue_reference 14 | { 15 | using type = T; 16 | }; 17 | 18 | template 19 | struct remove_rvalue_reference 20 | { 21 | using type = T; 22 | }; 23 | 24 | template 25 | using remove_rvalue_reference_t = typename remove_rvalue_reference::type; 26 | } // namespace detail 27 | } // namespace cppcoro 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/sync_wait_task.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_SYNC_WAIT_TASK_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_SYNC_WAIT_TASK_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace cppcoro 18 | { 19 | namespace detail 20 | { 21 | inline thread_local Allocator* sync_allocator{ nullptr }; 22 | 23 | template 24 | class sync_wait_task; 25 | 26 | template 27 | class sync_wait_task_promise final 28 | { 29 | using coroutine_handle_t = cppcoro::coroutine_handle>; 30 | 31 | public: 32 | using reference = RESULT&&; 33 | 34 | sync_wait_task_promise() noexcept {} 35 | 36 | static void* operator new(size_t sz) 37 | { 38 | if (sync_allocator) 39 | { 40 | return sync_allocator->allocate(sz); 41 | } 42 | else 43 | { 44 | return ::operator new(sz); 45 | } 46 | } 47 | 48 | static void operator delete(void* p, size_t sz) 49 | { 50 | if (sync_allocator) 51 | { 52 | sync_allocator->deallocate(p, sz); 53 | } 54 | else 55 | { 56 | ::operator delete(p, sz); 57 | } 58 | } 59 | 60 | void start(detail::lightweight_manual_reset_event& event) 61 | { 62 | m_event = &event; 63 | coroutine_handle_t::from_promise(*this).resume(); 64 | } 65 | 66 | auto get_return_object() noexcept { return coroutine_handle_t::from_promise(*this); } 67 | 68 | cppcoro::suspend_always initial_suspend() noexcept { return {}; } 69 | 70 | auto final_suspend() noexcept 71 | { 72 | class completion_notifier 73 | { 74 | public: 75 | bool await_ready() const noexcept { return false; } 76 | 77 | void await_suspend(coroutine_handle_t coroutine) const noexcept 78 | { 79 | coroutine.promise().m_event->set(); 80 | } 81 | 82 | void await_resume() noexcept {} 83 | }; 84 | 85 | return completion_notifier{}; 86 | } 87 | 88 | auto yield_value(reference result) noexcept 89 | { 90 | m_result = std::addressof(result); 91 | return final_suspend(); 92 | } 93 | 94 | void return_void() noexcept 95 | { 96 | // The coroutine should have either yielded a value or thrown 97 | // an exception in which case it should have bypassed return_void(). 98 | assert(false); 99 | } 100 | 101 | void unhandled_exception() { m_exception = std::current_exception(); } 102 | 103 | reference result() 104 | { 105 | if (m_exception) 106 | { 107 | std::rethrow_exception(m_exception); 108 | } 109 | 110 | return static_cast(*m_result); 111 | } 112 | 113 | private: 114 | detail::lightweight_manual_reset_event* m_event; 115 | std::remove_reference_t* m_result; 116 | std::exception_ptr m_exception; 117 | }; 118 | 119 | template<> 120 | class sync_wait_task_promise 121 | { 122 | using coroutine_handle_t = cppcoro::coroutine_handle>; 123 | 124 | public: 125 | sync_wait_task_promise() noexcept {} 126 | 127 | static void* operator new(size_t sz) 128 | { 129 | if (sync_allocator) 130 | { 131 | return sync_allocator->allocate(sz); 132 | } 133 | else 134 | { 135 | return ::operator new(sz); 136 | } 137 | } 138 | 139 | static void operator delete(void* p, size_t sz) 140 | { 141 | if (sync_allocator) 142 | { 143 | sync_allocator->deallocate(p, sz); 144 | } 145 | else 146 | { 147 | ::operator delete(p, sz); 148 | } 149 | } 150 | 151 | void start(detail::lightweight_manual_reset_event& event) 152 | { 153 | m_event = &event; 154 | coroutine_handle_t::from_promise(*this).resume(); 155 | } 156 | 157 | auto get_return_object() noexcept { return coroutine_handle_t::from_promise(*this); } 158 | 159 | cppcoro::suspend_always initial_suspend() noexcept { return {}; } 160 | 161 | auto final_suspend() noexcept 162 | { 163 | class completion_notifier 164 | { 165 | public: 166 | bool await_ready() const noexcept { return false; } 167 | 168 | void await_suspend(coroutine_handle_t coroutine) const noexcept 169 | { 170 | coroutine.promise().m_event->set(); 171 | } 172 | 173 | void await_resume() noexcept {} 174 | }; 175 | 176 | return completion_notifier{}; 177 | } 178 | 179 | void return_void() {} 180 | 181 | void unhandled_exception() { m_exception = std::current_exception(); } 182 | 183 | void result() 184 | { 185 | if (m_exception) 186 | { 187 | std::rethrow_exception(m_exception); 188 | } 189 | } 190 | 191 | private: 192 | detail::lightweight_manual_reset_event* m_event; 193 | std::exception_ptr m_exception; 194 | }; 195 | 196 | template 197 | class sync_wait_task final 198 | { 199 | public: 200 | using promise_type = sync_wait_task_promise; 201 | 202 | using coroutine_handle_t = cppcoro::coroutine_handle; 203 | 204 | sync_wait_task(coroutine_handle_t coroutine) noexcept 205 | : m_coroutine(coroutine) 206 | { 207 | } 208 | 209 | sync_wait_task(sync_wait_task&& other) noexcept 210 | : m_coroutine(std::exchange(other.m_coroutine, coroutine_handle_t{})) 211 | { 212 | } 213 | 214 | ~sync_wait_task() 215 | { 216 | if (m_coroutine) 217 | m_coroutine.destroy(); 218 | } 219 | 220 | sync_wait_task(const sync_wait_task&) = delete; 221 | sync_wait_task& operator=(const sync_wait_task&) = delete; 222 | 223 | void start(lightweight_manual_reset_event& event) noexcept 224 | { 225 | m_coroutine.promise().start(event); 226 | } 227 | 228 | decltype(auto) result() { return m_coroutine.promise().result(); } 229 | 230 | private: 231 | coroutine_handle_t m_coroutine; 232 | }; 233 | 234 | template< 235 | typename AWAITABLE, 236 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 237 | std::enable_if_t, int> = 0> 238 | sync_wait_task make_sync_wait_task(AWAITABLE&& awaitable) 239 | { 240 | co_yield co_await std::forward(awaitable); 241 | } 242 | 243 | template< 244 | typename AWAITABLE, 245 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 246 | std::enable_if_t, int> = 0> 247 | sync_wait_task make_sync_wait_task(AWAITABLE&& awaitable) 248 | { 249 | co_await std::forward(awaitable); 250 | } 251 | } // namespace detail 252 | } // namespace cppcoro 253 | 254 | #endif 255 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/unwrap_reference.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_UNWRAP_REFERENCE_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_UNWRAP_REFERENCE_HPP_INCLUDED 7 | 8 | #include 9 | 10 | namespace cppcoro 11 | { 12 | namespace detail 13 | { 14 | template 15 | struct unwrap_reference 16 | { 17 | using type = T; 18 | }; 19 | 20 | template 21 | struct unwrap_reference> 22 | { 23 | using type = T; 24 | }; 25 | 26 | template 27 | using unwrap_reference_t = typename unwrap_reference::type; 28 | } // namespace detail 29 | } // namespace cppcoro 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/void_value.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_VOID_VALUE_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_VOID_VALUE_HPP_INCLUDED 7 | 8 | namespace cppcoro 9 | { 10 | namespace detail 11 | { 12 | struct void_value 13 | { 14 | }; 15 | } // namespace detail 16 | } // namespace cppcoro 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/when_all_counter.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_WHEN_ALL_COUNTER_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_WHEN_ALL_COUNTER_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace cppcoro 13 | { 14 | namespace detail 15 | { 16 | class when_all_counter 17 | { 18 | public: 19 | when_all_counter(std::size_t count) noexcept 20 | : m_count(count + 1) 21 | , m_awaitingCoroutine(nullptr) 22 | { 23 | } 24 | 25 | bool is_ready() const noexcept 26 | { 27 | // We consider this complete if we're asking whether it's ready 28 | // after a coroutine has already been registered. 29 | return static_cast(m_awaitingCoroutine); 30 | } 31 | 32 | bool try_await(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 33 | { 34 | m_awaitingCoroutine = awaitingCoroutine; 35 | return m_count.fetch_sub(1, std::memory_order_acq_rel) > 1; 36 | } 37 | 38 | void notify_awaitable_completed() noexcept 39 | { 40 | if (m_count.fetch_sub(1, std::memory_order_acq_rel) == 1) 41 | { 42 | m_awaitingCoroutine.resume(); 43 | } 44 | } 45 | 46 | protected: 47 | std::atomic m_count; 48 | cppcoro::coroutine_handle<> m_awaitingCoroutine; 49 | }; 50 | } // namespace detail 51 | } // namespace cppcoro 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/when_all_ready_awaitable.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_WHEN_ALL_READY_AWAITABLE_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_WHEN_ALL_READY_AWAITABLE_HPP_INCLUDED 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | namespace cppcoro 14 | { 15 | namespace detail 16 | { 17 | template 18 | class when_all_ready_awaitable; 19 | 20 | template<> 21 | class when_all_ready_awaitable> 22 | { 23 | public: 24 | constexpr when_all_ready_awaitable() noexcept {} 25 | explicit constexpr when_all_ready_awaitable(std::tuple<>) noexcept {} 26 | 27 | constexpr bool await_ready() const noexcept { return true; } 28 | void await_suspend(cppcoro::coroutine_handle<>) noexcept {} 29 | std::tuple<> await_resume() const noexcept { return {}; } 30 | }; 31 | 32 | template 33 | class when_all_ready_awaitable> 34 | { 35 | public: 36 | explicit when_all_ready_awaitable(TASKS&&... tasks) noexcept( 37 | std::conjunction_v...>) 38 | : m_counter(sizeof...(TASKS)) 39 | , m_tasks(std::move(tasks)...) 40 | { 41 | } 42 | 43 | explicit when_all_ready_awaitable(std::tuple&& tasks) noexcept( 44 | std::is_nothrow_move_constructible_v>) 45 | : m_counter(sizeof...(TASKS)) 46 | , m_tasks(std::move(tasks)) 47 | { 48 | } 49 | 50 | when_all_ready_awaitable(when_all_ready_awaitable&& other) noexcept 51 | : m_counter(sizeof...(TASKS)) 52 | , m_tasks(std::move(other.m_tasks)) 53 | { 54 | } 55 | 56 | auto operator co_await() & noexcept 57 | { 58 | struct awaiter 59 | { 60 | awaiter(when_all_ready_awaitable& awaitable) noexcept 61 | : m_awaitable(awaitable) 62 | { 63 | } 64 | 65 | bool await_ready() const noexcept { return m_awaitable.is_ready(); } 66 | 67 | bool await_suspend(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 68 | { 69 | return m_awaitable.try_await(awaitingCoroutine); 70 | } 71 | 72 | std::tuple& await_resume() noexcept { return m_awaitable.m_tasks; } 73 | 74 | private: 75 | when_all_ready_awaitable& m_awaitable; 76 | }; 77 | 78 | return awaiter{ *this }; 79 | } 80 | 81 | auto operator co_await() && noexcept 82 | { 83 | struct awaiter 84 | { 85 | awaiter(when_all_ready_awaitable& awaitable) noexcept 86 | : m_awaitable(awaitable) 87 | { 88 | } 89 | 90 | bool await_ready() const noexcept { return m_awaitable.is_ready(); } 91 | 92 | bool await_suspend(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 93 | { 94 | return m_awaitable.try_await(awaitingCoroutine); 95 | } 96 | 97 | std::tuple&& await_resume() noexcept 98 | { 99 | return std::move(m_awaitable.m_tasks); 100 | } 101 | 102 | private: 103 | when_all_ready_awaitable& m_awaitable; 104 | }; 105 | 106 | return awaiter{ *this }; 107 | } 108 | 109 | private: 110 | bool is_ready() const noexcept { return m_counter.is_ready(); } 111 | 112 | bool try_await(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 113 | { 114 | start_tasks(std::make_integer_sequence{}); 115 | return m_counter.try_await(awaitingCoroutine); 116 | } 117 | 118 | template 119 | void start_tasks(std::integer_sequence) noexcept 120 | { 121 | (void)std::initializer_list{ ( 122 | std::get(m_tasks).start(m_counter), 0)... }; 123 | } 124 | 125 | when_all_counter m_counter; 126 | std::tuple m_tasks; 127 | }; 128 | 129 | template 130 | class when_all_ready_awaitable 131 | { 132 | public: 133 | explicit when_all_ready_awaitable(TASK_CONTAINER&& tasks) noexcept 134 | : m_counter(tasks.size()) 135 | , m_tasks(std::forward(tasks)) 136 | { 137 | } 138 | 139 | when_all_ready_awaitable(when_all_ready_awaitable&& other) noexcept( 140 | std::is_nothrow_move_constructible_v) 141 | : m_counter(other.m_tasks.size()) 142 | , m_tasks(std::move(other.m_tasks)) 143 | { 144 | } 145 | 146 | when_all_ready_awaitable(const when_all_ready_awaitable&) = delete; 147 | when_all_ready_awaitable& operator=(const when_all_ready_awaitable&) = delete; 148 | 149 | auto operator co_await() & noexcept 150 | { 151 | class awaiter 152 | { 153 | public: 154 | awaiter(when_all_ready_awaitable& awaitable) 155 | : m_awaitable(awaitable) 156 | { 157 | } 158 | 159 | bool await_ready() const noexcept { return m_awaitable.is_ready(); } 160 | 161 | bool await_suspend(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 162 | { 163 | return m_awaitable.try_await(awaitingCoroutine); 164 | } 165 | 166 | TASK_CONTAINER& await_resume() noexcept { return m_awaitable.m_tasks; } 167 | 168 | private: 169 | when_all_ready_awaitable& m_awaitable; 170 | }; 171 | 172 | return awaiter{ *this }; 173 | } 174 | 175 | auto operator co_await() && noexcept 176 | { 177 | class awaiter 178 | { 179 | public: 180 | awaiter(when_all_ready_awaitable& awaitable) 181 | : m_awaitable(awaitable) 182 | { 183 | } 184 | 185 | bool await_ready() const noexcept { return m_awaitable.is_ready(); } 186 | 187 | bool await_suspend(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 188 | { 189 | return m_awaitable.try_await(awaitingCoroutine); 190 | } 191 | 192 | TASK_CONTAINER&& await_resume() noexcept 193 | { 194 | return std::move(m_awaitable.m_tasks); 195 | } 196 | 197 | private: 198 | when_all_ready_awaitable& m_awaitable; 199 | }; 200 | 201 | return awaiter{ *this }; 202 | } 203 | 204 | private: 205 | bool is_ready() const noexcept { return m_counter.is_ready(); } 206 | 207 | bool try_await(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 208 | { 209 | for (auto&& task : m_tasks) 210 | { 211 | task.start(m_counter); 212 | } 213 | 214 | return m_counter.try_await(awaitingCoroutine); 215 | } 216 | 217 | when_all_counter m_counter; 218 | TASK_CONTAINER m_tasks; 219 | }; 220 | } // namespace detail 221 | } // namespace cppcoro 222 | 223 | #endif 224 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/detail/when_all_task.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_DETAIL_WHEN_ALL_TASK_HPP_INCLUDED 6 | #define CPPCORO_DETAIL_WHEN_ALL_TASK_HPP_INCLUDED 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace cppcoro 18 | { 19 | namespace detail 20 | { 21 | template 22 | class when_all_ready_awaitable; 23 | 24 | template 25 | class when_all_task; 26 | 27 | template 28 | class when_all_task_promise final 29 | { 30 | public: 31 | using coroutine_handle_t = cppcoro::coroutine_handle>; 32 | 33 | when_all_task_promise() noexcept {} 34 | 35 | auto get_return_object() noexcept { return coroutine_handle_t::from_promise(*this); } 36 | 37 | cppcoro::suspend_always initial_suspend() noexcept { return {}; } 38 | 39 | auto final_suspend() noexcept 40 | { 41 | class completion_notifier 42 | { 43 | public: 44 | bool await_ready() const noexcept { return false; } 45 | 46 | void await_suspend(coroutine_handle_t coro) const noexcept 47 | { 48 | coro.promise().m_counter->notify_awaitable_completed(); 49 | } 50 | 51 | void await_resume() const noexcept {} 52 | }; 53 | 54 | return completion_notifier{}; 55 | } 56 | 57 | void unhandled_exception() noexcept { m_exception = std::current_exception(); } 58 | 59 | void return_void() noexcept 60 | { 61 | // We should have either suspended at co_yield point or 62 | // an exception was thrown before running off the end of 63 | // the coroutine. 64 | assert(false); 65 | } 66 | 67 | auto yield_value(RESULT&& result) noexcept 68 | { 69 | m_result = std::addressof(result); 70 | return final_suspend(); 71 | } 72 | 73 | void start(when_all_counter& counter) noexcept 74 | { 75 | m_counter = &counter; 76 | coroutine_handle_t::from_promise(*this).resume(); 77 | } 78 | 79 | RESULT& result() & 80 | { 81 | rethrow_if_exception(); 82 | return *m_result; 83 | } 84 | 85 | RESULT&& result() && 86 | { 87 | rethrow_if_exception(); 88 | return std::forward(*m_result); 89 | } 90 | 91 | private: 92 | void rethrow_if_exception() 93 | { 94 | if (m_exception) 95 | { 96 | std::rethrow_exception(m_exception); 97 | } 98 | } 99 | 100 | when_all_counter* m_counter; 101 | std::exception_ptr m_exception; 102 | std::add_pointer_t m_result; 103 | }; 104 | 105 | template<> 106 | class when_all_task_promise final 107 | { 108 | public: 109 | using coroutine_handle_t = cppcoro::coroutine_handle>; 110 | 111 | when_all_task_promise() noexcept {} 112 | 113 | auto get_return_object() noexcept { return coroutine_handle_t::from_promise(*this); } 114 | 115 | cppcoro::suspend_always initial_suspend() noexcept { return {}; } 116 | 117 | auto final_suspend() noexcept 118 | { 119 | class completion_notifier 120 | { 121 | public: 122 | bool await_ready() const noexcept { return false; } 123 | 124 | void await_suspend(coroutine_handle_t coro) const noexcept 125 | { 126 | coro.promise().m_counter->notify_awaitable_completed(); 127 | } 128 | 129 | void await_resume() const noexcept {} 130 | }; 131 | 132 | return completion_notifier{}; 133 | } 134 | 135 | void unhandled_exception() noexcept { m_exception = std::current_exception(); } 136 | 137 | void return_void() noexcept {} 138 | 139 | void start(when_all_counter& counter) noexcept 140 | { 141 | m_counter = &counter; 142 | coroutine_handle_t::from_promise(*this).resume(); 143 | } 144 | 145 | void result() 146 | { 147 | if (m_exception) 148 | { 149 | std::rethrow_exception(m_exception); 150 | } 151 | } 152 | 153 | private: 154 | when_all_counter* m_counter; 155 | std::exception_ptr m_exception; 156 | }; 157 | 158 | template 159 | class when_all_task final 160 | { 161 | public: 162 | using promise_type = when_all_task_promise; 163 | 164 | using coroutine_handle_t = typename promise_type::coroutine_handle_t; 165 | 166 | when_all_task(coroutine_handle_t coroutine) noexcept 167 | : m_coroutine(coroutine) 168 | { 169 | } 170 | 171 | when_all_task(when_all_task&& other) noexcept 172 | : m_coroutine(std::exchange(other.m_coroutine, coroutine_handle_t{})) 173 | { 174 | } 175 | 176 | ~when_all_task() 177 | { 178 | if (m_coroutine) 179 | m_coroutine.destroy(); 180 | } 181 | 182 | when_all_task(const when_all_task&) = delete; 183 | when_all_task& operator=(const when_all_task&) = delete; 184 | 185 | decltype(auto) result() & { return m_coroutine.promise().result(); } 186 | 187 | decltype(auto) result() && { return std::move(m_coroutine.promise()).result(); } 188 | 189 | decltype(auto) non_void_result() & 190 | { 191 | if constexpr (std::is_void_vresult())>) 192 | { 193 | this->result(); 194 | return void_value{}; 195 | } 196 | else 197 | { 198 | return this->result(); 199 | } 200 | } 201 | 202 | decltype(auto) non_void_result() && 203 | { 204 | if constexpr (std::is_void_vresult())>) 205 | { 206 | std::move(*this).result(); 207 | return void_value{}; 208 | } 209 | else 210 | { 211 | return std::move(*this).result(); 212 | } 213 | } 214 | 215 | private: 216 | template 217 | friend class when_all_ready_awaitable; 218 | 219 | void start(when_all_counter& counter) noexcept { m_coroutine.promise().start(counter); } 220 | 221 | coroutine_handle_t m_coroutine; 222 | }; 223 | 224 | template< 225 | typename AWAITABLE, 226 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 227 | std::enable_if_t, int> = 0> 228 | when_all_task make_when_all_task(AWAITABLE awaitable) 229 | { 230 | co_yield co_await static_cast(awaitable); 231 | } 232 | 233 | template< 234 | typename AWAITABLE, 235 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 236 | std::enable_if_t, int> = 0> 237 | when_all_task make_when_all_task(AWAITABLE awaitable) 238 | { 239 | co_await static_cast(awaitable); 240 | } 241 | 242 | template< 243 | typename AWAITABLE, 244 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 245 | std::enable_if_t, int> = 0> 246 | when_all_task make_when_all_task(std::reference_wrapper awaitable) 247 | { 248 | co_yield co_await awaitable.get(); 249 | } 250 | 251 | template< 252 | typename AWAITABLE, 253 | typename RESULT = typename cppcoro::awaitable_traits::await_result_t, 254 | std::enable_if_t, int> = 0> 255 | when_all_task make_when_all_task(std::reference_wrapper awaitable) 256 | { 257 | co_await awaitable.get(); 258 | } 259 | } // namespace detail 260 | } // namespace cppcoro 261 | 262 | #endif 263 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/is_awaitable.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_IS_AWAITABLE_HPP_INCLUDED 6 | #define CPPCORO_IS_AWAITABLE_HPP_INCLUDED 7 | 8 | #include 9 | 10 | #include 11 | 12 | namespace cppcoro 13 | { 14 | template> 15 | struct is_awaitable : std::false_type 16 | { 17 | }; 18 | 19 | template 20 | struct is_awaitable()))>> 21 | : std::true_type 22 | { 23 | }; 24 | 25 | template 26 | constexpr bool is_awaitable_v = is_awaitable::value; 27 | } // namespace cppcoro 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/sync_wait.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_SYNC_WAIT_HPP_INCLUDED 6 | #define CPPCORO_SYNC_WAIT_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | namespace cppcoro 16 | { 17 | template 18 | auto sync_wait(AWAITABLE&& awaitable) -> 19 | typename cppcoro::awaitable_traits::await_result_t 20 | { 21 | auto task = detail::make_sync_wait_task(std::forward(awaitable)); 22 | detail::lightweight_manual_reset_event event; 23 | task.start(event); 24 | event.wait(); 25 | return task.result(); 26 | } 27 | } // namespace cppcoro 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/task.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_TASK_HPP_INCLUDED 6 | #define CPPCORO_TASK_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | namespace cppcoro 25 | { 26 | template 27 | class task; 28 | 29 | namespace detail 30 | { 31 | inline thread_local Allocator* allocator{ nullptr }; 32 | 33 | class task_promise_base 34 | { 35 | friend struct final_awaitable; 36 | 37 | struct final_awaitable 38 | { 39 | bool await_ready() const noexcept { return false; } 40 | 41 | template 42 | cppcoro::coroutine_handle<> 43 | await_suspend(cppcoro::coroutine_handle coro) noexcept 44 | { 45 | return coro.promise().m_continuation; 46 | } 47 | 48 | void await_resume() noexcept {} 49 | }; 50 | 51 | public: 52 | task_promise_base() noexcept {} 53 | 54 | static void* operator new(size_t sz) 55 | { 56 | if (allocator) 57 | { 58 | return allocator->allocate(sz); 59 | } 60 | else 61 | { 62 | return ::operator new(sz); 63 | } 64 | } 65 | 66 | static void operator delete(void* p, size_t sz) 67 | { 68 | if (allocator) 69 | { 70 | allocator->deallocate(p, sz); 71 | } 72 | else 73 | { 74 | ::operator delete(p, sz); 75 | } 76 | } 77 | 78 | // template 79 | // static void* 80 | // operator new(std::size_t sz, std::allocator_arg_t, async::Allocator& alloc, Args&...) 81 | // { 82 | // // Store a pointer to the allocator in the allocated memory 83 | // // region so that the allocator can be accessed in operator delete 84 | // std::size_t allocator_offset = (sz + alignof(void*) - 1u) & ~(alignof(void*) - 1u); 85 | // void* ptr = alloc.Allocate(allocator_offset + sizeof(void*)); 86 | // new (static_cast(ptr) + allocator_offset) 87 | // async::Allocator* { &alloc }; 88 | // return ptr; 89 | // } 90 | 91 | // static void operator delete(void* p, std::size_t sz) 92 | // { 93 | // std::size_t allocator_offset = (sz + alignof(void*) - 1u) & ~(alignof(void*) - 1u); 94 | // async::Allocator* alloc = *reinterpret_cast( 95 | // static_cast(p) + allocator_offset); 96 | // if (alloc) 97 | // { 98 | // alloc->Deallocate(p, allocator_offset + sizeof(void*)); 99 | // } 100 | // else 101 | // { 102 | // ::operator delete(p); 103 | // } 104 | // } 105 | 106 | auto initial_suspend() noexcept { return cppcoro::suspend_always{}; } 107 | 108 | auto final_suspend() noexcept { return final_awaitable{}; } 109 | 110 | void set_continuation(cppcoro::coroutine_handle<> continuation) noexcept 111 | { 112 | m_continuation = continuation; 113 | } 114 | 115 | private: 116 | cppcoro::coroutine_handle<> m_continuation; 117 | }; 118 | 119 | template 120 | class task_promise final : public task_promise_base 121 | { 122 | public: 123 | task_promise() noexcept {} 124 | 125 | ~task_promise() 126 | { 127 | switch (m_resultType) 128 | { 129 | case result_type::value: 130 | m_value.~T(); 131 | break; 132 | case result_type::exception: 133 | m_exception.~exception_ptr(); 134 | break; 135 | default: 136 | break; 137 | } 138 | } 139 | 140 | task get_return_object() noexcept; 141 | 142 | void unhandled_exception() noexcept 143 | { 144 | ::new (static_cast(std::addressof(m_exception))) 145 | std::exception_ptr(std::current_exception()); 146 | m_resultType = result_type::exception; 147 | } 148 | 149 | template>> 150 | void return_value(VALUE&& value) noexcept(std::is_nothrow_constructible_v) 151 | { 152 | ::new (static_cast(std::addressof(m_value))) T(std::forward(value)); 153 | m_resultType = result_type::value; 154 | } 155 | 156 | T& result() & 157 | { 158 | if (m_resultType == result_type::exception) 159 | { 160 | std::rethrow_exception(m_exception); 161 | } 162 | 163 | assert(m_resultType == result_type::value); 164 | 165 | return m_value; 166 | } 167 | 168 | // HACK: Need to have co_await of task return prvalue rather than 169 | // rvalue-reference to work around an issue with MSVC where returning 170 | // rvalue reference of a fundamental type from await_resume() will 171 | // cause the value to be copied to a temporary. This breaks the 172 | // sync_wait() implementation. 173 | // See https://github.com/lewissbaker/cppcoro/issues/40#issuecomment-326864107 174 | using rvalue_type = 175 | std::conditional_t || std::is_pointer_v, T, T&&>; 176 | 177 | rvalue_type result() && 178 | { 179 | if (m_resultType == result_type::exception) 180 | { 181 | std::rethrow_exception(m_exception); 182 | } 183 | 184 | assert(m_resultType == result_type::value); 185 | 186 | return std::move(m_value); 187 | } 188 | 189 | private: 190 | enum class result_type 191 | { 192 | empty, 193 | value, 194 | exception 195 | }; 196 | 197 | result_type m_resultType = result_type::empty; 198 | 199 | union 200 | { 201 | T m_value; 202 | std::exception_ptr m_exception; 203 | }; 204 | }; 205 | 206 | template<> 207 | class task_promise : public task_promise_base 208 | { 209 | public: 210 | task_promise() noexcept = default; 211 | 212 | task get_return_object() noexcept; 213 | 214 | void return_void() noexcept {} 215 | 216 | void unhandled_exception() noexcept { m_exception = std::current_exception(); } 217 | 218 | void result() 219 | { 220 | if (m_exception) 221 | { 222 | std::rethrow_exception(m_exception); 223 | } 224 | } 225 | 226 | private: 227 | std::exception_ptr m_exception; 228 | }; 229 | 230 | template 231 | class task_promise : public task_promise_base 232 | { 233 | public: 234 | task_promise() noexcept = default; 235 | 236 | task get_return_object() noexcept; 237 | 238 | void unhandled_exception() noexcept { m_exception = std::current_exception(); } 239 | 240 | void return_value(T& value) noexcept { m_value = std::addressof(value); } 241 | 242 | T& result() 243 | { 244 | if (m_exception) 245 | { 246 | std::rethrow_exception(m_exception); 247 | } 248 | 249 | return *m_value; 250 | } 251 | 252 | private: 253 | T* m_value = nullptr; 254 | std::exception_ptr m_exception; 255 | }; 256 | } // namespace detail 257 | 258 | /// \brief 259 | /// A task represents an operation that produces a result both lazily 260 | /// and asynchronously. 261 | /// 262 | /// When you call a coroutine that returns a task, the coroutine 263 | /// simply captures any passed parameters and returns exeuction to the 264 | /// caller. Execution of the coroutine body does not start until the 265 | /// coroutine is first co_await'ed. 266 | template 267 | class [[nodiscard]] task 268 | { 269 | public: 270 | using promise_type = detail::task_promise; 271 | 272 | using value_type = T; 273 | 274 | private: 275 | struct awaitable_base 276 | { 277 | cppcoro::coroutine_handle m_coroutine; 278 | 279 | awaitable_base(cppcoro::coroutine_handle coroutine) noexcept 280 | : m_coroutine(coroutine) 281 | { 282 | } 283 | 284 | bool await_ready() const noexcept { return !m_coroutine || m_coroutine.done(); } 285 | 286 | cppcoro::coroutine_handle<> 287 | await_suspend(cppcoro::coroutine_handle<> awaitingCoroutine) noexcept 288 | { 289 | m_coroutine.promise().set_continuation(awaitingCoroutine); 290 | return m_coroutine; 291 | } 292 | }; 293 | 294 | public: 295 | task() noexcept 296 | : m_coroutine(nullptr) 297 | { 298 | } 299 | 300 | explicit task(cppcoro::coroutine_handle coroutine) 301 | : m_coroutine(coroutine) 302 | { 303 | } 304 | 305 | task(task&& t) noexcept 306 | : m_coroutine(t.m_coroutine) 307 | { 308 | t.m_coroutine = nullptr; 309 | } 310 | 311 | /// Disable copy construction/assignment. 312 | task(const task&) = delete; 313 | task& operator=(const task&) = delete; 314 | 315 | /// Frees resources used by this task. 316 | ~task() 317 | { 318 | if (m_coroutine) 319 | { 320 | m_coroutine.destroy(); 321 | } 322 | } 323 | 324 | task& operator=(task&& other) noexcept 325 | { 326 | if (std::addressof(other) != this) 327 | { 328 | if (m_coroutine) 329 | { 330 | m_coroutine.destroy(); 331 | } 332 | 333 | m_coroutine = other.m_coroutine; 334 | other.m_coroutine = nullptr; 335 | } 336 | 337 | return *this; 338 | } 339 | 340 | /// \brief 341 | /// Query if the task result is complete. 342 | /// 343 | /// Awaiting a task that is ready is guaranteed not to block/suspend. 344 | bool is_ready() const noexcept { return !m_coroutine || m_coroutine.done(); } 345 | 346 | auto operator co_await() const& noexcept 347 | { 348 | struct awaitable : awaitable_base 349 | { 350 | using awaitable_base::awaitable_base; 351 | 352 | decltype(auto) await_resume() 353 | { 354 | if (!this->m_coroutine) 355 | { 356 | throw broken_promise{}; 357 | } 358 | 359 | return this->m_coroutine.promise().result(); 360 | } 361 | }; 362 | 363 | return awaitable{ m_coroutine }; 364 | } 365 | 366 | auto operator co_await() const&& noexcept 367 | { 368 | struct awaitable : awaitable_base 369 | { 370 | using awaitable_base::awaitable_base; 371 | 372 | decltype(auto) await_resume() 373 | { 374 | if (!this->m_coroutine) 375 | { 376 | throw broken_promise{}; 377 | } 378 | 379 | return std::move(this->m_coroutine.promise()).result(); 380 | } 381 | }; 382 | 383 | return awaitable{ m_coroutine }; 384 | } 385 | 386 | /// \brief 387 | /// Returns an awaitable that will await completion of the task without 388 | /// attempting to retrieve the result. 389 | auto when_ready() const noexcept 390 | { 391 | struct awaitable : awaitable_base 392 | { 393 | using awaitable_base::awaitable_base; 394 | 395 | void await_resume() const noexcept {} 396 | }; 397 | 398 | return awaitable{ m_coroutine }; 399 | } 400 | 401 | private: 402 | cppcoro::coroutine_handle m_coroutine; 403 | }; 404 | 405 | namespace detail 406 | { 407 | template 408 | task task_promise::get_return_object() noexcept 409 | { 410 | return task{ cppcoro::coroutine_handle::from_promise(*this) }; 411 | } 412 | 413 | inline task task_promise::get_return_object() noexcept 414 | { 415 | return task{ cppcoro::coroutine_handle::from_promise(*this) }; 416 | } 417 | 418 | template 419 | task task_promise::get_return_object() noexcept 420 | { 421 | return task{ cppcoro::coroutine_handle::from_promise(*this) }; 422 | } 423 | } // namespace detail 424 | 425 | template 426 | auto make_task(AWAITABLE awaitable) -> task< 427 | detail::remove_rvalue_reference_t::await_result_t>> 428 | { 429 | co_return co_await static_cast(awaitable); 430 | } 431 | } // namespace cppcoro 432 | 433 | #endif 434 | -------------------------------------------------------------------------------- /cppcoro/include/cppcoro/when_all_ready.hpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | #ifndef CPPCORO_WHEN_ALL_READY_HPP_INCLUDED 6 | #define CPPCORO_WHEN_ALL_READY_HPP_INCLUDED 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | namespace cppcoro 21 | { 22 | template< 23 | typename... AWAITABLES, 24 | std::enable_if_t< 25 | std::conjunction_v< 26 | is_awaitable>>...>, 27 | int> = 0> 28 | [[nodiscard]] auto when_all_ready(AWAITABLES&&... awaitables) 29 | { 30 | return detail::when_all_ready_awaitable< 31 | std::tuple>>::await_result_t>...>>( 33 | std::make_tuple(detail::make_when_all_task(std::forward(awaitables))...)); 34 | } 35 | 36 | // TODO: Generalise this from vector to arbitrary sequence of awaitable. 37 | 38 | template< 39 | typename AWAITABLE, 40 | typename RESULT = 41 | typename awaitable_traits>::await_result_t> 42 | [[nodiscard]] auto when_all_ready(std::vector awaitables) 43 | { 44 | std::vector> tasks; 45 | 46 | tasks.reserve(awaitables.size()); 47 | 48 | for (auto& awaitable : awaitables) 49 | { 50 | tasks.emplace_back(detail::make_when_all_task(std::move(awaitable))); 51 | } 52 | 53 | return detail::when_all_ready_awaitable>>( 54 | std::move(tasks)); 55 | } 56 | } // namespace cppcoro 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /cppcoro/lib/lightweight_manual_reset_event.cpp: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) Lewis Baker 3 | // Licenced under MIT license. See LICENSE.txt for details. 4 | /////////////////////////////////////////////////////////////////////////////// 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace 19 | { 20 | namespace local 21 | { 22 | // No futex() function provided by libc. 23 | // Wrap the syscall ourselves here. 24 | int futex( 25 | int* UserAddress, 26 | int FutexOperation, 27 | int Value, 28 | const struct timespec* timeout, 29 | int* UserAddress2, 30 | int Value3) 31 | { 32 | return syscall( 33 | SYS_futex, UserAddress, FutexOperation, Value, timeout, UserAddress2, Value3); 34 | } 35 | } // namespace local 36 | } // namespace 37 | 38 | cppcoro::detail::lightweight_manual_reset_event::lightweight_manual_reset_event(bool initiallySet) 39 | : m_value(initiallySet ? 1 : 0) 40 | { 41 | } 42 | 43 | cppcoro::detail::lightweight_manual_reset_event::~lightweight_manual_reset_event() 44 | { 45 | } 46 | 47 | void cppcoro::detail::lightweight_manual_reset_event::set() noexcept 48 | { 49 | m_value.store(1, std::memory_order_release); 50 | 51 | constexpr int numberOfWaitersToWakeUp = INT_MAX; 52 | 53 | [[maybe_unused]] int numberOfWaitersWokenUp = local::futex( 54 | reinterpret_cast(&m_value), 55 | FUTEX_WAKE_PRIVATE, 56 | numberOfWaitersToWakeUp, 57 | nullptr, 58 | nullptr, 59 | 0); 60 | 61 | // There are no errors expected here unless this class (or the caller) 62 | // has done something wrong. 63 | assert(numberOfWaitersWokenUp != -1); 64 | } 65 | 66 | void cppcoro::detail::lightweight_manual_reset_event::reset() noexcept 67 | { 68 | m_value.store(0, std::memory_order_relaxed); 69 | } 70 | 71 | void cppcoro::detail::lightweight_manual_reset_event::wait() noexcept 72 | { 73 | // Wait in a loop as futex() can have spurious wake-ups. 74 | int oldValue = m_value.load(std::memory_order_acquire); 75 | while (oldValue == 0) 76 | { 77 | int result = local::futex( 78 | reinterpret_cast(&m_value), FUTEX_WAIT_PRIVATE, oldValue, nullptr, nullptr, 0); 79 | if (result == -1) 80 | { 81 | if (errno == EAGAIN) 82 | { 83 | // The state was changed from zero before we could wait. 84 | // Must have been changed to 1. 85 | return; 86 | } 87 | 88 | // Other errors we'll treat as transient and just read the 89 | // value and go around the loop again. 90 | } 91 | 92 | oldValue = m_value.load(std::memory_order_acquire); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /queries/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(tpch_q1 tpch_q1.cc) 2 | target_link_libraries(tpch_q1 Threads::Threads storage) 3 | 4 | add_executable(tpch_q14 tpch_q14.cc) 5 | target_link_libraries(tpch_q14 Threads::Threads storage) -------------------------------------------------------------------------------- /queries/tpch_q1.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "cppcoro/sync_wait.hpp" 17 | #include "cppcoro/task.hpp" 18 | #include "cppcoro/when_all_ready.hpp" 19 | #include "storage/file.h" 20 | #include "storage/io_uring.h" 21 | #include "storage/schema.h" 22 | #include "storage/swip.h" 23 | #include "storage/types.h" 24 | 25 | namespace { 26 | using namespace storage; 27 | 28 | bool do_work = true; 29 | uint64_t num_tuples_per_morsel = 1'000; 30 | 31 | class Cache { 32 | public: 33 | Cache(std::span swips, const File &data_file) 34 | : swips_(swips), data_file_(data_file) { 35 | frames_.reserve(swips.size()); 36 | } 37 | 38 | void Populate(std::span swip_indexes) { 39 | constexpr uint64_t kNumConcurrentTasks = 64ull; 40 | IOUring ring(kNumConcurrentTasks); 41 | Countdown countdown(kNumConcurrentTasks); 42 | 43 | std::vector> tasks; 44 | tasks.reserve(kNumConcurrentTasks + 1); 45 | 46 | uint64_t partition_size = 47 | (swip_indexes.size() + kNumConcurrentTasks - 1) / kNumConcurrentTasks; 48 | 49 | for (uint64_t i = 0; i != kNumConcurrentTasks; ++i) { 50 | uint64_t begin = std::min(i * partition_size, swip_indexes.size()); 51 | auto end = std::min(begin + partition_size, swip_indexes.size()); 52 | tasks.emplace_back( 53 | AsyncLoadPages(ring, begin, end, countdown, swip_indexes)); 54 | } 55 | tasks.emplace_back(DrainRing(ring, countdown)); 56 | cppcoro::sync_wait(cppcoro::when_all_ready(std::move(tasks))); 57 | } 58 | 59 | private: 60 | cppcoro::task AsyncLoadPages(IOUring &ring, uint64_t begin, 61 | uint64_t end, Countdown &countdown, 62 | std::span swip_indexes) { 63 | for (uint64_t i = begin; i != end; ++i) { 64 | frames_.emplace_back(); 65 | LineitemPageQ1 &page = frames_.back(); 66 | co_await data_file_.AsyncReadPage(ring, 67 | swips_[swip_indexes[i]].GetPageIndex(), 68 | reinterpret_cast(&page)); 69 | swips_[swip_indexes[i]].SetPointer(&page); 70 | } 71 | countdown.Decrement(); 72 | } 73 | 74 | std::span swips_; 75 | const File &data_file_; 76 | std::vector frames_; 77 | }; 78 | 79 | struct HashTableEntry { 80 | Numeric<12, 2> sum_qty; 81 | Numeric<12, 2> sum_base_price; 82 | Numeric<12, 2> sum_disc; 83 | Numeric<12, 4> sum_disc_price; 84 | Numeric<12, 4> sum_charge; 85 | uint32_t count; 86 | Char l_returnflag; 87 | Char l_linestatus; 88 | }; 89 | 90 | using HashTable = std::vector>; 91 | using ValidHashTableIndexes = std::vector; 92 | 93 | // implementation idea for query 1 stolen from the MonetDB/X100 paper 94 | class QueryRunner { 95 | public: 96 | QueryRunner(uint32_t num_threads, std::span swips, 97 | const File &data_file, uint32_t num_ring_entries = 0) 98 | : thread_local_hash_tables_(num_threads), 99 | thread_local_valid_hash_table_indexes_(num_threads), 100 | high_date_(Date::FromString("1998-09-02|", '|').value), 101 | num_threads_(num_threads), 102 | swips_(swips), 103 | data_file_(data_file), 104 | num_ring_entries_(num_ring_entries) { 105 | for (auto &hash_table : thread_local_hash_tables_) { 106 | hash_table.resize(1ull << 16); 107 | } 108 | 109 | if (num_ring_entries > 0) { 110 | thread_local_rings_.reserve(num_threads); 111 | for (uint32_t i = 0; i != num_threads; ++i) { 112 | thread_local_rings_.emplace_back(num_ring_entries); 113 | } 114 | } 115 | } 116 | 117 | static void ProcessTuples(const LineitemPageQ1 &page, HashTable &hash_table, 118 | ValidHashTableIndexes &valid_hash_table_indexes, 119 | Date high_date) { 120 | Numeric<12, 2> one{int64_t{100}}; // assigns a raw value 121 | for (uint32_t i = 0; i != page.num_tuples; ++i) { 122 | if (page.l_shipdate[i] <= high_date) { 123 | uint32_t hash_table_index = page.l_returnflag[i]; 124 | hash_table_index = (hash_table_index << 8) + page.l_linestatus[i]; 125 | auto &entry = hash_table[hash_table_index]; 126 | if (!entry) { 127 | entry = std::make_unique(); 128 | entry->l_returnflag = page.l_returnflag[i]; 129 | entry->l_linestatus = page.l_linestatus[i]; 130 | entry->count = 0; 131 | valid_hash_table_indexes.push_back(hash_table_index); 132 | } 133 | 134 | ++entry->count; 135 | entry->sum_qty += page.l_quantity[i]; 136 | entry->sum_base_price += page.l_extendedprice[i]; 137 | entry->sum_disc += page.l_discount[i]; 138 | Numeric<12, 4> common_term = 139 | page.l_extendedprice[i] * (one - page.l_discount[i]); 140 | entry->sum_disc_price += common_term; 141 | entry->sum_charge += common_term.CastM2() * (one + page.l_tax[i]); 142 | } 143 | } 144 | } 145 | 146 | static void ProcessPages(LineitemPageQ1 &page, std::span swips, 147 | HashTable &hash_table, 148 | ValidHashTableIndexes &valid_hash_table_indexes, 149 | Date high_date, const File &data_file) { 150 | for (auto swip : swips) { 151 | LineitemPageQ1 *data; 152 | 153 | if (swip.IsPageIndex()) { 154 | data_file.ReadPage(swip.GetPageIndex(), 155 | reinterpret_cast(&page)); 156 | data = &page; 157 | } else { 158 | data = swip.GetPointer(); 159 | } 160 | if (do_work) { 161 | ProcessTuples(*data, hash_table, valid_hash_table_indexes, high_date); 162 | } 163 | } 164 | } 165 | 166 | static cppcoro::task AsyncProcessPages( 167 | LineitemPageQ1 &page, std::vector swips, HashTable &hash_table, 168 | ValidHashTableIndexes &valid_hash_table_indexes, Date high_date, 169 | const File &data_file, IOUring &ring, Countdown &countdown) { 170 | std::partition(swips.begin(), swips.end(), 171 | [](Swip swip) { return swip.IsPageIndex(); }); 172 | for (auto swip : swips) { 173 | LineitemPageQ1 *data; 174 | 175 | if (swip.IsPageIndex()) { 176 | co_await data_file.AsyncReadPage(ring, swip.GetPageIndex(), 177 | reinterpret_cast(&page)); 178 | data = &page; 179 | } else { 180 | data = swip.GetPointer(); 181 | } 182 | if (do_work) { 183 | ProcessTuples(*data, hash_table, valid_hash_table_indexes, high_date); 184 | } 185 | } 186 | countdown.Decrement(); 187 | } 188 | 189 | bool IsSynchronous() const noexcept { return num_ring_entries_ == 0; } 190 | 191 | void StartProcessing() { 192 | std::atomic current_swip{0ull}; 193 | std::vector threads; 194 | threads.reserve(num_threads_); 195 | 196 | for (uint32_t thread_index = 0; thread_index != num_threads_; 197 | ++thread_index) { 198 | threads.emplace_back( 199 | [&hash_table = thread_local_hash_tables_[thread_index], 200 | &valid_hash_table_indexes = 201 | thread_local_valid_hash_table_indexes_[thread_index], 202 | high_date = high_date_, ¤t_swip, num_swips = swips_.size(), 203 | &swips = swips_, &data_file = data_file_, 204 | is_synchronous = IsSynchronous(), 205 | &ring = thread_local_rings_[thread_index], 206 | num_ring_entries = num_ring_entries_] { 207 | if (!is_synchronous) { 208 | cppcoro::detail::allocator = new Allocator(num_ring_entries); 209 | cppcoro::detail::sync_allocator = new Allocator(1); 210 | } 211 | std::allocator alloc; 212 | auto pages = alloc.allocate(is_synchronous ? 1 : num_ring_entries); 213 | 214 | // process ceil(num_tuples_per_morsel / kMaxNumTuples) pages per 215 | // morsel 216 | // => each morsel contains circa num_tuples_per_morsel tuples 217 | const uint64_t kSyncFetchIncrement = 218 | (num_tuples_per_morsel + LineitemPageQ1::kMaxNumTuples - 1) / 219 | LineitemPageQ1::kMaxNumTuples; 220 | 221 | uint64_t fetch_increment; 222 | 223 | if (is_synchronous) { 224 | fetch_increment = kSyncFetchIncrement; 225 | } else { 226 | // process num_ring_entries morsels together 227 | fetch_increment = kSyncFetchIncrement * num_ring_entries; 228 | } 229 | 230 | while (true) { 231 | auto begin = current_swip.fetch_add(fetch_increment); 232 | if (begin >= num_swips) { 233 | return; 234 | } 235 | auto end = std::min(num_swips, begin + fetch_increment); 236 | auto size = end - begin; 237 | 238 | if (is_synchronous) { 239 | ProcessPages(pages[0], swips.subspan(begin, size), hash_table, 240 | valid_hash_table_indexes, high_date, data_file); 241 | } else { 242 | Countdown countdown(num_ring_entries); 243 | std::vector> tasks; 244 | tasks.reserve(num_ring_entries + 1); 245 | 246 | auto num_pages_per_task = 247 | (size + num_ring_entries - 1) / num_ring_entries; 248 | 249 | for (uint32_t i = 0; i != num_ring_entries; ++i) { 250 | auto local_begin = 251 | std::min(begin + i * num_pages_per_task, end); 252 | auto local_end = 253 | std::min(local_begin + num_pages_per_task, end); 254 | tasks.emplace_back(AsyncProcessPages( 255 | pages[i], {&swips[local_begin], &swips[local_end]}, 256 | hash_table, valid_hash_table_indexes, high_date, 257 | data_file, ring, countdown)); 258 | } 259 | tasks.emplace_back(DrainRing(ring, countdown)); 260 | cppcoro::sync_wait(cppcoro::when_all_ready(std::move(tasks))); 261 | } 262 | } 263 | 264 | alloc.deallocate(pages, is_synchronous ? 1 : num_ring_entries); 265 | if (!is_synchronous) { 266 | delete cppcoro::detail::allocator; 267 | cppcoro::detail::allocator = nullptr; 268 | delete cppcoro::detail::sync_allocator; 269 | cppcoro::detail::sync_allocator = nullptr; 270 | } 271 | }); 272 | } 273 | 274 | for (auto &t : threads) { 275 | t.join(); 276 | } 277 | } 278 | 279 | void DoPostProcessing(bool should_print_result) { 280 | if (do_work) { 281 | // post-processing happens in a single thread. That's okay, because there 282 | // are only four groups 283 | auto &result_hash_table = thread_local_hash_tables_.front(); 284 | auto &result_valid_hash_table_indexes = 285 | thread_local_valid_hash_table_indexes_.front(); 286 | 287 | for (uint32_t i = 1; i != num_threads_; ++i) { 288 | auto &local_hash_table = thread_local_hash_tables_[i]; 289 | for (auto valid_hash_table_index : 290 | thread_local_valid_hash_table_indexes_[i]) { 291 | auto &local_entry = local_hash_table[valid_hash_table_index]; 292 | auto &result_entry = result_hash_table[valid_hash_table_index]; 293 | if (result_entry) { 294 | result_entry->sum_qty += local_entry->sum_qty; 295 | result_entry->sum_base_price += local_entry->sum_base_price; 296 | result_entry->sum_disc += local_entry->sum_disc; 297 | result_entry->sum_disc_price += local_entry->sum_disc_price; 298 | result_entry->sum_charge += local_entry->sum_charge; 299 | result_entry->count += local_entry->count; 300 | } else { 301 | result_entry = std::move(local_entry); 302 | result_valid_hash_table_indexes.push_back(valid_hash_table_index); 303 | } 304 | } 305 | } 306 | 307 | std::vector result_entries; 308 | for (auto valid_hash_table_index : result_valid_hash_table_indexes) { 309 | result_entries.push_back( 310 | result_hash_table[valid_hash_table_index].get()); 311 | } 312 | std::sort(result_entries.begin(), result_entries.end(), 313 | [](HashTableEntry *lhs, HashTableEntry *rhs) { 314 | return std::pair(lhs->l_returnflag, lhs->l_linestatus) < 315 | std::pair(rhs->l_returnflag, rhs->l_linestatus); 316 | }); 317 | 318 | if (should_print_result) { 319 | std::cerr 320 | << "l_returnflag|l_linestatus|sum_qty|sum_base_price|sum_disc_" 321 | "price|sum_charge|avg_qty|avg_price|avg_disc|count_order\n"; 322 | for (auto *entry : result_entries) { 323 | std::cerr << entry->l_returnflag << "|" << entry->l_linestatus << "|" 324 | << entry->sum_qty << "|" << entry->sum_base_price << "|" 325 | << entry->sum_disc_price << "|" << entry->sum_charge << "|" 326 | << entry->sum_qty / entry->count << "|" 327 | << entry->sum_base_price / entry->count << "|" 328 | << entry->sum_disc / entry->count << "|" << entry->count 329 | << "\n"; 330 | } 331 | } 332 | } 333 | } 334 | 335 | private: 336 | std::vector thread_local_hash_tables_; 337 | std::vector thread_local_valid_hash_table_indexes_; 338 | std::vector thread_local_rings_; 339 | const Date high_date_; 340 | const uint32_t num_threads_; 341 | const std::span swips_; 342 | const File &data_file_; 343 | const uint32_t num_ring_entries_; 344 | }; 345 | 346 | std::vector GetSwips(uint64_t size_of_data_file) { 347 | auto num_pages = size_of_data_file / kPageSize; 348 | std::vector swips; 349 | swips.reserve(num_pages); 350 | for (PageIndex i = 0; i != num_pages; ++i) { 351 | swips.emplace_back(Swip::MakePageIndex(i)); 352 | } 353 | return swips; 354 | } 355 | 356 | } // namespace 357 | 358 | int main(int argc, char *argv[]) { 359 | if (argc != 9) { 360 | std::cerr << "Usage: " << argv[0] 361 | << " lineitem.dat num_threads num_entries_per_ring " 362 | "num_tuples_per_morsel do_work " 363 | "do_random_io print_result print_header\n"; 364 | return 1; 365 | } 366 | 367 | std::string path_to_lineitem{argv[1]}; 368 | unsigned num_threads = std::atoi(argv[2]); 369 | unsigned num_entries_per_ring = std::atoi(argv[3]); 370 | num_tuples_per_morsel = std::atoi(argv[4]); 371 | std::istringstream(argv[5]) >> std::boolalpha >> do_work; 372 | bool do_random_io; 373 | std::istringstream(argv[6]) >> std::boolalpha >> do_random_io; 374 | bool print_result; 375 | std::istringstream(argv[7]) >> std::boolalpha >> print_result; 376 | bool print_header; 377 | std::istringstream(argv[8]) >> std::boolalpha >> print_header; 378 | 379 | const File file{path_to_lineitem.c_str(), File::kRead, true}; 380 | auto file_size = file.ReadSize(); 381 | auto swips = GetSwips(file_size); 382 | 383 | std::vector swip_indexes(swips.size()); 384 | { 385 | std::random_device rd; 386 | std::mt19937 g(rd()); 387 | g.seed(42); 388 | 389 | if (do_random_io) { 390 | std::shuffle(swips.begin(), swips.end(), g); 391 | } 392 | 393 | std::iota(swip_indexes.begin(), swip_indexes.end(), 0ull); 394 | std::shuffle(swip_indexes.begin(), swip_indexes.end(), g); 395 | } 396 | 397 | Cache cache{swips, file}; 398 | 399 | auto partition_size = 400 | (swip_indexes.size() + 9) / 10; // divide in 10 partitions 401 | 402 | if (print_header) { 403 | std::cout << "kind_of_io,page_size_power,num_threads,num_cached_pages,num_" 404 | "total_pages,num_entries_per_ring,num_tuples_per_morsel,do_" 405 | "work,do_random_io,time,file_size,throughput\n"; 406 | } 407 | 408 | // Start with 0% cached, then 10%, then 20%, ... 409 | for (int i = 0; i != 11; ++i) { 410 | if (i > 0) { 411 | auto offset = std::min((i - 1) * partition_size, swip_indexes.size()); 412 | auto size = std::min(partition_size, swip_indexes.size() - offset); 413 | cache.Populate( 414 | std::span{swip_indexes}.subspan(offset, size)); 415 | } 416 | 417 | { 418 | QueryRunner synchronousRunner{num_threads, swips, file}; 419 | auto start = std::chrono::steady_clock::now(); 420 | synchronousRunner.StartProcessing(); 421 | synchronousRunner.DoPostProcessing(print_result); 422 | auto end = std::chrono::steady_clock::now(); 423 | auto milliseconds = 424 | std::chrono::duration_cast(end - start) 425 | .count(); 426 | std::cout << "synchronous," << kPageSizePower << "," << num_threads << "," 427 | << std::min(i * partition_size, swip_indexes.size()) << "," 428 | << swip_indexes.size() << ",0," << num_tuples_per_morsel << "," 429 | << std::boolalpha << do_work << "," << do_random_io << "," 430 | << milliseconds << "," << file_size << "," 431 | << (file_size / 1000000000.0) / (milliseconds / 1000.0) << "\n"; 432 | } 433 | 434 | { 435 | QueryRunner asynchronousRunner{num_threads, swips, file, 436 | num_entries_per_ring}; 437 | auto start = std::chrono::steady_clock::now(); 438 | asynchronousRunner.StartProcessing(); 439 | asynchronousRunner.DoPostProcessing(print_result); 440 | auto end = std::chrono::steady_clock::now(); 441 | auto milliseconds = 442 | std::chrono::duration_cast(end - start) 443 | .count(); 444 | std::cout << "asynchronous," << kPageSizePower << "," << num_threads 445 | << "," << std::min(i * partition_size, swip_indexes.size()) 446 | << "," << swip_indexes.size() << "," << num_entries_per_ring 447 | << "," << num_tuples_per_morsel << "," << std::boolalpha 448 | << do_work << "," << do_random_io << "," << milliseconds << "," 449 | << file_size << "," 450 | << (file_size / 1000000000.0) / (milliseconds / 1000.0) << "\n"; 451 | } 452 | } 453 | } -------------------------------------------------------------------------------- /queries/tpch_q14.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "cppcoro/sync_wait.hpp" 24 | #include "cppcoro/task.hpp" 25 | #include "cppcoro/when_all_ready.hpp" 26 | #include "storage/file.h" 27 | #include "storage/io_uring.h" 28 | #include "storage/schema.h" 29 | #include "storage/swip.h" 30 | #include "storage/types.h" 31 | 32 | namespace { 33 | using namespace storage; 34 | 35 | class InMemoryLineitemData { 36 | public: 37 | explicit InMemoryLineitemData(uint64_t capacity) 38 | : l_partkey(capacity), 39 | l_extendedprice(capacity), 40 | l_discount(capacity), 41 | l_shipdate(capacity), 42 | size_(0) {} 43 | 44 | std::vector l_partkey; 45 | std::vector> l_extendedprice; 46 | std::vector> l_discount; 47 | std::vector l_shipdate; 48 | 49 | uint64_t IncreaseSize(uint64_t increment) noexcept { 50 | return std::atomic_ref{size_}.fetch_add(increment); 51 | } 52 | 53 | uint64_t GetSize() const noexcept { return size_; } 54 | 55 | private: 56 | uint64_t size_; 57 | }; 58 | 59 | class LineitemHashTable { 60 | public: 61 | explicit LineitemHashTable(unsigned thread_count) 62 | : thread_local_entries_(thread_count) {} 63 | 64 | void InsertLocalEntries(const InMemoryLineitemData &data, 65 | uint64_t begin_tuple_index, uint64_t end_tuple_index, 66 | unsigned thread_index) { 67 | auto &entries = thread_local_entries_[thread_index]; 68 | auto lower_date_boundary = Date::FromString("1995-09-01|", '|').value; 69 | auto upper_date_boundary = Date::FromString("1995-09-30|", '|').value; 70 | 71 | for (auto tuple_index = begin_tuple_index; tuple_index != end_tuple_index; 72 | ++tuple_index) { 73 | if (lower_date_boundary <= data.l_shipdate[tuple_index] && 74 | data.l_shipdate[tuple_index] <= upper_date_boundary) { 75 | entries.emplace_back(data.l_partkey[tuple_index]); 76 | } 77 | } 78 | } 79 | 80 | void ResizeHashTable() { 81 | auto total_size = 0ull; 82 | for (const auto &entries : thread_local_entries_) { 83 | total_size += entries.size(); 84 | } 85 | hash_table_.resize(std::bit_ceil(total_size)); 86 | hash_table_mask_ = hash_table_.size() - 1; 87 | } 88 | 89 | void MergeLocalEntries(unsigned thread_index) noexcept { 90 | for (auto &entry : thread_local_entries_[thread_index]) { 91 | auto bucket_index = entry.partkey.hash() & hash_table_mask_; 92 | Entry *current = &hash_table_[bucket_index]; 93 | Entry *next = std::atomic_ref{current->next}.load(); 94 | while (true) { 95 | if (current->partkey == entry.partkey) { 96 | ++std::atomic_ref{current->count}; 97 | break; 98 | } else if (next == nullptr || entry.partkey < next->partkey) { 99 | // entry should be inserted after the current entry 100 | entry.next = next; 101 | if (std::atomic_ref{current->next}.compare_exchange_weak(next, 102 | &entry)) { 103 | break; 104 | } 105 | } else { // => entry.partkey >= next->partkey 106 | current = next; // we can even skip any elements that might have been 107 | // inserted between current and next in the meantime 108 | next = std::atomic_ref{current->next}.load(); 109 | } 110 | } 111 | } 112 | } 113 | 114 | uint32_t LookupCountForPartkey(Integer partkey) const noexcept { 115 | auto bucket_index = partkey.hash() & hash_table_mask_; 116 | for (Entry *current = hash_table_[bucket_index].next; current != nullptr; 117 | current = current->next) { 118 | if (current->partkey == partkey) { 119 | return current->count; 120 | } else if (partkey < current->partkey) { 121 | break; 122 | } 123 | } 124 | return 0u; 125 | } 126 | 127 | private: 128 | struct Entry { 129 | // partkey(0) is smaller than any partkey actually used 130 | Entry() noexcept : next(nullptr), partkey(0), count(0) {} 131 | 132 | explicit Entry(Integer partkey) noexcept 133 | : next(nullptr), partkey(partkey), count(1) {} 134 | 135 | Entry *next; 136 | const Integer partkey; 137 | uint32_t count; 138 | }; 139 | 140 | std::vector> thread_local_entries_; 141 | std::vector hash_table_; 142 | uint64_t hash_table_mask_; 143 | }; 144 | 145 | class PartHashTable { 146 | public: 147 | PartHashTable(unsigned thread_count, unsigned total_num_pages) 148 | : thread_local_entries_(thread_count), 149 | page_references_(total_num_pages), 150 | part_pages_buffer_(total_num_pages), 151 | num_used_buffer_pages_(0), 152 | num_cached_references_(0) { 153 | swips_.reserve(total_num_pages); 154 | for (PageIndex i{0}; i != total_num_pages; ++i) { 155 | swips_.emplace_back(Swip::MakePageIndex(i)); 156 | } 157 | } 158 | 159 | void InsertLocalEntries(const PartPage *begin, const PartPage *end, 160 | PageIndex begin_page_index, unsigned thread_index, 161 | const LineitemHashTable &lineitem_hash_table) { 162 | auto &entries = thread_local_entries_[thread_index]; 163 | auto current_page_index = begin_page_index; 164 | 165 | for (auto iter = begin; iter != end; ++iter, ++current_page_index) { 166 | uint32_t num_references = 0u; 167 | for (uint32_t i = 0, num_tuples = iter->num_tuples; i != num_tuples; 168 | ++i) { 169 | auto partkey = iter->p_partkey[i]; 170 | if (auto count = lineitem_hash_table.LookupCountForPartkey(partkey); 171 | count > 0) { 172 | entries.emplace_back(swips_[current_page_index], partkey, i); 173 | num_references += count; 174 | } 175 | } 176 | 177 | page_references_[current_page_index].num_references = num_references; 178 | } 179 | } 180 | 181 | void ResizeHashTable() { 182 | auto total_size = 0ull; 183 | for (const auto &entries : thread_local_entries_) { 184 | total_size += entries.size(); 185 | } 186 | hash_table_.resize(std::bit_ceil(total_size)); 187 | hash_table_mask_ = hash_table_.size() - 1; 188 | } 189 | 190 | void MergeLocalEntries(unsigned thread_index) noexcept { 191 | for (auto &entry : thread_local_entries_[thread_index]) { 192 | auto bucket_index = entry.partkey.hash() & hash_table_mask_; 193 | Entry *head = std::atomic_ref{hash_table_[bucket_index]}.load(); 194 | do { 195 | entry.next = head; 196 | } while (!std::atomic_ref{std::atomic_ref{hash_table_[bucket_index]}} 197 | .compare_exchange_weak(head, &entry)); 198 | } 199 | } 200 | 201 | struct LookupResult { 202 | Swip swip; 203 | uint32_t tuple_offset; 204 | }; 205 | 206 | LookupResult LookupPartkey(Integer partkey) const { 207 | auto bucket_index = partkey.hash() & hash_table_mask_; 208 | for (Entry *current = hash_table_[bucket_index]; current != nullptr; 209 | current = current->next) { 210 | if (current->partkey == partkey) { 211 | return {current->swip, current->tuple_offset}; 212 | } 213 | } 214 | throw "Unable to find partkey"; // this should never happen 215 | } 216 | 217 | uint64_t GetTotalNumPageReferences() const noexcept { 218 | uint64_t count = 0ull; 219 | for (const auto &page_reference : page_references_) { 220 | count += page_reference.num_references; 221 | } 222 | return count; 223 | } 224 | 225 | void CacheAtLeastNumReferences(File &part_data_file, 226 | uint64_t num_references_to_be_cached) { 227 | constexpr uint64_t kNumConcurrentTasks = 64ull; 228 | IOUring ring(kNumConcurrentTasks); 229 | Countdown countdown(kNumConcurrentTasks); 230 | 231 | std::vector> tasks; 232 | tasks.reserve(kNumConcurrentTasks + 1); 233 | 234 | auto global_begin = num_used_buffer_pages_; 235 | 236 | auto num_swips = swips_.size(); 237 | for (; num_cached_references_ < num_references_to_be_cached && 238 | num_used_buffer_pages_ != num_swips; 239 | ++num_used_buffer_pages_) { 240 | const auto &page_reference = page_references_[num_used_buffer_pages_]; 241 | assert(swips_[num_used_buffer_pages_].GetPageIndex() == 242 | num_used_buffer_pages_); 243 | num_cached_references_ += page_reference.num_references; 244 | } 245 | 246 | auto global_end = num_used_buffer_pages_; 247 | 248 | auto num_pages = global_end - global_begin; 249 | uint64_t partition_size = 250 | (num_pages + kNumConcurrentTasks - 1) / kNumConcurrentTasks; 251 | for (uint64_t i = 0; i != kNumConcurrentTasks; ++i) { 252 | uint64_t begin = std::min(global_begin + i * partition_size, global_end); 253 | auto end = std::min(begin + partition_size, global_end); 254 | tasks.emplace_back( 255 | AsyncLoadPages(ring, begin, end, countdown, part_data_file)); 256 | } 257 | tasks.emplace_back(DrainRing(ring, countdown)); 258 | cppcoro::sync_wait(cppcoro::when_all_ready(std::move(tasks))); 259 | } 260 | 261 | cppcoro::task AsyncLoadPages(IOUring &ring, uint64_t begin, 262 | uint64_t end, Countdown &countdown, 263 | File &part_data_file) { 264 | for (uint64_t i = begin; i != end; ++i) { 265 | auto *page = reinterpret_cast(&part_pages_buffer_[i]); 266 | co_await part_data_file.AsyncReadPage(ring, i, page); 267 | swips_[i].SetPointer(page); 268 | } 269 | countdown.Decrement(); 270 | } 271 | 272 | uint64_t GetNumAlreadyCachedReferences() const noexcept { 273 | return num_cached_references_; 274 | } 275 | 276 | private: 277 | struct Entry { 278 | Entry(const Swip &swip, Integer partkey, uint32_t tuple_offset) noexcept 279 | : next(nullptr), 280 | swip(swip), 281 | partkey(partkey), 282 | tuple_offset(tuple_offset) {} 283 | 284 | Entry *next; 285 | const Swip &swip; 286 | const Integer partkey; 287 | uint32_t tuple_offset; 288 | }; 289 | 290 | struct PageReferences { 291 | uint32_t num_references; 292 | }; 293 | 294 | std::vector> thread_local_entries_; 295 | std::vector swips_; 296 | std::vector hash_table_; 297 | std::vector page_references_; 298 | uint64_t hash_table_mask_; 299 | std::vector part_pages_buffer_; 300 | uint64_t num_used_buffer_pages_; 301 | uint64_t num_cached_references_; 302 | }; 303 | 304 | PartHashTable BuildHashTableForPart(const InMemoryLineitemData &lineitem_data, 305 | const char *path_to_part) { 306 | unsigned thread_count = std::thread::hardware_concurrency(); 307 | 308 | // First, we build a hash table on lineitem after applying the predicate used 309 | // in query 14. We need this hash table to figure out which partkeys are 310 | // actually required by the query and how often each page of the part relation 311 | // is accessed so that we can correctly implement caching for the benchmark 312 | // later. 313 | LineitemHashTable lineitem_hash_table{thread_count}; 314 | { 315 | auto total_num_tuples = lineitem_data.GetSize(); 316 | auto num_tuples_per_thread = 317 | (total_num_tuples + thread_count - 1) / thread_count; 318 | std::vector threads; 319 | threads.reserve(thread_count); 320 | 321 | std::once_flag flag; 322 | std::latch latch{thread_count}; 323 | 324 | for (unsigned thread_index = 0; thread_index != thread_count; 325 | ++thread_index) { 326 | threads.emplace_back([thread_index, total_num_tuples, 327 | num_tuples_per_thread, &lineitem_data, &flag, 328 | &latch, &lineitem_hash_table]() { 329 | auto begin = 330 | std::min(thread_index * num_tuples_per_thread, total_num_tuples); 331 | auto end = std::min(begin + num_tuples_per_thread, total_num_tuples); 332 | lineitem_hash_table.InsertLocalEntries(lineitem_data, begin, end, 333 | thread_index); 334 | latch.arrive_and_wait(); 335 | std::call_once(flag, [&lineitem_hash_table]() { 336 | lineitem_hash_table.ResizeHashTable(); 337 | }); 338 | lineitem_hash_table.MergeLocalEntries(thread_index); 339 | }); 340 | } 341 | 342 | for (auto &t : threads) { 343 | t.join(); 344 | } 345 | } 346 | 347 | // Now, we build a hash table for the part relation which contains only the 348 | // partkeys that are actually required to process query 14. While building 349 | // the hash table, we also remember how often each page of the part relation 350 | // will be accessed for processing query 14. 351 | int fd = open(path_to_part, O_RDONLY); 352 | uint64_t size_in_bytes = lseek(fd, 0, SEEK_END); 353 | auto *data = reinterpret_cast( 354 | mmap(nullptr, size_in_bytes, PROT_READ, MAP_SHARED, fd, 0)); 355 | madvise(data, size_in_bytes, MADV_SEQUENTIAL); 356 | madvise(data, size_in_bytes, MADV_WILLNEED); 357 | 358 | auto total_num_pages = size_in_bytes / kPageSize; 359 | auto num_pages_per_thread = 360 | (total_num_pages + thread_count - 1) / thread_count; 361 | std::vector threads; 362 | threads.reserve(thread_count); 363 | 364 | std::once_flag flag; 365 | std::latch latch{thread_count}; 366 | 367 | PartHashTable part_hash_table(thread_count, total_num_pages); 368 | 369 | for (unsigned thread_index = 0; thread_index != thread_count; 370 | ++thread_index) { 371 | threads.emplace_back([thread_index, total_num_pages, num_pages_per_thread, 372 | data, &part_hash_table, &lineitem_hash_table, &latch, 373 | &flag]() { 374 | auto begin = 375 | std::min(thread_index * num_pages_per_thread, total_num_pages); 376 | auto end = std::min(begin + num_pages_per_thread, total_num_pages); 377 | part_hash_table.InsertLocalEntries(&data[begin], &data[end], begin, 378 | thread_index, lineitem_hash_table); 379 | latch.arrive_and_wait(); 380 | std::call_once( 381 | flag, [&part_hash_table]() { part_hash_table.ResizeHashTable(); }); 382 | part_hash_table.MergeLocalEntries(thread_index); 383 | }); 384 | } 385 | 386 | for (auto &t : threads) { 387 | t.join(); 388 | } 389 | 390 | return part_hash_table; 391 | } 392 | 393 | class QueryRunner { 394 | public: 395 | QueryRunner(const PartHashTable &part_hash_table, File &part_data_file, 396 | const InMemoryLineitemData &lineitem_data, unsigned thread_count, 397 | uint32_t num_ring_entries = 0) 398 | : part_hash_table_(part_hash_table), 399 | part_data_file_(part_data_file), 400 | lineitem_data_(lineitem_data), 401 | thread_count_(thread_count), 402 | thread_local_sums_(thread_count), 403 | lower_date_boundary(Date::FromString("1995-09-01|", '|').value), 404 | upper_date_boundary(Date::FromString("1995-09-30|", '|').value), 405 | num_ring_entries_(num_ring_entries) { 406 | if (num_ring_entries_ > 0) { 407 | thread_local_rings_.reserve(thread_count_); 408 | for (unsigned i = 0; i != thread_count; ++i) { 409 | thread_local_rings_.emplace_back(num_ring_entries_); 410 | } 411 | } 412 | } 413 | 414 | void StartProcessing(uint64_t num_tuples_per_coroutine = 0) { 415 | std::atomic current_lineitem_tuple_offset{0ull}; 416 | std::vector threads; 417 | threads.reserve(thread_count_); 418 | 419 | for (unsigned thread_index = 0; thread_index != thread_count_; 420 | ++thread_index) { 421 | threads.emplace_back([is_synchronous = IsSynchronous(), 422 | num_coroutines = num_ring_entries_, 423 | ¤t_lineitem_tuple_offset, 424 | total_num_tuples_lineitem = 425 | lineitem_data_.GetSize(), 426 | this, thread_index, 427 | &ring = thread_local_rings_[thread_index], 428 | num_tuples_per_coroutine] { 429 | std::vector> tasks; 430 | if (!is_synchronous) { 431 | cppcoro::detail::allocator = new Allocator(num_coroutines); 432 | cppcoro::detail::sync_allocator = new Allocator(1); 433 | } 434 | std::allocator alloc; 435 | auto part_pages_buffer = 436 | alloc.allocate(is_synchronous ? 1 : num_coroutines); 437 | 438 | uint64_t fetch_increment = 439 | is_synchronous ? 100'000ull 440 | : std::max(num_coroutines * num_tuples_per_coroutine, 441 | 100'000ul); 442 | while (true) { 443 | uint64_t begin = 444 | current_lineitem_tuple_offset.fetch_add(fetch_increment); 445 | if (begin >= total_num_tuples_lineitem) { 446 | return; 447 | } 448 | auto end = 449 | std::min(begin + fetch_increment, total_num_tuples_lineitem); 450 | 451 | if (is_synchronous) { 452 | ProcessLineitems(begin, end, part_pages_buffer[0], thread_index); 453 | } else { 454 | Countdown countdown(0); 455 | auto local_begin = begin; 456 | auto local_end = local_begin + num_tuples_per_coroutine; 457 | for (; local_end <= end; local_begin = local_end, 458 | local_end += num_tuples_per_coroutine) { 459 | tasks.emplace_back(AsyncProcessLineitems( 460 | local_begin, local_end, part_pages_buffer[tasks.size()], 461 | thread_index, ring, countdown)); 462 | 463 | if (tasks.size() == num_coroutines) { 464 | countdown.Set(num_coroutines); 465 | tasks.emplace_back(DrainRing(ring, countdown)); 466 | cppcoro::sync_wait(cppcoro::when_all_ready(std::move(tasks))); 467 | } 468 | } 469 | if (tasks.empty()) { 470 | ProcessLineitems(local_begin, end, part_pages_buffer[0], 471 | thread_index); 472 | } else { 473 | tasks.emplace_back(AsyncProcessLineitems( 474 | local_begin, end, part_pages_buffer[tasks.size()], 475 | thread_index, ring, countdown)); 476 | countdown.Set(tasks.size()); 477 | tasks.emplace_back(DrainRing(ring, countdown)); 478 | cppcoro::sync_wait(cppcoro::when_all_ready(std::move(tasks))); 479 | } 480 | } 481 | } 482 | alloc.deallocate(part_pages_buffer, 483 | is_synchronous ? 1 : num_coroutines); 484 | if (!is_synchronous) { 485 | delete cppcoro::detail::allocator; 486 | cppcoro::detail::allocator = nullptr; 487 | delete cppcoro::detail::sync_allocator; 488 | cppcoro::detail::sync_allocator = nullptr; 489 | } 490 | }); 491 | } 492 | 493 | for (auto &t : threads) { 494 | t.join(); 495 | } 496 | } 497 | 498 | void DoPostProcessing(bool should_print_result) const { 499 | Numeric<12, 4> first_sum; 500 | Numeric<12, 4> second_sum; 501 | for (const auto &local_sums : thread_local_sums_) { 502 | first_sum += local_sums.first; 503 | second_sum += local_sums.second; 504 | } 505 | 506 | // 100 * first_sum / second_sum 507 | auto result = Numeric<12, 4>{1'000'000ll} * (first_sum / second_sum); 508 | 509 | if (should_print_result) { 510 | std::cerr << "promo_revenue\n" << result << "\n"; 511 | } 512 | } 513 | 514 | private: 515 | void ProcessLineitems(uint64_t begin_tuple_offset, uint64_t end_tuple_offset, 516 | PartPage &buffer, unsigned thread_index) { 517 | Numeric<12, 4> first_sum; 518 | Numeric<12, 4> second_sum; 519 | for (auto tuple_offset = begin_tuple_offset; 520 | tuple_offset != end_tuple_offset; ++tuple_offset) { 521 | if (lower_date_boundary <= lineitem_data_.l_shipdate[tuple_offset] && 522 | lineitem_data_.l_shipdate[tuple_offset] <= upper_date_boundary) { 523 | auto lookup_result = part_hash_table_.LookupPartkey( 524 | lineitem_data_.l_partkey[tuple_offset]); 525 | 526 | const PartPage *part_page; 527 | if (lookup_result.swip.IsPageIndex()) { 528 | part_data_file_.ReadPage(lookup_result.swip.GetPageIndex(), 529 | reinterpret_cast(&buffer)); 530 | part_page = &buffer; 531 | } else { 532 | part_page = lookup_result.swip.GetPointer(); 533 | } 534 | 535 | auto sum = 536 | lineitem_data_.l_extendedprice[tuple_offset] * 537 | (Numeric<12, 2>{100ll} - lineitem_data_.l_discount[tuple_offset]); 538 | std::string_view p_type( 539 | part_page->p_type[lookup_result.tuple_offset].Begin(), 540 | part_page->p_type[lookup_result.tuple_offset].Size()); 541 | if (p_type.starts_with("PROMO")) { 542 | first_sum += sum; 543 | } 544 | second_sum += sum; 545 | } 546 | } 547 | thread_local_sums_[thread_index].first += first_sum; 548 | thread_local_sums_[thread_index].second += second_sum; 549 | } 550 | 551 | cppcoro::task AsyncProcessLineitems( 552 | uint64_t begin_tuple_offset, uint64_t end_tuple_offset, PartPage &buffer, 553 | unsigned thread_index, IOUring &ring, Countdown &countdown) { 554 | Numeric<12, 4> first_sum; 555 | Numeric<12, 4> second_sum; 556 | for (auto tuple_offset = begin_tuple_offset; 557 | tuple_offset != end_tuple_offset; ++tuple_offset) { 558 | if (lower_date_boundary <= lineitem_data_.l_shipdate[tuple_offset] && 559 | lineitem_data_.l_shipdate[tuple_offset] <= upper_date_boundary) { 560 | auto lookup_result = part_hash_table_.LookupPartkey( 561 | lineitem_data_.l_partkey[tuple_offset]); 562 | 563 | const PartPage *part_page; 564 | if (lookup_result.swip.IsPageIndex()) { 565 | co_await part_data_file_.AsyncReadPage( 566 | ring, lookup_result.swip.GetPageIndex(), 567 | reinterpret_cast(&buffer)); 568 | part_page = &buffer; 569 | } else { 570 | part_page = lookup_result.swip.GetPointer(); 571 | } 572 | 573 | auto sum = 574 | lineitem_data_.l_extendedprice[tuple_offset] * 575 | (Numeric<12, 2>{100ll} - lineitem_data_.l_discount[tuple_offset]); 576 | std::string_view p_type( 577 | part_page->p_type[lookup_result.tuple_offset].Begin(), 578 | part_page->p_type[lookup_result.tuple_offset].Size()); 579 | if (p_type.starts_with("PROMO")) { 580 | first_sum += sum; 581 | } 582 | second_sum += sum; 583 | } 584 | } 585 | thread_local_sums_[thread_index].first += first_sum; 586 | thread_local_sums_[thread_index].second += second_sum; 587 | countdown.Decrement(); 588 | } 589 | 590 | bool IsSynchronous() const noexcept { return num_ring_entries_ == 0; } 591 | 592 | using NumericsPair = std::pair, Numeric<12, 4>>; 593 | 594 | const PartHashTable &part_hash_table_; 595 | File &part_data_file_; 596 | const InMemoryLineitemData &lineitem_data_; 597 | const uint32_t thread_count_; 598 | std::vector thread_local_sums_; 599 | const Date lower_date_boundary; 600 | const Date upper_date_boundary; 601 | std::vector thread_local_rings_; 602 | const uint32_t num_ring_entries_; 603 | }; 604 | 605 | InMemoryLineitemData LoadLineitemRelation(const char *path_to_lineitem) { 606 | int fd = open(path_to_lineitem, O_RDONLY); 607 | uint64_t size_in_bytes = lseek(fd, 0, SEEK_END); 608 | auto *data = reinterpret_cast( 609 | mmap(nullptr, size_in_bytes, PROT_READ, MAP_SHARED, fd, 0)); 610 | 611 | auto total_num_pages = size_in_bytes / kPageSize; 612 | auto max_num_tuples = total_num_pages * LineitemPageQ14::kMaxNumTuples; 613 | 614 | InMemoryLineitemData result(max_num_tuples); 615 | auto num_threads = std::thread::hardware_concurrency(); 616 | auto num_pages_per_thread = (total_num_pages + num_threads - 1) / num_threads; 617 | 618 | std::vector threads; 619 | threads.reserve(num_threads); 620 | for (unsigned thread_index = 0; thread_index != num_threads; ++thread_index) { 621 | threads.emplace_back( 622 | [thread_index, num_pages_per_thread, total_num_pages, &result, data]() { 623 | auto begin = 624 | std::min(thread_index * num_pages_per_thread, total_num_pages); 625 | auto end = std::min(begin + num_pages_per_thread, total_num_pages); 626 | for (auto page_index = begin; page_index != end; ++page_index) { 627 | const LineitemPageQ14 &page = data[page_index]; 628 | auto num_tuples = page.num_tuples; 629 | auto first_tuple_offset = result.IncreaseSize(num_tuples); 630 | std::memcpy(&result.l_partkey[first_tuple_offset], 631 | &page.l_partkey.front(), 632 | sizeof(page.l_partkey.front()) * num_tuples); 633 | std::memcpy(&result.l_extendedprice[first_tuple_offset], 634 | &page.l_extendedprice.front(), 635 | sizeof(page.l_extendedprice.front()) * num_tuples); 636 | std::memcpy(&result.l_discount[first_tuple_offset], 637 | &page.l_discount.front(), 638 | sizeof(page.l_discount.front()) * num_tuples); 639 | std::memcpy(&result.l_shipdate[first_tuple_offset], 640 | &page.l_shipdate.front(), 641 | sizeof(page.l_shipdate.front()) * num_tuples); 642 | } 643 | }); 644 | } 645 | for (auto &thread : threads) { 646 | thread.join(); 647 | } 648 | return result; 649 | } 650 | } // namespace 651 | 652 | int main(int argc, char *argv[]) { 653 | if (argc != 8) { 654 | std::cerr << "Usage: " << argv[0] 655 | << " lineitem.dat part.dat num_threads num_entries_per_ring " 656 | "num_tuples_per_coroutine " 657 | "print_result print_header\n"; 658 | return 1; 659 | } 660 | 661 | const char *path_to_lineitem = argv[1]; 662 | const char *path_to_part = argv[2]; 663 | unsigned num_threads = std::atoi(argv[3]); 664 | unsigned num_entries_per_ring = std::atoi(argv[4]); 665 | unsigned num_tuples_per_coroutine = std::atoi(argv[5]); 666 | bool print_result; 667 | std::istringstream(argv[6]) >> std::boolalpha >> print_result; 668 | bool print_header; 669 | std::istringstream(argv[7]) >> std::boolalpha >> print_header; 670 | 671 | InMemoryLineitemData lineitem_data = LoadLineitemRelation(path_to_lineitem); 672 | 673 | auto part_hash_table = BuildHashTableForPart(lineitem_data, path_to_part); 674 | 675 | File part_data_file{path_to_part, File::kRead, true}; 676 | 677 | auto total_num_references = part_hash_table.GetTotalNumPageReferences(); 678 | auto ten_percent = (total_num_references + 9) / 10; 679 | 680 | if (print_header) { 681 | std::cout << "kind_of_io,page_size_power,num_threads,num_cached_references," 682 | "num_total_references," 683 | "num_entries_per_ring,num_tuples_per_coroutine,time\n"; 684 | } 685 | 686 | for (int i = 0; i != 11; ++i) { 687 | { 688 | QueryRunner synchronousRunner{part_hash_table, part_data_file, 689 | lineitem_data, num_threads}; 690 | auto start = std::chrono::steady_clock::now(); 691 | synchronousRunner.StartProcessing(); 692 | synchronousRunner.DoPostProcessing(print_result); 693 | auto end = std::chrono::steady_clock::now(); 694 | auto milliseconds = 695 | std::chrono::duration_cast(end - start) 696 | .count(); 697 | std::cout << "synchronous," << kPageSizePower << "," << num_threads << "," 698 | << part_hash_table.GetNumAlreadyCachedReferences() << "," 699 | << total_num_references << ",0,0," << milliseconds << "\n"; 700 | } 701 | 702 | { 703 | QueryRunner asynchronousRunner{part_hash_table, part_data_file, 704 | lineitem_data, num_threads, 705 | num_entries_per_ring}; 706 | auto start = std::chrono::steady_clock::now(); 707 | asynchronousRunner.StartProcessing(num_tuples_per_coroutine); 708 | asynchronousRunner.DoPostProcessing(print_result); 709 | auto end = std::chrono::steady_clock::now(); 710 | auto milliseconds = 711 | std::chrono::duration_cast(end - start) 712 | .count(); 713 | std::cout << "asynchronous," << kPageSizePower << "," << num_threads 714 | << "," << part_hash_table.GetNumAlreadyCachedReferences() << "," 715 | << total_num_references << "," << num_entries_per_ring << "," 716 | << num_tuples_per_coroutine << "," << milliseconds << "\n"; 717 | } 718 | 719 | part_hash_table.CacheAtLeastNumReferences(part_data_file, 720 | (i + 1) * ten_percent); 721 | } 722 | } -------------------------------------------------------------------------------- /storage/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(STORAGE_SOURCES 2 | src/storage/file.cc 3 | src/storage/types.cc 4 | ) 5 | 6 | add_library(storage ${STORAGE_SOURCES}) 7 | target_include_directories(storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | target_link_libraries(storage PUBLIC uring cppcoro) 9 | 10 | add_executable(load_data src/storage/load_data.cc) 11 | target_link_libraries(load_data Threads::Threads storage) -------------------------------------------------------------------------------- /storage/src/storage/file.cc: -------------------------------------------------------------------------------- 1 | #include "storage/file.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace { 13 | [[noreturn]] static void ThrowErrno() { 14 | throw std::system_error{errno, std::system_category()}; 15 | } 16 | } // namespace 17 | 18 | namespace storage { 19 | 20 | File::File(const char *filename, Mode mode, bool use_direct_io_for_reading) { 21 | switch (mode) { 22 | case kRead: { 23 | fd_ = open(filename, O_RDONLY | O_NOATIME | 24 | (use_direct_io_for_reading ? O_DIRECT : 0)); 25 | break; 26 | } 27 | case kWrite: { 28 | fd_ = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_APPEND, 0600); 29 | break; 30 | } 31 | } 32 | if (fd_ < 0) { 33 | ThrowErrno(); 34 | } 35 | } 36 | 37 | File::~File() { 38 | if (close(fd_) == -1) { 39 | ThrowErrno(); 40 | } 41 | } 42 | 43 | size_t File::ReadSize() const { 44 | struct stat fileStat; 45 | if (fstat(fd_, &fileStat) < 0) { 46 | ThrowErrno(); 47 | } 48 | return fileStat.st_size; 49 | } 50 | 51 | void File::ReadBlock(std::byte *data, size_t offset, size_t size) const { 52 | size_t total_bytes_read = 0ull; 53 | while (total_bytes_read < size) { 54 | ssize_t bytes_read = 55 | pread(fd_, data + total_bytes_read, size - total_bytes_read, 56 | offset + total_bytes_read); 57 | if (bytes_read == 0) { 58 | // end of file, i.e. size was probably larger than the file 59 | // size 60 | return; 61 | } 62 | if (bytes_read < 0) { 63 | ThrowErrno(); 64 | } 65 | total_bytes_read += bytes_read; 66 | } 67 | } 68 | 69 | cppcoro::task File::AsyncReadBlock(IOUring &ring, std::byte *data, 70 | size_t offset, size_t size) const { 71 | size_t total_bytes_read = 0ull; 72 | while (total_bytes_read < size) { 73 | ssize_t bytes_read = co_await IOUringAwaiter( 74 | ring, data + total_bytes_read, size - total_bytes_read, 75 | offset + total_bytes_read, fd_); 76 | if (bytes_read == 0) { 77 | // end of file, i.e. size was probably larger than the file 78 | // size 79 | co_return; 80 | } 81 | if (bytes_read < 0) { 82 | ThrowErrno(); 83 | } 84 | total_bytes_read += bytes_read; 85 | } 86 | } 87 | 88 | void File::AppendBlock(const std::byte *data, size_t size) { 89 | ssize_t bytes_written = write(fd_, data, size); 90 | if (bytes_written == -1) { 91 | ThrowErrno(); 92 | } else if (static_cast(bytes_written) != size) { 93 | // Recovering from this situation is difficult because other threads can 94 | // write simultaneously 95 | throw std::runtime_error{"Unable to append full block"}; 96 | } 97 | } 98 | 99 | } // namespace storage -------------------------------------------------------------------------------- /storage/src/storage/file.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_FILE_H_ 2 | #define STORAGE_FILE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "cppcoro/task.hpp" 8 | #include "storage/io_uring.h" 9 | 10 | namespace storage { 11 | 12 | constexpr size_t kPageSizePower = ASYNCHRONOUS_IO_PAGE_SIZE_POWER; 13 | constexpr size_t kPageSize = 1ull << kPageSizePower; 14 | 15 | using PageIndex = size_t; 16 | 17 | class File { 18 | public: 19 | enum Mode { kRead, kWrite }; 20 | 21 | // Opens the file 22 | File(const char *filename, Mode mode, bool use_direct_io_for_reading = false); 23 | 24 | ~File(); 25 | 26 | size_t ReadSize() const; 27 | 28 | void ReadPage(PageIndex page_index, std::byte *data) const { 29 | auto offset = page_index * kPageSize; 30 | ReadBlock(data, offset, kPageSize); 31 | } 32 | 33 | void ReadBlock(std::byte *data, size_t offset, size_t size) const; 34 | 35 | cppcoro::task AsyncReadPage(IOUring &ring, PageIndex page_index, 36 | std::byte *data) const { 37 | auto offset = page_index * kPageSize; 38 | co_return co_await AsyncReadBlock(ring, data, offset, kPageSize); 39 | } 40 | 41 | cppcoro::task AsyncReadBlock(IOUring &ring, std::byte *data, 42 | size_t offset, size_t size) const; 43 | 44 | void AppendPages(const std::byte *data, size_t num_pages) { 45 | AppendBlock(data, kPageSize * num_pages); 46 | } 47 | 48 | void AppendBlock(const std::byte *data, size_t size); 49 | 50 | private: 51 | int fd_; 52 | }; 53 | 54 | } // namespace storage 55 | 56 | #endif // STORAGE_FILE_H_ -------------------------------------------------------------------------------- /storage/src/storage/find_pattern.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_FIND_PATTERN_H_ 2 | #define STORAGE_FIND_PATTERN_H_ 3 | 4 | #include 5 | 6 | namespace storage { 7 | // Returns the position of the pattern character within [iter, end), or end if 8 | // not found 9 | template 10 | const char *FindPatternFast(const char *iter, const char *end) { 11 | // Loop over the content in blocks of 32 characters 12 | auto end32 = end - 32; 13 | const auto expanded_pattern = _mm256_set1_epi8(kPattern); 14 | for (; iter < end32; iter += 32) { 15 | // Check the next 32 characters for the pattern 16 | auto block = _mm256_loadu_si256(reinterpret_cast(iter)); 17 | auto matches = 18 | _mm256_movemask_epi8(_mm256_cmpeq_epi8(block, expanded_pattern)); 19 | if (matches) { 20 | return iter + __builtin_ctzll(matches); 21 | } 22 | } 23 | 24 | // Check the last few characters explicitly 25 | while ((iter < end) && ((*iter) != kPattern)) { 26 | ++iter; 27 | } 28 | 29 | return iter; 30 | } 31 | 32 | // Returns the position of the pattern character within [iter, end), or end if 33 | // not found 34 | template 35 | const char *FindPatternSlow(const char *iter, const char *end) { 36 | while ((iter < end) && ((*iter) != kPattern)) { 37 | ++iter; 38 | } 39 | 40 | return iter; 41 | } 42 | 43 | // Returns the position of the n-th occurence of the pattern character within 44 | // [iter, end), or end if not found 45 | template 46 | const char *FindNthPatternFast(const char *iter, const char *end, unsigned n) { 47 | // Loop over the content in blocks of 32 characters 48 | auto end32 = end - 32; 49 | const auto expanded_pattern = _mm256_set1_epi8(kPattern); 50 | for (; iter < end32; iter += 32) { 51 | // Check the next 32 characters for the pattern 52 | auto block = _mm256_loadu_si256(reinterpret_cast(iter)); 53 | auto matches = 54 | _mm256_movemask_epi8(_mm256_cmpeq_epi8(block, expanded_pattern)); 55 | if (matches) { 56 | unsigned num_hits = __builtin_popcount(matches); 57 | if (num_hits >= n) { 58 | for (; n > 1; n--) { 59 | matches &= (matches - 1); 60 | } 61 | return iter + __builtin_ctzll(matches); 62 | } 63 | n -= num_hits; 64 | } 65 | } 66 | 67 | // Check the last few characters explicitly 68 | for (; iter < end; ++iter) { 69 | if ((*iter) == kPattern && (--n) == 0) { 70 | return iter; 71 | } 72 | } 73 | 74 | return end; 75 | } 76 | 77 | // Returns the beginning position of the index-th chunk when dividing the range 78 | // [begin, end) into chunkCount chunks 79 | template 80 | const char *FindBeginBoundary(const char *begin, const char *end, 81 | unsigned chunk_count, unsigned index) { 82 | if (index == 0) { 83 | return begin; 84 | } 85 | 86 | if (index == chunk_count) { 87 | return end; 88 | } 89 | 90 | const char *approx_chunk_begin = 91 | begin + ((end - begin) * index / chunk_count); 92 | return FindPatternFast(approx_chunk_begin, end) + 1; 93 | } 94 | 95 | } // namespace storage 96 | 97 | #endif // STORAGE_FIND_PATTERN_H_ -------------------------------------------------------------------------------- /storage/src/storage/io_uring.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_IO_URING_H_ 2 | #define STORAGE_IO_URING_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "cppcoro/coroutine.hpp" 9 | #include "cppcoro/task.hpp" 10 | #include "liburing.h" 11 | 12 | namespace storage { 13 | 14 | class IOUring; 15 | 16 | class IOUringAwaiter { 17 | public: 18 | IOUringAwaiter(IOUring &ring, void *buffer, size_t num_bytes, off_t offset, 19 | int fd) noexcept 20 | : ring_(ring), 21 | buffer_(buffer), 22 | num_bytes_(num_bytes), 23 | offset_(offset), 24 | fd_(fd) {} 25 | 26 | bool await_ready() const noexcept { return false; } 27 | 28 | void await_suspend(cppcoro::coroutine_handle<> handle); 29 | 30 | __s32 await_resume() const noexcept { return result_; } 31 | 32 | void SetResult(__s32 result) noexcept { result_ = result; } 33 | 34 | cppcoro::coroutine_handle<> GetHandle() const noexcept { return handle_; } 35 | 36 | private: 37 | cppcoro::coroutine_handle<> handle_; 38 | IOUring &ring_; 39 | void *buffer_; 40 | const size_t num_bytes_; 41 | const off_t offset_; 42 | const int fd_; 43 | __s32 result_; 44 | }; 45 | 46 | class IOUring { 47 | public: 48 | explicit IOUring(unsigned num_entries) : num_waiting_(0) { 49 | auto result = io_uring_queue_init(num_entries, &ring_, 0); 50 | if (result != 0) { 51 | throw std::system_error{-result, std::generic_category()}; 52 | } 53 | } 54 | 55 | ~IOUring() { io_uring_queue_exit(&ring_); } 56 | 57 | template 58 | void ProcessBatch() noexcept { 59 | std::array cqes; 60 | std::array, kBatchSize> handles; 61 | 62 | // collect up to kBatchSize handles 63 | unsigned num_returned = 64 | io_uring_peek_batch_cqe(&ring_, cqes.data(), kBatchSize); 65 | for (unsigned i = 0; i != num_returned; ++i) { 66 | auto *awaiter = 67 | reinterpret_cast(io_uring_cqe_get_data(cqes[i])); 68 | awaiter->SetResult(cqes[i]->res); 69 | io_uring_cqe_seen(&ring_, cqes[i]); 70 | handles[i] = awaiter->GetHandle(); 71 | } 72 | num_waiting_ -= num_returned; 73 | 74 | // resume all collected handles 75 | for (unsigned i = 0; i != num_returned; ++i) { 76 | handles[i].resume(); 77 | } 78 | } 79 | 80 | bool Empty() const noexcept { return num_waiting_ == 0; } 81 | 82 | private: 83 | friend class IOUringAwaiter; 84 | 85 | io_uring ring_; 86 | unsigned num_waiting_; 87 | }; 88 | 89 | class SubmissionQueueFullError : public std::exception { 90 | [[nodiscard]] const char *what() const noexcept override { 91 | return "Submission queue is full"; 92 | } 93 | }; 94 | 95 | inline void IOUringAwaiter::await_suspend(cppcoro::coroutine_handle<> handle) { 96 | handle_ = handle; 97 | 98 | io_uring_sqe *sqe = io_uring_get_sqe(&ring_.ring_); 99 | if (sqe == nullptr) { 100 | throw SubmissionQueueFullError{}; 101 | } 102 | 103 | io_uring_prep_read(sqe, fd_, buffer_, num_bytes_, offset_); 104 | 105 | io_uring_sqe_set_data(sqe, this); 106 | io_uring_submit(&ring_.ring_); 107 | ++ring_.num_waiting_; 108 | } 109 | 110 | class Countdown { 111 | public: 112 | explicit Countdown(std::uint64_t counter) noexcept : counter_(counter) {} 113 | 114 | void Decrement() noexcept { --counter_; } 115 | 116 | bool IsZero() const noexcept { return counter_ == 0; } 117 | 118 | void Set(std::uint64_t counter) noexcept { counter_ = counter; } 119 | 120 | private: 121 | std::uint64_t counter_; 122 | }; 123 | 124 | inline cppcoro::task DrainRing(IOUring &ring, 125 | const Countdown &countdown) { 126 | while (!countdown.IsZero()) { 127 | ring.ProcessBatch(); 128 | } 129 | co_return; 130 | } 131 | 132 | } // namespace storage 133 | 134 | #endif // STORAGE_IO_URING_H_ -------------------------------------------------------------------------------- /storage/src/storage/load_data.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "storage/file.h" 12 | #include "storage/find_pattern.h" 13 | #include "storage/schema.h" 14 | #include "storage/swip.h" 15 | #include "storage/types.h" 16 | 17 | namespace { 18 | 19 | using namespace storage; 20 | 21 | constexpr unsigned kNumThreads = 8u; 22 | constexpr uint64_t kWriteSize = 1ull << 22; 23 | static_assert(kWriteSize >= storage::kPageSize); 24 | constexpr uint64_t kWriteNumPages = kWriteSize / storage::kPageSize; 25 | 26 | template 27 | static const char *InsertLine(const char *begin, const char *end, 28 | uint64_t index, Page &page); 29 | 30 | template <> 31 | const char *InsertLine(const char *begin, const char *end, 32 | uint64_t index, LineitemPageQ1 &page) { 33 | auto iter = FindNthPatternFast<'|'>(begin, end, 4) + 1; 34 | auto parsed_quantity = storage::Numeric<12, 2>::FromString(iter, '|'); 35 | page.l_quantity[index] = parsed_quantity.value; 36 | auto parsed_extendedprice = 37 | storage::Numeric<12, 2>::FromString(parsed_quantity.end_it + 1, '|'); 38 | page.l_extendedprice[index] = parsed_extendedprice.value; 39 | auto parsed_discount = 40 | storage::Numeric<12, 2>::FromString(parsed_extendedprice.end_it + 1, '|'); 41 | page.l_discount[index] = parsed_discount.value; 42 | auto parsed_tax = 43 | storage::Numeric<12, 2>::FromString(parsed_discount.end_it + 1, '|'); 44 | page.l_tax[index] = parsed_tax.value; 45 | iter = parsed_tax.end_it + 1; 46 | page.l_returnflag[index] = *iter; 47 | iter += 2; 48 | page.l_linestatus[index] = *iter; 49 | iter += 2; 50 | page.l_shipdate[index] = storage::Date::FromString(iter, '|').value; 51 | return FindPatternFast<'\n'>(iter, end); 52 | } 53 | 54 | template <> 55 | const char *InsertLine(const char *begin, const char *end, 56 | uint64_t index, LineitemPageQ14 &page) { 57 | auto iter = FindPatternSlow<'|'>(begin, end) + 1; 58 | auto parsed_partkey = storage::Integer::FromString(iter, '|'); 59 | page.l_partkey[index] = parsed_partkey.value; 60 | iter = FindNthPatternFast<'|'>(parsed_partkey.end_it + 1, end, 3) + 1; 61 | auto parsed_extendedprice = storage::Numeric<12, 2>::FromString(iter, '|'); 62 | page.l_extendedprice[index] = parsed_extendedprice.value; 63 | auto parsed_discount = 64 | storage::Numeric<12, 2>::FromString(parsed_extendedprice.end_it + 1, '|'); 65 | page.l_discount[index] = parsed_discount.value; 66 | iter = FindNthPatternFast<'|'>(parsed_discount.end_it + 1, end, 3) + 1; 67 | page.l_shipdate[index] = storage::Date::FromString(iter, '|').value; 68 | return FindPatternFast<'\n'>(iter, end); 69 | } 70 | 71 | template <> 72 | const char *InsertLine(const char *begin, const char *end, 73 | uint64_t index, PartPage &page) { 74 | auto parsed_partkey = storage::Integer::FromString(begin, '|'); 75 | page.p_partkey[index] = parsed_partkey.value; 76 | auto type_begin = 77 | FindNthPatternFast<'|'>(parsed_partkey.end_it + 1, end, 3) + 1; 78 | auto type_end = FindPatternFast<'|'>(type_begin, end); 79 | new (&page.p_type[index]) typeof(page.p_type[index]){type_begin, type_end}; 80 | return FindPatternFast<'\n'>(type_end + 1, end); 81 | } 82 | 83 | template 84 | static void LoadChunk(const char *begin, const char *end, 85 | storage::File &data_file) { 86 | std::vector data(kWriteNumPages); 87 | 88 | while (begin < end) { 89 | for (uint64_t i = 0; i != kWriteNumPages; ++i) { 90 | auto &page = data[i]; 91 | uint64_t tuple_index = 0; 92 | for (; tuple_index != Page::kMaxNumTuples && begin < end; ++tuple_index) { 93 | begin = InsertLine(begin, end, tuple_index, page) + 1; 94 | } 95 | page.num_tuples = tuple_index; 96 | if (begin >= end) { 97 | // we have reached the end of our chunk 98 | // write the remaining pages 99 | auto num_used_pages = i + 1; 100 | data_file.AppendPages(reinterpret_cast(data.data()), 101 | num_used_pages); 102 | return; 103 | } 104 | } 105 | data_file.AppendPages(reinterpret_cast(data.data()), 106 | kWriteNumPages); 107 | } 108 | } 109 | 110 | template 111 | static void LoadFile(const char *path_to_data_in, 112 | const char *path_to_data_out) { 113 | int fd = open(path_to_data_in, O_RDONLY); 114 | auto length = lseek(fd, 0, SEEK_END); 115 | 116 | void *data = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0); 117 | madvise(data, length, MADV_SEQUENTIAL); 118 | madvise(data, length, MADV_WILLNEED); 119 | 120 | auto begin = static_cast(data); 121 | auto end = begin + length; 122 | 123 | storage::File output_file{path_to_data_out, storage::File::kWrite}; 124 | 125 | std::vector threads; 126 | threads.reserve(kNumThreads); 127 | 128 | auto start_time = std::chrono::steady_clock::now(); 129 | 130 | for (unsigned index = 0; index != kNumThreads; ++index) { 131 | threads.emplace_back([index, begin, end, &output_file]() { 132 | auto from = FindBeginBoundary<'\n'>(begin, end, kNumThreads, index); 133 | auto to = FindBeginBoundary<'\n'>(begin, end, kNumThreads, index + 1); 134 | LoadChunk(from, to, output_file); 135 | }); 136 | } 137 | 138 | for (auto &t : threads) { 139 | t.join(); 140 | } 141 | 142 | auto end_time = std::chrono::steady_clock::now(); 143 | double nanoseconds = std::chrono::duration_cast( 144 | end_time - start_time) 145 | .count(); 146 | std::cout << "Processed " << length / nanoseconds << " GB/s\n"; 147 | 148 | munmap(data, length); 149 | close(fd); 150 | } 151 | 152 | static void PrintUsage(const char *command) { 153 | std::cerr << "Usage: " << command 154 | << " lineitemQ1 lineitem.tbl lineitemQ1.dat |" 155 | " lineitemQ14 lineitem.tbl lineitemQ14.dat |" 156 | " part part.tbl part.dat\n"; 157 | } 158 | } // namespace 159 | 160 | int main(int argc, char *argv[]) { 161 | if (argc != 4) { 162 | PrintUsage(argv[0]); 163 | return 1; 164 | } 165 | 166 | std::string_view kind{argv[1]}; 167 | if (kind == "lineitemQ1") { 168 | LoadFile(argv[2], argv[3]); 169 | } else if (kind == "lineitemQ14") { 170 | LoadFile(argv[2], argv[3]); 171 | } else if (kind == "part") { 172 | LoadFile(argv[2], argv[3]); 173 | } else { 174 | PrintUsage(argv[0]); 175 | return 1; 176 | } 177 | } -------------------------------------------------------------------------------- /storage/src/storage/schema.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_SCHEMA_H_ 2 | #define STORAGE_SCHEMA_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "storage/file.h" 8 | #include "storage/types.h" 9 | 10 | namespace storage { 11 | 12 | static_assert(kPageSizePower >= 12 && kPageSizePower <= 22); 13 | 14 | constexpr std::array kLineitemPageQ1MaxNumTuples = { 15 | 107, 215, 430, 862, 1724, 3449, 6898, 13796, 27593, 55188, 110376}; 16 | 17 | struct alignas(kPageSize) LineitemPageQ1 { 18 | static constexpr uint64_t kMaxNumTuples = 19 | kLineitemPageQ1MaxNumTuples[kPageSizePower - 12]; 20 | uint32_t num_tuples; 21 | std::array, kMaxNumTuples> l_quantity; 22 | std::array, kMaxNumTuples> l_extendedprice; 23 | std::array, kMaxNumTuples> l_discount; 24 | std::array, kMaxNumTuples> l_tax; 25 | std::array l_returnflag; 26 | std::array l_linestatus; 27 | std::array l_shipdate; 28 | }; 29 | 30 | static_assert(sizeof(LineitemPageQ1) == kPageSize); 31 | 32 | constexpr std::array kLineitemPageQ14MaxNumTuples = { 33 | 170, 341, 682, 1365, 2730, 5461, 10922, 21845, 43690, 87381, 174762}; 34 | 35 | struct alignas(kPageSize) LineitemPageQ14 { 36 | static constexpr uint64_t kMaxNumTuples = 37 | kLineitemPageQ14MaxNumTuples[kPageSizePower - 12]; 38 | uint32_t num_tuples; 39 | std::array l_partkey; 40 | std::array, kMaxNumTuples> l_extendedprice; 41 | std::array, kMaxNumTuples> l_discount; 42 | std::array l_shipdate; 43 | }; 44 | 45 | static_assert(sizeof(LineitemPageQ14) == kPageSize); 46 | 47 | constexpr std::array kPartPageMaxNumTuples = { 48 | 24, 48, 96, 192, 385, 770, 1541, 3084, 6168, 12336, 24672}; 49 | 50 | struct alignas(kPageSize) PartPage { 51 | static constexpr uint64_t kMaxNumTuples = 52 | kPartPageMaxNumTuples[kPageSizePower - 12]; 53 | uint32_t num_tuples; 54 | std::array p_partkey; 55 | std::array, kMaxNumTuples> p_name; 56 | std::array, kMaxNumTuples> p_mfgr; 57 | std::array, kMaxNumTuples> p_brand; 58 | std::array, kMaxNumTuples> p_type; 59 | std::array p_size; 60 | std::array, kMaxNumTuples> p_container; 61 | std::array, kMaxNumTuples> p_retailprice; 62 | std::array, kMaxNumTuples> p_comment; 63 | }; 64 | 65 | static_assert(sizeof(PartPage) == kPageSize); 66 | 67 | } // namespace storage 68 | 69 | #endif // STORAGE_SCHEMA_H_ -------------------------------------------------------------------------------- /storage/src/storage/swip.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_SWIP_H_ 2 | #define STORAGE_SWIP_H_ 3 | 4 | #include 5 | 6 | #include "storage/file.h" 7 | 8 | namespace storage { 9 | 10 | static_assert(sizeof(uintptr_t) == 8); 11 | 12 | class Swip { 13 | public: 14 | bool IsPageIndex() const noexcept { return data_ >> 63; } 15 | 16 | bool IsPointer() const noexcept { return !IsPageIndex(); } 17 | 18 | void SetPointer(void *ptr) noexcept { 19 | data_ = reinterpret_cast(ptr); 20 | } 21 | 22 | void SetPageIndex(PageIndex index) noexcept { 23 | data_ = static_cast(index) | (1ull << 63); 24 | } 25 | 26 | template 27 | T *GetPointer() const noexcept { 28 | return reinterpret_cast(data_); 29 | } 30 | 31 | PageIndex GetPageIndex() const noexcept { 32 | constexpr std::uint64_t kMask = (1ull << 63) - 1; 33 | return data_ & kMask; 34 | } 35 | 36 | static Swip MakePointer(void *ptr) noexcept { 37 | Swip swip; 38 | swip.SetPointer(ptr); 39 | return swip; 40 | } 41 | 42 | static Swip MakePageIndex(PageIndex index) noexcept { 43 | Swip swip; 44 | swip.SetPageIndex(index); 45 | return swip; 46 | } 47 | 48 | private: 49 | uintptr_t data_; 50 | }; 51 | 52 | } // namespace storage 53 | 54 | #endif // STORAGE_SWIP_H_ -------------------------------------------------------------------------------- /storage/src/storage/types.cc: -------------------------------------------------------------------------------- 1 | #include "storage/types.h" 2 | 3 | #include 4 | 5 | namespace storage { 6 | 7 | static ParseResult ParseNumber(const char* iter, 8 | char delimiter) noexcept { 9 | uint32_t number = 0; 10 | for (; *iter != delimiter; ++iter) { 11 | number = 10 * number + (*iter - '0'); 12 | } 13 | return {number, iter}; 14 | } 15 | 16 | // Algorithm from the Calendar FAQ 17 | static uint32_t MergeJulianDay(uint32_t year, uint32_t month, 18 | uint32_t day) noexcept { 19 | uint32_t a = (14 - month) / 12; 20 | uint32_t y = year + 4800 - a; 21 | uint32_t m = month + (12 * a) - 3; 22 | 23 | return day + ((153 * m + 2) / 5) + (365 * y) + (y / 4) - (y / 100) + 24 | (y / 400) - 32045; 25 | } 26 | 27 | ParseResult Date::FromString(const char* iter, char delimiter) noexcept { 28 | auto parsed_year = ParseNumber(iter, '-'); 29 | auto parsed_month = ParseNumber(++parsed_year.end_it, '-'); 30 | auto parsed_day = ParseNumber(++parsed_month.end_it, delimiter); 31 | return {Date{MergeJulianDay(parsed_year.value, parsed_month.value, 32 | parsed_day.value)}, 33 | parsed_day.end_it}; 34 | } 35 | 36 | // Algorithm from the Calendar FAQ 37 | static void SplitJulianDay(unsigned jd, unsigned& year, unsigned& month, 38 | unsigned& day) { 39 | unsigned a = jd + 32044; 40 | unsigned b = (4 * a + 3) / 146097; 41 | unsigned c = a - ((146097 * b) / 4); 42 | unsigned d = (4 * c + 3) / 1461; 43 | unsigned e = c - ((1461 * d) / 4); 44 | unsigned m = (5 * e + 2) / 153; 45 | 46 | day = e - ((153 * m + 2) / 5) + 1; 47 | month = m + 3 - (12 * (m / 10)); 48 | year = (100 * b) + d - 4800 + (m / 10); 49 | } 50 | 51 | std::ostream& operator<<(std::ostream& out, const Date& value) { 52 | unsigned year, month, day; 53 | SplitJulianDay(value.raw_, year, month, day); 54 | 55 | char buffer[30]; 56 | snprintf(buffer, sizeof(buffer), "%04u-%02u-%02u", year, month, day); 57 | return out << buffer; 58 | } 59 | 60 | ParseResult Integer::FromString(const char* iter, 61 | char delimiter) noexcept { 62 | // Check for a sign 63 | bool is_negative = false; 64 | if ((*iter) == '-') { 65 | is_negative = true; 66 | ++iter; 67 | } else if ((*iter) == '+') { 68 | ++iter; 69 | } 70 | auto parsed_number = ParseNumber(iter, delimiter); 71 | return ParseResult{is_negative 72 | ? Integer{-int32_t(parsed_number.value)} 73 | : Integer(parsed_number.value), 74 | parsed_number.end_it}; 75 | } 76 | 77 | } // namespace storage -------------------------------------------------------------------------------- /storage/src/storage/types.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_TYPES_H_ 2 | #define STORAGE_TYPES_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace storage { 9 | 10 | template 11 | struct ParseResult { 12 | T value; 13 | const char* end_it; 14 | }; 15 | 16 | using Char = char; 17 | 18 | class Date { 19 | public: 20 | Date() noexcept = default; 21 | 22 | explicit Date(uint32_t raw) noexcept : raw_(raw) {} 23 | 24 | static ParseResult FromString(const char* iter, 25 | char delimiter) noexcept; 26 | 27 | bool operator<=(Date d) const noexcept { return raw_ <= d.raw_; } 28 | 29 | friend std::ostream& operator<<(std::ostream& out, const Date& value); 30 | 31 | private: 32 | uint32_t raw_{0}; 33 | }; 34 | 35 | template 36 | class Numeric { 37 | public: 38 | Numeric() noexcept : raw_(0) {} 39 | 40 | explicit Numeric(int64_t raw) noexcept : raw_(raw) {} 41 | 42 | static ParseResult FromString(const char* iter, 43 | char delimiter) noexcept { 44 | // Check for a sign 45 | bool negated = false; 46 | if ((*iter) == '-') { 47 | negated = true; 48 | ++iter; 49 | } else if ((*iter) == '+') { 50 | ++iter; 51 | } 52 | 53 | int64_t result = 0; 54 | bool fraction = false; 55 | uint32_t digits_seen_fraction = 0; 56 | for (; *iter != delimiter; ++iter) { 57 | char c = *iter; 58 | if (c == '.') { 59 | fraction = true; 60 | } else { 61 | result = (result * 10) + (c - '0'); 62 | if (fraction) { 63 | ++digits_seen_fraction; 64 | } 65 | } 66 | } 67 | 68 | static_assert(kPrecision <= 2, 69 | "Higher precision not supported for parsing"); 70 | constexpr int64_t shifts[] = {100ll, 10ll, 1ll}; 71 | result *= shifts[digits_seen_fraction]; 72 | 73 | if (negated) { 74 | return {Numeric{-result}, iter}; 75 | } else { 76 | return {Numeric{result}, iter}; 77 | } 78 | } 79 | 80 | Numeric& operator+=(Numeric n) noexcept { 81 | raw_ += n.raw_; 82 | return *this; 83 | } 84 | 85 | Numeric operator+(Numeric n) const noexcept { 86 | Numeric r; 87 | r.raw_ = raw_ + n.raw_; 88 | return r; 89 | } 90 | 91 | Numeric operator-(Numeric n) const noexcept { 92 | Numeric r; 93 | r.raw_ = raw_ - n.raw_; 94 | return r; 95 | } 96 | 97 | Numeric operator/(uint32_t n) const noexcept { 98 | Numeric r; 99 | r.raw_ = raw_ / n; 100 | return r; 101 | } 102 | 103 | Numeric operator*( 104 | Numeric n) const noexcept { 105 | Numeric r; 106 | r.raw_ = raw_ * n.raw_; 107 | return r; 108 | } 109 | 110 | template 111 | Numeric operator/(Numeric n) const noexcept { 112 | Numeric r; 113 | r.raw_ = raw_ * 10000 / n.raw_; 114 | return r; 115 | } 116 | 117 | Numeric CastM2() const noexcept { 118 | Numeric r; 119 | r.raw_ = raw_ / 100; 120 | return r; 121 | } 122 | 123 | int64_t GetRaw() const noexcept { return raw_; } 124 | 125 | private: 126 | template 127 | friend class Numeric; 128 | 129 | int64_t raw_; 130 | }; 131 | 132 | class Integer { 133 | public: 134 | Integer() noexcept : value_(0) {} 135 | 136 | explicit Integer(int32_t value) noexcept : value_(value) {} 137 | 138 | static ParseResult FromString(const char* iter, 139 | char delimiter) noexcept; 140 | 141 | uint64_t hash() const { 142 | uint64_t r = 88172645463325252ull ^ value_; 143 | r ^= (r << 13); 144 | r ^= (r >> 7); 145 | return (r ^ (r << 17)); 146 | } 147 | 148 | bool operator==(Integer other) const noexcept { 149 | return value_ == other.value_; 150 | } 151 | 152 | bool operator<(Integer other) const noexcept { return value_ < other.value_; } 153 | 154 | private: 155 | int32_t value_; 156 | }; 157 | 158 | template 159 | struct LengthSwitch {}; 160 | 161 | template <> 162 | struct LengthSwitch<1> { 163 | using Type = uint8_t; 164 | }; 165 | 166 | template <> 167 | struct LengthSwitch<2> { 168 | using Type = uint16_t; 169 | }; 170 | 171 | template <> 172 | struct LengthSwitch<4> { 173 | using Type = uint32_t; 174 | }; 175 | 176 | template 177 | struct LengthIndicator { 178 | using Type = typename LengthSwitch<((kMaxLen < 256) ? 1 179 | : (kMaxLen < 65536) ? 2 180 | : 4)>::Type; 181 | }; 182 | 183 | /// A variable length string 184 | template 185 | class Varchar { 186 | public: 187 | Varchar() noexcept = default; 188 | 189 | Varchar(const char* begin, const char* end) noexcept : size_(end - begin) { 190 | std::memcpy(data_, begin, size_); 191 | } 192 | 193 | const char* Begin() const noexcept { return data_; } 194 | 195 | typename LengthIndicator::Type Size() const noexcept { 196 | return size_; 197 | } 198 | 199 | private: 200 | typename LengthIndicator::Type size_; 201 | char data_[kMaxLen]; 202 | }; 203 | 204 | } // namespace storage 205 | 206 | template 207 | std::ostream& operator<<(std::ostream& out, 208 | storage::Numeric n) { 209 | int64_t raw = n.GetRaw(); 210 | if (raw < 0) { 211 | out << '-'; 212 | raw = -raw; 213 | } 214 | if (kPrecision == 0) { 215 | out << raw; 216 | } else { 217 | int64_t sep = 10; 218 | for (unsigned index = 1; index < kPrecision; ++index) { 219 | sep *= 10; 220 | } 221 | out << (raw / sep); 222 | out << '.'; 223 | raw = raw % sep; 224 | if (!raw) { 225 | for (unsigned index = 0; index < kPrecision; ++index) { 226 | out << '0'; 227 | } 228 | } else { 229 | while (sep > (10 * raw)) { 230 | out << '0'; 231 | sep /= 10; 232 | } 233 | out << raw; 234 | } 235 | } 236 | return out; 237 | } 238 | 239 | #endif // STORAGE_TYPES_H_ --------------------------------------------------------------------------------