├── .clang-format ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .gitmodules ├── AUTHORS ├── CITATION.cff ├── CMakeLists.txt ├── LICENSE ├── LICENSE.leveldb ├── LICENSE.rocksdb ├── README.md ├── bench ├── CMakeLists.txt ├── README.md ├── common │ ├── config.cc │ ├── config.h │ ├── data.cc │ ├── data.h │ ├── kvell_interface.h │ ├── leanstore_interface.h │ ├── load_data.cc │ ├── load_data.h │ ├── pg_treeline_interface.h │ ├── rocksdb_interface.h │ ├── startup.cc │ ├── startup.h │ ├── timing.h │ └── treeline_interface.h ├── data │ ├── common.h │ ├── taxi │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── contaxi.cc │ │ └── scripts │ │ │ ├── clean_year.sh │ │ │ ├── download_year.sh │ │ │ └── extract_year.sh │ └── wiki │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── genwiki.cc │ │ └── zipf_distribution.h ├── microbench │ ├── CMakeLists.txt │ ├── buffer_manager_benchmark.cc │ ├── deferred_io_simulation.cc │ ├── hashtable_ycsb.cc │ ├── memtable_benchmark.cc │ ├── packed_map_benchmark.cc │ ├── record_cache_benchmark.cc │ ├── synth_write.cc │ ├── thread_pool_benchmark.cc │ └── wal_benchmark.cc ├── run_custom.cc └── workload_configs │ └── phased_64B_A_B_A_B_A.yml ├── bufmgr ├── CMakeLists.txt ├── buffer_frame.cc ├── buffer_frame.h ├── buffer_manager.cc ├── buffer_manager.h ├── file.h ├── file_manager.cc ├── file_manager.h ├── lru_eviction.cc ├── lru_eviction.h ├── options.h ├── page_eviction_strategy.h ├── page_memory_allocator.cc ├── page_memory_allocator.h ├── physical_page_id.cc ├── physical_page_id.h ├── sync_hash_table.h ├── twoqueue_eviction.cc └── twoqueue_eviction.h ├── cond_config.toml ├── db ├── CMakeLists.txt ├── db_impl.cc ├── db_impl.h ├── db_impl_range.cc ├── format.h ├── logger.cc ├── logger.h ├── manifest.cc ├── manifest.h ├── memtable.cc ├── memtable.h ├── merge_iterators.h ├── options.cc ├── options.h ├── overflow_chain.cc ├── overflow_chain.h ├── page.cc ├── page.h ├── posix_logger.h └── slice.cc ├── include └── treeline │ ├── db.h │ ├── options.h │ ├── pg_db.h │ ├── pg_options.h │ ├── pg_stats.h │ ├── record_batch.h │ ├── slice.h │ ├── statistics.h │ └── status.h ├── model ├── CMakeLists.txt ├── btree_model.cc ├── btree_model.h ├── model.cc └── model.h ├── page_grouping ├── CMakeLists.txt ├── circular_page_buffer.h ├── config.cc ├── config.h ├── debug │ ├── CMakeLists.txt │ ├── pg_check.cc │ ├── pg_read.cc │ ├── pg_read2.cc │ └── pg_standalone.cc ├── free_list.cc ├── free_list.h ├── key.cc ├── key.h ├── lock_manager.cc ├── lock_manager.h ├── manager.cc ├── manager.h ├── manager_load.cc ├── manager_rewrite.cc ├── manager_scan.cc ├── manager_scan_prefetch.cc ├── persist │ ├── merge_iterator.h │ ├── packed_map-inl.h │ ├── packed_map.h │ ├── page.cc │ ├── page.h │ ├── segment_file.h │ ├── segment_id.cc │ ├── segment_id.h │ ├── segment_wrap.cc │ └── segment_wrap.h ├── pg_bench.cc ├── pg_db_impl.cc ├── pg_db_impl.h ├── pg_interface.h ├── pg_stats.cc ├── plr │ ├── data.h │ ├── greedy.h │ ├── pgm.h │ └── plr.h ├── rand_exp_backoff.cc ├── rand_exp_backoff.h ├── segment_builder.cc ├── segment_builder.h ├── segment_index.cc ├── segment_index.h ├── segment_info.h ├── tools │ ├── flatten.cc │ └── shuffle_pages.cc └── workspace.h ├── record_cache ├── CMakeLists.txt ├── record_cache.cc ├── record_cache.h ├── record_cache_entry.cc └── record_cache_entry.h ├── scripts ├── autotuning │ ├── COND │ ├── combine.py │ ├── phased_512B_B_A_B.yml │ ├── plot.py │ ├── preload.sh │ ├── preload512.yml │ └── run.sh ├── e2e_custom │ ├── insert │ │ ├── COND │ │ ├── combine_raw.py │ │ ├── plot.py │ │ └── workloads │ │ │ ├── middle-20.yml │ │ │ ├── middle-50.yml │ │ │ ├── middle-80.yml │ │ │ ├── middle-95.yml │ │ │ ├── middlescan-20.yml │ │ │ ├── middlescan-50.yml │ │ │ ├── middlescan-80.yml │ │ │ ├── setup.yml │ │ │ ├── uniform-100.yml │ │ │ ├── uniform-20.yml │ │ │ ├── uniform-50.yml │ │ │ ├── uniform-80.yml │ │ │ ├── uniform-95.yml │ │ │ ├── uniformscan-20.yml │ │ │ ├── uniformscan-50.yml │ │ │ └── uniformscan-80.yml │ ├── preload.sh │ ├── run.sh │ └── ycsb │ │ ├── COND │ │ ├── combine_raw.py │ │ ├── plot_bar.py │ │ ├── plot_common.py │ │ ├── plot_scale.py │ │ └── workloads │ │ ├── a.yml │ │ ├── b.yml │ │ ├── c.yml │ │ ├── d.yml │ │ ├── e.yml │ │ ├── f.yml │ │ ├── setup.yml │ │ ├── un_a.yml │ │ ├── un_b.yml │ │ ├── un_c.yml │ │ ├── un_d.yml │ │ ├── un_e.yml │ │ └── un_f.yml ├── e2e_perf │ ├── COND │ ├── README.md │ ├── build_benchmarks.sh │ ├── combine.py │ ├── generate_ycsb_traces.sh │ ├── run.sh │ ├── send_results_email.py │ ├── synth_write │ │ ├── COND │ │ ├── run_synth_write.sh │ │ └── summarize.py │ └── ycsb │ │ ├── COND │ │ ├── preload.sh │ │ ├── run_ycsb.sh │ │ └── summarize.py ├── experiment_config_example.sh ├── figures │ ├── COND │ ├── format_factor.py │ ├── plot_cdf.py │ ├── plot_cdf.sh │ ├── plot_common.py │ ├── plot_epoch_length.py │ ├── plot_forecasting.py │ ├── plot_greedyplr_compare.py │ ├── plot_lru_comparison.py │ ├── plot_pg_dist.py │ ├── plot_pg_sweep.py │ ├── plot_rand_io.py │ ├── plot_rw_sweep.py │ └── plot_ycsb.py ├── insert_v2 │ ├── COND │ ├── combine_raw.py │ ├── preload.sh │ ├── run.sh │ └── workloads │ │ ├── custom.yml │ │ └── setup.yml ├── iochar │ ├── COND │ ├── common.fio │ ├── fio_config_example.sh │ ├── mixed │ │ ├── COND │ │ ├── psync-16k │ │ ├── psync-16k-offset │ │ ├── psync-16k-unlim │ │ ├── psync-4k │ │ ├── psync-4k-offset │ │ ├── psync-4k-unlim │ │ ├── uring-16k │ │ ├── uring-16k-offset │ │ ├── uring-16k-unlim │ │ ├── uring-4k │ │ ├── uring-4k-offset │ │ └── uring-4k-unlim │ ├── read │ │ ├── COND │ │ ├── psync-16k │ │ ├── psync-4k │ │ ├── psync-64k │ │ ├── psync-8k │ │ ├── psync-sweep │ │ ├── seq-scale │ │ ├── uring-16k │ │ ├── uring-4k │ │ └── uring-64k │ ├── run_fio.sh │ └── write │ │ ├── COND │ │ ├── psync-16k │ │ ├── psync-4k │ │ ├── psync-64k │ │ ├── psync-8k │ │ ├── uring-16k │ │ ├── uring-4k │ │ └── uring-64k ├── pg │ ├── COND │ ├── insert_scan │ │ └── COND │ ├── preload.sh │ ├── preload_insert_scan.sh │ ├── process_fio.py │ ├── read │ │ ├── COND │ │ ├── run.sh │ │ └── run2.sh │ ├── run.sh │ └── workloads │ │ ├── 10M │ │ ├── long_scan-amzn.yml │ │ ├── long_scan-osm.yml │ │ ├── long_scan.yml │ │ ├── preload-amzn-insert.yml │ │ ├── preload-osm-insert.yml │ │ ├── preload-uniform-insert.yml │ │ └── preload.yml │ │ └── 20M │ │ ├── long_scan-amzn.yml │ │ ├── long_scan-osm.yml │ │ ├── long_scan.yml │ │ ├── preload-amzn-insert.yml │ │ ├── preload-osm-insert.yml │ │ ├── preload-uniform-insert.yml │ │ └── preload.yml ├── pg_analysis │ ├── COND │ ├── dump_dataset.py │ ├── process_segment_size.py │ ├── process_sweep.py │ ├── run_load.sh │ └── workload.yml ├── reorg_vs_split │ ├── COND │ ├── combine_raw.py │ ├── insert_middle.yml │ ├── insert_middle_amzn.yml │ ├── insert_middle_amzn_20m.yml │ ├── preload.sh │ └── run.sh ├── run_deferred_io_simulation.sh ├── run_ycsb_w_deferral.sh ├── rw_sweep │ ├── COND │ ├── combine_raw.py │ ├── generate_workload.py │ ├── preload.sh │ └── run.sh └── ycsb_v2 │ ├── COND │ ├── combine_raw.py │ ├── generate_workload.py │ ├── preload.sh │ ├── run.sh │ └── workloads │ ├── a.yml │ ├── b.yml │ ├── c.yml │ ├── d.yml │ ├── e.yml │ ├── f.yml │ ├── scan_only.yml │ └── setup.yml ├── tests ├── CMakeLists.txt ├── buffer_manager_test.cc ├── coding_test.cc ├── db_test.cc ├── file_manager_test.cc ├── insert_tracker_test.cc ├── manifest_test.cc ├── memtable_test.cc ├── packed_map_test.cc ├── page_test.cc ├── pg_datasets.cc ├── pg_datasets.h ├── pg_db_test.cc ├── pg_lock_manager_test.cc ├── pg_manager_rewrite_test.cc ├── pg_manager_test.cc ├── pg_segment_info_test.cc ├── pg_segment_test.cc ├── record_cache_test.cc ├── thread_pool_test.cc ├── wal_manager_test.cc └── wal_rw_test.cc ├── third_party ├── CMakeLists.txt ├── leanstore │ ├── .clang-format │ ├── .github │ │ └── workflows │ │ │ └── ccpp.yml │ ├── .gitignore │ ├── .projectile │ ├── CMakeLists.txt │ ├── LICENSE │ ├── README.md │ ├── backend │ │ ├── CMakeLists.txt │ │ └── leanstore │ │ │ ├── BTreeAdapter.hpp │ │ │ ├── Config.cpp │ │ │ ├── Config.hpp │ │ │ ├── LeanStore.cpp │ │ │ ├── LeanStore.hpp │ │ │ ├── concurrency-recovery │ │ │ ├── CRMG.cpp │ │ │ ├── CRMG.hpp │ │ │ ├── GroupCommiter.cpp │ │ │ ├── Transaction.hpp │ │ │ ├── WALEntry.hpp │ │ │ ├── Worker.cpp │ │ │ └── Worker.hpp │ │ │ ├── profiling │ │ │ ├── counters │ │ │ │ ├── CPUCounters.cpp │ │ │ │ ├── CPUCounters.hpp │ │ │ │ ├── CRCounters.cpp │ │ │ │ ├── CRCounters.hpp │ │ │ │ ├── PPCounters.cpp │ │ │ │ ├── PPCounters.hpp │ │ │ │ ├── WorkerCounters.cpp │ │ │ │ └── WorkerCounters.hpp │ │ │ └── tables │ │ │ │ ├── BMTable.cpp │ │ │ │ ├── BMTable.hpp │ │ │ │ ├── CPUTable.cpp │ │ │ │ ├── CPUTable.hpp │ │ │ │ ├── CRTable.cpp │ │ │ │ ├── CRTable.hpp │ │ │ │ ├── ConfigsTable.cpp │ │ │ │ ├── ConfigsTable.hpp │ │ │ │ ├── DTTable.cpp │ │ │ │ ├── DTTable.hpp │ │ │ │ └── ProfilingTable.hpp │ │ │ ├── storage │ │ │ ├── btree │ │ │ │ ├── BTreeLL.cpp │ │ │ │ ├── BTreeLL.hpp │ │ │ │ └── core │ │ │ │ │ ├── BTreeGeneric.cpp │ │ │ │ │ ├── BTreeGeneric.hpp │ │ │ │ │ ├── BTreeGenericIterator.cpp │ │ │ │ │ ├── BTreeGenericIterator.hpp │ │ │ │ │ ├── BTreeInterface.hpp │ │ │ │ │ ├── BTreeIteratorInterface.hpp │ │ │ │ │ ├── BTreeNode.cpp │ │ │ │ │ ├── BTreeNode.hpp │ │ │ │ │ └── WALMacros.hpp │ │ │ └── buffer-manager │ │ │ │ ├── AsyncWriteBuffer.cpp │ │ │ │ ├── AsyncWriteBuffer.hpp │ │ │ │ ├── BufferFrame.cpp │ │ │ │ ├── BufferFrame.hpp │ │ │ │ ├── BufferManager.cpp │ │ │ │ ├── BufferManager.hpp │ │ │ │ ├── DTRegistry.cpp │ │ │ │ ├── DTRegistry.hpp │ │ │ │ ├── DTTypes.hpp │ │ │ │ ├── FreeList.cpp │ │ │ │ ├── FreeList.hpp │ │ │ │ ├── PageProviderThread.cpp │ │ │ │ ├── Partition.cpp │ │ │ │ ├── Partition.hpp │ │ │ │ ├── Swip.cpp │ │ │ │ └── Swip.hpp │ │ │ ├── sync-primitives │ │ │ ├── Latch.hpp │ │ │ ├── PageGuard.hpp │ │ │ └── PlainGuard.hpp │ │ │ ├── threads │ │ │ ├── UT.cpp │ │ │ └── UT.hpp │ │ │ └── utils │ │ │ ├── FNVHash.cpp │ │ │ ├── FNVHash.hpp │ │ │ ├── FVector.hpp │ │ │ ├── Files.cpp │ │ │ ├── Files.hpp │ │ │ ├── JumpMU.cpp │ │ │ ├── JumpMU.hpp │ │ │ ├── MMapVector.cpp │ │ │ ├── Misc.cpp │ │ │ ├── Misc.hpp │ │ │ ├── Parallelize.cpp │ │ │ ├── Parallelize.hpp │ │ │ ├── RandomGenerator.cpp │ │ │ ├── RandomGenerator.hpp │ │ │ ├── ScrambledZipfGenerator.cpp │ │ │ ├── ScrambledZipfGenerator.hpp │ │ │ ├── StringsManipulation.hpp │ │ │ ├── ThreadLocalAggregator.hpp │ │ │ ├── ZipfGenerator.cpp │ │ │ └── ZipfGenerator.hpp │ ├── frontend │ │ ├── CMakeLists.txt │ │ ├── frontend.cpp │ │ ├── tpc-c │ │ │ ├── adapter.hpp │ │ │ ├── schema.hpp │ │ │ ├── tpcc.cpp │ │ │ ├── tpcc_workload.hpp │ │ │ └── types.hpp │ │ └── ycsb │ │ │ └── ycsb.cpp │ ├── libs │ │ ├── benchmark.cmake │ │ ├── croaring.cmake │ │ ├── fastpfor.cmake │ │ ├── gdouble.cmake │ │ ├── gflags.cmake │ │ ├── googletest.cmake │ │ ├── lz4.cmake │ │ ├── psql.cmake │ │ ├── rapidjson.cmake │ │ ├── spdlog.cmake │ │ ├── tabluate.cmake │ │ ├── tbb.cmake │ │ ├── turbo.cmake │ │ └── yaml-cpp.cmake │ └── shared-headers │ │ ├── CRC.hpp │ │ ├── Exceptions.hpp │ │ ├── PerfEvent.hpp │ │ ├── Units.hpp │ │ └── local.cmake ├── masstree │ ├── AUTHORS │ ├── CMakeLists.txt │ ├── GNUmakefile.in │ ├── LICENSE │ ├── README.md │ ├── bootstrap.sh │ ├── btree_leaflink.hh │ ├── checkpoint.cc │ ├── checkpoint.hh │ ├── circular_int.hh │ ├── clp.c │ ├── clp.h │ ├── compiler.cc │ ├── compiler.hh │ ├── configure.ac │ ├── doc │ │ ├── .gitignore │ │ ├── GNUmakefile │ │ ├── elements.mp │ │ ├── elemfig.sty │ │ ├── examples.mp │ │ ├── insert1.mp │ │ ├── masstree.mp │ │ ├── patches.mp │ │ ├── remove1.mp │ │ ├── remove2.mp │ │ └── spec.tex │ ├── file.cc │ ├── file.hh │ ├── hashcode.hh │ ├── json.cc │ ├── json.hh │ ├── jsontest.cc │ ├── kpermuter.hh │ ├── ksearch.hh │ ├── kvio.cc │ ├── kvio.hh │ ├── kvproto.hh │ ├── kvrandom.cc │ ├── kvrandom.hh │ ├── kvrow.hh │ ├── kvstats.hh │ ├── kvtest.hh │ ├── kvthread.cc │ ├── kvthread.hh │ ├── log.cc │ ├── log.hh │ ├── masstree.hh │ ├── masstree_get.hh │ ├── masstree_insert.hh │ ├── masstree_key.hh │ ├── masstree_print.hh │ ├── masstree_remove.hh │ ├── masstree_scan.hh │ ├── masstree_split.hh │ ├── masstree_stats.hh │ ├── masstree_struct.hh │ ├── masstree_tcursor.hh │ ├── memdebug.cc │ ├── memdebug.hh │ ├── misc.cc │ ├── misc.hh │ ├── msgpack.cc │ ├── msgpack.hh │ ├── msgpacktest.cc │ ├── mtclient.cc │ ├── mtclient.hh │ ├── mtcounters.hh │ ├── mtd.cc │ ├── mttest.cc │ ├── nodeversion.hh │ ├── perfstat.cc │ ├── perfstat.hh │ ├── query_masstree.cc │ ├── query_masstree.hh │ ├── scantest.cc │ ├── small_vector.hh │ ├── str.cc │ ├── str.hh │ ├── straccum.cc │ ├── straccum.hh │ ├── string.cc │ ├── string.hh │ ├── string_base.hh │ ├── string_slice.cc │ ├── string_slice.hh │ ├── stringbag.hh │ ├── test_atomics.cc │ ├── test_string.cc │ ├── testrunner.cc │ ├── testrunner.hh │ ├── timestamp.hh │ ├── unit-mt.cc │ ├── value_array.cc │ ├── value_array.hh │ ├── value_bag.hh │ ├── value_string.cc │ ├── value_string.hh │ ├── value_versioned_array.cc │ └── value_versioned_array.hh ├── masstree_wrapper │ ├── CMakeLists.txt │ ├── LICENSE │ ├── masstree_wrapper.cc │ └── masstree_wrapper.h ├── pgm │ └── piecewise_linear_model.hpp └── tlx │ ├── CMakeLists.txt │ ├── LICENSE │ ├── btree.h │ ├── btree_map.h │ ├── core.cc │ └── core.h ├── util ├── CMakeLists.txt ├── affinity.h ├── arena.cc ├── arena.h ├── calc.h ├── coding.cc ├── coding.h ├── crc32c.h ├── gcc_macros.h ├── hash_queue.h ├── inlineskiplist.h ├── insert_tracker.h ├── key.h ├── packed_map-inl.h ├── packed_map.h ├── random.cc ├── random.h ├── skiplist.h ├── status.cc ├── thread_pool.cc ├── thread_pool.h ├── timer.cc ├── timer.h └── tracking_allocator.h └── wal ├── CMakeLists.txt ├── format.h ├── manager.cc ├── manager.h ├── reader.cc ├── reader.h ├── writer.cc └── writer.h /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Testing Suite 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | workflow_dispatch: 9 | 10 | env: 11 | # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) 12 | BUILD_TYPE: Debug 13 | 14 | jobs: 15 | build: 16 | # The CMake configure and build commands are platform agnostic and should work equally 17 | # well on Windows or Mac. You can convert this to a matrix build if you need 18 | # cross-platform coverage. 19 | # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - uses: actions/checkout@v2 24 | 25 | - name: Install TBB 26 | run: sudo apt-get install libtbb-dev 27 | 28 | - name: Install jemalloc 29 | run: sudo apt-get install libjemalloc-dev 30 | 31 | - name: Configure Project using CMake 32 | run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DTL_BUILD_TESTS=ON -DTL_BUILD_BENCHMARKS=OFF 33 | 34 | - name: Compile Project 35 | working-directory: ${{github.workspace}}/build 36 | run: make -j 37 | timeout-minutes: 15 38 | 39 | - name: Run Tests using CTest 40 | working-directory: ${{github.workspace}}/build 41 | run: ctest -T Test --output-on-failure 42 | timeout-minutes: 15 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bench/hash_tables/libcuckoo/build 2 | bench/results/ 3 | bench/workloads/ 4 | build/ 5 | cmake-build-debug/ 6 | cmake-build-release/ 7 | cond-out* 8 | scripts/experiment_config.sh 9 | scripts/e2e_perf/results 10 | scripts/iochar/fio_config.sh 11 | third_party/masstree/autom4te.cache* 12 | third_party/masstree/config.* 13 | third_party/masstree/configure 14 | third_party/masstree/GNUmakefile 15 | 16 | .idea/ 17 | .vscode/ 18 | 19 | .DS_Store 20 | *.swp 21 | 22 | *.pyc 23 | __pycache__ 24 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/kvell"] 2 | path = third_party/kvell 3 | url = https://github.com/BLepers/KVell 4 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # The TreeLine project uses code and concepts from LevelDB and RocksDB. We 2 | # reproduce the LevelDB and RocksDB AUTHORS file below. 3 | 4 | Facebook Inc. 5 | Facebook Engineering Team 6 | 7 | Google Inc. 8 | # Initial version authors: 9 | Jeffrey Dean 10 | Sanjay Ghemawat 11 | 12 | # Partial list of contributors: 13 | Kevin Regan 14 | Johan Bilien 15 | Matthew Von-Maszewski (Basho Technologies) 16 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use TreeLine, please cite it as below." 3 | authors: 4 | - family-names: "Yu" 5 | given-names: "Geoffrey X." 6 | - family-names: "Markakis" 7 | given-names: "Markos" 8 | - family-names: "Kipf" 9 | given-names: "Andreas" 10 | - family-names: "Larson" 11 | given-names: "Per-Åke" 12 | - family-names: "Minhas" 13 | given-names: "Umar Farooq" 14 | - family-names: "Kraska" 15 | given-names: "Tim" 16 | title: "TreeLine: An Update-In-Place Key-Value Store for Modern Storage" 17 | version: 1.0.0 18 | date-released: 2022-09-25 19 | url: "https://github.com/mitdbg/treeline" 20 | preferred-citation: 21 | type: article 22 | authors: 23 | - family-names: "Yu" 24 | given-names: "Geoffrey X." 25 | - family-names: "Markakis" 26 | given-names: "Markos" 27 | - family-names: "Kipf" 28 | given-names: "Andreas" 29 | - family-names: "Larson" 30 | given-names: "Per-Åke" 31 | - family-names: "Minhas" 32 | given-names: "Umar Farooq" 33 | - family-names: "Kraska" 34 | given-names: "Tim" 35 | doi: "10.14778/3561261.3561270" 36 | journal: "Proceedings of the VLDB Endowment" 37 | month: 9 38 | start: 99 39 | end: 112 40 | title: "TreeLine: An Update-In-Place Key-Value Store for Modern Storage" 41 | issue: 1 42 | volume: 16 43 | year: 2022 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2022 Massachusetts Institute of Technology. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /LICENSE.leveldb: -------------------------------------------------------------------------------- 1 | Portions of the code in this project have been taken from or adapted from LevelDB. 2 | The files have their copyright notices retained and they carry the license below. 3 | 4 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | * Neither the name of Google Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://github.com/mitdbg/learnedlsm/actions/workflows/ci.yml/badge.svg) 2 | 3 | # TreeLine 4 | An embedded key-value store for modern SSDs. 5 | 6 | ## Building from source 7 | 8 | ### Install Dependencies 9 | 10 | A few packages that TreeLine depends on are 11 | 12 | - `libtbb-dev` 13 | - `autoconf` 14 | - `libjemalloc-dev` 15 | 16 | Depending on the distribution you have, ensure the above packages are installed. 17 | On Ubuntu, you can install the dependencies using `apt`: 18 | 19 | ``` 20 | sudo apt install libtbb-dev autoconf libjemalloc-dev 21 | ``` 22 | 23 | TreeLine's other dependencies are fetched by CMake during compilation. 24 | 25 | ### Compile 26 | 27 | CMake 3.17+ is required for building this project. 28 | 29 | ```bash 30 | mkdir build && cd build 31 | cmake -DCMAKE_BUILD_TYPE=Release .. && make -j 32 | ``` 33 | 34 | To build the tests, turn on the `TL_BUILD_TESTS` option when configuring. 35 | ```bash 36 | cmake -DCMAKE_BUILD_TYPE=Release -DTL_BUILD_TESTS=ON .. && make -j 37 | ``` 38 | 39 | To build the benchmarks, turn on the `TL_BUILD_BENCHMARKS` option when 40 | configuring. 41 | ```bash 42 | cmake -DCMAKE_BUILD_TYPE=Release -DTL_BUILD_BENCHMARKS=ON .. && make -j 43 | ``` 44 | 45 | ## Inspecting the codebase 46 | 47 | If you would like to read more about the internals of TreeLine, you can start at [this header file](https://github.com/mitdbg/treeline/blob/master/include/treeline/pg_db.h). 48 | 49 | The bulk of the code that comprises the current version of the system can be found in the [`page_grouping/`](https://github.com/mitdbg/treeline/tree/master/page_grouping) directory. 50 | 51 | Thank you for your interest in diving deeper in our work! 52 | 53 | 54 | -------------------------------------------------------------------------------- /bench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Common utilities used in the TreeLine benchmarks. 2 | add_library(bench_common 3 | common/data.cc 4 | common/data.h 5 | common/load_data.cc 6 | common/load_data.h 7 | common/startup.cc 8 | common/startup.h 9 | common/timing.h) 10 | # Any targets linking to `bench_common` will also link to `treeline` and will be 11 | # able to include TreeLine internal headers. 12 | target_link_libraries(bench_common PUBLIC treeline) 13 | target_include_directories(bench_common 14 | PUBLIC 15 | ${PROJECT_SOURCE_DIR} 16 | ${CMAKE_CURRENT_SOURCE_DIR} 17 | ) 18 | 19 | # All `bench_common` utilities plus configuration utilities used in benchmarks 20 | # comparing TreeLine against RocksDB. 21 | # 22 | # We use this additional library target because not all benchmark executables 23 | # need to link to RocksDB and gflags (e.g., `microbench`). 24 | add_library(bench_common_config 25 | common/config.cc 26 | common/config.h 27 | common/kvell_interface.h 28 | common/leanstore_interface.h 29 | common/treeline_interface.h 30 | common/pg_treeline_interface.h 31 | common/rocksdb_interface.h) 32 | target_link_libraries(bench_common_config PUBLIC 33 | bench_common 34 | gflags 35 | rocksdb 36 | leanstore 37 | pg_treeline 38 | libcuckoo) 39 | 40 | # The microbench directory contains microbenchmarks used to guide TreeLine's 41 | # development. 42 | add_subdirectory(microbench) 43 | 44 | # Run Custom: An executable that runs YSCBR-generated workloads against TreeLine 45 | # and RocksDB. 46 | add_executable(run_custom run_custom.cc) 47 | target_link_libraries(run_custom bench_common_config ycsbr-gen) 48 | -------------------------------------------------------------------------------- /bench/README.md: -------------------------------------------------------------------------------- 1 | # TreeLine Benchmarks 2 | 3 | This directory contains various benchmarks used to evaluate TreeLine. The 4 | primary executable is `run_custom`, which runs key-value workloads against 5 | TreeLine, RocksDB, and LeanStore. 6 | -------------------------------------------------------------------------------- /bench/common/load_data.cc: -------------------------------------------------------------------------------- 1 | #include "load_data.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace tl { 8 | namespace bench { 9 | 10 | std::vector LoadDatasetFromTextFile( 11 | const std::filesystem::path& dataset_path, const bool warn_on_duplicates) { 12 | std::unordered_set keys; 13 | std::ifstream in(dataset_path); 14 | uint64_t loaded_key; 15 | while (in >> loaded_key) { 16 | auto result = keys.insert(loaded_key); 17 | if (warn_on_duplicates && !result.second) { 18 | std::cerr << "WARNING: Ignored a duplicate key in the dataset: " 19 | << loaded_key << std::endl; 20 | } 21 | } 22 | 23 | std::vector keys_as_vector(keys.begin(), keys.end()); 24 | return keys_as_vector; 25 | } 26 | 27 | } // namespace bench 28 | } // namespace tl 29 | -------------------------------------------------------------------------------- /bench/common/load_data.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace tl { 8 | namespace bench { 9 | 10 | // The expected file format is a sequence of integers separated by newlines. 11 | // This function will remove any duplicates from the loaded dataset. 12 | std::vector LoadDatasetFromTextFile( 13 | const std::filesystem::path& dataset_path, bool warn_on_duplicates = false); 14 | 15 | } // namespace bench 16 | } // namespace tl 17 | -------------------------------------------------------------------------------- /bench/common/startup.cc: -------------------------------------------------------------------------------- 1 | #include "startup.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace { 8 | 9 | static constexpr int kReadySignal = SIGUSR1; 10 | 11 | } // namespace 12 | 13 | namespace tl { 14 | namespace bench { 15 | 16 | void SendReadySignalToParent() { 17 | const pid_t parent_pid = getppid(); 18 | if (kill(parent_pid, kReadySignal) < 0) { 19 | perror("SendReadySignalToParent():"); 20 | } 21 | } 22 | 23 | } 24 | } // namespace tl 25 | -------------------------------------------------------------------------------- /bench/common/startup.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace tl { 4 | namespace bench { 5 | 6 | // Called by the benchmark driver to notify its parent process when it has 7 | // finished initializing the database and will start running the workload. 8 | // This function will send the parent process a `SIGUSR1` signal. 9 | // 10 | // The parent process can use this signal to know when to start making 11 | // "physical" measurements (e.g., physical I/O while the workload runs). 12 | void SendReadySignalToParent(); 13 | 14 | } // namespace bench 15 | } // namespace tl 16 | -------------------------------------------------------------------------------- /bench/common/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace tl { 6 | namespace bench { 7 | 8 | template 9 | static std::chrono::nanoseconds MeasureRunTime(Callable callable) { 10 | auto start = std::chrono::steady_clock::now(); 11 | callable(); 12 | auto end = std::chrono::steady_clock::now(); 13 | return end - start; 14 | } 15 | 16 | } // namespace bench 17 | } // namespace tl 18 | -------------------------------------------------------------------------------- /bench/data/common.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | struct Record { 6 | uint64_t key; 7 | uint64_t value; 8 | }; 9 | 10 | // Loads values from binary file into vector. 11 | template 12 | static std::vector load_data(const std::string& filename) { 13 | std::vector data; 14 | 15 | std::ifstream in(filename, std::ios::binary); 16 | if (!in.is_open()) { 17 | std::cerr << "Unable to open " << filename << std::endl; 18 | exit(EXIT_FAILURE); 19 | } 20 | // Read size. 21 | uint64_t size; 22 | in.read(reinterpret_cast(&size), sizeof(uint64_t)); 23 | data.resize(size); 24 | // Read values. 25 | in.read(reinterpret_cast(data.data()), size * sizeof(T)); 26 | in.close(); 27 | 28 | std::cout << "Read " << data.size() << " values from " << filename 29 | << std::endl; 30 | 31 | return data; 32 | } 33 | 34 | // Writes `records` to a CSV file. 35 | static void write_to_csv(const std::vector& records, 36 | const std::string& filename, 37 | const bool key_only = false) { 38 | std::cout << "Writing records to CSV file " << filename << "..."; 39 | std::ofstream out(filename, std::ios_base::trunc); 40 | if (!out.is_open()) { 41 | std::cerr << "Unable to open " << filename << std::endl; 42 | exit(EXIT_FAILURE); 43 | } 44 | for (const auto& record : records) { 45 | if (key_only) { 46 | out << record.key << std::endl; 47 | } else { 48 | out << record.key << "," << record.value << std::endl; 49 | } 50 | } 51 | out.close(); 52 | 53 | std::cout << " done." << std::endl; 54 | } 55 | -------------------------------------------------------------------------------- /bench/data/taxi/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | project(data LANGUAGES CXX) 3 | set(CMAKE_CXX_STANDARD 17) 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") 5 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fsanitize=address") 6 | 7 | # S2Geometry 8 | include(ExternalProject) 9 | include(FetchContent) 10 | FetchContent_Declare( 11 | s2 12 | GIT_REPOSITORY https://github.com/google/s2geometry.git 13 | GIT_TAG v0.9.0 14 | ) 15 | FetchContent_MakeAvailable(s2) 16 | 17 | add_executable(contaxi contaxi.cc ../common.h) 18 | target_link_libraries(contaxi s2) 19 | -------------------------------------------------------------------------------- /bench/data/taxi/README.md: -------------------------------------------------------------------------------- 1 | # Download taxi data, extract location columns, and remove outliers 2 | ``` 3 | ./scripts/download_year.sh 2015 4 | 5 | Note that this dataset isn't sorted by time. 6 | One way to sort it is to import it into DuckDB, sort it by the pickup timestamp column, and write it back to CSV. 7 | https://duckdb.org/docs/data/csv 8 | 9 | ./scripts/extract_year.sh 2015 10 | 11 | Extracts the lng/lat columns. 12 | Produces yellow_tripdata_2015-01_lnglat.csv etc. 13 | 14 | ./scripts/clean_year.sh 2015 15 | ``` 16 | 17 | # Convert lat/lng coordinates to S2 cell ids (64-bit unsigned integers) 18 | ``` 19 | mkdir -p build 20 | cd build 21 | cmake -DCMAKE_BUILD_TYPE=Release .. 22 | make -j contaxi 23 | ./contaxi yellow_tripdata_2015-01_lnglat.csv 24 | ``` 25 | -------------------------------------------------------------------------------- /bench/data/taxi/scripts/clean_year.sh: -------------------------------------------------------------------------------- 1 | year=$1 2 | for i in {1..9}; do 3 | ./clean.sh yellow_tripdata_"$year"-0"$i"_lnglat.csv & 4 | done 5 | for i in {10..12}; do 6 | ./clean.sh yellow_tripdata_"$year"-"$i"_lnglat.csv & 7 | done 8 | wait 9 | -------------------------------------------------------------------------------- /bench/data/taxi/scripts/download_year.sh: -------------------------------------------------------------------------------- 1 | year=$1 2 | for i in {1..9}; do 3 | wget https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_$year-0$i.csv 4 | done 5 | for i in {10..12}; do 6 | wget https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_$year-$i.csv 7 | done 8 | -------------------------------------------------------------------------------- /bench/data/taxi/scripts/extract_year.sh: -------------------------------------------------------------------------------- 1 | year=$1 2 | for i in {1..9}; do 3 | ./extract.sh yellow_tripdata_"$year"-0"$i".csv yellow_tripdata_"$year"-0"$i"_lnglat.csv & 4 | done 5 | for i in {10..12}; do 6 | ./extract.sh yellow_tripdata_"$year"-"$i".csv yellow_tripdata_"$year"-"$i"_lnglat.csv & 7 | done 8 | wait 9 | -------------------------------------------------------------------------------- /bench/data/wiki/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | project(data LANGUAGES CXX) 3 | set(CMAKE_CXX_STANDARD 17) 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") 5 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fsanitize=address") 6 | 7 | add_executable(genwiki genwiki.cc ../common.h zipf_distribution.h) 8 | -------------------------------------------------------------------------------- /bench/data/wiki/README.md: -------------------------------------------------------------------------------- 1 | # Download Wikipedia edit timestamps 2 | ``` 3 | wget https://raw.githubusercontent.com/learnedsystems/SOSD/master/scripts/download.sh 4 | ./download.sh 5 | 6 | Or copy it from tebow:/spinning/kipf/sosd_data/data/wiki_ts_200M_uint64 7 | ``` 8 | 9 | # Generate Wikipedia trace 10 | ``` 11 | mkdir -p build 12 | cd build 13 | cmake -DCMAKE_BUILD_TYPE=Release .. 14 | make -j genwiki 15 | ./genwiki wiki_ts_200M_uint64 16 | ``` 17 | -------------------------------------------------------------------------------- /bench/microbench/memtable_benchmark.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "bench/common/data.h" 4 | #include "benchmark/benchmark.h" 5 | #include "db/memtable.h" 6 | 7 | namespace { 8 | 9 | using namespace tl; 10 | 11 | void MemTableInsert_64MiB(benchmark::State& state, bool shuffle) { 12 | constexpr size_t kDatasetSizeMiB = 64; 13 | bench::U64Dataset::GenerateOptions options; 14 | options.record_size = state.range(0); 15 | options.shuffle = shuffle; 16 | bench::U64Dataset dataset = 17 | bench::U64Dataset::Generate(kDatasetSizeMiB, options); 18 | Status s; 19 | MemTableOptions moptions; 20 | for (auto _ : state) { 21 | MemTable table(moptions); 22 | for (const auto& record : dataset) { 23 | s = table.Put(record.key(), record.value()); 24 | if (!s.ok()) { 25 | throw std::runtime_error("Failed to insert record into the memtable!"); 26 | } 27 | } 28 | } 29 | state.SetBytesProcessed(state.iterations() * kDatasetSizeMiB * 1024 * 1024); 30 | } 31 | 32 | BENCHMARK_CAPTURE(MemTableInsert_64MiB, in_order, /*shuffle=*/false) 33 | ->Arg(16) // Record size in bytes 34 | ->Arg(512) 35 | ->Unit(benchmark::kMillisecond); 36 | 37 | BENCHMARK_CAPTURE(MemTableInsert_64MiB, shuffled, /*shuffle=*/true) 38 | ->Arg(16) // Record size in bytes 39 | ->Arg(512) 40 | ->Unit(benchmark::kMillisecond); 41 | 42 | } // namespace 43 | -------------------------------------------------------------------------------- /bench/workload_configs/phased_64B_A_B_A_B_A.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: linspace 7 | start_key: 0 8 | step_size: 1 9 | 10 | run: 11 | - num_requests: 6000000 12 | read: 13 | proportion_pct: 50 14 | distribution: 15 | type: zipfian 16 | theta: 0.99 17 | salt: 12345 18 | update: 19 | proportion_pct: 50 20 | distribution: 21 | type: zipfian 22 | theta: 0.99 23 | salt: 12345 24 | 25 | - num_requests: 60000000 26 | read: 27 | proportion_pct: 95 28 | distribution: 29 | type: zipfian 30 | theta: 0.99 31 | salt: 12345 32 | update: 33 | proportion_pct: 5 34 | distribution: 35 | type: zipfian 36 | theta: 0.99 37 | salt: 12345 38 | 39 | - num_requests: 6000000 40 | read: 41 | proportion_pct: 50 42 | distribution: 43 | type: zipfian 44 | theta: 0.99 45 | salt: 12345 46 | update: 47 | proportion_pct: 50 48 | distribution: 49 | type: zipfian 50 | theta: 0.99 51 | salt: 12345 52 | 53 | - num_requests: 60000000 54 | read: 55 | proportion_pct: 95 56 | distribution: 57 | type: zipfian 58 | theta: 0.99 59 | salt: 12345 60 | update: 61 | proportion_pct: 5 62 | distribution: 63 | type: zipfian 64 | theta: 0.99 65 | salt: 12345 66 | 67 | - num_requests: 6000000 68 | read: 69 | proportion_pct: 50 70 | distribution: 71 | type: zipfian 72 | theta: 0.99 73 | salt: 12345 74 | update: 75 | proportion_pct: 50 76 | distribution: 77 | type: zipfian 78 | theta: 0.99 79 | salt: 12345 80 | -------------------------------------------------------------------------------- /bufmgr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | buffer_frame.cc 3 | buffer_frame.h 4 | buffer_manager.cc 5 | buffer_manager.h 6 | file_manager.cc 7 | file_manager.h 8 | file.h 9 | lru_eviction.cc 10 | lru_eviction.h 11 | options.h 12 | page_eviction_strategy.h 13 | page_memory_allocator.cc 14 | page_memory_allocator.h 15 | physical_page_id.cc 16 | physical_page_id.h 17 | twoqueue_eviction.cc 18 | twoqueue_eviction.h) 19 | -------------------------------------------------------------------------------- /bufmgr/lru_eviction.cc: -------------------------------------------------------------------------------- 1 | #include "lru_eviction.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace tl { 8 | 9 | // Create a new eviction strategy. 10 | LRUEviction::LRUEviction(size_t num_elements) { 11 | lru_ = std::make_unique>(num_elements); 12 | } 13 | 14 | // Free all resources. 15 | LRUEviction::~LRUEviction() {} 16 | 17 | // Makes the page held by `frame` a candidate for eviction. 18 | void LRUEviction::Insert(BufferFrame* frame) { 19 | lru_->Enqueue(frame); 20 | frame->SetEviction(kHasBeenInLru); 21 | } 22 | 23 | // Ensures the page held by `frame` is no longer a candidate for eviction. 24 | void LRUEviction::Delete(BufferFrame* frame) { 25 | uint8_t eviction_flags = frame->GetEviction(); 26 | 27 | if (HasBeenInLru(eviction_flags)) { 28 | lru_->Delete(frame); 29 | } 30 | } 31 | 32 | // Evicts a page previously marked as a candidate for eviction (if any), 33 | // following the LRU eviction strategy. 34 | BufferFrame* LRUEviction::Evict() { 35 | BufferFrame* evicted = nullptr; 36 | 37 | if (!lru_->IsEmpty()) { 38 | evicted = lru_->Dequeue(); 39 | evicted->UnsetEviction(); 40 | } 41 | 42 | return evicted; 43 | } 44 | 45 | // Prints the state of the eviction strategy, namely the contents of the LRU 46 | // queue. 47 | void LRUEviction::PrintState() { 48 | std::cout << "----------------------------" << std::endl 49 | << "Printing the state of LRU..." << std::endl; 50 | 51 | for (auto it = lru_->begin(); it != lru_->end(); ++it) { 52 | std::cout << (*it)->GetPageId() << std::endl; 53 | } 54 | 55 | std::cout << "----------------------------" << std::endl; 56 | } 57 | 58 | } // namespace tl 59 | -------------------------------------------------------------------------------- /bufmgr/options.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "db/page.h" 6 | #include "treeline/options.h" 7 | 8 | namespace tl { 9 | 10 | // Configuration options used by the `BufferManager`. 11 | struct BufMgrOptions { 12 | // Create `BufMgrOptions` using its default values (defined below). 13 | BufMgrOptions() {} 14 | 15 | // Sets the options that are identical in both `Options` and `BufMgrOptions` 16 | // using an existing `Options` instance. All other options are set to their 17 | // default values (defined below). 18 | explicit BufMgrOptions(const Options& options) 19 | : buffer_pool_size(options.buffer_pool_size), 20 | use_direct_io(options.use_direct_io) {} 21 | 22 | // The size of the buffer pool, in bytes. 23 | size_t buffer_pool_size = 64 * 1024 * 1024; 24 | 25 | // The number of segments to use to store the pages. The pages are equally 26 | // divided among all the segments. 27 | size_t num_segments = 1; 28 | 29 | // Whether or not the buffer manager should use direct I/O. 30 | bool use_direct_io = false; 31 | 32 | // Whether the buffer manager should be run without an underlying file manager, 33 | // for performance benchmarking purposes. 34 | bool simulation_mode = false; 35 | }; 36 | 37 | } // namespace tl 38 | -------------------------------------------------------------------------------- /bufmgr/page_eviction_strategy.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "buffer_frame.h" 4 | namespace tl { 5 | 6 | // A strategy for evicting in-memory pages back to disk. 7 | class PageEvictionStrategy { 8 | public: 9 | // Frees all resources. 10 | virtual ~PageEvictionStrategy() { return; } 11 | 12 | // Makes the page held by `frame` a candidate for eviction. 13 | virtual void Insert(BufferFrame* frame) = 0; 14 | 15 | // Ensures the page held by `frame` is no longer a candidate for eviction. 16 | virtual void Delete(BufferFrame* frame) = 0; 17 | 18 | // Evicts a page previously marked as a candidate for eviction (if any). 19 | virtual BufferFrame* Evict() = 0; 20 | 21 | // Prints the state of the eviction strategy 22 | virtual void PrintState() = 0; 23 | }; 24 | 25 | } // namespace tl 26 | -------------------------------------------------------------------------------- /bufmgr/page_memory_allocator.cc: -------------------------------------------------------------------------------- 1 | #include "bufmgr/page_memory_allocator.h" 2 | 3 | #include 4 | 5 | namespace tl { 6 | 7 | size_t PageMemoryAllocator::alignment_ = PageMemoryAllocator::kDefaultAlignment; 8 | 9 | void PageMemoryAllocator::SetAlignmentFor(const std::filesystem::path& path) { 10 | struct statvfs fs_stats; 11 | if (statvfs(path.c_str(), &fs_stats) == 0) { 12 | // We want memory to be aligned to the file system's block size to support 13 | // efficient direct I/O. 14 | alignment_ = fs_stats.f_bsize; 15 | } else { 16 | // The system call failed; use the default alignment instead. 17 | alignment_ = PageMemoryAllocator::kDefaultAlignment; 18 | } 19 | } 20 | 21 | } // namespace tl 22 | -------------------------------------------------------------------------------- /bufmgr/physical_page_id.cc: -------------------------------------------------------------------------------- 1 | #include "physical_page_id.h" 2 | 3 | namespace tl { 4 | 5 | // Define static variables (default to 1-segment case). 6 | size_t PhysicalPageId::offset_bits_ = (sizeof(size_t) << 3); 7 | 8 | size_t PhysicalPageId::segment_mask_ = 0ULL; 9 | size_t PhysicalPageId::offset_mask_ = ~(0ULL); 10 | 11 | } // namespace tl 12 | 13 | namespace std { 14 | 15 | ostream& operator<<(ostream& os, const tl::PhysicalPageId& id) { 16 | os << id.GetFileId() << "-" << id.GetOffset(); 17 | return os; 18 | } 19 | 20 | } // namespace std -------------------------------------------------------------------------------- /cond_config.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mitdbg/treeline/fcc220ea54beb3c951c80b9a2815ecb4292ef0be/cond_config.toml -------------------------------------------------------------------------------- /db/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | db_impl_range.cc 3 | db_impl.cc 4 | db_impl.h 5 | format.h 6 | logger.cc 7 | logger.h 8 | manifest.cc 9 | manifest.h 10 | memtable.cc 11 | memtable.h 12 | options.cc 13 | options.h 14 | overflow_chain.cc 15 | overflow_chain.h 16 | page.cc 17 | page.h 18 | posix_logger.h 19 | slice.cc) 20 | -------------------------------------------------------------------------------- /db/format.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace tl { 6 | namespace format { 7 | 8 | // Numeric values in this file should not be changed because they are part of 9 | // TL's on-disk format. 10 | 11 | // Used to disambiguate between inserting/updating a record and deleting a 12 | // record (both of which are treated as "writes"). 13 | enum class WriteType : uint8_t { 14 | kWrite = 0, 15 | kDelete = 1 16 | }; 17 | inline constexpr uint8_t kMaxWriteType = static_cast(WriteType::kDelete); 18 | 19 | } // namespace format 20 | } // namespace tl 21 | -------------------------------------------------------------------------------- /db/logger.cc: -------------------------------------------------------------------------------- 1 | #include "logger.h" 2 | 3 | #include 4 | 5 | namespace tl { 6 | 7 | void Logger::Initialize(const std::filesystem::path& db_path) { 8 | Logger& logger = Instance(); 9 | logger.logger_ = 10 | std::make_unique(std::fopen((db_path / "LOG").c_str(), "a")); 11 | } 12 | 13 | void Logger::Shutdown() { 14 | Logger& logger = Instance(); 15 | if (logger.logger_ == nullptr) return; 16 | logger.logger_.reset(nullptr); 17 | } 18 | 19 | Logger::Logger() : logger_(nullptr) {} 20 | 21 | Logger& Logger::Instance() { 22 | static Logger logger; 23 | return logger; 24 | } 25 | 26 | void Logger::Log(const char* format, ...) { 27 | Logger& logger = Instance(); 28 | if (logger.logger_ == nullptr) return; 29 | 30 | std::va_list ap; 31 | va_start(ap, format); 32 | logger.logger_->Logv(format, ap); 33 | va_end(ap); 34 | } 35 | 36 | } // namespace tl 37 | -------------------------------------------------------------------------------- /db/logger.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "posix_logger.h" 7 | 8 | namespace tl { 9 | 10 | // A singleton logger class for debug and other informative logging. 11 | // The `Log()` method is thread-safe. `Initialize()` and `Shutdown()` are not 12 | // thread-safe and should not be called concurrently. 13 | class Logger { 14 | public: 15 | // Log a message using printf-style formatting. 16 | static void Log(const char* format, ...) __attribute__((__format__(__printf__, 1, 2))); 17 | 18 | // Must be called first to enable logging. If this method is never called, 19 | // `Log()` calls will effectively be no-ops. 20 | static void Initialize(const std::filesystem::path& db_path); 21 | 22 | // Should be called when the database is shut down to stop logging. 23 | static void Shutdown(); 24 | 25 | private: 26 | Logger(); 27 | static Logger& Instance(); 28 | 29 | std::unique_ptr logger_; 30 | }; 31 | 32 | } // namespace tl 33 | -------------------------------------------------------------------------------- /db/options.cc: -------------------------------------------------------------------------------- 1 | #include "treeline/options.h" 2 | 3 | #include 4 | 5 | #include "db/page.h" 6 | 7 | namespace tl { 8 | 9 | size_t KeyDistHints::records_per_page() const { 10 | assert(page_fill_pct > 0 && page_fill_pct <= 100); 11 | const double fill_pct = page_fill_pct / 100.0; 12 | return fill_pct * Page::NumRecordsThatFit(record_size, 2 * key_size); 13 | } 14 | 15 | size_t KeyDistHints::num_pages() const { 16 | if (num_keys == 0) return 1; 17 | size_t rec_per_page = records_per_page(); 18 | return (num_keys / rec_per_page) + ((num_keys % rec_per_page) != 0); 19 | } 20 | 21 | } // namespace tl 22 | -------------------------------------------------------------------------------- /db/options.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace tl { 6 | 7 | // Options used to instruct the database on how to handle memtable flushing and 8 | // deferral. 9 | struct MemTableOptions { 10 | // The size above which this memtable will be flushed, in bytes. 11 | size_t flush_threshold = 64 * 1024 * 1024; 12 | 13 | // The maximum number of past deferrals for which this memtable will 14 | // remember the exact number of past deferrals. 15 | // 16 | // E.g. if set to 0, this memtable will lump all records inserted via 17 | // deferrals together, 18 | // if set to 1, this memtable will distinguish between records deferred 19 | // once, but lump all records deferred at least twice together, 20 | // etc. 21 | size_t deferral_granularity = 0; 22 | }; 23 | 24 | struct FlushOptions { 25 | // Disable I/O deferral during a MemTable flush. 26 | bool disable_deferred_io = false; 27 | 28 | // The minimum size in bytes of a memtable batch associated with a certain page 29 | // necessary to actually copy the entries out during this memtable flush. 30 | size_t deferred_io_batch_size = 1; 31 | 32 | // The maximum number of prior deferrals that this flush will accomodate (i.e. 33 | // all pages deferred at least that many times will not be deferred again in 34 | // this flush). 35 | size_t deferred_io_max_deferrals = 0; 36 | }; 37 | 38 | } // namespace tl 39 | -------------------------------------------------------------------------------- /db/slice.cc: -------------------------------------------------------------------------------- 1 | #include "treeline/slice.h" 2 | 3 | namespace std { 4 | 5 | ostream& operator<<(ostream& os, const tl::Slice& slice) { 6 | std::string s; 7 | for (int i = 0; i < slice.size(); ++i) { 8 | s += "("; 9 | s += std::to_string(static_cast(slice.data()[i])); 10 | s += ")"; 11 | } 12 | os << s; 13 | return os; 14 | } 15 | 16 | } // namespace std 17 | -------------------------------------------------------------------------------- /include/treeline/record_batch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | namespace tl { 6 | 7 | // Represents a complete record stored by TL. This is a convenience class used 8 | // to return range scan results. 9 | class Record { 10 | public: 11 | Record(std::string key, std::string value) 12 | : key_(std::move(key)), value_(std::move(value)) {} 13 | 14 | const std::string& key() const { return key_; } 15 | const std::string& value() const { return value_; } 16 | 17 | std::string&& ExtractKey() && { return std::move(key_); } 18 | std::string&& ExtractValue() && { return std::move(value_); } 19 | 20 | private: 21 | std::string key_, value_; 22 | }; 23 | 24 | using RecordBatch = std::vector; 25 | 26 | } // namespace tl 27 | -------------------------------------------------------------------------------- /model/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | btree_model.h 3 | btree_model.cc 4 | model.h 5 | model.cc) 6 | -------------------------------------------------------------------------------- /page_grouping/circular_page_buffer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../bufmgr/page_memory_allocator.h" 4 | #include "persist/page.h" 5 | 6 | namespace tl { 7 | namespace pg { 8 | 9 | class CircularPageBuffer { 10 | public: 11 | CircularPageBuffer(size_t num_pages) 12 | : buf_(PageMemoryAllocator::Allocate(num_pages)), 13 | base_(buf_.get()), 14 | num_pages_(num_pages), 15 | head_idx_(0), 16 | tail_idx_(0), 17 | free_pages_(num_pages) {} 18 | 19 | size_t NumFreePages() const { return free_pages_; } 20 | size_t NumAllocatedPages() const { return num_pages_ - free_pages_; } 21 | 22 | void* Allocate() { 23 | assert(free_pages_ > 0); 24 | void* to_return = base_ + (head_idx_ * pg::Page::kSize); 25 | ++head_idx_; 26 | --free_pages_; 27 | if (head_idx_ >= num_pages_) { 28 | head_idx_ = 0; 29 | } 30 | return to_return; 31 | } 32 | 33 | void Free() { 34 | assert(NumAllocatedPages() > 0); 35 | assert(tail_idx_ != head_idx_); 36 | ++tail_idx_; 37 | ++free_pages_; 38 | if (tail_idx_ >= num_pages_) { 39 | tail_idx_ = 0; 40 | } 41 | } 42 | 43 | private: 44 | PageBuffer buf_; 45 | char* base_; 46 | size_t num_pages_; 47 | // Pages are allocated at `head_idx_` and are freed at `tail_idx_`. 48 | // Having `head_idx_ == tail_idx_` means that no pages are allocated. 49 | size_t head_idx_, tail_idx_; 50 | size_t free_pages_; 51 | }; 52 | 53 | } // namespace pg 54 | } // namespace tl 55 | -------------------------------------------------------------------------------- /page_grouping/config.cc: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | 3 | DEFINE_string(db_path, "", "The path where the database(s) should be stored."); 4 | DEFINE_bool(disable_segments, false, 5 | "If set, the initial bulk load will not create segments."); 6 | DEFINE_uint32(records_per_page_goal, 44, 7 | "Aim to put this many records on a page."); 8 | DEFINE_double( 9 | records_per_page_epsilon, 5, 10 | "The number of records on a page can vary by +/- two times this value."); 11 | DEFINE_uint32(bg_threads, 16, 12 | "The number of background threads to use (for I/O)."); 13 | DEFINE_bool(use_memory_based_io, false, 14 | "Set to disable direct I/O AND to disable synchronous writes. This " 15 | "should NOT be set when running actual performance benchmarks."); 16 | DEFINE_uint32(write_batch_size, 1000000, 17 | "The number of records to batch before initiating a write."); 18 | -------------------------------------------------------------------------------- /page_grouping/config.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "gflags/gflags.h" 4 | 5 | DECLARE_string(db_path); 6 | DECLARE_bool(disable_segments); 7 | DECLARE_uint32(records_per_page_goal); 8 | DECLARE_double(records_per_page_epsilon); 9 | DECLARE_uint32(bg_threads); 10 | DECLARE_bool(use_memory_based_io); 11 | DECLARE_uint32(write_batch_size); 12 | -------------------------------------------------------------------------------- /page_grouping/debug/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file contains targets used for page grouping debugging. 2 | 3 | add_executable(pg_standalone 4 | pg_standalone.cc 5 | ../../bench/common/load_data.cc 6 | ../../bench/common/load_data.h 7 | ) 8 | target_link_libraries(pg_standalone PRIVATE pg gflags) 9 | 10 | add_executable(pg_read pg_read.cc) 11 | target_link_libraries(pg_read PRIVATE gflags) 12 | 13 | add_executable(pg_read2 pg_read2.cc) 14 | target_link_libraries(pg_read2 PRIVATE gflags) 15 | 16 | add_executable(pg_check pg_check.cc) 17 | target_link_libraries(pg_check PRIVATE pg gflags Threads::Threads) 18 | -------------------------------------------------------------------------------- /page_grouping/free_list.cc: -------------------------------------------------------------------------------- 1 | #include "free_list.h" 2 | 3 | #include 4 | 5 | #include "segment_builder.h" 6 | 7 | namespace tl { 8 | namespace pg { 9 | 10 | FreeList::FreeList() 11 | : bytes_allocated_(0), 12 | list_(TrackingAllocator(bytes_allocated_)) { 13 | list_.resize(SegmentBuilder::SegmentPageCounts().size()); 14 | } 15 | 16 | void FreeList::Add(SegmentId id) { 17 | std::unique_lock lock(mutex_); 18 | AddImpl(id); 19 | } 20 | 21 | std::optional FreeList::Get(const size_t page_count) { 22 | std::unique_lock lock(mutex_); 23 | const auto it = SegmentBuilder::PageCountToSegment().find(page_count); 24 | assert(it != SegmentBuilder::PageCountToSegment().end()); 25 | const size_t file_id = it->second; 26 | if (list_[file_id].empty()) { 27 | // No free segments. The caller should allocate a new one. 28 | return std::optional(); 29 | } 30 | const SegmentId free = list_[file_id].front(); 31 | list_[file_id].pop(); 32 | return free; 33 | } 34 | 35 | void FreeList::AddBatch(const std::vector& ids) { 36 | std::unique_lock lock(mutex_); 37 | for (const auto& id : ids) { 38 | AddImpl(id); 39 | } 40 | } 41 | 42 | void FreeList::AddImpl(SegmentId id) { list_[id.GetFileId()].push(id); } 43 | 44 | uint64_t FreeList::GetSizeFootprint() const { 45 | std::unique_lock lock(mutex_); 46 | return bytes_allocated_ + sizeof(*this); 47 | } 48 | 49 | uint64_t FreeList::GetNumEntries() const { 50 | std::unique_lock lock(mutex_); 51 | uint64_t total = 0; 52 | for (const auto& l : list_) { 53 | total += l.size(); 54 | } 55 | return total; 56 | } 57 | 58 | } // namespace pg 59 | } // namespace tl 60 | -------------------------------------------------------------------------------- /page_grouping/free_list.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "../util/tracking_allocator.h" 11 | #include "persist/segment_id.h" 12 | 13 | namespace tl { 14 | namespace pg { 15 | 16 | // Keeps track of available locations on disk for segments of different sizes. 17 | // This class' methods are thread-safe. 18 | class FreeList { 19 | public: 20 | FreeList(); 21 | void Add(SegmentId id); 22 | void AddBatch(const std::vector& ids); 23 | std::optional Get(size_t page_count); 24 | 25 | uint64_t GetSizeFootprint() const; 26 | uint64_t GetNumEntries() const; 27 | 28 | private: 29 | void AddImpl(SegmentId id); 30 | 31 | mutable std::mutex mutex_; 32 | using SegmentList = 33 | std::queue>>; 35 | uint64_t bytes_allocated_; 36 | std::vector>> 38 | list_; 39 | }; 40 | 41 | } // namespace pg 42 | } // namespace tl 43 | -------------------------------------------------------------------------------- /page_grouping/key.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "treeline/pg_db.h" 8 | #include "treeline/slice.h" 9 | #include "plr/data.h" 10 | 11 | namespace tl { 12 | namespace pg { 13 | 14 | // Returns an integer in the range [0, page_count) according to `model`. 15 | inline size_t PageForKey(const Key base_key, const plr::Line64& model, 16 | const size_t page_count, const Key candidate) { 17 | return std::min(page_count - 1, static_cast(std::max( 18 | 0.0, model(candidate - base_key)))); 19 | } 20 | 21 | // Returns the smallest key `k` such that 22 | // `PageForKey(base_key, model, page_count, k) == page_idx`. 23 | // 24 | // `model` should map keys to page indices. `model_inv` should be the result of 25 | // `model.Invert()`. 26 | Key FindLowerBoundary(const Key base_key, const plr::Line64& model, 27 | const size_t page_count, const plr::Line64& model_inv, 28 | size_t page_idx); 29 | 30 | } // namespace pg 31 | } // namespace tl 32 | -------------------------------------------------------------------------------- /page_grouping/persist/segment_id.cc: -------------------------------------------------------------------------------- 1 | #include "segment_id.h" 2 | 3 | namespace tl { 4 | namespace pg { 5 | 6 | // Most significant 4 bits reserved for the file (only 3 are actually used for 7 | // the file ID; the most significant bit is reserved). 8 | size_t SegmentId::offset_bits_ = 60; 9 | size_t SegmentId::file_mask_ = 15ULL << SegmentId::offset_bits_; 10 | size_t SegmentId::offset_mask_ = ~(SegmentId::file_mask_); 11 | 12 | } // namespace pg 13 | } // namespace tl 14 | 15 | namespace std { 16 | 17 | ostream& operator<<(ostream& os, const tl::pg::SegmentId& id) { 18 | os << id.GetFileId() << "-" << id.GetOffset(); 19 | return os; 20 | } 21 | 22 | } // namespace std 23 | -------------------------------------------------------------------------------- /page_grouping/persist/segment_wrap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "treeline/pg_db.h" 6 | #include "page.h" 7 | 8 | namespace tl { 9 | namespace pg { 10 | 11 | // A thin wrapper around a group of pages (a segment) to provide utility methods 12 | // for the segment as a whole. 13 | class SegmentWrap { 14 | public: 15 | SegmentWrap(void* data, const size_t pages_in_segment); 16 | 17 | uint32_t GetSequenceNumber() const; 18 | void SetSequenceNumber(uint32_t sequence); 19 | 20 | bool CheckChecksum() const; 21 | void ComputeAndSetChecksum(); 22 | 23 | // Sets all overflow values to "invalid" (indicating no overflow). 24 | void ClearAllOverflows(); 25 | 26 | // Returns true if there exists at least one page in the segment that has an 27 | // overflow. 28 | bool HasOverflow() const; 29 | 30 | // Returns the number of pages in this segment that have an overflow. 31 | size_t NumOverflows() const; 32 | 33 | template 34 | void ForEachPage(const Callable& callable) { 35 | for (size_t i = 0; i < pages_in_segment_; ++i) { 36 | callable(i, PageAtIndex(i)); 37 | } 38 | } 39 | 40 | // Retrieve the encoded "lower"/"upper" boundaries in the segment. 41 | Key EncodedBaseKey() const; 42 | Key EncodedUpperKey() const; 43 | 44 | private: 45 | Page PageAtIndex(size_t index) const; 46 | uint32_t ComputeChecksum() const; 47 | 48 | void* data_; 49 | size_t pages_in_segment_; 50 | }; 51 | 52 | } // namespace pg 53 | } // namespace tl 54 | -------------------------------------------------------------------------------- /page_grouping/pg_stats.cc: -------------------------------------------------------------------------------- 1 | #include "treeline/pg_stats.h" 2 | 3 | namespace tl { 4 | namespace pg { 5 | 6 | std::mutex PageGroupedDBStats::class_mutex_; 7 | PageGroupedDBStats PageGroupedDBStats::global_; 8 | 9 | PageGroupedDBStats::PageGroupedDBStats() { Reset(); } 10 | 11 | void PageGroupedDBStats::PostToGlobal() const { 12 | std::unique_lock lock(class_mutex_); 13 | global_.cache_hits_ += cache_hits_; 14 | global_.cache_misses_ += cache_misses_; 15 | global_.cache_clean_evictions_ += cache_clean_evictions_; 16 | global_.cache_dirty_evictions_ += cache_dirty_evictions_; 17 | 18 | global_.overflows_created_ += overflows_created_; 19 | global_.rewrites_ += rewrites_; 20 | global_.rewrite_input_pages_ += rewrite_input_pages_; 21 | global_.rewrite_output_pages_ += rewrite_output_pages_; 22 | 23 | global_.segments_ = segments_; 24 | global_.free_list_entries_ += free_list_entries_; 25 | global_.free_list_bytes_ += free_list_bytes_; 26 | global_.segment_index_bytes_ += segment_index_bytes_; 27 | global_.lock_manager_bytes_ += lock_manager_bytes_; 28 | global_.cache_bytes_ += cache_bytes_; 29 | 30 | global_.overfetched_pages_ += overfetched_pages_; 31 | } 32 | 33 | void PageGroupedDBStats::Reset() { 34 | cache_hits_ = 0; 35 | cache_misses_ = 0; 36 | cache_clean_evictions_ = 0; 37 | cache_dirty_evictions_ = 0; 38 | 39 | overflows_created_ = 0; 40 | rewrites_ = 0; 41 | rewrite_input_pages_ = 0; 42 | rewrite_output_pages_ = 0; 43 | 44 | segments_ = 0; 45 | free_list_entries_ = 0; 46 | free_list_bytes_ = 0; 47 | segment_index_bytes_ = 0; 48 | lock_manager_bytes_ = 0; 49 | cache_bytes_ = 0; 50 | 51 | overfetched_pages_ = 0; 52 | } 53 | 54 | } // namespace pg 55 | } // namespace tl 56 | -------------------------------------------------------------------------------- /page_grouping/plr/plr.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "data.h" 6 | 7 | namespace tl { 8 | namespace pg { 9 | namespace plr { 10 | 11 | // Common interface used by our PLR algorithms. 12 | template 13 | class PLRBuilder { 14 | public: 15 | virtual ~PLRBuilder() = default; 16 | 17 | // Extend the segment to include `p`. If the return value is not empty, it 18 | // will include a bounded line segment that is part of the piecewise linear 19 | // regression. 20 | virtual std::optional> Offer(const Point& p) = 0; 21 | 22 | // Retrieve the final built line. 23 | virtual std::optional> Finish() const = 0; 24 | }; 25 | 26 | } // namespace plr 27 | } // namespace pg 28 | } // namespace tl 29 | -------------------------------------------------------------------------------- /page_grouping/rand_exp_backoff.cc: -------------------------------------------------------------------------------- 1 | #include "rand_exp_backoff.h" 2 | 3 | namespace tl { 4 | namespace pg { 5 | 6 | inline thread_local std::mt19937 RandExpBackoff::prng_{}; 7 | 8 | } // namespace pg 9 | } // namespace tl 10 | -------------------------------------------------------------------------------- /page_grouping/rand_exp_backoff.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace tl { 8 | namespace pg { 9 | 10 | // Used for implementing randomized exponential backoff as a spin-wait. 11 | class RandExpBackoff { 12 | public: 13 | explicit RandExpBackoff(uint32_t saturate_at) 14 | : attempts_(0), saturate_at_(saturate_at) {} 15 | 16 | // Call this method to spin wait. The maximum number of spin waits will 17 | // increase exponentially as a function of how many times this method is 18 | // called, up to `saturate_at_` times. 19 | void Wait() { 20 | if (attempts_ < saturate_at_) { 21 | ++attempts_; 22 | } 23 | 24 | const uint32_t max_spin_cycles = 10 * (1UL << attempts_); 25 | std::uniform_int_distribution dist(1, max_spin_cycles); 26 | 27 | uint32_t spin_for = dist(prng_); 28 | while (spin_for > 0) { 29 | _mm_pause(); 30 | --spin_for; 31 | } 32 | } 33 | 34 | void Reset() { attempts_ = 0; } 35 | 36 | private: 37 | static thread_local std::mt19937 prng_; 38 | uint32_t attempts_; 39 | uint32_t saturate_at_; 40 | }; 41 | 42 | } // namespace pg 43 | } // namespace tl 44 | -------------------------------------------------------------------------------- /page_grouping/tools/flatten.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "gflags/gflags.h" 6 | #include "treeline/pg_db.h" 7 | 8 | DEFINE_string(db_path, "", "Path to the database directory."); 9 | DEFINE_uint64( 10 | start_key, 1, 11 | "The lower boundary of the key space to pass to `FlattenRange()`."); 12 | DEFINE_uint64(end_key, std::numeric_limits::max(), 13 | "The upper boundary (exclusive) of the key space to pass to " 14 | "`FlattenRange()`."); 15 | DEFINE_uint32(goal, 44, 16 | "Passed to PageGroupedDBOptions::records_per_page_goal."); 17 | DEFINE_double(epsilon, 5, 18 | "Passed to PageGroupedDBOptions::records_per_page_epsilon."); 19 | 20 | using namespace tl; 21 | using namespace tl::pg; 22 | 23 | int main(int argc, char* argv[]) { 24 | gflags::SetUsageMessage("Runs FlattenRange() on a page-grouped database."); 25 | gflags::ParseCommandLineFlags(&argc, &argv, /*remove_flags=*/true); 26 | if (FLAGS_db_path.empty()) { 27 | std::cerr << "ERROR: Must provide a database path." << std::endl; 28 | return 1; 29 | } 30 | 31 | PageGroupedDBOptions options; 32 | options.use_segments = true; 33 | options.records_per_page_goal = FLAGS_goal; 34 | options.records_per_page_epsilon = FLAGS_epsilon; 35 | options.use_memory_based_io = true; 36 | 37 | std::cerr << "Flattening PGTreeLine DB at: " << FLAGS_db_path << std::endl; 38 | PageGroupedDB* db = nullptr; 39 | PageGroupedDB::Open(options, FLAGS_db_path, &db); 40 | assert(db != nullptr); 41 | const auto status = db->FlattenRange(FLAGS_start_key, FLAGS_end_key); 42 | if (!status.ok()) { 43 | throw std::runtime_error(status.ToString()); 44 | } 45 | delete db; 46 | 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /record_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(record_cache_sources 2 | record_cache_entry.cc 3 | record_cache_entry.h 4 | record_cache.cc 5 | record_cache.h 6 | ) 7 | 8 | target_sources(treeline PRIVATE ${record_cache_sources}) 9 | target_sources(pg_treeline PRIVATE ${record_cache_sources}) 10 | -------------------------------------------------------------------------------- /scripts/autotuning/phased_512B_B_A_B.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 512 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: linspace 7 | start_key: 0 8 | step_size: 1 9 | 10 | run: 11 | - num_requests: 5000000 12 | read: 13 | proportion_pct: 95 14 | distribution: 15 | type: zipfian 16 | theta: 0.99 17 | update: 18 | proportion_pct: 5 19 | distribution: 20 | type: zipfian 21 | theta: 0.99 22 | 23 | - num_requests: 5000000 24 | read: 25 | proportion_pct: 50 26 | distribution: 27 | type: zipfian 28 | theta: 0.99 29 | update: 30 | proportion_pct: 50 31 | distribution: 32 | type: zipfian 33 | theta: 0.99 34 | 35 | - num_requests: 10000000 36 | read: 37 | proportion_pct: 95 38 | distribution: 39 | type: zipfian 40 | theta: 0.99 41 | update: 42 | proportion_pct: 5 43 | distribution: 44 | type: zipfian 45 | theta: 0.99 46 | -------------------------------------------------------------------------------- /scripts/autotuning/preload.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc 6 | source ../experiment_config.sh 7 | 8 | if [ -z $2 ]; then 9 | >&2 echo "Usage: $0 " 10 | exit 1 11 | fi 12 | 13 | checkpoint_name=$1 14 | preload_file=$2 15 | shift 2 16 | 17 | full_checkpoint_path=$DB_CHECKPOINT_PATH/$checkpoint_name 18 | 19 | # Check if the checkpoint already exists. If so, we do not need to rerun. 20 | if [ -d "$full_checkpoint_path" ]; then 21 | >&2 echo "Checkpoint $checkpoint_name already exists. No need to recreate." 22 | exit 0 23 | fi 24 | 25 | mkdir -p $DB_CHECKPOINT_PATH 26 | 27 | # RocksDB - Important to use 64 MiB memtables to ensure the emitted sstables 28 | # are small too. 29 | ../../build/bench/run_custom \ 30 | --db=rocksdb \ 31 | --db_path=$full_checkpoint_path \ 32 | --bg_threads=16 \ 33 | --bypass_wal=true \ 34 | --memtable_size_mib=64 \ 35 | --workload_config=$preload_file \ 36 | --use_direct_io=true \ 37 | --seed=$SEED \ 38 | --verbose 39 | 40 | # LLSM - Use a larger memtable to help the load run faster. 41 | ../../build/bench/run_custom \ 42 | --db=llsm \ 43 | --db_path=$full_checkpoint_path \ 44 | --bg_threads=16 \ 45 | --bypass_wal=true \ 46 | --tl_page_fill_pct=50 \ 47 | --memtable_size_mib=2048 \ 48 | --workload_config=$preload_file \ 49 | --seed=$SEED \ 50 | --verbose 51 | -------------------------------------------------------------------------------- /scripts/autotuning/preload512.yml: -------------------------------------------------------------------------------- 1 | # Should be the same as bench/workload_configs/phased_512B_A_B_A.yml 2 | record_size_bytes: 512 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: linspace 8 | start_key: 0 9 | step_size: 1 10 | 11 | run: 12 | - num_requests: 10000000 13 | update: 14 | proportion_pct: 100 15 | distribution: 16 | type: uniform 17 | -------------------------------------------------------------------------------- /scripts/autotuning/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc 6 | source ../experiment_config.sh 7 | 8 | if [ -z $1 ]; then 9 | >&2 echo "Usage: $0 [other args passed to run_custom]" 10 | exit 1 11 | fi 12 | 13 | checkpoint_name=$1 14 | shift 1 15 | 16 | # Evaluates any environment variables in this script's arguments. This script 17 | # should only be run on trusted input. 18 | orig_args=$@ 19 | args=() 20 | for val in "${orig_args[@]}"; do 21 | args+=$(eval "echo $val") 22 | done 23 | 24 | full_checkpoint_path=$DB_CHECKPOINT_PATH/$checkpoint_name 25 | 26 | rm -rf $DB_PATH 27 | cp -r $full_checkpoint_path $DB_PATH 28 | sync $DB_PATH 29 | 30 | set +e 31 | iostat -o JSON -d -y 1 > $COND_OUT/iostat.json & 32 | iostat_pid=$! 33 | 34 | ../../build/bench/run_custom \ 35 | --verbose \ 36 | --db_path=$DB_PATH \ 37 | --seed=$SEED \ 38 | --skip_load \ 39 | --output_path=$COND_OUT \ 40 | $args \ 41 | > $COND_OUT/results.csv 42 | 43 | cp $DB_PATH/llsm/LOG $COND_OUT/llsm.log 44 | cp $DB_PATH/rocksdb/LOG $COND_OUT/rocksdb.log 45 | 46 | kill -s SIGINT -- $iostat_pid 47 | wait 48 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middle-20.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 20 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 80 23 | distribution: 24 | type: uniform 25 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middle-50.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 50 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 50 23 | distribution: 24 | type: uniform 25 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middle-80.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 80 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 20 23 | distribution: 24 | type: uniform 25 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middle-95.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 95 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 5 23 | distribution: 24 | type: uniform 25 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middlescan-20.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 20 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 64 23 | distribution: 24 | type: uniform 25 | scan: 26 | proportion_pct: 16 27 | max_length: 100 28 | distribution: 29 | type: uniform 30 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middlescan-50.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 50 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 40 23 | distribution: 24 | type: uniform 25 | scan: 26 | proportion_pct: 10 27 | max_length: 100 28 | distribution: 29 | type: uniform 30 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/middlescan-80.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 80 14 | distribution: 15 | type: hotspot 16 | range_min: 0 17 | range_max: 2000000000 18 | hot_proportion_pct: 90 19 | hot_range_min: 990000000 # 990,000,000 20 | hot_range_max: 1010000000 # 1,010,000,000 21 | read: 22 | proportion_pct: 16 23 | distribution: 24 | type: uniform 25 | scan: 26 | proportion_pct: 4 27 | max_length: 100 28 | distribution: 29 | type: uniform 30 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/setup.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | # Should not exceed 2^31-1 because ALEX uses 32-bit signed ints internally. 9 | range_max: 2000000000 10 | 11 | # Ensures that RocksDB's LSM tree has multiple levels. 12 | run: 13 | - num_requests: 10000000 14 | update: 15 | proportion_pct: 100 16 | distribution: 17 | type: uniform 18 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniform-100.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 100 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniform-20.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 20 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 80 20 | distribution: 21 | type: uniform 22 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniform-50.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 50 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 50 20 | distribution: 21 | type: uniform 22 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniform-80.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 80 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 20 20 | distribution: 21 | type: uniform 22 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniform-95.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 95 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 5 20 | distribution: 21 | type: uniform 22 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniformscan-20.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 20 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 64 20 | distribution: 21 | type: uniform 22 | scan: 23 | proportion_pct: 16 24 | max_length: 100 25 | distribution: 26 | type: uniform 27 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniformscan-50.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 50 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 40 20 | distribution: 21 | type: uniform 22 | scan: 23 | proportion_pct: 10 24 | max_length: 100 25 | distribution: 26 | type: uniform 27 | -------------------------------------------------------------------------------- /scripts/e2e_custom/insert/workloads/uniformscan-80.yml: -------------------------------------------------------------------------------- 1 | record_size_bytes: 64 2 | 3 | load: 4 | num_records: 20000000 5 | distribution: 6 | type: uniform 7 | range_min: 0 8 | range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 80 14 | distribution: 15 | type: uniform 16 | range_min: 0 17 | range_max: 2000000000 18 | read: 19 | proportion_pct: 16 20 | distribution: 21 | type: uniform 22 | scan: 23 | proportion_pct: 4 24 | max_length: 100 25 | distribution: 26 | type: uniform 27 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/plot_common.py: -------------------------------------------------------------------------------- 1 | COLORS = { 2 | "llsm": "#023623", 3 | "rocksdb": "#08B676", 4 | "rocksdb-bf": "#59FBC1", 5 | } 6 | 7 | DATASET_MAP = { 8 | "ycsb-synthetic-64": "Synthetic", 9 | "ycsb-amzn-64": "Amazon", 10 | "ycsb-osm-64": "OSM", 11 | } 12 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/a.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 50 12 | distribution: 13 | type: zipfian 14 | theta: 0.99 15 | update: 16 | proportion_pct: 50 17 | distribution: 18 | type: zipfian 19 | theta: 0.99 20 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/b.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 95 12 | distribution: 13 | type: zipfian 14 | theta: 0.99 15 | update: 16 | proportion_pct: 5 17 | distribution: 18 | type: zipfian 19 | theta: 0.99 20 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/c.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 100 12 | distribution: 13 | type: zipfian 14 | theta: 0.99 15 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/d.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 95 12 | distribution: 13 | type: latest 14 | theta: 0.99 15 | insert: 16 | proportion_pct: 5 17 | distribution: 18 | type: uniform 19 | range_min: 0 20 | range_max: 2000000000 21 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/e.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 100000 10 | scan: 11 | proportion_pct: 95 12 | max_length: 100 13 | distribution: 14 | type: zipfian 15 | theta: 0.99 16 | insert: 17 | proportion_pct: 5 18 | distribution: 19 | type: uniform 20 | range_min: 0 21 | range_max: 2000000000 22 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/f.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 50 12 | distribution: 13 | type: zipfian 14 | theta: 0.99 15 | readmodifywrite: 16 | proportion_pct: 50 17 | distribution: 18 | type: zipfian 19 | theta: 0.99 20 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/setup.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | # Should not exceed 2^31-1 because ALEX uses 32-bit signed ints internally. 7 | range_max: 2000000000 8 | 9 | # Ensures that RocksDB's LSM tree has multiple levels. 10 | run: 11 | - num_requests: 10000000 12 | update: 13 | proportion_pct: 100 14 | distribution: 15 | type: uniform 16 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_a.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 50 12 | distribution: 13 | type: uniform 14 | update: 15 | proportion_pct: 50 16 | distribution: 17 | type: uniform 18 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_b.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 95 12 | distribution: 13 | type: uniform 14 | update: 15 | proportion_pct: 5 16 | distribution: 17 | type: uniform 18 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_c.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_d.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 95 12 | distribution: 13 | type: uniform 14 | insert: 15 | proportion_pct: 5 16 | distribution: 17 | type: uniform 18 | range_min: 0 19 | range_max: 2000000000 20 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_e.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 100000 10 | scan: 11 | proportion_pct: 95 12 | max_length: 100 13 | distribution: 14 | type: uniform 15 | insert: 16 | proportion_pct: 5 17 | distribution: 18 | type: uniform 19 | range_min: 0 20 | range_max: 2000000000 21 | -------------------------------------------------------------------------------- /scripts/e2e_custom/ycsb/workloads/un_f.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | read: 11 | proportion_pct: 50 12 | distribution: 13 | type: uniform 14 | readmodifywrite: 15 | proportion_pct: 50 16 | distribution: 17 | type: uniform 18 | -------------------------------------------------------------------------------- /scripts/e2e_perf/COND: -------------------------------------------------------------------------------- 1 | # Ensures the benchmark executables have been compiled. 2 | run_command( 3 | name="build_benchmarks", 4 | run="./build_benchmarks.sh", 5 | ) 6 | 7 | # An alias that runs all LLSM-related experiments. 8 | group( 9 | name="llsm", 10 | deps=[ 11 | "//scripts/e2e_perf/synth_write:llsm", 12 | "//scripts/e2e_perf/ycsb:llsm", 13 | ], 14 | ) 15 | 16 | # An alias that runs all RocksDB-related experiments. 17 | group( 18 | name="rocksdb", 19 | deps=[ 20 | "//scripts/e2e_perf/synth_write:rocksdb", 21 | "//scripts/e2e_perf/ycsb:rocksdb", 22 | ], 23 | ) 24 | 25 | # Combines all the results into one csv file. 26 | run_command( 27 | name="summarize_overall", 28 | run="python3 combine.py", 29 | deps=[ 30 | "//scripts/e2e_perf/synth_write:summarize_overall", 31 | "//scripts/e2e_perf/ycsb:summarize_overall", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /scripts/e2e_perf/build_benchmarks.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc/../.. 6 | 7 | mkdir -p build && cd build 8 | 9 | cmake -DCMAKE_BUILD_TYPE=Release -DLLSM_BUILD_BENCHMARKS=ON .. 10 | make -j synth_write ycsb 11 | -------------------------------------------------------------------------------- /scripts/e2e_perf/synth_write/run_synth_write.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc/.. 6 | source ../experiment_config.sh 7 | cd ../../build 8 | 9 | # The `synth_write` benchmark executable expects the database output directory 10 | # to not exist. 11 | rm -rf $DB_PATH 12 | 13 | # The experiment results will be saved to `results.csv`. 14 | ./bench/synth_write \ 15 | --db_path=$DB_PATH \ 16 | $@ \ 17 | > $COND_OUT/results.csv 18 | -------------------------------------------------------------------------------- /scripts/e2e_perf/ycsb/preload.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | if [ -z $2 ]; then 5 | echo "Usage: $0 [...other args passed to ./bench/ycsb]" 6 | exit 1 7 | fi 8 | 9 | script_loc=$(cd $(dirname $0) && pwd -P) 10 | cd $script_loc/.. 11 | source ../experiment_config.sh 12 | 13 | checkpoint_name=$1 14 | full_checkpoint_path=$DB_CHECKPOINT_PATH/$checkpoint_name 15 | trace_name=$2 16 | shift 2 17 | 18 | # Check if the checkpoint already exists. If so, we do not need to rerun. 19 | if [ -d "$full_checkpoint_path" ]; then 20 | >&2 echo "Checkpoint $checkpoint_name already exists. No need to recreate." 21 | exit 0 22 | fi 23 | 24 | mkdir -p $DB_CHECKPOINT_PATH 25 | 26 | cd ../../build 27 | ./bench/ycsb \ 28 | --db=all \ 29 | --db_path=$full_checkpoint_path \ 30 | --bg_threads=16 \ 31 | --bypass_wal=true \ 32 | --tl_page_fill_pct=50 \ 33 | --memtable_size_mib=64 \ 34 | --load_path=$YCSB_TRACE_PATH/$trace_name \ 35 | $@ 36 | -------------------------------------------------------------------------------- /scripts/e2e_perf/ycsb/run_ycsb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | if [ -z $2 ]; then 5 | echo "Usage: $0 [...other args passed to ./bench/ycsb]" 6 | exit 1 7 | fi 8 | 9 | script_loc=$(cd $(dirname $0) && pwd -P) 10 | cd $script_loc/.. 11 | source ../experiment_config.sh 12 | 13 | db=$1 14 | checkpoint_name=$2 15 | shift 2 16 | 17 | full_checkpoint_path=$DB_CHECKPOINT_PATH/$checkpoint_name 18 | 19 | rm -rf $DB_PATH && mkdir -p $DB_PATH 20 | cp -r $full_checkpoint_path/$db $DB_PATH 21 | 22 | # Evaluates any environment variables in this script's arguments. This script 23 | # should only be run on trusted input. 24 | orig_args=$@ 25 | args=() 26 | for val in "${orig_args[@]}"; do 27 | args+=$(eval "echo $val") 28 | done 29 | 30 | cd ../../build 31 | ./bench/ycsb \ 32 | --db=$db \ 33 | --db_path=$DB_PATH \ 34 | --verbose \ 35 | $args \ 36 | > $COND_OUT/ycsb.csv 37 | -------------------------------------------------------------------------------- /scripts/experiment_config_example.sh: -------------------------------------------------------------------------------- 1 | # The variables in this file are used when running LLSM's automated 2 | # experiments. Their values are specific to the environment where you run the 3 | # experiments, so you need to set them explicitly. 4 | # 5 | # To set the variables, make a copy of this configuration file and name it 6 | # `experiment_config.sh`. Then, set the variables below in that file as needed 7 | # for your own experimental setup. Note that the paths should be absolute. 8 | 9 | # Path where the YCSB traces are stored. 10 | YCSB_TRACE_PATH=/home/$USER/datasets 11 | 12 | # Path where preloaded databases should be stored. 13 | DB_CHECKPOINT_PATH=/flash1/$USER/llsm-checkpoint 14 | 15 | # Path where the database(s) should be stored. This path will dictate the 16 | # underlying storage device that is used to store the database, which will 17 | # affect the experimental results. 18 | DB_PATH=/flash1/$USER/llsm 19 | 20 | # The PRNG seed to use for all experiments (used to ensure reproducibility). 21 | SEED=42 22 | 23 | # Path to third-party datasets. 24 | TP_DATASET_PATH=/home/$USER/datasets 25 | -------------------------------------------------------------------------------- /scripts/figures/format_factor.py: -------------------------------------------------------------------------------- 1 | import conductor.lib as cond 2 | import pandas as pd 3 | 4 | 5 | def build_table(data, workload): 6 | df = data[data["workload"] == workload].copy() 7 | df["read_gib"] = df["phys_read_kb"] / 1024 / 1024 8 | df["write_gib"] = df["phys_written_kb"] / 1024 / 1024 9 | df = df[["read_gib", "write_gib", "krequests_per_s"]] 10 | df["speedup_over_prev"] = ( 11 | df["krequests_per_s"].rolling(2).apply(lambda x: x.iloc[1] / x.iloc[0]) 12 | ) 13 | df.insert(0, "name", ["TreeLine Base", "+ Record Cache", "+ Page Grouping"]) 14 | return df 15 | 16 | 17 | def main(): 18 | deps = cond.get_deps_paths() 19 | out_dir = cond.get_output_path() 20 | results = pd.read_csv(deps[0] / "all_results.csv") 21 | 22 | point = build_table(results, "a") 23 | scan = build_table(results, "scan_only") 24 | columns = [ 25 | "name", 26 | "read_gib", 27 | "write_gib", 28 | "krequests_per_s", 29 | "speedup_over_prev", 30 | ] 31 | 32 | with open(out_dir / "point.tex", "w") as file: 33 | point.to_latex( 34 | buf=file, 35 | columns=columns, 36 | na_rep="---", 37 | float_format="{:.2f}".format, 38 | index=False, 39 | header=False, 40 | ) 41 | 42 | with open(out_dir / "scan.tex", "w") as file: 43 | scan.to_latex( 44 | buf=file, 45 | columns=columns, 46 | na_rep="---", 47 | float_format="{:.2f}".format, 48 | index=False, 49 | header=False, 50 | ) 51 | 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /scripts/figures/plot_cdf.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc 6 | source ../experiment_config.sh 7 | 8 | # Evaluates any environment variables in this script's arguments. This script 9 | # should only be run on trusted input. 10 | orig_args=($@) 11 | args=() 12 | for val in "${orig_args[@]}"; do 13 | phys_arg=$(eval "echo $val") 14 | args+=($phys_arg) 15 | done 16 | 17 | python3 plot_cdf.py ${args[@]} 18 | -------------------------------------------------------------------------------- /scripts/figures/plot_common.py: -------------------------------------------------------------------------------- 1 | COLORS = { 2 | "pg_llsm": "#397624", 3 | "rocksdb": "#ffbf00", 4 | "leanstore": "#3d949d", 5 | } 6 | 7 | DATASET_MAP = { 8 | "synth": "Synthetic", 9 | "amzn": "Amazon", 10 | "osm": "OSM", 11 | } 12 | 13 | DATASET_COLORS = { 14 | "amzn": "#397624", 15 | "osm": "#9fc55a", 16 | "synth": "#958934", 17 | } 18 | 19 | DATASET_LINESTYLE = { 20 | "amzn": "solid", 21 | "osm": "dashed", 22 | "synth": "dotted", 23 | } 24 | 25 | RAND_WRITE_COLORS = { 26 | "4K": "#397624", 27 | "8K": "#9fc55a", 28 | "16K": "#958934", 29 | "peak": "#875826", 30 | } 31 | -------------------------------------------------------------------------------- /scripts/insert_v2/workloads/custom.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 1000000 3 | distribution: 4 | type: uniform 5 | range_min: 1 6 | range_max: 10000000 7 | 8 | # 8 M inserts, 8 M reads 9 | run: 10 | - num_requests: 16000000 11 | read: 12 | proportion_pct: 50 13 | distribution: 14 | type: uniform 15 | insert: 16 | proportion_pct: 50 17 | distribution: 18 | type: custom 19 | name: custom 20 | -------------------------------------------------------------------------------- /scripts/insert_v2/workloads/setup.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 1000000 3 | distribution: 4 | type: uniform 5 | range_min: 1 6 | range_max: 10000000 7 | 8 | # Dummy workload. We only use this config for the dataset. 9 | run: 10 | - num_requests: 1 11 | read: 12 | proportion_pct: 100 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/iochar/COND: -------------------------------------------------------------------------------- 1 | combine( 2 | name="iochar", 3 | deps=[ 4 | "//scripts/iochar/read:read", 5 | "//scripts/iochar/write:write", 6 | "//scripts/iochar/mixed:mixed", 7 | ], 8 | ) 9 | 10 | group( 11 | name="nvme_ssd_ext4", 12 | deps=[ 13 | "//scripts/iochar/read:read-nvme_ssd_ext4", 14 | "//scripts/iochar/write:write-nvme_ssd_ext4", 15 | "//scripts/iochar/mixed:mixed-nvme_ssd_ext4", 16 | ], 17 | ) 18 | 19 | group( 20 | name="nvme_ssd_xfs", 21 | deps=[ 22 | "//scripts/iochar/read:read-nvme_ssd_xfs", 23 | "//scripts/iochar/write:write-nvme_ssd_xfs", 24 | "//scripts/iochar/mixed:mixed-nvme_ssd_xfs", 25 | ], 26 | ) 27 | 28 | group( 29 | name="nvme_ssd_raw", 30 | deps=[ 31 | "//scripts/iochar/read:read-nvme_ssd_raw", 32 | "//scripts/iochar/write:write-nvme_ssd_raw", 33 | "//scripts/iochar/mixed:mixed-nvme_ssd_raw", 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /scripts/iochar/common.fio: -------------------------------------------------------------------------------- 1 | direct=1 2 | size=10g 3 | time_based 4 | runtime=10 5 | thread 6 | -------------------------------------------------------------------------------- /scripts/iochar/fio_config_example.sh: -------------------------------------------------------------------------------- 1 | # The path where fio should direct its workload. 2 | NVME_SSD_FS_OUT_DIR=/flash1/$USER/fio-out 3 | NVME_SSD_RAW_OUT_DIR=/dev 4 | NVME_SSD_RAW_FILE=nvme0n1p1 5 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/COND: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | 3 | JOBS = [ 4 | "psync-4k", 5 | "psync-16k", 6 | "psync-4k-unlim", 7 | "psync-16k-unlim", 8 | "psync-4k-offset", 9 | "psync-16k-offset", 10 | "uring-4k", 11 | "uring-16k", 12 | "uring-4k-unlim", 13 | "uring-16k-unlim", 14 | "uring-4k-offset", 15 | "uring-16k-offset", 16 | ] 17 | 18 | DEVICES = [ 19 | "nvme_ssd_ext4", 20 | "nvme_ssd_raw", 21 | "nvme_ssd_xfs", 22 | ] 23 | 24 | run_experiment_group( 25 | name="mixed", 26 | run="../run_fio.sh", 27 | experiments=[ 28 | ExperimentInstance( 29 | name="mixed-{}-{}".format(device, jobfile), 30 | args=[device, jobfile], 31 | ) 32 | for device, jobfile in product(DEVICES, JOBS) 33 | # Can't run the "unlim" workloads directly on the block device because they 34 | # rely on having the ability to use separate files. 35 | if not device.endswith("raw") or not jobfile.endswith("unlim") 36 | ], 37 | ) 38 | 39 | group( 40 | name="mixed-nvme_ssd_ext4", 41 | deps=[ 42 | ":mixed-nvme_ssd_ext4-{}".format(jobfile) 43 | for jobfile in JOBS 44 | ], 45 | ) 46 | 47 | group( 48 | name="mixed-nvme_ssd_xfs", 49 | deps=[ 50 | ":mixed-nvme_ssd_xfs-{}".format(jobfile) 51 | for jobfile in JOBS 52 | ], 53 | ) 54 | 55 | group( 56 | name="mixed-nvme_ssd_raw", 57 | deps=[ 58 | ":mixed-nvme_ssd_raw-{}".format(jobfile) 59 | for jobfile in JOBS 60 | if not jobfile.endswith("unlim") 61 | ], 62 | ) 63 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randrw 4 | direct=1 5 | bs=16k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randrw-16k-1] 15 | numjobs=1 16 | 17 | [randrw-16k-2] 18 | numjobs=2 19 | 20 | [randrw-16k-4] 21 | numjobs=4 22 | 23 | [randrw-16k-8] 24 | numjobs=8 25 | 26 | [randrw-16k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-16k-offset: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | direct=1 4 | bs=16k 5 | time_based 6 | runtime=10 7 | overwrite=1 8 | filesize=10g 9 | thread 10 | 11 | [psync-16k-unlim-r] 12 | numjobs=8 13 | rw=randread 14 | 15 | [psync-16k-unlim-w] 16 | numjobs=8 17 | rw=randwrite 18 | fdatasync=1 19 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-16k-unlim: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | direct=1 4 | bs=16k 5 | size=10g 6 | time_based 7 | runtime=10 8 | overwrite=1 9 | thread 10 | 11 | [psync-16k-unlim-r] 12 | numjobs=8 13 | rw=randread 14 | filename=randrw-psync-16k-unlim-r 15 | 16 | [psync-16k-unlim-w] 17 | numjobs=8 18 | rw=randwrite 19 | filename=randrw-psync-16k-unlim-w 20 | fdatasync=1 21 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randrw 4 | direct=1 5 | bs=4k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randrw-4k-1] 15 | numjobs=1 16 | 17 | [randrw-4k-2] 18 | numjobs=2 19 | 20 | [randrw-4k-4] 21 | numjobs=4 22 | 23 | [randrw-4k-8] 24 | numjobs=8 25 | 26 | [randrw-4k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-4k-offset: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | direct=1 4 | bs=4k 5 | time_based 6 | runtime=10 7 | overwrite=1 8 | filesize=10g 9 | thread 10 | 11 | [psync-4k-unlim-r] 12 | numjobs=8 13 | rw=randread 14 | 15 | [psync-4k-unlim-w] 16 | numjobs=8 17 | rw=randwrite 18 | fdatasync=1 19 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/psync-4k-unlim: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | direct=1 4 | bs=4k 5 | size=10g 6 | time_based 7 | runtime=10 8 | overwrite=1 9 | thread 10 | 11 | [psync-4k-unlim-r] 12 | numjobs=8 13 | rw=randread 14 | filename=randrw-psync-4k-unlim-r 15 | 16 | [psync-4k-unlim-w] 17 | numjobs=8 18 | rw=randwrite 19 | filename=randrw-psync-4k-unlim-w 20 | fdatasync=1 21 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | rw=randrw 4 | direct=1 5 | bs=16k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randrw-16k-1] 15 | iodepth=1 16 | 17 | [randrw-16k-4] 18 | iodepth=4 19 | 20 | [randrw-16k-16] 21 | iodepth=16 22 | 23 | [randrw-16k-64] 24 | iodepth=64 25 | 26 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-16k-offset: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | direct=1 4 | bs=16k 5 | time_based 6 | runtime=10 7 | overwrite=1 8 | filesize=10g 9 | thread 10 | 11 | [uring-16k-unlim-r] 12 | iodepth=8 13 | rw=randread 14 | 15 | [uring-16k-unlim-w] 16 | iodepth=8 17 | rw=randwrite 18 | fdatasync=1 19 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-16k-unlim: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | direct=1 4 | bs=16k 5 | size=10g 6 | time_based 7 | runtime=10 8 | overwrite=1 9 | thread 10 | 11 | [uring-16k-unlim-r] 12 | iodepth=8 13 | rw=randread 14 | filename=randrw-uring-16k-unlim-r 15 | 16 | [uring-16k-unlim-w] 17 | iodepth=8 18 | rw=randwrite 19 | filename=randrw-uring-16k-unlim-w 20 | fdatasync=1 21 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | rw=randrw 4 | direct=1 5 | bs=4k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randrw-4k-1] 15 | iodepth=1 16 | 17 | [randrw-4k-4] 18 | iodepth=4 19 | 20 | [randrw-4k-16] 21 | iodepth=16 22 | 23 | [randrw-4k-64] 24 | iodepth=64 25 | 26 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-4k-offset: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | direct=1 4 | bs=4k 5 | time_based 6 | runtime=10 7 | overwrite=1 8 | filesize=10g 9 | thread 10 | 11 | [uring-4k-unlim-r] 12 | iodepth=8 13 | rw=randread 14 | 15 | [uring-4k-unlim-w] 16 | iodepth=8 17 | rw=randwrite 18 | fdatasync=1 19 | -------------------------------------------------------------------------------- /scripts/iochar/mixed/uring-4k-unlim: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | direct=1 4 | bs=4k 5 | size=10g 6 | time_based 7 | runtime=10 8 | overwrite=1 9 | thread 10 | 11 | [uring-4k-unlim-r] 12 | iodepth=8 13 | rw=randread 14 | filename=randrw-uring-4k-unlim-r 15 | 16 | [uring-4k-unlim-w] 17 | iodepth=8 18 | rw=randwrite 19 | filename=randrw-uring-4k-unlim-w 20 | fdatasync=1 21 | -------------------------------------------------------------------------------- /scripts/iochar/read/COND: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | 3 | JOBS = [ 4 | "psync-4k", 5 | "psync-8k", 6 | "psync-16k", 7 | "psync-64k", 8 | "uring-4k", 9 | "uring-16k", 10 | "uring-64k", 11 | ] 12 | 13 | DEVICES = [ 14 | "nvme_ssd_ext4", 15 | "nvme_ssd_raw", 16 | "nvme_ssd_xfs", 17 | ] 18 | 19 | run_experiment_group( 20 | name="read", 21 | run="../run_fio.sh", 22 | experiments=[ 23 | ExperimentInstance( 24 | name="read-{}-{}".format(device, jobfile), 25 | args=[device, jobfile], 26 | ) 27 | for device, jobfile in product(DEVICES, JOBS) 28 | ], 29 | ) 30 | 31 | run_experiment_group( 32 | name="read_seq_scale", 33 | run="../run_fio.sh", 34 | experiments=[ 35 | ExperimentInstance( 36 | name="read_seq_scale-{}".format(device), 37 | args=[device, "seq-scale"], 38 | ) 39 | for device in DEVICES 40 | ], 41 | ) 42 | 43 | group( 44 | name="read-nvme_ssd_ext4", 45 | deps=[ 46 | ":read-nvme_ssd_ext4-{}".format(jobfile) 47 | for jobfile in JOBS 48 | ], 49 | ) 50 | 51 | group( 52 | name="read-nvme_ssd_xfs", 53 | deps=[ 54 | ":read-nvme_ssd_xfs-{}".format(jobfile) 55 | for jobfile in JOBS 56 | ], 57 | ) 58 | 59 | group( 60 | name="read-nvme_ssd_raw", 61 | deps=[ 62 | ":read-nvme_ssd_raw-{}".format(jobfile) 63 | for jobfile in JOBS 64 | ], 65 | ) 66 | 67 | run_experiment( 68 | name="read_rand_sweep", 69 | run="../run_fio.sh", 70 | args=["nvme_ssd_ext4", "psync-sweep"], 71 | ) 72 | 73 | combine( 74 | name="rand-sweep", 75 | deps=[ 76 | ":read-nvme_ssd_ext4-{}".format(jobfile) 77 | for jobfile in JOBS if "psync" in jobfile 78 | ] 79 | ) 80 | -------------------------------------------------------------------------------- /scripts/iochar/read/psync-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=psync 4 | bs=16k 5 | stonewall 6 | 7 | ; Random Read 8 | 9 | [randread-16k-1] 10 | rw=randread 11 | numjobs=1 12 | 13 | [randread-16k-2] 14 | rw=randread 15 | numjobs=2 16 | 17 | [randread-16k-4] 18 | rw=randread 19 | numjobs=4 20 | 21 | [randread-16k-8] 22 | rw=randread 23 | numjobs=8 24 | 25 | [randread-16k-16] 26 | rw=randread 27 | numjobs=16 28 | 29 | ; Sequential Read 30 | 31 | [seqread-16k-1] 32 | rw=read 33 | numjobs=1 34 | 35 | [seqread-16k-2] 36 | rw=read 37 | numjobs=2 38 | 39 | [seqread-16k-4] 40 | rw=read 41 | numjobs=4 42 | 43 | [seqread-16k-8] 44 | rw=read 45 | numjobs=8 46 | 47 | [seqread-16k-16] 48 | rw=read 49 | numjobs=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/psync-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=psync 4 | bs=4k 5 | stonewall 6 | 7 | ; --- Random read --- 8 | 9 | [randread-4k-1] 10 | rw=randread 11 | numjobs=1 12 | 13 | [randread-4k-2] 14 | rw=randread 15 | numjobs=2 16 | 17 | [randread-4k-4] 18 | rw=randread 19 | numjobs=4 20 | 21 | [randread-4k-8] 22 | rw=randread 23 | numjobs=8 24 | 25 | [randread-4k-16] 26 | rw=randread 27 | numjobs=16 28 | 29 | ; --- Sequential read --- 30 | 31 | [seqread-4k-1] 32 | rw=read 33 | numjobs=1 34 | 35 | [seqread-4k-2] 36 | rw=read 37 | numjobs=2 38 | 39 | [seqread-4k-4] 40 | rw=read 41 | numjobs=4 42 | 43 | [seqread-4k-8] 44 | rw=read 45 | numjobs=8 46 | 47 | [seqread-4k-16] 48 | rw=read 49 | numjobs=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/psync-64k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=psync 4 | bs=64k 5 | stonewall 6 | 7 | ; Random Read 8 | 9 | [randread-64k-1] 10 | rw=randread 11 | numjobs=1 12 | 13 | [randread-64k-2] 14 | rw=randread 15 | numjobs=2 16 | 17 | [randread-64k-4] 18 | rw=randread 19 | numjobs=4 20 | 21 | [randread-64k-8] 22 | rw=randread 23 | numjobs=8 24 | 25 | [randread-64k-16] 26 | rw=randread 27 | numjobs=16 28 | 29 | ; Sequential Read 30 | 31 | [seqread-64k-1] 32 | rw=read 33 | numjobs=1 34 | 35 | [seqread-64k-2] 36 | rw=read 37 | numjobs=2 38 | 39 | [seqread-64k-4] 40 | rw=read 41 | numjobs=4 42 | 43 | [seqread-64k-8] 44 | rw=read 45 | numjobs=8 46 | 47 | [seqread-64k-16] 48 | rw=read 49 | numjobs=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/psync-8k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=psync 4 | bs=8k 5 | stonewall 6 | 7 | ; --- Random read --- 8 | 9 | [randread-8k-1] 10 | rw=randread 11 | numjobs=1 12 | 13 | [randread-8k-2] 14 | rw=randread 15 | numjobs=2 16 | 17 | [randread-8k-4] 18 | rw=randread 19 | numjobs=4 20 | 21 | [randread-8k-8] 22 | rw=randread 23 | numjobs=8 24 | 25 | [randread-8k-16] 26 | rw=randread 27 | numjobs=16 28 | 29 | ; --- Sequential read --- 30 | 31 | [seqread-8k-1] 32 | rw=read 33 | numjobs=1 34 | 35 | [seqread-8k-2] 36 | rw=read 37 | numjobs=2 38 | 39 | [seqread-8k-4] 40 | rw=read 41 | numjobs=4 42 | 43 | [seqread-8k-8] 44 | rw=read 45 | numjobs=8 46 | 47 | [seqread-8k-16] 48 | rw=read 49 | numjobs=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/psync-sweep: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=psync 4 | numjobs=1 5 | stonewall 6 | 7 | ; --- Random read --- 8 | 9 | [randread-1] 10 | rw=randread 11 | bs=4k 12 | 13 | [randread-2] 14 | rw=randread 15 | bs=8k 16 | 17 | [randread-3] 18 | rw=randread 19 | bs=12k 20 | 21 | [randread-4] 22 | rw=randread 23 | bs=16k 24 | 25 | [randread-5] 26 | rw=randread 27 | bs=20k 28 | 29 | [randread-6] 30 | rw=randread 31 | bs=24k 32 | 33 | [randread-7] 34 | rw=randread 35 | bs=28k 36 | 37 | [randread-8] 38 | rw=randread 39 | bs=32k 40 | 41 | [randread-9] 42 | rw=randread 43 | bs=36k 44 | 45 | [randread-10] 46 | rw=randread 47 | bs=40k 48 | 49 | [randread-11] 50 | rw=randread 51 | bs=44k 52 | 53 | [randread-12] 54 | rw=randread 55 | bs=48k 56 | 57 | [randread-13] 58 | rw=randread 59 | bs=52k 60 | 61 | [randread-14] 62 | rw=randread 63 | bs=56k 64 | 65 | [randread-15] 66 | rw=randread 67 | bs=60k 68 | 69 | [randread-16] 70 | rw=randread 71 | bs=64k 72 | -------------------------------------------------------------------------------- /scripts/iochar/read/seq-scale: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | rw=read 4 | ioengine=psync 5 | size=10g 6 | stonewall 7 | 8 | [seqread-4k-1] 9 | bs=4k 10 | numjobs=1 11 | 12 | [seqread-8k-1] 13 | bs=8k 14 | numjobs=1 15 | 16 | [seqread-16k-1] 17 | bs=16k 18 | numjobs=1 19 | 20 | [seqread-32k-1] 21 | bs=32k 22 | numjobs=1 23 | 24 | [seqread-64k-1] 25 | bs=64k 26 | numjobs=1 27 | 28 | [seqread-128k-1] 29 | bs=128k 30 | numjobs=1 31 | 32 | [seqread-256k-1] 33 | bs=256k 34 | numjobs=1 35 | 36 | [seqread-512k-1] 37 | bs=512k 38 | numjobs=1 39 | 40 | [seqread-1024k-1] 41 | bs=1024k 42 | numjobs=1 43 | 44 | [seqread-2048k-1] 45 | bs=2048k 46 | numjobs=1 47 | 48 | [seqread-4096k-1] 49 | bs=4096k 50 | numjobs=1 51 | 52 | [seqread-8192k-1] 53 | bs=8192k 54 | numjobs=1 55 | 56 | [seqread-16384k-1] 57 | bs=16384k 58 | numjobs=1 59 | 60 | [seqread-32768k-1] 61 | bs=32768k 62 | numjobs=1 63 | 64 | [seqread-65536k-1] 65 | bs=65536k 66 | numjobs=1 67 | -------------------------------------------------------------------------------- /scripts/iochar/read/uring-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=io_uring 4 | bs=16k 5 | stonewall 6 | 7 | ; Random Read 8 | 9 | [randread-16k-1] 10 | rw=randread 11 | iodepth=1 12 | 13 | [randread-16k-2] 14 | rw=randread 15 | iodepth=2 16 | 17 | [randread-16k-4] 18 | rw=randread 19 | iodepth=4 20 | 21 | [randread-16k-8] 22 | rw=randread 23 | iodepth=8 24 | 25 | [randread-16k-16] 26 | rw=randread 27 | iodepth=16 28 | 29 | ; Sequential Read 30 | 31 | [seqread-16k-1] 32 | rw=read 33 | iodepth=1 34 | 35 | [seqread-16k-2] 36 | rw=read 37 | iodepth=2 38 | 39 | [seqread-16k-4] 40 | rw=read 41 | iodepth=4 42 | 43 | [seqread-16k-8] 44 | rw=read 45 | iodepth=8 46 | 47 | [seqread-16k-16] 48 | rw=read 49 | iodepth=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/uring-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=io_uring 4 | bs=4k 5 | stonewall 6 | 7 | ; Random Read 8 | 9 | [randread-4k-1] 10 | rw=randread 11 | iodepth=1 12 | 13 | [randread-4k-2] 14 | rw=randread 15 | iodepth=2 16 | 17 | [randread-4k-4] 18 | rw=randread 19 | iodepth=4 20 | 21 | [randread-4k-8] 22 | rw=randread 23 | iodepth=8 24 | 25 | [randread-4k-16] 26 | rw=randread 27 | iodepth=16 28 | 29 | ; Sequential Read 30 | 31 | [seqread-4k-1] 32 | rw=read 33 | iodepth=1 34 | 35 | [seqread-4k-2] 36 | rw=read 37 | iodepth=2 38 | 39 | [seqread-4k-4] 40 | rw=read 41 | iodepth=4 42 | 43 | [seqread-4k-8] 44 | rw=read 45 | iodepth=8 46 | 47 | [seqread-4k-16] 48 | rw=read 49 | iodepth=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/read/uring-64k: -------------------------------------------------------------------------------- 1 | [global] 2 | include ../common.fio 3 | ioengine=io_uring 4 | bs=64k 5 | stonewall 6 | 7 | ; Random Read 8 | 9 | [randread-64k-1] 10 | rw=randread 11 | iodepth=1 12 | 13 | [randread-64k-2] 14 | rw=randread 15 | iodepth=2 16 | 17 | [randread-64k-4] 18 | rw=randread 19 | iodepth=4 20 | 21 | [randread-64k-8] 22 | rw=randread 23 | iodepth=8 24 | 25 | [randread-64k-16] 26 | rw=randread 27 | iodepth=16 28 | 29 | ; Sequential Read 30 | 31 | [seqread-64k-1] 32 | rw=read 33 | iodepth=1 34 | 35 | [seqread-64k-2] 36 | rw=read 37 | iodepth=2 38 | 39 | [seqread-64k-4] 40 | rw=read 41 | iodepth=4 42 | 43 | [seqread-64k-8] 44 | rw=read 45 | iodepth=8 46 | 47 | [seqread-64k-16] 48 | rw=read 49 | iodepth=16 50 | -------------------------------------------------------------------------------- /scripts/iochar/run_fio.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Acknowledgement: 4 | # https://stackoverflow.com/questions/25288194/dont-display-pushd-popd-stack-across-several-bash-scripts-quiet-pushd-popd 5 | pushd () { 6 | command pushd "$@" > /dev/null 7 | } 8 | popd () { 9 | command popd "$@" > /dev/null 10 | } 11 | 12 | script_loc=$(cd $(dirname $0) && pwd -P) 13 | pushd $script_loc 14 | source fio_config.sh 15 | popd 16 | 17 | device=$1 18 | jobfile=$2 19 | shift 2 20 | 21 | args=() 22 | args+=("--output=$COND_OUT/fio.json") 23 | args+=("--output-format=json") 24 | if [ $device = "nvme_ssd_raw" ]; then 25 | args+=("--directory=$NVME_SSD_RAW_OUT_DIR") 26 | args+=("--filename=$NVME_SSD_RAW_FILE") 27 | else 28 | args+=("--directory=$NVME_SSD_FS_OUT_DIR") 29 | args+=("--filename=$jobfile") 30 | mkdir -p $NVME_SSD_FS_OUT_DIR 31 | rm -f $NVME_SSD_FS_OUT_DIR/$jobfile 32 | fi 33 | args+=($jobfile) 34 | 35 | # All jobs in a jobfile share the same physical file. 36 | fio ${args[@]} 37 | -------------------------------------------------------------------------------- /scripts/iochar/write/COND: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | 3 | JOBS = [ 4 | "psync-4k", 5 | "psync-8k", 6 | "psync-16k", 7 | "psync-64k", 8 | "uring-4k", 9 | "uring-16k", 10 | "uring-64k", 11 | ] 12 | 13 | DEVICES = [ 14 | "nvme_ssd_ext4", 15 | "nvme_ssd_raw", 16 | "nvme_ssd_xfs", 17 | ] 18 | 19 | run_experiment_group( 20 | name="write", 21 | run="../run_fio.sh", 22 | experiments=[ 23 | ExperimentInstance( 24 | name="write-{}-{}".format(device, jobfile), 25 | args=[device, jobfile], 26 | ) 27 | for device, jobfile in product(DEVICES, JOBS) 28 | ], 29 | ) 30 | 31 | group( 32 | name="write-nvme_ssd_ext4", 33 | deps=[ 34 | ":write-nvme_ssd_ext4-{}".format(jobfile) 35 | for jobfile in JOBS 36 | ], 37 | ) 38 | 39 | group( 40 | name="write-nvme_ssd_xfs", 41 | deps=[ 42 | ":write-nvme_ssd_xfs-{}".format(jobfile) 43 | for jobfile in JOBS 44 | ], 45 | ) 46 | 47 | group( 48 | name="write-nvme_ssd_raw", 49 | deps=[ 50 | ":write-nvme_ssd_raw-{}".format(jobfile) 51 | for jobfile in JOBS 52 | ], 53 | ) 54 | 55 | combine( 56 | name="rand-sweep", 57 | deps=[ 58 | ":write-nvme_ssd_ext4-{}".format(jobfile) 59 | for jobfile in JOBS if "psync" in jobfile 60 | ] 61 | ) 62 | -------------------------------------------------------------------------------- /scripts/iochar/write/psync-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randwrite 4 | direct=1 5 | bs=16k 6 | size=10g 7 | time_based 8 | runtime=30 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-16k-1] 15 | numjobs=1 16 | 17 | [randwrite-16k-2] 18 | numjobs=2 19 | 20 | [randwrite-16k-4] 21 | numjobs=4 22 | 23 | [randwrite-16k-8] 24 | numjobs=8 25 | 26 | [randwrite-16k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/write/psync-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randwrite 4 | direct=1 5 | bs=4k 6 | size=10g 7 | time_based 8 | runtime=30 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-4k-1] 15 | numjobs=1 16 | 17 | [randwrite-4k-2] 18 | numjobs=2 19 | 20 | [randwrite-4k-4] 21 | numjobs=4 22 | 23 | [randwrite-4k-8] 24 | numjobs=8 25 | 26 | [randwrite-4k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/write/psync-64k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randwrite 4 | direct=1 5 | bs=64k 6 | size=10g 7 | time_based 8 | runtime=30 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-64k-1] 15 | numjobs=1 16 | 17 | [randwrite-64k-2] 18 | numjobs=2 19 | 20 | [randwrite-64k-4] 21 | numjobs=4 22 | 23 | [randwrite-64k-8] 24 | numjobs=8 25 | 26 | [randwrite-64k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/write/psync-8k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=psync 3 | rw=randwrite 4 | direct=1 5 | bs=8k 6 | size=10g 7 | time_based 8 | runtime=30 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-8k-1] 15 | numjobs=1 16 | 17 | [randwrite-8k-2] 18 | numjobs=2 19 | 20 | [randwrite-8k-4] 21 | numjobs=4 22 | 23 | [randwrite-8k-8] 24 | numjobs=8 25 | 26 | [randwrite-8k-16] 27 | numjobs=16 28 | -------------------------------------------------------------------------------- /scripts/iochar/write/uring-16k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | rw=randwrite 4 | direct=1 5 | bs=16k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-16k-1] 15 | iodepth=1 16 | 17 | [randwrite-16k-4] 18 | iodepth=4 19 | 20 | [randwrite-16k-16] 21 | iodepth=16 22 | 23 | [randwrite-16k-64] 24 | iodepth=64 25 | 26 | -------------------------------------------------------------------------------- /scripts/iochar/write/uring-4k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | rw=randwrite 4 | direct=1 5 | bs=4k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-4k-1] 15 | iodepth=1 16 | 17 | [randwrite-4k-4] 18 | iodepth=4 19 | 20 | [randwrite-4k-16] 21 | iodepth=16 22 | 23 | [randwrite-4k-64] 24 | iodepth=64 25 | 26 | -------------------------------------------------------------------------------- /scripts/iochar/write/uring-64k: -------------------------------------------------------------------------------- 1 | [global] 2 | ioengine=io_uring 3 | rw=randwrite 4 | direct=1 5 | bs=64k 6 | size=10g 7 | time_based 8 | runtime=10 9 | overwrite=1 10 | fdatasync=1 11 | thread 12 | stonewall 13 | 14 | [randwrite-64k-1] 15 | iodepth=1 16 | 17 | [randwrite-64k-4] 18 | iodepth=4 19 | 20 | [randwrite-64k-16] 21 | iodepth=16 22 | 23 | [randwrite-64k-64] 24 | iodepth=64 25 | 26 | -------------------------------------------------------------------------------- /scripts/pg/process_fio.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import json 3 | import pandas as pd 4 | import conductor.lib as cond 5 | 6 | 7 | def aggregate_bw(df): 8 | results = ( 9 | df.groupby(["punits"]) 10 | .agg({"bw_bytes": "sum", "lat_ns": "median"}) 11 | .reset_index() 12 | ) 13 | results["bw_mib"] = results["bw_bytes"] / 1024 / 1024 14 | results["lat_us"] = results["lat_ns"] / 1000 15 | return results 16 | 17 | 18 | def process_read(raw_json): 19 | punits = [] 20 | bw_bytes = [] 21 | lat_ns = [] 22 | for job in raw_json["jobs"]: 23 | name = job["jobname"] 24 | parts = name.split("-") 25 | punits.append(int(parts[-1])) 26 | bw_bytes.append(int(job["read"]["bw_bytes"])) 27 | lat_ns.append(int(job["read"]["lat_ns"]["mean"])) 28 | 29 | df = pd.DataFrame({"punits": punits, "bw_bytes": bw_bytes, "lat_ns": lat_ns}) 30 | return df 31 | 32 | 33 | def load_and_process(path): 34 | with open(pathlib.Path(path) / "fio.json", "r") as file: 35 | raw_json = json.load(file) 36 | 37 | return aggregate_bw(process_read(raw_json)) 38 | 39 | 40 | def main(): 41 | deps = cond.get_deps_paths() 42 | assert len(deps) == 1 43 | processed_data = load_and_process(deps[0]) 44 | out_dir = cond.get_output_path() 45 | processed_data.to_csv(out_dir / "fio.csv", index=False) 46 | 47 | 48 | if __name__ == "__main__": 49 | main() 50 | -------------------------------------------------------------------------------- /scripts/pg/read/COND: -------------------------------------------------------------------------------- 1 | MODES = [ 2 | "all", 3 | "power_two", 4 | "single", 5 | ] 6 | 7 | READ_PAGES = 10000000 # 10 million 8 | SIZE_MIB = 40960 # 40 GiB 9 | WARMUP = 1000 10 | 11 | OUT_FILE = "/flash1/geoffxy/pg_read_test_rand" 12 | WRITE_PATH = "/flash1/geoffxy/pg_read_dir" 13 | 14 | run_experiment_group( 15 | name="simple", 16 | run="./run.sh", 17 | experiments=[ 18 | ExperimentInstance( 19 | name="simple-{}".format(mode), 20 | options={ 21 | "mode": mode, 22 | "iterations": READ_PAGES, 23 | "warmup": WARMUP, 24 | "size_mib": SIZE_MIB, 25 | "file": OUT_FILE, 26 | "seed": 42, 27 | "trials": 1, 28 | "rand_init": True, 29 | }, 30 | ) 31 | for mode in MODES 32 | ], 33 | ) 34 | 35 | run_experiment_group( 36 | name="multi", 37 | run="./run2.sh", 38 | experiments=[ 39 | ExperimentInstance( 40 | name="multi-{}".format(mode), 41 | options={ 42 | "mode": mode, 43 | "iterations": READ_PAGES, 44 | "warmup": WARMUP, 45 | "size_mib": SIZE_MIB, 46 | "write_path": WRITE_PATH, 47 | "seed": 42, 48 | "trials": 1, 49 | "interleave_alloc": True, 50 | }, 51 | ) 52 | for mode in MODES 53 | ], 54 | ) 55 | -------------------------------------------------------------------------------- /scripts/pg/read/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | ../../../build/page_grouping/pg_read --out_path=$COND_OUT $@ 4 | -------------------------------------------------------------------------------- /scripts/pg/read/run2.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Record physical I/O activity. 4 | iostat -o JSON -d -y 1 >$COND_OUT/iostat.json & 5 | iostat_pid=$! 6 | 7 | ../../../build/page_grouping/pg_read2 --out_path=$COND_OUT $@ 8 | 9 | kill -s SIGINT -- $iostat_pid 10 | wait 11 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/long_scan-amzn.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | # Key bounds of the Amazon dataset. 6 | range_min: 10001 7 | range_max: 53096592 8 | 9 | run: 10 | - num_requests: 100000 11 | scan: 12 | proportion_pct: 100 13 | max_length: 500 14 | distribution: 15 | type: uniform 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/long_scan-osm.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | # Key bounds of the OSM dataset. 6 | range_min: 699540 7 | range_max: 6820987374 8 | 9 | run: 10 | - num_requests: 100000 11 | scan: 12 | proportion_pct: 100 13 | max_length: 500 14 | distribution: 15 | type: uniform 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/long_scan.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 1000000000 7 | 8 | run: 9 | - num_requests: 100000 10 | scan: 11 | proportion_pct: 100 12 | max_length: 500 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/preload-amzn-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 1000000000 7 | 8 | run: 9 | - num_requests: 5000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | # Key bounds of the Amazon dataset. 15 | range_min: 10001 16 | range_max: 53096592 17 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/preload-osm-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 1000000000 7 | 8 | run: 9 | - num_requests: 5000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | # Key bounds of the OSM dataset. 15 | range_min: 699540 16 | range_max: 6820987374 17 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/preload-uniform-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 1000000000 7 | 8 | run: 9 | - num_requests: 5000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | range_min: 0 15 | range_max: 1000000000 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/10M/preload.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 10000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 1000000000 7 | 8 | # We use this workload for the dataset only. 9 | run: 10 | - num_requests: 1 11 | read: 12 | proportion_pct: 100 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/long_scan-amzn.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | # Key bounds of the Amazon dataset. 6 | range_min: 10001 7 | range_max: 53096592 8 | 9 | run: 10 | - num_requests: 100000 11 | scan: 12 | proportion_pct: 100 13 | max_length: 500 14 | distribution: 15 | type: uniform 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/long_scan-osm.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | # Key bounds of the OSM dataset. 6 | range_min: 699540 7 | range_max: 6820987374 8 | 9 | run: 10 | - num_requests: 100000 11 | scan: 12 | proportion_pct: 100 13 | max_length: 500 14 | distribution: 15 | type: uniform 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/long_scan.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 100000 10 | scan: 11 | proportion_pct: 100 12 | max_length: 500 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/preload-amzn-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | # Key bounds of the Amazon dataset. 15 | range_min: 10001 16 | range_max: 53096592 17 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/preload-osm-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | # Key bounds of the OSM dataset. 15 | range_min: 699540 16 | range_max: 6820987374 17 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/preload-uniform-insert.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | range_min: 0 15 | range_max: 2000000000 16 | -------------------------------------------------------------------------------- /scripts/pg/workloads/20M/preload.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | # We use this workload for the dataset only. 9 | run: 10 | - num_requests: 1 11 | read: 12 | proportion_pct: 100 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/pg_analysis/run_load.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | script_loc=$(cd $(dirname $0) && pwd -P) 5 | cd $script_loc 6 | source ../experiment_config.sh 7 | 8 | # Evaluates any environment variables in this script's arguments. This script 9 | # should only be run on trusted input. 10 | orig_args=($@) 11 | args=() 12 | for val in "${orig_args[@]}"; do 13 | phys_arg=$(eval "echo $val") 14 | args+=($phys_arg) 15 | done 16 | 17 | if [ -z $COND_OUT ]; then 18 | echo >&2 "ERROR: This script is meant to be launched by Conductor." 19 | exit 1 20 | fi 21 | 22 | # Used when multiple experiments are running at the same time. 23 | if [ ! -z $COND_SLOT ]; then 24 | full_db_path=$DB_PATH-$COND_SLOT 25 | else 26 | full_db_path=$DB_PATH 27 | fi 28 | 29 | rm -rf $full_db_path 30 | mkdir $full_db_path 31 | sync $full_db_path 32 | 33 | # Add common arguments. 34 | args+=("--verbose") 35 | args+=("--db_path=$full_db_path") 36 | args+=("--seed=$SEED") 37 | 38 | set +e 39 | ../../build/bench/run_custom \ 40 | --db=pg_llsm \ 41 | ${args[@]} \ 42 | --workload_config=workload.yml \ 43 | --output_path=$COND_OUT \ 44 | > $COND_OUT/results.csv 45 | code=$? 46 | set -e 47 | 48 | du -b $full_db_path > $COND_OUT/db_space.log 49 | 50 | # Store the generated debug information. Note that this information is only 51 | # generated when segments are used. We retrieve the number of pages in the 52 | # non-segment case by consulting the debug counters file. 53 | if [ -d "${full_db_path}/pg_llsm/debug" ]; then 54 | cp -r $full_db_path/pg_llsm/debug $COND_OUT 55 | elif [ -d "${full_db_path}/pg_tl/debug" ]; then 56 | cp -r $full_db_path/pg_tl/debug $COND_OUT 57 | fi 58 | 59 | # Report that the experiment failed if the `run_custom` exit code is not 0 60 | if [ $code -ne 0 ]; then 61 | exit $code 62 | fi 63 | -------------------------------------------------------------------------------- /scripts/pg_analysis/workload.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: 0 6 | range_max: 2000000000 7 | 8 | # We use this workload for the dataset only. 9 | run: 10 | - num_requests: 1 11 | read: 12 | proportion_pct: 100 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /scripts/reorg_vs_split/insert_middle.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 # 20,000,000 3 | distribution: 4 | type: linspace 5 | start_key: 1 6 | step_size: 100 7 | # range_min: 1 8 | # range_max: 2000000000 9 | 10 | run: 11 | - num_requests: 20000000 12 | insert: 13 | proportion_pct: 100 14 | distribution: 15 | type: uniform 16 | range_min: 990000000 # 990,000,000 17 | range_max: 1010000000 # 1,010,000,000 18 | 19 | -------------------------------------------------------------------------------- /scripts/reorg_vs_split/insert_middle_amzn.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 # 20,000,000 3 | distribution: 4 | type: uniform 5 | range_min: 1 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 10000000 # 10,000,000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | range_min: 21543296 # 21,543,296 15 | range_max: 31543296 # 31,543,296 16 | 17 | -------------------------------------------------------------------------------- /scripts/reorg_vs_split/insert_middle_amzn_20m.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 # 20,000,000 3 | distribution: 4 | type: uniform 5 | range_min: 1 6 | range_max: 2000000000 7 | 8 | run: 9 | - num_requests: 20000000 # 20,000,000 10 | insert: 11 | proportion_pct: 100 12 | distribution: 13 | type: uniform 14 | range_min: 21543296 # 21,543,296 15 | range_max: 31543296 # 31,543,296 16 | 17 | -------------------------------------------------------------------------------- /scripts/run_deferred_io_simulation.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | ########## Experiment Configuration ########## 4 | WORKLOAD_PATH="../bench/workloads" 5 | 6 | declare -a WORKLOAD_PAIRS=( 7 | "-load_path=${WORKLOAD_PATH}/ycsb-a-load-20m -workload_path=${WORKLOAD_PATH}/ycsb-a-run-20m-uniform" 8 | "-load_path=${WORKLOAD_PATH}/ycsb-a-load-20m -workload_path=${WORKLOAD_PATH}/ycsb-a-run-20m-zipfian" 9 | "-load_path=${WORKLOAD_PATH}/ycsb-a-load-100m -workload_path=${WORKLOAD_PATH}/ycsb-a-run-100m-uniform" 10 | "-load_path=${WORKLOAD_PATH}/ycsb-a-load-100m -workload_path=${WORKLOAD_PATH}/ycsb-a-run-100m-zipfian" 11 | ) 12 | declare -a THRES=( 13 | "0" 14 | "1" 15 | "100" 16 | "200" 17 | "400" 18 | "800" 19 | "1600" 20 | "3200" 21 | ) 22 | 23 | declare -a DEFERS=( 24 | "0" 25 | "1" 26 | "2" 27 | "3" 28 | "4" 29 | ) 30 | 31 | 32 | ############################################## 33 | 34 | NOW=$(date "+%F_%H_%M") 35 | 36 | LOG_FILE=results/experiment_$NOW 37 | touch $LOG_FILE 38 | echo "${LOG_FILE}" > $LOG_FILE 39 | 40 | for workload in "${WORKLOAD_PAIRS[@]}"; do 41 | for threshold in "${THRES[@]}"; do 42 | for deferrals in "${DEFERS[@]}"; do 43 | name="${workload} -io_threshold=${threshold} -max_deferrals=${deferrals}" 44 | echo "[$(date +"%D %T")] Running $name" 45 | ~/LearnedLSM/learnedlsm/build/bench/deferred_io $name >> $LOG_FILE 46 | done 47 | done 48 | done 49 | -------------------------------------------------------------------------------- /scripts/run_ycsb_w_deferral.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | ########## Experiment Configuration ########## 4 | WORKLOAD_PATH="/home/markakis/LearnedLSM/workloads" 5 | 6 | declare -a WORKLOAD_PAIRS=( 7 | "-load_path=${WORKLOAD_PATH}/ycsb-20M-load -workload_path=${WORKLOAD_PATH}/ycsb-20M-20Mop-wa" 8 | # "-load_path=${WORKLOAD_PATH}/ycsb-20M-load -workload_path=${WORKLOAD_PATH}/ycsb-20M-10Mop-wa-updateonly" 9 | ) 10 | declare -a THRES=( 11 | "1" 12 | "1600" 13 | "3200" 14 | "6400" 15 | "12800" 16 | "25600" 17 | "1 -deferral_autotuning=true" 18 | # "3200" 19 | # "6400" 20 | ) 21 | 22 | declare -a DEFERS=( 23 | "1" 24 | # "2" 25 | ) 26 | 27 | declare -a THREADS=( 28 | #"1" 29 | "4" 30 | ) 31 | 32 | declare -a DIRECT_IO=( 33 | # "false" 34 | "true" 35 | ) 36 | 37 | ############################################## 38 | 39 | NOW=$(date "+%F_%H_%M") 40 | 41 | LOG_FILE=~/LearnedLSM/learnedlsm/bench/results/experiment_$NOW 42 | STATS_FILE=${LOG_FILE}_stats 43 | touch $LOG_FILE 44 | touch $STATS_FILE 45 | echo "${LOG_FILE}" > $LOG_FILE 46 | echo "${STATS_FILE}" > $STATS_FILE 47 | 48 | for workload in "${WORKLOAD_PAIRS[@]}"; do 49 | for threads in "${THREADS[@]}"; do 50 | for direct in "${DIRECT_IO[@]}"; do 51 | for threshold in "${THRES[@]}"; do 52 | for deferrals in "${DEFERS[@]}"; do 53 | name="${workload} -threads=${threads} -use_direct_io=${direct} -io_min_batch_size=${threshold} -max_deferrals=${deferrals}" 54 | echo "[$(date +"%D %T")] Running $name" 55 | rm -rf test 56 | ~/LearnedLSM/learnedlsm/build/release/bench/ycsb --db_path=test -db=llsm -cache_size_mib=256 -bg_threads=16 $name >> $LOG_FILE 2>> $STATS_FILE 57 | done 58 | done 59 | done 60 | done 61 | done 62 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/generate_workload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import conductor.lib as cond 3 | import pathlib 4 | 5 | ZIPFIAN_DIST = """\ 6 | type: zipfian 7 | theta: 0.99\ 8 | """ 9 | 10 | UNIFORM_DIST = """\ 11 | type: uniform\ 12 | """ 13 | 14 | 15 | def main(): 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--gen_template", type=str, required=True) 18 | parser.add_argument("--gen_num_requests", type=int, required=True) 19 | parser.add_argument("--gen_range_min", type=int, required=True) 20 | parser.add_argument("--gen_range_max", type=int, required=True) 21 | parser.add_argument( 22 | "--gen_distribution", choices=["zipfian", "uniform"], default="zipfian" 23 | ) 24 | args, unknown = parser.parse_known_args() 25 | 26 | # Load the template. 27 | template_path = pathlib.Path(args.gen_template) 28 | with open(template_path, "r") as template_file: 29 | template = template_file.read() 30 | 31 | # If the template does not contain a key in the config, the config value is 32 | # just ignored. 33 | config = { 34 | "range_min": args.gen_range_min, 35 | "range_max": args.gen_range_max, 36 | "num_requests": args.gen_num_requests, 37 | "distribution": ZIPFIAN_DIST 38 | if args.gen_distribution == "zipfian" 39 | else UNIFORM_DIST, 40 | } 41 | 42 | workload = template.format(**config) 43 | 44 | # Write out the generated workload. 45 | out_dir = cond.get_output_path() 46 | with open(out_dir / "workload.yml", "w") as file: 47 | file.write(workload) 48 | 49 | # Pass through unknown arguments. 50 | print(" ".join(unknown)) 51 | 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/a.yml: -------------------------------------------------------------------------------- 1 | # YCSB A 2 | # 50% Read, 50% Update 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | read: 14 | proportion_pct: 50 15 | distribution: 16 | {distribution} 17 | update: 18 | proportion_pct: 50 19 | distribution: 20 | {distribution} 21 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/b.yml: -------------------------------------------------------------------------------- 1 | # YCSB B 2 | # 95% Read, 5% Update 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | read: 14 | proportion_pct: 95 15 | distribution: 16 | {distribution} 17 | update: 18 | proportion_pct: 5 19 | distribution: 20 | {distribution} 21 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/c.yml: -------------------------------------------------------------------------------- 1 | # YCSB C 2 | # 100% Read 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | read: 14 | proportion_pct: 100 15 | distribution: 16 | {distribution} 17 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/d.yml: -------------------------------------------------------------------------------- 1 | # YCSB D 2 | # 95% Read Latest, 5% Insert 3 | 4 | # NOTE: This workload's request distribution is always fixed since it is a "read 5 | # latest" workload. Changing the read distribution to zipfian or uniform would 6 | # change the workload's semantics. 7 | 8 | load: 9 | num_records: 20000000 10 | distribution: 11 | type: uniform 12 | range_min: {range_min} 13 | range_max: {range_max} 14 | 15 | run: 16 | - num_requests: {num_requests} 17 | read: 18 | proportion_pct: 95 19 | distribution: 20 | type: latest 21 | theta: 0.99 22 | insert: 23 | proportion_pct: 5 24 | distribution: 25 | type: uniform 26 | range_min: {range_min} 27 | range_max: {range_max} 28 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/e.yml: -------------------------------------------------------------------------------- 1 | # YCSB E 2 | # 95% Scans of lengths 1 to 100 (uniformly chosen length), 5% Insert 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | insert: 14 | proportion_pct: 5 15 | distribution: 16 | type: uniform 17 | range_min: {range_min} 18 | range_max: {range_max} 19 | scan: 20 | proportion_pct: 95 21 | max_length: 100 22 | distribution: 23 | {distribution} 24 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/f.yml: -------------------------------------------------------------------------------- 1 | # YCSB F 2 | # 50% Read, 50% Read-modify-write 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | read: 14 | proportion_pct: 50 15 | distribution: 16 | {distribution} 17 | readmodifywrite: 18 | proportion_pct: 50 19 | distribution: 20 | {distribution} 21 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/scan_only.yml: -------------------------------------------------------------------------------- 1 | # YCSB "Scan Only" 2 | # 100% Scans of lengths 1 to 100 (uniformly chosen length). 3 | 4 | load: 5 | num_records: 20000000 6 | distribution: 7 | type: uniform 8 | range_min: {range_min} 9 | range_max: {range_max} 10 | 11 | run: 12 | - num_requests: {num_requests} 13 | scan: 14 | proportion_pct: 100 15 | max_length: 100 16 | distribution: 17 | {distribution} 18 | -------------------------------------------------------------------------------- /scripts/ycsb_v2/workloads/setup.yml: -------------------------------------------------------------------------------- 1 | load: 2 | num_records: 20000000 3 | distribution: 4 | type: uniform 5 | range_min: {range_min} 6 | range_max: {range_max} 7 | 8 | # Ensures that RocksDB's LSM tree has multiple levels. 9 | run: 10 | - num_requests: {num_requests} 11 | update: 12 | proportion_pct: 100 13 | distribution: 14 | type: uniform 15 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(GoogleTest) 2 | 3 | add_executable(tester) 4 | target_sources(tester 5 | PRIVATE 6 | buffer_manager_test.cc 7 | coding_test.cc 8 | db_test.cc 9 | file_manager_test.cc 10 | insert_tracker_test.cc 11 | manifest_test.cc 12 | memtable_test.cc 13 | packed_map_test.cc 14 | page_test.cc 15 | pg_datasets.cc 16 | pg_datasets.h 17 | pg_db_test.cc 18 | pg_lock_manager_test.cc 19 | pg_manager_rewrite_test.cc 20 | pg_manager_test.cc 21 | pg_segment_info_test.cc 22 | pg_segment_test.cc 23 | record_cache_test.cc 24 | thread_pool_test.cc 25 | wal_manager_test.cc 26 | wal_rw_test.cc 27 | ) 28 | target_link_libraries(tester 29 | PRIVATE 30 | treeline 31 | pg_treeline 32 | pg 33 | gtest 34 | gtest_main 35 | Threads::Threads 36 | crc32c 37 | ) 38 | # Allow tests to also include TreeLine internal headers 39 | target_include_directories(tester PRIVATE ${PROJECT_SOURCE_DIR}) 40 | 41 | # Register our tests with CTest 42 | gtest_add_tests(TARGET tester) 43 | -------------------------------------------------------------------------------- /tests/pg_datasets.cc: -------------------------------------------------------------------------------- 1 | #include "pg_datasets.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace tl { 10 | namespace pg { 11 | 12 | // Uniformly selects `num_samples` samples from the range [min_val, max_val] 13 | // without replacement. 14 | std::vector Datasets::FloydSample(const size_t num_samples, 15 | uint64_t min_val, uint64_t max_val, 16 | std::mt19937& rng) { 17 | std::unordered_set samples; 18 | samples.reserve(num_samples); 19 | for (uint64_t curr = max_val - num_samples + 1; curr <= max_val; ++curr) { 20 | std::uniform_int_distribution dist(min_val, curr); 21 | const uint64_t next = dist(rng); 22 | auto res = samples.insert(next); 23 | if (!res.second) { 24 | samples.insert(curr); 25 | } 26 | } 27 | assert(samples.size() == num_samples); 28 | return std::vector(samples.begin(), samples.end()); 29 | } 30 | 31 | const std::vector Datasets::kSequentialKeys = 32 | ([](const size_t range) { 33 | std::vector results; 34 | results.resize(range); 35 | std::iota(results.begin(), results.end(), 1ULL); 36 | return results; 37 | })(1000); 38 | 39 | const std::vector Datasets::kUniformKeys = 40 | ([](const size_t num_samples, uint64_t min_val, uint64_t max_val) { 41 | std::mt19937 prng(42); 42 | auto res = Datasets::FloydSample(num_samples, min_val, max_val, prng); 43 | std::sort(res.begin(), res.end()); 44 | return res; 45 | })(1000, 1, 1000000); 46 | 47 | } // namespace pg 48 | } // namespace tl 49 | -------------------------------------------------------------------------------- /tests/pg_datasets.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace tl { 8 | namespace pg { 9 | 10 | // Fixture datasets for the tests. 11 | class Datasets { 12 | public: 13 | static std::vector FloydSample(const size_t num_samples, 14 | uint64_t min_val, uint64_t max_val, 15 | std::mt19937& rng); 16 | 17 | static const std::vector kSequentialKeys; 18 | static const std::vector kUniformKeys; 19 | }; 20 | 21 | } // namespace pg 22 | } // namespace tl 23 | -------------------------------------------------------------------------------- /tests/pg_segment_info_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "gtest/gtest.h" 3 | #include "page_grouping/persist/segment_id.h" 4 | #include "page_grouping/plr/data.h" 5 | #include "page_grouping/segment_info.h" 6 | 7 | namespace { 8 | 9 | using namespace tl; 10 | using namespace tl::pg; 11 | 12 | TEST(SegmentInfoTest, Invalid) { 13 | SegmentInfo invalid; 14 | ASSERT_FALSE(invalid.id().IsValid()); 15 | } 16 | 17 | TEST(SegmentInfoTest, NoOverflowDefault) { 18 | SegmentId id(0, 16081); 19 | SegmentInfo info(id, std::optional()); 20 | ASSERT_FALSE(info.HasOverflow()); 21 | } 22 | 23 | TEST(SegmentInfoTest, NoOverflowDefault2) { 24 | SegmentId id(0, 125); 25 | SegmentInfo info(id, plr::Line64(1.0, 1.0)); 26 | ASSERT_FALSE(info.HasOverflow()); 27 | } 28 | 29 | TEST(SegmentInfoTest, SetOverflow) { 30 | SegmentId id(0, 100); 31 | SegmentInfo info(id, plr::Line64(1.0, 1.0)); 32 | ASSERT_FALSE(info.HasOverflow()); 33 | info.SetOverflow(true); 34 | ASSERT_TRUE(info.HasOverflow()); 35 | } 36 | 37 | } // namespace 38 | -------------------------------------------------------------------------------- /tests/thread_pool_test.cc: -------------------------------------------------------------------------------- 1 | #include "util/thread_pool.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | static int Square(int val) { return val * val; } 8 | 9 | static int SafeIntDivide(int a, int b) { 10 | if (b == 0) throw std::invalid_argument("Division by zero."); 11 | return a / b; 12 | } 13 | 14 | TEST(ThreadPoolTest, Submit) { 15 | tl::ThreadPool pool(2); 16 | auto f1 = pool.Submit(Square, 5); 17 | auto f2 = pool.Submit(Square, 1); 18 | auto f3 = pool.Submit(Square, 10); 19 | ASSERT_EQ(f1.get(), 25); 20 | ASSERT_EQ(f2.get(), 1); 21 | ASSERT_EQ(f3.get(), 100); 22 | } 23 | 24 | TEST(ThreadPoolTest, SubmitWithException) { 25 | tl::ThreadPool pool(3); 26 | auto f1 = pool.Submit(SafeIntDivide, 5, 2); 27 | auto f2 = pool.Submit(SafeIntDivide, 3, 0); 28 | auto f3 = pool.Submit(SafeIntDivide, 10, 2); 29 | ASSERT_EQ(f1.get(), 2); 30 | ASSERT_EQ(f3.get(), 5); // Can retrieve results out of order 31 | ASSERT_THROW(f2.get(), std::invalid_argument); 32 | 33 | auto f4 = pool.Submit(Square, 11); 34 | auto f5 = pool.Submit(SafeIntDivide, 100, 10); 35 | ASSERT_EQ(f4.get(), 121); 36 | ASSERT_EQ(f5.get(), 10); 37 | } 38 | 39 | TEST(ThreadPoolTest, SubmitNoWait) { 40 | int val1 = 1; 41 | int val2 = 10; 42 | { 43 | tl::ThreadPool pool(2); 44 | pool.SubmitNoWait([&val1]() { val1 += 1; }); 45 | pool.SubmitNoWait([&val2]() { val2 *= val2; }); 46 | } 47 | ASSERT_EQ(val1, 2); 48 | ASSERT_EQ(val2, 100); 49 | } 50 | -------------------------------------------------------------------------------- /third_party/leanstore/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: Chromium 3 | BreakBeforeBraces: Linux 4 | SpaceInEmptyParentheses: 'false' 5 | ColumnLimit: 150 6 | IndentWidth: 3 7 | ... 8 | -------------------------------------------------------------------------------- /third_party/leanstore/.github/workflows/ccpp.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Install dependencies 17 | run: sudo apt install -y libtbb-dev libaio-dev 18 | - name: cmake 19 | run: mkdir release && cd release && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo .. && make -j 20 | -------------------------------------------------------------------------------- /third_party/leanstore/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | bin/ 3 | build*/ 4 | cmake-*/ 5 | .DS_Store 6 | zipf_cache_* 7 | a.out 8 | data/ 9 | log/ 10 | adhoc/ 11 | #release/ 12 | #debug/ 13 | log.txt 14 | git-ignore 15 | *# 16 | debug* 17 | release* 18 | clang* 19 | paper* -------------------------------------------------------------------------------- /third_party/leanstore/.projectile: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /third_party/leanstore/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021, FSU Jena 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /third_party/leanstore/README.md: -------------------------------------------------------------------------------- 1 | **Note:** This is a fork of [LeanStore](https://github.com/leanstore/leanstore) 2 | from commit `d3d83143ee74c54c901fe5431512a46965377f4e` with a few patches to (i) 3 | set the page size and alignment to match TreeLine (4 KiB), and (ii) to enable 4 | compilation as a part of TreeLine's benchmarks (fixed some naming conflicts). 5 | 6 | ---------- 7 | 8 | # LeanStore 9 | [LeanStore](https://db.in.tum.de/~leis/papers/leanstore.pdf) is a high-performance OLTP storage engine optimized for many-core CPUs and NVMe SSDs. Our goal is to achieve performance comparable to in-memory systems when the data set fits into RAM, while being able to fully exploit the bandwidth of fast NVMe SSDs for large data sets. While LeanStore is currently a research prototype, we hope to make it usable in production in the future. 10 | 11 | ## Compiling 12 | Install dependencies: 13 | 14 | `sudo apt-get install cmake libaio-dev libtbb-dev` 15 | 16 | `mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo .. && make -j` 17 | 18 | ## TPC-C Example 19 | `build/frontend/tpcc --ssd_path=./ssd_block_device_or_file --worker_threads=120 --pp_threads=4 --dram_gib=240 --tpcc_warehouse_count=100 --notpcc_warehouse_affinity --csv_path=./log --cool_pct=40 --free_pct=1 --contention_split --xmerge --print_tx_console --run_for_seconds=60` 20 | 21 | check `build/frontend/tpcc --help` for other options 22 | 23 | ## Cite 24 | The code we used for our CIDR 2021 paper is in a different (and outdated) [branch](https://github.com/leanstore/leanstore/tree/cidr). 25 | 26 | ``` 27 | @inproceedings{alhomssi21, 28 | author = {Adnan Alhomssi and Viktor Leis}, 29 | title = {Contention and Space Management in B-Trees}, 30 | booktitle = {CIDR}, 31 | year = {2021} 32 | } 33 | ``` 34 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/concurrency-recovery/Transaction.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Exceptions.hpp" 3 | #include "Units.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | // ------------------------------------------------------------------------------------- 7 | namespace leanstore 8 | { 9 | namespace cr 10 | { 11 | // ------------------------------------------------------------------------------------- 12 | struct Transaction { 13 | enum class TYPE : u8 { USER, SYSTEM }; 14 | enum class STATE { IDLE, STARTED, READY_TO_COMMIT, COMMITED, ABORTED }; 15 | STATE state = STATE::IDLE; 16 | u64 tts = 0; 17 | LID min_gsn, max_gsn; 18 | }; 19 | // ------------------------------------------------------------------------------------- 20 | } // namespace cr 21 | } // namespace leanstore 22 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/CPUCounters.cpp: -------------------------------------------------------------------------------- 1 | #include "CPUCounters.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | std::mutex CPUCounters::mutex; 8 | u64 CPUCounters::id = 0; 9 | std::unordered_map CPUCounters::threads; 10 | // ------------------------------------------------------------------------------------- 11 | u64 CPUCounters::registerThread(string name, bool perf_inherit) 12 | { 13 | std::unique_lock guard(mutex); 14 | threads[id] = {.e = std::make_unique(perf_inherit), .name = name}; 15 | return id++; 16 | } 17 | void CPUCounters::removeThread(u64 id) 18 | { 19 | std::unique_lock guard(mutex); 20 | threads.erase(id); 21 | } 22 | } // namespace leanstore 23 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/CPUCounters.hpp: -------------------------------------------------------------------------------- 1 | #include "Units.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | #include "PerfEvent.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | #include 6 | #include 7 | #include 8 | #include 9 | // ------------------------------------------------------------------------------------- 10 | // Sadly, we can not use TBB enumerable thread specific here because it does not automatically remove its entry upon thread destroy 11 | namespace leanstore 12 | { 13 | struct CPUCounters { 14 | std::unique_ptr e; 15 | string name; 16 | // ------------------------------------------------------------------------------------- 17 | static u64 id; 18 | static std::unordered_map threads; 19 | static std::mutex mutex; 20 | static u64 registerThread(string name, bool perf_inherit = false); 21 | static void removeThread(u64 id); 22 | }; 23 | } // namespace leanstore 24 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/CRCounters.cpp: -------------------------------------------------------------------------------- 1 | #include "CRCounters.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | tbb::enumerable_thread_specific CRCounters::cr_counters; 8 | } // namespace leanstore 9 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/CRCounters.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Units.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | #include 5 | 6 | // ------------------------------------------------------------------------------------- 7 | #include 8 | // ------------------------------------------------------------------------------------- 9 | namespace leanstore 10 | { 11 | struct CRCounters { 12 | atomic worker_id = -1; 13 | atomic written_log_bytes = 0; 14 | atomic wal_reserve_blocked = 0; 15 | atomic wal_reserve_immediate = 0; 16 | // ------------------------------------------------------------------------------------- 17 | atomic gct_total_ms = 0; 18 | atomic gct_phase_1_ms = 0; 19 | atomic gct_phase_2_ms = 0; 20 | atomic gct_write_ms = 0; 21 | atomic gct_write_bytes = 0; 22 | // ------------------------------------------------------------------------------------- 23 | atomic gct_rounds = 0; 24 | atomic gct_committed_tx = 0; 25 | // ------------------------------------------------------------------------------------- 26 | CRCounters() {} 27 | // ------------------------------------------------------------------------------------- 28 | static tbb::enumerable_thread_specific cr_counters; 29 | static tbb::enumerable_thread_specific::reference myCounters() { return cr_counters.local(); } 30 | }; 31 | } // namespace leanstore 32 | // ------------------------------------------------------------------------------------- 33 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/PPCounters.cpp: -------------------------------------------------------------------------------- 1 | #include "PPCounters.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | tbb::enumerable_thread_specific PPCounters::pp_counters; 8 | } -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/counters/WorkerCounters.cpp: -------------------------------------------------------------------------------- 1 | #include "WorkerCounters.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | atomic WorkerCounters::workers_counter = 0; 8 | tbb::enumerable_thread_specific WorkerCounters::worker_counters; 9 | } // namespace leanstore 10 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/tables/BMTable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ProfilingTable.hpp" 3 | #include "leanstore/storage/buffer-manager/BufferManager.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | // ------------------------------------------------------------------------------------- 7 | namespace leanstore 8 | { 9 | namespace profiling 10 | { 11 | using namespace storage; 12 | class BMTable : public ProfilingTable 13 | { 14 | private: 15 | BufferManager& bm; 16 | s64 local_phase_1_ms = 0, local_phase_2_ms = 0, local_phase_3_ms = 0, local_poll_ms = 0, total; 17 | u64 local_total_free, local_total_cool; 18 | 19 | public: 20 | BMTable(BufferManager& bm); 21 | // ------------------------------------------------------------------------------------- 22 | virtual std::string getName(); 23 | virtual void open(); 24 | virtual void next(); 25 | }; 26 | } // namespace profiling 27 | } // namespace leanstore 28 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/tables/CPUTable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ProfilingTable.hpp" 3 | #include "leanstore/storage/buffer-manager/BufferManager.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | // ------------------------------------------------------------------------------------- 7 | namespace leanstore 8 | { 9 | namespace profiling 10 | { 11 | using namespace storage; 12 | class CPUTable : public ProfilingTable 13 | { 14 | public: 15 | std::unordered_map workers_agg_events, pp_agg_events, ww_agg_events; 16 | virtual std::string getName(); 17 | virtual void open(); 18 | virtual void next(); 19 | }; 20 | } // namespace profiling 21 | } // namespace leanstore 22 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/tables/CRTable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ProfilingTable.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | namespace leanstore 7 | { 8 | namespace profiling 9 | { 10 | class CRTable : public ProfilingTable 11 | { 12 | private: 13 | u64 wal_hits, wal_miss; 14 | double p1, p2, total, write, wal_total, wal_hit_pct, wal_miss_pct; 15 | 16 | public: 17 | virtual std::string getName(); 18 | virtual void open(); 19 | virtual void next(); 20 | }; 21 | } // namespace profiling 22 | } // namespace leanstore 23 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/tables/ConfigsTable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ProfilingTable.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | namespace leanstore 7 | { 8 | namespace profiling 9 | { 10 | class ConfigsTable : public ProfilingTable 11 | { 12 | public: 13 | virtual std::string getName(); 14 | virtual void open(); 15 | virtual void next(); 16 | u64 hash(); 17 | void add(string name, string value); 18 | }; 19 | } // namespace profiling 20 | } // namespace leanstore 21 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/profiling/tables/DTTable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "ProfilingTable.hpp" 3 | #include "leanstore/storage/buffer-manager/BufferManager.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | // ------------------------------------------------------------------------------------- 7 | namespace leanstore 8 | { 9 | namespace profiling 10 | { 11 | using namespace storage; 12 | class DTTable : public ProfilingTable 13 | { 14 | private: 15 | string dt_name; 16 | u64 dt_id; 17 | BufferManager& bm; 18 | 19 | public: 20 | DTTable(BufferManager& bm); 21 | // ------------------------------------------------------------------------------------- 22 | virtual std::string getName(); 23 | virtual void open(); 24 | virtual void next(); 25 | }; 26 | } // namespace profiling 27 | } // namespace leanstore 28 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/storage/btree/core/BTreeGenericIterator.cpp: -------------------------------------------------------------------------------- 1 | #include "BTreeGenericIterator.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | namespace storage 8 | { 9 | namespace btree 10 | { 11 | // ------------------------------------------------------------------------------------- 12 | 13 | // ------------------------------------------------------------------------------------- 14 | } 15 | } // namespace storage 16 | } // namespace leanstore 17 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/storage/buffer-manager/BufferFrame.cpp: -------------------------------------------------------------------------------- 1 | #include "BufferFrame.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | namespace leanstore 4 | { 5 | namespace storage 6 | { 7 | // ------------------------------------------------------------------------------------- 8 | } 9 | } // namespace leanstore 10 | // ------------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/storage/buffer-manager/DTTypes.hpp: -------------------------------------------------------------------------------- 1 | #include "Units.hpp" 2 | #pragma once 3 | namespace leanstore 4 | { 5 | namespace storage 6 | { 7 | // ------------------------------------------------------------------------------------- 8 | using DTType = u8; 9 | // enum class DTType : u8 { 10 | // BTREE, 11 | // LSM 12 | //}; 13 | // ------------------------------------------------------------------------------------- 14 | } // namespace storage 15 | } // namespace leanstore 16 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/storage/buffer-manager/FreeList.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "BufferFrame.hpp" 3 | #include "Units.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | #include 7 | // ------------------------------------------------------------------------------------- 8 | namespace leanstore 9 | { 10 | namespace storage 11 | { 12 | // ------------------------------------------------------------------------------------- 13 | struct FreeList { 14 | atomic head = nullptr; 15 | atomic counter = 0; 16 | BufferFrame& tryPop(JMUW>& lock); 17 | BufferFrame& pop(); 18 | void batchPush(BufferFrame* head, BufferFrame* tail, u64 counter); 19 | void push(BufferFrame& bf); 20 | }; 21 | // ------------------------------------------------------------------------------------- 22 | } // namespace storage 23 | } // namespace leanstore 24 | 25 | // ------------------------------------------------------------------------------------- 26 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/storage/buffer-manager/Swip.cpp: -------------------------------------------------------------------------------- 1 | #include "Swip.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | namespace leanstore 4 | { 5 | namespace storage 6 | { 7 | // ------------------------------------------------------------------------------------- 8 | } 9 | } // namespace leanstore 10 | // ------------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/FNVHash.cpp: -------------------------------------------------------------------------------- 1 | #include "FNVHash.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | namespace utils 8 | // ------------------------------------------------------------------------------------- 9 | { 10 | u64 FNV::hash(u64 val) 11 | { 12 | // from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash 13 | u64 hash_val = FNV_OFFSET_BASIS_64; 14 | for (int i = 0; i < 8; i++) { 15 | u64 octet = val & 0x00ff; 16 | val = val >> 8; 17 | 18 | hash_val = hash_val ^ octet; 19 | hash_val = hash_val * FNV_PRIME_64; 20 | } 21 | return hash_val; 22 | } 23 | // ------------------------------------------------------------------------------------- 24 | } // namespace utils 25 | } // namespace leanstore 26 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/FNVHash.hpp: -------------------------------------------------------------------------------- 1 | #include "Units.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | namespace utils 8 | { 9 | // ------------------------------------------------------------------------------------- 10 | class FNV 11 | { 12 | private: 13 | static constexpr u64 FNV_OFFSET_BASIS_64 = 0xCBF29CE484222325L; 14 | static constexpr u64 FNV_PRIME_64 = 1099511628211L; 15 | 16 | public: 17 | static u64 hash(u64 val); 18 | }; 19 | // ------------------------------------------------------------------------------------- 20 | } // namespace utils 21 | } // namespace leanstore 22 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/JumpMU.cpp: -------------------------------------------------------------------------------- 1 | #include "JumpMU.hpp" 2 | 3 | #include 4 | // ------------------------------------------------------------------------------------- 5 | namespace jumpmu 6 | { 7 | __thread int checkpoint_counter = 0; 8 | __thread jmp_buf env[JUMPMU_STACK_SIZE]; 9 | __thread int checkpoint_stacks_counter[JUMPMU_STACK_SIZE]; 10 | __thread void (*de_stack_arr[JUMPMU_STACK_SIZE])(void*); 11 | __thread void* de_stack_obj[JUMPMU_STACK_SIZE]; 12 | __thread int de_stack_counter = 0; 13 | __thread bool in_jump = false; 14 | void jump() 15 | { 16 | assert(checkpoint_counter > 0); 17 | assert(de_stack_counter >= 0); 18 | auto c_c_stacks_counter = checkpoint_stacks_counter[checkpoint_counter - 1]; 19 | if (de_stack_counter > c_c_stacks_counter) { 20 | int begin = de_stack_counter - 1; // inc 21 | int till = c_c_stacks_counter; // inc 22 | assert(begin >= 0); 23 | assert(till >= 0); 24 | in_jump = true; 25 | for (int i = begin; i >= till; i--) { 26 | de_stack_arr[i](de_stack_obj[i]); 27 | } 28 | in_jump = false; 29 | } 30 | auto& env_to_jump = jumpmu::env[jumpmu::checkpoint_counter - 1]; 31 | checkpoint_counter--; 32 | longjmp(env_to_jump, 1); 33 | } 34 | } // namespace jumpmu 35 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/MMapVector.cpp: -------------------------------------------------------------------------------- 1 | #include "FVector.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | namespace utils 8 | { 9 | void writeBinary(const char* pathname, std::vector& v) 10 | { 11 | std::cout << "Writing binary file : " << pathname << std::endl; 12 | int fd = open(pathname, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 13 | posix_check(fd != -1); 14 | uint64_t fileSize = 8 + 16 * v.size(); 15 | for (auto s : v) 16 | fileSize += s.size() + 1; 17 | posix_check(posix_fallocate(fd, 0, fileSize) == 0); 18 | auto data = reinterpret_cast::Data*>(mmap(nullptr, fileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); 19 | data->count = v.size(); 20 | posix_check(data != MAP_FAILED); 21 | uint64_t offset = 8 + 16 * v.size(); 22 | char* dst = reinterpret_cast(data); 23 | uint64_t slot = 0; 24 | for (auto s : v) { 25 | data->slot[slot].size = s.size(); 26 | data->slot[slot].offset = offset; 27 | memcpy(dst + offset, s.data(), s.size()); 28 | offset += s.size(); 29 | slot++; 30 | } 31 | posix_check(close(fd) == 0); 32 | } 33 | } // namespace utils 34 | } // namespace leanstore -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/Misc.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Units.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | #include 6 | #include 7 | // ------------------------------------------------------------------------------------- 8 | namespace leanstore 9 | { 10 | namespace utils 11 | { 12 | // ------------------------------------------------------------------------------------- 13 | u32 getBitsNeeded(u64 input); 14 | // ------------------------------------------------------------------------------------- 15 | double calculateMTPS(std::chrono::high_resolution_clock::time_point begin, std::chrono::high_resolution_clock::time_point end, u64 factor); 16 | // ------------------------------------------------------------------------------------- 17 | void pinThisThreadRome(); 18 | void pinThisThreadRome(const u64 t_i); 19 | void pinThisThread(const u64 t_i); 20 | // ------------------------------------------------------------------------------------- 21 | void printBackTrace(); 22 | // ------------------------------------------------------------------------------------- 23 | inline u64 upAlign(u64 x) 24 | { 25 | return (x + 511) & ~511ul; 26 | } 27 | // ------------------------------------------------------------------------------------- 28 | inline u64 downAlign(u64 x) 29 | { 30 | return x - (x & 511); 31 | } 32 | // ------------------------------------------------------------------------------------- 33 | u32 CRC(const u8* src, u64 size); 34 | // ------------------------------------------------------------------------------------- 35 | } // namespace utils 36 | } // namespace leanstore 37 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/Parallelize.hpp: -------------------------------------------------------------------------------- 1 | #include "Units.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | #include 5 | // ------------------------------------------------------------------------------------- 6 | namespace leanstore 7 | { 8 | namespace utils 9 | { 10 | class Parallelize 11 | { 12 | public: 13 | static void parallelRange(u64 n, std::function); 14 | // [begin, end] 15 | static void parallelRange(u64 begin, u64 end, u64 n_threads, std::function); 16 | }; 17 | } // namespace utils 18 | } // namespace leanstore 19 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/ScrambledZipfGenerator.cpp: -------------------------------------------------------------------------------- 1 | #include "ScrambledZipfGenerator.hpp" 2 | // ------------------------------------------------------------------------------------- 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | namespace leanstore 6 | { 7 | namespace utils 8 | { 9 | // ------------------------------------------------------------------------------------- 10 | u64 ScrambledZipfGenerator::rand() 11 | { 12 | u64 zipf_value = zipf_generator.rand(); 13 | return min + (FNV::hash(zipf_value) % n); 14 | } 15 | // ------------------------------------------------------------------------------------- 16 | } // namespace utils 17 | } // namespace leanstore 18 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/ScrambledZipfGenerator.hpp: -------------------------------------------------------------------------------- 1 | #include "FNVHash.hpp" 2 | #include "Units.hpp" 3 | #include "ZipfGenerator.hpp" 4 | // ------------------------------------------------------------------------------------- 5 | // ------------------------------------------------------------------------------------- 6 | // ------------------------------------------------------------------------------------- 7 | namespace leanstore 8 | { 9 | namespace utils 10 | { 11 | // ------------------------------------------------------------------------------------- 12 | class ScrambledZipfGenerator 13 | { 14 | public: 15 | u64 min, max, n; 16 | double theta; 17 | ZipfGenerator zipf_generator; 18 | // 10000000000ul 19 | // [min, max) 20 | ScrambledZipfGenerator(u64 min, u64 max, double theta) : min(min), max(max), n(max - min), zipf_generator((max - min) * 2, theta) {} 21 | u64 rand(); 22 | }; 23 | // ------------------------------------------------------------------------------------- 24 | } // namespace utils 25 | } // namespace leanstore 26 | -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/StringsManipulation.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mitdbg/treeline/fcc220ea54beb3c951c80b9a2815ecb4292ef0be/third_party/leanstore/backend/leanstore/utils/StringsManipulation.hpp -------------------------------------------------------------------------------- /third_party/leanstore/backend/leanstore/utils/ZipfGenerator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "Units.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | // ------------------------------------------------------------------------------------- 5 | #include 6 | #include 7 | // ------------------------------------------------------------------------------------- 8 | namespace leanstore 9 | { 10 | namespace utils 11 | { 12 | // ------------------------------------------------------------------------------------- 13 | // A Zipf distributed random number generator 14 | // Based on Jim Gray Algorithm as described in "Quickly Generating Billion-Record..." 15 | // ------------------------------------------------------------------------------------- 16 | class ZipfGenerator 17 | { 18 | // ------------------------------------------------------------------------------------- 19 | private: 20 | u64 n; 21 | double theta; 22 | // ------------------------------------------------------------------------------------- 23 | double alpha, zetan, eta; 24 | // ------------------------------------------------------------------------------------- 25 | double zeta(u64 n, double theta); 26 | 27 | public: 28 | // [0, n) 29 | ZipfGenerator(uint64_t ex_n, double theta); 30 | // uint64_t rand(u64 new_n); 31 | uint64_t rand(); 32 | }; 33 | // ------------------------------------------------------------------------------------- 34 | } // namespace utils 35 | } // namespace leanstore 36 | // ------------------------------------------------------------------------------------- 37 | -------------------------------------------------------------------------------- /third_party/leanstore/frontend/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(frontend frontend.cpp) 2 | add_dependencies(frontend leanstore) 3 | target_link_libraries(frontend leanstore) 4 | 5 | 6 | add_executable(leanstore_ycsb ycsb/ycsb.cpp) 7 | target_link_libraries(leanstore_ycsb leanstore Threads::Threads) 8 | target_include_directories(leanstore_ycsb PRIVATE ${SHARED_INCLUDE_DIRECTORY}) 9 | 10 | add_executable(tpcc tpc-c/tpcc.cpp) 11 | target_link_libraries(tpcc leanstore Threads::Threads) 12 | target_include_directories(tpcc PRIVATE ${SHARED_INCLUDE_DIRECTORY}) 13 | -------------------------------------------------------------------------------- /third_party/leanstore/frontend/frontend.cpp: -------------------------------------------------------------------------------- 1 | #include "Units.hpp" 2 | #include "leanstore/LeanStore.hpp" 3 | // ------------------------------------------------------------------------------------- 4 | #include 5 | // ------------------------------------------------------------------------------------- 6 | #include 7 | // ------------------------------------------------------------------------------------- 8 | using namespace leanstore; 9 | int main(int argc, char** argv) 10 | { 11 | gflags::SetUsageMessage("Leanstore Frontend"); 12 | gflags::ParseCommandLineFlags(&argc, &argv, true); 13 | // ------------------------------------------------------------------------------------- 14 | return 0; 15 | } -------------------------------------------------------------------------------- /third_party/leanstore/libs/benchmark.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get gflags 9 | ExternalProject_Add( 10 | benchmark_src 11 | PREFIX "vendor/gbenchmark" 12 | GIT_REPOSITORY "https://github.com/google/benchmark.git" 13 | GIT_TAG 336bb8db986cc52cdf0cefa0a7378b9567d1afee 14 | TIMEOUT 10 15 | CMAKE_ARGS 16 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/gbenchmark 17 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 18 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 19 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 20 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 21 | UPDATE_COMMAND "" 22 | ) 23 | 24 | # Prepare gflags 25 | ExternalProject_Get_Property(benchmark_src install_dir) 26 | set(BENCHMARK_INCLUDE_DIR ${install_dir}/include) 27 | set(BENCHMARK_LIBRARY_PATH ${install_dir}/lib/libbenchmark.a) 28 | file(MAKE_DIRECTORY ${BENCHMARK_INCLUDE_DIR}) 29 | add_library(gbenchmark STATIC IMPORTED) 30 | set_property(TARGET gbenchmark PROPERTY IMPORTED_LOCATION ${BENCHMARK_LIBRARY_PATH}) 31 | set_property(TARGET gbenchmark APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${BENCHMARK_INCLUDE_DIR}) 32 | 33 | # Dependencies 34 | add_dependencies(gbenchmark benchmark_src) 35 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/croaring.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get croaring 9 | ExternalProject_Add( 10 | croaring_src 11 | PREFIX "vendor/croaring" 12 | GIT_REPOSITORY "https://github.com/RoaringBitmap/CRoaring.git" 13 | GIT_TAG b88b002407b42fafaea23ea5009a54a24d1c1ed4 14 | TIMEOUT 10 15 | CMAKE_ARGS 16 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/croaring 17 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 18 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 19 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 20 | UPDATE_COMMAND "" 21 | ) 22 | 23 | # Prepare croaring 24 | ExternalProject_Get_Property(croaring_src install_dir) 25 | set(CROARING_INCLUDE_DIR ${install_dir}/include) 26 | set(CROARING_LIBRARY_PATH ${install_dir}/lib/libroaring.so) 27 | file(MAKE_DIRECTORY ${CROARING_INCLUDE_DIR}) 28 | add_library(croaring SHARED IMPORTED) 29 | set_property(TARGET croaring PROPERTY IMPORTED_LOCATION ${CROARING_LIBRARY_PATH}) 30 | set_property(TARGET croaring APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CROARING_INCLUDE_DIR}) 31 | 32 | # Dependencies 33 | add_dependencies(croaring croaring_src) 34 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/fastpfor.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get rapidjson 9 | ExternalProject_Add( 10 | fastpfor_src 11 | PREFIX "vendor/lemire/fastpfor" 12 | GIT_REPOSITORY "https://github.com/lemire/FastPFor.git" 13 | GIT_TAG dfc88a9f775958f325982a2699db08324fca0568 14 | TIMEOUT 10 15 | BUILD_COMMAND make FastPFor 16 | UPDATE_COMMAND "" # to prevent rebuilding everytime 17 | INSTALL_COMMAND "" 18 | CMAKE_ARGS 19 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/fastpfor_cpp 20 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 21 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 22 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 23 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 24 | ) 25 | 26 | # Prepare json 27 | ExternalProject_Get_Property(fastpfor_src source_dir) 28 | ExternalProject_Get_Property(fastpfor_src binary_dir) 29 | 30 | set(FASTPFOR_INCLUDE_DIR ${source_dir}/) 31 | set(FASTPFOR_LIBRARY_PATH ${binary_dir}/libFastPFor.a) 32 | 33 | file(MAKE_DIRECTORY ${FASTPFOR_INCLUDE_DIR}) 34 | 35 | add_library(fastpfor STATIC IMPORTED) 36 | add_dependencies(fastpfor fastpfor_src) 37 | 38 | set_property(TARGET fastpfor PROPERTY IMPORTED_LOCATION ${FASTPFOR_LIBRARY_PATH}) 39 | set_property(TARGET fastpfor APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${FASTPFOR_INCLUDE_DIR}) 40 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/gdouble.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Import google double converisons https://github.com/google/double-conversion 9 | ExternalProject_Add(gdouble_src 10 | PREFIX vendor/gdouble 11 | GIT_REPOSITORY https://github.com/google/double-conversion.git 12 | GIT_TAG cb5cf996d123014a2420c853c4db60e4500973b1 13 | URL_MD5 ${GDOUGDOUBLE_URL_MD5} 14 | CONFIGURE_COMMAND "cmake" "." "-DBUILD_TESTING=ON" 15 | BUILD_COMMAND make 16 | BUILD_IN_SOURCE 1 17 | INSTALL_COMMAND "" 18 | LOG_DOWNLOAD 1 19 | LOG_BUILD 1 20 | UPDATE_COMMAND "" # to prevent rebuilding everytime 21 | STEP_TARGETS ${GDOUBLE_PREFIX}_info ${GDOUBLE_EXAMPLES_STEP} 22 | ) 23 | 24 | ExternalProject_Get_property(gdouble_src SOURCE_DIR) 25 | set(GDOUBLE_ROOT_DIR ${SOURCE_DIR}) 26 | 27 | # Prepare gdouble 28 | set(GFLAGS_INCLUDE_DIR ${GDOUBLE_ROOT_DIR}) 29 | set(GFLAGS_LIBRARY_PATH ${GDOUBLE_ROOT_DIR}/libdouble-conversion.a) 30 | file(MAKE_DIRECTORY ${GFLAGS_INCLUDE_DIR}) 31 | add_library(gdouble STATIC IMPORTED) 32 | set_property(TARGET gdouble PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARY_PATH}) 33 | set_property(TARGET gdouble APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}) 34 | 35 | # Dependencies 36 | add_dependencies(gdouble gdouble_src) 37 | message(STATUS "Tata" ${GFLAGS_INCLUDE_DIR}) -------------------------------------------------------------------------------- /third_party/leanstore/libs/gflags.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get gflags 9 | ExternalProject_Add( 10 | gflags_src 11 | PREFIX "vendor/gflags" 12 | GIT_REPOSITORY "https://github.com/gflags/gflags.git" 13 | GIT_TAG f8a0efe03aa69b3336d8e228b37d4ccb17324b88 14 | TIMEOUT 10 15 | CMAKE_ARGS 16 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/gflags 17 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 18 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 19 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 20 | UPDATE_COMMAND "" 21 | ) 22 | 23 | # Prepare gflags 24 | ExternalProject_Get_Property(gflags_src install_dir) 25 | set(GFLAGS_INCLUDE_DIR ${install_dir}/include) 26 | set(GFLAGS_LIBRARY_PATH ${install_dir}/lib/libgflags.a) 27 | file(MAKE_DIRECTORY ${GFLAGS_INCLUDE_DIR}) 28 | add_library(gflags STATIC IMPORTED) 29 | set_property(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARY_PATH}) 30 | set_property(TARGET gflags APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}) 31 | 32 | # Dependencies 33 | add_dependencies(gflags gflags_src) 34 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/googletest.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | find_package(Threads REQUIRED) 8 | 9 | # Get googletest 10 | ExternalProject_Add( 11 | googletest 12 | PREFIX "vendor/gtm" 13 | GIT_REPOSITORY "https://github.com/google/googletest.git" 14 | GIT_TAG release-1.8.0 15 | TIMEOUT 10 16 | CONFIGURE_COMMAND "" 17 | BUILD_COMMAND "" 18 | INSTALL_COMMAND "" 19 | UPDATE_COMMAND "" 20 | ) 21 | 22 | # Build gtest 23 | ExternalProject_Add( 24 | gtest_src 25 | PREFIX "vendor/gtm" 26 | SOURCE_DIR "vendor/gtm/src/googletest/googletest" 27 | INSTALL_DIR "vendor/gtm/gtest" 28 | CMAKE_ARGS 29 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/gtm/gtest 30 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 31 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 32 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 33 | DOWNLOAD_COMMAND "" 34 | UPDATE_COMMAND "" 35 | ) 36 | 37 | # Prepare gtest 38 | ExternalProject_Get_Property(gtest_src install_dir) 39 | set(GTEST_INCLUDE_DIR ${install_dir}/include) 40 | set(GTEST_LIBRARY_PATH ${install_dir}/lib/libgtest.a) 41 | file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIR}) 42 | add_library(gtest STATIC IMPORTED) 43 | set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARY_PATH}) 44 | set_property(TARGET gtest APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIR}) 45 | 46 | 47 | # Dependencies 48 | add_dependencies(gtest_src googletest) 49 | add_dependencies(gtest gtest_src) 50 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/lz4.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | find_program(MAKE_EXE NAMES gmake nmake make) 8 | 9 | # Get lz4 10 | ExternalProject_Add( 11 | lz4_src 12 | PREFIX "vendor/lz4" 13 | GIT_REPOSITORY "https://github.com/lz4/lz4.git" 14 | GIT_TAG 798301b4e144fab5d25fc34566c1419685f5f1eb 15 | TIMEOUT 10 16 | CONFIGURE_COMMAND "" 17 | BUILD_IN_SOURCE TRUE 18 | BUILD_COMMAND make 19 | UPDATE_COMMAND "" 20 | INSTALL_COMMAND "" 21 | ) 22 | 23 | # Prepare lz4 24 | ExternalProject_Get_Property(lz4_src source_dir) 25 | set(lz4_INCLUDE_DIR ${source_dir}/lib) 26 | set(lz4_LIBRARY_PATH ${source_dir}/lib/liblz4.a) 27 | file(MAKE_DIRECTORY ${lz4_INCLUDE_DIR}) 28 | add_library(lz4 STATIC IMPORTED) 29 | 30 | set_property(TARGET lz4 PROPERTY IMPORTED_LOCATION ${lz4_LIBRARY_PATH}) 31 | set_property(TARGET lz4 APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIR}) 32 | 33 | # Dependencies 34 | add_dependencies(lz4 lz4_src) 35 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/psql.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get psql 9 | ExternalProject_Add( 10 | psql_src 11 | PREFIX "vendor/psql" 12 | GIT_REPOSITORY "https://github.com/jtv/libpqxx.git" 13 | GIT_TAG 520113bc790d5f44726830c23018668cb5241e67 14 | TIMEOUT 10 15 | CMAKE_ARGS 16 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/psql 17 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 18 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 19 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 20 | UPDATE_COMMAND "" 21 | ) 22 | 23 | # Prepare psql 24 | ExternalProject_Get_Property(psql_src install_dir) 25 | set(psql_INCLUDE_DIR ${install_dir}/include) 26 | set(psql_LIBRARY_PATH ${install_dir}/lib/libpqxx.so) 27 | file(MAKE_DIRECTORY ${psql_INCLUDE_DIR}) 28 | add_library(psql SHARED IMPORTED) 29 | set_property(TARGET psql PROPERTY IMPORTED_LOCATION ${psql_LIBRARY_PATH}) 30 | set_property(TARGET psql APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${psql_INCLUDE_DIR}) 31 | 32 | # Dependencies 33 | add_dependencies(psql psql_src) 34 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/rapidjson.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get rapidjson 9 | ExternalProject_Add( 10 | rapidjson_src 11 | PREFIX "vendor/rapidjson" 12 | GIT_REPOSITORY "https://github.com/tencent/rapidjson" 13 | GIT_TAG v1.1.0 14 | TIMEOUT 10 15 | CONFIGURE_COMMAND "" 16 | BUILD_COMMAND "" 17 | INSTALL_COMMAND "" 18 | UPDATE_COMMAND "" 19 | ) 20 | 21 | # Prepare json 22 | ExternalProject_Get_Property(rapidjson_src source_dir) 23 | set(RAPIDJSON_INCLUDE_DIR ${source_dir}/include) 24 | file(MAKE_DIRECTORY ${RAPIDJSON_INCLUDE_DIR}) 25 | add_library(rapidjson INTERFACE) 26 | set_property(TARGET rapidjson APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${RAPIDJSON_INCLUDE_DIR}) 27 | 28 | # Dependencies 29 | add_dependencies(rapidjson rapidjson_src) 30 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/spdlog.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get rapidjson 9 | ExternalProject_Add( 10 | spdlog_src 11 | PREFIX "vendor/spdlog" 12 | GIT_REPOSITORY "https://github.com/gabime/spdlog.git" 13 | GIT_TAG 366935142780e3754b358542f0fd5ed83793b263 14 | TIMEOUT 10 15 | BUILD_COMMAND "" 16 | UPDATE_COMMAND "" # to prevent rebuilding everytime 17 | INSTALL_COMMAND "" 18 | CMAKE_ARGS 19 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/spdlog_cpp 20 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 21 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 22 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 23 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 24 | ) 25 | 26 | # Prepare json 27 | ExternalProject_Get_Property(spdlog_src source_dir) 28 | ExternalProject_Get_Property(spdlog_src binary_dir) 29 | set(SPDLOG_INCLUDE_DIR ${source_dir}/include/) 30 | file(MAKE_DIRECTORY ${SPDLOG_INCLUDE_DIR}) 31 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/tabluate.cmake: -------------------------------------------------------------------------------- 1 | include(ExternalProject) 2 | find_package(Git REQUIRED) 3 | 4 | ExternalProject_Add( 5 | tabluate_src 6 | PREFIX "vendor/tabluate" 7 | GIT_REPOSITORY "https://github.com/p-ranav/tabulate.git" 8 | GIT_TAG 718d827cf05c2e9bba17e926cac2d7ab2356621c 9 | TIMEOUT 10 10 | BUILD_COMMAND "" 11 | UPDATE_COMMAND "" # to prevent rebuilding everytime 12 | CMAKE_ARGS 13 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/tabulate 14 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 15 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 16 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 17 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 18 | ) 19 | 20 | ExternalProject_Get_Property(tabluate_src source_dir) 21 | set(TABLUATE_INCLUDE_DIR ${source_dir}/include) 22 | file(MAKE_DIRECTORY ${TABLUATE_INCLUDE_DIR}) 23 | add_library(tabluate INTERFACE IMPORTED) 24 | set_property(TARGET tabluate APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TABLUATE_INCLUDE_DIR}) 25 | add_dependencies(tabluate tabluate_src) 26 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/tbb.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get rapidjson 9 | ExternalProject_Add( 10 | tbb_src 11 | PREFIX "vendor/intel/tbb" 12 | GIT_REPOSITORY "https://github.com/wjakob/tbb.git" 13 | GIT_TAG b066defc0229a1e92d7a200eb3fe0f7e35945d95 14 | TIMEOUT 10 15 | BUILD_COMMAND make 16 | UPDATE_COMMAND "" # to prevent rebuilding everytime 17 | INSTALL_COMMAND "" 18 | CMAKE_ARGS 19 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/tbb_cpp 20 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 21 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 22 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 23 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 24 | ) 25 | 26 | # Prepare json 27 | ExternalProject_Get_Property(tbb_src source_dir) 28 | ExternalProject_Get_Property(tbb_src binary_dir) 29 | 30 | set(TBB_INCLUDE_DIR ${source_dir}/include) 31 | set(TBB_LIBRARY_PATH ${binary_dir}/libtbb.so) 32 | 33 | file(MAKE_DIRECTORY ${TBB_INCLUDE_DIR}) 34 | 35 | add_library(tbb SHARED IMPORTED) 36 | add_dependencies(tbb tbb_src) 37 | 38 | set_property(TARGET tbb PROPERTY IMPORTED_LOCATION ${TBB_LIBRARY_PATH}) 39 | set_property(TARGET tbb APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIR}) 40 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/turbo.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | find_program(MAKE_EXE NAMES gmake nmake make) 8 | 9 | # Get turbo 10 | ExternalProject_Add( 11 | turbo_src 12 | PREFIX "vendor/turbo" 13 | GIT_REPOSITORY "https://github.com/powturbo/TurboPFor" 14 | GIT_TAG 58f49912603d85b60b176b8e5e905f69c5a259a8 15 | TIMEOUT 10 16 | CONFIGURE_COMMAND "" 17 | BUILD_IN_SOURCE TRUE 18 | BUILD_COMMAND AVX2=1 make -j20 COMMAND ar rcs libic.a bitpack_avx2.o bitpack_sse.o bitunpack.o bitutil.o fp.o icbench.o idxqry.o plugins.o transpose.o vint.o vp4c.o vp4d_avx2.o vp4d_sse.o bitpack.o bitunpack_avx2.o bitunpack_sse.o eliasfano.o icapp.o idxcr.o idxseg.o transpose_avx2.o transpose_sse.o vp4c_avx2.o vp4c_sse.o vp4d.o vsimple.o 19 | UPDATE_COMMAND "" 20 | INSTALL_COMMAND "" 21 | ) 22 | 23 | # Prepare turbo 24 | ExternalProject_Get_Property(turbo_src source_dir) 25 | set(TURBO_INCLUDE_DIR ${source_dir}/) 26 | set(TURBO_LIBRARY_PATH ${source_dir}/libic.a) 27 | file(MAKE_DIRECTORY ${TURBO_INCLUDE_DIR}) 28 | add_library(turbo STATIC IMPORTED) 29 | 30 | set_property(TARGET turbo PROPERTY IMPORTED_LOCATION ${TURBO_LIBRARY_PATH}) 31 | set_property(TARGET turbo APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TURBO_INCLUDE_DIR}) 32 | 33 | # Dependencies 34 | add_dependencies(turbo turbo_src) 35 | -------------------------------------------------------------------------------- /third_party/leanstore/libs/yaml-cpp.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | include(ExternalProject) 6 | find_package(Git REQUIRED) 7 | 8 | # Get rapidjson 9 | ExternalProject_Add( 10 | yaml_src 11 | PREFIX "vendor/yaml_cpp" 12 | GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git" 13 | GIT_TAG bd7f8c60c82614bb0bd1c526db2cbc39dac02fec 14 | TIMEOUT 10 15 | BUILD_COMMAND make 16 | UPDATE_COMMAND "" # to prevent rebuilding everytime 17 | CMAKE_ARGS 18 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/vendor/yaml_cpp 19 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 20 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 21 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 22 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 23 | ) 24 | 25 | # Prepare json 26 | ExternalProject_Get_Property(yaml_src source_dir) 27 | ExternalProject_Get_Property(yaml_src binary_dir) 28 | 29 | set(YAML_INCLUDE_DIR ${source_dir}/include) 30 | set(YAML_LIBRARY_PATH ${binary_dir}/libyaml-cpp.a) 31 | 32 | file(MAKE_DIRECTORY ${YAML_INCLUDE_DIR}) 33 | 34 | add_library(yaml STATIC IMPORTED) 35 | add_dependencies(yaml yaml_src) 36 | 37 | set_property(TARGET yaml PROPERTY IMPORTED_LOCATION ${YAML_LIBRARY_PATH}) 38 | set_property(TARGET yaml APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${YAML_INCLUDE_DIR}) 39 | -------------------------------------------------------------------------------- /third_party/leanstore/shared-headers/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # cengine 3 | # --------------------------------------------------------------------------- 4 | 5 | # --------------------------------------------------------------------------- 6 | # Files 7 | # --------------------------------------------------------------------------- 8 | 9 | set(SHARED_INCLUDE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) 10 | -------------------------------------------------------------------------------- /third_party/masstree/AUTHORS: -------------------------------------------------------------------------------- 1 | Eddie Kohler 2 | kohler@seas.harvard.edu 3 | 4 | Yandong Mao 5 | ydmao@csail.mit.edu 6 | 7 | Robert Morris 8 | rtm@csail.mit.edu 9 | -------------------------------------------------------------------------------- /third_party/masstree/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_library( 2 | libjemalloc 3 | jemalloc 4 | HINTS 5 | $ENV{HOME}/lib 6 | $ENV{HOME}/opt/lib 7 | REQUIRED) 8 | 9 | set(masstree_sources 10 | checkpoint.cc 11 | clp.c 12 | compiler.cc 13 | file.cc 14 | json.cc 15 | kvio.cc 16 | kvrandom.cc 17 | kvthread.cc 18 | log.cc 19 | memdebug.cc 20 | misc.cc 21 | msgpack.cc 22 | perfstat.cc 23 | query_masstree.cc 24 | str.cc 25 | straccum.cc 26 | string_slice.cc 27 | string.cc 28 | value_array.cc 29 | value_string.cc 30 | value_versioned_array.cc 31 | ) 32 | 33 | set(common_optimization_options 34 | -funroll-loops 35 | -fno-omit-frame-pointer) 36 | 37 | # Configure masstree to generate config.h 38 | get_filename_component(libjemalloc_dir ${libjemalloc} DIRECTORY) 39 | add_custom_command( 40 | OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/config.h 41 | COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && 42 | ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.sh && 43 | ${CMAKE_CURRENT_SOURCE_DIR}/configure 44 | --with-malloc=jemalloc 45 | --disable-assertions 46 | --disable-invariants 47 | --disable-preconditions 48 | --enable-max-key-len=1024 49 | LDFLAGS=-L${libjemalloc_dir} 50 | VERBATIM) 51 | 52 | add_library(masstree STATIC) 53 | set_source_files_properties(clp.c PROPERTIES LANGUAGE CXX ) 54 | target_sources(masstree PRIVATE ${masstree_sources} ${CMAKE_CURRENT_SOURCE_DIR}/config.h) 55 | target_compile_options(masstree 56 | PRIVATE 57 | -include ${CMAKE_CURRENT_SOURCE_DIR}/config.h 58 | ${common_optimization_options}) 59 | -------------------------------------------------------------------------------- /third_party/masstree/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh -e 2 | 3 | autoreconf -i 4 | 5 | echo "Now, run ./configure." 6 | -------------------------------------------------------------------------------- /third_party/masstree/checkpoint.cc: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #include "checkpoint.hh" 17 | 18 | // add one key/value to a checkpoint. 19 | // called by checkpoint_tree() for each node. 20 | bool ckstate::visit_value(Str key, const row_type* value, threadinfo&) { 21 | if (endkey && key >= endkey) 22 | return false; 23 | if (!row_is_marker(value)) { 24 | msgpack::unparser up(*vals); 25 | up.write(key).write_wide(value->timestamp()); 26 | value->checkpoint_write(up); 27 | ++count; 28 | } 29 | return true; 30 | } 31 | -------------------------------------------------------------------------------- /third_party/masstree/doc/.gitignore: -------------------------------------------------------------------------------- 1 | *.aux 2 | *.bbl 3 | *.blg 4 | *.dvi 5 | *.log 6 | *.mpx 7 | *_[0-9]*.eps 8 | *_[0-9]*.pdf 9 | !GNUmakefile 10 | mpxerr.tex 11 | spec.pdf 12 | -------------------------------------------------------------------------------- /third_party/masstree/doc/patches.mp: -------------------------------------------------------------------------------- 1 | input elements; 2 | 3 | def thediamond_(suffix $) = 4 | $.e -- $.n -- $.w -- $s -- cycle 5 | enddef; 6 | 7 | vardef sizediamond_(suffix $) = 8 | save a_, b_; 9 | (a_,b_) = .5*(urcorner pic_$ - llcorner pic_$); 10 | if (unknown $dy or unknown $dx) and unknown $squatness: $squatness = 1; fi 11 | if unknown $dy: $dy = $squatness * ($dx + a_) - b_; fi 12 | if unknown $dy: 13 | $dy = b_ + circmargin * sqrt($squatness+1) + $squatness * a_; 14 | fi 15 | sizeelement_($); 16 | enddef; 17 | 18 | vardef cleardiamond_(suffix $) = 19 | _n_ := str $; 20 | generic_redeclare(numeric) _n.squatness; 21 | enddef; 22 | 23 | vardef patchit@#(expr s, type) = 24 | _elementit.@#(s, 0, 0, push, false, false); 25 | if type = 0: 26 | pproc_.@# := "thecirc_"; 27 | % sproc_.@# := "sizecirc_"; 28 | elseif type = 2: 29 | _n_ := str @#; 30 | generic_declare(numeric) _n.squatness, _n.dx, _n.dy; 31 | pproc_.@# := "thediamond_"; 32 | sproc_.@# := "sizediamond_"; 33 | expandafter def expandafter clearboxes expandafter = 34 | clearboxes cleardiamond_(@#); 35 | enddef 36 | elseif type = 3: 37 | pproc_.@# := "rboxpath_"; 38 | fi; 39 | enddef; 40 | 41 | vardef drawconnpatch(suffix $,$$)(text conn) text rest = 42 | interim linejoin := mitered; 43 | drawarrow ($.c conn $$.c) cutbefore bpath.$ cutafter bpath.$$ withpen connectionpen rest 44 | enddef; 45 | -------------------------------------------------------------------------------- /third_party/masstree/kvproto.hh: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #ifndef KVPROTO_HH 17 | #define KVPROTO_HH 18 | #include "compiler.hh" 19 | 20 | enum { 21 | Cmd_None = 0, 22 | Cmd_Get = 2, 23 | Cmd_Scan = 4, 24 | Cmd_Put = 6, 25 | Cmd_Replace = 8, 26 | Cmd_Remove = 10, 27 | Cmd_Checkpoint = 12, 28 | Cmd_Handshake = 14, 29 | Cmd_Max 30 | }; 31 | 32 | enum result_t { 33 | NotFound = -2, 34 | Retry, 35 | OutOfDate, 36 | Inserted, 37 | Updated, 38 | Found, 39 | ScanDone 40 | }; 41 | 42 | enum ckptrav_order_t { 43 | ckptrav_inorder = 0, 44 | ckptrav_preorder 45 | }; 46 | 47 | struct row_marker { 48 | enum { mt_remove = 1, mt_delta = 2 }; 49 | int marker_type_; 50 | }; 51 | 52 | template 53 | inline bool row_is_marker(const R* row) { 54 | return row->timestamp() & 1; 55 | } 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /third_party/masstree/kvrandom.cc: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #include "kvrandom.hh" 17 | #include "compiler.hh" 18 | #include 19 | 20 | const uint32_t kvrandom_psdes_nr::c1[] = { 21 | 0xBAA96887U, 0x1E17D32CU, 0x03BCDC3CU, 0x0F33D1B2U 22 | }; 23 | const uint32_t kvrandom_psdes_nr::c2[] = { 24 | 0x4B0F3B58U, 0xE874F0C3U, 0x6955C5A6U, 0x55A7CA46U 25 | }; 26 | 27 | uint32_t kvrandom_psdes_nr::psdes(uint32_t lword, uint32_t irword) { 28 | for (int i = 0; i < niter; ++i) { 29 | uint32_t iswap = irword; 30 | uint32_t ia = irword ^ c1[i]; 31 | uint32_t il = ia & 0xFFFF, ih = ia >> 16; 32 | uint32_t ib = il * il + ~(ih * ih); 33 | ia = (ib >> 16) | (ib << 16); 34 | irword = lword ^ ((ia ^ c2[i]) + il * ih); 35 | lword = iswap; 36 | } 37 | return irword; 38 | } 39 | -------------------------------------------------------------------------------- /third_party/masstree/misc.cc: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #include "misc.hh" 17 | #include 18 | #include "kvthread.hh" 19 | 20 | int clp_parse_suffixdouble(Clp_Parser *clp, const char *vstr, 21 | int complain, void *) 22 | { 23 | const char *post; 24 | if (*vstr == 0 || isspace((unsigned char) *vstr)) 25 | post = vstr; 26 | else 27 | clp->val.d = strtod(vstr, (char **) &post); 28 | if (vstr != post && (*post == 'K' || *post == 'k')) 29 | clp->val.d *= 1000, ++post; 30 | else if (vstr != post && (*post == 'M' || *post == 'm')) 31 | clp->val.d *= 1000000, ++post; 32 | else if (vstr != post && (*post == 'B' || *post == 'b' || *post == 'G' || *post == 'g')) 33 | clp->val.d *= 1000000000, ++post; 34 | if (*vstr != 0 && *post == 0) 35 | return 1; 36 | else if (complain) 37 | return Clp_OptionError(clp, "%<%O%> expects a real number, not %<%s%>", vstr); 38 | else 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /third_party/masstree/perfstat.hh: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #ifndef PERF_STAT_HH 17 | #define PERF_STAT_HH 1 18 | #include "compiler.hh" 19 | #include "misc.hh" 20 | #include 21 | #include 22 | 23 | namespace Perf { 24 | struct stat { 25 | /** @brief An initialization call from main function 26 | */ 27 | static void initmain(bool pinthreads); 28 | #if GCSTATS 29 | int gc_nfree; 30 | #endif 31 | void initialize(int cid) { 32 | this->cid = cid; 33 | } 34 | static void print(const stat **s, int n); 35 | int cid; // core index 36 | }; 37 | } 38 | #endif 39 | -------------------------------------------------------------------------------- /third_party/masstree/scantest.cc: -------------------------------------------------------------------------------- 1 | #include "query_masstree.hh" 2 | 3 | using namespace Masstree; 4 | 5 | kvepoch_t global_log_epoch = 0; 6 | volatile mrcu_epoch_type globalepoch = 1; // global epoch, updated by main thread regularly 7 | volatile bool recovering = false; // so don't add log entries, and free old value immediately 8 | kvtimestamp_t initial_timestamp; 9 | 10 | int 11 | main(int argc, char *argv[]) 12 | { 13 | (void) argc; 14 | (void) argv; 15 | 16 | threadinfo* ti = threadinfo::make(threadinfo::TI_MAIN, -1); 17 | default_table::test(*ti); 18 | } 19 | -------------------------------------------------------------------------------- /third_party/masstree/string_slice.cc: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2013 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2013 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #include "string_slice.hh" 17 | -------------------------------------------------------------------------------- /third_party/masstree/value_string.cc: -------------------------------------------------------------------------------- 1 | /* Masstree 2 | * Eddie Kohler, Yandong Mao, Robert Morris 3 | * Copyright (c) 2012-2014 President and Fellows of Harvard College 4 | * Copyright (c) 2012-2014 Massachusetts Institute of Technology 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a 7 | * copy of this software and associated documentation files (the "Software"), 8 | * to deal in the Software without restriction, subject to the conditions 9 | * listed in the Masstree LICENSE file. These conditions include: you must 10 | * preserve this copyright notice, and you cannot mention the copyright 11 | * holders in advertising related to the Software without their permission. 12 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 13 | * notice is a summary of the Masstree LICENSE file; the license in that file 14 | * is legally binding. 15 | */ 16 | #include "kvrow.hh" 17 | #include "value_string.hh" 18 | #include 19 | -------------------------------------------------------------------------------- /third_party/masstree_wrapper/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(masstree_wrapper_sources 2 | masstree_wrapper.cc 3 | masstree_wrapper.h 4 | ) 5 | 6 | target_sources(treeline PRIVATE ${masstree_wrapper_sources}) 7 | target_sources(pg_treeline PRIVATE ${masstree_wrapper_sources}) 8 | -------------------------------------------------------------------------------- /third_party/masstree_wrapper/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Riki Otaki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /third_party/masstree_wrapper/masstree_wrapper.cc: -------------------------------------------------------------------------------- 1 | #include "masstree_wrapper.h" 2 | 3 | volatile mrcu_epoch_type active_epoch = 1; 4 | volatile std::uint64_t globalepoch = 1; 5 | volatile bool recovering = false; 6 | 7 | 8 | -------------------------------------------------------------------------------- /third_party/tlx/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | btree.h 3 | btree_map.h 4 | core.cc 5 | core.h) 6 | -------------------------------------------------------------------------------- /third_party/tlx/LICENSE: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | affinity.h 3 | arena.cc 4 | arena.h 5 | calc.h 6 | coding.cc 7 | coding.h 8 | crc32c.h 9 | gcc_macros.h 10 | hash_queue.h 11 | inlineskiplist.h 12 | insert_tracker.h 13 | key.h 14 | packed_map.h 15 | packed_map-inl.h 16 | random.cc 17 | random.h 18 | skiplist.h 19 | status.cc 20 | thread_pool.cc 21 | thread_pool.h 22 | timer.cc 23 | timer.h) 24 | -------------------------------------------------------------------------------- /util/affinity.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | namespace tl { 10 | namespace affinity { 11 | 12 | // Pins the current thread to core `core_id`, returning true if the pin 13 | // succeeded. 14 | static bool PinToCore(uint32_t core_id) { 15 | cpu_set_t cpuset; 16 | CPU_ZERO(&cpuset); 17 | CPU_SET(core_id, &cpuset); 18 | 19 | pthread_t thread = pthread_self(); 20 | return pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset) == 0; 21 | } 22 | 23 | // Unpins the current thread, returning true if the unpin succeeded. 24 | static bool Unpin() { 25 | cpu_set_t cpuset; 26 | CPU_ZERO(&cpuset); 27 | 28 | pthread_t thread = pthread_self(); 29 | return pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset) == 0; 30 | } 31 | 32 | // RAII-helper that pins the current thread to `core_id`. 33 | class PinInScope { 34 | public: 35 | PinInScope(uint32_t core_id) { 36 | if (!PinToCore(core_id)) { 37 | throw std::runtime_error("Failed to pin thread to core " + 38 | std::to_string(core_id)); 39 | } 40 | } 41 | ~PinInScope() { Unpin(); } 42 | }; 43 | 44 | } // namespace affinity 45 | } // namespace tl 46 | -------------------------------------------------------------------------------- /util/calc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace tl { 6 | 7 | // Find the smallest power of 2 larger than or equal to `num` 8 | inline size_t Pow2Ceil(size_t num) { 9 | size_t count = 0; 10 | 11 | if (num && !(num & (num - 1))) return num; 12 | 13 | while (num > 0) { 14 | num >>= 1; 15 | ++count; 16 | } 17 | 18 | return 1ULL << count; 19 | } 20 | 21 | } // namespace tl 22 | -------------------------------------------------------------------------------- /util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "crc32c/crc32c.h" 11 | 12 | namespace tl { 13 | namespace crc32c { 14 | 15 | // Return the crc32c of `concat(A, data[0,n-1])` where `init_crc` is the crc32c 16 | // of some string A. `Extend()` is often used to maintain the crc32c of a stream 17 | // of data. 18 | inline uint32_t Extend(uint32_t init_crc, const uint8_t* data, size_t n) { 19 | return ::crc32c::Extend(init_crc, data, n); 20 | } 21 | 22 | // Return the crc32c of `data[0,n-1]`. 23 | inline uint32_t Value(const uint8_t* data, size_t n) { 24 | return Extend(0, data, n); 25 | } 26 | 27 | inline constexpr uint32_t kMaskDelta = 0xa282ead8ul; 28 | 29 | // Return a masked representation of crc. 30 | // 31 | // Motivation: it is problematic to compute the CRC of a string that 32 | // contains embedded CRCs. Therefore we recommend that CRCs stored 33 | // somewhere (e.g., in files) should be masked before being stored. 34 | inline uint32_t Mask(uint32_t crc) { 35 | // Rotate right by 15 bits and add a constant. 36 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 37 | } 38 | 39 | // Return the crc whose masked representation is masked_crc. 40 | inline uint32_t Unmask(uint32_t masked_crc) { 41 | uint32_t rot = masked_crc - kMaskDelta; 42 | return ((rot >> 17) | (rot << 15)); 43 | } 44 | 45 | } // namespace crc32c 46 | } // namespace tl 47 | -------------------------------------------------------------------------------- /util/gcc_macros.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under both the GPLv2 (found in the 3 | // COPYING file in the root directory) and Apache 2.0 License 4 | // (found in the LICENSE.Apache file in the root directory). 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | 12 | #define LIKELY(x) (__builtin_expect((x), 1)) 13 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 14 | #define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality) 15 | -------------------------------------------------------------------------------- /util/random.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under both the GPLv2 (found in the 3 | // COPYING file in the root directory) and Apache 2.0 License 4 | // (found in the LICENSE.Apache file in the root directory). 5 | // 6 | 7 | #include "util/gcc_macros.h" 8 | #include "util/random.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace tl { 16 | 17 | Random* Random::GetTLSInstance() { 18 | static thread_local Random* tls_instance; 19 | static thread_local std::aligned_storage::type tls_instance_bytes; 20 | 21 | auto rv = tls_instance; 22 | if (UNLIKELY(rv == nullptr)) { 23 | size_t seed = std::hash()(std::this_thread::get_id()); 24 | rv = new (&tls_instance_bytes) Random((uint32_t)seed); 25 | tls_instance = rv; 26 | } 27 | return rv; 28 | } 29 | 30 | std::string Random::HumanReadableString(int len) { 31 | std::string ret; 32 | ret.resize(len); 33 | for (int i = 0; i < len; ++i) { 34 | ret[i] = static_cast('a' + Uniform(26)); 35 | } 36 | return ret; 37 | } 38 | 39 | std::string Random::RandomString(int len) { 40 | std::string ret; 41 | ret.resize(len); 42 | for (int i = 0; i < len; i++) { 43 | ret[i] = static_cast(' ' + Uniform(95)); // ' ' .. '~' 44 | } 45 | return ret; 46 | } 47 | 48 | } // namespace tl 49 | -------------------------------------------------------------------------------- /util/timer.cc: -------------------------------------------------------------------------------- 1 | // Acknowledgement: The code in this file was adapted from 2 | // https://github.com/google/cuckoo-index/blob/master/common/profiling.cc 3 | // 4 | // This code is licensed under the Apache 2.0 License. See 5 | // https://github.com/google/cuckoo-index/blob/master/LICENSE 6 | 7 | #include "timer.h" 8 | 9 | namespace tl { 10 | 11 | Timer& Timer::GetThreadInstance() { 12 | thread_local static Timer static_timer; 13 | return static_timer; 14 | } 15 | 16 | } // namespace tl 17 | -------------------------------------------------------------------------------- /util/tracking_allocator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | // The boilerplate in this file is CC4 licenced code. 6 | // https://howardhinnant.github.io/allocator_boilerplate.html 7 | 8 | namespace tl { 9 | 10 | template 11 | class TrackingAllocator { 12 | public: 13 | using value_type = T; 14 | 15 | template 16 | struct rebind { 17 | typedef TrackingAllocator other; 18 | }; 19 | 20 | TrackingAllocator(uint64_t& currently_allocated_bytes) noexcept 21 | : currently_allocated_bytes_(currently_allocated_bytes) {} 22 | 23 | template 24 | TrackingAllocator(TrackingAllocator const& other) noexcept 25 | : currently_allocated_bytes_(other.currently_allocated_bytes_) {} 26 | 27 | value_type* allocate(std::size_t n) { 28 | currently_allocated_bytes_ += n * sizeof(value_type); 29 | return static_cast(::operator new(n * sizeof(value_type))); 30 | } 31 | 32 | void deallocate(value_type* p, std::size_t n) noexcept { 33 | currently_allocated_bytes_ -= n * sizeof(value_type); 34 | ::operator delete(p); 35 | } 36 | 37 | template 38 | void destroy(U* p) noexcept { 39 | p->~U(); 40 | } 41 | 42 | private: 43 | template 44 | friend class TrackingAllocator; 45 | 46 | uint64_t& currently_allocated_bytes_; 47 | }; 48 | 49 | } // namespace tl 50 | -------------------------------------------------------------------------------- /wal/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | target_sources(treeline PRIVATE 2 | format.h 3 | manager.cc 4 | manager.h 5 | reader.cc 6 | reader.h 7 | writer.cc 8 | writer.h) 9 | --------------------------------------------------------------------------------