├── .github └── workflows │ └── release.yml ├── .gitignore ├── CMakeLists.txt ├── Dockerfile ├── README.md ├── VERSION ├── assets ├── lsm-tree.png ├── slc-mlc-tlc-shape.jpg ├── slc-mlc-tlc-specs.png └── unum.png ├── bench ├── configs │ ├── leveldb │ │ ├── 10TB.cfg │ │ ├── 1TB.cfg │ │ └── default.cfg │ ├── lmdb │ │ ├── 10TB.cfg │ │ ├── 1TB.cfg │ │ └── default.cfg │ ├── mongodb │ │ ├── 100GB.cfg │ │ ├── 100MB.cfg │ │ ├── 10GB.cfg │ │ ├── 10TB.cfg │ │ ├── 1GB.cfg │ │ └── 1TB.cfg │ ├── redis │ │ ├── 100GB.cfg │ │ ├── 100GB.cfg.redis │ │ ├── 100MB.cfg │ │ ├── 100MB.cfg.redis │ │ ├── 10GB.cfg │ │ ├── 10GB.cfg.redis │ │ ├── 10TB.cfg │ │ ├── 10TB.cfg.redis │ │ ├── 1GB.cfg │ │ ├── 1GB.cfg.redis │ │ ├── 1TB.cfg │ │ ├── 1TB.cfg.redis │ │ └── redis.cfg │ ├── rocksdb │ │ ├── 100MB.cfg │ │ ├── 10TB.cfg │ │ ├── 1TB.cfg │ │ ├── additional.cfg │ │ └── default.cfg │ ├── ustore │ │ ├── 100GB.cfg │ │ ├── 100MB.cfg │ │ ├── 10GB.cfg │ │ ├── 10TB.cfg │ │ ├── 1GB.cfg │ │ ├── 1TB.cfg │ │ └── default.cfg │ └── wiredtiger │ │ ├── 10GB.cfg │ │ ├── 10TB.cfg │ │ ├── 1TB.cfg │ │ └── default.cfg └── workloads │ ├── 100GB.json │ ├── 100MB.json │ ├── 10GB.json │ ├── 10TB.json │ ├── 1GB.json │ ├── 1TB.json │ └── example.json ├── build_debug.sh ├── build_docker_image.sh ├── build_release.sh ├── cmake ├── argparse.cmake ├── benchmark.cmake ├── fmt.cmake ├── hiredis.cmake ├── leveldb.cmake ├── lmdb.cmake ├── mongodb.cmake ├── nlohmannjson.cmake ├── redis.cmake ├── rocksdb.cmake ├── tabulate.cmake ├── uring.cmake ├── ustore.cmake └── wiredtiger.cmake ├── docs ├── Makefile ├── _static │ ├── custom.css │ └── custom.js ├── conf.dox ├── conf.py ├── index.rst └── reference.rst ├── run.py └── src ├── bench.cxx ├── core ├── aligned_buffer.hpp ├── data_accessor.hpp ├── db.hpp ├── db_brand.hpp ├── db_hint.hpp ├── distribution.hpp ├── exception.hpp ├── generators │ ├── acknowledged_counter_generator.hpp │ ├── const_generator.hpp │ ├── counter_generator.hpp │ ├── generator.hpp │ ├── random_generator.hpp │ ├── scrambled_zipfian_generator.hpp │ ├── skewed_zipfian_generator.hpp │ ├── uniform_generator.hpp │ └── zipfian_generator.hpp ├── helper.hpp ├── operation.hpp ├── printable.hpp ├── profiler.hpp ├── reporter.hpp ├── settings.hpp ├── threads_fence.hpp ├── timer.hpp ├── types.hpp ├── worker.hpp └── workload.hpp ├── leveldb └── leveldb.hpp ├── lmdb └── lmdb.hpp ├── mongodb └── mongodb.hpp ├── redis └── redis.hpp ├── rocksdb ├── rocksdb.hpp └── rocksdb_transaction.hpp ├── ustore ├── ustore.hpp └── ustore_transaction.hpp └── wiredtiger └── wiredtiger.hpp /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | 7 | env: 8 | BUILD_TYPE: Release 9 | GH_TOKEN: ${{ secrets.SEMANTIC_RELEASE_TOKEN }} 10 | 11 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 12 | permissions: 13 | contents: read 14 | pages: write 15 | id-token: write 16 | 17 | 18 | jobs: 19 | 20 | deploy: 21 | environment: 22 | name: github-pages 23 | url: ${{ steps.deployment.outputs.page_url }} 24 | runs-on: ubuntu-22.04 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v3 28 | with: 29 | ref: 'main' 30 | - name: Setup GitHub Pages 31 | uses: actions/configure-pages@v2 32 | - name: Install dependencies 33 | run: sudo apt update && sudo apt install -y doxygen graphviz dia git && pip install sphinx breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery 34 | - name: Build documentation 35 | run: cd docs && doxygen conf.dox && make html 36 | - name: Copy assets 37 | run: cp -r assets build/docs/html/ 38 | - name: Upload artifacts 39 | uses: actions/upload-pages-artifact@v1 40 | with: 41 | # Upload entire repository 42 | path: "./build/docs/html/" 43 | - name: Deploy to GitHub Pages 44 | id: deployment 45 | uses: actions/deploy-pages@v1 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build files and folders 2 | build/ 3 | build_debug/ 4 | build_release/ 5 | .DS_Store 6 | bin/ 7 | CMakeFiles 8 | Makefile 9 | !docs/Makefile 10 | CMakeCache.txt 11 | *.ninja 12 | .clang-format 13 | .vscode/ 14 | 15 | # Temporary files and folders 16 | db_main/ 17 | tmp/ 18 | .scripts/.log 19 | code-review*.bak 20 | 21 | # General Settings 22 | 23 | # Dependency graphs 24 | *.dot.* 25 | *.dot 26 | 27 | # Prerequisites 28 | *.d 29 | 30 | # Compiled Object files 31 | *.slo 32 | *.lo 33 | *.o 34 | *.obj 35 | 36 | # Precompiled Headers 37 | *.gch 38 | *.pch 39 | 40 | # Compiled Dynamic libraries 41 | *.so 42 | *.dylib 43 | *.dll 44 | 45 | # Fortran module files 46 | *.mod 47 | *.smod 48 | 49 | # Compiled Static libraries 50 | *.lai 51 | *.la 52 | *.a 53 | *.lib 54 | 55 | # Executables 56 | *.exe 57 | *.out 58 | *.app 59 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ####################################################################################################################### 2 | # Set cmake minimum required version 3 | ####################################################################################################################### 4 | 5 | cmake_minimum_required(VERSION 3.20.1) 6 | 7 | ####################################################################################################################### 8 | # Enable/Disable Databases 9 | ####################################################################################################################### 10 | 11 | option(UCSB_BUILD_USTORE "Build USTORE for the benchmark" ON) 12 | option(UCSB_BUILD_ROCKSDB "Build RocksDB for the benchmark" ON) 13 | option(UCSB_BUILD_LEVELDB "Build LevelDB for the benchmark" ON) 14 | option(UCSB_BUILD_WIREDTIGER "Build WiredTiger for the benchmark" ON) 15 | option(UCSB_BUILD_MONGODB "Build MongoDB for the benchmark" OFF) 16 | option(UCSB_BUILD_REDIS "Build Redis for the benchmark" OFF) 17 | option(UCSB_BUILD_LMDB "Build LMDB for the benchmark" OFF) 18 | 19 | ####################################################################################################################### 20 | # Set compiler 21 | ####################################################################################################################### 22 | 23 | message(STATUS "CMAKE_C_COMPILER: ${CMAKE_C_COMPILER}") 24 | message(STATUS "CMAKE_C_COMPILER_ID: ${CMAKE_C_COMPILER_ID}") 25 | message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") 26 | message(STATUS "CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") 27 | 28 | ####################################################################################################################### 29 | # Set project environment 30 | ####################################################################################################################### 31 | 32 | project(UCSB) 33 | include_directories(${PROJECT_SOURCE_DIR} ) 34 | link_directories("/usr/local/lib/") 35 | 36 | set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") 37 | set(CMAKE_CACHEFILE_DIR "${CMAKE_SOURCE_DIR}/build") 38 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/build/lib") 39 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/build/lib") 40 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/build/bin") 41 | 42 | message(STATUS "CMAKE_ROOT: ${CMAKE_ROOT}") 43 | message(STATUS "CMAKE_SYSTEM_VERSION: ${CMAKE_SYSTEM_VERSION}") 44 | message(STATUS "CMAKE_BINARY_DIR: ${CMAKE_BINARY_DIR}") 45 | message(STATUS "CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}") 46 | message(STATUS "CMAKE_SOURCE_DIR: ${CMAKE_SOURCE_DIR}") 47 | message(STATUS "CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") 48 | message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") 49 | message(STATUS "Python3_SITELIB: ${Python3_SITELIB}") 50 | 51 | ####################################################################################################################### 52 | # Selecting a modern standard version 53 | ####################################################################################################################### 54 | 55 | # Export the compilation report, that can be later be reused by static 56 | # analysis tools and other apps like SourceTrail. 57 | set(CMAKE_EXPORT_COMPILE_COMMANDS OFF) 58 | 59 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 60 | 61 | set(CMAKE_CXX_STANDARD 20) 62 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 63 | set(CMAKE_CXX_EXTENSIONS ON) 64 | 65 | if(NOT CMAKE_BUILD_TYPE) 66 | set(CMAKE_BUILD_TYPE RELEASE) 67 | message(STATUS "Build type not specified: Use RELEASE by default") 68 | endif() 69 | 70 | if(CMAKE_BUILD_TYPE EQUAL "RELEASE") 71 | set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) 72 | set(CMAKE_UNITY_BUILD ON) 73 | endif() 74 | 75 | ####################################################################################################################### 76 | # Macros and additional flags 77 | ####################################################################################################################### 78 | 79 | # General flags. 80 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wnon-virtual-dtor -Woverloaded-virtual") 81 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers") 82 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp") 83 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wreorder -Wunused-parameter") 84 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-unknown-pragmas") 85 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti -fPIC") 86 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wattributes") 87 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wdeprecated-copy") 88 | 89 | # Debugging flags. 90 | # https://code.visualstudio.com/docs/cpp/faq-cpp#_how-do-i-enable-debug-symbols 91 | set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG_SAFE_MODE -g") 92 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG_SAFE_MODE -g") 93 | set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG -O2 -g") 94 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -O2 -g") 95 | 96 | if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") 97 | message("- Enabling SIMD for ARM") 98 | else() 99 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ") 100 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ") 101 | endif() 102 | 103 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ") 104 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ") 105 | 106 | # Compiler-specific flags. 107 | if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 108 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fstrict-vtable-pointers") 109 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ferror-limit=1") 110 | # Riscky Performance Optimization, that require tuning. 111 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -momit-leaf-frame-pointer") 112 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funsafe-math-optimizations") 113 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-omit-frame-pointer") 114 | elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 115 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=1") 116 | elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") 117 | elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") 118 | else () 119 | message(STATUS "Unknown CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") 120 | endif() 121 | 122 | include("${CMAKE_MODULE_PATH}/benchmark.cmake") 123 | include("${CMAKE_MODULE_PATH}/tabulate.cmake") 124 | include("${CMAKE_MODULE_PATH}/fmt.cmake") 125 | include("${CMAKE_MODULE_PATH}/argparse.cmake") 126 | include("${CMAKE_MODULE_PATH}/nlohmannjson.cmake") 127 | include("${CMAKE_MODULE_PATH}/uring.cmake") 128 | 129 | set(UCSB_DB_LIBS) 130 | add_executable(ucsb_bench ./src/bench.cxx) 131 | 132 | if(${UCSB_BUILD_USTORE}) 133 | # Choose engine. Available engines: UCSET, ROCKSDB, LEVELDB, UDISK 134 | set(USTORE_ENGINE_NAME UCSET) 135 | 136 | include("${CMAKE_MODULE_PATH}/ustore.cmake") 137 | list(APPEND UCSB_DB_LIBS "ustore") 138 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_USTORE=1) 139 | endif() 140 | 141 | if(${UCSB_BUILD_ROCKSDB}) 142 | include("${CMAKE_MODULE_PATH}/rocksdb.cmake") 143 | list(APPEND UCSB_DB_LIBS "rocksdb") 144 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_ROCKSDB=1) 145 | endif() 146 | 147 | if(${UCSB_BUILD_LEVELDB}) 148 | include("${CMAKE_MODULE_PATH}/leveldb.cmake") 149 | list(APPEND UCSB_DB_LIBS "leveldb") 150 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_LEVELDB=1) 151 | endif() 152 | 153 | if(${UCSB_BUILD_WIREDTIGER}) 154 | include("${CMAKE_MODULE_PATH}/wiredtiger.cmake") 155 | list(APPEND UCSB_DB_LIBS "wiredtiger") 156 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_WIREDTIGER=1) 157 | endif() 158 | 159 | if(${UCSB_BUILD_MONGODB}) 160 | include("${CMAKE_MODULE_PATH}/mongodb.cmake") 161 | list(APPEND UCSB_DB_LIBS "mongodb") 162 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_MONGODB=1) 163 | endif() 164 | 165 | if(${UCSB_BUILD_REDIS}) 166 | include("${CMAKE_MODULE_PATH}/redis.cmake") 167 | list(APPEND UCSB_DB_LIBS "redis") 168 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_REDIS=1) 169 | endif() 170 | 171 | if(${UCSB_BUILD_LMDB}) 172 | include("${CMAKE_MODULE_PATH}/lmdb.cmake") 173 | list(APPEND UCSB_DB_LIBS "lmdb") 174 | target_compile_definitions(ucsb_bench PUBLIC UCSB_HAS_LMDB=1) 175 | endif() 176 | 177 | set(CXX_TARGET_LINK_LIBRARIES z uring benchmark fmt ${UCSB_DB_LIBS}) 178 | 179 | set(CMAKE_THREAD_LIBS_INIT "-lpthread") 180 | set(CMAKE_HAVE_THREADS_LIBRARY 1) 181 | set(CMAKE_USE_WIN32_THREADS_INIT 0) 182 | set(CMAKE_USE_PTHREADS_INIT 1) 183 | set(THREADS_PREFER_PTHREAD_FLAG ON) 184 | find_package(Threads REQUIRED) 185 | 186 | ####################################################################################################################### 187 | # Build UCSB 188 | ####################################################################################################################### 189 | 190 | target_link_libraries(ucsb_bench${ARTIFACT_SUFFIX} 191 | ${CXX_TARGET_LINK_LIBRARIES} 192 | ${PKG_PACKAGES_LIBRARIES} 193 | ${CMAKE_DL_LIBS} 194 | ) 195 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | 3 | MAINTAINER unum@cloud.com 4 | 5 | WORKDIR ./ 6 | 7 | RUN apt-get update 8 | ENV DEBIAN_FRONTEND noninteractive 9 | 10 | # Install tools 11 | RUN apt install -y python3-pip 12 | RUN pip3 install cmake 13 | RUN pip3 install conan 14 | RUN apt install -y gcc-10 15 | RUN apt install -y g++-10 16 | RUN apt-get install -y libexplain-dev 17 | RUN apt-get install -y libsnappy-dev 18 | RUN apt-get install -yq pkg-config 19 | RUN apt-get install -y git 20 | 21 | # Build WiredTiger (latest) 22 | RUN git clone git://github.com/wiredtiger/wiredtiger.git 23 | RUN mkdir ./wiredtiger/build 24 | WORKDIR "./wiredtiger/build" 25 | RUN cmake ../. 26 | RUN make install 27 | 28 | WORKDIR / 29 | 30 | # Build UCSB 31 | RUN git clone https://github.com/unum-cloud/ucsb.git 32 | WORKDIR "./ucsb/" 33 | RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 34 | RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 35 | RUN bash -i setup.sh 36 | RUN bash -i build_release.sh 37 | RUN rm -rf ./bench 38 | 39 | ENTRYPOINT ["./build_release/bin/ucsb_bench"] 40 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.0 -------------------------------------------------------------------------------- /assets/lsm-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unum-cloud/ucsb/d74c0efd963bea2a907cc8353a92016aaddb3b61/assets/lsm-tree.png -------------------------------------------------------------------------------- /assets/slc-mlc-tlc-shape.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unum-cloud/ucsb/d74c0efd963bea2a907cc8353a92016aaddb3b61/assets/slc-mlc-tlc-shape.jpg -------------------------------------------------------------------------------- /assets/slc-mlc-tlc-specs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unum-cloud/ucsb/d74c0efd963bea2a907cc8353a92016aaddb3b61/assets/slc-mlc-tlc-specs.png -------------------------------------------------------------------------------- /assets/unum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unum-cloud/ucsb/d74c0efd963bea2a907cc8353a92016aaddb3b61/assets/unum.png -------------------------------------------------------------------------------- /bench/configs/leveldb/10TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "write_buffer_size": 268435456, 3 | "max_file_size": 268435456, 4 | "max_open_files": -1, 5 | "compression": "none", 6 | "cache_size": 200000 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/leveldb/1TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "write_buffer_size": 134217728, 3 | "max_file_size": 134217728, 4 | "max_open_files": -1, 5 | "compression": "none", 6 | "cache_size": 20000 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/leveldb/default.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "write_buffer_size": 134217728, 3 | "max_file_size": 134217728, 4 | "max_open_files": -1, 5 | "compression": "none", 6 | "cache_size": 2000 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/lmdb/10TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "map_size": 20737418240000, 3 | "no_sync": true, 4 | "no_meta_sync": false, 5 | "no_read_a_head": false, 6 | "write_map": false 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/lmdb/1TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "map_size": 20737418240000, 3 | "no_sync": true, 4 | "no_meta_sync": false, 5 | "no_read_a_head": false, 6 | "write_map": false 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/lmdb/default.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "map_size": 2737418240000, 3 | "no_sync": true, 4 | "no_meta_sync": false, 5 | "no_read_a_head": false, 6 | "write_map": false 7 | } 8 | -------------------------------------------------------------------------------- /bench/configs/mongodb/100GB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/100GB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/100GB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | maxIncomingConnections: 64 15 | 16 | processManagement: 17 | timeZoneInfo: /usr/share/zoneinfo 18 | -------------------------------------------------------------------------------- /bench/configs/mongodb/100MB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/100MB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/100MB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | maxIncomingConnections: 64 15 | 16 | processManagement: 17 | timeZoneInfo: /usr/share/zoneinfo -------------------------------------------------------------------------------- /bench/configs/mongodb/10GB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/10GB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/10GB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | maxIncomingConnections: 64 15 | 16 | processManagement: 17 | timeZoneInfo: /usr/share/zoneinfo -------------------------------------------------------------------------------- /bench/configs/mongodb/10TB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/10GB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/10GB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | maxIncomingConnections: 64 15 | 16 | processManagement: 17 | timeZoneInfo: /usr/share/zoneinfo -------------------------------------------------------------------------------- /bench/configs/mongodb/1GB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/1GB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/1GB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | maxIncomingConnections: 64 15 | 16 | processManagement: 17 | timeZoneInfo: /usr/share/zoneinfo -------------------------------------------------------------------------------- /bench/configs/mongodb/1TB.cfg: -------------------------------------------------------------------------------- 1 | storage: 2 | dbPath: ./db_main/mongodb/1TB 3 | journal: 4 | enabled: true 5 | 6 | systemLog: 7 | destination: file 8 | logAppend: true 9 | path: ./db_main/mongodb/1TB/mongod.log 10 | 11 | net: 12 | port: 27017 13 | bindIp: 127.0.0.1 14 | 15 | processManagement: 16 | timeZoneInfo: /usr/share/zoneinfo 17 | -------------------------------------------------------------------------------- /bench/configs/redis/100GB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/100GB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/100GB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/100GB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/100MB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/100MB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/100MB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/100MB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/10GB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/10GB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/10GB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/10GB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/10TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/10TB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/10TB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/10TB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/1GB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/1GB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/1GB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/1GB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/1TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "connection_type": "UNIX", 3 | "path": "./db_main/redis/1TB/redis.sock", 4 | "host": "127.0.0.1", 5 | "port": 6999, 6 | "pool_size": 64, 7 | "wait_timeout": 10 8 | } 9 | -------------------------------------------------------------------------------- /bench/configs/redis/1TB.cfg.redis: -------------------------------------------------------------------------------- 1 | include bench/configs/redis/redis.cfg 2 | # Directory 3 | dir db_main/redis/1TB/ 4 | pidfile redis.pid 5 | logfile redis.log 6 | 7 | # Unix socket 8 | unixsocket redis.sock 9 | unixsocketperm 777 10 | -------------------------------------------------------------------------------- /bench/configs/redis/redis.cfg: -------------------------------------------------------------------------------- 1 | 2 | # General 3 | daemonize yes 4 | timeout 0 5 | 6 | bind 127.0.0.1 ::1 7 | port 6999 8 | 9 | tcp-backlog 511 10 | tcp-keepalive 0 11 | 12 | maxclients 100 13 | 14 | supervised no 15 | loglevel notice 16 | databases 1 17 | always-show-logo yes 18 | stop-writes-on-bgsave-error yes 19 | appendonly no 20 | lua-time-limit 5000 21 | slowlog-log-slower-than 10000 22 | slowlog-max-len 128 23 | latency-monitor-threshold 0 24 | notify-keyspace-events "" 25 | hash-max-ziplist-entries 512 26 | hash-max-ziplist-value 1024 27 | stream-node-max-bytes 4096 28 | stream-node-max-entries 100 29 | activerehashing yes 30 | client-output-buffer-limit normal 0 0 0 31 | client-output-buffer-limit replica 256mb 64mb 60 32 | client-output-buffer-limit pubsub 32mb 8mb 60 33 | hz 10 34 | dynamic-hz yes 35 | rdb-save-incremental-fsync yes 36 | save "" 37 | stop-writes-on-bgsave-error no 38 | rdbcompression no 39 | rdbchecksum no 40 | repl-diskless-sync yes 41 | lazyfree-lazy-eviction yes 42 | lazyfree-lazy-expire yes 43 | lazyfree-lazy-server-del yes 44 | replica-lazy-flush yes 45 | tcp-backlog 65536 -------------------------------------------------------------------------------- /bench/configs/rocksdb/100MB.cfg: -------------------------------------------------------------------------------- 1 | [Version] 2 | rocksdb_version=7.2.9 3 | options_file_version=1.1 4 | 5 | [DBOptions] 6 | create_if_missing=true 7 | writable_file_max_buffer_size=67108864 8 | max_open_files=-1 9 | max_file_opening_threads=4 10 | 11 | [CFOptions "default"] 12 | max_write_buffer_number=2 13 | write_buffer_size=67108864 14 | target_file_size_base=67108864 15 | max_bytes_for_level_base=268435456 16 | max_compaction_bytes=536870912 17 | level_compaction_dynamic_level_bytes=false 18 | level0_stop_writes_trigger=4 19 | target_file_size_multiplier=2 20 | max_bytes_for_level_multiplier=4 21 | compression=kNoCompression 22 | compaction_style=kCompactionStyleLevel 23 | -------------------------------------------------------------------------------- /bench/configs/rocksdb/10TB.cfg: -------------------------------------------------------------------------------- 1 | [Version] 2 | rocksdb_version=7.2.9 3 | options_file_version=1.1 4 | 5 | [DBOptions] 6 | create_if_missing=true 7 | writable_file_max_buffer_size=268435456 8 | max_open_files=-1 9 | max_file_opening_threads=128 10 | 11 | [CFOptions "default"] 12 | max_write_buffer_number=8 13 | write_buffer_size=268435456 14 | target_file_size_base=268435456 15 | max_bytes_for_level_base=17179869184 16 | max_compaction_bytes=34359738368 17 | level_compaction_dynamic_level_bytes=false 18 | level0_stop_writes_trigger=40 19 | target_file_size_multiplier=2 20 | max_bytes_for_level_multiplier=4 21 | compression=kNoCompression 22 | compaction_style=kCompactionStyleLevel 23 | -------------------------------------------------------------------------------- /bench/configs/rocksdb/1TB.cfg: -------------------------------------------------------------------------------- 1 | [Version] 2 | rocksdb_version=7.2.9 3 | options_file_version=1.1 4 | 5 | [DBOptions] 6 | create_if_missing=true 7 | writable_file_max_buffer_size=134217728 8 | max_open_files=-1 9 | max_file_opening_threads=64 10 | 11 | [CFOptions "default"] 12 | max_write_buffer_number=8 13 | write_buffer_size=134217728 14 | target_file_size_base=134217728 15 | max_bytes_for_level_base=4294967296 16 | max_compaction_bytes=8589934592 17 | level_compaction_dynamic_level_bytes=false 18 | level0_stop_writes_trigger=32 19 | target_file_size_multiplier=2 20 | max_bytes_for_level_multiplier=4 21 | compression=kNoCompression 22 | compaction_style=kCompactionStyleLevel 23 | -------------------------------------------------------------------------------- /bench/configs/rocksdb/additional.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "default_write_batch_flush_threshold": 10 3 | } 4 | -------------------------------------------------------------------------------- /bench/configs/rocksdb/default.cfg: -------------------------------------------------------------------------------- 1 | [Version] 2 | rocksdb_version=7.2.9 3 | options_file_version=1.1 4 | 5 | [DBOptions] 6 | create_if_missing=true 7 | writable_file_max_buffer_size=134217728 8 | max_open_files=-1 9 | max_file_opening_threads=32 10 | 11 | [CFOptions "default"] 12 | max_write_buffer_number=4 13 | write_buffer_size=134217728 14 | target_file_size_base=134217728 15 | max_bytes_for_level_base=2147483648 16 | max_compaction_bytes=4294967296 17 | level_compaction_dynamic_level_bytes=false 18 | level0_stop_writes_trigger=16 19 | target_file_size_multiplier=2 20 | max_bytes_for_level_multiplier=4 21 | compression=kNoCompression 22 | compaction_style=kCompactionStyleLevel 23 | -------------------------------------------------------------------------------- /bench/configs/ustore/100GB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/100MB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/10GB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/10TB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/1GB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/1TB.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/ustore/default.cfg: -------------------------------------------------------------------------------- 1 | // Warning: Values of 'directory', 'data_directories' and 'engine.config_file_path' filled by the benchmark and based on the run.py arguments. 2 | // The engine_config_path resolved based on the UKV config file path and the engine name: **/configs/$ENGINE/$THIS_FILE_NAME 3 | // But if you set the values here the above stuff will be disabled. 4 | { 5 | "version": "1.0", 6 | "directory": "", 7 | "data_directories": [], 8 | "engine": { 9 | "config_file_path": "" 10 | } 11 | } -------------------------------------------------------------------------------- /bench/configs/wiredtiger/10GB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "cache_size": 200000000 3 | } 4 | -------------------------------------------------------------------------------- /bench/configs/wiredtiger/10TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "cache_size": 200000000000 3 | } 4 | -------------------------------------------------------------------------------- /bench/configs/wiredtiger/1TB.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "cache_size": 20000000000 3 | } 4 | -------------------------------------------------------------------------------- /bench/configs/wiredtiger/default.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "cache_size": 2000000000 3 | } 4 | -------------------------------------------------------------------------------- /bench/workloads/100GB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 100000000, 5 | "operations_count": 1000, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 100000, 9 | "bulk_load_max_length": 100000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 100000000, 14 | "operations_count": 10000000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 100000000, 22 | "operations_count": 80000, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 100000000, 33 | "operations_count": 40000, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 100000000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 100000000, 52 | "operations_count": 10000000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 100000000, 61 | "operations_count": 10000000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 100500000, 70 | "operations_count": 5000, 71 | "start_key": 100500000, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 105500000, 80 | "operations_count": 5500000, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/100MB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 100000, 5 | "operations_count": 10, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 10000, 9 | "bulk_load_max_length": 10000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 100000, 14 | "operations_count": 10000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 100000, 22 | "operations_count": 80, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 100000, 33 | "operations_count": 40, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 100000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 100000, 52 | "operations_count": 10000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 100000, 61 | "operations_count": 10000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 100500, 70 | "operations_count": 5, 71 | "start_key": 100500, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 105500, 80 | "operations_count": 5500, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/10GB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 10000000, 5 | "operations_count": 100, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 100000, 9 | "bulk_load_max_length": 100000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 10000000, 14 | "operations_count": 1000000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 10000000, 22 | "operations_count": 8000, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 10000000, 33 | "operations_count": 4000, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 10000000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 10000000, 52 | "operations_count": 1000000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 10000000, 61 | "operations_count": 1000000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 10050000, 70 | "operations_count": 500, 71 | "start_key": 10050000, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 10550000, 80 | "operations_count": 550000, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/10TB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 10000000000, 5 | "operations_count": 50000, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 200000, 9 | "bulk_load_max_length": 200000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 10000000000, 14 | "operations_count": 1000000000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 10000000000, 22 | "operations_count": 8000000, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 10000000000, 33 | "operations_count": 4000000, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 10000000000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 10000000000, 52 | "operations_count": 1000000000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 10000000000, 61 | "operations_count": 1000000000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 10050000000, 70 | "operations_count": 500000, 71 | "start_key": 10050000000, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 10550000000, 80 | "operations_count": 550000000, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/1GB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 1000000, 5 | "operations_count": 100, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 10000, 9 | "bulk_load_max_length": 10000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 1000000, 14 | "operations_count": 100000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 1000000, 22 | "operations_count": 800, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 1000000, 33 | "operations_count": 400, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 1000000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 1000000, 52 | "operations_count": 100000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 1000000, 61 | "operations_count": 100000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 1005000, 70 | "operations_count": 50, 71 | "start_key": 1005000, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 1055000, 80 | "operations_count": 55000, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/1TB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Init", 4 | "records_count": 1000000000, 5 | "operations_count": 10000, 6 | "value_length": 1024, 7 | "bulk_load_proportion": 1.0, 8 | "bulk_load_min_length": 100000, 9 | "bulk_load_max_length": 100000 10 | }, 11 | { 12 | "name": "Read", 13 | "records_count": 1000000000, 14 | "operations_count": 100000000, 15 | "read_proportion": 1.0, 16 | "key_dist": "zipfian", 17 | "value_length": 1024 18 | }, 19 | { 20 | "name": "BatchRead", 21 | "records_count": 1000000000, 22 | "operations_count": 800000, 23 | "batch_read_proportion": 1.0, 24 | "key_dist": "zipfian", 25 | "value_length": 1024, 26 | "batch_read_min_length": 256, 27 | "batch_read_max_length": 256, 28 | "batch_read_length_dist": "uniform" 29 | }, 30 | { 31 | "name": "RangeSelect", 32 | "records_count": 1000000000, 33 | "operations_count": 400000, 34 | "range_select_proportion": 1.0, 35 | "key_dist": "zipfian", 36 | "value_length": 1024, 37 | "range_select_min_length": 256, 38 | "range_select_max_length": 256, 39 | "range_select_length_dist": "uniform" 40 | }, 41 | { 42 | "name": "Scan", 43 | "records_count": 1000000000, 44 | "operations_count": 1, 45 | "scan_proportion": 1.0, 46 | "key_dist": "zipfian", 47 | "value_length": 1024 48 | }, 49 | { 50 | "name": "ReadUpdate_50_50", 51 | "records_count": 1000000000, 52 | "operations_count": 100000000, 53 | "read_proportion": 0.5, 54 | "update_proportion": 0.5, 55 | "key_dist": "zipfian", 56 | "value_length": 1024 57 | }, 58 | { 59 | "name": "ReadUpsert_95_5", 60 | "records_count": 1000000000, 61 | "operations_count": 100000000, 62 | "read_proportion": 0.95, 63 | "upsert_proportion": 0.05, 64 | "key_dist": "latest", 65 | "value_length": 1024 66 | }, 67 | { 68 | "name": "BatchUpsert", 69 | "records_count": 1005000000, 70 | "operations_count": 50000, 71 | "start_key": 1005000000, 72 | "value_length": 1024, 73 | "batch_upsert_proportion": 1.0, 74 | "batch_upsert_min_length": 1000, 75 | "batch_upsert_max_length": 1000 76 | }, 77 | { 78 | "name": "Remove", 79 | "records_count": 1055000000, 80 | "operations_count": 55000000, 81 | "value_length": 1024, 82 | "remove_proportion": 1.0, 83 | "key_dist": "zipfian" 84 | } 85 | ] -------------------------------------------------------------------------------- /bench/workloads/example.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "", 4 | "records_count": 1000, 5 | "operations_count": 1000, 6 | "upsert_proportion": 0.1, 7 | "update_proportion": 0.2, 8 | "remove_proportion": 0.1, 9 | "read_proportion": 0.6, 10 | "batch_upsert_proportion": 0.0, 11 | "batch_read_proportion": 0.0, 12 | "bulk_load_proportion": 0.0, 13 | "range_select_proportion": 0.1, 14 | "scan_proportion": 0.0, 15 | "key_dist": "uniform", 16 | "value_length": 1024, 17 | "value_length_dist": "const", 18 | "batch_upsert_max_length": 10, 19 | "batch_upsert_min_length": 10, 20 | "batch_upsert_length_dist": "uniform", 21 | "batch_read_min_length": 256, 22 | "batch_read_max_length": 256, 23 | "batch_read_length_dist": "uniform", 24 | "bulk_load_max_length": 100, 25 | "bulk_load_min_length": 100, 26 | "bulk_load_length_dist": "uniform", 27 | "range_select_min_length": 256, 28 | "range_select_max_length": 256, 29 | "range_select_length_dist": "uniform" 30 | } 31 | ] -------------------------------------------------------------------------------- /build_debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Visualizing the dependencies graph. 3 | # https://cmake.org/cmake/help/latest/module/CMakeGraphVizOptions.html#module:CMakeGraphVizOptions 4 | # Debug vs Release: 5 | # https://stackoverflow.com/a/7725055 6 | 7 | cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Debug -B ./build_debug && 8 | make -j16 -C ./build_debug --silent && 9 | # Congratulate the user! 10 | echo 'Congrats, UCSB is ready for use!' 11 | -------------------------------------------------------------------------------- /build_docker_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build --tag=ucsb-image . 4 | -------------------------------------------------------------------------------- /build_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Visualizing the dependencies graph. 3 | # https://cmake.org/cmake/help/latest/module/CMakeGraphVizOptions.html#module:CMakeGraphVizOptions 4 | # Debug vs Release: 5 | # https://stackoverflow.com/a/7725055 6 | 7 | cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -B ./build_release && 8 | make -j16 -C ./build_release --silent && 9 | # Congratulate the user! 10 | echo 'Congrats, UCSB is ready for use!' 11 | -------------------------------------------------------------------------------- /cmake/argparse.cmake: -------------------------------------------------------------------------------- 1 | # argparse: 2 | # https://github.com/p-ranav/argparse/blob/master/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | argparse 7 | GIT_REPOSITORY https://github.com/p-ranav/argparse.git 8 | GIT_TAG v2.9 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(argparse) 13 | 14 | if(NOT argparse_POPULATED) 15 | set(ARGPARSE_BUILD_TESTS OFF CACHE INTERNAL "") 16 | 17 | # Should not be set globally, but just for this target! 18 | FetchContent_Populate(argparse) 19 | add_subdirectory(${argparse_SOURCE_DIR} ${argparse_BINARY_DIR} EXCLUDE_FROM_ALL) 20 | endif() 21 | 22 | include_directories(${argparse_SOURCE_DIR}/include) 23 | -------------------------------------------------------------------------------- /cmake/benchmark.cmake: -------------------------------------------------------------------------------- 1 | # benchmark: 2 | # https://github.com/google/benchmark/blob/main/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | benchmark 7 | GIT_REPOSITORY https://github.com/google/benchmark.git 8 | GIT_TAG v1.7.0 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(benchmark) 13 | 14 | if(NOT benchmark_POPULATED) 15 | set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "") 16 | set(BENCHMARK_ENABLE_INSTALL OFF CACHE INTERNAL "") 17 | set(BENCHMARK_ENABLE_DOXYGEN OFF CACHE INTERNAL "") 18 | set(BENCHMARK_INSTALL_DOCS OFF CACHE INTERNAL "") 19 | set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON CACHE INTERNAL "") 20 | set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE INTERNAL "") 21 | set(BENCHMARK_USE_BUNDLED_GTEST ON CACHE INTERNAL "") 22 | 23 | # Should not be set globally, but just for this target! 24 | FetchContent_Populate(benchmark) 25 | add_subdirectory(${benchmark_SOURCE_DIR} ${benchmark_BINARY_DIR} EXCLUDE_FROM_ALL) 26 | endif() 27 | 28 | include_directories(${benchmark_SOURCE_DIR}/include) 29 | -------------------------------------------------------------------------------- /cmake/fmt.cmake: -------------------------------------------------------------------------------- 1 | # fmt: 2 | # https://github.com/fmtlib/fmt/blob/master/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | fmt 7 | GIT_REPOSITORY https://github.com/fmtlib/fmt.git 8 | GIT_TAG 9.1.0 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(fmt) 13 | 14 | if(NOT fmt_POPULATED) 15 | set(FMT_MASTER_PROJECT OFF CACHE INTERNAL "") 16 | 17 | # Should not be set globally, but just for this target! 18 | FetchContent_Populate(fmt) 19 | add_subdirectory(${fmt_SOURCE_DIR} ${fmt_BINARY_DIR} EXCLUDE_FROM_ALL) 20 | endif() 21 | 22 | include_directories(${fmt_SOURCE_DIR}/include) 23 | -------------------------------------------------------------------------------- /cmake/hiredis.cmake: -------------------------------------------------------------------------------- 1 | # hiredis: 2 | # https://github.com/redis/hiredis/blob/master/CMakeLists.txt 3 | 4 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 5 | 6 | include(ExternalProject) 7 | 8 | ExternalProject_Add( 9 | hiredis_external 10 | 11 | GIT_REPOSITORY "https://github.com/redis/hiredis.git" 12 | GIT_TAG v1.1.0 13 | GIT_SHALLOW 1 14 | GIT_PROGRESS 0 15 | 16 | PREFIX "${PREFIX_DIR}" 17 | DOWNLOAD_DIR "${PREFIX_DIR}/hiredis-src" 18 | LOG_DIR "${PREFIX_DIR}/hiredis-log" 19 | STAMP_DIR "${PREFIX_DIR}/hiredis-stamp" 20 | TMP_DIR "${PREFIX_DIR}/hiredis-tmp" 21 | SOURCE_DIR "${PREFIX_DIR}/hiredis-src" 22 | INSTALL_DIR "${PREFIX_DIR}/hiredis-install" 23 | BINARY_DIR "${PREFIX_DIR}/hiredis-build" 24 | 25 | BUILD_ALWAYS 0 26 | UPDATE_COMMAND "" 27 | 28 | CMAKE_ARGS 29 | -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX_DIR}/hiredis-install 30 | -DCMAKE_INSTALL_LIBDIR=lib 31 | -DCMAKE_INSTALL_RPATH:PATH=/lib 32 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 33 | -DENABLE_STATIC:STRING=ON 34 | -DENABLE_CPPSUITE:BOOL=OFF 35 | -DCMAKE_C_FLAGS=-Wno-maybe-uninitialized 36 | -DCMAKE_CXX_FLAGS=-Wno-unused-variable 37 | -DENABLE_SSL:BOOL=OFF 38 | -DDISABLE_TESTS:BOOL=ON 39 | -DENABLE_SSL_TESTS:BOOL=OFF 40 | -DENABLE_ASYNC_TESTS:BOOL=OFF 41 | ) 42 | 43 | set(hiredis_INCLUDE_DIR ${PREFIX_DIR}/hiredis-install/include) 44 | if(CMAKE_BUILD_TYPE MATCHES "Debug") 45 | set(hiredis_LIBRARY_PATH ${PREFIX_DIR}/hiredis-install/lib/libhiredisd.a) 46 | else() 47 | set(hiredis_LIBRARY_PATH ${PREFIX_DIR}/hiredis-install/lib/libhiredis.a) 48 | endif() 49 | 50 | file(MAKE_DIRECTORY ${hiredis_INCLUDE_DIR}) 51 | add_library(hiredis STATIC IMPORTED) 52 | 53 | set_property(TARGET hiredis PROPERTY IMPORTED_LOCATION ${hiredis_LIBRARY_PATH}) 54 | set_property(TARGET hiredis APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${hiredis_INCLUDE_DIR}) 55 | 56 | include_directories(${hiredis_INCLUDE_DIR}) 57 | add_dependencies(hiredis hiredis_external) -------------------------------------------------------------------------------- /cmake/leveldb.cmake: -------------------------------------------------------------------------------- 1 | # LevelDB: 2 | # https://github.com/google/leveldb/blob/main/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | leveldb 7 | GIT_REPOSITORY https://github.com/google/leveldb.git 8 | GIT_TAG main 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(leveldb) 13 | 14 | if(NOT leveldb_POPULATED) 15 | set(LEVELDB_BUILD_TESTS OFF CACHE INTERNAL "") 16 | set(LEVELDB_BUILD_BENCHMARKS OFF CACHE INTERNAL "") 17 | set(HAVE_SNAPPY OFF CACHE INTERNAL "") 18 | 19 | # Enable RTTI (Note: LevelDB forcibly disables RTTI in CMakeList.txt:CMAKE_CXX_FLAGS) 20 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -frtti") 21 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -frtti") 22 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") 23 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-implicit-fallthrough") 24 | 25 | # Should not be set globally, but just for this target! 26 | FetchContent_Populate(leveldb) 27 | add_subdirectory(${leveldb_SOURCE_DIR} ${leveldb_BINARY_DIR} EXCLUDE_FROM_ALL) 28 | endif() 29 | 30 | include_directories(${leveldb_SOURCE_DIR}/include) 31 | -------------------------------------------------------------------------------- /cmake/lmdb.cmake: -------------------------------------------------------------------------------- 1 | # lmdb: 2 | # https://github.com/LMDB/lmdb/blob/mdb.master/libraries/liblmdb/Makefile 3 | 4 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 5 | 6 | include(ExternalProject) 7 | 8 | ExternalProject_Add( 9 | lmdb_external 10 | 11 | GIT_REPOSITORY "https://github.com/LMDB/lmdb.git" 12 | GIT_TAG LMDB_0.9.29 13 | GIT_SHALLOW 1 14 | GIT_PROGRESS 0 15 | 16 | PREFIX "${PREFIX_DIR}" 17 | DOWNLOAD_DIR "${PREFIX_DIR}/lmdb-src" 18 | LOG_DIR "${PREFIX_DIR}/lmdb-log" 19 | STAMP_DIR "${PREFIX_DIR}/lmdb-stamp" 20 | TMP_DIR "${PREFIX_DIR}/lmdb-tmp" 21 | SOURCE_DIR "${PREFIX_DIR}/lmdb-src" 22 | INSTALL_DIR "${PREFIX_DIR}/lmdb-install" 23 | BINARY_DIR "${PREFIX_DIR}/lmdb-build" 24 | 25 | CONFIGURE_COMMAND "" 26 | UPDATE_COMMAND "" 27 | INSTALL_COMMAND "" 28 | BUILD_ALWAYS 0 29 | 30 | BUILD_COMMAND make install prefix=${PREFIX_DIR}/lmdb-install CXXFLAGS="-Wno-implicit-fallthrough" -C "${PREFIX_DIR}/lmdb-src/libraries/liblmdb" 31 | ) 32 | 33 | set(lmdb_INCLUDE_DIR ${PREFIX_DIR}/lmdb-src/libraries/liblmdb) 34 | set(lmdb_LIBRARY_PATH ${PREFIX_DIR}/lmdb-src/libraries/liblmdb/liblmdb.a) 35 | 36 | file(MAKE_DIRECTORY ${lmdb_INCLUDE_DIR}) 37 | add_library(lmdb STATIC IMPORTED) 38 | 39 | set_property(TARGET lmdb PROPERTY IMPORTED_LOCATION ${lmdb_LIBRARY_PATH}) 40 | set_property(TARGET lmdb APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${lmdb_INCLUDE_DIR}) 41 | 42 | include_directories(${lmdb_INCLUDE_DIR}) 43 | add_dependencies(lmdb lmdb_external) -------------------------------------------------------------------------------- /cmake/mongodb.cmake: -------------------------------------------------------------------------------- 1 | # mongodb: 2 | # https://github.com/mongodb/mongo-c-driver/blob/master/CMakeLists.txt 3 | # https://github.com/mongodb/mongo-cxx-driver/blob/master/CMakeLists.txt 4 | 5 | include(FetchContent) 6 | 7 | # Build mongo-c-driver 8 | FetchContent_Declare( 9 | mongoc 10 | GIT_REPOSITORY https://github.com/mongodb/mongo-c-driver.git 11 | GIT_TAG 1.23.2 12 | GIT_SHALLOW TRUE 13 | ) 14 | 15 | FetchContent_GetProperties(mongoc) 16 | 17 | if(NOT mongoc_POPULATED) 18 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-implicit-function-declaration") 19 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-deprecated-declarations") 20 | 21 | # Should not be set globally, but just for this target! 22 | FetchContent_Populate(mongoc) 23 | add_subdirectory(${mongoc_SOURCE_DIR} ${mongoc_BINARY_DIR} EXCLUDE_FROM_ALL) 24 | endif() 25 | 26 | 27 | # Build mongo-cxx-driver 28 | FetchContent_Declare( 29 | mongocxx 30 | GIT_REPOSITORY https://github.com/mongodb/mongo-cxx-driver.git 31 | GIT_TAG r3.7.0 32 | GIT_SHALLOW TRUE 33 | ) 34 | 35 | FetchContent_GetProperties(mongocxx) 36 | 37 | if(NOT mongocxx_POPULATED) 38 | set(ENABLE_TESTS OFF CACHE INTERNAL "") 39 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated") 40 | 41 | # Should not be set globally, but just for this target! 42 | FetchContent_Populate(mongocxx) 43 | add_subdirectory(${mongocxx_SOURCE_DIR} ${mongocxx_BINARY_DIR} EXCLUDE_FROM_ALL) 44 | endif() 45 | 46 | include_directories(${mongocxx_SOURCE_DIR}/src) 47 | include_directories(${mongocxx_BINARY_DIR}/src) 48 | 49 | add_library(mongodb ALIAS mongocxx_shared) -------------------------------------------------------------------------------- /cmake/nlohmannjson.cmake: -------------------------------------------------------------------------------- 1 | # nlohmannjson: 2 | # https://github.com/nlohmann/json/blob/develop/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | nlohmannjson 7 | GIT_REPOSITORY https://github.com/nlohmann/json.git 8 | GIT_TAG v3.11.2 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(nlohmannjson) 13 | 14 | if(NOT nlohmannjson_POPULATED) 15 | # Should not be set globally, but just for this target! 16 | FetchContent_Populate(nlohmannjson) 17 | add_subdirectory(${nlohmannjson_SOURCE_DIR} ${nlohmannjson_BINARY_DIR} EXCLUDE_FROM_ALL) 18 | endif() 19 | 20 | include_directories(${nlohmannjson_SOURCE_DIR}/include) 21 | -------------------------------------------------------------------------------- /cmake/redis.cmake: -------------------------------------------------------------------------------- 1 | # redis: 2 | # https://github.com/sewenew/redis-plus-plus/blob/master/CMakeLists.txt 3 | 4 | include("${CMAKE_MODULE_PATH}/hiredis.cmake") 5 | 6 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 7 | 8 | include(ExternalProject) 9 | 10 | ExternalProject_Add( 11 | redis_external 12 | 13 | GIT_REPOSITORY "https://github.com/sewenew/redis-plus-plus.git" 14 | GIT_TAG 1.3.7 15 | GIT_SHALLOW 1 16 | GIT_PROGRESS 0 17 | 18 | PREFIX "${PREFIX_DIR}" 19 | DOWNLOAD_DIR "${PREFIX_DIR}/redis-src" 20 | LOG_DIR "${PREFIX_DIR}/redis-log" 21 | STAMP_DIR "${PREFIX_DIR}/redis-stamp" 22 | TMP_DIR "${PREFIX_DIR}/redis-tmp" 23 | SOURCE_DIR "${PREFIX_DIR}/redis-src" 24 | INSTALL_DIR "${PREFIX_DIR}/redis-install" 25 | BINARY_DIR "${PREFIX_DIR}/redis-build" 26 | 27 | BUILD_ALWAYS 0 28 | UPDATE_COMMAND "" 29 | 30 | CMAKE_ARGS 31 | -DCMAKE_PREFIX_PATH:PATH=${PREFIX_DIR}/hiredis-install 32 | -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX_DIR}/redis-install 33 | -DCMAKE_INSTALL_LIBDIR=lib 34 | -DCMAKE_INSTALL_RPATH:PATH=/lib 35 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 36 | -DENABLE_STATIC:STRING=ON 37 | -DENABLE_CPPSUITE:BOOL=OFF 38 | -DCMAKE_C_FLAGS=-Wno-maybe-uninitialized 39 | -DCMAKE_CXX_FLAGS=-Wno-unused-variable 40 | -DREDIS_PLUS_PLUS_BUILD_TEST:BOOL=OFF 41 | ) 42 | 43 | set(redis_INCLUDE_DIR ${PREFIX_DIR}/redis-install/include) 44 | set(redis_LIBRARY_PATH ${PREFIX_DIR}/redis-install/lib/libredis++.a) 45 | 46 | file(MAKE_DIRECTORY ${redis_INCLUDE_DIR}) 47 | add_library(redis STATIC IMPORTED) 48 | set_target_properties(redis PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES hiredis) 49 | 50 | set_property(TARGET redis PROPERTY IMPORTED_LOCATION ${redis_LIBRARY_PATH}) 51 | set_property(TARGET redis APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${redis_INCLUDE_DIR}) 52 | 53 | include_directories(${redis_INCLUDE_DIR}) 54 | add_dependencies(redis_external hiredis) 55 | add_dependencies(redis redis_external) -------------------------------------------------------------------------------- /cmake/rocksdb.cmake: -------------------------------------------------------------------------------- 1 | # RocksDB: 2 | # https://github.com/facebook/rocksdb/blob/main/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | rocksdb 7 | GIT_REPOSITORY https://github.com/facebook/rocksdb.git 8 | GIT_TAG v7.9.2 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(rocksdb) 13 | 14 | if(NOT rocksdb_POPULATED) 15 | set(WITH_LIBURING OFF CACHE INTERNAL "") 16 | set(WITH_SNAPPY OFF CACHE INTERNAL "") 17 | set(WITH_LZ4 OFF CACHE INTERNAL "") 18 | set(WITH_GFLAGS OFF CACHE INTERNAL "") 19 | set(WITH_JEMALLOC OFF CACHE INTERNAL "") 20 | 21 | set(FAIL_ON_WARNINGS OFF CACHE INTERNAL "") 22 | set(USE_RTTI 1 CACHE INTERNAL "") 23 | set(FORCE_SSE42 OFF CACHE INTERNAL "") 24 | 25 | set(PORTABLE ON CACHE INTERNAL "") 26 | set(BUILD_SHARED OFF CACHE INTERNAL "") 27 | 28 | set(WITH_JNI OFF CACHE INTERNAL "") 29 | set(WITH_CORE_TOOLS OFF CACHE INTERNAL "") 30 | set(WITH_TOOLS OFF CACHE INTERNAL "") 31 | set(WITH_TESTS OFF CACHE INTERNAL "") 32 | set(WITH_ALL_TESTS OFF CACHE INTERNAL "") 33 | set(WITH_BENCHMARK_TOOLS OFF CACHE INTERNAL "") 34 | 35 | set(CMAKE_ENABLE_SHARED OFF CACHE INTERNAL "") 36 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-implicit-fallthrough") 37 | 38 | FetchContent_Populate(rocksdb) 39 | add_subdirectory(${rocksdb_SOURCE_DIR} ${rocksdb_BINARY_DIR} EXCLUDE_FROM_ALL) 40 | endif() 41 | 42 | include_directories(${rocksdb_SOURCE_DIR}/include) 43 | -------------------------------------------------------------------------------- /cmake/tabulate.cmake: -------------------------------------------------------------------------------- 1 | # tabulate: 2 | # https://github.com/p-ranav/tabulate/blob/master/CMakeLists.txt 3 | 4 | include(FetchContent) 5 | FetchContent_Declare( 6 | tabulate 7 | GIT_REPOSITORY https://github.com/p-ranav/tabulate.git 8 | GIT_TAG v1.5 9 | GIT_SHALLOW TRUE 10 | ) 11 | 12 | FetchContent_GetProperties(tabulate) 13 | 14 | if(NOT tabulate_POPULATED) 15 | set(tabulate_BUILD_TESTS OFF CACHE INTERNAL "") 16 | 17 | # Should not be set globally, but just for this target! 18 | FetchContent_Populate(tabulate) 19 | add_subdirectory(${tabulate_SOURCE_DIR} ${tabulate_BINARY_DIR} EXCLUDE_FROM_ALL) 20 | endif() 21 | 22 | include_directories(${tabulate_SOURCE_DIR}/include) 23 | -------------------------------------------------------------------------------- /cmake/uring.cmake: -------------------------------------------------------------------------------- 1 | # uring: 2 | # https://github.com/axboe/liburing/blob/master/Makefile 3 | 4 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 5 | 6 | include(ExternalProject) 7 | 8 | ExternalProject_Add( 9 | uring_external 10 | 11 | GIT_REPOSITORY "https://github.com/axboe/liburing.git" 12 | GIT_TAG liburing-2.3 13 | GIT_SHALLOW 1 14 | GIT_PROGRESS 0 15 | 16 | PREFIX "${PREFIX_DIR}" 17 | DOWNLOAD_DIR "${PREFIX_DIR}/uring-src" 18 | LOG_DIR "${PREFIX_DIR}/uring-log" 19 | STAMP_DIR "${PREFIX_DIR}/uring-stamp" 20 | TMP_DIR "${PREFIX_DIR}/uring-tmp" 21 | SOURCE_DIR "${PREFIX_DIR}/uring-src" 22 | INSTALL_DIR "${PREFIX_DIR}/uring-install" 23 | BINARY_DIR "${PREFIX_DIR}/uring-build" 24 | 25 | CONFIGURE_COMMAND "" 26 | UPDATE_COMMAND "" 27 | INSTALL_COMMAND "" 28 | BUILD_ALWAYS 0 29 | 30 | BUILD_COMMAND make install prefix=${PREFIX_DIR}/uring-install -C "${PREFIX_DIR}/uring-src" 31 | ) 32 | 33 | set(uring_INCLUDE_DIR ${PREFIX_DIR}/uring-src/src/include) 34 | set(uring_LIBRARY_PATH ${PREFIX_DIR}/uring-install/lib/liburing.a) 35 | 36 | file(MAKE_DIRECTORY ${uring_INCLUDE_DIR}) 37 | add_library(uring STATIC IMPORTED) 38 | 39 | set_property(TARGET uring PROPERTY IMPORTED_LOCATION ${uring_LIBRARY_PATH}) 40 | set_property(TARGET uring APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${uring_INCLUDE_DIR}) 41 | 42 | include_directories(${uring_INCLUDE_DIR}) 43 | add_dependencies(uring uring_external) -------------------------------------------------------------------------------- /cmake/ustore.cmake: -------------------------------------------------------------------------------- 1 | # USTORE: 2 | # https://github.com/unum-cloud/ustore/blob/main/CMakeLists.txt 3 | 4 | # Note: This is set in top CMakeLists.txt 5 | option(USTORE_ENGINE_NAME "Choose engine") 6 | 7 | set(REPOSITORY_BRANCH "main-dev") 8 | list(APPEND BUILD_ARGS "-DUSTORE_BUILD_BUNDLES=1" "-DUSTORE_BUILD_TESTS=0" "-DUSTORE_BUILD_BENCHMARKS=0" "-DUSTORE_BUILD_SANITIZE=0") 9 | 10 | if(USTORE_ENGINE_NAME STREQUAL "FLIGHT_CLIENT") 11 | list(APPEND BUILD_ARGS "-DUSTORE_BUILD_API_FLIGHT=1") 12 | 13 | target_compile_definitions(ucsb_bench PUBLIC USTORE_ENGINE_IS_FLIGHT_CLIENT=1) 14 | elseif(USTORE_ENGINE_NAME STREQUAL "UDISK") 15 | set(ENGINE_UDISK_PATH "${CMAKE_BINARY_DIR}/build/lib/libudisk.a") 16 | list(APPEND BUILD_ARGS "-DUSTORE_BUILD_ENGINE_${USTORE_ENGINE_NAME}=1" "-DUSTORE_ENGINE_NAME_UDISK_PATH=${ENGINE_UDISK_PATH}") 17 | else() 18 | list(APPEND BUILD_ARGS "-DUSTORE_BUILD_ENGINE_${USTORE_ENGINE_NAME}=1") 19 | endif() 20 | 21 | string(TOLOWER ${USTORE_ENGINE_NAME} LOWERCASE_ENGINE_NAME) 22 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 23 | 24 | set(VERSION_URL "https://raw.githubusercontent.com/unum-cloud/ustore/${REPOSITORY_BRANCH}/VERSION") 25 | file(DOWNLOAD "${VERSION_URL}" "${PREFIX_DIR}/ustore-src/VERSION") 26 | file(STRINGS "${PREFIX_DIR}/ustore-src/VERSION" USTORE_VERSION) 27 | 28 | include(ExternalProject) 29 | 30 | ExternalProject_Add( 31 | ustore_external 32 | 33 | GIT_REPOSITORY "https://github.com/unum-cloud/ukv" 34 | GIT_TAG "${REPOSITORY_BRANCH}" 35 | GIT_SHALLOW 1 36 | GIT_PROGRESS 0 37 | GIT_SUBMODULES "" 38 | 39 | PREFIX "${PREFIX_DIR}" 40 | DOWNLOAD_DIR "${PREFIX_DIR}/ustore-src" 41 | LOG_DIR "${PREFIX_DIR}/ustore-log" 42 | STAMP_DIR "${PREFIX_DIR}/ustore-stamp" 43 | TMP_DIR "${PREFIX_DIR}/ustore-tmp" 44 | SOURCE_DIR "${PREFIX_DIR}/ustore-src" 45 | INSTALL_DIR "${PREFIX_DIR}/ustore-install" 46 | BINARY_DIR "${PREFIX_DIR}/ustore-build" 47 | 48 | BUILD_ALWAYS 0 49 | UPDATE_COMMAND "" 50 | 51 | CMAKE_ARGS 52 | -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX_DIR}/ustore-install 53 | -DCMAKE_INSTALL_LIBDIR=lib 54 | -DCMAKE_INSTALL_RPATH:PATH=/lib 55 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 56 | -DENABLE_STATIC:STRING=ON 57 | -DENABLE_CPPSUITE:BOOL=OFF 58 | -DCMAKE_C_FLAGS=-Wno-maybe-uninitialized -Wno-implicit-fallthrough 59 | 60 | "${BUILD_ARGS}" 61 | ) 62 | 63 | list(APPEND ustore_INCLUDE_DIRS ${PREFIX_DIR}/ustore-src/include ${PREFIX_DIR}/ustore-src/src) 64 | set(ustore_LIBRARY_PATH ${PREFIX_DIR}/ustore-build/build/lib/libustore_${LOWERCASE_ENGINE_NAME}_bundle.a) 65 | file(MAKE_DIRECTORY ${ustore_INCLUDE_DIRS}) 66 | 67 | add_library(ustore_intermediate STATIC IMPORTED) 68 | if(USTORE_ENGINE_NAME STREQUAL "UDISK") 69 | target_link_libraries(ustore_intermediate INTERFACE dl pthread explain uring numa tbb) 70 | endif() 71 | 72 | target_compile_definitions(ustore_intermediate INTERFACE USTORE_VERSION="${USTORE_VERSION}") 73 | target_compile_definitions(ustore_intermediate INTERFACE USTORE_ENGINE_NAME_IS_${USTORE_ENGINE_NAME}=1) 74 | target_compile_definitions(ustore_intermediate INTERFACE USTORE_ENGINE_NAME="${LOWERCASE_ENGINE_NAME}") 75 | 76 | set_property(TARGET ustore_intermediate PROPERTY IMPORTED_LOCATION ${ustore_LIBRARY_PATH}) 77 | set_property(TARGET ustore_intermediate APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ustore_INCLUDE_DIRS}) 78 | 79 | include_directories(${ustore_INCLUDE_DIRS}) 80 | add_dependencies(ustore_intermediate ustore_external) 81 | 82 | # Create intermidiat library to hide dependences 83 | add_library(ustore INTERFACE) 84 | target_compile_definitions(ustore INTERFACE ${ustore_DEFINITIONS}) 85 | target_include_directories(ustore INTERFACE ${ustore_INCLUDE_DIRS}) 86 | target_link_libraries(ustore INTERFACE ustore_intermediate) -------------------------------------------------------------------------------- /cmake/wiredtiger.cmake: -------------------------------------------------------------------------------- 1 | # WiredTiger: 2 | # https://github.com/wiredtiger/wiredtiger/blob/develop/CMakeLists.txt 3 | 4 | set(PREFIX_DIR ${CMAKE_BINARY_DIR}/_deps) 5 | 6 | include(ExternalProject) 7 | 8 | ExternalProject_Add( 9 | wiredtiger_external 10 | 11 | GIT_REPOSITORY "https://github.com/wiredtiger/wiredtiger.git" 12 | GIT_TAG 11.1.0 13 | GIT_SHALLOW 1 14 | GIT_PROGRESS 0 15 | 16 | PREFIX "${PREFIX_DIR}" 17 | DOWNLOAD_DIR "${PREFIX_DIR}/wiredtiger-src" 18 | LOG_DIR "${PREFIX_DIR}/wiredtiger-log" 19 | STAMP_DIR "${PREFIX_DIR}/wiredtiger-stamp" 20 | TMP_DIR "${PREFIX_DIR}/wiredtiger-tmp" 21 | SOURCE_DIR "${PREFIX_DIR}/wiredtiger-src" 22 | INSTALL_DIR "${PREFIX_DIR}/wiredtiger-install" 23 | BINARY_DIR "${PREFIX_DIR}/wiredtiger-build" 24 | 25 | BUILD_ALWAYS 0 26 | UPDATE_COMMAND "" 27 | 28 | CMAKE_ARGS 29 | -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX_DIR}/wiredtiger-install 30 | -DCMAKE_INSTALL_LIBDIR=lib 31 | -DCMAKE_INSTALL_RPATH:PATH=/lib 32 | -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} 33 | -DENABLE_STATIC:STRING=ON 34 | -DENABLE_CPPSUITE:BOOL=OFF 35 | -DCMAKE_C_FLAGS=-Wno-maybe-uninitialized 36 | -DCMAKE_CXX_FLAGS=-Wno-unused-variable 37 | -DENABLE_PYTHON:BOOL=OFF 38 | -DENABLE_LLVM:BOOL=OFF 39 | -DHAVE_UNITTEST:BOOL=OFF 40 | -DENABLE_SHARED:BOOL=OFF 41 | ) 42 | 43 | set(wiredtiger_INCLUDE_DIR ${PREFIX_DIR}/wiredtiger-install/include) 44 | set(wiredtiger_LIBRARY_PATH ${PREFIX_DIR}/wiredtiger-install/lib/libwiredtiger.a) 45 | 46 | file(MAKE_DIRECTORY ${wiredtiger_INCLUDE_DIR}) 47 | add_library(wiredtiger STATIC IMPORTED) 48 | 49 | set_property(TARGET wiredtiger PROPERTY IMPORTED_LOCATION ${wiredtiger_LIBRARY_PATH}) 50 | set_property(TARGET wiredtiger APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${wiredtiger_INCLUDE_DIR}) 51 | 52 | include_directories(${wiredtiger_INCLUDE_DIR}) 53 | add_dependencies(wiredtiger wiredtiger_external) -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = ../build/docs 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | p.caption { 2 | font-size: 0 !important; 3 | margin: 8px 0px !important; 4 | padding: 0 !important; 5 | border-bottom: 1px solid #8b7f8b12; 6 | } 7 | 8 | article>section>h1:nth-child(1) { 9 | display: none; 10 | } 11 | 12 | .sidebar-brand-text { 13 | cursor: initial; 14 | } 15 | 16 | table>tbody>tr>td { 17 | text-align: center; 18 | } 19 | 20 | table>tbody>tr>td:first-child { 21 | text-align: left; 22 | } 23 | 24 | #overview>p>a>img { 25 | height: 25px !important; 26 | } -------------------------------------------------------------------------------- /docs/_static/custom.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function(){ 2 | const github_logo = ` 3 | 4 | ` 5 | 6 | $(".sidebar-brand-text").html("Unum · UCSB
$(VERSION)" + github_logo) 7 | }) 8 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = 'Unum · UCSB' 10 | copyright = '2023, Unum' 11 | author = 'Unum' 12 | release = open('../VERSION', 'r').read().strip() 13 | with open('_static/custom.js', 'r+') as js: 14 | content = js.read() 15 | js.seek(0) 16 | js.truncate() 17 | js.write(content.replace('$(VERSION)', release)) 18 | 19 | # -- General configuration --------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 21 | 22 | extensions = [ 23 | 'breathe', 'm2r2', 24 | 'sphinx.ext.autodoc', 25 | 'sphinx.ext.autosummary', 26 | 'sphinx.ext.intersphinx', 27 | 'sphinx.ext.napoleon', 28 | 'sphinxcontrib.jquery', 29 | 'sphinxcontrib.googleanalytics'] 30 | 31 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '*.md'] 32 | 33 | googleanalytics_id = '341385789' 34 | googleanalytics_enabled = True 35 | 36 | # -- Options for HTML output ------------------------------------------------- 37 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 38 | 39 | html_logo = '../assets/unum.png' 40 | html_theme = 'furo' 41 | html_static_path = ['_static'] 42 | html_css_files = ['custom.css'] 43 | html_js_files = ['custom.js'] 44 | 45 | breathe_projects = {'UCSB': '../build/xml'} 46 | breathe_default_project = 'UCSB' 47 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Overview 3 | ========== 4 | .. mdinclude:: ../README.md 5 | 6 | .. toctree:: 7 | :hidden: 8 | 9 | self 10 | 11 | .. toctree:: 12 | :hidden: 13 | :caption: � 14 | 15 | reference 16 | 17 | .. toctree:: 18 | :hidden: 19 | :caption: � 20 | 21 | genindex -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | =============== 3 | 4 | 5 | .. doxygenfile:: db.hpp 6 | .. doxygenfile:: types.hpp -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import time 6 | import shutil 7 | import signal 8 | import pexpect 9 | import pathlib 10 | import argparse 11 | import termcolor 12 | 13 | """ 14 | Below are listed the supported databases, benchmark sizes, workloads and more settings. 15 | All of them are optional and treated as defaults, so you can select settings here or pass arguments from outside. 16 | Usage: sudo ./run.py [arguments], type -h or --help for help. 17 | Note there are some databases and sizes commented by default just to minimize the sample run, so select them according to your needs. 18 | """ 19 | 20 | db_names = [ 21 | "ustore", 22 | "rocksdb", 23 | "leveldb", 24 | "wiredtiger", 25 | # "mongodb", 26 | # "redis", 27 | # "lmdb", 28 | ] 29 | 30 | sizes = [ 31 | "100MB", 32 | # "1GB", 33 | # "10GB", 34 | # "100GB", 35 | # "1TB", 36 | # "10TB", 37 | ] 38 | 39 | workload_names = [ 40 | "Init", 41 | "Read", 42 | "BatchRead", 43 | "RangeSelect", 44 | "Scan", 45 | "ReadUpdate_50_50", 46 | "ReadUpsert_95_5", 47 | "BatchUpsert", 48 | "Remove", 49 | ] 50 | 51 | threads_count = 1 52 | transactional = False 53 | 54 | drop_caches = False 55 | run_in_docker_container = False 56 | 57 | main_dir_path = "./db_main/" 58 | storage_disk_paths = [ 59 | # '/disk1/db_storage/', 60 | # '/disk2/db_storage/', 61 | ] 62 | 63 | 64 | def get_db_config_file_path(db_name: str, size: str) -> str: 65 | path = os.path.join("./bench/configs", db_name, f"{size}.cfg") 66 | if not pathlib.Path(path).exists(): 67 | path = os.path.join("./bench/configs", db_name, "default.cfg") 68 | return path 69 | 70 | 71 | def get_workloads_file_path(size: str) -> str: 72 | return os.path.join("./bench/workloads", f"{size}.json") 73 | 74 | 75 | def get_db_main_dir_path(db_name: str, size: str, main_dir_path: str) -> str: 76 | return os.path.join(main_dir_path, db_name, size, "") 77 | 78 | 79 | def get_db_storage_dir_paths(db_name: str, size: str, storage_disk_paths: str) -> list: 80 | db_storage_dir_paths = [] 81 | for storage_disk_path in storage_disk_paths: 82 | db_storage_dir_paths.append(os.path.join(storage_disk_path, db_name, size, "")) 83 | return db_storage_dir_paths 84 | 85 | 86 | def get_results_file_path( 87 | db_name: str, 88 | size: str, 89 | drop_caches: bool, 90 | transactional: bool, 91 | storage_disk_paths: str, 92 | threads_count: int, 93 | ) -> str: 94 | root_dir_path = "" 95 | if drop_caches: 96 | if transactional: 97 | root_dir_path = "./bench/results/without_caches/transactional/" 98 | else: 99 | root_dir_path = "./bench/results/without_caches/" 100 | else: 101 | if transactional: 102 | root_dir_path = "./bench/results/transactional/" 103 | else: 104 | root_dir_path = "./bench/results/" 105 | 106 | disks_count = max(len(storage_disk_paths), 1) 107 | return os.path.join( 108 | f"{root_dir_path}cores_{threads_count}", 109 | f"disks_{disks_count}", 110 | db_name, 111 | f"{size}.json", 112 | ) 113 | 114 | 115 | def drop_system_caches(): 116 | print(end="\x1b[1K\r") 117 | print(" [✱] Dropping system caches...", end="\r") 118 | try: 119 | with open("/proc/sys/vm/drop_caches", "w") as stream: 120 | stream.write("3\n") 121 | time.sleep(8) # Wait for other apps to reload its caches 122 | except KeyboardInterrupt: 123 | print(termcolor.colored("Terminated by user", "yellow")) 124 | exit(1) 125 | except: 126 | print(termcolor.colored("Failed to drop system caches", "red")) 127 | exit(1) 128 | 129 | 130 | def run( 131 | db_name: str, 132 | size: str, 133 | workload_names: list, 134 | main_dir_path: str, 135 | storage_disk_paths: str, 136 | transactional: bool, 137 | drop_caches: bool, 138 | run_in_docker_container: bool, 139 | threads_count: bool, 140 | run_index: int, 141 | runs_count: int, 142 | ) -> None: 143 | db_config_file_path = get_db_config_file_path(db_name, size) 144 | workloads_file_path = get_workloads_file_path(size) 145 | db_main_dir_path = get_db_main_dir_path(db_name, size, main_dir_path) 146 | db_storage_dir_paths = get_db_storage_dir_paths(db_name, size, storage_disk_paths) 147 | results_file_path = get_results_file_path( 148 | db_name, size, drop_caches, transactional, storage_disk_paths, threads_count 149 | ) 150 | 151 | transactional_flag = "-t" if transactional else "" 152 | filter = ",".join(workload_names) 153 | db_storage_dir_paths = ",".join(db_storage_dir_paths) 154 | 155 | runner: str 156 | if run_in_docker_container: 157 | runner = f"docker run -v {os.getcwd()}/bench:/ucsb/bench -v {os.getcwd()}/tmp:/ucsb/tmp -it ucsb-image-dev" 158 | else: 159 | runner = "./build_release/build/bin/ucsb_bench" 160 | if not os.path.exists(runner): 161 | raise Exception("First, please build the runner: `build_release.sh`") 162 | 163 | process = pexpect.spawn( 164 | f'{runner} -db {db_name} {transactional_flag} -cfg "{db_config_file_path}" -wl "{workloads_file_path}" -md "{db_main_dir_path}" -sd "{db_storage_dir_paths}" -res "{results_file_path}" -th {threads_count} -fl {filter} -ri {run_index} -rc {runs_count}' 165 | ) 166 | process.interact() 167 | process.close() 168 | 169 | # Handle signal 170 | if process.signalstatus: 171 | sig = signal.Signals(process.signalstatus) 172 | if sig == signal.SIGINT: 173 | print(termcolor.colored("Benchmark terminated by user", "yellow")) 174 | else: 175 | print( 176 | termcolor.colored(f"Benchmark terminated (signal: {sig.name})", "red") 177 | ) 178 | exit(process.signalstatus) 179 | 180 | 181 | def parse_args(): 182 | global db_names 183 | global sizes 184 | global workload_names 185 | global main_dir_path 186 | global storage_disk_paths 187 | global threads_count 188 | global transactional 189 | global drop_caches 190 | global run_in_docker_container 191 | 192 | parser = argparse.ArgumentParser() 193 | 194 | parser.add_argument( 195 | "-db", 196 | "--db-names", 197 | help="Database name(s)", 198 | nargs="+", 199 | required=False, 200 | default=db_names, 201 | ) 202 | parser.add_argument( 203 | "-sz", 204 | "--sizes", 205 | help="Benchmark size(s)", 206 | nargs="+", 207 | required=False, 208 | default=sizes, 209 | ) 210 | parser.add_argument( 211 | "-wl", 212 | "--workload-names", 213 | help="Workload name(s)", 214 | nargs="+", 215 | required=False, 216 | default=workload_names, 217 | ) 218 | parser.add_argument( 219 | "-md", 220 | "--main-dir", 221 | help="Main directory", 222 | type=str, 223 | required=False, 224 | default=main_dir_path, 225 | ) 226 | parser.add_argument( 227 | "-sd", 228 | "--storage-dirs", 229 | help="Storage directories", 230 | nargs="+", 231 | required=False, 232 | default=storage_disk_paths, 233 | ) 234 | parser.add_argument( 235 | "-th", 236 | "--threads", 237 | help="Threads count", 238 | type=int, 239 | required=False, 240 | default=threads_count, 241 | ) 242 | parser.add_argument( 243 | "-tx", 244 | "--transactional", 245 | help="Transactional benchmark", 246 | action=argparse.BooleanOptionalAction, 247 | default=transactional, 248 | ) 249 | parser.add_argument( 250 | "-dp", 251 | "--drop-caches", 252 | help="Drops system cashes before each benchmark", 253 | action=argparse.BooleanOptionalAction, 254 | default=drop_caches, 255 | ) 256 | parser.add_argument( 257 | "-rd", 258 | "--run-docker", 259 | help="Runs the benchmark in a docker container", 260 | action=argparse.BooleanOptionalAction, 261 | default=run_in_docker_container, 262 | ) 263 | 264 | args = parser.parse_args() 265 | db_names = args.db_names 266 | sizes = args.sizes 267 | workload_names = args.workload_names 268 | main_dir_path = args.main_dir 269 | storage_disk_paths = args.storage_dirs 270 | threads_count = args.threads 271 | transactional = args.transactional 272 | drop_caches = args.drop_caches 273 | run_in_docker_container = args.run_docker 274 | 275 | 276 | def check_args(): 277 | global db_names 278 | global sizes 279 | global workload_names 280 | 281 | if not db_names: 282 | sys.exit("Database(s) not specified") 283 | if not sizes: 284 | sys.exit("Database size(s) not specified") 285 | if not workload_names: 286 | sys.exit("Workload name(s) not specified") 287 | 288 | 289 | def main() -> None: 290 | if os.geteuid() != 0: 291 | print(termcolor.colored(f"Run as sudo!", "red")) 292 | sys.exit(-1) 293 | 294 | parse_args() 295 | check_args() 296 | 297 | # Cleanup old DBs (Note: It actually cleanups if the first workload is `Init`) 298 | if workload_names[0] == "Init": 299 | print(end="\x1b[1K\r") 300 | print(" [✱] Cleanup...", end="\r") 301 | for size in sizes: 302 | for db_name in db_names: 303 | # Remove DB main directory 304 | db_main_dir_path = get_db_main_dir_path(db_name, size, main_dir_path) 305 | if pathlib.Path(db_main_dir_path).exists(): 306 | shutil.rmtree(db_main_dir_path) 307 | 308 | # Remove DB storage directories 309 | db_storage_dir_paths = get_db_storage_dir_paths( 310 | db_name, size, storage_disk_paths 311 | ) 312 | for db_storage_dir_path in db_storage_dir_paths: 313 | if pathlib.Path(db_storage_dir_path).exists(): 314 | shutil.rmtree(db_storage_dir_path) 315 | 316 | # Run benchmarks 317 | for size in sizes: 318 | for db_name in db_names: 319 | # Create DB main directory 320 | db_main_dir_path = get_db_main_dir_path(db_name, size, main_dir_path) 321 | pathlib.Path(db_main_dir_path).mkdir(parents=True, exist_ok=True) 322 | 323 | # Create DB storage directories 324 | db_storage_dir_paths = get_db_storage_dir_paths( 325 | db_name, size, storage_disk_paths 326 | ) 327 | for db_storage_dir_path in db_storage_dir_paths: 328 | pathlib.Path(db_storage_dir_path).mkdir(parents=True, exist_ok=True) 329 | 330 | # Create results dir paths 331 | results_file_path = get_results_file_path( 332 | db_name, 333 | size, 334 | drop_caches, 335 | transactional, 336 | storage_disk_paths, 337 | threads_count, 338 | ) 339 | pathlib.Path(results_file_path).parent.mkdir(parents=True, exist_ok=True) 340 | 341 | # Run benchmark 342 | if drop_caches: 343 | for i, workload_name in enumerate(workload_names): 344 | if not run_in_docker_container: 345 | drop_system_caches() 346 | run( 347 | db_name, 348 | size, 349 | [workload_name], 350 | main_dir_path, 351 | storage_disk_paths, 352 | transactional, 353 | drop_caches, 354 | run_in_docker_container, 355 | threads_count, 356 | i, 357 | len(workload_names), 358 | ) 359 | else: 360 | run( 361 | db_name, 362 | size, 363 | workload_names, 364 | main_dir_path, 365 | storage_disk_paths, 366 | transactional, 367 | drop_caches, 368 | run_in_docker_container, 369 | threads_count, 370 | 0, 371 | 1, 372 | ) 373 | 374 | 375 | if __name__ == "__main__": 376 | main() 377 | -------------------------------------------------------------------------------- /src/core/aligned_buffer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace ucsb { 6 | 7 | /** 8 | * @brief Aligned buffer for direct file I/O 9 | */ 10 | class aligned_buffer_t { 11 | public: 12 | constexpr static size_t alignment_k = 4096; 13 | 14 | inline aligned_buffer_t() noexcept : buffer_(nullptr), size_(0) {} 15 | inline aligned_buffer_t(size_t size) : buffer_(nullptr), size_(size) { 16 | assert(size % alignment_k == 0); 17 | buffer_ = reinterpret_cast(std::aligned_alloc(alignment_k, size)); 18 | } 19 | 20 | inline aligned_buffer_t(aligned_buffer_t const& other) : aligned_buffer_t(other.size_) { 21 | memcpy(buffer_, other.buffer_, size_); 22 | } 23 | inline aligned_buffer_t(aligned_buffer_t&& other) noexcept 24 | : buffer_(std::move(other.buffer_)), size_(std::move(other.size_)) { 25 | other.buffer_ = nullptr; 26 | other.size_ = 0; 27 | } 28 | 29 | ~aligned_buffer_t() { std::free(buffer_); } 30 | 31 | inline aligned_buffer_t operator=(aligned_buffer_t const& other) { 32 | aligned_buffer_t copy(other); 33 | std::swap(buffer_, copy.buffer_); 34 | std::swap(size_, copy.size_); 35 | return *this; 36 | } 37 | inline aligned_buffer_t operator=(aligned_buffer_t&& other) { 38 | std::swap(buffer_, other.buffer_); 39 | std::swap(size_, other.size_); 40 | return *this; 41 | } 42 | 43 | inline size_t size() const noexcept { return size_; } 44 | 45 | inline std::byte& operator[](size_t idx) noexcept { return buffer_[idx]; } 46 | inline std::byte const& operator[](size_t idx) const noexcept { return buffer_[idx]; } 47 | 48 | inline std::byte* data() noexcept { return buffer_; } 49 | inline std::byte const* data() const noexcept { return buffer_; } 50 | 51 | private: 52 | std::byte* buffer_; 53 | size_t size_; 54 | }; 55 | 56 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/data_accessor.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/types.hpp" 6 | #include "src/core/operation.hpp" 7 | 8 | namespace ucsb { 9 | 10 | /** 11 | * @brief A base class for data accessing: on DBs and Transactions state. 12 | * 13 | * @section Keys. 14 | * The key type is set to a 64-bit unsigned integer. 15 | * Todays engines often support string keys of variable length, 16 | * but generally work faster if keys are of identical length. 17 | * To avoid extra heap allocations and expensive integer formatting, 18 | * under the hood, we pass a view to raw bytes forming the integer key. 19 | * For order consistency it's also recommended to provide a custom 20 | * comparator. 21 | * @section Values. 22 | * The Value buffer can be bigger than actual value because of memory alignment 23 | * Note: This is done for read workloads only 24 | */ 25 | class data_accessor_t { 26 | public: 27 | virtual ~data_accessor_t() {} 28 | 29 | virtual operation_result_t upsert(key_t key, value_spanc_t value) = 0; 30 | virtual operation_result_t update(key_t key, value_spanc_t value) = 0; 31 | virtual operation_result_t remove(key_t key) = 0; 32 | virtual operation_result_t read(key_t key, value_span_t value) const = 0; 33 | 34 | /** 35 | * @brief Performs many upsert at once in a batch-asynchronous fashion. 36 | * 37 | * @param keys Keys are in strict ascending order. 38 | * @param values Values are written in continuous form. 39 | * @param sizes Value sizes. 40 | */ 41 | virtual operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) = 0; 42 | 43 | /** 44 | * @brief Performs many reads at once in a batch-asynchronous fashion. 45 | * This means, that the order of lookups within the batch is irrelevant 46 | * and the engine can reorganize them for faster execution. 47 | * 48 | * Every DB engine implements the interface for this operation in a different 49 | * way, making designing generic interfaces cumbersome and costly (performance- 50 | * wise). For this benchmark we don't return the retrieved the values and only 51 | * check them under the hood. 52 | * 53 | * @param keys Keys are random. 54 | * @param values A single buffer for all concatenated values. 55 | */ 56 | virtual operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const = 0; 57 | 58 | /** 59 | * @brief Performs many insert at once. 60 | * In contrast to `batch_upsert` this interface used to initialize DB, 61 | * assume that after every `batch_upsert` DB flushes, but in this case 62 | * DB should do many `bulk_load`s and periodically flushes 63 | * 64 | * @param keys Keys are in strict ascending order. 65 | * @param values Values are written in continuous form. 66 | * @param sizes Value sizes. 67 | */ 68 | virtual operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) = 0; 69 | 70 | /** 71 | * @brief Performs many reads at once in an ordered fashion, 72 | * starting from a specified `key` location. 73 | * 74 | * @param key The first entry to find and read. 75 | * @param length The number of consecutive entries to read. 76 | * @param values A temporary buffer big enough for a all values. 77 | */ 78 | virtual operation_result_t range_select(key_t key, size_t length, values_span_t values) const = 0; 79 | 80 | /** 81 | * @brief Performs many reads in an ordered fashion, 82 | * starting from a specified `key` location. 83 | * 84 | * @param key The first entry to find and read. 85 | * @param length The number of consecutive entries to read. 86 | * @param values A temporary buffer big enough for a all values. 87 | */ 88 | virtual operation_result_t scan(key_t key, size_t length, value_span_t single_value) const = 0; 89 | }; 90 | 91 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/db.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "src/core/types.hpp" 8 | #include "src/core/db_hint.hpp" 9 | #include "src/core/data_accessor.hpp" 10 | 11 | namespace ucsb { 12 | 13 | using transaction_t = data_accessor_t; 14 | 15 | /** 16 | * @brief A base class for benchmarking key-value stores. 17 | * This doesn't apply to transactional benchmarks. 18 | * 19 | * General usage procedure is the following: 20 | * 1. Configure `.set_config` and a custom file, 21 | * 2. Recover the state from disk via `.open`, 22 | * 3. Perform all the needed benchmarks, 23 | * 4. Flush the state via `.close`, 24 | * 5. Remove all the data, if needed, via `.destroy`. 25 | */ 26 | class db_t : public data_accessor_t { 27 | public: 28 | virtual ~db_t() {} 29 | 30 | virtual bool open(std::string& error) = 0; 31 | virtual void close() = 0; 32 | 33 | /** 34 | * @brief Returns high level description about the DB 35 | */ 36 | virtual std::string info() = 0; 37 | 38 | /** 39 | * @brief Initializes the DB before usage. 40 | * 41 | * This function can't be used more than once. 42 | * Every DB has it's own format for configuration files. 43 | * LMDB and LevelDB use `.json`, RocksDB uses `.ini`. 44 | * The internal contents and available settings also differ. 45 | * 46 | * @param config_path The path of configuration files. 47 | * @param main_dir_path The target directory, where DB should store metadata. 48 | * @param storage_dir_paths The target directories, where DB should store data. 49 | * @param hints hints for the DB to prepare to work better. 50 | */ 51 | virtual void set_config(fs::path const& config_path, 52 | fs::path const& main_dir_path, 53 | std::vector const& storage_dir_paths, 54 | db_hints_t const& hints) = 0; 55 | 56 | virtual void flush() = 0; 57 | 58 | /** 59 | * @brief Accumulates the size (in bytes) of all the files the engine persisted on disk. 60 | */ 61 | virtual size_t size_on_disk() const = 0; 62 | 63 | virtual std::unique_ptr create_transaction() = 0; 64 | }; 65 | 66 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/db_brand.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "src/core/types.hpp" 4 | #include "src/core/db.hpp" 5 | 6 | #if defined(UCSB_HAS_USTORE) 7 | #include "src/ustore/ustore.hpp" 8 | #endif 9 | #if defined(UCSB_HAS_ROCKSDB) 10 | #include "src/rocksdb/rocksdb.hpp" 11 | #endif 12 | #if defined(UCSB_HAS_LEVELDB) 13 | #include "src/leveldb/leveldb.hpp" 14 | #endif 15 | #if defined(UCSB_HAS_WIREDTIGER) 16 | #include "src/wiredtiger/wiredtiger.hpp" 17 | #endif 18 | #if defined(UCSB_HAS_MONGODB) 19 | #include "src/mongodb/mongodb.hpp" 20 | #endif 21 | #if defined(UCSB_HAS_REDIS) 22 | #include "src/redis/redis.hpp" 23 | #endif 24 | #if defined(UCSB_HAS_LMDB) 25 | #include "src/lmdb/lmdb.hpp" 26 | #endif 27 | 28 | namespace ucsb { 29 | 30 | enum class db_brand_t { 31 | unknown_k, 32 | 33 | ustore_k, 34 | rocksdb_k, 35 | leveldb_k, 36 | wiredtiger_k, 37 | mongodb_k, 38 | redis_k, 39 | lmdb_k, 40 | }; 41 | 42 | std::shared_ptr make_db(db_brand_t db_brand, bool transactional) { 43 | if (transactional) { 44 | switch (db_brand) { 45 | #if defined(UCSB_HAS_USTORE) 46 | case db_brand_t::ustore_k: return std::make_shared(); 47 | #endif 48 | #if defined(UCSB_HAS_ROCKSDB) 49 | case db_brand_t::rocksdb_k: return std::make_shared(facebook::db_mode_t::transactional_k); 50 | #endif 51 | default: break; 52 | } 53 | } 54 | else { 55 | switch (db_brand) { 56 | #if defined(UCSB_HAS_USTORE) 57 | case db_brand_t::ustore_k: return std::make_shared(); 58 | #endif 59 | #if defined(UCSB_HAS_ROCKSDB) 60 | case db_brand_t::rocksdb_k: return std::make_shared(facebook::db_mode_t::regular_k); 61 | #endif 62 | #if defined(UCSB_HAS_LEVELDB) 63 | case db_brand_t::leveldb_k: return std::make_shared(); 64 | #endif 65 | #if defined(UCSB_HAS_WIREDTIGER) 66 | case db_brand_t::wiredtiger_k: return std::make_shared(); 67 | #endif 68 | #if defined(UCSB_HAS_MONGODB) 69 | case db_brand_t::mongodb_k: return std::make_shared(); 70 | #endif 71 | #if defined(UCSB_HAS_REDIS) 72 | case db_brand_t::redis_k: return std::make_shared(); 73 | #endif 74 | #if defined(UCSB_HAS_LMDB) 75 | case db_brand_t::lmdb_k: return std::make_shared(); 76 | #endif 77 | default: break; 78 | } 79 | } 80 | return {}; 81 | } 82 | 83 | inline db_brand_t parse_db_brand(std::string const& name) { 84 | if (name == "ustore") 85 | return db_brand_t::ustore_k; 86 | if (name == "rocksdb") 87 | return db_brand_t::rocksdb_k; 88 | if (name == "leveldb") 89 | return db_brand_t::leveldb_k; 90 | if (name == "wiredtiger") 91 | return db_brand_t::wiredtiger_k; 92 | if (name == "mongodb") 93 | return db_brand_t::mongodb_k; 94 | if (name == "redis") 95 | return db_brand_t::redis_k; 96 | if (name == "lmdb") 97 | return db_brand_t::lmdb_k; 98 | return db_brand_t::unknown_k; 99 | } 100 | 101 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/db_hint.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace ucsb { 6 | 7 | /** 8 | * @brief hints for a DB 9 | */ 10 | struct db_hints_t { 11 | size_t threads_count = 0; 12 | size_t records_count = 0; 13 | size_t value_length = 0; 14 | }; 15 | 16 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/distribution.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace ucsb { 4 | 5 | enum class distribution_kind_t { 6 | unknown_k, 7 | 8 | const_k, 9 | counter_k, 10 | uniform_k, 11 | zipfian_k, 12 | scrambled_zipfian_k, 13 | skewed_latest_k, 14 | acknowledged_counter_k, 15 | }; 16 | 17 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/exception.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "src/core/types.hpp" 4 | 5 | namespace ucsb { 6 | 7 | struct exception_t : public std::exception { 8 | public: 9 | inline exception_t(std::string const& message) : message_(message) {} 10 | inline const char* what() const noexcept override { return message_.c_str(); } 11 | 12 | private: 13 | std::string message_; 14 | }; 15 | 16 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/generators/acknowledged_counter_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/exception.hpp" 6 | #include "src/core/generators/counter_generator.hpp" 7 | 8 | namespace ucsb::core { 9 | 10 | class acknowledged_counter_generator_t : public counter_generator_t { 11 | public: 12 | static const size_t window_size_k = (1 << 16); 13 | static const size_t window_mask_k = window_size_k - 1; 14 | 15 | inline acknowledged_counter_generator_t(uint64_t start) 16 | : counter_generator_t(start), ack_window_(window_size_k, false), limit_(start - 1) {} 17 | 18 | inline size_t last() override { return limit_; } 19 | 20 | void acknowledge(uint64_t value) { 21 | size_t cur_slot = value & window_mask_k; 22 | if (ack_window_[cur_slot]) { 23 | throw exception_t("Not enough window size"); 24 | } 25 | ack_window_[cur_slot] = true; 26 | size_t until = limit_ + window_size_k; 27 | size_t i; 28 | for (i = limit_ + 1; i < until; i++) { 29 | size_t slot = i & window_mask_k; 30 | if (!ack_window_[slot]) { 31 | break; 32 | } 33 | ack_window_[slot] = false; 34 | } 35 | limit_ = i - 1; 36 | } 37 | 38 | private: 39 | std::vector ack_window_; 40 | uint64_t limit_; 41 | }; 42 | 43 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/const_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/generator.hpp" 6 | 7 | namespace ucsb::core { 8 | 9 | template 10 | class const_generator_gt : public generator_gt { 11 | public: 12 | using value_t = value_at; 13 | 14 | inline const_generator_gt(value_t constant) : constant_(constant) {} 15 | 16 | inline value_t generate() override { return constant_; } 17 | inline value_t last() override { return constant_; } 18 | 19 | private: 20 | value_t constant_; 21 | }; 22 | 23 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/counter_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/generator.hpp" 6 | 7 | namespace ucsb::core { 8 | 9 | class counter_generator_t : public generator_gt { 10 | public: 11 | inline counter_generator_t(size_t start) : counter_(start) {} 12 | 13 | inline size_t generate() override { return counter_++; } 14 | inline size_t last() override { return counter_ - 1; } 15 | 16 | protected: 17 | size_t counter_; 18 | }; 19 | 20 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "src/core/types.hpp" 4 | 5 | namespace ucsb::core { 6 | 7 | template 8 | class generator_gt { 9 | public: 10 | using value_t = value_at; 11 | 12 | virtual ~generator_gt() {} 13 | 14 | virtual value_t generate() = 0; 15 | virtual value_t last() = 0; 16 | }; 17 | 18 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/random_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/generator.hpp" 6 | 7 | namespace ucsb::core { 8 | 9 | class random_int_generator_t final : public generator_gt { 10 | public: 11 | inline random_int_generator_t() : device_(), rand_(device_()), last_(0) { generate(); } 12 | 13 | inline uint32_t generate() override { return last_ = rand_(); } 14 | inline uint32_t last() override { return last_; } 15 | 16 | private: 17 | std::random_device device_; 18 | std::minstd_rand rand_; 19 | float last_; 20 | }; 21 | 22 | class random_double_generator_t final : public generator_gt { 23 | public: 24 | inline random_double_generator_t(float min, float max) 25 | : device_(), rand_(device_()), uniform_(min, max), last_(0.0) { 26 | generate(); 27 | } 28 | ~random_double_generator_t() override = default; 29 | 30 | inline float generate() override { return last_ = uniform_(rand_); } 31 | inline float last() override { return last_; } 32 | 33 | private: 34 | std::random_device device_; 35 | std::minstd_rand rand_; 36 | std::uniform_real_distribution uniform_; 37 | float last_; 38 | }; 39 | 40 | class random_byte_generator_t final : public generator_gt { 41 | public: 42 | inline random_byte_generator_t() : off_(6) {} 43 | ~random_byte_generator_t() override = default; 44 | 45 | inline char generate() override; 46 | inline char last() override { return buf_[(off_ - 1 + 6) % 6]; } 47 | 48 | private: 49 | random_int_generator_t generator_; 50 | char buf_[6]; 51 | int off_; 52 | }; 53 | 54 | inline char random_byte_generator_t::generate() { 55 | if (off_ == 6) { 56 | uint32_t bytes = generator_.generate(); 57 | buf_[0] = static_cast((bytes & 31) + ' '); 58 | buf_[1] = static_cast(((bytes >> 5) & 63) + ' '); 59 | buf_[2] = static_cast(((bytes >> 10) & 95) + ' '); 60 | buf_[3] = static_cast(((bytes >> 15) & 31) + ' '); 61 | buf_[4] = static_cast(((bytes >> 20) & 63) + ' '); 62 | buf_[5] = static_cast(((bytes >> 25) & 95) + ' '); 63 | off_ = 0; 64 | } 65 | 66 | return buf_[off_++]; 67 | } 68 | 69 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/scrambled_zipfian_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/zipfian_generator.hpp" 6 | 7 | namespace ucsb::core { 8 | 9 | class scrambled_zipfian_generator_t : public generator_gt { 10 | public: 11 | inline scrambled_zipfian_generator_t(size_t min, size_t max, float zipfian_const) 12 | : base_(min), num_items_(max - min + 1), generator_(0, 10'000'000'000LL, zipfian_const) {} 13 | inline scrambled_zipfian_generator_t(size_t min, size_t max) 14 | : base_(min), num_items_(max - min + 1), 15 | generator_(0, 10'000'000'000LL, zipfian_generator_t::zipfian_const_k, zetan_k) {} 16 | inline scrambled_zipfian_generator_t(size_t num_items) : scrambled_zipfian_generator_t(0, num_items - 1) {} 17 | 18 | inline size_t generate() override { return scramble(generator_.generate()); } 19 | inline size_t last() override { return scramble(generator_.last()); } 20 | 21 | private: 22 | static constexpr float zetan_k = 26.46902820178302; 23 | 24 | inline size_t scramble(size_t value) const noexcept { return base_ + fnv_hash64(value) % num_items_; } 25 | 26 | inline size_t fnv_hash64(size_t val) const noexcept { 27 | size_t constexpr fnv_offset_basis64 = 0xCBF29CE484222325ull; 28 | size_t constexpr fnv_prime64 = 1099511628211ull; 29 | size_t hash = fnv_offset_basis64; 30 | #pragma unroll 31 | for (int i = 0; i < 8; ++i) { 32 | size_t octet = val & 0x00ff; 33 | val = val >> 8; 34 | hash = hash ^ octet; 35 | hash = hash * fnv_prime64; 36 | } 37 | return hash; 38 | } 39 | 40 | size_t const base_; 41 | size_t const num_items_; 42 | zipfian_generator_t generator_; 43 | }; 44 | 45 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/skewed_zipfian_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/counter_generator.hpp" 6 | #include "src/core/generators/zipfian_generator.hpp" 7 | 8 | namespace ucsb::core { 9 | 10 | class skewed_latest_generator_t : public generator_gt { 11 | public: 12 | skewed_latest_generator_t(counter_generator_t& counter) : basis_(&counter), zipfian_(basis_->last()) { generate(); } 13 | 14 | inline size_t generate() override; 15 | inline size_t last() override { return last_; } 16 | 17 | private: 18 | counter_generator_t* basis_; 19 | zipfian_generator_t zipfian_; 20 | size_t last_; 21 | }; 22 | 23 | inline size_t skewed_latest_generator_t::generate() { 24 | 25 | size_t max = basis_->last(); 26 | return last_ = max - zipfian_.generate(max); 27 | } 28 | 29 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/uniform_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/generators/generator.hpp" 6 | 7 | namespace ucsb::core { 8 | 9 | template 10 | class uniform_generator_gt : public generator_gt { 11 | public: 12 | using value_t = value_at; 13 | static_assert(std::is_integral()); 14 | 15 | inline uniform_generator_gt(value_t min, value_t max) : dist_(min, max), last_(0) { generate(); } 16 | inline value_t generate() override { return last_ = dist_(generator_); } 17 | inline value_t last() override { return last_; } 18 | 19 | private: 20 | std::mt19937_64 generator_; 21 | std::uniform_int_distribution dist_; 22 | value_t last_; 23 | }; 24 | 25 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/generators/zipfian_generator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "src/core/generators/generator.hpp" 7 | #include "src/core/generators/random_generator.hpp" 8 | 9 | namespace ucsb::core { 10 | 11 | class zipfian_generator_t : public generator_gt { 12 | public: 13 | static constexpr float zipfian_const_k = 0.99; 14 | static constexpr size_t items_max_count = (UINT64_MAX >> 24); 15 | 16 | zipfian_generator_t(size_t items_count) : zipfian_generator_t(0, items_count - 1) {} 17 | zipfian_generator_t(size_t min, size_t max, float zipfian_const = zipfian_const_k) 18 | : zipfian_generator_t(min, max, zipfian_const, zeta(max - min + 1, zipfian_const)) {} 19 | zipfian_generator_t(size_t min, size_t max, float zipfian_const, float zeta_n); 20 | 21 | inline size_t generate() override { return generate(items_count_); } 22 | inline size_t last() override { return last_; } 23 | 24 | size_t generate(size_t items_count); 25 | 26 | private: 27 | inline float eta() { return (1 - std::pow(2.f / items_count_, 1 - theta_)) / (1 - zeta_2_ / zeta_n_); } 28 | inline float zeta(size_t num, float theta) { return zeta(0, num, theta, 0.f); } 29 | inline float zeta(size_t last_num, size_t cur_num, float theta, float last_zeta); 30 | 31 | random_double_generator_t generator_; 32 | size_t items_count_; 33 | size_t base_; 34 | size_t count_for_zeta_; 35 | size_t last_; 36 | float theta_; 37 | float zeta_n_; 38 | float eta_; 39 | float alpha_; 40 | float zeta_2_; 41 | bool allow_count_decrease_; 42 | }; 43 | 44 | zipfian_generator_t::zipfian_generator_t(size_t min, size_t max, float zipfian_const, float zeta_n) 45 | : generator_(0.0, 1.0), items_count_(max - min + 1), base_(min), theta_(zipfian_const), 46 | allow_count_decrease_(false) { 47 | assert(items_count_ >= 2 && items_count_ < items_max_count); 48 | 49 | zeta_2_ = zeta(2, theta_); 50 | alpha_ = 1.0 / (1.0 - theta_); 51 | zeta_n_ = zeta_n; 52 | count_for_zeta_ = items_count_; 53 | eta_ = eta(); 54 | 55 | generate(); 56 | } 57 | 58 | size_t zipfian_generator_t::generate(size_t num) { 59 | assert(num >= 2 && num < items_max_count); 60 | if (num != count_for_zeta_) { 61 | if (num > count_for_zeta_) { 62 | zeta_n_ = zeta(count_for_zeta_, num, theta_, zeta_n_); 63 | count_for_zeta_ = num; 64 | eta_ = eta(); 65 | } 66 | else if (num < count_for_zeta_ && allow_count_decrease_) { 67 | // TODO 68 | } 69 | } 70 | 71 | float u = generator_.generate(); 72 | float uz = u * zeta_n_; 73 | 74 | if (uz < 1.0) 75 | return last_ = base_; 76 | if (uz < 1.0 + std::pow(0.5, theta_)) 77 | return last_ = base_ + 1; 78 | return last_ = base_ + num * std::pow(eta_ * u - eta_ + 1, alpha_); 79 | } 80 | 81 | inline float zipfian_generator_t::zeta(size_t last_num, size_t cur_num, float theta, float last_zeta) { 82 | float zeta = last_zeta; 83 | for (size_t i = last_num + 1; i <= cur_num; ++i) 84 | zeta += 1 / std::pow(i, theta); 85 | return zeta; 86 | } 87 | 88 | } // namespace ucsb::core -------------------------------------------------------------------------------- /src/core/helper.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "src/core/types.hpp" 7 | 8 | namespace ucsb { 9 | 10 | template 11 | inline at atomic_add_fetch(at& value, at delta) noexcept { 12 | return __atomic_add_fetch(&value, delta, __ATOMIC_RELAXED); 13 | } 14 | 15 | template 16 | inline at atomic_load(at& value) noexcept { 17 | return __atomic_load_n(&value, __ATOMIC_RELAXED); 18 | } 19 | 20 | template 21 | inline void atomic_store(at& value, at desired) noexcept { 22 | __atomic_store_n(&value, desired, __ATOMIC_RELAXED); 23 | } 24 | 25 | template 26 | constexpr size_t roundup_to_multiple(size_t number) noexcept { 27 | static_assert((multiple_ak != 0) && !(multiple_ak & (multiple_ak - 1))); 28 | constexpr size_t one_less = multiple_ak - 1; 29 | constexpr size_t negative_mask = -multiple_ak; 30 | return (number + one_less) & negative_mask; 31 | } 32 | 33 | inline bool start_with(const char* str, const char* prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } 34 | 35 | std::vector split(std::string const& str, char delimiter) { 36 | size_t start = 0; 37 | size_t end = 0; 38 | std::vector tokens; 39 | 40 | while ((start = str.find_first_not_of(delimiter, end)) != std::string::npos) { 41 | end = str.find(delimiter, start); 42 | tokens.push_back(str.substr(start, end - start)); 43 | } 44 | 45 | return tokens; 46 | } 47 | 48 | size_t size_on_disk(fs::path const& path) { 49 | size_t total_size = 0; 50 | for (auto const& entry : fs::directory_iterator(path)) { 51 | if (entry.is_directory()) 52 | total_size += size_on_disk(entry.path()); 53 | else 54 | total_size += fs::file_size(entry.path()); 55 | } 56 | return total_size; 57 | } 58 | 59 | void clear_directory(fs::path const& dir_path) { 60 | for (auto const& entry : fs::directory_iterator(dir_path)) 61 | fs::remove_all(entry.path()); 62 | } 63 | 64 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/operation.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "src/core/generators/random_generator.hpp" 9 | 10 | namespace ucsb { 11 | 12 | enum class operation_kind_t { 13 | upsert_k, 14 | update_k, 15 | remove_k, 16 | read_k, 17 | read_modify_write_k, 18 | batch_upsert_k, 19 | batch_read_k, 20 | bulk_load_k, 21 | range_select_k, 22 | scan_k, 23 | }; 24 | 25 | enum class operation_status_t : int { 26 | ok_k = 1, 27 | error_k = -1, 28 | not_found_k = -2, 29 | not_implemented_k = -3, 30 | }; 31 | 32 | struct operation_result_t { 33 | /** 34 | * @brief The number of entries touched during the operation. 35 | * 36 | * For basic single read/writes will be 1. 37 | * For batch operations is equal to the size of batch. 38 | * For global scans is equal to the total number of entries in DB. 39 | */ 40 | size_t entries_touched = 0; 41 | 42 | operation_status_t status = operation_status_t::ok_k; 43 | }; 44 | 45 | class operation_chooser_t { 46 | public: 47 | inline operation_chooser_t() : generator_(0.0, 1.0), sum_(0) {} 48 | 49 | inline void add(operation_kind_t op, float weight); 50 | inline operation_kind_t choose(); 51 | 52 | private: 53 | std::vector> ops_; 54 | core::random_double_generator_t generator_; 55 | float sum_; 56 | }; 57 | 58 | inline void operation_chooser_t::add(operation_kind_t op, float weight) { 59 | ops_.push_back(std::make_pair(op, weight)); 60 | sum_ += weight; 61 | } 62 | 63 | inline operation_kind_t operation_chooser_t::choose() { 64 | float chooser = generator_.generate(); 65 | for (auto op = ops_.cbegin(); op != ops_.cend(); ++op) { 66 | float part = op->second / sum_; 67 | if (chooser < part) 68 | return op->first; 69 | chooser -= part; 70 | } 71 | 72 | assert(false); 73 | return operation_kind_t::read_k; 74 | } 75 | 76 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/printable.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace ucsb { 6 | 7 | /** 8 | * @brief Formatting sugar for "fmt" library. 9 | */ 10 | struct printable_bytes_t { 11 | size_t bytes = 0; 12 | }; 13 | 14 | struct printable_float_t { 15 | double value = 0; 16 | }; 17 | 18 | struct printable_duration_t { 19 | size_t duration = 0; // In milliseconds 20 | }; 21 | 22 | } // namespace ucsb 23 | 24 | template <> 25 | class fmt::formatter { 26 | public: 27 | size_t suffix_idx = 0; 28 | unsigned char precision = 2; 29 | 30 | template 31 | inline constexpr auto parse(ctx_at& ctx) { 32 | auto it = ctx.begin(); 33 | if (it == ctx.end()) 34 | return it; 35 | 36 | if (*it == '.') { 37 | ++it; 38 | if (it != ctx.end() && *it >= '0' && *it <= '9') 39 | precision = (int)(*it) - (int)48; 40 | else 41 | throw format_error("Invalid precision"); 42 | ++it; 43 | } 44 | 45 | if (it == ctx.end()) 46 | return it; 47 | 48 | switch (*it) { 49 | case 'B': suffix_idx = 1; break; 50 | case 'K': suffix_idx = 2; break; 51 | case 'M': suffix_idx = 3; break; 52 | case 'G': suffix_idx = 4; break; 53 | case 'T': suffix_idx = 5; break; 54 | default: throw format_error("Invalid unit"); 55 | } 56 | ++it; 57 | 58 | if (it != ctx.end() && *it == 'B') 59 | ++it; 60 | if (it != ctx.end() && *it != '}') 61 | throw format_error("Invalid format"); 62 | 63 | return it; 64 | } 65 | 66 | template 67 | inline auto format(ucsb::printable_bytes_t const& v, ctx_at& ctx) { 68 | 69 | char const* suffix_k[] = {"", "B", "KB", "MB", "GB", "TB"}; 70 | 71 | size_t bytes = v.bytes; 72 | float float_bytes = bytes; 73 | char const length = sizeof(suffix_k) / sizeof(suffix_k[0]); 74 | if (suffix_idx == 0) { 75 | ++suffix_idx; 76 | while (bytes > 1024 && suffix_idx < length - 1) { 77 | ++suffix_idx; 78 | float_bytes = bytes / 1024.0; 79 | bytes /= 1024; 80 | } 81 | } 82 | else 83 | float_bytes /= std::pow(1024, suffix_idx - 1); 84 | 85 | if (suffix_idx == 1) 86 | return fmt::format_to(ctx.out(), "{}B", bytes); 87 | else { 88 | std::string format = fmt::format("{{:.{}f}}{{}}", precision); 89 | return fmt::format_to(ctx.out(), fmt::runtime(format), float_bytes, suffix_k[suffix_idx]); 90 | } 91 | } 92 | }; 93 | 94 | template <> 95 | class fmt::formatter { 96 | public: 97 | size_t suffix_idx = 0; 98 | unsigned char precision = 2; 99 | 100 | template 101 | constexpr auto parse(ctx_at& ctx) { 102 | auto it = ctx.begin(); 103 | if (it == ctx.end()) 104 | return it; 105 | 106 | if (*it == '.') { 107 | ++it; 108 | if (it != ctx.end() && *it >= '0' && *it <= '9') 109 | precision = (int)(*it) - (int)48; 110 | else 111 | throw format_error("Invalid precision"); 112 | ++it; 113 | } 114 | 115 | if (it == ctx.end()) 116 | return it; 117 | 118 | switch (*it) { 119 | case 'k': suffix_idx = 1; break; 120 | case 'M': suffix_idx = 2; break; 121 | case 'B': suffix_idx = 3; break; 122 | case 'T': suffix_idx = 4; break; 123 | default: throw format_error("Invalid unit"); 124 | } 125 | it++; 126 | 127 | if (it != ctx.end() && *it != '}') 128 | throw format_error("Invalid format"); 129 | 130 | return it; 131 | } 132 | 133 | template 134 | auto format(ucsb::printable_float_t const& v, ctx_at& ctx) { 135 | 136 | char const* suffix_k[] = {"", "k", "M", "B", "T"}; 137 | 138 | double value = v.value; 139 | char const length = sizeof(suffix_k) / sizeof(suffix_k[0]); 140 | if (suffix_idx == 0) { 141 | while (value > 1'000.0 && suffix_idx < length - 1) { 142 | ++suffix_idx; 143 | value /= 1'000.0; 144 | } 145 | } 146 | else 147 | value /= std::pow(1'000, suffix_idx); 148 | 149 | std::string format; 150 | if (suffix_idx == 0) 151 | format = fmt::format("{{:.{}f}}", precision); 152 | else 153 | format = fmt::format("{{:.{}f}}{{}}", precision); 154 | 155 | return fmt::format_to(ctx.out(), fmt::runtime(format), value, suffix_k[suffix_idx]); 156 | } 157 | }; 158 | 159 | template <> 160 | class fmt::formatter { 161 | public: 162 | template 163 | constexpr auto parse(ctx_at& ctx) { 164 | return ctx.begin(); 165 | } 166 | 167 | template 168 | auto format(ucsb::printable_duration_t const& v, ctx_at& ctx) { 169 | 170 | // Resolve human readable duration 171 | double duration = v.duration; 172 | size_t major = 0; 173 | size_t minor = 0; 174 | std::string major_unit; 175 | std::string minor_unit; 176 | auto resolve = [&](double& duration, size_t period, std::string const& maj_unit, std::string const& min_unit) { 177 | if (duration > period) { 178 | duration /= period; 179 | return false; 180 | } 181 | major = duration; 182 | minor = (duration - major) * period; 183 | major_unit = maj_unit; 184 | minor_unit = min_unit; 185 | return true; 186 | }; 187 | // 188 | do { 189 | if (resolve(duration, 1'000, "ms", "")) 190 | break; 191 | if (resolve(duration, 60, "s", "ms")) 192 | break; 193 | if (resolve(duration, 60, "m", "s")) 194 | break; 195 | if (resolve(duration, 24, "h", "m")) 196 | break; 197 | resolve(duration, std::numeric_limits::max(), "d", "h"); 198 | } while (false); 199 | 200 | // Format 201 | std::string str_duration = fmt::format("{}{}", major, major_unit); 202 | if (!minor_unit.empty()) 203 | str_duration = fmt::format("{} {}{}", str_duration, minor, minor_unit); 204 | 205 | return fmt::format_to(ctx.out(), "{}", str_duration); 206 | } 207 | }; -------------------------------------------------------------------------------- /src/core/profiler.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace ucsb { 13 | 14 | /** 15 | * @brief Manages a sibling thread, that samples CPU time and real time from OS. 16 | * Uses similar methodology to Python package `psutil`, to estimate CPU load 17 | * from the aforementioned timers. 18 | * 19 | * @see psutil: https://pypi.org/project/psutil/ 20 | */ 21 | class cpu_profiler_t { 22 | public: 23 | inline cpu_profiler_t(size_t request_delay = 100) 24 | : time_to_die_(true), request_delay_(request_delay), requests_count_(0) {} 25 | ~cpu_profiler_t() { stop(); } 26 | 27 | struct stats_t { 28 | float min = std::numeric_limits::max(); 29 | float max = 0; 30 | float avg = 0; 31 | }; 32 | 33 | inline void start() { 34 | if (!time_to_die_.load()) 35 | return; 36 | 37 | stats_.min = std::numeric_limits::max(); 38 | stats_.max = 0; 39 | stats_.avg = 0; 40 | 41 | requests_count_ = 0; 42 | time_to_die_.store(false); 43 | thread_ = std::thread(&cpu_profiler_t::request_cpu_usage, this); 44 | } 45 | inline void stop() { 46 | if (time_to_die_.load()) 47 | return; 48 | 49 | // Wait to calculate one more times for to get more accuracy 50 | std::this_thread::sleep_for(std::chrono::milliseconds(request_delay_ + 1)); 51 | 52 | time_to_die_.store(true); 53 | thread_.join(); 54 | } 55 | 56 | inline stats_t percent() const { return stats_; } 57 | 58 | private: 59 | inline void recalculate(float percent) { 60 | stats_.min = std::min(percent, stats_.min); 61 | stats_.max = std::max(percent, stats_.max); 62 | stats_.avg = (stats_.avg * (requests_count_ - 1) + percent) / requests_count_; 63 | } 64 | 65 | void request_cpu_usage() { 66 | bool first_time = true; 67 | clock_t last_cpu = 0; 68 | clock_t last_proc_user = 0; 69 | clock_t last_proc_sys = 0; 70 | 71 | while (!time_to_die_.load(std::memory_order_relaxed)) { 72 | tms time_sample; 73 | clock_t cpu = times(&time_sample); 74 | clock_t proc_user = time_sample.tms_utime; 75 | clock_t proc_sys = time_sample.tms_stime; 76 | clock_t delta_proc = (proc_user - last_proc_user) + (proc_sys - last_proc_sys); 77 | clock_t delta_cpu = cpu - last_cpu; 78 | 79 | if (!first_time && delta_cpu > 0) { 80 | float percent = 100.0 * delta_proc / delta_cpu; 81 | ++requests_count_; 82 | recalculate(percent); 83 | } 84 | else 85 | first_time = false; 86 | 87 | last_cpu = cpu; 88 | last_proc_user = proc_user; 89 | last_proc_sys = proc_sys; 90 | 91 | std::this_thread::sleep_for(std::chrono::milliseconds(request_delay_)); 92 | } 93 | } 94 | 95 | std::thread thread_; 96 | std::atomic_bool time_to_die_; 97 | 98 | stats_t stats_; 99 | size_t request_delay_; 100 | size_t requests_count_; 101 | }; 102 | 103 | /** 104 | * @brief Manages a sibling thread, that sample the virtual "/proc/self/stat" file 105 | * to estimate memory usage stats of the current process, similar to Valgrind. 106 | * Collects both Resident Set Size and Virtual Memory Size. 107 | * ! To avoid reimplementing STL for a purposes of benchmark, this uses `std::ifstream` 108 | * ! and does numerous heap allocations. Not a recommended practice :) 109 | * 110 | * @see valgrind: https://valgrind.org/ 111 | */ 112 | class mem_profiler_t { 113 | public: 114 | inline mem_profiler_t(size_t request_delay = 100) 115 | : time_to_die_(true), request_delay_(request_delay), requests_count_(0), page_size_(sysconf(_SC_PAGE_SIZE)) {} 116 | ~mem_profiler_t() { stop(); } 117 | 118 | struct stats_t { 119 | size_t min = std::numeric_limits::max(); 120 | size_t max = 0; 121 | size_t avg = 0; 122 | }; 123 | 124 | inline void start() { 125 | if (!time_to_die_.load()) 126 | return; 127 | 128 | stats_vms_.min = std::numeric_limits::max(); 129 | stats_vms_.max = 0; 130 | stats_vms_.avg = 0; 131 | 132 | stats_rss_.min = std::numeric_limits::max(); 133 | stats_rss_.max = 0; 134 | stats_rss_.avg = 0; 135 | 136 | requests_count_ = 0; 137 | time_to_die_.store(false); 138 | thread_ = std::thread(&mem_profiler_t::request_mem_usage, this); 139 | } 140 | inline void stop() { 141 | if (time_to_die_.load()) 142 | return; 143 | 144 | time_to_die_.store(true); 145 | thread_.join(); 146 | } 147 | 148 | inline stats_t vm() const { return stats_vms_; } 149 | inline stats_t rss() const { return stats_rss_; } 150 | 151 | private: 152 | inline void recalculate(size_t vm, size_t rss) { 153 | stats_vms_.min = std::min(vm, stats_vms_.min); 154 | stats_vms_.max = std::max(vm, stats_vms_.max); 155 | stats_vms_.avg = (stats_vms_.avg * (requests_count_ - 1) + vm) / requests_count_; 156 | 157 | stats_rss_.min = std::min(rss, stats_rss_.min); 158 | stats_rss_.max = std::max(rss, stats_rss_.max); 159 | stats_rss_.avg = (stats_rss_.avg * (requests_count_ - 1) + rss) / requests_count_; 160 | } 161 | 162 | inline void request_mem_usage() { 163 | while (!time_to_die_.load(std::memory_order_relaxed)) { 164 | size_t vm = 0, rss = 0; 165 | mem_usage(vm, rss); 166 | ++requests_count_; 167 | recalculate(vm, rss); 168 | std::this_thread::sleep_for(std::chrono::milliseconds(request_delay_)); 169 | } 170 | } 171 | 172 | inline void mem_usage(size_t& vm, size_t& rss) { 173 | vm = 0; 174 | rss = 0; 175 | 176 | std::ifstream stat("/proc/self/stat", std::ios_base::in); 177 | std::string pid, comm, state, ppid, pgrp, session, tty_nr; 178 | std::string tpgid, flags, minflt, cminflt, majflt, cmajflt; 179 | std::string utime, stime, cutime, cstime, priority, nice; 180 | std::string num_threads, itrealvalue, starttime; 181 | stat >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr >> tpgid >> flags >> minflt >> cminflt >> 182 | majflt >> cmajflt >> utime >> stime >> cutime >> cstime >> priority >> nice >> num_threads >> itrealvalue >> 183 | starttime >> vm >> rss; 184 | stat.close(); 185 | 186 | rss = rss * page_size_; 187 | } 188 | 189 | std::thread thread_; 190 | std::atomic_bool time_to_die_; 191 | 192 | stats_t stats_vms_; 193 | stats_t stats_rss_; 194 | 195 | size_t request_delay_; 196 | size_t requests_count_; 197 | size_t page_size_; 198 | }; 199 | 200 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/reporter.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | #include "src/core/types.hpp" 13 | #include "src/core/printable.hpp" 14 | 15 | using ordered_json = nlohmann::ordered_json; 16 | 17 | namespace ucsb { 18 | 19 | namespace bm = benchmark; 20 | namespace fs = ucsb::fs; 21 | 22 | class console_reporter_t : public bm::BenchmarkReporter { 23 | 24 | using base_t = bm::BenchmarkReporter; 25 | 26 | public: 27 | enum sections_t { 28 | header_k = 0x01, 29 | result_k = 0x02, 30 | logo_k = 0x04, 31 | 32 | all_k = header_k | result_k | logo_k, 33 | }; 34 | 35 | public: 36 | inline console_reporter_t(std::string const& title, sections_t sections); 37 | 38 | public: 39 | // Prints environment information 40 | bool ReportContext(Context const&) override; 41 | // Prints results 42 | void ReportRuns(std::vector const& reports) override; 43 | // Prints logo 44 | void Finalize() override; 45 | 46 | private: 47 | double convert_duration(double duration, bm::TimeUnit from, bm::TimeUnit to); 48 | 49 | private: 50 | std::string title_; 51 | sections_t sections_; 52 | bool has_header_printed_; 53 | 54 | tabulate::Table::Row_t columns_; 55 | size_t fails_column_idx_; 56 | size_t column_width_; 57 | size_t workload_column_width_; 58 | size_t columns_total_width_; 59 | }; 60 | 61 | inline console_reporter_t::console_reporter_t(std::string const& title, sections_t sections) 62 | : base_t(), title_(title), sections_(sections), has_header_printed_(true), fails_column_idx_(0), column_width_(0), 63 | workload_column_width_(0), columns_total_width_(0) { 64 | 65 | columns_ = { 66 | "Workload", 67 | "Throughput", 68 | "Data Processed", 69 | "Disk Usage", 70 | "Memory (avg)", 71 | "Memory (max)", 72 | "CPU (avg,%)", 73 | "CPU (max,%)", 74 | "Fails (%)", 75 | "Duration", 76 | }; 77 | 78 | fails_column_idx_ = 8; 79 | 80 | column_width_ = 13; 81 | workload_column_width_ = 18; 82 | columns_total_width_ = workload_column_width_ + (columns_.size() - 1) * column_width_ + columns_.size() - 1; 83 | } 84 | 85 | bool console_reporter_t::ReportContext(Context const&) { 86 | 87 | if (sections_ & sections_t::header_k) { 88 | tabulate::Table table; 89 | table.add_row({title_}); 90 | table.column(0) 91 | .format() 92 | .width(columns_total_width_) 93 | .font_align(tabulate::FontAlign::center) 94 | .font_color(tabulate::Color::blue) 95 | .locale("C"); 96 | std::cout << table << std::endl; 97 | } 98 | 99 | return true; 100 | } 101 | 102 | void console_reporter_t::ReportRuns(std::vector const& reports) { 103 | 104 | // Print header 105 | if ((sections_ & sections_t::header_k) && has_header_printed_) { 106 | has_header_printed_ = false; 107 | tabulate::Table table; 108 | table.add_row({columns_}); 109 | table.row(0) 110 | .format() 111 | .width(column_width_) 112 | .font_align(tabulate::FontAlign::center) 113 | .font_color(tabulate::Color::blue) 114 | .hide_border_top() 115 | .locale("C"); 116 | table.column(0).format().width(workload_column_width_); 117 | std::cout << table << std::endl; 118 | } 119 | 120 | if (sections_ & sections_t::result_k) { 121 | if (reports.size() != 1) { 122 | fmt::print("Each benchmark should be in separate group"); 123 | return; 124 | } 125 | auto const& report = reports.front(); 126 | 127 | // Counters 128 | double throughput = report.counters.at("operations/s").value; 129 | // 130 | size_t data_processed = report.counters.at("processed,bytes").value; 131 | size_t disk_usage = report.counters.at("disk,bytes").value; 132 | // 133 | size_t mem_avg = report.counters.at("mem_avg(rss),bytes").value; 134 | size_t mem_max = report.counters.at("mem_max(rss),bytes").value; 135 | double cpu_avg = report.counters.at("cpu_avg,%").value; 136 | double cpu_max = report.counters.at("cpu_max,%").value; 137 | // 138 | double fails = report.counters.at("fails,%").value; 139 | double duration = 140 | convert_duration(report.real_accumulated_time, bm::TimeUnit::kSecond, bm::TimeUnit::kMillisecond); 141 | 142 | // Build table 143 | tabulate::Table table; 144 | table.add_row({report.run_name.function_name, 145 | fmt::format("{}/s", printable_float_t {throughput}), 146 | fmt::format("{}", printable_bytes_t {data_processed}), 147 | fmt::format("{}", printable_bytes_t {disk_usage}), 148 | fmt::format("{}", printable_bytes_t {mem_avg}), 149 | fmt::format("{}", printable_bytes_t {mem_max}), 150 | fmt::format("{:.1f}", cpu_avg), 151 | fmt::format("{:.1f}", cpu_max), 152 | fmt::format("{:g}", fails), 153 | fmt::format("{}", printable_duration_t {size_t(duration)})}); 154 | table.row(0).format().width(column_width_).font_align(tabulate::FontAlign::right).hide_border_top().locale("C"); 155 | table.column(0) 156 | .format() 157 | .width(workload_column_width_) 158 | .font_align(tabulate::FontAlign::left) 159 | .font_color(tabulate::Color::green); 160 | 161 | // Highlight cells 162 | if (fails > 0.0) 163 | table[0][fails_column_idx_].format().font_color(tabulate::Color::red); 164 | 165 | // Print 166 | std::cout << table << std::endl; 167 | } 168 | } 169 | 170 | void console_reporter_t::Finalize() { 171 | 172 | if (sections_ & sections_t::logo_k) { 173 | tabulate::Table table; 174 | table.add_row({"C 2015-2023 UCSB, Unum Cloud"}); 175 | table.row(0) 176 | .format() 177 | .width(columns_total_width_) 178 | .font_align(tabulate::FontAlign::center) 179 | .font_color(tabulate::Color::blue) 180 | .hide_border_top() 181 | .locale("C"); 182 | std::cout << table << std::endl; 183 | } 184 | } 185 | 186 | double console_reporter_t::convert_duration(double duration, bm::TimeUnit from, bm::TimeUnit to) { 187 | // First convert to nanoseconds 188 | switch (from) { 189 | case bm::TimeUnit::kSecond: duration *= 1'000; [[fallthrough]]; 190 | case bm::TimeUnit::kMillisecond: duration *= 1'000; [[fallthrough]]; 191 | case bm::TimeUnit::kMicrosecond: duration *= 1'000; [[fallthrough]]; 192 | case bm::TimeUnit::kNanosecond: [[fallthrough]]; 193 | default: break; 194 | } 195 | 196 | // Convert to specified 197 | switch (to) { 198 | case bm::TimeUnit::kSecond: duration /= 1'000; [[fallthrough]]; 199 | case bm::TimeUnit::kMillisecond: duration /= 1'000; [[fallthrough]]; 200 | case bm::TimeUnit::kMicrosecond: duration /= 1'000; [[fallthrough]]; 201 | case bm::TimeUnit::kNanosecond: [[fallthrough]]; 202 | default: break; 203 | } 204 | 205 | return duration; 206 | } 207 | 208 | class file_reporter_t { 209 | public: 210 | static void merge_results(fs::path const& source_file_path, fs::path const& destination_file_path); 211 | 212 | private: 213 | static std::string parse_workload_name(std::string const& benchmark_name); 214 | }; 215 | 216 | std::string file_reporter_t::parse_workload_name(std::string const& benchmark_name) { 217 | std::string name; 218 | size_t pos = benchmark_name.find('/'); 219 | if (pos != std::string::npos) 220 | name = benchmark_name.substr(0, pos); 221 | else 222 | name = benchmark_name; 223 | return name; 224 | } 225 | 226 | void file_reporter_t::merge_results(fs::path const& source_file_path, fs::path const& destination_file_path) { 227 | 228 | if (!fs::exists(source_file_path)) 229 | return; 230 | 231 | std::ifstream ifstream(source_file_path); 232 | ordered_json j_source; 233 | ifstream >> j_source; 234 | 235 | ordered_json j_destination; 236 | if (fs::exists(destination_file_path)) { 237 | ifstream = std::ifstream(destination_file_path); 238 | ifstream >> j_destination; 239 | } 240 | 241 | if (!j_destination.empty()) { 242 | auto j_source_benchmarks = j_source["benchmarks"]; 243 | auto j_destination_benchmarks = j_destination["benchmarks"]; 244 | std::vector results; 245 | // Take olds 246 | for (auto it = j_destination_benchmarks.begin(); it != j_destination_benchmarks.end(); ++it) 247 | results.push_back(*it); 248 | // Update with new 249 | for (auto it = j_source_benchmarks.begin(); it != j_source_benchmarks.end(); ++it) { 250 | auto src_name = (*it)["name"].get(); 251 | src_name = parse_workload_name(src_name); 252 | 253 | size_t idx = 0; 254 | for (; idx < results.size(); ++idx) { 255 | auto dest_name = results[idx]["name"].get(); 256 | dest_name = parse_workload_name(dest_name); 257 | if (src_name == dest_name) { 258 | results[idx] = *it; 259 | break; 260 | } 261 | } 262 | if (idx == results.size()) 263 | results.push_back(*it); 264 | } 265 | j_destination["benchmarks"] = results; 266 | } 267 | else 268 | j_destination = j_source; 269 | 270 | std::ofstream ofstream(destination_file_path); 271 | ofstream << std::setw(2) << j_destination << std::endl; 272 | } 273 | 274 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/settings.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "src/core/types.hpp" 6 | 7 | namespace ucsb { 8 | 9 | struct settings_t { 10 | std::string db_name; 11 | bool transactional = false; 12 | fs::path db_config_file_path; 13 | fs::path db_main_dir_path; 14 | std::vector db_storage_dir_paths; 15 | 16 | fs::path workloads_file_path; 17 | std::string workload_filter; 18 | size_t threads_count = 0; 19 | 20 | fs::path results_file_path; 21 | size_t run_idx = 0; 22 | size_t runs_count = 0; 23 | }; 24 | 25 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/threads_fence.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace ucsb { 8 | 9 | /** 10 | * @brief Synchronization primitive to isolate workers across 11 | * threads from operation on uninitialized or closing DB. 12 | */ 13 | class threads_fence_t { 14 | public: 15 | inline threads_fence_t(size_t threads_count) 16 | : threads_count_(threads_count), waiting_threads_count_(0), released_threads_count_(0) {} 17 | 18 | inline void sync() { 19 | while (released_threads_count_.load()) 20 | ; 21 | 22 | ++waiting_threads_count_; 23 | while (waiting_threads_count_.load() != threads_count_) 24 | ; 25 | 26 | ++released_threads_count_; 27 | if (released_threads_count_.load() == threads_count_) { 28 | size_t tmp_waiting = threads_count_; 29 | size_t tmp_released = threads_count_; 30 | waiting_threads_count_.compare_exchange_weak(tmp_waiting, size_t(0)); 31 | released_threads_count_.compare_exchange_weak(tmp_released, size_t(0)); 32 | } 33 | } 34 | 35 | private: 36 | size_t const threads_count_; 37 | std::atomic_size_t waiting_threads_count_; 38 | std::atomic_size_t released_threads_count_; 39 | }; 40 | 41 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/timer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace bm = benchmark; 7 | 8 | namespace ucsb { 9 | 10 | using high_resolution_clock_t = std::chrono::high_resolution_clock; 11 | using time_point_t = std::chrono::time_point; 12 | using elapsed_time_t = std::chrono::nanoseconds; 13 | 14 | /** 15 | * @brief Trivial Google Benchmark wrapper. 16 | * No added value here :) 17 | */ 18 | class timer_t { 19 | public: 20 | enum class state_t { 21 | stopped_k, 22 | running_k, 23 | paused_k, 24 | }; 25 | 26 | inline timer_t(bm::State& bench) : bench_(&bench), state_(state_t::stopped_k) {} 27 | 28 | // Google benchmark timer methods 29 | inline void pause() { 30 | bench_->PauseTiming(); 31 | 32 | assert(state_ == state_t::running_k); 33 | recalculate_operations_elapsed_time(); 34 | state_ = state_t::paused_k; 35 | } 36 | inline void resume() { 37 | assert(state_ == state_t::paused_k); 38 | operations_start_time_ = high_resolution_clock_t::now(); 39 | state_ = state_t::running_k; 40 | 41 | bench_->ResumeTiming(); 42 | } 43 | 44 | // Helper methods to calculate real time statistics 45 | inline void start() { 46 | assert(state_ == state_t::stopped_k); 47 | elapsed_time_ = elapsed_time_t(0); 48 | operations_elapsed_time_ = elapsed_time_t(0); 49 | auto now = high_resolution_clock_t::now(); 50 | start_time_ = now; 51 | operations_start_time_ = now; 52 | state_ = state_t::running_k; 53 | } 54 | inline void stop() { 55 | assert(state_ == state_t::running_k); 56 | recalculate_operations_elapsed_time(); 57 | state_ = state_t::stopped_k; 58 | } 59 | inline auto operations_elapsed_time() { 60 | if (state_ == state_t::running_k) 61 | recalculate_operations_elapsed_time(); 62 | return operations_elapsed_time_; 63 | } 64 | inline auto elapsed_time() { 65 | if (state_ != state_t::stopped_k) 66 | recalculate_elapsed_time(); 67 | return elapsed_time_; 68 | } 69 | 70 | private: 71 | void recalculate_operations_elapsed_time() { 72 | auto now = high_resolution_clock_t::now(); 73 | operations_elapsed_time_ += std::chrono::duration_cast(now - operations_start_time_); 74 | operations_start_time_ = now; 75 | } 76 | void recalculate_elapsed_time() { 77 | auto now = high_resolution_clock_t::now(); 78 | elapsed_time_ += std::chrono::duration_cast(now - start_time_); 79 | start_time_ = now; 80 | } 81 | 82 | // Bench state 83 | bm::State* bench_; 84 | 85 | // Note: These are not needed if we were able to get elapsed times from bm::State 86 | state_t state_; 87 | // 88 | time_point_t start_time_; 89 | elapsed_time_t elapsed_time_; 90 | // 91 | time_point_t operations_start_time_; 92 | elapsed_time_t operations_elapsed_time_; 93 | }; 94 | 95 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/types.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "src/core/aligned_buffer.hpp" 11 | 12 | namespace ucsb { 13 | 14 | namespace fs = std::filesystem; 15 | 16 | using key_t = size_t; 17 | using keys_t = std::vector; 18 | using keys_span_t = std::span; 19 | using keys_spanc_t = std::span; 20 | using value_length_t = uint32_t; 21 | using value_t = std::vector; 22 | using values_t = std::vector; 23 | using value_span_t = std::span; 24 | using value_spanc_t = std::span; 25 | using values_buffer_t = aligned_buffer_t; 26 | using values_span_t = std::span; 27 | using values_spanc_t = std::span; 28 | using value_lengths_t = std::vector; 29 | using value_lengths_span_t = std::span; 30 | using value_lengths_spanc_t = std::span; 31 | 32 | } // namespace ucsb -------------------------------------------------------------------------------- /src/core/workload.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "src/core/types.hpp" 11 | #include "src/core/distribution.hpp" 12 | 13 | using json = nlohmann::json; 14 | 15 | namespace ucsb { 16 | 17 | /** 18 | * @brief A description of a single benchmark. 19 | * It's post-processed version will divide the task 20 | * by the number of threads. 21 | */ 22 | struct workload_t { 23 | std::string name; 24 | 25 | /** 26 | * @brief Qualitative reference number of entries in the DB. 27 | * Doesn't change during the benchmark. 28 | * Defines the number of entries after initialization, 29 | * but it's outdated after insertions and deletions. 30 | */ 31 | size_t db_records_count = 0; 32 | /** 33 | * @brief Number of entries to be changed/inserted/deleted 34 | * for this specific workload, divided by the number of threads. 35 | */ 36 | size_t records_count = 0; 37 | /** 38 | * @brief Number of operations which will be done by all threads 39 | * Loads from workload file, doesn't change during the benchmark. 40 | */ 41 | size_t db_operations_count = 0; 42 | /** 43 | * @brief Number of operations for this specific workload, 44 | * which will be done by a single thread, divided by the number of threads. 45 | */ 46 | size_t operations_count = 0; 47 | 48 | float upsert_proportion = 0; 49 | float update_proportion = 0; 50 | float remove_proportion = 0; 51 | float read_proportion = 0; 52 | float read_modify_write_proportion = 0; 53 | float batch_upsert_proportion = 0; 54 | float batch_read_proportion = 0; 55 | float bulk_load_proportion = 0; 56 | float range_select_proportion = 0; 57 | float scan_proportion = 0; 58 | 59 | key_t start_key = 0; 60 | distribution_kind_t key_dist = distribution_kind_t::uniform_k; 61 | 62 | value_length_t value_length = 0; 63 | distribution_kind_t value_length_dist = distribution_kind_t::const_k; 64 | 65 | size_t batch_upsert_min_length = 0; 66 | size_t batch_upsert_max_length = 0; 67 | distribution_kind_t batch_upsert_length_dist = distribution_kind_t::uniform_k; 68 | 69 | size_t batch_read_min_length = 0; 70 | size_t batch_read_max_length = 0; 71 | distribution_kind_t batch_read_length_dist = distribution_kind_t::uniform_k; 72 | 73 | size_t bulk_load_min_length = 0; 74 | size_t bulk_load_max_length = 0; 75 | distribution_kind_t bulk_load_length_dist = distribution_kind_t::uniform_k; 76 | 77 | size_t range_select_min_length = 0; 78 | size_t range_select_max_length = 0; 79 | distribution_kind_t range_select_length_dist = distribution_kind_t::uniform_k; 80 | }; 81 | 82 | using workloads_t = std::vector; 83 | 84 | inline distribution_kind_t parse_distribution(std::string const& name) { 85 | distribution_kind_t dist = distribution_kind_t::unknown_k; 86 | if (name == "const") 87 | dist = distribution_kind_t::const_k; 88 | else if (name == "counter") 89 | dist = distribution_kind_t::counter_k; 90 | else if (name == "uniform") 91 | dist = distribution_kind_t::uniform_k; 92 | else if (name == "zipfian") 93 | dist = distribution_kind_t::zipfian_k; 94 | else if (name == "scrambled") 95 | dist = distribution_kind_t::scrambled_zipfian_k; 96 | else if (name == "latest") 97 | dist = distribution_kind_t::skewed_latest_k; 98 | else if (name == "acknowledged") 99 | dist = distribution_kind_t::acknowledged_counter_k; 100 | return dist; 101 | } 102 | 103 | bool load(fs::path const& path, workloads_t& workloads) { 104 | 105 | workloads.clear(); 106 | if (!fs::exists(path)) 107 | return false; 108 | 109 | std::ifstream ifstream(path); 110 | json j_workloads; 111 | ifstream >> j_workloads; 112 | 113 | for (auto j_workload = j_workloads.begin(); j_workload != j_workloads.end(); ++j_workload) { 114 | workload_t workload; 115 | 116 | workload.name = (*j_workload)["name"].get(); 117 | 118 | workload.db_records_count = (*j_workload)["records_count"].get(); 119 | workload.db_operations_count = (*j_workload)["operations_count"].get(); 120 | 121 | workload.upsert_proportion = (*j_workload).value("upsert_proportion", 0.0); 122 | workload.update_proportion = (*j_workload).value("update_proportion", 0.0); 123 | workload.remove_proportion = (*j_workload).value("remove_proportion", 0.0); 124 | workload.read_proportion = (*j_workload).value("read_proportion", 0.0); 125 | workload.read_modify_write_proportion = (*j_workload).value("read_modify_write_proportion", 0.0); 126 | workload.batch_upsert_proportion = (*j_workload).value("batch_upsert_proportion", 0.0); 127 | workload.batch_read_proportion = (*j_workload).value("batch_read_proportion", 0.0); 128 | workload.bulk_load_proportion = (*j_workload).value("bulk_load_proportion", 0.0); 129 | workload.range_select_proportion = (*j_workload).value("range_select_proportion", 0.0); 130 | workload.scan_proportion = (*j_workload).value("scan_proportion", 0.0); 131 | 132 | workload.start_key = (*j_workload).value("start_key", 0); 133 | workload.key_dist = parse_distribution((*j_workload).value("key_dist", "uniform")); 134 | if (workload.key_dist == distribution_kind_t::unknown_k) { 135 | workloads.clear(); 136 | return false; 137 | } 138 | 139 | workload.value_length = (*j_workload).value("value_length", 0); 140 | workload.value_length_dist = parse_distribution((*j_workload).value("value_length_dist", "const")); 141 | if (workload.value_length_dist == distribution_kind_t::unknown_k) { 142 | workloads.clear(); 143 | return false; 144 | } 145 | 146 | workload.batch_upsert_min_length = (*j_workload).value("batch_upsert_min_length", 0); 147 | workload.batch_upsert_max_length = (*j_workload).value("batch_upsert_max_length", 0); 148 | workload.batch_upsert_length_dist = 149 | parse_distribution((*j_workload).value("batch_upsert_length_dist", "uniform")); 150 | if (workload.batch_upsert_length_dist == distribution_kind_t::unknown_k) { 151 | workloads.clear(); 152 | return false; 153 | } 154 | 155 | workload.batch_read_min_length = (*j_workload).value("batch_read_min_length", 0); 156 | workload.batch_read_max_length = (*j_workload).value("batch_read_max_length", 0); 157 | workload.batch_read_length_dist = parse_distribution((*j_workload).value("batch_read_length_dist", "uniform")); 158 | if (workload.batch_read_length_dist == distribution_kind_t::unknown_k) { 159 | workloads.clear(); 160 | return false; 161 | } 162 | 163 | workload.bulk_load_min_length = (*j_workload).value("bulk_load_min_length", 0); 164 | workload.bulk_load_max_length = (*j_workload).value("bulk_load_max_length", 0); 165 | workload.bulk_load_length_dist = parse_distribution((*j_workload).value("bulk_load_length_dist", "uniform")); 166 | if (workload.bulk_load_length_dist == distribution_kind_t::unknown_k) { 167 | workloads.clear(); 168 | return false; 169 | } 170 | 171 | workload.range_select_min_length = (*j_workload).value("range_select_min_length", 0); 172 | workload.range_select_max_length = (*j_workload).value("range_select_max_length", 0); 173 | workload.range_select_length_dist = 174 | parse_distribution((*j_workload).value("range_select_length_dist", "uniform")); 175 | if (workload.key_dist == distribution_kind_t::unknown_k) { 176 | workloads.clear(); 177 | return false; 178 | } 179 | 180 | workloads.push_back(workload); 181 | } 182 | 183 | return true; 184 | } 185 | 186 | } // namespace ucsb -------------------------------------------------------------------------------- /src/leveldb/leveldb.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "src/core/types.hpp" 19 | #include "src/core/db.hpp" 20 | #include "src/core/helper.hpp" 21 | 22 | namespace ucsb::google { 23 | 24 | namespace fs = ucsb::fs; 25 | 26 | using key_t = ucsb::key_t; 27 | using keys_spanc_t = ucsb::keys_spanc_t; 28 | using value_span_t = ucsb::value_span_t; 29 | using value_spanc_t = ucsb::value_spanc_t; 30 | using values_span_t = ucsb::values_span_t; 31 | using values_spanc_t = ucsb::values_spanc_t; 32 | using value_lengths_spanc_t = ucsb::value_lengths_spanc_t; 33 | using operation_status_t = ucsb::operation_status_t; 34 | using operation_result_t = ucsb::operation_result_t; 35 | using db_hints_t = ucsb::db_hints_t; 36 | using transaction_t = ucsb::transaction_t; 37 | 38 | inline leveldb::Slice to_slice(key_t& key) { return {reinterpret_cast(&key), sizeof(key_t)}; } 39 | 40 | inline leveldb::Slice to_slice(value_spanc_t value) { 41 | return {reinterpret_cast(value.data()), value.size()}; 42 | } 43 | 44 | /** 45 | * @brief LevelDB wrapper for the UCSB benchmark. 46 | * It's the precursor of RocksDB by Facebook. 47 | * https://github.com/google/leveldb 48 | */ 49 | class leveldb_t : public ucsb::db_t { 50 | public: 51 | inline leveldb_t() : db_(nullptr) {} 52 | ~leveldb_t() { close(); } 53 | 54 | void set_config(fs::path const& config_path, 55 | fs::path const& main_dir_path, 56 | std::vector const& storage_dir_paths, 57 | db_hints_t const& hints) override; 58 | bool open(std::string& error) override; 59 | void close() override; 60 | 61 | std::string info() override; 62 | 63 | operation_result_t upsert(key_t key, value_spanc_t value) override; 64 | operation_result_t update(key_t key, value_spanc_t value) override; 65 | operation_result_t remove(key_t key) override; 66 | operation_result_t read(key_t key, value_span_t value) const override; 67 | 68 | operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 69 | operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const override; 70 | 71 | operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 72 | 73 | operation_result_t range_select(key_t key, size_t length, values_span_t values) const override; 74 | operation_result_t scan(key_t key, size_t length, value_span_t single_value) const override; 75 | 76 | void flush() override; 77 | 78 | size_t size_on_disk() const override; 79 | 80 | std::unique_ptr create_transaction() override; 81 | 82 | private: 83 | struct config_t { 84 | size_t write_buffer_size = 0; 85 | size_t max_file_size = 0; 86 | size_t max_open_files = -1; 87 | std::string compression; 88 | size_t cache_size = 0; 89 | size_t filter_bits = -1; 90 | }; 91 | 92 | inline bool load_config(config_t& config); 93 | 94 | class key_comparator_t final : public leveldb::Comparator { 95 | public: 96 | int Compare(leveldb::Slice const& left, leveldb::Slice const& right) const /*override*/ { 97 | assert(left.size() == sizeof(key_t)); 98 | assert(right.size() == sizeof(key_t)); 99 | 100 | key_t left_key = *reinterpret_cast(left.data()); 101 | key_t right_key = *reinterpret_cast(right.data()); 102 | return left_key < right_key ? -1 : left_key > right_key; 103 | } 104 | const char* Name() const { return "KeyComparator"; } 105 | void FindShortestSeparator(std::string*, const leveldb::Slice&) const {} 106 | void FindShortSuccessor(std::string*) const {} 107 | }; 108 | 109 | fs::path config_path_; 110 | fs::path main_dir_path_; 111 | std::vector storage_dir_paths_; 112 | 113 | leveldb::Options options_; 114 | leveldb::ReadOptions read_options_; 115 | leveldb::WriteOptions write_options_; 116 | 117 | std::unique_ptr db_; 118 | key_comparator_t key_cmp_; 119 | }; 120 | 121 | void leveldb_t::set_config(fs::path const& config_path, 122 | fs::path const& main_dir_path, 123 | std::vector const& storage_dir_paths, 124 | [[maybe_unused]] db_hints_t const& hints) { 125 | config_path_ = config_path; 126 | main_dir_path_ = main_dir_path; 127 | storage_dir_paths_ = storage_dir_paths; 128 | } 129 | 130 | bool leveldb_t::open(std::string& error) { 131 | if (db_) 132 | return true; 133 | 134 | if (!storage_dir_paths_.empty()) { 135 | error = "Doesn't support multiple disks"; 136 | return false; 137 | } 138 | 139 | config_t config; 140 | if (!load_config(config)) { 141 | error = "Failed to load config"; 142 | return false; 143 | } 144 | 145 | options_ = leveldb::Options(); 146 | options_.create_if_missing = true; 147 | // options_.comparator = &key_cmp_; 148 | if (config.write_buffer_size > 0) 149 | options_.write_buffer_size = config.write_buffer_size; 150 | if (config.max_file_size > 0) 151 | options_.max_file_size = config.max_file_size; 152 | if (config.max_open_files > 0) 153 | options_.max_open_files = config.max_open_files; 154 | if (config.compression == "snappy") 155 | options_.compression = leveldb::kSnappyCompression; 156 | else 157 | options_.compression = leveldb::kNoCompression; 158 | if (config.cache_size > 0) 159 | options_.block_cache = leveldb::NewLRUCache(config.cache_size); 160 | if (config.filter_bits > 0) 161 | options_.filter_policy = leveldb::NewBloomFilterPolicy(config.filter_bits); 162 | 163 | leveldb::DB* db_raw = nullptr; 164 | leveldb::Status status = leveldb::DB::Open(options_, main_dir_path_.string(), &db_raw); 165 | db_.reset(db_raw); 166 | 167 | error = status.ok() ? std::string() : status.ToString(); 168 | return status.ok(); 169 | } 170 | 171 | void leveldb_t::close() { db_.reset(nullptr); } 172 | 173 | operation_result_t leveldb_t::upsert(key_t key, value_spanc_t value) { 174 | leveldb::Status status = db_->Put(write_options_, to_slice(key), to_slice(value)); 175 | return {size_t(status.ok()), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 176 | } 177 | 178 | operation_result_t leveldb_t::update(key_t key, value_spanc_t value) { 179 | 180 | std::string data; 181 | leveldb::Status status = db_->Get(read_options_, to_slice(key), &data); 182 | if (status.IsNotFound()) 183 | return {0, operation_status_t::not_found_k}; 184 | else if (!status.ok()) 185 | return {0, operation_status_t::error_k}; 186 | 187 | status = db_->Put(write_options_, to_slice(key), to_slice(value)); 188 | return {size_t(status.ok()), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 189 | } 190 | 191 | operation_result_t leveldb_t::remove(key_t key) { 192 | leveldb::Status status = db_->Delete(write_options_, to_slice(key)); 193 | return {size_t(status.ok()), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 194 | } 195 | 196 | operation_result_t leveldb_t::read(key_t key, value_span_t value) const { 197 | 198 | // Unlike RocksDB, we can't read into some form fo a `PinnableSlice`, 199 | // just `std::string`, causing heap allocations. 200 | std::string data; 201 | leveldb::Status status = db_->Get(read_options_, to_slice(key), &data); 202 | if (status.IsNotFound()) 203 | return {0, operation_status_t::not_found_k}; 204 | else if (!status.ok()) 205 | return {0, operation_status_t::error_k}; 206 | 207 | memcpy(value.data(), data.data(), data.size()); 208 | return {1, operation_status_t::ok_k}; 209 | } 210 | 211 | operation_result_t leveldb_t::batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 212 | 213 | size_t offset = 0; 214 | leveldb::WriteBatch batch; 215 | for (size_t idx = 0; idx < keys.size(); ++idx) { 216 | key_t key = keys[idx]; 217 | batch.Put(to_slice(key), to_slice(values.subspan(offset, sizes[idx]))); 218 | offset += sizes[idx]; 219 | } 220 | 221 | leveldb::Status status = db_->Write(leveldb::WriteOptions(), &batch); 222 | return {keys.size(), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 223 | } 224 | 225 | operation_result_t leveldb_t::batch_read(keys_spanc_t keys, values_span_t values) const { 226 | 227 | // Note: imitation of batch read! 228 | size_t offset = 0; 229 | size_t found_cnt = 0; 230 | for (auto key : keys) { 231 | std::string data; 232 | leveldb::Status status = db_->Get(read_options_, to_slice(key), &data); 233 | if (status.ok()) { 234 | memcpy(values.data() + offset, data.data(), data.size()); 235 | offset += data.size(); 236 | ++found_cnt; 237 | } 238 | } 239 | return {found_cnt, operation_status_t::ok_k}; 240 | } 241 | 242 | operation_result_t leveldb_t::bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 243 | // Technically, LevelDB has a `TableBuilder` class, which is public, but once built - 244 | // can't be imported into the DB itself. 245 | // https://github.com/google/leveldb/blob/main/include/leveldb/table_builder.h 246 | // The most efficient alternative is to use `WriteBatch`, which comes with a very 247 | // scarce set of options. 248 | // https://github.com/google/leveldb/blob/main/include/leveldb/options.h 249 | return batch_upsert(keys, values, sizes); 250 | } 251 | 252 | operation_result_t leveldb_t::range_select(key_t key, size_t length, values_span_t values) const { 253 | 254 | size_t i = 0; 255 | size_t exported_bytes = 0; 256 | std::unique_ptr it(db_->NewIterator(read_options_)); 257 | it->Seek(to_slice(key)); 258 | for (; it->Valid() && i != length; i++, it->Next()) { 259 | memcpy(values.data() + exported_bytes, it->value().data(), it->value().size()); 260 | exported_bytes += it->value().size(); 261 | } 262 | return {i, operation_status_t::ok_k}; 263 | } 264 | 265 | operation_result_t leveldb_t::scan(key_t key, size_t length, value_span_t single_value) const { 266 | 267 | size_t i = 0; 268 | leveldb::ReadOptions scan_options = read_options_; 269 | scan_options.fill_cache = false; 270 | std::unique_ptr it(db_->NewIterator(scan_options)); 271 | it->Seek(to_slice(key)); 272 | for (; it->Valid() && i != length; i++, it->Next()) 273 | memcpy(single_value.data(), it->value().data(), it->value().size()); 274 | return {i, operation_status_t::ok_k}; 275 | } 276 | 277 | std::string leveldb_t::info() { return fmt::format("v{}.{}", leveldb::kMajorVersion, leveldb::kMinorVersion); } 278 | 279 | void leveldb_t::flush() { 280 | // Nothing to do 281 | } 282 | 283 | size_t leveldb_t::size_on_disk() const { return ucsb::size_on_disk(main_dir_path_); } 284 | 285 | std::unique_ptr leveldb_t::create_transaction() { return {}; } 286 | 287 | bool leveldb_t::load_config(config_t& config) { 288 | if (!fs::exists(config_path_)) 289 | return false; 290 | 291 | std::ifstream i_config(config_path_); 292 | nlohmann::json j_config; 293 | i_config >> j_config; 294 | 295 | config.write_buffer_size = j_config.value("write_buffer_size", size_t(67'108'864)); 296 | config.max_file_size = j_config.value("max_file_size", size_t(67'108'864)); 297 | config.max_open_files = j_config.value("max_open_files", size_t(1'000)); 298 | config.compression = j_config.value("compression", "none"); 299 | config.cache_size = j_config.value("cache_size", size_t(134'217'728)); 300 | config.filter_bits = j_config.value("filter_bits", size_t(10)); 301 | 302 | return true; 303 | } 304 | 305 | } // namespace ucsb::google -------------------------------------------------------------------------------- /src/mongodb/mongodb.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "src/core/types.hpp" 9 | #include "src/core/db.hpp" 10 | #include "src/core/helper.hpp" 11 | 12 | namespace ucsb::mongo { 13 | 14 | namespace fs = ucsb::fs; 15 | 16 | using key_t = ucsb::key_t; 17 | using keys_spanc_t = ucsb::keys_spanc_t; 18 | using value_t = ucsb::value_t; 19 | using value_span_t = ucsb::value_span_t; 20 | using value_spanc_t = ucsb::value_spanc_t; 21 | using values_span_t = ucsb::values_span_t; 22 | using values_spanc_t = ucsb::values_spanc_t; 23 | using value_length_t = ucsb::value_length_t; 24 | using value_lengths_spanc_t = ucsb::value_lengths_spanc_t; 25 | using operation_status_t = ucsb::operation_status_t; 26 | using operation_result_t = ucsb::operation_result_t; 27 | using db_hints_t = ucsb::db_hints_t; 28 | using transaction_t = ucsb::transaction_t; 29 | 30 | using bsoncxx::builder::basic::kvp; 31 | using bsoncxx::builder::basic::make_document; 32 | 33 | /** 34 | * @brief MongoDB wrapper for the UCSB benchmark. 35 | * https://github.com/mongodb/mongo-cxx-driver 36 | */ 37 | 38 | /* 39 | * @brief Preallocated buffers used for batch operations. 40 | * Globals and especially `thread_local`s are a bad practice. 41 | */ 42 | 43 | struct oid_hash_t { 44 | size_t operator()(const bsoncxx::oid& oid) const { 45 | return std::hash {}({oid.bytes(), oid.size()}); 46 | } 47 | }; 48 | thread_local std::unordered_map batch_keys_map; 49 | thread_local bsoncxx::builder::basic::array batch_keys_array; 50 | 51 | class mongodb_t : public ucsb::db_t { 52 | public: 53 | inline mongodb_t() : inst_(mongocxx::instance {}) {} 54 | 55 | void set_config(fs::path const& config_path, 56 | fs::path const& main_dir_path, 57 | std::vector const& storage_dir_paths, 58 | db_hints_t const& hints) override; 59 | bool open(std::string& error) override; 60 | void close() override; 61 | 62 | std::string info() override; 63 | 64 | operation_result_t upsert(key_t key, value_spanc_t value) override; 65 | operation_result_t update(key_t key, value_spanc_t value) override; 66 | operation_result_t remove(key_t key) override; 67 | operation_result_t read(key_t key, value_span_t value) const override; 68 | 69 | operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 70 | operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const override; 71 | 72 | operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 73 | 74 | operation_result_t range_select(key_t key, size_t length, values_span_t values) const override; 75 | operation_result_t scan(key_t key, size_t length, value_span_t single_value) const override; 76 | 77 | void flush() override; 78 | 79 | size_t size_on_disk() const override; 80 | 81 | std::unique_ptr create_transaction() override; 82 | 83 | private: 84 | fs::path config_path_; 85 | fs::path main_dir_path_; 86 | std::vector storage_dir_paths_; 87 | 88 | mongocxx::instance inst_; 89 | std::unique_ptr pool_; 90 | std::string coll_name; 91 | }; 92 | 93 | static bsoncxx::oid make_oid(key_t key) { 94 | size_t len_k = bsoncxx::oid::size(); 95 | char padded_key[len_k] = {0}; 96 | memcpy(padded_key + len_k - sizeof(key_t), &key, sizeof(key_t)); 97 | return bsoncxx::oid {&padded_key[0], len_k}; 98 | } 99 | 100 | static bsoncxx::types::b_binary make_binary(auto value, size_t size) { 101 | bsoncxx::types::b_binary bin_val; 102 | bin_val.sub_type = bsoncxx::binary_sub_type::k_binary; 103 | bin_val.bytes = reinterpret_cast(value); 104 | bin_val.size = size; 105 | return bin_val; 106 | } 107 | 108 | static void exec_cmd(const char* cmd) { 109 | using namespace std::chrono_literals; 110 | FILE* pipe = popen(cmd, "r"); 111 | if (!pipe) 112 | throw std::runtime_error("popen() failed!"); 113 | std::this_thread::sleep_for(2s); 114 | } 115 | 116 | void mongodb_t::set_config(fs::path const& config_path, 117 | fs::path const& main_dir_path, 118 | std::vector const& storage_dir_paths, 119 | [[maybe_unused]] db_hints_t const& hints) { 120 | config_path_ = config_path; 121 | main_dir_path_ = main_dir_path; 122 | storage_dir_paths_ = storage_dir_paths; 123 | coll_name = main_dir_path.parent_path().filename(); 124 | }; 125 | 126 | bool mongodb_t::open(std::string& error) { 127 | 128 | if (!storage_dir_paths_.empty()) { 129 | error = "Doesn't support multiple disks"; 130 | return false; 131 | } 132 | 133 | std::string start_cmd = "mongod --config "; 134 | start_cmd += config_path_; 135 | exec_cmd(start_cmd.c_str()); 136 | pool_ = std::make_unique(mongocxx::uri {"mongodb://127.0.0.1:27017/?minPoolSize=1&maxPoolSize=64"}); 137 | return true; 138 | } 139 | 140 | void mongodb_t::close() { 141 | batch_keys_array.clear(); 142 | batch_keys_map.clear(); 143 | std::string stop_cmd = "sudo mongod -f "; 144 | stop_cmd += config_path_; 145 | stop_cmd += " --shutdown"; 146 | exec_cmd(stop_cmd.c_str()); 147 | } 148 | 149 | operation_result_t mongodb_t::upsert(key_t key, value_spanc_t value) { 150 | auto client = (*pool_).acquire(); 151 | auto coll = (*client)["mongodb"][coll_name]; 152 | auto bin_val = make_binary(value.data(), value.size()); 153 | mongocxx::options::update opts; 154 | opts.upsert(true); 155 | if (coll.update_one(make_document(kvp("_id", make_oid(key))), 156 | make_document(kvp("$set", make_document(kvp("data", bin_val)))), 157 | opts) 158 | ->modified_count()) 159 | return {1, operation_status_t::ok_k}; 160 | return {0, operation_status_t::error_k}; 161 | } 162 | 163 | operation_result_t mongodb_t::update(key_t key, value_spanc_t value) { 164 | auto client = (*pool_).acquire(); 165 | auto coll = (*client)["mongodb"][coll_name]; 166 | // TODO: Do we need upsert here? 167 | mongocxx::options::update opts; 168 | opts.upsert(true); 169 | auto bin_val = make_binary(value.data(), value.size()); 170 | if (coll.update_one(make_document(kvp("_id", make_oid(key))), 171 | make_document(kvp("$set", make_document(kvp("data", bin_val)))), 172 | opts) 173 | ->modified_count()) 174 | return {1, operation_status_t::ok_k}; 175 | return {0, operation_status_t::error_k}; 176 | }; 177 | 178 | operation_result_t mongodb_t::remove(key_t key) { 179 | auto client = (*pool_).acquire(); 180 | auto coll = (*client)["mongodb"][coll_name]; 181 | if (coll.delete_one(make_document(kvp("_id", make_oid(key))))->deleted_count()) 182 | return {1, operation_status_t::ok_k}; 183 | return {0, operation_status_t::not_found_k}; 184 | }; 185 | 186 | operation_result_t mongodb_t::read(key_t key, value_span_t value) const { 187 | auto client = (*pool_).acquire(); 188 | auto coll = (*client)["mongodb"][coll_name]; 189 | bsoncxx::stdx::optional doc = coll.find_one(make_document(kvp("_id", make_oid(key)))); 190 | if (!doc) 191 | return {0, operation_status_t::not_found_k}; 192 | auto data = (*doc).view()["data"].get_binary(); 193 | memcpy(value.data(), data.bytes, data.size); 194 | return {1, operation_status_t::ok_k}; 195 | } 196 | 197 | operation_result_t mongodb_t::batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 198 | auto client = (*pool_).acquire(); 199 | auto coll = (*client)["mongodb"][coll_name]; 200 | auto bulk = mongocxx::bulk_write(coll.create_bulk_write()); 201 | size_t data_offset = 0; 202 | for (size_t index = 0; index < keys.size(); index++) { 203 | auto bin_val = make_binary(values.data() + data_offset, sizes[index]); 204 | bsoncxx::document::value doc1 = make_document(kvp("_id", make_oid(keys[index]))); 205 | bsoncxx::document::value doc2 = make_document(kvp("$set", make_document(kvp("data", bin_val)))); 206 | mongocxx::model::update_one upsert_op {doc1.view(), doc2.view()}; 207 | upsert_op.upsert(true); 208 | bulk.append(upsert_op); 209 | data_offset += sizes[index]; 210 | } 211 | size_t modified_count = bulk.execute()->modified_count(); 212 | if (modified_count == keys.size()) 213 | return {keys.size(), operation_status_t::ok_k}; 214 | return {0, operation_status_t::error_k}; 215 | } 216 | 217 | operation_result_t mongodb_t::batch_read(keys_spanc_t keys, values_span_t values) const { 218 | batch_keys_map.reserve(keys.size()); 219 | 220 | for (size_t index = 0; index < keys.size(); index++) { 221 | auto oid = make_oid(keys[index]); 222 | batch_keys_map.emplace(oid, index); 223 | batch_keys_array.append(oid); 224 | } 225 | 226 | size_t found_cnt = 0; 227 | 228 | auto client = (*pool_).acquire(); 229 | auto coll = (*client)["mongodb"][coll_name]; 230 | auto cursor = coll.find(make_document(kvp("_id", make_document(kvp("$in", batch_keys_array))))); 231 | 232 | for (auto&& doc : cursor) { 233 | found_cnt++; 234 | auto key = doc["_id"].get_oid().value; 235 | auto data = doc["data"].get_binary(); 236 | auto idx = batch_keys_map[key]; 237 | memcpy(&values[idx], data.bytes, data.size); 238 | } 239 | 240 | batch_keys_array.clear(); 241 | batch_keys_map.clear(); 242 | 243 | if (found_cnt == keys.size()) 244 | return {keys.size(), operation_status_t::ok_k}; 245 | 246 | return {0, operation_status_t::error_k}; 247 | } 248 | 249 | operation_result_t mongodb_t::bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 250 | auto client = (*pool_).acquire(); 251 | auto coll = (*client)["mongodb"][coll_name]; 252 | auto bulk = mongocxx::bulk_write(coll.create_bulk_write()); 253 | size_t data_offset = 0; 254 | for (size_t index = 0; index < keys.size(); index++) { 255 | auto bin_val = make_binary(values.data() + data_offset, sizes[index]); 256 | bsoncxx::document::value doc = make_document(kvp("_id", make_oid(keys[index])), kvp("data", bin_val)); 257 | mongocxx::model::insert_one insert_op {doc.view()}; 258 | bulk.append(insert_op); 259 | data_offset += sizes[index]; 260 | } 261 | 262 | size_t inserted_count = bulk.execute()->inserted_count(); 263 | if (inserted_count == keys.size()) 264 | return {keys.size(), operation_status_t::ok_k}; 265 | return {0, operation_status_t::error_k}; 266 | } 267 | 268 | operation_result_t mongodb_t::range_select(key_t key, size_t length, [[maybe_unused]] values_span_t values) const { 269 | size_t i = 0; 270 | auto client = (*pool_).acquire(); 271 | auto coll = (*client)["mongodb"][coll_name]; 272 | mongocxx::options::find opts; 273 | opts.limit(length); 274 | auto cursor = coll.find(make_document(kvp("_id", make_document(kvp("$gt", make_oid(key))))), opts); 275 | 276 | if (cursor.begin() == cursor.end()) 277 | return {0, operation_status_t::error_k}; 278 | 279 | for (auto&& doc : cursor) { 280 | (void)doc; 281 | i++; 282 | } 283 | 284 | return {i, operation_status_t::ok_k}; 285 | } 286 | 287 | operation_result_t mongodb_t::scan([[maybe_unused]] key_t key, size_t length, value_span_t single_value) const { 288 | auto client = (*pool_).acquire(); 289 | auto coll = (*client)["mongodb"][coll_name]; 290 | auto cursor = coll.find({}); 291 | size_t i = 0; 292 | for (auto doc = cursor.begin(); doc != cursor.end() && i++ < length; doc++) { 293 | auto data = (*doc)["data"].get_binary(); 294 | memcpy(single_value.data(), data.bytes, data.size); 295 | } 296 | return {i, operation_status_t::ok_k}; 297 | } 298 | 299 | std::string mongodb_t::info() { return {}; } 300 | 301 | void mongodb_t::flush() {} 302 | 303 | size_t mongodb_t::size_on_disk() const { return ucsb::size_on_disk(main_dir_path_); } 304 | 305 | std::unique_ptr mongodb_t::create_transaction() { return {}; } 306 | 307 | } // namespace ucsb::mongo -------------------------------------------------------------------------------- /src/redis/redis.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include "src/core/types.hpp" 10 | #include "src/core/db.hpp" 11 | #include "src/core/helper.hpp" 12 | 13 | namespace ucsb::redis { 14 | 15 | namespace fs = ucsb::fs; 16 | 17 | using key_t = ucsb::key_t; 18 | using value_length_t = ucsb::value_length_t; 19 | using keys_spanc_t = ucsb::keys_spanc_t; 20 | using value_span_t = ucsb::value_span_t; 21 | using value_spanc_t = ucsb::value_spanc_t; 22 | using values_span_t = ucsb::values_span_t; 23 | using values_spanc_t = ucsb::values_spanc_t; 24 | using value_lengths_spanc_t = ucsb::value_lengths_spanc_t; 25 | using operation_status_t = ucsb::operation_status_t; 26 | using operation_result_t = ucsb::operation_result_t; 27 | using db_hints_t = ucsb::db_hints_t; 28 | using transaction_t = ucsb::transaction_t; 29 | 30 | /** 31 | * @brief Redis wrapper for the UCSB benchmark. 32 | * Using redis-plus-plus client, based on hiredis. 33 | * https://github.com/sewenew/redis-plus-plus 34 | */ 35 | 36 | struct redis_t : public ucsb::db_t { 37 | public: 38 | ~redis_t() { 39 | std::string stop_cmd = "redis-cli -s "; 40 | stop_cmd += connection_options_.path; 41 | stop_cmd += " shutdown"; 42 | exec_cmd(stop_cmd.c_str()); 43 | } 44 | void set_config(fs::path const& config_path, 45 | fs::path const& main_dir_path, 46 | std::vector const& storage_dir_paths, 47 | db_hints_t const& hints) override; 48 | bool open(std::string& error) override; 49 | void close() override; 50 | 51 | std::string info() override; 52 | 53 | operation_result_t upsert(key_t key, value_spanc_t value) override; 54 | operation_result_t update(key_t key, value_spanc_t value) override; 55 | operation_result_t remove(key_t key) override; 56 | operation_result_t read(key_t key, value_span_t value) const override; 57 | 58 | operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 59 | operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const override; 60 | 61 | operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 62 | 63 | operation_result_t range_select(key_t key, size_t length, values_span_t values) const override; 64 | operation_result_t scan(key_t key, size_t length, value_span_t single_value) const override; 65 | 66 | void flush() override; 67 | 68 | size_t size_on_disk() const override; 69 | 70 | std::unique_ptr create_transaction() override; 71 | 72 | void get_options(fs::path const& path); 73 | std::string exec_cmd(const char* cmd); 74 | 75 | private: 76 | fs::path config_path_; 77 | fs::path main_dir_path_; 78 | std::vector storage_dir_paths_; 79 | 80 | std::unique_ptr redis_; 81 | sw::redis::ConnectionOptions connection_options_; 82 | sw::redis::ConnectionPoolOptions connection_pool_options_; 83 | bool is_opened_ = false; 84 | }; 85 | 86 | inline sw::redis::StringView to_string_view(std::byte const* p, size_t size_bytes) noexcept { 87 | return {reinterpret_cast(p), size_bytes}; 88 | } 89 | 90 | inline sw::redis::StringView to_string_view(key_t const& k) noexcept { 91 | return {reinterpret_cast(&k), sizeof(key_t)}; 92 | } 93 | 94 | std::string redis_t::exec_cmd(const char* cmd) { 95 | using namespace std::chrono_literals; 96 | std::array buffer; 97 | std::string result; 98 | std::unique_ptr pipe(popen(cmd, "r"), pclose); 99 | if (!pipe) 100 | throw std::runtime_error("popen() failed!"); 101 | 102 | while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) 103 | result += buffer.data(); 104 | 105 | std::this_thread::sleep_for(2s); 106 | return result; 107 | } 108 | 109 | void redis_t::get_options(fs::path const& path) { 110 | std::ifstream cfg_file(path); 111 | nlohmann::json j_config; 112 | cfg_file >> j_config; 113 | auto connection_type = j_config["connection_type"].get(); 114 | if (connection_type == "TCP") { 115 | connection_options_.host = j_config["host"].get(); 116 | connection_options_.port = j_config["port"].get(); 117 | } 118 | else if (connection_type == "UNIX") { 119 | connection_options_.type = sw::redis::ConnectionType::UNIX; 120 | connection_options_.path = j_config["path"].get(); 121 | } 122 | connection_pool_options_.wait_timeout = std::chrono::milliseconds(j_config["wait_timeout"].get()); 123 | connection_pool_options_.size = j_config["pool_size"].get(); 124 | } 125 | 126 | void redis_t::set_config(fs::path const& config_path, 127 | fs::path const& main_dir_path, 128 | std::vector const& storage_dir_paths, 129 | [[maybe_unused]] db_hints_t const& hints) { 130 | config_path_ = config_path; 131 | main_dir_path_ = main_dir_path; 132 | storage_dir_paths_ = storage_dir_paths; 133 | } 134 | 135 | bool redis_t::open(std::string& error) { 136 | if (is_opened_) 137 | return true; 138 | 139 | if (!storage_dir_paths_.empty()) { 140 | error = "Doesn't support multiple disks"; 141 | return false; 142 | } 143 | 144 | std::string start_cmd("redis-server "); 145 | start_cmd += config_path_; 146 | start_cmd += ".redis"; 147 | exec_cmd(start_cmd.c_str()); 148 | 149 | get_options(config_path_); 150 | redis_ = std::make_unique(sw::redis::Redis(connection_options_, connection_pool_options_)); 151 | is_opened_ = true; 152 | return true; 153 | } 154 | 155 | void redis_t::close() {} 156 | 157 | operation_result_t redis_t::upsert(key_t key, value_spanc_t value) { 158 | auto status = (*redis_).hset("hash", to_string_view(key), to_string_view(value.data(), value.size())); 159 | return {size_t(status), status ? operation_status_t::ok_k : operation_status_t::error_k}; 160 | } 161 | 162 | operation_result_t redis_t::update(key_t key, value_spanc_t value) { 163 | auto status = (*redis_).hset("hash", to_string_view(key), to_string_view(value.data(), value.size())); 164 | return {status, status ? operation_status_t::ok_k : operation_status_t::not_found_k}; 165 | } 166 | 167 | operation_result_t redis_t::remove(key_t key) { 168 | size_t count = (*redis_).hdel("hash", to_string_view(key)); 169 | return {count, count ? operation_status_t::ok_k : operation_status_t::not_found_k}; 170 | } 171 | 172 | operation_result_t redis_t::read(key_t key, value_span_t value) const { 173 | auto val = (*redis_).hget("hash", to_string_view(key)); 174 | if (!val) 175 | return {0, operation_status_t::not_found_k}; 176 | 177 | memcpy(value.data(), &val, sizeof(val)); 178 | return {1, operation_status_t::ok_k}; 179 | } 180 | 181 | operation_result_t redis_t::batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 182 | struct kv_iterator_t { 183 | using pair_t = std::pair; 184 | using val_t = typename values_spanc_t::element_type; 185 | key_t const* key_ptr_; 186 | val_t* val_ptr_; 187 | value_length_t const* size_ptr_; 188 | pair_t pair_; 189 | 190 | kv_iterator_t(key_t const* key_ptr, val_t const* val_ptr, value_length_t const* size_ptr) noexcept 191 | : key_ptr_(key_ptr), val_ptr_(val_ptr), size_ptr_(size_ptr), 192 | pair_(std::make_pair(to_string_view(*key_ptr_), to_string_view(val_ptr_, *size_ptr_))) {} 193 | 194 | pair_t const& operator*() const noexcept { return pair_; } 195 | pair_t const* operator->() const noexcept { return &pair_; } 196 | bool operator==(kv_iterator_t const& other) const noexcept { return key_ptr_ == other.key_ptr_; } 197 | 198 | kv_iterator_t& operator++() noexcept { 199 | key_ptr_++; 200 | val_ptr_ += *size_ptr_; 201 | size_ptr_++; 202 | pair_.first = to_string_view(*key_ptr_); 203 | pair_.second = to_string_view(val_ptr_, *size_ptr_); 204 | return *this; 205 | } 206 | }; 207 | 208 | (*redis_).hmset( 209 | "hash", 210 | kv_iterator_t(keys.data(), values.data(), sizes.data()), 211 | kv_iterator_t(keys.data() + keys.size(), values.data() + values.size(), sizes.data() + sizes.size())); 212 | return {keys.size(), operation_status_t::ok_k}; 213 | } 214 | 215 | operation_result_t redis_t::batch_read(keys_spanc_t keys, values_span_t values) const { 216 | struct key_iterator_t { 217 | key_t const* key_ptr_; 218 | 219 | key_iterator_t(key_t const* key_ptr) noexcept : key_ptr_(key_ptr) {} 220 | sw::redis::StringView operator*() const noexcept { return to_string_view(*key_ptr_); } 221 | bool operator==(key_iterator_t const& other) const noexcept { return key_ptr_ == other.key_ptr_; } 222 | 223 | key_iterator_t& operator++() noexcept { 224 | key_ptr_++; 225 | return *this; 226 | } 227 | }; 228 | 229 | struct value_getter_t { 230 | using value_type = sw::redis::Optional; 231 | using iterator = values_span_t::pointer; 232 | 233 | values_span_t values; 234 | size_t count = 0; 235 | size_t offset = 0; 236 | 237 | iterator push_back(value_type value) noexcept { 238 | if (!value) 239 | return values.data() + offset; 240 | memcpy(values.data() + offset, &value, sizeof(value)); 241 | offset += sizeof(value); 242 | count++; 243 | return values.data() + offset; 244 | } 245 | }; 246 | 247 | value_getter_t getter(values); 248 | (*redis_).hmget("hash", 249 | key_iterator_t(keys.data()), 250 | key_iterator_t(keys.data() + keys.size()), 251 | std::back_inserter(getter)); 252 | return {getter.count, getter.count ? operation_status_t::ok_k : operation_status_t::error_k}; 253 | } 254 | 255 | operation_result_t redis_t::bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 256 | auto data_offset = 0; 257 | auto pipe = (*redis_).pipeline(false); 258 | for (size_t i = 0; i != keys.size(); ++i) { 259 | pipe.hset("hash", to_string_view(keys[i]), to_string_view(values.data() + data_offset, sizes[i])); 260 | data_offset += sizes[i]; 261 | } 262 | 263 | auto pipe_replies = pipe.exec(); 264 | 265 | size_t count = 0; 266 | for (size_t i = 0; i != keys.size(); ++i) 267 | count += pipe_replies.get(i); 268 | 269 | return {count, operation_status_t::ok_k}; 270 | } 271 | 272 | operation_result_t redis_t::range_select(key_t /* key */, size_t /* length */, values_span_t /* values */) const { 273 | return {0, operation_status_t::not_implemented_k}; 274 | } 275 | 276 | operation_result_t redis_t::scan(key_t /* key */, size_t /* length */, value_span_t /* single_value */) const { 277 | return {0, operation_status_t::not_implemented_k}; 278 | } 279 | 280 | std::string redis_t::info() { return {}; } 281 | 282 | void redis_t::flush() {} 283 | 284 | size_t redis_t::size_on_disk() const { return 0; } 285 | 286 | std::unique_ptr redis_t::create_transaction() { return {}; } 287 | 288 | } // namespace ucsb::redis 289 | -------------------------------------------------------------------------------- /src/rocksdb/rocksdb_transaction.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include "src/core/types.hpp" 9 | #include "src/core/data_accessor.hpp" 10 | 11 | namespace ucsb::facebook { 12 | 13 | namespace fs = ucsb::fs; 14 | 15 | using key_t = ucsb::key_t; 16 | using keys_spanc_t = ucsb::keys_spanc_t; 17 | using value_span_t = ucsb::value_span_t; 18 | using value_spanc_t = ucsb::value_spanc_t; 19 | using values_span_t = ucsb::values_span_t; 20 | using values_spanc_t = ucsb::values_spanc_t; 21 | using value_lengths_spanc_t = ucsb::value_lengths_spanc_t; 22 | using operation_status_t = ucsb::operation_status_t; 23 | using operation_result_t = ucsb::operation_result_t; 24 | 25 | inline rocksdb::Slice to_slice(key_t& key) { 26 | key = __builtin_bswap64(key); 27 | static_assert(sizeof(key_t) == sizeof(uint64_t), "Check `__builtin_bswap64`"); 28 | return {reinterpret_cast(&key), sizeof(key_t)}; 29 | } 30 | 31 | inline rocksdb::Slice to_slice(value_spanc_t value) { 32 | return {reinterpret_cast(value.data()), value.size()}; 33 | } 34 | 35 | /* 36 | * @brief Preallocated buffers used for batch operations. 37 | * Globals and especially `thread_local`s are a bad practice. 38 | */ 39 | thread_local std::vector transaction_batch_keys; 40 | thread_local std::vector transaction_key_slices; 41 | thread_local std::vector transaction_value_slices; 42 | thread_local std::vector transaction_statuses; 43 | 44 | /** 45 | * @brief RocksDB transactional wrapper for the UCSB benchmark. 46 | * Wraps all of our operations into transactions or just 47 | * snapshots if read-only workloads run. 48 | */ 49 | class rocksdb_transaction_t : public ucsb::transaction_t { 50 | public: 51 | inline rocksdb_transaction_t(std::unique_ptr transaction, 52 | std::vector const& cf_handles) 53 | : transaction_(std::move(transaction)), cf_handles_(cf_handles) { 54 | read_options_.verify_checksums = false; 55 | } 56 | ~rocksdb_transaction_t(); 57 | 58 | operation_result_t upsert(key_t key, value_spanc_t value) override; 59 | operation_result_t update(key_t key, value_spanc_t value) override; 60 | operation_result_t remove(key_t key) override; 61 | operation_result_t read(key_t key, value_span_t value) const override; 62 | 63 | operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 64 | operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const override; 65 | 66 | operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 67 | 68 | operation_result_t range_select(key_t key, size_t length, values_span_t values) const override; 69 | operation_result_t scan(key_t key, size_t length, value_span_t single_value) const override; 70 | 71 | private: 72 | std::unique_ptr transaction_; 73 | std::vector cf_handles_; 74 | 75 | rocksdb::ReadOptions read_options_; 76 | }; 77 | 78 | rocksdb_transaction_t::~rocksdb_transaction_t() { 79 | transaction_batch_keys.clear(); 80 | transaction_key_slices.clear(); 81 | transaction_value_slices.clear(); 82 | transaction_statuses.clear(); 83 | 84 | auto status = transaction_->Commit(); 85 | assert(status.ok()); 86 | } 87 | 88 | operation_result_t rocksdb_transaction_t::upsert(key_t key, value_spanc_t value) { 89 | auto key_slice = to_slice(key); 90 | rocksdb::Status status = transaction_->Put(key_slice, to_slice(value)); 91 | if (!status.ok()) { 92 | assert(status.IsTryAgain()); 93 | status = transaction_->Commit(); 94 | assert(status.ok()); 95 | status = transaction_->Put(key_slice, to_slice(value)); 96 | assert(status.ok()); 97 | } 98 | return {size_t(status.ok()), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 99 | } 100 | 101 | operation_result_t rocksdb_transaction_t::update(key_t key, value_spanc_t value) { 102 | key_t original_key = key; 103 | rocksdb::PinnableSlice data; 104 | rocksdb::Status status = transaction_->Get(read_options_, to_slice(key), &data); 105 | if (status.IsNotFound()) 106 | return {0, operation_status_t::not_found_k}; 107 | else if (!status.ok()) 108 | return {0, operation_status_t::error_k}; 109 | 110 | return upsert(original_key, value); 111 | } 112 | 113 | operation_result_t rocksdb_transaction_t::remove(key_t key) { 114 | auto key_slice = to_slice(key); 115 | rocksdb::Status status = transaction_->Delete(key_slice); 116 | if (!status.ok()) { 117 | assert(status.IsTryAgain()); 118 | status = transaction_->Commit(); 119 | assert(status.ok()); 120 | status = transaction_->Delete(key_slice); 121 | assert(status.ok()); 122 | } 123 | 124 | return {size_t(status.ok()), status.ok() ? operation_status_t::ok_k : operation_status_t::error_k}; 125 | } 126 | 127 | operation_result_t rocksdb_transaction_t::read(key_t key, value_span_t value) const { 128 | rocksdb::PinnableSlice data; 129 | rocksdb::Status status = transaction_->Get(read_options_, to_slice(key), &data); 130 | if (status.IsNotFound()) 131 | return {0, operation_status_t::not_found_k}; 132 | else if (!status.ok()) 133 | return {0, operation_status_t::error_k}; 134 | 135 | memcpy(value.data(), data.data(), data.size()); 136 | return {1, operation_status_t::ok_k}; 137 | } 138 | 139 | operation_result_t rocksdb_transaction_t::batch_upsert(keys_spanc_t keys, 140 | values_spanc_t values, 141 | value_lengths_spanc_t sizes) { 142 | 143 | size_t offset = 0; 144 | for (size_t idx = 0; idx < keys.size(); ++idx) { 145 | auto key = keys[idx]; 146 | auto key_slice = to_slice(key); 147 | rocksdb::Status status = transaction_->Put(key_slice, to_slice(values.subspan(offset, sizes[idx]))); 148 | if (!status.ok()) { 149 | assert(status.IsTryAgain()); 150 | status = transaction_->Commit(); 151 | assert(status.ok()); 152 | status = transaction_->Put(key_slice, to_slice(values.subspan(offset, sizes[idx]))); 153 | assert(status.ok()); 154 | } 155 | offset += sizes[idx]; 156 | } 157 | return {keys.size(), operation_status_t::ok_k}; 158 | } 159 | 160 | operation_result_t rocksdb_transaction_t::batch_read(keys_spanc_t keys, values_span_t values) const { 161 | 162 | if (keys.size() > transaction_batch_keys.size()) { 163 | transaction_batch_keys.resize(keys.size()); 164 | transaction_key_slices.resize(keys.size()); 165 | transaction_value_slices.resize(keys.size()); 166 | transaction_statuses.resize(keys.size()); 167 | } 168 | 169 | for (size_t idx = 0; idx < keys.size(); ++idx) 170 | transaction_key_slices[idx] = to_slice(transaction_batch_keys[idx] = keys[idx]); 171 | 172 | transaction_->MultiGet(read_options_, 173 | cf_handles_.front(), 174 | transaction_key_slices.size(), 175 | transaction_key_slices.data(), 176 | transaction_value_slices.data(), 177 | transaction_statuses.data()); 178 | 179 | size_t offset = 0; 180 | size_t found_cnt = 0; 181 | for (size_t i = 0; i < transaction_statuses.size(); ++i) { 182 | if (!transaction_statuses[i].ok()) 183 | continue; 184 | 185 | memcpy(values.data() + offset, transaction_value_slices[i].data(), transaction_value_slices[i].size()); 186 | offset += transaction_value_slices[i].size(); 187 | ++found_cnt; 188 | } 189 | 190 | return {found_cnt, operation_status_t::ok_k}; 191 | } 192 | 193 | operation_result_t rocksdb_transaction_t::bulk_load(keys_spanc_t keys, 194 | values_spanc_t values, 195 | value_lengths_spanc_t sizes) { 196 | return batch_upsert(keys, values, sizes); 197 | } 198 | 199 | operation_result_t rocksdb_transaction_t::range_select(key_t key, size_t length, values_span_t values) const { 200 | 201 | size_t i = 0; 202 | size_t exported_bytes = 0; 203 | std::unique_ptr it(transaction_->GetIterator(read_options_)); 204 | it->Seek(to_slice(key)); 205 | for (; it->Valid() && i != length; i++, it->Next()) { 206 | memcpy(values.data() + exported_bytes, it->value().data(), it->value().size()); 207 | exported_bytes += it->value().size(); 208 | } 209 | return {i, operation_status_t::ok_k}; 210 | } 211 | 212 | operation_result_t rocksdb_transaction_t::scan(key_t key, size_t length, value_span_t single_value) const { 213 | 214 | size_t i = 0; 215 | rocksdb::ReadOptions scan_options = read_options_; 216 | // It's recommended to disable caching on long scans. 217 | // https://github.com/facebook/rocksdb/blob/49a10feb21dc5c766bb272406136667e1d8a969e/include/rocksdb/options.h#L1462 218 | scan_options.fill_cache = false; 219 | std::unique_ptr it(transaction_->GetIterator(scan_options)); 220 | it->Seek(to_slice(key)); 221 | for (; it->Valid() && i != length; i++, it->Next()) 222 | memcpy(single_value.data(), it->value().data(), it->value().size()); 223 | return {i, operation_status_t::ok_k}; 224 | } 225 | 226 | } // namespace ucsb::facebook 227 | -------------------------------------------------------------------------------- /src/ustore/ustore_transaction.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "src/core/types.hpp" 7 | #include "src/core/data_accessor.hpp" 8 | 9 | namespace ucsb::ustore { 10 | 11 | namespace fs = ucsb::fs; 12 | namespace ustore = unum::ustore; 13 | 14 | using key_t = ucsb::key_t; 15 | using keys_spanc_t = ucsb::keys_spanc_t; 16 | using value_span_t = ucsb::value_span_t; 17 | using value_spanc_t = ucsb::value_spanc_t; 18 | using values_span_t = ucsb::values_span_t; 19 | using values_spanc_t = ucsb::values_spanc_t; 20 | using value_lengths_spanc_t = ucsb::value_lengths_spanc_t; 21 | using operation_status_t = ucsb::operation_status_t; 22 | using operation_result_t = ucsb::operation_result_t; 23 | 24 | thread_local ustore::arena_t arena_(nullptr); 25 | 26 | inline ustore::value_view_t make_value(std::byte const* ptr, size_t length) { 27 | return {reinterpret_cast(ptr), static_cast(length)}; 28 | } 29 | 30 | class ustore_transact_t : public ucsb::transaction_t { 31 | public: 32 | inline ustore_transact_t(ustore_database_t db, ustore_transaction_t transaction) 33 | : db_(db), transaction_(transaction), arena_(db_) {} 34 | ~ustore_transact_t(); 35 | 36 | operation_result_t upsert(key_t key, value_spanc_t value) override; 37 | operation_result_t update(key_t key, value_spanc_t value) override; 38 | operation_result_t remove(key_t key) override; 39 | operation_result_t read(key_t key, value_span_t value) const override; 40 | 41 | operation_result_t batch_upsert(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 42 | operation_result_t batch_read(keys_spanc_t keys, values_span_t values) const override; 43 | 44 | operation_result_t bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) override; 45 | 46 | operation_result_t range_select(key_t key, size_t length, values_span_t values) const override; 47 | operation_result_t scan(key_t key, size_t length, value_span_t single_value) const override; 48 | 49 | private: 50 | inline ustore::status_t commit() { 51 | ustore::status_t status; 52 | ustore_transaction_commit_t txn_commit {}; 53 | txn_commit.db = db_; 54 | txn_commit.transaction = transaction_; 55 | txn_commit.error = status.member_ptr(); 56 | txn_commit.options = options_; 57 | ustore_transaction_commit(&txn_commit); 58 | return status; 59 | } 60 | 61 | ustore_database_t db_; 62 | ustore_transaction_t transaction_; 63 | ustore_collection_t collection_ = ustore_collection_main_k; 64 | ustore_options_t options_ = ustore_options_default_k; 65 | ustore::arena_t mutable arena_; 66 | }; 67 | 68 | ustore_transact_t::~ustore_transact_t() { 69 | [[maybe_unused]] auto status = commit(); 70 | assert(status); 71 | ustore_transaction_free(transaction_); 72 | } 73 | 74 | operation_result_t ustore_transact_t::upsert(key_t key, value_spanc_t value) { 75 | ustore::status_t status; 76 | ustore_key_t key_ = key; 77 | ustore_length_t length = value.size(); 78 | auto value_ = make_value(value.data(), value.size()); 79 | 80 | ustore_write_t write {}; 81 | write.db = db_; 82 | write.transaction = transaction_; 83 | write.error = status.member_ptr(); 84 | write.arena = arena_.member_ptr(); 85 | write.options = options_; 86 | write.tasks_count = 1; 87 | write.collections = &collection_; 88 | write.keys = &key_; 89 | write.lengths = reinterpret_cast(&length); 90 | write.values = value_.member_ptr(); 91 | ustore_write(&write); 92 | if (!status && commit()) { 93 | status.release_exception(); 94 | ustore_write(&write); 95 | } 96 | 97 | return {size_t(status), status ? operation_status_t::ok_k : operation_status_t::error_k}; 98 | } 99 | 100 | operation_result_t ustore_transact_t::update(key_t key, value_spanc_t value) { 101 | ustore::status_t status; 102 | ustore_key_t key_ = key; 103 | ustore_byte_t* value_ = nullptr; 104 | 105 | ustore_read_t read {}; 106 | read.db = db_; 107 | read.transaction = transaction_; 108 | read.error = status.member_ptr(); 109 | read.arena = arena_.member_ptr(); 110 | read.options = options_; 111 | read.tasks_count = 1; 112 | read.collections = &collection_; 113 | read.keys = &key_; 114 | read.values = &value_; 115 | ustore_read(&read); 116 | if (!status) 117 | return {0, operation_status_t::not_found_k}; 118 | 119 | return upsert(key, value); 120 | } 121 | 122 | operation_result_t ustore_transact_t::remove(key_t key) { 123 | ustore::status_t status; 124 | ustore_key_t key_ = key; 125 | 126 | ustore_write_t write {}; 127 | write.db = db_; 128 | write.transaction = transaction_; 129 | write.error = status.member_ptr(); 130 | write.arena = arena_.member_ptr(); 131 | write.options = options_; 132 | write.tasks_count = 1; 133 | write.collections = &collection_; 134 | write.keys = &key_; 135 | ustore_write(&write); 136 | if (!status && commit()) { 137 | status.release_exception(); 138 | ustore_write(&write); 139 | } 140 | 141 | return {status ? size_t(1) : 0, status ? operation_status_t::ok_k : operation_status_t::error_k}; 142 | } 143 | 144 | operation_result_t ustore_transact_t::read(key_t key, value_span_t value) const { 145 | ustore::status_t status; 146 | ustore_key_t key_ = key; 147 | ustore_byte_t* value_ = nullptr; 148 | ustore_length_t* lengths = nullptr; 149 | 150 | ustore_read_t read {}; 151 | read.db = db_; 152 | read.transaction = transaction_; 153 | read.error = status.member_ptr(); 154 | read.arena = arena_.member_ptr(); 155 | read.options = options_; 156 | read.tasks_count = 1; 157 | read.collections = &collection_; 158 | read.keys = &key_; 159 | read.lengths = &lengths; 160 | read.values = &value_; 161 | ustore_read(&read); 162 | if (!status) 163 | return {0, operation_status_t::error_k}; 164 | if (lengths[0] == ustore_length_missing_k) 165 | return {0, operation_status_t::not_found_k}; 166 | 167 | memcpy(value.data(), value_, lengths[0]); 168 | return {1, operation_status_t::ok_k}; 169 | } 170 | 171 | operation_result_t ustore_transact_t::batch_upsert(keys_spanc_t keys, 172 | values_spanc_t values, 173 | value_lengths_spanc_t sizes) { 174 | ustore::status_t status; 175 | std::vector offsets; 176 | offsets.reserve(sizes.size() + 1); 177 | offsets.push_back(0); 178 | for (auto size : sizes) 179 | offsets.push_back(offsets.back() + size); 180 | 181 | auto values_ = make_value(values.data(), values.size()); 182 | ustore_write_t write {}; 183 | write.db = db_; 184 | write.transaction = transaction_; 185 | write.error = status.member_ptr(); 186 | write.arena = arena_.member_ptr(); 187 | write.options = options_; 188 | write.tasks_count = keys.size(); 189 | write.collections = &collection_; 190 | write.keys = reinterpret_cast(keys.data()); 191 | write.keys_stride = sizeof(ustore_key_t); 192 | write.offsets = offsets.data(); 193 | write.offsets_stride = sizeof(ustore_length_t); 194 | write.lengths = reinterpret_cast(sizes.data()); 195 | write.lengths_stride = sizeof(ustore_length_t); 196 | write.values = values_.member_ptr(); 197 | ustore_write(&write); 198 | if (!status && commit()) { 199 | status.release_exception(); 200 | ustore_write(&write); 201 | } 202 | 203 | return {status ? keys.size() : 0, status ? operation_status_t::ok_k : operation_status_t::error_k}; 204 | } 205 | 206 | operation_result_t ustore_transact_t::batch_read(keys_spanc_t keys, values_span_t values) const { 207 | ustore::status_t status; 208 | ustore_octet_t* presences = nullptr; 209 | ustore_length_t* offsets = nullptr; 210 | ustore_length_t* lengths = nullptr; 211 | ustore_byte_t* values_ = nullptr; 212 | 213 | ustore_read_t read {}; 214 | read.db = db_; 215 | read.transaction = transaction_; 216 | read.error = status.member_ptr(); 217 | read.arena = arena_.member_ptr(); 218 | read.options = options_; 219 | read.tasks_count = keys.size(); 220 | read.collections = &collection_; 221 | read.keys = reinterpret_cast(keys.data()); 222 | read.keys_stride = sizeof(ustore_key_t); 223 | read.presences = &presences; 224 | read.offsets = &offsets; 225 | read.lengths = &lengths; 226 | read.values = &values_; 227 | ustore_read(&read); 228 | if (!status) 229 | return {0, operation_status_t::error_k}; 230 | 231 | size_t offset = 0; 232 | size_t found_cnt = 0; 233 | for (size_t idx = 0; idx < keys.size(); ++idx) { 234 | if (lengths[idx] == ustore_length_missing_k) 235 | continue; 236 | memcpy(values.data() + offset, values_ + offsets[idx], lengths[idx]); 237 | offset += lengths[idx]; 238 | ++found_cnt; 239 | } 240 | 241 | return {found_cnt, found_cnt > 0 ? operation_status_t::ok_k : operation_status_t::not_found_k}; 242 | } 243 | 244 | operation_result_t ustore_transact_t::bulk_load(keys_spanc_t keys, values_spanc_t values, value_lengths_spanc_t sizes) { 245 | return batch_upsert(keys, values, sizes); 246 | } 247 | 248 | operation_result_t ustore_transact_t::range_select(key_t key, size_t length, values_span_t values) const { 249 | ustore::status_t status; 250 | ustore_key_t key_ = key; 251 | ustore_length_t len = length; 252 | ustore_length_t* found_counts = nullptr; 253 | ustore_key_t* found_keys = nullptr; 254 | 255 | // First scan keys 256 | ustore_scan_t scan {}; 257 | scan.db = db_; 258 | scan.transaction = transaction_; 259 | scan.error = status.member_ptr(); 260 | scan.arena = arena_.member_ptr(); 261 | scan.options = options_; 262 | scan.tasks_count = 1; 263 | scan.collections = &collection_; 264 | scan.start_keys = &key_; 265 | scan.count_limits = &len; 266 | scan.counts = &found_counts; 267 | scan.keys = &found_keys; 268 | ustore_scan(&scan); 269 | if (!status) 270 | return {0, operation_status_t::error_k}; 271 | 272 | ustore_length_t* offsets = nullptr; 273 | ustore_length_t* lengths = nullptr; 274 | ustore_byte_t* values_ = nullptr; 275 | 276 | // Then do batch read 277 | ustore_read_t read {}; 278 | read.db = db_; 279 | read.transaction = transaction_; 280 | read.error = status.member_ptr(); 281 | read.arena = arena_.member_ptr(); 282 | read.options = ustore_options_t(options_ | ustore_option_dont_discard_memory_k); 283 | read.tasks_count = *found_counts; 284 | read.collections = &collection_; 285 | read.keys = found_keys; 286 | read.keys_stride = sizeof(ustore_key_t); 287 | read.offsets = &offsets; 288 | read.lengths = &lengths; 289 | read.values = &values_; 290 | ustore_read(&read); 291 | if (!status) 292 | return {0, operation_status_t::error_k}; 293 | 294 | size_t offset = 0; 295 | for (size_t idx = 0; idx < *found_counts; ++idx) { 296 | if (lengths[idx] == ustore_length_missing_k) 297 | continue; 298 | memcpy(values.data() + offset, values_ + offsets[idx], lengths[idx]); 299 | offset += lengths[idx]; 300 | } 301 | 302 | return {*found_counts, *found_counts > 0 ? operation_status_t::ok_k : operation_status_t::not_found_k}; 303 | } 304 | 305 | operation_result_t ustore_transact_t::scan(key_t key, size_t length, value_span_t single_value) const { 306 | ustore::status_t status; 307 | ustore_key_t key_ = key; 308 | ustore_length_t len = 309 | std::min(length, 1'000'000); // Note: Don't scan all at once because the DB might be very big 310 | ustore_length_t* found_counts = nullptr; 311 | ustore_key_t* found_keys = nullptr; 312 | 313 | ustore_length_t* offsets = nullptr; 314 | ustore_length_t* lengths = nullptr; 315 | ustore_byte_t* values_ = nullptr; 316 | 317 | // Init scan 318 | ustore_scan_t scan {}; 319 | scan.db = db_; 320 | scan.transaction = transaction_; 321 | scan.error = status.member_ptr(); 322 | scan.arena = arena_.member_ptr(); 323 | scan.options = options_; 324 | scan.tasks_count = 1; 325 | scan.collections = &collection_; 326 | scan.start_keys = &key_; 327 | scan.count_limits = &len; 328 | scan.counts = &found_counts; 329 | scan.keys = &found_keys; 330 | 331 | // Init batch read 332 | ustore_read_t read {}; 333 | read.db = db_; 334 | read.transaction = transaction_; 335 | read.error = status.member_ptr(); 336 | read.arena = arena_.member_ptr(); 337 | read.options = ustore_options_t(options_ | ustore_option_dont_discard_memory_k); 338 | read.collections = &collection_; 339 | read.keys_stride = sizeof(ustore_key_t); 340 | read.offsets = &offsets; 341 | read.lengths = &lengths; 342 | read.values = &values_; 343 | 344 | ustore_length_t scanned = 0; 345 | ustore_length_t remaining_keys_cnt = length; 346 | while (remaining_keys_cnt) { 347 | // First scan 348 | ustore_scan(&scan); 349 | if (!status) 350 | return {0, operation_status_t::error_k}; 351 | 352 | // Then read 353 | read.tasks_count = *found_counts; 354 | read.keys = found_keys; 355 | ustore_read(&read); 356 | if (!status) 357 | return {0, operation_status_t::error_k}; 358 | 359 | scanned += *found_counts; 360 | for (size_t idx = 0; idx < *found_counts; ++idx) 361 | if (lengths[idx] != ustore_length_missing_k) 362 | memcpy(single_value.data(), values_ + offsets[idx], lengths[idx]); 363 | 364 | key_ += len; 365 | remaining_keys_cnt = remaining_keys_cnt - len; 366 | len = std::min(len, remaining_keys_cnt); 367 | } 368 | 369 | return {scanned, scanned > 0 ? operation_status_t::ok_k : operation_status_t::not_found_k}; 370 | } 371 | 372 | } // namespace ucsb::ustore --------------------------------------------------------------------------------