├── .clang-format ├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── LICENSE ├── build.sh ├── cmake └── Modules │ ├── Coveralls.cmake │ ├── CoverallsClear.cmake │ ├── CoverallsGenerateGcov.cmake │ └── Utils.cmake ├── include ├── bucket.h ├── bucket_header.h ├── cursor.h ├── db.h ├── fnv │ └── fnv.h ├── memory_pool.h ├── meta.h ├── node.h ├── rwlock.h ├── txn.h ├── types.h └── util.h ├── readme.md ├── script ├── formatting │ ├── formatter.py │ └── iwyu.md ├── git-hooks │ ├── README.md │ └── pre-commit ├── helpers.py ├── readme └── validators │ └── source_validator.py ├── src ├── bucket.cpp ├── cursor.cpp ├── db.cpp ├── fnv │ └── hash_64a.c ├── meta.cpp ├── node.cpp ├── txn.cpp ├── types.cpp └── util.cpp ├── test ├── CMakeLists.txt ├── bucket │ └── bucket_test.cpp ├── db │ └── db_test.cpp ├── flock │ └── flock_test.cpp ├── include │ └── test_util.h └── txn │ └── txn_test.cpp └── third_party ├── gmock ├── gmock-gtest-all.cc ├── gmock │ └── gmock.h ├── gmock_main.cc └── gtest │ └── gtest.h └── valgrind └── valgrind.supp /.clang-format: -------------------------------------------------------------------------------- 1 | # Options for Clang Formatter 2 | 3 | # We'll use defaults from the Google style, 4 | BasedOnStyle: Google 5 | 6 | # 2 columns tab indentation. 7 | TabWidth: 2 8 | 9 | # No tabs 10 | UseTab: Never 11 | 12 | # Don't automatically derive the alignment of & and * 13 | DerivePointerAlignment: false 14 | 15 | # Always align pointer and reference to the right 16 | PointerAlignment: Right 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmake-build-debug/ 2 | build/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: cpp 3 | 4 | #before_script: 5 | # - sh install_grpc.sh 6 | 7 | script: sh build.sh -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.9) 2 | project(boltDB_in_cpp) 3 | 4 | set(CMAKE_CXX_STANDARD 11) 5 | 6 | # ---[ CTest 7 | include(CTest) 8 | 9 | # ---[ Dependencies 10 | find_package(Threads REQUIRED) 11 | 12 | # ---[ Using cmake scripts and modules 13 | list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) 14 | # -- [ Coverage 15 | option(COVERALLS "Generate coveralls data" OFF) 16 | 17 | if (COVERALLS) 18 | include(Coveralls) 19 | include(Utils) 20 | coveralls_turn_on_coverage() 21 | 22 | # Create the coveralls target. 23 | file(GLOB_RECURSE srcs ${PROJECT_SOURCE_DIR}/src/*.cpp) 24 | 25 | peloton_convert_absolute_paths(srcs) 26 | peloton_convert_absolute_paths(parser_srcs) 27 | 28 | #message(STATUS "Coverage srcs : ${srcs}" ) 29 | set(COVERAGE_SRCS ${srcs}) 30 | 31 | coveralls_setup("${COVERAGE_SRCS}" ON) 32 | endif () 33 | 34 | include_directories(${CMAKE_SOURCE_DIR}/include) 35 | include_directories(${CMAKE_SOURCE_DIR}/test/include) 36 | include_directories(${CMAKE_SOURCE_DIR}/third_party) 37 | include_directories(BEFORE src) # This is needed for gtest. 38 | file(GLOB SRC_BOLT ${PROJECT_SOURCE_DIR}/src/*.cpp) 39 | file(GLOB INC_BOLT ${PROJECT_SOURCE_DIR}/include/*.h) 40 | set(FNV ${CMAKE_SOURCE_DIR}/src/fnv/hash_64a.c ${CMAKE_SOURCE_DIR}/include/fnv/fnv.h) 41 | set(SRC 42 | ${SRC_BOLT} 43 | ${INC_BOLT} 44 | ) 45 | add_library(boltDB_in_cpp ${SRC} ${FNV}) 46 | add_subdirectory(test) 47 | 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Ben Johnson, Nov11 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [ ! -d "build" ] ; then 3 | mkdir build 4 | fi 5 | 6 | cd build 7 | cmake .. -DCMAKE_BUILD_TYPE=Debug -DCOVERALLS=ON 8 | make -j4 9 | make check 10 | make coveralls -------------------------------------------------------------------------------- /cmake/Modules/Coveralls.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # The MIT License (MIT) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in all 12 | # copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | # SOFTWARE. 21 | # 22 | # Copyright (C) 2014 Joakim Söderberg 23 | # 24 | 25 | set(_CMAKE_SCRIPT_PATH ${CMAKE_CURRENT_LIST_DIR}) # must be outside coveralls_setup() to get correct path 26 | 27 | # 28 | # Param _COVERAGE_SRCS A list of source files that coverage should be collected for. 29 | # Param _COVERALLS_UPLOAD Upload the result to coveralls? 30 | # 31 | 32 | function(coveralls_setup _COVERAGE_SRCS _COVERALLS_UPLOAD) 33 | 34 | if (ARGC GREATER 2) 35 | set(_CMAKE_SCRIPT_PATH ${ARGN}) 36 | message(STATUS "Coveralls: Using alternate CMake script dir: ${_CMAKE_SCRIPT_PATH}") 37 | endif() 38 | 39 | if (NOT EXISTS "${_CMAKE_SCRIPT_PATH}/CoverallsClear.cmake") 40 | message(FATAL_ERROR "Coveralls: Missing ${_CMAKE_SCRIPT_PATH}/CoverallsClear.cmake") 41 | endif() 42 | 43 | if (NOT EXISTS "${_CMAKE_SCRIPT_PATH}/CoverallsGenerateGcov.cmake") 44 | message(FATAL_ERROR "Coveralls: Missing ${_CMAKE_SCRIPT_PATH}/CoverallsGenerateGcov.cmake") 45 | endif() 46 | 47 | # When passing a CMake list to an external process, the list 48 | # will be converted from the format "1;2;3" to "1 2 3". 49 | # This means the script we're calling won't see it as a list 50 | # of sources, but rather just one long path. We remedy this 51 | # by replacing ";" with "*" and then reversing that in the script 52 | # that we're calling. 53 | # http://cmake.3232098.n2.nabble.com/Passing-a-CMake-list-quot-as-is-quot-to-a-custom-target-td6505681.html 54 | set(COVERAGE_SRCS_TMP ${_COVERAGE_SRCS}) 55 | set(COVERAGE_SRCS "") 56 | foreach (COVERAGE_SRC ${COVERAGE_SRCS_TMP}) 57 | set(COVERAGE_SRCS "${COVERAGE_SRCS}*${COVERAGE_SRC}") 58 | endforeach() 59 | 60 | #message("Coverage sources: ${COVERAGE_SRCS}") 61 | set(COVERALLS_FILE ${PROJECT_BINARY_DIR}/coveralls.json) 62 | 63 | add_custom_target(coveralls_generate 64 | 65 | # Zero the coverage counters. 66 | # COMMAND ${CMAKE_COMMAND} -DPROJECT_BINARY_DIR="${PROJECT_BINARY_DIR}" -P "${_CMAKE_SCRIPT_PATH}/CoverallsClear.cmake" 67 | 68 | # Run regress tests. 69 | # COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure 70 | 71 | # Since we are running tests in a serial manner, it will take a long time rerunning all the test cases again, which 72 | # cause a time out error in the travis ci server. So We just use the .gcda file generated from the previous 73 | # make check to evaluate the coverage. Note that now we can only run make coveralls after running make check. 74 | 75 | # Generate Gcov and translate it into coveralls JSON. 76 | # We do this by executing an external CMake script. 77 | # (We don't want this to run at CMake generation time, but after compilation and everything has run). 78 | COMMAND ${CMAKE_COMMAND} 79 | -DCOVERAGE_SRCS="${COVERAGE_SRCS}" # TODO: This is passed like: "a b c", not "a;b;c" 80 | -DCOVERALLS_OUTPUT_FILE="${COVERALLS_FILE}" 81 | -DCOV_PATH="${PROJECT_BINARY_DIR}" 82 | -DPROJECT_ROOT="${PROJECT_SOURCE_DIR}" 83 | -P "${_CMAKE_SCRIPT_PATH}/CoverallsGenerateGcov.cmake" 84 | 85 | WORKING_DIRECTORY ${PROJECT_BINARY_DIR} 86 | COMMENT "Generating coveralls output..." 87 | ) 88 | 89 | if (_COVERALLS_UPLOAD) 90 | message("COVERALLS UPLOAD: ON") 91 | 92 | find_program(CURL_EXECUTABLE curl) 93 | 94 | if (NOT CURL_EXECUTABLE) 95 | message(FATAL_ERROR "Coveralls: curl not found! Aborting") 96 | endif() 97 | 98 | add_custom_target(coveralls_upload 99 | # Upload the JSON to coveralls. 100 | COMMAND ${CURL_EXECUTABLE} 101 | -S -F json_file=@${COVERALLS_FILE} 102 | https://coveralls.io/api/v1/jobs 103 | 104 | DEPENDS coveralls_generate 105 | 106 | WORKING_DIRECTORY ${PROJECT_BINARY_DIR} 107 | COMMENT "Uploading coveralls output...") 108 | 109 | add_custom_target(coveralls DEPENDS coveralls_upload) 110 | else() 111 | message("COVERALLS UPLOAD: OFF") 112 | add_custom_target(coveralls DEPENDS coveralls_generate) 113 | endif() 114 | 115 | endfunction() 116 | 117 | macro(coveralls_turn_on_coverage) 118 | if(NOT (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) 119 | AND (NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")) 120 | message(FATAL_ERROR "Coveralls: Compiler ${CMAKE_C_COMPILER_ID} is not GNU gcc! Aborting... You can set this on the command line using CC=/usr/bin/gcc CXX=/usr/bin/g++ cmake ..") 121 | endif() 122 | 123 | if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") 124 | message(FATAL_ERROR "Coveralls: Code coverage results with an optimised (non-Debug) build may be misleading! Add -DCMAKE_BUILD_TYPE=Debug") 125 | endif() 126 | 127 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") 128 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") 129 | set(CMAKE_EXE_LINKER_FLAGS " ${CMAKE_EXE_LINKER_FLAGS} -lgcov -fprofile-arcs --coverage") 130 | endmacro() 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /cmake/Modules/CoverallsClear.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # The MIT License (MIT) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in all 12 | # copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | # SOFTWARE. 21 | # 22 | # Copyright (C) 2014 Joakim Söderberg 23 | # 24 | 25 | # do not follow symlinks in file(GLOB_RECURSE ...) 26 | cmake_policy(SET CMP0009 NEW) 27 | 28 | file(GLOB_RECURSE GCDA_FILES "${PROJECT_BINARY_DIR}/*.gcda") 29 | if(NOT GCDA_FILES STREQUAL "") 30 | file(REMOVE ${GCDA_FILES}) 31 | endif() 32 | -------------------------------------------------------------------------------- /cmake/Modules/CoverallsGenerateGcov.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # The MIT License (MIT) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in all 12 | # copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | # SOFTWARE. 21 | # 22 | # Copyright (C) 2014 Joakim Söderberg 23 | # 24 | # This is intended to be run by a custom target in a CMake project like this. 25 | # 0. Compile program with coverage support. 26 | # 1. Clear coverage data. (Recursively delete *.gcda in build dir) 27 | # 2. Run the unit tests. 28 | # 3. Run this script specifying which source files the coverage should be performed on. 29 | # 30 | # This script will then use gcov to generate .gcov files in the directory specified 31 | # via the COV_PATH var. This should probably be the same as your cmake build dir. 32 | # 33 | # It then parses the .gcov files to convert them into the Coveralls JSON format: 34 | # https://coveralls.io/docs/api 35 | # 36 | # Example for running as standalone CMake script from the command line: 37 | # (Note it is important the -P is at the end...) 38 | # $ cmake -DCOV_PATH=$(pwd) 39 | # -DCOVERAGE_SRCS="catcierge_rfid.c;catcierge_timer.c" 40 | # -P ../cmake/CoverallsGcovUpload.cmake 41 | # 42 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8) 43 | 44 | 45 | # 46 | # Make sure we have the needed arguments. 47 | # 48 | if (NOT COVERALLS_OUTPUT_FILE) 49 | message(FATAL_ERROR "Coveralls: No coveralls output file specified. Please set COVERALLS_OUTPUT_FILE") 50 | endif () 51 | 52 | if (NOT COV_PATH) 53 | message(FATAL_ERROR "Coveralls: Missing coverage directory path where gcov files will be generated. Please set COV_PATH") 54 | endif () 55 | 56 | if (NOT COVERAGE_SRCS) 57 | message(FATAL_ERROR "Coveralls: Missing the list of source files that we should get the coverage data for COVERAGE_SRCS") 58 | endif () 59 | 60 | if (NOT PROJECT_ROOT) 61 | message(FATAL_ERROR "Coveralls: Missing PROJECT_ROOT.") 62 | endif () 63 | 64 | # Since it's not possible to pass a CMake list properly in the 65 | # "1;2;3" format to an external process, we have replaced the 66 | # ";" with "*", so reverse that here so we get it back into the 67 | # CMake list format. 68 | string(REGEX REPLACE "\\*" ";" COVERAGE_SRCS ${COVERAGE_SRCS}) 69 | 70 | if (NOT DEFINED ENV{GCOV}) 71 | find_program(GCOV_EXECUTABLE gcov) 72 | else () 73 | find_program(GCOV_EXECUTABLE $ENV{GCOV}) 74 | endif () 75 | 76 | # convert all paths in COVERAGE_SRCS to absolute paths 77 | set(COVERAGE_SRCS_TMP "") 78 | foreach (COVERAGE_SRC ${COVERAGE_SRCS}) 79 | if (NOT "${COVERAGE_SRC}" MATCHES "^/") 80 | set(COVERAGE_SRC ${PROJECT_ROOT}/${COVERAGE_SRC}) 81 | endif () 82 | list(APPEND COVERAGE_SRCS_TMP ${COVERAGE_SRC}) 83 | endforeach () 84 | set(COVERAGE_SRCS ${COVERAGE_SRCS_TMP}) 85 | unset(COVERAGE_SRCS_TMP) 86 | 87 | if (NOT GCOV_EXECUTABLE) 88 | message(FATAL_ERROR "gcov not found! Aborting...") 89 | endif () 90 | 91 | find_package(Git) 92 | 93 | set(JSON_REPO_TEMPLATE 94 | "{ 95 | \"head\": { 96 | \"id\": \"\@GIT_COMMIT_HASH\@\", 97 | \"author_name\": \"\@GIT_AUTHOR_NAME\@\", 98 | \"author_email\": \"\@GIT_AUTHOR_EMAIL\@\", 99 | \"committer_name\": \"\@GIT_COMMITTER_NAME\@\", 100 | \"committer_email\": \"\@GIT_COMMITTER_EMAIL\@\", 101 | \"message\": \"\@GIT_COMMIT_MESSAGE\@\" 102 | }, 103 | \"branch\": \"@GIT_BRANCH@\", 104 | \"remotes\": [] 105 | }" 106 | ) 107 | 108 | # TODO: Fill in git remote data 109 | if (GIT_FOUND) 110 | # Branch. 111 | execute_process( 112 | COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD 113 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 114 | OUTPUT_VARIABLE GIT_BRANCH 115 | OUTPUT_STRIP_TRAILING_WHITESPACE 116 | ) 117 | 118 | macro(git_log_format FORMAT_CHARS VAR_NAME) 119 | execute_process( 120 | COMMAND ${GIT_EXECUTABLE} log -1 --pretty=format:%${FORMAT_CHARS} 121 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 122 | OUTPUT_VARIABLE ${VAR_NAME} 123 | OUTPUT_STRIP_TRAILING_WHITESPACE 124 | ) 125 | endmacro() 126 | 127 | git_log_format(an GIT_AUTHOR_NAME) 128 | git_log_format(ae GIT_AUTHOR_EMAIL) 129 | git_log_format(cn GIT_COMMITTER_NAME) 130 | git_log_format(ce GIT_COMMITTER_EMAIL) 131 | git_log_format(B GIT_COMMIT_MESSAGE) 132 | git_log_format(H GIT_COMMIT_HASH) 133 | 134 | if (GIT_COMMIT_MESSAGE) 135 | string(REPLACE "\n" "\\n" GIT_COMMIT_MESSAGE ${GIT_COMMIT_MESSAGE}) 136 | endif () 137 | 138 | message("Git exe: ${GIT_EXECUTABLE}") 139 | message("Git branch: ${GIT_BRANCH}") 140 | message("Git author: ${GIT_AUTHOR_NAME}") 141 | message("Git e-mail: ${GIT_AUTHOR_EMAIL}") 142 | message("Git commiter name: ${GIT_COMMITTER_NAME}") 143 | message("Git commiter e-mail: ${GIT_COMMITTER_EMAIL}") 144 | message("Git commit hash: ${GIT_COMMIT_HASH}") 145 | message("Git commit message: ${GIT_COMMIT_MESSAGE}") 146 | 147 | string(CONFIGURE ${JSON_REPO_TEMPLATE} JSON_REPO_DATA) 148 | else () 149 | set(JSON_REPO_DATA "{}") 150 | endif () 151 | 152 | ############################# Macros ######################################### 153 | 154 | # 155 | # This macro converts from the full path format gcov outputs: 156 | # 157 | # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov 158 | # 159 | # to the original source file path the .gcov is for: 160 | # 161 | # /path/to/project/root/subdir/the_file.c 162 | # 163 | macro(get_source_path_from_gcov_filename _SRC_FILENAME _GCOV_FILENAME) 164 | 165 | # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov 166 | # -> 167 | # #path#to#project#root#subdir#the_file.c.gcov 168 | get_filename_component(_GCOV_FILENAME_WEXT ${_GCOV_FILENAME} NAME) 169 | 170 | # #path#to#project#root#subdir#the_file.c.gcov -> /path/to/project/root/subdir/the_file.c 171 | string(REGEX REPLACE "\\.gcov$" "" SRC_FILENAME_TMP ${_GCOV_FILENAME_WEXT}) 172 | string(REGEX REPLACE "\#" "/" SRC_FILENAME_TMP ${SRC_FILENAME_TMP}) 173 | set(${_SRC_FILENAME} "${SRC_FILENAME_TMP}") 174 | endmacro() 175 | 176 | ############################################################################## 177 | 178 | # Get the coverage data. 179 | file(GLOB_RECURSE GCDA_FILES "${COV_PATH}/*.gcda") 180 | message("GCDA files:") 181 | 182 | # Get a list of all the object directories needed by gcov 183 | # (The directories the .gcda files and .o files are found in) 184 | # and run gcov on those. 185 | foreach (GCDA ${GCDA_FILES}) 186 | message("Process: ${GCDA}") 187 | message("------------------------------------------------------------------------------") 188 | get_filename_component(GCDA_DIR ${GCDA} PATH) 189 | 190 | # 191 | # The -p below refers to "Preserve path components", 192 | # This means that the generated gcov filename of a source file will 193 | # keep the original files entire filepath, but / is replaced with #. 194 | # Example: 195 | # 196 | # /path/to/project/root/build/CMakeFiles/the_file.dir/subdir/the_file.c.gcda 197 | # ------------------------------------------------------------------------------ 198 | # File '/path/to/project/root/subdir/the_file.c' 199 | # Lines executed:68.34% of 199 200 | # /path/to/project/root/subdir/the_file.c:creating '#path#to#project#root#subdir#the_file.c.gcov' 201 | # 202 | # If -p is not specified then the file is named only "the_file.c.gcov" 203 | # 204 | execute_process( 205 | COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} 206 | WORKING_DIRECTORY ${COV_PATH} 207 | OUTPUT_QUIET 208 | ) 209 | endforeach () 210 | 211 | # TODO: Make these be absolute path 212 | file(GLOB ALL_GCOV_FILES ${COV_PATH}/*.gcov) 213 | 214 | # Get only the filenames to use for filtering. 215 | #set(COVERAGE_SRCS_NAMES "") 216 | #foreach (COVSRC ${COVERAGE_SRCS}) 217 | # get_filename_component(COVSRC_NAME ${COVSRC} NAME) 218 | # message("${COVSRC} -> ${COVSRC_NAME}") 219 | # list(APPEND COVERAGE_SRCS_NAMES "${COVSRC_NAME}") 220 | #endforeach() 221 | 222 | # 223 | # Filter out all but the gcov files we want. 224 | # 225 | # We do this by comparing the list of COVERAGE_SRCS filepaths that the 226 | # user wants the coverage data for with the paths of the generated .gcov files, 227 | # so that we only keep the relevant gcov files. 228 | # 229 | # Example: 230 | # COVERAGE_SRCS = 231 | # /path/to/project/root/subdir/the_file.c 232 | # 233 | # ALL_GCOV_FILES = 234 | # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov 235 | # /path/to/project/root/build/#path#to#project#root#subdir#other_file.c.gcov 236 | # 237 | # Result should be: 238 | # GCOV_FILES = 239 | # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov 240 | # 241 | set(GCOV_FILES "") 242 | #message("Look in coverage sources: ${COVERAGE_SRCS}") 243 | message("\nFilter out unwanted GCOV files:") 244 | message("===============================") 245 | 246 | set(COVERAGE_SRCS_REMAINING ${COVERAGE_SRCS}) 247 | 248 | foreach (GCOV_FILE ${ALL_GCOV_FILES}) 249 | 250 | # 251 | # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov 252 | # -> 253 | # /path/to/project/root/subdir/the_file.c 254 | get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) 255 | 256 | # skip if full path is not present 257 | # can happen if files are generated for external libraries 258 | if (IS_ABSOLUTE ${GCOV_SRC_PATH}) 259 | file(RELATIVE_PATH GCOV_SRC_REL_PATH "${PROJECT_ROOT}" "${GCOV_SRC_PATH}") 260 | 261 | # Is this in the list of source files? 262 | # TODO: We want to match against relative path filenames from the source file root... 263 | list(FIND COVERAGE_SRCS ${GCOV_SRC_PATH} WAS_FOUND) 264 | 265 | if (NOT WAS_FOUND EQUAL -1) 266 | message("YES: ${GCOV_FILE}") 267 | list(APPEND GCOV_FILES ${GCOV_FILE}) 268 | 269 | # We remove it from the list, so we don't bother searching for it again. 270 | # Also files left in COVERAGE_SRCS_REMAINING after this loop ends should 271 | # have coverage data generated from them (no lines are covered). 272 | list(REMOVE_ITEM COVERAGE_SRCS_REMAINING ${GCOV_SRC_PATH}) 273 | else () 274 | message("NO: ${GCOV_FILE}") 275 | endif () 276 | endif () 277 | endforeach () 278 | 279 | # TODO: Enable setting these 280 | set(JSON_SERVICE_NAME "travis-ci") 281 | set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID}) 282 | set(JSON_REPO_TOKEN $ENV{COVERALLS_REPO_TOKEN}) 283 | 284 | set(JSON_TEMPLATE 285 | "{ 286 | \"repo_token\": \"\@JSON_REPO_TOKEN\@\", 287 | \"service_name\": \"\@JSON_SERVICE_NAME\@\", 288 | \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\", 289 | \"source_files\": \@JSON_GCOV_FILES\@, 290 | \"git\": \@JSON_REPO_DATA\@ 291 | }" 292 | ) 293 | message("======@@@@@@@@@@@@@==========") 294 | #message("${JSON_TEMPLATE}") 295 | 296 | set(SRC_FILE_TEMPLATE 297 | "{ 298 | \"name\": \"\@GCOV_SRC_REL_PATH\@\", 299 | \"source_digest\": \"\@GCOV_CONTENTS_MD5\@\", 300 | \"coverage\": \@GCOV_FILE_COVERAGE\@ 301 | }" 302 | ) 303 | 304 | message("\nGenerate JSON for files:") 305 | message("=========================") 306 | 307 | set(JSON_GCOV_FILES "[") 308 | 309 | # Read the GCOV files line by line and get the coverage data. 310 | foreach (GCOV_FILE ${GCOV_FILES}) 311 | 312 | get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) 313 | file(RELATIVE_PATH GCOV_SRC_REL_PATH "${PROJECT_ROOT}" "${GCOV_SRC_PATH}") 314 | 315 | # The new coveralls API doesn't need the entire source (Yay!) 316 | # However, still keeping that part for now. Will cleanup in the future. 317 | file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5) 318 | message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}") 319 | 320 | # Loads the gcov file as a list of lines. 321 | # (We first open the file and replace all occurences of [] with _ 322 | # because CMake will fail to parse a line containing unmatched brackets... 323 | # also the \ to escaped \n in macros screws up things.) 324 | # https://public.kitware.com/Bug/view.php?id=15369 325 | file(READ ${GCOV_FILE} GCOV_CONTENTS) 326 | string(REPLACE "[" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") 327 | string(REPLACE "]" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") 328 | string(REPLACE "\\" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") 329 | 330 | # Remove file contents to avoid encoding issues (cmake 2.8 has no ENCODING option) 331 | string(REGEX REPLACE "([^:]*):([^:]*):([^\n]*)\n" "\\1:\\2: \n" GCOV_CONTENTS "${GCOV_CONTENTS}") 332 | file(WRITE ${GCOV_FILE}_tmp "${GCOV_CONTENTS}") 333 | 334 | file(STRINGS ${GCOV_FILE}_tmp GCOV_LINES) 335 | list(LENGTH GCOV_LINES LINE_COUNT) 336 | 337 | # Instead of trying to parse the source from the 338 | # gcov file, simply read the file contents from the source file. 339 | # (Parsing it from the gcov is hard because C-code uses ; in many places 340 | # which also happens to be the same as the CMake list delimeter). 341 | file(READ ${GCOV_SRC_PATH} GCOV_FILE_SOURCE) 342 | 343 | string(REPLACE "\\" "\\\\" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 344 | string(REGEX REPLACE "\"" "\\\\\"" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 345 | string(REPLACE "\t" "\\\\t" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 346 | string(REPLACE "\r" "\\\\r" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 347 | string(REPLACE "\n" "\\\\n" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 348 | # According to http://json.org/ these should be escaped as well. 349 | # Don't know how to do that in CMake however... 350 | #string(REPLACE "\b" "\\\\b" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 351 | #string(REPLACE "\f" "\\\\f" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 352 | #string(REGEX REPLACE "\u([a-fA-F0-9]{4})" "\\\\u\\1" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") 353 | 354 | # We want a json array of coverage data as a single string 355 | # start building them from the contents of the .gcov 356 | set(GCOV_FILE_COVERAGE "[") 357 | 358 | set(GCOV_LINE_COUNT 1) # Line number for the .gcov. 359 | set(DO_SKIP 0) 360 | foreach (GCOV_LINE ${GCOV_LINES}) 361 | #message("${GCOV_LINE}") 362 | # Example of what we're parsing: 363 | # Hitcount |Line | Source 364 | # " 8: 26: if (!allowed || (strlen(allowed) == 0))" 365 | string(REGEX REPLACE 366 | "^([^:]*):([^:]*):(.*)$" 367 | "\\1;\\2;\\3" 368 | RES 369 | "${GCOV_LINE}") 370 | 371 | # Check if we should exclude lines using the Lcov syntax. 372 | string(REGEX MATCH "LCOV_EXCL_START" START_SKIP "${GCOV_LINE}") 373 | string(REGEX MATCH "LCOV_EXCL_END" END_SKIP "${GCOV_LINE}") 374 | string(REGEX MATCH "LCOV_EXCL_LINE" LINE_SKIP "${GCOV_LINE}") 375 | 376 | set(RESET_SKIP 0) 377 | if (LINE_SKIP AND NOT DO_SKIP) 378 | set(DO_SKIP 1) 379 | set(RESET_SKIP 1) 380 | endif () 381 | 382 | if (START_SKIP) 383 | set(DO_SKIP 1) 384 | message("${GCOV_LINE_COUNT}: Start skip") 385 | endif () 386 | 387 | if (END_SKIP) 388 | set(DO_SKIP 0) 389 | endif () 390 | 391 | list(LENGTH RES RES_COUNT) 392 | 393 | if (RES_COUNT GREATER 2) 394 | list(GET RES 0 HITCOUNT) 395 | list(GET RES 1 LINE) 396 | list(GET RES 2 SOURCE) 397 | 398 | string(STRIP ${HITCOUNT} HITCOUNT) 399 | string(STRIP ${LINE} LINE) 400 | 401 | # Lines with 0 line numbers are metadata and can be ignored. 402 | if (NOT ${LINE} EQUAL 0) 403 | 404 | if (DO_SKIP) 405 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") 406 | else () 407 | # Translate the hitcount into valid JSON values. 408 | if (${HITCOUNT} STREQUAL "#####" OR ${HITCOUNT} STREQUAL "=====") 409 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") 410 | elseif (${HITCOUNT} STREQUAL "-") 411 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") 412 | else () 413 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}${HITCOUNT}, ") 414 | endif () 415 | endif () 416 | endif () 417 | else () 418 | message(WARNING "Failed to properly parse line (RES_COUNT = ${RES_COUNT}) ${GCOV_FILE}:${GCOV_LINE_COUNT}\n-->${GCOV_LINE}") 419 | endif () 420 | 421 | if (RESET_SKIP) 422 | set(DO_SKIP 0) 423 | endif () 424 | math(EXPR GCOV_LINE_COUNT "${GCOV_LINE_COUNT}+1") 425 | endforeach () 426 | 427 | message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!") 428 | 429 | # Advanced way of removing the trailing comma in the JSON array. 430 | # "[1, 2, 3, " -> "[1, 2, 3" 431 | string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) 432 | 433 | # Append the trailing ] to complete the JSON array. 434 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") 435 | 436 | # Generate the final JSON for this file. 437 | message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...") 438 | string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) 439 | 440 | set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") 441 | endforeach () 442 | 443 | # Loop through all files we couldn't find any coverage for 444 | # as well, and generate JSON for those as well with 0% coverage. 445 | foreach (NOT_COVERED_SRC ${COVERAGE_SRCS_REMAINING}) 446 | 447 | # Set variables for json replacement 448 | set(GCOV_SRC_PATH ${NOT_COVERED_SRC}) 449 | file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5) 450 | file(RELATIVE_PATH GCOV_SRC_REL_PATH "${PROJECT_ROOT}" "${GCOV_SRC_PATH}") 451 | 452 | # Loads the source file as a list of lines. 453 | file(STRINGS ${NOT_COVERED_SRC} SRC_LINES) 454 | 455 | set(GCOV_FILE_COVERAGE "[") 456 | set(GCOV_FILE_SOURCE "") 457 | 458 | foreach (SOURCE ${SRC_LINES}) 459 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") 460 | 461 | string(REPLACE "\\" "\\\\" SOURCE "${SOURCE}") 462 | string(REGEX REPLACE "\"" "\\\\\"" SOURCE "${SOURCE}") 463 | string(REPLACE "\t" "\\\\t" SOURCE "${SOURCE}") 464 | string(REPLACE "\r" "\\\\r" SOURCE "${SOURCE}") 465 | set(GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}${SOURCE}\\n") 466 | endforeach () 467 | 468 | # Remove trailing comma, and complete JSON array with ] 469 | string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) 470 | set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") 471 | 472 | # Generate the final JSON for this file. 473 | message("Generate JSON for non-gcov file: ${NOT_COVERED_SRC}...") 474 | string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) 475 | set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") 476 | endforeach () 477 | 478 | # Get rid of trailing comma. 479 | string(REGEX REPLACE ",[ ]*$" "" JSON_GCOV_FILES ${JSON_GCOV_FILES}) 480 | set(JSON_GCOV_FILES "${JSON_GCOV_FILES}]") 481 | 482 | # Generate the final complete JSON! 483 | message("Generate final JSON...") 484 | string(CONFIGURE ${JSON_TEMPLATE} JSON) 485 | 486 | file(WRITE "${COVERALLS_OUTPUT_FILE}" "${JSON}") 487 | message("###########################################################################") 488 | message("Generated coveralls JSON containing coverage data:") 489 | #message("${COVERALLS_OUTPUT_FILE}") 490 | message("###########################################################################") 491 | 492 | #message("content: ${JSON}") -------------------------------------------------------------------------------- /cmake/Modules/Utils.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################################ 2 | # Command alias for debugging messages 3 | # Usage: 4 | # dmsg() 5 | function(dmsg) 6 | message(STATUS ${ARGN}) 7 | endfunction() 8 | 9 | ################################################################################################ 10 | # Removes duplicates from list(s) 11 | # Usage: 12 | # peloton_list_unique( [] [...]) 13 | macro(peloton_list_unique) 14 | foreach(__lst ${ARGN}) 15 | if(${__lst}) 16 | list(REMOVE_DUPLICATES ${__lst}) 17 | endif() 18 | endforeach() 19 | endmacro() 20 | 21 | ################################################################################################ 22 | # Clears variables from list 23 | # Usage: 24 | # peloton_clear_vars() 25 | macro(peloton_clear_vars) 26 | foreach(_var ${ARGN}) 27 | unset(${_var}) 28 | endforeach() 29 | endmacro() 30 | 31 | ################################################################################################ 32 | # Removes duplicates from string 33 | # Usage: 34 | # peloton_string_unique() 35 | function(peloton_string_unique __string) 36 | if(${__string}) 37 | set(__list ${${__string}}) 38 | separate_arguments(__list) 39 | list(REMOVE_DUPLICATES __list) 40 | foreach(__e ${__list}) 41 | set(__str "${__str} ${__e}") 42 | endforeach() 43 | set(${__string} ${__str} PARENT_SCOPE) 44 | endif() 45 | endfunction() 46 | 47 | ################################################################################################ 48 | # Prints list element per line 49 | # Usage: 50 | # peloton_print_list() 51 | function(peloton_print_list) 52 | foreach(e ${ARGN}) 53 | message(STATUS ${e}) 54 | endforeach() 55 | endfunction() 56 | 57 | ################################################################################################ 58 | # Function merging lists of compiler flags to single string. 59 | # Usage: 60 | # peloton_merge_flag_lists(out_variable [] [] ...) 61 | function(peloton_merge_flag_lists out_var) 62 | set(__result "") 63 | foreach(__list ${ARGN}) 64 | foreach(__flag ${${__list}}) 65 | string(STRIP ${__flag} __flag) 66 | set(__result "${__result} ${__flag}") 67 | endforeach() 68 | endforeach() 69 | string(STRIP ${__result} __result) 70 | set(${out_var} ${__result} PARENT_SCOPE) 71 | endfunction() 72 | 73 | ################################################################################################ 74 | # Converts all paths in list to absolute 75 | # Usage: 76 | # peloton_convert_absolute_paths() 77 | function(peloton_convert_absolute_paths variable) 78 | set(__dlist "") 79 | foreach(__s ${${variable}}) 80 | get_filename_component(__abspath ${__s} ABSOLUTE) 81 | list(APPEND __list ${__abspath}) 82 | endforeach() 83 | set(${variable} ${__list} PARENT_SCOPE) 84 | endfunction() 85 | 86 | ################################################################################################ 87 | # Reads set of version defines from the header file 88 | # Usage: 89 | # peloton_parse_header( ..) 90 | macro(peloton_parse_header FILENAME FILE_VAR) 91 | set(vars_regex "") 92 | set(__parnet_scope OFF) 93 | set(__add_cache OFF) 94 | foreach(name ${ARGN}) 95 | if("${name}" STREQUAL "PARENT_SCOPE") 96 | set(__parnet_scope ON) 97 | elseif("${name}" STREQUAL "CACHE") 98 | set(__add_cache ON) 99 | elseif(vars_regex) 100 | set(vars_regex "${vars_regex}|${name}") 101 | else() 102 | set(vars_regex "${name}") 103 | endif() 104 | endforeach() 105 | if(EXISTS "${FILENAME}") 106 | file(STRINGS "${FILENAME}" ${FILE_VAR} REGEX "#define[ \t]+(${vars_regex})[ \t]+[0-9]+" ) 107 | else() 108 | unset(${FILE_VAR}) 109 | endif() 110 | foreach(name ${ARGN}) 111 | if(NOT "${name}" STREQUAL "PARENT_SCOPE" AND NOT "${name}" STREQUAL "CACHE") 112 | if(${FILE_VAR}) 113 | if(${FILE_VAR} MATCHES ".+[ \t]${name}[ \t]+([0-9]+).*") 114 | string(REGEX REPLACE ".+[ \t]${name}[ \t]+([0-9]+).*" "\\1" ${name} "${${FILE_VAR}}") 115 | else() 116 | set(${name} "") 117 | endif() 118 | if(__add_cache) 119 | set(${name} ${${name}} CACHE INTERNAL "${name} parsed from ${FILENAME}" FORCE) 120 | elseif(__parnet_scope) 121 | set(${name} "${${name}}" PARENT_SCOPE) 122 | endif() 123 | else() 124 | unset(${name} CACHE) 125 | endif() 126 | endif() 127 | endforeach() 128 | endmacro() 129 | 130 | ################################################################################################ 131 | # Reads single version define from the header file and parses it 132 | # Usage: 133 | # peloton_parse_header_single_define( ) 134 | function(peloton_parse_header_single_define LIBNAME HDR_PATH VARNAME) 135 | set(${LIBNAME}_H "") 136 | if(EXISTS "${HDR_PATH}") 137 | file(STRINGS "${HDR_PATH}" ${LIBNAME}_H REGEX "^#define[ \t]+${VARNAME}[ \t]+\"[^\"]*\".*$" LIMIT_COUNT 1) 138 | endif() 139 | 140 | if(${LIBNAME}_H) 141 | string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_MAJOR "${${LIBNAME}_H}") 142 | string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_MINOR "${${LIBNAME}_H}") 143 | string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.[0-9]+\\.([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_PATCH "${${LIBNAME}_H}") 144 | set(${LIBNAME}_VERSION_MAJOR ${${LIBNAME}_VERSION_MAJOR} ${ARGN} PARENT_SCOPE) 145 | set(${LIBNAME}_VERSION_MINOR ${${LIBNAME}_VERSION_MINOR} ${ARGN} PARENT_SCOPE) 146 | set(${LIBNAME}_VERSION_PATCH ${${LIBNAME}_VERSION_PATCH} ${ARGN} PARENT_SCOPE) 147 | set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}.${${LIBNAME}_VERSION_PATCH}" PARENT_SCOPE) 148 | 149 | # append a TWEAK version if it exists: 150 | set(${LIBNAME}_VERSION_TWEAK "") 151 | if("${${LIBNAME}_H}" MATCHES "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.[0-9]+\\.[0-9]+\\.([0-9]+).*$") 152 | set(${LIBNAME}_VERSION_TWEAK "${CMAKE_MATCH_1}" ${ARGN} PARENT_SCOPE) 153 | endif() 154 | if(${LIBNAME}_VERSION_TWEAK) 155 | set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_STRING}.${${LIBNAME}_VERSION_TWEAK}" ${ARGN} PARENT_SCOPE) 156 | else() 157 | set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_STRING}" ${ARGN} PARENT_SCOPE) 158 | endif() 159 | endif() 160 | endfunction() 161 | 162 | ######################################################################################################## 163 | # An option that the user can select. Can accept condition to control when option is available for user. 164 | # Usage: 165 | # peloton_option( "doc string" [IF ]) 166 | function(peloton_option variable description value) 167 | set(__value ${value}) 168 | set(__condition "") 169 | set(__varname "__value") 170 | foreach(arg ${ARGN}) 171 | if(arg STREQUAL "IF" OR arg STREQUAL "if") 172 | set(__varname "__condition") 173 | else() 174 | list(APPEND ${__varname} ${arg}) 175 | endif() 176 | endforeach() 177 | unset(__varname) 178 | if("${__condition}" STREQUAL "") 179 | set(__condition 2 GREATER 1) 180 | endif() 181 | 182 | if(${__condition}) 183 | if("${__value}" MATCHES ";") 184 | if(${__value}) 185 | option(${variable} "${description}" ON) 186 | else() 187 | option(${variable} "${description}" OFF) 188 | endif() 189 | elseif(DEFINED ${__value}) 190 | if(${__value}) 191 | option(${variable} "${description}" ON) 192 | else() 193 | option(${variable} "${description}" OFF) 194 | endif() 195 | else() 196 | option(${variable} "${description}" ${__value}) 197 | endif() 198 | else() 199 | unset(${variable} CACHE) 200 | endif() 201 | endfunction() 202 | 203 | ################################################################################################ 204 | # Utility macro for comparing two lists. Used for CMake debugging purposes 205 | # Usage: 206 | # peloton_compare_lists( [description]) 207 | function(peloton_compare_lists list1 list2 desc) 208 | set(__list1 ${${list1}}) 209 | set(__list2 ${${list2}}) 210 | list(SORT __list1) 211 | list(SORT __list2) 212 | list(LENGTH __list1 __len1) 213 | list(LENGTH __list2 __len2) 214 | 215 | if(NOT ${__len1} EQUAL ${__len2}) 216 | message(FATAL_ERROR "Lists are not equal. ${__len1} != ${__len2}. ${desc}") 217 | endif() 218 | 219 | foreach(__i RANGE 1 ${__len1}) 220 | math(EXPR __index "${__i}- 1") 221 | list(GET __list1 ${__index} __item1) 222 | list(GET __list2 ${__index} __item2) 223 | if(NOT ${__item1} STREQUAL ${__item2}) 224 | message(FATAL_ERROR "Lists are not equal. Differ at element ${__index}. ${desc}") 225 | endif() 226 | endforeach() 227 | endfunction() 228 | 229 | ################################################################################################ 230 | # Command for disabling warnings for different platforms (see below for gcc and VisualStudio) 231 | # Usage: 232 | # peloton_warnings_disable( -Wshadow /wd4996 ..,) 233 | macro(peloton_warnings_disable) 234 | set(_flag_vars "") 235 | set(_msvc_warnings "") 236 | set(_gxx_warnings "") 237 | 238 | foreach(arg ${ARGN}) 239 | if(arg MATCHES "^CMAKE_") 240 | list(APPEND _flag_vars ${arg}) 241 | elseif(arg MATCHES "^/wd") 242 | list(APPEND _msvc_warnings ${arg}) 243 | elseif(arg MATCHES "^-W") 244 | list(APPEND _gxx_warnings ${arg}) 245 | endif() 246 | endforeach() 247 | 248 | if(NOT _flag_vars) 249 | set(_flag_vars CMAKE_C_FLAGS CMAKE_CXX_FLAGS) 250 | endif() 251 | 252 | if(MSVC AND _msvc_warnings) 253 | foreach(var ${_flag_vars}) 254 | foreach(warning ${_msvc_warnings}) 255 | set(${var} "${${var}} ${warning}") 256 | endforeach() 257 | endforeach() 258 | elseif((CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX) AND _gxx_warnings) 259 | foreach(var ${_flag_vars}) 260 | foreach(warning ${_gxx_warnings}) 261 | if(NOT warning MATCHES "^-Wno-") 262 | string(REPLACE "${warning}" "" ${var} "${${var}}") 263 | string(REPLACE "-W" "-Wno-" warning "${warning}") 264 | endif() 265 | set(${var} "${${var}} ${warning}") 266 | endforeach() 267 | endforeach() 268 | endif() 269 | peloton_clear_vars(_flag_vars _msvc_warnings _gxx_warnings) 270 | endmacro() 271 | 272 | ################################################################################################ 273 | # Helper function get current definitions 274 | # Usage: 275 | # peloton_get_current_definitions() 276 | function(peloton_get_current_definitions definitions_var) 277 | get_property(current_definitions DIRECTORY PROPERTY COMPILE_DEFINITIONS) 278 | set(result "") 279 | 280 | foreach(d ${current_definitions}) 281 | list(APPEND result -D${d}) 282 | endforeach() 283 | 284 | peloton_list_unique(result) 285 | set(${definitions_var} ${result} PARENT_SCOPE) 286 | endfunction() 287 | 288 | ################################################################################################ 289 | # Helper function get current includes/definitions 290 | # Usage: 291 | # peloton_get_current_cflags() 292 | function(peloton_get_current_cflags cflags_var) 293 | get_property(current_includes DIRECTORY PROPERTY INCLUDE_DIRECTORIES) 294 | peloton_convert_absolute_paths(current_includes) 295 | peloton_get_current_definitions(cflags) 296 | 297 | foreach(i ${current_includes}) 298 | list(APPEND cflags "-I${i}") 299 | endforeach() 300 | 301 | peloton_list_unique(cflags) 302 | set(${cflags_var} ${cflags} PARENT_SCOPE) 303 | endfunction() 304 | 305 | ################################################################################################ 306 | # Helper function to parse current linker libs into link directories, libflags and osx frameworks 307 | # Usage: 308 | # peloton_parse_linker_libs( ) 309 | function(peloton_parse_linker_libs Peloton_LINKER_LIBS_variable folders_var flags_var frameworks_var) 310 | 311 | set(__unspec "") 312 | set(__debug "") 313 | set(__optimized "") 314 | set(__framework "") 315 | set(__varname "__unspec") 316 | 317 | # split libs into debug, optimized, unspecified and frameworks 318 | foreach(list_elem ${${Peloton_LINKER_LIBS_variable}}) 319 | if(list_elem STREQUAL "debug") 320 | set(__varname "__debug") 321 | elseif(list_elem STREQUAL "optimized") 322 | set(__varname "__optimized") 323 | elseif(list_elem MATCHES "^-framework[ \t]+([^ \t].*)") 324 | list(APPEND __framework -framework ${CMAKE_MATCH_1}) 325 | else() 326 | list(APPEND ${__varname} ${list_elem}) 327 | set(__varname "__unspec") 328 | endif() 329 | endforeach() 330 | 331 | # attach debug or optimized libs to unspecified according to current configuration 332 | if(CMAKE_BUILD_TYPE MATCHES "Debug") 333 | set(__libs ${__unspec} ${__debug}) 334 | else() 335 | set(__libs ${__unspec} ${__optimized}) 336 | endif() 337 | 338 | set(libflags "") 339 | set(folders "") 340 | 341 | # convert linker libraries list to link flags 342 | foreach(lib ${__libs}) 343 | if(TARGET ${lib}) 344 | list(APPEND folders $) 345 | list(APPEND libflags -l${lib}) 346 | elseif(lib MATCHES "^-l.*") 347 | list(APPEND libflags ${lib}) 348 | elseif(IS_ABSOLUTE ${lib}) 349 | get_filename_component(folder ${lib} PATH) 350 | get_filename_component(filename ${lib} NAME) 351 | string(REGEX REPLACE "\\.[^.]*$" "" filename_without_shortest_ext ${filename}) 352 | 353 | string(REGEX MATCH "^lib(.*)" __match ${filename_without_shortest_ext}) 354 | list(APPEND libflags -l${CMAKE_MATCH_1}) 355 | list(APPEND folders ${folder}) 356 | else() 357 | message(FATAL_ERROR "Logic error. Need to update cmake script") 358 | endif() 359 | endforeach() 360 | 361 | peloton_list_unique(libflags folders) 362 | 363 | set(${folders_var} ${folders} PARENT_SCOPE) 364 | set(${flags_var} ${libflags} PARENT_SCOPE) 365 | set(${frameworks_var} ${__framework} PARENT_SCOPE) 366 | endfunction() 367 | 368 | ################################################################################################ 369 | # Helper function to detect Darwin version, i.e. 10.8, 10.9, 10.10, .... 370 | # Usage: 371 | # peloton_detect_darwin_version() 372 | function(peloton_detect_darwin_version output_var) 373 | if(APPLE) 374 | execute_process(COMMAND /usr/bin/sw_vers -productVersion 375 | RESULT_VARIABLE __sw_vers OUTPUT_VARIABLE __sw_vers_out 376 | ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) 377 | 378 | set(${output_var} ${__sw_vers_out} PARENT_SCOPE) 379 | else() 380 | set(${output_var} "" PARENT_SCOPE) 381 | endif() 382 | endfunction() 383 | -------------------------------------------------------------------------------- /include/bucket.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_BUCKET_H 6 | #define BOLTDB_IN_CPP_BUCKET_H 7 | 8 | #include 9 | #include "bucket_header.h" 10 | 11 | namespace boltDB_CPP { 12 | const uint32_t MAXKEYSIZE = 32768; 13 | const uint32_t MAXVALUESIZE = (1L << 31) - 2; 14 | const double MINFILLPERCENT = 0.1; 15 | const double MAXFILLPERCENT = 1.0; 16 | const double DEFAULTFILLPERCENT = 0.5; 17 | 18 | class Page; 19 | class Node; 20 | class Cursor; 21 | class Txn; 22 | class Bucket { 23 | Txn *tx = nullptr; 24 | Page *page = nullptr; // useful for inline buckets, page points to beginning 25 | // of the serialized value i.e. a page' header 26 | Node *rootNode = nullptr; 27 | BucketHeader bucketHeader; 28 | std::unordered_map 29 | buckets; // subbucket cache. used if txn is writable. k:bucket name 30 | std::unordered_map 31 | nodes; // node cache. used if txn is writable 32 | double fillpercent = 0.5; 33 | 34 | friend class Txn; 35 | 36 | public: 37 | explicit Bucket(Txn *tx_p) : tx(tx_p) {} 38 | void setBucketHeader(BucketHeader &bh) { bucketHeader = bh; } 39 | void setTxn(Txn *txn) { tx = txn; } 40 | Txn *getTxn() const { return tx; } 41 | page_id getRootPage() const { return bucketHeader.rootPageId; } 42 | size_t getTotalPageNumber() const; 43 | MemoryPool &getPool(); 44 | Node *getCachedNode(page_id pageId) { 45 | auto iter = nodes.find(pageId); 46 | if (iter != nodes.end()) { 47 | return iter->second; 48 | } 49 | return nullptr; 50 | } 51 | void eraseCachedNode(page_id pid) { nodes.erase(pid); } 52 | 53 | bool isWritable() const; 54 | double getFillPercent() const { return fillpercent; } 55 | Cursor *createCursor(); 56 | Bucket *getBucketByName(const Item &searchKey); 57 | Bucket *openBucket(const Item &value); 58 | Bucket *createBucket(const Item &key); 59 | Bucket *createBucketIfNotExists(const Item &key); 60 | int deleteBucket(const Item &key); 61 | void getPageNode(page_id pageId, Node *&node, Page *&page); 62 | Node *getNode(page_id pageId, Node *parent); 63 | Item write(); // serialize bucket into a byte array 64 | int for_each(std::function); 65 | void for_each_page(std::function); 66 | void for_each_page_node(std::function); 67 | void for_each_page_node_impl(page_id page, int depth, 68 | std::function); 69 | void free(); 70 | void dereference(); 71 | void rebalance(); 72 | char *cloneBytes(const Item &key, size_t *retSz = nullptr); 73 | Item get(const Item &key); 74 | int put(const Item &key, const Item &value); 75 | int remove(const Item &key); 76 | uint64_t sequence(); 77 | int setSequence(uint64_t v); 78 | int nextSequence(uint64_t &v); 79 | size_t maxInlineBucketSize(); 80 | bool isInlineable(); 81 | int spill(); // write dirty pages 82 | void reset(); 83 | static Bucket *newBucket(Txn *tx); 84 | bool isInlineBucket() const { 85 | return bucketHeader.rootPageId == 0; 86 | } 87 | }; 88 | 89 | const uint32_t BUCKETHEADERSIZE = sizeof(boltDB_CPP::BucketHeader); 90 | } // namespace boltDB_CPP 91 | #endif // BOLTDB_IN_CPP_BUCKET_H 92 | -------------------------------------------------------------------------------- /include/bucket_header.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-3. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_BUCKETHEADER_H 6 | #define BOLTDB_IN_CPP_BUCKETHEADER_H 7 | #include "types.h" 8 | namespace boltDB_CPP { 9 | struct BucketHeader { 10 | page_id rootPageId = 0; 11 | uint64_t sequence = 0; 12 | void reset() { 13 | rootPageId = 0; 14 | sequence = 0; 15 | } 16 | }; 17 | } // namespace boltDB_CPP 18 | #endif // BOLTDB_IN_CPP_BUCKETHEADER_H 19 | -------------------------------------------------------------------------------- /include/cursor.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-27. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_CURSOR_H 6 | #define BOLTDB_IN_CPP_CURSOR_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include "types.h" 12 | namespace boltDB_CPP { 13 | class Page; 14 | class Node; 15 | class Bucket; 16 | // reference to an element on a given page/node 17 | struct ElementRef { 18 | Page *page = nullptr; 19 | Node *node = nullptr; 20 | uint64_t index = 0; // DO NOT change this default ctor build up a ref to the 21 | // first element in a page 22 | // is this a leaf page/node 23 | bool isLeaf() const; 24 | 25 | // return the number of inodes or page elements 26 | size_t count() const; 27 | 28 | ElementRef(Page *page_p, Node *node_p) : page(page_p), node(node_p) {} 29 | }; 30 | 31 | struct Cursor { 32 | Bucket *bucket = nullptr; 33 | std::deque dq; 34 | 35 | Cursor() = delete; 36 | explicit Cursor(Bucket *bucket1) : bucket(bucket1) {} 37 | 38 | Bucket *getBucket() const { return bucket; } 39 | void search(const Item &key, page_id pageId); 40 | // search leaf node (which is on the top of the stack) for a Key 41 | void searchLeaf(const Item &key); 42 | void searchBranchNode(const Item &key, Node *node); 43 | void searchBranchPage(const Item &key, Page *page); 44 | void keyValue(Item &key, Item &value, uint32_t &flag); 45 | 46 | // weird function signature 47 | // return kv of the search Key if searchkey exists 48 | // or return the next Key 49 | void do_seek(Item searchKey, Item &key, Item &value, uint32_t &flag); 50 | void seek(const Item &searchKey, Item &key, Item &value, uint32_t &flag); 51 | 52 | // return the node the cursor is currently on 53 | Node *getNode() const; 54 | 55 | void do_next(Item &key, Item &value, uint32_t &flag); 56 | 57 | void do_first(); 58 | void do_last(); 59 | int remove(); 60 | void prev(Item &key, Item &value); 61 | void next(Item &key, Item &value); 62 | void last(Item &key, Item &value); 63 | void first(Item &key, Item &value); 64 | }; 65 | 66 | } // namespace boltDB_CPP 67 | 68 | #endif // BOLTDB_IN_CPP_CURSOR_H 69 | -------------------------------------------------------------------------------- /include/db.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_DATABASE_H 6 | #define BOLTDB_IN_CPP_DATABASE_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "bucket_header.h" 18 | #include "rwlock.h" 19 | #include "types.h" 20 | #include "txn.h" 21 | 22 | namespace boltDB_CPP { 23 | 24 | /** 25 | * constant definations 26 | */ 27 | 28 | // for ease of debug, use constants of x86 on x86_64 machine 29 | 30 | const uint64_t MAXMAPSIZE = 0x7FFFFFFF; // 2GB on x86 31 | // const uint64_t MAXMAPSIZE = 0xFFFFFFFFFFFF; // 256TB on x86_64 32 | const uint64_t MAXALLOCSIZE = 0xFFFFFFF; // used on x86 33 | // const uint64_t MAXALLOCSIZE = 0x7FFFFFFF; // on x86_64 used when creating 34 | // array pointers x86/x86_64 will not break on unaligned load/store 35 | 36 | const uint64_t MAXMMAPSTEP = 1 << 30; // 1GB used when remapping the mmap 37 | const uint64_t VERSION = 2; // v1.3.1 38 | const uint32_t MAGIC = 0xED0CDAED; 39 | 40 | // default configuration values 41 | 42 | const int DEFAULTMAXBATCHSIZE = 100; 43 | const int DEFAULTMAXBATCHDELAYMILLIIONSEC = 10; 44 | const int DEFAULTALLOCATIONSIZE = 16 * 1024 * 1024; 45 | const int DEFAULTPAGESIZE = 4096; // this value is returned by `getconf 46 | // PAGE_SIZE` on ubuntu 17.10 x86_64 47 | 48 | /** 49 | * forward declaration 50 | */ 51 | struct Page; 52 | struct DB; 53 | struct TxStat; 54 | struct Txn; 55 | struct Meta; 56 | 57 | struct FreeList { 58 | std::vector pageIds; // free & available 59 | std::map> pending; // soon to be free 60 | std::map cache; // all free & pending page_ids 61 | 62 | void free(txn_id tid, Page *page); 63 | size_t size() const; // size in bytes after serialization 64 | size_t count() const; 65 | size_t free_count() const; 66 | size_t pending_count() const; 67 | void copyall(std::vector &dest); 68 | page_id allocate(size_t sz); 69 | void release(txn_id tid); 70 | void rollback(txn_id tid); 71 | bool freed(page_id pageId); 72 | void read(Page *page); 73 | int write(Page *page); 74 | void reindex(); 75 | void reload(Page *page); 76 | void reset(); 77 | }; 78 | 79 | struct Stat { 80 | // about free list 81 | uint64_t freePageNumber = 0; 82 | uint64_t pendingPageNumber = 0; 83 | // in byte 84 | uint64_t freeAlloc = 0; 85 | // in byte 86 | uint64_t freeListInUse = 0; 87 | 88 | // txn stat 89 | uint64_t txnNumber = 0; 90 | uint64_t opened_txnNumber = 0; 91 | TxStat *txStat; 92 | }; 93 | 94 | struct Options { 95 | uint32_t timeOut = 0; // currently not supporting this parameter 96 | bool noGrowSync = false; 97 | bool readOnly = false; 98 | uint32_t mmapFlag = 0; 99 | size_t initalMmapSize = 0; 100 | }; 101 | const Options DEFAULTOPTION{}; 102 | 103 | class Batch { 104 | DB *database; 105 | // timer 106 | // call once 107 | // a function list 108 | }; 109 | 110 | class DB { 111 | static uint32_t pageSize; 112 | bool strictMode = false; 113 | bool noSync = false; 114 | bool noGrowSync = false; 115 | uint32_t mmapFlags = 0; 116 | uint32_t maxBatchSize = 0; 117 | uint32_t maxBatchDelayMillionSeconds = 0; 118 | uint32_t allocSize = 0; 119 | off_t fileSize = 0; 120 | std::string path; 121 | int fd = -1; 122 | void *dataref = nullptr; // readonly . this is mmap data 123 | char *data = nullptr; // data is a pointer to block of memory if sizeof 124 | // MAXMAPSIZE 125 | uint64_t dataSize = 0; 126 | Meta *meta0 = nullptr; 127 | Meta *meta1 = nullptr; 128 | bool opened = false; 129 | std::unique_ptr rwtx; 130 | std::vector> txs; 131 | FreeList freeList; 132 | Stat stat; 133 | 134 | std::mutex batchMtx; 135 | Batch *batch = nullptr; 136 | 137 | std::mutex readWriteAccessMutex; // this is writer's mutex. writers must 138 | // acquire this lock before proceeding. 139 | std::mutex metaLock; // this protects the database object. 140 | RWLock mmapLock; 141 | RWLock statLock; 142 | 143 | bool readOnly = false; 144 | 145 | int munmap_db_file(); 146 | void closeTx(Txn *txn); 147 | public: 148 | const std::function writeAt = 149 | [this](char *buf, size_t len, off_t offset) { 150 | auto ret = ::pwrite(fd, buf, len, offset); 151 | if (ret == -1) { 152 | perror("pwrite"); 153 | } 154 | return ret; 155 | }; 156 | bool isNoSync() const { return noSync; } 157 | void writerEnter() { readWriteAccessMutex.lock(); } 158 | void writerLeave() { readWriteAccessMutex.unlock(); } 159 | void resetRWTX(); 160 | int getFd() const { return this->fd; } 161 | size_t freeListSerialSize() const { return freeList.size(); } 162 | void resetData(); 163 | void resetData(void *data_p, void *dataref_p, size_t datasz_p); 164 | bool hasMappingData() const { return dataref != nullptr; } 165 | Page *pagePointer(page_id pageId); 166 | FreeList &getFreeLIst(); 167 | static uint64_t getPageSize(); 168 | Page *allocate(size_t count, Txn *txn); 169 | Meta *meta(); 170 | void removeTxn(Txn *txn); 171 | int grow(size_t sz); 172 | int init(); 173 | Page *pageInBuffer(char *ptr, size_t length, page_id pageId); 174 | DB *openDB(const std::string &path, uint16_t mode, 175 | const Options &options = DEFAULTOPTION); 176 | void closeDB(); 177 | void do_closeDB(); 178 | int initMeta(off_t minMmapSize); 179 | int mmapSize(off_t &targetSize); // targetSize is a hint. calculate the mmap 180 | // size based on input param 181 | int update(std::function fn); 182 | int view(std::function fn); 183 | Txn *beginRWTx(); 184 | Txn *beginTx(); 185 | int commitTxn(Txn *txn); 186 | void rollbackTxn(Txn *txn); 187 | int mmap_db_file(DB *database, size_t sz); 188 | }; 189 | 190 | struct BranchPageElement { 191 | size_t pos = 0; 192 | size_t ksize = 0; 193 | page_id pageId = 0; 194 | 195 | Item Key() const { 196 | auto ptr = reinterpret_cast(this); 197 | // return std::string(&ptr[pos], &ptr[pos + ksize]); 198 | return {&ptr[pos], ksize}; 199 | } 200 | }; 201 | 202 | struct LeafPageElement { 203 | uint32_t flag = 0; // is this element a bucket? yes:1 no:0 204 | size_t pos = 0; 205 | size_t ksize = 0; 206 | size_t vsize = 0; 207 | 208 | Item read(uint32_t p, uint32_t s) const { 209 | const auto *ptr = reinterpret_cast(this); 210 | // return std::string(&ptr[p], &ptr[p + s]); 211 | return {&ptr[p], s}; 212 | } 213 | 214 | Item Key() const { return read(pos, ksize); } 215 | 216 | Item Value() const { return read(pos + ksize, vsize); } 217 | }; 218 | 219 | struct Page { 220 | friend class ElementRef; 221 | page_id pageId = 0; 222 | uint16_t flag = 0; 223 | uint16_t count = 0; 224 | uint32_t overflow = 0; 225 | char *ptr = nullptr; 226 | 227 | LeafPageElement *getLeafPageElement(uint64_t index) const { 228 | //it's not about ptr, but the memory address of ptr 229 | // assert(ptr); 230 | const auto *list = reinterpret_cast(&ptr); 231 | return const_cast(&list[index]); 232 | } 233 | 234 | BranchPageElement *getBranchPageElement(uint64_t index) const { 235 | // assert(ptr); 236 | const auto *list = reinterpret_cast(&ptr); 237 | return const_cast(&list[index]); 238 | } 239 | 240 | uint16_t getFlag() const { return flag; } 241 | void setFlag(uint16_t flags) { Page::flag = flags; } 242 | uint16_t getCount() const { return count; } 243 | 244 | Meta *metaPointer() { return reinterpret_cast(&ptr); } 245 | }; 246 | 247 | // transverse all kv pairs in a bucket in sorted order 248 | // valid only if related txn is valid 249 | enum class PageFlag : uint16_t { 250 | branchPageFlag = 0x01, 251 | leafPageFlag = 0x02, 252 | bucketLeafFlag = 0x03, 253 | metaPageFlag = 0x04, 254 | freelistPageFlag = 0x10, 255 | }; 256 | static inline uint16_t pageFlagValue(PageFlag pf) { 257 | return static_cast(pf); 258 | } 259 | 260 | static inline bool isSet(uint32_t flag, PageFlag flag1) { 261 | return static_cast(flag & static_cast(flag1)); 262 | } 263 | 264 | static inline bool isBucketLeaf(uint32_t flag) { 265 | return isSet(flag, PageFlag::bucketLeafFlag); 266 | } 267 | 268 | const size_t PAGEHEADERSIZE = offsetof(Page, ptr); 269 | const size_t MINKEYSPERPAGE = 2; 270 | const size_t BRANCHPAGEELEMENTSIZE = sizeof(BranchPageElement); 271 | const size_t LEAFPAGEELEMENTSIZE = sizeof(LeafPageElement); 272 | 273 | } // namespace boltDB_CPP 274 | #endif // BOLTDB_IN_CPP_DATABASE_H 275 | -------------------------------------------------------------------------------- /include/fnv/fnv.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-9. 3 | // 4 | /* 5 | * fnv - Fowler/Noll/Vo- hash code 6 | * 7 | * @(#) $Revision: 5.4 $ 8 | * @(#) $Id: fnv.h,v 5.4 2009/07/30 22:49:13 chongo Exp $ 9 | * @(#) $Source: /usr/local/src/cmd/fnv/RCS/fnv.h,v $ 10 | * 11 | *** 12 | * 13 | * Fowler/Noll/Vo- hash 14 | * 15 | * The basis of this hash algorithm was taken from an idea sent 16 | * as reviewer comments to the IEEE POSIX P1003.2 committee by: 17 | * 18 | * Phong Vo (http://www.research.att.com/info/kpv/) 19 | * Glenn Fowler (http://www.research.att.com/~gsf/) 20 | * 21 | * In a subsequent ballot round: 22 | * 23 | * Landon Curt Noll (http://www.isthe.com/chongo/) 24 | * 25 | * improved on their algorithm. Some people tried this hash 26 | * and found that it worked rather well. In an EMail message 27 | * to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash. 28 | * 29 | * FNV hashes are designed to be fast while maintaining a low 30 | * collision rate. The FNV speed allows one to quickly hash lots 31 | * of data while maintaining a reasonable collision rate. See: 32 | * 33 | * http://www.isthe.com/chongo/tech/comp/fnv/index.html 34 | * 35 | * for more details as well as other forms of the FNV hash. 36 | * 37 | *** 38 | * 39 | * NOTE: The FNV-0 historic hash is not recommended. One should use 40 | * the FNV-1 hash instead. 41 | * 42 | * To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the 43 | * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). 44 | * 45 | * To use the 64 bit FNV-0 historic hash, pass FNV0_64_INIT as the 46 | * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). 47 | * 48 | * To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the 49 | * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). 50 | * 51 | * To use the recommended 64 bit FNV-1 hash, pass FNV1_64_INIT as the 52 | * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). 53 | * 54 | * To use the recommended 32 bit FNV-1a hash, pass FNV1_32A_INIT as the 55 | * Fnv32_t hashval argument to fnv_32a_buf() or fnv_32a_str(). 56 | * 57 | * To use the recommended 64 bit FNV-1a hash, pass FNV1A_64_INIT as the 58 | * Fnv64_t hashval argument to fnv_64a_buf() or fnv_64a_str(). 59 | * 60 | *** 61 | * 62 | * Please do not copyright this code. This code is in the public domain. 63 | * 64 | * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 65 | * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO 66 | * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR 67 | * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF 68 | * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 69 | * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 70 | * PERFORMANCE OF THIS SOFTWARE. 71 | * 72 | * By: 73 | * chongo /\oo/\ 74 | * http://www.isthe.com/chongo/ 75 | * 76 | * Share and Enjoy! :-) 77 | */ 78 | 79 | #if !defined(__FNV_H__) 80 | #define __FNV_H__ 81 | 82 | #include 83 | 84 | #define FNV_VERSION "5.0.2" /* @(#) FNV Version */ 85 | 86 | 87 | /* 88 | * 32 bit FNV-0 hash type 89 | */ 90 | typedef u_int32_t Fnv32_t; 91 | 92 | 93 | /* 94 | * 32 bit FNV-0 zero initial basis 95 | * 96 | * This historic hash is not recommended. One should use 97 | * the FNV-1 hash and initial basis instead. 98 | */ 99 | #define FNV0_32_INIT ((Fnv32_t)0) 100 | 101 | 102 | /* 103 | * 32 bit FNV-1 and FNV-1a non-zero initial basis 104 | * 105 | * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: 106 | * 107 | * chongo /\../\ 108 | * 109 | * NOTE: The \'s above are not back-slashing escape characters. 110 | * They are literal ASCII backslash 0x5c characters. 111 | * 112 | * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. 113 | */ 114 | #define FNV1_32_INIT ((Fnv32_t)0x811c9dc5) 115 | #define FNV1_32A_INIT FNV1_32_INIT 116 | 117 | 118 | /* 119 | * determine how 64 bit unsigned values are represented 120 | */ 121 | /* 122 | * DO NOT EDIT -- generated by the Makefile 123 | */ 124 | 125 | 126 | 127 | /* do we have/want to use a long long type? */ 128 | #define HAVE_64BIT_LONG_LONG /* yes */ 129 | 130 | /* 131 | * NO64BIT_LONG_LONG undef HAVE_64BIT_LONG_LONG 132 | */ 133 | #if defined(NO64BIT_LONG_LONG) 134 | #undef HAVE_64BIT_LONG_LONG 135 | #endif /* NO64BIT_LONG_LONG */ 136 | 137 | 138 | 139 | 140 | /* 141 | * 64 bit FNV-0 hash 142 | */ 143 | #if defined(HAVE_64BIT_LONG_LONG) 144 | typedef u_int64_t Fnv64_t; 145 | #else /* HAVE_64BIT_LONG_LONG */ 146 | typedef struct { 147 | u_int32_t w32[2]; /* w32[0] is low order, w32[1] is high order word */ 148 | } Fnv64_t; 149 | #endif /* HAVE_64BIT_LONG_LONG */ 150 | 151 | 152 | /* 153 | * 64 bit FNV-0 zero initial basis 154 | * 155 | * This historic hash is not recommended. One should use 156 | * the FNV-1 hash and initial basis instead. 157 | */ 158 | #if defined(HAVE_64BIT_LONG_LONG) 159 | #define FNV0_64_INIT ((Fnv64_t)0) 160 | #else /* HAVE_64BIT_LONG_LONG */ 161 | extern const Fnv64_t fnv0_64_init; 162 | #define FNV0_64_INIT (fnv0_64_init) 163 | #endif /* HAVE_64BIT_LONG_LONG */ 164 | 165 | 166 | /* 167 | * 64 bit FNV-1 non-zero initial basis 168 | * 169 | * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: 170 | * 171 | * chongo /\../\ 172 | * 173 | * NOTE: The \'s above are not back-slashing escape characters. 174 | * They are literal ASCII backslash 0x5c characters. 175 | * 176 | * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. 177 | */ 178 | #if defined(HAVE_64BIT_LONG_LONG) 179 | #define FNV1_64_INIT ((Fnv64_t)0xcbf29ce484222325ULL) 180 | #define FNV1A_64_INIT FNV1_64_INIT 181 | #else /* HAVE_64BIT_LONG_LONG */ 182 | extern const fnv1_64_init; 183 | extern const Fnv64_t fnv1a_64_init; 184 | #define FNV1_64_INIT (fnv1_64_init) 185 | #define FNV1A_64_INIT (fnv1a_64_init) 186 | #endif /* HAVE_64BIT_LONG_LONG */ 187 | 188 | 189 | /* 190 | * hash types 191 | */ 192 | enum fnv_type { 193 | FNV_NONE = 0, /* invalid FNV hash type */ 194 | FNV0_32 = 1, /* FNV-0 32 bit hash */ 195 | FNV1_32 = 2, /* FNV-1 32 bit hash */ 196 | FNV1a_32 = 3, /* FNV-1a 32 bit hash */ 197 | FNV0_64 = 4, /* FNV-0 64 bit hash */ 198 | FNV1_64 = 5, /* FNV-1 64 bit hash */ 199 | FNV1a_64 = 6, /* FNV-1a 64 bit hash */ 200 | }; 201 | 202 | 203 | /* 204 | * these test vectors are used as part o the FNV test suite 205 | */ 206 | struct test_vector { 207 | void *buf; /* start of test vector buffer */ 208 | int len; /* length of test vector */ 209 | }; 210 | struct fnv0_32_test_vector { 211 | struct test_vector *test; /* test vector buffer to hash */ 212 | Fnv32_t fnv0_32; /* expected FNV-0 32 bit hash value */ 213 | }; 214 | struct fnv1_32_test_vector { 215 | struct test_vector *test; /* test vector buffer to hash */ 216 | Fnv32_t fnv1_32; /* expected FNV-1 32 bit hash value */ 217 | }; 218 | struct fnv1a_32_test_vector { 219 | struct test_vector *test; /* test vector buffer to hash */ 220 | Fnv32_t fnv1a_32; /* expected FNV-1a 32 bit hash value */ 221 | }; 222 | struct fnv0_64_test_vector { 223 | struct test_vector *test; /* test vector buffer to hash */ 224 | Fnv64_t fnv0_64; /* expected FNV-0 64 bit hash value */ 225 | }; 226 | struct fnv1_64_test_vector { 227 | struct test_vector *test; /* test vector buffer to hash */ 228 | Fnv64_t fnv1_64; /* expected FNV-1 64 bit hash value */ 229 | }; 230 | struct fnv1a_64_test_vector { 231 | struct test_vector *test; /* test vector buffer to hash */ 232 | Fnv64_t fnv1a_64; /* expected FNV-1a 64 bit hash value */ 233 | }; 234 | 235 | 236 | /* 237 | * external functions 238 | */ 239 | /* hash_32.c */ 240 | extern Fnv32_t fnv_32_buf(void *buf, size_t len, Fnv32_t hashval); 241 | extern Fnv32_t fnv_32_str(char *buf, Fnv32_t hashval); 242 | 243 | /* hash_32a.c */ 244 | extern Fnv32_t fnv_32a_buf(void *buf, size_t len, Fnv32_t hashval); 245 | extern Fnv32_t fnv_32a_str(char *buf, Fnv32_t hashval); 246 | 247 | /* hash_64.c */ 248 | extern Fnv64_t fnv_64_buf(void *buf, size_t len, Fnv64_t hashval); 249 | extern Fnv64_t fnv_64_str(char *buf, Fnv64_t hashval); 250 | 251 | /* hash_64a.c */ 252 | extern Fnv64_t fnv_64a_buf(void *buf, size_t len, Fnv64_t hashval); 253 | extern Fnv64_t fnv_64a_str(char *buf, Fnv64_t hashval); 254 | 255 | /* test_fnv.c */ 256 | extern struct test_vector fnv_test_str[]; 257 | extern struct fnv0_32_test_vector fnv0_32_vector[]; 258 | extern struct fnv1_32_test_vector fnv1_32_vector[]; 259 | extern struct fnv1a_32_test_vector fnv1a_32_vector[]; 260 | extern struct fnv0_64_test_vector fnv0_64_vector[]; 261 | extern struct fnv1_64_test_vector fnv1_64_vector[]; 262 | extern struct fnv1a_64_test_vector fnv1a_64_vector[]; 263 | extern void unknown_hash_type(char *prog, enum fnv_type type, int code); 264 | extern void print_fnv32(Fnv32_t hval, Fnv32_t mask, int verbose, char *arg); 265 | extern void print_fnv64(Fnv64_t hval, Fnv64_t mask, int verbose, char *arg); 266 | 267 | 268 | #endif /* __FNV_H__ */ -------------------------------------------------------------------------------- /include/memory_pool.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-1. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_MEMORYPOOL_H 6 | #define BOLTDB_IN_CPP_MEMORYPOOL_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | namespace boltDB_CPP { 17 | 18 | /** 19 | * this is needed by zero copy. 20 | * it allocate/deallocate memory, but DOES NOT INVOKE DESTRUCTOR. 21 | * destructors are ignored when an object is created in memory pool. 22 | * nor will this works with hierarchy. 23 | * it CANNOT be used with new / shared / unique pointer. 24 | * 25 | * MemoryPool may not be a descriptive name since it doesn't have a 'pool' of 26 | * memory spaces at all. But it fits my need for now. 27 | */ 28 | class MemoryPool { 29 | //this is for char arrays 30 | std::vector arrays; 31 | //this is for typed objects 32 | //I need to separate this from plain arrays 33 | std::map > objs; 34 | public: 35 | ~MemoryPool() { 36 | for (auto item : arrays) { 37 | // std::cout << "delete " << std::showbase << std::hex << (void *) item << std::endl; 38 | //call destructor first if it has one 39 | auto iter = objs.find(item); 40 | if (iter != objs.end()) { 41 | iter->second(item); 42 | } 43 | //release the memory 44 | delete[] item; 45 | } 46 | } 47 | char *allocateByteArray(size_t sz) { 48 | auto ret = new char[sz]; 49 | for (size_t i = 0; i < sz; i++) { 50 | ret[i] = 0; 51 | } 52 | arrays.push_back(ret); 53 | // std::cout << "allocate " << std::showbase << std::hex << (void *) ret << std::endl; 54 | return ret; 55 | } 56 | template 57 | T *allocate(Args &&... args) { 58 | auto ret = allocateByteArray(sizeof(T)); 59 | new(ret) T(std::forward(args)...); 60 | //register destructors into objs 61 | 62 | auto fn = [](void *pointer) { 63 | reinterpret_cast(pointer)->~T(); 64 | }; 65 | objs[ret] = fn; 66 | 67 | return reinterpret_cast(ret); 68 | } 69 | void deallocateByteArray(char *ptr) { 70 | assert(objs.find(ptr) == objs.end()); 71 | auto iter = std::find(arrays.begin(), arrays.end(), ptr); 72 | assert(iter != arrays.end()); 73 | delete[] ptr; 74 | arrays.erase(iter); 75 | } 76 | template 77 | void deallocate(T *ptr) { 78 | //there should be a destructor 79 | auto dtor = objs.find(ptr); 80 | assert(dtor != objs.end()); 81 | //ptr should be in 'arrays' 82 | auto iter = std::find(arrays.begin(), arrays.end(), ptr); 83 | assert(iter != arrays.end()); 84 | //call destructor first 85 | dtor->second(ptr); 86 | //release memory 87 | char *cptr = reinterpret_cast(ptr); 88 | deallocateByteArray(cptr); 89 | } 90 | char *arrayCopy(const char *src, size_t len) { 91 | auto ret = allocateByteArray(len); 92 | std::memcpy(ret, src, len); 93 | return ret; 94 | } 95 | MemoryPool operator=(const MemoryPool &) = delete; 96 | MemoryPool(const MemoryPool &) = delete; 97 | MemoryPool() = default; 98 | }; 99 | } // namespace boltDB_CPP 100 | #endif // BOLTDB_IN_CPP_MEMORYPOOL_H 101 | -------------------------------------------------------------------------------- /include/meta.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-4. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_METADATA_H 6 | #define BOLTDB_IN_CPP_METADATA_H 7 | #include 8 | #include "util.h" 9 | #include "bucket_header.h" 10 | namespace boltDB_CPP { 11 | class Page; 12 | struct Meta { 13 | uint32_t magic = 0; 14 | uint32_t version = 0; 15 | uint32_t pageSize = 0; 16 | uint32_t reservedFlag = 0; 17 | BucketHeader rootBucketHeader; 18 | page_id freeListPageNumber = 0; 19 | page_id totalPageNumber = 0; 20 | txn_id txnId = 0; 21 | uint64_t checkSum = 0; 22 | static Meta *copyCreateFrom(Meta *other, MemoryPool &pool); 23 | bool validate(); 24 | uint64_t sum64(); 25 | 26 | void write(Page *page); 27 | }; 28 | } // namespace boltDB_CPP 29 | 30 | #endif // BOLTDB_IN_CPP_METADATA_H 31 | -------------------------------------------------------------------------------- /include/node.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-27. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_NODE_H 6 | #define BOLTDB_IN_CPP_NODE_H 7 | #include 8 | #include 9 | #include "types.h" 10 | #include "util.h" 11 | namespace boltDB_CPP { 12 | class Node; 13 | typedef std::vector NodeList; 14 | 15 | // this is a pointer to element. The element can be in a page or not added to a 16 | // page yet. 1.points to an element in a page 2.points to an element not yet in 17 | // a page this can be pointing to kv pair. in this case, pageId is meaningless. 18 | // if the inode is comprised in a branch node, then pageId is the page starts 19 | // with key value equals to 'key' member and the value is meaningless. may use 20 | // an union to wrap up pageId and value 21 | struct Inode { 22 | uint32_t flag = 0;//==bucketleaf if this is an inline bucket. otherwise == 0 23 | page_id pageId = 0; 24 | Item key; 25 | Item value; 26 | Item Key() const { return key; } 27 | Item Value() const { return value; } 28 | }; 29 | 30 | typedef std::vector InodeList; 31 | 32 | class Page; 33 | class Bucket; 34 | // this is a in-memory deserialized page 35 | class Node { 36 | friend class ElementRef; 37 | 38 | Bucket *bucket = nullptr; 39 | bool isLeaf = false; 40 | bool unbalanced = false; 41 | bool spilled = false; 42 | Item key; 43 | page_id pageId = 0; 44 | Node *parentNode = nullptr; 45 | NodeList children; 46 | InodeList inodeList; 47 | 48 | public: 49 | explicit Node(Bucket *b, Node *p) : bucket(b), parentNode(p) {} 50 | /** 51 | * setter 52 | */ 53 | void markLeaf() { isLeaf = true; } 54 | void setBucket(Bucket *b) { bucket = b; } 55 | void setParent(Node *p) { parentNode = p; } 56 | void addChild(Node *c) { children.push_back(c); } 57 | 58 | /** 59 | * getter 60 | */ 61 | page_id getPageId() const { return pageId; } 62 | Inode getInode(size_t idx) { return inodeList[idx]; } 63 | bool isLeafNode() const { return isLeaf; } 64 | std::vector branchPageIds(); 65 | 66 | size_t search(const Item &key, bool &found); 67 | bool isinlineable(size_t maxInlineBucketSize) const; 68 | 69 | void read(Page *page); 70 | Node *childAt(uint64_t index); 71 | void do_remove(const Item &key); 72 | // return size of deserialized node 73 | size_t size() const; 74 | size_t pageElementSize() const; 75 | Node *root(); 76 | size_t minKeys() const; 77 | bool sizeLessThan(size_t s) const; 78 | size_t childIndex(Node *child) const; 79 | size_t numChildren() const; 80 | Node *nextSibling(); 81 | Node *prevSibling(); 82 | void put(const Item &oldKey, const Item &newKey, const Item &value, 83 | page_id pageId, uint32_t flag); 84 | void del(const Item &key); 85 | void write(Page *page); 86 | std::vector split(size_t pageSize); 87 | void splitTwo(size_t pageSize, Node *&a, Node *&b); 88 | size_t splitIndex(size_t threshold); // sz is return value. it's the size of the first page. 89 | void free(); 90 | void removeChild(Node *target); 91 | void dereference(); 92 | int spill(); 93 | void rebalance(); 94 | }; 95 | 96 | } // namespace boltDB_CPP 97 | 98 | #endif // BOLTDB_IN_CPP_NODE_H 99 | -------------------------------------------------------------------------------- /include/rwlock.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_RWLOCK_H 6 | #define BOLTDB_IN_CPP_RWLOCK_H 7 | #include 8 | #include 9 | 10 | namespace boltDB_CPP { 11 | class RWLock { 12 | bool write_entered; 13 | size_t reader_count; 14 | std::mutex mtx; 15 | std::condition_variable reader; 16 | std::condition_variable writer; 17 | 18 | public: 19 | RWLock() : write_entered(false), reader_count(0) {} 20 | RWLock(const RWLock &) = delete; 21 | RWLock &operator=(const RWLock &) = delete; 22 | 23 | void readLock() { 24 | std::unique_lock lock(mtx); 25 | while (write_entered) { 26 | reader.wait(lock); 27 | } 28 | if (reader_count == INT_MAX) { 29 | throw std::runtime_error("possibly request too many read locks"); 30 | } 31 | reader_count++; 32 | } 33 | void readUnlock() { 34 | std::lock_guard guard(mtx); 35 | if (reader_count == 0) { 36 | throw std::runtime_error( 37 | "try release read lock without any reader holding the lock"); 38 | } 39 | reader_count--; 40 | if (write_entered && reader_count == 0) { 41 | writer.notify_one(); 42 | } 43 | } 44 | void writeLock() { 45 | std::unique_lock lock(mtx); 46 | while (write_entered) { 47 | reader.wait(lock); 48 | } 49 | write_entered = true; 50 | while (reader_count) { 51 | writer.wait(lock); 52 | } 53 | } 54 | void writeUnlock() { 55 | std::lock_guard lock(mtx); 56 | write_entered = false; 57 | reader.notify_all(); 58 | } 59 | }; 60 | } // namespace boltDB_CPP 61 | 62 | #endif // BOLTDB_IN_CPP_RWLOCK_H 63 | -------------------------------------------------------------------------------- /include/txn.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_TRANSACTION_H 6 | #define BOLTDB_IN_CPP_TRANSACTION_H 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "bucket.h" 12 | #include "memory_pool.h" 13 | #include "meta.h" 14 | #include "types.h" 15 | namespace boltDB_CPP { 16 | 17 | class DB; 18 | class Meta; 19 | class Bucket; 20 | class Page; 21 | class Node; 22 | 23 | struct TxStat { 24 | uint64_t pageCount = 0; 25 | uint64_t pageAlloc = 0; // in bytes 26 | 27 | uint64_t cursorCount = 0; 28 | 29 | uint64_t nodeCount = 0; 30 | uint64_t nodeDereferenceCount = 0; 31 | 32 | uint64_t rebalanceCount = 0; 33 | uint64_t rebalanceTime = 0; 34 | 35 | uint64_t splitCount = 0; 36 | uint64_t spillCount = 0; 37 | uint64_t spillTime = 0; 38 | 39 | uint64_t writeCount = 0; 40 | uint64_t writeTime = 0; 41 | }; 42 | 43 | class Txn { 44 | friend class DB; 45 | friend class Bucket; 46 | friend class Cursor; 47 | 48 | bool writable = false; 49 | bool managed = false; 50 | DB *db = nullptr; 51 | Meta *metaData = nullptr; 52 | Bucket rootBucket; 53 | std::map dirtyPageTable; // this is dirty page table 54 | std::vector> commitHandlers; 55 | bool writeFlag = false; 56 | TxStat stats; 57 | MemoryPool pool; 58 | 59 | public: 60 | Txn() : rootBucket(this) {} 61 | txn_id txnId() const; 62 | void free(txn_id tid, Page *page); 63 | size_t getTotalPageNumber() { return metaData->totalPageNumber; } 64 | 65 | bool isWritable() const { return writable; } 66 | void increaseCurserCount() { stats.cursorCount++; } 67 | void increaseNodeCount() { stats.nodeCount++; } 68 | Page *getPage(page_id pageId); 69 | Page *allocate(size_t count); 70 | void for_each_page(page_id pageId, int depth, 71 | std::function); 72 | 73 | void init(DB *db); 74 | Bucket *getBucket(const Item &name); 75 | Bucket *createBucket(const Item &name); 76 | Bucket *createBucketIfNotExists(const Item &name); 77 | int deleteBucket(const Item &name); 78 | int for_each(std::function); 79 | void OnCommit(std::function fn); 80 | int commit(); 81 | int rollback(); 82 | // void closeTxn(); 83 | int writeMeta(); 84 | int write(); 85 | int isFreelistCheckOK(); 86 | bool isBucketsRemainConsistent(Bucket &bucket, std::map &reachable, 87 | std::map &freed); 88 | }; 89 | 90 | } // namespace boltDB_CPP 91 | 92 | #endif // BOLTDB_IN_CPP_TRANSACTION_H 93 | -------------------------------------------------------------------------------- /include/types.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_BOLTDB_TYPES_H 6 | #define BOLTDB_IN_CPP_BOLTDB_TYPES_H 7 | #include 8 | #include 9 | #include 10 | namespace boltDB_CPP { 11 | 12 | typedef uint64_t txn_id; 13 | typedef uint64_t page_id; 14 | 15 | class MemoryPool; 16 | // this is used to hold values from page element 17 | // it doesn't own any memory resource 18 | // value copy is exactly what I want 19 | struct Item { 20 | const char *pointer = nullptr; 21 | size_t length = 0; 22 | Item() = default; 23 | Item(const char *p, size_t sz) : pointer(p), length(sz) {} 24 | bool operator==(const Item &other) const; 25 | bool operator!=(const Item &other) const; 26 | bool operator<(const Item &other) const; 27 | 28 | void reset(); 29 | bool empty() const; 30 | Item clone(MemoryPool *pool); 31 | static Item make_item(const char *p) { 32 | if (p == nullptr || *p == 0) { 33 | return {}; 34 | } 35 | return {p, strlen(p)}; 36 | } 37 | }; 38 | } // namespace boltDB_CPP 39 | 40 | namespace std { 41 | template<> 42 | struct hash { 43 | std::size_t operator()(const boltDB_CPP::Item &k) const { 44 | return hash()(reinterpret_cast(k.pointer)) ^ 45 | hash()(k.length); 46 | } 47 | }; 48 | } // namespace std 49 | #endif // BOLTDB_IN_CPP_BOLTDB_TYPES_H 50 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_UTILITY_H 6 | #define BOLTDB_IN_CPP_UTILITY_H 7 | #include 8 | #include "types.h" 9 | namespace boltDB_CPP { 10 | int file_Wlock(int fd); 11 | int file_WlockBlocking(int fd); 12 | int file_Rlock(int fd); 13 | int file_Unlock(int fd); 14 | 15 | int file_data_sync(int fd); 16 | template 17 | size_t binary_search(T &target, V &key, CMP cmp, size_t e_p, bool &found) { 18 | found = false; 19 | size_t b = 0; 20 | size_t e = e_p; 21 | while (b < e) { 22 | size_t mid = b + (e - b) / 2; 23 | int ret = cmp(target[mid], key); 24 | 25 | if (ret == 0) { 26 | found = true; 27 | return mid; 28 | } else if (ret < 0) { 29 | b = mid + 1; 30 | } else { 31 | e = mid; 32 | } 33 | } 34 | return b; 35 | }; 36 | 37 | template 38 | int cmp_wrapper(const T &t, const Item &p) { 39 | if (t.Key() < p) { 40 | return -1; 41 | } 42 | if (t.Key() == p) { 43 | return 0; 44 | } 45 | return 1; 46 | } 47 | 48 | // template 49 | // std::unique_ptr make_unique(Args &&... args) { 50 | // return std::unique_ptr(new T(std::forward(args)...)); 51 | //} 52 | //} 53 | 54 | void mergePageIds(std::vector &dest, 55 | const std::vector &a, 56 | const std::vector &b); 57 | 58 | std::vector merge( 59 | const std::vector &a, 60 | const std::vector &b); 61 | 62 | off_t file_size(int fd); 63 | } // namespace boltDB_CPP 64 | #endif // BOLTDB_IN_CPP_UTILITY_H 65 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/Nov11/boltdb_in_cpp.svg?branch=master)](https://travis-ci.org/Nov11/boltdb_in_cpp) 2 | [![Coverage Status](https://coveralls.io/repos/github/Nov11/boltdb_in_cpp/badge.svg?branch=master&service=github)](https://coveralls.io/github/Nov11/boltdb_in_cpp?branch=master) 3 | ### A C++ implementation of [BoltDB](https://github.com/boltdb/bolt) 4 | * under-construction 5 | * based on version tagged 1.3.1 6 | * supports x86_64 linux only 7 | * compiles against C++ 11 8 | * will be tested on ubuntu 18 9 | * do performance comparision with [LMDB](https://github.com/LMDB/lmdb) when all test cases are ported & passed 10 | * refactory is needed as C++ enforce encapsulation in language feature 11 | 12 | 13 | ### todo 14 | - [ ] uniform pointer/shared pointer usage(this will not be needed providing memory pool) 15 | - [ ] create a per txn memory pool and replace raw 'new' to calls of it (in progress) 16 | - [ ] re-encapsulate : avoid annoying getters and provide a proper user interface 17 | ### progress 18 | - [x] node 19 | - [x] page 20 | - [x] cursor 21 | - [x] bucket 22 | - [x] tx 23 | - [x] freelist 24 | - [x] db 25 | - [ ] test cases 26 | 27 | ### later 28 | - [ ] batch support 29 | - [ ] status 30 | - [ ] better error message 31 | 32 | ### on disk file layouts 33 | #### file 34 | * |page 0|page 1|page 2 |page 3 |...| <- general 35 | * |meta 0|meta 1|freelist|leaf page|...| <- typical 36 | #### page 37 | * page layout : |page header | page content| 38 | * page header : |page id | flag | count | overflow| 39 | * page type : meta / freelist / leaf node / branch node 40 | 41 | #### node 42 | * leaf node serialized: 43 | |page header | leaf element .... | kv pair ... | 44 | * branch node serialized: 45 | |page header | branch element .... | kv pair ... | 46 | 47 | ### mvcc 48 | 1. isolation level(close to) : read committed 49 | 2. exclusive remapping 50 | 3. no exclusion on writing meta page 51 | 52 | ### great resource: 53 | * [a series of source code reading blogs(by oceanken)](https://www.jianshu.com/p/b86a69892990) 54 | * [bucket data structure explanation (2016)](http://www.d-kai.me/boltdb%E4%B9%8Bbucket%E4%B8%80/) 55 | 56 | ### lock lifetime 57 | * file lock on db file 58 | * (acquire)when opening db in read only way, it grabs a share lock. An exclusive lock if in read write mode. 59 | * (release)when 'closeDB' is invoked, it releases the lock if grabbed an exclusive lock. Do nothing if grabbed a shared lock. 60 | As shared lock will be released implicitly when closing the related file descriptor, a invocation of flock with LOCK_UN will be 61 | redundant. 62 | * readWriteAccessMutex 63 | * (acquire)when starting a read/write txn 64 | * (release)when committing a txn(it must be a read write txn) 65 | * mmaplock 66 | * (acquire)1.remap db file(full lock) 2. start a ro txn(read lock) 67 | * (release)1.remapped db file 2. remove a txn 68 | * metalock(this is actually db data structure mutex) 69 | * (acquire)1.begin a new rw/ro txn 2.remove a txn 70 | * (release)1.txn being created 2. removed a txn 71 | * rwmtx, mmaplock, metalock are acquired and released during database shutdown 72 | 73 | ### memory management 74 | * one txn maintains a local memory manager which is in charge of every allocation/deallocation of the txn's. the memory manager releases all the memory 75 | on txn destruction which happens when the txn is removed from its database(for ro txn) or the txn is reset so that it is not the 76 | current rw txn any more. 77 | * the database object's memory is maintained by user application. typically should be used with a smart pointer. this implementation assumes that 78 | users take care of memory recycling of a database object. 79 | 80 | ### issues with source code 81 | 1. isn't meta page a waste of space? as it only needs a page header and a meta data which will be significantly smaller than a page(4k). 82 | 2. top level bucket of a txn is kind of merely a concept. it will never be returned to user code. it always has at least one sub bucket. 83 | 3. calling (i forgot about this. let me leave it here) 84 | 4. memory management. if there is a item copy, from txn to user space, the memory an item points to is invalid when the txn ends. user must 85 | do additional copy to maintain it. there might be a way to force this kind of copy in compilation in cpp. 86 | -------------------------------------------------------------------------------- /script/formatting/formatter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # encoding: utf-8 3 | """Format the ill-formatted code.""" 4 | ## ============================================== 5 | ## GOAL : Format code, Update headers 6 | ## ============================================== 7 | 8 | import argparse 9 | import logging 10 | import os 11 | import re 12 | import sys 13 | import datetime 14 | import subprocess 15 | 16 | from functools import reduce 17 | 18 | # Following is done so that we can import from ../helpers.py 19 | sys.path.append( 20 | os.path.abspath(os.path.dirname(__file__)).replace('/formatting', '') 21 | ) 22 | from helpers import CLANG_FORMAT, PELOTON_DIR, CLANG_FORMAT_FILE, LOG,\ 23 | clang_format 24 | 25 | ## ============================================== 26 | ## CONFIGURATION 27 | ## ============================================== 28 | 29 | # NOTE: absolute path to peloton directory is calculated from current directory 30 | # directory structure: peloton/scripts/formatting/ 31 | # PELOTON_DIR needs to be redefined if the directory structure is changed 32 | 33 | #other directory paths used are relative to peloton_dir 34 | PELOTON_SRC_DIR = os.path.join(PELOTON_DIR, "src") 35 | PELOTON_TESTS_DIR = os.path.join(PELOTON_DIR, "test") 36 | 37 | # DEFAULT DIRS 38 | DEFAULT_DIRS = [] 39 | DEFAULT_DIRS.append(PELOTON_SRC_DIR) 40 | DEFAULT_DIRS.append(PELOTON_TESTS_DIR) 41 | 42 | ## ============================================== 43 | ## HEADER CONFIGURATION 44 | ## ============================================== 45 | 46 | #header framework, dynamic information will be added inside function 47 | header_comment_line_1 = "//===----------------------------------------------------------------------===//\n" 48 | header_comment_line_1 += "//\n" 49 | header_comment_line_1 += "// Peloton\n" 50 | header_comment_line_2 = "//\n" 51 | header_comment_line_3 = "// " 52 | header_comment_line_4 = "//\n" 53 | header_comment_line_5 = "// Identification: " 54 | header_comment_line_6 = "//\n" 55 | header_comment_line_7 = "// Copyright (c) 2015-%d, Carnegie Mellon University Database Group\n" % datetime.datetime.now().year 56 | header_comment_line_8 = "//\n" 57 | header_comment_line_9 = "//===----------------------------------------------------------------------===//\n\n" 58 | 59 | header_comment_1 = header_comment_line_1 + header_comment_line_2 60 | header_comment_3 = header_comment_line_4 61 | header_comment_5 = header_comment_line_6 + header_comment_line_7 \ 62 | + header_comment_line_8 + header_comment_line_9 63 | 64 | #regular expresseion used to track header 65 | HEADER_REGEX = re.compile(r"((\/\/===-*===\/\/\n(\/\/.*\n)*\/\/===-*===\/\/[\n]*)\n\n)*") 66 | 67 | ## ============================================== 68 | ## UTILITY FUNCTION DEFINITIONS 69 | ## ============================================== 70 | 71 | 72 | def format_file(file_path, update_header, clang_format_code): 73 | """Formats the file passed as argument.""" 74 | file_name = os.path.basename(file_path) 75 | abs_path = os.path.abspath(file_path) 76 | rel_path_from_peloton_dir = os.path.relpath(abs_path, PELOTON_DIR) 77 | 78 | with open(file_path, "r+") as file: 79 | file_data = file.read() 80 | 81 | if update_header: 82 | # strip old header if it exists 83 | header_match = HEADER_REGEX.match(file_data) 84 | if not header_match is None: 85 | LOG.info("Strip header from %s", file_name) 86 | header_comment = header_match.group() 87 | LOG.debug("Header comment : %s", header_comment) 88 | file_data = file_data.replace(header_comment,"") 89 | 90 | # add new header 91 | LOG.info("Add header to %s", file_name) 92 | header_comment_2 = header_comment_line_3 + file_name + "\n" 93 | header_comment_4 = header_comment_line_5\ 94 | + rel_path_from_peloton_dir + "\n" 95 | header_comment = header_comment_1 + header_comment_2 \ 96 | + header_comment_3 + header_comment_4 \ 97 | + header_comment_5 98 | #print header_comment 99 | 100 | file_data = header_comment + file_data 101 | 102 | file.seek(0, 0) 103 | file.truncate() 104 | file.write(file_data) 105 | 106 | elif clang_format_code: 107 | clang_format(file_path) 108 | 109 | #END WITH 110 | #END FORMAT__FILE(FILE_NAME) 111 | 112 | 113 | def format_dir(dir_path, update_header, clang_format_code): 114 | """Formats all the files in the dir passed as argument.""" 115 | for subdir, _, files in os.walk(dir_path): # _ is for directories. 116 | for file in files: 117 | #print os.path.join(subdir, file) 118 | file_path = subdir + os.path.sep + file 119 | 120 | if file_path.endswith(".h") or file_path.endswith(".cpp"): 121 | format_file(file_path, update_header, clang_format_code) 122 | #END IF 123 | #END FOR [file] 124 | #END FOR [os.walk] 125 | #END ADD_HEADERS_DIR(DIR_PATH) 126 | 127 | 128 | ## ============================================== 129 | ## Main Function 130 | ## ============================================== 131 | 132 | if __name__ == '__main__': 133 | 134 | PARSER = argparse.ArgumentParser( 135 | description='Update headers and/or format source code' 136 | ) 137 | 138 | PARSER.add_argument( 139 | "-u", "--update-header", 140 | help='Action: Update existing headers or add new ones', 141 | action='store_true' 142 | ) 143 | PARSER.add_argument( 144 | "-c", "--clang-format-code", 145 | help='Action: Apply clang-format to source code', 146 | action='store_true' 147 | ) 148 | PARSER.add_argument( 149 | "-f", "--staged-files", 150 | help='Action: Apply the selected action(s) to all staged files (git)', 151 | action='store_true' 152 | ) 153 | PARSER.add_argument( 154 | 'paths', metavar='PATH', type=str, nargs='*', 155 | help='Files or directories to (recursively) apply the actions to' 156 | ) 157 | 158 | ARGS = PARSER.parse_args() 159 | 160 | if ARGS.staged_files: 161 | PELOTON_DIR_bytes = bytes(PELOTON_DIR, 'utf-8') 162 | TARGETS = [ 163 | str(os.path.abspath(os.path.join(PELOTON_DIR_bytes, f)), 'utf-8') \ 164 | for f in subprocess.check_output( 165 | ["git", "diff", "--name-only", "HEAD", "--cached", 166 | "--diff-filter=d" 167 | ] 168 | ).split()] 169 | 170 | if not TARGETS: 171 | LOG.error( 172 | "no staged files or not calling from a repository -- exiting" 173 | ) 174 | sys.exit("no staged files or not calling from a repository") 175 | elif not ARGS.paths: 176 | LOG.error("no files or directories given -- exiting") 177 | sys.exit("no files or directories given") 178 | else: 179 | TARGETS = ARGS.paths 180 | 181 | for x in TARGETS: 182 | if os.path.isfile(x): 183 | LOG.info("Scanning file: %s", x) 184 | format_file(x, ARGS.update_header, ARGS.clang_format_code) 185 | elif os.path.isdir(x): 186 | LOG.info("Scanning directory %s", x) 187 | format_dir(x, ARGS.update_header, ARGS.clang_format_code) 188 | ## FOR 189 | ## IF 190 | -------------------------------------------------------------------------------- /script/formatting/iwyu.md: -------------------------------------------------------------------------------- 1 | # Include-What-You-Use 2 | 3 | Need LLVM 4 | 5 | sudo apt-get install iwyu 6 | 7 | make -k CXX=/usr/bin/include-what-you-use -j4 > iwyu.txt 2>&1 8 | 9 | 10 | -------------------------------------------------------------------------------- /script/git-hooks/README.md: -------------------------------------------------------------------------------- 1 | # Git Hooks 2 | 3 | This directory contains all Git hooks used in Peloton. Git hooks provide 4 | a mechanism to execute custom scripts when important events occur during 5 | your interaction with this Git repository. More details on Git hooks can 6 | be found [here](https://git-scm.com/book/gr/v2/Customizing-Git-Git-Hooks). 7 | 8 | ### Pre-commit 9 | 10 | A pre-commit hook is fired on every commit into your local Git 11 | repository. Peloton's pre-commit hook collects all modified files 12 | (excluding deleted files) and runs them through our source code 13 | validator script in `script/validator/source_validator.py`. *Ideally*, 14 | we should also ensure every commit successfully compiles and, to a 15 | lesser extent, that the commit passes the tests, but this isn't 16 | reasonable for now. 17 | 18 | **Installation:** Under the peloton root directory, run 19 | `ln -s ../../script/git-hooks/pre-commit .git/hooks/pre-commit` to 20 | install locally. The pre-commit file should already have executable 21 | permission, but check again to make sure. -------------------------------------------------------------------------------- /script/git-hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Source validation pre-commit hook 3 | # 4 | # This script collects all modified files and runs it through our source code 5 | # validation script. The validation script returns 0 on success and 1 on any 6 | # failure. 7 | # 8 | # To enable, symlink this file to '.git/hooks/pre-commit' like so: 9 | # ln -s ../../script/git-hooks/pre-commit .git/hooks/pre-commit 10 | 11 | FORMATTER_COMMAND="./script/formatting/formatter.py" 12 | 13 | FILES=$(git diff --name-only HEAD --cached --diff-filter=d | grep '\.\(cpp\|h\)$') 14 | if [ -n "$FILES" ]; then 15 | ./script/validators/source_validator.py --files $FILES 16 | RESULT=$? 17 | if [ $RESULT -ne 0 ]; then 18 | echo "***************************************" 19 | echo "******* Peloton Pre-Commit Hook *******" 20 | echo "***************************************" 21 | echo "Use \"$FORMATTER_PATH -c -f\" to format all staged files." 22 | echo "Or use \"git commit --no-verify\" to temporarily bypass the pre-commit hook." 23 | 24 | echo 25 | echo "Be aware that changed files have to be staged again!" 26 | echo "***************************************" 27 | fi 28 | exit $RESULT 29 | fi 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /script/helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Common helper functions to be used in different Python scripts.""" 3 | import difflib 4 | import distutils.spawn 5 | import logging 6 | import os 7 | import subprocess 8 | 9 | from functools import reduce 10 | 11 | CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__)) 12 | PELOTON_DIR = CODE_SOURCE_DIR.replace('/script', '') 13 | CLANG_FORMAT_FILE = os.path.join(PELOTON_DIR, ".clang-format") 14 | 15 | FORMATTING_FILE_WHITELIST = [ 16 | # Fill me 17 | ] 18 | 19 | ## ============================================== 20 | ## LOGGING CONFIGURATION 21 | ## ============================================== 22 | 23 | LOG = logging.getLogger(__name__) 24 | LOG_HANDLER = logging.StreamHandler() 25 | LOG_FORMATTER = logging.Formatter( 26 | fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s', 27 | datefmt='%m-%d-%Y %H:%M:%S' 28 | ) 29 | LOG_HANDLER.setFormatter(LOG_FORMATTER) 30 | LOG.addHandler(LOG_HANDLER) 31 | LOG.setLevel(logging.INFO) 32 | 33 | def find_clangformat(): 34 | """Finds appropriate clang-format executable.""" 35 | #check for possible clang-format versions 36 | for exe in ["clang-format", "clang-format-3.6", "clang-format-3.7", 37 | "clang-format-3.8" 38 | ]: 39 | path = distutils.spawn.find_executable(exe) 40 | if not path is None: 41 | break 42 | return path 43 | 44 | CLANG_FORMAT = find_clangformat() 45 | CLANG_COMMAND_PREFIX = [CLANG_FORMAT, "-style=file"] 46 | 47 | def clang_check(file_path): 48 | """Checks and reports bad code formatting.""" 49 | rel_path_from_peloton_dir = os.path.relpath(file_path, PELOTON_DIR) 50 | 51 | if rel_path_from_peloton_dir in FORMATTING_FILE_WHITELIST: 52 | return True 53 | 54 | file_status = True 55 | 56 | # Run clang-format on the file 57 | if CLANG_FORMAT is None: 58 | LOG.error("clang-format seems not installed") 59 | exit() 60 | clang_format_cmd = CLANG_COMMAND_PREFIX + [file_path] 61 | formatted_src = subprocess.check_output(clang_format_cmd).splitlines(True) 62 | 63 | # For Python 3, the above command gives a list of binary sequences, each 64 | # of which has to be converted to string for diff to operate correctly. 65 | # Otherwise, strings would be compared with binary sequences and there 66 | # will always be a big difference. 67 | formatted_src = [line.decode('utf-8') for line in formatted_src] 68 | # Load source file 69 | with open(file_path, "r") as file: 70 | src = file.readlines() 71 | 72 | # Do the diff 73 | difference = difflib.Differ() 74 | diff = difference.compare(src, formatted_src) 75 | line_num = 0 76 | for line in diff: 77 | code = line[:2] 78 | if code in (" ", "- "): 79 | line_num += 1 80 | if code == '- ': 81 | if file_status: 82 | LOG.info("Invalid formatting in file : " + file_path) 83 | LOG.info("Line %d: %s", line_num, line[2:].strip()) 84 | file_status = False 85 | 86 | return file_status 87 | 88 | 89 | def clang_format(file_path): 90 | """Formats the file at file_path""" 91 | if CLANG_FORMAT is None: 92 | LOG.error("clang-format seems not installed") 93 | exit() 94 | 95 | formatting_command = CLANG_COMMAND_PREFIX + ["-i", file_path] 96 | LOG.info(' '.join(formatting_command)) 97 | subprocess.call(formatting_command) 98 | -------------------------------------------------------------------------------- /script/readme: -------------------------------------------------------------------------------- 1 | these files in this directory and.clang - 2 | format in top level directory are from https: // github.com/cmu-db/peloton. -------------------------------------------------------------------------------- /script/validators/source_validator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Validates source files and reports compliance anomalies.""" 3 | 4 | import argparse 5 | import os 6 | import re 7 | import sys 8 | import mmap 9 | import glob 10 | 11 | # Following is done so that we can import from ../helpers.py 12 | sys.path.append( 13 | os.path.abspath(os.path.dirname(__file__)).replace('/validators', '') 14 | ) 15 | from helpers import clang_check, PELOTON_DIR, LOG 16 | 17 | ## ============================================== 18 | ## CONFIGURATION 19 | ## ============================================== 20 | 21 | # NOTE: absolute path to peloton directory is calculated from current directory 22 | # directory structure: peloton/scripts/formatting/ 23 | # PELOTON_DIR needs to be redefined if the directory structure is changed 24 | 25 | # Other directory paths used are relative to PELOTON_DIR 26 | DEFAULT_DIRS = [ 27 | os.path.join(PELOTON_DIR, "src"), 28 | os.path.join(PELOTON_DIR, "test") 29 | ] 30 | 31 | # To be used in check_includes. 32 | PATHS = set([path.replace('src/include/', '') for path in glob.glob('src/include/**/*.h')]) 33 | 34 | EXIT_SUCCESS = 0 35 | EXIT_FAILURE = -1 36 | 37 | # Source patterns to check for 38 | VALIDATOR_PATTERNS = [re.compile(patt) for patt in [ 39 | r"std\:\:cout", 40 | r" printf\(", 41 | r"cout", 42 | r" malloc\(", 43 | r" free\(", 44 | r" memset\(", 45 | r" memcpy\(", 46 | r" \:\:memset\(", 47 | r" \:\:memcpy\(", 48 | r" std\:\:memset\(", 49 | r" std\:\:memcpy\(", 50 | r'^#include "include/' 51 | ] 52 | ] 53 | 54 | # Files that should not be checked 55 | SKIP_FILES_LIST = [ 56 | "src/common/allocator.cpp", 57 | "src/network/socket_base.cpp", 58 | "src/network/protocol.cpp", 59 | "src/include/common/macros.h", 60 | "src/common/stack_trace.cpp", 61 | "src/include/parser/sql_scanner.h", # There is a free() in comments 62 | "src/include/index/bloom_filter.h", 63 | "src/include/index/compact_ints_key.h", 64 | "src/include/index/bwtree.h", 65 | "src/codegen/util/oa_hash_table.cpp", 66 | "src/codegen/util/cc_hash_table.cpp" 67 | ] 68 | 69 | ## ============================================== 70 | ## UTILITY FUNCTION DEFINITIONS 71 | ## ============================================== 72 | 73 | 74 | def check_common_patterns(file_path): 75 | """Checks for unwanted patterns in source files.""" 76 | rel_path_from_peloton_dir = os.path.relpath(file_path, PELOTON_DIR) 77 | 78 | # Skip some files 79 | if rel_path_from_peloton_dir in SKIP_FILES_LIST: 80 | return True 81 | 82 | with open(file_path, 'r') as opened_file: 83 | file_status = True 84 | line_ctr = 1 85 | for line in opened_file: 86 | for validator_pattern in VALIDATOR_PATTERNS: 87 | # Check for patterns one at a time 88 | if validator_pattern.search(line): 89 | if file_status: 90 | LOG.info("Invalid pattern -- " + 91 | validator_pattern.pattern + 92 | " -- found in : " + file_path 93 | ) 94 | LOG.info("Line %d: %s", line_ctr, line.strip()) 95 | file_status = False 96 | line_ctr += 1 97 | 98 | return file_status 99 | 100 | 101 | def check_namespaces(file_path): 102 | """Scans namespace openings and closings.""" 103 | # only check for src files 104 | if not file_path.startswith(DEFAULT_DIRS[0]): 105 | return True 106 | 107 | # get required namespaces from path 108 | required_namespaces = ['peloton'] + file_path.replace(DEFAULT_DIRS[0] + "/", "").split("/") 109 | 110 | # for the include files, remove the include item in the list 111 | if 'include' in required_namespaces: 112 | required_namespaces.remove('include') 113 | 114 | # cut off the file name at the end of the list 115 | required_namespaces = required_namespaces[:-1] 116 | 117 | with open(file_path, 'r') as file: 118 | data = mmap.mmap(file.fileno(), 0, prot=mmap.PROT_READ) 119 | 120 | # scan for all namespace openings and closings 121 | matches = re.findall( 122 | r'^ *namespace ([a-z_-]+) {$|^ *} +\/\/ namespace ([a-z_-]+)$', 123 | data, 124 | flags=re.MULTILINE 125 | ) 126 | 127 | open_namespaces = list() 128 | namespace_errors = list() 129 | 130 | for match in matches: 131 | # assert the match is either an opening or a closing 132 | assert match[0] or match[1] 133 | 134 | # 1. namespace opening 135 | if match[0]: 136 | # add to list of open namespaces 137 | open_namespaces.append(match[0]) 138 | 139 | # remove from list of required namespaces 140 | if required_namespaces and required_namespaces[0] == match[0]: 141 | required_namespaces.pop(0) 142 | 143 | # 2. namespace closing 144 | else: 145 | # check if correct order 146 | if open_namespaces and open_namespaces[-1] != match[1]: 147 | namespace_errors.append( 148 | "This namespace was closed in wrong order: '" + 149 | match[1] + 150 | "' -- in " + file_path 151 | ) 152 | # check if present at all 153 | if not match[1] in open_namespaces: 154 | namespace_errors.append( 155 | "This namespace was closed, but is missing a correct " 156 | "opening: '" + match[1] + "' -- in " + file_path 157 | ) 158 | else: 159 | # remove from open list 160 | open_namespaces.remove(match[1]) 161 | 162 | if required_namespaces: 163 | namespace_errors.append( 164 | "Required namespaces are missing or in wrong order: " + 165 | str(required_namespaces) + " -- in " + file_path 166 | ) 167 | 168 | if open_namespaces: 169 | namespace_errors.append( 170 | "These namespaces were not closed properly: " + 171 | str(open_namespaces) + " -- in " + file_path 172 | ) 173 | 174 | if namespace_errors: 175 | LOG.info("Invalid namespace style -- in " + file_path) 176 | for error in namespace_errors: 177 | LOG.info(" " + error) 178 | return False 179 | return True 180 | 181 | 182 | def check_includes(file_path): 183 | """Checks whether local includes are done via #include<...>""" 184 | with open(file_path, "r") as file: 185 | path_pattern = re.compile(r'^#include <(include/)?(.*?)>') 186 | linenum = 0 187 | file_status = True 188 | for line in file: 189 | linenum += 1 190 | res = path_pattern.match(line) 191 | if res: 192 | path = res.groups()[1] 193 | if path in PATHS: 194 | if file_status: 195 | LOG.info("Invalid include in %s", file_path) 196 | file_status = False 197 | LOG.info("Line %s: %s", linenum, line.strip()) 198 | if not file_status: 199 | LOG.info('includes for peloton header files have must not have ' 200 | 'brackets' 201 | ) 202 | return file_status 203 | 204 | 205 | VALIDATORS = [ 206 | check_common_patterns, 207 | check_includes, 208 | 209 | # Uncomment the below validator once the namespace refactoring is done 210 | #check_namespaces, 211 | 212 | # Uncomment the below validator when all files are clang-format-compliant 213 | #check_format 214 | ] 215 | 216 | 217 | def validate_file(file_path): 218 | """Validates the source file that is passed as argument.""" 219 | if not file_path.endswith(".h") and not file_path.endswith(".cpp"): 220 | return True 221 | 222 | file_status = True 223 | for validator in VALIDATORS: 224 | if not validator(file_path): 225 | file_status = False 226 | 227 | return file_status 228 | 229 | 230 | def validate_dir(dir_path): 231 | """Validates all the files in the directory passed as argument.""" 232 | dir_status = True 233 | for subdir, _, files in os.walk(dir_path): # _ represents directories. 234 | for file in files: 235 | file_path = subdir + os.path.sep + file 236 | 237 | if not validate_file(file_path): 238 | dir_status = False 239 | return dir_status 240 | 241 | #END IF 242 | #END FOR [file] 243 | #END FOR [os.walk] 244 | #END VALIDATE_DIR(DIR_PATH) 245 | 246 | 247 | ## ============================================== 248 | ## Main Function 249 | ## ============================================== 250 | 251 | if __name__ == '__main__': 252 | 253 | PARSER = argparse.ArgumentParser( 254 | description='Perform source code validation on Peloton source' 255 | ) 256 | PARSER.add_argument("-f", "--files", nargs='*', 257 | help="A list of files to validate" 258 | ) 259 | ARGS = PARSER.parse_args() 260 | 261 | LOG.info("Running source validator ...") 262 | LOG.info("Peloton root : " + PELOTON_DIR) 263 | 264 | 265 | if ARGS.files: 266 | # Validate just the provided files. 267 | 268 | # In this mode, we perform explicit clang-format checks 269 | VALIDATORS.append(clang_check) 270 | for each_file in ARGS.files: 271 | each_file = os.path.abspath(each_file.lower()) 272 | 273 | # Fail if the file isn't really a file 274 | if not os.path.isfile(each_file): 275 | LOG.info("ERROR: " + each_file + " isn't a file") 276 | sys.exit(EXIT_FAILURE) 277 | 278 | # Skip files not in 'src' or 'test' 279 | if not each_file.startswith(DEFAULT_DIRS[0]) and \ 280 | not each_file.startswith(DEFAULT_DIRS[1]): 281 | LOG.info("Skipping non-Peloton source : " + each_file) 282 | continue 283 | 284 | # Looks good, let's validate 285 | LOG.info("Scanning file : " + each_file) 286 | status = validate_file(each_file) 287 | if not status: 288 | LOG.info("Validation NOT successful") 289 | sys.exit(EXIT_FAILURE) 290 | #END FOR 291 | 292 | else: 293 | ## Validate all files in source and test directories 294 | for directory in DEFAULT_DIRS: 295 | LOG.info("Scanning directory : " + directory) 296 | 297 | status = validate_dir(directory) 298 | if not status: 299 | LOG.info("Validation NOT successful") 300 | sys.exit(EXIT_FAILURE) 301 | #END FOR 302 | 303 | LOG.info("Validation successful") 304 | sys.exit(EXIT_SUCCESS) 305 | -------------------------------------------------------------------------------- /src/bucket.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | #include "bucket.h" 5 | #include 6 | #include 7 | #include "cursor.h" 8 | #include "db.h" 9 | #include "meta.h" 10 | #include "node.h" 11 | #include "txn.h" 12 | 13 | namespace boltDB_CPP { 14 | 15 | Bucket *Bucket::newBucket(Txn *tx_p) { 16 | auto bucket = tx_p->pool.allocate(tx_p); 17 | return bucket; 18 | } 19 | 20 | Cursor *Bucket::createCursor() { 21 | tx->increaseCurserCount(); 22 | auto ret = tx->pool.allocate(this); 23 | // std::cout << std::showbase << std::hex << (void *) ret << std::endl; 24 | return ret; 25 | } 26 | 27 | void Bucket::getPageNode(page_id pageId_p, Node *&node_p, Page *&page_p) { 28 | node_p = nullptr; 29 | page_p = nullptr; 30 | 31 | if (bucketHeader.rootPageId == 0) { 32 | if (pageId_p) { 33 | assert(false); 34 | } 35 | if (rootNode) { 36 | node_p = rootNode; 37 | return; 38 | } 39 | page_p = page; 40 | return; 41 | } 42 | if (!nodes.empty()) { 43 | auto iter = nodes.find(pageId_p); 44 | if (iter != nodes.end()) { 45 | node_p = iter->second; 46 | return; 47 | } 48 | } 49 | 50 | page_p = tx->getPage(pageId_p); 51 | return; 52 | } 53 | 54 | // create a node from a page and associate it with a given parent 55 | Node *Bucket::getNode(page_id pageId, Node *parent) { 56 | //if the page is in the cache, return it 57 | auto iter = nodes.find(pageId); 58 | if (iter != nodes.end()) { 59 | return iter->second; 60 | } 61 | 62 | //make a new node 63 | auto node = tx->pool.allocate(this, parent); 64 | 65 | if (parent == nullptr) { 66 | assert(rootNode == nullptr); 67 | rootNode = node; 68 | } else { 69 | parent->addChild(node); 70 | } 71 | 72 | Page *p = page; 73 | if (p == nullptr) { 74 | p = tx->getPage(pageId); 75 | } 76 | 77 | node->read(p); 78 | //add this new created node to node cache 79 | nodes[pageId] = node; 80 | 81 | tx->increaseNodeCount(); 82 | 83 | return node; 84 | } 85 | 86 | Bucket *Bucket::getBucketByName(const Item &searchKey) { 87 | auto iter = buckets.find(searchKey); 88 | if (iter != buckets.end()) { 89 | return iter->second; 90 | } 91 | 92 | auto cursor = createCursor(); 93 | Item key; 94 | Item value; 95 | uint32_t flag = 0; 96 | cursor->seek(searchKey, key, value, flag); 97 | if (searchKey != key || 98 | (flag & static_cast(PageFlag::bucketLeafFlag)) == 0) { 99 | return nullptr; 100 | } 101 | 102 | auto result = openBucket(value); 103 | buckets[searchKey] = result; 104 | return result; 105 | } 106 | 107 | Bucket *Bucket::createBucket(const Item &key) { 108 | if (tx->db == nullptr || !tx->writable || key.length == 0) { 109 | std::cerr << "invalid param " << std::endl; 110 | return nullptr; 111 | } 112 | auto c = Cursor(this); 113 | Item k; 114 | Item v; 115 | uint32_t flag; 116 | c.seek(key, k, v, flag); 117 | 118 | if (k == key) { 119 | if (flag & static_cast(PageFlag::bucketLeafFlag)) { 120 | std::cerr << "key already exists" << std::endl; 121 | return getBucketByName(key); 122 | } 123 | return nullptr; 124 | } 125 | 126 | // create an empty inline bucket 127 | Bucket bucket(this->tx); 128 | bucket.rootNode = tx->pool.allocate(&bucket, nullptr); 129 | bucket.rootNode->markLeaf(); 130 | 131 | // todo:use memory pool to fix memory leak 132 | Item putValue = bucket.write(); 133 | 134 | c.getNode()->put(key, key, putValue, 0, 135 | static_cast(PageFlag::bucketLeafFlag)); 136 | 137 | // this is not inline bucket any more 138 | page = nullptr; 139 | return getBucketByName(key); 140 | } 141 | 142 | Bucket *Bucket::openBucket(const Item &value) { 143 | auto child = newBucket(tx); 144 | // 145 | // this may result in un-equivalent to the original purpose 146 | // in boltDB, it saves the pointer to mmapped file. 147 | // it's reinterpreting value on read-only txn, make a copy in writable one 148 | // here I don't make 'value' a pointer. 149 | // reimplementation is needed if value is shared among read txns 150 | // and update in mmap file is reflected through 'bucket' field 151 | // 152 | 153 | if (child->tx->isWritable()) { 154 | std::memcpy((char *) &child->bucketHeader, value.pointer, 155 | sizeof(bucketHeader)); 156 | } else { 157 | child->bucketHeader = 158 | *(reinterpret_cast(const_cast(value.pointer))); 159 | } 160 | 161 | // is this a inline bucket? 162 | if (child->bucketHeader.rootPageId == 0) { 163 | child->page = reinterpret_cast( 164 | const_cast(&value.pointer[BUCKETHEADERSIZE])); 165 | } 166 | return child; 167 | } 168 | 169 | // serialize bucket header & rootNode 170 | Item Bucket::write() { 171 | size_t length = BUCKETHEADERSIZE + rootNode->size(); 172 | assert(tx); 173 | char *result = tx->pool.allocateByteArray(length); 174 | 175 | // write bucketHeader in the front 176 | *(reinterpret_cast(result)) = bucketHeader; 177 | 178 | // serialize node after bucketHeader 179 | auto pageInBuffer = (Page *) &result[BUCKETHEADERSIZE]; 180 | rootNode->write(pageInBuffer); 181 | 182 | return Item{result, length}; 183 | } 184 | 185 | Bucket *Bucket::createBucketIfNotExists(const Item &key) { 186 | auto child = createBucket(key); 187 | return child; 188 | } 189 | 190 | int Bucket::deleteBucket(const Item &key) { 191 | if (tx->db == nullptr || !isWritable()) { 192 | return -1; 193 | } 194 | auto c = createCursor(); 195 | Item k; 196 | Item v; 197 | uint32_t flag; 198 | c->seek(key, k, v, flag); 199 | if (k != key || flag & static_cast(PageFlag::bucketLeafFlag)) { 200 | return -1; 201 | } 202 | 203 | auto child = getBucketByName(key); 204 | auto ret = for_each([&child](const Item &k, const Item &v) { 205 | if (v.length == 0) { 206 | auto ret = child->deleteBucket(k); 207 | if (ret != 0) { 208 | return ret; 209 | } 210 | } 211 | return 0; 212 | }); 213 | if (ret != 0) { 214 | return ret; 215 | } 216 | 217 | // remove cache 218 | buckets.erase(key); 219 | 220 | child->nodes.clear(); 221 | child->rootNode = nullptr; 222 | child->free(); 223 | 224 | c->getNode()->del(key); 225 | 226 | return 0; 227 | } 228 | 229 | int Bucket::for_each(std::function fn) { 230 | if (tx->db == nullptr) { 231 | return -1; 232 | } 233 | auto c = createCursor(); 234 | Item k; 235 | Item v; 236 | c->first(k, v); 237 | while (k.length != 0) { 238 | auto ret = fn(k, v); 239 | if (ret != 0) { 240 | return ret; 241 | } 242 | c->next(k, v); 243 | } 244 | return 0; 245 | } 246 | 247 | void Bucket::free() { 248 | if (bucketHeader.rootPageId == 0) { 249 | return; 250 | } 251 | 252 | for_each_page_node([this](Page *p, Node *n, int) { 253 | if (p) { 254 | tx->free(tx->metaData->txnId, p); 255 | } else { 256 | assert(n); 257 | n->free(); 258 | } 259 | }); 260 | 261 | bucketHeader.rootPageId = 0; 262 | } 263 | 264 | void Bucket::for_each_page_node(std::function fn) { 265 | if (page) { 266 | fn(page, nullptr, 0); 267 | return; 268 | } 269 | for_each_page_node_impl(getRootPage(), 0, fn); 270 | } 271 | 272 | void Bucket::for_each_page_node_impl( 273 | page_id pid, int depth, std::function fn) { 274 | Node *node; 275 | Page *page; 276 | getPageNode(pid, node, page); 277 | 278 | fn(page, node, depth); 279 | if (page) { 280 | if (isSet(page->flag, PageFlag::branchPageFlag)) { 281 | for (size_t i = 0; i < page->getCount(); i++) { 282 | auto element = page->getBranchPageElement(i); 283 | for_each_page_node_impl(element->pageId, depth + 1, fn); 284 | } 285 | } 286 | } else { 287 | if (!node->isLeafNode()) { 288 | for (auto pid : node->branchPageIds()) { 289 | for_each_page_node_impl(pid, depth + 1, fn); 290 | } 291 | } 292 | } 293 | } 294 | 295 | void Bucket::dereference() { 296 | if (rootNode) { 297 | rootNode->root()->dereference(); 298 | } 299 | 300 | for (auto item : buckets) { 301 | item.second->dereference(); 302 | } 303 | } 304 | 305 | /** 306 | * this is merging node which has elements below threshold 307 | */ 308 | void Bucket::rebalance() { 309 | for (auto &item : nodes) { 310 | item.second->rebalance(); 311 | } 312 | 313 | for (auto &item : buckets) { 314 | item.second->rebalance(); 315 | } 316 | } 317 | 318 | char *Bucket::cloneBytes(const Item &key, size_t *retSz) { 319 | if (retSz) { 320 | *retSz = key.length; 321 | } 322 | auto result(new char[key.length]); 323 | std::memcpy(result, key.pointer, key.length); 324 | return result; 325 | } 326 | 327 | Item Bucket::get(const Item &key) { 328 | Item k; 329 | Item v; 330 | uint32_t flag = 0; 331 | createCursor()->seek(key, k, v, flag); 332 | if (isSet(flag, PageFlag::bucketLeafFlag) || k != key) { 333 | return {}; 334 | } 335 | return v; 336 | } 337 | 338 | int Bucket::put(const Item &key, const Item &value) { 339 | if (tx->db == nullptr || !isWritable() || key.length == 0 || 340 | key.length > MAXKEYSIZE || value.length > MAXVALUESIZE) { 341 | return -1; 342 | } 343 | 344 | auto c = createCursor(); 345 | Item k; 346 | Item v; 347 | uint32_t flag = 0; 348 | 349 | c->seek(key, k, v, flag); 350 | 351 | if (k == key && isSet(flag, PageFlag::bucketLeafFlag)) { 352 | return -1; 353 | } 354 | 355 | c->getNode()->put(key, key, value, 0, 0); 356 | 357 | return 0; 358 | } 359 | 360 | int Bucket::remove(const Item &key) { 361 | if (tx->db == nullptr || !isWritable()) { 362 | return -1; 363 | } 364 | 365 | auto c = createCursor(); 366 | Item k; 367 | Item v; 368 | uint32_t flag = 0; 369 | 370 | c->seek(key, k, v, flag); 371 | 372 | if (isBucketLeaf(flag)) { 373 | return -1; 374 | } 375 | 376 | c->getNode()->del(key); 377 | return 0; 378 | } 379 | 380 | uint64_t Bucket::sequence() { return bucketHeader.sequence; } 381 | 382 | int Bucket::setSequence(uint64_t v) { 383 | if (tx->db == nullptr || !isWritable()) { 384 | return -1; 385 | } 386 | if (rootNode == nullptr) { 387 | getNode(getRootPage(), nullptr); 388 | } 389 | 390 | bucketHeader.sequence = v; 391 | return 0; 392 | } 393 | 394 | int Bucket::nextSequence(uint64_t &v) { 395 | if (tx->db == nullptr || !isWritable()) { 396 | return -1; 397 | } 398 | if (rootNode == nullptr) { 399 | getNode(getRootPage(), nullptr); 400 | } 401 | bucketHeader.sequence++; 402 | v = bucketHeader.sequence; 403 | return 0; 404 | } 405 | 406 | void Bucket::for_each_page(std::function fn) { 407 | if (page) { 408 | fn(page, 0); 409 | return; 410 | } 411 | 412 | tx->for_each_page(getRootPage(), 0, fn); 413 | } 414 | 415 | size_t Bucket::maxInlineBucketSize() { 416 | return boltDB_CPP::DB::getPageSize() / 4; 417 | } 418 | 419 | bool Bucket::isInlineable() { 420 | auto r = rootNode; 421 | if (r == nullptr || !r->isLeafNode()) { 422 | return false; 423 | } 424 | 425 | return r->isinlineable(maxInlineBucketSize()); 426 | } 427 | 428 | int Bucket::spill() { 429 | for (auto item : buckets) { 430 | auto name = item.first; 431 | auto child = item.second; 432 | 433 | Item newValue; 434 | 435 | if (child->isInlineable()) { 436 | child->free(); 437 | newValue = child->write(); 438 | } else { 439 | if (child->spill()) { 440 | return -1; 441 | } 442 | 443 | newValue.length = sizeof(bucketHeader); 444 | auto ptr = tx->pool.allocateByteArray(newValue.length); 445 | *(reinterpret_cast(ptr)) = child->bucketHeader; 446 | newValue.pointer = ptr; 447 | } 448 | 449 | if (child->rootNode == nullptr) { 450 | continue; 451 | } 452 | 453 | auto c = createCursor(); 454 | Item k; 455 | Item v; 456 | uint32_t flag = 0; 457 | 458 | c->seek(name, k, v, flag); 459 | 460 | if (k != name) { 461 | assert(false); 462 | } 463 | 464 | if (!isBucketLeaf(flag)) { 465 | assert(false); 466 | } 467 | 468 | c->getNode()->put(name, name, newValue, 0, 469 | static_cast(PageFlag::bucketLeafFlag)); 470 | } 471 | 472 | if (rootNode == nullptr) { 473 | return 0; 474 | } 475 | 476 | auto ret = rootNode->spill(); 477 | if (ret) { 478 | return ret; 479 | } 480 | 481 | rootNode = rootNode->root(); 482 | 483 | if (rootNode->getPageId() >= tx->metaData->totalPageNumber) { 484 | assert(false); 485 | } 486 | 487 | bucketHeader.rootPageId = rootNode->getPageId(); 488 | return 0; 489 | } 490 | 491 | bool Bucket::isWritable() const { return tx->isWritable(); } 492 | 493 | void Bucket::reset() { 494 | tx = nullptr; 495 | page = nullptr; // useful for inline buckets, page points to beginning of the 496 | // serialized value i.e. a page' header 497 | rootNode = nullptr; 498 | bucketHeader.reset(); 499 | buckets.clear(); // subbucket cache. used if txn is writable. k:bucket name 500 | nodes.clear(); // node cache. used if txn is writable 501 | } 502 | size_t Bucket::getTotalPageNumber() const { 503 | return tx->metaData->totalPageNumber; 504 | } 505 | MemoryPool &Bucket::getPool() { return tx->pool; } 506 | } // namespace boltDB_CPP -------------------------------------------------------------------------------- /src/cursor.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-27. 3 | // 4 | 5 | #include "cursor.h" 6 | #include 7 | #include 8 | #include 9 | #include "bucket.h" 10 | #include "db.h" 11 | #include "node.h" 12 | #include "txn.h" 13 | #include "util.h" 14 | 15 | namespace boltDB_CPP { 16 | 17 | bool ElementRef::isLeaf() const { 18 | if (node != nullptr) { 19 | return node->isLeaf; 20 | } 21 | assert(page); 22 | return (page->flag & static_cast(PageFlag::leafPageFlag)) != 0; 23 | } 24 | 25 | size_t ElementRef::count() const { 26 | if (node != nullptr) { 27 | return node->inodeList.size(); 28 | } 29 | assert(page); 30 | return page->count; 31 | } 32 | 33 | void Cursor::keyValue(Item &key, Item &value, uint32_t &flag) { 34 | if (dq.empty()) { 35 | key.reset(); 36 | value.reset(); 37 | flag = 0; 38 | return; 39 | } 40 | 41 | auto ref = dq.back(); 42 | if (ref.count() == 0 || ref.index >= ref.count()) { 43 | std::cerr << "get Key/value from empty bucket / index out of range" 44 | << std::endl; 45 | return; 46 | } 47 | 48 | // are those values sitting a node? 49 | if (ref.node) { 50 | auto inode = ref.node->getInode(ref.index); 51 | key = inode.Key(); 52 | value = inode.Value(); 53 | flag = inode.flag; 54 | return; 55 | } 56 | 57 | // let's get them from page 58 | auto ret = ref.page->getLeafPageElement(ref.index); 59 | key = ret->Key(); 60 | value = ret->Value(); 61 | flag = ret->flag; 62 | return; 63 | } 64 | 65 | void Cursor::search(const Item &key, page_id pageId) { 66 | Node *node = nullptr; 67 | Page *page = nullptr; 68 | bucket->getPageNode(pageId, node, page); 69 | if (page && 70 | (page->getFlag() & (static_cast(PageFlag::branchPageFlag) | 71 | static_cast(PageFlag::leafPageFlag))) == 0) { 72 | assert(false); 73 | } 74 | ElementRef ref{page, node}; 75 | dq.push_back(ref); 76 | if (ref.isLeaf()) { 77 | searchLeaf(key); 78 | return; 79 | } 80 | 81 | if (node) { 82 | searchBranchNode(key, node); 83 | return; 84 | } 85 | searchBranchPage(key, page); 86 | } 87 | 88 | void Cursor::searchLeaf(const Item &key) { 89 | assert(!dq.empty()); 90 | ElementRef &ref = dq.back(); 91 | 92 | bool found = false; 93 | if (ref.node) { 94 | // search through inodeList for a matching Key 95 | // inodelist should be sorted in ascending order 96 | ref.index = static_cast(ref.node->search(key, found)); 97 | return; 98 | } 99 | 100 | auto ptr = ref.page->getLeafPageElement(0); 101 | ref.index = static_cast(binary_search( 102 | ptr, key, cmp_wrapper, ref.page->count, found)); 103 | } 104 | 105 | void Cursor::searchBranchNode(const Item &key, Node *node) { 106 | bool found = false; 107 | auto index = node->search(key, found); 108 | if (!found && index > 0) { 109 | index--; 110 | } 111 | assert(!dq.empty()); 112 | dq.back().index = index; 113 | search(key, node->getInode(index).pageId); 114 | } 115 | 116 | void Cursor::searchBranchPage(const Item &key, Page *page) { 117 | auto branchElements = page->getBranchPageElement(0); 118 | bool found = false; 119 | auto index = binary_search( 120 | branchElements, key, cmp_wrapper, page->count, found); 121 | if (!found && index > 0) { 122 | index--; 123 | } 124 | assert(!dq.empty()); 125 | dq.back().index = index; 126 | search(key, branchElements[index].pageId); 127 | } 128 | 129 | void Cursor::do_seek(Item searchKey, Item &key, Item &value, uint32_t &flag) { 130 | dq.clear(); 131 | search(searchKey, bucket->getRootPage()); 132 | 133 | auto &ref = dq.back(); 134 | if (ref.index >= ref.count()) { 135 | key.reset(); 136 | value.reset(); 137 | flag = 0; 138 | return; 139 | } 140 | keyValue(key, value, flag); 141 | } 142 | 143 | /** 144 | * refactory this after main components are implemented 145 | * @return 146 | */ 147 | Node *Cursor::getNode() const { 148 | if (!dq.empty() && dq.back().node && dq.back().isLeaf()) { 149 | return dq.back().node; 150 | } 151 | 152 | std::vector v(dq.begin(), dq.end()); 153 | 154 | assert(!v.empty()); 155 | Node *node = v[0].node; 156 | if (node == nullptr) { 157 | node = bucket->getNode(v[0].page->pageId, nullptr); 158 | } 159 | 160 | //the last one should be a leaf node 161 | //transverse every branch node 162 | for (size_t i = 0; i + 1 < v.size(); i++) { 163 | assert(!node->isLeafNode()); 164 | node = node->childAt(v[i].index); 165 | } 166 | 167 | assert(node->isLeafNode()); 168 | return node; 169 | } 170 | 171 | void Cursor::do_next(Item &key, Item &value, uint32_t &flag) { 172 | while (true) { 173 | while (!dq.empty()) { 174 | auto &ref = dq.back(); 175 | // not the last element 176 | if (ref.index + 1 < ref.count()) { 177 | ref.index++; 178 | break; 179 | } 180 | dq.pop_back(); 181 | } 182 | 183 | if (dq.empty()) { 184 | key.reset(); 185 | value.reset(); 186 | flag = 0; 187 | return; 188 | } 189 | 190 | do_first(); 191 | // not sure what this intends to do 192 | if (dq.back().count() == 0) { 193 | continue; 194 | } 195 | 196 | keyValue(key, value, flag); 197 | return; 198 | } 199 | } 200 | 201 | // get to first leaf element under the last page in the stack 202 | void Cursor::do_first() { 203 | while (true) { 204 | assert(!dq.empty()); 205 | if (dq.back().isLeaf()) { 206 | break; 207 | } 208 | 209 | auto &ref = dq.back(); 210 | page_id pageId = 0; 211 | if (ref.node != nullptr) { 212 | pageId = ref.node->getInode(ref.index).pageId; 213 | } else { 214 | pageId = ref.page->getBranchPageElement(ref.index)->pageId; 215 | } 216 | 217 | Page *page = nullptr; 218 | Node *node = nullptr; 219 | bucket->getPageNode(pageId, node, page); 220 | ElementRef element(page, node); 221 | dq.push_back(element); 222 | } 223 | } 224 | 225 | void Cursor::do_last() { 226 | while (true) { 227 | assert(!dq.empty()); 228 | auto &ref = dq.back(); 229 | if (ref.isLeaf()) { 230 | break; 231 | } 232 | 233 | page_id pageId = 0; 234 | if (ref.node != nullptr) { 235 | pageId = ref.node->getInode(ref.index).pageId; 236 | } else { 237 | pageId = ref.page->getBranchPageElement(ref.index)->pageId; 238 | } 239 | 240 | Page *page = nullptr; 241 | Node *node = nullptr; 242 | bucket->getPageNode(pageId, node, page); 243 | ElementRef element(page, node); 244 | element.index = element.count() - 1; 245 | dq.push_back(element); 246 | } 247 | } 248 | 249 | int Cursor::remove() { 250 | if (bucket->getTxn()->db == nullptr) { 251 | std::cerr << "db closed" << std::endl; 252 | return -1; 253 | } 254 | 255 | if (!bucket->isWritable()) { 256 | std::cerr << "txn not writable" << std::endl; 257 | return -1; 258 | } 259 | 260 | Item key; 261 | Item value; 262 | uint32_t flag; 263 | keyValue(key, value, flag); 264 | 265 | if (flag & static_cast(PageFlag::bucketLeafFlag)) { 266 | std::cerr << "current value is a bucket| try removing a branch bucket " 267 | "other than kv in leaf node" 268 | << std::endl; 269 | return -1; 270 | } 271 | 272 | getNode()->do_remove(key); 273 | return 0; 274 | } 275 | 276 | void Cursor::seek(const Item &searchKey, Item &key, Item &value, 277 | uint32_t &flag) { 278 | key.reset(); 279 | value.reset(); 280 | flag = 0; 281 | do_seek(searchKey, key, value, flag); 282 | auto &ref = dq.back(); 283 | if (ref.index >= ref.count()) { 284 | flag = 0; 285 | key.reset(); 286 | value.reset(); 287 | return; 288 | } 289 | keyValue(key, value, flag); 290 | } 291 | 292 | void Cursor::prev(Item &key, Item &value) { 293 | key.reset(); 294 | value.reset(); 295 | while (!dq.empty()) { 296 | auto &ref = dq.back(); 297 | if (ref.index > 0) { 298 | ref.index--; 299 | break; 300 | } 301 | dq.pop_back(); 302 | } 303 | 304 | if (dq.empty()) { 305 | return; 306 | } 307 | 308 | do_last(); 309 | uint32_t flag = 0; 310 | keyValue(key, value, flag); 311 | // I think there's no need to clear value if current node is a branch node 312 | } 313 | 314 | void Cursor::next(Item &key, Item &value) { 315 | key.reset(); 316 | value.reset(); 317 | uint32_t flag = 0; 318 | do_next(key, value, flag); 319 | } 320 | 321 | void Cursor::last(Item &key, Item &value) { 322 | key.reset(); 323 | value.reset(); 324 | dq.clear(); 325 | Page *page = nullptr; 326 | Node *node = nullptr; 327 | bucket->getPageNode(bucket->getRootPage(), node, page); 328 | ElementRef element{page, node}; 329 | element.index = element.count() - 1; 330 | dq.push_back(element); 331 | do_last(); 332 | uint32_t flag = 0; 333 | keyValue(key, value, flag); 334 | } 335 | 336 | void Cursor::first(Item &key, Item &value) { 337 | key.reset(); 338 | value.reset(); 339 | dq.clear(); 340 | Page *page = nullptr; 341 | Node *node = nullptr; 342 | bucket->getPageNode(bucket->getRootPage(), node, page); 343 | ElementRef element{page, node}; 344 | 345 | dq.push_back(element); 346 | do_first(); 347 | 348 | uint32_t flag = 0; 349 | // what does this do? 350 | if (dq.back().count() == 0) { 351 | do_next(key, value, flag); 352 | } 353 | 354 | keyValue(key, value, flag); 355 | } 356 | } // namespace boltDB_CPP 357 | -------------------------------------------------------------------------------- /src/db.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-27. 3 | // 4 | 5 | #include "db.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "meta.h" 15 | #include "txn.h" 16 | #include "util.h" 17 | namespace boltDB_CPP { 18 | uint32_t DB::pageSize = DEFAULTPAGESIZE; 19 | Page *boltDB_CPP::DB::pagePointer(page_id pageId) { 20 | assert(pageSize != 0); 21 | uint64_t pos = pageId * pageSize; 22 | return reinterpret_cast(&data[pos]); 23 | } 24 | FreeList &DB::getFreeLIst() { return freeList; } 25 | uint64_t DB::getPageSize() { return pageSize; } 26 | 27 | /** 28 | * return the meta that has maximal txn id, or current txn will not be able to read most recent updates 29 | */ 30 | Meta *DB::meta() { 31 | auto m0 = meta0; 32 | auto m1 = meta1; 33 | if (m0->txnId < m1->txnId) { 34 | m0 = meta1; 35 | m1 = meta0; 36 | } 37 | if (m0->validate()) { 38 | return m0; 39 | } 40 | if (m1->validate()) { 41 | return m1; 42 | } 43 | assert(false); 44 | } 45 | 46 | void DB::removeTxn(Txn *txn) { 47 | mmapLock.readUnlock(); 48 | std::lock_guard guard(metaLock); 49 | 50 | for (auto iter = txs.begin(); iter != txs.end(); iter++) { 51 | if (iter->get() == txn) { 52 | txs.erase(iter); 53 | break; 54 | } 55 | } 56 | } 57 | 58 | int DB::grow(size_t sz) { 59 | if (sz <= fileSize) { 60 | return 0; 61 | } 62 | 63 | if (dataSize <= allocSize) { 64 | sz = dataSize; 65 | } else { 66 | sz += allocSize; 67 | } 68 | 69 | if (!noGrowSync && !readOnly) { 70 | // increase file's size 71 | if (ftruncate(fd, sz)) { 72 | return -1; 73 | } 74 | // make sure that file size is written into metadata 75 | if (fsync(fd)) { 76 | return -1; 77 | } 78 | } 79 | 80 | fileSize = sz; 81 | return 0; 82 | } 83 | 84 | int DB::init() { 85 | //2 meta pages, 1 freelist page, 1 leaf node page 86 | std::vector buf(pageSize * 4); 87 | for (page_id i = 0; i < 2; i++) { 88 | auto p = pageInBuffer(buf.data(), buf.size(), i); 89 | p->pageId = i; 90 | p->flag = pageFlagValue(PageFlag::metaPageFlag); 91 | 92 | auto m = p->metaPointer(); 93 | m->magic = MAGIC; 94 | m->version = VERSION; 95 | m->pageSize = pageSize; 96 | m->freeListPageNumber = 2; 97 | m->rootBucketHeader.rootPageId = 3; 98 | m->totalPageNumber = 4; 99 | m->txnId = i; 100 | 101 | m->checkSum = m->sum64(); 102 | } 103 | 104 | { 105 | //free list page 106 | auto p = pageInBuffer(buf.data(), buf.size(), 2); 107 | p->pageId = 2; 108 | p->flag |= pageFlagValue(PageFlag::freelistPageFlag); 109 | p->count = 0; 110 | } 111 | 112 | { 113 | //leaf node page 114 | auto p = pageInBuffer(buf.data(), buf.size(), 3); 115 | p->pageId = 3; 116 | p->flag |= pageFlagValue(PageFlag::leafPageFlag); 117 | p->count = 0; 118 | } 119 | 120 | ssize_t writtenOut = writeAt(buf.data(), buf.size(), 0); 121 | if (buf.size() != writtenOut) { 122 | return -1; 123 | } 124 | 125 | if (file_data_sync(fd) != 0) { 126 | return -1; 127 | } 128 | 129 | fileSize = 4 * pageSize; 130 | return 0; 131 | } 132 | 133 | Page *DB::pageInBuffer(char *ptr, size_t length, page_id pageId) { 134 | assert(length > pageId * pageSize); 135 | return reinterpret_cast(ptr + pageId * pageSize); 136 | } 137 | 138 | struct OnClose { 139 | explicit OnClose(std::function &&function) : fn(std::move(function)) {} 140 | std::function fn; 141 | ~OnClose() { 142 | if (fn) { 143 | fn(); 144 | } 145 | } 146 | }; 147 | DB *DB::openDB(const std::string &path_p, uint16_t mode, 148 | const Options &options) { 149 | opened = true; 150 | 151 | noGrowSync = options.noGrowSync; 152 | mmapFlags = options.mmapFlag; 153 | 154 | maxBatchDelayMillionSeconds = DEFAULTMAXBATCHDELAYMILLIIONSEC; 155 | maxBatchSize = DEFAULTMAXBATCHSIZE; 156 | allocSize = DEFAULTALLOCATIONSIZE; 157 | 158 | uint32_t flag = O_RDWR; 159 | if (options.readOnly) { 160 | flag = O_RDONLY; 161 | readOnly = true; 162 | } 163 | 164 | this->path = path_p; 165 | 166 | { 167 | // open db file 168 | auto ret = ::open(path.c_str(), flag | O_CREAT, mode); 169 | if (ret == -1) { 170 | perror("open db file"); 171 | return nullptr; 172 | } 173 | this->fd = ret; 174 | } 175 | 176 | { 177 | if (readOnly) { 178 | auto ret = file_Rlock(fd); 179 | if (ret == -1) { 180 | perror("flock read"); 181 | do_closeDB(); 182 | return nullptr; 183 | } 184 | } else { 185 | auto ret = file_Wlock(fd); 186 | if (ret == -1) { 187 | perror("flock write"); 188 | do_closeDB(); 189 | return nullptr; 190 | } 191 | } 192 | } 193 | 194 | fileSize = file_size(fd); 195 | if (fileSize == -1) { 196 | do_closeDB(); 197 | return nullptr; 198 | } 199 | 200 | if (fileSize == 0) { 201 | if (init()) { 202 | do_closeDB(); 203 | return nullptr; 204 | } 205 | } else { 206 | //it is not necessary to read out pagesize, since this implementation use fixed page size 207 | // currently not dealing with corrupted db 208 | std::vector localBuff(0x1000); // 4k page 209 | auto ret = ::pread(fd, localBuff.data(), localBuff.size(), 0); 210 | if (ret == -1) { 211 | do_closeDB(); 212 | return nullptr; 213 | } 214 | auto m = pageInBuffer(localBuff.data(), localBuff.size(), 0)->metaPointer(); 215 | if (!m->validate()) { 216 | pageSize = DEFAULTPAGESIZE; 217 | } else { 218 | pageSize = m->pageSize; 219 | } 220 | } 221 | 222 | // init meta 223 | if (initMeta(options.initalMmapSize)) { 224 | do_closeDB(); 225 | return nullptr; 226 | } 227 | 228 | // init freelist 229 | freeList.read(pagePointer(meta()->freeListPageNumber)); 230 | return this; 231 | } 232 | void DB::closeDB() { 233 | std::lock_guard guard1(readWriteAccessMutex); 234 | std::lock_guard guard2(metaLock); 235 | mmapLock.readLock(); 236 | do_closeDB(); 237 | mmapLock.readUnlock(); 238 | } 239 | 240 | void DB::do_closeDB() { 241 | if (!opened) { 242 | return; 243 | } 244 | 245 | opened = false; 246 | freeList.reset(); 247 | munmap_db_file(); 248 | 249 | if (fd) { 250 | /** 251 | * do not need to unlock SH/EX lock 252 | * it will be released implicitly when fd is closed 253 | */ 254 | // if (!readOnly) { 255 | // file_Unlock(fd); 256 | // } 257 | close(fd); 258 | fd = -1; 259 | } 260 | path.clear(); 261 | } 262 | 263 | int DB::initMeta(off_t minMmapSize) { 264 | mmapLock.writeLock(); 265 | OnClose onClose(std::bind(&RWLock::writeUnlock, &mmapLock)); 266 | 267 | if (fileSize < pageSize * 2) { 268 | // there should be at least 2 page of meta page 269 | return -1; 270 | } 271 | 272 | auto targetSize = std::max(fileSize, minMmapSize); 273 | if (mmapSize(targetSize) == -1) { 274 | return -1; 275 | } 276 | 277 | // dereference before unmapping. deference? 278 | // dereference means make a copy 279 | // clone data which nodes are pointing to 280 | // if not doing this on unmapping, nodes these data points to will be undefined value 281 | if (rwtx) { 282 | rwtx->rootBucket.dereference(); 283 | } 284 | 285 | // unmapping current db file 286 | if (munmap_db_file()) { 287 | return -1; 288 | } 289 | 290 | assert(targetSize > 0); 291 | if (mmap_db_file(this, static_cast(targetSize))) { 292 | return -1; 293 | } 294 | 295 | meta0 = pagePointer(0)->metaPointer(); 296 | meta1 = pagePointer(1)->metaPointer(); 297 | 298 | // if one fails validation, it can be recovered from the other one. 299 | // fail when both meta pages are broken 300 | if (!meta0->validate() && !meta1->validate()) { 301 | return -1; 302 | } 303 | return 0; 304 | } 305 | 306 | int DB::mmapSize(off_t &targetSize) { 307 | // get lowest size not less than targetSize 308 | // from 32k to 1g, double every try 309 | for (size_t i = 15; i <= 30; i++) { 310 | if (targetSize <= (1UL << i)) { 311 | targetSize = 1UL << i; 312 | return 0; 313 | } 314 | } 315 | 316 | if (targetSize > MAXMAPSIZE) { 317 | return -1; 318 | } 319 | 320 | // not dealing with file size larger than 1GB now 321 | assert(false); 322 | std::cerr << "not dealing with db file larger than 1GB now" << std::endl; 323 | exit(1); 324 | return 0; 325 | } 326 | 327 | int DB::update(std::function fn) { 328 | auto tx = beginRWTx(); 329 | if (tx == nullptr) { 330 | return -1; 331 | } 332 | tx->managed = true; 333 | auto ret = fn(tx); 334 | tx->managed = false; 335 | if (ret != 0) { 336 | tx->rollback(); 337 | return -1; 338 | } 339 | 340 | auto result = tx->commit(); 341 | closeTx(tx); 342 | return result; 343 | } 344 | 345 | // when commit a rw txn, readWriteAccessMutex must be released 346 | Txn *DB::beginRWTx() { 347 | // this property will only be set once 348 | if (readOnly) { 349 | return nullptr; 350 | } 351 | 352 | // unlock on commit/rollback; only one writer transaction at a time 353 | readWriteAccessMutex.lock(); 354 | 355 | // exclusively update transaction 356 | std::lock_guard guard(metaLock); 357 | 358 | // this needs to be protected under readWriteAccessMutex/not sure if it needs 359 | // metaLock 360 | if (!opened) { 361 | readWriteAccessMutex.unlock(); 362 | return nullptr; 363 | } 364 | rwtx.reset(new Txn); 365 | rwtx->writable = true; 366 | rwtx->init(this); 367 | 368 | // release pages of finished read only txns 369 | auto minId = UINT64_MAX; 370 | for (auto &item : txs) { 371 | minId = std::min(minId, item->metaData->txnId); 372 | } 373 | 374 | if (minId > 0) { 375 | freeList.release(minId - 1); 376 | } 377 | return rwtx.get(); 378 | } 379 | 380 | // when commit/abort a read only txn, mmaplock must be released 381 | Txn *DB::beginTx() { 382 | std::lock_guard guard(metaLock); 383 | mmapLock.readLock(); 384 | if (!opened) { 385 | mmapLock.readUnlock(); 386 | return nullptr; 387 | } 388 | 389 | txs.emplace_back(new Txn); 390 | auto &txn = txs.back(); 391 | txn->init(this); 392 | return txn.get(); 393 | } 394 | 395 | Page *DB::allocate(size_t count, Txn *txn) { 396 | // buffer len for continuous page 397 | size_t len = count * pageSize; 398 | assert(count < UINT32_MAX); 399 | // allocate memory for these pages 400 | auto page = reinterpret_cast(txn->pool.allocateByteArray(len)); 401 | // set up overflow. overflow + 1 is the total page count 402 | page->overflow = static_cast(count) - 1; 403 | // set up page id 404 | 405 | // 1.allocate page numbers from freelist 406 | auto pid = freeList.allocate(count); 407 | if (pid != 0) { 408 | page->pageId = pid; 409 | return page; 410 | } 411 | 412 | // 2.need to expand mmap file 413 | page->pageId = rwtx->metaData->totalPageNumber; 414 | // no sure what the '1' indicates 415 | size_t minLen = (page->pageId + count + 1) * pageSize; 416 | if (minLen > dataSize) { 417 | struct stat stat1; 418 | { 419 | auto ret = fstat(fd, &stat1); 420 | if (ret == -1) { 421 | perror("stat"); 422 | // closeDB(); 423 | return nullptr; 424 | } 425 | } 426 | if (initMeta(minLen)) { 427 | return nullptr; 428 | } 429 | } 430 | 431 | rwtx->metaData->totalPageNumber += count; 432 | return page; 433 | } 434 | int DB::view(std::function fn) { 435 | auto tx = beginTx(); 436 | if (tx == nullptr) { 437 | return -1; 438 | } 439 | 440 | tx->managed = true; 441 | 442 | auto ret = fn(tx); 443 | 444 | tx->managed = false; 445 | 446 | if (ret != 0) { 447 | tx->rollback(); 448 | closeTx(tx); 449 | return -1; 450 | } 451 | 452 | tx->rollback(); 453 | closeTx(tx); 454 | return 0; 455 | } 456 | void DB::resetData() { 457 | dataSize = 0; 458 | data = nullptr; 459 | dataref = nullptr; 460 | } 461 | void DB::resetData(void *data_p, void *dataref_p, size_t datasz_p) { 462 | data = reinterpret_cast(data_p); 463 | dataref = dataref_p; 464 | dataSize = datasz_p; 465 | } 466 | int DB::munmap_db_file() { 467 | // return if it has no mapping data 468 | if (dataref == nullptr) { 469 | return 0; 470 | } 471 | int ret = ::munmap(dataref, dataSize); 472 | if (ret == -1) { 473 | perror("munmap"); 474 | return ret; 475 | } 476 | 477 | resetData(); 478 | return 0; 479 | } 480 | 481 | void DB::resetRWTX() { 482 | rwtx = nullptr; 483 | } 484 | 485 | void DB::closeTx(Txn *txn) { 486 | if (txn == nullptr) { 487 | return; 488 | } 489 | if (rwtx && rwtx.get() == txn) { 490 | resetRWTX(); 491 | writerLeave(); 492 | } else { 493 | removeTxn(txn); 494 | } 495 | } 496 | int DB::mmap_db_file(DB *database, size_t sz) { 497 | void *ret = ::mmap(nullptr, sz, PROT_READ, MAP_SHARED, database->getFd(), 0); 498 | if (ret == MAP_FAILED) { 499 | perror("mmap"); 500 | return -1; 501 | } 502 | 503 | int retAdvise = ::madvise(ret, sz, MADV_RANDOM); 504 | if (retAdvise == -1) { 505 | perror("madvise"); 506 | return -1; 507 | } 508 | 509 | // database->data = reinterpret_castdata)>(ret); 510 | // database->dataref = (ret); 511 | // database->dataSize = (sz); 512 | database->resetData(ret, ret, sz); 513 | return 0; 514 | } 515 | 516 | /** 517 | * commit the txn, rollback & return -1 on failure 518 | * @param txn 519 | * @return commited ? 0 : -1 520 | */ 521 | int DB::commitTxn(Txn *txn) { 522 | assert(txn); 523 | auto ret = txn->commit(); 524 | closeTx(txn); 525 | return ret; 526 | } 527 | 528 | void DB::rollbackTxn(Txn *txn) { 529 | assert(txn); 530 | txn->rollback(); 531 | } 532 | 533 | void FreeList::free(txn_id tid, Page *page) { 534 | // meta page will never be freed 535 | if (page->pageId <= 1) { 536 | assert(false); 537 | } 538 | auto &idx = pending[tid]; 539 | for (auto iter = page->pageId; iter <= page->pageId + page->overflow; 540 | iter++) { 541 | if (cache[iter]) { 542 | assert(false); 543 | } 544 | 545 | idx.push_back(iter); 546 | cache[iter] = true; 547 | } 548 | } 549 | 550 | size_t FreeList::size() const { 551 | auto ret = count(); 552 | 553 | if (ret >= 0xffff) { 554 | ret++; 555 | } 556 | 557 | return PAGEHEADERSIZE + ret * sizeof(page_id); 558 | } 559 | 560 | size_t FreeList::count() const { return free_count() + pending_count(); } 561 | 562 | size_t FreeList::free_count() const { return pageIds.size(); } 563 | 564 | size_t FreeList::pending_count() const { 565 | size_t result = 0; 566 | for (auto &item : pending) { 567 | result += item.second.size(); 568 | } 569 | return result; 570 | } 571 | 572 | void FreeList::copyall(std::vector &dest) { 573 | std::vector tmp; 574 | for (auto item : pending) { 575 | std::copy(item.second.begin(), item.second.end(), std::back_inserter(tmp)); 576 | } 577 | std::sort(tmp.begin(), tmp.end()); 578 | mergePageIds(dest, tmp, pageIds); 579 | } 580 | 581 | page_id FreeList::allocate(size_t sz) { 582 | if (pageIds.empty()) { 583 | return 0; 584 | } 585 | 586 | page_id init = 0; 587 | page_id prev = 0; 588 | 589 | for (size_t i = 0; i < pageIds.size(); i++) { 590 | page_id id = pageIds[i]; 591 | if (id <= 1) { 592 | assert(false); 593 | } 594 | 595 | if (prev == 0 || id - prev != 1) { 596 | init = id; 597 | } 598 | 599 | if (id - init + 1 == sz ) { 600 | for (size_t j = 0; j < sz; j++) { 601 | pageIds[i + 1 - sz + j] = pageIds[i + 1 + j]; 602 | } 603 | 604 | for(size_t j = 0; j < sz; j++){ 605 | pageIds.pop_back(); 606 | } 607 | 608 | for (size_t j = 0; j < sz; j++) { 609 | cache.erase(init + j); 610 | } 611 | 612 | return init; 613 | } 614 | 615 | prev = id; 616 | } 617 | 618 | return 0; 619 | } 620 | 621 | /** 622 | * release pages belong to txns of tid from zero to parameter tid 623 | * @param tid : largest tid of txn whose pages are to be freed 624 | */ 625 | void FreeList::release(txn_id tid) { 626 | std::vector list; 627 | for (auto iter = pending.begin(); iter != pending.end();) { 628 | if (iter->first <= tid) { 629 | std::copy(iter->second.begin(), iter->second.end(), 630 | std::back_inserter(list)); 631 | iter = pending.erase(iter); 632 | } else { 633 | iter++; 634 | } 635 | } 636 | 637 | std::sort(list.begin(), list.end()); 638 | pageIds = merge(pageIds, list); 639 | } 640 | 641 | void FreeList::rollback(txn_id tid) { 642 | for (auto item : pending[tid]) { 643 | cache.erase(item); 644 | } 645 | 646 | pending.erase(tid); 647 | } 648 | 649 | bool FreeList::freed(page_id pageId) { return cache[pageId]; } 650 | 651 | void FreeList::read(Page *page) { 652 | size_t idx = 0; 653 | size_t count = page->count; 654 | if (count == 0xffff) { 655 | idx = 1; 656 | count = *reinterpret_cast(&page->ptr); 657 | } 658 | 659 | if (count == 0) { 660 | pageIds.clear(); 661 | } else { 662 | pageIds.clear(); 663 | page_id *ptr = reinterpret_cast(&page->ptr) + idx; 664 | for (size_t i = 0; i < count; i++) { 665 | pageIds.push_back(*ptr); 666 | ptr++; 667 | } 668 | 669 | std::sort(pageIds.begin(), pageIds.end()); 670 | } 671 | 672 | reindex(); 673 | } 674 | 675 | void FreeList::reindex() { 676 | cache.clear(); 677 | for (auto item : pageIds) { 678 | cache[item] = true; 679 | } 680 | 681 | for (auto &item : pending) { 682 | for (auto inner : item.second) { 683 | cache[inner] = true; 684 | } 685 | } 686 | } 687 | 688 | // curious about how to deal with overflow pages 689 | // allocate will return a continuous block of memory 690 | // if the write overflow one page, it should be written into the next page 691 | // but I havn't find out in which procedure the overflow is updated 692 | int FreeList::write(Page *page) { 693 | page->flag |= static_cast(PageFlag::freelistPageFlag); 694 | auto count = this->count(); 695 | if (count == 0) { 696 | page->count = static_cast(count); 697 | } else if (count < 0xffff) { 698 | page->count = static_cast(count); 699 | // re-implement it to avoid copying later 700 | std::vector dest; 701 | copyall(dest); 702 | auto ptr = reinterpret_cast(&page->ptr); 703 | for (auto item : dest) { 704 | *ptr = item; 705 | ptr++; 706 | } 707 | } else { 708 | page->count = 0xffff; 709 | auto ptr = reinterpret_cast(&page->ptr); 710 | *ptr = count; 711 | std::vector dest; 712 | copyall(dest); 713 | ptr++; 714 | for (auto item : dest) { 715 | *ptr = item; 716 | ptr++; 717 | } 718 | } 719 | return 0; 720 | } 721 | 722 | void FreeList::reload(Page *page) { 723 | read(page); 724 | 725 | // filter out current pending pages from those just read from page file 726 | std::map curPending; 727 | for (auto item : pending) { 728 | for (auto inner : item.second) { 729 | curPending[inner] = true; 730 | } 731 | } 732 | 733 | std::vector newIds; 734 | for (auto item : pageIds) { 735 | if (curPending.find(item) == curPending.end()) { 736 | newIds.push_back(item); 737 | } 738 | } 739 | 740 | pageIds = newIds; 741 | reindex(); 742 | } 743 | void FreeList::reset() { 744 | pageIds.clear(); 745 | pending.clear(); 746 | cache.clear(); 747 | } 748 | } // namespace boltDB_CPP -------------------------------------------------------------------------------- /src/fnv/hash_64a.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-9. 3 | // 4 | 5 | /* 6 | * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code 7 | * 8 | * @(#) $Revision: 5.1 $ 9 | * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ 10 | * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ 11 | * 12 | *** 13 | * 14 | * Fowler/Noll/Vo hash 15 | * 16 | * The basis of this hash algorithm was taken from an idea sent 17 | * as reviewer comments to the IEEE POSIX P1003.2 committee by: 18 | * 19 | * Phong Vo (http://www.research.att.com/info/kpv/) 20 | * Glenn Fowler (http://www.research.att.com/~gsf/) 21 | * 22 | * In a subsequent ballot round: 23 | * 24 | * Landon Curt Noll (http://www.isthe.com/chongo/) 25 | * 26 | * improved on their algorithm. Some people tried this hash 27 | * and found that it worked rather well. In an EMail message 28 | * to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash. 29 | * 30 | * FNV hashes are designed to be fast while maintaining a low 31 | * collision rate. The FNV speed allows one to quickly hash lots 32 | * of data while maintaining a reasonable collision rate. See: 33 | * 34 | * http://www.isthe.com/chongo/tech/comp/fnv/index.html 35 | * 36 | * for more details as well as other forms of the FNV hash. 37 | * 38 | *** 39 | * 40 | * To use the recommended 64 bit FNV-1a hash, pass FNV1A_64_INIT as the 41 | * Fnv64_t hashval argument to fnv_64a_buf() or fnv_64a_str(). 42 | * 43 | *** 44 | * 45 | * Please do not copyright this code. This code is in the public domain. 46 | * 47 | * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 48 | * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO 49 | * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR 50 | * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF 51 | * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 52 | * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 53 | * PERFORMANCE OF THIS SOFTWARE. 54 | * 55 | * By: 56 | * chongo /\oo/\ 57 | * http://www.isthe.com/chongo/ 58 | * 59 | * Share and Enjoy! :-) 60 | */ 61 | 62 | #include 63 | #include "fnv/fnv.h" 64 | 65 | /* 66 | * FNV-1a defines the initial basis to be non-zero 67 | */ 68 | #if !defined(HAVE_64BIT_LONG_LONG) 69 | const Fnv64_t fnv1a_64_init = {0x84222325, 0xcbf29ce4}; 70 | #endif /* ! HAVE_64BIT_LONG_LONG */ 71 | 72 | 73 | /* 74 | * 64 bit magic FNV-1a prime 75 | */ 76 | #if defined(HAVE_64BIT_LONG_LONG) 77 | #define FNV_64_PRIME ((Fnv64_t)0x100000001b3ULL) 78 | #else /* HAVE_64BIT_LONG_LONG */ 79 | #define FNV_64_PRIME_LOW ((unsigned long)0x1b3) /* lower bits of FNV prime */ 80 | #define FNV_64_PRIME_SHIFT (8) /* top FNV prime shift above 2^32 */ 81 | #endif /* HAVE_64BIT_LONG_LONG */ 82 | 83 | /* 84 | * fnv_64a_buf - perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer 85 | * 86 | * input: 87 | * buf - start of buffer to hash 88 | * len - length of buffer in octets 89 | * hval - previous hash value or 0 if first call 90 | * 91 | * returns: 92 | * 64 bit hash as a static hash type 93 | * 94 | * NOTE: To use the recommended 64 bit FNV-1a hash, use FNV1A_64_INIT as the 95 | * hval arg on the first call to either fnv_64a_buf() or fnv_64a_str(). 96 | */ 97 | Fnv64_t 98 | fnv_64a_buf(void *buf, size_t len, Fnv64_t hval) { 99 | unsigned char *bp = (unsigned char *) buf; /* start of buffer */ 100 | unsigned char *be = bp + len; /* beyond end of buffer */ 101 | 102 | #if defined(HAVE_64BIT_LONG_LONG) 103 | /* 104 | * FNV-1a hash each octet of the buffer 105 | */ 106 | while (bp < be) { 107 | 108 | /* xor the bottom with the current octet */ 109 | hval ^= (Fnv64_t)*bp++; 110 | 111 | /* multiply by the 64 bit FNV magic prime mod 2^64 */ 112 | #if defined(NO_FNV_GCC_OPTIMIZATION) 113 | hval *= FNV_64_PRIME; 114 | #else /* NO_FNV_GCC_OPTIMIZATION */ 115 | hval += (hval << 1) + (hval << 4) + (hval << 5) + 116 | (hval << 7) + (hval << 8) + (hval << 40); 117 | #endif /* NO_FNV_GCC_OPTIMIZATION */ 118 | } 119 | 120 | #else /* HAVE_64BIT_LONG_LONG */ 121 | 122 | unsigned long val[4]; /* hash value in base 2^16 */ 123 | unsigned long tmp[4]; /* tmp 64 bit value */ 124 | 125 | /* 126 | * Convert Fnv64_t hval into a base 2^16 array 127 | */ 128 | val[0] = hval.w32[0]; 129 | val[1] = (val[0] >> 16); 130 | val[0] &= 0xffff; 131 | val[2] = hval.w32[1]; 132 | val[3] = (val[2] >> 16); 133 | val[2] &= 0xffff; 134 | 135 | /* 136 | * FNV-1a hash each octet of the buffer 137 | */ 138 | while (bp < be) { 139 | 140 | /* xor the bottom with the current octet */ 141 | val[0] ^= (unsigned long) *bp++; 142 | 143 | /* 144 | * multiply by the 64 bit FNV magic prime mod 2^64 145 | * 146 | * Using 0x100000001b3 we have the following digits base 2^16: 147 | * 148 | * 0x0 0x100 0x0 0x1b3 149 | * 150 | * which is the same as: 151 | * 152 | * 0x0 1<> 16); 164 | val[0] = tmp[0] & 0xffff; 165 | tmp[2] += (tmp[1] >> 16); 166 | val[1] = tmp[1] & 0xffff; 167 | val[3] = tmp[3] + (tmp[2] >> 16); 168 | val[2] = tmp[2] & 0xffff; 169 | /* 170 | * Doing a val[3] &= 0xffff; is not really needed since it simply 171 | * removes multiples of 2^64. We can discard these excess bits 172 | * outside of the loop when we convert to Fnv64_t. 173 | */ 174 | } 175 | 176 | /* 177 | * Convert base 2^16 array back into an Fnv64_t 178 | */ 179 | hval.w32[1] = ((val[3] << 16) | val[2]); 180 | hval.w32[0] = ((val[1] << 16) | val[0]); 181 | 182 | #endif /* HAVE_64BIT_LONG_LONG */ 183 | 184 | /* return our new hash value */ 185 | return hval; 186 | } 187 | 188 | /* 189 | * fnv_64a_str - perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer 190 | * 191 | * input: 192 | * buf - start of buffer to hash 193 | * hval - previous hash value or 0 if first call 194 | * 195 | * returns: 196 | * 64 bit hash as a static hash type 197 | * 198 | * NOTE: To use the recommended 64 bit FNV-1a hash, use FNV1A_64_INIT as the 199 | * hval arg on the first call to either fnv_64a_buf() or fnv_64a_str(). 200 | */ 201 | Fnv64_t 202 | fnv_64a_str(char *str, Fnv64_t hval) { 203 | unsigned char *s = (unsigned char *) str; /* unsigned string */ 204 | 205 | #if defined(HAVE_64BIT_LONG_LONG) 206 | 207 | /* 208 | * FNV-1a hash each octet of the string 209 | */ 210 | while (*s) { 211 | 212 | /* xor the bottom with the current octet */ 213 | hval ^= (Fnv64_t)*s++; 214 | 215 | /* multiply by the 64 bit FNV magic prime mod 2^64 */ 216 | #if defined(NO_FNV_GCC_OPTIMIZATION) 217 | hval *= FNV_64_PRIME; 218 | #else /* NO_FNV_GCC_OPTIMIZATION */ 219 | hval += (hval << 1) + (hval << 4) + (hval << 5) + 220 | (hval << 7) + (hval << 8) + (hval << 40); 221 | #endif /* NO_FNV_GCC_OPTIMIZATION */ 222 | } 223 | 224 | #else /* !HAVE_64BIT_LONG_LONG */ 225 | 226 | unsigned long val[4]; /* hash value in base 2^16 */ 227 | unsigned long tmp[4]; /* tmp 64 bit value */ 228 | 229 | /* 230 | * Convert Fnv64_t hval into a base 2^16 array 231 | */ 232 | val[0] = hval.w32[0]; 233 | val[1] = (val[0] >> 16); 234 | val[0] &= 0xffff; 235 | val[2] = hval.w32[1]; 236 | val[3] = (val[2] >> 16); 237 | val[2] &= 0xffff; 238 | 239 | /* 240 | * FNV-1a hash each octet of the string 241 | */ 242 | while (*s) { 243 | 244 | /* xor the bottom with the current octet */ 245 | 246 | /* 247 | * multiply by the 64 bit FNV magic prime mod 2^64 248 | * 249 | * Using 1099511628211, we have the following digits base 2^16: 250 | * 251 | * 0x0 0x100 0x0 0x1b3 252 | * 253 | * which is the same as: 254 | * 255 | * 0x0 1<> 16); 267 | val[0] = tmp[0] & 0xffff; 268 | tmp[2] += (tmp[1] >> 16); 269 | val[1] = tmp[1] & 0xffff; 270 | val[3] = tmp[3] + (tmp[2] >> 16); 271 | val[2] = tmp[2] & 0xffff; 272 | /* 273 | * Doing a val[3] &= 0xffff; is not really needed since it simply 274 | * removes multiples of 2^64. We can discard these excess bits 275 | * outside of the loop when we convert to Fnv64_t. 276 | */ 277 | val[0] ^= (unsigned long) (*s++); 278 | } 279 | 280 | /* 281 | * Convert base 2^16 array back into an Fnv64_t 282 | */ 283 | hval.w32[1] = ((val[3] << 16) | val[2]); 284 | hval.w32[0] = ((val[1] << 16) | val[0]); 285 | 286 | #endif /* !HAVE_64BIT_LONG_LONG */ 287 | 288 | /* return our new hash value */ 289 | return hval; 290 | } 291 | -------------------------------------------------------------------------------- /src/meta.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-4. 3 | // 4 | extern "C" { 5 | #include "fnv/fnv.h" 6 | } 7 | #include 8 | #include "meta.h" 9 | #include "memory_pool.h" 10 | #include "db.h" 11 | namespace boltDB_CPP { 12 | 13 | bool Meta::validate() { 14 | if (magic != MAGIC) { 15 | return false; 16 | } 17 | 18 | if (version != VERSION) { 19 | return false; 20 | } 21 | 22 | return !(checkSum != 0 && checkSum != sum64()); 23 | } 24 | 25 | void Meta::write(Page *page) { 26 | if (rootBucketHeader.rootPageId >= totalPageNumber) { 27 | assert(false); 28 | } 29 | if (freeListPageNumber >= totalPageNumber) { 30 | assert(false); 31 | } 32 | 33 | page->pageId = txnId % 2; 34 | page->flag |= static_cast(PageFlag::metaPageFlag); 35 | 36 | checkSum = sum64(); 37 | 38 | std::memcpy(page->metaPointer(), this, sizeof(Meta)); 39 | } 40 | 41 | uint64_t Meta::sum64() { 42 | uint64_t result = 0; 43 | result = ::fnv_64a_buf(this, offsetof(Meta, checkSum), FNV1A_64_INIT); 44 | return result; 45 | } 46 | 47 | Meta *Meta::copyCreateFrom(Meta *other, MemoryPool &pool) { 48 | if (other == nullptr) { 49 | return other; 50 | } 51 | auto ret = pool.allocate(); 52 | // member wise copy 53 | *ret = *other; 54 | return ret; 55 | } 56 | } // namespace boltDB_CPP -------------------------------------------------------------------------------- /src/node.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-27. 3 | // 4 | 5 | #include "node.h" 6 | #include 7 | #include 8 | #include "bucket.h" 9 | #include "db.h" 10 | #include "meta.h" 11 | #include "txn.h" 12 | #include "util.h" 13 | namespace boltDB_CPP { 14 | template<> 15 | int cmp_wrapper(const Inode &t, const Item &p) { 16 | if (t.key < p) { 17 | return -1; 18 | } 19 | if (t.key == p) { 20 | return 0; 21 | } 22 | return 1; 23 | } 24 | void Node::read(boltDB_CPP::Page *page) { 25 | // this is called inside a function, should not receive nullptr 26 | assert(page); 27 | this->pageId = page->pageId; 28 | this->isLeaf = 29 | static_cast(page->flag & static_cast(PageFlag::leafPageFlag)); 30 | this->inodeList.resize(page->count); 31 | 32 | for (size_t i = 0; i < page->count; i++) { 33 | auto& item = this->inodeList[i]; 34 | if (this->isLeaf) { 35 | auto element = page->getLeafPageElement(i); 36 | item.flag = element->flag; 37 | item.key = element->Key(); 38 | item.value = element->Value(); 39 | } else { 40 | auto element = page->getBranchPageElement(i); 41 | item.pageId = element->pageId; 42 | item.key = element->Key(); 43 | } 44 | assert(item.Key().length != 0); 45 | } 46 | 47 | if (!inodeList.empty()) { 48 | key = inodeList.front().Key(); 49 | assert(!key.empty()); 50 | } else { 51 | key.reset(); 52 | } 53 | } 54 | Node *Node::childAt(uint64_t index) { 55 | if (isLeaf) { 56 | assert(false); 57 | } 58 | return bucket->getNode(inodeList[index].pageId, this); 59 | } 60 | void Node::do_remove(const Item &key) { 61 | bool found = false; 62 | size_t index = binary_search(inodeList, key, cmp_wrapper, 63 | inodeList.size(), found); 64 | if (!found) { 65 | return; 66 | } 67 | 68 | auto b = inodeList.begin(); 69 | std::advance(b, index); 70 | inodeList.erase(b); 71 | 72 | unbalanced = true; // need re-balance 73 | } 74 | 75 | /** 76 | * \start |end 77 | * |page header| leafPageElement or branchPageElement ... | key & value ...| 78 | * @return 79 | */ 80 | size_t Node::size() const { 81 | size_t result = PAGEHEADERSIZE; 82 | for (size_t i = 0; i < inodeList.size(); i++) { 83 | result += 84 | pageElementSize() + inodeList[i].key.length + inodeList[i].value.length; 85 | } 86 | return result; 87 | } 88 | size_t Node::pageElementSize() const { 89 | if (isLeaf) { 90 | return LEAFPAGEELEMENTSIZE; 91 | } 92 | return BUCKETHEADERSIZE; 93 | } 94 | Node *Node::root() { 95 | if (parentNode == nullptr) { 96 | return this; 97 | } 98 | return parentNode->root(); 99 | } 100 | size_t Node::minKeys() const { 101 | if (isLeaf) { 102 | return 1; 103 | } 104 | return 2; 105 | } 106 | bool Node::sizeLessThan(size_t s) const { 107 | size_t sz = PAGEHEADERSIZE; 108 | for (size_t i = 0; i < inodeList.size(); i++) { 109 | sz += 110 | pageElementSize() + inodeList[i].key.length + inodeList[i].value.length; 111 | if (sz >= s) { 112 | return false; 113 | } 114 | } 115 | return true; 116 | } 117 | size_t Node::childIndex(Node *child) const { 118 | bool found = false; 119 | auto ret = binary_search(inodeList, child->key, cmp_wrapper, 120 | inodeList.size(), found); 121 | return ret; 122 | } 123 | size_t Node::numChildren() const { return inodeList.size(); } 124 | Node *Node::nextSibling() { 125 | if (parentNode == nullptr) { 126 | return nullptr; 127 | } 128 | auto idx = parentNode->childIndex(this); 129 | if (idx == 0) { 130 | return nullptr; 131 | } 132 | return parentNode->childAt(idx + 1); 133 | } 134 | 135 | //55c3 daec 12c8 136 | //5557 e4a2 a7f0 137 | //55c1 89c7 c7f0 138 | void Node::put(const Item &oldKey, const Item &newKey, const Item &value, 139 | page_id pageId, uint32_t flag) { 140 | if (pageId >= bucket->getTotalPageNumber()) { 141 | assert(false); 142 | } 143 | if (oldKey.length <= 0 || newKey.length <= 0) { 144 | assert(false); 145 | } 146 | 147 | bool found = false; 148 | auto ret = binary_search(inodeList, oldKey, cmp_wrapper, 149 | inodeList.size(), found); 150 | if (!found) { 151 | inodeList.insert(inodeList.begin() + ret, Inode()); 152 | } 153 | auto &ref = inodeList[ret]; 154 | ref.flag = flag; 155 | ref.key = newKey; 156 | ref.value = value; 157 | ref.pageId = pageId; 158 | } 159 | 160 | void Node::del(const Item &key) { 161 | bool found = false; 162 | auto ret = binary_search(inodeList, key, cmp_wrapper, inodeList.size(), 163 | found); 164 | 165 | if (!found) { 166 | return; 167 | } 168 | // for (size_t i = ret; i + 1 < inodeList.size(); i++) { 169 | // inodeList[i] = inodeList[i + 1]; 170 | // } 171 | inodeList.erase(inodeList.begin() + ret); 172 | 173 | // need re-balance 174 | unbalanced = true; 175 | } 176 | // serialize itself into a given page 177 | void Node::write(Page *page) { 178 | if (isLeaf) { 179 | page->flag |= static_cast(PageFlag::leafPageFlag); 180 | } else { 181 | page->flag |= static_cast(PageFlag::branchPageFlag); 182 | } 183 | 184 | // why it exceed 0xffff ? 185 | if (inodeList.size() > 0xffff) { 186 | assert(false); 187 | } 188 | 189 | page->count = inodeList.size(); 190 | if (page->count == 0) { 191 | return; 192 | } 193 | 194 | //|page header | leaf/branch element .... | kv pair ... | 195 | //|<-page start| &page->ptr |<-contentPtr |<-page end 196 | auto contentPtr = &(reinterpret_cast(&page->ptr)[inodeList.size() * pageElementSize()]); 197 | for (size_t i = 0; i < inodeList.size(); i++) { 198 | if (isLeaf) { 199 | auto item = page->getLeafPageElement(i); 200 | item->pos = contentPtr - (char *) item; 201 | item->flag = inodeList[i].flag; 202 | item->ksize = inodeList[i].key.length; 203 | item->vsize = inodeList[i].value.length; 204 | } else { 205 | auto item = page->getBranchPageElement(i); 206 | item->pos = contentPtr - (char *) &item; 207 | item->ksize = inodeList[i].key.length; 208 | item->pageId = inodeList[i].pageId; 209 | } 210 | 211 | std::memcpy(contentPtr, inodeList[i].key.pointer, inodeList[i].key.length); 212 | contentPtr += inodeList[i].key.length; 213 | std::memcpy(contentPtr, inodeList[i].value.pointer, 214 | inodeList[i].value.length); 215 | contentPtr += inodeList[i].value.length; 216 | } 217 | } 218 | std::vector Node::split(size_t pageSize) { 219 | std::vector result; 220 | 221 | auto cur = this; 222 | while (true) { 223 | Node *a; 224 | Node *b; 225 | cur->splitTwo(pageSize, a, b); 226 | result.push_back(a); 227 | if (b == nullptr) { 228 | break; 229 | } 230 | cur = b; 231 | } 232 | return result; 233 | } 234 | 235 | // used only inside split 236 | void Node::splitTwo(size_t pageSize, Node *&a, Node *&b) { 237 | if (inodeList.size() <= MINKEYSPERPAGE * 2 || sizeLessThan(pageSize)) { 238 | a = this; 239 | b = nullptr; 240 | return; 241 | } 242 | 243 | //calculate threshold 244 | double fill = bucket->getFillPercent(); 245 | if (fill < MINFILLPERCENT) { 246 | fill = MINFILLPERCENT; 247 | } 248 | if (fill > MAXFILLPERCENT) { 249 | fill = MAXFILLPERCENT; 250 | } 251 | 252 | auto threshold = static_cast(pageSize * fill); 253 | 254 | //determinate split position 255 | auto index = splitIndex(threshold); 256 | 257 | if (parentNode == nullptr) { 258 | // using share pointer to deal with this 259 | parentNode = bucket->getPool().allocate(bucket, nullptr); 260 | parentNode->children.push_back(this); 261 | } 262 | 263 | auto newNode = bucket->getPool().allocate(bucket, parentNode); 264 | newNode->isLeaf = isLeaf; 265 | parentNode->children.push_back(newNode); 266 | 267 | for (size_t i = index; i < inodeList.size(); i++) { 268 | newNode->inodeList.push_back(inodeList[i]); 269 | } 270 | inodeList.erase(inodeList.begin() + index, inodeList.end()); 271 | 272 | a = this; 273 | b = newNode; 274 | } 275 | 276 | size_t Node::splitIndex(size_t threshold) { 277 | size_t index = 0; 278 | size_t sz = PAGEHEADERSIZE; 279 | for (size_t i = 0; i < inodeList.size() - MINKEYSPERPAGE; i++) { 280 | index = i; 281 | auto &ref = inodeList[i]; 282 | auto elementSize = pageElementSize() + ref.key.length + ref.value.length; 283 | if (i >= MINKEYSPERPAGE && sz + elementSize > threshold) { 284 | break; 285 | } 286 | sz += elementSize; 287 | } 288 | return index; 289 | } 290 | 291 | void Node::free() { 292 | if (pageId) { 293 | auto txn = bucket->getTxn(); 294 | txn->free(txn->txnId(), txn->getPage(pageId)); 295 | pageId = 0; 296 | } 297 | } 298 | void Node::removeChild(Node *target) { 299 | for (auto iter = children.begin(); iter != children.end(); iter++) { 300 | if (*iter == target) { 301 | children.erase(iter); 302 | return; 303 | } 304 | } 305 | } 306 | void Node::dereference() { 307 | // 308 | // node lives in heap 309 | // nothing to be done here 310 | // 311 | // 2 kinds of nodes lives in inodeslist 312 | // 1.value pointers to mmap address 313 | // 2.value pointers to heap/memory pool allocated object 314 | // when remapping is needed, the first kind needs to be saved to somewhere. 315 | // or it will pointing to undefined values after a new mmap 316 | // the second case will not need to be saved 317 | // may provide a method in memorypool to distinguish with pointer should be 318 | // saved for now, just copy every value to memory pool duplicate values exist. 319 | // they will be freed when memorypool clears itself 320 | 321 | // clone current node's key 322 | if (!key.empty()) { 323 | key = key.clone(&bucket->getPool()); 324 | } 325 | 326 | // clone current node's kv pairs 327 | for (auto &item : inodeList) { 328 | item.key = item.key.clone(&bucket->getPool()); 329 | item.value = item.value.clone(&bucket->getPool()); 330 | } 331 | 332 | // do copy recursively 333 | for (auto &child : children) { 334 | child->dereference(); 335 | } 336 | } 337 | int Node::spill() { 338 | if (spilled) { 339 | return 0; 340 | } 341 | auto tx = bucket->getTxn(); 342 | 343 | // by pointer value for now 344 | std::sort(children.begin(), children.end()); 345 | // spill will modify children's size, no range loop here 346 | for (size_t i = 0; i < children.size(); i++) { 347 | if (children[i]->spill()) { 348 | return -1; 349 | } 350 | } 351 | 352 | children.clear(); 353 | auto nodes = split(DB::getPageSize()); 354 | 355 | for (auto &item : nodes) { 356 | assert(item); 357 | if (item->pageId > 0) { 358 | tx->free(tx->txnId(), tx->getPage(item->pageId)); 359 | } 360 | 361 | auto page = tx->allocate((size() / DB::getPageSize()) + 1); 362 | if (page == nullptr) { 363 | return -1; 364 | } 365 | 366 | if (page->pageId >= tx->getTotalPageNumber()) { 367 | assert(false); 368 | } 369 | item->pageId = page->pageId; 370 | item->write(page); 371 | item->spilled = true; 372 | 373 | if (item->parentNode) { 374 | auto k = item->key; 375 | if (k.length == 0) { 376 | k = inodeList.front().key; 377 | } 378 | Item emptyValue; 379 | item->parentNode->put(k, item->inodeList.front().key, emptyValue, 380 | item->pageId, 0); 381 | item->key = item->inodeList[0].key; 382 | } 383 | 384 | // tx->stats.spillCount++; 385 | } 386 | 387 | if (parentNode && parentNode->pageId == 0) { 388 | children.clear(); 389 | return parentNode->spill(); 390 | } 391 | return 0; 392 | } 393 | 394 | /** 395 | * this should be named by 'merge sibling' 396 | */ 397 | void Node::rebalance() { 398 | if (!unbalanced) { 399 | return; 400 | } 401 | unbalanced = false; 402 | // bucket->getTxn()->stats.rebalanceCount++; 403 | 404 | auto threshold = DB::getPageSize() / 4; 405 | if (size() > threshold && inodeList.size() > minKeys()) { 406 | return; 407 | } 408 | 409 | if (parentNode == nullptr) { 410 | // root node has only one branch, need to collapse it 411 | // assign current node to child, and remove child node 412 | if (!isLeaf && inodeList.size() == 1) { 413 | auto child = bucket->getNode(inodeList[0].pageId, this); 414 | isLeaf = child->isLeaf; 415 | inodeList = child->inodeList; 416 | children = child->children; 417 | 418 | for (auto &item : inodeList) { 419 | // branch node will have a meaningful value in pageId field 420 | auto n = bucket->getCachedNode(item.pageId); 421 | // what about those nodes that are not cached? 422 | // only in memory nodes will be write out to db file 423 | // that means if a branch node is not found/not accessed before 424 | // there should be something unexpected happening 425 | if (n) { 426 | n->parentNode = this; 427 | } else { 428 | assert(false); 429 | } 430 | } 431 | 432 | child->parentNode = nullptr; 433 | bucket->eraseCachedNode(child->pageId); 434 | child->free(); 435 | } 436 | 437 | return; 438 | } 439 | 440 | if (numChildren() == 0) { 441 | parentNode->del(key); 442 | parentNode->removeChild(this); 443 | bucket->eraseCachedNode(pageId); 444 | free(); 445 | parentNode->rebalance(); 446 | return; 447 | } 448 | 449 | assert(parentNode->numChildren() > 1); 450 | 451 | if (parentNode->childIndex(this) == 0) { 452 | auto target = nextSibling(); 453 | 454 | // this should move inodes of target into current node 455 | // and re set up between node's parent&child link 456 | 457 | // set sibling node's children's parent to current node 458 | for (auto &item : target->inodeList) { 459 | auto childNode = bucket->getCachedNode(item.pageId); 460 | if (childNode) { 461 | childNode->parentNode->removeChild(childNode); 462 | childNode->parentNode = this; 463 | childNode->parentNode->children.push_back(childNode); 464 | } 465 | } 466 | 467 | // copy sibling node's children to current node 468 | std::copy(target->inodeList.begin(), target->inodeList.end(), 469 | std::back_inserter(inodeList)); 470 | // remove sibling node 471 | parentNode->del(target->key); 472 | parentNode->removeChild(target); 473 | bucket->eraseCachedNode(target->pageId); 474 | target->free(); 475 | } else { 476 | auto target = prevSibling(); 477 | 478 | for (auto &item : target->inodeList) { 479 | auto childNode = bucket->getCachedNode(item.pageId); 480 | if (childNode) { 481 | childNode->parentNode->removeChild(childNode); 482 | childNode->parentNode = this; 483 | childNode->parentNode->children.push_back(childNode); 484 | } 485 | } 486 | 487 | std::copy(target->inodeList.begin(), target->inodeList.end(), 488 | std::back_inserter(inodeList)); 489 | parentNode->del(this->key); 490 | parentNode->removeChild(this); 491 | bucket->eraseCachedNode(this->pageId); 492 | this->free(); 493 | } 494 | 495 | // as parent node has one element removed, re-balance it 496 | parentNode->rebalance(); 497 | } 498 | Node *Node::prevSibling() { 499 | if (parentNode == nullptr) { 500 | return nullptr; 501 | } 502 | auto idx = parentNode->childIndex(this); 503 | if (idx == 0) { 504 | return nullptr; 505 | } 506 | return parentNode->childAt(idx - 1); 507 | } 508 | size_t Node::search(const Item &key, bool &found) { 509 | auto ret = binary_search(inodeList, key, cmp_wrapper, inodeList.size(), 510 | found); 511 | return ret; 512 | } 513 | bool Node::isinlineable(size_t maxInlineBucketSize) const { 514 | size_t s = PAGEHEADERSIZE; 515 | for (auto item : inodeList) { 516 | s += LEAFPAGEELEMENTSIZE + item.key.length + item.value.length; 517 | 518 | if (isBucketLeaf(item.flag)) { 519 | return false; 520 | } 521 | if (s > maxInlineBucketSize) { 522 | return false; 523 | } 524 | } 525 | return true; 526 | } 527 | std::vector Node::branchPageIds() { 528 | std::vector result; 529 | for (auto &item : inodeList) { 530 | result.push_back(item.pageId); 531 | } 532 | return result; 533 | } 534 | 535 | } // namespace boltDB_CPP 536 | -------------------------------------------------------------------------------- /src/txn.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | #include "txn.h" 5 | #include 6 | #include 7 | #include "bucket.h" 8 | #include "db.h" 9 | #include "meta.h" 10 | #include "util.h" 11 | namespace boltDB_CPP { 12 | 13 | Page *boltDB_CPP::Txn::getPage(page_id pageId) { 14 | if (pageId >= metaData->totalPageNumber) { 15 | return nullptr; 16 | } 17 | if (!dirtyPageTable.empty()) { 18 | auto iter = dirtyPageTable.find(pageId); 19 | if (iter != dirtyPageTable.end()) { 20 | return iter->second; 21 | } 22 | } 23 | return db->pagePointer(pageId); 24 | } 25 | 26 | Page *Txn::allocate(size_t count) { 27 | auto ret = db->allocate(count, this); 28 | if (ret == nullptr) { 29 | return ret; 30 | } 31 | dirtyPageTable[ret->pageId] = ret; 32 | 33 | stats.pageCount++; 34 | stats.pageAlloc += count * db->getPageSize(); 35 | 36 | return ret; 37 | } 38 | 39 | void Txn::for_each_page(page_id pageId, int depth, 40 | std::function fn) { 41 | auto p = getPage(pageId); 42 | fn(p, depth); 43 | 44 | if (p->flag & static_cast(PageFlag::branchPageFlag)) { 45 | for (int i = 0; i < p->getCount(); i++) { 46 | auto element = p->getBranchPageElement(i); 47 | for_each_page(element->pageId, depth + 1, fn); 48 | } 49 | } 50 | } 51 | 52 | void Txn::init(DB *db) { 53 | this->db = db; 54 | metaData = Meta::copyCreateFrom(db->meta(), pool); 55 | rootBucket.setBucketHeader(metaData->rootBucketHeader); 56 | if (writable) { 57 | metaData->txnId += 1; 58 | } 59 | } 60 | 61 | Bucket *Txn::getBucket(const Item &name) { 62 | return rootBucket.getBucketByName(name); 63 | } 64 | 65 | Bucket *Txn::createBucket(const Item &name) { 66 | return rootBucket.createBucket(name); 67 | } 68 | 69 | Bucket *Txn::createBucketIfNotExists(const Item &name) { 70 | return rootBucket.createBucketIfNotExists(name); 71 | } 72 | 73 | int Txn::deleteBucket(const Item &name) { rootBucket.deleteBucket(name); } 74 | 75 | int Txn::for_each(std::function fn) { 76 | return rootBucket.for_each([&fn, this](const Item &k, const Item &v) -> int { 77 | auto ret = fn(k, rootBucket.getBucketByName(k)); 78 | return ret; 79 | }); 80 | } 81 | 82 | void Txn::OnCommit(std::function fn) { commitHandlers.push_back(fn); } 83 | 84 | /** 85 | * deal with txn level work. db will release this txn' resources in upper level 86 | * @return disk write error / called on read only txn (what? I think read only 87 | * txn should succeed on doing commit) 88 | */ 89 | int Txn::commit() { 90 | // txn should not call commit in Update/View 91 | if (managed) { 92 | return -1; 93 | } 94 | if (db == nullptr || !isWritable()) { 95 | return -1; 96 | } 97 | 98 | // recursively merge nodes of which number of elements is below threshold 99 | rootBucket.rebalance(); 100 | if (rootBucket.spill()) { 101 | rollback(); 102 | return -1; 103 | } 104 | 105 | metaData->rootBucketHeader.rootPageId = rootBucket.getRootPage(); 106 | auto pgid = metaData->totalPageNumber; 107 | 108 | free(metaData->txnId, db->pagePointer(metaData->freeListPageNumber)); 109 | auto page = allocate((db->freeListSerialSize() / boltDB_CPP::DB::getPageSize()) + 1); 110 | if (page == nullptr) { 111 | rollback(); 112 | return -1; 113 | } 114 | if (db->getFreeLIst().write(page)) { 115 | rollback(); 116 | return -1; 117 | } 118 | 119 | metaData->freeListPageNumber = page->pageId; 120 | 121 | if (metaData->totalPageNumber > pgid) { 122 | if (db->grow((metaData->totalPageNumber + 1) * db->getPageSize())) { 123 | rollback(); 124 | return -1; 125 | } 126 | } 127 | 128 | if (write() != 0) { 129 | rollback(); 130 | return -1; 131 | } 132 | 133 | if (writeMeta() != 0) { 134 | rollback(); 135 | return -1; 136 | } 137 | 138 | for (auto &item : commitHandlers) { 139 | item(); 140 | } 141 | 142 | return 0; 143 | } 144 | 145 | /** 146 | * this should be move to db class other than placed here 147 | */ 148 | int Txn::rollback() { 149 | if (managed) { 150 | return -1; 151 | } 152 | if (db == nullptr) { 153 | return -1; 154 | } 155 | if (isWritable()) { 156 | db->getFreeLIst().rollback(metaData->txnId); 157 | db->getFreeLIst().reload(db->pagePointer(metaData->freeListPageNumber)); 158 | } 159 | return 0; 160 | } 161 | 162 | int Txn::writeMeta() { 163 | std::vector tmp(boltDB_CPP::DB::getPageSize()); 164 | Page *page = reinterpret_cast(tmp.data()); 165 | metaData->write(page); 166 | 167 | if (db->writeAt(tmp.data(), tmp.size(), page->pageId * boltDB_CPP::DB::getPageSize()) != tmp.size()) { 168 | return -1; 169 | } 170 | 171 | if (!db->isNoSync()) { 172 | if (file_data_sync(db->getFd())) { 173 | return -1; 174 | } 175 | } 176 | return 0; 177 | } 178 | 179 | int Txn::write() { 180 | std::vector pages; 181 | for (auto item : dirtyPageTable) { 182 | pages.push_back(item.second); 183 | } 184 | dirtyPageTable.clear(); 185 | std::sort(pages.begin(), pages.end()); 186 | 187 | for (auto p : pages) { 188 | auto length = (p->overflow + 1) * db->getPageSize(); 189 | auto offset = p->pageId * db->getPageSize(); 190 | if (db->writeAt(reinterpret_cast(p), length, offset) != length) { 191 | return -1; 192 | } 193 | } 194 | 195 | if (!db->isNoSync()) { 196 | if (file_data_sync(db->getFd())) { 197 | return -1; 198 | } 199 | } 200 | 201 | return 0; 202 | } 203 | 204 | int Txn::isFreelistCheckOK() { 205 | std::map freePageIds; 206 | for (auto item : db->getFreeLIst().pageIds) { 207 | if (freePageIds.find(item) != freePageIds.end()) { 208 | return -1; 209 | } 210 | freePageIds[item] = true; 211 | } 212 | 213 | for (auto &p : db->getFreeLIst().pending) { 214 | for (auto item : p.second) { 215 | if (freePageIds.find(item) != freePageIds.end()) { 216 | return -1; 217 | } 218 | freePageIds[item] = true; 219 | } 220 | } 221 | 222 | std::map occupiedPageIds; 223 | occupiedPageIds[0] = getPage(0); 224 | occupiedPageIds[1] = getPage(1); 225 | //add all pages included in free list page, it could be many consecutive pages there 226 | for (size_t i = 0; i <= getPage(metaData->freeListPageNumber)->overflow; 227 | i++) { 228 | occupiedPageIds[metaData->freeListPageNumber + i] = 229 | getPage(metaData->freeListPageNumber); 230 | } 231 | 232 | if (!isBucketsRemainConsistent(rootBucket, occupiedPageIds, freePageIds)) { 233 | return -1; 234 | } 235 | 236 | for (size_t i = 0; i < metaData->totalPageNumber; i++) { 237 | if ((occupiedPageIds.find(i) == occupiedPageIds.end()) && (freePageIds.find(i) == freePageIds.end())) { 238 | return -1; 239 | } 240 | } 241 | return 0; 242 | } 243 | 244 | bool Txn::isBucketsRemainConsistent(Bucket &bucket, std::map &reachable, 245 | std::map &freed) { 246 | if (bucket.isInlineBucket()) { 247 | return true; 248 | } 249 | bool ret = true; 250 | bucket.tx->for_each_page( 251 | bucket.bucketHeader.rootPageId, 0, [&, this](Page *page, int i) { 252 | if (page->pageId >= metaData->totalPageNumber) { 253 | ret = false; 254 | return; 255 | } 256 | 257 | for (size_t i = 0; i <= page->overflow; i++) { 258 | auto id = page->pageId + i; 259 | if (reachable.find(id) != reachable.end()) { 260 | // multiple reference 261 | ret = false; 262 | return; 263 | } 264 | reachable[id] = page; 265 | } 266 | 267 | if (freed.find(page->pageId) != freed.end()) { 268 | // this page appeared in free list 269 | ret = false; 270 | return; 271 | } 272 | 273 | PageFlag pf = static_cast(page->flag); 274 | //one page of a bucket must be either branch page or leaf page 275 | if (pf != PageFlag::branchPageFlag && pf != PageFlag::leafPageFlag) { 276 | ret = false; 277 | return; 278 | } 279 | }); 280 | 281 | if (!ret) { 282 | return ret; 283 | } 284 | 285 | //check every sub buckets of current bucket 286 | auto bucketCheck = bucket.for_each([&, this](const Item &k, const Item &v) { 287 | auto child = bucket.getBucketByName(k); 288 | if (child) { 289 | if (!this->isBucketsRemainConsistent(*child, reachable, freed)) { 290 | return 1; 291 | } 292 | } 293 | return 0; 294 | }); 295 | return bucketCheck == 0; 296 | } 297 | 298 | txn_id Txn::txnId() const { return metaData->txnId; } 299 | void Txn::free(txn_id tid, Page *page) { db->getFreeLIst().free(tid, page); } 300 | } // namespace boltDB_CPP 301 | -------------------------------------------------------------------------------- /src/types.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-11. 3 | // 4 | 5 | #include "types.h" 6 | #include "memory_pool.h" 7 | namespace boltDB_CPP { 8 | 9 | bool Item::operator==(const Item &other) const { 10 | if (this == &other) { 11 | return true; 12 | } 13 | if (length != other.length) { 14 | return false; 15 | } 16 | for (size_t i = 0; i < length; i++) { 17 | if (pointer[i] != other.pointer[i]) { 18 | return false; 19 | } 20 | } 21 | return true; 22 | } 23 | bool Item::operator!=(const Item &other) const { 24 | return !(this->operator==(other)); 25 | } 26 | bool Item::operator<(const Item &other) const { 27 | size_t i = 0; 28 | size_t j = 0; 29 | while (i < length && j < other.length) { 30 | if (pointer[i] == other.pointer[j]) { 31 | i++; 32 | j++; 33 | continue; 34 | } 35 | return pointer[i] < other.pointer[j]; 36 | } 37 | return i == length && j != other.length; 38 | } 39 | void Item::reset() { 40 | pointer = nullptr; 41 | length = 0; 42 | } 43 | bool Item::empty() const { return length == 0; } 44 | Item Item::clone(MemoryPool *pool) { 45 | char *ptr = pool->arrayCopy(pointer, length); 46 | return Item{ptr, length}; 47 | } 48 | } -------------------------------------------------------------------------------- /src/util.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-4-26. 3 | // 4 | 5 | #include "util.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "db.h" 12 | 13 | namespace boltDB_CPP { 14 | 15 | static int file_lock_nonblocking(int fd, int operation) { 16 | int flockRet = flock(fd, operation); 17 | if (flockRet == -1) { 18 | perror("flock file"); 19 | } 20 | return flockRet; 21 | } 22 | 23 | int file_Wlock(int fd) { return file_lock_nonblocking(fd, LOCK_EX | LOCK_NB); } 24 | 25 | int file_WlockBlocking(int fd) { return file_lock_nonblocking(fd, LOCK_EX); } 26 | 27 | int file_Rlock(int fd) { return file_lock_nonblocking(fd, LOCK_SH | LOCK_NB); } 28 | 29 | int file_Unlock(int fd) { return file_lock_nonblocking(fd, LOCK_UN | LOCK_NB); } 30 | 31 | int file_data_sync(int fd) { 32 | int ret = fdatasync(fd); 33 | if (ret != 0) { 34 | perror("fdatasync"); 35 | } 36 | return ret; 37 | } 38 | 39 | void mergePageIds(std::vector &dest, 40 | const std::vector &a, 41 | const std::vector &b) { 42 | if (a.empty()) { 43 | dest = b; 44 | return; 45 | } 46 | if (b.empty()) { 47 | dest = a; 48 | return; 49 | } 50 | 51 | dest.clear(); 52 | size_t ia = 0; 53 | size_t ib = 0; 54 | while (ia != a.size() || ib != b.size()) { 55 | if (ia == a.size()) { 56 | dest.push_back(b[ib++]); 57 | } else if (ib == b.size()) { 58 | dest.push_back(a[ia++]); 59 | } else { 60 | if (b[ib] < a[ia]) { 61 | dest.push_back(b[ib++]); 62 | } else { 63 | dest.push_back(a[ia++]); 64 | } 65 | } 66 | } 67 | } 68 | 69 | std::vector merge( 70 | const std::vector &a, 71 | const std::vector &b) { 72 | std::vector result; 73 | mergePageIds(result, a, b); 74 | return result; 75 | } 76 | 77 | off_t file_size(int fd) { 78 | struct stat stat1; 79 | auto ret = fstat(fd, &stat1); 80 | if (ret == -1) { 81 | perror("stat"); 82 | return 0; 83 | } 84 | return stat1.st_size; 85 | } 86 | } // namespace boltDB_CPP 87 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################################################################################## 2 | #TEST CMAKELISTS 3 | ################################################################################## 4 | 5 | #--[Tests lists 6 | file(GLOB test_srcs_temp ${PROJECT_SOURCE_DIR}/test/*/*test.cpp) 7 | 8 | set(test_srcs "") 9 | 10 | foreach(test_src_temp ${test_srcs_temp} ) 11 | string(REPLACE "//" "/" test_src ${test_src_temp}) 12 | list(APPEND test_srcs ${test_src}) 13 | endforeach(test_src_temp ${test_srcs_temp}) 14 | 15 | ################################################################################## 16 | 17 | # --[ Gmock 18 | set(GMOCK_DIR "${PROJECT_SOURCE_DIR}/third_party/gmock") 19 | file(GLOB gmock_srcs ${GMOCK_DIR}/*.cc) 20 | include_directories(SYSTEM ${GMOCK_DIR}) 21 | add_library(gtest EXCLUDE_FROM_ALL ${gmock_srcs}) 22 | target_link_libraries(gtest ${CMAKE_THREAD_LIBS_INIT}) 23 | 24 | ################################################################################## 25 | 26 | # --[ Add "make check" target 27 | set(CTEST_FLAGS "") 28 | add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} ${CTEST_FLAGS} --verbose) 29 | 30 | ################################################################################## 31 | # --[ Memcheck 32 | find_program(MEMORYCHECK_COMMAND valgrind REQUIRED) 33 | # Note you can add '--gen-suppressions=all' to MEMORYCHECK_COMMAND_OPTIONS 34 | # if you want valgrind to print out the syntax to use to suppress a particular 35 | # memory leak 36 | set(MEMORYCHECK_COMMAND_OPTIONS 37 | --trace-children=yes 38 | --leak-check=full 39 | --track-origins=yes 40 | --soname-synonyms=somalloc=*jemalloc* 41 | --error-exitcode=1) 42 | set(MEMORYCHECK_SUPPRESSIONS_FILE ${PROJECT_SOURCE_DIR}/third_party/valgrind/valgrind.supp) 43 | 44 | ################################################################################## 45 | # --[ Functionality Tests 46 | foreach(test_src ${test_srcs} ) 47 | # get test file name 48 | get_filename_component(test_bare_name ${test_src} NAME) 49 | string(REPLACE ".cpp" "" test_bare_name_without_extension ${test_bare_name}) 50 | string(REPLACE "\"" "" test_name ${test_bare_name_without_extension}) 51 | 52 | # create executable 53 | add_executable(${test_name} EXCLUDE_FROM_ALL ${test_src}) 54 | add_dependencies(check ${test_name}) 55 | 56 | # link libraries 57 | target_link_libraries(${test_name} boltDB_in_cpp gtest) 58 | 59 | # set target properties 60 | set_target_properties(${test_name} 61 | PROPERTIES 62 | RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test" 63 | COMMAND ${test_name} 64 | ) 65 | 66 | # add test 67 | add_test(${test_name} ${MEMORYCHECK_COMMAND} ${MEMORYCHECK_COMMAND_OPTIONS} 68 | --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE} ${CMAKE_BINARY_DIR}/test/${test_name} 69 | --gtest_color=yes --gtest_output=xml:${CMAKE_BINARY_DIR}/test/unit_${test_name}.xml) 70 | add_test(${test_name} ${CMAKE_BINARY_DIR}/test/${test_name} --gtest_color=yes 71 | --gtest_output=xml:${CMAKE_BINARY_DIR}/test/${test_name}.xml) 72 | 73 | endforeach(test_src ${test_srcs}) 74 | -------------------------------------------------------------------------------- /test/bucket/bucket_test.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-6-2. 3 | // 4 | 5 | #include "test_util.h" 6 | namespace boltDB_CPP { 7 | TEST_F(TmpFile, get_nonexist_bucket) { 8 | auto updateFunc = [](Txn *txn) -> int { 9 | auto b = txn->createBucket(Item::make_item("widgets")); 10 | EXPECT_NE(b, nullptr); 11 | auto v = b->get(Item::make_item("foo")); 12 | EXPECT_EQ(v.empty(), true); 13 | return 0; 14 | }; 15 | db->update(updateFunc); 16 | } 17 | 18 | TEST_F(TmpFile, get_value_from_bucket) { 19 | auto updateFunc = [](Txn *txn) -> int { 20 | auto b = txn->createBucket(Item::make_item("widgets")); 21 | EXPECT_NE(b, nullptr); 22 | auto r = b->put(Item::make_item("foo"), Item::make_item("bar")); 23 | EXPECT_EQ(0, r); 24 | auto v = b->get(Item::make_item("foo")); 25 | EXPECT_EQ(v, Item::make_item("bar")); 26 | return 0; 27 | }; 28 | db->update(updateFunc); 29 | } 30 | 31 | TEST_F(TmpFile, subbucket_should_be_null) { 32 | auto updateFunc = [](Txn *txn) -> int { 33 | auto b1 = txn->createBucket(Item::make_item("widgets")); 34 | EXPECT_NE(b1, nullptr); 35 | auto b2 = txn->getBucket(Item::make_item("widgets"))->createBucket(Item::make_item("foo")); 36 | EXPECT_NE(b2, nullptr); 37 | auto ret = txn->getBucket(Item::make_item("widgets"))->get(Item::make_item("foo")); 38 | EXPECT_EQ(ret.empty(), true); 39 | return 0; 40 | }; 41 | db->update(updateFunc); 42 | } 43 | 44 | TEST_F(TmpFile, bucket_put) { 45 | auto updateFunc = [](Txn *txn) -> int { 46 | auto b1 = txn->createBucket(Item::make_item("widgets")); 47 | EXPECT_NE(b1, nullptr); 48 | auto ret = b1->put(Item::make_item("foo"), Item::make_item("bar")); 49 | EXPECT_EQ(0, ret); 50 | auto val = b1->get(Item::make_item("foo")); 51 | EXPECT_EQ(val == Item::make_item("bar"), true); 52 | return 0; 53 | }; 54 | db->update(updateFunc); 55 | } 56 | 57 | TEST_F(TmpFile, bucket_repeat_put) { 58 | auto updateFunc = [](Txn *txn) -> int { 59 | auto b1 = txn->createBucket(Item::make_item("widgets")); 60 | EXPECT_NE(b1, nullptr); 61 | auto ret = b1->put(Item::make_item("foo"), Item::make_item("bar")); 62 | EXPECT_EQ(0, ret); 63 | auto ret2 = b1->put(Item::make_item("foo"), Item::make_item("bar")); 64 | EXPECT_EQ(0, ret2); 65 | auto val = b1->get(Item::make_item("foo")); 66 | EXPECT_EQ(val == Item::make_item("bar"), true); 67 | return 0; 68 | }; 69 | db->update(updateFunc); 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /test/db/db_test.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-7. 3 | // 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "db.h" 14 | namespace boltDB_CPP { 15 | static const int SLEEP_WAIT_TIME = 1; 16 | //open an empty file as db 17 | TEST(dbtest, opentest) { 18 | std::unique_ptr ptr(new DB); 19 | auto ret = ptr->openDB(newFileName(), S_IRWXU); 20 | EXPECT_EQ(ret, ptr.get()); 21 | ptr->closeDB(); 22 | } 23 | //open a file that is not a valid db file 24 | TEST(dbtest, open_invalid_db_file) { 25 | auto name = newFileName(); 26 | auto fd = ::open(name.c_str(), O_CREAT | O_RDWR, S_IRWXU); 27 | EXPECT_NE(fd, -1); 28 | auto wcnt = ::write(fd, "abc", 4); 29 | EXPECT_EQ(wcnt, 4); 30 | ::close(fd); 31 | std::unique_ptr ptr(new DB); 32 | auto ret = ptr->openDB(name, S_IRWXU); 33 | EXPECT_EQ(ret, nullptr); 34 | ptr->closeDB(); 35 | } 36 | //re-open a db 37 | TEST(dbtest, dbtest_reopendb_Test) { 38 | auto name = newFileName(); 39 | std::unique_ptr db1(new DB); 40 | db1->openDB(name, S_IRWXU); 41 | int ret = 0; 42 | db1->view([&ret](Txn *txn) { 43 | ret = txn->isFreelistCheckOK(); 44 | return ret; 45 | }); 46 | EXPECT_EQ(ret, 0); 47 | db1->closeDB(); 48 | db1.reset(); 49 | 50 | std::unique_ptr db2(new DB); 51 | db2->openDB(name, S_IRWXU); 52 | db2->view([&ret](Txn *txn) { 53 | ret = txn->isFreelistCheckOK(); 54 | return ret; 55 | }); 56 | EXPECT_EQ(ret, 0); 57 | db2->closeDB(); 58 | } 59 | 60 | //open a truncated db 61 | TEST(dbtest, dbtest_open_truncatefile_Test) { 62 | auto name = newFileName(); 63 | std::unique_ptr db1(new DB); 64 | db1->openDB(name, S_IRWXU); 65 | db1->closeDB(); 66 | int fd = open(name.c_str(), O_RDWR); 67 | EXPECT_NE(fd, -1); 68 | auto ret = ftruncate(fd, 0x1000); 69 | EXPECT_EQ(ret, 0); 70 | std::unique_ptr db2(new DB); 71 | EXPECT_EQ(db2->openDB(name, S_IRWXU), nullptr); 72 | db2->closeDB(); 73 | } 74 | 75 | TEST_F(TmpFile, dbtest_start_rw_txn_Test) { 76 | auto txn = db->beginRWTx(); 77 | EXPECT_NE(txn, nullptr); 78 | EXPECT_EQ(db->commitTxn(txn), 0); 79 | db->closeDB(); 80 | } 81 | 82 | //any not closed txn should stop the db from closing 83 | TEST(dbtest, dbtest_closedb_while_rw_txn_is_active_Test) { 84 | std::cout << std::endl; 85 | auto name = newFileName(); 86 | std::unique_ptr db1(new DB); 87 | EXPECT_NE(db1->openDB(name, S_IRWXU), nullptr); 88 | //1.rw txn 89 | auto rw = db1->beginRWTx(); 90 | EXPECT_NE(rw, nullptr); 91 | std::thread th([&]() { 92 | sleep(SLEEP_WAIT_TIME); 93 | std::cout << "committing txn" << std::endl; 94 | EXPECT_EQ(db1->commitTxn(rw), 0); 95 | }); 96 | std::cout << "close in main thread" << std::endl; 97 | db1->closeDB(); 98 | th.join(); 99 | } 100 | TEST(dbtest, dbtest_closedb_while_ro_txn_is_active_Test) { 101 | std::cout << std::endl; 102 | auto name = newFileName(); 103 | std::unique_ptr db1(new DB); 104 | EXPECT_NE(db1->openDB(name, S_IRWXU), nullptr); 105 | //1.rw txn 106 | auto ro = db1->beginTx(); 107 | EXPECT_NE(ro, nullptr); 108 | std::thread th([&]() { 109 | sleep(SLEEP_WAIT_TIME); 110 | std::cout << "rollback ro txn" << std::endl; 111 | db1->rollbackTxn(ro); 112 | }); 113 | std::cout << "close in main thread" << std::endl; 114 | db1->closeDB(); 115 | th.join(); 116 | } 117 | TEST_F(TmpFile, dbtest_udpate_and_read_Test) { 118 | std::string s = "penapplepen"; 119 | Item bname(s.c_str(), s.size()); 120 | auto updateFunc = [&bname](Txn *txn) -> int { 121 | auto b = txn->createBucket(bname); 122 | EXPECT_NE(b, nullptr); 123 | EXPECT_EQ(b->put(Item::make_item("foo"), Item::make_item("bar")), 0); 124 | EXPECT_EQ(b->put(Item::make_item("foo1"), Item::make_item("bar1")), 0); 125 | EXPECT_EQ(b->put(Item::make_item("foo2"), Item::make_item("bar2")), 0); 126 | EXPECT_EQ(b->remove(Item::make_item("foo2")), 0); 127 | return 0; 128 | }; 129 | auto updateRet = db->update(updateFunc); 130 | EXPECT_EQ(updateRet, 0); 131 | auto viewFunc = [&bname](Txn *txn) -> int { 132 | auto b = txn->getBucket(bname); 133 | EXPECT_NE(b, nullptr); 134 | auto item1 = b->get(Item::make_item("foo2")); 135 | EXPECT_EQ(item1.length, 0); 136 | auto item2 = b->get(Item::make_item("foo")); 137 | EXPECT_EQ(item2, Item::make_item("bar")); 138 | return 0; 139 | }; 140 | EXPECT_EQ(db->view(viewFunc), 0); 141 | } 142 | 143 | TEST(dbtest, cmp_test) { 144 | std::vector inodeList; 145 | auto in1 = Inode{}; 146 | in1.key = Item::make_item("foo"); 147 | inodeList.push_back(in1); 148 | auto in2 = Inode{}; 149 | in2.key = Item::make_item("foo1"); 150 | inodeList.push_back(in2); 151 | 152 | auto in3 = Inode{}; 153 | auto t = Item::make_item("foo2"); 154 | in3.key = t; 155 | bool found = false; 156 | auto ret = binary_search(inodeList, t, cmp_wrapper, 157 | inodeList.size(), found); 158 | EXPECT_EQ(ret, 2); 159 | } 160 | 161 | TEST(dbtest, update_underlying_file) { 162 | auto name = newFileName(); 163 | std::unique_ptr db1(new DB); 164 | EXPECT_NE(db1->openDB(name, S_IRWXU), nullptr); 165 | int fd = db1->getFd(); 166 | lseek(fd, SEEK_SET, 0); 167 | auto ret = write(fd, "1234567890", 10); 168 | EXPECT_NE(ret, -1); 169 | db1->closeDB(); 170 | } 171 | 172 | TEST(dbtest, mmap_update_underlying_file) { 173 | auto name = newFileName(); 174 | auto fd = ::open(name.c_str(), O_CREAT | O_RDWR, S_IRWXU); 175 | EXPECT_NE(fd, -1); 176 | auto r1 = ::write(fd, "123456789", 10); 177 | EXPECT_EQ(r1, 10); 178 | auto mmapRet = ::mmap(nullptr, 10, PROT_READ, MAP_SHARED, fd, 0); 179 | EXPECT_NE(mmapRet, MAP_FAILED); 180 | for (size_t i = 0; i < 10; i++) { 181 | std::cout << ((char *) mmapRet)[i] << " "; 182 | } 183 | std::cout << std::endl; 184 | 185 | // lseek(fd, SEEK_SET, 0); 186 | auto ret = pwrite(fd, "ABCDEFG", 8, 0); 187 | EXPECT_NE(ret, -1); 188 | 189 | for (size_t i = 0; i < 10; i++) { 190 | std::cout << ((char *) mmapRet)[i] << " " << std::endl; 191 | } 192 | close(fd); 193 | } 194 | 195 | TEST(dbtest, update_closed) { 196 | auto name = newFileName(); 197 | std::unique_ptr db1(new DB); 198 | auto updateFunc = [](Txn *txn) -> int { 199 | auto b = txn->createBucket(Item::make_item("anewbucket")); 200 | return b != nullptr; 201 | }; 202 | auto ret = db1->update(updateFunc); 203 | EXPECT_EQ(ret, -1); 204 | } 205 | 206 | TEST(dbtest, commit_managed_txn) { 207 | auto name = newFileName(); 208 | std::unique_ptr db1(new DB); 209 | auto updateFunc = [](Txn *txn) -> int { 210 | return txn->commit(); 211 | }; 212 | auto ret = db1->update(updateFunc); 213 | EXPECT_EQ(ret, -1); 214 | } 215 | 216 | TEST(dbtest, rollback_managed_txn) { 217 | auto name = newFileName(); 218 | std::unique_ptr db1(new DB); 219 | auto updateFunc = [](Txn *txn) -> int { 220 | return txn->rollback(); 221 | }; 222 | auto ret = db1->update(updateFunc); 223 | EXPECT_EQ(ret, -1); 224 | } 225 | 226 | TEST(dbtest, view_commit_managed_txn) { 227 | auto name = newFileName(); 228 | std::unique_ptr db1(new DB); 229 | auto viewFunc = [](Txn *txn) -> int { 230 | return txn->commit(); 231 | }; 232 | auto ret = db1->view(viewFunc); 233 | EXPECT_EQ(ret, -1); 234 | } 235 | 236 | TEST(dbtest, view_rollback_managed_txn) { 237 | auto name = newFileName(); 238 | std::unique_ptr db1(new DB); 239 | auto viewFunc = [](Txn *txn) -> int { 240 | return txn->rollback(); 241 | }; 242 | auto ret = db1->view(viewFunc); 243 | EXPECT_EQ(ret, -1); 244 | } 245 | 246 | TEST_F(TmpFile, dbtest_consistency_test) { 247 | auto create = [](Txn *txn) { 248 | auto ret = txn->createBucket(Item::make_item("widgets")); 249 | return !(ret != nullptr); 250 | }; 251 | EXPECT_EQ(db->update(create), 0); 252 | 253 | auto updateFunc = [](Txn *txn) { 254 | return txn->getBucket(Item::make_item("widgets"))->put(Item::make_item("foo"), Item::make_item("bar")); 255 | }; 256 | for (int i = 0; i < 10; i++) { 257 | db->update(updateFunc); 258 | } 259 | 260 | auto updateRet = db->update([this](Txn *txn) -> int { 261 | auto p0 = txn->getPage(0); 262 | EXPECT_NE(p0, nullptr); 263 | EXPECT_EQ(isSet(p0->flag, PageFlag::metaPageFlag), true); 264 | 265 | auto p1 = txn->getPage(1); 266 | EXPECT_NE(p1, nullptr); 267 | EXPECT_EQ(isSet(p1->flag, PageFlag::metaPageFlag), true); 268 | 269 | auto p2 = txn->getPage(2); 270 | EXPECT_NE(p2, nullptr); 271 | EXPECT_EQ(db->getFreeLIst().freed(2), true); 272 | 273 | auto p3 = txn->getPage(3); 274 | EXPECT_NE(p3, nullptr); 275 | EXPECT_EQ(db->getFreeLIst().freed(3), true); 276 | 277 | auto p4 = txn->getPage(4); 278 | EXPECT_NE(p4, nullptr); 279 | EXPECT_EQ(isSet(p4->flag, PageFlag::leafPageFlag), true); 280 | 281 | auto p5 = txn->getPage(5); 282 | EXPECT_NE(p5, nullptr); 283 | EXPECT_EQ(isSet(p5->flag, PageFlag::freelistPageFlag), true); 284 | 285 | auto p6 = txn->getPage(6); 286 | EXPECT_EQ(p6, nullptr); 287 | 288 | return 0; 289 | }); 290 | 291 | EXPECT_EQ(updateRet, 0); 292 | } 293 | 294 | TEST(db_test, leafpageelement_padding_test) { 295 | 296 | } 297 | } -------------------------------------------------------------------------------- /test/flock/flock_test.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-7. 3 | // 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | namespace boltDB_CPP { 11 | TEST(flockteset, flockWLockOn2RLocksGranted) { 12 | std::cout << std::endl; 13 | /** 14 | * flock works on open file descriptions 15 | * see what happens if acquiring a write lock when there're multiple descriptions and two read locks are granted. 16 | * 17 | * write lock can not be granted until the read locks are released 18 | */ 19 | auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 20 | EXPECT_NE(fd1, -1); 21 | auto fd2 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 22 | EXPECT_NE(fd2, -1); 23 | auto fd3 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 24 | EXPECT_NE(fd3, -1); 25 | 26 | std::atomic granted{false}; 27 | //2 read locks 28 | EXPECT_EQ(file_Rlock(fd1), 0); 29 | EXPECT_EQ(file_Rlock(fd2), 0); 30 | std::cout << "grabbed 2 read locks" << std::endl; 31 | std::thread th([fd3, &granted]() { 32 | EXPECT_EQ(file_WlockBlocking(fd3), 0); 33 | std::cout << "grabbed write lock" << std::endl; 34 | granted.store(true); 35 | }); 36 | //release 2 read locks 37 | sleep(1); 38 | file_Unlock(fd1); 39 | EXPECT_EQ(granted.load(), false); 40 | file_Unlock(fd2); 41 | std::cout << "2 read locks released" << std::endl; 42 | th.join(); 43 | EXPECT_EQ(granted.load(), true); 44 | file_Unlock(fd3); 45 | close(fd1); 46 | close(fd2); 47 | close(fd3); 48 | std::cout << "test ends" << std::endl; 49 | } 50 | 51 | TEST(flockteset, flockWLockOn2RLocksGranted_closeButNotFunlock) { 52 | std::cout << std::endl; 53 | /** 54 | * flock works on open file descriptions 55 | * see what happens if acquiring a write lock when there're multiple descriptions and two read locks are granted. 56 | * 57 | * write lock can not be granted until the read locks are released 58 | * 59 | * close will release the lock implicitly 60 | * 61 | * this is a proof that read only db does not need to funlock when closing it 62 | */ 63 | auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 64 | EXPECT_NE(fd1, -1); 65 | auto fd2 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 66 | EXPECT_NE(fd2, -1); 67 | auto fd3 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 68 | EXPECT_NE(fd3, -1); 69 | 70 | //2 read locks 71 | EXPECT_EQ(file_Rlock(fd1), 0); 72 | EXPECT_EQ(file_Rlock(fd2), 0); 73 | std::cout << "grabbed 2 read locks" << std::endl; 74 | std::thread th([fd3]() { 75 | EXPECT_EQ(file_WlockBlocking(fd3), 0); 76 | std::cout << "grabbed write lock" << std::endl; 77 | }); 78 | //release 2 read locks 79 | sleep(1); 80 | close(fd1); 81 | close(fd2); 82 | std::cout << "2 read locks closed" << std::endl; 83 | th.join(); 84 | file_Unlock(fd3); 85 | close(fd3); 86 | std::cout << "test ends" << std::endl; 87 | } 88 | 89 | TEST(flockteset, flockLockTypeChange) { 90 | std::cout << std::endl; 91 | /** 92 | * flock calls on an already locked file will do lock type conversion. share <-> exclusive 93 | */ 94 | auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 95 | EXPECT_NE(fd1, -1); 96 | 97 | EXPECT_EQ(file_Rlock(fd1), 0); 98 | std::cout << "grabbed 1 read lock" << std::endl; 99 | EXPECT_EQ(file_WlockBlocking(fd1), 0); 100 | std::cout << "grabbed 1 write lock" << std::endl; 101 | EXPECT_EQ(file_Rlock(fd1), 0); 102 | std::cout << "grabbed 1 read lock" << std::endl; 103 | file_Unlock(fd1); 104 | close(fd1); 105 | std::cout << "test ends" << std::endl; 106 | } 107 | 108 | //this will block 109 | //TEST(dbteset, flockAcquireWLockOn2RLocksBeingGranted) { 110 | // std::cout << std::endl; 111 | // /** 112 | // * flock works on open file descriptions 113 | // * see what happens if acquiring a write lock when there're multiple descriptions of the same underlying file and two read locks are granted. 114 | // * 115 | // * write lock can not be granted 116 | // */ 117 | // auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 118 | // EXPECT_NE(fd1, -1); 119 | // auto fd2 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 120 | // EXPECT_NE(fd2, -1); 121 | // auto fd3 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 122 | // EXPECT_NE(fd3, -1); 123 | // 124 | // //2 read lock 125 | // EXPECT_EQ(file_Rlock(fd1), 0); 126 | // EXPECT_EQ(file_Rlock(fd2), 0); 127 | // std::cout << "grabbed 2 read locks" << std::endl; 128 | // EXPECT_EQ(file_WlockBlocking(fd3), 0); 129 | // std::cout << "grabbed write lock" << std::endl; 130 | // std::cout << "test ends" << std::endl; 131 | //} 132 | 133 | //this will block 134 | TEST(flockteset, flockAcquireWLockOn1WLockBeingGranted) { 135 | std::cout << std::endl; 136 | /** 137 | * flock works on open file descriptions 138 | * see what happens if acquiring a write lock when there is a write lock granted on a different file description 139 | * of the same underlying file. 140 | * 141 | * write lock can not be granted 142 | */ 143 | auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 144 | assert(fd1 != -1); 145 | auto fd2 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 146 | assert(fd1 != -1); 147 | 148 | //2 write lock 149 | EXPECT_EQ(file_WlockBlocking(fd1), 0); 150 | std::cout << "grabbed 1 write lock" << std::endl; 151 | std::thread th([fd1](){ 152 | sleep(1); 153 | close(fd1); 154 | }); 155 | EXPECT_EQ(file_WlockBlocking(fd2), 0); 156 | std::cout << "grabbed 1 write lock" << std::endl; 157 | th.join(); 158 | close(fd2); 159 | close(fd1); 160 | std::cout << "test ends" << std::endl; 161 | } 162 | 163 | TEST(flockteset, flockTwoWLocks) { 164 | std::cout << std::endl; 165 | /** 166 | * open one fd 167 | * apply two wlock on it 168 | * all write locks can be granted 169 | */ 170 | auto fd1 = ::open("dbtest", O_CREAT | O_RDWR, 0666); 171 | EXPECT_NE((fd1), -1); 172 | 173 | //2 write locks 174 | EXPECT_EQ(file_WlockBlocking(fd1), 0); 175 | std::cout << "grabbed 1 write lock" << std::endl; 176 | EXPECT_EQ(file_WlockBlocking(fd1), 0); 177 | std::cout << "grabbed 1 write lock" << std::endl; 178 | file_Unlock(fd1); 179 | close(fd1); 180 | std::cout << "test ends" << std::endl; 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /test/include/test_util.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-9. 3 | // 4 | 5 | #ifndef BOLTDB_IN_CPP_TEST_UTIL_H 6 | #define BOLTDB_IN_CPP_TEST_UTIL_H 7 | #include "gtest/gtest.h" 8 | #include "db.h" 9 | namespace boltDB_CPP { 10 | std::string newFileName() { 11 | auto ret = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()); 12 | int64_t currentTime = ret.count(); 13 | return std::to_string(currentTime); 14 | } 15 | 16 | struct TmpFile : testing::Test { 17 | std::unique_ptr db; 18 | void SetUp() override { 19 | //create a tmp db file 20 | db.reset(new DB); 21 | auto ret = db->openDB(newFileName(), 0666); 22 | assert(ret); 23 | } 24 | void TearDown() override { 25 | //close the tmp db file 26 | db->closeDB(); 27 | } 28 | 29 | }; 30 | } 31 | #endif //BOLTDB_IN_CPP_TEST_UTIL_H 32 | -------------------------------------------------------------------------------- /test/txn/txn_test.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by c6s on 18-5-5. 3 | // 4 | #include 5 | #include 6 | #include 7 | #include "db.h" 8 | #include "gtest/gtest.h" 9 | 10 | namespace boltDB_CPP { 11 | 12 | TEST(TypeTests, helloworldtest) { 13 | std::cout << "hello world" << std::endl; 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /third_party/gmock/gmock_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: wan@google.com (Zhanyong Wan) 31 | 32 | #include 33 | #include "gmock/gmock.h" 34 | #include "gtest/gtest.h" 35 | 36 | // MS C++ compiler/linker has a bug on Windows (not on Windows CE), which 37 | // causes a link error when _tmain is defined in a static library and UNICODE 38 | // is enabled. For this reason instead of _tmain, main function is used on 39 | // Windows. See the following link to track the current status of this bug: 40 | // http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=394464 // NOLINT 41 | #if GTEST_OS_WINDOWS_MOBILE 42 | # include // NOLINT 43 | 44 | GTEST_API_ int _tmain(int argc, TCHAR** argv) { 45 | #else 46 | GTEST_API_ int main(int argc, char** argv) { 47 | #endif // GTEST_OS_WINDOWS_MOBILE 48 | std::cout << "Running main() from gmock_main.cc\n"; 49 | // Since Google Mock depends on Google Test, InitGoogleMock() is 50 | // also responsible for initializing Google Test. Therefore there's 51 | // no need for calling testing::InitGoogleTest() separately. 52 | testing::InitGoogleMock(&argc, argv); 53 | return RUN_ALL_TESTS(); 54 | } 55 | -------------------------------------------------------------------------------- /third_party/valgrind/valgrind.supp: -------------------------------------------------------------------------------- 1 | { 2 | Handle GFLAGS leaks - Debug mode 3 | Memcheck:Leak 4 | ... 5 | fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag 6 | fun:_ZNSsC1EPKcRKSaIcE 7 | fun:_ZN3fLS25dont_pass0toDEFINE_stringEPcPKc 8 | ... 9 | } 10 | 11 | { 12 | Handle GFLAGS leaks - Release mode 13 | Memcheck:Leak 14 | ... 15 | fun:_ZNSsC1EPKcRKSaIcE 16 | fun:_GLOBAL__sub_I_config.cpp 17 | ... 18 | } 19 | 20 | { 21 | Handle init leak (32 bytes) 22 | Memcheck:Leak 23 | ... 24 | fun:_GLOBAL__sub_I_configuration.cpp 25 | fun:call_init.part.0 26 | fun:call_init 27 | fun:_dl_init 28 | ... 29 | } 30 | 31 | { 32 | Handles invalid free in NetworkAddress::Parse because of 'getaddrinfo' 33 | Memcheck:Free 34 | fun:free 35 | fun:__libc_freeres 36 | fun:_vgnU_freeres 37 | fun:__run_exit_handlers 38 | fun:exit 39 | fun:(below main) 40 | } 41 | 42 | { 43 | Handles leaks in libpg_query 44 | Memcheck:Leak 45 | match-leak-kinds: definite 46 | ... 47 | fun:AllocSetContextCreate 48 | fun:MemoryContextInit 49 | fun:pg_query_init 50 | fun:pg_query_enter_memory_context 51 | ... 52 | } 53 | 54 | --------------------------------------------------------------------------------