├── src ├── config.cmake.h ├── breakpad_dummy.cc ├── log_message.h ├── filemgr_ops.cc ├── breakpad.h ├── btree_var_kv_ops.h ├── hash_functions.h ├── configuration.h ├── btree_str_kv.h ├── btree_fast_str_kv.h ├── bgflusher.h ├── encryption_bogus.cc ├── version.h ├── hash_functions.cc ├── btree_kv.h ├── blockcache.h ├── list.h ├── hash.h ├── breakpad_linux.cc ├── filemgr_ops.h ├── avltree.h ├── encryption_aes.cc ├── breakpad_win32.cc ├── encryption.h ├── common.h ├── version.cc ├── encryption.cc ├── btreeblock.h ├── forestdb_endian.h ├── checksum.h ├── compactor.h ├── log_message.cc ├── checksum.cc ├── staleblock.h ├── api_wrapper.cc └── list.cc ├── tests ├── fdbench-micro │ ├── README.md │ ├── CMakeLists.txt │ └── config.h ├── e2e │ ├── CMakeLists.txt │ └── e2espec.h ├── usecase │ └── CMakeLists.txt ├── CMakeLists.txt ├── functional │ ├── functional_util.h │ └── functional_util.cc ├── stats-agg │ └── stat_aggregator.h ├── unit │ ├── filemgr_test.cc │ ├── list_test.cc │ ├── docio_test.cc │ └── hash_test.cc ├── anomaly │ ├── CMakeLists.txt │ └── filemgr_anomalous_ops.h └── include │ └── test.h ├── tools ├── threshold.stats ├── dump_option.ini └── dump_common.h ├── utils ├── crc32.h ├── gettimeofday_vs.h ├── debug.h ├── system_resource_stats.h ├── gettimeofday_vs.cc ├── time_utils.h ├── partiallock.h ├── timing.h ├── iniparser.h ├── time_utils.cc ├── memleak.h ├── bitwise_utils.h ├── system_resource_stats.cc ├── crypto_primitives.h └── partiallock.cc ├── .gitignore ├── cmake └── Modules │ ├── tsan.suppressions │ ├── FindJemalloc.cmake │ ├── FindAsyncIOLib.cmake │ ├── FindSnappy.cmake │ ├── FindEncryptionLib.cmake │ ├── MemoryCheck.cmake │ ├── ThreadSanitizer.cmake │ ├── CouchbaseCodeCoverage.cmake │ └── CodeCoverage.cmake ├── github_action_build.sh ├── .github └── workflows │ └── cmake.yml ├── README.md ├── conanfile.py ├── option └── option.h └── INSTALL.md /src/config.cmake.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* various */ 4 | #define FORESTDB_VERSION "${FORESTDB_VERSION}" -------------------------------------------------------------------------------- /tests/fdbench-micro/README.md: -------------------------------------------------------------------------------- 1 | # fdbench-micro 2 | forestdb micro benchmark testing 3 | 4 | **dependencies** 5 | See: https://github.com/couchbase/forestdb/blob/master/INSTALL.md#dependencies 6 | 7 | *NOTE: make install* of forestdb libraries is required prior to run 8 | 9 | **Standalone** 10 | ```bash 11 | mkdir build && cd build 12 | cmake ../ 13 | make 14 | ./fdb_bench 15 | ``` 16 | -------------------------------------------------------------------------------- /tools/threshold.stats: -------------------------------------------------------------------------------- 1 | set 100 2 | get 100 3 | delete 100 4 | iterator_init 100 5 | iterator_get 100 6 | iterator_next 100 7 | iterator_close 100 8 | snap_open 100 9 | kv_close 100 10 | file_close 10000 11 | snap_close 10000 12 | shutdown 100000 13 | commit_wal 100000 14 | commit_norm 100000 15 | compact 1000000 16 | -------------------------------------------------------------------------------- /utils/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_CRC32_H 2 | #define _JSAHN_CRC32_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | uint32_t crc32_1(const void* data, size_t len, uint32_t prev_value); 9 | uint32_t crc32_8(const void* data, size_t len, uint32_t prev_value); 10 | uint32_t crc32_8_last8(const void *data, size_t len, uint32_t prev_value); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /utils/gettimeofday_vs.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_GETTIMEOFDAY_VS 2 | #define _JSAHN_GETTIMEOFDAY_VS 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | struct timezone 12 | { 13 | int tz_minuteswest; /* minutes W of Greenwich */ 14 | int tz_dsttime; /* type of dst correction */ 15 | }; 16 | 17 | int gettimeofday_vs(struct timeval *tv, struct timezone *tz); 18 | 19 | #ifdef __cplusplus 20 | } 21 | #endif 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Keep the entries sorted to reduce the risk for a merge conflict 2 | *.[ao] 3 | couchstore_api/couchbench_fdb 4 | build 5 | coverage 6 | platform 7 | dummy2 8 | errorlog.txt 9 | forestdb_test 10 | libforestdb.so 11 | libforestdb_couch.so 12 | src/wc 13 | tests/bcache_test 14 | tests/btreeblock_test 15 | tests/crc_test 16 | tests/docio_test 17 | tests/filemgr_test 18 | tests/hash_test 19 | tests/hbtrie_test 20 | tests/list_test 21 | tests/mempool_test 22 | wt2 23 | CMakeUserPresets.json 24 | compile_commands.json 25 | .cache/ 26 | .build_success* 27 | -------------------------------------------------------------------------------- /cmake/Modules/tsan.suppressions: -------------------------------------------------------------------------------- 1 | # ThreadSanitizer suppressions file for Couchbase 2 | # https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions 3 | 4 | # In forestdb plock_lock is invoked from a lock structure that is sure to not 5 | # overlap with that of another thread by use of is_overlapped() test. 6 | # However since this function is not trusted by ThreadSanitizer it reports 7 | # a lock inversion since the locks in the race are from the same struct definition 8 | # but different memory addresses. This is hence a false positive. 9 | deadlock:plock_lock 10 | -------------------------------------------------------------------------------- /github_action_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | rm -rf build 5 | mkdir build 6 | cd build 7 | 8 | echo "head branch: ${GITHUB_HEAD_REF}" 9 | echo "base branch: ${GITHUB_BASE_REF}" 10 | echo "branch: ${GITHUB_REF}" 11 | 12 | BRANCH_NAME="${GITHUB_REF#refs/heads/}" 13 | echo "branch name: ${BRANCH_NAME}" 14 | 15 | if [ -z "${GITHUB_HEAD_REF}" ]; then 16 | # TODO: Code coverage 17 | cmake -DCMAKE_BUILD_TYPE=Debug ../ 18 | make -j2 19 | ctest --verbose 20 | else 21 | # TODO: ASAN 22 | echo "pull request sha: ${GITHUB_SHA}" 23 | cmake -DCMAKE_BUILD_TYPE=Debug ../ 24 | make -j2 25 | ctest --verbose 26 | fi 27 | -------------------------------------------------------------------------------- /tests/e2e/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(e2etest 2 | e2etest.cc 3 | e2espec.cc 4 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(e2etest ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(e2etest PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | # add test target 16 | add_test(e2etest e2etest) 17 | ADD_CUSTOM_TARGET(e2etests 18 | COMMAND ctest 19 | ) 20 | -------------------------------------------------------------------------------- /tests/usecase/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(usecase_test 2 | usecase_test.cc 3 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 4 | $ 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(usecase_test ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(usecase_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | # add test target 16 | add_test(usecase_test usecase_test) 17 | ADD_CUSTOM_TARGET(usecase_tests 18 | COMMAND ctest 19 | ) 20 | -------------------------------------------------------------------------------- /tests/fdbench-micro/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(fdb_microbench 2 | fdb_bench.cc 3 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 4 | $ 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(fdb_microbench ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(fdb_microbench PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | 16 | # add test target 17 | add_test(fdb_microbench fdb_microbench) 18 | ADD_CUSTOM_TARGET(fdbench-micro 19 | COMMAND ctest 20 | ) 21 | -------------------------------------------------------------------------------- /src/breakpad_dummy.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | void initialize_breakpad(const char* minidump_dir) { 21 | } 22 | 23 | void destroy_breakpad(void) { 24 | } 25 | -------------------------------------------------------------------------------- /utils/debug.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_DEBUG_H 2 | #define _JSAHN_DEBUG_H 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __DEBUG 8 | #include 9 | 10 | #define DBG(...) printf(__VA_ARGS__) 11 | #define DBGCMD(...) __VA_ARGS__ 12 | #define DBGSW(n, ...) if (_dbg_is_sw_set(n)) {__VA_ARGS__; } 13 | #else 14 | #define DBG(...) 15 | #define DBGCMD(...) 16 | #define DBGSW(n, ...) 17 | #endif 18 | 19 | void _dbg_sw_set(int n); 20 | void _dbg_sw_clear(int n); 21 | int _dbg_is_sw_set(int n); 22 | 23 | void _dbg_set_addr(int n, void *addr); 24 | void * _dbg_get_addr(int n); 25 | 26 | void _dbg_set_uint64_t(int n, uint64_t val); 27 | uint64_t _dbg_get_uint64_t(int n); 28 | 29 | fdb_status _dbg_install_handler(void); 30 | fdb_status _dbg_destroy_altstack(void); 31 | fdb_status _dbg_handle_crashes(const char *pathname); 32 | 33 | void dbg_print_buf(void *buf, uint64_t buflen, bool hex, int align); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(STAT_AGGREGATOR 2 | ${PROJECT_SOURCE_DIR}/tests/stats-agg/stat_aggregator.cc) 3 | add_library(TEST_STAT_AGG OBJECT ${STAT_AGGREGATOR}) 4 | 5 | set(ROOT_SRC ${PROJECT_SOURCE_DIR}/src) 6 | set(ROOT_UTILS ${PROJECT_SOURCE_DIR}/utils) 7 | set(UNIT_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/unit) 8 | set(FUNCTIONAL_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/functional) 9 | set(ANOMALY_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/anomaly) 10 | set(E2E_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/e2e) 11 | set(MICRO_BENCH_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fdbench-micro) 12 | set(USECASE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/usecase) 13 | 14 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include) 15 | include_directories(BEFORE ${PROJECT_SOURCE_DIR}/tests/stats-agg) 16 | 17 | # import each test suite 18 | add_subdirectory("${UNIT_TEST_DIR}") 19 | add_subdirectory("${FUNCTIONAL_TEST_DIR}") 20 | add_subdirectory("${ANOMALY_TEST_DIR}") 21 | add_subdirectory("${E2E_TEST_DIR}") 22 | add_subdirectory("${MICRO_BENCH_DIR}") 23 | add_subdirectory("${USECASE_TEST_DIR}") 24 | -------------------------------------------------------------------------------- /src/log_message.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "internal_types.h" 4 | 5 | typedef size_t fdb_log_levels; 6 | enum { 7 | FDB_LOG_FATAL = 1, 8 | FDB_LOG_ERROR = 2, 9 | FDB_LOG_WARNING = 3, 10 | FDB_LOG_INFO = 4, 11 | FDB_LOG_DEBUG = 5, 12 | FDB_LOG_TRACE = 6 13 | }; 14 | 15 | fdb_status fdb_log_init(struct fdb_log_config log_config); 16 | 17 | #define fdb_log(cb, lv, s, ...) \ 18 | fdb_log_impl(cb, lv, s, __FILE__, __func__, __LINE__, __VA_ARGS__) 19 | 20 | fdb_status fdb_log_set_global_callback(err_log_callback* log_callback); 21 | 22 | fdb_status fdb_log_impl(err_log_callback* log_callback, 23 | fdb_log_levels given_log_level, 24 | fdb_status status, 25 | const char* source_file, 26 | const char* func_name, 27 | size_t line_number, 28 | const char *format, ...); 29 | 30 | struct fdb_log_config { 31 | fdb_log_config(): log_msg_level(1) {} 32 | 33 | size_t log_msg_level; 34 | }; 35 | 36 | -------------------------------------------------------------------------------- /tools/dump_option.ini: -------------------------------------------------------------------------------- 1 | [doc] 2 | # Flag to enable displaying the body of document. 3 | # It is set to 'y' by default. 4 | print_body = y 5 | 6 | # Flag to enable displaying the metadata of document. 7 | # It is set to 'y' by default. 8 | print_meta = y 9 | 10 | # Flag to print the key of document in hex. 11 | # It is set to 'n' by default. 12 | print_key_in_hex = n 13 | 14 | # Flag to print the meatadata of document in hex. 15 | # It is set to 'y' by default. 16 | print_meta_in_hex = y 17 | 18 | # Flag to print the body of document in hex. 19 | # It is set to 'n' by default. 20 | print_body_in_hex = n 21 | 22 | [hex] 23 | # The number of bytes to be printed in a line in hex mode. 24 | # It is set to 16 by default. 25 | hex_align = 16 26 | 27 | [scan] 28 | # Mode of scanning of the documents in a file. 29 | # wal_first_index_next: scan WAL first and the main index next 30 | # key: scan all documents in an ascending order of key 31 | # seq: scan all documents in an ascending order of sequence number 32 | # It is set to 'wal_first_index_next' by default 33 | scan_mode = wal_first_index_next 34 | -------------------------------------------------------------------------------- /utils/system_resource_stats.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _SYSTEM_RESOURCE_STATS_H 19 | #define _SYSTEM_RESOURCE_STATS_H 20 | 21 | #include 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | int64_t get_memory_size(void); 28 | 29 | size_t get_num_cores(void); 30 | 31 | #ifdef __cplusplus 32 | } 33 | #endif 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /cmake/Modules/FindJemalloc.cmake: -------------------------------------------------------------------------------- 1 | # Locate jemalloc libraries on a host OS. 2 | 3 | if(UNIX) 4 | find_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h 5 | PATH_SUFFIXES include 6 | PATHS 7 | ~/Library/Frameworks 8 | /Library/Frameworks 9 | /usr/local 10 | /opt/local 11 | /opt/csw 12 | /opt) 13 | 14 | find_library(JEMALLOC_LIBRARIES 15 | NAMES jemalloc 16 | PATHS 17 | ~/Library/Frameworks 18 | /Library/Frameworks 19 | /usr/local 20 | /opt/local 21 | /opt/csw 22 | /opt) 23 | elseif(WIN32) 24 | endif() 25 | 26 | if(JEMALLOC_LIBRARIES) 27 | message(STATUS "Found jemalloc libraries in ${JEMALLOC_INCLUDE_DIR} : 28 | ${JEMALLOC_LIBRARIES}") 29 | add_compile_definitions(HAVE_JEMALLOC=1) 30 | set(MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) 31 | include_directories(AFTER ${JEMALLOC_INCLUDE_DIR}) 32 | mark_as_advanced(MALLOC_INCLUDE_DIR JEMALLOC_LIBRARIES) 33 | else() 34 | message(FATAL_ERROR "Can't find jemalloc libraries") 35 | endif(JEMALLOC_LIBRARIES) 36 | -------------------------------------------------------------------------------- /utils/gettimeofday_vs.cc: -------------------------------------------------------------------------------- 1 | #include "gettimeofday_vs.h" 2 | 3 | #if defined(_MSC_VER) || defined(_MSC_EXTENSIONS) 4 | #define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 5 | #else 6 | #define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL 7 | #endif 8 | 9 | int gettimeofday_vs(struct timeval *tv, struct timezone *tz) 10 | { 11 | FILETIME ft; 12 | unsigned __int64 tmpres = 0; 13 | static int tzflag; 14 | 15 | if (NULL != tv) { 16 | GetSystemTimeAsFileTime(&ft); 17 | 18 | tmpres |= ft.dwHighDateTime; 19 | tmpres <<= 32; 20 | tmpres |= ft.dwLowDateTime; 21 | 22 | /*converting file time to unix epoch*/ 23 | tmpres -= DELTA_EPOCH_IN_MICROSECS; 24 | tmpres /= 10; /*convert into microseconds*/ 25 | tv->tv_sec = (long)(tmpres / 1000000UL); 26 | tv->tv_usec = (long)(tmpres % 1000000UL); 27 | } 28 | 29 | if (NULL != tz) { 30 | if (!tzflag) { 31 | _tzset(); 32 | tzflag++; 33 | } 34 | tz->tz_minuteswest = _timezone / 60; 35 | tz->tz_dsttime = _daylight; 36 | } 37 | 38 | return 0; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/filemgr_ops.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "filemgr_ops.h" 19 | 20 | struct filemgr_ops * get_win_filemgr_ops(); 21 | struct filemgr_ops * get_linux_filemgr_ops(); 22 | 23 | struct filemgr_ops * get_filemgr_ops() 24 | { 25 | #if defined(WIN32) || defined(_WIN32) 26 | // windows 27 | return get_win_filemgr_ops(); 28 | #else 29 | // linux, mac os x 30 | return get_linux_filemgr_ops(); 31 | #endif 32 | } 33 | -------------------------------------------------------------------------------- /src/breakpad.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #pragma once 18 | 19 | /** 20 | * Initialize breakpad based on the specified minidump_dir 21 | * 22 | * The function may be called multiple times and allow for reconfiguration 23 | * of the breakpad's minidump_dir. 24 | */ 25 | void initialize_breakpad(const char* minidump_dir); 26 | 27 | /** 28 | * Cleaning up when breakpad no longer needed 29 | * (Assuming it is enabled and has been initialized) 30 | */ 31 | void destroy_breakpad(void); 32 | -------------------------------------------------------------------------------- /src/btree_var_kv_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_BTREE_VAR_KV_OPS 19 | #define _FDB_BTREE_VAR_KV_OPS 20 | 21 | #include "btree_fast_str_kv.h" 22 | #define _get_var_kv_ops btree_fast_str_kv_get_kb64_vb64 23 | #define _get_var_key btree_fast_str_kv_get_key 24 | #define _set_var_key btree_fast_str_kv_set_key 25 | #define _set_var_inf_key btree_fast_str_kv_set_inf_key 26 | #define _is_inf_key btree_fast_str_kv_is_inf_key 27 | #define _free_var_key btree_fast_str_kv_free_key 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/hash_functions.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_HASH_FUNCTIONS_H 19 | #define _JSAHN_HASH_FUNCTIONS_H 20 | 21 | #include 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | uint32_t hash_djb2(uint8_t *value, int len); 28 | uint32_t hash_djb2_last8(uint8_t *value, int len); 29 | uint32_t hash_uint_modular(uint64_t value, uint64_t mod); 30 | uint32_t hash_shuffle_2uint(uint64_t a, uint64_t b); 31 | 32 | #ifdef __cplusplus 33 | } 34 | #endif 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /src/configuration.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _CONFIGURATION_H 19 | #define _CONFIGURATION_H 20 | 21 | #include 22 | 23 | #include "common.h" 24 | #include "internal_types.h" 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | fdb_config get_default_config(void); 31 | fdb_kvs_config get_default_kvs_config(void); 32 | 33 | bool validate_fdb_config(fdb_config *fconfig); 34 | bool validate_fdb_kvs_config(fdb_kvs_config *kvs_config); 35 | 36 | #ifdef __cplusplus 37 | } 38 | #endif 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /cmake/Modules/FindAsyncIOLib.cmake: -------------------------------------------------------------------------------- 1 | # Locate async I/O libraries on a host OS. 2 | 3 | IF (UNIX) 4 | FIND_PATH(ASYNC_IO_INCLUDE_DIR libaio.h 5 | PATH_SUFFIXES include 6 | PATHS 7 | ~/Library/Frameworks 8 | /Library/Frameworks 9 | /usr/local 10 | /opt/local 11 | /opt/csw 12 | /opt) 13 | 14 | FIND_LIBRARY(ASYNC_IO_LIBRARIES 15 | NAMES aio 16 | PATHS 17 | ~/Library/Frameworks 18 | /Library/Frameworks 19 | /usr/local 20 | /opt/local 21 | /opt/csw 22 | /opt) 23 | ELSEIF (WIN32) 24 | ENDIF() 25 | 26 | IF (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 27 | MESSAGE(STATUS "Found async I/O libraries in ${ASYNC_IO_INCLUDE_DIR} : 28 | ${ASYNC_IO_LIBRARIES}") 29 | ADD_DEFINITIONS(-D_ASYNC_IO=1) 30 | set(ASYNC_IO_LIB ${ASYNC_IO_LIBRARIES}) 31 | include_directories(AFTER ${ASYNC_IO_INCLUDE_DIR}) 32 | MARK_AS_ADVANCED(ASYNC_IO_INCLUDE_DIR ASYNC_IO_LIBRARIES) 33 | ELSE (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 34 | MESSAGE(STATUS "Can't find async I/O libraries") 35 | ENDIF (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 36 | -------------------------------------------------------------------------------- /tests/functional/functional_util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FUNCTIONAL_UTIL_H 19 | #define _FUNCTIONAL_UTIL_H 20 | 21 | #include 22 | #include "common.h" 23 | #include "filemgr_ops.h" 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | void _set_random_string(char *str, int len); 30 | 31 | void _set_random_string_smallabt(char *str, int len); 32 | 33 | int _disk_dump(const char *filepath, const size_t pos, const size_t bytes); 34 | 35 | void logCallbackFunc(int err_code, 36 | const char *err_msg, 37 | void *pCtxData); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/btree_str_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_STR_KV_H 19 | #define _JSAHN_BTREE_STR_KV_H 20 | 21 | #include 22 | #include "common.h" 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | void btree_str_kv_set_key(void *key, void *str, size_t len); 29 | void btree_str_kv_set_inf_key(void *key); 30 | int btree_str_kv_is_inf_key(void *key); 31 | void btree_str_kv_get_key(void *key, void *strbuf, size_t *len); 32 | void btree_str_kv_free_key(void *key); 33 | 34 | struct btree_kv_ops; 35 | struct btree_kv_ops *btree_str_kv_get_kb64_vb64(struct btree_kv_ops *kv_ops); 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /.github/workflows/cmake.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | # Trigger the workflow on push or pull request, 5 | # but only for the main branch 6 | push: 7 | branches: 8 | - master 9 | pull_request: 10 | branches: 11 | - master 12 | 13 | env: 14 | # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) 15 | BUILD_TYPE: Debug 16 | 17 | jobs: 18 | build: 19 | # The CMake configure and build commands are platform agnostic and should work equally 20 | # well on Windows or Mac. You can convert this to a matrix build if you need 21 | # cross-platform coverage. 22 | # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix 23 | runs-on: ubuntu-latest 24 | timeout-minutes: 30 25 | 26 | steps: 27 | - uses: actions/checkout@v2 28 | 29 | - name: Dependency Install 30 | shell: bash 31 | # Some projects don't allow in-source building, so create a separate build directory 32 | # We'll use this as our working directory for all subsequent commands 33 | run: sudo apt update; sudo apt install -y lcov 34 | 35 | - name: Build and Test 36 | shell: bash 37 | # Execute tests defined by the CMake configuration. 38 | # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail 39 | run: ./github_action_build.sh 40 | -------------------------------------------------------------------------------- /src/btree_fast_str_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_FAST_STR_KV_H 19 | #define _JSAHN_BTREE_FAST_STR_KV_H 20 | 21 | #include 22 | #include "common.h" 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | void btree_fast_str_kv_set_key(void *key, void *str, size_t len); 29 | void btree_fast_str_kv_set_inf_key(void *key); 30 | int btree_fast_str_kv_is_inf_key(void *key); 31 | void btree_fast_str_kv_get_key(void *key, void *strbuf, size_t *len); 32 | void btree_fast_str_kv_free_key(void *key); 33 | 34 | struct btree_kv_ops; 35 | struct btree_kv_ops *btree_fast_str_kv_get_kb64_vb64(struct btree_kv_ops *kv_ops); 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /utils/time_utils.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_TIME_UTILS_H 19 | #define _JSAHN_TIME_UTILS_H 20 | 21 | #include 22 | #if defined(WIN32) || defined(_WIN32) 23 | #include 24 | #else 25 | #include 26 | #include 27 | #endif 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | struct timeval _utime_gap(struct timeval a, struct timeval b); 34 | 35 | #if defined(WIN32) || defined(_WIN32) 36 | void usleep(unsigned int useconds); 37 | #endif 38 | 39 | void decaying_usleep(unsigned int *sleep_time, unsigned int max_sleep_time); 40 | 41 | #if !defined(WIN32) && !defined(_WIN32) 42 | struct timespec convert_reltime_to_abstime(unsigned int ms); 43 | #endif 44 | 45 | #ifdef __cplusplus 46 | } 47 | #endif 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /src/bgflusher.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_BGFLUSHER_H 19 | #define _FDB_BGFLUSHER_H 20 | 21 | #include 22 | 23 | #include "internal_types.h" 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | struct bgflusher_config{ 30 | size_t num_threads; 31 | }; 32 | 33 | void bgflusher_init(struct bgflusher_config *config); 34 | void bgflusher_shutdown(); 35 | fdb_status bgflusher_register_file(struct filemgr *file, 36 | fdb_config *config, 37 | err_log_callback *log_callback); 38 | void bgflusher_switch_file(struct filemgr *old_file, struct filemgr *new_file, 39 | err_log_callback *log_callback); 40 | void bgflusher_deregister_file(struct filemgr *file); 41 | 42 | #ifdef __cplusplus 43 | } 44 | #endif 45 | 46 | #endif // _FDB_BGFLUSHER_H 47 | -------------------------------------------------------------------------------- /tests/fdbench-micro/config.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifdef __sun 19 | #include 20 | #endif 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "stat_aggregator.h" 28 | 29 | //#define __DEBUG_E2E 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | static const char BENCHDB_NAME[] = "fdb_bench_dbfile"; 36 | static const char BENCHKV_NAME[] = "fdb_bench_kv"; 37 | static const int KEY_SIZE = 16; 38 | static const int PERMUTED_BYTES = 4; 39 | 40 | // custom stats 41 | static const char ST_ITR_INIT[] = "iterator_init"; 42 | static const char ST_ITR_GET[] = "iterator_get"; 43 | static const char ST_ITR_NEXT[] = "iterator_next"; 44 | static const char ST_ITR_CLOSE[] = "iterator_close"; 45 | 46 | struct reader_context { 47 | fdb_kvs_handle *handle; 48 | stat_history_t *stat_itr_init; 49 | stat_history_t *stat_itr_get; 50 | stat_history_t *stat_itr_next; 51 | stat_history_t *stat_itr_close; 52 | }; 53 | 54 | #define alca(type, n) ((type*)alloca(sizeof(type) * (n))) 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | -------------------------------------------------------------------------------- /src/encryption_bogus.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "encryption.h" 19 | 20 | // Bogus encryption for test purposes. Simply adds the same number to each byte in a block. 21 | // The number is the LSB of the block number xor'ed with the first byte of the key. 22 | 23 | fdb_status bogus_setup(encryptor *e) { 24 | return FDB_RESULT_SUCCESS; 25 | } 26 | 27 | static fdb_status bogus_crypt(encryptor *e, 28 | bool encrypt, 29 | void *dst_buf, 30 | const void *src_buf, 31 | size_t size, 32 | bid_t bid) 33 | { 34 | int8_t delta = (bid & 0xFF) ^ e->key.bytes[0]; 35 | if (!encrypt) 36 | delta = -delta; 37 | const uint8_t *src = (const uint8_t *)src_buf; 38 | uint8_t *dst = (uint8_t *)dst_buf; 39 | while (size-- > 0) { 40 | *dst++ = *src++ + delta; 41 | } 42 | return FDB_RESULT_SUCCESS; 43 | } 44 | 45 | static encryption_ops bogus_ops = { 46 | bogus_setup, 47 | bogus_crypt 48 | }; 49 | 50 | const encryption_ops* const fdb_encryption_ops_bogus = &bogus_ops; 51 | -------------------------------------------------------------------------------- /src/version.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_VERSION_H 19 | #define _FDB_VERSION_H 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "libforestdb/fdb_errors.h" 23 | #include "common.h" 24 | 25 | #include "filemgr.h" 26 | 27 | INLINE filemgr_magic_t ver_get_latest_magic() { 28 | return FILEMGR_MAGIC_002; 29 | } 30 | bool ver_is_valid_magic(filemgr_magic_t magic); 31 | bool ver_is_magic_000(filemgr_magic_t magic); 32 | bool ver_is_atleast_magic_001(filemgr_magic_t magic); 33 | bool ver_staletree_support(filemgr_magic_t magic); 34 | bool ver_superblock_support(filemgr_magic_t magic); 35 | bool ver_non_consecutive_doc(filemgr_magic_t magic); 36 | size_t ver_get_new_filename_off(filemgr_magic_t magic); 37 | 38 | /** 39 | * Return the version of a given file's magic value 40 | * 41 | * @param magic ForestDB file magic value 42 | * @return Version of a given file's magic value 43 | */ 44 | const char* ver_get_version_string(filemgr_magic_t magic); 45 | 46 | /** 47 | * Return the offset of last_wal_flush_header field in a commit header 48 | */ 49 | size_t ver_get_last_wal_flush_hdr_off(filemgr_magic_t magic); 50 | 51 | #endif /* _FDB_VERSION_H */ 52 | 53 | -------------------------------------------------------------------------------- /cmake/Modules/FindSnappy.cmake: -------------------------------------------------------------------------------- 1 | # Locate snappy library 2 | # This module defines 3 | # SNAPPY_FOUND, if false, do not try to link with snappy 4 | # LIBSNAPPY, Library path and libs 5 | # SNAPPY_INCLUDE_DIR, where to find the ICU headers 6 | 7 | FIND_PATH(SNAPPY_INCLUDE_DIR snappy.h 8 | HINTS 9 | ENV SNAPPY_DIR 10 | PATH_SUFFIXES include 11 | PATHS 12 | ~/Library/Frameworks 13 | /Library/Frameworks 14 | /usr/local 15 | /opt/local 16 | /opt/csw 17 | /opt/snappy 18 | /opt) 19 | 20 | FIND_LIBRARY(SNAPPY_LIBRARIES 21 | NAMES snappy 22 | HINTS 23 | ENV SNAPPY_DIR 24 | PATHS 25 | ~/Library/Frameworks 26 | /Library/Frameworks 27 | /usr/local 28 | /opt/local 29 | /opt/csw 30 | /opt/snappy 31 | /opt) 32 | 33 | FIND_LIBRARY(SNAPPY_STATIC_LIBRARIES 34 | NAMES libsnappy.a 35 | HINTS 36 | ENV SNAPPY_DIR 37 | PATHS 38 | ~/Library/Frameworks 39 | /Library/Frameworks 40 | /usr/local 41 | /opt/local 42 | /opt/csw 43 | /opt/snappy 44 | /opt) 45 | 46 | 47 | IF (SNAPPY_LIBRARIES) 48 | include_directories(AFTER ${SNAPPY_INCLUDE_DIR}) 49 | MESSAGE(STATUS "Found snappy in ${SNAPPY_INCLUDE_DIR} : ${SNAPPY_LIBRARIES}") 50 | IF (SNAPPY_STATIC_LIBRARIES) 51 | MESSAGE(STATUS "Found snappy static library : ${SNAPPY_STATIC_LIBRARIES}") 52 | ENDIF() 53 | ELSE (SNAPPY_LIBRARIES) 54 | MESSAGE(FATAL_ERROR "Can't build forestdb without Snappy") 55 | ENDIF (SNAPPY_LIBRARIES) 56 | 57 | MARK_AS_ADVANCED(SNAPPY_INCLUDE_DIR SNAPPY_LIBRARIES) 58 | -------------------------------------------------------------------------------- /src/hash_functions.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Hash Functions 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #include "hash_functions.h" 8 | #include "common.h" 9 | 10 | // djb2 hashing using last LEN digits in VALUE 11 | uint32_t hash_djb2(uint8_t *value, int len) 12 | { 13 | unsigned hash = 5381; 14 | while(len--){ 15 | hash = ((hash << 5) + hash) + *((uint8_t*)value + len); 16 | } 17 | return hash; 18 | } 19 | 20 | uint32_t hash_djb2_last8(uint8_t *value, int len) 21 | { 22 | int min = MIN(len, 8), c; 23 | unsigned hash = 5381; 24 | c = min; 25 | while(c--){ 26 | hash = ((hash << 5) + hash) + *((uint8_t*)value + (len - min) + c); 27 | } 28 | return hash; 29 | } 30 | 31 | // LCOV_EXCL_START 32 | uint32_t hash_uint_modular(uint64_t value, uint64_t mod) 33 | { 34 | return value % mod; 35 | } 36 | // LCOV_EXCL_STOP 37 | 38 | // LCOV_EXCL_START 39 | uint32_t hash_shuffle_2uint(uint64_t a, uint64_t b) 40 | { 41 | uint32_t c; 42 | 43 | a ^= bitswap64(a ^ UINT64_C(0xffffffffffffffff)); 44 | b ^= bitswap64(b ^ UINT64_C(0xffffffffffffffff)); 45 | 46 | a = (a & 0xffff) ^ ((a & 0xffff0000) >> 16) ^ 47 | ((a & UINT64_C(0xffff00000000)) >> 32) ^ 48 | ((a & UINT64_C(0xffff000000000000)) >> 48); 49 | b = (b & 0xffff) ^ ((b & 0xffff0000) >> 16) ^ 50 | ((b & UINT64_C(0xffff00000000)) >> 32) ^ 51 | ((b & UINT64_C(0xffff000000000000)) >> 48); 52 | 53 | c = (((a & 0x0000000f) << 0) | 54 | ((b & 0x0000000f) << 4) | 55 | ((a & 0x000000f0) << 4) | 56 | ((b & 0x000000f0) << 8) | 57 | ((a & 0x00000f00) << 8) | 58 | ((b & 0x00000f00) << 12) | 59 | ((a & 0x0000f000) << 12) | 60 | ((b & 0x0000f000) << 16)); 61 | 62 | return (((c << 5) + c) << 5) + c; 63 | } 64 | // LCOV_EXCL_STOP 65 | 66 | -------------------------------------------------------------------------------- /utils/partiallock.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Generic Partial Lock 4 | * see https://github.com/greensky00/partiallock 5 | */ 6 | 7 | #ifndef _JSAHN_PARITIAL_LOCK_H 8 | #define _JSAHN_PARITIAL_LOCK_H 9 | 10 | #include 11 | #include 12 | 13 | #include "list.h" 14 | 15 | #ifdef __cplusplus 16 | extern "C" { 17 | #endif 18 | 19 | typedef uint64_t plock_range_t; 20 | typedef struct plock_node plock_entry_t; // opaque reference 21 | 22 | struct plock_ops { 23 | void (*init_user)(void *lock); 24 | void (*lock_user)(void *lock); 25 | void (*unlock_user)(void *lock); 26 | void (*destroy_user)(void *lock); 27 | void (*init_internal)(void *lock); 28 | void (*lock_internal)(void *lock); 29 | void (*unlock_internal)(void *lock); 30 | void (*destroy_internal)(void *lock); 31 | int (*is_overlapped)(void *start1, void *len1, void *start2, void *len2, void *aux); 32 | }; 33 | 34 | struct plock_config { 35 | struct plock_ops *ops; 36 | size_t sizeof_lock_user; 37 | size_t sizeof_lock_internal; 38 | size_t sizeof_range; 39 | void *aux; 40 | }; 41 | 42 | struct plock { 43 | struct list active; // list of active locks 44 | struct list inactive; // list of inactive (freed) locks 45 | struct plock_ops *ops; 46 | size_t sizeof_lock_user; 47 | size_t sizeof_lock_internal; 48 | size_t sizeof_range; 49 | void *lock; 50 | void *aux; 51 | }; 52 | 53 | #define PLOCK_RESULT_SUCCESS (0) 54 | #define PLOCK_RESULT_INVALID_ARGS (-1) 55 | #define PLOCK_RESULT_ALLOC_FAIL (-2) 56 | 57 | int plock_init(struct plock *plock, struct plock_config *config); 58 | plock_entry_t *plock_lock(struct plock *plock, void *start, void *len); 59 | int plock_unlock(struct plock *plock, plock_entry_t *plock_entry); 60 | int plock_destroy(struct plock *plock); 61 | 62 | #ifdef __cplusplus 63 | } 64 | #endif 65 | 66 | #endif 67 | 68 | -------------------------------------------------------------------------------- /tests/functional/functional_util.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "functional_util.h" 19 | 20 | void _set_random_string(char *str, int len) 21 | { 22 | str[len--] = 0; 23 | do { 24 | str[len] = '!' + random('~'-'!'); 25 | } while (len--); 26 | } 27 | 28 | void _set_random_string_smallabt(char *str, int len) 29 | { 30 | str[len--] = 0; 31 | do { 32 | str[len] = 'a' + random('z'-'a'); 33 | } while (len--); 34 | } 35 | 36 | int _disk_dump(const char *filepath, size_t pos, size_t bytes) { 37 | struct filemgr_ops *ops = get_filemgr_ops(); 38 | int fd = ops->open(filepath, O_CREAT| O_RDWR, 0666); 39 | if (fd < 0) { 40 | fprintf(stderr, "failure to open %s\n", filepath); 41 | return fd; 42 | } 43 | char *buf = (char *)malloc(bytes); 44 | if (!buf) { 45 | return -2; 46 | } 47 | if (ops->pwrite(fd, buf, bytes, pos) != (int) bytes) { 48 | return -1; 49 | } 50 | ops->close(fd); 51 | free(buf); 52 | return fd; 53 | } 54 | 55 | void logCallbackFunc(int err_code, 56 | const char *err_msg, 57 | void *pCtxData) { 58 | fprintf(stderr, "%s - error code: %d, error message: %s\n", 59 | (char *) pCtxData, err_code, err_msg); 60 | } 61 | 62 | -------------------------------------------------------------------------------- /utils/timing.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | //#define __DEBUG_E2E 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #if !defined(WIN32) && !defined(_WIN32) 25 | #include 26 | #endif 27 | 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | static const long int ERR_NS = 0xFFFFFFFF; 34 | typedef long int ts_nsec; 35 | ts_nsec get_monotonic_ts(); 36 | ts_nsec ts_diff(ts_nsec start, ts_nsec end); 37 | ts_nsec timed_fdb_get(fdb_kvs_handle *kv, fdb_doc *doc); 38 | ts_nsec timed_fdb_set(fdb_kvs_handle *kv, fdb_doc *doc); 39 | ts_nsec timed_fdb_delete(fdb_kvs_handle *kv, fdb_doc *doc); 40 | ts_nsec timed_fdb_compact(fdb_file_handle *fhandle); 41 | ts_nsec timed_fdb_commit(fdb_file_handle *fhandle, bool walflush); 42 | ts_nsec timed_fdb_snapshot(fdb_kvs_handle *kv, fdb_kvs_handle **snap_kv); 43 | ts_nsec timed_fdb_iterator_init(fdb_kvs_handle *kv, fdb_iterator **it); 44 | ts_nsec timed_fdb_iterator_get(fdb_iterator *it, fdb_doc **doc); 45 | ts_nsec timed_fdb_iterator_next(fdb_iterator *it); 46 | ts_nsec timed_fdb_iterator_close(fdb_iterator *it); 47 | ts_nsec timed_fdb_kvs_close(fdb_kvs_handle *kv); 48 | ts_nsec timed_fdb_close(fdb_file_handle *fhandle); 49 | ts_nsec timed_fdb_shutdown(); 50 | 51 | #ifdef __cplusplus 52 | } 53 | #endif 54 | -------------------------------------------------------------------------------- /tests/stats-agg/stat_aggregator.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | typedef struct { 25 | std::string name; 26 | std::vector latencies; 27 | } stat_history_t; 28 | 29 | struct Stats { 30 | // Stat name 31 | std::string name; 32 | // Calculated mean 33 | double mean; 34 | // Calculated median 35 | double median; 36 | // Estimated standard deviation 37 | double stddev; 38 | // Calculated 5th percentile 39 | double pct5; 40 | // Calculated 95th percentile 41 | double pct95; 42 | // Calculated 99th percentile 43 | double pct99; 44 | // Vector of samples 45 | std::vector* values; 46 | }; 47 | 48 | typedef stat_history_t** StatMatrix_t; 49 | typedef std::vector*> > samples_t; 50 | 51 | class StatAggregator { 52 | public: 53 | StatAggregator(int _num_stats, int _num_samples); 54 | 55 | ~StatAggregator(); 56 | 57 | void aggregateAndPrintStats(const char* title, int count, const char* unit); 58 | 59 | StatMatrix_t t_stats; 60 | 61 | private: 62 | 63 | void printValues(samples_t values, std::string unit); 64 | 65 | void fillLineWith(const char c, int spaces); 66 | 67 | int num_stats; 68 | int num_samples; 69 | }; 70 | -------------------------------------------------------------------------------- /src/btree_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_KV_H 19 | #define _JSAHN_BTREE_KV_H 20 | 21 | #include 22 | #include 23 | #include "common.h" 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | INLINE uint32_t deref32(const void *ptr) 30 | { 31 | #ifdef _ALIGN_MEM_ACCESS 32 | // 4-byte align check (rightmost 2 bits must be '00') 33 | if ( (size_t)ptr & 0x3 ) { 34 | uint32_t value; 35 | memcpy(&value, ptr, sizeof(uint32_t)); 36 | return value; 37 | } 38 | #endif 39 | return *(uint32_t*)ptr; 40 | } 41 | 42 | INLINE uint64_t deref64(const void *ptr) 43 | { 44 | #ifdef _ALIGN_MEM_ACCESS 45 | // 8-byte align check (rightmost 3 bits must be '000') 46 | // Not sure whether 8-byte integer should be aligned in 47 | // 8-byte boundary or just 4-byte boundary. 48 | if ( (size_t)ptr & 0x7 ) { 49 | uint64_t value; 50 | memcpy(&value, ptr, sizeof(uint64_t)); 51 | return value; 52 | } 53 | #endif 54 | return *(uint64_t*)ptr; 55 | } 56 | 57 | struct btree_kv_ops; 58 | struct btree_kv_ops * btree_kv_get_ku64_vu64(); 59 | struct btree_kv_ops * btree_kv_get_ku32_vu64(); 60 | struct btree_kv_ops * btree_kv_get_kb64_vb64(struct btree_kv_ops *kv_ops); 61 | struct btree_kv_ops * btree_kv_get_kb32_vb64(struct btree_kv_ops *kv_ops); 62 | struct btree_kv_ops * btree_kv_get_kbn_vb64(struct btree_kv_ops *kv_ops); 63 | 64 | #ifdef __cplusplus 65 | } 66 | #endif 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /src/blockcache.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BLOCKCACHE_H 19 | #define _JSAHN_BLOCKCACHE_H 20 | 21 | #include "filemgr.h" 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | typedef enum { 28 | BCACHE_REQ_CLEAN, 29 | BCACHE_REQ_DIRTY 30 | } bcache_dirty_t; 31 | 32 | struct bcache_config { 33 | bcache_config() : do_not_cache_doc_blocks(false) {} 34 | bool do_not_cache_doc_blocks; 35 | }; 36 | 37 | void bcache_init(int nblock, int blocksize, const bcache_config& bconfig); 38 | int bcache_read(struct filemgr *file, bid_t bid, void *buf); 39 | bool bcache_invalidate_block(struct filemgr *file, bid_t bid); 40 | int bcache_write(struct filemgr *file, bid_t bid, void *buf, 41 | bcache_dirty_t dirty, bool final_write, bool ignore_if_exist); 42 | int bcache_write_partial(struct filemgr *file, bid_t bid, void *buf, 43 | size_t offset, size_t len, bool final_write); 44 | void bcache_remove_dirty_blocks(struct filemgr *file); 45 | void bcache_remove_clean_blocks(struct filemgr *file); 46 | bool bcache_remove_file(struct filemgr *file); 47 | uint64_t bcache_get_num_blocks(struct filemgr *file); 48 | fdb_status bcache_flush(struct filemgr *file); 49 | uint64_t bcache_get_num_immutable(struct filemgr *file); 50 | fdb_status bcache_flush_immutable(struct filemgr *file); 51 | void bcache_shutdown(); 52 | uint64_t bcache_get_num_free_blocks(); 53 | void bcache_print_items(); 54 | 55 | #ifdef __cplusplus 56 | } 57 | #endif 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /src/list.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Doubly Linked List 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #ifndef _JSAHN_LIST_H 8 | #define _JSAHN_LIST_H 9 | 10 | #include 11 | #include 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | //#define _LIST_LOCK 18 | #include "arch.h" 19 | 20 | struct list_elem { 21 | struct list_elem *prev; 22 | struct list_elem *next; 23 | }; 24 | 25 | struct list { 26 | struct list_elem *head; 27 | struct list_elem *tail; 28 | #ifdef _LIST_LOCK 29 | spin_t lock; 30 | #endif 31 | }; 32 | 33 | #ifndef _get_entry 34 | #define _get_entry(ELEM, STRUCT, MEMBER) \ 35 | ((STRUCT *) ((uint8_t *) (ELEM) - offsetof (STRUCT, MEMBER))) 36 | #endif 37 | 38 | #ifdef LIST_LOCK 39 | void list_init(struct list *list); 40 | #else 41 | INLINE void list_init(struct list *list) 42 | { 43 | list->head = NULL; 44 | list->tail = NULL; 45 | } 46 | #endif 47 | 48 | void list_push_front(struct list *list, struct list_elem *e); 49 | void list_push_back(struct list *list, struct list_elem *e); 50 | void list_insert_before(struct list *list, struct list_elem *before, struct list_elem *e); 51 | void list_insert_after(struct list *list, struct list_elem *after, struct list_elem *e); 52 | 53 | struct list_elem *list_remove(struct list *list, struct list_elem *e); 54 | struct list_elem *list_remove_reverse(struct list *list, struct list_elem *e); 55 | 56 | struct list_elem *list_pop_front(struct list *list); 57 | struct list_elem *list_pop_back(struct list *list); 58 | 59 | #ifdef _LIST_LOCK 60 | struct list_elem *list_begin(struct list *list); 61 | struct list_elem *list_end(struct list *list); 62 | #else 63 | INLINE struct list_elem *list_begin(struct list *list) 64 | { 65 | return list->head; 66 | } 67 | 68 | INLINE struct list_elem *list_end(struct list *list) 69 | { 70 | return list->tail; 71 | } 72 | #endif 73 | 74 | INLINE struct list_elem *list_next(struct list_elem *e) 75 | { 76 | return e->next; 77 | } 78 | 79 | INLINE struct list_elem *list_prev(struct list_elem *e) 80 | { 81 | return e->prev; 82 | } 83 | 84 | #ifdef __cplusplus 85 | } 86 | #endif 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /src/hash.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Hash Table 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #ifndef _JSAHN_HASH_H 8 | #define _JSAHN_HASH_H 9 | 10 | #include 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | #define _HASH_TREE 17 | #ifdef _HASH_TREE 18 | #include "avltree.h" 19 | #else 20 | #include "list.h" 21 | #endif 22 | 23 | //#define _HASH_LOCK 24 | #ifdef _HASH_LOCK 25 | #include "arch.h" 26 | #endif 27 | 28 | struct hash_elem { 29 | #ifdef _HASH_TREE 30 | struct avl_node avl; 31 | #else 32 | struct list_elem list_elem; 33 | #endif 34 | }; 35 | 36 | struct hash; 37 | 38 | typedef uint32_t hash_hash_func(struct hash *hash, struct hash_elem *e); 39 | typedef int hash_cmp_func(struct hash_elem *a, struct hash_elem *b); 40 | //typedef int hash_cmp_func(void *a, void *b); 41 | typedef void hash_free_func(struct hash_elem *e); 42 | typedef void *hash_check_func(struct hash_elem *e, void *ctx); 43 | 44 | struct hash { 45 | size_t nbuckets; 46 | #ifdef _HASH_TREE 47 | struct avl_tree *buckets; 48 | #else 49 | struct list *buckets; 50 | #endif 51 | 52 | hash_hash_func *hash_func; 53 | hash_cmp_func *cmp; 54 | #ifdef _HASH_TREE 55 | avl_cmp_func *avl_cmp; 56 | #endif 57 | 58 | #ifdef _HASH_LOCK 59 | // define locks for each bucket 60 | spin_t *locks; 61 | #endif 62 | }; 63 | 64 | void hash_init(struct hash *hash, int nbuckets, hash_hash_func *hash_func, hash_cmp_func *cmp_func); 65 | void hash_insert(struct hash *hash, struct hash_elem *e); 66 | void hash_insert_by_hash_val(struct hash *hash, struct hash_elem *e, uint32_t hash_val); 67 | struct hash_elem * hash_find(struct hash *hash, struct hash_elem *e); 68 | struct hash_elem * hash_find_by_hash_val(struct hash *hash, struct hash_elem *e, 69 | uint32_t hash_val); 70 | void *hash_scan(struct hash *hash, hash_check_func *check_func, void *ctx); 71 | struct hash_elem * hash_remove(struct hash *hash, struct hash_elem *e); 72 | void hash_free(struct hash *hash); 73 | void hash_free_active(struct hash *hash, hash_free_func *free_func); 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /cmake/Modules/FindEncryptionLib.cmake: -------------------------------------------------------------------------------- 1 | # Locate the encryption library given by the build environment 2 | 3 | IF ("${_ENCRYPTION}" STREQUAL "commoncrypto") 4 | IF (APPLE) 5 | ADD_DEFINITIONS(-D_CRYPTO_CC=1) 6 | ELSE (APPLE) 7 | MESSAGE(FATAL_ERROR "commoncrypto lib is only available in Apple systems") 8 | ENDIF (APPLE) 9 | 10 | ELSEIF ("${_ENCRYPTION}" STREQUAL "openssl") 11 | INCLUDE(FindOpenSSL) 12 | 13 | IF (OPENSSL_FOUND) 14 | include_directories(AFTER ${OPENSSL_INCLUDE_DIR}) 15 | MESSAGE(STATUS "Found openssl in ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") 16 | ELSE (OPENSSL_FOUND) 17 | MESSAGE(FATAL_ERROR "Can't find openssl library") 18 | ENDIF (OPENSSL_FOUND) 19 | 20 | MARK_AS_ADVANCED(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES) 21 | ADD_DEFINITIONS(-D_CRYPTO_OPENSSL=1) 22 | set(CRYPTO_LIB ${OPENSSL_LIBRARIES}) 23 | 24 | ELSEIF ("${_ENCRYPTION}" STREQUAL "libtomcrypt") 25 | FIND_PATH(LIBTOMCRYPT_INCLUDE_DIR tomcrypt.h 26 | PATH_SUFFIXES include 27 | PATHS 28 | ~/Library/Frameworks 29 | /Library/Frameworks 30 | /usr/local 31 | /usr 32 | /opt/local 33 | /opt/csw 34 | /opt) 35 | 36 | FIND_LIBRARY(LIBTOMCRYPT_LIBRARIES 37 | NAMES tomcrypt 38 | PATHS 39 | ~/Library/Frameworks 40 | /Library/Frameworks 41 | /usr/local 42 | /usr 43 | /opt/local 44 | /opt/csw 45 | /opt) 46 | 47 | IF (LIBTOMCRYPT_LIBRARIES) 48 | include_directories(AFTER ${LIBTOMCRYPT_INCLUDE_DIR}) 49 | MESSAGE(STATUS "Found libtomcrypt in ${LIBTOMCRYPT_INCLUDE_DIR} : ${LIBTOMCRYPT_LIBRARIES}") 50 | ELSE (LIBTOMCRYPT_LIBRARIES) 51 | MESSAGE(FATAL_ERROR "Can't find libtomcrypt library") 52 | ENDIF (LIBTOMCRYPT_LIBRARIES) 53 | 54 | MARK_AS_ADVANCED(LIBTOMCRYPT_INCLUDE_DIR LIBTOMCRYPT_LIBRARIES) 55 | ADD_DEFINITIONS(-D_CRYPTO_LIBTOMCRYPT=1) 56 | set(CRYPTO_LIB ${LIBTOMCRYPT_LIBRARIES}) 57 | 58 | ELSE() 59 | MESSAGE(FATAL_ERROR "Can't find the cryto library ${_ENCRYPTION}") 60 | ENDIF() 61 | -------------------------------------------------------------------------------- /cmake/Modules/MemoryCheck.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # - Enable Valgrind Check 3 | # 4 | # Build a Valgrind build: 5 | # cmake -DCMAKE_BUILD_TYPE=Valgrind .. 6 | # make 7 | # make _targetname 8 | # 9 | # 10 | 11 | # Check prereqs 12 | FIND_PROGRAM( VALGRIND_PATH valgrind ) 13 | 14 | IF(NOT VALGRIND_PATH) 15 | MESSAGE(FATAL_ERROR "valgrind not found! Aborting...") 16 | ENDIF() # NOT VALGRIND_PATH 17 | 18 | SET(VALGRIND_OPTIONS "") 19 | 20 | SET(CMAKE_CXX_FLAGS_MEMCHECK 21 | "-g -O0 -fprofile-arcs " 22 | CACHE STRING "Flags used by the C++ compiler during valgrind builds." 23 | FORCE ) 24 | SET(CMAKE_C_FLAGS_MEMCHECK 25 | "-g -O0 -fprofile-arcs" 26 | CACHE STRING "Flags used by the C compiler during valgrind builds." 27 | FORCE ) 28 | SET(CMAKE_EXE_LINKER_FLAGS_MEMCHECK 29 | "" 30 | CACHE STRING "Flags used for linking binaries during valgrind builds." 31 | FORCE ) 32 | SET(CMAKE_SHARED_LINKER_FLAGS_MEMCHECK 33 | "" 34 | CACHE STRING "Flags used by the shared libraries linker during valgrind builds." 35 | FORCE ) 36 | MARK_AS_ADVANCED( 37 | CMAKE_CXX_FLAGS_MEMCHECK 38 | CMAKE_C_FLAGS_MEMCHECK 39 | CMAKE_EXE_LINKER_FLAGS_MEMCHECK 40 | CMAKE_SHARED_LINKER_FLAGS_MEMCHECK ) 41 | 42 | IF ( NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "Valgrind")) 43 | MESSAGE( WARNING "Valgrind results with an optimized (non-Debug) build may be misleading" ) 44 | ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" 45 | 46 | 47 | # Param _targetname The name of new the custom make target 48 | # Param list of target tests 49 | FUNCTION(SETUP_TARGET_FOR_MEMCHECK _targetname _test) 50 | 51 | IF(NOT VALGRIND_PATH) 52 | MESSAGE(FATAL_ERROR "valgrind not found! Aborting...") 53 | ENDIF() # NOT VALGRIND_PATH 54 | 55 | # Setup target 56 | ADD_CUSTOM_TARGET(${_targetname} 57 | 58 | COMMENT "Test: ${I}" 59 | 60 | # Run tests 61 | COMMAND ${VALGRIND_PATH} --trace-children=yes --quiet --tool=memcheck 62 | --leak-check=yes --show-reachable=yes 63 | --num-callers=100 --verbose --demangle=yes 64 | ${_test} 65 | 66 | COMMENT "Valgrind run complete for ${_test}" 67 | ) 68 | 69 | ENDFUNCTION() # SETUP_TARGET_FOR_MEMCHECK 70 | -------------------------------------------------------------------------------- /src/breakpad_linux.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | #include "client/linux/handler/exception_handler.h" 21 | 22 | #include 23 | #include 24 | 25 | using namespace google_breakpad; 26 | static ExceptionHandler* handler = nullptr; 27 | 28 | /* Called when an exception triggers a dump, outputs details to caller's logs */ 29 | static bool dumpCallback(const MinidumpDescriptor& descriptor, 30 | void* context, bool succeeded) { 31 | fprintf(stderr, "Breakpad caught a crash in forestdb. Writing crash dump " 32 | "to %s before terminating.\n", descriptor.path()); 33 | 34 | return succeeded; 35 | } 36 | 37 | static void create_breakpad(const char* minidump_dir) { 38 | MinidumpDescriptor descriptor(minidump_dir); 39 | handler = new ExceptionHandler(descriptor, 40 | /*filter*/nullptr, 41 | dumpCallback, 42 | /*callback-context*/nullptr, 43 | /*install_handler*/true, 44 | /*server_fd*/-1); 45 | } 46 | 47 | void initialize_breakpad(const char* minidump_dir) { 48 | // We cannot actually change any of breakpad's setings once created, only 49 | // remove it and re-create with new settings. 50 | destroy_breakpad(); 51 | 52 | if (minidump_dir != nullptr && minidump_dir[0] != '\0') { 53 | create_breakpad(minidump_dir); 54 | } 55 | } 56 | 57 | void destroy_breakpad(void) { 58 | delete handler; 59 | handler = nullptr; 60 | } 61 | -------------------------------------------------------------------------------- /src/filemgr_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_FILEMGR_OPS 19 | #define _JSAHN_FILEMGR_OPS 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "arch.h" 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | // Note: Please try to ensure that the following filemgr ops also have 29 | // equivalent test/filemgr_anomalous_ops.h/cc test apis for failure testing 30 | struct filemgr_ops { 31 | int (*open)(const char *pathname, int flags, mode_t mode); 32 | ssize_t (*pwrite)(int fd, void *buf, size_t count, cs_off_t offset); 33 | ssize_t (*pread)(int fd, void *buf, size_t count, cs_off_t offset); 34 | int (*close)(int fd); 35 | cs_off_t (*goto_eof)(int fd); 36 | cs_off_t (*file_size)(const char *filename); 37 | int (*fdatasync)(int fd); 38 | int (*fsync)(int fd); 39 | void (*get_errno_str)(char *buf, size_t size); 40 | 41 | // Async I/O operations 42 | int (*aio_init)(struct async_io_handle *aio_handle); 43 | int (*aio_prep_read)(struct async_io_handle *aio_handle, size_t aio_idx, 44 | size_t read_size, uint64_t offset); 45 | int (*aio_submit)(struct async_io_handle *aio_handle, int num_subs); 46 | int (*aio_getevents)(struct async_io_handle *aio_handle, int min, 47 | int max, unsigned int timeout); 48 | int (*aio_destroy)(struct async_io_handle *aio_handle); 49 | 50 | int (*get_fs_type)(int src_fd); 51 | int (*copy_file_range)(int fs_type, int src_fd, int dst_fd, 52 | uint64_t src_off, uint64_t dst_off, uint64_t len); 53 | }; 54 | 55 | struct filemgr_ops * get_filemgr_ops(); 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/avltree.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * AVL Tree 4 | * (C) 2014 Jung-Sang Ahn 5 | * see https://github.com/greensky00/avltree 6 | */ 7 | 8 | #ifndef _JSAHN_AVL_TREE_H 9 | #define _JSAHN_AVL_TREE_H 10 | 11 | #include "stddef.h" 12 | #include "stdint.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | struct avl_node { 19 | struct avl_node *parent, *left, *right; 20 | 21 | #ifdef _AVL_SEPARATE_PARENT_BF 22 | int bf; 23 | #endif 24 | #ifdef _AVL_NEXT_POINTER 25 | struct avl_node *prev, *next; 26 | #endif 27 | }; 28 | 29 | struct avl_tree{ 30 | struct avl_node *root; 31 | void *aux; 32 | }; 33 | 34 | #ifndef _get_entry 35 | #define _get_entry(ELEM, STRUCT, MEMBER) \ 36 | ((STRUCT *) ((uint8_t *) (ELEM) - offsetof (STRUCT, MEMBER))) 37 | #endif 38 | 39 | #define avl_parent(node) \ 40 | ((struct avl_node *)((uint64_t)(node)->parent & ~0x3)) 41 | 42 | #ifdef _AVL_SEPARATE_PARENT_BF 43 | #define avl_bf(node) ((node)->bf) 44 | #else 45 | #define avl_bf(node) (((int)((uint64_t)(node)->parent & 0x3)) - 1) 46 | #endif 47 | 48 | // *a < *b : return neg 49 | // *a == *b : return 0 50 | // *a > *b : return pos 51 | typedef int avl_cmp_func (struct avl_node *a, struct avl_node *b, void *aux); 52 | 53 | void avl_init(struct avl_tree *tree, void *aux); 54 | void avl_set_aux(struct avl_tree *tree, void *aux); 55 | struct avl_node* avl_insert(struct avl_tree *tree, 56 | struct avl_node *node, 57 | avl_cmp_func *func); 58 | struct avl_node* avl_search(struct avl_tree *tree, 59 | struct avl_node *node, 60 | avl_cmp_func *func); 61 | struct avl_node* avl_search_greater(struct avl_tree *tree, 62 | struct avl_node *node, 63 | avl_cmp_func *func); 64 | struct avl_node* avl_search_smaller(struct avl_tree *tree, 65 | struct avl_node *node, 66 | avl_cmp_func *func); 67 | void avl_remove(struct avl_tree *tree, 68 | struct avl_node *node); 69 | struct avl_node* avl_first(struct avl_tree *tree); 70 | struct avl_node* avl_last(struct avl_tree *tree); 71 | struct avl_node* avl_next(struct avl_node *node); 72 | struct avl_node* avl_prev(struct avl_node *node); 73 | #ifdef __cplusplus 74 | } 75 | #endif 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ForestDB 2 | 3 | ForestDB is a key-value storage engine that is developed by Couchbase Caching and Storage Team, and its main index structure is built from [Hierarchical B+-Tree based Trie](http://db.csail.mit.edu/sigmod11contest/sigmod_2011_contest_poster_jungsang_ahn.pdf), called HB+-Trie. [HB+-Trie](http://db.csail.mit.edu/sigmod11contest/sigmod_2011_contest_poster_jungsang_ahn.pdf) was originally presented at [ACM SIGMOD 2011 Programming Contest](http://db.csail.mit.edu/sigmod11contest/), by [Jung-Sang Ahn](http://cagsky.kaist.ac.kr/jsahn/) who works at Couchbase Caching and Storage Team. 4 | 5 | Compared with traditional B+-Tree based storage engines, ForestDB shows significantly better read and write performance with less storage overhead. ForestDB has been tested on various server OS environments (Centos, Ubuntu, Mac OS x, Windows) and mobile OSs (iOS, Android). 6 | 7 | ForestDB is currently in [1.0 Beta](https://github.com/couchbaselabs/forestdb/wiki/ForestDB-1.0-Beta) and its GA will be released separately soon. The test coverage stats for ForestDB are available in [ForestDB Code Coverage Report](http://labs.couchbase.com/fdbcoverage/index.html). 8 | 9 | [ForestDB benchmark program](https://github.com/couchbaselabs/ForestDB-Benchmark) is also available for performance comparisons with other key-value storage engines. 10 | 11 | Please visit the [ForestDB wiki](https://github.com/couchbaselabs/forestdb/wiki) for more details. 12 | 13 | ## Main Features 14 | 15 | - Keys and values are treated as an arbitrary binary. 16 | - Applications can supply a custom compare function to support a customized key order. 17 | - A value can be retrieved by its sequence number or disk offset in addition to a key. 18 | - Write-Ahead Logging (WAL) and its in-memory index are used to reduce the main index lookup / update overhead. 19 | - Multi-Version Concurrency Control (MVCC) support and append-only storage layer. 20 | - Multiple snapshot instances can be created from a given ForestDB instance to provide different views of database. 21 | - Rollback is supported to revert the database to a specific point. 22 | - Ranged iteration by keys or sequence numbers is supported for a partial or full range lookup operation. 23 | - Manual or auto compaction can be configured per ForestDB database file. 24 | - Transactional support with read\_committed or read\_uncommitted isolation level. 25 | 26 | ## How to build 27 | 28 | See INSTALL.MD 29 | 30 | ## How to Use 31 | 32 | Please refer to [Public APIs](https://github.com/couchbaselabs/forestdb/wiki/Public-APIs) and tests/fdb\_functional\_test.cc in ForestDB source directory. 33 | -------------------------------------------------------------------------------- /utils/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #if !defined(WIN32) && !defined(_WIN32) 37 | #include 38 | #endif 39 | #include 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | 46 | typedef struct _dictionary_ { 47 | /** Number of entries in dictionary */ 48 | int n; 49 | /** Storage size */ 50 | int size; 51 | /** List of string values */ 52 | char **val; 53 | /** List of string keys */ 54 | char **key ; 55 | /** List of hash values for keys */ 56 | unsigned *hash; 57 | } dictionary ; 58 | 59 | 60 | /* generated by genproto */ 61 | 62 | dictionary * iniparser_new(char *ininame); 63 | void iniparser_free(dictionary * d); 64 | 65 | int iniparser_getnsec(dictionary * d); 66 | char * iniparser_getsecname(dictionary * d, int n); 67 | void iniparser_dump(dictionary * d, FILE * f); 68 | void iniparser_dump_ini(dictionary * d, FILE * f); 69 | char * iniparser_getkey(dictionary *d, char *section, char *key); 70 | char * iniparser_getstr(dictionary * d, char * key); 71 | char * iniparser_getstring(dictionary * d, char * key, char * def); 72 | int iniparser_getint(dictionary * d, char * key, int notfound); 73 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 74 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 75 | int iniparser_find_entry(dictionary * ini, char * entry); 76 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 77 | void iniparser_unset(dictionary * ini, char * entry); 78 | 79 | #ifdef __cplusplus 80 | } 81 | #endif 82 | 83 | #endif 84 | 85 | -------------------------------------------------------------------------------- /tools/dump_common.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_DUMP_COMMON_H 19 | #define _FDB_DUMP_COMMON_H 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #if !defined(WIN32) && !defined(_WIN32) 27 | #include 28 | #include 29 | #endif 30 | 31 | #include "libforestdb/forestdb.h" 32 | #include "fdb_internal.h" 33 | #include "filemgr.h" 34 | #include "list.h" 35 | #include "hbtrie.h" 36 | #include "btree.h" 37 | #include "btree_var_kv_ops.h" 38 | #include "docio.h" 39 | #include "btreeblock.h" 40 | #include "common.h" 41 | #include "wal.h" 42 | #include "filemgr_ops.h" 43 | #include "configuration.h" 44 | #include "internal_types.h" 45 | #include "compactor.h" 46 | #include "memleak.h" 47 | #include "time_utils.h" 48 | 49 | #ifdef __DEBUG 50 | #ifndef __DEBUG_FDB 51 | #undef DBG 52 | #undef DBGCMD 53 | #undef DBGSW 54 | #define DBG(...) 55 | #define DBGCMD(...) 56 | #define DBGSW(n, ...) 57 | #endif 58 | #endif 59 | 60 | #ifdef __cplusplus 61 | extern "C" { 62 | #endif 63 | 64 | INLINE int is_subblock(bid_t subbid) 65 | { 66 | uint8_t flag; 67 | flag = (subbid >> (8 * (sizeof(bid_t)-2))) & 0x00ff; 68 | return flag; 69 | } 70 | 71 | INLINE void subbid2bid(bid_t subbid, size_t *subblock_no, size_t *idx, 72 | bid_t *bid) 73 | { 74 | uint8_t flag; 75 | flag = (subbid >> (8 * (sizeof(bid_t)-2))) & 0x00ff; 76 | *subblock_no = flag >> 5; 77 | // to distinguish subblock_no==0 to non-subblock 78 | *subblock_no -= 1; 79 | *idx = flag & (0x20 - 0x01); 80 | *bid = ((bid_t)(subbid << 16)) >> 16; 81 | } 82 | 83 | int _kvs_cmp_name_fdb_dump(struct avl_node *a, 84 | struct avl_node *b, 85 | void *aux); 86 | void print_header(fdb_kvs_handle *db); 87 | 88 | 89 | #ifdef __cplusplus 90 | } 91 | #endif 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /cmake/Modules/ThreadSanitizer.cmake: -------------------------------------------------------------------------------- 1 | # Support for building with ThreadSanitizer (tsan) - 2 | # https://code.google.com/p/thread-sanitizer/ 3 | 4 | INCLUDE(CheckCCompilerFlag) 5 | INCLUDE(CheckCXXCompilerFlag) 6 | INCLUDE(CMakePushCheckState) 7 | 8 | OPTION(CB_THREADSANITIZER "Enable ThreadSanitizer data race detector." 9 | OFF) 10 | 11 | IF (CB_THREADSANITIZER) 12 | CMAKE_PUSH_CHECK_STATE(RESET) 13 | SET(CMAKE_REQUIRED_FLAGS "-fsanitize=thread") # Also needs to be a link flag for test to pass 14 | CHECK_C_COMPILER_FLAG("-fsanitize=thread" HAVE_FLAG_SANITIZE_THREAD_C) 15 | CHECK_CXX_COMPILER_FLAG("-fsanitize=thread" HAVE_FLAG_SANITIZE_THREAD_CXX) 16 | CMAKE_POP_CHECK_STATE() 17 | 18 | IF(HAVE_FLAG_SANITIZE_THREAD_C AND HAVE_FLAG_SANITIZE_THREAD_CXX) 19 | SET(THREAD_SANITIZER_FLAG "-fsanitize=thread") 20 | 21 | SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${THREAD_SANITIZER_FLAG}") 22 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THREAD_SANITIZER_FLAG}") 23 | SET(CMAKE_CGO_LDFLAGS "${CMAKE_CGO_LDFLAGS} ${THREAD_SANITIZER_FLAG}") 24 | 25 | # TC/jemalloc are incompatible with ThreadSanitizer - force 26 | # the use of the system allocator. 27 | SET(COUCHBASE_MEMORY_ALLOCATOR system CACHE STRING "Memory allocator to use") 28 | 29 | # Configure CTest's MemCheck to ThreadSanitizer. 30 | SET(MEMORYCHECK_TYPE ThreadSanitizer) 31 | 32 | ADD_DEFINITIONS(-DTHREAD_SANITIZER) 33 | 34 | # Override the normal ADD_TEST macro to set the TSAN_OPTIONS 35 | # environment variable - this allows us to specify the 36 | # suppressions file to use. 37 | FUNCTION(ADD_TEST name) 38 | IF(${ARGV0} STREQUAL "NAME") 39 | SET(_name ${ARGV1}) 40 | ELSE() 41 | SET(_name ${ARGV0}) 42 | ENDIF() 43 | _ADD_TEST(${ARGV}) 44 | if (COUCHBASE_SERVER_BUILD) 45 | SET_TESTS_PROPERTIES(${_name} PROPERTIES ENVIRONMENT 46 | "TSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/tlm/tsan.suppressions") 47 | else (COUCHBASE_SERVER_BUILD) 48 | SET_TESTS_PROPERTIES(${_name} PROPERTIES ENVIRONMENT 49 | "TSAN_OPTIONS=suppressions=${PROJECT_SOURCE_DIR}/cmake/Modules/tsan.suppressions") 50 | endif (COUCHBASE_SERVER_BUILD) 51 | ENDFUNCTION() 52 | 53 | MESSAGE(STATUS "ThreadSanitizer enabled - forcing use of 'system' memory allocator.") 54 | ELSE() 55 | MESSAGE(FATAL_ERROR "CB_THREADSANITIZER enabled but compiler doesn't support ThreadSanitizer - cannot continue.") 56 | ENDIF() 57 | ENDIF() 58 | 59 | -------------------------------------------------------------------------------- /src/encryption_aes.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | #include "encryption.h" 20 | #include "crypto_primitives.h" 21 | #include 22 | 23 | #if AES256_AVAILABLE && SHA256_AVAILABLE 24 | 25 | static fdb_status aes_setup(encryptor *e) { 26 | // There must be room enough for AES keys in the provided structs: 27 | // (This would be a compile-time assert if C supported those.) 28 | assert(sizeof(e->key.bytes) == 32); 29 | assert(sizeof(e->extra) >= 32); 30 | // Precompute an auxiliary key by generating a SHA256 digest of the main key: 31 | sha256(&e->key.bytes, sizeof(e->key.bytes), e->extra); 32 | return FDB_RESULT_SUCCESS; 33 | } 34 | 35 | static fdb_status aes_crypt(encryptor *e, 36 | bool encrypt, 37 | void *dst_buf, 38 | const void *src_buf, 39 | size_t size, 40 | bid_t bid) 41 | { 42 | // Derive an IV as per the Encrypted Salt-Sector Initialization Value (ESSIV) algorithm 43 | // by encrypting the block number using the auxiliary key (a digest of the key.) 44 | // See https://en.wikipedia.org/wiki/Disk_encryption_theory 45 | uint8_t iv[16] = {0}; 46 | uint64_t bigBlockNo = _endian_encode(bid); 47 | memcpy(&iv, &bigBlockNo, sizeof(bigBlockNo)); 48 | if (!aes256(true, e->extra, NULL, &iv, &iv, sizeof(iv))) 49 | return FDB_RESULT_CRYPTO_ERROR; 50 | 51 | // Now encrypt/decrypt the block using the main key and the IV: 52 | if (!aes256(encrypt, e->key.bytes, iv, dst_buf, src_buf, size)) 53 | return FDB_RESULT_CRYPTO_ERROR; 54 | return FDB_RESULT_SUCCESS; 55 | } 56 | 57 | static encryption_ops aes_ops = { 58 | aes_setup, 59 | aes_crypt 60 | }; 61 | 62 | const encryption_ops* const fdb_encryption_ops_aes = &aes_ops; 63 | 64 | #else // AES not available: 65 | 66 | const encryption_ops* const fdb_encryption_ops_aes = NULL; 67 | 68 | #endif // AES256_AVAILABLE && SHA256_AVAILABLE 69 | -------------------------------------------------------------------------------- /utils/time_utils.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "time_utils.h" 19 | #include "stdint.h" 20 | #include "string.h" 21 | 22 | struct timeval _utime_gap(struct timeval a, struct timeval b) 23 | { 24 | struct timeval ret; 25 | if (b.tv_usec >= a.tv_usec) { 26 | ret.tv_usec = b.tv_usec - a.tv_usec; 27 | ret.tv_sec = b.tv_sec - a.tv_sec; 28 | }else{ 29 | ret.tv_usec = 1000000 + b.tv_usec - a.tv_usec; 30 | ret.tv_sec = b.tv_sec - a.tv_sec - 1; 31 | } 32 | return ret; 33 | } 34 | 35 | #if defined(WIN32) || defined(_WIN32) 36 | 37 | void usleep(unsigned int usec) 38 | { 39 | HANDLE timer; 40 | LARGE_INTEGER ft; 41 | 42 | // Convert to 100 nanosecond interval, negative value indicates relative time 43 | ft.QuadPart = -(10*usec); 44 | 45 | timer = CreateWaitableTimer(NULL, TRUE, NULL); 46 | SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); 47 | WaitForSingleObject(timer, INFINITE); 48 | CloseHandle(timer); 49 | } 50 | 51 | #else 52 | 53 | struct timespec convert_reltime_to_abstime(unsigned int ms) { 54 | struct timespec ts; 55 | struct timeval tp; 56 | uint64_t wakeup; 57 | 58 | memset(&ts, 0, sizeof(ts)); 59 | 60 | /* 61 | * Unfortunately pthread_cond_timedwait doesn't support relative sleeps 62 | * so we need to convert back to an absolute time. 63 | */ 64 | gettimeofday(&tp, NULL); 65 | wakeup = ((uint64_t)(tp.tv_sec) * 1000) + (tp.tv_usec / 1000) + ms; 66 | /* Round up for sub ms */ 67 | if ((tp.tv_usec % 1000) > 499) { 68 | ++wakeup; 69 | } 70 | 71 | ts.tv_sec = wakeup / 1000; 72 | wakeup %= 1000; 73 | ts.tv_nsec = wakeup * 1000000; 74 | return ts; 75 | } 76 | #endif //!defined(WIN32) && !defined(_WIN32) 77 | 78 | void decaying_usleep(unsigned int *sleep_time, unsigned int max_sleep_time) { 79 | usleep(*sleep_time); 80 | *sleep_time = *sleep_time << 1; 81 | if (max_sleep_time < *sleep_time) { 82 | *sleep_time = max_sleep_time; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/breakpad_win32.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | #include "client/windows/handler/exception_handler.h" 21 | 22 | #include 23 | #include 24 | 25 | using namespace google_breakpad; 26 | static ExceptionHandler* handler = nullptr; 27 | 28 | /* Called when an exception triggers a dump, outputs details to caller's logs */ 29 | static bool dumpCallback(const wchar_t* dump_path, const wchar_t* minidump_id, 30 | void* context, EXCEPTION_POINTERS* exinfo, 31 | MDRawAssertionInfo* assertion, bool succeeded) { 32 | fprintf(stderr, "Breakpad caught a crash in forestdb. Writing crash dump " 33 | "to %S\\%S before terminating.\n", dump_path, minidump_id); 34 | 35 | return succeeded; 36 | } 37 | 38 | static void create_breakpad(const char* minidump_dir) { 39 | // Takes a wchar_t* on Windows. 40 | size_t len = strlen(minidump_dir) + 1; 41 | wchar_t* wc_minidump_dir = new wchar_t[len]; 42 | size_t wlen = 0; 43 | mbstowcs_s(&wlen, wc_minidump_dir, len, minidump_dir, _TRUNCATE); 44 | 45 | handler = new ExceptionHandler(wc_minidump_dir, 46 | /*filter*/nullptr, 47 | dumpCallback, 48 | /*callback-context*/NULL, 49 | ExceptionHandler::HANDLER_ALL, 50 | MiniDumpNormal, 51 | /*pipe*/(wchar_t*) nullptr, 52 | /*custom_info*/nullptr); 53 | 54 | delete[] wc_minidump_dir; 55 | } 56 | 57 | void initialize_breakpad(const char* minidump_dir) { 58 | // We cannot actually change any of breakpad's setings once created, only 59 | // remove it and re-create with new settings. 60 | destroy_breakpad(); 61 | 62 | if (minidump_dir != nullptr && minidump_dir[0] != '\0') { 63 | create_breakpad(minidump_dir); 64 | } 65 | } 66 | 67 | void destroy_breakpad(void) { 68 | delete handler; 69 | handler = nullptr; 70 | } 71 | -------------------------------------------------------------------------------- /tests/unit/filemgr_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "filemgr.h" 23 | #include "filemgr_ops.h" 24 | #include "test.h" 25 | 26 | void basic_test(fdb_encryption_algorithm_t encryption) 27 | { 28 | TEST_INIT(); 29 | 30 | struct filemgr *file; 31 | struct filemgr_config config; 32 | const char *dbheader = "dbheader"; 33 | const char *dbheader2 = "dbheader2222222222"; 34 | char buf[256]; 35 | 36 | memset(&config, 0, sizeof(config)); 37 | config.blocksize = 4096; 38 | config.ncacheblock = 1024; 39 | config.options = FILEMGR_CREATE; 40 | config.num_wal_shards = 8; 41 | 42 | config.encryption_key.algorithm = encryption; 43 | memset(&config.encryption_key.bytes, 0x55, sizeof(config.encryption_key.bytes)); 44 | 45 | filemgr_open_result result = filemgr_open((char *) "./filemgr_testfile", 46 | get_filemgr_ops(), &config, NULL); 47 | result = filemgr_open((char *) "./filemgr_testfile", get_filemgr_ops(), &config, NULL); 48 | file = result.file; 49 | 50 | filemgr_update_header(file, (void*)dbheader, strlen(dbheader)+1, true); 51 | 52 | filemgr_close(file, true, NULL, NULL); 53 | result = filemgr_open((char *) "./filemgr_testfile", get_filemgr_ops(), &config, NULL); 54 | file = result.file; 55 | 56 | memcpy(buf, file->header.data, file->header.size); 57 | printf("%s\n", buf); 58 | 59 | filemgr_update_header(file, (void*)dbheader2, strlen(dbheader2) + 1, true); 60 | 61 | filemgr_close(file, true, NULL, NULL); 62 | 63 | sprintf(buf, "basic test, encryption=%d", (int)encryption); 64 | TEST_RESULT(buf); 65 | } 66 | 67 | void mt_init_test() 68 | { 69 | TEST_INIT(); 70 | 71 | TEST_RESULT("multi threaded initialization test"); 72 | } 73 | 74 | int main() 75 | { 76 | int r = system(SHELL_DEL" filemgr_testfile"); 77 | (void)r; 78 | 79 | basic_test(FDB_ENCRYPTION_NONE); 80 | basic_test(FDB_ENCRYPTION_BOGUS); 81 | mt_init_test(); 82 | 83 | return 0; 84 | } 85 | -------------------------------------------------------------------------------- /utils/memleak.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Simple Memory Leakage Detection Tool 4 | * (C) 2013 Jung-Sang Ahn 5 | * see https://github.com/greensky00/memleak 6 | */ 7 | 8 | #ifndef _JSAHN_MEMLEAK_H 9 | #define _JSAHN_MEMLEAK_H 10 | 11 | #include 12 | 13 | #ifdef _MSC_VER 14 | #ifdef forestdb_EXPORTS 15 | #define LIBMEMLEAK_API extern __declspec(dllexport) 16 | #else 17 | #define LIBMEMLEAK_API 18 | #endif 19 | #elif __GNUC__ 20 | #define LIBMEMLEAK_API __attribute ((visibility("default"))) 21 | #else 22 | #define LIBMEMLEAK_API 23 | #endif 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | #ifndef _MEMLEAK_ENABLE 30 | #define _MALLOC_OVERRIDE 31 | #endif 32 | #ifndef _MALLOC_OVERRIDE 33 | #define _MALLOC_OVERRIDE 34 | #define malloc(size) memleak_alloc(size, (char*)__FILE__, __LINE__) 35 | #define calloc(nmemb, size) memleak_calloc(nmemb, size, (char*)__FILE__, __LINE__) 36 | #define realloc(ptr, size) memleak_realloc(ptr, size); 37 | #define free(addr) memleak_free(addr, (char*)__FILE__, __LINE__) 38 | 39 | #if !defined(WIN32) 40 | 41 | #if !defined(__ANDROID__) 42 | #define posix_memalign(memptr, alignment, size) \ 43 | memleak_posix_memalign(memptr, alignment, size, (char*)__FILE__, __LINE__) 44 | #else // not __ANDROID__ 45 | #define memalign(alignment, size) \ 46 | memleak_memalign(alignment, size, (char*)__FILE__, __LINE__) 47 | #endif // not __ANDROID__ 48 | 49 | #else // not WIN32 50 | #define _aligned_malloc(size, align) \ 51 | memleak_aligned_malloc(size, align, (char*)__FILE__, __LINE__) 52 | #define _aligned_free(addr) \ 53 | memleak_aligned_free(addr, (char*)__FILE__, __LINE__) 54 | #endif // not WIN32 55 | 56 | #endif // not _MALLOC_OVERRIDE 57 | 58 | LIBMEMLEAK_API 59 | void memleak_start(); 60 | 61 | LIBMEMLEAK_API 62 | void memleak_end(); 63 | 64 | LIBMEMLEAK_API 65 | void * memleak_alloc(size_t size, char *file, size_t line); 66 | 67 | LIBMEMLEAK_API 68 | void * memleak_calloc(size_t nmemb, size_t size, char *file, size_t line); 69 | 70 | LIBMEMLEAK_API 71 | void * memleak_memalign(size_t alignment, size_t size, char *file, size_t line); 72 | 73 | LIBMEMLEAK_API 74 | void *memleak_realloc(void *ptr, size_t size); 75 | 76 | LIBMEMLEAK_API 77 | void memleak_free(void *addr, char *file, size_t line); 78 | 79 | #ifndef WIN32 80 | LIBMEMLEAK_API 81 | int memleak_posix_memalign(void **memptr, size_t alignment, size_t size, char *file, size_t line); 82 | #else 83 | LIBMEMLEAK_API 84 | void * memleak_aligned_malloc(size_t size, size_t alignment, char *file, size_t line); 85 | LIBMEMLEAK_API 86 | void memleak_aligned_free(void *addr, char *file, size_t line); 87 | #endif 88 | 89 | #ifdef __cplusplus 90 | } 91 | #endif 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /utils/bitwise_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_BITWISE_UTILS_H 2 | #define _JSAHN_BITWISE_UTILS_H 3 | 4 | #ifndef INT64_C 5 | #define INT64_C(c) (c ## LL) 6 | #define UINT64_C(c) (c ## ULL) 7 | #endif 8 | 9 | #define MAX(a,b) (((a)>(b))?(a):(b)) 10 | #define MIN(a,b) (((a)<(b))?(a):(b)) 11 | #define CHK_POW2(v) (!((uint64_t)v & ((uint64_t)v - 0x1))) 12 | 13 | #ifndef bitswap64 14 | #define bitswap64(v) \ 15 | ( (((v) & 0xff00000000000000ULL) >> 56) \ 16 | | (((v) & 0x00ff000000000000ULL) >> 40) \ 17 | | (((v) & 0x0000ff0000000000ULL) >> 24) \ 18 | | (((v) & 0x000000ff00000000ULL) >> 8) \ 19 | | (((v) & 0x00000000ff000000ULL) << 8) \ 20 | | (((v) & 0x0000000000ff0000ULL) << 24) \ 21 | | (((v) & 0x000000000000ff00ULL) << 40) \ 22 | | (((v) & 0x00000000000000ffULL) << 56) ) 23 | #endif 24 | 25 | #ifndef bitswap32 26 | #define bitswap32(v) \ 27 | ( (((v) & 0xff000000) >> 24) \ 28 | | (((v) & 0x00ff0000) >> 8) \ 29 | | (((v) & 0x0000ff00) << 8) \ 30 | | (((v) & 0x000000ff) << 24) ) 31 | #endif 32 | 33 | #ifndef bitswap16 34 | #define bitswap16(v) \ 35 | ( (((v) & 0xff00) >> 8) \ 36 | | (((v) & 0x00ff) << 8) ) 37 | #endif 38 | 39 | // can be faster under O3 optimization 40 | //#ifdef __BIT_CMP 41 | 42 | // 64-bit sign mask 43 | #define _64_SM (UINT64_C(0x8000000000000000)) 44 | // 32-bit sign mask 45 | #define _32_SM (0x80000000) 46 | // 32-bit value mask 47 | #define _32_M (0xffffffff) 48 | 49 | // 64-bit sign bit check 50 | #define _64_SC(a,b) ((uint64_t)(((a) & _64_SM)^((b) & _64_SM))>>63) 51 | // 64-bit sign bit check and convert to 32-bit 52 | #define _64_SC_32(a,b) ((uint64_t)(((a) & _64_SM)^((b) & _64_SM))>>32) 53 | 54 | // 32-bit sign bit check 55 | #define _32_SC(a,b) ((uint32_t)(((a) & _32_SM)^((b) & _32_SM))>>31) 56 | 57 | #define _U64_V(ptr) ( *(uint64_t*)(ptr) ) 58 | #define _U32_V(ptr) ( *(uint32_t*)(ptr) ) 59 | 60 | // check whether V is non-zero or not (return 1 when non-zero, otherwise 0) 61 | #define _NZ(v) ( (( (v) | (~(v) + 1)) >> 31) & 0x1 ) 62 | #define _NZ_64(v) ( (( (v) | (~(v) + 1)) >> 63) & 0x1 ) 63 | 64 | // convert 64-bit value to 32-bit value preserving sign bit (but not value) 65 | //#define _CSB(v) ( ((v)>>32) | (((v)&_32_M)>>1) | ((v)&0x1) ) 66 | #define _CSB(v) ( ((v)>>32) | _NZ_64((uint64_t)v) ) 67 | 68 | // map from (32-bit signed integer){neg, 0, pos} to {-1, 0, 1} 69 | #define _CS(v) ((~(((v) & _32_SM)>>31)+1) | _NZ(v)) 70 | #define _MAP(v) (int32_t)(_CS((uint32_t)v)) 71 | 72 | #define _CMP_U32(a, b) \ 73 | (int32_t)( _CSB((int64_t)(a)-(int64_t)(b)) ) 74 | #define _CMP_U32_P(a, b) _CMP_U32(_U32_V(a), _U32_V(b)) 75 | 76 | #define _CMP_U64(a, b) \ 77 | (int32_t) ( \ 78 | ( (_64_SC(a,b)-1) & _CSB((a)-(b)) ) | /* a and b have same sign */ \ 79 | ( (_64_SC_32(a,b) | _64_SC(a,b)) & ( (((b) & _64_SM) >> 32) | 0x1))) /* a and b have different sign */ 80 | #define _CMP_U64_P(a, b) _CMP_U64(_U64_V(a), _U64_V(b)) 81 | 82 | //#endif 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /src/encryption.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_ENCRYPTION_H 19 | #define _FDB_ENCRYPTION_H 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "libforestdb/fdb_errors.h" 23 | #include "common.h" 24 | 25 | // A very simple but insecure algorithm for testing purposes only. 26 | enum { 27 | FDB_ENCRYPTION_BOGUS = -1 28 | }; 29 | 30 | // An "object" that can perform encryption. 31 | typedef struct { 32 | const struct encryption_ops *ops; // callbacks 33 | fdb_encryption_key key; // key + algorithm 34 | uint8_t extra[32]; // scratch space for encryptor to use 35 | } encryptor; 36 | 37 | // Initializes an encryptor given a key. 38 | fdb_status fdb_init_encryptor(encryptor*, 39 | const fdb_encryption_key*); 40 | 41 | // Decrypts a block of data. 42 | fdb_status fdb_decrypt_block(encryptor*, 43 | void *buf, 44 | size_t blocksize, 45 | bid_t bid); 46 | 47 | // Encrypts one or more consecutive blocks of data. 48 | fdb_status fdb_encrypt_blocks(encryptor*, 49 | void *dst_buf, 50 | const void *src_buf, 51 | size_t blocksize, 52 | unsigned num_blocks, 53 | bid_t start_bid); 54 | 55 | // Callbacks provided by an encryption implementation. 56 | typedef struct encryption_ops { 57 | fdb_status (*setup)(encryptor*); 58 | fdb_status (*crypt)(encryptor*, 59 | bool encrypt, 60 | void *dst_buf, 61 | const void *src_buf, 62 | size_t size, 63 | bid_t bid); 64 | } encryption_ops; 65 | 66 | // Provides the encryption_ops (callbacks) for a particular algorithm. 67 | const encryption_ops* get_encryption_ops(fdb_encryption_algorithm_t); 68 | 69 | // Declarations of encryption_ops for specific algorithms. 70 | // Will be NULL if not implemented on the current platform. 71 | extern const encryption_ops* const fdb_encryption_ops_aes; 72 | extern const encryption_ops* const fdb_encryption_ops_bogus; 73 | 74 | #endif /* _FDB_ENCRYPTION_H */ 75 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_COMMON_H 19 | #define _JSAHN_COMMON_H 20 | 21 | #include 22 | #include 23 | 24 | #undef NDEBUG 25 | #include 26 | 27 | #if !defined(__APPLE__) && !defined(__FreeBSD__) 28 | #include 29 | #endif 30 | 31 | #include "option.h" 32 | #include "arch.h" 33 | #include "debug.h" 34 | #include "bitwise_utils.h" 35 | 36 | #ifndef _MEMPOOL 37 | #define mempool_alloc malloc 38 | #define mempool_free free 39 | #endif 40 | 41 | #define _MEMORY_OVERRIDE 42 | 43 | #define alca(type, n) ((type*)alloca(sizeof(type) * (n))) 44 | 45 | #define seq_memcpy(dest, src, size, offset_var) \ 46 | memcpy(dest, src, size); \ 47 | offset_var += size 48 | 49 | typedef uint64_t bid_t; 50 | #define BLK_NOT_FOUND (UINT64_C(0xffffffffffffffff)) 51 | 52 | typedef uint8_t file_status_t; 53 | enum{ 54 | // No compaction now or before or in progress 55 | FILE_NORMAL = 0, 56 | // fdb_compact has begun - switch to using the new_file for new fdb_set()s 57 | FILE_COMPACT_OLD = 1, 58 | // this is a new file in the compaction process 59 | FILE_COMPACT_NEW = 2, 60 | // all open handles on the file has been closed 61 | FILE_CLOSED = 3, 62 | // compaction completed successfully, and file needs to be removed once all 63 | // open handles refering to this file are closed or switched to new_file 64 | FILE_REMOVED_PENDING = 4, 65 | }; 66 | 67 | #define BLK_MARKER_BNODE (0xff) 68 | #define BLK_MARKER_DBHEADER (0xee) 69 | #define BLK_MARKER_DOC (0xdd) 70 | #define BLK_MARKER_SB (0xcc) // superblock 71 | #define BLK_MARKER_SIZE (1) 72 | #define DOCBLK_META_SIZE (16) 73 | #define BMP_REVNUM_MASK 0xffff 74 | 75 | struct docblk_meta { 76 | bid_t next_bid; 77 | uint16_t sb_bmp_revnum_hash; 78 | uint8_t reserved[5]; 79 | uint8_t marker; 80 | }; 81 | 82 | #define randomize() srand((unsigned)time(NULL)) 83 | #define random(num) ((rand())%(num)) 84 | 85 | #define random_custom(prev, num) (prev) = ((prev)+811)&((num)-1) 86 | 87 | 88 | /* Custom assert */ 89 | 90 | void fdb_assert_die(const char* expression, const char* file, int line, 91 | uint64_t val, uint64_t expected); 92 | 93 | #define fdb_assert(cond, val, expected) \ 94 | ((void)((cond) ? (void)0 : fdb_assert_die(#cond, __FILE__, __LINE__,\ 95 | (uint64_t)(val), (uint64_t)(expected)))) 96 | 97 | #endif // _JSAHN_COMMON_H 98 | -------------------------------------------------------------------------------- /src/version.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "version.h" 19 | #include 20 | 21 | bool ver_is_valid_magic(filemgr_magic_t magic) 22 | { 23 | if ( magic == FILEMGR_MAGIC_000 || 24 | (magic >= FILEMGR_MAGIC_001 && magic <= FILEMGR_LATEST_MAGIC)) { 25 | return true; 26 | } 27 | return false; 28 | } 29 | 30 | bool ver_is_magic_000(filemgr_magic_t magic) 31 | { 32 | if (magic == FILEMGR_MAGIC_000) { 33 | return true; 34 | } 35 | return false; 36 | } 37 | 38 | bool ver_is_atleast_magic_001(filemgr_magic_t magic) 39 | { 40 | // All magic numbers since FILEMGR_MAGIC_001 41 | if (magic >= FILEMGR_MAGIC_001 && magic <= FILEMGR_LATEST_MAGIC) { 42 | return true; 43 | } 44 | return false; 45 | } 46 | 47 | bool ver_staletree_support(filemgr_magic_t magic) 48 | { 49 | // All magic numbers since FILEMGR_MAGIC_002 50 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 51 | return true; 52 | } 53 | return false; 54 | } 55 | 56 | bool ver_non_consecutive_doc(filemgr_magic_t magic) 57 | { 58 | // All magic numbers since FILEMGR_MAGIC_002 59 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 60 | return true; 61 | } 62 | return false; 63 | } 64 | 65 | bool ver_superblock_support(filemgr_magic_t magic) 66 | { 67 | // All magic numbers since FILEMGR_MAGIC_002 68 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 69 | return true; 70 | } 71 | return false; 72 | } 73 | 74 | size_t ver_get_new_filename_off(filemgr_magic_t magic) { 75 | switch(magic) { 76 | case FILEMGR_MAGIC_000: return 64; 77 | case FILEMGR_MAGIC_001: return 72; 78 | case FILEMGR_MAGIC_002: return 80; 79 | } 80 | return (size_t) -1; 81 | } 82 | 83 | size_t ver_get_last_wal_flush_hdr_off(filemgr_magic_t magic) { 84 | switch(magic) { 85 | case FILEMGR_MAGIC_000: return 40; 86 | case FILEMGR_MAGIC_001: return 48; 87 | case FILEMGR_MAGIC_002: return 56; 88 | } 89 | return (size_t) -1; 90 | } 91 | 92 | const char* ver_get_version_string(filemgr_magic_t magic) { 93 | switch (magic) { 94 | case FILEMGR_MAGIC_000: 95 | return "ForestDB v1.x format"; 96 | case FILEMGR_MAGIC_001: 97 | return "ForestDB v1.x format"; 98 | case FILEMGR_MAGIC_002: 99 | return "ForestDB v2.x format"; 100 | } 101 | return "unknown"; 102 | } 103 | -------------------------------------------------------------------------------- /src/encryption.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "encryption.h" 19 | #include 20 | 21 | //#define FDB_LOG_CRYPTO 22 | 23 | fdb_status fdb_init_encryptor(encryptor *e, 24 | const fdb_encryption_key *key) 25 | { 26 | if (key->algorithm == FDB_ENCRYPTION_NONE) { 27 | e->ops = NULL; 28 | return FDB_RESULT_SUCCESS; 29 | } 30 | #ifdef FDB_LOG_CRYPTO 31 | fprintf(stderr, "CRYPT: Initializing context for key %d:%llx\n", 32 | key->algorithm, *(uint64_t*)key->bytes); 33 | #endif 34 | e->ops = get_encryption_ops(key->algorithm); 35 | if (!e->ops) 36 | return FDB_RESULT_CRYPTO_ERROR; // unsupported algorithm 37 | e->key = *key; 38 | return e->ops->setup(e); 39 | } 40 | 41 | fdb_status fdb_decrypt_block(encryptor *e, 42 | void *buf, 43 | size_t blocksize, 44 | bid_t bid) 45 | { 46 | #ifdef FDB_LOG_CRYPTO 47 | fprintf(stderr, "CRYPT: Decrypting block #%llu with key %d:%llx\n", 48 | bid, e->key.algorithm, *(uint64_t*)e->key.bytes); 49 | #endif 50 | return e->ops->crypt(e, false, buf, buf, blocksize, bid); 51 | } 52 | 53 | fdb_status fdb_encrypt_blocks(encryptor *e, 54 | void *dst_buf, 55 | const void *src_buf, 56 | size_t blocksize, 57 | unsigned num_blocks, 58 | bid_t start_bid) 59 | { 60 | #ifdef FDB_LOG_CRYPTO 61 | fprintf(stderr, "CRYPT: Encrypting blocks #%llu-%llu with key %d:%llx\n", 62 | start_bid, start_bid+num_blocks-1, 63 | e->key.algorithm, *(uint64_t*)e->key.bytes); 64 | #endif 65 | fdb_status status = FDB_RESULT_SUCCESS; 66 | for (unsigned i = 0; i < num_blocks; i++) { 67 | status = e->ops->crypt(e, 68 | true, 69 | (uint8_t*)dst_buf + i*blocksize, 70 | (const uint8_t*)src_buf + i*blocksize, 71 | blocksize, 72 | start_bid + i); 73 | if (status != FDB_RESULT_SUCCESS) 74 | break; 75 | } 76 | return status; 77 | } 78 | 79 | const encryption_ops* get_encryption_ops(fdb_encryption_algorithm_t algorithm) { 80 | switch (algorithm) { 81 | case FDB_ENCRYPTION_AES256: 82 | return fdb_encryption_ops_aes; 83 | case FDB_ENCRYPTION_BOGUS: 84 | return fdb_encryption_ops_bogus; 85 | default: 86 | return NULL; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/btreeblock.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREEBLOCK_H 19 | #define _JSAHN_BTREEBLOCK_H 20 | 21 | #include "filemgr.h" 22 | #include "list.h" 23 | #include "avltree.h" 24 | #include "btree.h" 25 | #include "libforestdb/fdb_errors.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | struct btreeblk_block; 32 | 33 | struct btreeblk_subblocks{ 34 | bid_t bid; 35 | uint32_t sb_size; 36 | uint16_t nblocks; 37 | uint8_t *bitmap; 38 | }; 39 | 40 | struct dirty_snapshot_t { 41 | spin_t lock; 42 | int ref_cnt; 43 | struct avl_tree *snap_tree; 44 | }; 45 | 46 | struct btreeblk_handle{ 47 | uint32_t nodesize; 48 | uint16_t nnodeperblock; 49 | int64_t nlivenodes; 50 | int64_t ndeltanodes; 51 | struct list alc_list; 52 | struct list read_list; 53 | struct filemgr *file; 54 | err_log_callback *log_callback; 55 | 56 | #ifdef __BTREEBLK_BLOCKPOOL 57 | struct list blockpool; 58 | #endif 59 | 60 | uint32_t nsb; 61 | struct btreeblk_subblocks *sb; 62 | // dirty update entry for read 63 | struct filemgr_dirty_update_node *dirty_update; 64 | // dirty update entry for the current WAL flushing 65 | struct filemgr_dirty_update_node *dirty_update_writer; 66 | }; 67 | 68 | struct btree_blk_ops *btreeblk_get_ops(); 69 | void btreeblk_init(struct btreeblk_handle *handle, struct filemgr *file, 70 | uint32_t nodesize); 71 | 72 | INLINE void btreeblk_set_dirty_update(struct btreeblk_handle *handle, 73 | struct filemgr_dirty_update_node *node) 74 | { 75 | handle->dirty_update = node; 76 | } 77 | 78 | INLINE void btreeblk_set_dirty_update_writer(struct btreeblk_handle *handle, 79 | struct filemgr_dirty_update_node *node) 80 | { 81 | handle->dirty_update_writer = node; 82 | } 83 | 84 | INLINE void btreeblk_clear_dirty_update(struct btreeblk_handle *handle) 85 | { 86 | handle->dirty_update = handle->dirty_update_writer = NULL; 87 | } 88 | 89 | INLINE struct filemgr_dirty_update_node* 90 | btreeblk_get_dirty_update(struct btreeblk_handle *handle) 91 | { 92 | return handle->dirty_update; 93 | } 94 | 95 | void btreeblk_reset_subblock_info(struct btreeblk_handle *handle); 96 | void btreeblk_free(struct btreeblk_handle *handle); 97 | void btreeblk_discard_blocks(struct btreeblk_handle *handle); 98 | fdb_status btreeblk_end(struct btreeblk_handle *handle); 99 | void btreeblk_write_done(void* voidhandle, bid_t bid); 100 | 101 | #ifdef __cplusplus 102 | } 103 | #endif 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /tests/anomaly/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # every file same as FORESTDB_CORE_SRC execept filemgr_ops.cc replaced 2 | # with filemgr_anomalous_ops.cc 3 | SET(FORESTDB_COMMON_CORE_SRC 4 | ${PROJECT_SOURCE_DIR}/src/api_wrapper.cc 5 | ${PROJECT_SOURCE_DIR}/src/avltree.cc 6 | ${PROJECT_SOURCE_DIR}/src/bgflusher.cc 7 | ${PROJECT_SOURCE_DIR}/src/blockcache.cc 8 | ${PROJECT_SOURCE_DIR}/${BREAKPAD_SRC} 9 | ${PROJECT_SOURCE_DIR}/src/btree.cc 10 | ${PROJECT_SOURCE_DIR}/src/btree_kv.cc 11 | ${PROJECT_SOURCE_DIR}/src/btree_str_kv.cc 12 | ${PROJECT_SOURCE_DIR}/src/btree_fast_str_kv.cc 13 | ${PROJECT_SOURCE_DIR}/src/btreeblock.cc 14 | ${PROJECT_SOURCE_DIR}/src/checksum.cc 15 | ${PROJECT_SOURCE_DIR}/src/compactor.cc 16 | ${PROJECT_SOURCE_DIR}/src/configuration.cc 17 | ${PROJECT_SOURCE_DIR}/src/docio.cc 18 | ${PROJECT_SOURCE_DIR}/src/encryption.cc 19 | ${PROJECT_SOURCE_DIR}/src/encryption_aes.cc 20 | ${PROJECT_SOURCE_DIR}/src/encryption_bogus.cc 21 | ${PROJECT_SOURCE_DIR}/src/fdb_errors.cc 22 | ${PROJECT_SOURCE_DIR}/src/filemgr.cc 23 | ${PROJECT_SOURCE_DIR}/src/forestdb.cc 24 | ${PROJECT_SOURCE_DIR}/src/hash.cc 25 | ${PROJECT_SOURCE_DIR}/src/hash_functions.cc 26 | ${PROJECT_SOURCE_DIR}/src/hbtrie.cc 27 | ${PROJECT_SOURCE_DIR}/src/iterator.cc 28 | ${PROJECT_SOURCE_DIR}/src/kv_instance.cc 29 | ${PROJECT_SOURCE_DIR}/src/list.cc 30 | ${PROJECT_SOURCE_DIR}/src/log_message.cc 31 | ${PROJECT_SOURCE_DIR}/src/staleblock.cc 32 | ${PROJECT_SOURCE_DIR}/src/superblock.cc 33 | ${PROJECT_SOURCE_DIR}/src/transaction.cc 34 | ${PROJECT_SOURCE_DIR}/src/version.cc 35 | ${PROJECT_SOURCE_DIR}/src/wal.cc) 36 | 37 | add_library(FDB_TOOLS_CCORE OBJECT ${FORESTDB_COMMON_CORE_SRC}) 38 | set_target_properties(FDB_TOOLS_CCORE PROPERTIES 39 | COMPILE_FLAGS "-D_FDB_TOOLS ${CB_GNU_CXX11_OPTION}") 40 | 41 | add_executable(fdb_anomaly_test 42 | $ 43 | filemgr_anomalous_ops.cc 44 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 45 | fdb_anomaly_test.cc 46 | ${GETTIMEOFDAY_VS} 47 | $) 48 | target_link_libraries(fdb_anomaly_test ${PTHREAD_LIB} ${LIBM} 49 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 50 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 51 | ${LIBRT} ${CRYPTO_LIB} 52 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 53 | set_target_properties(fdb_anomaly_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 54 | 55 | add_executable(disk_sim_test 56 | $ 57 | filemgr_anomalous_ops.cc 58 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 59 | disk_sim_test.cc 60 | ${GETTIMEOFDAY_VS} 61 | $) 62 | target_link_libraries(disk_sim_test ${PTHREAD_LIB} ${LIBM} 63 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 64 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 65 | ${LIBRT} ${CRYPTO_LIB} 66 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 67 | set_target_properties(disk_sim_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 68 | 69 | # add test target 70 | add_test(fdb_anomaly_test fdb_anomaly_test) 71 | add_test(disk_sim_test disk_sim_test) 72 | 73 | ADD_CUSTOM_TARGET(anomaly_tests 74 | COMMAND ctest 75 | ) 76 | -------------------------------------------------------------------------------- /src/forestdb_endian.h: -------------------------------------------------------------------------------- 1 | #ifndef _FDB_ENDIAN_H 2 | #define _FDB_ENDIAN_H 3 | 4 | #if defined(WIN32) || defined(_WIN32) 5 | #ifndef _LITTLE_ENDIAN 6 | #define _LITTLE_ENDIAN 7 | #endif 8 | 9 | #elif __APPLE__ 10 | #include 11 | #if BYTE_ORDER == LITTLE_ENDIAN 12 | #ifndef _LITTLE_ENDIAN 13 | #define _LITTLE_ENDIAN 14 | #endif 15 | #elif BYTE_ORDER == BIG_ENDIAN 16 | #ifndef _BIG_ENDIAN 17 | #define _BIG_ENDIAN 18 | #endif 19 | #else 20 | #error "not supported endian" 21 | #endif 22 | 23 | #elif __ANDROID__ 24 | #include 25 | #if _BYTE_ORDER == _LITTLE_ENDIAN 26 | #ifndef _LITTLE_ENDIAN 27 | #define _LITTLE_ENDIAN 28 | #endif 29 | #ifdef _BIG_ENDIAN 30 | #undef _BIG_ENDIAN 31 | #endif 32 | #elif _BYTE_ORDER == _BIG_ENDIAN 33 | #ifndef _BIG_ENDIAN 34 | #define _BIG_ENDIAN 35 | #endif 36 | #ifdef _LITTLE_ENDIAN 37 | #undef _LITTLE_ENDIAN 38 | #endif 39 | #else 40 | #error "not supported endian" 41 | #endif 42 | 43 | #elif __linux__ 44 | #include 45 | #if __BYTE_ORDER == __LITTLE_ENDIAN 46 | #ifndef _LITTLE_ENDIAN 47 | #define _LITTLE_ENDIAN 48 | #endif 49 | #elif __BYTE_ORDER == __BIG_ENDIAN 50 | #ifndef _BIG_ENDIAN 51 | #define _BIG_ENDIAN 52 | #endif 53 | #else 54 | #error "not supported endian" 55 | #endif 56 | 57 | #endif 58 | 59 | #ifndef bitswap64 60 | #define bitswap64(v) \ 61 | ( (((v) & 0xff00000000000000ULL) >> 56) \ 62 | | (((v) & 0x00ff000000000000ULL) >> 40) \ 63 | | (((v) & 0x0000ff0000000000ULL) >> 24) \ 64 | | (((v) & 0x000000ff00000000ULL) >> 8) \ 65 | | (((v) & 0x00000000ff000000ULL) << 8) \ 66 | | (((v) & 0x0000000000ff0000ULL) << 24) \ 67 | | (((v) & 0x000000000000ff00ULL) << 40) \ 68 | | (((v) & 0x00000000000000ffULL) << 56) ) 69 | #endif 70 | 71 | #ifndef bitswap32 72 | #define bitswap32(v) \ 73 | ( (((v) & 0xff000000) >> 24) \ 74 | | (((v) & 0x00ff0000) >> 8) \ 75 | | (((v) & 0x0000ff00) << 8) \ 76 | | (((v) & 0x000000ff) << 24) ) 77 | #endif 78 | 79 | #ifndef bitswap16 80 | #define bitswap16(v) \ 81 | ( (((v) & 0xff00) >> 8) \ 82 | | (((v) & 0x00ff) << 8) ) 83 | #endif 84 | 85 | #if defined(_LITTLE_ENDIAN) 86 | // convert to big endian 87 | #define _enc64(v) bitswap64(v) 88 | #define _dec64(v) bitswap64(v) 89 | #define _enc32(v) bitswap32(v) 90 | #define _dec32(v) bitswap32(v) 91 | #define _enc16(v) bitswap16(v) 92 | #define _dec16(v) bitswap16(v) 93 | #else 94 | // big endian .. do nothing 95 | #define _enc64(v) (v) 96 | #define _dec64(v) (v) 97 | #define _enc32(v) (v) 98 | #define _dec32(v) (v) 99 | #define _enc16(v) (v) 100 | #define _dec16(v) (v) 101 | #endif 102 | 103 | #ifdef __ENDIAN_SAFE 104 | #define _endian_encode(v) \ 105 | ((sizeof(v) == 8)?(_enc64(v)):( \ 106 | (sizeof(v) == 4)?(_enc32(v)):( \ 107 | (sizeof(v) == 2)?(_enc16(v)):(v)))) 108 | #define _endian_decode(v) \ 109 | ((sizeof(v) == 8)?(_dec64(v)):( \ 110 | (sizeof(v) == 4)?(_dec32(v)):( \ 111 | (sizeof(v) == 2)?(_dec16(v)):(v)))) 112 | #else 113 | #define _endian_encode(v) (v) 114 | #define _endian_decode(v) (v) 115 | #endif 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /src/checksum.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * ForestDB evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | #pragma once 27 | 28 | #include 29 | #include 30 | 31 | enum crc_mode_e { 32 | CRC_UNKNOWN, 33 | CRC32, 34 | CRC32C, 35 | #ifdef _CRC32C 36 | CRC_DEFAULT = CRC32C // Couchbase builds pickup crc32c from plaform 37 | #else 38 | CRC_DEFAULT = CRC32 // Non-couchbase falls back to bundled utils/crc32.cc 39 | #endif 40 | }; 41 | 42 | /* 43 | * Get a checksum of buf for buf_len bytes. 44 | * 45 | * mode = UNKNOWN is an invalid input (triggers assert). 46 | */ 47 | uint32_t get_checksum(const uint8_t* buf, 48 | size_t buf_len, 49 | uint32_t pre, 50 | crc_mode_e mode); 51 | 52 | /* 53 | * Get a checksum of buf for buf_len bytes. 54 | * 55 | * The pre value is set to 0. 56 | * 57 | * mode = UNKNOWN is an invalid input (triggers assert). 58 | */ 59 | uint32_t get_checksum(const uint8_t* buf, 60 | size_t buf_len, 61 | crc_mode_e mode); 62 | 63 | /* 64 | * Get a checksum of buf for buf_len bytes. 65 | * 66 | * The CRC is generated using the default mode for the build of ForestDB. 67 | * The pre value is set to 0. 68 | * 69 | */ 70 | inline uint32_t get_checksum(const uint8_t* buf, 71 | size_t buf_len) { 72 | return get_checksum(buf, buf_len, 0, CRC_DEFAULT); 73 | } 74 | 75 | /* 76 | * Perform an integrity check of buf for buf_len bytes. 77 | * 78 | * A checksum of buf is created and compared against checksum argument. 79 | * 80 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 81 | * 82 | * Returns true success (checksums match), false for failure. 83 | */ 84 | bool perform_integrity_check(const uint8_t* buf, 85 | size_t buf_len, 86 | uint32_t checksum, 87 | crc_mode_e mode); 88 | 89 | /* 90 | * Detect the CRC mode by performing an integrity check against the 91 | * two CRC functions ForestDB files could be written with. 92 | * 93 | * Returns true if a match is found and sets mode to the correct mode. 94 | * Returns false if no match and sets mode to CRC_UNKNOWN. 95 | */ 96 | bool detect_and_check_crc(const uint8_t* buf, 97 | size_t buf_len, 98 | uint32_t checksum, 99 | crc_mode_e* mode); 100 | -------------------------------------------------------------------------------- /src/compactor.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_COMPACTOR_H 19 | #define _FDB_COMPACTOR_H 20 | 21 | #include 22 | 23 | #include "internal_types.h" 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | struct compactor_config { 30 | compactor_config() 31 | : sleep_duration(FDB_COMPACTOR_SLEEP_DURATION) 32 | , num_threads(DEFAULT_NUM_COMPACTOR_THREADS) 33 | , spawn_threads(true) 34 | {} 35 | compactor_config& operator=(const fdb_config& src) { 36 | sleep_duration = src.compactor_sleep_duration; 37 | num_threads = src.num_compactor_threads; 38 | spawn_threads = src.enable_background_compactor; 39 | return *this; 40 | } 41 | size_t sleep_duration; 42 | size_t num_threads; 43 | bool spawn_threads; 44 | }; 45 | 46 | void compactor_init(struct compactor_config *config); 47 | void compactor_shutdown(); 48 | bool compactor_switch_compaction_flag(struct filemgr *file, bool flag); 49 | fdb_status compactor_register_file(struct filemgr *file, 50 | fdb_config *config, 51 | err_log_callback *log_callback); 52 | 53 | fdb_status compactor_register_file_removing(struct filemgr *file, 54 | err_log_callback *log_callback); 55 | bool compactor_is_file_removed(const char *filename); 56 | 57 | void compactor_deregister_file(struct filemgr *file); 58 | void compactor_change_threshold(struct filemgr *file, size_t new_threshold); 59 | void compactor_switch_file(struct filemgr *old_file, struct filemgr *new_file, 60 | err_log_callback *log_callback); 61 | void compactor_get_virtual_filename(const char *filename, 62 | char *virtual_filename); 63 | fdb_status compactor_get_actual_filename(const char *filename, 64 | char *actual_filename, 65 | fdb_compaction_mode_t comp_mode, 66 | err_log_callback *log_callback); 67 | void compactor_get_next_filename(char *file, char *nextfile); 68 | bool compactor_is_valid_mode(const char *filename, fdb_config *config); 69 | 70 | fdb_status compactor_destroy_file(char *filename, 71 | fdb_config *config); 72 | 73 | /** 74 | * Set the daemon compaction interval for a given file. 75 | * 76 | * @param file Pointer to the file manager instance 77 | * @param interval Daemon compaction interval to be set 78 | * @return FDB_RESULT_SUCCESS upon successful interval change 79 | */ 80 | fdb_status compactor_set_compaction_interval(struct filemgr *file, 81 | size_t interval); 82 | 83 | #ifdef __cplusplus 84 | } 85 | #endif 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /cmake/Modules/CouchbaseCodeCoverage.cmake: -------------------------------------------------------------------------------- 1 | # Support for running code coverage reporting. 2 | # 3 | # Usage: 4 | # 1). Add a call to ADD_COVERAGE_REPORT() to the module(s) you wish to 5 | # obtain code coverage reports on. 6 | # 2). Enable the option CB_CODE_COVERAGE (e.g. pass -DCB_CODE_COVERAGE=ON to cmake). 7 | # 3). Build as normal. 8 | # 4). Run unit test(s) to exercise the codebase. 9 | # 5). Run `make coverage-report-html` and/or `coverage-report-xml` 10 | # (from the selected module subdirectory) to generate the reports. 11 | # 6) (Optional) to zero coverage counters before a re-run run `make coverage-zero-counters`. 12 | 13 | 14 | OPTION(CB_CODE_COVERAGE "Enable code coverage testing." 15 | OFF) 16 | 17 | IF (CB_CODE_COVERAGE) 18 | FIND_PROGRAM(GCOV_PATH gcov) 19 | FIND_PROGRAM(GCOVR_PATH gcovr) 20 | 21 | IF (NOT GCOV_PATH) 22 | MESSAGE(STATUS "gcov not found.") 23 | ENDIF (NOT GCOV_PATH) 24 | 25 | IF (NOT GCOVR_PATH) 26 | MESSAGE(STATUS "gcovr [www.gcovr.com] not found.") 27 | ENDIF () 28 | 29 | IF (NOT GCOV_PATH OR NOT GCOVR_PATH) 30 | MESSAGE(FATAL_ERROR "CB_CODE_COVERAGE enabled but one of more required tools not found - cannot continue.") 31 | ENDIF() 32 | ENDIF(CB_CODE_COVERAGE) 33 | 34 | # Defines a coverage report for the current module. If CB_CODE_COVERAGE is enabled, 35 | # adds three new targets to that module: 36 | # -coverage-zero-counters: Zeros the code coverage counters for the module. 37 | # -coverage-report-html: Generates a code coverage report in HTML. 38 | # -coverage-report-xml: Generates a code coverage report in XML. 39 | # Usage: 40 | # 1) `make -coverage-zero-counters` to clear any counters from 41 | # previously-executed programs. 42 | # 2) Run whatever programs to excercise the code (unit tests, etc). 43 | # 3) `make -coverage-report-{html,xml}` to generate a report. 44 | # 45 | FUNCTION(ENABLE_CODE_COVERAGE_REPORT) 46 | GET_FILENAME_COMPONENT(_cc_project ${CMAKE_CURRENT_BINARY_DIR} NAME) 47 | 48 | IF (CB_CODE_COVERAGE) 49 | MESSAGE(STATUS "Setting up code coverage for ${PROJECT_NAME}") 50 | 51 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-zero-counters 52 | COMMAND find . -name *.gcda -exec rm {} \; 53 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 54 | COMMENT "Zeroing coverage counters for objects in ${CMAKE_CURRENT_BINARY_DIR}" 55 | VERBATIM) 56 | 57 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-report-html 58 | COMMAND ${CMAKE_COMMAND} -E remove_directory coverage 59 | COMMAND ${CMAKE_COMMAND} -E make_directory coverage 60 | COMMAND ${GCOVR_PATH} --root=${CMAKE_SOURCE_DIR} --filter="${CMAKE_CURRENT_SOURCE_DIR}/.*" --html --html-details -o coverage/index.html 61 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 62 | COMMENT "Generating code coverage report for ${PROJECT_NAME} to ${CMAKE_CURRENT_BINARY_DIR}/coverage/index.html") 63 | 64 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-report-xml 65 | COMMAND ${GCOVR_PATH} --root=${CMAKE_SOURCE_DIR} --filter="${CMAKE_CURRENT_SOURCE_DIR}/.*" --xml -o coverage.xml 66 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 67 | COMMENT "Generating code coverage report for ${PROJECT_NAME} to ${CMAKE_CURRENT_BINARY_DIR}/coverage.xml") 68 | ENDIF (CB_CODE_COVERAGE) 69 | ENDFUNCTION() 70 | -------------------------------------------------------------------------------- /tests/include/test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_TEST_H 19 | #define _JSAHN_TEST_H 20 | 21 | #include 22 | #include 23 | #if !defined(WIN32) && !defined(_WIN32) 24 | #include 25 | #endif 26 | #include "time_utils.h" 27 | 28 | #include "common.h" 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif 33 | 34 | #define _TEST_GLOBAL 35 | #ifdef _TEST_GLOBAL 36 | 37 | #define TEST_INIT() \ 38 | static int __test_pass=1; \ 39 | struct timeval __test_begin, __test_prev, __test_cur, __test_interval_total, __test_interval_ins; \ 40 | (void)__test_prev; \ 41 | (void)__test_interval_total; \ 42 | (void)__test_interval_ins; \ 43 | (void)__test_pass; \ 44 | gettimeofday(&__test_begin, NULL); \ 45 | __test_cur = __test_begin 46 | 47 | #define TEST_CHK(cond) {if (!(cond)) {fprintf(stderr, "Test failed: %s %d\n", __FILE__, __LINE__); __test_pass=0; assert(cond);}} 48 | #define TEST_CMP(str1, str2, len) {if (memcmp(str1, str2, len)) {fprintf(stderr, "Test expected %s but got %s failed: %s %d\n", (char*)str2,(char*) str1, __FILE__, __LINE__); __test_pass=0; assert(false);}} 49 | #define TEST_STATUS(status) {fdb_status s = (status); if (s != FDB_RESULT_SUCCESS) {fprintf(stderr, "Test failed with fdb_status %d (%s) at %s %d\n", s, fdb_error_msg(s), __FILE__, __LINE__); __test_pass=0; assert(false);}} 50 | #define TEST_RESULT(name) {if ((__test_pass)) fprintf(stderr, "%s PASSED\n", (name)); else fprintf(stderr, "%s FAILED\n", (name)); } 51 | 52 | #define TEST_TIME() {\ 53 | __test_prev = __test_cur; \ 54 | gettimeofday(&__test_cur, NULL); \ 55 | __test_interval_total = _utime_gap(__test_begin, __test_cur); \ 56 | __test_interval_ins = _utime_gap(__test_prev, __test_cur); \ 57 | DBG("Time elapsed: total %" _FSEC ".%06" _FUSEC " , interval %" _FSEC ".%06" _FUSEC "\n", \ 58 | __test_interval_total.tv_sec, __test_interval_total.tv_usec, \ 59 | __test_interval_ins.tv_sec, __test_interval_ins.tv_usec); } 60 | 61 | 62 | #else 63 | 64 | #define TEST_CHK(cond, sw) {if (!(cond)) {fprintf(stderr, "Test failed: %s %d\n", __FILE__, __LINE__); sw=0; assert(cond);}} 65 | #define TEST_RESULT(name, sw) {if ((sw)) fprintf(stderr, "%s PASSED\n", (name)); else fprintf(stderr, "%s FAILED\n", (name)); } 66 | 67 | #endif 68 | 69 | #ifdef __cplusplus 70 | } 71 | #endif 72 | 73 | #if defined(WIN32) || defined(_WIN32) 74 | #define SHELL_DEL "del /f " 75 | #define SHELL_COPY "copy " 76 | #define SHELL_MOVE "move " 77 | #define SHELL_MKDIR "mkdir " 78 | #define SHELL_RMDIR "rd /s/q " 79 | #define SHELL_DMT "\\" 80 | #define SHELL_MAX_PATHLEN (256) 81 | #else 82 | #define SHELL_DEL "rm -rf " 83 | #define SHELL_COPY "cp " 84 | #define SHELL_MOVE "mv " 85 | #define SHELL_MKDIR "mkdir " 86 | #define SHELL_RMDIR SHELL_DEL 87 | #define SHELL_DMT "/" 88 | #define SHELL_MAX_PATHLEN (1024) 89 | #endif 90 | 91 | #include "memleak.h" 92 | 93 | #endif 94 | 95 | 96 | -------------------------------------------------------------------------------- /tests/anomaly/filemgr_anomalous_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FILEMGR_ANOMALOUS_OPS 19 | #define _FILEMGR_ANOMALOUS_OPS 20 | 21 | #ifdef __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | void filemgr_ops_set_anomalous(int behavior); 26 | 27 | struct filemgr_ops *get_normal_ops_ptr(); 28 | 29 | // These callbacks allow test-suite to control how the file ops should behave 30 | // If these return 0, then normal operation will happen, 31 | // If these return a non-zero value, then the file ops will return the same result 32 | struct anomalous_callbacks { 33 | int (*open_cb)(void *ctx, struct filemgr_ops *normal_ops, 34 | const char *pathname, int flags, mode_t mode); 35 | ssize_t (*pwrite_cb)(void *ctx, struct filemgr_ops *normal_ops, 36 | int fd, void *buf, size_t count, cs_off_t offset); 37 | ssize_t (*pread_cb)(void *ctx, struct filemgr_ops *normal_ops, 38 | int fd, void *buf, size_t count, cs_off_t offset); 39 | int (*close_cb)(void *ctx, struct filemgr_ops *normal_ops, int fd); 40 | cs_off_t (*goto_eof_cb)(void *ctx, struct filemgr_ops *normal_ops, int fd); 41 | cs_off_t (*file_size_cb)(void *ctx, struct filemgr_ops *normal_ops, 42 | const char *filename); 43 | int (*fdatasync_cb)(void *ctx, struct filemgr_ops *normal_ops, int fd); 44 | int (*fsync_cb)(void *ctx, struct filemgr_ops *normal_ops, int fd); 45 | void (*get_errno_str_cb)(void *ctx, struct filemgr_ops *normal_ops, 46 | char *buf, size_t size); 47 | int (*aio_init_cb)(void *ctx, struct filemgr_ops *normal_ops, 48 | struct async_io_handle *aio_handle); 49 | int (*aio_prep_read_cb)(void *ctx, struct filemgr_ops *normal_ops, 50 | struct async_io_handle *aio_handle, 51 | size_t aio_idx, size_t read_size, uint64_t offset); 52 | int (*aio_submit_cb)(void *ctx, struct filemgr_ops *normal_ops, 53 | struct async_io_handle *aio_handle, 54 | int num_subs); 55 | int (*aio_getevents_cb)(void *ctx, struct filemgr_ops *normal_ops, 56 | struct async_io_handle *aio_handle, 57 | int min, int max, unsigned int timeout); 58 | int (*aio_destroy_cb)(void *ctx, struct filemgr_ops *normal_ops, 59 | struct async_io_handle *aio_handle); 60 | int (*get_fs_type_cb)(void *ctx, struct filemgr_ops *normal_ops, 61 | int src_fd); 62 | int (*copy_file_range_cb)(void *ctx, struct filemgr_ops *normal_ops, 63 | int fs_type, int src_fd, int dst_fd, 64 | uint64_t src_off, uint64_t dst_off, uint64_t len); 65 | }; 66 | 67 | struct anomalous_callbacks * get_default_anon_cbs(); 68 | void filemgr_ops_anomalous_init(struct anomalous_callbacks *cbs, void *ctx); 69 | 70 | #ifdef __cplusplus 71 | } 72 | #endif 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /utils/system_resource_stats.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: David Robert Nadeau 3 | * Site: http://NadeauSoftware.com/ 4 | * License: Creative Commons Attribution 3.0 Unported License 5 | * http://creativecommons.org/licenses/by/3.0/deed.en_US 6 | */ 7 | 8 | #if defined(WIN32) || defined(_WIN32) 9 | #include 10 | #else 11 | #include 12 | #include 13 | #include 14 | #if defined(BSD) 15 | #include 16 | #endif 17 | #endif 18 | 19 | #include "system_resource_stats.h" 20 | 21 | /** 22 | * Returns the size of physical memory (RAM) in bytes. 23 | */ 24 | int64_t get_memory_size(void) 25 | { 26 | #if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__)) 27 | /* Cygwin under Windows. ------------------------------------ */ 28 | /* New 64-bit MEMORYSTATUSEX isn't available. Use old 32.bit */ 29 | MEMORYSTATUS status; 30 | status.dwLength = sizeof(status); 31 | GlobalMemoryStatus( &status ); 32 | return (int64_t)status.dwTotalPhys; 33 | 34 | #elif defined(WIN32) || defined(_WIN32) 35 | /* Windows. ------------------------------------------------- */ 36 | /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */ 37 | MEMORYSTATUSEX status; 38 | status.dwLength = sizeof(status); 39 | GlobalMemoryStatusEx( &status ); 40 | return (int64_t)status.ullTotalPhys; 41 | 42 | #else 43 | /* UNIX variants. ------------------------------------------- */ 44 | /* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */ 45 | 46 | #if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64)) 47 | int mib[2]; 48 | mib[0] = CTL_HW; 49 | #if defined(HW_MEMSIZE) 50 | mib[1] = HW_MEMSIZE; /* OSX. --------------------- */ 51 | #elif defined(HW_PHYSMEM64) 52 | mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */ 53 | #endif 54 | int64_t size = 0; /* 64-bit */ 55 | size_t len = sizeof(size); 56 | if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { 57 | return size; 58 | } 59 | return 0; /* Failed? */ 60 | 61 | #elif defined(_SC_AIX_REALMEM) 62 | /* AIX. ----------------------------------------------------- */ 63 | return (int64_t)(sysconf( _SC_AIX_REALMEM ) * 1024); 64 | 65 | #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) 66 | /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */ 67 | return (int64_t)sysconf( _SC_PHYS_PAGES ) 68 | * (int64_t)sysconf( _SC_PAGESIZE ); 69 | 70 | #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE) 71 | /* Legacy. -------------------------------------------------- */ 72 | return (int64_t)sysconf( _SC_PHYS_PAGES ) 73 | * (int64_t)sysconf( _SC_PAGE_SIZE ); 74 | 75 | #elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM)) 76 | /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */ 77 | int mib[2]; 78 | mib[0] = CTL_HW; 79 | #if defined(HW_REALMEM) 80 | mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */ 81 | #elif defined(HW_PYSMEM) 82 | mib[1] = HW_PHYSMEM; /* Others. ------------------ */ 83 | #endif 84 | unsigned int size = 0; /* 32-bit */ 85 | size_t len = sizeof( size ); 86 | if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { 87 | return (int64_t) size; 88 | } 89 | return 0; /* Failed? */ 90 | #endif /* sysctl and sysconf variants */ 91 | 92 | #endif 93 | } 94 | 95 | size_t get_num_cores(void) { 96 | size_t num_cores; 97 | #if defined(WIN32) || defined(_WIN32) 98 | SYSTEM_INFO sysinfo; 99 | GetSystemInfo(&sysinfo); 100 | num_cores = (size_t)sysinfo.dwNumberOfProcessors; 101 | #else 102 | num_cores = (size_t)sysconf(_SC_NPROCESSORS_ONLN); 103 | #endif 104 | 105 | return num_cores; 106 | } 107 | -------------------------------------------------------------------------------- /tests/e2e/e2espec.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "libforestdb/forestdb.h" 19 | 20 | //#define __DEBUG_E2E 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | static const char E2EDB_MAIN[] = "e2edb_main"; 27 | static const char E2EDB_RECORDS[] = "e2edb_records"; 28 | static const char E2EKV_ALLDOCS[] = "e2ekv_alldocs"; 29 | static const char E2EKV_INDEX1[] = "e2ekv_index1"; 30 | static const char E2EKV_INDEX2[] = "e2ekv_index2"; 31 | static const char E2EKV_RECORDS[] = "e2ekv_rtx"; 32 | static const char E2EKV_CHECKPOINTS[] = "e2ekv_chk"; 33 | static const char RECORD_DOCKEY[] = "e2edoc_records"; 34 | static const int KEYSPACE_LEN = 6; 35 | static const int MAXKEY_LEN = 1024; 36 | static const int MAXITR_LEN = 12; 37 | 38 | static const int LOAD_FACTOR = 1; 39 | 40 | typedef uint16_t tx_type_t; 41 | enum { 42 | SET_PERSON = 0x01, 43 | DEL_PERSON = 0x02, 44 | ACC_DEPOSIT = 0x03, 45 | ACC_WITHDRAW = 0x04, 46 | START_CHECKPOINT = 0x05, 47 | END_CHECKPOINT = 0x06 48 | }; 49 | 50 | typedef struct { 51 | char key[MAXKEY_LEN]; 52 | char name[MAXKEY_LEN]; 53 | char city[256]; 54 | char state[256]; 55 | char desc[1024]; 56 | char keyspace[KEYSPACE_LEN]; 57 | int age; 58 | }person_t; 59 | 60 | typedef struct { 61 | char min[MAXITR_LEN]; 62 | char max[MAXITR_LEN]; 63 | }idx_prams_t; 64 | 65 | typedef struct { 66 | int amount; 67 | char refkey[MAXKEY_LEN]; 68 | size_t refkey_len; 69 | tx_type_t type; 70 | }transaction_t; 71 | 72 | typedef struct { 73 | char key[MAXKEY_LEN]; 74 | int ndocs; 75 | int num_indexed; 76 | int sum_age_indexed; 77 | uint16_t balance; 78 | fdb_seqnum_t seqnum_all; 79 | fdb_seqnum_t seqnum_idx1; 80 | fdb_seqnum_t seqnum_idx2; 81 | tx_type_t type; 82 | }checkpoint_t; 83 | 84 | 85 | typedef struct { 86 | 87 | fdb_file_handle *main; 88 | fdb_kvs_handle *all_docs; 89 | fdb_kvs_handle *index1; 90 | fdb_kvs_handle *index2; 91 | 92 | fdb_file_handle *records; 93 | fdb_kvs_handle *rtx; 94 | fdb_kvs_handle *chk; 95 | checkpoint_t *v_chk; 96 | idx_prams_t *index_params; 97 | 98 | char keyspace[KEYSPACE_LEN]; 99 | bool walflush; 100 | bool verify_set; 101 | }storage_t; 102 | 103 | // generators 104 | void gen_random(char *s, const int len); 105 | void gen_index_params(idx_prams_t *params); 106 | void gen_person(person_t *p); 107 | 108 | // fdb wrappers 109 | void e2e_fdb_set_person(storage_t *st, person_t *p); 110 | void e2e_fdb_del_person(storage_t *st, person_t *p); 111 | void e2e_fdb_cancel_checkpoint(storage_t *st); 112 | void e2e_fdb_commit(fdb_file_handle* fhandle, bool walflush); 113 | void e2e_fdb_close(storage_t *st); 114 | void e2e_fdb_shutdown(storage_t *st); 115 | 116 | // checkpointing 117 | void start_checkpoint(storage_t *st); 118 | void end_checkpoint(storage_t *st); 119 | checkpoint_t* create_checkpoint(storage_t *st, tx_type_t type); 120 | 121 | // storage 122 | storage_t *init_storage(fdb_config *m_fconfig, fdb_config *r_fconfig, 123 | fdb_kvs_config *kvs_config, idx_prams_t *idxp, 124 | checkpoint_t *v_chk, bool walflush); 125 | 126 | void reset_storage_index(storage_t *st); 127 | void rm_storage_fs(); 128 | 129 | // utility 130 | bool is_indexed(idx_prams_t *idxp, person_t *p); 131 | void save_tx(storage_t *st, void *key, size_t keylen, tx_type_t type); 132 | fdb_kvs_handle *scan(storage_t *st, fdb_kvs_handle *reuse_kv); 133 | 134 | #ifdef __cplusplus 135 | } 136 | #endif 137 | -------------------------------------------------------------------------------- /src/log_message.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #if !defined(WIN32) && !defined(_WIN32) 3 | #include 4 | #endif 5 | #include 6 | 7 | #include "log_message.h" 8 | 9 | 10 | void printISOTime(char* buffer, size_t buffer_len) { 11 | struct tm* tm_now; 12 | time_t rawtime; 13 | time(&rawtime); 14 | tm_now = localtime(&rawtime); 15 | 16 | // 2017-06-22T10:00:00 17 | size_t time_len = strftime(buffer, buffer_len, 18 | "%Y-%m-%dT%H:%M:%S", tm_now); 19 | 20 | // Add milliseconds 21 | timeval cur_time; 22 | gettimeofday(&cur_time, NULL); 23 | size_t milli = cur_time.tv_usec / 1000; 24 | // 2017-06-22T10:00:00.123 25 | sprintf(buffer + time_len, ".%03d", (int)milli); 26 | time_len += 4; 27 | 28 | // timezone offset format: -0500 29 | char tz_offset_str[6]; 30 | size_t offset_len = strftime(tz_offset_str, 6, 31 | "%z", tm_now); 32 | if (offset_len < 5) { 33 | // Time zone info is not supported, skip it. 34 | return; 35 | } 36 | 37 | // hour 38 | strncat(buffer, tz_offset_str, 3); 39 | // : 40 | strcat(buffer, ":"); 41 | // min 42 | strncat(buffer, tz_offset_str + 3, 2); 43 | // final format: 2017-06-22T10:00:00.123-05:00 44 | } 45 | 46 | // Log config that is globally used for this process. 47 | static fdb_log_config global_log_config; 48 | 49 | static err_log_callback global_log_cb; 50 | 51 | fdb_status fdb_log_init(struct fdb_log_config log_config) { 52 | if (log_config.log_msg_level > 6) { 53 | return FDB_RESULT_INVALID_CONFIG; 54 | } 55 | global_log_config = log_config; 56 | return FDB_RESULT_SUCCESS; 57 | } 58 | 59 | fdb_status fdb_log_set_global_callback(err_log_callback* log_callback) { 60 | global_log_cb = *log_callback; 61 | return FDB_RESULT_SUCCESS; 62 | } 63 | 64 | fdb_status fdb_log_impl(err_log_callback* log_callback, 65 | fdb_log_levels given_log_level, 66 | fdb_status status, 67 | const char* source_file, 68 | const char* func_name, 69 | size_t line_number, 70 | const char *format, ...) 71 | { 72 | // 1: Fatal [FATL] 73 | // 2: Error [ERRO] 74 | // 3: Warning [WARN] 75 | // 4: Info [INFO] 76 | // 5: Debug [DEBG] 77 | // 6: Trace [TRAC] 78 | size_t cur_log_level = given_log_level; 79 | 80 | if (global_log_config.log_msg_level < cur_log_level) { 81 | // Configuration doesn't allow to print out 82 | // log message of this level. 83 | return status; 84 | } 85 | 86 | char msg[4096]; 87 | va_list args; 88 | va_start(args, format); 89 | vsnprintf(msg, 4095, format, args); 90 | va_end(args); 91 | 92 | if (log_callback && log_callback->callback_ex) { 93 | // If extended callback exist, use it. 94 | log_callback->callback_ex 95 | ( cur_log_level, status, source_file, func_name, line_number, 96 | msg, log_callback->ctx_data ); 97 | return status; 98 | 99 | } else if (log_callback && log_callback->callback) { 100 | // Normal callback. 101 | log_callback->callback(status, msg, log_callback->ctx_data); 102 | return status; 103 | 104 | } else if (global_log_cb.callback_ex) { 105 | // Global log callback is given. 106 | global_log_cb.callback_ex 107 | ( cur_log_level, status, source_file, func_name, line_number, 108 | msg, global_log_cb.ctx_data ); 109 | return status; 110 | } 111 | 112 | // Callback is not given. 113 | char ISO_time_buffer[64]; 114 | char log_abbr[7][8] = {"XXXX", "FATL", "ERRO", "WARN", 115 | "INFO", "DEBG", "TRAC"}; 116 | printISOTime(ISO_time_buffer, 64); 117 | if (status != FDB_RESULT_SUCCESS) { 118 | fprintf(stderr, "%s [%s][FDB] %s\n", 119 | ISO_time_buffer, log_abbr[cur_log_level], msg); 120 | } else { 121 | fprintf(stderr, "%s [%s][FDB] %s\n", 122 | ISO_time_buffer, log_abbr[cur_log_level], msg); 123 | } 124 | return status; 125 | } 126 | 127 | -------------------------------------------------------------------------------- /src/checksum.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * ForestDB evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | 27 | #include 28 | #include 29 | #ifdef _CRC32C 30 | // Linking with platform for crc32c 31 | # include 32 | #else 33 | #include 34 | // Define a crc32c which does nothing but assert. 35 | static uint32_t crc32c(const uint8_t* buf, 36 | size_t buf_len, 37 | uint32_t pre) { 38 | // Non couchbase builds are configured to never turn on crc32c. 39 | assert(false); 40 | return 0; 41 | } 42 | #endif 43 | # include "checksum.h" 44 | #include "crc32.h" 45 | 46 | /* 47 | * Get a checksum of buf for buf_len bytes. 48 | * 49 | * mode = UNKNOWN is an invalid input (triggers assert). 50 | */ 51 | uint32_t get_checksum(const uint8_t* buf, 52 | size_t buf_len, 53 | uint32_t pre, 54 | crc_mode_e mode) { 55 | if (mode == CRC32C) { 56 | return crc32c(buf, buf_len, pre); 57 | } else { 58 | assert(mode == CRC32); 59 | return crc32_8((void *)buf, buf_len, pre); 60 | } 61 | } 62 | 63 | /* 64 | * Get a checksum of buf for buf_len bytes. 65 | * 66 | * The pre value is set to 0. 67 | * 68 | * mode = UNKNOWN is an invalid input (triggers assert). 69 | */ 70 | uint32_t get_checksum(const uint8_t* buf, 71 | size_t buf_len, 72 | crc_mode_e mode) { 73 | return get_checksum(buf, buf_len, 0, mode); 74 | } 75 | 76 | /* 77 | * Perform an integrity check of buf for buf_len bytes. 78 | * 79 | * A checksum of buf is created and compared against checksum argument. 80 | * 81 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 82 | * 83 | * 84 | * Returns true success (checksums match), false for failure. 85 | */ 86 | bool perform_integrity_check(const uint8_t* buf, 87 | size_t buf_len, 88 | uint32_t checksum, 89 | crc_mode_e mode) { 90 | bool success = false; 91 | #ifdef _CRC32C 92 | if (mode == CRC_UNKNOWN || mode == CRC32C) { 93 | success = checksum == crc32c(buf, buf_len, 0); 94 | if (!success && mode == CRC_UNKNOWN) { 95 | success = checksum == crc32_8((void *)buf, buf_len, 0); 96 | } 97 | } else 98 | #endif 99 | { 100 | success = checksum == crc32_8((void *)buf, buf_len, 0); 101 | } 102 | return success; 103 | } 104 | 105 | /* 106 | * Detect the CRC mode by performing an integrity check against the 107 | * two CRC functions ForestDB files could be written with. 108 | * 109 | * Returns true if a match is found and sets mode to the correct mode. 110 | * Returns false if no match and sets mode to CRC_UNKNOWN. 111 | */ 112 | bool detect_and_check_crc(const uint8_t* buf, 113 | size_t buf_len, 114 | uint32_t checksum, 115 | crc_mode_e* mode) { 116 | *mode = CRC_UNKNOWN; 117 | #ifdef _CRC32C 118 | if (perform_integrity_check(buf, buf_len, checksum, CRC32C)) { 119 | *mode = CRC32C; 120 | return true; 121 | } else 122 | #endif 123 | if (perform_integrity_check(buf, buf_len, checksum, CRC32)) { 124 | *mode = CRC32; 125 | return true; 126 | } 127 | 128 | return false; 129 | } 130 | -------------------------------------------------------------------------------- /src/staleblock.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_STALEBLOCK_H 19 | #define _FDB_STALEBLOCK_H 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "libforestdb/fdb_errors.h" 23 | #include "common.h" 24 | 25 | #include "filemgr.h" 26 | #include "avltree.h" 27 | 28 | /** 29 | * Gather stale region info from stale list and store it as a system doc. 30 | * 31 | * @param handle Pointer to ForestDB KV store handle. 32 | * @param revnum Header revision number that will be stored as a key in stale tree. 33 | * @param prev_hdr Currently up-to-date header BID. 34 | * @param kv_info_offset Currently up-to-date KVS header doc offset. 35 | * @param seqnum Currently up-to-date seq number of the default KVS. 36 | * @param e_last Last (rightmost) stale region that should not be gathered at this time. 37 | * @param from_mergetree If true, gather stale regions from merge-tree (which contains 38 | * remaining items after block reclaim) instead of stale list. 39 | * @return void. 40 | */ 41 | void fdb_gather_stale_blocks(fdb_kvs_handle *handle, 42 | filemgr_header_revnum_t revnum, 43 | bid_t prev_hdr, 44 | uint64_t kv_info_offset, 45 | fdb_seqnum_t seqnum, 46 | struct list_elem *e_last, 47 | bool from_mergetree); 48 | 49 | struct reusable_block { 50 | bid_t bid; 51 | bid_t count; 52 | }; 53 | 54 | typedef struct { 55 | size_t n_blocks; 56 | struct reusable_block *blocks; 57 | } reusable_block_list; 58 | 59 | typedef struct { 60 | filemgr_header_revnum_t revnum; 61 | bid_t bid; 62 | } stale_header_info; 63 | 64 | // in-memory structure for stale info 65 | // (corresponding to a commit, an entry of stale-block-tree) 66 | struct stale_info_commit { 67 | struct avl_node avl; 68 | // commit revision number 69 | filemgr_header_revnum_t revnum; 70 | // list of system docs (there will be more than one doc) 71 | struct list doc_list; 72 | }; 73 | 74 | // in-memory structure for stale info 75 | // (corresponding to a system doc) 76 | struct stale_info_entry { 77 | struct list_elem le; 78 | // document body 79 | void *ctx; 80 | // document offset 81 | uint64_t offset; 82 | // actual document length on disk 83 | uint64_t doclen; 84 | // length of 'ctx' 85 | uint32_t ctxlen; 86 | // length of compressed 'ctx' 87 | uint32_t comp_ctxlen; 88 | }; 89 | 90 | /** 91 | * Gather and merge all stale regions up to 'stale_header', and then return the list 92 | * of reusable blocks. 93 | * 94 | * @param handle Pointer to ForestDB KV store handle. 95 | * @param stale_header Revision number and block ID of a header. All stale regions 96 | * corresponding to commit headers whose seq number is equal to or smaller 97 | * than that of 'stale_header' are gathered and merged for block reusing. 98 | * @return List of reusable blocks. 99 | */ 100 | reusable_block_list fdb_get_reusable_block(fdb_kvs_handle *handle, 101 | stale_header_info stale_header); 102 | 103 | /** 104 | * Load all system documents pointed to by stale tree into memory. 105 | * 106 | * @param handle Pointer to ForestDB KV store handle. 107 | * @return void. 108 | */ 109 | void fdb_load_inmem_stale_info(fdb_kvs_handle *handle); 110 | 111 | /** 112 | * Remove all stale-tree entries since the rollback point. 113 | * 114 | * @param handle Pointer to ForestDB KV store handle. 115 | * @param cur_revnum Revision number of the header that will be appended next. 116 | * @return void. 117 | */ 118 | void fdb_rollback_stale_blocks(fdb_kvs_handle *handle, 119 | filemgr_header_revnum_t cur_revnum); 120 | 121 | #endif /* _FDB_STALEBLOCK_H */ 122 | 123 | -------------------------------------------------------------------------------- /tests/unit/list_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #include "list.h" 21 | #include "test.h" 22 | 23 | void basic_test() 24 | { 25 | TEST_INIT(); 26 | 27 | struct dummy_t { 28 | int a; 29 | struct list_elem e; 30 | }; 31 | 32 | int n=10, i; 33 | struct dummy_t dummy[n], *d; 34 | struct list list; 35 | struct list_elem *ee; 36 | 37 | list_init(&list); 38 | 39 | // push front 0 ~ 9 40 | for (i=0;ia == i--); 51 | ee = list_next(ee); 52 | } 53 | // check in reversed order 54 | ee = list_end(&list); 55 | i=0; 56 | while(ee) { 57 | d = _get_entry(ee, struct dummy_t, e); 58 | TEST_CHK(d->a == i++); 59 | ee = list_prev(ee); 60 | } 61 | 62 | // remove even numbers 63 | ee = list_begin(&list); 64 | while(ee){ 65 | d = _get_entry(ee, struct dummy_t, e); 66 | if (d->a % 2 == 0) { 67 | ee = list_remove(&list, ee); 68 | }else 69 | ee = list_next(ee); 70 | } 71 | 72 | // check 73 | ee = list_begin(&list); 74 | i=9; 75 | while(ee) { 76 | d = _get_entry(ee, struct dummy_t, e); 77 | TEST_CHK(d->a == i); 78 | i-=2; 79 | ee = list_next(ee); 80 | } 81 | 82 | // remove head 83 | list_remove(&list, list_begin(&list)); 84 | 85 | // check 86 | ee = list_begin(&list); 87 | i=7; 88 | while(ee) { 89 | d = _get_entry(ee, struct dummy_t, e); 90 | TEST_CHK(d->a == i); 91 | i-=2; 92 | ee = list_next(ee); 93 | } 94 | 95 | // pop front 96 | ee = list_pop_front(&list); 97 | // check 98 | d = _get_entry(ee, struct dummy_t, e); 99 | TEST_CHK(d->a == 7); 100 | ee = list_begin(&list); 101 | i=5; 102 | while(ee) { 103 | d = _get_entry(ee, struct dummy_t, e); 104 | TEST_CHK(d->a == i); 105 | i-=2; 106 | ee = list_next(ee); 107 | } 108 | 109 | // pop back 110 | ee = list_pop_back(&list); 111 | // check 112 | d = _get_entry(ee, struct dummy_t, e); 113 | TEST_CHK(d->a == 1); 114 | ee = list_begin(&list); 115 | i=5; 116 | while(ee) { 117 | d = _get_entry(ee, struct dummy_t, e); 118 | TEST_CHK(d->a == i); 119 | i-=2; 120 | ee = list_next(ee); 121 | } 122 | 123 | TEST_RESULT("basic test"); 124 | } 125 | 126 | 127 | void insert_test() 128 | { 129 | TEST_INIT(); 130 | 131 | int n= 5; 132 | int i; 133 | struct list list; 134 | struct list_elem *e; 135 | struct dummy_t { 136 | int value; 137 | struct list_elem e; 138 | } dummy[n], dummy_a, dummy_b, dummy_c, dummy_d, *dummy_ptr; 139 | 140 | list_init(&list); 141 | 142 | for (i=0;ivalue); 161 | e = list_next(e); 162 | } 163 | 164 | TEST_RESULT("insert test"); 165 | } 166 | 167 | int main(){ 168 | basic_test(); 169 | insert_test(); 170 | 171 | return 0; 172 | } 173 | -------------------------------------------------------------------------------- /tests/unit/docio_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "docio.h" 23 | #include "filemgr.h" 24 | #include "filemgr_ops.h" 25 | #include "test.h" 26 | 27 | uint32_t _set_doc(struct docio_object *doc, char *key, char *meta, char *body) 28 | { 29 | strcpy((char*)doc->key, key); 30 | doc->length.keylen = strlen((char*)doc->key) + 1; 31 | strcpy((char*)doc->meta, meta); 32 | doc->length.metalen = strlen((char*)doc->meta) + 1; 33 | strcpy((char*)doc->body, body); 34 | doc->length.bodylen = strlen((char*)doc->body) + 1; 35 | 36 | return sizeof(struct docio_length) + doc->length.keylen + doc->length.metalen + doc->length.bodylen; 37 | } 38 | 39 | void basic_test() 40 | { 41 | TEST_INIT(); 42 | 43 | uint64_t offset; 44 | uint32_t docsize; 45 | int r; 46 | int blocksize = 128; 47 | struct docio_handle handle; 48 | struct filemgr *file; 49 | char keybuf[1024]; 50 | char metabuf[1024]; 51 | char bodybuf[4096]; 52 | struct docio_object doc; 53 | struct filemgr_config config; 54 | char *fname = (char *) "./docio_testfile"; 55 | 56 | handle.log_callback = NULL; 57 | 58 | memset(keybuf, 0x0, 1024); 59 | memset(metabuf, 0x0, 1024); 60 | memset(bodybuf, 0x0, 4096); 61 | doc.key = (void*)keybuf; 62 | doc.meta = (void*)metabuf; 63 | doc.body = (void*)bodybuf; 64 | 65 | memset(&config, 0, sizeof(config)); 66 | config.blocksize = blocksize; 67 | config.ncacheblock = 1024; 68 | config.options = FILEMGR_CREATE; 69 | config.num_wal_shards = 8; 70 | r = system(SHELL_DEL " docio_testfile"); 71 | (void)r; 72 | filemgr_open_result result = filemgr_open(fname, get_filemgr_ops(), &config, NULL); 73 | file = result.file; 74 | docio_init(&handle, file, false); 75 | 76 | docsize = _set_doc(&doc, (char *) "this_is_key", (char *) "this_is_metadata", 77 | (char *) "this_is_body_lawiefjaawleif"); 78 | (void)docsize; 79 | offset = docio_append_doc(&handle, &doc, 0, 0); 80 | (void)offset; 81 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 82 | 83 | docsize = _set_doc(&doc, (char *) "this_is_key2", (char *) "this_is_metadata2", 84 | (char *) "hello_world"); 85 | (void)docsize; 86 | offset = docio_append_doc(&handle, &doc, 0, 0); 87 | (void)offset; 88 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 89 | 90 | docsize = _set_doc(&doc, (char *) "key3", (char *) "a", (char *) "b"); 91 | (void)docsize; 92 | offset = docio_append_doc(&handle, &doc, 0, 0); 93 | (void)offset; 94 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 95 | 96 | docsize = _set_doc(&doc, (char *) "key4", (char *) "a", (char *) "b"); 97 | (void)docsize; 98 | offset = docio_append_doc(&handle, &doc, 0, 0); 99 | (void)offset; 100 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 101 | 102 | docsize = _set_doc(&doc, (char *) "key5", (char *) "a", (char *) "b"); 103 | (void)docsize; 104 | offset = docio_append_doc(&handle, &doc, 0, 0); 105 | (void)offset; 106 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 107 | 108 | doc.length.keylen = 1; 109 | doc.length.metalen = 1; 110 | doc.length.bodylen = 190; 111 | docsize = 12 + 182; 112 | (void)docsize; 113 | offset = docio_append_doc(&handle, &doc, 0, 0); 114 | (void)offset; 115 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 116 | 117 | keylen_t keylen; 118 | docio_read_doc_key(&handle, 81, &keylen, (void*)keybuf); 119 | DBG("keylen %d %s\n", keylen, keybuf); 120 | 121 | filemgr_commit(file, true, NULL); 122 | filemgr_close(file, true, NULL, NULL); 123 | 124 | TEST_RESULT("basic test"); 125 | } 126 | 127 | int main() 128 | { 129 | #ifdef _MEMPOOL 130 | mempool_init(); 131 | #endif 132 | 133 | 134 | basic_test(); 135 | 136 | return 0; 137 | } 138 | -------------------------------------------------------------------------------- /conanfile.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from conan import ConanFile 4 | from conan.tools.cmake import cmake_layout, CMake, CMakeToolchain 5 | from conan.tools.files import copy,save,load 6 | from conan.tools.scm import Git 7 | 8 | required_conan_version = ">=2.12.2" 9 | 10 | ''' 11 | Notes: 12 | Snappy is disabled. 13 | Build static lib only, no shared lib. 14 | Package contains bin forestdb_dump. 15 | ''' 16 | class ForestdbConan(ConanFile): 17 | name = "forestdb" 18 | license = "Apache License 2.0" 19 | description = "forestdb kv store" 20 | homepage = "https://github.com/ForestDB-KVStore/forestdb" 21 | 22 | generators = "CMakeDeps" 23 | settings = "os", "compiler", "build_type", "arch" 24 | options = { 25 | "shared": [True,False], 26 | "fPIC": [True, False], 27 | "coverage": [True, False], 28 | "with_snappy" : [True, False], 29 | "with_jemalloc" : [True, False], 30 | "build_tests": [True, False] 31 | } 32 | default_options = { 33 | "shared": False, 34 | "fPIC": True, 35 | "coverage": False, 36 | "with_snappy": False, 37 | "with_jemalloc": False, 38 | "build_tests": False 39 | } 40 | 41 | exports_sources = "CMakeLists.txt", "src/*", "include/*", "cmake/*", "tools/*", "option/*", "utils/*", "LICENSE" 42 | 43 | def set_version(self): 44 | if self.version: 45 | return 46 | 47 | git = Git(self, folder=self.recipe_folder) 48 | version = git.run(cmd="describe --tags --long") 49 | # the string returned by the git command above looks like this: 50 | # v1.2.3-4-g5bfa09b 51 | 52 | # if the tag starts with "v" - strip it 53 | version = version[1:] if version.startswith("v") else version 54 | 55 | # replace the second dash with "." in order to ensure proper semantic version comparison 56 | items = version.split("-", 2) 57 | assert len(items) == 3, f"Unexpected version format: {version}" 58 | version = items[0] + "-" + items[1] + "." + items[2] 59 | 60 | # the final version string will look like this: 61 | # "1.2.3-4.g5bfa09b" 62 | # 1 is the major version 63 | # 2 is the minor version 64 | # 3 is the patch version 65 | # 4.g5bfa09b is the pre-release version wich will also be parsed 66 | # and 4 will be the major version of the pre-release version 67 | self.version = version 68 | 69 | def requirements(self): 70 | if self.options.with_snappy: 71 | self.requires("snappy/[~1]") 72 | 73 | if self.options.with_jemalloc: 74 | self.requires("jemalloc/[*]") 75 | 76 | def _computeCommitHash(self): 77 | hash_file = os.path.join(self.recipe_folder, "COMMIT_HASH") 78 | if (os.path.exists(hash_file)): 79 | hash = load(self,path=hash_file) 80 | self.output.info(f"Fetched commit hash from {hash_file}") 81 | else: # we are building from local source, i.e. in editable mode 82 | git = Git(self, folder=self.recipe_folder) 83 | hash = git.get_commit() 84 | diff = git.run(cmd="diff --stat") 85 | if diff: 86 | hash +="-dirty" 87 | self.output.info(f"Fetched commit hash {hash} from local Git in {self.recipe_folder}") 88 | return hash 89 | 90 | def export(self): 91 | save(self, path=os.path.join(self.export_folder, "COMMIT_HASH"), content=self._computeCommitHash()) 92 | 93 | def layout(self): 94 | cmake_layout(self, generator="CMakeDeps") 95 | self.cpp.package.libs = [f"lib{self.name}.so" if self.options.shared else f"lib{self.name}.a"] 96 | self.cpp.package.defines = self.cpp.build.defines = ["_FDB_COMMIT_HASH=%s" % self._computeCommitHash()] 97 | 98 | def generate(self): 99 | tc = CMakeToolchain(self) 100 | tc.variables["WITH_CONAN"] = True 101 | tc.variables["CONAN_BUILD_COVERAGE"] = False 102 | tc.variables["CODE_COVERAGE"] = self.options.coverage 103 | tc.variables["SNAPPY_OPTION"] = self.options.with_snappy 104 | tc.variables["_JEMALLOC"] = self.options.with_snappy 105 | tc.variables["BUILD_TESTING"] = self.options.build_tests 106 | tc.variables["CMAKE_VERBOSE_MAKEFILE"] = False 107 | tc.variables["CMAKE_EXPORT_COMPILE_COMMANDS"] = True 108 | tc.generate() 109 | 110 | def build(self): 111 | cmake = CMake(self) 112 | cmake.configure() 113 | cmake.build() 114 | if self.options.build_tests: 115 | cmake.ctest() 116 | copy(self, "compile_commands.json", self.build_folder, self.source_folder, keep_path=False) 117 | 118 | def package(self): 119 | cmake = CMake(self) 120 | cmake.install() 121 | assert copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")), "Copy failed" 122 | 123 | -------------------------------------------------------------------------------- /tests/unit/hash_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "hash.h" 23 | #include "test.h" 24 | #include "common.h" 25 | #include "hash_functions.h" 26 | 27 | struct item { 28 | int val; 29 | struct hash_elem e; 30 | }; 31 | 32 | unsigned hash_func(struct hash *hash, struct hash_elem *e) 33 | { 34 | struct item *item = _get_entry(e, struct item, e); 35 | return item->val % hash->nbuckets; 36 | } 37 | 38 | int hash_cmp(struct hash_elem *a, struct hash_elem *b) 39 | { 40 | struct item *aa, *bb; 41 | aa = _get_entry(a, struct item, e); 42 | bb = _get_entry(b, struct item, e); 43 | if (aa->val < bb->val) return -1; 44 | else if (aa->val > bb->val) return 1; 45 | else return 0; 46 | } 47 | 48 | void basic_test() 49 | { 50 | TEST_INIT(); 51 | 52 | struct hash hash; 53 | struct item item[16], query, *result; 54 | struct hash_elem *e; 55 | int i; 56 | 57 | hash_init(&hash, 4, hash_func, hash_cmp); 58 | 59 | for (i=0;i<16;++i){ 60 | item[i].val = i; 61 | hash_insert(&hash, &item[i].e); 62 | } 63 | 64 | for (i=0;i<16;++i){ 65 | query.val = i; 66 | e = hash_find(&hash, &query.e); 67 | result = _get_entry(e, struct item, e); 68 | TEST_CHK(result->val == query.val); 69 | } 70 | 71 | for (i=1;i<16;i+=2){ 72 | query.val = i; 73 | e = hash_remove(&hash, &query.e); 74 | result = _get_entry(e, struct item, e); 75 | TEST_CHK(result->val == query.val); 76 | } 77 | 78 | for (i=0;i<16;++i){ 79 | query.val = i; 80 | e = hash_find(&hash, &query.e); 81 | TEST_CHK((i%2==1 && e==NULL) || (i%2==0 && e)); 82 | if (e) { 83 | result = _get_entry(e, struct item, e); 84 | TEST_CHK(result->val == query.val); 85 | } 86 | } 87 | 88 | TEST_RESULT("basic test"); 89 | } 90 | 91 | void string_hash_test() 92 | { 93 | TEST_INIT(); 94 | 95 | randomize(); 96 | unsigned a, b; 97 | char str[32]; 98 | 99 | int i,j; 100 | 101 | for (i=0;i<16;++i){ 102 | for (j=0;j<8;++j){ 103 | str[j] = 'a' + random('z'-'a'); 104 | } 105 | str[j] = 0; 106 | b = hash_djb2((uint8_t *)str, strlen(str)); 107 | (void)b; 108 | DBG("%s %10u %5u\n",str, b, b&0xfff); 109 | } 110 | 111 | for (i=0;i<16;++i){ 112 | sprintf(str, "asdf%d.%d",i,random(100)); 113 | b = hash_djb2((uint8_t *)str, strlen(str)); 114 | (void)b; 115 | DBG("%s %10u %5u\n",str, b, b&((unsigned)1023)); 116 | } 117 | 118 | sprintf(str, "1234aaaaaaaa"); 119 | a = hash_djb2((uint8_t *)str, strlen(str)); 120 | (void)a; 121 | sprintf(str, "5678aaaaaaaa"); 122 | b = hash_djb2((uint8_t *)str, strlen(str)); 123 | (void)b; 124 | DBG("%u %u\n", a, b); 125 | 126 | sprintf(str, "1234aaaaaaaa"); 127 | a = hash_djb2_last8((uint8_t *)str, strlen(str)); 128 | (void)a; 129 | sprintf(str, "5678aaaaaaaa"); 130 | b = hash_djb2_last8((uint8_t *)str, strlen(str)); 131 | (void)b; 132 | DBG("%u %u\n", a, b); 133 | 134 | sprintf(str, "./dummy0"); 135 | a = hash_djb2_last8((uint8_t *)str, strlen(str)); 136 | (void)a; 137 | sprintf(str, "./dummy01"); 138 | b = hash_djb2_last8((uint8_t *)str, strlen(str)); 139 | (void)b; 140 | DBG("%u %u\n", a, b); 141 | 142 | TEST_RESULT("string hash test"); 143 | } 144 | 145 | void twohash_test() 146 | { 147 | TEST_INIT(); 148 | 149 | randomize(); 150 | 151 | int n = 64; 152 | int i,j; 153 | uint32_t h; 154 | int *array = alca(int, n); 155 | 156 | for (i=100;i<108;++i) { 157 | for (j=333;j<341;++j){ 158 | printf("(%d,%d) = %u\n", i, j, hash_shuffle_2uint(i,j)); 159 | } 160 | } 161 | for (i=0;i 19 | #include 20 | #include 21 | #include 22 | 23 | #include "libforestdb/forestdb.h" 24 | #include "common.h" 25 | #include "internal_types.h" 26 | #include "btree_var_kv_ops.h" 27 | #include "hbtrie.h" 28 | #include "fdb_internal.h" 29 | 30 | #include "memleak.h" 31 | 32 | #ifdef __DEBUG 33 | #ifndef __DEBUG_FDB 34 | #undef DBG 35 | #undef DBGCMD 36 | #undef DBGSW 37 | #define DBG(...) 38 | #define DBGCMD(...) 39 | #define DBGSW(n, ...) 40 | #endif 41 | #endif 42 | 43 | LIBFDB_API 44 | fdb_status fdb_get_kv(fdb_kvs_handle *handle, 45 | const void *key, size_t keylen, 46 | void **value_out, size_t *valuelen_out) 47 | { 48 | fdb_doc *doc = NULL; 49 | fdb_status fs; 50 | 51 | if (key == NULL || keylen == 0 || keylen > FDB_MAX_KEYLEN || 52 | value_out == NULL || valuelen_out == NULL || 53 | (handle->kvs_config.custom_cmp && 54 | keylen > handle->config.blocksize - HBTRIE_HEADROOM)) { 55 | return FDB_RESULT_INVALID_ARGS; 56 | } 57 | 58 | fs = fdb_doc_create(&doc, key, keylen, NULL, 0, NULL, 0); 59 | if (fs != FDB_RESULT_SUCCESS) { // LCOV_EXCL_START 60 | if (doc) { 61 | fdb_doc_free(doc); 62 | } 63 | fdb_log(&handle->log_callback, FDB_LOG_ERROR, fs, 64 | "Failed to allocate fdb_doc instance for key '%s' in " 65 | "fdb_get_kv API.", (const char *)key); 66 | return fs; 67 | } // LCOV_EXCL_STOP 68 | 69 | fs = fdb_get(handle, doc); 70 | if (fs != FDB_RESULT_SUCCESS) { 71 | if (doc) { 72 | fdb_doc_free(doc); 73 | } 74 | return fs; 75 | } 76 | 77 | *value_out = doc->body; 78 | *valuelen_out = doc->bodylen; 79 | if (doc->key) free(doc->key); 80 | if (doc->meta) free(doc->meta); 81 | free(doc); 82 | 83 | return fs; 84 | } 85 | 86 | LIBFDB_API 87 | fdb_status fdb_set_kv(fdb_kvs_handle *handle, 88 | const void *key, size_t keylen, 89 | const void *value, size_t valuelen) 90 | { 91 | fdb_doc *doc; 92 | fdb_status fs; 93 | 94 | if (key == NULL || keylen == 0 || keylen > FDB_MAX_KEYLEN || 95 | (handle->kvs_config.custom_cmp && 96 | keylen > handle->config.blocksize - HBTRIE_HEADROOM)) { 97 | return FDB_RESULT_INVALID_ARGS; 98 | } 99 | 100 | fs = fdb_doc_create(&doc, key, keylen, NULL, 0, value, valuelen); 101 | if (fs != FDB_RESULT_SUCCESS) { // LCOV_EXCL_START 102 | if (doc) { 103 | fdb_doc_free(doc); 104 | } 105 | fdb_log(&handle->log_callback, FDB_LOG_ERROR, fs, 106 | "Warning: Failed to allocate fdb_doc instance for key '%s' in " 107 | "fdb_set_kv API.", (const char *)key); 108 | return fs; 109 | } // LCOV_EXCL_STOP 110 | 111 | fs = fdb_set(handle, doc); 112 | if (fs != FDB_RESULT_SUCCESS) { 113 | if (doc) { 114 | fdb_doc_free(doc); 115 | } 116 | return fs; 117 | } 118 | fdb_doc_free(doc); 119 | 120 | return fs; 121 | } 122 | 123 | LIBFDB_API 124 | fdb_status fdb_del_kv(fdb_kvs_handle *handle, 125 | const void *key, size_t keylen) 126 | { 127 | fdb_doc *doc; 128 | fdb_status fs; 129 | 130 | if (key == NULL || keylen == 0 || keylen > FDB_MAX_KEYLEN || 131 | (handle->kvs_config.custom_cmp && 132 | keylen > handle->config.blocksize - HBTRIE_HEADROOM)) { 133 | return FDB_RESULT_INVALID_ARGS; 134 | } 135 | 136 | fs = fdb_doc_create(&doc, key, keylen, NULL, 0, NULL, 0); 137 | if (fs != FDB_RESULT_SUCCESS) { // LCOV_EXCL_START 138 | if (doc) { 139 | fdb_doc_free(doc); 140 | } 141 | fdb_log(&handle->log_callback, FDB_LOG_ERROR, fs, 142 | "Warning: Failed to allocate fdb_doc instance for key '%s' in " 143 | "fdb_del_kv API.", (const char *)key); 144 | return fs; 145 | } // LCOV_EXCL_STOP 146 | 147 | fs = fdb_del(handle, doc); 148 | if (fs != FDB_RESULT_SUCCESS) { 149 | fdb_doc_free(doc); 150 | return fs; 151 | } 152 | fdb_doc_free(doc); 153 | 154 | return fs; 155 | } 156 | 157 | LIBFDB_API 158 | fdb_status fdb_free_block(void *ptr) 159 | { 160 | free(ptr); 161 | return FDB_RESULT_SUCCESS; 162 | } 163 | 164 | -------------------------------------------------------------------------------- /src/list.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Doubly Linked List 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #include 8 | #include "list.h" 9 | 10 | #ifdef _LIST_LOCK 11 | #define IFDEF_LOCK(...) __VA_ARGS__ 12 | #else 13 | #define IFDEF_LOCK(...) 14 | #endif 15 | 16 | 17 | #ifdef LIST_LOCK 18 | void list_init(struct list *list) 19 | { 20 | list->head = NULL; 21 | list->tail = NULL; 22 | IFDEF_LOCK( spin_init(&list->lock); ); 23 | } 24 | #endif 25 | 26 | void list_push_front(struct list *list, struct list_elem *e) 27 | { 28 | IFDEF_LOCK( spin_lock(&list->lock); ); 29 | if (list->head == NULL) { 30 | list->head = e; 31 | list->tail = e; 32 | e->next = e->prev = NULL; 33 | }else{ 34 | list->head->prev = e; 35 | e->prev = NULL; 36 | e->next = list->head; 37 | list->head = e; 38 | } 39 | IFDEF_LOCK( spin_unlock(&list->lock); ); 40 | } 41 | 42 | void list_push_back(struct list *list, struct list_elem *e) 43 | { 44 | IFDEF_LOCK( spin_lock(&list->lock); ); 45 | if (list->tail == NULL) { 46 | list->head = e; 47 | list->tail = e; 48 | e->next = e->prev = NULL; 49 | }else{ 50 | list->tail->next = e; 51 | e->prev = list->tail; 52 | e->next = NULL; 53 | list->tail = e; 54 | } 55 | IFDEF_LOCK( spin_unlock(&list->lock); ); 56 | } 57 | 58 | // insert E just before BEFORE 59 | void list_insert_before(struct list *list, struct list_elem *before, struct list_elem *e) 60 | { 61 | IFDEF_LOCK( spin_lock(&list->lock); ); 62 | e->prev = before->prev; 63 | e->next = before; 64 | if (before->prev) before->prev->next = e; 65 | else list->head = e; 66 | before->prev = e; 67 | IFDEF_LOCK( spin_unlock(&list->lock); ); 68 | } 69 | 70 | // insert E just after AFTER 71 | // LCOV_EXCL_START 72 | void list_insert_after(struct list *list, struct list_elem *after, struct list_elem *e) 73 | { 74 | IFDEF_LOCK( spin_lock(&list->lock); ); 75 | e->next = after->next; 76 | e->prev = after; 77 | if (after->next) after->next->prev = e; 78 | else list->tail = e; 79 | after->next = e; 80 | IFDEF_LOCK( spin_unlock(&list->lock); ); 81 | } 82 | // LCOV_EXCL_STOP 83 | 84 | struct list_elem *list_remove(struct list *list, struct list_elem *e) 85 | { 86 | IFDEF_LOCK( spin_lock(&list->lock); ); 87 | if (e) { 88 | // if not NULL 89 | if (e->next) e->next->prev = e->prev; 90 | if (e->prev) e->prev->next = e->next; 91 | 92 | if (list->head == e) list->head = e->next; 93 | if (list->tail == e) list->tail = e->prev; 94 | 95 | struct list_elem *next = e->next; 96 | 97 | IFDEF_LOCK( spin_unlock(&list->lock); ); 98 | return next; 99 | } 100 | // NULL .. do nothing 101 | IFDEF_LOCK( spin_unlock(&list->lock); ); 102 | return NULL; 103 | } 104 | 105 | struct list_elem *list_remove_reverse(struct list *list, struct list_elem *e) 106 | { 107 | IFDEF_LOCK( spin_lock(&list->lock); ); 108 | if (e) { 109 | // if not NULL 110 | if (e->next) e->next->prev = e->prev; 111 | if (e->prev) e->prev->next = e->next; 112 | 113 | if (list->head == e) list->head = e->next; 114 | if (list->tail == e) list->tail = e->prev; 115 | 116 | IFDEF_LOCK( spin_unlock(&list->lock); ); 117 | return e->prev; 118 | } 119 | // NULL .. do nothing 120 | IFDEF_LOCK( spin_unlock(&list->lock); ); 121 | return NULL; 122 | } 123 | 124 | struct list_elem *list_pop_front(struct list *list) 125 | { 126 | IFDEF_LOCK( spin_lock(&list->lock); ); 127 | struct list_elem *e = list->head; 128 | if (e) { 129 | // if not NULL 130 | if (e->next) e->next->prev = e->prev; 131 | if (e->prev) e->prev->next = e->next; 132 | 133 | if (list->head == e) list->head = e->next; 134 | if (list->tail == e) list->tail = e->prev; 135 | 136 | IFDEF_LOCK( spin_unlock(&list->lock); ); 137 | return e; 138 | } 139 | // NULL .. do nothing 140 | IFDEF_LOCK( spin_unlock(&list->lock); ); 141 | return NULL; 142 | } 143 | 144 | struct list_elem *list_pop_back(struct list *list) 145 | { 146 | IFDEF_LOCK( spin_lock(&list->lock); ); 147 | struct list_elem *e = list->tail; 148 | if (e) { 149 | // if not NULL 150 | if (e->next) e->next->prev = e->prev; 151 | if (e->prev) e->prev->next = e->next; 152 | 153 | if (list->head == e) list->head = e->next; 154 | if (list->tail == e) list->tail = e->prev; 155 | 156 | IFDEF_LOCK( spin_unlock(&list->lock); ); 157 | return e; 158 | } 159 | // NULL .. do nothing 160 | IFDEF_LOCK( spin_unlock(&list->lock); ); 161 | return NULL; 162 | } 163 | 164 | #ifdef LIST_LOCK 165 | struct list_elem *list_begin(struct list *list) 166 | { 167 | IFDEF_LOCK( spin_lock(&list->lock); ); 168 | struct list_elem *e = list->head; 169 | IFDEF_LOCK( spin_unlock(&list->lock); ); 170 | return e; 171 | } 172 | 173 | struct list_elem *list_end(struct list *list) 174 | { 175 | IFDEF_LOCK( spin_lock(&list->lock); ); 176 | struct list_elem *e = list->tail; 177 | IFDEF_LOCK( spin_unlock(&list->lock); ); 178 | return e; 179 | } 180 | #endif 181 | -------------------------------------------------------------------------------- /option/option.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_OPTION_H 19 | #define _JSAHN_OPTION_H 20 | 21 | //#include "mempool.h" 22 | 23 | #define SEQNUM_NOT_USED (UINT64_C(0xffffffffffffffff)) 24 | #define DEFAULT_KVS_NAME "default" 25 | 26 | #define __FDB_BCACHE_USE 27 | #ifdef __FDB_BCACHE_USE 28 | //#define __FDB_SORTED_COMPACTION 29 | #endif 30 | #define FDB_SECTOR_SIZE (512) 31 | 32 | //#define DOCIO_BLOCK_ALIGN 33 | #define DOCIO_LEN_STRUCT_ALIGN 34 | 35 | //#define __RAW_BLOCK 36 | 37 | #define __CRC32 38 | #ifdef __CRC32 39 | #define BTREE_CRC_OFFSET (8) 40 | #define BTREE_CRC_FIELD_LEN (8) 41 | #define BTREE_LEVEL_OFFSET (2) 42 | //#define __CHECKSUM_ADLER32 43 | #endif 44 | 45 | #define __BIT_CMP 46 | 47 | #define __ENDIAN_SAFE 48 | 49 | //#define __DEBUG_FDB 50 | //#define __DEBUG_WAL 51 | //#define __DEBUG_HBTRIE 52 | //#define __DEBUG_BTREE 53 | //#define __DEBUG_BTREEBLOCK 54 | //#define __DEBUG_BCACHE 55 | //#define __DEBUG_FILEMGR 56 | //#define __DEBUG_COUCHBENCH 57 | 58 | #define FDB_BLOCKSIZE (4096) 59 | #define FDB_WAL_NBUCKET (4099) // a prime number 60 | #define FDB_MAX_FILENAME_LEN (1024) 61 | #define FDB_MAX_KVINS_NAME_LEN (65536) 62 | #define FDB_WAL_THRESHOLD (4*1024) 63 | #define FDB_COMP_BUF_MINSIZE (67108864) // 64 MB, 8M offsets 64 | #define FDB_COMP_BUF_MAXSIZE (1073741824) // 1 GB, 128M offsets 65 | // Minimum window size for compaction sorting window, 128K offsets. 66 | #define FDB_COMP_WINDOW_MINSIZE (131072) 67 | #define FDB_COMP_BATCHSIZE (131072) // 128K docs 68 | // Minimum batch size for compaction, 1024 docs. 69 | #define FDB_COMP_BATCHSIZE_MIN (1024) 70 | #define FDB_COMP_MOVE_UNIT (134217728) // 128 MB 71 | #define FDB_COMP_RATIO_MIN (40) // 40% (writer speed / compactor speed) 72 | #define FDB_COMP_RATIO_MAX (60) // 60% (writer speed / compactor speed) 73 | #define FDB_COMP_PROB_UNIT_INC (5) // 5% (probability delta unit for increase) 74 | #define FDB_COMP_PROB_UNIT_DEC (5) // 5% (probability delta unit for decrease) 75 | 76 | // full compaction internval in secs when the circular block reusing is enabled 77 | #define FDB_COMPACTOR_SLEEP_DURATION (28800) 78 | #define FDB_DEFAULT_COMPACTION_THRESHOLD (30) 79 | 80 | #define FDB_BGFLUSHER_SLEEP_DURATION (2) 81 | #define FDB_BGFLUSHER_DIRTY_THRESHOLD (1024) //if more than this 4MB dirty 82 | // wake up any sleeping bgflusher 83 | 84 | //#define BCACHE_NBUCKET (4099) // a prime number 85 | #define BCACHE_NBUCKET (257) // a prime number 86 | //#define BCACHE_NBUCKET (31) // a prime number 87 | #define BCACHE_NDICBUCKET (4099) // a prime number 88 | #define BCACHE_FLUSH_UNIT (1048576) // 1MB 89 | #define BCACHE_EVICT_UNIT (1) 90 | #define BCACHE_MEMORY_THRESHOLD (0.8) // 80% of physical RAM 91 | #define __BCACHE_SECOND_CHANCE 92 | 93 | #define FILEMGR_PREFETCH_UNIT (4194304) // 4MB 94 | #define FILEMGR_RESIDENT_THRESHOLD (0.9) // 90 % of file is in buffer cache 95 | #define __FILEMGR_DATA_PARTIAL_LOCK 96 | //#define __FILEMGR_DATA_MUTEX_LOCK 97 | 98 | #define SB_DEFAULT_NUM_SUPERBLOCKS (4) // 4 superblocks for crash recovery 99 | #define SB_MAX_BITMAP_DOC_SIZE (1048576) // 1MB, 4M bitmaps per doc 100 | // Minimum file size for the condition that block reusing is triggered 101 | #define SB_MIN_BLOCK_REUSING_FILESIZE (16777216) // 16MB 102 | // Period that superblock is written into the file 103 | #define SB_SYNC_PERIOD (4194304) // sync for every 4MB update 104 | // Time limit for reusable block reclaim 105 | #define SB_RECLAIM_TIMELIMIT (100000) // 100 ms 106 | // Threshold for pre-reclaiming 107 | #define SB_PRE_RECLAIM_RATIO (10) // 10 % 108 | 109 | #define __BTREEBLK_BLOCKPOOL 110 | //#define __BTREEBLK_SUBBLOCK 111 | //#define __BTREEBLK_READ_TREE // not used now, for future use 112 | #define BTREEBLK_AGE_LIMIT (8) 113 | #define BTREEBLK_MIN_SUBBLOCK (128) 114 | //#define __BTREEBLK_CACHE 115 | #ifdef __BTREEBLK_CACHE 116 | #define BTREEBLK_CACHE_LIMIT (8) 117 | #endif 118 | 119 | //#define __UTREE 120 | #ifdef __UTREE 121 | #define __UTREE_HEADER_SIZE (16) 122 | #undef BTREE_CRC_OFFSET 123 | #define BTREE_CRC_OFFSET (__UTREE_HEADER_SIZE+8) 124 | #endif 125 | 126 | // WAL parition sizes 127 | #define DEFAULT_NUM_WAL_PARTITIONS (11) // a prime number 128 | #define MAX_NUM_WAL_PARTITIONS (512) 129 | 130 | // Buffer cache partition size 131 | #define DEFAULT_NUM_BCACHE_PARTITIONS (11) // a prime number 132 | #define MAX_NUM_BCACHE_PARTITIONS (512) 133 | 134 | // Asynchronous I/O queue depth 135 | #define ASYNC_IO_QUEUE_DEPTH (64) 136 | 137 | // Number of daemon compactor threads 138 | #define DEFAULT_NUM_COMPACTOR_THREADS (4) 139 | #define MAX_NUM_COMPACTOR_THREADS (128) 140 | 141 | #define DEFAULT_NUM_BGFLUSHER_THREADS (2) 142 | #define MAX_NUM_BGFLUSHER_THREADS (64) 143 | #endif 144 | -------------------------------------------------------------------------------- /utils/crypto_primitives.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_CRYPTO_PRIMITIVES_H 19 | #define _FDB_CRYPTO_PRIMITIVES_H 20 | 21 | 22 | // Defines inline functions sha256 and aes256 if implementations are available. 23 | // Callers can use "#if AES_AVAILABLE" to conditionalize code based on availability. 24 | 25 | #if defined(_CRYPTO_CC) 26 | 27 | // iOS and Mac OS implementation based on system-level CommonCrypto library: 28 | #include 29 | #include 30 | #include 31 | 32 | static inline void sha256(const void *src_buf, 33 | size_t size, 34 | void *digest) 35 | { 36 | CC_SHA256(src_buf, size, (uint8_t*)digest); 37 | } 38 | 39 | static bool aes256(bool encrypt, 40 | const void *key, 41 | const void *iv, 42 | void *dst_buf, 43 | const void *src_buf, 44 | size_t size) 45 | { 46 | size_t outSize; 47 | CCCryptorStatus status = CCCrypt((encrypt ? kCCEncrypt : kCCDecrypt), 48 | kCCAlgorithmAES128, 0, 49 | key, kCCKeySizeAES256, 50 | iv, 51 | src_buf, size, dst_buf, size, &outSize); 52 | assert(status != kCCParamError && status != kCCBufferTooSmall && 53 | status != kCCUnimplemented); 54 | return status == kCCSuccess; 55 | } 56 | 57 | #define AES256_AVAILABLE 1 58 | #define SHA256_AVAILABLE 1 59 | 60 | #elif defined(_CRYPTO_LIBTOMCRYPT) 61 | 62 | #include 63 | 64 | static inline void sha256(const void *src_buf, 65 | size_t size, 66 | void *digest) 67 | { 68 | unsigned long dummy = size; 69 | register_hash(&sha256_desc); 70 | hash_memory(find_hash("sha256"), (const unsigned char*)src_buf, 71 | (unsigned long)size, (unsigned char*)digest, &dummy); 72 | } 73 | 74 | static bool aes256(bool encrypt, 75 | const void *key, 76 | const void *iv, 77 | void *dst_buf, 78 | const void *src_buf, 79 | size_t size) 80 | { 81 | if(register_cipher(&rijndael_desc) != CRYPT_OK) { 82 | return false; 83 | } 84 | 85 | int cipher_idx = find_cipher("rijndael"); 86 | symmetric_CBC cbc; 87 | uint8_t tmpiv[16] = {0}; 88 | if (iv == NULL) { 89 | iv = tmpiv; 90 | } 91 | 92 | int err = cbc_start(cipher_idx, (const unsigned char*)iv, 93 | (const unsigned char*)key, 32, 0, &cbc); 94 | if (err != CRYPT_OK) { 95 | return false; 96 | } 97 | 98 | if (encrypt) { 99 | err = cbc_encrypt((const unsigned char*)src_buf, (unsigned char*)dst_buf, 100 | size, &cbc); 101 | } else { 102 | err = cbc_decrypt((const unsigned char*)src_buf, (unsigned char*)dst_buf, 103 | size, &cbc); 104 | } 105 | 106 | if(err != CRYPT_OK) { 107 | return false; 108 | } 109 | err = cbc_done(&cbc); 110 | 111 | return err == CRYPT_OK; 112 | } 113 | 114 | #define AES256_AVAILABLE 1 115 | #define SHA256_AVAILABLE 1 116 | 117 | #elif defined(_CRYPTO_OPENSSL) 118 | 119 | #include 120 | #include 121 | 122 | static inline void sha256(const void *src_buf, 123 | size_t size, 124 | void *digest) 125 | { 126 | SHA256_CTX sha256; 127 | SHA256_Init(&sha256); 128 | SHA256_Update(&sha256, src_buf, size); 129 | SHA256_Final((unsigned char*)digest, &sha256); 130 | } 131 | 132 | static inline bool aes256(bool encrypt, 133 | const void *key, 134 | const void *iv, 135 | void *dst_buf, 136 | const void *src_buf, 137 | size_t size) 138 | { 139 | AES_KEY aes_key; 140 | uint8_t tmpiv[16] = {0}; 141 | 142 | if(iv == NULL) { 143 | iv = tmpiv; 144 | } 145 | 146 | if(encrypt) { 147 | AES_set_encrypt_key((const unsigned char*)key, 256, &aes_key); 148 | AES_cbc_encrypt((const unsigned char*)src_buf, (unsigned char*)dst_buf, 149 | size, &aes_key, (unsigned char *)iv, AES_ENCRYPT); 150 | } else { 151 | AES_set_decrypt_key((const unsigned char*)key, 256, &aes_key); 152 | AES_cbc_encrypt((const unsigned char*)src_buf, (unsigned char*)dst_buf, 153 | size, &aes_key, (unsigned char *)iv, AES_DECRYPT); 154 | } 155 | 156 | return true; 157 | } 158 | 159 | #define AES256_AVAILABLE 1 160 | #define SHA256_AVAILABLE 1 161 | 162 | #else 163 | 164 | #define AES256_AVAILABLE 0 165 | #define SHA256_AVAILABLE 0 166 | 167 | #endif 168 | 169 | 170 | #endif /* _FDB_CRYPTO_PRIMITIVES_H */ 171 | -------------------------------------------------------------------------------- /utils/partiallock.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Generic Partial Lock 4 | * see https://github.com/greensky00/partiallock 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "partiallock.h" 12 | 13 | struct plock_node { 14 | void *lock; 15 | void *start; 16 | void *len; 17 | volatile uint32_t wcount; // waiting count 18 | struct list_elem le; 19 | }; 20 | 21 | int plock_init(struct plock *plock, struct plock_config *config) 22 | { 23 | if (!plock || !config) { 24 | return PLOCK_RESULT_INVALID_ARGS; 25 | } 26 | 27 | plock->ops = (struct plock_ops *)malloc(sizeof(struct plock_ops)); 28 | if (!plock->ops) { 29 | return PLOCK_RESULT_ALLOC_FAIL; 30 | } 31 | *plock->ops = *(config->ops); 32 | 33 | // allocate and init lock 34 | plock->sizeof_lock_user = config->sizeof_lock_user; 35 | plock->sizeof_lock_internal = config->sizeof_lock_internal; 36 | plock->sizeof_range = config->sizeof_range; 37 | plock->aux = config->aux; 38 | plock->lock = (void *)malloc(plock->sizeof_lock_internal); 39 | plock->ops->init_internal(plock->lock); 40 | 41 | // init list and tree 42 | list_init(&plock->active); 43 | list_init(&plock->inactive); 44 | 45 | return PLOCK_RESULT_SUCCESS; 46 | } 47 | 48 | plock_entry_t *plock_lock(struct plock *plock, void *start, void *len) 49 | { 50 | struct list_elem *le = NULL; 51 | struct plock_node *node = NULL; 52 | 53 | if (!plock || !start || !len) { 54 | return NULL; 55 | } 56 | 57 | // grab plock's lock 58 | plock->ops->lock_internal(plock->lock); 59 | 60 | // find existing overlapped lock 61 | le = list_begin(&plock->active); 62 | while (le) { 63 | node = _get_entry(le, struct plock_node, le); 64 | if (plock->ops->is_overlapped(node->start, node->len, 65 | start, len, plock->aux)) { 66 | // overlapped 67 | // increase waiting count 68 | node->wcount++; 69 | // release plock's lock 70 | plock->ops->unlock_internal(plock->lock); 71 | 72 | // grab node's lock 73 | plock->ops->lock_user(node->lock); 74 | // got control .. that means the owner released the lock 75 | // grab plock's lock 76 | plock->ops->lock_internal(plock->lock); 77 | // decrease waiting count 78 | le = list_next(&node->le); 79 | node->wcount--; 80 | if (node->wcount == 0) { 81 | // no other thread refers this node 82 | // move from active to inactive 83 | list_remove(&plock->active, &node->le); 84 | list_push_front(&plock->inactive, &node->le); 85 | } 86 | // release node's lock 87 | plock->ops->unlock_user(node->lock); 88 | } else { 89 | le = list_next(le); 90 | } 91 | } 92 | 93 | // get a free lock 94 | le = list_pop_front(&plock->inactive); 95 | if (le == NULL) { 96 | // no free lock .. create one 97 | node = (struct plock_node *)malloc(sizeof(struct plock_node)); 98 | if (!node) { 99 | plock->ops->unlock_internal(plock->lock); 100 | return NULL; 101 | } 102 | node->lock = (void *)malloc(plock->sizeof_lock_user); 103 | plock->ops->init_user(node->lock); 104 | node->start = (void *)malloc(plock->sizeof_range); 105 | node->len = (void *)malloc(plock->sizeof_range); 106 | if (!node->lock || !node->start || !node->len) { 107 | free(node); 108 | plock->ops->unlock_internal(plock->lock); 109 | return NULL; 110 | } 111 | } else { 112 | node = _get_entry(le, struct plock_node ,le); 113 | } 114 | node->wcount = 0; 115 | 116 | // copy start & len value 117 | memcpy(node->start, start, plock->sizeof_range); 118 | memcpy(node->len, len, plock->sizeof_range); 119 | // insert into active list 120 | list_push_back(&plock->active, &node->le); 121 | 122 | // grab node's lock & release plock's lock 123 | plock->ops->lock_user(node->lock); 124 | plock->ops->unlock_internal(plock->lock); 125 | 126 | return node; 127 | } 128 | 129 | int plock_unlock(struct plock *plock, plock_entry_t *plock_entry) 130 | { 131 | struct plock_node *node = plock_entry; 132 | 133 | if (!plock || !plock_entry) { 134 | return PLOCK_RESULT_INVALID_ARGS; 135 | } 136 | 137 | // grab plock's lock 138 | plock->ops->lock_internal(plock->lock); 139 | 140 | if (node->wcount == 0) { 141 | // no other thread refers this node 142 | // move from active to inactive 143 | list_remove(&plock->active, &node->le); 144 | list_push_front(&plock->inactive, &node->le); 145 | } 146 | plock->ops->unlock_user(node->lock); 147 | 148 | // release plock's lock 149 | plock->ops->unlock_internal(plock->lock); 150 | 151 | return PLOCK_RESULT_SUCCESS; 152 | } 153 | 154 | int plock_destroy(struct plock *plock) 155 | { 156 | struct list_elem *le; 157 | struct plock_node *node; 158 | 159 | if (!plock) { 160 | return PLOCK_RESULT_INVALID_ARGS; 161 | } 162 | 163 | plock->ops->destroy_internal(plock->lock); 164 | 165 | // free all active locks 166 | le = list_begin(&plock->active); 167 | while(le) { 168 | node = _get_entry(le, struct plock_node, le); 169 | le = list_next(le); 170 | 171 | // unlock and destroy 172 | plock->ops->unlock_user(node->lock); 173 | plock->ops->destroy_user(node->lock); 174 | free(node->start); 175 | free(node->len); 176 | free(node->lock); 177 | free(node); 178 | } 179 | 180 | // free all inactive locks 181 | le = list_begin(&plock->inactive); 182 | while(le) { 183 | node = _get_entry(le, struct plock_node, le); 184 | le = list_next(le); 185 | 186 | // destroy 187 | plock->ops->destroy_user(node->lock); 188 | free(node->start); 189 | free(node->len); 190 | free(node->lock); 191 | free(node); 192 | } 193 | 194 | // free plock 195 | free(plock->lock); 196 | free(plock->ops); 197 | 198 | return PLOCK_RESULT_SUCCESS; 199 | } 200 | 201 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | ## Dependencies 2 | 3 | On non-Windows platforms, there is a dependency on Snappy library because ForestDB supports an option to compress a document body using Snappy. 4 | Please visit [Snappy site](https://code.google.com/p/snappy/) for more details. 5 | 6 | * **Ubuntu** 7 | 8 | `sudo apt-get install libsnappy-dev` 9 | 10 | * **CentOS** 11 | 12 | `wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz` 13 | 14 | `tar -xvfz snappy-1.1.1.tar.gz` 15 | 16 | `cd snappy-1.1.1` 17 | 18 | `./configure && make && sudo make install` 19 | 20 | * **OS X** 21 | 22 | `sudo brew install snappy` 23 | 24 | We also use the asynchronous I/O library `libaio` on Linux if it is available, to submit multiple I/O requests at once to speed up fetching non-resident blocks from disk. As of this time, we use `libaio` to read data blocks from the old file during the compaction. 25 | 26 | * **Ubuntu** 27 | 28 | `sudo apt-get install libaio-devel` 29 | 30 | * **CentOS** 31 | 32 | `sudo yum install libaio-devel` 33 | 34 | We plan to support asynchronous I/O in other operating systems such as Windows and OS X, soon. 35 | 36 | For better memory fragmentation and concurrency management, jemalloc can be linked to ForestDB if available on the host environment. Please visit [jemalloc site](http://www.canonware.com/jemalloc/) for more information and install guideline. 37 | 38 | Database encryption is also supported and can be enabled at compilation time. Currently, we support CommonCrypto library on iOS and OS X, [OpenSSL](https://www.openssl.org/), and [LibTomCrypt](https://github.com/libtom/libtomcrypt). 39 | 40 | ## Compilation and Build 41 | 42 | We use [CMake](http://www.cmake.org/cmake/) to provide the build support for a wide range of platforms. Cmake version 3.1 is required at least to have C++11 support. Please follow the instructions below to install CMake in your target platform. 43 | 44 | * **Ubuntu** 45 | 46 | `sudo apt-get install cmake` 47 | 48 | * **Centos** 49 | 50 | `wget http://www.cmake.org/files/v2.8/cmake-3.4.3.tar.gz` 51 | 52 | `tar xvfz cmake-3.4.3.tar.gz` 53 | 54 | `cd cmake-3.4.3` 55 | 56 | `./bootstrap && make && sudo make install` 57 | 58 | * **OS X** 59 | 60 | `brew install cmake` 61 | 62 | * **Windows** 63 | 64 | Please download and install CMake binary for Windows from [CMake download page](http://www.cmake.org/cmake/resources/software.html). 65 | 66 | Once CMake is installed, please follow the instructions below to compile and build ForestDB on Ubuntu, Centos, or OS X: 67 | 68 | `git clone git://github.com/couchbase/forestdb` 69 | 70 | `cd forestdb` 71 | 72 | `mkdir build` 73 | 74 | `cd build` 75 | 76 | `cmake ../` 77 | 78 | (The default value of `CMAKE_BUILD_TYPE` is `RelWithDebInfo`. If you want to build with optimizations disabled for debugging, type `cmake -DCMAKE_BUILD_TYPE=Debug ../` instead.) 79 | 80 | `_JEMALLOC` variable can be optionally set to 1 and passed to link jemalloc to ForestDB (`cmake -D_JEMALLOC=1 ../`) 81 | 82 | `_ENCRYPTION` macro can be also optionally passed to specify which crypto library (`commoncrypto`, `openssl`, or `libtomcrypt`) is used to support database encryption (e.g., `cmake -D_ENCRYPTION=commoncrypto ../`) 83 | 84 | `make all` 85 | 86 | On Windows (using Visual Studio's CL compiler), the instructions are as follows: 87 | 88 | `git clone git://github.com/couchbase/forestdb` (or clone repository using [TortoiseGit](http://code.google.com/p/tortoisegit/)) 89 | 90 | `cd forestdb` 91 | 92 | `mkdir build` 93 | 94 | `cd build` 95 | 96 | Note that the path and environment variables for command-line builds need to be set before moving to the next step. Please refer to the [MSDN Page](http://msdn.microsoft.com/en-us/library/f2ccy3wt.aspx). 97 | 98 | `cmake -G "NMake Makefiles" ..\` 99 | 100 | (The default value of `CMAKE_BUILD_TYPE` is `Debug`. If you want to build with optimizations enabled for better performance, type `cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ../` instead.) 101 | 102 | `nmake all` 103 | 104 | ## Test 105 | 106 | To run all the unit tests: 107 | 108 | * **Ubuntu**, **Centos**, and **OS X** 109 | 110 | `make test` 111 | 112 | * **Windows** 113 | 114 | `nmake test` 115 | 116 | * To enable code-coverage reporting for tests, install ```gcov``` and ```lcov``` for your OS. (Windows not supported). Also make sure build type is **Coverage** 117 | 118 | `cmake -DCMAKE_BUILD_TYPE=Coverage ../` 119 | 120 | `make all` 121 | 122 | `make test_coverage` 123 | 124 | This target will run the tests and output coverage report to ```/coverage/index.html``` 125 | 126 | * To enable valgrind tests, install ```valgrind``` for your OS. (Windows not supported). Also make sure build type is **Valgrind** 127 | 128 | `cmake -DCMAKE_BUILD_TYPE=Valgrind ../` 129 | 130 | `make all` 131 | 132 | `make test_valgrind` 133 | 134 | This target will run the tests and output the memory report to the console 135 | 136 | * By default, every run will produce an additional .lat file which indicates the latencies 137 | of the forestdb api. This is disabled if CMAKE_BUILD_TYPE=Release 138 | 139 | * To enable the use of threadsanitizer for tests, install clang 3.5+ (currently only Ubuntu 14.04 is supported). Also make sure to set flag **CB_THREADSANITIZER** 140 | 141 | `cmake -DCMAKE_C_COMPILER=clang-3.5 -DCMAKE_CXX_COMPILER=clang++-3.5 -DCB_THREADSANITIZER=1 ..` 142 | 143 | `make all` 144 | 145 | `make test` 146 | 147 | Alternatively you can manually run tests under build/tests// for real-time analysis. When running as standalone it is recommended to include false error suppressions: 148 | 149 | `TSAN_OPTIONS="second_deadlock_stack=1 suppressions=../../../cmake/Modules/tsan.suppressions" ./fdb_functional_test` 150 | 151 | * For hardware accelerated CRC32C which can improve ForestDB performance, please fetch the optional couchbase platform repository into the forestdb source directory before building. 152 | 153 | `cd forestdb` 154 | 155 | `git clone https://github.com/couchbase/platform.git` 156 | 157 | `Follow the rest of the platform specific build instructions from above` 158 | 159 | 160 | 161 | Alternative build method - with Conan 162 | ---------- 163 | 1. Install conan using one of the methods desribed here: https://conan.io/downloads 164 | 1. `conan build .` - builds Release configuration 165 | 1. `conan build . -s build_type=Debug` builds an alternative configuration supported by CMake 166 | 1. You can control supported build options via command line as well, ex: 167 | `conan build . -o build_tests=True` 168 | **NOTE**: this conan recipe supports conan **editable** mode which make it convenient to embedd this library in other projects built with conan 169 | -------------------------------------------------------------------------------- /cmake/Modules/CodeCoverage.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # 2012-01-31, Lars Bilke 3 | # - Enable Code Coverage 4 | # 5 | # 2013-09-17, Joakim Söderberg 6 | # - Added support for Clang. 7 | # - Some additional usage instructions. 8 | # 9 | # USAGE: 10 | 11 | # 0. (Mac only) If you use Xcode 5.1 make sure to patch geninfo as described here: 12 | # http://stackoverflow.com/a/22404544/80480 13 | # 14 | # 1. Copy this file into your cmake modules path. 15 | # 16 | # 2. Add the following line to your CMakeLists.txt: 17 | # INCLUDE(CodeCoverage) 18 | # 19 | # 3. Set compiler flags to turn off optimization and enable coverage: 20 | # SET(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") 21 | # SET(CMAKE_C_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") 22 | # 23 | # 3. Use the function SETUP_TARGET_FOR_COVERAGE to create a custom make target 24 | # which runs your test executable and produces a lcov code coverage report: 25 | # Example: 26 | # SETUP_TARGET_FOR_COVERAGE( 27 | # my_coverage_target # Name for custom target. 28 | # test_driver # Name of the test driver executable that runs the tests. 29 | # # NOTE! This should always have a ZERO as exit code 30 | # # otherwise the coverage generation will not complete. 31 | # coverage # Name of output directory. 32 | # ) 33 | # 34 | # 4. Build a Debug build: 35 | # cmake -DCMAKE_BUILD_TYPE=Debug .. 36 | # make 37 | # make my_coverage_target 38 | # 39 | # 40 | 41 | # Check prereqs 42 | FIND_PROGRAM( GCOV_PATH gcov ) 43 | FIND_PROGRAM( LCOV_PATH lcov ) 44 | FIND_PROGRAM( GENHTML_PATH genhtml ) 45 | FIND_PROGRAM( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/tests) 46 | 47 | IF(NOT GCOV_PATH) 48 | MESSAGE(FATAL_ERROR "gcov not found! Aborting...") 49 | ENDIF() # NOT GCOV_PATH 50 | 51 | IF(NOT CMAKE_COMPILER_IS_GNUCXX) 52 | # Clang version 3.0.0 and greater now supports gcov as well. 53 | MESSAGE(WARNING "Compiler is not GNU gcc! Clang Version 3.0.0 and greater supports gcov as well, but older versions don't.") 54 | 55 | IF(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 56 | MESSAGE(FATAL_ERROR "Compiler is not GNU gcc! Aborting...") 57 | ENDIF() 58 | ENDIF() # NOT CMAKE_COMPILER_IS_GNUCXX 59 | 60 | SET(CMAKE_CXX_FLAGS_COVERAGE 61 | "-Wunused-variable -g -O0 --coverage -fprofile-arcs -ftest-coverage" 62 | CACHE STRING "Flags used by the C++ compiler during coverage builds." 63 | FORCE ) 64 | SET(CMAKE_C_FLAGS_COVERAGE 65 | "-Wunused-variable -g -O0 --coverage -fprofile-arcs -ftest-coverage" 66 | CACHE STRING "Flags used by the C compiler during coverage builds." 67 | FORCE ) 68 | SET(CMAKE_EXE_LINKER_FLAGS_COVERAGE 69 | "" 70 | CACHE STRING "Flags used for linking binaries during coverage builds." 71 | FORCE ) 72 | SET(CMAKE_SHARED_LINKER_FLAGS_COVERAGE 73 | "" 74 | CACHE STRING "Flags used by the shared libraries linker during coverage builds." 75 | FORCE ) 76 | MARK_AS_ADVANCED( 77 | CMAKE_CXX_FLAGS_COVERAGE 78 | CMAKE_C_FLAGS_COVERAGE 79 | CMAKE_EXE_LINKER_FLAGS_COVERAGE 80 | CMAKE_SHARED_LINKER_FLAGS_COVERAGE ) 81 | 82 | IF ( NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "Coverage")) 83 | MESSAGE( WARNING "Code coverage results with an optimized (non-Debug) build may be misleading" ) 84 | ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" 85 | 86 | 87 | # Param _targetname The name of new the custom make target 88 | # Param _testrunner The name of the target which runs the tests. 89 | # MUST return ZERO always, even on errors. 90 | # If not, no coverage report will be created! 91 | # Param _outputname lcov output is generated as _outputname.info 92 | # HTML report is generated in _outputname/index.html 93 | # Optional fourth parameter is passed as arguments to _testrunner 94 | # Pass them in list form, e.g.: "-j;2" for -j 2 95 | FUNCTION(SETUP_TARGET_FOR_COVERAGE _targetname _testrunner _outputname) 96 | 97 | IF(NOT LCOV_PATH) 98 | MESSAGE(FATAL_ERROR "lcov not found! Aborting...") 99 | ENDIF() # NOT LCOV_PATH 100 | 101 | IF(NOT GENHTML_PATH) 102 | MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") 103 | ENDIF() # NOT GENHTML_PATH 104 | 105 | # Setup target 106 | ADD_CUSTOM_TARGET(${_targetname} 107 | 108 | # Cleanup lcov 109 | ${LCOV_PATH} --directory . --zerocounters 110 | 111 | # Run tests 112 | COMMAND ${_testrunner} ${ARGV3} 113 | 114 | # Capturing lcov counters and generating report 115 | COMMAND ${LCOV_PATH} --rc lcov_branch_coverage=1 --directory . --capture --output-file ${_outputname}.info 116 | COMMAND ${LCOV_PATH} --rc lcov_branch_coverage=1 --remove ${_outputname}.info 'tests/*' '/usr/*' --output-file ${_outputname}.info.cleaned 117 | COMMAND ${GENHTML_PATH} --rc lcov_branch_coverage=1 -o ${_outputname} ${_outputname}.info.cleaned 118 | COMMAND ${CMAKE_COMMAND} -E remove ${_outputname}.info ${_outputname}.info.cleaned 119 | 120 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 121 | COMMENT "Resetting code coverage counters to zero.\nProcessing code coverage counters and generating report." 122 | ) 123 | 124 | # Show info where to find the report 125 | ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD 126 | COMMAND ; 127 | COMMENT "Open ./${_outputname}/index.html in your browser to view the coverage report." 128 | ) 129 | 130 | ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE 131 | 132 | # Param _targetname The name of new the custom make target 133 | # Param _testrunner The name of the target which runs the tests 134 | # Param _outputname cobertura output is generated as _outputname.xml 135 | # Optional fourth parameter is passed as arguments to _testrunner 136 | # Pass them in list form, e.g.: "-j;2" for -j 2 137 | FUNCTION(SETUP_TARGET_FOR_COVERAGE_COBERTURA _targetname _testrunner _outputname) 138 | 139 | IF(NOT PYTHON_EXECUTABLE) 140 | MESSAGE(FATAL_ERROR "Python not found! Aborting...") 141 | ENDIF() # NOT PYTHON_EXECUTABLE 142 | 143 | IF(NOT GCOVR_PATH) 144 | MESSAGE(FATAL_ERROR "gcovr not found! Aborting...") 145 | ENDIF() # NOT GCOVR_PATH 146 | 147 | ADD_CUSTOM_TARGET(${_targetname} 148 | 149 | # Run tests 150 | ${_testrunner} ${ARGV3} 151 | 152 | # Running gcovr 153 | COMMAND ${GCOVR_PATH} -x -r ${CMAKE_SOURCE_DIR} -e '${CMAKE_SOURCE_DIR}/tests/' -o ${_outputname}.xml 154 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 155 | COMMENT "Running gcovr to produce Cobertura code coverage report." 156 | ) 157 | 158 | # Show info where to find the report 159 | ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD 160 | COMMAND ; 161 | COMMENT "Cobertura code coverage report saved in ${_outputname}.xml." 162 | ) 163 | 164 | ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE_COBERTURA 165 | --------------------------------------------------------------------------------