├── src ├── config.cmake.h ├── breakpad_dummy.cc ├── tasks.h ├── task_type.h ├── filemgr_ops.cc ├── breakpad.h ├── btree_var_kv_ops.h ├── hash_functions.h ├── configuration.h ├── filemgr_ops.h ├── btree_str_kv.h ├── task_priority.cc ├── globaltask.cc ├── sync_object.h ├── encryption_bogus.cc ├── hash_functions.cc ├── version.h ├── memory_pool.cc ├── list.h ├── hash.h ├── bgflusher.h ├── breakpad_linux.cc ├── taskable.h ├── workload.h ├── avltree.h ├── task_priority.h ├── ringbuffer.h ├── encryption_aes.cc ├── breakpad_win32.cc ├── btree_fast_str_kv.h ├── taskqueue.h ├── encryption.h ├── memory_pool.h ├── encryption.cc ├── checksum.h ├── version.cc ├── btree_kv.h ├── common.h ├── forestdb_endian.h ├── executorthread.h ├── checksum.cc ├── globaltask.h ├── futurequeue.h ├── file_handle.h ├── atomicqueue.h └── list.cc ├── utils ├── gettimeofday_vs.h ├── crc32.h ├── debug.h ├── system_resource_stats.h ├── gettimeofday_vs.cc ├── partiallock.h ├── timing.h ├── time_utils.h ├── iniparser.h ├── memleak.h ├── bitwise_utils.h ├── system_resource_stats.cc └── time_utils.cc ├── tests ├── fdbench-micro │ ├── README.md │ ├── CMakeLists.txt │ └── config.h ├── e2e │ ├── CMakeLists.txt │ └── e2espec.h ├── CMakeLists.txt ├── functional │ ├── functional_util.h │ └── functional_util.cc ├── usecase │ └── CMakeLists.txt ├── stats-agg │ └── stat_aggregator.h ├── unit │ ├── filemgr_test.cc │ ├── docio_test.cc │ ├── mempool_test.cc │ └── hash_test.cc ├── include │ └── test.h └── anomaly │ ├── CMakeLists.txt │ └── filemgr_anomalous_ops.h ├── tools ├── threshold.stats ├── dump_option.ini └── dump_common.h ├── .gitignore ├── cmake └── Modules │ ├── tsan.suppressions │ ├── FindJemalloc.cmake │ ├── FindAsyncIOLib.cmake │ ├── FindSnappy.cmake │ ├── FindEncryptionLib.cmake │ ├── MemoryCheck.cmake │ ├── ThreadSanitizer.cmake │ └── CouchbaseCodeCoverage.cmake ├── CONTRIBUTING.md ├── README.md └── option └── option.h /src/config.cmake.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* various */ 4 | #define FORESTDB_VERSION "${FORESTDB_VERSION}" -------------------------------------------------------------------------------- /utils/gettimeofday_vs.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define NOMINMAX 1 4 | #include 5 | #undef NOMINMAX 6 | #include 7 | #include 8 | 9 | struct timezone 10 | { 11 | int tz_minuteswest; /* minutes W of Greenwich */ 12 | int tz_dsttime; /* type of dst correction */ 13 | }; 14 | 15 | int gettimeofday_vs(struct timeval *tv, void *tz); 16 | 17 | -------------------------------------------------------------------------------- /tests/fdbench-micro/README.md: -------------------------------------------------------------------------------- 1 | # fdbench-micro 2 | forestdb micro benchmark testing 3 | 4 | **dependencies** 5 | See: https://github.com/couchbase/forestdb/blob/master/INSTALL.md#dependencies 6 | 7 | *NOTE: make install* of forestdb libraries is required prior to run 8 | 9 | **Standalone** 10 | ```bash 11 | mkdir build && cd build 12 | cmake ../ 13 | make 14 | ./fdb_bench 15 | ``` 16 | -------------------------------------------------------------------------------- /tools/threshold.stats: -------------------------------------------------------------------------------- 1 | set 100 2 | get 100 3 | delete 100 4 | iterator_init 100 5 | iterator_get 100 6 | iterator_next 100 7 | iterator_close 100 8 | snap_open 100 9 | kv_close 100 10 | file_close 10000 11 | snap_close 10000 12 | shutdown 100000 13 | commit_wal 100000 14 | commit_norm 100000 15 | compact 1000000 16 | -------------------------------------------------------------------------------- /utils/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_CRC32_H 2 | #define _JSAHN_CRC32_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | uint32_t crc32_1(const void* data, size_t len, uint32_t prev_value); 9 | uint32_t crc32_8(const void* data, size_t len, uint32_t prev_value); 10 | uint32_t crc32_8_last8(const void *data, size_t len, uint32_t prev_value); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Keep the entries sorted to reduce the risk for a merge conflict 2 | *.[ao] 3 | couchstore_api/couchbench_fdb 4 | build 5 | coverage 6 | platform 7 | dummy2 8 | errorlog.txt 9 | forestdb_test 10 | libforestdb.so 11 | libforestdb_couch.so 12 | src/wc 13 | tests/bcache_test 14 | tests/btreeblock_test 15 | tests/crc_test 16 | tests/docio_test 17 | tests/filemgr_test 18 | tests/hash_test 19 | tests/hbtrie_test 20 | tests/list_test 21 | tests/mempool_test 22 | wt2 23 | -------------------------------------------------------------------------------- /cmake/Modules/tsan.suppressions: -------------------------------------------------------------------------------- 1 | # ThreadSanitizer suppressions file for Couchbase 2 | # https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions 3 | 4 | # In forestdb plock_lock is invoked from a lock structure that is sure to not 5 | # overlap with that of another thread by use of is_overlapped() test. 6 | # However since this function is not trusted by ThreadSanitizer it reports 7 | # a lock inversion since the locks in the race are from the same struct definition 8 | # but different memory addresses. This is hence a false positive. 9 | deadlock:plock_lock 10 | -------------------------------------------------------------------------------- /tests/e2e/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(e2etest 2 | e2etest.cc 3 | e2espec.cc 4 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(e2etest ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(e2etest PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | # add test target 16 | add_test(e2etest e2etest) 17 | ADD_CUSTOM_TARGET(e2etests 18 | COMMAND ctest 19 | ) 20 | -------------------------------------------------------------------------------- /tests/fdbench-micro/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(fdb_microbench 2 | fdb_bench.cc 3 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 4 | $ 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(fdb_microbench ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(fdb_microbench PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | 16 | # add test target 17 | add_test(fdb_microbench fdb_microbench) 18 | ADD_CUSTOM_TARGET(fdbench-micro 19 | COMMAND ctest 20 | ) 21 | -------------------------------------------------------------------------------- /src/breakpad_dummy.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | void initialize_breakpad(const char* minidump_dir) { 21 | } 22 | 23 | void destroy_breakpad(void) { 24 | } 25 | -------------------------------------------------------------------------------- /utils/debug.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "time_utils.h" 5 | #include 6 | 7 | #ifdef __DEBUG 8 | #include 9 | 10 | #define DBG(...) printf(__VA_ARGS__) 11 | #define DBGCMD(...) __VA_ARGS__ 12 | #define DBGSW(n, ...) if (_dbg_is_sw_set(n)) {__VA_ARGS__; } 13 | #else 14 | #define DBG(...) 15 | #define DBGCMD(...) 16 | #define DBGSW(n, ...) 17 | #endif 18 | 19 | void _dbg_sw_set(int n); 20 | void _dbg_sw_clear(int n); 21 | int _dbg_is_sw_set(int n); 22 | 23 | void _dbg_set_addr(int n, void *addr); 24 | void * _dbg_get_addr(int n); 25 | 26 | void _dbg_set_uint64_t(int n, uint64_t val); 27 | uint64_t _dbg_get_uint64_t(int n); 28 | 29 | fdb_status _dbg_install_handler(void); 30 | fdb_status _dbg_destroy_altstack(void); 31 | fdb_status _dbg_handle_crashes(const char *pathname); 32 | 33 | void dbg_print_buf(void *buf, uint64_t buflen, bool hex, int align); 34 | -------------------------------------------------------------------------------- /src/tasks.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_TASKS_H_ 19 | #define SRC_TASKS_H_ 1 20 | 21 | #include "common.h" 22 | 23 | #include 24 | #include 25 | #include "globaltask.h" 26 | 27 | #endif // SRC_TASKS_H_ 28 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Forestdb 2 | 3 | We look forward to your contributions, but ask that you first review these guidelines. 4 | 5 | ### Sign the CLA 6 | 7 | As Forestdb is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Forestdb project does use this code review tool and it is also used to track acceptance of the contributor license agreements. 8 | 9 | ### Submitting a Patch 10 | 11 | All types of contributions are welcome, but please keep the following in mind: 12 | 13 | - If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged. 14 | - Existing tests should continue to pass, new tests for the contribution are nice to have. 15 | -------------------------------------------------------------------------------- /src/task_type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef SRC_TASK_TYPE_H_ 18 | #define SRC_TASK_TYPE_H_ 1 19 | 20 | enum task_type_t { 21 | NO_TASK_TYPE=-1, 22 | WRITER_TASK_IDX=0, 23 | READER_TASK_IDX=1, 24 | AUXIO_TASK_IDX=2, 25 | NONIO_TASK_IDX=3, 26 | NUM_TASK_GROUPS=4 // keep this as last element of the enum 27 | }; 28 | 29 | #endif // SRC_TASK_TYPE_H_ 30 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(STAT_AGGREGATOR 2 | ${PROJECT_SOURCE_DIR}/tests/stats-agg/stat_aggregator.cc) 3 | add_library(TEST_STAT_AGG OBJECT ${STAT_AGGREGATOR}) 4 | 5 | set(ROOT_SRC ${PROJECT_SOURCE_DIR}/src) 6 | set(ROOT_UTILS ${PROJECT_SOURCE_DIR}/utils) 7 | set(UNIT_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/unit) 8 | set(FUNCTIONAL_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/functional) 9 | set(ANOMALY_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/anomaly) 10 | set(E2E_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/e2e) 11 | set(MICRO_BENCH_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fdbench-micro) 12 | set(USECASE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/usecase) 13 | 14 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include) 15 | include_directories(BEFORE ${PROJECT_SOURCE_DIR}/tests/stats-agg) 16 | 17 | # import each test suite 18 | add_subdirectory("${UNIT_TEST_DIR}") 19 | add_subdirectory("${FUNCTIONAL_TEST_DIR}") 20 | add_subdirectory("${ANOMALY_TEST_DIR}") 21 | add_subdirectory("${E2E_TEST_DIR}") 22 | add_subdirectory("${MICRO_BENCH_DIR}") 23 | add_subdirectory("${USECASE_TEST_DIR}") 24 | -------------------------------------------------------------------------------- /tools/dump_option.ini: -------------------------------------------------------------------------------- 1 | [doc] 2 | # Flag to enable displaying the body of document. 3 | # It is set to 'y' by default. 4 | print_body = y 5 | 6 | # Flag to enable displaying the metadata of document. 7 | # It is set to 'y' by default. 8 | print_meta = y 9 | 10 | # Flag to print the key of document in hex. 11 | # It is set to 'n' by default. 12 | print_key_in_hex = n 13 | 14 | # Flag to print the meatadata of document in hex. 15 | # It is set to 'y' by default. 16 | print_meta_in_hex = y 17 | 18 | # Flag to print the body of document in hex. 19 | # It is set to 'n' by default. 20 | print_body_in_hex = n 21 | 22 | [hex] 23 | # The number of bytes to be printed in a line in hex mode. 24 | # It is set to 16 by default. 25 | hex_align = 16 26 | 27 | [scan] 28 | # Mode of scanning of the documents in a file. 29 | # wal_first_index_next: scan WAL first and the main index next 30 | # key: scan all documents in an ascending order of key 31 | # seq: scan all documents in an ascending order of sequence number 32 | # It is set to 'wal_first_index_next' by default 33 | scan_mode = wal_first_index_next 34 | -------------------------------------------------------------------------------- /utils/system_resource_stats.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _SYSTEM_RESOURCE_STATS_H 19 | #define _SYSTEM_RESOURCE_STATS_H 20 | 21 | #include 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | int64_t get_memory_size(void); 28 | 29 | size_t get_num_cores(void); 30 | 31 | #ifdef __cplusplus 32 | } 33 | #endif 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/filemgr_ops.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "filemgr_ops.h" 19 | 20 | struct filemgr_ops * get_win_filemgr_ops(); 21 | struct filemgr_ops * get_linux_filemgr_ops(); 22 | 23 | struct filemgr_ops * get_filemgr_ops() 24 | { 25 | #if defined(WIN32) || defined(_WIN32) 26 | // windows 27 | return get_win_filemgr_ops(); 28 | #else 29 | // linux, mac os x 30 | return get_linux_filemgr_ops(); 31 | #endif 32 | } 33 | -------------------------------------------------------------------------------- /src/breakpad.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #pragma once 18 | 19 | /** 20 | * Initialize breakpad based on the specified minidump_dir 21 | * 22 | * The function may be called multiple times and allow for reconfiguration 23 | * of the breakpad's minidump_dir. 24 | */ 25 | void initialize_breakpad(const char* minidump_dir); 26 | 27 | /** 28 | * Cleaning up when breakpad no longer needed 29 | * (Assuming it is enabled and has been initialized) 30 | */ 31 | void destroy_breakpad(void); 32 | -------------------------------------------------------------------------------- /src/btree_var_kv_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_BTREE_VAR_KV_OPS 19 | #define _FDB_BTREE_VAR_KV_OPS 20 | 21 | #include "btree_fast_str_kv.h" 22 | #define _get_var_kv_ops btree_fast_str_kv_get_kb64_vb64 23 | #define _get_var_key btree_fast_str_kv_get_key 24 | #define _set_var_key btree_fast_str_kv_set_key 25 | #define _set_var_inf_key btree_fast_str_kv_set_inf_key 26 | #define _is_inf_key btree_fast_str_kv_is_inf_key 27 | #define _free_var_key btree_fast_str_kv_free_key 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/hash_functions.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_HASH_FUNCTIONS_H 19 | #define _JSAHN_HASH_FUNCTIONS_H 20 | 21 | #include 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | uint32_t hash_djb2(uint8_t *value, int len); 28 | uint32_t hash_djb2_last8(uint8_t *value, int len); 29 | uint32_t hash_uint_modular(uint64_t value, uint64_t mod); 30 | uint32_t hash_shuffle_2uint(uint64_t a, uint64_t b); 31 | 32 | #ifdef __cplusplus 33 | } 34 | #endif 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /cmake/Modules/FindJemalloc.cmake: -------------------------------------------------------------------------------- 1 | # Locate jemalloc libraries on a host OS. 2 | 3 | IF (UNIX) 4 | FIND_PATH(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h 5 | PATH_SUFFIXES include 6 | PATHS 7 | ~/Library/Frameworks 8 | /Library/Frameworks 9 | /usr/local 10 | /opt/local 11 | /opt/csw 12 | /opt) 13 | 14 | FIND_LIBRARY(JEMALLOC_LIBRARIES 15 | NAMES jemalloc 16 | PATHS 17 | ~/Library/Frameworks 18 | /Library/Frameworks 19 | /usr/local 20 | /opt/local 21 | /opt/csw 22 | /opt) 23 | ELSEIF (WIN32) 24 | ENDIF() 25 | 26 | IF (JEMALLOC_LIBRARIES) 27 | MESSAGE(STATUS "Found jemalloc libraries in ${JEMALLOC_INCLUDE_DIR} : 28 | ${JEMALLOC_LIBRARIES}") 29 | ADD_DEFINITIONS(-D_JEMALLOC=1) 30 | set(MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) 31 | include_directories(AFTER ${JEMALLOC_INCLUDE_DIR}) 32 | MARK_AS_ADVANCED(MALLOC_INCLUDE_DIR JEMALLOC_LIBRARIES) 33 | ELSE (JEMALLOC_LIBRARIES) 34 | MESSAGE(FATAL_ERROR "Can't find jemalloc libraries") 35 | ENDIF (JEMALLOC_LIBRARIES) 36 | -------------------------------------------------------------------------------- /src/configuration.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _CONFIGURATION_H 19 | #define _CONFIGURATION_H 20 | 21 | #include 22 | 23 | #include "common.h" 24 | #include "internal_types.h" 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | fdb_config get_default_config(void); 31 | fdb_kvs_config get_default_kvs_config(void); 32 | 33 | bool validate_fdb_config(fdb_config *fconfig); 34 | bool validate_fdb_kvs_config(fdb_kvs_config *kvs_config); 35 | 36 | #ifdef __cplusplus 37 | } 38 | #endif 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/filemgr_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_FILEMGR_OPS 19 | #define _JSAHN_FILEMGR_OPS 20 | 21 | #include "libforestdb/fdb_types.h" 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | struct filemgr_ops * get_filemgr_ops(); 28 | 29 | static inline int handle_to_fd(fdb_fileops_handle handle) { 30 | return (int)(intptr_t)handle; 31 | } 32 | 33 | static inline fdb_fileops_handle fd_to_handle(int fd) { 34 | return (fdb_fileops_handle)(intptr_t)fd; 35 | } 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /cmake/Modules/FindAsyncIOLib.cmake: -------------------------------------------------------------------------------- 1 | # Locate async I/O libraries on a host OS. 2 | 3 | IF (UNIX) 4 | FIND_PATH(ASYNC_IO_INCLUDE_DIR libaio.h 5 | PATH_SUFFIXES include 6 | PATHS 7 | ~/Library/Frameworks 8 | /Library/Frameworks 9 | /usr/local 10 | /opt/local 11 | /opt/csw 12 | /opt) 13 | 14 | FIND_LIBRARY(ASYNC_IO_LIBRARIES 15 | NAMES aio 16 | PATHS 17 | ~/Library/Frameworks 18 | /Library/Frameworks 19 | /usr/local 20 | /opt/local 21 | /opt/csw 22 | /opt) 23 | ELSEIF (WIN32) 24 | ENDIF() 25 | 26 | IF (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 27 | MESSAGE(STATUS "Found async I/O libraries in ${ASYNC_IO_INCLUDE_DIR} : 28 | ${ASYNC_IO_LIBRARIES}") 29 | ADD_DEFINITIONS(-D_ASYNC_IO=1) 30 | set(ASYNC_IO_LIB ${ASYNC_IO_LIBRARIES}) 31 | include_directories(AFTER ${ASYNC_IO_INCLUDE_DIR}) 32 | MARK_AS_ADVANCED(ASYNC_IO_INCLUDE_DIR ASYNC_IO_LIBRARIES) 33 | ELSE (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 34 | MESSAGE(STATUS "Can't find async I/O libraries") 35 | ENDIF (ASYNC_IO_LIBRARIES AND ASYNC_IO_INCLUDE_DIR) 36 | -------------------------------------------------------------------------------- /cmake/Modules/FindSnappy.cmake: -------------------------------------------------------------------------------- 1 | # Locate snappy library 2 | # This module defines 3 | # SNAPPY_FOUND, if false, do not try to link with snappy 4 | # LIBSNAPPY, Library path and libs 5 | # SNAPPY_INCLUDE_DIR, where to find the ICU headers 6 | 7 | FIND_PATH(SNAPPY_INCLUDE_DIR snappy.h 8 | HINTS 9 | ENV SNAPPY_DIR 10 | PATH_SUFFIXES include 11 | PATHS 12 | ~/Library/Frameworks 13 | /Library/Frameworks 14 | /usr/local 15 | /opt/local 16 | /opt/csw 17 | /opt/snappy 18 | /opt) 19 | 20 | FIND_LIBRARY(SNAPPY_LIBRARIES 21 | NAMES snappy 22 | HINTS 23 | ENV SNAPPY_DIR 24 | PATHS 25 | ~/Library/Frameworks 26 | /Library/Frameworks 27 | /usr/local 28 | /opt/local 29 | /opt/csw 30 | /opt/snappy 31 | /opt) 32 | 33 | IF (SNAPPY_LIBRARIES) 34 | include_directories(AFTER ${SNAPPY_INCLUDE_DIR}) 35 | MESSAGE(STATUS "Found snappy in ${SNAPPY_INCLUDE_DIR} : ${SNAPPY_LIBRARIES}") 36 | ELSE (SNAPPY_LIBRARIES) 37 | MESSAGE(FATAL_ERROR "Can't build forestdb without Snappy") 38 | ENDIF (SNAPPY_LIBRARIES) 39 | 40 | MARK_AS_ADVANCED(SNAPPY_INCLUDE_DIR SNAPPY_LIBRARIES) 41 | -------------------------------------------------------------------------------- /utils/gettimeofday_vs.cc: -------------------------------------------------------------------------------- 1 | #include "gettimeofday_vs.h" 2 | 3 | #if defined(_MSC_VER) || defined(_MSC_EXTENSIONS) 4 | #define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 5 | #else 6 | #define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL 7 | #endif 8 | 9 | #ifndef _PLATFORM_LIB_AVAILABLE 10 | // In case of Couchbase Server Builds, platform library 11 | // is included, which already contains the following 12 | // definition. 13 | int gettimeofday_vs(struct timeval *tv, void *_tz) 14 | { 15 | FILETIME ft; 16 | unsigned __int64 tmpres = 0; 17 | static int tzflag; 18 | struct timezone *tz = (struct timezone *)_tz; 19 | 20 | if (NULL != tv) { 21 | GetSystemTimeAsFileTime(&ft); 22 | 23 | tmpres |= ft.dwHighDateTime; 24 | tmpres <<= 32; 25 | tmpres |= ft.dwLowDateTime; 26 | 27 | /*converting file time to unix epoch*/ 28 | tmpres -= DELTA_EPOCH_IN_MICROSECS; 29 | tmpres /= 10; /*convert into microseconds*/ 30 | tv->tv_sec = (long)(tmpres / 1000000UL); 31 | tv->tv_usec = (long)(tmpres % 1000000UL); 32 | } 33 | 34 | if (NULL != tz) { 35 | if (!tzflag) { 36 | _tzset(); 37 | tzflag++; 38 | } 39 | tz->tz_minuteswest = _timezone / 60; 40 | tz->tz_dsttime = _daylight; 41 | } 42 | 43 | return 0; 44 | } 45 | #endif // _PLATFORM_LIB_AVAILABLE 46 | -------------------------------------------------------------------------------- /tests/functional/functional_util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FUNCTIONAL_UTIL_H 19 | #define _FUNCTIONAL_UTIL_H 20 | 21 | #include 22 | #include "common.h" 23 | #include "filemgr_ops.h" 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | void _set_random_string(char *str, int len); 30 | 31 | void _set_random_string_smallabt(char *str, int len); 32 | 33 | int _disk_dump(const char *filepath, const size_t pos, const size_t bytes); 34 | 35 | void logCallbackFunc(int err_code, 36 | const char *err_msg, 37 | void *pCtxData); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/btree_str_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_STR_KV_H 19 | #define _JSAHN_BTREE_STR_KV_H 20 | 21 | #include 22 | #include "common.h" 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | void btree_str_kv_set_key(void *key, void *str, size_t len); 29 | void btree_str_kv_set_inf_key(void *key); 30 | int btree_str_kv_is_inf_key(void *key); 31 | void btree_str_kv_get_key(void *key, void *strbuf, size_t *len); 32 | void btree_str_kv_free_key(void *key); 33 | 34 | struct btree_kv_ops; 35 | struct btree_kv_ops *btree_str_kv_get_kb64_vb64(struct btree_kv_ops *kv_ops); 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/task_priority.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "common.h" 19 | 20 | #include "task_priority.h" 21 | 22 | // Priorities for Read-only IO tasks 23 | 24 | // Priorities for Auxiliary IO tasks 25 | 26 | // Priorities for Read-Write IO tasks 27 | const Priority Priority::CompactorPriority(COMPACTOR_ID, 2); 28 | const Priority Priority::BgFlusherPriority(BGFLUSHER_ID, 1); 29 | 30 | // Priorities for NON-IO tasks 31 | 32 | const char *Priority::getTypeName(const type_id_t i) { 33 | switch (i) { 34 | case COMPACTOR_ID: 35 | return "compactor_tasks"; 36 | case BGFLUSHER_ID: 37 | return "bgflusher_tasks"; 38 | default: break; 39 | } 40 | 41 | return "error"; 42 | } 43 | -------------------------------------------------------------------------------- /tests/usecase/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(usecase_test 2 | usecase_test.cc 3 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 4 | $ 5 | $ 6 | $ 7 | ${GETTIMEOFDAY_VS}) 8 | target_link_libraries(usecase_test ${PTHREAD_LIB} ${LIBM} 9 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 10 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 11 | ${LIBRT} ${CRYPTO_LIB} 12 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 13 | set_target_properties(usecase_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 14 | 15 | add_executable(endurance_test 16 | endurance_test.cc 17 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 18 | $ 19 | $ 20 | $ 21 | ${GETTIMEOFDAY_VS}) 22 | target_link_libraries(endurance_test ${PTHREAD_LIB} ${LIBM} 23 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 24 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 25 | ${LIBRT} ${CRYPTO_LIB} 26 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 27 | set_target_properties(endurance_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 28 | 29 | # add test target 30 | add_test(usecase_test usecase_test) 31 | add_test(endurance_test endurance_test) 32 | 33 | ADD_CUSTOM_TARGET(usecase_tests 34 | COMMAND ctest 35 | ) 36 | -------------------------------------------------------------------------------- /src/globaltask.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #include "globaltask.h" 21 | 22 | std::atomic GlobalTask::task_id_counter(1); 23 | 24 | GlobalTask::GlobalTask(Taskable& t, const Priority &p, 25 | double sleeptime, bool completeBeforeShutdown) : 26 | RCValue(), priority(p), 27 | blockShutdown(completeBeforeShutdown), 28 | state(TASK_RUNNING), taskId(nextTaskId()), taskable(t) { 29 | snooze(sleeptime); 30 | } 31 | 32 | void GlobalTask::snooze(const double secs) { 33 | if (secs == INT_MAX) { 34 | setState(TASK_SNOOZED, TASK_RUNNING); 35 | updateWaketime(hrtime_t(-1)); 36 | return; 37 | } 38 | 39 | hrtime_t curTime = gethrtime(); 40 | if (secs) { 41 | setState(TASK_SNOOZED, TASK_RUNNING); 42 | waketime.store(curTime + hrtime_t(secs * 1000000000)); 43 | } else { 44 | waketime.store(curTime); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/sync_object.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | 23 | #include "atomic.h" 24 | 25 | /** 26 | * Abstraction built on top of std::condition_variable & std::mutex 27 | */ 28 | class SyncObject : public std::mutex { 29 | public: 30 | SyncObject() { 31 | } 32 | 33 | ~SyncObject() { 34 | } 35 | 36 | void wait(UniqueLock& lock) { 37 | cond.wait(lock); 38 | } 39 | 40 | void wait_for(UniqueLock& lock, 41 | const double secs) { 42 | cond.wait_for(lock, std::chrono::milliseconds(int64_t(secs * 1000.0))); 43 | } 44 | 45 | void wait_for(UniqueLock& lock, 46 | const uint64_t nanoSecs) { 47 | cond.wait_for(lock, std::chrono::nanoseconds(nanoSecs)); 48 | } 49 | 50 | void notify_all() { 51 | cond.notify_all(); 52 | } 53 | 54 | void notify_one() { 55 | cond.notify_one(); 56 | } 57 | 58 | private: 59 | std::condition_variable cond; 60 | 61 | DISALLOW_COPY_AND_ASSIGN(SyncObject); 62 | }; 63 | 64 | -------------------------------------------------------------------------------- /tests/fdbench-micro/config.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifdef __sun 19 | #include 20 | #endif 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include "stat_aggregator.h" 28 | 29 | //#define __DEBUG_E2E 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | static const char BENCHDB_NAME[] = "fdb_bench_dbfile"; 36 | static const char BENCHKV_NAME[] = "fdb_bench_kv"; 37 | static const int KEY_SIZE = 16; 38 | static const int PERMUTED_BYTES = 4; 39 | 40 | // custom stats 41 | static const char ST_ITR_INIT[] = "iterator_init"; 42 | static const char ST_ITR_GET[] = "iterator_get"; 43 | static const char ST_ITR_NEXT[] = "iterator_next"; 44 | static const char ST_ITR_CLOSE[] = "iterator_close"; 45 | 46 | struct reader_context { 47 | fdb_kvs_handle *handle; 48 | stat_history_t *stat_itr_init; 49 | stat_history_t *stat_itr_get; 50 | stat_history_t *stat_itr_next; 51 | stat_history_t *stat_itr_close; 52 | }; 53 | 54 | #define alca(type, n) ((type*)alloca(sizeof(type) * (n))) 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | -------------------------------------------------------------------------------- /src/encryption_bogus.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "encryption.h" 19 | 20 | // Bogus encryption for test purposes. Simply adds the same number to each byte in a block. 21 | // The number is the LSB of the block number xor'ed with the first byte of the key. 22 | 23 | fdb_status bogus_setup(encryptor *e) { 24 | return FDB_RESULT_SUCCESS; 25 | } 26 | 27 | static fdb_status bogus_crypt(encryptor *e, 28 | bool encrypt, 29 | void *dst_buf, 30 | const void *src_buf, 31 | size_t size, 32 | bid_t bid) 33 | { 34 | int8_t delta = (bid & 0xFF) ^ e->key.bytes[0]; 35 | if (!encrypt) 36 | delta = -delta; 37 | const uint8_t *src = (const uint8_t *)src_buf; 38 | uint8_t *dst = (uint8_t *)dst_buf; 39 | while (size-- > 0) { 40 | *dst++ = *src++ + delta; 41 | } 42 | return FDB_RESULT_SUCCESS; 43 | } 44 | 45 | static encryption_ops bogus_ops = { 46 | bogus_setup, 47 | bogus_crypt 48 | }; 49 | 50 | const encryption_ops* const fdb_encryption_ops_bogus = &bogus_ops; 51 | -------------------------------------------------------------------------------- /src/hash_functions.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Hash Functions 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #include "hash_functions.h" 8 | #include "common.h" 9 | 10 | // djb2 hashing using last LEN digits in VALUE 11 | uint32_t hash_djb2(uint8_t *value, int len) 12 | { 13 | unsigned hash = 5381; 14 | while(len--){ 15 | hash = ((hash << 5) + hash) + *((uint8_t*)value + len); 16 | } 17 | return hash; 18 | } 19 | 20 | uint32_t hash_djb2_last8(uint8_t *value, int len) 21 | { 22 | int min = MIN(len, 8), c; 23 | unsigned hash = 5381; 24 | c = min; 25 | while(c--){ 26 | hash = ((hash << 5) + hash) + *((uint8_t*)value + (len - min) + c); 27 | } 28 | return hash; 29 | } 30 | 31 | // LCOV_EXCL_START 32 | uint32_t hash_uint_modular(uint64_t value, uint64_t mod) 33 | { 34 | return value % mod; 35 | } 36 | // LCOV_EXCL_STOP 37 | 38 | // LCOV_EXCL_START 39 | uint32_t hash_shuffle_2uint(uint64_t a, uint64_t b) 40 | { 41 | uint32_t c; 42 | 43 | a ^= bitswap64(a ^ UINT64_C(0xffffffffffffffff)); 44 | b ^= bitswap64(b ^ UINT64_C(0xffffffffffffffff)); 45 | 46 | a = (a & 0xffff) ^ ((a & 0xffff0000) >> 16) ^ 47 | ((a & UINT64_C(0xffff00000000)) >> 32) ^ 48 | ((a & UINT64_C(0xffff000000000000)) >> 48); 49 | b = (b & 0xffff) ^ ((b & 0xffff0000) >> 16) ^ 50 | ((b & UINT64_C(0xffff00000000)) >> 32) ^ 51 | ((b & UINT64_C(0xffff000000000000)) >> 48); 52 | 53 | c = (((a & 0x0000000f) << 0) | 54 | ((b & 0x0000000f) << 4) | 55 | ((a & 0x000000f0) << 4) | 56 | ((b & 0x000000f0) << 8) | 57 | ((a & 0x00000f00) << 8) | 58 | ((b & 0x00000f00) << 12) | 59 | ((a & 0x0000f000) << 12) | 60 | ((b & 0x0000f000) << 16)); 61 | 62 | return (((c << 5) + c) << 5) + c; 63 | } 64 | // LCOV_EXCL_STOP 65 | 66 | -------------------------------------------------------------------------------- /utils/partiallock.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Generic Partial Lock 4 | * see https://github.com/greensky00/partiallock 5 | */ 6 | 7 | #ifndef _JSAHN_PARITIAL_LOCK_H 8 | #define _JSAHN_PARITIAL_LOCK_H 9 | 10 | #include 11 | #include 12 | 13 | #include "list.h" 14 | 15 | #ifdef __cplusplus 16 | extern "C" { 17 | #endif 18 | 19 | typedef uint64_t plock_range_t; 20 | typedef struct plock_node plock_entry_t; // opaque reference 21 | 22 | struct plock_ops { 23 | void (*init_user)(void *lock); 24 | void (*lock_user)(void *lock); 25 | void (*unlock_user)(void *lock); 26 | void (*destroy_user)(void *lock); 27 | void (*init_internal)(void *lock); 28 | void (*lock_internal)(void *lock); 29 | void (*unlock_internal)(void *lock); 30 | void (*destroy_internal)(void *lock); 31 | int (*is_overlapped)(void *start1, void *len1, void *start2, void *len2, void *aux); 32 | }; 33 | 34 | struct plock_config { 35 | struct plock_ops *ops; 36 | size_t sizeof_lock_user; 37 | size_t sizeof_lock_internal; 38 | size_t sizeof_range; 39 | void *aux; 40 | }; 41 | 42 | struct plock { 43 | struct list active; // list of active locks 44 | struct list inactive; // list of inactive (freed) locks 45 | struct plock_ops *ops; 46 | size_t sizeof_lock_user; 47 | size_t sizeof_lock_internal; 48 | size_t sizeof_range; 49 | void *lock; 50 | void *aux; 51 | }; 52 | 53 | #define PLOCK_RESULT_SUCCESS (0) 54 | #define PLOCK_RESULT_INVALID_ARGS (-1) 55 | #define PLOCK_RESULT_ALLOC_FAIL (-2) 56 | 57 | int plock_init(struct plock *plock, struct plock_config *config); 58 | plock_entry_t *plock_lock(struct plock *plock, void *start, void *len); 59 | int plock_unlock(struct plock *plock, plock_entry_t *plock_entry); 60 | int plock_destroy(struct plock *plock); 61 | 62 | #ifdef __cplusplus 63 | } 64 | #endif 65 | 66 | #endif 67 | 68 | -------------------------------------------------------------------------------- /src/version.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_VERSION_H 19 | #define _FDB_VERSION_H 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "libforestdb/fdb_errors.h" 23 | #include "common.h" 24 | 25 | #include "filemgr.h" 26 | 27 | INLINE filemgr_magic_t ver_get_latest_magic() { 28 | return FILEMGR_LATEST_MAGIC; 29 | } 30 | bool ver_is_valid_magic(filemgr_magic_t magic); 31 | bool ver_is_magic_000(filemgr_magic_t magic); 32 | bool ver_is_atleast_magic_001(filemgr_magic_t magic); 33 | bool ver_staletree_support(filemgr_magic_t magic); 34 | bool ver_superblock_support(filemgr_magic_t magic); 35 | bool ver_non_consecutive_doc(filemgr_magic_t magic); 36 | bool ver_btreev2_format(filemgr_magic_t magic); 37 | size_t ver_get_new_filename_off(filemgr_magic_t magic); 38 | 39 | /** 40 | * Return the version of a given file's magic value 41 | * 42 | * @param magic ForestDB file magic value 43 | * @return Version of a given file's magic value 44 | */ 45 | const char* ver_get_version_string(filemgr_magic_t magic); 46 | 47 | /** 48 | * Return the offset of last_wal_flush_header field in a commit header 49 | */ 50 | size_t ver_get_last_wal_flush_hdr_off(filemgr_magic_t magic); 51 | 52 | #endif /* _FDB_VERSION_H */ 53 | 54 | -------------------------------------------------------------------------------- /src/memory_pool.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | MemoryPool::MemoryPool(int num_bins, size_t bin_size) { 21 | spin_init(&lock); 22 | for (int i = 0; i < num_bins; ++i) { 23 | memPool.push_back((uint8_t *) malloc(bin_size)); 24 | enQueue(i); 25 | } 26 | } 27 | 28 | MemoryPool::~MemoryPool() { 29 | spin_destroy(&lock); 30 | for (auto &it : memPool) { 31 | free(it); 32 | } 33 | } 34 | 35 | const int MemoryPool::fetchBlock(uint8_t **buf) { 36 | int ret = deQueue(); 37 | if (ret == -1) { 38 | *buf = nullptr; 39 | } else { 40 | *buf = memPool.at(ret); 41 | } 42 | return ret; 43 | } 44 | 45 | void MemoryPool::returnBlock(int index) { 46 | if (index >= 0 && index < static_cast(memPool.size())) { 47 | enQueue(index); 48 | } 49 | } 50 | 51 | inline void MemoryPool::enQueue(int index) { 52 | spin_lock(&lock); 53 | indexQ.push(index); 54 | spin_unlock(&lock); 55 | } 56 | 57 | inline int MemoryPool::deQueue() { 58 | int data = -1; 59 | spin_lock(&lock); 60 | if (indexQ.size()) { 61 | data = indexQ.front(); 62 | indexQ.pop(); 63 | } 64 | spin_unlock(&lock); 65 | return data; 66 | } 67 | -------------------------------------------------------------------------------- /utils/timing.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | //#define __DEBUG_E2E 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #if !defined(WIN32) && !defined(_WIN32) 25 | #include 26 | #endif 27 | #include "time_utils.h" 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | // Forestdb APIs wrappers where time taken in nano secs is returned on success 34 | 35 | static const long int ERR_NS = 0xFFFFFFFF; 36 | ts_nsec timed_fdb_get(fdb_kvs_handle *kv, fdb_doc *doc); 37 | ts_nsec timed_fdb_set(fdb_kvs_handle *kv, fdb_doc *doc); 38 | ts_nsec timed_fdb_delete(fdb_kvs_handle *kv, fdb_doc *doc); 39 | ts_nsec timed_fdb_compact(fdb_file_handle *fhandle); 40 | ts_nsec timed_fdb_commit(fdb_file_handle *fhandle, bool walflush); 41 | ts_nsec timed_fdb_snapshot(fdb_kvs_handle *kv, fdb_kvs_handle **snap_kv); 42 | ts_nsec timed_fdb_iterator_init(fdb_kvs_handle *kv, fdb_iterator **it); 43 | ts_nsec timed_fdb_iterator_get(fdb_iterator *it, fdb_doc **doc); 44 | ts_nsec timed_fdb_iterator_next(fdb_iterator *it); 45 | ts_nsec timed_fdb_iterator_close(fdb_iterator *it); 46 | ts_nsec timed_fdb_kvs_close(fdb_kvs_handle *kv); 47 | ts_nsec timed_fdb_close(fdb_file_handle *fhandle); 48 | ts_nsec timed_fdb_shutdown(); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | -------------------------------------------------------------------------------- /utils/time_utils.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #if defined(WIN32) || defined(_WIN32) 21 | #include 22 | #include 23 | #else 24 | #include 25 | #include 26 | #endif 27 | #include 28 | #include "stdint.h" 29 | 30 | #ifndef hrtime_t 31 | #include 32 | typedef uint64_t hrtime_t; 33 | #endif //ifndef hrtime_t 34 | 35 | #ifndef _PLATFORM_LIB_AVAILABLE 36 | extern "C" hrtime_t gethrtime(void); 37 | extern "C" hrtime_t gethrtime_period(void); 38 | #endif 39 | 40 | typedef long int ts_nsec; 41 | ts_nsec get_monotonic_ts(); 42 | ts_nsec ts_diff(ts_nsec start, ts_nsec end); 43 | 44 | struct timeval _utime_gap(struct timeval a, struct timeval b); 45 | uint64_t timeval_to_us(struct timeval tv); 46 | 47 | #if defined(WIN32) || defined(_WIN32) 48 | #ifdef _PLATFORM_LIB_AVAILABLE 49 | #include 50 | #else 51 | // If platform library has not been included, usleep 52 | // needs to be explicitly defined for windows. 53 | void usleep(unsigned int useconds); 54 | #endif // _PLATFORM_LIB_AVAILABLE 55 | #endif // defined(WIN32) || defined(_WIN32) 56 | 57 | void decaying_usleep(unsigned int *sleep_time, unsigned int max_sleep_time); 58 | 59 | #if !defined(WIN32) && !defined(_WIN32) 60 | struct timespec convert_reltime_to_abstime(unsigned int ms); 61 | #endif 62 | 63 | -------------------------------------------------------------------------------- /tests/stats-agg/stat_aggregator.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | typedef struct { 26 | std::string name; 27 | std::vector latencies; 28 | } stat_history_t; 29 | 30 | struct Stats { 31 | // Stat name 32 | std::string name; 33 | // Calculated mean 34 | double mean; 35 | // Calculated median 36 | double median; 37 | // Estimated standard deviation 38 | double stddev; 39 | // Calculated 5th percentile 40 | double pct5; 41 | // Calculated 95th percentile 42 | double pct95; 43 | // Calculated 99th percentile 44 | double pct99; 45 | // Vector of samples 46 | std::vector* values; 47 | }; 48 | 49 | typedef stat_history_t** StatMatrix_t; 50 | typedef std::vector*> > samples_t; 51 | 52 | class StatAggregator { 53 | public: 54 | StatAggregator(int _num_stats, int _num_samples); 55 | 56 | ~StatAggregator(); 57 | 58 | void aggregateAndPrintStats(const char* title, int count, const char* unit); 59 | 60 | StatMatrix_t t_stats; 61 | 62 | private: 63 | 64 | void printValues(samples_t values, std::string unit); 65 | 66 | void fillLineWith(const char c, int spaces); 67 | 68 | int num_stats; 69 | int num_samples; 70 | }; 71 | -------------------------------------------------------------------------------- /src/list.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Doubly Linked List 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include "common.h" 12 | 13 | //#define _LIST_LOCK 14 | 15 | struct list_elem { 16 | struct list_elem *prev; 17 | struct list_elem *next; 18 | }; 19 | 20 | struct list { 21 | struct list_elem *head; 22 | struct list_elem *tail; 23 | #ifdef _LIST_LOCK 24 | spin_t lock; 25 | #endif 26 | }; 27 | 28 | #ifndef _get_entry 29 | #define _get_entry(ELEM, STRUCT, MEMBER) \ 30 | ((STRUCT *) ((uint8_t *) (ELEM) - offsetof (STRUCT, MEMBER))) 31 | #endif 32 | 33 | #ifdef LIST_LOCK 34 | void list_init(struct list *list); 35 | #else 36 | INLINE void list_init(struct list *list) 37 | { 38 | list->head = NULL; 39 | list->tail = NULL; 40 | } 41 | #endif 42 | 43 | void list_push_front(struct list *list, struct list_elem *e); 44 | void list_push_back(struct list *list, struct list_elem *e); 45 | void list_insert_before(struct list *list, struct list_elem *before, struct list_elem *e); 46 | void list_insert_after(struct list *list, struct list_elem *after, struct list_elem *e); 47 | 48 | struct list_elem *list_remove(struct list *list, struct list_elem *e); 49 | struct list_elem *list_remove_reverse(struct list *list, struct list_elem *e); 50 | 51 | struct list_elem *list_pop_front(struct list *list); 52 | struct list_elem *list_pop_back(struct list *list); 53 | 54 | #ifdef _LIST_LOCK 55 | struct list_elem *list_begin(struct list *list); 56 | struct list_elem *list_end(struct list *list); 57 | #else 58 | INLINE struct list_elem *list_begin(struct list *list) 59 | { 60 | return list->head; 61 | } 62 | 63 | INLINE struct list_elem *list_end(struct list *list) 64 | { 65 | return list->tail; 66 | } 67 | #endif 68 | 69 | INLINE struct list_elem *list_next(struct list_elem *e) 70 | { 71 | return e->next; 72 | } 73 | 74 | INLINE struct list_elem *list_prev(struct list_elem *e) 75 | { 76 | return e->prev; 77 | } 78 | 79 | INLINE bool list_empty(struct list *list) { 80 | return list->head == NULL; 81 | } 82 | 83 | -------------------------------------------------------------------------------- /src/hash.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Hash Table 4 | * (C) 2013 Jung-Sang Ahn 5 | */ 6 | 7 | #ifndef _JSAHN_HASH_H 8 | #define _JSAHN_HASH_H 9 | 10 | #include 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | #define _HASH_TREE 17 | #ifdef _HASH_TREE 18 | #include "avltree.h" 19 | #else 20 | #include "list.h" 21 | #endif 22 | 23 | //#define _HASH_LOCK 24 | #ifdef _HASH_LOCK 25 | #include "arch.h" 26 | #endif 27 | 28 | struct hash_elem { 29 | #ifdef _HASH_TREE 30 | struct avl_node avl; 31 | #else 32 | struct list_elem list_elem; 33 | #endif 34 | }; 35 | 36 | struct hash; 37 | 38 | typedef uint32_t hash_hash_func(struct hash *hash, struct hash_elem *e); 39 | typedef int hash_cmp_func(struct hash_elem *a, struct hash_elem *b); 40 | //typedef int hash_cmp_func(void *a, void *b); 41 | typedef void hash_free_func(struct hash_elem *e); 42 | typedef void *hash_check_func(struct hash_elem *e, void *ctx); 43 | 44 | struct hash { 45 | size_t nbuckets; 46 | #ifdef _HASH_TREE 47 | struct avl_tree *buckets; 48 | #else 49 | struct list *buckets; 50 | #endif 51 | 52 | hash_hash_func *hash_func; 53 | hash_cmp_func *cmp; 54 | #ifdef _HASH_TREE 55 | avl_cmp_func *avl_cmp; 56 | #endif 57 | 58 | #ifdef _HASH_LOCK 59 | // define locks for each bucket 60 | spin_t *locks; 61 | #endif 62 | }; 63 | 64 | void hash_init(struct hash *hash, int nbuckets, hash_hash_func *hash_func, hash_cmp_func *cmp_func); 65 | void hash_insert(struct hash *hash, struct hash_elem *e); 66 | void hash_insert_by_hash_val(struct hash *hash, struct hash_elem *e, uint32_t hash_val); 67 | struct hash_elem * hash_find(struct hash *hash, struct hash_elem *e); 68 | struct hash_elem * hash_find_by_hash_val(struct hash *hash, struct hash_elem *e, 69 | uint32_t hash_val); 70 | void *hash_scan(struct hash *hash, hash_check_func *check_func, void *ctx); 71 | void hash_remove(struct hash *hash, struct hash_elem *e); 72 | void hash_free(struct hash *hash); 73 | void hash_free_active(struct hash *hash, hash_free_func *free_func); 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /src/bgflusher.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "internal_types.h" 25 | 26 | struct bgflusher_config{ 27 | size_t num_threads; 28 | }; 29 | 30 | // Singleton Instance of Background Flusher 31 | class BgFlusher { 32 | public: 33 | static BgFlusher *getBgfInstance(); 34 | static BgFlusher *createBgFlusher(struct bgflusher_config *config); 35 | static void destroyBgFlusher(); 36 | 37 | fdb_status registerFile_BgFlusher(FileMgr *file, 38 | fdb_config *config, 39 | ErrLogCallback *log_callback); 40 | void switchFile_BgFlusher(FileMgr *old_file, 41 | FileMgr *new_file, 42 | ErrLogCallback *log_callback); 43 | void deregisterFile_BgFlusher(FileMgr *file); 44 | 45 | private: 46 | BgFlusher(size_t num_threads); 47 | ~BgFlusher(); 48 | 49 | friend void *bgflusher_thread(void *voidargs); 50 | 51 | void * bgflusherThread(); 52 | 53 | static std::atomic bgflusherInstance; 54 | static std::mutex bgfLock; 55 | 56 | size_t numBgFlusherThreads; 57 | thread_t *bgflusherThreadIds; 58 | 59 | size_t bgFlusherSleepInSecs; 60 | 61 | mutex_t syncMutex; 62 | thread_cond_t syncCond; 63 | 64 | std::atomic bgflusherTerminateSignal; 65 | 66 | struct avl_tree openFiles; 67 | 68 | DISALLOW_COPY_AND_ASSIGN(BgFlusher); 69 | }; 70 | -------------------------------------------------------------------------------- /cmake/Modules/FindEncryptionLib.cmake: -------------------------------------------------------------------------------- 1 | # Locate the encryption library given by the build environment 2 | 3 | IF ("${_ENCRYPTION}" STREQUAL "commoncrypto") 4 | IF (APPLE) 5 | ADD_DEFINITIONS(-D_CRYPTO_CC=1) 6 | ELSE (APPLE) 7 | MESSAGE(FATAL_ERROR "commoncrypto lib is only available in Apple systems") 8 | ENDIF (APPLE) 9 | 10 | ELSEIF ("${_ENCRYPTION}" STREQUAL "openssl") 11 | INCLUDE(FindOpenSSL) 12 | 13 | IF (OPENSSL_FOUND) 14 | include_directories(AFTER ${OPENSSL_INCLUDE_DIR}) 15 | MESSAGE(STATUS "Found openssl in ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") 16 | ELSE (OPENSSL_FOUND) 17 | MESSAGE(FATAL_ERROR "Can't find openssl library") 18 | ENDIF (OPENSSL_FOUND) 19 | 20 | MARK_AS_ADVANCED(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES) 21 | ADD_DEFINITIONS(-D_CRYPTO_OPENSSL=1) 22 | set(CRYPTO_LIB ${OPENSSL_CRYPTO_LIBRARY}) 23 | 24 | ELSEIF ("${_ENCRYPTION}" STREQUAL "libtomcrypt") 25 | FIND_PATH(LIBTOMCRYPT_INCLUDE_DIR tomcrypt.h 26 | PATH_SUFFIXES include 27 | PATHS 28 | ~/Library/Frameworks 29 | /Library/Frameworks 30 | /usr/local 31 | /usr 32 | /opt/local 33 | /opt/csw 34 | /opt) 35 | 36 | FIND_LIBRARY(LIBTOMCRYPT_LIBRARIES 37 | NAMES tomcrypt 38 | PATHS 39 | ~/Library/Frameworks 40 | /Library/Frameworks 41 | /usr/local 42 | /usr 43 | /opt/local 44 | /opt/csw 45 | /opt) 46 | 47 | IF (LIBTOMCRYPT_LIBRARIES) 48 | include_directories(AFTER ${LIBTOMCRYPT_INCLUDE_DIR}) 49 | MESSAGE(STATUS "Found libtomcrypt in ${LIBTOMCRYPT_INCLUDE_DIR} : ${LIBTOMCRYPT_LIBRARIES}") 50 | ELSE (LIBTOMCRYPT_LIBRARIES) 51 | MESSAGE(FATAL_ERROR "Can't find libtomcrypt library") 52 | ENDIF (LIBTOMCRYPT_LIBRARIES) 53 | 54 | MARK_AS_ADVANCED(LIBTOMCRYPT_INCLUDE_DIR LIBTOMCRYPT_LIBRARIES) 55 | ADD_DEFINITIONS(-D_CRYPTO_LIBTOMCRYPT=1) 56 | set(CRYPTO_LIB ${LIBTOMCRYPT_LIBRARIES}) 57 | 58 | ELSE() 59 | MESSAGE(FATAL_ERROR "Can't find the cryto library ${_ENCRYPTION}") 60 | ENDIF() 61 | -------------------------------------------------------------------------------- /cmake/Modules/MemoryCheck.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # - Enable Valgrind Check 3 | # 4 | # Build a Valgrind build: 5 | # cmake -DCMAKE_BUILD_TYPE=Valgrind .. 6 | # make 7 | # make _targetname 8 | # 9 | # 10 | 11 | # Check prereqs 12 | FIND_PROGRAM( VALGRIND_PATH valgrind ) 13 | 14 | IF(NOT VALGRIND_PATH) 15 | MESSAGE(FATAL_ERROR "valgrind not found! Aborting...") 16 | ENDIF() # NOT VALGRIND_PATH 17 | 18 | SET(VALGRIND_OPTIONS "") 19 | 20 | SET(CMAKE_CXX_FLAGS_MEMCHECK 21 | "-g -O0 -fprofile-arcs " 22 | CACHE STRING "Flags used by the C++ compiler during valgrind builds." 23 | FORCE ) 24 | SET(CMAKE_C_FLAGS_MEMCHECK 25 | "-g -O0 -fprofile-arcs" 26 | CACHE STRING "Flags used by the C compiler during valgrind builds." 27 | FORCE ) 28 | SET(CMAKE_EXE_LINKER_FLAGS_MEMCHECK 29 | "" 30 | CACHE STRING "Flags used for linking binaries during valgrind builds." 31 | FORCE ) 32 | SET(CMAKE_SHARED_LINKER_FLAGS_MEMCHECK 33 | "" 34 | CACHE STRING "Flags used by the shared libraries linker during valgrind builds." 35 | FORCE ) 36 | MARK_AS_ADVANCED( 37 | CMAKE_CXX_FLAGS_MEMCHECK 38 | CMAKE_C_FLAGS_MEMCHECK 39 | CMAKE_EXE_LINKER_FLAGS_MEMCHECK 40 | CMAKE_SHARED_LINKER_FLAGS_MEMCHECK ) 41 | 42 | IF ( NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "Valgrind")) 43 | MESSAGE( WARNING "Valgrind results with an optimized (non-Debug) build may be misleading" ) 44 | ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" 45 | 46 | 47 | # Param _targetname The name of new the custom make target 48 | # Param list of target tests 49 | FUNCTION(SETUP_TARGET_FOR_MEMCHECK _targetname _test) 50 | 51 | IF(NOT VALGRIND_PATH) 52 | MESSAGE(FATAL_ERROR "valgrind not found! Aborting...") 53 | ENDIF() # NOT VALGRIND_PATH 54 | 55 | # Setup target 56 | ADD_CUSTOM_TARGET(${_targetname} 57 | 58 | COMMENT "Test: ${I}" 59 | 60 | # Run tests 61 | COMMAND ${VALGRIND_PATH} --trace-children=yes --quiet --tool=memcheck 62 | --leak-check=yes --show-reachable=yes 63 | --num-callers=100 --verbose --demangle=yes 64 | ${_test} 65 | 66 | COMMENT "Valgrind run complete for ${_test}" 67 | ) 68 | 69 | ENDFUNCTION() # SETUP_TARGET_FOR_MEMCHECK 70 | -------------------------------------------------------------------------------- /src/breakpad_linux.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | #include "client/linux/handler/exception_handler.h" 21 | 22 | #include 23 | #include 24 | 25 | using namespace google_breakpad; 26 | static ExceptionHandler* handler = nullptr; 27 | 28 | /* Called when an exception triggers a dump, outputs details to caller's logs */ 29 | static bool dumpCallback(const MinidumpDescriptor& descriptor, 30 | void* context, bool succeeded) { 31 | fprintf(stderr, "Breakpad caught a crash in forestdb. Writing crash dump " 32 | "to %s before terminating.\n", descriptor.path()); 33 | 34 | return succeeded; 35 | } 36 | 37 | static void create_breakpad(const char* minidump_dir) { 38 | MinidumpDescriptor descriptor(minidump_dir); 39 | handler = new ExceptionHandler(descriptor, 40 | /*filter*/nullptr, 41 | dumpCallback, 42 | /*callback-context*/nullptr, 43 | /*install_handler*/true, 44 | /*server_fd*/-1); 45 | } 46 | 47 | void initialize_breakpad(const char* minidump_dir) { 48 | // We cannot actually change any of breakpad's setings once created, only 49 | // remove it and re-create with new settings. 50 | destroy_breakpad(); 51 | 52 | if (minidump_dir != nullptr && minidump_dir[0] != '\0') { 53 | create_breakpad(minidump_dir); 54 | } 55 | } 56 | 57 | void destroy_breakpad(void) { 58 | delete handler; 59 | handler = nullptr; 60 | } 61 | -------------------------------------------------------------------------------- /src/taskable.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | An abstract baseclass for classes which wish to create and own tasks. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "workload.h" 25 | #include "task_priority.h" 26 | 27 | /* 28 | A type for identifying all tasks belonging to a task owner. 29 | */ 30 | typedef uintptr_t task_gid_t; 31 | 32 | class Taskable { 33 | public: 34 | /* 35 | Return a name for the task, used for logging 36 | */ 37 | virtual const std::string& getName() const = 0; 38 | 39 | /* 40 | Return a 'group' ID for the task. 41 | 42 | The use-case here is to allow the lookup of all tasks 43 | owned by this taskable. 44 | 45 | The address of the object is a safe GID. 46 | */ 47 | virtual task_gid_t getGID() const = 0; 48 | 49 | /* 50 | Return the workload priority for the task 51 | */ 52 | virtual bucket_priority_t getWorkloadPriority() const = 0; 53 | 54 | /* 55 | Set the taskable object's workload priority. 56 | */ 57 | virtual void setWorkloadPriority(bucket_priority_t prio) = 0; 58 | 59 | /* 60 | Return the taskable object's workload policy. 61 | */ 62 | virtual WorkLoadPolicy& getWorkLoadPolicy() = 0; 63 | 64 | /* 65 | Called with the time spent queued 66 | */ 67 | virtual void logQTime(type_id_t taskType, hrtime_t enqTime) = 0; 68 | 69 | /* 70 | Called with the time spent running 71 | */ 72 | virtual void logRunTime(type_id_t taskType, hrtime_t runTime) = 0; 73 | 74 | protected: 75 | virtual ~Taskable() {} 76 | }; 77 | -------------------------------------------------------------------------------- /tests/functional/functional_util.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "functional_util.h" 19 | #include "filemgr.h" 20 | 21 | void _set_random_string(char *str, int len) 22 | { 23 | str[len--] = 0; 24 | do { 25 | str[len] = '!' + random('~'-'!'); 26 | } while (len--); 27 | } 28 | 29 | void _set_random_string_smallabt(char *str, int len) 30 | { 31 | str[len--] = 0; 32 | do { 33 | str[len] = 'a' + random('z'-'a'); 34 | } while (len--); 35 | } 36 | 37 | int _disk_dump(const char *filepath, size_t pos, size_t bytes) { 38 | struct filemgr_ops *ops = get_filemgr_ops(); 39 | fdb_fileops_handle fileops_handle; 40 | fdb_status fs = FileMgr::fileOpen(filepath, ops, &fileops_handle, 41 | O_CREAT| O_RDWR, 0666); 42 | if (fs != FDB_RESULT_SUCCESS) { 43 | fprintf(stderr, "failure to open %s\n", filepath); 44 | return (int)fs; 45 | } 46 | char *buf = (char *)malloc(bytes); 47 | if (!buf) { 48 | FileMgr::fileClose(ops, fileops_handle); 49 | return -2; 50 | } 51 | if (ops->pwrite(fileops_handle, buf, bytes, pos) != (int) bytes) { 52 | FileMgr::fileClose(ops, fileops_handle); 53 | return -1; 54 | } 55 | FileMgr::fileClose(ops, fileops_handle); 56 | free(buf); 57 | return (int)fs; 58 | } 59 | 60 | void logCallbackFunc(int err_code, 61 | const char *err_msg, 62 | void *pCtxData) { 63 | fprintf(stderr, "%s - error code: %d, error message: %s\n", 64 | (char *) pCtxData, err_code, err_msg); 65 | } 66 | 67 | -------------------------------------------------------------------------------- /src/workload.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2013 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_WORKLOAD_H_ 19 | #define SRC_WORKLOAD_H_ 1 20 | 21 | #include "common.h" 22 | 23 | #include 24 | 25 | #include 26 | 27 | enum bucket_priority_t { 28 | HIGH_BUCKET_PRIORITY=6, 29 | LOW_BUCKET_PRIORITY=2, 30 | NO_BUCKET_PRIORITY=0 31 | }; 32 | 33 | enum workload_pattern_t { 34 | READ_HEAVY, 35 | WRITE_HEAVY, 36 | MIXED 37 | }; 38 | 39 | /** 40 | * Workload optimization policy 41 | */ 42 | class WorkLoadPolicy { 43 | public: 44 | WorkLoadPolicy(int m, int s) 45 | : maxNumWorkers(m), maxNumShards(s), workloadPattern(READ_HEAVY) { } 46 | 47 | size_t getNumShards(void) { 48 | return maxNumShards; 49 | } 50 | 51 | bucket_priority_t getBucketPriority(void) { 52 | if (maxNumWorkers < HIGH_BUCKET_PRIORITY) { 53 | return LOW_BUCKET_PRIORITY; 54 | } 55 | return HIGH_BUCKET_PRIORITY; 56 | } 57 | 58 | size_t getNumWorkers(void) { 59 | return maxNumWorkers; 60 | } 61 | 62 | workload_pattern_t getWorkLoadPattern(void) { 63 | return workloadPattern.load(); 64 | } 65 | 66 | std::string stringOfWorkLoadPattern(void) { 67 | switch (workloadPattern.load()) { 68 | case READ_HEAVY: 69 | return "read_heavy"; 70 | case WRITE_HEAVY: 71 | return "write_heavy"; 72 | default: 73 | return "mixed"; 74 | } 75 | } 76 | 77 | void setWorkLoadPattern(workload_pattern_t pattern) { 78 | workloadPattern.store(pattern); 79 | } 80 | 81 | private: 82 | 83 | int maxNumWorkers; 84 | int maxNumShards; 85 | std::atomic workloadPattern; 86 | }; 87 | 88 | #endif // SRC_WORKLOAD_H_ 89 | -------------------------------------------------------------------------------- /src/avltree.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * AVL Tree 4 | * (C) 2014 Jung-Sang Ahn 5 | * see https://github.com/greensky00/avltree 6 | */ 7 | 8 | #ifndef _JSAHN_AVL_TREE_H 9 | #define _JSAHN_AVL_TREE_H 10 | 11 | #include "stddef.h" 12 | #include "stdint.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | struct avl_node { 19 | struct avl_node *parent, *left, *right; 20 | 21 | #ifdef _AVL_SEPARATE_PARENT_BF 22 | int bf; 23 | #endif 24 | #ifdef _AVL_NEXT_POINTER 25 | struct avl_node *prev, *next; 26 | #endif 27 | }; 28 | 29 | struct avl_tree{ 30 | struct avl_node *root; 31 | void *aux; 32 | }; 33 | 34 | #ifndef _get_entry 35 | #define _get_entry(ELEM, STRUCT, MEMBER) \ 36 | ((STRUCT *) ((uint8_t *) (ELEM) - offsetof (STRUCT, MEMBER))) 37 | #endif 38 | 39 | #define avl_parent(node) \ 40 | ((struct avl_node *)((uint64_t)(node)->parent & ~0x3)) 41 | 42 | #ifdef _AVL_SEPARATE_PARENT_BF 43 | #define avl_bf(node) ((node)->bf) 44 | #else 45 | #define avl_bf(node) (((int)((uint64_t)(node)->parent & 0x3)) - 1) 46 | #endif 47 | 48 | // *a < *b : return neg 49 | // *a == *b : return 0 50 | // *a > *b : return pos 51 | typedef int avl_cmp_func (struct avl_node *a, struct avl_node *b, void *aux); 52 | 53 | void avl_init(struct avl_tree *tree, void *aux); 54 | void avl_set_aux(struct avl_tree *tree, void *aux); 55 | struct avl_node* avl_insert(struct avl_tree *tree, 56 | struct avl_node *node, 57 | avl_cmp_func *func); 58 | struct avl_node* avl_search(struct avl_tree *tree, 59 | struct avl_node *node, 60 | avl_cmp_func *func); 61 | struct avl_node* avl_search_greater(struct avl_tree *tree, 62 | struct avl_node *node, 63 | avl_cmp_func *func); 64 | struct avl_node* avl_search_smaller(struct avl_tree *tree, 65 | struct avl_node *node, 66 | avl_cmp_func *func); 67 | void avl_remove(struct avl_tree *tree, 68 | struct avl_node *node); 69 | struct avl_node* avl_first(struct avl_tree *tree); 70 | struct avl_node* avl_last(struct avl_tree *tree); 71 | struct avl_node* avl_next(struct avl_node *node); 72 | struct avl_node* avl_prev(struct avl_node *node); 73 | #ifdef __cplusplus 74 | } 75 | #endif 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /utils/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #if !defined(WIN32) && !defined(_WIN32) 37 | #include 38 | #endif 39 | #include 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | 46 | typedef struct _dictionary_ { 47 | /** Number of entries in dictionary */ 48 | int n; 49 | /** Storage size */ 50 | int size; 51 | /** List of string values */ 52 | char **val; 53 | /** List of string keys */ 54 | char **key ; 55 | /** List of hash values for keys */ 56 | unsigned *hash; 57 | } dictionary ; 58 | 59 | 60 | /* generated by genproto */ 61 | 62 | dictionary * iniparser_new(char *ininame); 63 | void iniparser_free(dictionary * d); 64 | 65 | int iniparser_getnsec(dictionary * d); 66 | char * iniparser_getsecname(dictionary * d, int n); 67 | void iniparser_dump(dictionary * d, FILE * f); 68 | void iniparser_dump_ini(dictionary * d, FILE * f); 69 | char * iniparser_getkey(dictionary *d, char *section, char *key); 70 | char * iniparser_getstr(dictionary * d, char * key); 71 | char * iniparser_getstring(dictionary * d, char * key, char * def); 72 | int iniparser_getint(dictionary * d, char * key, int notfound); 73 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 74 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 75 | int iniparser_find_entry(dictionary * ini, char * entry); 76 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 77 | void iniparser_unset(dictionary * ini, char * entry); 78 | 79 | #ifdef __cplusplus 80 | } 81 | #endif 82 | 83 | #endif 84 | 85 | -------------------------------------------------------------------------------- /tests/unit/filemgr_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "filemgr.h" 23 | #include "filemgr_ops.h" 24 | #include "test.h" 25 | 26 | void basic_test(fdb_encryption_algorithm_t encryption) 27 | { 28 | TEST_INIT(); 29 | 30 | FileMgr *file; 31 | FileMgrConfig config(4096, 1024, 1048576, 0, 0, FILEMGR_CREATE, 32 | FDB_SEQTREE_NOT_USE, 0, 8, 0, encryption, 33 | 0x55, 0, 0); 34 | const char *dbheader = "dbheader"; 35 | const char *dbheader2 = "dbheader2222222222"; 36 | char buf[256]; 37 | 38 | std::string fname("./filemgr_testfile"); 39 | filemgr_open_result result = FileMgr::open(fname, 40 | get_filemgr_ops(), 41 | &config, NULL); 42 | result = FileMgr::open(fname, get_filemgr_ops(), &config, NULL); 43 | file = result.file; 44 | 45 | file->updateHeader((void*)dbheader, strlen(dbheader)+1); 46 | 47 | FileMgr::close(file, true, NULL, NULL); 48 | result = FileMgr::open(fname, get_filemgr_ops(), &config, NULL); 49 | file = result.file; 50 | 51 | memcpy(buf, file->accessHeader()->data, file->accessHeader()->size); 52 | printf("%s\n", buf); 53 | 54 | file->updateHeader((void*)dbheader2, strlen(dbheader2) + 1); 55 | 56 | FileMgr::close(file, true, NULL, NULL); 57 | 58 | sprintf(buf, "basic test, encryption=%d", (int)encryption); 59 | TEST_RESULT(buf); 60 | } 61 | 62 | void mt_init_test() 63 | { 64 | TEST_INIT(); 65 | 66 | TEST_RESULT("multi threaded initialization test"); 67 | } 68 | 69 | int main() 70 | { 71 | int r = system(SHELL_DEL" filemgr_testfile"); 72 | (void)r; 73 | 74 | basic_test(FDB_ENCRYPTION_NONE); 75 | basic_test(FDB_ENCRYPTION_BOGUS); 76 | mt_init_test(); 77 | 78 | return 0; 79 | } 80 | -------------------------------------------------------------------------------- /cmake/Modules/ThreadSanitizer.cmake: -------------------------------------------------------------------------------- 1 | # Support for building with ThreadSanitizer (tsan) - 2 | # https://code.google.com/p/thread-sanitizer/ 3 | 4 | INCLUDE(CheckCCompilerFlag) 5 | INCLUDE(CheckCXXCompilerFlag) 6 | INCLUDE(CMakePushCheckState) 7 | 8 | OPTION(CB_THREADSANITIZER "Enable ThreadSanitizer data race detector." 9 | OFF) 10 | 11 | IF (CB_THREADSANITIZER) 12 | CMAKE_PUSH_CHECK_STATE(RESET) 13 | SET(CMAKE_REQUIRED_FLAGS "-fsanitize=thread") # Also needs to be a link flag for test to pass 14 | CHECK_C_COMPILER_FLAG("-fsanitize=thread" HAVE_FLAG_SANITIZE_THREAD_C) 15 | CHECK_CXX_COMPILER_FLAG("-fsanitize=thread" HAVE_FLAG_SANITIZE_THREAD_CXX) 16 | CMAKE_POP_CHECK_STATE() 17 | 18 | IF(HAVE_FLAG_SANITIZE_THREAD_C AND HAVE_FLAG_SANITIZE_THREAD_CXX) 19 | SET(THREAD_SANITIZER_FLAG "-fsanitize=thread") 20 | 21 | SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${THREAD_SANITIZER_FLAG}") 22 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${THREAD_SANITIZER_FLAG}") 23 | SET(CMAKE_CGO_LDFLAGS "${CMAKE_CGO_LDFLAGS} ${THREAD_SANITIZER_FLAG}") 24 | 25 | # TC/jemalloc are incompatible with ThreadSanitizer - force 26 | # the use of the system allocator. 27 | SET(COUCHBASE_MEMORY_ALLOCATOR system CACHE STRING "Memory allocator to use") 28 | 29 | # Configure CTest's MemCheck to ThreadSanitizer. 30 | SET(MEMORYCHECK_TYPE ThreadSanitizer) 31 | 32 | ADD_DEFINITIONS(-DTHREAD_SANITIZER) 33 | 34 | # Override the normal ADD_TEST macro to set the TSAN_OPTIONS 35 | # environment variable - this allows us to specify the 36 | # suppressions file to use. 37 | FUNCTION(ADD_TEST name) 38 | IF(${ARGV0} STREQUAL "NAME") 39 | SET(_name ${ARGV1}) 40 | ELSE() 41 | SET(_name ${ARGV0}) 42 | ENDIF() 43 | _ADD_TEST(${ARGV}) 44 | if (COUCHBASE_SERVER_BUILD) 45 | SET_TESTS_PROPERTIES(${_name} PROPERTIES ENVIRONMENT 46 | "TSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/tlm/tsan.suppressions") 47 | else (COUCHBASE_SERVER_BUILD) 48 | SET_TESTS_PROPERTIES(${_name} PROPERTIES ENVIRONMENT 49 | "TSAN_OPTIONS=suppressions=${PROJECT_SOURCE_DIR}/cmake/Modules/tsan.suppressions") 50 | endif (COUCHBASE_SERVER_BUILD) 51 | ENDFUNCTION() 52 | 53 | MESSAGE(STATUS "ThreadSanitizer enabled - forcing use of 'system' memory allocator.") 54 | ELSE() 55 | MESSAGE(FATAL_ERROR "CB_THREADSANITIZER enabled but compiler doesn't support ThreadSanitizer - cannot continue.") 56 | ENDIF() 57 | ENDIF() 58 | 59 | -------------------------------------------------------------------------------- /tools/dump_common.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_DUMP_COMMON_H 19 | #define _FDB_DUMP_COMMON_H 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #if !defined(WIN32) && !defined(_WIN32) 27 | #include 28 | #include 29 | #endif 30 | 31 | #include "libforestdb/forestdb.h" 32 | #include "fdb_internal.h" 33 | #include "filemgr.h" 34 | #include "file_handle.h" 35 | #include "list.h" 36 | #include "hbtrie.h" 37 | #include "btree.h" 38 | #include "btree_var_kv_ops.h" 39 | #include "docio.h" 40 | #include "btreeblock.h" 41 | #include "common.h" 42 | #include "wal.h" 43 | #include "filemgr_ops.h" 44 | #include "configuration.h" 45 | #include "internal_types.h" 46 | #include "compactor.h" 47 | #include "memleak.h" 48 | #include "time_utils.h" 49 | 50 | #ifdef __DEBUG 51 | #ifndef __DEBUG_FDB 52 | #undef DBG 53 | #undef DBGCMD 54 | #undef DBGSW 55 | #define DBG(...) 56 | #define DBGCMD(...) 57 | #define DBGSW(n, ...) 58 | #endif 59 | #endif 60 | 61 | #ifdef __cplusplus 62 | extern "C" { 63 | #endif 64 | 65 | INLINE int is_subblock(bid_t subbid) 66 | { 67 | uint8_t flag; 68 | flag = (subbid >> (8 * (sizeof(bid_t)-2))) & 0x00ff; 69 | return flag; 70 | } 71 | 72 | INLINE void subbid2bid(bid_t subbid, size_t *subblock_no, size_t *idx, 73 | bid_t *bid) 74 | { 75 | uint8_t flag; 76 | flag = (subbid >> (8 * (sizeof(bid_t)-2))) & 0x00ff; 77 | *subblock_no = flag >> 5; 78 | // to distinguish subblock_no==0 to non-subblock 79 | *subblock_no -= 1; 80 | *idx = flag & (0x20 - 0x01); 81 | *bid = ((bid_t)(subbid << 16)) >> 16; 82 | } 83 | 84 | int _kvs_cmp_name_fdb_dump(struct avl_node *a, 85 | struct avl_node *b, 86 | void *aux); 87 | void print_header(fdb_kvs_handle *db); 88 | 89 | 90 | #ifdef __cplusplus 91 | } 92 | #endif 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /src/task_priority.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_PRIORITY_H_ 19 | #define SRC_PRIORITY_H_ 1 20 | 21 | #include "common.h" 22 | 23 | enum type_id_t { 24 | COMPACTOR_ID, 25 | BGFLUSHER_ID, 26 | MAX_TYPE_ID // Keep this as the last enum value 27 | }; 28 | 29 | /** 30 | * Task priority definition. 31 | */ 32 | class Priority { 33 | public: 34 | // Priorities for Read-only tasks 35 | 36 | // Priorities for Read-Write tasks 37 | static const Priority CompactorPriority; 38 | static const Priority BgFlusherPriority; 39 | 40 | // Priorities for NON-IO tasks 41 | 42 | bool operator==(const Priority &other) const { 43 | return other.getPriorityValue() == this->priority; 44 | } 45 | 46 | bool operator<(const Priority &other) const { 47 | return this->priority > other.getPriorityValue(); 48 | } 49 | 50 | bool operator>(const Priority &other) const { 51 | return this->priority < other.getPriorityValue(); 52 | } 53 | 54 | /** 55 | * Return the task name. 56 | * 57 | * @return a task name 58 | */ 59 | static const char *getTypeName(const type_id_t i); 60 | 61 | /** 62 | * Return the type id representing a task 63 | * 64 | * @return type id 65 | */ 66 | type_id_t getTypeId() const { 67 | return t_id; 68 | } 69 | 70 | /** 71 | * Return an integer value that represents a priority. 72 | * 73 | * @return a priority value 74 | */ 75 | int getPriorityValue() const { 76 | return priority; 77 | } 78 | 79 | // gcc didn't like the idea of having a class with no constructor 80 | // available to anyone.. let's make it protected instead to shut 81 | // gcc up :( 82 | protected: 83 | Priority(type_id_t id, int p) : t_id(id), priority(p) { } 84 | type_id_t t_id; 85 | int priority; 86 | 87 | private: 88 | DISALLOW_COPY_AND_ASSIGN(Priority); 89 | }; 90 | 91 | #endif // SRC_PRIORITY_H_ 92 | -------------------------------------------------------------------------------- /src/ringbuffer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_RINGBUFFER_H_ 19 | #define SRC_RINGBUFFER_H_ 1 20 | 21 | #include "common.h" 22 | 23 | #include 24 | #include 25 | 26 | /** 27 | * A RingBuffer holds a fixed number of elements of type T. 28 | */ 29 | template 30 | class RingBuffer { 31 | public: 32 | 33 | /** 34 | * Construct a RingBuffer to hold the given number of elements. 35 | */ 36 | explicit RingBuffer(size_t s) : pos(0), max(s), wrapped(false) { 37 | storage = new T[max]; 38 | } 39 | 40 | ~RingBuffer() { 41 | delete[] storage; 42 | } 43 | 44 | RingBuffer(const RingBuffer&) = delete; 45 | RingBuffer& operator=(const RingBuffer&) = delete; 46 | 47 | /** 48 | * How many elements are currently stored in this ring buffer? 49 | */ 50 | size_t size() { 51 | return wrapped ? max : pos; 52 | } 53 | 54 | /** 55 | * Add an object to the RingBuffer. 56 | */ 57 | void add(T ob) { 58 | if (pos == max) { 59 | wrapped = true; 60 | pos = 0; 61 | } 62 | storage[pos++] = ob; 63 | } 64 | 65 | /** 66 | * Remove all items. 67 | */ 68 | void reset() { 69 | pos = 0; 70 | wrapped = false; 71 | } 72 | 73 | /** 74 | * Copy out the contents of this RingBuffer into the a vector. 75 | */ 76 | std::vector contents() { 77 | std::vector rv; 78 | size_t lsize = wrapped ? max : pos; 79 | rv.resize(lsize); 80 | size_t copied(0); 81 | if (wrapped && pos != max) { 82 | std::copy(storage + pos, storage + max, rv.begin()); 83 | copied = max - pos; 84 | } 85 | std::copy(storage, storage + pos, rv.begin() + copied); 86 | return rv; 87 | } 88 | 89 | private: 90 | T *storage; 91 | size_t pos; 92 | size_t max; 93 | bool wrapped; 94 | }; 95 | 96 | #endif // SRC_RINGBUFFER_H_ 97 | -------------------------------------------------------------------------------- /src/encryption_aes.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | #include "encryption.h" 20 | #include "crypto_primitives.h" 21 | #include 22 | 23 | #if AES256_AVAILABLE && SHA256_AVAILABLE 24 | 25 | static fdb_status aes_setup(encryptor *e) { 26 | // There must be room enough for AES keys in the provided structs: 27 | // (This would be a compile-time assert if C supported those.) 28 | assert(sizeof(e->key.bytes) == 32); 29 | assert(sizeof(e->extra) >= 32); 30 | // Precompute an auxiliary key by generating a SHA256 digest of the main key: 31 | sha256(&e->key.bytes, sizeof(e->key.bytes), e->extra); 32 | return FDB_RESULT_SUCCESS; 33 | } 34 | 35 | static fdb_status aes_crypt(encryptor *e, 36 | bool encrypt, 37 | void *dst_buf, 38 | const void *src_buf, 39 | size_t size, 40 | bid_t bid) 41 | { 42 | // Derive an IV as per the Encrypted Salt-Sector Initialization Value (ESSIV) algorithm 43 | // by encrypting the block number using the auxiliary key (a digest of the key.) 44 | // See https://en.wikipedia.org/wiki/Disk_encryption_theory 45 | uint8_t iv[16] = {0}; 46 | uint64_t bigBlockNo = _endian_encode(bid); 47 | memcpy(&iv, &bigBlockNo, sizeof(bigBlockNo)); 48 | if (!aes256(true, e->extra, NULL, &iv, &iv, sizeof(iv))) 49 | return FDB_RESULT_CRYPTO_ERROR; 50 | 51 | // Now encrypt/decrypt the block using the main key and the IV: 52 | if (!aes256(encrypt, e->key.bytes, iv, dst_buf, src_buf, size)) 53 | return FDB_RESULT_CRYPTO_ERROR; 54 | return FDB_RESULT_SUCCESS; 55 | } 56 | 57 | static encryption_ops aes_ops = { 58 | aes_setup, 59 | aes_crypt 60 | }; 61 | 62 | const encryption_ops* const fdb_encryption_ops_aes = &aes_ops; 63 | 64 | #else // AES not available: 65 | 66 | const encryption_ops* const fdb_encryption_ops_aes = NULL; 67 | 68 | #endif // AES256_AVAILABLE && SHA256_AVAILABLE 69 | -------------------------------------------------------------------------------- /src/breakpad_win32.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "breakpad.h" 19 | 20 | #include "client/windows/handler/exception_handler.h" 21 | 22 | #include 23 | #include 24 | 25 | using namespace google_breakpad; 26 | static ExceptionHandler* handler = nullptr; 27 | 28 | /* Called when an exception triggers a dump, outputs details to caller's logs */ 29 | static bool dumpCallback(const wchar_t* dump_path, const wchar_t* minidump_id, 30 | void* context, EXCEPTION_POINTERS* exinfo, 31 | MDRawAssertionInfo* assertion, bool succeeded) { 32 | fprintf(stderr, "Breakpad caught a crash in forestdb. Writing crash dump " 33 | "to %S\\%S before terminating.\n", dump_path, minidump_id); 34 | 35 | return succeeded; 36 | } 37 | 38 | static void create_breakpad(const char* minidump_dir) { 39 | // Takes a wchar_t* on Windows. 40 | size_t len = strlen(minidump_dir) + 1; 41 | wchar_t* wc_minidump_dir = new wchar_t[len]; 42 | size_t wlen = 0; 43 | mbstowcs_s(&wlen, wc_minidump_dir, len, minidump_dir, _TRUNCATE); 44 | 45 | handler = new ExceptionHandler(wc_minidump_dir, 46 | /*filter*/nullptr, 47 | dumpCallback, 48 | /*callback-context*/NULL, 49 | ExceptionHandler::HANDLER_ALL, 50 | MiniDumpNormal, 51 | /*pipe*/(wchar_t*) nullptr, 52 | /*custom_info*/nullptr); 53 | 54 | delete[] wc_minidump_dir; 55 | } 56 | 57 | void initialize_breakpad(const char* minidump_dir) { 58 | // We cannot actually change any of breakpad's setings once created, only 59 | // remove it and re-create with new settings. 60 | destroy_breakpad(); 61 | 62 | if (minidump_dir != nullptr && minidump_dir[0] != '\0') { 63 | create_breakpad(minidump_dir); 64 | } 65 | } 66 | 67 | void destroy_breakpad(void) { 68 | delete handler; 69 | handler = nullptr; 70 | } 71 | -------------------------------------------------------------------------------- /utils/memleak.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Simple Memory Leakage Detection Tool 4 | * (C) 2013 Jung-Sang Ahn 5 | * see https://github.com/greensky00/memleak 6 | */ 7 | 8 | #ifndef _JSAHN_MEMLEAK_H 9 | #define _JSAHN_MEMLEAK_H 10 | 11 | #include 12 | 13 | #ifdef _MSC_VER 14 | #ifdef forestdb_EXPORTS 15 | #define LIBMEMLEAK_API extern __declspec(dllexport) 16 | #else 17 | #define LIBMEMLEAK_API 18 | #endif 19 | #elif __GNUC__ 20 | #define LIBMEMLEAK_API __attribute ((visibility("default"))) 21 | #else 22 | #define LIBMEMLEAK_API 23 | #endif 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | #ifndef _MEMLEAK_ENABLE 30 | #define _MALLOC_OVERRIDE 31 | #endif 32 | #ifndef _MALLOC_OVERRIDE 33 | #define _MALLOC_OVERRIDE 34 | #define malloc(size) memleak_alloc(size, (char*)__FILE__, __LINE__) 35 | #define calloc(nmemb, size) memleak_calloc(nmemb, size, (char*)__FILE__, __LINE__) 36 | #define realloc(ptr, size) memleak_realloc(ptr, size); 37 | #define free(addr) memleak_free(addr, (char*)__FILE__, __LINE__) 38 | 39 | #if !defined(WIN32) 40 | 41 | #if !defined(__ANDROID__) 42 | #define posix_memalign(memptr, alignment, size) \ 43 | memleak_posix_memalign(memptr, alignment, size, (char*)__FILE__, __LINE__) 44 | #else // not __ANDROID__ 45 | #define memalign(alignment, size) \ 46 | memleak_memalign(alignment, size, (char*)__FILE__, __LINE__) 47 | #endif // not __ANDROID__ 48 | 49 | #else // not WIN32 50 | #define _aligned_malloc(size, align) \ 51 | memleak_aligned_malloc(size, align, (char*)__FILE__, __LINE__) 52 | #define _aligned_free(addr) \ 53 | memleak_aligned_free(addr, (char*)__FILE__, __LINE__) 54 | #endif // not WIN32 55 | 56 | #endif // not _MALLOC_OVERRIDE 57 | 58 | LIBMEMLEAK_API 59 | void memleak_start(); 60 | 61 | LIBMEMLEAK_API 62 | void memleak_end(); 63 | 64 | LIBMEMLEAK_API 65 | void * memleak_alloc(size_t size, char *file, size_t line); 66 | 67 | LIBMEMLEAK_API 68 | void * memleak_calloc(size_t nmemb, size_t size, char *file, size_t line); 69 | 70 | LIBMEMLEAK_API 71 | void * memleak_memalign(size_t alignment, size_t size, char *file, size_t line); 72 | 73 | LIBMEMLEAK_API 74 | void *memleak_realloc(void *ptr, size_t size); 75 | 76 | LIBMEMLEAK_API 77 | void memleak_free(void *addr, char *file, size_t line); 78 | 79 | #ifndef WIN32 80 | LIBMEMLEAK_API 81 | int memleak_posix_memalign(void **memptr, size_t alignment, size_t size, char *file, size_t line); 82 | #else 83 | LIBMEMLEAK_API 84 | void * memleak_aligned_malloc(size_t size, size_t alignment, char *file, size_t line); 85 | LIBMEMLEAK_API 86 | void memleak_aligned_free(void *addr, char *file, size_t line); 87 | #endif 88 | 89 | #ifdef __cplusplus 90 | } 91 | #endif 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /src/btree_fast_str_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_FAST_STR_KV_H 19 | #define _JSAHN_BTREE_FAST_STR_KV_H 20 | 21 | #include 22 | #include "common.h" 23 | 24 | #include "btree.h" 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | /** 31 | * B+tree key-value operation class for variable-length string key. 32 | * Note that it can be also used for custom (non-lexicographical) order operations. 33 | */ 34 | class FastStrKVOps : public BTreeKVOps { 35 | public: 36 | FastStrKVOps(); 37 | FastStrKVOps(size_t _ksize, size_t _vsize); 38 | FastStrKVOps(size_t _ksize, size_t _vsize, btree_cmp_func _cmp_func); 39 | 40 | virtual ~FastStrKVOps() { } 41 | 42 | void init(size_t _ksize, size_t _vsize, btree_cmp_func _cmp_func); 43 | 44 | void getKV(struct bnode *node, idx_t idx, void *key, void *value); 45 | void setKV(struct bnode *node, idx_t idx, void *key, void *value); 46 | void insKV(struct bnode *node, idx_t idx, void *key, void *value); 47 | void copyKV(struct bnode *node_dst, 48 | struct bnode *node_src, 49 | idx_t dst_idx, 50 | idx_t src_idx, 51 | idx_t len); 52 | size_t getDataSize(struct bnode *node, 53 | void *new_minkey, 54 | void *key_arr, 55 | void *value_arr, 56 | size_t len); 57 | size_t getKVSize(void *key, void *value); 58 | void initKVVar(void *key, void *value); 59 | void freeKVVar(void *key, void *value); 60 | void setKey(void *dst, void *src); 61 | void setValue(void *dst, void *src); 62 | idx_t getNthIdx(struct bnode *node, idx_t num, idx_t den); 63 | void getNthSplitter(struct bnode *prev_node, 64 | struct bnode *node, 65 | void *key); 66 | 67 | void setVarKey(void *key, void *str, size_t len); 68 | void setInfVarKey(void *key); 69 | bool isInfVarKey(void *key); 70 | void getVarKey(void *key, void *strbuf, size_t& len); 71 | void freeVarKey(void *key); 72 | }; 73 | 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /src/taskqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef SRC_TASKQUEUE_H_ 18 | #define SRC_TASKQUEUE_H_ 1 19 | 20 | #include "common.h" 21 | 22 | #include 23 | #include 24 | 25 | #include "futurequeue.h" 26 | #include "ringbuffer.h" 27 | #include "task_type.h" 28 | #include "sync_object.h" 29 | #include "tasks.h" 30 | 31 | class ExecutorPool; 32 | class ExecutorThread; 33 | 34 | class TaskQueue { 35 | friend class ExecutorPool; 36 | public: 37 | TaskQueue(ExecutorPool *m, task_type_t t, const char *nm); 38 | ~TaskQueue(); 39 | 40 | void schedule(ExTask &task); 41 | 42 | hrtime_t reschedule(ExTask &task, task_type_t &curTaskType); 43 | 44 | void checkPendingQueue(void); 45 | 46 | void doWake(size_t &numToWake); 47 | 48 | bool fetchNextTask(ExecutorThread &thread, bool toSleep); 49 | 50 | void wake(ExTask &task); 51 | 52 | static const std::string taskType2Str(task_type_t type); 53 | 54 | const std::string getName() const; 55 | 56 | const task_type_t getQueueType() const { return queueType; } 57 | 58 | size_t getReadyQueueSize(); 59 | 60 | size_t getFutureQueueSize(); 61 | 62 | size_t getPendingQueueSize(); 63 | 64 | void snooze(ExTask& task, const double secs) { 65 | futureQueue.snooze(task, secs); 66 | } 67 | 68 | private: 69 | void _schedule(ExTask &task); 70 | hrtime_t _reschedule(ExTask &task, task_type_t &curTaskType); 71 | void _checkPendingQueue(void); 72 | bool _fetchNextTask(ExecutorThread &thread, bool toSleep); 73 | void _wake(ExTask &task); 74 | bool _doSleep(ExecutorThread &thread, UniqueLock& lock); 75 | void _doWake_UNLOCKED(size_t &numToWake); 76 | size_t _moveReadyTasks(hrtime_t tv); 77 | ExTask _popReadyTask(void); 78 | 79 | SyncObject mutex; 80 | const std::string name; 81 | task_type_t queueType; 82 | ExecutorPool *manager; 83 | size_t sleepers; // number of threads sleeping in this taskQueue 84 | 85 | // sorted by task priority. 86 | std::priority_queue, 87 | CompareByPriority> readyQueue; 88 | 89 | // sorted by waketime. 90 | FutureQueue<> futureQueue; 91 | 92 | std::list pendingQueue; 93 | }; 94 | 95 | #endif // SRC_TASKQUEUE_H_ 96 | -------------------------------------------------------------------------------- /utils/bitwise_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_BITWISE_UTILS_H 2 | #define _JSAHN_BITWISE_UTILS_H 3 | 4 | #ifndef INT64_C 5 | #define INT64_C(c) (c ## LL) 6 | #define UINT64_C(c) (c ## ULL) 7 | #endif 8 | 9 | #define MAX(a,b) (((a)>(b))?(a):(b)) 10 | #define MIN(a,b) (((a)<(b))?(a):(b)) 11 | #define CHK_POW2(v) (!((uint64_t)v & ((uint64_t)v - 0x1))) 12 | 13 | #ifndef bitswap64 14 | #define bitswap64(v) \ 15 | ( (((v) & 0xff00000000000000ULL) >> 56) \ 16 | | (((v) & 0x00ff000000000000ULL) >> 40) \ 17 | | (((v) & 0x0000ff0000000000ULL) >> 24) \ 18 | | (((v) & 0x000000ff00000000ULL) >> 8) \ 19 | | (((v) & 0x00000000ff000000ULL) << 8) \ 20 | | (((v) & 0x0000000000ff0000ULL) << 24) \ 21 | | (((v) & 0x000000000000ff00ULL) << 40) \ 22 | | (((v) & 0x00000000000000ffULL) << 56) ) 23 | #endif 24 | 25 | #ifndef bitswap32 26 | #define bitswap32(v) \ 27 | ( (((v) & 0xff000000) >> 24) \ 28 | | (((v) & 0x00ff0000) >> 8) \ 29 | | (((v) & 0x0000ff00) << 8) \ 30 | | (((v) & 0x000000ff) << 24) ) 31 | #endif 32 | 33 | #ifndef bitswap16 34 | #define bitswap16(v) \ 35 | ( (((v) & 0xff00) >> 8) \ 36 | | (((v) & 0x00ff) << 8) ) 37 | #endif 38 | 39 | // can be faster under O3 optimization 40 | //#ifdef __BIT_CMP 41 | 42 | // 64-bit sign mask 43 | #define _64_SM (UINT64_C(0x8000000000000000)) 44 | // 32-bit sign mask 45 | #define _32_SM (0x80000000) 46 | // 32-bit value mask 47 | #define _32_M (0xffffffff) 48 | 49 | // 64-bit sign bit check 50 | #define _64_SC(a,b) ((uint64_t)(((a) & _64_SM)^((b) & _64_SM))>>63) 51 | // 64-bit sign bit check and convert to 32-bit 52 | #define _64_SC_32(a,b) ((uint64_t)(((a) & _64_SM)^((b) & _64_SM))>>32) 53 | 54 | // 32-bit sign bit check 55 | #define _32_SC(a,b) ((uint32_t)(((a) & _32_SM)^((b) & _32_SM))>>31) 56 | 57 | #define _U64_V(ptr) ( *(uint64_t*)(ptr) ) 58 | #define _U32_V(ptr) ( *(uint32_t*)(ptr) ) 59 | 60 | // check whether V is non-zero or not (return 1 when non-zero, otherwise 0) 61 | #define _NZ(v) ( (( (v) | (~(v) + 1)) >> 31) & 0x1 ) 62 | #define _NZ_64(v) ( (( (v) | (~(v) + 1)) >> 63) & 0x1 ) 63 | 64 | // convert 64-bit value to 32-bit value preserving sign bit (but not value) 65 | //#define _CSB(v) ( ((v)>>32) | (((v)&_32_M)>>1) | ((v)&0x1) ) 66 | #define _CSB(v) ( ((v)>>32) | _NZ_64((uint64_t)v) ) 67 | 68 | // map from (32-bit signed integer){neg, 0, pos} to {-1, 0, 1} 69 | #define _CS(v) ((~(((v) & _32_SM)>>31)+1) | _NZ(v)) 70 | #define _MAP(v) (int32_t)(_CS((uint32_t)v)) 71 | 72 | #define _CMP_U32(a, b) \ 73 | (int32_t)( _CSB((int64_t)(a)-(int64_t)(b)) ) 74 | #define _CMP_U32_P(a, b) _CMP_U32(_U32_V(a), _U32_V(b)) 75 | 76 | #define _CMP_U64(a, b) \ 77 | (int32_t) ( \ 78 | ( (_64_SC(a,b)-1) & _CSB((a)-(b)) ) | /* a and b have same sign */ \ 79 | ( (_64_SC_32(a,b) | _64_SC(a,b)) & ( (((b) & _64_SM) >> 32) | 0x1))) /* a and b have different sign */ 80 | #define _CMP_U64_P(a, b) _CMP_U64(_U64_V(a), _U64_V(b)) 81 | 82 | //#endif 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /src/encryption.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FDB_ENCRYPTION_H 19 | #define _FDB_ENCRYPTION_H 20 | 21 | #include "libforestdb/fdb_types.h" 22 | #include "libforestdb/fdb_errors.h" 23 | #include "common.h" 24 | 25 | // A very simple but insecure algorithm for testing purposes only. 26 | enum { 27 | FDB_ENCRYPTION_BOGUS = -1 28 | }; 29 | 30 | // An "object" that can perform encryption. 31 | typedef struct { 32 | const struct encryption_ops *ops; // callbacks 33 | fdb_encryption_key key; // key + algorithm 34 | uint8_t extra[32]; // scratch space for encryptor to use 35 | } encryptor; 36 | 37 | // Initializes an encryptor given a key. 38 | fdb_status fdb_init_encryptor(encryptor*, 39 | const fdb_encryption_key*); 40 | 41 | // Decrypts a block of data. 42 | fdb_status fdb_decrypt_block(encryptor*, 43 | void *buf, 44 | size_t blocksize, 45 | bid_t bid); 46 | 47 | // Encrypts one or more consecutive blocks of data. 48 | fdb_status fdb_encrypt_blocks(encryptor*, 49 | void *dst_buf, 50 | const void *src_buf, 51 | size_t blocksize, 52 | unsigned num_blocks, 53 | bid_t start_bid); 54 | 55 | // Callbacks provided by an encryption implementation. 56 | typedef struct encryption_ops { 57 | fdb_status (*setup)(encryptor*); 58 | fdb_status (*crypt)(encryptor*, 59 | bool encrypt, 60 | void *dst_buf, 61 | const void *src_buf, 62 | size_t size, 63 | bid_t bid); 64 | } encryption_ops; 65 | 66 | // Provides the encryption_ops (callbacks) for a particular algorithm. 67 | const encryption_ops* get_encryption_ops(fdb_encryption_algorithm_t); 68 | 69 | // Declarations of encryption_ops for specific algorithms. 70 | // Will be NULL if not implemented on the current platform. 71 | extern const encryption_ops* const fdb_encryption_ops_aes; 72 | extern const encryption_ops* const fdb_encryption_ops_bogus; 73 | 74 | #endif /* _FDB_ENCRYPTION_H */ 75 | -------------------------------------------------------------------------------- /src/memory_pool.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include "common.h" 26 | 27 | class MemoryPool { 28 | /** 29 | The memory pool is a memory container designed for concurrent access and 30 | it contains: 31 | - A vector of pre-allocated heap memory (bins). 32 | - Each bin will be of the initially provided bin size. 33 | - A queue containing the indexes of the available bins from the vector. 34 | - A fetchBlock() operation gets an available resource and returns the index 35 | to the acquired bin, removes the index entry from the queue. 36 | - A returnBlock() operation requeues the provided index back into the queue, 37 | thereby making the bin pointed to by the index available again. 38 | 39 | +---+---+---+---+---+---+---- 40 | queue: | 0 | 1 | 2 | 3 | 4 | 5 | ... 41 | +---+---+---+---+---+---+---- 42 | | ____________ 43 | +-----> | bin1 | 44 | |____________| 45 | | bin2 | 46 | vector: |____________| 47 | | bin3 | 48 | |____________| 49 | | .. | 50 | */ 51 | 52 | public: 53 | MemoryPool(int num_bins, size_t bin_size); 54 | 55 | ~MemoryPool(); 56 | 57 | /** 58 | * Fetches an available block of memory (bin), making it unavailable for 59 | * other clients. 60 | * 61 | * @param buf Pointer to where the memory block is initialized. 62 | * @return index of bin in the vector, -1 if in case of no available bin. 63 | */ 64 | const int fetchBlock(uint8_t **buf); 65 | 66 | /* 67 | * Return a block of memory (bin), making it available for other clients. 68 | * 69 | * @param index Index of the bin being returned back. 70 | */ 71 | void returnBlock(int index); 72 | 73 | private: 74 | 75 | /** 76 | * Pushes entry into queue. 77 | */ 78 | void enQueue(int index); 79 | 80 | /** 81 | * Pops entry from queue. 82 | */ 83 | int deQueue(); 84 | 85 | // Spin lock for queue ops 86 | spin_t lock; 87 | // Queue of indexes 88 | std::queue indexQ; 89 | // Vector of pre-allocated memory 90 | std::vector memPool; 91 | }; 92 | -------------------------------------------------------------------------------- /src/encryption.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "encryption.h" 19 | #include 20 | 21 | //#define FDB_LOG_CRYPTO 22 | 23 | fdb_status fdb_init_encryptor(encryptor *e, 24 | const fdb_encryption_key *key) 25 | { 26 | if (key->algorithm == FDB_ENCRYPTION_NONE) { 27 | e->ops = NULL; 28 | return FDB_RESULT_SUCCESS; 29 | } 30 | #ifdef FDB_LOG_CRYPTO 31 | fprintf(stderr, "CRYPT: Initializing context for key %d:%llx\n", 32 | key->algorithm, *(uint64_t*)key->bytes); 33 | #endif 34 | e->ops = get_encryption_ops(key->algorithm); 35 | if (!e->ops) 36 | return FDB_RESULT_CRYPTO_ERROR; // unsupported algorithm 37 | e->key = *key; 38 | return e->ops->setup(e); 39 | } 40 | 41 | fdb_status fdb_decrypt_block(encryptor *e, 42 | void *buf, 43 | size_t blocksize, 44 | bid_t bid) 45 | { 46 | #ifdef FDB_LOG_CRYPTO 47 | fprintf(stderr, "CRYPT: Decrypting block #%llu with key %d:%llx\n", 48 | bid, e->key.algorithm, *(uint64_t*)e->key.bytes); 49 | #endif 50 | return e->ops->crypt(e, false, buf, buf, blocksize, bid); 51 | } 52 | 53 | fdb_status fdb_encrypt_blocks(encryptor *e, 54 | void *dst_buf, 55 | const void *src_buf, 56 | size_t blocksize, 57 | unsigned num_blocks, 58 | bid_t start_bid) 59 | { 60 | #ifdef FDB_LOG_CRYPTO 61 | fprintf(stderr, "CRYPT: Encrypting blocks #%llu-%llu with key %d:%llx\n", 62 | start_bid, start_bid+num_blocks-1, 63 | e->key.algorithm, *(uint64_t*)e->key.bytes); 64 | #endif 65 | fdb_status status = FDB_RESULT_SUCCESS; 66 | for (unsigned i = 0; i < num_blocks; i++) { 67 | status = e->ops->crypt(e, 68 | true, 69 | (uint8_t*)dst_buf + i*blocksize, 70 | (const uint8_t*)src_buf + i*blocksize, 71 | blocksize, 72 | start_bid + i); 73 | if (status != FDB_RESULT_SUCCESS) 74 | break; 75 | } 76 | return status; 77 | } 78 | 79 | const encryption_ops* get_encryption_ops(fdb_encryption_algorithm_t algorithm) { 80 | switch (algorithm) { 81 | case FDB_ENCRYPTION_AES256: 82 | return fdb_encryption_ops_aes; 83 | case FDB_ENCRYPTION_BOGUS: 84 | return fdb_encryption_ops_bogus; 85 | default: 86 | return NULL; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/checksum.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * ForestDB evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | #pragma once 27 | 28 | #include 29 | #include 30 | 31 | enum crc_mode_e { 32 | CRC_UNKNOWN, 33 | CRC32, 34 | CRC32C, 35 | #ifdef _CRC32C 36 | CRC_DEFAULT = CRC32C // Couchbase builds pickup crc32c from plaform 37 | #else 38 | CRC_DEFAULT = CRC32 // Non-couchbase falls back to bundled utils/crc32.cc 39 | #endif 40 | }; 41 | 42 | /* 43 | * Get a checksum of buf for buf_len bytes. 44 | * 45 | * mode = UNKNOWN is an invalid input (triggers assert). 46 | */ 47 | uint32_t get_checksum(const uint8_t* buf, 48 | size_t buf_len, 49 | uint32_t pre, 50 | crc_mode_e mode); 51 | 52 | /* 53 | * Get a checksum of buf for buf_len bytes. 54 | * 55 | * The pre value is set to 0. 56 | * 57 | * mode = UNKNOWN is an invalid input (triggers assert). 58 | */ 59 | uint32_t get_checksum(const uint8_t* buf, 60 | size_t buf_len, 61 | crc_mode_e mode); 62 | 63 | /* 64 | * Get a checksum of buf for buf_len bytes. 65 | * 66 | * The CRC is generated using the default mode for the build of ForestDB. 67 | * The pre value is set to 0. 68 | * 69 | */ 70 | inline uint32_t get_checksum(const uint8_t* buf, 71 | size_t buf_len) { 72 | return get_checksum(buf, buf_len, 0, CRC_DEFAULT); 73 | } 74 | 75 | /* 76 | * Perform an integrity check of buf for buf_len bytes. 77 | * 78 | * A checksum of buf is created and compared against checksum argument. 79 | * 80 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 81 | * 82 | * Returns true success (checksums match), false for failure. 83 | */ 84 | bool perform_integrity_check(const uint8_t* buf, 85 | size_t buf_len, 86 | uint32_t checksum, 87 | crc_mode_e mode); 88 | 89 | /* 90 | * Detect the CRC mode by performing an integrity check against the 91 | * two CRC functions ForestDB files could be written with. 92 | * 93 | * Returns true if a match is found and sets mode to the correct mode. 94 | * Returns false if no match and sets mode to CRC_UNKNOWN. 95 | */ 96 | bool detect_and_check_crc(const uint8_t* buf, 97 | size_t buf_len, 98 | uint32_t checksum, 99 | crc_mode_e* mode); 100 | -------------------------------------------------------------------------------- /cmake/Modules/CouchbaseCodeCoverage.cmake: -------------------------------------------------------------------------------- 1 | # Support for running code coverage reporting. 2 | # 3 | # Usage: 4 | # 1). Add a call to ADD_COVERAGE_REPORT() to the module(s) you wish to 5 | # obtain code coverage reports on. 6 | # 2). Enable the option CB_CODE_COVERAGE (e.g. pass -DCB_CODE_COVERAGE=ON to cmake). 7 | # 3). Build as normal. 8 | # 4). Run unit test(s) to exercise the codebase. 9 | # 5). Run `make coverage-report-html` and/or `coverage-report-xml` 10 | # (from the selected module subdirectory) to generate the reports. 11 | # 6) (Optional) to zero coverage counters before a re-run run `make coverage-zero-counters`. 12 | 13 | 14 | OPTION(CB_CODE_COVERAGE "Enable code coverage testing." 15 | OFF) 16 | 17 | IF (CB_CODE_COVERAGE) 18 | FIND_PROGRAM(GCOV_PATH gcov) 19 | FIND_PROGRAM(GCOVR_PATH gcovr) 20 | 21 | IF (NOT GCOV_PATH) 22 | MESSAGE(STATUS "gcov not found.") 23 | ENDIF (NOT GCOV_PATH) 24 | 25 | IF (NOT GCOVR_PATH) 26 | MESSAGE(STATUS "gcovr [www.gcovr.com] not found.") 27 | ENDIF () 28 | 29 | IF (NOT GCOV_PATH OR NOT GCOVR_PATH) 30 | MESSAGE(FATAL_ERROR "CB_CODE_COVERAGE enabled but one of more required tools not found - cannot continue.") 31 | ENDIF() 32 | ENDIF(CB_CODE_COVERAGE) 33 | 34 | # Defines a coverage report for the current module. If CB_CODE_COVERAGE is enabled, 35 | # adds three new targets to that module: 36 | # -coverage-zero-counters: Zeros the code coverage counters for the module. 37 | # -coverage-report-html: Generates a code coverage report in HTML. 38 | # -coverage-report-xml: Generates a code coverage report in XML. 39 | # Usage: 40 | # 1) `make -coverage-zero-counters` to clear any counters from 41 | # previously-executed programs. 42 | # 2) Run whatever programs to excercise the code (unit tests, etc). 43 | # 3) `make -coverage-report-{html,xml}` to generate a report. 44 | # 45 | FUNCTION(ENABLE_CODE_COVERAGE_REPORT) 46 | GET_FILENAME_COMPONENT(_cc_project ${CMAKE_CURRENT_BINARY_DIR} NAME) 47 | 48 | IF (CB_CODE_COVERAGE) 49 | MESSAGE(STATUS "Setting up code coverage for ${PROJECT_NAME}") 50 | 51 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-zero-counters 52 | COMMAND find . -name *.gcda -exec rm {} \; 53 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 54 | COMMENT "Zeroing coverage counters for objects in ${CMAKE_CURRENT_BINARY_DIR}" 55 | VERBATIM) 56 | 57 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-report-html 58 | COMMAND ${CMAKE_COMMAND} -E remove_directory coverage 59 | COMMAND ${CMAKE_COMMAND} -E make_directory coverage 60 | COMMAND ${GCOVR_PATH} --root=${CMAKE_SOURCE_DIR} --filter="${CMAKE_CURRENT_SOURCE_DIR}/.*" --html --html-details -o coverage/index.html 61 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 62 | COMMENT "Generating code coverage report for ${PROJECT_NAME} to ${CMAKE_CURRENT_BINARY_DIR}/coverage/index.html") 63 | 64 | ADD_CUSTOM_TARGET(${_cc_project}-coverage-report-xml 65 | COMMAND ${GCOVR_PATH} --root=${CMAKE_SOURCE_DIR} --filter="${CMAKE_CURRENT_SOURCE_DIR}/.*" --xml -o coverage.xml 66 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 67 | COMMENT "Generating code coverage report for ${PROJECT_NAME} to ${CMAKE_CURRENT_BINARY_DIR}/coverage.xml") 68 | ENDIF (CB_CODE_COVERAGE) 69 | ENDFUNCTION() 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ForestDB 2 | 3 | ForestDB is a key-value storage engine developed by Couchbase Caching and Storage Team, and its main index structure is built from [Hierarchical B+-Tree based Trie](http://db.csail.mit.edu/sigmod11contest/sigmod_2011_contest_poster_jungsang_ahn.pdf), called HB+-Trie. [ForestDB paper](https://www.computer.org/csdl/trans/tc/preprint/07110563.pdf) has been published in IEEE Transactions on Computers. 4 | 5 | Compared with traditional B+-Tree based storage engines, ForestDB shows significantly better read and write performance with less storage overhead. ForestDB has been tested on various server OS environments (Centos, Ubuntu, Mac OS x, Windows) and mobile OSs (iOS, Android). The test coverage stats for ForestDB are available in [ForestDB Code Coverage Report](http://labs.couchbase.com/fdbcoverage/index.html). 6 | 7 | [ForestDB benchmark program](https://github.com/couchbaselabs/ForestDB-Benchmark) is also available for performance comparisons with other key-value storage engines. 8 | 9 | Please visit the [ForestDB wiki](https://github.com/couchbaselabs/forestdb/wiki) for more details. 10 | 11 | ## Main Features 12 | 13 | - Keys and values are treated as an arbitrary binary. 14 | - Applications can supply a custom compare function to support a customized key order. 15 | - A value can be retrieved by its sequence number or disk offset in addition to a key. 16 | - Write-Ahead Logging (WAL) and its in-memory index are used to reduce the main index lookup / update overhead. 17 | - Multi-Version Concurrency Control (MVCC) support and append-only storage layer. 18 | - Multiple snapshot instances can be created from a given ForestDB instance to provide different views of database. 19 | - Rollback is supported to revert the database to a specific point. 20 | - Ranged iteration by keys or sequence numbers is supported for a partial or full range lookup operation. 21 | - Manual or auto compaction can be configured per ForestDB database file. 22 | - Transactional support with read\_committed or read\_uncommitted isolation level. 23 | 24 | ## How to build 25 | 26 | See INSTALL.MD 27 | 28 | ## How to Use 29 | 30 | Please refer to [Public APIs](https://github.com/couchbaselabs/forestdb/wiki/Public-APIs) and tests/fdb\_functional\_test.cc in ForestDB source directory. 31 | 32 | ## How to contribute code 33 | 34 | 1. Sign the [Couchbase Contributor License 35 | Agreement](http://review.couchbase.org/static/individual_agreement.html) 36 | 1. Submit code changes via either a Github PR or via Gerrit (for Gerrit usage, see [Instructions](https://github.com/couchbase/couchbase-spark-connector/blob/master/CONTRIBUTING.md#preparing-for-contribution) from the couchbase-spark-connector project.) 37 | 38 | # Note regarding master branch 39 | The 'master' git branch of forestdb contains a number of changes which ultimately were not kept for production builds of Couchbase Server. Production builds were kept on an earlier release branch named 'watson' corresponding to Couchbase Server 4.5. Couchbase Server 5.0, 5.1, 5.5, and 6.0 added some bug fixes on branches made from 'watson', namely 'spock' and 'vulcan'. For Couchbase Server 6.5 and forward, a new branch 'cb-master' was created from the then-current 'vulcan' branch. 40 | 41 | 'cb-master' should be seen as the equivalent of 'master' for all Couchbase Server production build purposes. Any additional production bug fixes will go there, and release-specific branches will be made from there when necessary. 42 | 43 | The current 'master' branch is left untouched and unsupported, for use by community users who may depend on the work done there. 44 | -------------------------------------------------------------------------------- /src/version.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "version.h" 19 | #include 20 | 21 | bool ver_is_valid_magic(filemgr_magic_t magic) 22 | { 23 | if ( magic == FILEMGR_MAGIC_000 || 24 | (magic >= FILEMGR_MAGIC_001 && magic <= FILEMGR_LATEST_MAGIC)) { 25 | return true; 26 | } 27 | return false; 28 | } 29 | 30 | bool ver_is_magic_000(filemgr_magic_t magic) 31 | { 32 | if (magic == FILEMGR_MAGIC_000) { 33 | return true; 34 | } 35 | return false; 36 | } 37 | 38 | bool ver_is_atleast_magic_001(filemgr_magic_t magic) 39 | { 40 | // All magic numbers since FILEMGR_MAGIC_001 41 | if (magic >= FILEMGR_MAGIC_001 && magic <= FILEMGR_LATEST_MAGIC) { 42 | return true; 43 | } 44 | return false; 45 | } 46 | 47 | bool ver_staletree_support(filemgr_magic_t magic) 48 | { 49 | // All magic numbers since FILEMGR_MAGIC_002 50 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 51 | return true; 52 | } 53 | return false; 54 | } 55 | 56 | bool ver_non_consecutive_doc(filemgr_magic_t magic) 57 | { 58 | // All magic numbers since FILEMGR_MAGIC_002 59 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 60 | return true; 61 | } 62 | return false; 63 | } 64 | 65 | bool ver_superblock_support(filemgr_magic_t magic) 66 | { 67 | // All magic numbers since FILEMGR_MAGIC_002 68 | if (magic >= FILEMGR_MAGIC_002 && magic <= FILEMGR_LATEST_MAGIC) { 69 | return true; 70 | } 71 | return false; 72 | } 73 | 74 | bool ver_btreev2_format(filemgr_magic_t magic) 75 | { 76 | // All magic numbers since FILEMGR_MAGIC_003 77 | if (magic >= FILEMGR_MAGIC_003) { 78 | return true; 79 | } 80 | return false; 81 | } 82 | 83 | size_t ver_get_new_filename_off(filemgr_magic_t magic) { 84 | switch(magic) { 85 | case FILEMGR_MAGIC_000: return 64; 86 | case FILEMGR_MAGIC_001: return 72; 87 | case FILEMGR_MAGIC_002: return 80; 88 | case FILEMGR_MAGIC_003: return 80; 89 | } 90 | return (size_t) -1; 91 | } 92 | 93 | size_t ver_get_last_wal_flush_hdr_off(filemgr_magic_t magic) { 94 | switch(magic) { 95 | case FILEMGR_MAGIC_000: return 40; 96 | case FILEMGR_MAGIC_001: return 48; 97 | case FILEMGR_MAGIC_002: return 56; 98 | case FILEMGR_MAGIC_003: return 56; 99 | } 100 | return (size_t) -1; 101 | } 102 | 103 | const char* ver_get_version_string(filemgr_magic_t magic) { 104 | switch (magic) { 105 | case FILEMGR_MAGIC_000: 106 | return "ForestDB v1.x format"; 107 | case FILEMGR_MAGIC_001: 108 | return "ForestDB v1.x format"; 109 | case FILEMGR_MAGIC_002: 110 | return "ForestDB v2.x format"; 111 | case FILEMGR_MAGIC_003: 112 | return "ForestDB v3.x format"; 113 | } 114 | return "unknown"; 115 | } 116 | -------------------------------------------------------------------------------- /src/btree_kv.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_BTREE_KV_H 19 | #define _JSAHN_BTREE_KV_H 20 | 21 | #include 22 | #include 23 | #include "common.h" 24 | 25 | #include "btree.h" 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | INLINE uint32_t deref32(const void *ptr) 32 | { 33 | #ifdef _ALIGN_MEM_ACCESS 34 | // 4-byte align check (rightmost 2 bits must be '00') 35 | if ( (size_t)ptr & 0x3 ) { 36 | uint32_t value; 37 | memcpy(&value, ptr, sizeof(uint32_t)); 38 | return value; 39 | } 40 | #endif 41 | return *(uint32_t*)ptr; 42 | } 43 | 44 | INLINE uint64_t deref64(const void *ptr) 45 | { 46 | #ifdef _ALIGN_MEM_ACCESS 47 | // 8-byte align check (rightmost 3 bits must be '000') 48 | // Not sure whether 8-byte integer should be aligned in 49 | // 8-byte boundary or just 4-byte boundary. 50 | if ( (size_t)ptr & 0x7 ) { 51 | uint64_t value; 52 | memcpy(&value, ptr, sizeof(uint64_t)); 53 | return value; 54 | } 55 | #endif 56 | return *(uint64_t*)ptr; 57 | } 58 | 59 | /** 60 | * B+tree key-value operation class for fixed chunk key. 61 | */ 62 | class FixedKVOps : public BTreeKVOps { 63 | public: 64 | FixedKVOps(); 65 | FixedKVOps(size_t _ksize, size_t _vsize); 66 | FixedKVOps(size_t _ksize, size_t _vsize, btree_cmp_func _cmp_func); 67 | 68 | virtual ~FixedKVOps() { } 69 | 70 | void init(size_t _ksize, size_t _vsize, btree_cmp_func _cmp_func); 71 | 72 | void getKV(struct bnode *node, idx_t idx, void *key, void *value); 73 | void setKV(struct bnode *node, idx_t idx, void *key, void *value); 74 | void insKV(struct bnode *node, idx_t idx, void *key, void *value); 75 | void copyKV(struct bnode *node_dst, 76 | struct bnode *node_src, 77 | idx_t dst_idx, 78 | idx_t src_idx, 79 | idx_t len); 80 | size_t getDataSize(struct bnode *node, 81 | void *new_minkey, 82 | void *key_arr, 83 | void *value_arr, 84 | size_t len); 85 | size_t getKVSize(void *key, void *value); 86 | void initKVVar(void *key, void *value); 87 | void freeKVVar(void *key, void *value) { } 88 | void setKey(void *dst, void *src); 89 | void setValue(void *dst, void *src); 90 | idx_t getNthIdx(struct bnode *node, idx_t num, idx_t den); 91 | void getNthSplitter(struct bnode *prev_node, 92 | struct bnode *node, 93 | void *key); 94 | 95 | void setVarKey(void *key, void *str, size_t len) { } 96 | void setInfVarKey(void *key) { } 97 | bool isInfVarKey(void *key) { return false; } 98 | void getVarKey(void *key, void *strbuf, size_t& len) { } 99 | void freeVarKey(void *key) { } 100 | }; 101 | 102 | #ifdef __cplusplus 103 | } 104 | #endif 105 | 106 | #endif 107 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_COMMON_H 19 | #define _JSAHN_COMMON_H 20 | 21 | #include 22 | #include 23 | 24 | #undef NDEBUG 25 | #include 26 | 27 | #if !defined(__APPLE__) && !defined(__FreeBSD__) 28 | #include 29 | #endif 30 | 31 | #include "option.h" 32 | #include "arch.h" 33 | #include "debug.h" 34 | #include "bitwise_utils.h" 35 | 36 | // Stolen from http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml 37 | // A macro to disallow the copy constructor and operator= functions 38 | // This should be used in the private: declarations for a class 39 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 40 | TypeName(const TypeName&); \ 41 | void operator=(const TypeName&) 42 | 43 | #define DISALLOW_ASSIGN(TypeName) \ 44 | void operator=(const TypeName&) 45 | 46 | #ifndef _MEMPOOL 47 | #define mempool_alloc malloc 48 | #define mempool_free free 49 | #endif 50 | 51 | #define _MEMORY_OVERRIDE 52 | 53 | #define alca(type, n) ((type*)alloca(sizeof(type) * (n))) 54 | 55 | #define seq_memcpy(dest, src, size, offset_var) \ 56 | memcpy(dest, src, size); \ 57 | offset_var += size 58 | 59 | typedef uint64_t hrtime_t; 60 | typedef uint64_t bid_t; 61 | #define BLK_NOT_FOUND (UINT64_C(0xffffffffffffffff)) 62 | 63 | typedef uint8_t file_status_t; 64 | enum{ 65 | // No compaction now or before or in progress 66 | FILE_NORMAL = 0, 67 | // fdb_compact has begun - switch to using the new_file for new fdb_set()s 68 | FILE_COMPACT_OLD = 1, 69 | // this is a new file in the compaction process 70 | FILE_COMPACT_NEW = 2, 71 | // all open handles on the file has been closed 72 | FILE_CLOSED = 3, 73 | // compaction completed successfully, and file needs to be removed once all 74 | // open handles refering to this file are closed or switched to new_file 75 | FILE_REMOVED_PENDING = 4, 76 | }; 77 | 78 | #define BLK_MARKER_BNODE (0xff) 79 | #define BLK_MARKER_DBHEADER (0xee) 80 | #define BLK_MARKER_DOC (0xdd) 81 | #define BLK_MARKER_SB (0xcc) // superblock 82 | #define BLK_MARKER_SIZE (1) 83 | #define DOCBLK_META_SIZE (16) 84 | #define BMP_REVNUM_MASK 0xffff 85 | 86 | struct docblk_meta { 87 | bid_t next_bid; 88 | uint16_t sb_bmp_revnum_hash; 89 | uint8_t reserved[5]; 90 | uint8_t marker; 91 | }; 92 | 93 | #define randomize() srand((unsigned)time(NULL)) 94 | #define random(num) ((rand())%(num)) 95 | 96 | #define random_custom(prev, num) (prev) = ((prev)+811)&((num)-1) 97 | 98 | 99 | /* Custom assert */ 100 | 101 | void fdb_assert_die(const char* expression, const char* file, int line, 102 | uint64_t val, uint64_t expected); 103 | 104 | #define fdb_assert(cond, val, expected) \ 105 | ((void)((cond) ? (void)0 : fdb_assert_die(#cond, __FILE__, __LINE__,\ 106 | (uint64_t)(val), (uint64_t)(expected)))) 107 | 108 | #endif // _JSAHN_COMMON_H 109 | -------------------------------------------------------------------------------- /tests/include/test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #if !defined(WIN32) && !defined(_WIN32) 23 | #include 24 | #endif 25 | #include "time_utils.h" 26 | 27 | #include "common.h" 28 | 29 | #undef THREAD_SANITIZER 30 | #undef ADDRESS_SANITIZER 31 | #if __clang__ 32 | # if defined(__has_feature) 33 | # if __has_feature(thread_sanitizer) 34 | # define THREAD_SANITIZER 35 | # endif 36 | # if __has_feature(address_sanitizer) 37 | # define ADDRESS_SANITIZER 38 | # endif 39 | # endif 40 | #endif 41 | 42 | #define _TEST_GLOBAL 43 | #ifdef _TEST_GLOBAL 44 | 45 | #define TEST_INIT() \ 46 | static int __test_pass=1; \ 47 | struct timeval __test_begin, __test_prev, __test_cur, __test_interval_total, __test_interval_ins; \ 48 | (void)__test_prev; \ 49 | (void)__test_interval_total; \ 50 | (void)__test_interval_ins; \ 51 | (void)__test_pass; \ 52 | gettimeofday(&__test_begin, NULL); \ 53 | __test_cur = __test_begin 54 | 55 | #define TEST_CHK(cond) {if (!(cond)) {fprintf(stderr, "Test failed: %s %d\n", __FILE__, __LINE__); __test_pass=0; assert(cond);}} 56 | #define TEST_CMP(str1, str2, len) {if (memcmp(str1, str2, len)) {fprintf(stderr, "Test expected %s but got %s failed: %s %d\n", (char*)str2,(char*) str1, __FILE__, __LINE__); __test_pass=0; assert(false);}} 57 | #define TEST_STATUS(status) {fdb_status s = (status); if (s != FDB_RESULT_SUCCESS) {fprintf(stderr, "Test failed with fdb_status %d (%s) at %s %d\n", s, fdb_error_msg(s), __FILE__, __LINE__); __test_pass=0; assert(false);}} 58 | #define TEST_RESULT(name) {if ((__test_pass)) fprintf(stderr, "%s PASSED\n", (name)); else fprintf(stderr, "%s FAILED\n", (name)); } 59 | 60 | #define TEST_TIME() {\ 61 | __test_prev = __test_cur; \ 62 | gettimeofday(&__test_cur, NULL); \ 63 | __test_interval_total = _utime_gap(__test_begin, __test_cur); \ 64 | __test_interval_ins = _utime_gap(__test_prev, __test_cur); \ 65 | DBG("Time elapsed: total %" _FSEC ".%06" _FUSEC " , interval %" _FSEC ".%06" _FUSEC "\n", \ 66 | __test_interval_total.tv_sec, __test_interval_total.tv_usec, \ 67 | __test_interval_ins.tv_sec, __test_interval_ins.tv_usec); } 68 | 69 | 70 | #else 71 | 72 | #define TEST_CHK(cond, sw) {if (!(cond)) {fprintf(stderr, "Test failed: %s %d\n", __FILE__, __LINE__); sw=0; assert(cond);}} 73 | #define TEST_RESULT(name, sw) {if ((sw)) fprintf(stderr, "%s PASSED\n", (name)); else fprintf(stderr, "%s FAILED\n", (name)); } 74 | 75 | #endif 76 | 77 | #if defined(WIN32) || defined(_WIN32) 78 | #define SHELL_DEL "del /f " 79 | #define SHELL_COPY "copy " 80 | #define SHELL_MOVE "move " 81 | #define SHELL_MKDIR "mkdir " 82 | #define SHELL_RMDIR "rd /s/q " 83 | #define SHELL_DMT "\\" 84 | #define SHELL_MAX_PATHLEN (256) 85 | #else 86 | #define SHELL_DEL "rm -rf " 87 | #define SHELL_COPY "cp " 88 | #define SHELL_MOVE "mv " 89 | #define SHELL_MKDIR "mkdir " 90 | #define SHELL_RMDIR SHELL_DEL 91 | #define SHELL_DMT "/" 92 | #define SHELL_MAX_PATHLEN (1024) 93 | #endif 94 | 95 | #include "memleak.h" 96 | 97 | -------------------------------------------------------------------------------- /utils/system_resource_stats.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: David Robert Nadeau 3 | * Site: http://NadeauSoftware.com/ 4 | * License: Creative Commons Attribution 3.0 Unported License 5 | * http://creativecommons.org/licenses/by/3.0/deed.en_US 6 | */ 7 | 8 | #if defined(WIN32) || defined(_WIN32) 9 | #include 10 | #else 11 | #include 12 | #include 13 | #include 14 | #if defined(BSD) 15 | #include 16 | #endif 17 | #endif 18 | 19 | #include "system_resource_stats.h" 20 | 21 | /** 22 | * Returns the size of physical memory (RAM) in bytes. 23 | */ 24 | int64_t get_memory_size(void) 25 | { 26 | #if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__)) 27 | /* Cygwin under Windows. ------------------------------------ */ 28 | /* New 64-bit MEMORYSTATUSEX isn't available. Use old 32.bit */ 29 | MEMORYSTATUS status; 30 | status.dwLength = sizeof(status); 31 | GlobalMemoryStatus( &status ); 32 | return (int64_t)status.dwTotalPhys; 33 | 34 | #elif defined(WIN32) || defined(_WIN32) 35 | /* Windows. ------------------------------------------------- */ 36 | /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */ 37 | MEMORYSTATUSEX status; 38 | status.dwLength = sizeof(status); 39 | GlobalMemoryStatusEx( &status ); 40 | return (int64_t)status.ullTotalPhys; 41 | 42 | #else 43 | /* UNIX variants. ------------------------------------------- */ 44 | /* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */ 45 | 46 | #if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64)) 47 | int mib[2]; 48 | mib[0] = CTL_HW; 49 | #if defined(HW_MEMSIZE) 50 | mib[1] = HW_MEMSIZE; /* OSX. --------------------- */ 51 | #elif defined(HW_PHYSMEM64) 52 | mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */ 53 | #endif 54 | int64_t size = 0; /* 64-bit */ 55 | size_t len = sizeof(size); 56 | if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { 57 | return size; 58 | } 59 | return 0; /* Failed? */ 60 | 61 | #elif defined(_SC_AIX_REALMEM) 62 | /* AIX. ----------------------------------------------------- */ 63 | return (int64_t)(sysconf( _SC_AIX_REALMEM ) * 1024); 64 | 65 | #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) 66 | /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */ 67 | return (int64_t)sysconf( _SC_PHYS_PAGES ) 68 | * (int64_t)sysconf( _SC_PAGESIZE ); 69 | 70 | #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE) 71 | /* Legacy. -------------------------------------------------- */ 72 | return (int64_t)sysconf( _SC_PHYS_PAGES ) 73 | * (int64_t)sysconf( _SC_PAGE_SIZE ); 74 | 75 | #elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM)) 76 | /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */ 77 | int mib[2]; 78 | mib[0] = CTL_HW; 79 | #if defined(HW_REALMEM) 80 | mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */ 81 | #elif defined(HW_PYSMEM) 82 | mib[1] = HW_PHYSMEM; /* Others. ------------------ */ 83 | #endif 84 | unsigned int size = 0; /* 32-bit */ 85 | size_t len = sizeof( size ); 86 | if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { 87 | return (int64_t) size; 88 | } 89 | return 0; /* Failed? */ 90 | #endif /* sysctl and sysconf variants */ 91 | 92 | #endif 93 | } 94 | 95 | size_t get_num_cores(void) { 96 | size_t num_cores; 97 | #if defined(WIN32) || defined(_WIN32) 98 | SYSTEM_INFO sysinfo; 99 | GetSystemInfo(&sysinfo); 100 | num_cores = (size_t)sysinfo.dwNumberOfProcessors; 101 | #else 102 | num_cores = (size_t)sysconf(_SC_NPROCESSORS_ONLN); 103 | #endif 104 | 105 | return num_cores; 106 | } 107 | -------------------------------------------------------------------------------- /src/forestdb_endian.h: -------------------------------------------------------------------------------- 1 | #ifndef _FDB_ENDIAN_H 2 | #define _FDB_ENDIAN_H 3 | 4 | #if defined(WIN32) || defined(_WIN32) 5 | #ifndef _LITTLE_ENDIAN 6 | #define _LITTLE_ENDIAN 7 | #endif 8 | 9 | #elif __APPLE__ 10 | #include 11 | #if BYTE_ORDER == LITTLE_ENDIAN 12 | #ifndef _LITTLE_ENDIAN 13 | #define _LITTLE_ENDIAN 14 | #endif 15 | #elif BYTE_ORDER == BIG_ENDIAN 16 | #ifndef _BIG_ENDIAN 17 | #define _BIG_ENDIAN 18 | #endif 19 | #else 20 | #error "not supported endian" 21 | #endif 22 | 23 | #elif __ANDROID__ 24 | #include 25 | #if _BYTE_ORDER == _LITTLE_ENDIAN 26 | #ifndef _LITTLE_ENDIAN 27 | #define _LITTLE_ENDIAN 28 | #endif 29 | #ifdef _BIG_ENDIAN 30 | #undef _BIG_ENDIAN 31 | #endif 32 | #elif _BYTE_ORDER == _BIG_ENDIAN 33 | #ifndef _BIG_ENDIAN 34 | #define _BIG_ENDIAN 35 | #endif 36 | #ifdef _LITTLE_ENDIAN 37 | #undef _LITTLE_ENDIAN 38 | #endif 39 | #else 40 | #error "not supported endian" 41 | #endif 42 | 43 | #elif __linux__ 44 | #include 45 | #if __BYTE_ORDER == __LITTLE_ENDIAN 46 | #ifndef _LITTLE_ENDIAN 47 | #define _LITTLE_ENDIAN 48 | #endif 49 | #elif __BYTE_ORDER == __BIG_ENDIAN 50 | #ifndef _BIG_ENDIAN 51 | #define _BIG_ENDIAN 52 | #endif 53 | #else 54 | #error "not supported endian" 55 | #endif 56 | 57 | #elif __FreeBSD__ 58 | #include 59 | #if _BYTE_ORDER == _LITTLE_ENDIAN 60 | #ifdef _BIG_ENDIAN 61 | #undef _BIG_ENDIAN 62 | #endif 63 | #elif _BYTE_ORDER == _BIG_ENDIAN 64 | #ifdef _LITTLE_ENDIAN 65 | #undef _LITTLE_ENDIAN 66 | #endif 67 | #else 68 | #error "not supported endian" 69 | #endif 70 | #endif 71 | 72 | #ifndef bitswap64 73 | #define bitswap64(v) \ 74 | ( (((v) & 0xff00000000000000ULL) >> 56) \ 75 | | (((v) & 0x00ff000000000000ULL) >> 40) \ 76 | | (((v) & 0x0000ff0000000000ULL) >> 24) \ 77 | | (((v) & 0x000000ff00000000ULL) >> 8) \ 78 | | (((v) & 0x00000000ff000000ULL) << 8) \ 79 | | (((v) & 0x0000000000ff0000ULL) << 24) \ 80 | | (((v) & 0x000000000000ff00ULL) << 40) \ 81 | | (((v) & 0x00000000000000ffULL) << 56) ) 82 | #endif 83 | 84 | #ifndef bitswap32 85 | #define bitswap32(v) \ 86 | ( (((v) & 0xff000000) >> 24) \ 87 | | (((v) & 0x00ff0000) >> 8) \ 88 | | (((v) & 0x0000ff00) << 8) \ 89 | | (((v) & 0x000000ff) << 24) ) 90 | #endif 91 | 92 | #ifndef bitswap16 93 | #define bitswap16(v) \ 94 | ( (((v) & 0xff00) >> 8) \ 95 | | (((v) & 0x00ff) << 8) ) 96 | #endif 97 | 98 | #if defined(_LITTLE_ENDIAN) 99 | // convert to big endian 100 | #define _enc64(v) bitswap64(v) 101 | #define _dec64(v) bitswap64(v) 102 | #define _enc32(v) bitswap32(v) 103 | #define _dec32(v) bitswap32(v) 104 | #define _enc16(v) bitswap16(v) 105 | #define _dec16(v) bitswap16(v) 106 | #else 107 | // big endian .. do nothing 108 | #define _enc64(v) (v) 109 | #define _dec64(v) (v) 110 | #define _enc32(v) (v) 111 | #define _dec32(v) (v) 112 | #define _enc16(v) (v) 113 | #define _dec16(v) (v) 114 | #endif 115 | 116 | #ifdef __ENDIAN_SAFE 117 | #define _endian_encode(v) \ 118 | ((sizeof(v) == 8)?(_enc64(v)):( \ 119 | (sizeof(v) == 4)?(_enc32(v)):( \ 120 | (sizeof(v) == 2)?(_enc16(v)):(v)))) 121 | #define _endian_decode(v) \ 122 | ((sizeof(v) == 8)?(_dec64(v)):( \ 123 | (sizeof(v) == 4)?(_dec32(v)):( \ 124 | (sizeof(v) == 2)?(_dec16(v)):(v)))) 125 | #else 126 | #define _endian_encode(v) (v) 127 | #define _endian_decode(v) (v) 128 | #endif 129 | 130 | #endif 131 | -------------------------------------------------------------------------------- /src/executorthread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef SRC_SCHEDULER_H_ 18 | #define SRC_SCHEDULER_H_ 1 19 | 20 | #include "common.h" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "tasks.h" 32 | #include "task_type.h" 33 | 34 | #define LOG(...) 35 | #define MIN_SLEEP_TIME 2.0 36 | 37 | class ExecutorPool; 38 | class ExecutorThread; 39 | class TaskQueue; 40 | class WorkLoadPolicy; 41 | 42 | enum executor_state_t { 43 | EXECUTOR_RUNNING, 44 | EXECUTOR_WAITING, 45 | EXECUTOR_SLEEPING, 46 | EXECUTOR_SHUTDOWN, 47 | EXECUTOR_DEAD 48 | }; 49 | 50 | 51 | class ExecutorThread { 52 | friend class ExecutorPool; 53 | friend class TaskQueue; 54 | public: 55 | 56 | ExecutorThread(ExecutorPool *m, int startingQueue, 57 | const std::string nm) : manager(m), 58 | startIndex(startingQueue), name(nm), 59 | state(EXECUTOR_RUNNING), taskStart(0), 60 | currentTask(NULL), curTaskType(NO_TASK_TYPE) { 61 | now = gethrtime(); 62 | waketime = hrtime_t(-1); 63 | } 64 | 65 | ~ExecutorThread() { 66 | LOG(EXTENSION_LOG_INFO, "Executor killing %s", name.c_str()); 67 | } 68 | 69 | void start(void); 70 | 71 | void run(void); 72 | 73 | void stop(bool wait=true); 74 | 75 | void schedule(ExTask &task); 76 | 77 | void reschedule(ExTask &task); 78 | 79 | void wake(ExTask &task); 80 | 81 | // Changes this threads' current task to the specified task 82 | void setCurrentTask(ExTask newTask); 83 | 84 | const std::string& getName() const { return name; } 85 | 86 | const std::string getTaskName() { 87 | LockHolder lh(currentTaskMutex); 88 | if (currentTask) { 89 | return currentTask->getDescription(); 90 | } else { 91 | return std::string("Not currently running any task"); 92 | } 93 | } 94 | 95 | const std::string getTaskableName() { 96 | LockHolder lh(currentTaskMutex); 97 | if (currentTask) { 98 | return currentTask->getTaskable().getName(); 99 | } else { 100 | return std::string(); 101 | } 102 | } 103 | 104 | hrtime_t getTaskStart() const { return taskStart; } 105 | 106 | const std::string getStateName(); 107 | 108 | const hrtime_t getWaketime(void) { return waketime; } 109 | 110 | const hrtime_t getCurTime(void) { return now; } 111 | 112 | protected: 113 | 114 | thread_t thread; 115 | ExecutorPool *manager; 116 | int startIndex; 117 | const std::string name; 118 | std::atomic state; 119 | 120 | std::atomic now; // record of current time 121 | std::atomic waketime; // set to the earliest 122 | 123 | std::atomic taskStart; 124 | 125 | std::mutex currentTaskMutex; // Protects currentTask 126 | ExTask currentTask; 127 | 128 | task_type_t curTaskType; 129 | 130 | std::mutex logMutex; 131 | }; 132 | 133 | #endif // SRC_SCHEDULER_H_ 134 | -------------------------------------------------------------------------------- /tests/anomaly/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # every file same as FORESTDB_CORE_SRC execept filemgr_ops.cc replaced 2 | # with filemgr_anomalous_ops.cc 3 | SET(FORESTDB_COMMON_CORE_SRC 4 | ${PROJECT_SOURCE_DIR}/src/api_wrapper.cc 5 | ${PROJECT_SOURCE_DIR}/src/avltree.cc 6 | ${PROJECT_SOURCE_DIR}/src/bgflusher.cc 7 | ${PROJECT_SOURCE_DIR}/src/blockcache.cc 8 | ${PROJECT_SOURCE_DIR}/src/bnode.cc 9 | ${PROJECT_SOURCE_DIR}/src/bnodecache.cc 10 | ${PROJECT_SOURCE_DIR}/${BREAKPAD_SRC} 11 | ${PROJECT_SOURCE_DIR}/src/btree.cc 12 | ${PROJECT_SOURCE_DIR}/src/btree_kv.cc 13 | ${PROJECT_SOURCE_DIR}/src/btree_new.cc 14 | ${PROJECT_SOURCE_DIR}/src/bnode.cc 15 | ${PROJECT_SOURCE_DIR}/src/bnodemgr.cc 16 | ${PROJECT_SOURCE_DIR}/src/bnodecache.cc 17 | ${PROJECT_SOURCE_DIR}/src/btree_fast_str_kv.cc 18 | ${PROJECT_SOURCE_DIR}/src/btreeblock.cc 19 | ${PROJECT_SOURCE_DIR}/src/checksum.cc 20 | ${PROJECT_SOURCE_DIR}/src/compaction.cc 21 | ${PROJECT_SOURCE_DIR}/src/compactor.cc 22 | ${PROJECT_SOURCE_DIR}/src/configuration.cc 23 | ${PROJECT_SOURCE_DIR}/src/docio.cc 24 | ${PROJECT_SOURCE_DIR}/src/encryption.cc 25 | ${PROJECT_SOURCE_DIR}/src/encryption_aes.cc 26 | ${PROJECT_SOURCE_DIR}/src/encryption_bogus.cc 27 | ${PROJECT_SOURCE_DIR}/src/executorpool.cc 28 | ${PROJECT_SOURCE_DIR}/src/executorthread.cc 29 | ${PROJECT_SOURCE_DIR}/src/fdb_errors.cc 30 | ${PROJECT_SOURCE_DIR}/src/filemgr.cc 31 | ${PROJECT_SOURCE_DIR}/src/file_handle.cc 32 | ${PROJECT_SOURCE_DIR}/src/forestdb.cc 33 | ${PROJECT_SOURCE_DIR}/src/hash.cc 34 | ${PROJECT_SOURCE_DIR}/src/hash_functions.cc 35 | ${PROJECT_SOURCE_DIR}/src/hbtrie.cc 36 | ${PROJECT_SOURCE_DIR}/src/iterator.cc 37 | ${PROJECT_SOURCE_DIR}/src/kvs_handle.cc 38 | ${PROJECT_SOURCE_DIR}/src/kv_instance.cc 39 | ${PROJECT_SOURCE_DIR}/src/list.cc 40 | ${PROJECT_SOURCE_DIR}/src/memory_pool.cc 41 | ${PROJECT_SOURCE_DIR}/src/staleblock.cc 42 | ${PROJECT_SOURCE_DIR}/src/superblock.cc 43 | ${PROJECT_SOURCE_DIR}/src/taskqueue.cc 44 | ${PROJECT_SOURCE_DIR}/src/transaction.cc 45 | ${PROJECT_SOURCE_DIR}/src/version.cc 46 | ${PROJECT_SOURCE_DIR}/src/wal.cc) 47 | 48 | add_library(FDB_TOOLS_CCORE OBJECT ${FORESTDB_COMMON_CORE_SRC}) 49 | set_target_properties(FDB_TOOLS_CCORE PROPERTIES 50 | COMPILE_FLAGS "-D_FDB_TOOLS ${CB_GNU_CXX11_OPTION}") 51 | 52 | add_executable(fdb_anomaly_test 53 | $ 54 | filemgr_anomalous_ops.cc 55 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 56 | fdb_anomaly_test.cc 57 | ${GETTIMEOFDAY_VS} 58 | $) 59 | target_link_libraries(fdb_anomaly_test ${PTHREAD_LIB} ${LIBM} 60 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 61 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 62 | ${LIBRT} ${CRYPTO_LIB} 63 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 64 | set_target_properties(fdb_anomaly_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 65 | 66 | add_executable(disk_sim_test 67 | $ 68 | filemgr_anomalous_ops.cc 69 | ${PROJECT_SOURCE_DIR}/${FORESTDB_FILE_OPS} 70 | disk_sim_test.cc 71 | ${GETTIMEOFDAY_VS} 72 | $) 73 | target_link_libraries(disk_sim_test ${PTHREAD_LIB} ${LIBM} 74 | ${SNAPPY_LIBRARIES} ${ASYNC_IO_LIB} 75 | ${MALLOC_LIBRARIES} ${PLATFORM_LIBRARY} 76 | ${LIBRT} ${CRYPTO_LIB} 77 | ${DL_LIBRARIES} ${BREAKPAD_LIBRARIES}) 78 | set_target_properties(disk_sim_test PROPERTIES COMPILE_FLAGS "-D_FDB_TOOLS") 79 | 80 | # add test target 81 | add_test(fdb_anomaly_test fdb_anomaly_test) 82 | add_test(disk_sim_test disk_sim_test) 83 | 84 | ADD_CUSTOM_TARGET(anomaly_tests 85 | COMMAND ctest 86 | ) 87 | -------------------------------------------------------------------------------- /tests/e2e/e2espec.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "libforestdb/forestdb.h" 19 | 20 | //#define __DEBUG_E2E 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | static const char E2EDB_MAIN[] = "e2edb_main"; 27 | static const char E2EDB_RECORDS[] = "e2edb_records"; 28 | static const char E2EKV_ALLDOCS[] = "e2ekv_alldocs"; 29 | static const char E2EKV_INDEX1[] = "e2ekv_index1"; 30 | static const char E2EKV_INDEX2[] = "e2ekv_index2"; 31 | static const char E2EKV_RECORDS[] = "e2ekv_rtx"; 32 | static const char E2EKV_CHECKPOINTS[] = "e2ekv_chk"; 33 | static const char RECORD_DOCKEY[] = "e2edoc_records"; 34 | static const int KEYSPACE_LEN = 6; 35 | static const int MAXKEY_LEN = 1024; 36 | static const int MAXITR_LEN = 12; 37 | 38 | static const int LOAD_FACTOR = 1; 39 | 40 | typedef uint16_t tx_type_t; 41 | enum { 42 | SET_PERSON = 0x01, 43 | DEL_PERSON = 0x02, 44 | ACC_DEPOSIT = 0x03, 45 | ACC_WITHDRAW = 0x04, 46 | START_CHECKPOINT = 0x05, 47 | END_CHECKPOINT = 0x06 48 | }; 49 | 50 | typedef struct { 51 | char key[MAXKEY_LEN]; 52 | char name[MAXKEY_LEN]; 53 | char city[256]; 54 | char state[256]; 55 | char desc[1024]; 56 | char keyspace[KEYSPACE_LEN]; 57 | int age; 58 | }person_t; 59 | 60 | typedef struct { 61 | char min[MAXITR_LEN]; 62 | char max[MAXITR_LEN]; 63 | }idx_prams_t; 64 | 65 | typedef struct { 66 | int amount; 67 | char refkey[MAXKEY_LEN]; 68 | size_t refkey_len; 69 | tx_type_t type; 70 | }transaction_t; 71 | 72 | typedef struct { 73 | char key[MAXKEY_LEN]; 74 | int ndocs; 75 | int num_indexed; 76 | int sum_age_indexed; 77 | uint16_t balance; 78 | fdb_seqnum_t seqnum_all; 79 | fdb_seqnum_t seqnum_idx1; 80 | fdb_seqnum_t seqnum_idx2; 81 | tx_type_t type; 82 | }checkpoint_t; 83 | 84 | 85 | typedef struct { 86 | 87 | fdb_file_handle *main; 88 | fdb_kvs_handle *all_docs; 89 | fdb_kvs_handle *index1; 90 | fdb_kvs_handle *index2; 91 | 92 | fdb_file_handle *records; 93 | fdb_kvs_handle *rtx; 94 | fdb_kvs_handle *chk; 95 | checkpoint_t *v_chk; 96 | idx_prams_t *index_params; 97 | 98 | char keyspace[KEYSPACE_LEN]; 99 | bool walflush; 100 | bool verify_set; 101 | }storage_t; 102 | 103 | // generators 104 | void gen_random(char *s, const int len); 105 | void gen_index_params(idx_prams_t *params); 106 | void gen_person(person_t *p); 107 | 108 | // fdb wrappers 109 | void e2e_fdb_set_person(storage_t *st, person_t *p); 110 | void e2e_fdb_del_person(storage_t *st, person_t *p); 111 | void e2e_fdb_cancel_checkpoint(storage_t *st); 112 | void e2e_fdb_commit(fdb_file_handle* fhandle, bool walflush); 113 | void e2e_fdb_close(storage_t *st); 114 | void e2e_fdb_shutdown(storage_t *st); 115 | 116 | // checkpointing 117 | void start_checkpoint(storage_t *st); 118 | void end_checkpoint(storage_t *st); 119 | checkpoint_t* create_checkpoint(storage_t *st, tx_type_t type); 120 | 121 | // storage 122 | storage_t *init_storage(fdb_config *m_fconfig, fdb_config *r_fconfig, 123 | fdb_kvs_config *kvs_config, idx_prams_t *idxp, 124 | checkpoint_t *v_chk, bool walflush); 125 | 126 | void reset_storage_index(storage_t *st); 127 | void rm_storage_fs(); 128 | 129 | // utility 130 | bool is_indexed(idx_prams_t *idxp, person_t *p); 131 | void save_tx(storage_t *st, void *key, size_t keylen, tx_type_t type); 132 | fdb_kvs_handle *scan(storage_t *st, fdb_kvs_handle *reuse_kv); 133 | 134 | #ifdef __cplusplus 135 | } 136 | #endif 137 | -------------------------------------------------------------------------------- /src/checksum.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * ForestDB evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | 27 | #include 28 | #include 29 | #ifdef _CRC32C 30 | // Linking with platform for crc32c 31 | # include 32 | #else 33 | #include 34 | // Define a crc32c which does nothing but assert. 35 | static uint32_t crc32c(const uint8_t* buf, 36 | size_t buf_len, 37 | uint32_t pre) { 38 | // Non couchbase builds are configured to never turn on crc32c. 39 | assert(false); 40 | return 0; 41 | } 42 | #endif 43 | # include "checksum.h" 44 | #include "crc32.h" 45 | 46 | /* 47 | * Get a checksum of buf for buf_len bytes. 48 | * 49 | * mode = UNKNOWN is an invalid input (triggers assert). 50 | */ 51 | uint32_t get_checksum(const uint8_t* buf, 52 | size_t buf_len, 53 | uint32_t pre, 54 | crc_mode_e mode) { 55 | if (mode == CRC32C) { 56 | return crc32c(buf, buf_len, pre); 57 | } else { 58 | assert(mode == CRC32); 59 | return crc32_8((void *)buf, buf_len, pre); 60 | } 61 | } 62 | 63 | /* 64 | * Get a checksum of buf for buf_len bytes. 65 | * 66 | * The pre value is set to 0. 67 | * 68 | * mode = UNKNOWN is an invalid input (triggers assert). 69 | */ 70 | uint32_t get_checksum(const uint8_t* buf, 71 | size_t buf_len, 72 | crc_mode_e mode) { 73 | return get_checksum(buf, buf_len, 0, mode); 74 | } 75 | 76 | /* 77 | * Perform an integrity check of buf for buf_len bytes. 78 | * 79 | * A checksum of buf is created and compared against checksum argument. 80 | * 81 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 82 | * 83 | * 84 | * Returns true success (checksums match), false for failure. 85 | */ 86 | bool perform_integrity_check(const uint8_t* buf, 87 | size_t buf_len, 88 | uint32_t checksum, 89 | crc_mode_e mode) { 90 | bool success = false; 91 | #ifdef _CRC32C 92 | if (mode == CRC_UNKNOWN || mode == CRC32C) { 93 | success = checksum == crc32c(buf, buf_len, 0); 94 | if (!success && mode == CRC_UNKNOWN) { 95 | success = checksum == crc32_8((void *)buf, buf_len, 0); 96 | } 97 | } else 98 | #endif 99 | { 100 | success = checksum == crc32_8((void *)buf, buf_len, 0); 101 | } 102 | return success; 103 | } 104 | 105 | /* 106 | * Detect the CRC mode by performing an integrity check against the 107 | * two CRC functions ForestDB files could be written with. 108 | * 109 | * Returns true if a match is found and sets mode to the correct mode. 110 | * Returns false if no match and sets mode to CRC_UNKNOWN. 111 | */ 112 | bool detect_and_check_crc(const uint8_t* buf, 113 | size_t buf_len, 114 | uint32_t checksum, 115 | crc_mode_e* mode) { 116 | *mode = CRC_UNKNOWN; 117 | #ifdef _CRC32C 118 | if (perform_integrity_check(buf, buf_len, checksum, CRC32C)) { 119 | *mode = CRC32C; 120 | return true; 121 | } else 122 | #endif 123 | if (perform_integrity_check(buf, buf_len, checksum, CRC32)) { 124 | *mode = CRC32; 125 | return true; 126 | } 127 | 128 | return false; 129 | } 130 | -------------------------------------------------------------------------------- /tests/unit/docio_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "docio.h" 23 | #include "filemgr.h" 24 | #include "filemgr_ops.h" 25 | #include "test.h" 26 | 27 | uint32_t _set_doc(struct docio_object *doc, char *key, char *meta, char *body) 28 | { 29 | strcpy((char*)doc->key, key); 30 | doc->length.keylen = strlen((char*)doc->key) + 1; 31 | strcpy((char*)doc->meta, meta); 32 | doc->length.metalen = strlen((char*)doc->meta) + 1; 33 | strcpy((char*)doc->body, body); 34 | doc->length.bodylen = strlen((char*)doc->body) + 1; 35 | 36 | return sizeof(struct docio_length) + doc->length.keylen + doc->length.metalen + doc->length.bodylen; 37 | } 38 | 39 | void basic_test() 40 | { 41 | TEST_INIT(); 42 | 43 | uint64_t offset; 44 | uint32_t docsize; 45 | int r; 46 | int blocksize = 128; 47 | FileMgr *file; 48 | char keybuf[1024]; 49 | char metabuf[1024]; 50 | char bodybuf[4096]; 51 | struct docio_object doc; 52 | FileMgrConfig config(blocksize, 1024, 1048576, 0, 0, FILEMGR_CREATE, 53 | FDB_SEQTREE_NOT_USE, 0, 8, 0, FDB_ENCRYPTION_NONE, 54 | 0x00, 0, 0); 55 | std::string fname("./docio_testfile"); 56 | 57 | doc.key = (void*)keybuf; 58 | doc.meta = (void*)metabuf; 59 | doc.body = (void*)bodybuf; 60 | 61 | r = system(SHELL_DEL " docio_testfile"); 62 | (void)r; 63 | filemgr_open_result result = FileMgr::open(fname, get_filemgr_ops(), &config, NULL); 64 | file = result.file; 65 | DocioHandle handle(file, false, NULL); 66 | 67 | docsize = _set_doc(&doc, (char *) "this_is_key", (char *) "this_is_metadata", 68 | (char *) "this_is_body_lawiefjaawleif"); 69 | (void)docsize; 70 | offset = handle.appendDoc_Docio(&doc, 0, 0); 71 | (void)offset; 72 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 73 | 74 | docsize = _set_doc(&doc, (char *) "this_is_key2", (char *) "this_is_metadata2", 75 | (char *) "hello_world"); 76 | (void)docsize; 77 | offset = handle.appendDoc_Docio(&doc, 0, 0); 78 | (void)offset; 79 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 80 | 81 | docsize = _set_doc(&doc, (char *) "key3", (char *) "a", (char *) "b"); 82 | (void)docsize; 83 | offset = handle.appendDoc_Docio(&doc, 0, 0); 84 | (void)offset; 85 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 86 | 87 | docsize = _set_doc(&doc, (char *) "key4", (char *) "a", (char *) "b"); 88 | (void)docsize; 89 | offset = handle.appendDoc_Docio(&doc, 0, 0); 90 | (void)offset; 91 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 92 | 93 | docsize = _set_doc(&doc, (char *) "key5", (char *) "a", (char *) "b"); 94 | (void)docsize; 95 | offset = handle.appendDoc_Docio(&doc, 0, 0); 96 | (void)offset; 97 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 98 | 99 | doc.length.keylen = 1; 100 | doc.length.metalen = 1; 101 | doc.length.bodylen = 190; 102 | docsize = 12 + 182; 103 | (void)docsize; 104 | offset = handle.appendDoc_Docio(&doc, 0, 0); 105 | (void)offset; 106 | DBG("docsize %d written at %" _F64 "\n", docsize, offset); 107 | 108 | keylen_t keylen; 109 | handle.readDocKey_Docio(81, &keylen, (void*)keybuf); 110 | DBG("keylen %d %s\n", keylen, keybuf); 111 | 112 | file->commit_FileMgr(true, NULL); 113 | FileMgr::close(file, true, NULL, NULL); 114 | 115 | TEST_RESULT("basic test"); 116 | } 117 | 118 | int main() 119 | { 120 | #ifdef _MEMPOOL 121 | mempool_init(); 122 | #endif 123 | 124 | 125 | basic_test(); 126 | 127 | return 0; 128 | } 129 | -------------------------------------------------------------------------------- /src/globaltask.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "common.h" 21 | #include "atomic.h" 22 | #include "task_priority.h" 23 | 24 | enum task_state_t { 25 | TASK_RUNNING, 26 | TASK_SNOOZED, 27 | TASK_DEAD 28 | }; 29 | 30 | enum class TaskId : int { 31 | NOTASK=0 // TODO: implement mapping from taskId to strigified name 32 | }; 33 | 34 | class Taskable; 35 | 36 | class GlobalTask : public RCValue { 37 | friend class CompareByDueDate; 38 | friend class CompareByPriority; 39 | friend class ExecutorPool; 40 | friend class ExecutorThread; 41 | public: 42 | 43 | GlobalTask(Taskable& t, const Priority &p, 44 | double sleeptime = 0, bool completeBeforeShutdown = true); 45 | 46 | /* destructor */ 47 | virtual ~GlobalTask(void) { 48 | } 49 | 50 | /** 51 | * The invoked function when the task is executed. 52 | * 53 | * @return Whether or not this task should be rescheduled 54 | */ 55 | virtual bool run(void) = 0; 56 | 57 | /** 58 | * Gives a description of this task. 59 | * 60 | * @return A description of this task 61 | */ 62 | virtual std::string getDescription(void) = 0; 63 | 64 | virtual int maxExpectedDuration(void) { 65 | return 3600; 66 | } 67 | 68 | /** 69 | * test if a task is dead 70 | */ 71 | bool isdead(void) { 72 | return (state == TASK_DEAD); 73 | } 74 | 75 | 76 | /** 77 | * Cancels this task by marking it dead. 78 | */ 79 | void cancel(void) { 80 | state = TASK_DEAD; 81 | } 82 | 83 | /** 84 | * Puts the task to sleep for a given duration. 85 | */ 86 | virtual void snooze(const double secs); 87 | 88 | /** 89 | * Returns the id of this task. 90 | * 91 | * @return A unique task id number. 92 | */ 93 | size_t getId() const { return taskId; } 94 | 95 | /** 96 | * Returns the type id of this task. 97 | * 98 | * @return A type id of the task. 99 | */ 100 | type_id_t getTypeId() { return priority.getTypeId(); } 101 | 102 | task_state_t getState(void) { 103 | return state.load(); 104 | } 105 | 106 | void setState(task_state_t tstate, task_state_t expected) { 107 | state.compare_exchange_strong(expected, tstate); 108 | } 109 | 110 | Taskable& getTaskable() const { 111 | return taskable; 112 | } 113 | 114 | hrtime_t getWaketime() const { 115 | return waketime.load(); 116 | } 117 | 118 | void updateWaketime(hrtime_t to) { 119 | waketime.store(to); 120 | } 121 | 122 | void updateWaketimeIfLessThan(hrtime_t to) { 123 | atomic_setIfBigger(waketime, to); 124 | } 125 | 126 | protected: 127 | const Priority &priority; 128 | bool blockShutdown; 129 | std::atomic state; 130 | const size_t taskId; 131 | Taskable& taskable; 132 | 133 | static std::atomic task_id_counter; 134 | static size_t nextTaskId() { return task_id_counter.fetch_add(1); } 135 | 136 | 137 | private: 138 | std::atomic waketime; // used for priority_queue 139 | }; 140 | 141 | typedef SingleThreadedRCPtr ExTask; 142 | 143 | /** 144 | * Order tasks by their priority and taskId (try to ensure FIFO) 145 | */ 146 | class CompareByPriority { 147 | public: 148 | bool operator()(ExTask &t1, ExTask &t2) { 149 | return (t1->priority == t2->priority) ? 150 | (t1->taskId > t2->taskId) : 151 | (t1->priority < t2->priority); 152 | } 153 | }; 154 | 155 | /** 156 | * Order tasks by their ready date. 157 | */ 158 | class CompareByDueDate { 159 | public: 160 | bool operator()(ExTask &t1, ExTask &t2) { 161 | return t2->waketime < t1->waketime; 162 | } 163 | }; 164 | -------------------------------------------------------------------------------- /src/futurequeue.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * The FutureQueue provides a std::priority_queue style interface 20 | * onto a queue of ExTask objects that are sorted by the tasks wakeTime. 21 | * The lowest wakeTime will be the top() task. 22 | * 23 | * FutureQueue provides methods that allow a task's wakeTime to be mutated 24 | * whilst maintaining the priority ordering. 25 | */ 26 | 27 | #pragma once 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | #include "tasks.h" 34 | 35 | template , 36 | class Compare = CompareByDueDate> 37 | class FutureQueue { 38 | public: 39 | 40 | void push(ExTask task) { 41 | LockHolder lock(queueMutex); 42 | queue.push(task); 43 | } 44 | 45 | void pop() { 46 | LockHolder lock(queueMutex); 47 | queue.pop(); 48 | } 49 | 50 | ExTask top() { 51 | LockHolder lock(queueMutex); 52 | return queue.top(); 53 | } 54 | 55 | size_t size() { 56 | LockHolder lock(queueMutex); 57 | return queue.size(); 58 | } 59 | 60 | bool empty() { 61 | LockHolder lock(queueMutex); 62 | return queue.empty(); 63 | } 64 | 65 | /* 66 | * Update the wakeTime of task and ensure the heap property is 67 | * maintained. 68 | * @returns true if 'task' is in the FutureQueue. 69 | */ 70 | bool updateWaketime(const ExTask& task, hrtime_t newTime) { 71 | LockHolder lock(queueMutex); 72 | task->updateWaketime(newTime); 73 | // After modifiying the task's wakeTime, rebuild the heap 74 | return queue.heapify(task); 75 | } 76 | 77 | /* 78 | * snooze the task (by altering its wakeTime) and ensure the 79 | * heap property is maintained. 80 | * @returns true if 'task' is in the FutureQueue. 81 | */ 82 | bool snooze(const ExTask& task, const double secs) { 83 | LockHolder lock(queueMutex); 84 | task->snooze(secs); 85 | // After modifiying the task's wakeTime, rebuild the heap 86 | return queue.heapify(task); 87 | } 88 | 89 | protected: 90 | 91 | /* 92 | * HeapifiableQueue exposes a method to maintain the heap ordering 93 | * of the underlying queue. 94 | * 95 | * This class is deliberately hidden inside FutureQueue so that any 96 | * extensions made to priority_queue can't be accessed without work. 97 | * I.e. correct locking and any need to 'heapify'. 98 | */ 99 | class HeapifiableQueue : public std::priority_queue { 100 | public: 101 | /* 102 | * Ensure the heap property is maintained 103 | * @returns true if 'task' is in the queue and heapify() did something. 104 | */ 105 | bool heapify(const ExTask& task) { 106 | // if the task exists, rebuild 107 | if (exists(task)) { 108 | if (this->c.back()->getId() == task->getId()) { 109 | std::push_heap(this->c.begin(), 110 | this->c.end(), 111 | this->comp); 112 | } else { 113 | std::make_heap(this->c.begin(), 114 | this->c.end(), 115 | this->comp); 116 | } 117 | return true; 118 | } else { 119 | return false; 120 | } 121 | } 122 | 123 | protected: 124 | bool exists(const ExTask& task) { 125 | return std::find_if(this->c.begin(), 126 | this->c.end(), 127 | [&task](const ExTask& qTask) { 128 | return task->getId() == qTask->getId(); 129 | }) != this->c.end(); 130 | } 131 | 132 | } queue; 133 | 134 | // All access to queue must be done with the queueMutex 135 | std::mutex queueMutex; 136 | }; 137 | -------------------------------------------------------------------------------- /src/file_handle.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include "arch.h" 23 | #include "common.h" 24 | #include "internal_types.h" 25 | #include "list.h" 26 | #include "staleblock.h" 27 | 28 | class FdbKvsHandle; 29 | 30 | // list element for custom cmp functions in FdbFileHandle 31 | struct cmp_func_node { 32 | char *kvs_name; 33 | fdb_custom_cmp_variable func; 34 | struct list_elem le; 35 | }; 36 | 37 | // list element for opened KV store handles 38 | // (in-memory data: managed by the file handle) 39 | struct kvs_opened_node { 40 | FdbKvsHandle *handle; 41 | struct list_elem le; 42 | }; 43 | 44 | #define FHANDLE_ROOT_OPENED (0x1) 45 | #define FHANDLE_ROOT_INITIALIZED (0x2) 46 | #define FHANDLE_ROOT_CUSTOM_CMP (0x4) 47 | 48 | /** 49 | * ForestDB file handle definition. 50 | */ 51 | class FdbFileHandle { 52 | public: 53 | FdbFileHandle(); 54 | 55 | FdbFileHandle(FdbKvsHandle *_root); 56 | 57 | ~FdbFileHandle(); 58 | 59 | FdbKvsHandle* getRootHandle() const { 60 | return root; 61 | } 62 | 63 | struct list *getHandleList() const { 64 | return handles; 65 | } 66 | 67 | struct list *getCmpFunctionList() const { 68 | return cmpFuncList; 69 | } 70 | 71 | fdb_custom_cmp_variable getCmpFunctionByName(char *kvs_name); 72 | 73 | uint64_t getFlags() const { 74 | return flags; 75 | } 76 | 77 | void setRootHandle(FdbKvsHandle* _root) { 78 | root = _root; 79 | } 80 | 81 | void setFlags(uint64_t _flags) { 82 | spin_lock(&lock); 83 | flags = _flags; 84 | spin_unlock(&lock); 85 | } 86 | 87 | void setCmpFunctionList(size_t n_func, 88 | char **kvs_names, 89 | fdb_custom_cmp_variable *functions); 90 | 91 | void setCmpFunctionList(struct list *cmp_func_list); 92 | 93 | void addCmpFunction(char *kvs_name, 94 | fdb_custom_cmp_variable cmp_func); 95 | 96 | void addKVHandle(struct list_elem *kv_handle) { 97 | spin_lock(&lock); 98 | list_push_back(handles, kv_handle); 99 | spin_unlock(&lock); 100 | } 101 | 102 | /** 103 | * Create new node for 'handle' & link it in the file's list of KVS handles 104 | */ 105 | struct kvs_opened_node *createNLinkKVHandle(FdbKvsHandle *handle); 106 | 107 | void removeKVHandle(struct list_elem *kv_handle) { 108 | spin_lock(&lock); 109 | list_remove(handles, kv_handle); 110 | spin_unlock(&lock); 111 | } 112 | 113 | bool activateRootHandle(const char *kvs_name, fdb_kvs_config &config); 114 | 115 | fdb_status closeAllKVHandles(); 116 | 117 | /** 118 | * Check if any handle for a given KV store Id is still active. 119 | * 120 | * @param kv_id Id of a given KV store 121 | * @return true if there is at least one active handle for a given KV store Id. 122 | */ 123 | bool checkAnyActiveKVHandle(fdb_kvs_id_t kv_id); 124 | 125 | /** 126 | * Check if the KV handle list is empty or not. 127 | * 128 | * @return true if the KV handle list is empty. 129 | */ 130 | bool isKVHandleListEmpty(); 131 | 132 | /** 133 | * Return the oldest stable commit header that is still being accessed by 134 | * any KV store handles created with this file handle. 135 | */ 136 | stale_header_info getOldestActiveHeader(); 137 | 138 | private: 139 | /** 140 | * The root KV store handle. 141 | */ 142 | FdbKvsHandle *root; 143 | /** 144 | * List of opened default KV store handles 145 | * (except for the root handle). 146 | */ 147 | struct list *handles; 148 | /** 149 | * List of custom compare functions assigned by user 150 | */ 151 | struct list *cmpFuncList; 152 | /** 153 | * Flags for the file handle. 154 | */ 155 | uint64_t flags; 156 | /** 157 | * Spin lock for the file handle. 158 | */ 159 | spin_t lock; 160 | 161 | DISALLOW_COPY_AND_ASSIGN(FdbFileHandle); 162 | }; 163 | 164 | -------------------------------------------------------------------------------- /tests/unit/mempool_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "atomic.h" 25 | #include "memory_pool.h" 26 | 27 | #include "test.h" 28 | #include "stat_aggregator.h" 29 | 30 | int samples(0); 31 | static std::mutex guard; 32 | 33 | void collect_stat(StatAggregator *sa, uint64_t diff) { 34 | LockHolder lh(guard); 35 | sa->t_stats[0][0].latencies.push_back(diff); 36 | ++samples; 37 | } 38 | 39 | struct worker_args{ 40 | int num_runs; 41 | int num_bins; 42 | size_t bin_size; 43 | MemoryPool *mp; 44 | StatAggregator *sa; 45 | }; 46 | 47 | void *basic_tester(void *args_) 48 | { 49 | TEST_INIT(); 50 | struct worker_args *args = (struct worker_args *)args_; 51 | size_t bin_size = args->bin_size; 52 | for (int i = args->num_runs; i; --i) { 53 | uint8_t *buf; 54 | ts_nsec start = get_monotonic_ts(); 55 | const int idx = args->mp->fetchBlock(&buf); 56 | ts_nsec end = get_monotonic_ts(); 57 | TEST_CHK(idx != -1); 58 | for (int j = 100; j; --j) { 59 | buf[rand() % (bin_size - 1)] = 'X'; 60 | } 61 | collect_stat(args->sa, (end - start)); 62 | args->mp->returnBlock(idx); 63 | } 64 | return NULL; 65 | } 66 | 67 | void basic_test(int iterations, int num_bins, size_t bin_size) 68 | { 69 | TEST_INIT(); 70 | struct timeval ts_begin, ts_cur, ts_gap; 71 | 72 | StatAggregator *sa = new StatAggregator(1, 1); 73 | sa->t_stats[0][0].name = "wait_times"; 74 | samples = 0; 75 | 76 | gettimeofday(&ts_begin, NULL); 77 | 78 | struct worker_args mpool; 79 | mpool.num_runs = iterations; 80 | mpool.num_bins = num_bins; 81 | mpool.bin_size = bin_size; 82 | mpool.mp = new MemoryPool(mpool.num_bins, mpool.bin_size); 83 | mpool.sa = sa; 84 | 85 | void *ret = basic_tester(&mpool); 86 | TEST_CHK(!ret); 87 | delete mpool.mp; 88 | 89 | gettimeofday(&ts_cur, NULL); 90 | 91 | sa->aggregateAndPrintStats("BASIC_TEST", samples, "ns"); 92 | delete sa; 93 | 94 | ts_gap = _utime_gap(ts_begin, ts_cur); 95 | char res[128]; 96 | sprintf(res, 97 | "basic test with %d runs of %d bins x %" _F64 "bytes in %ld µs", 98 | iterations, num_bins, uint64_t(bin_size), 99 | ts_gap.tv_sec*1000000 + ts_gap.tv_usec); 100 | TEST_RESULT(res); 101 | } 102 | 103 | void multi_thread_test(int num_threads, int iterations, int num_bins, 104 | size_t bin_size) 105 | { 106 | TEST_INIT(); 107 | thread_t *tid = alca(thread_t, num_threads); 108 | struct timeval ts_begin, ts_cur, ts_gap; 109 | 110 | StatAggregator *sa = new StatAggregator(1, 1); 111 | sa->t_stats[0][0].name = "wait_times"; 112 | samples = 0; 113 | 114 | gettimeofday(&ts_begin, NULL); 115 | 116 | struct worker_args mpool; 117 | mpool.num_runs = iterations; 118 | mpool.num_bins = num_bins; 119 | mpool.bin_size = bin_size; 120 | mpool.mp = new MemoryPool(mpool.num_bins, mpool.bin_size); 121 | mpool.sa = sa; 122 | 123 | for (int i = num_threads - 1; i >= 0; --i) { 124 | thread_create(&tid[i], basic_tester, &mpool); 125 | } 126 | 127 | for (int i = num_threads - 1; i >= 0; --i) { 128 | void *ret; 129 | thread_join(tid[i], &ret); 130 | TEST_CHK(!ret); 131 | } 132 | 133 | delete mpool.mp; 134 | 135 | gettimeofday(&ts_cur, NULL); 136 | 137 | sa->aggregateAndPrintStats("MULTI_THREAD_TEST", samples, "ns"); 138 | delete sa; 139 | 140 | ts_gap = _utime_gap(ts_begin, ts_cur); 141 | char res[128]; 142 | sprintf(res, 143 | "multi-thread test with %d threads each with " 144 | "%d runs of %d bins x %" _F64 "bytes in %ld µs", 145 | num_threads, iterations, num_bins, uint64_t(bin_size), 146 | ts_gap.tv_sec*1000000 + ts_gap.tv_usec); 147 | TEST_RESULT(res); 148 | } 149 | 150 | int main() 151 | { 152 | basic_test(10000, 8, 10485760); //1000 runs of 8 x 10MB buffers 153 | multi_thread_test(8, 10000, 8, 10485760); // repeat with 8 threads 154 | return 0; 155 | } 156 | -------------------------------------------------------------------------------- /tests/anomaly/filemgr_anomalous_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _FILEMGR_ANOMALOUS_OPS 19 | #define _FILEMGR_ANOMALOUS_OPS 20 | 21 | #ifdef __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | void filemgr_ops_set_anomalous(int behavior); 26 | 27 | struct filemgr_ops *get_normal_ops_ptr(); 28 | 29 | // These callbacks allow test-suite to control how the file ops should behave 30 | // If these return 0, then normal operation will happen, 31 | // If these return a non-zero value, then the file ops will return the same result 32 | struct anomalous_callbacks { 33 | fdb_fileops_handle (*constructor_cb) (void *ctx, struct filemgr_ops *normal_ops); 34 | int (*open_cb)(void *ctx, struct filemgr_ops *normal_ops, 35 | fdb_fileops_handle *fileops_handle, const char *pathname, 36 | int flags, mode_t mode); 37 | fdb_ssize_t (*pwrite_cb)(void *ctx, struct filemgr_ops *normal_ops, 38 | fdb_fileops_handle fileops_handle, void *buf, size_t count, 39 | cs_off_t offset); 40 | fdb_ssize_t (*pread_cb)(void *ctx, struct filemgr_ops *normal_ops, 41 | fdb_fileops_handle fileops_handle, void *buf, size_t count, 42 | cs_off_t offset); 43 | int (*close_cb)(void *ctx, struct filemgr_ops *normal_ops, 44 | fdb_fileops_handle fileops_handle); 45 | cs_off_t (*goto_eof_cb)(void *ctx, struct filemgr_ops *normal_ops, 46 | fdb_fileops_handle fileops_handle); 47 | cs_off_t (*file_size_cb)(void *ctx, struct filemgr_ops *normal_ops, 48 | fdb_fileops_handle fileops_handle, const char *filename); 49 | int (*fdatasync_cb)(void *ctx, struct filemgr_ops *normal_ops, 50 | fdb_fileops_handle fileops_handle); 51 | int (*fsync_cb)(void *ctx, struct filemgr_ops *normal_ops, 52 | fdb_fileops_handle fileops_handle); 53 | void (*get_errno_str_cb)(void *ctx, struct filemgr_ops *normal_ops, 54 | fdb_fileops_handle fileops_handle, char *buf, size_t size); 55 | voidref (*mmap_cb)(void *ctx, struct filemgr_ops *normal_ops, 56 | fdb_fileops_handle fops_handle, size_t length, void **aux); 57 | int (*munmap_cb)(void *ctx, struct filemgr_ops *normal_ops, 58 | fdb_fileops_handle fops_handle, void *addr, 59 | size_t length, void *aux); 60 | int (*aio_init_cb)(void *ctx, struct filemgr_ops *normal_ops, 61 | fdb_fileops_handle fops_handle, struct async_io_handle *aio_handle); 62 | int (*aio_prep_read_cb)(void *ctx, struct filemgr_ops *normal_ops, 63 | fdb_fileops_handle fops_handle, struct async_io_handle *aio_handle, 64 | size_t aio_idx, size_t read_size, uint64_t offset); 65 | int (*aio_submit_cb)(void *ctx, struct filemgr_ops *normal_ops, 66 | fdb_fileops_handle fops_handle, struct async_io_handle *aio_handle, 67 | int num_subs); 68 | int (*aio_getevents_cb)(void *ctx, struct filemgr_ops *normal_ops, 69 | fdb_fileops_handle fops_handle, struct async_io_handle *aio_handle, 70 | int min, int max, unsigned int timeout); 71 | int (*aio_destroy_cb)(void *ctx, struct filemgr_ops *normal_ops, 72 | fdb_fileops_handle fops_handle, struct async_io_handle *aio_handle); 73 | int (*get_fs_type_cb)(void *ctx, struct filemgr_ops *normal_ops, 74 | fdb_fileops_handle src_fileops_handle); 75 | int (*copy_file_range_cb)(void *ctx, struct filemgr_ops *normal_ops, 76 | int fs_type, fdb_fileops_handle src_fileops_handle, 77 | fdb_fileops_handle dst_fileops_handle, uint64_t src_off, 78 | uint64_t dst_off, uint64_t len); 79 | void (*destructor_cb)(void *ctx, struct filemgr_ops *normal_ops, 80 | fdb_fileops_handle fileops_handle); 81 | }; 82 | 83 | struct anomalous_callbacks * get_default_anon_cbs(); 84 | void filemgr_ops_anomalous_init(struct anomalous_callbacks *cbs, void *ctx); 85 | 86 | #ifdef __cplusplus 87 | } 88 | #endif 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /utils/time_utils.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "time_utils.h" 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #if defined(__APPLE__) 27 | #include 28 | #endif 29 | 30 | #ifndef _PLATFORM_LIB_AVAILABLE 31 | extern "C" hrtime_t gethrtime(void) 32 | { 33 | auto now = std::chrono::high_resolution_clock::now(); 34 | return std::chrono::duration_cast(now.time_since_epoch()).count(); 35 | } 36 | 37 | extern "C" hrtime_t gethrtime_period(void) 38 | { 39 | std::chrono::nanoseconds ns = std::chrono::high_resolution_clock::duration(1); 40 | return ns.count(); 41 | } 42 | #endif // _PLATFORM_LIB_AVAILABLE 43 | 44 | #if defined(WIN32) || defined(_WIN32) 45 | 46 | #ifndef _PLATFORM_LIB_AVAILABLE 47 | void usleep(unsigned int usec) 48 | { 49 | HANDLE timer; 50 | LARGE_INTEGER ft; 51 | 52 | // Convert to 100 nanosecond interval, negative value indicates relative time 53 | ft.QuadPart = -(10*usec); 54 | 55 | timer = CreateWaitableTimer(NULL, TRUE, NULL); 56 | SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); 57 | WaitForSingleObject(timer, INFINITE); 58 | CloseHandle(timer); 59 | } 60 | #endif // _PLATFORM_LIB_AVAILABLE 61 | 62 | #else 63 | #include 64 | 65 | struct timespec convert_reltime_to_abstime(unsigned int ms) { 66 | struct timespec ts; 67 | struct timeval tp; 68 | uint64_t wakeup; 69 | 70 | memset(&ts, 0, sizeof(ts)); 71 | 72 | /* 73 | * Unfortunately pthread_cond_timedwait doesn't support relative sleeps 74 | * so we need to convert back to an absolute time. 75 | */ 76 | gettimeofday(&tp, NULL); 77 | wakeup = ((uint64_t)(tp.tv_sec) * 1000) + (tp.tv_usec / 1000) + ms; 78 | /* Round up for sub ms */ 79 | if ((tp.tv_usec % 1000) > 499) { 80 | ++wakeup; 81 | } 82 | 83 | ts.tv_sec = wakeup / 1000; 84 | wakeup %= 1000; 85 | ts.tv_nsec = wakeup * 1000000; 86 | return ts; 87 | } 88 | #endif //!defined(WIN32) && !defined(_WIN32) 89 | 90 | struct timeval _utime_gap(struct timeval a, struct timeval b) 91 | { 92 | struct timeval ret; 93 | if (b.tv_usec >= a.tv_usec) { 94 | ret.tv_usec = b.tv_usec - a.tv_usec; 95 | ret.tv_sec = b.tv_sec - a.tv_sec; 96 | }else{ 97 | ret.tv_usec = 1000000 + b.tv_usec - a.tv_usec; 98 | ret.tv_sec = b.tv_sec - a.tv_sec - 1; 99 | } 100 | return ret; 101 | } 102 | 103 | uint64_t timeval_to_us(struct timeval tv) 104 | { 105 | return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec; 106 | } 107 | 108 | void decaying_usleep(unsigned int *sleep_time, unsigned int max_sleep_time) { 109 | usleep(*sleep_time); 110 | *sleep_time = *sleep_time << 1; 111 | if (max_sleep_time < *sleep_time) { 112 | *sleep_time = max_sleep_time; 113 | } 114 | } 115 | 116 | /* 117 | return a monotonically increasing value with a seconds frequency. 118 | */ 119 | ts_nsec get_monotonic_ts() { 120 | ts_nsec ts = 0; 121 | #if defined(WIN32) 122 | LARGE_INTEGER _ts; 123 | QueryPerformanceCounter(&_ts); 124 | ts = _ts.QuadPart; 125 | #elif defined(__APPLE__) 126 | long time = mach_absolute_time(); 127 | 128 | static mach_timebase_info_data_t timebase; 129 | if (timebase.denom == 0) { 130 | mach_timebase_info(&timebase); 131 | } 132 | 133 | ts = (double)time * timebase.numer / timebase.denom; 134 | #elif defined(__linux__) || defined(__sun) || defined(__FreeBSD__) 135 | /* Linux and Solaris can use clock_gettime */ 136 | struct timespec tm; 137 | if (clock_gettime(CLOCK_MONOTONIC, &tm) == -1) { 138 | abort(); 139 | } 140 | ts = tm.tv_nsec; 141 | #else 142 | #error "Don't know how to build get_monotonic_ts" 143 | #endif 144 | 145 | return ts; 146 | } 147 | 148 | ts_nsec ts_diff(ts_nsec start, ts_nsec end) 149 | { 150 | ts_nsec diff = 0; 151 | if ((end-start)<0) { 152 | diff = 1000000000+end-start; 153 | } else { 154 | diff = end-start; 155 | } 156 | #if defined(WIN32) 157 | LARGE_INTEGER Pf; 158 | QueryPerformanceFrequency(&Pf); 159 | return diff / (Pf.QuadPart/1000000); 160 | #else 161 | return diff/1000; 162 | #endif // defined(WIN32) 163 | } 164 | -------------------------------------------------------------------------------- /src/atomicqueue.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_ATOMICQUEUE_H_ 19 | #define SRC_ATOMICQUEUE_H_ 1 20 | 21 | #ifdef _MSC_VER 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "atomic.h" 28 | 29 | /** 30 | * Create a simple version of the AtomicQueue for windows right now to 31 | * avoid the threadlocal usage which is currently using pthreads 32 | */ 33 | template 34 | class AtomicQueue { 35 | public: 36 | void push(T &value) { 37 | LockHolder lock(mutex); 38 | queue.push(value); 39 | } 40 | 41 | void getAll(std::queue &outQueue) { 42 | LockHolder lock(mutex); 43 | while (!queue.empty()) { 44 | outQueue.push(queue.front()); 45 | queue.pop(); 46 | } 47 | } 48 | 49 | bool empty() { 50 | LockHolder lock(mutex); 51 | return queue.empty(); 52 | } 53 | 54 | size_t size() { 55 | LockHolder lock(mutex); 56 | return queue.size(); 57 | } 58 | 59 | private: 60 | std::queue queue; 61 | std::mutex mutex; 62 | }; 63 | 64 | #else 65 | 66 | #include 67 | 68 | #include 69 | #include "threadlocal.h" 70 | #include "utility.h" 71 | 72 | /** 73 | * Efficient approximate-FIFO queue optimize for concurrent writers. 74 | */ 75 | template 76 | class AtomicQueue { 77 | public: 78 | AtomicQueue() : counter(0), numItems(0) {} 79 | 80 | ~AtomicQueue() { 81 | size_t i; 82 | for (i = 0; i < counter; ++i) { 83 | delete queues[i].load(); 84 | } 85 | } 86 | 87 | /** 88 | * Place an item in the queue. 89 | */ 90 | void push(T &value) { 91 | std::queue *q = swapQueue(); // steal our queue 92 | q->push(value); 93 | ++numItems; 94 | q = swapQueue(q); 95 | } 96 | 97 | /** 98 | * Grab all items from this queue an place them into the provided 99 | * output queue. 100 | * 101 | * @param outQueue a destination queue to fill 102 | */ 103 | void getAll(std::queue &outQueue) { 104 | std::queue *q(swapQueue()); // Grab my own queue 105 | std::queue *newQueue(NULL); 106 | int count(0); 107 | 108 | // Will start empty unless this thread is adding stuff 109 | while (!q->empty()) { 110 | outQueue.push(q->front()); 111 | q->pop(); 112 | ++count; 113 | } 114 | 115 | size_t c(counter); 116 | for (size_t i = 0; i < c; ++i) { 117 | // Swap with another thread 118 | std::queue *nullQueue(NULL); 119 | newQueue = atomic_swapIfNot(queues[i], nullQueue, q); 120 | // Empty the queue 121 | if (newQueue != NULL) { 122 | q = newQueue; 123 | while (!q->empty()) { 124 | outQueue.push(q->front()); 125 | q->pop(); 126 | ++count; 127 | } 128 | } 129 | } 130 | 131 | q = swapQueue(q); 132 | numItems.fetch_sub(count); 133 | } 134 | 135 | /** 136 | * True if this queue is empty. 137 | */ 138 | bool empty() const { 139 | return size() == 0; 140 | } 141 | 142 | /** 143 | * Return the number of queued items. 144 | */ 145 | size_t size() const { 146 | return numItems; 147 | } 148 | private: 149 | AtomicPtr > *initialize() { 150 | std::queue *q = new std::queue; 151 | size_t i(counter++); 152 | if (counter > MAX_THREADS) { 153 | throw std::overflow_error("AtomicQueue::initialize: exceeded maximum allowed threads"); 154 | } 155 | queues[i].store(q); 156 | threadQueue = &queues[i]; 157 | return &queues[i]; 158 | } 159 | 160 | std::queue *swapQueue(std::queue *newQueue = NULL) { 161 | AtomicPtr > *qPtr(threadQueue); 162 | if (qPtr == NULL) { 163 | qPtr = initialize(); 164 | } 165 | return qPtr->exchange(newQueue); 166 | } 167 | 168 | ThreadLocalPtr > > threadQueue; 169 | AtomicPtr > queues[MAX_THREADS]; 170 | std::atomic counter; 171 | std::atomic numItems; 172 | DISALLOW_COPY_AND_ASSIGN(AtomicQueue); 173 | }; 174 | #endif 175 | 176 | 177 | #endif // SRC_ATOMICQUEUE_H_ 178 | -------------------------------------------------------------------------------- /tests/unit/hash_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "hash.h" 23 | #include "test.h" 24 | #include "common.h" 25 | #include "hash_functions.h" 26 | 27 | struct item { 28 | int val; 29 | struct hash_elem e; 30 | }; 31 | 32 | unsigned hash_func(struct hash *hash, struct hash_elem *e) 33 | { 34 | struct item *item = _get_entry(e, struct item, e); 35 | return item->val % hash->nbuckets; 36 | } 37 | 38 | int hash_cmp(struct hash_elem *a, struct hash_elem *b) 39 | { 40 | struct item *aa, *bb; 41 | aa = _get_entry(a, struct item, e); 42 | bb = _get_entry(b, struct item, e); 43 | if (aa->val < bb->val) return -1; 44 | else if (aa->val > bb->val) return 1; 45 | else return 0; 46 | } 47 | 48 | void basic_test() 49 | { 50 | TEST_INIT(); 51 | 52 | struct hash hash; 53 | struct item item[16], query, *result; 54 | struct hash_elem *e; 55 | int i; 56 | 57 | hash_init(&hash, 4, hash_func, hash_cmp); 58 | 59 | for (i=0;i<16;++i){ 60 | item[i].val = i; 61 | hash_insert(&hash, &item[i].e); 62 | } 63 | 64 | for (i=0;i<16;++i){ 65 | query.val = i; 66 | e = hash_find(&hash, &query.e); 67 | result = _get_entry(e, struct item, e); 68 | TEST_CHK(result->val == query.val); 69 | } 70 | 71 | for (i=1;i<16;i+=2){ 72 | query.val = i; 73 | e = hash_find(&hash, &query.e); 74 | result = _get_entry(e, struct item, e); 75 | TEST_CHK(result->val == query.val); 76 | hash_remove(&hash, e); 77 | } 78 | 79 | for (i=0;i<16;++i){ 80 | query.val = i; 81 | e = hash_find(&hash, &query.e); 82 | TEST_CHK((i%2==1 && e==NULL) || (i%2==0 && e)); 83 | if (e) { 84 | result = _get_entry(e, struct item, e); 85 | TEST_CHK(result->val == query.val); 86 | } 87 | } 88 | hash_free(&hash); 89 | 90 | TEST_RESULT("basic test"); 91 | } 92 | 93 | void string_hash_test() 94 | { 95 | TEST_INIT(); 96 | 97 | randomize(); 98 | unsigned a, b; 99 | char str[32]; 100 | 101 | int i,j; 102 | 103 | for (i=0;i<16;++i){ 104 | for (j=0;j<8;++j){ 105 | str[j] = 'a' + random('z'-'a'); 106 | } 107 | str[j] = 0; 108 | b = hash_djb2((uint8_t *)str, strlen(str)); 109 | (void)b; 110 | DBG("%s %10u %5u\n",str, b, b&0xfff); 111 | } 112 | 113 | for (i=0;i<16;++i){ 114 | sprintf(str, "asdf%d.%d",i,random(100)); 115 | b = hash_djb2((uint8_t *)str, strlen(str)); 116 | (void)b; 117 | DBG("%s %10u %5u\n",str, b, b&((unsigned)1023)); 118 | } 119 | 120 | sprintf(str, "1234aaaaaaaa"); 121 | a = hash_djb2((uint8_t *)str, strlen(str)); 122 | (void)a; 123 | sprintf(str, "5678aaaaaaaa"); 124 | b = hash_djb2((uint8_t *)str, strlen(str)); 125 | (void)b; 126 | DBG("%u %u\n", a, b); 127 | 128 | sprintf(str, "1234aaaaaaaa"); 129 | a = hash_djb2_last8((uint8_t *)str, strlen(str)); 130 | (void)a; 131 | sprintf(str, "5678aaaaaaaa"); 132 | b = hash_djb2_last8((uint8_t *)str, strlen(str)); 133 | (void)b; 134 | DBG("%u %u\n", a, b); 135 | 136 | sprintf(str, "./dummy0"); 137 | a = hash_djb2_last8((uint8_t *)str, strlen(str)); 138 | (void)a; 139 | sprintf(str, "./dummy01"); 140 | b = hash_djb2_last8((uint8_t *)str, strlen(str)); 141 | (void)b; 142 | DBG("%u %u\n", a, b); 143 | 144 | TEST_RESULT("string hash test"); 145 | } 146 | 147 | void twohash_test() 148 | { 149 | TEST_INIT(); 150 | 151 | randomize(); 152 | 153 | int n = 64; 154 | int i,j; 155 | uint32_t h; 156 | int *array = alca(int, n); 157 | 158 | for (i=100;i<108;++i) { 159 | for (j=333;j<341;++j){ 160 | printf("(%d,%d) = %u\n", i, j, hash_shuffle_2uint(i,j)); 161 | } 162 | } 163 | for (i=0;i 5 | */ 6 | 7 | #include 8 | #include "list.h" 9 | 10 | #ifdef _LIST_LOCK 11 | #define IFDEF_LOCK(...) __VA_ARGS__ 12 | #else 13 | #define IFDEF_LOCK(...) 14 | #endif 15 | 16 | 17 | #ifdef LIST_LOCK 18 | void list_init(struct list *list) 19 | { 20 | list->head = NULL; 21 | list->tail = NULL; 22 | IFDEF_LOCK( spin_init(&list->lock); ); 23 | } 24 | #endif 25 | 26 | void list_push_front(struct list *list, struct list_elem *e) 27 | { 28 | IFDEF_LOCK( spin_lock(&list->lock); ); 29 | if (list->head == NULL) { 30 | list->head = e; 31 | list->tail = e; 32 | e->next = e->prev = NULL; 33 | }else{ 34 | list->head->prev = e; 35 | e->prev = NULL; 36 | e->next = list->head; 37 | list->head = e; 38 | } 39 | IFDEF_LOCK( spin_unlock(&list->lock); ); 40 | } 41 | 42 | void list_push_back(struct list *list, struct list_elem *e) 43 | { 44 | IFDEF_LOCK( spin_lock(&list->lock); ); 45 | if (list->tail == NULL) { 46 | list->head = e; 47 | list->tail = e; 48 | e->next = e->prev = NULL; 49 | }else{ 50 | list->tail->next = e; 51 | e->prev = list->tail; 52 | e->next = NULL; 53 | list->tail = e; 54 | } 55 | IFDEF_LOCK( spin_unlock(&list->lock); ); 56 | } 57 | 58 | // insert E just before BEFORE 59 | void list_insert_before(struct list *list, struct list_elem *before, struct list_elem *e) 60 | { 61 | IFDEF_LOCK( spin_lock(&list->lock); ); 62 | e->prev = before->prev; 63 | e->next = before; 64 | if (before->prev) before->prev->next = e; 65 | else list->head = e; 66 | before->prev = e; 67 | IFDEF_LOCK( spin_unlock(&list->lock); ); 68 | } 69 | 70 | // insert E just after AFTER 71 | // LCOV_EXCL_START 72 | void list_insert_after(struct list *list, struct list_elem *after, struct list_elem *e) 73 | { 74 | IFDEF_LOCK( spin_lock(&list->lock); ); 75 | e->next = after->next; 76 | e->prev = after; 77 | if (after->next) after->next->prev = e; 78 | else list->tail = e; 79 | after->next = e; 80 | IFDEF_LOCK( spin_unlock(&list->lock); ); 81 | } 82 | // LCOV_EXCL_STOP 83 | 84 | struct list_elem *list_remove(struct list *list, struct list_elem *e) 85 | { 86 | IFDEF_LOCK( spin_lock(&list->lock); ); 87 | if (e) { 88 | // if not NULL 89 | if (e->next) e->next->prev = e->prev; 90 | if (e->prev) e->prev->next = e->next; 91 | 92 | if (list->head == e) list->head = e->next; 93 | if (list->tail == e) list->tail = e->prev; 94 | 95 | struct list_elem *next = e->next; 96 | 97 | IFDEF_LOCK( spin_unlock(&list->lock); ); 98 | return next; 99 | } 100 | // NULL .. do nothing 101 | IFDEF_LOCK( spin_unlock(&list->lock); ); 102 | return NULL; 103 | } 104 | 105 | struct list_elem *list_remove_reverse(struct list *list, struct list_elem *e) 106 | { 107 | IFDEF_LOCK( spin_lock(&list->lock); ); 108 | if (e) { 109 | // if not NULL 110 | if (e->next) e->next->prev = e->prev; 111 | if (e->prev) e->prev->next = e->next; 112 | 113 | if (list->head == e) list->head = e->next; 114 | if (list->tail == e) list->tail = e->prev; 115 | 116 | IFDEF_LOCK( spin_unlock(&list->lock); ); 117 | return e->prev; 118 | } 119 | // NULL .. do nothing 120 | IFDEF_LOCK( spin_unlock(&list->lock); ); 121 | return NULL; 122 | } 123 | 124 | struct list_elem *list_pop_front(struct list *list) 125 | { 126 | IFDEF_LOCK( spin_lock(&list->lock); ); 127 | struct list_elem *e = list->head; 128 | if (e) { 129 | // if not NULL 130 | if (e->next) e->next->prev = e->prev; 131 | if (e->prev) e->prev->next = e->next; 132 | 133 | if (list->head == e) list->head = e->next; 134 | if (list->tail == e) list->tail = e->prev; 135 | 136 | IFDEF_LOCK( spin_unlock(&list->lock); ); 137 | return e; 138 | } 139 | // NULL .. do nothing 140 | IFDEF_LOCK( spin_unlock(&list->lock); ); 141 | return NULL; 142 | } 143 | 144 | struct list_elem *list_pop_back(struct list *list) 145 | { 146 | IFDEF_LOCK( spin_lock(&list->lock); ); 147 | struct list_elem *e = list->tail; 148 | if (e) { 149 | // if not NULL 150 | if (e->next) e->next->prev = e->prev; 151 | if (e->prev) e->prev->next = e->next; 152 | 153 | if (list->head == e) list->head = e->next; 154 | if (list->tail == e) list->tail = e->prev; 155 | 156 | IFDEF_LOCK( spin_unlock(&list->lock); ); 157 | return e; 158 | } 159 | // NULL .. do nothing 160 | IFDEF_LOCK( spin_unlock(&list->lock); ); 161 | return NULL; 162 | } 163 | 164 | #ifdef LIST_LOCK 165 | struct list_elem *list_begin(struct list *list) 166 | { 167 | IFDEF_LOCK( spin_lock(&list->lock); ); 168 | struct list_elem *e = list->head; 169 | IFDEF_LOCK( spin_unlock(&list->lock); ); 170 | return e; 171 | } 172 | 173 | struct list_elem *list_end(struct list *list) 174 | { 175 | IFDEF_LOCK( spin_lock(&list->lock); ); 176 | struct list_elem *e = list->tail; 177 | IFDEF_LOCK( spin_unlock(&list->lock); ); 178 | return e; 179 | } 180 | #endif 181 | -------------------------------------------------------------------------------- /option/option.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_OPTION_H 19 | #define _JSAHN_OPTION_H 20 | 21 | //#include "mempool.h" 22 | 23 | #define SEQNUM_NOT_USED (UINT64_C(0xffffffffffffffff)) 24 | #define DEFAULT_KVS_NAME "default" 25 | 26 | #define __FDB_BCACHE_USE 27 | #ifdef __FDB_BCACHE_USE 28 | //#define __FDB_SORTED_COMPACTION 29 | #endif 30 | #define FDB_SECTOR_SIZE (512) 31 | 32 | //#define DOCIO_BLOCK_ALIGN 33 | #define DOCIO_LEN_STRUCT_ALIGN 34 | 35 | //#define __RAW_BLOCK 36 | 37 | #define __CRC32 38 | #ifdef __CRC32 39 | #define BTREE_CRC_OFFSET (8) 40 | #define BTREE_CRC_FIELD_LEN (8) 41 | #define BTREE_LEVEL_OFFSET (2) 42 | //#define __CHECKSUM_ADLER32 43 | #endif 44 | 45 | #define __BIT_CMP 46 | 47 | #define __ENDIAN_SAFE 48 | 49 | //#define __DEBUG_FDB 50 | //#define __DEBUG_WAL 51 | //#define __DEBUG_HBTRIE 52 | //#define __DEBUG_BTREE 53 | //#define __DEBUG_BTREEBLOCK 54 | //#define __DEBUG_BCACHE 55 | //#define __DEBUG_FILEMGR 56 | //#define __DEBUG_COUCHBENCH 57 | 58 | #define FDB_BLOCKSIZE (4096) 59 | #define FDB_WAL_NBUCKET (4099) // a prime number 60 | #define FDB_MAX_FILENAME_LEN (1024) 61 | #define FDB_MAX_KVINS_NAME_LEN (65536) 62 | #define FDB_WAL_THRESHOLD (4*1024) 63 | #define FDB_COMP_BUF_MINSIZE (67108864) // 64 MB, 8M offsets 64 | #define FDB_COMP_BUF_MAXSIZE (1073741824) // 1 GB, 128M offsets 65 | #define FDB_COMP_BATCHSIZE (131072) // 128K docs 66 | #define FDB_COMP_MOVE_UNIT (134217728) // 128 MB 67 | #define FDB_COMP_RATIO_MIN (40) // 40% (writer speed / compactor speed) 68 | #define FDB_COMP_RATIO_MAX (60) // 60% (writer speed / compactor speed) 69 | #define FDB_COMP_PROB_UNIT_INC (5) // 5% (probability delta unit for increase) 70 | #define FDB_COMP_PROB_UNIT_DEC (5) // 5% (probability delta unit for decrease) 71 | 72 | // full compaction internval in secs when the circular block reusing is enabled 73 | #define FDB_COMPACTOR_SLEEP_DURATION (28800) 74 | #define FDB_DEFAULT_COMPACTION_THRESHOLD (30) 75 | 76 | #define FDB_BGFLUSHER_SLEEP_DURATION (2) 77 | #define FDB_BGFLUSHER_DIRTY_THRESHOLD (1024) //if more than this 4MB dirty 78 | // wake up any sleeping bgflusher 79 | 80 | #define FDB_DEFAULT_COMMIT_LOG_SIZE (16777216) // 16MB 81 | 82 | #define BCACHE_NBUCKET (4099) // a prime number 83 | #define BCACHE_NDICBUCKET (4099) // a prime number 84 | #define BCACHE_FLUSH_UNIT (1048576) // 1MB 85 | #define BCACHE_EVICT_UNIT (1) 86 | #define BCACHE_MEMORY_THRESHOLD (0.8) // 80% of physical RAM 87 | #define __BCACHE_SECOND_CHANCE 88 | 89 | #define FILEMGR_PREFETCH_UNIT (4194304) // 4MB 90 | #define FILEMGR_RESIDENT_THRESHOLD (0.9) // 90 % of file is in buffer cache 91 | #define __FILEMGR_DATA_PARTIAL_LOCK 92 | //#define __FILEMGR_DATA_MUTEX_LOCK 93 | 94 | #define SB_DEFAULT_NUM_SUPERBLOCKS (4) // 4 superblocks for crash recovery 95 | #define SB_MAX_BITMAP_DOC_SIZE (1048576) // 1MB, 4M bitmaps per doc 96 | // Minimum file size for the condition that block reusing is triggered 97 | #define SB_MIN_BLOCK_REUSING_FILESIZE (16777216) // 16MB 98 | // Period that superblock is written into the file 99 | #define SB_SYNC_PERIOD (4194304) // sync for every 4MB update 100 | // Time limit for reusable block reclaim 101 | #define SB_RECLAIM_TIMELIMIT (100000) // 100 ms 102 | // Threshold for pre-reclaiming 103 | #define SB_PRE_RECLAIM_RATIO (10) // 10 % 104 | 105 | #define __BTREEBLK_BLOCKPOOL 106 | #define __BTREEBLK_SUBBLOCK 107 | //#define __BTREEBLK_READ_TREE // not used now, for future use 108 | #define BTREEBLK_AGE_LIMIT (10) 109 | #define BTREEBLK_MIN_SUBBLOCK (128) 110 | //#define __BTREEBLK_CACHE 111 | #ifdef __BTREEBLK_CACHE 112 | #define BTREEBLK_CACHE_LIMIT (8) 113 | #endif 114 | 115 | //#define __UTREE 116 | #ifdef __UTREE 117 | #define __UTREE_HEADER_SIZE (16) 118 | #undef BTREE_CRC_OFFSET 119 | #define BTREE_CRC_OFFSET (__UTREE_HEADER_SIZE+8) 120 | #endif 121 | 122 | // WAL parition sizes 123 | #define DEFAULT_NUM_WAL_PARTITIONS (11) // a prime number 124 | #define MAX_NUM_WAL_PARTITIONS (512) 125 | 126 | // Buffer cache partition size 127 | #define DEFAULT_NUM_BCACHE_PARTITIONS (11) // a prime number 128 | #define MAX_NUM_BCACHE_PARTITIONS (512) 129 | 130 | // Asynchronous I/O queue depth 131 | #define ASYNC_IO_QUEUE_DEPTH (64) 132 | 133 | // Number of daemon compactor threads 134 | #define DEFAULT_NUM_COMPACTOR_THREADS (4) 135 | #define MAX_NUM_COMPACTOR_THREADS (128) 136 | 137 | #define DEFAULT_NUM_BGFLUSHER_THREADS (0) // temporarily disable bgflusher 138 | #define MAX_NUM_BGFLUSHER_THREADS (64) 139 | 140 | #define FDB_EXPOOL_NUM_THREADS (4) 141 | #define FDB_EXPOOL_MAX_THREADS (128) 142 | #define FDB_EXPOOL_NUM_QUEUES (4) 143 | #define FDB_EXPOOL_NUM_WRITERS FDB_EXPOOL_NUM_THREADS 144 | 145 | #endif 146 | --------------------------------------------------------------------------------