├── CMakeLists.txt ├── INSTALL.md ├── LICENSE ├── README.md ├── bench ├── arch.h ├── couch_bench.cc └── forestdb_endian.h ├── bench_config.ini ├── include └── libcouchstore │ ├── couch_common.h │ ├── couch_db.h │ ├── couch_index.h │ ├── error.h │ ├── file_ops.h │ └── visibility.h ├── utils ├── adv_random.h ├── avltree.cc ├── avltree.h ├── crc32.cc ├── crc32.h ├── iniparser.cc ├── iniparser.h ├── keygen.cc ├── keygen.h ├── keyloader.cc ├── keyloader.h ├── memleak.cc ├── memleak.h ├── stopwatch.cc ├── stopwatch.h ├── zipfian_random.cc └── zipfian_random.h └── wrappers ├── couch_fdb.cc ├── couch_leveldb.cc ├── couch_rocksdb.cc └── couch_wt.cc /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | include_directories(BEFORE ${CMAKE_SOURCE_DIR}/utils) 4 | include_directories(BEFORE ${CMAKE_SOURCE_DIR}/bench) 5 | include_directories(BEFORE ${CMAKE_SOURCE_DIR}/include) 6 | include_directories(BEFORE ${CMAKE_SOURCE_DIR}/wrappers) 7 | include_directories(BEFORE ${CMAKE_INCLUDE_PATH}) 8 | 9 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE -D__DEBUG") 10 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -L${CMAKE_LIBRARY_PATH} -std=c++11") 11 | 12 | if (NOT CMAKE_BUILD_TYPE) 13 | set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) 14 | endif(NOT CMAKE_BUILD_TYPE) 15 | 16 | if (NOT WIN32) 17 | set(PTHREAD_LIB pthread) 18 | set(LIBM m) 19 | set(LIBZ z) 20 | set(LIBSNAPPY snappy) 21 | set(LIBBZ2 bz2) 22 | set(LIBFDB forestdb) 23 | set(LIBLDB leveldb) 24 | set(LIBRDB rocksdb) 25 | set(LIBCOUCH couchstore) 26 | set(LIBWT wiredtiger) 27 | endif(NOT WIN32) 28 | 29 | if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") 30 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments -g -fomit-frame-pointer -pthread") 31 | elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") 32 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fomit-frame-pointer -pthread") 33 | set(LIBRT rt) 34 | endif() 35 | 36 | add_executable(fdb_bench 37 | bench/couch_bench.cc 38 | wrappers/couch_fdb.cc 39 | utils/avltree.cc 40 | utils/stopwatch.cc 41 | utils/iniparser.cc 42 | utils/crc32.cc 43 | utils/memleak.cc 44 | utils/zipfian_random.cc 45 | utils/keyloader.cc 46 | utils/keygen.cc) 47 | target_link_libraries(fdb_bench ${PTHREAD_LIB} ${LIBM} ${LIBSNAPPY} ${LIBFDB}) 48 | set_target_properties(fdb_bench PROPERTIES COMPILE_FLAGS "-D__FDB_BENCH") 49 | file(COPY ${CMAKE_SOURCE_DIR}/bench_config.ini DESTINATION ./) 50 | 51 | add_executable(couch_bench 52 | bench/couch_bench.cc 53 | utils/avltree.cc 54 | utils/stopwatch.cc 55 | utils/iniparser.cc 56 | utils/crc32.cc 57 | utils/memleak.cc 58 | utils/zipfian_random.cc 59 | utils/keyloader.cc 60 | utils/keygen.cc) 61 | target_link_libraries(couch_bench ${PTHREAD_LIB} ${LIBM} ${LIBSNAPPY} ${LIBCOUCH}) 62 | set_target_properties(couch_bench PROPERTIES COMPILE_FLAGS "-D__COUCH_BENCH") 63 | file(COPY ${CMAKE_SOURCE_DIR}/bench_config.ini DESTINATION ./) 64 | 65 | add_executable(leveldb_bench 66 | bench/couch_bench.cc 67 | wrappers/couch_leveldb.cc 68 | utils/avltree.cc 69 | utils/stopwatch.cc 70 | utils/iniparser.cc 71 | utils/crc32.cc 72 | utils/memleak.cc 73 | utils/zipfian_random.cc 74 | utils/keyloader.cc 75 | utils/keygen.cc) 76 | target_link_libraries(leveldb_bench ${PTHREAD_LIB} ${LIBM} ${LIBSNAPPY} ${LIBLDB}) 77 | set_target_properties(leveldb_bench PROPERTIES COMPILE_FLAGS "-D__LEVEL_BENCH") 78 | file(COPY ${CMAKE_SOURCE_DIR}/bench_config.ini DESTINATION ./) 79 | 80 | add_executable(wt_bench 81 | bench/couch_bench.cc 82 | wrappers/couch_wt.cc 83 | utils/avltree.cc 84 | utils/stopwatch.cc 85 | utils/iniparser.cc 86 | utils/crc32.cc 87 | utils/memleak.cc 88 | utils/zipfian_random.cc 89 | utils/keyloader.cc 90 | utils/keygen.cc) 91 | target_link_libraries(wt_bench ${PTHREAD_LIB} ${LIBM} ${LIBSNAPPY} ${LIBWT}) 92 | set_target_properties(wt_bench PROPERTIES COMPILE_FLAGS "-D__WT_BENCH") 93 | file(COPY ${CMAKE_SOURCE_DIR}/bench_config.ini DESTINATION ./) 94 | 95 | add_executable(rocksdb_bench 96 | bench/couch_bench.cc 97 | wrappers/couch_rocksdb.cc 98 | utils/avltree.cc 99 | utils/stopwatch.cc 100 | utils/iniparser.cc 101 | utils/crc32.cc 102 | utils/memleak.cc 103 | utils/zipfian_random.cc 104 | utils/keyloader.cc 105 | utils/keygen.cc) 106 | target_link_libraries(rocksdb_bench 107 | ${PTHREAD_LIB} ${LIBM} ${LIBSNAPPY} ${LIBRT} ${LIBZ} ${LIBBZ2} ${LIBRDB}) 108 | set_target_properties(rocksdb_bench PROPERTIES COMPILE_FLAGS "-D__ROCKS_BENCH") 109 | file(COPY ${CMAKE_SOURCE_DIR}/bench_config.ini DESTINATION ./) 110 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | 2 | ## Compilation and Build 3 | 4 | We use [CMake](http://www.cmake.org/cmake/) to provide the build support for a wide range of platforms. Please follow the instructions below to install CMake in your target platform. 5 | 6 | * **Ubuntu** 7 | 8 | `sudo apt-get install cmake` 9 | 10 | * **Centos** 11 | 12 | `wget http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz` 13 | 14 | `tar xvfz cmake-2.8.12.1.tar.gz` 15 | 16 | `cd cmake-2.8.12.1` 17 | 18 | `./bootstrap && make && sudo make install` 19 | 20 | * **OS X** 21 | 22 | `brew install cmake` 23 | 24 | 25 | Once CMake is installed, please follow the instructions below to compile and build the benchmark program on Ubuntu, Centos, or OS X: 26 | 27 | `git clone ForestDB-Benchmark_repo_url` 28 | 29 | `cd ForestDB-Benchmark` 30 | 31 | `mkdir build` 32 | 33 | `cd build` 34 | 35 | `cmake ../` 36 | 37 | Note that DB libraries that you want to evaluate with the benchmark program should be installed before executing `cmake` command. 38 | 39 | If you have libraries or include header files in non-standard locations, you can assign custom paths as follows: 40 | 41 | `cmake -DCMAKE_INCLUDE_PATH=[your_include_path] -DCMAKE_LIBRARY_PATH=[your_library_path] ../` 42 | 43 | (We recommend that all custom paths should be absolute paths to avoid potential problems.) 44 | 45 | After that, you can build each benchmark program using commands below: 46 | 47 | `make fdb_bench`: ForestDB benchmark 48 | 49 | `make couch_bench`: Couchstore benchmark 50 | 51 | `make leveldb_bench`: LevelDB benchmark 52 | 53 | `make rocksdb_bench`: RocksDB benchmark 54 | 55 | `make wt_bench`: WiredTiger benchmark 56 | 57 | If the following error occurs due to the custom library path, 58 | 59 | `error while loading shared libraries: [library_filename]: cannot open shared object file: No such file or directory` 60 | 61 | you need to manually set the environmental variable `LD_LIBRARY_PATH` as follows: 62 | 63 | `export LD_LIBRARY_PATH=[your_library_path]` 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ForestDB-Benchmark 2 | ================== 3 | ForestDB-Benchmark is a benchmark program for embedded key-value storage engines, based on a sophisticated workload generation which is more realistic than performing a bunch of read/write operations. It generates key-value store operations using the APIs of Couchstore, which is the current storage engine of Couchbase Server. We currently provide API-wrappers for ForestDB, LevelDB, RocksDB, and WiredTiger. 4 | 5 | How to Build 6 | ---- 7 | Please refer to [INSTALL.MD](https://github.com/couchbaselabs/ForestDB-Benchmark/blob/master/INSTALL.md) 8 | 9 | 10 | How to Use 11 | ---- 12 | Please visit [wiki pages](https://github.com/couchbaselabs/ForestDB-Benchmark/wiki). -------------------------------------------------------------------------------- /bench/arch.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_ARCH_H 19 | #define _JSAHN_ARCH_H 20 | 21 | #define __STDC_FORMAT_MACROS 22 | 23 | #include 24 | #include 25 | 26 | #include "forestdb_endian.h" 27 | /* Large File Support */ 28 | #define _LARGE_FILE 1 29 | #ifndef _FILE_OFFSET_BITS 30 | # define _FILE_OFFSET_BITS 64 31 | #elif (_FILE_OFFSET_BITS != 64) 32 | #error "bad things" 33 | #endif 34 | #define _LARGEFILE_SOURCE 1 35 | #ifndef O_LARGEFILE 36 | # define O_LARGEFILE 0 37 | #endif 38 | 39 | #ifndef _MSC_VER 40 | #include 41 | #else 42 | #ifndef __cplusplus 43 | #pragma once 44 | #define false (0) 45 | #define true (1) 46 | #define bool int 47 | #endif 48 | #endif 49 | 50 | #ifdef __APPLE__ 51 | #include 52 | #include 53 | #include 54 | 55 | #define INLINE extern inline 56 | 57 | #define _X64 "llx" 58 | #define _F64 "lld" 59 | #define _FSEC "ld" 60 | #define _FUSEC "d" 61 | 62 | #define _ARCH_O_DIRECT (0x0) 63 | 64 | #if TARGET_CPU_ARM 65 | #define _ALIGN_MEM_ACCESS 66 | #endif 67 | 68 | #define malloc_align(addr, align, size) \ 69 | {int __ret__=0; __ret__=posix_memalign(&(addr), (align), (size));} 70 | #define free_align(addr) free(addr) 71 | 72 | #ifndef spin_t 73 | // spinlock 74 | #include 75 | #define spin_t OSSpinLock 76 | #define spin_lock(arg) OSSpinLockLock(arg) 77 | #define spin_unlock(arg) OSSpinLockUnlock(arg) 78 | #define SPIN_INITIALIZER (spin_t)(0) 79 | #define spin_init(arg) *(arg) = (spin_t)(0) 80 | #define spin_destroy(arg) 81 | #endif 82 | #ifndef mutex_t 83 | // mutex 84 | #include 85 | #define mutex_t pthread_mutex_t 86 | #define mutex_init(arg) pthread_mutex_init(arg, NULL) 87 | #define mutex_lock(arg) pthread_mutex_lock(arg) 88 | #define mutex_unlock(arg) pthread_mutex_unlock(arg) 89 | #define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER 90 | #define mutex_destroy(arg) pthread_mutex_destroy(arg) 91 | #endif 92 | #ifndef thread_t 93 | // thread 94 | #include 95 | #define thread_t pthread_t 96 | #define thread_cond_t pthread_cond_t 97 | #define thread_create(tid, func, args) \ 98 | pthread_create((tid), NULL, (func), (args)) 99 | #define thread_join(tid, ret) pthread_join(tid, ret) 100 | #define thread_cancel(tid) pthread_cancel(tid) 101 | #define thread_exit(code) pthread_exit(code) 102 | #define thread_cond_init(cond) pthread_cond_init(cond, NULL) 103 | #define thread_cond_destroy(cond) pthread_cond_destroy(cond) 104 | #define thread_cond_wait(cond, mutex) pthread_cond_wait(cond, mutex) 105 | #define thread_cond_timedwait(cond, mutex, ms) \ 106 | { \ 107 | struct timespec ts = convert_reltime_to_abstime(ms); \ 108 | pthread_cond_timedwait(cond, mutex, &ts); \ 109 | } 110 | #define thread_cond_signal(cond) pthread_cond_signal(cond) 111 | #define thread_cond_broadcast(cond) pthread_cond_broadcast(cond) 112 | #endif 113 | 114 | #elif __ANDROID__ 115 | #include 116 | #include 117 | 118 | #define INLINE __inline 119 | 120 | #define _X64 "llx" 121 | #define _F64 "lld" 122 | #define _FSEC "ld" 123 | #define _FUSEC "ld" 124 | 125 | #define _ARCH_O_DIRECT (O_DIRECT) 126 | #define malloc_align(addr, align, size) \ 127 | (addr = memalign((align), (size))) 128 | #define free_align(addr) free(addr) 129 | 130 | #ifndef spin_t 131 | // spinlock 132 | #include 133 | #define spin_t pthread_mutex_t 134 | #define spin_init(arg) pthread_mutex_init(arg, NULL) 135 | #define spin_lock(arg) pthread_mutex_lock(arg) 136 | #define spin_unlock(arg) pthread_mutex_unlock(arg) 137 | #define spin_destroy(arg) pthread_mutex_destroy(arg) 138 | #define SPIN_INITIALIZER ((spin_t)PTHREAD_MUTEX_INITIALIZER) 139 | #endif 140 | #ifndef mutex_t 141 | // mutex 142 | #include 143 | #define mutex_t pthread_mutex_t 144 | #define mutex_init(arg) pthread_mutex_init(arg, NULL) 145 | #define mutex_lock(arg) pthread_mutex_lock(arg) 146 | #define mutex_unlock(arg) pthread_mutex_unlock(arg) 147 | #define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER 148 | #define mutex_destroy(arg) pthread_mutex_destroy(arg) 149 | #endif 150 | #ifndef thread_t 151 | // thread 152 | #include 153 | #define thread_t pthread_t 154 | #define thread_cond_t pthread_cond_t 155 | #define thread_create(tid, func, args) \ 156 | pthread_create((tid), NULL, (func), (args)) 157 | #define thread_join(tid, ret) pthread_join(tid, ret) 158 | #define thread_cancel(tid) pthread_cancel(tid) 159 | #define thread_exit(code) pthread_exit(code) 160 | #define thread_cond_init(cond) pthread_cond_init(cond, NULL) 161 | #define thread_cond_destroy(cond) pthread_cond_destroy(cond) 162 | #define thread_cond_wait(cond, mutex) pthread_cond_wait(cond, mutex) 163 | #define thread_cond_timedwait(cond, mutex, ms) \ 164 | { \ 165 | struct timespec ts = convert_reltime_to_abstime(ms); \ 166 | pthread_cond_timedwait(cond, mutex, &ts); \ 167 | } 168 | #define thread_cond_signal(cond) pthread_cond_signal(cond) 169 | #define thread_cond_broadcast(cond) pthread_cond_broadcast(cond) 170 | #endif 171 | 172 | #ifdef assert 173 | #undef assert 174 | #endif 175 | #define assert(a) (a) 176 | 177 | #elif __linux__ 178 | #include 179 | #include 180 | 181 | #define INLINE __inline 182 | 183 | #define _X64 PRIx64 184 | #define _F64 PRIu64 185 | #define _FSEC "ld" 186 | #define _FUSEC "ld" 187 | 188 | #define _ARCH_O_DIRECT (O_DIRECT) 189 | 190 | #define malloc_align(addr, align, size) \ 191 | {int __ret__=0; __ret__=posix_memalign(&(addr), (align), (size));} 192 | #define free_align(addr) free(addr) 193 | 194 | #ifndef spin_t 195 | // spinlock 196 | #include 197 | #define spin_t pthread_spinlock_t 198 | #define spin_init(arg) pthread_spin_init(arg, PTHREAD_PROCESS_SHARED) 199 | #define spin_lock(arg) pthread_spin_lock(arg) 200 | #define spin_unlock(arg) pthread_spin_unlock(arg) 201 | #define spin_destroy(arg) pthread_spin_destroy(arg) 202 | #define SPIN_INITIALIZER (spin_t)(1) 203 | #endif 204 | #ifndef mutex_t 205 | // mutex 206 | #include 207 | #define mutex_t pthread_mutex_t 208 | #define mutex_init(arg) pthread_mutex_init(arg, NULL) 209 | #define mutex_lock(arg) pthread_mutex_lock(arg) 210 | #define mutex_unlock(arg) pthread_mutex_unlock(arg) 211 | #define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER 212 | #define mutex_destroy(arg) pthread_mutex_destroy(arg) 213 | #endif 214 | #ifndef thread_t 215 | // thread 216 | #include 217 | #define thread_t pthread_t 218 | #define thread_cond_t pthread_cond_t 219 | #define thread_create(tid, func, args) \ 220 | pthread_create((tid), NULL, (func), (args)) 221 | #define thread_join(tid, ret) pthread_join(tid, ret) 222 | #define thread_cancel(tid) pthread_cancel(tid) 223 | #define thread_exit(code) pthread_exit(code) 224 | #define thread_cond_init(cond) pthread_cond_init(cond, NULL) 225 | #define thread_cond_destroy(cond) pthread_cond_destroy(cond) 226 | #define thread_cond_wait(cond, mutex) pthread_cond_wait(cond, mutex) 227 | #define thread_cond_timedwait(cond, mutex, ms) \ 228 | { \ 229 | struct timespec ts = convert_reltime_to_abstime(ms); \ 230 | pthread_cond_timedwait(cond, mutex, &ts); \ 231 | } 232 | #define thread_cond_signal(cond) pthread_cond_signal(cond) 233 | #define thread_cond_broadcast(cond) pthread_cond_broadcast(cond) 234 | #endif 235 | 236 | #elif defined(WIN32) || defined(_WIN32) 237 | // mingw compatiable 238 | 239 | #define _FSEC "ld" 240 | #define _FUSEC "ld" 241 | 242 | #define _ARCH_O_DIRECT (0x0) 243 | 244 | #ifdef _MSC_VER 245 | // visual studio CL compiler 246 | #include 247 | #include "gettimeofday_vs.h" 248 | #define INLINE static inline 249 | //#define alloca(size) _alloca(size) 250 | #define _X64 "llx" 251 | #define _F64 "llu" 252 | #define _CRT_SECURE_NO_WARNINGS 253 | #define gettimeofday gettimeofday_vs 254 | #define sleep(sec) Sleep((sec)*1000) 255 | typedef unsigned long mode_t; 256 | #include 257 | typedef SSIZE_T ssize_t; 258 | #else 259 | #include 260 | #include 261 | #define _X64 PRIx64 262 | #define _F64 PRIu64 263 | #define INLINE __inline 264 | #endif 265 | #include 266 | #include 267 | #define malloc_align(addr, align, size) \ 268 | (addr = (void*)_aligned_malloc((size), (align))) 269 | #define free_align(addr) _aligned_free(addr) 270 | 271 | #ifndef spin_t 272 | // spinlock 273 | #define spin_t CRITICAL_SECTION 274 | #define spin_init(arg) InitializeCriticalSection(arg) 275 | #define spin_lock(arg) EnterCriticalSection(arg) 276 | #define spin_unlock(arg) LeaveCriticalSection(arg) 277 | #define spin_destroy(arg) DeleteCriticalSection(arg) 278 | #endif 279 | #ifndef mutex_t 280 | // mutex 281 | #define mutex_t CRITICAL_SECTION 282 | #define mutex_init(arg) InitializeCriticalSection(arg) 283 | #define mutex_lock(arg) EnterCriticalSection(arg) 284 | #define mutex_unlock(arg) LeaveCriticalSection(arg) 285 | #define mutex_destroy(arg) DeleteCriticalSection(arg) 286 | #endif 287 | #ifndef thread_t 288 | // thread 289 | #define thread_t HANDLE 290 | #define thread_cond_t CONDITION_VARIABLE 291 | #define thread_create(tid, func, args) \ 292 | { \ 293 | DWORD __dt__; \ 294 | *(tid) = CreateThread(NULL, 0, \ 295 | (LPTHREAD_START_ROUTINE)(func), (args), 0, &__dt__); \ 296 | } 297 | #define thread_join(tid, ret) WaitForSingleObject(tid, INFINITE) 298 | #define thread_cancel(tid) TerminateThread(tid, 0); 299 | #define thread_exit(code) ExitThread(code) 300 | #define thread_cond_init(cond) InitializeConditionVariable(cond) 301 | #define thread_cond_destroy(cond) (void)cond 302 | #define thread_cond_wait(cond, mutex) SleepConditionVariableCS(cond, mutex, INFINITE) 303 | #define thread_cond_timedwait(cond, mutex, msec) \ 304 | SleepConditionVariableCS(cond, mutex, msec) 305 | #define thread_cond_signal(cond) WakeConditionVariable(cond) 306 | #define thread_cond_broadcast(cond) WakeAllConditionVariable(cond) 307 | #endif 308 | 309 | #elif __CYGWIN__ 310 | // cygwin compatiable 311 | #include 312 | #include 313 | 314 | #define INLINE __inline 315 | 316 | #define _X64 PRIx64 317 | #define _F64 PRIu64 318 | #define _FSEC "ld" 319 | #define _FUSEC "ld" 320 | 321 | #define _ARCH_O_DIRECT (0x0) 322 | 323 | #define malloc_align(addr, align, size) \ 324 | (addr = memalign((align), (size))) 325 | #define free_align(addr) free(addr) 326 | 327 | #ifndef spin_t 328 | // spinlock 329 | #include 330 | #define spin_t pthread_spinlock_t 331 | #define spin_init(arg) pthread_spin_init(arg, PTHREAD_PROCESS_SHARED) 332 | #define spin_lock(arg) pthread_spin_lock(arg) 333 | #define spin_unlock(arg) pthread_spin_unlock(arg) 334 | #define spin_destroy(arg) pthread_spin_destroy(arg) 335 | #define SPIN_INITIALIZER (spin_t)(1) 336 | #endif 337 | #ifndef mutex_t 338 | // mutex 339 | #include 340 | #define mutex_t pthread_mutex_t 341 | #define mutex_init(arg) pthread_mutex_init(arg, NULL) 342 | #define mutex_lock(arg) pthread_mutex_lock(arg) 343 | #define mutex_unlock(arg) pthread_mutex_unlock(arg) 344 | #define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER 345 | #define mutex_destroy(arg) pthread_mutex_destroy(arg) 346 | #endif 347 | #ifndef thread_t 348 | // thread 349 | #include 350 | #define thread_t pthread_t 351 | #define thread_cond_t pthread_cond_t 352 | #define thread_create(tid, func, args) \ 353 | pthread_create((tid), NULL, (func), (args)) 354 | #define thread_join(tid, ret) pthread_join(tid, ret) 355 | #define thread_cancel(tid) pthread_cancel(tid) 356 | #define thread_exit(code) pthread_exit(code) 357 | #define thread_cond_init(cond) pthread_cond_init(cond, NULL) 358 | #define thread_cond_destroy(cond) pthread_cond_destroy(cond) 359 | #define thread_cond_wait(cond, mutex) pthread_cond_wait(cond, mutex) 360 | #define thread_cond_timedwait(cond, mutex, ms) \ 361 | { \ 362 | struct timespec ts = convert_reltime_to_abstime(ms); \ 363 | pthread_cond_timedwait(cond, mutex, &ts); \ 364 | } 365 | #define thread_cond_signal(cond) pthread_cond_signal(cond) 366 | #define thread_cond_broadcast(cond) pthread_cond_broadcast(cond) 367 | #endif 368 | 369 | #else 370 | #define INLINE make_error 371 | #endif 372 | 373 | #endif 374 | -------------------------------------------------------------------------------- /bench/forestdb_endian.h: -------------------------------------------------------------------------------- 1 | #ifndef _FDB_ENDIAN_H 2 | #define _FDB_ENDIAN_H 3 | 4 | #if defined(WIN32) || defined(_WIN32) 5 | #ifndef _LITTLE_ENDIAN 6 | #define _LITTLE_ENDIAN 7 | #endif 8 | 9 | #elif __APPLE__ 10 | #include 11 | #if BYTE_ORDER == LITTLE_ENDIAN 12 | #ifndef _LITTLE_ENDIAN 13 | #define _LITTLE_ENDIAN 14 | #endif 15 | #elif BYTE_ORDER == BIG_ENDIAN 16 | #ifndef _BIG_ENDIAN 17 | #define _BIG_ENDIAN 18 | #endif 19 | #else 20 | #error "not supported endian" 21 | #endif 22 | 23 | #elif __linux__ 24 | #include 25 | #if __BYTE_ORDER == __LITTLE_ENDIAN 26 | #ifndef _LITTLE_ENDIAN 27 | #define _LITTLE_ENDIAN 28 | #endif 29 | #elif __BYTE_ORDER == __BIG_ENDIAN 30 | #ifndef _BIG_ENDIAN 31 | #define _BIG_ENDIAN 32 | #endif 33 | #else 34 | #error "not supported endian" 35 | #endif 36 | 37 | #endif 38 | 39 | #ifndef bitswap64 40 | #define bitswap64(v) \ 41 | ( (((v) & 0xff00000000000000ULL) >> 56) \ 42 | | (((v) & 0x00ff000000000000ULL) >> 40) \ 43 | | (((v) & 0x0000ff0000000000ULL) >> 24) \ 44 | | (((v) & 0x000000ff00000000ULL) >> 8) \ 45 | | (((v) & 0x00000000ff000000ULL) << 8) \ 46 | | (((v) & 0x0000000000ff0000ULL) << 24) \ 47 | | (((v) & 0x000000000000ff00ULL) << 40) \ 48 | | (((v) & 0x00000000000000ffULL) << 56) ) 49 | #endif 50 | 51 | #ifndef bitswap32 52 | #define bitswap32(v) \ 53 | ( (((v) & 0xff000000) >> 24) \ 54 | | (((v) & 0x00ff0000) >> 8) \ 55 | | (((v) & 0x0000ff00) << 8) \ 56 | | (((v) & 0x000000ff) << 24) ) 57 | #endif 58 | 59 | #ifndef bitswap16 60 | #define bitswap16(v) \ 61 | ( (((v) & 0xff00) >> 8) \ 62 | | (((v) & 0x00ff) << 8) ) 63 | #endif 64 | 65 | #if defined(_LITTLE_ENDIAN) 66 | // convert to big endian 67 | #define _enc64(v) bitswap64(v) 68 | #define _dec64(v) bitswap64(v) 69 | #define _enc32(v) bitswap32(v) 70 | #define _dec32(v) bitswap32(v) 71 | #define _enc16(v) bitswap16(v) 72 | #define _dec16(v) bitswap16(v) 73 | #else 74 | // big endian .. do nothing 75 | #define _enc64(v) (v) 76 | #define _dec64(v) (v) 77 | #define _enc32(v) (v) 78 | #define _dec32(v) (v) 79 | #define _enc16(v) (v) 80 | #define _dec16(v) (v) 81 | #endif 82 | 83 | #ifdef __ENDIAN_SAFE 84 | #define _endian_encode(v) \ 85 | ((sizeof(v) == 8)?(_enc64(v)):( \ 86 | (sizeof(v) == 4)?(_enc32(v)):( \ 87 | (sizeof(v) == 2)?(_enc16(v)):(v)))) 88 | #define _endian_decode(v) \ 89 | ((sizeof(v) == 8)?(_dec64(v)):( \ 90 | (sizeof(v) == 4)?(_dec32(v)):( \ 91 | (sizeof(v) == 2)?(_dec16(v)):(v)))) 92 | #else 93 | #define _endian_encode(v) (v) 94 | #define _endian_decode(v) (v) 95 | #endif 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /bench_config.ini: -------------------------------------------------------------------------------- 1 | [document] 2 | ndocs = 1000000 3 | 4 | [log] 5 | filename = logs/ops_log 6 | 7 | [db_config] 8 | cache_size_MB = 2048 9 | compaction_mode = auto 10 | auto_compaction_threads = 4 11 | wbs_init_MB = 256 12 | wbs_bench_MB = 4 13 | bloom_bits_per_key = 0 14 | compaction_style = level 15 | fdb_wal = 4096 16 | wt_type = b-tree 17 | compression = false 18 | split_pct = 100 19 | leaf_pg_size_KB = 4 20 | int_pg_size_KB = 4 21 | 22 | [db_file] 23 | filename = data/dummy 24 | nfiles = 1 25 | 26 | [population] 27 | nthreads = 8 28 | batchsize = 4096 29 | 30 | [threads] 31 | readers = 1 32 | iterators = 0 33 | writers = 1 34 | reader_ops = 0 35 | writer_ops = 0 36 | disjoint_write = false 37 | 38 | [key_length] 39 | distribution = normal 40 | median = 32 41 | standard_deviation = 2 42 | 43 | [prefix] 44 | level = 0 45 | nprefixes = 100 46 | distribution = uniform 47 | lower_bound = 4 48 | upper_bound = 12 49 | 50 | [body_length] 51 | distribution = normal 52 | median = 512 53 | standard_deviation = 32 54 | compressibility = 30 55 | 56 | [operation] 57 | warmingup = 0 58 | duration = 60 59 | #nops = 1000000 60 | 61 | batch_distribution = zipfian 62 | batch_parameter1 = 0.0 63 | batch_parameter2 = 8 64 | 65 | batchsize_distribution = normal 66 | 67 | read_batchsize_median = 5 68 | read_batchsize_standard_deviation = 1 69 | 70 | iterate_batchsize_median = 1000 71 | iterate_batchsize_standard_deviation = 100 72 | 73 | write_batchsize_median = 1024 74 | write_batchsize_standard_deviation = 2 75 | 76 | write_ratio_percent = 1000 77 | write_type = sync 78 | 79 | [compaction] 80 | threshold = 50 81 | period = 60 82 | block_reuse = 70 83 | 84 | [latency_monitor] 85 | rate = 100 86 | max_samples = 1000000 87 | -------------------------------------------------------------------------------- /include/libcouchstore/couch_common.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef COUCH_COMMON_H 3 | #define COUCH_COMMON_H 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef __cplusplus 11 | extern "C" { 12 | #endif 13 | 14 | /** 15 | * Using off_t turned out to be a real challenge. On "unix-like" systems 16 | * its size is set by a combination of #defines like: _LARGE_FILE, 17 | * _FILE_OFFSET_BITS and/or _LARGEFILE_SOURCE etc. The interesting 18 | * part is however Windows. 19 | * 20 | * Windows follows the LLP64 data model: 21 | * http://en.wikipedia.org/wiki/LLP64#64-bit_data_models 22 | * 23 | * This means both the int and long int types have a size of 32 bits 24 | * regardless if it's a 32 or 64 bits Windows system. 25 | * 26 | * And Windows defines the type off_t as being a signed long integer: 27 | * http://msdn.microsoft.com/en-us/library/323b6b3k.aspx 28 | * 29 | * This means we can't use off_t on Windows if we deal with files 30 | * that can have a size of 2Gb or more. 31 | */ 32 | typedef int64_t cs_off_t; 33 | 34 | /** Document content metadata flags */ 35 | typedef uint8_t couchstore_content_meta_flags; 36 | enum { 37 | COUCH_DOC_IS_COMPRESSED = 128, /**< Document contents compressed via Snappy */ 38 | /* Content Type Reasons (content_meta & 0x0F): */ 39 | COUCH_DOC_IS_JSON = 0, /**< Document is valid JSON data */ 40 | COUCH_DOC_INVALID_JSON = 1, /**< Document was checked, and was not valid JSON */ 41 | COUCH_DOC_INVALID_JSON_KEY = 2, /**< Document was checked, and contained reserved keys, 42 | was not inserted as JSON. */ 43 | COUCH_DOC_NON_JSON_MODE = 3 /**< Document was not checked (DB running in non-JSON mode) */ 44 | }; 45 | 46 | typedef enum { 47 | #ifdef POSIX_FADV_NORMAL 48 | /* Evict this range from FS caches if possible */ 49 | COUCHSTORE_FILE_ADVICE_EVICT = POSIX_FADV_DONTNEED 50 | #else 51 | /* Assign these whatever values, we'll be ignoring them.. */ 52 | COUCHSTORE_FILE_ADVICE_EVICT 53 | #endif 54 | } couchstore_file_advice_t; 55 | 56 | /** A generic data blob. Nothing is implied about ownership of the block pointed to. */ 57 | typedef struct _sized_buf { 58 | char *buf; 59 | size_t size; 60 | } sized_buf; 61 | 62 | /** A CouchStore document, consisting of an ID (key) and data, each of which is a blob. */ 63 | typedef struct _doc { 64 | sized_buf id; 65 | sized_buf data; 66 | } Doc; 67 | 68 | /** Metadata of a CouchStore document. */ 69 | typedef struct _docinfo { 70 | sized_buf id; /**< Document ID (key) */ 71 | uint64_t db_seq; /**< Sequence number in database */ 72 | uint64_t rev_seq; /**< Revision number of document */ 73 | sized_buf rev_meta; /**< Revision metadata; uninterpreted by CouchStore. 74 | Needs to be kept small enough to fit in a B-tree index.*/ 75 | int deleted; /**< Is this a deleted revision? */ 76 | couchstore_content_meta_flags content_meta; /**< Content metadata flags */ 77 | uint64_t bp; /**< Byte offset of document data in file */ 78 | size_t size; /**< Data size in bytes */ 79 | } DocInfo; 80 | 81 | #define DOC_INFO_INITIALIZER { {0, 0}, 0, 0, {0, 0}, 0, 0, 0, 0 } 82 | 83 | /** Contents of a 'local' (unreplicated) document. */ 84 | typedef struct _local_doc { 85 | sized_buf id; 86 | sized_buf json; 87 | int deleted; 88 | } LocalDoc; 89 | 90 | /** Information about the database as a whole. */ 91 | typedef struct { 92 | const char* filename; /**< Filesystem path */ 93 | uint64_t last_sequence; /**< Last sequence number allocated */ 94 | uint64_t doc_count; /**< Total number of (non-deleted) documents */ 95 | uint64_t deleted_count; /**< Total number of deleted documents */ 96 | uint64_t space_used; /**< Disk space actively used by docs */ 97 | uint64_t file_size; /**< Total disk space used by database */ 98 | cs_off_t header_position; /**< File offset of current header */ 99 | uint64_t purge_seq; /**< Last Purge sequence number */ 100 | } DbInfo; 101 | 102 | 103 | /** Opaque reference to an open database. */ 104 | typedef struct _db Db; 105 | 106 | #ifdef __cplusplus 107 | } 108 | #endif 109 | 110 | #endif 111 | -------------------------------------------------------------------------------- /include/libcouchstore/couch_index.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef COUCHSTORE_COUCH_INDEX_H 3 | #define COUCHSTORE_COUCH_INDEX_H 4 | 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /** 12 | * Opaque reference to an open index file. 13 | */ 14 | typedef struct _CouchStoreIndex CouchStoreIndex; 15 | 16 | /* 17 | * Types of indexes. 18 | */ 19 | typedef uint64_t couchstore_index_type; 20 | enum { 21 | COUCHSTORE_VIEW_PRIMARY_INDEX = 0, /**< Primary index, maps emitted keys to values */ 22 | COUCHSTORE_VIEW_BACK_INDEX = 1, /**< Back-index, maps doc IDs to emitted keys */ 23 | }; 24 | 25 | /* 26 | * Available built-in reduce functions to use on the JSON values in primary indexes. 27 | * These are documented at 28 | */ 29 | typedef uint64_t couchstore_json_reducer; 30 | enum { 31 | COUCHSTORE_REDUCE_NONE = 0, /**< No reduction */ 32 | COUCHSTORE_REDUCE_COUNT = 1, /**< Count rows */ 33 | COUCHSTORE_REDUCE_SUM = 2, /**< Sum numeric values */ 34 | COUCHSTORE_REDUCE_STATS = 3, /**< Compute count, min, max, sum, sum of squares */ 35 | }; 36 | 37 | 38 | /** 39 | * Create a new index file. 40 | * 41 | * The file should be closed with couchstore_close_index(). 42 | * 43 | * @param filename The name of the file containing the index. Any existing file at this path 44 | * will be deleted. 45 | * @param index Pointer to where you want the handle to the index to be 46 | * stored. 47 | * @return COUCHSTORE_SUCCESS for success 48 | */ 49 | LIBCOUCHSTORE_API 50 | couchstore_error_t couchstore_create_index(const char *filename, 51 | CouchStoreIndex** index); 52 | 53 | /** 54 | * Close an open index file. 55 | * 56 | * @param index Pointer to the index handle to free. 57 | * @return COUCHSTORE_SUCCESS upon success 58 | */ 59 | LIBCOUCHSTORE_API 60 | couchstore_error_t couchstore_close_index(CouchStoreIndex* index); 61 | 62 | /** 63 | * Read an unsorted key-value file and add its contents to an index file. 64 | * Each file added will create a new independent index within the file; they are not merged. 65 | * 66 | * The key-value file is a sequence of zero or more records, each of which consists of: 67 | * key length (16 bits, big-endian) 68 | * value length (32 bits, big-endian) 69 | * key data 70 | * value data 71 | * The data formats inside the actual keys and values differ with the index types, and are 72 | * documented in "The Binary (Termless) Format for Views" (view_format.md). 73 | * 74 | * @param inputPath The path to the key-value file 75 | * @param index_type The type of index keys/values in the input file, and the type of index 76 | * to generate in the index file. Note: CouchStore can currently add only one back index. 77 | * @param reduce_function The type of JSON reduce function to apply to the data. Valid only 78 | * for primary indexes; ignored in back-indexes. 79 | * @param index The index file to write to 80 | * @return COUCHSTORE_SUCCESS on success, else an error code 81 | */ 82 | LIBCOUCHSTORE_API 83 | couchstore_error_t couchstore_index_add(const char *inputPath, 84 | couchstore_index_type index_type, 85 | couchstore_json_reducer reduce_function, 86 | CouchStoreIndex* index); 87 | 88 | #ifdef __cplusplus 89 | } 90 | #endif 91 | #endif 92 | -------------------------------------------------------------------------------- /include/libcouchstore/error.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef LIBCOUCHSTORE_ERROR_H 3 | #define LIBCOUCHSTORE_ERROR_H 1 4 | 5 | #ifndef COUCHSTORE_COUCH_DB_H 6 | #error "You should include instead" 7 | #endif 8 | 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | 13 | /** Error values returned by CouchStore API calls. */ 14 | typedef enum { 15 | COUCHSTORE_SUCCESS = 0, 16 | COUCHSTORE_ERROR_OPEN_FILE = -1, 17 | COUCHSTORE_ERROR_CORRUPT = -2, 18 | COUCHSTORE_ERROR_ALLOC_FAIL = -3, 19 | COUCHSTORE_ERROR_READ = -4, 20 | COUCHSTORE_ERROR_DOC_NOT_FOUND = -5, 21 | COUCHSTORE_ERROR_NO_HEADER = -6, 22 | COUCHSTORE_ERROR_WRITE = -7, 23 | COUCHSTORE_ERROR_HEADER_VERSION = -8, 24 | COUCHSTORE_ERROR_CHECKSUM_FAIL = -9, 25 | COUCHSTORE_ERROR_INVALID_ARGUMENTS = -10, 26 | COUCHSTORE_ERROR_NO_SUCH_FILE = -11, 27 | COUCHSTORE_ERROR_CANCEL = -12, 28 | COUCHSTORE_ERROR_REDUCTION_TOO_LARGE = -13, 29 | COUCHSTORE_ERROR_REDUCER_FAILURE = -14, 30 | COUCHSTORE_ERROR_FILE_CLOSED = -15, 31 | COUCHSTORE_ERROR_DB_NO_LONGER_VALID = -16, 32 | COUCHSTORE_ERROR_FILE_CLOSE = -17, 33 | } couchstore_error_t; 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /include/libcouchstore/file_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef LIBCOUCHSTORE_FILE_OPS_H 19 | #define LIBCOUCHSTORE_FILE_OPS_H 20 | 21 | #include 22 | #include 23 | #include "couch_common.h" 24 | 25 | /** 26 | * Abstract file handle. Implementations can use it for anything 27 | * they want, whether a pointer to an allocated data structure, or 28 | * an integer such as a Unix file descriptor. 29 | */ 30 | typedef struct couch_file_handle_opaque* couch_file_handle; 31 | 32 | typedef struct { 33 | #ifdef WIN32 34 | DWORD error; 35 | #else 36 | int error; 37 | #endif 38 | } couchstore_error_info_t; 39 | 40 | 41 | 42 | #ifdef __cplusplus 43 | /** 44 | * An abstract base class that defines the interface of the file 45 | * I/O primitives used by CouchStore. Passed to couchstore_open_db_ex(). 46 | */ 47 | class FileOpsInterface { 48 | public: 49 | /** 50 | * Virtual destructor used for optional cleanup 51 | */ 52 | virtual ~FileOpsInterface() {} 53 | 54 | /** 55 | * Initialize state (e.g. allocate memory) for a file handle 56 | * before opening a file. This method is optional and 57 | * doesn't need to do anything at all; it can just return NULL 58 | * if there isn't anything to do. 59 | * 60 | * Note: No error checking is done on the result of this call 61 | * so any failure should be handled accordingly (e.g. error 62 | * when calling the `open` method). 63 | */ 64 | virtual couch_file_handle constructor(couchstore_error_info_t* errinfo) = 0; 65 | 66 | /** 67 | * Open a file. 68 | * 69 | * @param on input, a pointer to the file handle that was 70 | * returned by the constructor function. The function 71 | * can change this value if it wants to; the value 72 | * stored here on return is the one that will be passed 73 | * to the other functions. 74 | * @param path the name of the file 75 | * @param flags flags as specified by UNIX open(2) system call 76 | * @return COUCHSTORE_SUCCESS upon success. 77 | */ 78 | virtual couchstore_error_t open(couchstore_error_info_t* errinfo, 79 | couch_file_handle* handle, const char* path, 80 | int oflag) = 0; 81 | 82 | /** 83 | * Close file associated with this handle. 84 | * 85 | * @param handle file handle to close 86 | * @return COUCHSTORE_SUCCESS upon success, COUCHSTORE_ERROR_FILE_CLOSE if 87 | * there was an error. 88 | */ 89 | virtual couchstore_error_t close(couchstore_error_info_t* errinfo, 90 | couch_file_handle handle) = 0; 91 | 92 | /** 93 | * Read a chunk of data from a given offset in the file. 94 | * 95 | * @param handle file handle to read from 96 | * @param buf where to store data 97 | * @param nbyte number of bytes to read 98 | * @param offset where to read from 99 | * @return number of bytes read (which may be less than nbytes), 100 | * or a value <= 0 if an error occurred 101 | */ 102 | virtual ssize_t pread(couchstore_error_info_t* errinfo, 103 | couch_file_handle handle, void* buf, size_t nbytes, 104 | cs_off_t offset) = 0; 105 | 106 | /** 107 | * Write a chunk of data to a given offset in the file. 108 | * 109 | * @param handle file handle to write to 110 | * @param buf where to read data 111 | * @param nbyte number of bytes to write 112 | * @param offset where to write to 113 | * @return number of bytes written (which may be less than nbytes), 114 | * or a value <= 0 if an error occurred 115 | */ 116 | virtual ssize_t pwrite(couchstore_error_info_t* errinfo, 117 | couch_file_handle handle, const void* buf, 118 | size_t nbytes, cs_off_t offset) = 0; 119 | 120 | /** 121 | * Find the end of the file. 122 | * 123 | * @param handle file handle to find the offset for 124 | * @return the offset (from beginning of the file), or -1 if 125 | * the operation failed 126 | */ 127 | virtual cs_off_t goto_eof(couchstore_error_info_t* errinfo, 128 | couch_file_handle handle) = 0; 129 | 130 | /** 131 | * Flush the buffers to disk 132 | * 133 | * @param handle file handle to flush 134 | * @return COUCHSTORE_SUCCESS upon success 135 | */ 136 | virtual couchstore_error_t sync(couchstore_error_info_t* errinfo, 137 | couch_file_handle handle) = 0; 138 | 139 | /** 140 | * Give filesystem caching advice. 141 | * @param handle file handle to give advice on 142 | * @param offset offset to start at 143 | * @param len length of range to advise on 144 | * @param advice the advice type, see couchstore_file_advice_t 145 | * in couch_common.h 146 | */ 147 | virtual couchstore_error_t advise(couchstore_error_info_t* errinfo, 148 | couch_file_handle handle, cs_off_t offset, 149 | cs_off_t len, 150 | couchstore_file_advice_t advice) = 0; 151 | 152 | /** 153 | * Called as part of shutting down the db instance this instance was 154 | * passed to. A hook to for releasing allocated resources 155 | * 156 | * @param handle file handle to be released 157 | */ 158 | virtual void destructor(couch_file_handle handle) = 0; 159 | }; 160 | 161 | #else 162 | /** 163 | * Opaque reference to a FileOpsInterface instance 164 | */ 165 | typedef struct couch_file_ops_opaque* FileOpsInterface; 166 | 167 | #endif 168 | 169 | #endif 170 | -------------------------------------------------------------------------------- /include/libcouchstore/visibility.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef LIBCOUCHSTORE_VISIBILITY_H 3 | #define LIBCOUCHSTORE_VISIBILITY_H 4 | 5 | #if defined(LIBCOUCHSTORE_INTERNAL) 6 | 7 | #if defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) 8 | #define LIBCOUCHSTORE_API __global 9 | #elif defined __GNUC__ 10 | #define LIBCOUCHSTORE_API __attribute__ ((visibility("default"))) 11 | #elif defined(_MSC_VER) 12 | #define LIBCOUCHSTORE_API extern __declspec(dllexport) 13 | #else 14 | #define LIBCOUCHSTORE_API 15 | #endif 16 | 17 | #else 18 | 19 | #if defined(_MSC_VER) && !defined(LIBCOUCHSTORE_NO_VISIBILITY) 20 | #define LIBCOUCHSTORE_API extern __declspec(dllimport) 21 | #else 22 | #define LIBCOUCHSTORE_API 23 | #endif 24 | 25 | #endif 26 | 27 | 28 | #if defined(TESTAPP) 29 | #define STATIC 30 | #else 31 | #define STATIC static 32 | #endif 33 | 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /utils/adv_random.h: -------------------------------------------------------------------------------- 1 | /** Copyright (c) 2011 TU Dresden - Database Technology Group 2 | * 3 | * Permission is hereby granted, free of charge, to any person obtaining 4 | * a copy of this software and associated documentation files 5 | * (the "Software"), to deal in the Software without restriction, 6 | * including without limitation the rights to use, copy, modify, merge, 7 | * publish, distribute, sublicense, and/or sell copies of the Software, 8 | * and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * The above copyright notice and this permission notice shall be included 11 | * in all copies or substantial portions of the Software. 12 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 13 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 14 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 15 | * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 16 | * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 17 | * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | * 20 | * Author: T. Kissinger 21 | */ 22 | 23 | // Modified by Jung-Sang Ahn in 2013 24 | 25 | #ifndef _JSAHN_ADV_RANDOM_H 26 | #define _JSAHN_ADV_RANDOM_H 27 | 28 | #include 29 | #include 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | #ifndef UINT64_MAX 36 | #define UINT64_MAX ((uint64_t)0xffffffffffffffff) 37 | #endif 38 | 39 | #define BDR_RNG_VARS \ 40 | uint64_t rngx=rand(), rngy=362436069, rngz=521288629; \ 41 | uint64_t rngt, rngz2; (void)rngt; (void)rngz2; 42 | 43 | #define BDR_RNG_VARS_SET(x) \ 44 | uint64_t rngx, rngy, rngz; \ 45 | rngx = (x); \ 46 | rngy = rngx; rngy ^= rngy << 16; rngy ^= rngy >> 5; rngy ^= rngy << 1; \ 47 | rngz = rngy; rngz ^= rngz << 16; rngz ^= rngz >> 5; rngz ^= rngz << 1; \ 48 | uint64_t rngt, rngz2; (void)rngt; (void)rngz2; 49 | 50 | //rngz contains the new value 51 | #define BDR_RNG_NEXT \ 52 | rngx ^= rngx << 16; \ 53 | rngx ^= rngx >> 5; \ 54 | rngx ^= rngx << 1; \ 55 | rngt = rngx; \ 56 | rngx = rngy; \ 57 | rngy = rngz; \ 58 | rngz = rngt ^ rngx ^ rngy; 59 | 60 | //rngz, rngz2 contains the new values 61 | #define BDR_RNG_NEXTPAIR \ 62 | rngx ^= rngx << 16; \ 63 | rngx ^= rngx >> 5; \ 64 | rngx ^= rngx << 1; \ 65 | rngt = rngx; \ 66 | rngx = rngy; \ 67 | rngy = rngz; \ 68 | rngz = rngt ^ rngx ^ rngy; \ 69 | rngz2 = rngz; \ 70 | rngx ^= rngx << 16; \ 71 | rngx ^= rngx >> 5; \ 72 | rngx ^= rngx << 1; \ 73 | rngt = rngx; \ 74 | rngx = rngy; \ 75 | rngy = rngz; \ 76 | rngz = rngt ^ rngx ^ rngy; 77 | 78 | #define BDR_RNG_GET_INT(number) (rngz % (number)) 79 | #define BDR_RNG_GET_INT_PAIR(a, b, number) \ 80 | (a) = (rngz % (number)); \ 81 | (b) = (rngz2 % (number)); 82 | 83 | typedef enum { 84 | RND_UNIFORM, 85 | RND_NORMAL, 86 | RND_ZIPFIAN, 87 | } rndtype_t; 88 | 89 | struct rndinfo{ 90 | rndtype_t type; 91 | // for uniform: lower bound of range (including itself) 92 | // for normal: average (or median) 93 | int64_t a; 94 | // for uniform: upper bound of range (including itself but extremely rare (probability == 1/(2^64)) 95 | // for normal: standard deviation (=sigma) 96 | int64_t b; 97 | }; 98 | 99 | static double __PI = 3.141592654; 100 | 101 | static int64_t get_random(struct rndinfo* ri, uint64_t rv1, uint64_t rv2) 102 | { 103 | if (ri->type == RND_UNIFORM) 104 | { 105 | double anorm = ((double)rv1) / UINT64_MAX; 106 | return (int64_t)(anorm * (ri->b - ri->a) + ri->a); 107 | } 108 | else if (ri->type == RND_NORMAL){ 109 | double r1, r2; 110 | r1 = -log(1-(((double)rv1) / UINT64_MAX )); 111 | r2 = 2 * __PI * (((double)rv2) / UINT64_MAX ); 112 | r1 = sqrt(2*r1); 113 | return (int64_t)(ri->b * r1 * cos(r2) + ri->a); 114 | } 115 | return 0; 116 | } 117 | 118 | #ifdef __RAND_GEN_TEST 119 | 120 | void _rand_gen_test() 121 | { 122 | int n = 32, m = 1<<20; 123 | int arr[n]; 124 | int i; 125 | double cdf; 126 | int64_t r; 127 | struct rndinfo ri; 128 | BDR_RNG_VARS; 129 | 130 | memset(arr, 0, sizeof(int)*n); 131 | cdf = 0; 132 | ri.type = RND_UNIFORM; 133 | ri.a = 0; 134 | ri.b = 32; 135 | 136 | for (i=0;i=n) r=n-1; 158 | arr[r]++; 159 | } 160 | 161 | for (i=0;i 5 | * see https://github.com/greensky00/avltree 6 | */ 7 | 8 | #ifndef INLINE 9 | #ifdef __APPLE__ 10 | #define INLINE extern inline 11 | #elif __linux__ 12 | #define INLINE __inline 13 | #else 14 | #define INLINE 15 | #endif 16 | #endif 17 | 18 | #include "avltree.h" 19 | 20 | #define max(a,b) (((a) > (b)) ? (a) : (b)) 21 | 22 | INLINE int _abs(int n) { 23 | int mask = n >> ((sizeof(int)*8) -1); 24 | return (mask + n)^mask; 25 | } 26 | 27 | INLINE void avl_set_parent(struct avl_node *node, struct avl_node *parent) 28 | { 29 | node->parent = (struct avl_node *)( 30 | (uint64_t)parent | ((uint64_t)node->parent & 0x3)); 31 | } 32 | 33 | #ifdef __AVL_DEBUG 34 | #include 35 | #include 36 | #include "avltree_debug.h" 37 | #else 38 | #define __AVL_DEBUG_BF_CHECK(bf) 39 | #define __AVL_DEBUG_LL(p, c, pb, cb) 40 | #define __AVL_DEBUG_RR(p, c, pb, cb) 41 | #define __AVL_DEBUG_BAL_BEGIN(node, bf, height_diff) 42 | #define __AVL_DEBUG_BAL_END(node) 43 | #define __AVL_DEBUG_INSERT(node) 44 | #define __AVL_DEBUG_REMOVE(node) 45 | #define __AVL_DEBUG_DISPLAY(tree) 46 | #endif 47 | 48 | INLINE void avl_set_bf(struct avl_node *node, int bf) 49 | { 50 | __AVL_DEBUG_BF_CHECK(bf); 51 | 52 | #ifdef _AVL_SEPARATE_PARENT_BF 53 | node->bf = bf; 54 | #else 55 | node->parent = (struct avl_node *)( 56 | (uint64_t)avl_parent(node) | (uint64_t)(bf+1)); 57 | #endif 58 | } 59 | 60 | INLINE struct avl_node* _rotate_LL(struct avl_node *parent, 61 | int parent_bf, 62 | int *child_bf, 63 | int *height_delta) 64 | // MUST ensure that parent_bf <= 0 65 | { 66 | int p_right, c_left, c_right; 67 | struct avl_node *child = parent->left; 68 | 69 | __AVL_DEBUG_LL(parent, child, parent_bf, *child_bf); 70 | 71 | c_left = (child->left)?(1):(0); 72 | c_right = (child->right)?(1):(0); 73 | if (*child_bf < 0) { 74 | // child->left > child->right 75 | c_left = c_right - (*child_bf); 76 | p_right = c_left + 1 + parent_bf; 77 | if (height_delta) 78 | *height_delta = max(c_left, max(c_right, p_right)+1) - (c_left + 1); 79 | 80 | } else { 81 | // child->left <= child->right 82 | c_right = c_left + (*child_bf); 83 | p_right = c_right + 1 + parent_bf; 84 | if (height_delta) 85 | *height_delta = max(c_left, max(c_right, p_right)+1) - (c_right + 1); 86 | } 87 | *child_bf = (max(c_right, p_right) + 1) - c_left; 88 | avl_set_bf(parent, p_right - c_right); 89 | 90 | parent->left = child->right; 91 | if (child->right) 92 | avl_set_parent(child->right, parent); 93 | child->right = parent; 94 | avl_set_parent(child, avl_parent(parent)); 95 | avl_set_parent(parent, child); 96 | 97 | return child; 98 | } 99 | 100 | INLINE struct avl_node* _rotate_RR(struct avl_node *parent, 101 | int parent_bf, 102 | int *child_bf, 103 | int *height_delta) 104 | // MUST ensure that parent_bf >= 0 105 | { 106 | int p_left, c_left, c_right; 107 | struct avl_node *child = parent->right; 108 | 109 | __AVL_DEBUG_RR(parent, child, parent_bf, *child_bf); 110 | 111 | c_left = (child->left)?(1):(0); 112 | c_right = (child->right)?(1):(0); 113 | if (*child_bf < 0) { 114 | // child->left > child->right 115 | c_left = c_right - (*child_bf); 116 | p_left = c_left + 1 - parent_bf; 117 | if (height_delta) 118 | *height_delta = max(c_right, max(c_left, p_left)+1) - (c_left + 1); 119 | 120 | } else { 121 | // child->left <= child->right 122 | c_right = c_left + (*child_bf); 123 | p_left = c_right + 1 - parent_bf; 124 | if (height_delta) 125 | *height_delta = max(c_right, max(c_left, p_left)+1) - (c_right + 1); 126 | 127 | } 128 | *child_bf = c_right - (max(c_left, p_left) + 1); 129 | avl_set_bf(parent, c_left - p_left); 130 | 131 | parent->right = child->left; 132 | if (child->left) 133 | avl_set_parent(child->left, parent); 134 | child->left = parent; 135 | avl_set_parent(child, avl_parent(parent)); 136 | avl_set_parent(parent, child); 137 | 138 | return child; 139 | } 140 | 141 | INLINE struct avl_node* _rotate_LR(struct avl_node *parent, int parent_bf) 142 | { 143 | int child_bf, height_delta = 0; 144 | struct avl_node *child = parent->left; 145 | struct avl_node *ret; 146 | 147 | if (child->right) { 148 | child_bf = avl_bf(child->right); 149 | parent->left = _rotate_RR(child, avl_bf(child), &child_bf, &height_delta); 150 | } else { 151 | child_bf = avl_bf(child); 152 | } 153 | 154 | ret = _rotate_LL(parent, parent_bf-height_delta, &child_bf, NULL); 155 | avl_set_bf(ret, child_bf); 156 | return ret; 157 | } 158 | 159 | INLINE struct avl_node* _rotate_RL(struct avl_node *parent, int parent_bf) 160 | { 161 | int child_bf, height_delta = 0; 162 | struct avl_node *child = parent->right; 163 | struct avl_node *ret; 164 | 165 | if (child->left) { 166 | child_bf = avl_bf(child->left); 167 | parent->right = _rotate_LL(child, avl_bf(child), &child_bf, &height_delta); 168 | } else { 169 | child_bf = avl_bf(child); 170 | } 171 | 172 | ret = _rotate_RR(parent, parent_bf+height_delta, &child_bf, NULL); 173 | avl_set_bf(ret, child_bf); 174 | return ret; 175 | } 176 | 177 | #define _get_balance(node) ((node)?(avl_bf(node)):(0)) 178 | 179 | static struct avl_node* _balance_tree(struct avl_node *node, int bf) 180 | { 181 | int child_bf; 182 | int height_diff= _get_balance(node) + bf; 183 | 184 | if (node) { 185 | __AVL_DEBUG_BAL_BEGIN(node, bf, height_diff); 186 | 187 | if(height_diff < -1 && node->left) { 188 | // balance left sub tree 189 | if(_get_balance(node->left) <= 0) { 190 | child_bf = avl_bf(node->left); 191 | node = _rotate_LL(node, height_diff, &child_bf, NULL); 192 | avl_set_bf(node, child_bf); 193 | } else { 194 | node = _rotate_LR(node, height_diff); 195 | } 196 | } else if(height_diff > 1 && node->right) { 197 | // balance right sub tree 198 | if(_get_balance(node->right) >= 0) { 199 | child_bf = avl_bf(node->right); 200 | node = _rotate_RR(node, height_diff, &child_bf, NULL); 201 | avl_set_bf(node, child_bf); 202 | } else { 203 | node = _rotate_RL(node, height_diff); 204 | } 205 | } else { 206 | avl_set_bf(node, avl_bf(node) + bf); 207 | } 208 | 209 | __AVL_DEBUG_BAL_END(node); 210 | } 211 | 212 | return node; 213 | } 214 | 215 | struct avl_node* avl_first(struct avl_tree *tree) 216 | { 217 | struct avl_node *p = NULL; 218 | struct avl_node *node = tree->root; 219 | 220 | while(node) { 221 | p = node; 222 | node = node->left; 223 | } 224 | return p; 225 | } 226 | 227 | struct avl_node* avl_last(struct avl_tree *tree) 228 | { 229 | struct avl_node *p = NULL; 230 | struct avl_node *node = tree->root; 231 | 232 | while(node) { 233 | p = node; 234 | node = node->right; 235 | } 236 | return p; 237 | } 238 | 239 | struct avl_node* avl_next(struct avl_node *node) 240 | { 241 | if (node == NULL) return NULL; 242 | 243 | #ifdef _AVL_NEXT_POINTER 244 | return node->next; 245 | #else 246 | 247 | struct avl_node *p; 248 | 249 | // smallest value of right subtree 250 | if (node->right) { 251 | p = node; 252 | node = node->right; 253 | while (node) { 254 | p = node; 255 | node = node->left; 256 | } 257 | return p; 258 | } 259 | 260 | // node does not have right child 261 | if (avl_parent(node)) { 262 | // find first parent that has right child 263 | p = node; 264 | node = avl_parent(node); 265 | while(node) { 266 | if (node->left == p) { 267 | return node; 268 | } 269 | p = node; 270 | node = avl_parent(node); 271 | } 272 | } 273 | #endif 274 | return NULL; 275 | } 276 | 277 | struct avl_node* avl_prev(struct avl_node *node) 278 | { 279 | if (node == NULL) return NULL; 280 | 281 | #ifdef _AVL_NEXT_POINTER 282 | return node->prev; 283 | #else 284 | 285 | struct avl_node *p; 286 | 287 | // largest value of left subtree 288 | if (node->left) { 289 | p = node; 290 | node = node->left; 291 | while (node) { 292 | p = node; 293 | node = node->right; 294 | } 295 | return p; 296 | } 297 | 298 | // node does not have left child 299 | if (avl_parent(node)) { 300 | // find first parent that has left child 301 | p = node; 302 | node = avl_parent(node); 303 | while(node) { 304 | if (node->right == p) { 305 | return node; 306 | } 307 | p = node; 308 | node = avl_parent(node); 309 | } 310 | } 311 | #endif 312 | return NULL; 313 | } 314 | 315 | struct avl_node* avl_search(struct avl_tree *tree, 316 | struct avl_node *node, 317 | avl_cmp_func *func) 318 | // exact match 319 | { 320 | struct avl_node *p = tree->root; 321 | int cmp; 322 | 323 | while(p) 324 | { 325 | cmp = func(p, node, tree->aux); 326 | if (cmp > 0) { 327 | p = p->left; 328 | }else if (cmp < 0){ 329 | p = p->right; 330 | }else { 331 | // search success 332 | return p; 333 | } 334 | } 335 | // search fail 336 | return NULL; 337 | } 338 | 339 | struct avl_node* avl_search_greater(struct avl_tree *tree, 340 | struct avl_node *node, 341 | avl_cmp_func *func) 342 | // if an exact match does not exist, 343 | // return smallest node greater than NODE 344 | { 345 | struct avl_node *p = tree->root; 346 | struct avl_node *pp = NULL; 347 | int cmp; 348 | 349 | while(p) 350 | { 351 | cmp = func(p, node, tree->aux); 352 | pp = p; 353 | 354 | if (cmp > 0) { 355 | p = p->left; 356 | }else if (cmp < 0){ 357 | p = p->right; 358 | }else { 359 | // search success 360 | return p; 361 | } 362 | } 363 | 364 | if (!pp) { 365 | return pp; 366 | } 367 | 368 | cmp = func(pp, node, tree->aux); 369 | if (cmp > 0) { 370 | return pp; 371 | }else{ 372 | return avl_next(pp); 373 | } 374 | } 375 | 376 | struct avl_node* avl_search_smaller(struct avl_tree *tree, 377 | struct avl_node *node, 378 | avl_cmp_func *func) 379 | // if an exact match does not exist, 380 | // return greatest node smaller than NODE 381 | { 382 | struct avl_node *p = tree->root; 383 | struct avl_node *pp = NULL; 384 | int cmp; 385 | 386 | while(p) 387 | { 388 | cmp = func(p, node, tree->aux); 389 | pp = p; 390 | 391 | if (cmp > 0) { 392 | p = p->left; 393 | }else if (cmp < 0){ 394 | p = p->right; 395 | }else { 396 | // search success 397 | return p; 398 | } 399 | } 400 | 401 | if (!pp) { 402 | return pp; 403 | } 404 | 405 | cmp = func(pp, node, tree->aux); 406 | if (cmp < 0) { 407 | return pp; 408 | }else{ 409 | return avl_prev(pp); 410 | } 411 | } 412 | 413 | void avl_init(struct avl_tree *tree, void *aux) 414 | { 415 | tree->root = NULL; 416 | tree->aux = aux; 417 | } 418 | 419 | struct avl_node* avl_insert(struct avl_tree *tree, 420 | struct avl_node *node, 421 | avl_cmp_func *func) 422 | { 423 | __AVL_DEBUG_INSERT(node); 424 | 425 | struct avl_node *p=NULL,*cur; 426 | int cmp, bf, bf_old; 427 | 428 | cur = tree->root; 429 | while(cur) 430 | { 431 | cmp = func(cur, node, tree->aux); 432 | p = cur; 433 | 434 | if(cmp > 0) { 435 | cur = cur->left; 436 | }else if (cmp < 0){ 437 | cur = cur->right; 438 | }else { 439 | // duplicated key -> return 440 | return cur; 441 | } 442 | } 443 | 444 | avl_set_parent(node, p); 445 | avl_set_bf(node, 0); 446 | node->left = node->right = NULL; 447 | #ifdef _AVL_NEXT_POINTER 448 | node->prev = node->next = NULL; 449 | #endif 450 | 451 | // P is parent node of CUR 452 | if(p) { 453 | if(func(p, node, tree->aux) > 0) { 454 | p->left = node; 455 | #ifdef _AVL_NEXT_POINTER 456 | node->next = p; 457 | node->prev = p->prev; 458 | if (p->prev) p->prev->next = node; 459 | p->prev = node; 460 | #endif 461 | 462 | }else { 463 | p->right = node; 464 | #ifdef _AVL_NEXT_POINTER 465 | node->prev = p; 466 | node->next = p->next; 467 | if (p->next) p->next->prev = node; 468 | p->next = node; 469 | #endif 470 | } 471 | 472 | } else { 473 | // no parent .. make NODE as root 474 | tree->root = node; 475 | } 476 | 477 | // recursive balancing process .. scan from leaf to root 478 | bf = 0; 479 | while(node) { 480 | p = avl_parent(node); 481 | 482 | if (p) { 483 | // if parent exists 484 | bf_old = avl_bf(node); 485 | 486 | if (p->right == node) { 487 | node = _balance_tree(node, bf); 488 | p->right = node; 489 | }else { 490 | node = _balance_tree(node, bf); 491 | p->left = node; 492 | } 493 | 494 | // calculate balance facter BF for parent 495 | if (node->left == NULL && node->right == NULL) { 496 | // leaf node 497 | if (p->left == node) bf = -1; 498 | else bf = 1; 499 | } else { 500 | // index ndoe 501 | bf = 0; 502 | if (_abs(bf_old) < _abs(avl_bf(node))) { 503 | // if ABS of balance factor increases 504 | // cascade to parent 505 | if (p->left == node) bf = -1; 506 | else bf = 1; 507 | } 508 | } 509 | 510 | } else if(node == tree->root){ 511 | tree->root = _balance_tree(tree->root, bf); 512 | break; 513 | } 514 | if (bf == 0) break; 515 | 516 | node = p; 517 | } 518 | 519 | __AVL_DEBUG_DISPLAY(tree); 520 | 521 | return node; 522 | } 523 | 524 | void avl_remove(struct avl_tree *tree, 525 | struct avl_node *node) 526 | { 527 | __AVL_DEBUG_REMOVE(node); 528 | 529 | // not found 530 | if (node == NULL) return; 531 | 532 | struct avl_tree right_subtree; 533 | struct avl_node *p=NULL,*cur, *next=NULL; 534 | int bf = 0, bf_old; 535 | 536 | 537 | #ifdef _AVL_NEXT_POINTER 538 | if (node->prev) node->prev->next = node->next; 539 | if (node->next) node->next->prev = node->prev; 540 | #endif 541 | 542 | // find smallest node in right sub-tree 543 | right_subtree.root = node->right; 544 | next = avl_first(&right_subtree); 545 | 546 | if (next) { 547 | // 1. NEXT exists 548 | if (avl_parent(next)) { 549 | if (avl_parent(next) != node) { 550 | // NODE is not NEXT's direct parent 551 | // MUST ensure NEXT should be *left child* of its parent 552 | // MUST ensure NEXT doesn't have right child 553 | avl_parent(next)->left = next->right; 554 | if (next->right) 555 | avl_set_parent(next->right, avl_parent(next)); 556 | } 557 | } 558 | if (avl_parent(node)) { 559 | // replace NODE by NEXT 560 | if (avl_parent(node)->left == node) { 561 | avl_parent(node)->left = next; 562 | } else { 563 | avl_parent(node)->right = next; 564 | } 565 | } 566 | 567 | // re-link pointers 568 | if (node->right != next) { 569 | next->right = node->right; 570 | if (node->right) avl_set_parent(node->right, next); 571 | cur = avl_parent(next); 572 | bf = 1; 573 | }else{ 574 | cur = next; 575 | bf = -1; 576 | } 577 | 578 | next->left = node->left; 579 | if (node->left) avl_set_parent(node->left, next); 580 | avl_set_parent(next, avl_parent(node)); 581 | 582 | // inherit NODE's balance factor 583 | avl_set_bf(next, avl_bf(node)); 584 | 585 | } else { 586 | // 2. NEXT == NULL (only when there's no right sub-tree) 587 | p = avl_parent(node); 588 | if (p) { 589 | if (p->left == node) { 590 | p->left = node->left; 591 | bf = 1; 592 | } else { 593 | p->right = node->left; 594 | bf = -1; 595 | } 596 | } 597 | if (node->left) 598 | avl_set_parent(node->left, p); 599 | 600 | cur = avl_parent(node); 601 | } 602 | 603 | // reset root 604 | if (tree->root == node) { 605 | tree->root = next; 606 | if (next == NULL) { 607 | if (node->left) tree->root = node->left; 608 | } 609 | } 610 | 611 | // recursive balancing process .. scan from CUR to root 612 | while(cur) { 613 | p = avl_parent(cur); 614 | if (p) { 615 | // if parent exists 616 | bf_old = avl_bf(cur); 617 | 618 | if (p->right == cur) { 619 | cur = _balance_tree(cur, bf); 620 | p->right = cur; 621 | }else { 622 | cur = _balance_tree(cur, bf); 623 | p->left = cur; 624 | } 625 | 626 | // calculate balance facter BF for parent 627 | if (cur->left == NULL && cur->right == NULL) { 628 | // leaf node 629 | if (p->left == cur) bf = 1; 630 | else bf = -1; 631 | } else { 632 | // index ndoe 633 | bf = 0; 634 | if (_abs(bf_old) > _abs(avl_bf(cur))) { 635 | // if ABS of balance factor decreases 636 | // cascade to parent 637 | if (p->left == cur) bf = 1; 638 | else bf = -1; 639 | } 640 | } 641 | 642 | } else if(cur == tree->root){ 643 | tree->root = _balance_tree(tree->root, bf); 644 | break; 645 | } 646 | if (bf == 0) break; 647 | 648 | cur = p; 649 | } 650 | 651 | __AVL_DEBUG_DISPLAY(tree); 652 | } 653 | 654 | -------------------------------------------------------------------------------- /utils/avltree.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * AVL Tree 4 | * (C) 2014 Jung-Sang Ahn 5 | * see https://github.com/greensky00/avltree 6 | */ 7 | 8 | #ifndef _JSAHN_AVL_TREE_H 9 | #define _JSAHN_AVL_TREE_H 10 | 11 | #include "stddef.h" 12 | #include "stdint.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | struct avl_node { 19 | struct avl_node *parent, *left, *right; 20 | 21 | #ifdef _AVL_SEPARATE_PARENT_BF 22 | int bf; 23 | #endif 24 | #ifdef _AVL_NEXT_POINTER 25 | struct avl_node *prev, *next; 26 | #endif 27 | }; 28 | 29 | struct avl_tree{ 30 | struct avl_node *root; 31 | void *aux; 32 | }; 33 | 34 | #ifndef _get_entry 35 | #define _get_entry(ELEM, STRUCT, MEMBER) \ 36 | ((STRUCT *) ((uint8_t *) (ELEM) - offsetof (STRUCT, MEMBER))) 37 | #endif 38 | 39 | #define avl_parent(node) \ 40 | ((struct avl_node *)((uint64_t)(node)->parent & ~0x3)) 41 | 42 | #ifdef _AVL_SEPARATE_PARENT_BF 43 | #define avl_bf(node) ((node)->bf) 44 | #else 45 | #define avl_bf(node) (((int)((uint64_t)(node)->parent & 0x3)) - 1) 46 | #endif 47 | 48 | // *a < *b : return neg 49 | // *a == *b : return 0 50 | // *a > *b : return pos 51 | typedef int avl_cmp_func (struct avl_node *a, struct avl_node *b, void *aux); 52 | 53 | void avl_init(struct avl_tree *tree, void *aux); 54 | struct avl_node* avl_insert(struct avl_tree *tree, 55 | struct avl_node *node, 56 | avl_cmp_func *func); 57 | struct avl_node* avl_search(struct avl_tree *tree, 58 | struct avl_node *node, 59 | avl_cmp_func *func); 60 | struct avl_node* avl_search_greater(struct avl_tree *tree, 61 | struct avl_node *node, 62 | avl_cmp_func *func); 63 | struct avl_node* avl_search_smaller(struct avl_tree *tree, 64 | struct avl_node *node, 65 | avl_cmp_func *func); 66 | void avl_remove(struct avl_tree *tree, 67 | struct avl_node *node); 68 | struct avl_node* avl_first(struct avl_tree *tree); 69 | struct avl_node* avl_last(struct avl_tree *tree); 70 | struct avl_node* avl_next(struct avl_node *node); 71 | struct avl_node* avl_prev(struct avl_node *node); 72 | 73 | #ifdef __cplusplus 74 | } 75 | #endif 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /utils/crc32.cc: -------------------------------------------------------------------------------- 1 | // ////////////////////////////////////////////////////////// 2 | // Crc32.cpp 3 | // Copyright (c) 2011-2013 Stephan Brumme. All rights reserved. 4 | // see http://create.stephan-brumme.com/disclaimer.html 5 | // 6 | 7 | /* 8 | * modified by Jung-Sang Ahn in 2013. 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "crc32.h" 16 | #include "arch.h" 17 | 18 | 19 | /// look-up table, already declared above 20 | const uint32_t crc_lookup[8][256] = 21 | { 22 | //// same algorithm as crc32_bitwise 23 | //for (int i = 0; i <= 0xFF; i++) 24 | //{ 25 | // uint32_t crc = i; 26 | // for (int j = 0; j < 8; j++) 27 | // crc = (crc >> 1) ^ ((crc & 1) * Polynomial); 28 | // Crc32Lookup[0][i] = crc; 29 | //} 30 | //// ... and the following slicing-by-8 algorithm (from Intel): 31 | //// http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf 32 | //// http://sourceforge.net/projects/slicing-by-8/ 33 | //for (int i = 0; i <= 0xFF; i++) 34 | //{ 35 | // Crc32Lookup[1][i] = (Crc32Lookup[0][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[0][i] & 0xFF]; 36 | // Crc32Lookup[2][i] = (Crc32Lookup[1][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[1][i] & 0xFF]; 37 | // Crc32Lookup[3][i] = (Crc32Lookup[2][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[2][i] & 0xFF]; 38 | 39 | // Crc32Lookup[4][i] = (Crc32Lookup[3][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[3][i] & 0xFF]; 40 | // Crc32Lookup[5][i] = (Crc32Lookup[4][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[4][i] & 0xFF]; 41 | // Crc32Lookup[6][i] = (Crc32Lookup[5][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[5][i] & 0xFF]; 42 | // Crc32Lookup[7][i] = (Crc32Lookup[6][i] >> 8) ^ Crc32Lookup[0][Crc32Lookup[6][i] & 0xFF]; 43 | //} 44 | { 0x00000000,0x77073096,0xEE0E612C,0x990951BA,0x076DC419,0x706AF48F,0xE963A535,0x9E6495A3, 45 | 0x0EDB8832,0x79DCB8A4,0xE0D5E91E,0x97D2D988,0x09B64C2B,0x7EB17CBD,0xE7B82D07,0x90BF1D91, 46 | 0x1DB71064,0x6AB020F2,0xF3B97148,0x84BE41DE,0x1ADAD47D,0x6DDDE4EB,0xF4D4B551,0x83D385C7, 47 | 0x136C9856,0x646BA8C0,0xFD62F97A,0x8A65C9EC,0x14015C4F,0x63066CD9,0xFA0F3D63,0x8D080DF5, 48 | 0x3B6E20C8,0x4C69105E,0xD56041E4,0xA2677172,0x3C03E4D1,0x4B04D447,0xD20D85FD,0xA50AB56B, 49 | 0x35B5A8FA,0x42B2986C,0xDBBBC9D6,0xACBCF940,0x32D86CE3,0x45DF5C75,0xDCD60DCF,0xABD13D59, 50 | 0x26D930AC,0x51DE003A,0xC8D75180,0xBFD06116,0x21B4F4B5,0x56B3C423,0xCFBA9599,0xB8BDA50F, 51 | 0x2802B89E,0x5F058808,0xC60CD9B2,0xB10BE924,0x2F6F7C87,0x58684C11,0xC1611DAB,0xB6662D3D, 52 | 0x76DC4190,0x01DB7106,0x98D220BC,0xEFD5102A,0x71B18589,0x06B6B51F,0x9FBFE4A5,0xE8B8D433, 53 | 0x7807C9A2,0x0F00F934,0x9609A88E,0xE10E9818,0x7F6A0DBB,0x086D3D2D,0x91646C97,0xE6635C01, 54 | 0x6B6B51F4,0x1C6C6162,0x856530D8,0xF262004E,0x6C0695ED,0x1B01A57B,0x8208F4C1,0xF50FC457, 55 | 0x65B0D9C6,0x12B7E950,0x8BBEB8EA,0xFCB9887C,0x62DD1DDF,0x15DA2D49,0x8CD37CF3,0xFBD44C65, 56 | 0x4DB26158,0x3AB551CE,0xA3BC0074,0xD4BB30E2,0x4ADFA541,0x3DD895D7,0xA4D1C46D,0xD3D6F4FB, 57 | 0x4369E96A,0x346ED9FC,0xAD678846,0xDA60B8D0,0x44042D73,0x33031DE5,0xAA0A4C5F,0xDD0D7CC9, 58 | 0x5005713C,0x270241AA,0xBE0B1010,0xC90C2086,0x5768B525,0x206F85B3,0xB966D409,0xCE61E49F, 59 | 0x5EDEF90E,0x29D9C998,0xB0D09822,0xC7D7A8B4,0x59B33D17,0x2EB40D81,0xB7BD5C3B,0xC0BA6CAD, 60 | 0xEDB88320,0x9ABFB3B6,0x03B6E20C,0x74B1D29A,0xEAD54739,0x9DD277AF,0x04DB2615,0x73DC1683, 61 | 0xE3630B12,0x94643B84,0x0D6D6A3E,0x7A6A5AA8,0xE40ECF0B,0x9309FF9D,0x0A00AE27,0x7D079EB1, 62 | 0xF00F9344,0x8708A3D2,0x1E01F268,0x6906C2FE,0xF762575D,0x806567CB,0x196C3671,0x6E6B06E7, 63 | 0xFED41B76,0x89D32BE0,0x10DA7A5A,0x67DD4ACC,0xF9B9DF6F,0x8EBEEFF9,0x17B7BE43,0x60B08ED5, 64 | 0xD6D6A3E8,0xA1D1937E,0x38D8C2C4,0x4FDFF252,0xD1BB67F1,0xA6BC5767,0x3FB506DD,0x48B2364B, 65 | 0xD80D2BDA,0xAF0A1B4C,0x36034AF6,0x41047A60,0xDF60EFC3,0xA867DF55,0x316E8EEF,0x4669BE79, 66 | 0xCB61B38C,0xBC66831A,0x256FD2A0,0x5268E236,0xCC0C7795,0xBB0B4703,0x220216B9,0x5505262F, 67 | 0xC5BA3BBE,0xB2BD0B28,0x2BB45A92,0x5CB36A04,0xC2D7FFA7,0xB5D0CF31,0x2CD99E8B,0x5BDEAE1D, 68 | 0x9B64C2B0,0xEC63F226,0x756AA39C,0x026D930A,0x9C0906A9,0xEB0E363F,0x72076785,0x05005713, 69 | 0x95BF4A82,0xE2B87A14,0x7BB12BAE,0x0CB61B38,0x92D28E9B,0xE5D5BE0D,0x7CDCEFB7,0x0BDBDF21, 70 | 0x86D3D2D4,0xF1D4E242,0x68DDB3F8,0x1FDA836E,0x81BE16CD,0xF6B9265B,0x6FB077E1,0x18B74777, 71 | 0x88085AE6,0xFF0F6A70,0x66063BCA,0x11010B5C,0x8F659EFF,0xF862AE69,0x616BFFD3,0x166CCF45, 72 | 0xA00AE278,0xD70DD2EE,0x4E048354,0x3903B3C2,0xA7672661,0xD06016F7,0x4969474D,0x3E6E77DB, 73 | 0xAED16A4A,0xD9D65ADC,0x40DF0B66,0x37D83BF0,0xA9BCAE53,0xDEBB9EC5,0x47B2CF7F,0x30B5FFE9, 74 | 0xBDBDF21C,0xCABAC28A,0x53B39330,0x24B4A3A6,0xBAD03605,0xCDD70693,0x54DE5729,0x23D967BF, 75 | 0xB3667A2E,0xC4614AB8,0x5D681B02,0x2A6F2B94,0xB40BBE37,0xC30C8EA1,0x5A05DF1B,0x2D02EF8D }, 76 | 77 | { 0x00000000,0x191B3141,0x32366282,0x2B2D53C3,0x646CC504,0x7D77F445,0x565AA786,0x4F4196C7, 78 | 0xC8D98A08,0xD1C2BB49,0xFAEFE88A,0xE3F4D9CB,0xACB54F0C,0xB5AE7E4D,0x9E832D8E,0x87981CCF, 79 | 0x4AC21251,0x53D92310,0x78F470D3,0x61EF4192,0x2EAED755,0x37B5E614,0x1C98B5D7,0x05838496, 80 | 0x821B9859,0x9B00A918,0xB02DFADB,0xA936CB9A,0xE6775D5D,0xFF6C6C1C,0xD4413FDF,0xCD5A0E9E, 81 | 0x958424A2,0x8C9F15E3,0xA7B24620,0xBEA97761,0xF1E8E1A6,0xE8F3D0E7,0xC3DE8324,0xDAC5B265, 82 | 0x5D5DAEAA,0x44469FEB,0x6F6BCC28,0x7670FD69,0x39316BAE,0x202A5AEF,0x0B07092C,0x121C386D, 83 | 0xDF4636F3,0xC65D07B2,0xED705471,0xF46B6530,0xBB2AF3F7,0xA231C2B6,0x891C9175,0x9007A034, 84 | 0x179FBCFB,0x0E848DBA,0x25A9DE79,0x3CB2EF38,0x73F379FF,0x6AE848BE,0x41C51B7D,0x58DE2A3C, 85 | 0xF0794F05,0xE9627E44,0xC24F2D87,0xDB541CC6,0x94158A01,0x8D0EBB40,0xA623E883,0xBF38D9C2, 86 | 0x38A0C50D,0x21BBF44C,0x0A96A78F,0x138D96CE,0x5CCC0009,0x45D73148,0x6EFA628B,0x77E153CA, 87 | 0xBABB5D54,0xA3A06C15,0x888D3FD6,0x91960E97,0xDED79850,0xC7CCA911,0xECE1FAD2,0xF5FACB93, 88 | 0x7262D75C,0x6B79E61D,0x4054B5DE,0x594F849F,0x160E1258,0x0F152319,0x243870DA,0x3D23419B, 89 | 0x65FD6BA7,0x7CE65AE6,0x57CB0925,0x4ED03864,0x0191AEA3,0x188A9FE2,0x33A7CC21,0x2ABCFD60, 90 | 0xAD24E1AF,0xB43FD0EE,0x9F12832D,0x8609B26C,0xC94824AB,0xD05315EA,0xFB7E4629,0xE2657768, 91 | 0x2F3F79F6,0x362448B7,0x1D091B74,0x04122A35,0x4B53BCF2,0x52488DB3,0x7965DE70,0x607EEF31, 92 | 0xE7E6F3FE,0xFEFDC2BF,0xD5D0917C,0xCCCBA03D,0x838A36FA,0x9A9107BB,0xB1BC5478,0xA8A76539, 93 | 0x3B83984B,0x2298A90A,0x09B5FAC9,0x10AECB88,0x5FEF5D4F,0x46F46C0E,0x6DD93FCD,0x74C20E8C, 94 | 0xF35A1243,0xEA412302,0xC16C70C1,0xD8774180,0x9736D747,0x8E2DE606,0xA500B5C5,0xBC1B8484, 95 | 0x71418A1A,0x685ABB5B,0x4377E898,0x5A6CD9D9,0x152D4F1E,0x0C367E5F,0x271B2D9C,0x3E001CDD, 96 | 0xB9980012,0xA0833153,0x8BAE6290,0x92B553D1,0xDDF4C516,0xC4EFF457,0xEFC2A794,0xF6D996D5, 97 | 0xAE07BCE9,0xB71C8DA8,0x9C31DE6B,0x852AEF2A,0xCA6B79ED,0xD37048AC,0xF85D1B6F,0xE1462A2E, 98 | 0x66DE36E1,0x7FC507A0,0x54E85463,0x4DF36522,0x02B2F3E5,0x1BA9C2A4,0x30849167,0x299FA026, 99 | 0xE4C5AEB8,0xFDDE9FF9,0xD6F3CC3A,0xCFE8FD7B,0x80A96BBC,0x99B25AFD,0xB29F093E,0xAB84387F, 100 | 0x2C1C24B0,0x350715F1,0x1E2A4632,0x07317773,0x4870E1B4,0x516BD0F5,0x7A468336,0x635DB277, 101 | 0xCBFAD74E,0xD2E1E60F,0xF9CCB5CC,0xE0D7848D,0xAF96124A,0xB68D230B,0x9DA070C8,0x84BB4189, 102 | 0x03235D46,0x1A386C07,0x31153FC4,0x280E0E85,0x674F9842,0x7E54A903,0x5579FAC0,0x4C62CB81, 103 | 0x8138C51F,0x9823F45E,0xB30EA79D,0xAA1596DC,0xE554001B,0xFC4F315A,0xD7626299,0xCE7953D8, 104 | 0x49E14F17,0x50FA7E56,0x7BD72D95,0x62CC1CD4,0x2D8D8A13,0x3496BB52,0x1FBBE891,0x06A0D9D0, 105 | 0x5E7EF3EC,0x4765C2AD,0x6C48916E,0x7553A02F,0x3A1236E8,0x230907A9,0x0824546A,0x113F652B, 106 | 0x96A779E4,0x8FBC48A5,0xA4911B66,0xBD8A2A27,0xF2CBBCE0,0xEBD08DA1,0xC0FDDE62,0xD9E6EF23, 107 | 0x14BCE1BD,0x0DA7D0FC,0x268A833F,0x3F91B27E,0x70D024B9,0x69CB15F8,0x42E6463B,0x5BFD777A, 108 | 0xDC656BB5,0xC57E5AF4,0xEE530937,0xF7483876,0xB809AEB1,0xA1129FF0,0x8A3FCC33,0x9324FD72 }, 109 | 110 | { 0x00000000,0x01C26A37,0x0384D46E,0x0246BE59,0x0709A8DC,0x06CBC2EB,0x048D7CB2,0x054F1685, 111 | 0x0E1351B8,0x0FD13B8F,0x0D9785D6,0x0C55EFE1,0x091AF964,0x08D89353,0x0A9E2D0A,0x0B5C473D, 112 | 0x1C26A370,0x1DE4C947,0x1FA2771E,0x1E601D29,0x1B2F0BAC,0x1AED619B,0x18ABDFC2,0x1969B5F5, 113 | 0x1235F2C8,0x13F798FF,0x11B126A6,0x10734C91,0x153C5A14,0x14FE3023,0x16B88E7A,0x177AE44D, 114 | 0x384D46E0,0x398F2CD7,0x3BC9928E,0x3A0BF8B9,0x3F44EE3C,0x3E86840B,0x3CC03A52,0x3D025065, 115 | 0x365E1758,0x379C7D6F,0x35DAC336,0x3418A901,0x3157BF84,0x3095D5B3,0x32D36BEA,0x331101DD, 116 | 0x246BE590,0x25A98FA7,0x27EF31FE,0x262D5BC9,0x23624D4C,0x22A0277B,0x20E69922,0x2124F315, 117 | 0x2A78B428,0x2BBADE1F,0x29FC6046,0x283E0A71,0x2D711CF4,0x2CB376C3,0x2EF5C89A,0x2F37A2AD, 118 | 0x709A8DC0,0x7158E7F7,0x731E59AE,0x72DC3399,0x7793251C,0x76514F2B,0x7417F172,0x75D59B45, 119 | 0x7E89DC78,0x7F4BB64F,0x7D0D0816,0x7CCF6221,0x798074A4,0x78421E93,0x7A04A0CA,0x7BC6CAFD, 120 | 0x6CBC2EB0,0x6D7E4487,0x6F38FADE,0x6EFA90E9,0x6BB5866C,0x6A77EC5B,0x68315202,0x69F33835, 121 | 0x62AF7F08,0x636D153F,0x612BAB66,0x60E9C151,0x65A6D7D4,0x6464BDE3,0x662203BA,0x67E0698D, 122 | 0x48D7CB20,0x4915A117,0x4B531F4E,0x4A917579,0x4FDE63FC,0x4E1C09CB,0x4C5AB792,0x4D98DDA5, 123 | 0x46C49A98,0x4706F0AF,0x45404EF6,0x448224C1,0x41CD3244,0x400F5873,0x4249E62A,0x438B8C1D, 124 | 0x54F16850,0x55330267,0x5775BC3E,0x56B7D609,0x53F8C08C,0x523AAABB,0x507C14E2,0x51BE7ED5, 125 | 0x5AE239E8,0x5B2053DF,0x5966ED86,0x58A487B1,0x5DEB9134,0x5C29FB03,0x5E6F455A,0x5FAD2F6D, 126 | 0xE1351B80,0xE0F771B7,0xE2B1CFEE,0xE373A5D9,0xE63CB35C,0xE7FED96B,0xE5B86732,0xE47A0D05, 127 | 0xEF264A38,0xEEE4200F,0xECA29E56,0xED60F461,0xE82FE2E4,0xE9ED88D3,0xEBAB368A,0xEA695CBD, 128 | 0xFD13B8F0,0xFCD1D2C7,0xFE976C9E,0xFF5506A9,0xFA1A102C,0xFBD87A1B,0xF99EC442,0xF85CAE75, 129 | 0xF300E948,0xF2C2837F,0xF0843D26,0xF1465711,0xF4094194,0xF5CB2BA3,0xF78D95FA,0xF64FFFCD, 130 | 0xD9785D60,0xD8BA3757,0xDAFC890E,0xDB3EE339,0xDE71F5BC,0xDFB39F8B,0xDDF521D2,0xDC374BE5, 131 | 0xD76B0CD8,0xD6A966EF,0xD4EFD8B6,0xD52DB281,0xD062A404,0xD1A0CE33,0xD3E6706A,0xD2241A5D, 132 | 0xC55EFE10,0xC49C9427,0xC6DA2A7E,0xC7184049,0xC25756CC,0xC3953CFB,0xC1D382A2,0xC011E895, 133 | 0xCB4DAFA8,0xCA8FC59F,0xC8C97BC6,0xC90B11F1,0xCC440774,0xCD866D43,0xCFC0D31A,0xCE02B92D, 134 | 0x91AF9640,0x906DFC77,0x922B422E,0x93E92819,0x96A63E9C,0x976454AB,0x9522EAF2,0x94E080C5, 135 | 0x9FBCC7F8,0x9E7EADCF,0x9C381396,0x9DFA79A1,0x98B56F24,0x99770513,0x9B31BB4A,0x9AF3D17D, 136 | 0x8D893530,0x8C4B5F07,0x8E0DE15E,0x8FCF8B69,0x8A809DEC,0x8B42F7DB,0x89044982,0x88C623B5, 137 | 0x839A6488,0x82580EBF,0x801EB0E6,0x81DCDAD1,0x8493CC54,0x8551A663,0x8717183A,0x86D5720D, 138 | 0xA9E2D0A0,0xA820BA97,0xAA6604CE,0xABA46EF9,0xAEEB787C,0xAF29124B,0xAD6FAC12,0xACADC625, 139 | 0xA7F18118,0xA633EB2F,0xA4755576,0xA5B73F41,0xA0F829C4,0xA13A43F3,0xA37CFDAA,0xA2BE979D, 140 | 0xB5C473D0,0xB40619E7,0xB640A7BE,0xB782CD89,0xB2CDDB0C,0xB30FB13B,0xB1490F62,0xB08B6555, 141 | 0xBBD72268,0xBA15485F,0xB853F606,0xB9919C31,0xBCDE8AB4,0xBD1CE083,0xBF5A5EDA,0xBE9834ED }, 142 | 143 | { 0x00000000,0xB8BC6765,0xAA09C88B,0x12B5AFEE,0x8F629757,0x37DEF032,0x256B5FDC,0x9DD738B9, 144 | 0xC5B428EF,0x7D084F8A,0x6FBDE064,0xD7018701,0x4AD6BFB8,0xF26AD8DD,0xE0DF7733,0x58631056, 145 | 0x5019579F,0xE8A530FA,0xFA109F14,0x42ACF871,0xDF7BC0C8,0x67C7A7AD,0x75720843,0xCDCE6F26, 146 | 0x95AD7F70,0x2D111815,0x3FA4B7FB,0x8718D09E,0x1ACFE827,0xA2738F42,0xB0C620AC,0x087A47C9, 147 | 0xA032AF3E,0x188EC85B,0x0A3B67B5,0xB28700D0,0x2F503869,0x97EC5F0C,0x8559F0E2,0x3DE59787, 148 | 0x658687D1,0xDD3AE0B4,0xCF8F4F5A,0x7733283F,0xEAE41086,0x525877E3,0x40EDD80D,0xF851BF68, 149 | 0xF02BF8A1,0x48979FC4,0x5A22302A,0xE29E574F,0x7F496FF6,0xC7F50893,0xD540A77D,0x6DFCC018, 150 | 0x359FD04E,0x8D23B72B,0x9F9618C5,0x272A7FA0,0xBAFD4719,0x0241207C,0x10F48F92,0xA848E8F7, 151 | 0x9B14583D,0x23A83F58,0x311D90B6,0x89A1F7D3,0x1476CF6A,0xACCAA80F,0xBE7F07E1,0x06C36084, 152 | 0x5EA070D2,0xE61C17B7,0xF4A9B859,0x4C15DF3C,0xD1C2E785,0x697E80E0,0x7BCB2F0E,0xC377486B, 153 | 0xCB0D0FA2,0x73B168C7,0x6104C729,0xD9B8A04C,0x446F98F5,0xFCD3FF90,0xEE66507E,0x56DA371B, 154 | 0x0EB9274D,0xB6054028,0xA4B0EFC6,0x1C0C88A3,0x81DBB01A,0x3967D77F,0x2BD27891,0x936E1FF4, 155 | 0x3B26F703,0x839A9066,0x912F3F88,0x299358ED,0xB4446054,0x0CF80731,0x1E4DA8DF,0xA6F1CFBA, 156 | 0xFE92DFEC,0x462EB889,0x549B1767,0xEC277002,0x71F048BB,0xC94C2FDE,0xDBF98030,0x6345E755, 157 | 0x6B3FA09C,0xD383C7F9,0xC1366817,0x798A0F72,0xE45D37CB,0x5CE150AE,0x4E54FF40,0xF6E89825, 158 | 0xAE8B8873,0x1637EF16,0x048240F8,0xBC3E279D,0x21E91F24,0x99557841,0x8BE0D7AF,0x335CB0CA, 159 | 0xED59B63B,0x55E5D15E,0x47507EB0,0xFFEC19D5,0x623B216C,0xDA874609,0xC832E9E7,0x708E8E82, 160 | 0x28ED9ED4,0x9051F9B1,0x82E4565F,0x3A58313A,0xA78F0983,0x1F336EE6,0x0D86C108,0xB53AA66D, 161 | 0xBD40E1A4,0x05FC86C1,0x1749292F,0xAFF54E4A,0x322276F3,0x8A9E1196,0x982BBE78,0x2097D91D, 162 | 0x78F4C94B,0xC048AE2E,0xD2FD01C0,0x6A4166A5,0xF7965E1C,0x4F2A3979,0x5D9F9697,0xE523F1F2, 163 | 0x4D6B1905,0xF5D77E60,0xE762D18E,0x5FDEB6EB,0xC2098E52,0x7AB5E937,0x680046D9,0xD0BC21BC, 164 | 0x88DF31EA,0x3063568F,0x22D6F961,0x9A6A9E04,0x07BDA6BD,0xBF01C1D8,0xADB46E36,0x15080953, 165 | 0x1D724E9A,0xA5CE29FF,0xB77B8611,0x0FC7E174,0x9210D9CD,0x2AACBEA8,0x38191146,0x80A57623, 166 | 0xD8C66675,0x607A0110,0x72CFAEFE,0xCA73C99B,0x57A4F122,0xEF189647,0xFDAD39A9,0x45115ECC, 167 | 0x764DEE06,0xCEF18963,0xDC44268D,0x64F841E8,0xF92F7951,0x41931E34,0x5326B1DA,0xEB9AD6BF, 168 | 0xB3F9C6E9,0x0B45A18C,0x19F00E62,0xA14C6907,0x3C9B51BE,0x842736DB,0x96929935,0x2E2EFE50, 169 | 0x2654B999,0x9EE8DEFC,0x8C5D7112,0x34E11677,0xA9362ECE,0x118A49AB,0x033FE645,0xBB838120, 170 | 0xE3E09176,0x5B5CF613,0x49E959FD,0xF1553E98,0x6C820621,0xD43E6144,0xC68BCEAA,0x7E37A9CF, 171 | 0xD67F4138,0x6EC3265D,0x7C7689B3,0xC4CAEED6,0x591DD66F,0xE1A1B10A,0xF3141EE4,0x4BA87981, 172 | 0x13CB69D7,0xAB770EB2,0xB9C2A15C,0x017EC639,0x9CA9FE80,0x241599E5,0x36A0360B,0x8E1C516E, 173 | 0x866616A7,0x3EDA71C2,0x2C6FDE2C,0x94D3B949,0x090481F0,0xB1B8E695,0xA30D497B,0x1BB12E1E, 174 | 0x43D23E48,0xFB6E592D,0xE9DBF6C3,0x516791A6,0xCCB0A91F,0x740CCE7A,0x66B96194,0xDE0506F1 }, 175 | 176 | { 0x00000000,0x3D6029B0,0x7AC05360,0x47A07AD0,0xF580A6C0,0xC8E08F70,0x8F40F5A0,0xB220DC10, 177 | 0x30704BC1,0x0D106271,0x4AB018A1,0x77D03111,0xC5F0ED01,0xF890C4B1,0xBF30BE61,0x825097D1, 178 | 0x60E09782,0x5D80BE32,0x1A20C4E2,0x2740ED52,0x95603142,0xA80018F2,0xEFA06222,0xD2C04B92, 179 | 0x5090DC43,0x6DF0F5F3,0x2A508F23,0x1730A693,0xA5107A83,0x98705333,0xDFD029E3,0xE2B00053, 180 | 0xC1C12F04,0xFCA106B4,0xBB017C64,0x866155D4,0x344189C4,0x0921A074,0x4E81DAA4,0x73E1F314, 181 | 0xF1B164C5,0xCCD14D75,0x8B7137A5,0xB6111E15,0x0431C205,0x3951EBB5,0x7EF19165,0x4391B8D5, 182 | 0xA121B886,0x9C419136,0xDBE1EBE6,0xE681C256,0x54A11E46,0x69C137F6,0x2E614D26,0x13016496, 183 | 0x9151F347,0xAC31DAF7,0xEB91A027,0xD6F18997,0x64D15587,0x59B17C37,0x1E1106E7,0x23712F57, 184 | 0x58F35849,0x659371F9,0x22330B29,0x1F532299,0xAD73FE89,0x9013D739,0xD7B3ADE9,0xEAD38459, 185 | 0x68831388,0x55E33A38,0x124340E8,0x2F236958,0x9D03B548,0xA0639CF8,0xE7C3E628,0xDAA3CF98, 186 | 0x3813CFCB,0x0573E67B,0x42D39CAB,0x7FB3B51B,0xCD93690B,0xF0F340BB,0xB7533A6B,0x8A3313DB, 187 | 0x0863840A,0x3503ADBA,0x72A3D76A,0x4FC3FEDA,0xFDE322CA,0xC0830B7A,0x872371AA,0xBA43581A, 188 | 0x9932774D,0xA4525EFD,0xE3F2242D,0xDE920D9D,0x6CB2D18D,0x51D2F83D,0x167282ED,0x2B12AB5D, 189 | 0xA9423C8C,0x9422153C,0xD3826FEC,0xEEE2465C,0x5CC29A4C,0x61A2B3FC,0x2602C92C,0x1B62E09C, 190 | 0xF9D2E0CF,0xC4B2C97F,0x8312B3AF,0xBE729A1F,0x0C52460F,0x31326FBF,0x7692156F,0x4BF23CDF, 191 | 0xC9A2AB0E,0xF4C282BE,0xB362F86E,0x8E02D1DE,0x3C220DCE,0x0142247E,0x46E25EAE,0x7B82771E, 192 | 0xB1E6B092,0x8C869922,0xCB26E3F2,0xF646CA42,0x44661652,0x79063FE2,0x3EA64532,0x03C66C82, 193 | 0x8196FB53,0xBCF6D2E3,0xFB56A833,0xC6368183,0x74165D93,0x49767423,0x0ED60EF3,0x33B62743, 194 | 0xD1062710,0xEC660EA0,0xABC67470,0x96A65DC0,0x248681D0,0x19E6A860,0x5E46D2B0,0x6326FB00, 195 | 0xE1766CD1,0xDC164561,0x9BB63FB1,0xA6D61601,0x14F6CA11,0x2996E3A1,0x6E369971,0x5356B0C1, 196 | 0x70279F96,0x4D47B626,0x0AE7CCF6,0x3787E546,0x85A73956,0xB8C710E6,0xFF676A36,0xC2074386, 197 | 0x4057D457,0x7D37FDE7,0x3A978737,0x07F7AE87,0xB5D77297,0x88B75B27,0xCF1721F7,0xF2770847, 198 | 0x10C70814,0x2DA721A4,0x6A075B74,0x576772C4,0xE547AED4,0xD8278764,0x9F87FDB4,0xA2E7D404, 199 | 0x20B743D5,0x1DD76A65,0x5A7710B5,0x67173905,0xD537E515,0xE857CCA5,0xAFF7B675,0x92979FC5, 200 | 0xE915E8DB,0xD475C16B,0x93D5BBBB,0xAEB5920B,0x1C954E1B,0x21F567AB,0x66551D7B,0x5B3534CB, 201 | 0xD965A31A,0xE4058AAA,0xA3A5F07A,0x9EC5D9CA,0x2CE505DA,0x11852C6A,0x562556BA,0x6B457F0A, 202 | 0x89F57F59,0xB49556E9,0xF3352C39,0xCE550589,0x7C75D999,0x4115F029,0x06B58AF9,0x3BD5A349, 203 | 0xB9853498,0x84E51D28,0xC34567F8,0xFE254E48,0x4C059258,0x7165BBE8,0x36C5C138,0x0BA5E888, 204 | 0x28D4C7DF,0x15B4EE6F,0x521494BF,0x6F74BD0F,0xDD54611F,0xE03448AF,0xA794327F,0x9AF41BCF, 205 | 0x18A48C1E,0x25C4A5AE,0x6264DF7E,0x5F04F6CE,0xED242ADE,0xD044036E,0x97E479BE,0xAA84500E, 206 | 0x4834505D,0x755479ED,0x32F4033D,0x0F942A8D,0xBDB4F69D,0x80D4DF2D,0xC774A5FD,0xFA148C4D, 207 | 0x78441B9C,0x4524322C,0x028448FC,0x3FE4614C,0x8DC4BD5C,0xB0A494EC,0xF704EE3C,0xCA64C78C }, 208 | 209 | { 0x00000000,0xCB5CD3A5,0x4DC8A10B,0x869472AE,0x9B914216,0x50CD91B3,0xD659E31D,0x1D0530B8, 210 | 0xEC53826D,0x270F51C8,0xA19B2366,0x6AC7F0C3,0x77C2C07B,0xBC9E13DE,0x3A0A6170,0xF156B2D5, 211 | 0x03D6029B,0xC88AD13E,0x4E1EA390,0x85427035,0x9847408D,0x531B9328,0xD58FE186,0x1ED33223, 212 | 0xEF8580F6,0x24D95353,0xA24D21FD,0x6911F258,0x7414C2E0,0xBF481145,0x39DC63EB,0xF280B04E, 213 | 0x07AC0536,0xCCF0D693,0x4A64A43D,0x81387798,0x9C3D4720,0x57619485,0xD1F5E62B,0x1AA9358E, 214 | 0xEBFF875B,0x20A354FE,0xA6372650,0x6D6BF5F5,0x706EC54D,0xBB3216E8,0x3DA66446,0xF6FAB7E3, 215 | 0x047A07AD,0xCF26D408,0x49B2A6A6,0x82EE7503,0x9FEB45BB,0x54B7961E,0xD223E4B0,0x197F3715, 216 | 0xE82985C0,0x23755665,0xA5E124CB,0x6EBDF76E,0x73B8C7D6,0xB8E41473,0x3E7066DD,0xF52CB578, 217 | 0x0F580A6C,0xC404D9C9,0x4290AB67,0x89CC78C2,0x94C9487A,0x5F959BDF,0xD901E971,0x125D3AD4, 218 | 0xE30B8801,0x28575BA4,0xAEC3290A,0x659FFAAF,0x789ACA17,0xB3C619B2,0x35526B1C,0xFE0EB8B9, 219 | 0x0C8E08F7,0xC7D2DB52,0x4146A9FC,0x8A1A7A59,0x971F4AE1,0x5C439944,0xDAD7EBEA,0x118B384F, 220 | 0xE0DD8A9A,0x2B81593F,0xAD152B91,0x6649F834,0x7B4CC88C,0xB0101B29,0x36846987,0xFDD8BA22, 221 | 0x08F40F5A,0xC3A8DCFF,0x453CAE51,0x8E607DF4,0x93654D4C,0x58399EE9,0xDEADEC47,0x15F13FE2, 222 | 0xE4A78D37,0x2FFB5E92,0xA96F2C3C,0x6233FF99,0x7F36CF21,0xB46A1C84,0x32FE6E2A,0xF9A2BD8F, 223 | 0x0B220DC1,0xC07EDE64,0x46EAACCA,0x8DB67F6F,0x90B34FD7,0x5BEF9C72,0xDD7BEEDC,0x16273D79, 224 | 0xE7718FAC,0x2C2D5C09,0xAAB92EA7,0x61E5FD02,0x7CE0CDBA,0xB7BC1E1F,0x31286CB1,0xFA74BF14, 225 | 0x1EB014D8,0xD5ECC77D,0x5378B5D3,0x98246676,0x852156CE,0x4E7D856B,0xC8E9F7C5,0x03B52460, 226 | 0xF2E396B5,0x39BF4510,0xBF2B37BE,0x7477E41B,0x6972D4A3,0xA22E0706,0x24BA75A8,0xEFE6A60D, 227 | 0x1D661643,0xD63AC5E6,0x50AEB748,0x9BF264ED,0x86F75455,0x4DAB87F0,0xCB3FF55E,0x006326FB, 228 | 0xF135942E,0x3A69478B,0xBCFD3525,0x77A1E680,0x6AA4D638,0xA1F8059D,0x276C7733,0xEC30A496, 229 | 0x191C11EE,0xD240C24B,0x54D4B0E5,0x9F886340,0x828D53F8,0x49D1805D,0xCF45F2F3,0x04192156, 230 | 0xF54F9383,0x3E134026,0xB8873288,0x73DBE12D,0x6EDED195,0xA5820230,0x2316709E,0xE84AA33B, 231 | 0x1ACA1375,0xD196C0D0,0x5702B27E,0x9C5E61DB,0x815B5163,0x4A0782C6,0xCC93F068,0x07CF23CD, 232 | 0xF6999118,0x3DC542BD,0xBB513013,0x700DE3B6,0x6D08D30E,0xA65400AB,0x20C07205,0xEB9CA1A0, 233 | 0x11E81EB4,0xDAB4CD11,0x5C20BFBF,0x977C6C1A,0x8A795CA2,0x41258F07,0xC7B1FDA9,0x0CED2E0C, 234 | 0xFDBB9CD9,0x36E74F7C,0xB0733DD2,0x7B2FEE77,0x662ADECF,0xAD760D6A,0x2BE27FC4,0xE0BEAC61, 235 | 0x123E1C2F,0xD962CF8A,0x5FF6BD24,0x94AA6E81,0x89AF5E39,0x42F38D9C,0xC467FF32,0x0F3B2C97, 236 | 0xFE6D9E42,0x35314DE7,0xB3A53F49,0x78F9ECEC,0x65FCDC54,0xAEA00FF1,0x28347D5F,0xE368AEFA, 237 | 0x16441B82,0xDD18C827,0x5B8CBA89,0x90D0692C,0x8DD55994,0x46898A31,0xC01DF89F,0x0B412B3A, 238 | 0xFA1799EF,0x314B4A4A,0xB7DF38E4,0x7C83EB41,0x6186DBF9,0xAADA085C,0x2C4E7AF2,0xE712A957, 239 | 0x15921919,0xDECECABC,0x585AB812,0x93066BB7,0x8E035B0F,0x455F88AA,0xC3CBFA04,0x089729A1, 240 | 0xF9C19B74,0x329D48D1,0xB4093A7F,0x7F55E9DA,0x6250D962,0xA90C0AC7,0x2F987869,0xE4C4ABCC }, 241 | 242 | { 0x00000000,0xA6770BB4,0x979F1129,0x31E81A9D,0xF44F2413,0x52382FA7,0x63D0353A,0xC5A73E8E, 243 | 0x33EF4E67,0x959845D3,0xA4705F4E,0x020754FA,0xC7A06A74,0x61D761C0,0x503F7B5D,0xF64870E9, 244 | 0x67DE9CCE,0xC1A9977A,0xF0418DE7,0x56368653,0x9391B8DD,0x35E6B369,0x040EA9F4,0xA279A240, 245 | 0x5431D2A9,0xF246D91D,0xC3AEC380,0x65D9C834,0xA07EF6BA,0x0609FD0E,0x37E1E793,0x9196EC27, 246 | 0xCFBD399C,0x69CA3228,0x582228B5,0xFE552301,0x3BF21D8F,0x9D85163B,0xAC6D0CA6,0x0A1A0712, 247 | 0xFC5277FB,0x5A257C4F,0x6BCD66D2,0xCDBA6D66,0x081D53E8,0xAE6A585C,0x9F8242C1,0x39F54975, 248 | 0xA863A552,0x0E14AEE6,0x3FFCB47B,0x998BBFCF,0x5C2C8141,0xFA5B8AF5,0xCBB39068,0x6DC49BDC, 249 | 0x9B8CEB35,0x3DFBE081,0x0C13FA1C,0xAA64F1A8,0x6FC3CF26,0xC9B4C492,0xF85CDE0F,0x5E2BD5BB, 250 | 0x440B7579,0xE27C7ECD,0xD3946450,0x75E36FE4,0xB044516A,0x16335ADE,0x27DB4043,0x81AC4BF7, 251 | 0x77E43B1E,0xD19330AA,0xE07B2A37,0x460C2183,0x83AB1F0D,0x25DC14B9,0x14340E24,0xB2430590, 252 | 0x23D5E9B7,0x85A2E203,0xB44AF89E,0x123DF32A,0xD79ACDA4,0x71EDC610,0x4005DC8D,0xE672D739, 253 | 0x103AA7D0,0xB64DAC64,0x87A5B6F9,0x21D2BD4D,0xE47583C3,0x42028877,0x73EA92EA,0xD59D995E, 254 | 0x8BB64CE5,0x2DC14751,0x1C295DCC,0xBA5E5678,0x7FF968F6,0xD98E6342,0xE86679DF,0x4E11726B, 255 | 0xB8590282,0x1E2E0936,0x2FC613AB,0x89B1181F,0x4C162691,0xEA612D25,0xDB8937B8,0x7DFE3C0C, 256 | 0xEC68D02B,0x4A1FDB9F,0x7BF7C102,0xDD80CAB6,0x1827F438,0xBE50FF8C,0x8FB8E511,0x29CFEEA5, 257 | 0xDF879E4C,0x79F095F8,0x48188F65,0xEE6F84D1,0x2BC8BA5F,0x8DBFB1EB,0xBC57AB76,0x1A20A0C2, 258 | 0x8816EAF2,0x2E61E146,0x1F89FBDB,0xB9FEF06F,0x7C59CEE1,0xDA2EC555,0xEBC6DFC8,0x4DB1D47C, 259 | 0xBBF9A495,0x1D8EAF21,0x2C66B5BC,0x8A11BE08,0x4FB68086,0xE9C18B32,0xD82991AF,0x7E5E9A1B, 260 | 0xEFC8763C,0x49BF7D88,0x78576715,0xDE206CA1,0x1B87522F,0xBDF0599B,0x8C184306,0x2A6F48B2, 261 | 0xDC27385B,0x7A5033EF,0x4BB82972,0xEDCF22C6,0x28681C48,0x8E1F17FC,0xBFF70D61,0x198006D5, 262 | 0x47ABD36E,0xE1DCD8DA,0xD034C247,0x7643C9F3,0xB3E4F77D,0x1593FCC9,0x247BE654,0x820CEDE0, 263 | 0x74449D09,0xD23396BD,0xE3DB8C20,0x45AC8794,0x800BB91A,0x267CB2AE,0x1794A833,0xB1E3A387, 264 | 0x20754FA0,0x86024414,0xB7EA5E89,0x119D553D,0xD43A6BB3,0x724D6007,0x43A57A9A,0xE5D2712E, 265 | 0x139A01C7,0xB5ED0A73,0x840510EE,0x22721B5A,0xE7D525D4,0x41A22E60,0x704A34FD,0xD63D3F49, 266 | 0xCC1D9F8B,0x6A6A943F,0x5B828EA2,0xFDF58516,0x3852BB98,0x9E25B02C,0xAFCDAAB1,0x09BAA105, 267 | 0xFFF2D1EC,0x5985DA58,0x686DC0C5,0xCE1ACB71,0x0BBDF5FF,0xADCAFE4B,0x9C22E4D6,0x3A55EF62, 268 | 0xABC30345,0x0DB408F1,0x3C5C126C,0x9A2B19D8,0x5F8C2756,0xF9FB2CE2,0xC813367F,0x6E643DCB, 269 | 0x982C4D22,0x3E5B4696,0x0FB35C0B,0xA9C457BF,0x6C636931,0xCA146285,0xFBFC7818,0x5D8B73AC, 270 | 0x03A0A617,0xA5D7ADA3,0x943FB73E,0x3248BC8A,0xF7EF8204,0x519889B0,0x6070932D,0xC6079899, 271 | 0x304FE870,0x9638E3C4,0xA7D0F959,0x01A7F2ED,0xC400CC63,0x6277C7D7,0x539FDD4A,0xF5E8D6FE, 272 | 0x647E3AD9,0xC209316D,0xF3E12BF0,0x55962044,0x90311ECA,0x3646157E,0x07AE0FE3,0xA1D90457, 273 | 0x579174BE,0xF1E67F0A,0xC00E6597,0x66796E23,0xA3DE50AD,0x05A95B19,0x34414184,0x92364A30 }, 274 | 275 | { 0x00000000,0xCCAA009E,0x4225077D,0x8E8F07E3,0x844A0EFA,0x48E00E64,0xC66F0987,0x0AC50919, 276 | 0xD3E51BB5,0x1F4F1B2B,0x91C01CC8,0x5D6A1C56,0x57AF154F,0x9B0515D1,0x158A1232,0xD92012AC, 277 | 0x7CBB312B,0xB01131B5,0x3E9E3656,0xF23436C8,0xF8F13FD1,0x345B3F4F,0xBAD438AC,0x767E3832, 278 | 0xAF5E2A9E,0x63F42A00,0xED7B2DE3,0x21D12D7D,0x2B142464,0xE7BE24FA,0x69312319,0xA59B2387, 279 | 0xF9766256,0x35DC62C8,0xBB53652B,0x77F965B5,0x7D3C6CAC,0xB1966C32,0x3F196BD1,0xF3B36B4F, 280 | 0x2A9379E3,0xE639797D,0x68B67E9E,0xA41C7E00,0xAED97719,0x62737787,0xECFC7064,0x205670FA, 281 | 0x85CD537D,0x496753E3,0xC7E85400,0x0B42549E,0x01875D87,0xCD2D5D19,0x43A25AFA,0x8F085A64, 282 | 0x562848C8,0x9A824856,0x140D4FB5,0xD8A74F2B,0xD2624632,0x1EC846AC,0x9047414F,0x5CED41D1, 283 | 0x299DC2ED,0xE537C273,0x6BB8C590,0xA712C50E,0xADD7CC17,0x617DCC89,0xEFF2CB6A,0x2358CBF4, 284 | 0xFA78D958,0x36D2D9C6,0xB85DDE25,0x74F7DEBB,0x7E32D7A2,0xB298D73C,0x3C17D0DF,0xF0BDD041, 285 | 0x5526F3C6,0x998CF358,0x1703F4BB,0xDBA9F425,0xD16CFD3C,0x1DC6FDA2,0x9349FA41,0x5FE3FADF, 286 | 0x86C3E873,0x4A69E8ED,0xC4E6EF0E,0x084CEF90,0x0289E689,0xCE23E617,0x40ACE1F4,0x8C06E16A, 287 | 0xD0EBA0BB,0x1C41A025,0x92CEA7C6,0x5E64A758,0x54A1AE41,0x980BAEDF,0x1684A93C,0xDA2EA9A2, 288 | 0x030EBB0E,0xCFA4BB90,0x412BBC73,0x8D81BCED,0x8744B5F4,0x4BEEB56A,0xC561B289,0x09CBB217, 289 | 0xAC509190,0x60FA910E,0xEE7596ED,0x22DF9673,0x281A9F6A,0xE4B09FF4,0x6A3F9817,0xA6959889, 290 | 0x7FB58A25,0xB31F8ABB,0x3D908D58,0xF13A8DC6,0xFBFF84DF,0x37558441,0xB9DA83A2,0x7570833C, 291 | 0x533B85DA,0x9F918544,0x111E82A7,0xDDB48239,0xD7718B20,0x1BDB8BBE,0x95548C5D,0x59FE8CC3, 292 | 0x80DE9E6F,0x4C749EF1,0xC2FB9912,0x0E51998C,0x04949095,0xC83E900B,0x46B197E8,0x8A1B9776, 293 | 0x2F80B4F1,0xE32AB46F,0x6DA5B38C,0xA10FB312,0xABCABA0B,0x6760BA95,0xE9EFBD76,0x2545BDE8, 294 | 0xFC65AF44,0x30CFAFDA,0xBE40A839,0x72EAA8A7,0x782FA1BE,0xB485A120,0x3A0AA6C3,0xF6A0A65D, 295 | 0xAA4DE78C,0x66E7E712,0xE868E0F1,0x24C2E06F,0x2E07E976,0xE2ADE9E8,0x6C22EE0B,0xA088EE95, 296 | 0x79A8FC39,0xB502FCA7,0x3B8DFB44,0xF727FBDA,0xFDE2F2C3,0x3148F25D,0xBFC7F5BE,0x736DF520, 297 | 0xD6F6D6A7,0x1A5CD639,0x94D3D1DA,0x5879D144,0x52BCD85D,0x9E16D8C3,0x1099DF20,0xDC33DFBE, 298 | 0x0513CD12,0xC9B9CD8C,0x4736CA6F,0x8B9CCAF1,0x8159C3E8,0x4DF3C376,0xC37CC495,0x0FD6C40B, 299 | 0x7AA64737,0xB60C47A9,0x3883404A,0xF42940D4,0xFEEC49CD,0x32464953,0xBCC94EB0,0x70634E2E, 300 | 0xA9435C82,0x65E95C1C,0xEB665BFF,0x27CC5B61,0x2D095278,0xE1A352E6,0x6F2C5505,0xA386559B, 301 | 0x061D761C,0xCAB77682,0x44387161,0x889271FF,0x825778E6,0x4EFD7878,0xC0727F9B,0x0CD87F05, 302 | 0xD5F86DA9,0x19526D37,0x97DD6AD4,0x5B776A4A,0x51B26353,0x9D1863CD,0x1397642E,0xDF3D64B0, 303 | 0x83D02561,0x4F7A25FF,0xC1F5221C,0x0D5F2282,0x079A2B9B,0xCB302B05,0x45BF2CE6,0x89152C78, 304 | 0x50353ED4,0x9C9F3E4A,0x121039A9,0xDEBA3937,0xD47F302E,0x18D530B0,0x965A3753,0x5AF037CD, 305 | 0xFF6B144A,0x33C114D4,0xBD4E1337,0x71E413A9,0x7B211AB0,0xB78B1A2E,0x39041DCD,0xF5AE1D53, 306 | 0x2C8E0FFF,0xE0240F61,0x6EAB0882,0xA201081C,0xA8C40105,0x646E019B,0xEAE10678,0x264B06E6 } 307 | }; 308 | 309 | #define MIN(a,b) (((a)<(b))?(a):(b)) 310 | 311 | uint32_t crc32_1(void* data, size_t len, uint32_t prev_value) 312 | { 313 | uint32_t crc = ~prev_value; 314 | const uint8_t* cur = (const uint8_t*) data; 315 | 316 | while (len-- > 0) 317 | crc = (crc >> 8) ^ crc_lookup[0][(crc & 0xFF) ^ *cur++]; 318 | 319 | return ~crc; 320 | } 321 | 322 | uint32_t crc32_8(void* data, size_t len, uint32_t prev_value) 323 | { 324 | uint32_t *cur = (uint32_t*) data; 325 | uint32_t crc = ~prev_value; 326 | 327 | while (len >= 8) { 328 | #ifdef _BIG_ENDIAN 329 | uint32_t one = *cur++ ^ bitswap32(crc); 330 | uint32_t two = *cur++; 331 | crc = 332 | crc_lookup[7][(one>>24) & 0xFF] ^ 333 | crc_lookup[6][(one>>16) & 0xFF] ^ 334 | crc_lookup[5][(one>> 8) & 0xFF] ^ 335 | crc_lookup[4][(one ) & 0xFF] ^ 336 | crc_lookup[3][(two>>24) & 0xFF] ^ 337 | crc_lookup[2][(two>>16) & 0xFF] ^ 338 | crc_lookup[1][(two>> 8) & 0xFF] ^ 339 | crc_lookup[0][(two ) & 0xFF]; 340 | #else 341 | uint32_t one = *cur++ ^ crc; 342 | uint32_t two = *cur++; 343 | crc = 344 | crc_lookup[7][(one ) & 0xFF] ^ 345 | crc_lookup[6][(one>> 8) & 0xFF] ^ 346 | crc_lookup[5][(one>>16) & 0xFF] ^ 347 | crc_lookup[4][(one>>24) & 0xFF] ^ 348 | crc_lookup[3][(two ) & 0xFF] ^ 349 | crc_lookup[2][(two>> 8) & 0xFF] ^ 350 | crc_lookup[1][(two>>16) & 0xFF] ^ 351 | crc_lookup[0][(two>>24) & 0xFF]; 352 | #endif 353 | len -= 8; 354 | } 355 | 356 | unsigned char *cur_byte = (unsigned char*) cur; 357 | while (len--) 358 | crc = (crc >> 8) ^ crc_lookup[0][(crc & 0xFF) ^ *cur_byte++]; 359 | 360 | return ~crc; 361 | } 362 | 363 | uint32_t crc32_8_last8(void *data, size_t len, uint32_t prev_value) 364 | { 365 | size_t min = MIN(len, 8); 366 | void *src = (char*)data + (len-min); 367 | #ifdef _ALIGN_MEM_ACCESS 368 | uint64_t temp; // aligned 369 | memcpy(&temp, src, min); 370 | src = &temp; 371 | #endif 372 | return crc32_8(src, min, prev_value); 373 | } 374 | 375 | -------------------------------------------------------------------------------- /utils/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_CRC32_H 2 | #define _JSAHN_CRC32_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | uint32_t crc32_1(void* data, size_t len, uint32_t prev_value); 9 | uint32_t crc32_8(void* data, size_t len, uint32_t prev_value); 10 | uint32_t crc32_8_last8(void *data, size_t len, uint32_t prev_value); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /utils/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #if !defined(WIN32) && !defined(_WIN32) 37 | #include 38 | #endif 39 | #include 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | 46 | typedef struct _dictionary_ { 47 | /** Number of entries in dictionary */ 48 | int n; 49 | /** Storage size */ 50 | int size; 51 | /** List of string values */ 52 | char **val; 53 | /** List of string keys */ 54 | char **key ; 55 | /** List of hash values for keys */ 56 | unsigned *hash; 57 | } dictionary ; 58 | 59 | 60 | /* generated by genproto */ 61 | 62 | dictionary * iniparser_new(char *ininame); 63 | void iniparser_free(dictionary * d); 64 | 65 | int iniparser_getnsec(dictionary * d); 66 | char * iniparser_getsecname(dictionary * d, int n); 67 | void iniparser_dump(dictionary * d, FILE * f); 68 | void iniparser_dump_ini(dictionary * d, FILE * f); 69 | char * iniparser_getkey(dictionary *d, char *section, char *key); 70 | char * iniparser_getstr(dictionary * d, char * key); 71 | char * iniparser_getstring(dictionary * d, char * key, char * def); 72 | int iniparser_getint(dictionary * d, char * key, int notfound); 73 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 74 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 75 | int iniparser_find_entry(dictionary * ini, char * entry); 76 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 77 | void iniparser_unset(dictionary * ini, char * entry); 78 | 79 | #ifdef __cplusplus 80 | } 81 | #endif 82 | 83 | #endif 84 | 85 | -------------------------------------------------------------------------------- /utils/keygen.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "keygen.h" 7 | #include "crc32.h" 8 | 9 | static char *abt_array = 10 | (char*)"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; 11 | 12 | void keygen_init( 13 | struct keygen *keygen, 14 | size_t nprefix, 15 | struct rndinfo *prefix_len, 16 | struct rndinfo *prefix_dist, 17 | struct keygen_option *opt) 18 | { 19 | keygen->nprefix = nprefix; 20 | keygen->prefix_len = (struct rndinfo *)malloc(sizeof(struct rndinfo) * nprefix); 21 | keygen->prefix_dist = (struct rndinfo *)malloc(sizeof(struct rndinfo) * nprefix); 22 | keygen->opt = *opt; 23 | keygen->abt_array_size = strlen(abt_array); 24 | 25 | memcpy(keygen->prefix_len, prefix_len, sizeof(struct rndinfo) * nprefix); 26 | memcpy(keygen->prefix_dist, prefix_dist, sizeof(struct rndinfo) * nprefix); 27 | } 28 | 29 | void keygen_free(struct keygen *keygen) 30 | { 31 | free(keygen->prefix_len); 32 | free(keygen->prefix_dist); 33 | } 34 | 35 | void _crc2key(struct keygen *keygen, uint64_t crc, char *buf, size_t len, uint8_t abt_only) 36 | { 37 | size_t i; 38 | BDR_RNG_VARS_SET(crc); 39 | BDR_RNG_NEXTPAIR; 40 | BDR_RNG_NEXT; 41 | 42 | for (i=0;iabt_array_size))]; 46 | //buf[i] = 'a' + (rngz%('z'-'a')); 47 | } else { 48 | buf[i] = rngz & 0xff; 49 | } 50 | } 51 | } 52 | 53 | size_t _crc2keylen(struct rndinfo *prefix_len, uint64_t crc) 54 | { 55 | size_t r; 56 | BDR_RNG_VARS_SET(crc); 57 | BDR_RNG_NEXTPAIR; 58 | BDR_RNG_NEXT; 59 | 60 | r = get_random(prefix_len, rngz, rngz2); 61 | return r; 62 | } 63 | 64 | uint32_t keygen_idx2crc(uint64_t idx, uint32_t seed) 65 | { 66 | uint32_t crc; 67 | uint64_t idx64 = idx; 68 | crc = crc32_8(&idx64, sizeof(idx64), seed); 69 | crc = crc32_8(&idx64, sizeof(idx64), crc); 70 | return crc; 71 | } 72 | 73 | uint64_t MurmurHash64A ( const void * key, int len, unsigned int seed ) 74 | { 75 | const uint64_t m = 0xc6a4a7935bd1e995; 76 | const int r = 47; 77 | 78 | uint64_t h = seed ^ (len * m); 79 | 80 | const uint64_t * data = (const uint64_t *)key; 81 | const uint64_t * end = data + (len/8); 82 | 83 | while(data != end) 84 | { 85 | uint64_t k = *data++; 86 | 87 | k *= m; 88 | k ^= k >> r; 89 | k *= m; 90 | 91 | h ^= k; 92 | h *= m; 93 | } 94 | 95 | const unsigned char * data2 = (const unsigned char*)data; 96 | 97 | switch(len & 7) 98 | { 99 | case 7: h ^= uint64_t(data2[6]) << 48; 100 | case 6: h ^= uint64_t(data2[5]) << 40; 101 | case 5: h ^= uint64_t(data2[4]) << 32; 102 | case 4: h ^= uint64_t(data2[3]) << 24; 103 | case 3: h ^= uint64_t(data2[2]) << 16; 104 | case 2: h ^= uint64_t(data2[1]) << 8; 105 | case 1: h ^= uint64_t(data2[0]); 106 | h *= m; 107 | }; 108 | 109 | h ^= h >> r; 110 | h *= m; 111 | h ^= h >> r; 112 | 113 | return h; 114 | } 115 | 116 | size_t keygen_seed2key(struct keygen *keygen, uint64_t seed, char *buf) 117 | { 118 | uint64_t i; 119 | size_t len, cursor; 120 | uint64_t seed_local, seed64, rnd_sel; 121 | 122 | seed64 = MurmurHash64A(&seed, sizeof(uint64_t), 0); 123 | BDR_RNG_VARS_SET(seed64); 124 | BDR_RNG_NEXTPAIR; 125 | BDR_RNG_NEXT; 126 | 127 | cursor = 0; 128 | for (i=0;inprefix;++i){ 129 | if (i+1 == keygen->nprefix) { 130 | len = _crc2keylen(&keygen->prefix_len[i], seed64); 131 | _crc2key(keygen, seed64, buf + cursor, len, keygen->opt.abt_only); 132 | } else { 133 | BDR_RNG_NEXTPAIR; 134 | BDR_RNG_NEXT; 135 | rnd_sel = get_random(&keygen->prefix_dist[i], rngz, rngz2); 136 | seed_local = MurmurHash64A(&rnd_sel, sizeof(rnd_sel), 0); 137 | 138 | len = _crc2keylen(&keygen->prefix_len[i], seed_local); 139 | _crc2key(keygen, seed_local, buf + cursor, len, keygen->opt.abt_only); 140 | } 141 | 142 | cursor += len; 143 | 144 | if (keygen->opt.delimiter && i != keygen->nprefix-1) { 145 | buf[cursor] = '/'; 146 | cursor++; 147 | } 148 | } 149 | buf[cursor] = 0; 150 | return cursor; 151 | } 152 | 153 | -------------------------------------------------------------------------------- /utils/keygen.h: -------------------------------------------------------------------------------- 1 | #ifndef _JSAHN_KEYGEN_H 2 | #define _JSAHN_KEYGEN_H 3 | 4 | #include "adv_random.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | struct keygen_option { 11 | uint8_t delimiter; 12 | uint8_t abt_only; 13 | }; 14 | 15 | struct keygen { 16 | size_t nprefix; 17 | size_t abt_array_size; 18 | struct rndinfo *prefix_len; 19 | struct rndinfo *prefix_dist; 20 | struct keygen_option opt; 21 | }; 22 | 23 | void keygen_init( 24 | struct keygen *keygen, 25 | size_t nprefix, 26 | struct rndinfo *prefix_len, 27 | struct rndinfo *prefix_dist, 28 | struct keygen_option *opt); 29 | 30 | uint64_t MurmurHash64A( const void * key, int len, unsigned int seed ); 31 | 32 | void keygen_free(struct keygen *keygen); 33 | uint32_t keygen_idx2crc(uint64_t idx, uint32_t seed); 34 | size_t keygen_seed2key(struct keygen *keygen, uint64_t seed, char *buf); 35 | 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /utils/keyloader.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "sys/mman.h" 9 | #include "keyloader.h" 10 | #include "memleak.h" 11 | 12 | int keyloader_init(struct keyloader *handle, 13 | char *filename, 14 | struct keyloader_option *option) 15 | { 16 | int r_flag, first_key_found; 17 | uint64_t i, begin_pos, last_pos; 18 | uint64_t segsize = 256; 19 | 20 | begin_pos = last_pos = 0; 21 | 22 | // file open 23 | handle->fd = open(filename, O_RDONLY); 24 | if (handle->fd < 0) { 25 | return -1; 26 | } 27 | handle->nkeys = 0; 28 | handle->avg_keysize = 0; 29 | handle->filesize = lseek(handle->fd, 0, SEEK_END); 30 | 31 | handle->map = (char*)mmap(0, handle->filesize, PROT_READ, 32 | MAP_SHARED, handle->fd, 0); 33 | if (handle->map == MAP_FAILED) { 34 | close(handle->fd); 35 | return -2; 36 | } 37 | 38 | handle->arr = (struct keyloader_array*) 39 | malloc(sizeof(struct keyloader_array) * segsize); 40 | 41 | r_flag = first_key_found = 0; 42 | for (i=0;ifilesize;++i) { 43 | if (handle->map[i] == 0x0d || handle->map[i] == 0x0a) { 44 | // return character 45 | if (!r_flag && first_key_found) { 46 | // the end of key 47 | handle->arr[handle->nkeys].len = last_pos - begin_pos + 1; 48 | handle->avg_keysize += handle->arr[handle->nkeys].len; 49 | handle->nkeys++; 50 | if (option->max_nkeys && 51 | handle->nkeys >= option->max_nkeys) { 52 | r_flag = 1; 53 | break; 54 | } 55 | 56 | if (handle->nkeys >= segsize) { 57 | segsize *= 2; 58 | handle->arr = (struct keyloader_array*) 59 | realloc(handle->arr, sizeof(struct keyloader_array) * 60 | segsize); 61 | } 62 | } 63 | r_flag = 1; 64 | } else { 65 | if (!first_key_found) { 66 | // the first key 67 | handle->arr[0].offset = i; 68 | begin_pos = i; 69 | first_key_found = 1; 70 | } else if (r_flag) { 71 | handle->arr[handle->nkeys].offset = i; 72 | begin_pos = i; 73 | } 74 | last_pos = i; 75 | r_flag = 0; 76 | } 77 | } 78 | 79 | if (!r_flag) { 80 | handle->arr[handle->nkeys].len = last_pos - begin_pos + 1; 81 | handle->avg_keysize += handle->arr[handle->nkeys].len; 82 | handle->nkeys++; 83 | } 84 | if (handle->nkeys) { 85 | handle->avg_keysize /= handle->nkeys; 86 | } 87 | 88 | return 0; 89 | } 90 | 91 | uint64_t keyloader_get_nkeys(struct keyloader *handle) 92 | { 93 | return handle->nkeys; 94 | } 95 | 96 | size_t keyloader_get_avg_keylen(struct keyloader *handle) 97 | { 98 | return handle->avg_keysize; 99 | } 100 | 101 | size_t keyloader_get_key(struct keyloader *handle, uint64_t idx, char *buf) 102 | { 103 | size_t len; 104 | 105 | if (idx >= handle->nkeys) { 106 | return 0; 107 | } 108 | 109 | len = handle->arr[idx].len; 110 | memcpy(buf, handle->map + handle->arr[idx].offset, len); 111 | buf[len] = 0; 112 | 113 | return len; 114 | } 115 | 116 | void keyloader_free(struct keyloader *handle) 117 | { 118 | munmap(handle->map, handle->filesize); 119 | free(handle->arr); 120 | close(handle->fd); 121 | } 122 | 123 | -------------------------------------------------------------------------------- /utils/keyloader.h: -------------------------------------------------------------------------------- 1 | #ifndef _FDBBENCH_KEYLOADER_H 2 | #define _FDBBENCH_KEYLOADER_H 3 | 4 | #include 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | struct keyloader_option { 11 | uint64_t max_nkeys; 12 | }; 13 | 14 | struct keyloader_array { 15 | uint64_t offset; 16 | uint32_t len; 17 | }; 18 | 19 | struct keyloader { 20 | uint64_t nkeys; 21 | uint64_t filesize; 22 | int fd; 23 | char *map; 24 | struct keyloader_array *arr; 25 | uint64_t avg_keysize; 26 | }; 27 | 28 | int keyloader_init(struct keyloader *handle, 29 | char *filename, 30 | struct keyloader_option *option); 31 | 32 | uint64_t keyloader_get_nkeys(struct keyloader *handle); 33 | size_t keyloader_get_avg_keylen(struct keyloader *handle); 34 | 35 | size_t keyloader_get_key(struct keyloader *handle, uint64_t idx, char *buf); 36 | void keyloader_free(struct keyloader *handle); 37 | 38 | #ifdef __cplusplus 39 | } 40 | #endif 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /utils/memleak.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Simple Memory Leakage Detection Tool 4 | * (C) 2013 Jung-Sang Ahn 5 | * see https://github.com/greensky00/memleak 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #if !defined(__APPLE__) 14 | #include 15 | #endif 16 | 17 | #include "arch.h" 18 | 19 | #define _MALLOC_OVERRIDE 20 | #define INIT_VAL (0x77) 21 | #define FREE_VAL (0xff) 22 | //#define _WARN_NOT_ALLOCATED_MEMORY 23 | //#define _PRINT_DBG 24 | #if !defined(_WIN32) && !defined(WIN32) 25 | //#define _STACK_BACKTRACE 26 | //#define _LIBBFD 27 | #endif 28 | 29 | #include "memleak.h" 30 | 31 | #ifdef _STACK_BACKTRACE 32 | #include 33 | 34 | #ifdef _LIBBFD 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | /* globals retained across calls to resolve. */ 41 | static bfd* abfd = 0; 42 | static asymbol **syms = 0; 43 | static asection *text = 0; 44 | 45 | static void resolve(void *address, char **file, char **func, unsigned *line) { 46 | if (!abfd) { 47 | char ename[1024]; 48 | int l = readlink("/proc/self/exe",ename,sizeof(ename)); 49 | if (l == -1) { 50 | perror("failed to find executable\n"); 51 | return; 52 | } 53 | ename[l] = 0; 54 | 55 | bfd_init(); 56 | 57 | abfd = bfd_openr(ename, 0); 58 | if (!abfd) { 59 | perror("bfd_openr failed: "); 60 | return; 61 | } 62 | 63 | bfd_check_format(abfd,bfd_object); 64 | 65 | unsigned storage_needed = bfd_get_symtab_upper_bound(abfd); 66 | syms = (asymbol **) malloc(storage_needed); 67 | unsigned cSymbols = bfd_canonicalize_symtab(abfd, syms); 68 | 69 | text = bfd_get_section_by_name(abfd, ".text"); 70 | } 71 | 72 | long offset = ((long)address) - text->vma; 73 | if (offset > 0) { 74 | bfd_find_nearest_line(abfd, text, syms, offset, 75 | (const char**)file, (const char**)func, line); 76 | } 77 | } 78 | 79 | #endif // _LIBBFD 80 | 81 | #endif // _STACK_BACKTRACE 82 | 83 | 84 | #ifdef _PRINT_DBG 85 | #define DBG(...) fprintf(stderr, __VA_ARGS__) 86 | #else 87 | #define DBG(...) 88 | #endif 89 | 90 | #include "avltree.h" 91 | 92 | struct memleak_item { 93 | uint64_t addr; 94 | char *file; 95 | size_t size; 96 | size_t line; 97 | struct avl_node avl; 98 | #ifdef _STACK_BACKTRACE 99 | size_t bt_size; 100 | void **btrace; 101 | #endif 102 | #ifdef _CHK_MODIFY_AFTER_FREE 103 | uint8_t freed; 104 | #endif 105 | }; 106 | 107 | static struct avl_tree tree_index; 108 | static uint8_t start_sw = 0; 109 | static spin_t lock; 110 | 111 | int memleak_cmp(struct avl_node *a, struct avl_node *b, void *aux) 112 | { 113 | struct memleak_item *aa, *bb; 114 | aa = _get_entry(a, struct memleak_item, avl); 115 | bb = _get_entry(b, struct memleak_item, avl); 116 | if (aa->addr < bb->addr) return -1; 117 | else if (aa->addr > bb->addr) return 1; 118 | else return 0; 119 | } 120 | 121 | LIBMEMLEAK_API 122 | void memleak_start() 123 | { 124 | spin_init(&lock); 125 | avl_init(&tree_index, NULL); 126 | start_sw = 1; 127 | } 128 | 129 | LIBMEMLEAK_API 130 | void memleak_end() 131 | { 132 | uint8_t is_leaked; 133 | size_t count = 0; 134 | struct avl_node *a; 135 | struct memleak_item *item; 136 | #ifdef _STACK_BACKTRACE 137 | int i; 138 | char **strs; 139 | char *file, *func; 140 | unsigned line; 141 | #endif 142 | 143 | spin_lock(&lock); 144 | 145 | start_sw = 0; 146 | 147 | a = avl_first(&tree_index); 148 | while(a){ 149 | item = _get_entry(a, struct memleak_item, avl); 150 | a = avl_next(a); 151 | avl_remove(&tree_index, &item->avl); 152 | is_leaked = 1; 153 | 154 | #ifdef _CHK_MODIFY_AFTER_FREE 155 | int i; 156 | uint8_t *addr; 157 | if (item->freed) { 158 | is_leaked = 0; 159 | addr = (uint8_t*)item->addr; 160 | for (i=0;isize;++i){ 161 | if (*(addr + i) != FREE_VAL) { 162 | fprintf(stderr, 163 | "address 0x%016lx (allocated at %s:%lu, size %lu) " 164 | "has been modified after being freed\n", 165 | (unsigned long)item->addr, item->file, 166 | item->line, item->size); 167 | break; 168 | } 169 | } 170 | free(addr); 171 | } 172 | #endif // _CHK_MODIFY_AFTER_FREE 173 | 174 | if (is_leaked) { 175 | fprintf(stderr, "address 0x%016lx (allocated at %s:%lu, size %lu) " 176 | "is not freed\n", 177 | (unsigned long)item->addr, item->file, 178 | (unsigned long)item->line, (unsigned long)item->size); 179 | count++; 180 | #ifdef _STACK_BACKTRACE 181 | strs = backtrace_symbols(item->btrace, item->bt_size); 182 | for (i=0;ibt_size;++i){ 183 | #ifdef _LIBBFD 184 | resolve(item->btrace[i], &file, &func, &line); 185 | fprintf(stderr, " %s [%s:%d]\n", func, file, line); 186 | #else // _LIBBFD 187 | fprintf(stderr, " %s\n", strs[i]); 188 | #endif // _LIBBFD 189 | } 190 | free(item->btrace); 191 | #endif // _STACK_BACKTRACE 192 | } 193 | free(item); 194 | } 195 | if (count > 0) fprintf(stderr, "total %d objects\n", (int)count); 196 | 197 | spin_unlock(&lock); 198 | } 199 | 200 | void _memleak_add_to_index(void *addr, size_t size, char *file, size_t line, uint8_t init_val) 201 | { 202 | DBG("malloc at %s:%ld, size %ld\n", file, line, size); 203 | struct memleak_item *item = (struct memleak_item *)malloc(sizeof(struct memleak_item)); 204 | item->addr = (uint64_t)addr; 205 | item->file = file; 206 | item->line = line; 207 | item->size = size; 208 | #ifdef INIT_VAL 209 | memset(addr, init_val, size); 210 | #endif 211 | #ifdef _STACK_BACKTRACE 212 | void *temp_stack[256]; 213 | item->bt_size = backtrace(temp_stack, 256); 214 | item->btrace = (void**)malloc(sizeof(void*) * item->bt_size); 215 | memcpy(item->btrace, temp_stack, sizeof(void*) * item->bt_size); 216 | #endif 217 | #ifdef _CHK_MODIFY_AFTER_FREE 218 | item->freed = 0; 219 | #endif 220 | avl_insert(&tree_index, &item->avl, memleak_cmp); 221 | } 222 | 223 | LIBMEMLEAK_API 224 | void * memleak_alloc(size_t size, char *file, size_t line) 225 | { 226 | void *addr = (void*)malloc(size); 227 | if (addr && start_sw) { 228 | spin_lock(&lock); 229 | _memleak_add_to_index(addr, size, file, line, INIT_VAL); 230 | spin_unlock(&lock); 231 | } 232 | 233 | return addr; 234 | } 235 | 236 | LIBMEMLEAK_API 237 | void * memleak_calloc(size_t nmemb, size_t size, char *file, size_t line) 238 | { 239 | void *addr = (void *)calloc(nmemb, size); 240 | if (addr && start_sw) { 241 | spin_lock(&lock); 242 | _memleak_add_to_index(addr, size, file, line, 0x0); 243 | spin_unlock(&lock); 244 | } 245 | 246 | return addr; 247 | } 248 | 249 | #ifndef WIN32 250 | // posix only 251 | LIBMEMLEAK_API 252 | int memleak_posix_memalign(void **memptr, size_t alignment, size_t size, char *file, size_t line) 253 | { 254 | 255 | int ret = posix_memalign(memptr, alignment, size); 256 | if (ret==0 && start_sw) 257 | { 258 | spin_lock(&lock); 259 | _memleak_add_to_index(*memptr, size, file, line, INIT_VAL); 260 | spin_unlock(&lock); 261 | } 262 | 263 | return ret; 264 | } 265 | #else // WIN32 266 | 267 | LIBMEMLEAK_API 268 | void * memleak_aligned_malloc(size_t size, size_t alignment, char *file, size_t line) 269 | { 270 | void *addr = (void*)_aligned_malloc(size, alignment); 271 | if (addr && start_sw) { 272 | spin_lock(&lock); 273 | _memleak_add_to_index(addr, size, file, line, INIT_VAL); 274 | spin_unlock(&lock); 275 | } 276 | return addr; 277 | } 278 | 279 | LIBMEMLEAK_API 280 | void memleak_aligned_free(void *addr, char *file, size_t line) 281 | { 282 | struct avl_node *a; 283 | struct memleak_item *item, query; 284 | 285 | if (start_sw) { 286 | spin_lock(&lock); 287 | 288 | query.addr = (uint64_t)addr; 289 | a = avl_search(&tree_index, &query.avl, memleak_cmp); 290 | if (!a) { 291 | #ifdef _WARN_NOT_ALLOCATED_MEMORY 292 | fprintf(stderr, "try to free not allocated memory address 0x%016lx at %s:%ld\n", 293 | (long unsigned int)addr, file, line); 294 | #endif 295 | spin_unlock(&lock); 296 | return; 297 | } 298 | 299 | item = _get_entry(a, struct memleak_item, avl); 300 | DBG("free address 0x%016lx (allocated at %s:%ld, size %ld)\n", 301 | item->addr, item->file, item->line, item->size); 302 | #ifdef FREE_VAL 303 | memset(addr, FREE_VAL, item->size); 304 | #endif 305 | 306 | avl_remove(&tree_index, a); 307 | #ifdef _STACK_BACKTRACE 308 | free(item->btrace); 309 | #endif 310 | free(item); 311 | spin_unlock(&lock); 312 | } 313 | _aligned_free(addr); 314 | } 315 | 316 | #endif // WIN32 317 | 318 | LIBMEMLEAK_API 319 | void *memleak_realloc(void *ptr, size_t size) 320 | { 321 | 322 | void *addr = (void *)realloc(ptr, size); 323 | if (addr && start_sw) { 324 | spin_lock(&lock); 325 | struct avl_node *a; 326 | struct memleak_item *item, query; 327 | 328 | query.addr = (uint64_t)ptr; 329 | a = avl_search(&tree_index, &query.avl, memleak_cmp); 330 | if (a) { 331 | item = _get_entry(a, struct memleak_item, avl); 332 | DBG("realloc from address 0x%016lx (allocated at %s:%ld, size %ld)\n\tto address 0x%016lx (size %ld)\n", 333 | item->addr, item->file, item->line, item->size, (uint64_t)addr, size); 334 | avl_remove(&tree_index, a); 335 | _memleak_add_to_index(addr, size, item->file, item->line, INIT_VAL); 336 | #ifdef _STACK_BACKTRACE 337 | free(item->btrace); 338 | #endif 339 | free(item); 340 | } 341 | spin_unlock(&lock); 342 | } 343 | 344 | return addr; 345 | } 346 | 347 | LIBMEMLEAK_API 348 | void memleak_free(void *addr, char *file, size_t line) 349 | { 350 | struct avl_node *a; 351 | struct memleak_item *item, query; 352 | 353 | if (start_sw) { 354 | spin_lock(&lock); 355 | 356 | query.addr = (uint64_t)addr; 357 | a = avl_search(&tree_index, &query.avl, memleak_cmp); 358 | if (!a) { 359 | #ifdef _WARN_NOT_ALLOCATED_MEMORY 360 | fprintf(stderr, "try to free not allocated memory address 0x%016lx at %s:%ld\n", 361 | (long unsigned int)addr, file, line); 362 | #endif 363 | spin_unlock(&lock); 364 | return; 365 | } 366 | 367 | item = _get_entry(a, struct memleak_item, avl); 368 | DBG("free address 0x%016lx (allocated at %s:%ld, size %ld)\n", 369 | item->addr, item->file, item->line, item->size); 370 | #ifdef FREE_VAL 371 | memset(addr, FREE_VAL, item->size); 372 | #endif 373 | 374 | #ifdef _STACK_BACKTRACE 375 | free(item->btrace); 376 | #endif 377 | 378 | #ifndef _CHK_MODIFY_AFTER_FREE 379 | avl_remove(&tree_index, a); 380 | free(item); 381 | #else 382 | item->freed = 1; 383 | #endif 384 | spin_unlock(&lock); 385 | } 386 | #ifndef _CHK_MODIFY_AFTER_FREE 387 | free(addr); 388 | #endif 389 | } 390 | 391 | -------------------------------------------------------------------------------- /utils/memleak.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Simple Memory Leakage Detection Tool 4 | * (C) 2013 Jung-Sang Ahn 5 | * see https://github.com/greensky00/memleak 6 | */ 7 | 8 | #ifndef _JSAHN_MEMLEAK_H 9 | #define _JSAHN_MEMLEAK_H 10 | 11 | #include 12 | 13 | #ifdef _MSC_VER 14 | #ifdef forestdb_EXPORTS 15 | #define LIBMEMLEAK_API extern __declspec(dllexport) 16 | #else 17 | #define LIBMEMLEAK_API 18 | #endif 19 | #else 20 | #define LIBMEMLEAK_API 21 | #endif 22 | 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | 27 | //#define _MEMLEAK_ENABLE 28 | #ifndef _MEMLEAK_ENABLE 29 | #define _MALLOC_OVERRIDE 30 | #endif 31 | #ifndef _MALLOC_OVERRIDE 32 | #define _MALLOC_OVERRIDE 33 | #define malloc(size) memleak_alloc(size, (char*)__FILE__, __LINE__) 34 | #define calloc(nmemb, size) memleak_calloc(nmemb, size, (char*)__FILE__, __LINE__) 35 | #define realloc(ptr, size) memleak_realloc(ptr, size); 36 | #define free(addr) memleak_free(addr, (char*)__FILE__, __LINE__) 37 | #ifndef WIN32 38 | #define posix_memalign(memptr, alignment, size) \ 39 | memleak_posix_memalign(memptr, alignment, size, (char*)__FILE__, __LINE__) 40 | #else // WIN32 41 | #define _aligned_malloc(size, align) \ 42 | memleak_aligned_malloc(size, align, (char*)__FILE__, __LINE__) 43 | #define _aligned_free(addr) \ 44 | memleak_aligned_free(addr, (char*)__FILE__, __LINE__) 45 | #endif // WIN32 46 | #endif 47 | 48 | LIBMEMLEAK_API 49 | void memleak_start(); 50 | 51 | LIBMEMLEAK_API 52 | void memleak_end(); 53 | 54 | LIBMEMLEAK_API 55 | void * memleak_alloc(size_t size, char *file, size_t line); 56 | 57 | LIBMEMLEAK_API 58 | void * memleak_calloc(size_t nmemb, size_t size, char *file, size_t line); 59 | 60 | LIBMEMLEAK_API 61 | void * memleak_memalign(size_t alignment, size_t size, char *file, size_t line); 62 | 63 | LIBMEMLEAK_API 64 | void *memleak_realloc(void *ptr, size_t size); 65 | 66 | LIBMEMLEAK_API 67 | void memleak_free(void *addr, char *file, size_t line); 68 | 69 | #ifndef WIN32 70 | LIBMEMLEAK_API 71 | int memleak_posix_memalign(void **memptr, size_t alignment, size_t size, char *file, size_t line); 72 | #else 73 | LIBMEMLEAK_API 74 | void * memleak_aligned_malloc(size_t size, size_t alignment, char *file, size_t line); 75 | LIBMEMLEAK_API 76 | void memleak_aligned_free(void *addr, char *file, size_t line); 77 | #endif 78 | 79 | #ifdef __cplusplus 80 | } 81 | #endif 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /utils/stopwatch.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #include "stopwatch.h" 21 | 22 | static struct timeval _utime_gap(struct timeval a, struct timeval b) 23 | { 24 | struct timeval ret; 25 | if (b.tv_usec >= a.tv_usec) { 26 | ret.tv_usec = b.tv_usec - a.tv_usec; 27 | ret.tv_sec = b.tv_sec - a.tv_sec; 28 | }else{ 29 | ret.tv_usec = 1000000 + b.tv_usec - a.tv_usec; 30 | ret.tv_sec = b.tv_sec - a.tv_sec - 1; 31 | } 32 | return ret; 33 | } 34 | 35 | uint64_t _timeval_to_us(struct timeval tv) 36 | { 37 | return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec; 38 | } 39 | 40 | void stopwatch_init(struct stopwatch *sw) 41 | { 42 | sw->elapsed.tv_sec = 0; 43 | sw->elapsed.tv_usec = 0; 44 | } 45 | 46 | void stopwatch_start(struct stopwatch *sw) 47 | { 48 | gettimeofday(&sw->start, NULL); 49 | } 50 | 51 | void stopwatch_init_start(struct stopwatch *sw) 52 | { 53 | stopwatch_init(sw); 54 | stopwatch_start(sw); 55 | } 56 | 57 | int stopwatch_check_ms(struct stopwatch *sw, size_t ms) 58 | { 59 | struct timeval cur, gap; 60 | gettimeofday(&cur, NULL); 61 | gap = _utime_gap(sw->start, cur); 62 | if ((uint64_t)gap.tv_sec * 1000 + (uint64_t)gap.tv_usec / 1000 >= ms) { 63 | return 1; 64 | } 65 | return 0; 66 | } 67 | 68 | int stopwatch_check_us(struct stopwatch *sw, size_t us) 69 | { 70 | struct timeval cur, gap; 71 | gettimeofday(&cur, NULL); 72 | gap = _utime_gap(sw->start, cur); 73 | if ((uint64_t)gap.tv_sec * 1000000 + (uint64_t)gap.tv_usec >= us) { 74 | return 1; 75 | } 76 | return 0; 77 | } 78 | 79 | struct timeval stopwatch_get_curtime(struct stopwatch *sw) 80 | { 81 | struct timeval end, gap; 82 | gettimeofday(&end, NULL); 83 | gap = _utime_gap(sw->start, end); 84 | return gap; 85 | } 86 | 87 | struct timeval stopwatch_get_elapsed(struct stopwatch *sw) 88 | { 89 | return sw->elapsed; 90 | } 91 | 92 | struct timeval stopwatch_stop(struct stopwatch *sw) 93 | { 94 | struct timeval end, gap; 95 | gettimeofday(&end, NULL); 96 | gap = _utime_gap(sw->start, end); 97 | sw->elapsed.tv_sec += gap.tv_sec; 98 | sw->elapsed.tv_usec += gap.tv_usec; 99 | if (sw->elapsed.tv_usec >= 1000000) { 100 | sw->elapsed.tv_usec -= 1000000; 101 | sw->elapsed.tv_sec++; 102 | } 103 | 104 | return gap; 105 | } 106 | -------------------------------------------------------------------------------- /utils/stopwatch.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _JSAHN_STOPWATCH_H 19 | #define _JSAHN_STOPWATCH_H 20 | 21 | #include 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | struct stopwatch { 29 | struct timeval elapsed; 30 | struct timeval start; 31 | }; 32 | 33 | uint64_t _timeval_to_us(struct timeval tv); 34 | void stopwatch_init(struct stopwatch *sw); 35 | void stopwatch_start(struct stopwatch *sw); 36 | void stopwatch_init_start(struct stopwatch *sw); 37 | int stopwatch_check_ms(struct stopwatch *sw, size_t ms); 38 | int stopwatch_check_us(struct stopwatch *sw, size_t us); 39 | struct timeval stopwatch_get_curtime(struct stopwatch *sw); 40 | struct timeval stopwatch_get_elapsed(struct stopwatch *sw); 41 | struct timeval stopwatch_stop(struct stopwatch *sw); 42 | 43 | #ifdef __cplusplus 44 | } 45 | #endif 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /utils/zipfian_random.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "adv_random.h" 6 | #include "zipfian_random.h" 7 | 8 | #include "memleak.h" 9 | 10 | double _get_den_sum(struct zipf_rnd *zipf) 11 | { 12 | uint64_t i; 13 | double s, sum; 14 | 15 | s = zipf->s; 16 | sum = 0; 17 | 18 | for (i=0;in;++i){ 19 | sum += 1/pow((i+1), s); 20 | } 21 | return sum; 22 | } 23 | 24 | double _get_zeta(struct zipf_rnd *zipf, uint64_t k, double s, uint64_t n) 25 | { 26 | return (1/pow(k,s)) / zipf->sum; 27 | } 28 | 29 | void zipf_rnd_init(struct zipf_rnd *zipf, uint64_t n, double s, uint32_t resolution) 30 | { 31 | uint64_t i, j, temp, a, b; 32 | uint32_t *table; 33 | double cum, zeta; 34 | BDR_RNG_VARS; 35 | 36 | zipf->n = n; 37 | zipf->s = s; 38 | zipf->resolution = resolution; 39 | zipf->turn = 0; 40 | zipf->table = (uint32_t*)malloc(sizeof(uint32_t) * zipf->n); 41 | zipf->map = (uint32_t*)malloc(sizeof(uint32_t) * resolution); 42 | zipf->sum = _get_den_sum(zipf); 43 | 44 | memset(zipf->map, 0, sizeof(uint32_t) * resolution); 45 | table = zipf->table; 46 | for (i=0; in; ++i){ 47 | table[i] = i; 48 | } 49 | 50 | // shuffle zipfian table 51 | for (i=0; in; ++i){ 52 | BDR_RNG_NEXTPAIR; 53 | a = rngz % zipf->n; 54 | b = rngz2 % zipf->n; 55 | 56 | temp = table[a]; 57 | table[a] = table[b]; 58 | table[b] = temp; 59 | } 60 | 61 | // create map table 62 | cum = 0; 63 | for (i=0;in;++i){ 64 | zeta = _get_zeta(zipf, i+1, zipf->s, zipf->n); 65 | 66 | a = (uint64_t)(cum * zipf->resolution); 67 | b = (uint64_t)((cum+zeta) * zipf->resolution); 68 | if (b >= zipf->resolution) b = zipf->resolution-1; 69 | cum += zeta; 70 | 71 | for (j = a; j <= b; ++j){ 72 | zipf->map[j] = i; 73 | } 74 | } 75 | } 76 | 77 | uint32_t zipf_rnd_get(struct zipf_rnd *zipf) 78 | { 79 | uint32_t idx, r; 80 | r = rand() % zipf->resolution; 81 | idx = (zipf->map[r] + zipf->turn) % zipf->n; 82 | 83 | return zipf->table[idx]; 84 | } 85 | 86 | void zipf_rnd_shift(struct zipf_rnd *zipf, uint32_t shift) 87 | { 88 | zipf->turn += shift; 89 | zipf->turn = zipf->turn % zipf->n; 90 | } 91 | 92 | void zipf_rnd_free(struct zipf_rnd *zipf) 93 | { 94 | free(zipf->table); 95 | free(zipf->map); 96 | } 97 | 98 | -------------------------------------------------------------------------------- /utils/zipfian_random.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSAHN_ZIPFIAN_RANDOM 2 | #define __JSAHN_ZIPFIAN_RANDOM 3 | 4 | #include 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | struct zipf_rnd{ 11 | uint64_t n; 12 | uint32_t resolution; 13 | uint32_t *table; 14 | // map from [0, resolution) -> table 15 | uint32_t *map; 16 | uint64_t turn; 17 | double s; 18 | double sum; 19 | }; 20 | 21 | void zipf_rnd_init(struct zipf_rnd *zipf, uint64_t n, double s, uint32_t resolution); 22 | uint32_t zipf_rnd_get(struct zipf_rnd *zipf); 23 | void zipf_rnd_shift(struct zipf_rnd *zipf, uint32_t shift); 24 | void zipf_rnd_free(struct zipf_rnd *zipf); 25 | 26 | #ifdef __cplusplus 27 | } 28 | #endif 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /wrappers/couch_fdb.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "libforestdb/forestdb.h" 8 | #include "libcouchstore/couch_db.h" 9 | 10 | #include "memleak.h" 11 | 12 | #define META_BUF_MAXLEN (256) 13 | #define SEQNUM_NOT_USED (0xffffffffffffffff) 14 | #define MAX_KEYLEN (4096) 15 | extern int64_t DATABUF_MAXLEN; 16 | 17 | struct _db { 18 | fdb_file_handle *dbfile; 19 | fdb_kvs_handle *fdb; 20 | char *filename; 21 | }; 22 | 23 | static uint64_t config_flags = 0x0; 24 | static uint64_t cache_size = 0; 25 | static int c_auto = 1; 26 | static size_t c_threshold = 83; 27 | static size_t br_threshold = 65; 28 | static size_t wal_size = 4096; 29 | static size_t c_period = 15; 30 | static int compression = 0; 31 | static int indexing_type = 0; 32 | static int auto_compaction_threads = 4; 33 | 34 | couchstore_error_t couchstore_set_flags(uint64_t flags) { 35 | config_flags = flags; 36 | return COUCHSTORE_SUCCESS; 37 | } 38 | couchstore_error_t couchstore_set_idx_type(int type) { 39 | indexing_type = type; 40 | return COUCHSTORE_SUCCESS; 41 | } 42 | couchstore_error_t couchstore_set_cache(uint64_t size) { 43 | cache_size = size; 44 | return COUCHSTORE_SUCCESS; 45 | } 46 | couchstore_error_t couchstore_set_compaction(int mode, 47 | size_t compact_thres, 48 | size_t block_reuse_thres) { 49 | c_auto = mode; 50 | c_threshold = compact_thres; 51 | br_threshold = block_reuse_thres; 52 | return COUCHSTORE_SUCCESS; 53 | } 54 | couchstore_error_t couchstore_set_auto_compaction_threads(int num_threads) { 55 | auto_compaction_threads = num_threads; 56 | return COUCHSTORE_SUCCESS; 57 | } 58 | couchstore_error_t couchstore_set_wal_size(size_t size) { 59 | wal_size = size; 60 | return COUCHSTORE_SUCCESS; 61 | } 62 | couchstore_error_t couchstore_close_conn() { 63 | fdb_shutdown(); 64 | return COUCHSTORE_SUCCESS; 65 | } 66 | couchstore_error_t couchstore_set_chk_period(size_t seconds) { 67 | c_period = seconds; 68 | return COUCHSTORE_SUCCESS; 69 | } 70 | couchstore_error_t couchstore_set_compression(int opt) { 71 | compression = opt; 72 | return COUCHSTORE_SUCCESS; 73 | } 74 | 75 | void logCallbackFunc(int err_code, 76 | const char *err_msg, 77 | void *pCtxData) { 78 | fprintf(stderr, "%s - error code: %d, error message: %s\n", 79 | (char *) pCtxData, err_code, err_msg); 80 | } 81 | 82 | LIBCOUCHSTORE_API 83 | couchstore_error_t couchstore_open_db(const char *filename, 84 | couchstore_open_flags flags, 85 | Db **pDb) 86 | { 87 | return couchstore_open_db_ex(filename, flags, 88 | NULL, pDb); 89 | } 90 | 91 | // lexicographically compares two variable-length binary streams 92 | #define MIN(a,b) (((a)<(b))?(a):(b)) 93 | static int _bench_keycmp(void *key1, size_t keylen1, void *key2, size_t keylen2) 94 | { 95 | if (keylen1 == keylen2) { 96 | return memcmp(key1, key2, keylen1); 97 | }else { 98 | size_t len = MIN(keylen1, keylen2); 99 | int cmp = memcmp(key1, key2, len); 100 | if (cmp != 0) return cmp; 101 | else { 102 | return (int)((int)keylen1 - (int)keylen2); 103 | } 104 | } 105 | } 106 | 107 | LIBCOUCHSTORE_API 108 | couchstore_error_t couchstore_open_db_ex(const char *filename, 109 | couchstore_open_flags flags, 110 | FileOpsInterface *ops, 111 | Db **pDb) 112 | { 113 | fdb_config config; 114 | fdb_kvs_config kvs_config; 115 | fdb_status status; 116 | fdb_file_handle *dbfile; 117 | fdb_kvs_handle *fdb; 118 | char *fname = (char *)filename; 119 | 120 | memset(&config, 0, sizeof(fdb_config)); 121 | config = fdb_get_default_config(); 122 | if (c_auto) { 123 | config.compaction_mode = FDB_COMPACTION_AUTO; 124 | config.compaction_threshold = c_threshold; 125 | } else { 126 | config.compaction_mode = FDB_COMPACTION_MANUAL; 127 | } 128 | config.block_reusing_threshold = br_threshold; 129 | config.num_compactor_threads = auto_compaction_threads; 130 | config.compactor_sleep_duration = c_period; 131 | config.chunksize = sizeof(uint64_t); 132 | config.buffercache_size = (uint64_t)cache_size; 133 | config.wal_threshold = wal_size; 134 | config.num_wal_partitions = 31; 135 | config.num_bcache_partitions = 31; 136 | config.seqtree_opt = FDB_SEQTREE_NOT_USE; 137 | if (flags & 0x10) { 138 | config.durability_opt = FDB_DRB_NONE; 139 | } else { 140 | config.durability_opt = FDB_DRB_ASYNC; 141 | } 142 | config.compress_document_body = (compression)?true:false; 143 | if (config_flags & 0x1) { 144 | config.wal_flush_before_commit = true; 145 | } else { 146 | config.wal_flush_before_commit = false; 147 | } 148 | if (config_flags & 0x10) { 149 | config.auto_commit = true; 150 | } else { 151 | config.auto_commit = false; 152 | } 153 | config.prefetch_duration = 0; 154 | config.multi_kv_instances = false; 155 | 156 | kvs_config = fdb_get_default_kvs_config(); 157 | 158 | *pDb = (Db*)calloc(1, sizeof(Db)); 159 | (*pDb)->filename = (char *)malloc(strlen(filename)+1); 160 | strcpy((*pDb)->filename, filename); 161 | 162 | if (indexing_type == 1) { 163 | // naive B+tree 164 | char *kvs_names[] = {(char*)"default"}; 165 | fdb_custom_cmp_variable functions[] = {_bench_keycmp}; 166 | config.multi_kv_instances = true; 167 | status = fdb_open_custom_cmp(&dbfile, fname, &config, 168 | 1, kvs_names, functions); 169 | } else { 170 | status = fdb_open(&dbfile, fname, &config); 171 | } 172 | status = fdb_kvs_open_default(dbfile, &fdb, &kvs_config); 173 | 174 | (*pDb)->dbfile = dbfile; 175 | (*pDb)->fdb = fdb; 176 | 177 | if (status == FDB_RESULT_SUCCESS) { 178 | return COUCHSTORE_SUCCESS; 179 | } else { 180 | free((*pDb)->filename); 181 | free(*pDb); 182 | return COUCHSTORE_ERROR_OPEN_FILE; 183 | } 184 | } 185 | 186 | LIBCOUCHSTORE_API 187 | couchstore_error_t couchstore_close_db(Db *db) 188 | { 189 | fdb_close(db->dbfile); 190 | free(db->filename); 191 | free(db); 192 | 193 | return COUCHSTORE_SUCCESS; 194 | } 195 | 196 | LIBCOUCHSTORE_API 197 | couchstore_error_t couchstore_db_info(Db *db, DbInfo* info) 198 | { 199 | fdb_file_info fdb_info; 200 | 201 | fdb_get_file_info(db->dbfile, &fdb_info); 202 | 203 | info->file_size = fdb_info.file_size; 204 | info->space_used = fdb_info.space_used; 205 | info->filename = fdb_info.filename; 206 | 207 | return COUCHSTORE_SUCCESS; 208 | } 209 | 210 | size_t _docinfo_to_buf(DocInfo *docinfo, void *buf) 211 | { 212 | // [db_seq,] rev_seq, deleted, content_meta, rev_meta (size), rev_meta (buf) 213 | size_t offset = 0; 214 | 215 | memcpy((uint8_t*)buf + offset, &docinfo->rev_seq, sizeof(docinfo->rev_seq)); 216 | offset += sizeof(docinfo->rev_seq); 217 | 218 | memcpy((uint8_t*)buf + offset, &docinfo->deleted, sizeof(docinfo->deleted)); 219 | offset += sizeof(docinfo->deleted); 220 | 221 | memcpy((uint8_t*)buf + offset, &docinfo->content_meta, sizeof(docinfo->content_meta)); 222 | offset += sizeof(docinfo->content_meta); 223 | 224 | memcpy((uint8_t*)buf + offset, &docinfo->rev_meta.size, sizeof(docinfo->rev_meta.size)); 225 | offset += sizeof(docinfo->rev_meta.size); 226 | 227 | if (docinfo->rev_meta.size > 0) { 228 | memcpy((uint8_t*)buf + offset, docinfo->rev_meta.buf, docinfo->rev_meta.size); 229 | offset += docinfo->rev_meta.size; 230 | } 231 | 232 | return offset; 233 | } 234 | 235 | LIBCOUCHSTORE_API 236 | couchstore_error_t couchstore_save_documents(Db *db, Doc* const docs[], DocInfo *infos[], 237 | unsigned numdocs, couchstore_save_options options) 238 | { 239 | unsigned i; 240 | fdb_doc _doc; 241 | fdb_status status = FDB_RESULT_SUCCESS; 242 | uint8_t buf[META_BUF_MAXLEN]; 243 | 244 | memset(&_doc, 0, sizeof(_doc)); 245 | for (i=0;iid.buf; 247 | _doc.keylen = docs[i]->id.size; 248 | _doc.body = docs[i]->data.buf; 249 | _doc.bodylen = docs[i]->data.size; 250 | _doc.metalen = _docinfo_to_buf(infos[i], buf); 251 | _doc.meta = buf; 252 | _doc.deleted = 0; 253 | 254 | status = fdb_set(db->fdb, &_doc); 255 | assert(status == FDB_RESULT_SUCCESS); 256 | 257 | infos[i]->db_seq = _doc.seqnum; 258 | infos[i]->bp = _doc.offset; 259 | } 260 | 261 | if (status == FDB_RESULT_SUCCESS) 262 | return COUCHSTORE_SUCCESS; 263 | else 264 | return COUCHSTORE_ERROR_ALLOC_FAIL; 265 | } 266 | 267 | LIBCOUCHSTORE_API 268 | couchstore_error_t couchstore_save_document(Db *db, const Doc *doc, DocInfo *info, 269 | couchstore_save_options options) 270 | { 271 | return couchstore_save_documents(db, (Doc**)&doc, (DocInfo**)&info, 1, options); 272 | } 273 | 274 | void _buf_to_docinfo(void *buf, size_t size, DocInfo *docinfo) 275 | { 276 | size_t offset = 0; 277 | 278 | memcpy(&docinfo->rev_seq, (uint8_t*)buf + offset, sizeof(docinfo->rev_seq)); 279 | offset += sizeof(docinfo->rev_seq); 280 | 281 | memcpy(&docinfo->deleted, (uint8_t*)buf + offset, sizeof(docinfo->deleted)); 282 | offset += sizeof(docinfo->deleted); 283 | 284 | memcpy(&docinfo->content_meta, (uint8_t*)buf + offset, sizeof(docinfo->content_meta)); 285 | offset += sizeof(docinfo->content_meta); 286 | 287 | memcpy(&docinfo->rev_meta.size, (uint8_t*)buf + offset, sizeof(docinfo->rev_meta.size)); 288 | offset += sizeof(docinfo->rev_meta.size); 289 | 290 | if (docinfo->rev_meta.size > 0) { 291 | //docinfo->rev_meta.buf = (char *)malloc(docinfo->rev_meta.size); 292 | docinfo->rev_meta.buf = ((char *)docinfo) + sizeof(DocInfo); 293 | memcpy(docinfo->rev_meta.buf, (uint8_t*)buf + offset, docinfo->rev_meta.size); 294 | offset += docinfo->rev_meta.size; 295 | }else{ 296 | docinfo->rev_meta.buf = NULL; 297 | } 298 | } 299 | 300 | LIBCOUCHSTORE_API 301 | couchstore_error_t couchstore_docinfo_by_id(Db *db, const void *id, size_t idlen, DocInfo **pInfo) 302 | { 303 | fdb_doc _doc; 304 | fdb_status status; (void)status; 305 | size_t rev_meta_size; 306 | size_t meta_offset; 307 | 308 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + sizeof(couchstore_content_meta_flags); 309 | 310 | _doc.key = (void *)id; 311 | _doc.keylen = idlen; 312 | _doc.seqnum = SEQNUM_NOT_USED; 313 | _doc.meta = _doc.body = NULL; 314 | 315 | status = fdb_get_metaonly(db->fdb, &_doc); 316 | memcpy(&rev_meta_size, (uint8_t*)_doc.meta + meta_offset, sizeof(size_t)); 317 | 318 | *pInfo = (DocInfo *)malloc(sizeof(DocInfo) + rev_meta_size); 319 | (*pInfo)->id.buf = (char *)id; 320 | (*pInfo)->id.size = idlen; 321 | (*pInfo)->size = _doc.bodylen; 322 | (*pInfo)->bp = _doc.offset; 323 | (*pInfo)->db_seq = _doc.seqnum; 324 | _buf_to_docinfo(_doc.meta, _doc.metalen, (*pInfo)); 325 | 326 | free(_doc.meta); 327 | 328 | return COUCHSTORE_SUCCESS; 329 | } 330 | 331 | LIBCOUCHSTORE_API 332 | couchstore_error_t couchstore_docinfos_by_id(Db *db, const sized_buf ids[], unsigned numDocs, 333 | couchstore_docinfos_options options, couchstore_changes_callback_fn callback, void *ctx) 334 | { 335 | size_t i; 336 | fdb_doc _doc; 337 | fdb_status status; 338 | DocInfo *docinfo; 339 | size_t rev_meta_size, max_meta_size = 256; 340 | size_t meta_offset; 341 | 342 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + sizeof(couchstore_content_meta_flags); 343 | 344 | docinfo = (DocInfo*)malloc(sizeof(DocInfo) + max_meta_size); 345 | 346 | for (i=0;ifdb, &_doc); 353 | assert(status == FDB_RESULT_SUCCESS); 354 | 355 | memcpy(&rev_meta_size, (uint8_t*)_doc.meta + meta_offset, sizeof(size_t)); 356 | if (rev_meta_size > max_meta_size) { 357 | max_meta_size = rev_meta_size; 358 | docinfo = (DocInfo*)realloc(docinfo, sizeof(DocInfo) + max_meta_size); 359 | } 360 | 361 | memset(docinfo, 0, sizeof(DocInfo)); 362 | docinfo->id.buf = ids[i].buf; 363 | docinfo->id.size = ids[i].size; 364 | docinfo->size = _doc.bodylen; 365 | docinfo->bp = _doc.offset; 366 | docinfo->db_seq = _doc.seqnum; 367 | _buf_to_docinfo(_doc.meta, _doc.metalen, docinfo); 368 | free(_doc.meta); 369 | 370 | callback(db, docinfo, ctx); 371 | } 372 | 373 | free(docinfo); 374 | 375 | return COUCHSTORE_SUCCESS; 376 | } 377 | 378 | LIBCOUCHSTORE_API 379 | couchstore_error_t couchstore_docinfos_by_sequence(Db *db, 380 | const uint64_t sequence[], 381 | unsigned numDocs, 382 | couchstore_docinfos_options options, 383 | couchstore_changes_callback_fn callback, 384 | void *ctx) 385 | { 386 | size_t i; 387 | fdb_doc _doc; 388 | fdb_status status; 389 | DocInfo *docinfo; 390 | size_t rev_meta_size, max_meta_size = 256; 391 | size_t meta_offset; 392 | uint8_t keybuf[MAX_KEYLEN]; 393 | 394 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + sizeof(couchstore_content_meta_flags); 395 | 396 | docinfo = (DocInfo*)malloc(sizeof(DocInfo) + max_meta_size); 397 | 398 | for (i=0;ifdb, &_doc); 404 | assert(status == FDB_RESULT_SUCCESS); 405 | 406 | memcpy(&rev_meta_size, (uint8_t*)_doc.meta + meta_offset, sizeof(size_t)); 407 | if (rev_meta_size > max_meta_size) { 408 | max_meta_size = rev_meta_size; 409 | docinfo = (DocInfo*)realloc(docinfo, sizeof(DocInfo) + max_meta_size); 410 | } 411 | 412 | memset(docinfo, 0, sizeof(DocInfo)); 413 | docinfo->id.buf = (char *)keybuf; 414 | docinfo->id.size = _doc.keylen; 415 | docinfo->size = _doc.bodylen; 416 | docinfo->bp = _doc.offset; 417 | docinfo->db_seq = _doc.seqnum; 418 | _buf_to_docinfo(_doc.meta, _doc.metalen, docinfo); 419 | free(_doc.meta); 420 | 421 | callback(db, docinfo, ctx); 422 | } 423 | 424 | free(docinfo); 425 | 426 | return COUCHSTORE_SUCCESS; 427 | } 428 | 429 | LIBCOUCHSTORE_API 430 | couchstore_error_t couchstore_open_document(Db *db, 431 | const void *id, 432 | size_t idlen, 433 | Doc **pDoc, 434 | couchstore_open_options options) 435 | { 436 | fdb_doc _doc; 437 | fdb_status status; 438 | couchstore_error_t ret = COUCHSTORE_SUCCESS; 439 | 440 | memset(&_doc, 0, sizeof(_doc)); 441 | _doc.key = (void *)id; 442 | _doc.keylen = idlen; 443 | _doc.seqnum = SEQNUM_NOT_USED; 444 | _doc.meta = _doc.body = NULL; 445 | 446 | status = fdb_get(db->fdb, &_doc); 447 | if (status != FDB_RESULT_SUCCESS) { 448 | printf("\nget error %.*s\n", (int)idlen, (char*)id); 449 | ret = COUCHSTORE_ERROR_DOC_NOT_FOUND; 450 | } 451 | //assert(status == FDB_RESULT_SUCCESS); 452 | 453 | *pDoc = (Doc *)malloc(sizeof(Doc)); 454 | (*pDoc)->id.buf = (char*)_doc.key; 455 | (*pDoc)->id.size = _doc.keylen; 456 | (*pDoc)->data.buf = (char*)_doc.body; 457 | (*pDoc)->data.size = _doc.bodylen; 458 | 459 | free(_doc.meta); 460 | 461 | return ret; 462 | } 463 | 464 | LIBCOUCHSTORE_API 465 | couchstore_error_t couchstore_walk_id_tree(Db *db, 466 | const sized_buf* startDocID, 467 | couchstore_docinfos_options options, 468 | couchstore_walk_tree_callback_fn callback, 469 | void *ctx) 470 | { 471 | int c_ret = 0; 472 | fdb_iterator *fit = NULL; 473 | fdb_status fs; 474 | fdb_doc *doc; 475 | DocInfo doc_info; 476 | 477 | fs = fdb_iterator_init(db->fdb, &fit, startDocID->buf, startDocID->size, NULL, 0, 0x0); 478 | if (fs != FDB_RESULT_SUCCESS) { 479 | return COUCHSTORE_ERROR_DOC_NOT_FOUND; 480 | } 481 | 482 | do { 483 | doc = NULL; 484 | fs = fdb_iterator_get(fit, &doc); 485 | if (fs == FDB_RESULT_SUCCESS) { 486 | doc_info.id.buf = (char *)doc->key; 487 | doc_info.id.size = doc->keylen; 488 | c_ret = callback(db, 0, &doc_info, 0, NULL, ctx); 489 | fs = fdb_doc_free(doc); 490 | if (c_ret != 0) { 491 | break; 492 | } 493 | } else { 494 | break; 495 | } 496 | } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL); 497 | 498 | fs = fdb_iterator_close(fit); 499 | 500 | return COUCHSTORE_SUCCESS; 501 | } 502 | 503 | LIBCOUCHSTORE_API 504 | void couchstore_free_document(Doc *doc) 505 | { 506 | if (doc->id.buf) free(doc->id.buf); 507 | if (doc->data.buf) free(doc->data.buf); 508 | free(doc); 509 | } 510 | 511 | LIBCOUCHSTORE_API 512 | void couchstore_free_docinfo(DocInfo *docinfo) 513 | { 514 | free(docinfo); 515 | } 516 | 517 | LIBCOUCHSTORE_API 518 | couchstore_error_t couchstore_commit(Db *db) 519 | { 520 | fdb_commit(db->dbfile, FDB_COMMIT_NORMAL); 521 | return COUCHSTORE_SUCCESS; 522 | } 523 | 524 | LIBCOUCHSTORE_API 525 | couchstore_error_t couchstore_compact_db_ex(Db* source, const char* target_filename, 526 | uint64_t flags, FileOpsInterface *ops) 527 | { 528 | char *new_filename = (char *)target_filename; 529 | fdb_compact(source->dbfile, new_filename); 530 | 531 | return COUCHSTORE_SUCCESS; 532 | } 533 | 534 | LIBCOUCHSTORE_API 535 | couchstore_error_t couchstore_compact_db(Db* source, const char* target_filename) 536 | { 537 | return couchstore_compact_db_ex(source, target_filename, 0x0, NULL); 538 | } 539 | 540 | -------------------------------------------------------------------------------- /wrappers/couch_leveldb.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "leveldb/c.h" 9 | #include "libcouchstore/couch_db.h" 10 | 11 | #define METABUF_MAXLEN (256) 12 | extern int64_t DATABUF_MAXLEN; 13 | 14 | struct _db { 15 | leveldb_t *db; 16 | leveldb_options_t *options; 17 | leveldb_readoptions_t *read_options; 18 | leveldb_writeoptions_t *write_options; 19 | char *filename; 20 | }; 21 | 22 | static uint64_t cache_size = 0; 23 | static uint64_t wbs_size = 4*1024*1024; 24 | static int bloom_bits_per_key = 0; 25 | static int compression = 0; 26 | 27 | couchstore_error_t couchstore_set_cache(uint64_t size) 28 | { 29 | cache_size = size; 30 | return COUCHSTORE_SUCCESS; 31 | } 32 | couchstore_error_t couchstore_set_wbs_size(uint64_t size) { 33 | wbs_size = size; 34 | return COUCHSTORE_SUCCESS; 35 | } 36 | couchstore_error_t couchstore_set_bloom(int bits_per_key) { 37 | bloom_bits_per_key = bits_per_key; 38 | return COUCHSTORE_SUCCESS; 39 | } 40 | couchstore_error_t couchstore_set_compression(int opt) { 41 | compression = opt; 42 | return COUCHSTORE_SUCCESS; 43 | } 44 | 45 | LIBCOUCHSTORE_API 46 | couchstore_error_t couchstore_open_db(const char *filename, 47 | couchstore_open_flags flags, 48 | Db **pDb) 49 | { 50 | return couchstore_open_db_ex(filename, flags, 51 | NULL, pDb); 52 | } 53 | 54 | LIBCOUCHSTORE_API 55 | couchstore_error_t couchstore_open_db_ex(const char *filename, 56 | couchstore_open_flags flags, 57 | FileOpsInterface *ops, 58 | Db **pDb) 59 | { 60 | Db *ppdb; 61 | char *err; 62 | leveldb_filterpolicy_t *bloom; 63 | 64 | *pDb = (Db*)malloc(sizeof(Db)); 65 | ppdb = *pDb; 66 | 67 | ppdb->filename = (char*)malloc(strlen(filename)+1); 68 | strcpy(ppdb->filename, filename); 69 | 70 | ppdb->options = leveldb_options_create(); 71 | leveldb_options_set_create_if_missing(ppdb->options, 1); 72 | leveldb_options_set_compression(ppdb->options, compression); 73 | leveldb_options_set_write_buffer_size(ppdb->options, wbs_size); 74 | if (bloom_bits_per_key) { 75 | // set bloom filter 76 | bloom = leveldb_filterpolicy_create_bloom(bloom_bits_per_key); 77 | leveldb_options_set_filter_policy(ppdb->options, bloom); 78 | } 79 | 80 | if (cache_size) { 81 | leveldb_options_set_cache(ppdb->options, 82 | leveldb_cache_create_lru((uint64_t)cache_size)); 83 | } 84 | 85 | leveldb_options_set_max_open_files(ppdb->options, 1000); 86 | ppdb->db = leveldb_open(ppdb->options, ppdb->filename, &err); 87 | 88 | ppdb->read_options = leveldb_readoptions_create(); 89 | ppdb->write_options = leveldb_writeoptions_create(); 90 | leveldb_writeoptions_set_sync(ppdb->write_options, 1); 91 | 92 | return COUCHSTORE_SUCCESS; 93 | } 94 | 95 | couchstore_error_t couchstore_set_sync(Db *db, int sync) 96 | { 97 | leveldb_writeoptions_set_sync(db->write_options, sync); 98 | return COUCHSTORE_SUCCESS; 99 | } 100 | 101 | LIBCOUCHSTORE_API 102 | couchstore_error_t couchstore_close_db(Db *db) 103 | { 104 | leveldb_close(db->db); 105 | free(db->filename); 106 | free(db); 107 | 108 | return COUCHSTORE_SUCCESS; 109 | } 110 | 111 | LIBCOUCHSTORE_API 112 | couchstore_error_t couchstore_db_info(Db *db, DbInfo* info) 113 | { 114 | struct stat filestat; 115 | 116 | info->filename = db->filename; 117 | info->doc_count = 0; 118 | info->deleted_count = 0; 119 | info->header_position = 0; 120 | info->last_sequence = 0; 121 | 122 | stat(db->filename, &filestat); 123 | info->space_used = filestat.st_size; 124 | 125 | return COUCHSTORE_SUCCESS; 126 | } 127 | 128 | size_t _docinfo_to_buf(DocInfo *docinfo, void *buf) 129 | { 130 | // [db_seq,] rev_seq, deleted, content_meta, rev_meta (size), rev_meta (buf) 131 | size_t offset = 0; 132 | 133 | memcpy((uint8_t*)buf + offset, &docinfo->rev_seq, sizeof(docinfo->rev_seq)); 134 | offset += sizeof(docinfo->rev_seq); 135 | 136 | memcpy((uint8_t*)buf + offset, &docinfo->deleted, sizeof(docinfo->deleted)); 137 | offset += sizeof(docinfo->deleted); 138 | 139 | memcpy((uint8_t*)buf + offset, &docinfo->content_meta, 140 | sizeof(docinfo->content_meta)); 141 | offset += sizeof(docinfo->content_meta); 142 | 143 | memcpy((uint8_t*)buf + offset, &docinfo->rev_meta.size, 144 | sizeof(docinfo->rev_meta.size)); 145 | offset += sizeof(docinfo->rev_meta.size); 146 | 147 | if (docinfo->rev_meta.size > 0) { 148 | memcpy((uint8_t*)buf + offset, docinfo->rev_meta.buf, docinfo->rev_meta.size); 149 | offset += docinfo->rev_meta.size; 150 | } 151 | 152 | return offset; 153 | } 154 | 155 | LIBCOUCHSTORE_API 156 | couchstore_error_t couchstore_save_documents(Db *db, Doc* const docs[], DocInfo *infos[], 157 | unsigned numdocs, couchstore_save_options options) 158 | { 159 | unsigned i; 160 | uint16_t metalen; 161 | uint8_t *buf; 162 | char *err = NULL; 163 | leveldb_writebatch_t *wb; 164 | wb = leveldb_writebatch_create(); 165 | buf = (uint8_t*)malloc(sizeof(metalen) + METABUF_MAXLEN + DATABUF_MAXLEN); 166 | 167 | for (i=0;idata.buf, docs[i]->data.size); 171 | 172 | leveldb_writebatch_put(wb, docs[i]->id.buf, docs[i]->id.size, (char*)buf, 173 | sizeof(metalen) + metalen + docs[i]->data.size); 174 | 175 | infos[i]->db_seq = 0; 176 | } 177 | free(buf); 178 | leveldb_write(db->db, db->write_options, wb, &err); 179 | if (err) { 180 | printf("ERR %s\n", err); 181 | } 182 | assert(err == NULL); 183 | leveldb_writebatch_destroy(wb); 184 | 185 | return COUCHSTORE_SUCCESS; 186 | } 187 | 188 | LIBCOUCHSTORE_API 189 | couchstore_error_t couchstore_save_document(Db *db, const Doc *doc, DocInfo *info, 190 | couchstore_save_options options) 191 | { 192 | return couchstore_save_documents(db, (Doc**)&doc, (DocInfo**)&info, 1, options); 193 | } 194 | 195 | void _buf_to_docinfo(void *buf, size_t size, DocInfo *docinfo) 196 | { 197 | size_t offset = 0; 198 | 199 | memcpy(&docinfo->rev_seq, (uint8_t*)buf + offset, sizeof(docinfo->rev_seq)); 200 | offset += sizeof(docinfo->rev_seq); 201 | 202 | memcpy(&docinfo->deleted, (uint8_t*)buf + offset, sizeof(docinfo->deleted)); 203 | offset += sizeof(docinfo->deleted); 204 | 205 | memcpy(&docinfo->content_meta, (uint8_t*)buf + offset, 206 | sizeof(docinfo->content_meta)); 207 | offset += sizeof(docinfo->content_meta); 208 | 209 | memcpy(&docinfo->rev_meta.size, (uint8_t*)buf + offset, 210 | sizeof(docinfo->rev_meta.size)); 211 | offset += sizeof(docinfo->rev_meta.size); 212 | 213 | if (docinfo->rev_meta.size > 0) { 214 | //docinfo->rev_meta.buf = (char *)malloc(docinfo->rev_meta.size); 215 | docinfo->rev_meta.buf = ((char *)docinfo) + sizeof(DocInfo); 216 | memcpy(docinfo->rev_meta.buf, (uint8_t*)buf + offset, docinfo->rev_meta.size); 217 | offset += docinfo->rev_meta.size; 218 | }else{ 219 | docinfo->rev_meta.buf = NULL; 220 | } 221 | } 222 | 223 | LIBCOUCHSTORE_API 224 | couchstore_error_t couchstore_docinfo_by_id(Db *db, const void *id, size_t idlen, DocInfo **pInfo) 225 | { 226 | char *err; 227 | void *value; 228 | size_t valuelen; 229 | size_t rev_meta_size; 230 | size_t meta_offset; 231 | 232 | value = leveldb_get(db->db, db->read_options, (char*)id, idlen, &valuelen, &err); 233 | 234 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + sizeof(couchstore_content_meta_flags); 235 | memcpy(&rev_meta_size, (uint8_t*)value + sizeof(uint16_t) + meta_offset, 236 | sizeof(size_t)); 237 | 238 | *pInfo = (DocInfo *)malloc(sizeof(DocInfo) + rev_meta_size); 239 | (*pInfo)->id.buf = (char *)id; 240 | (*pInfo)->id.size = idlen; 241 | (*pInfo)->size = idlen + valuelen; 242 | (*pInfo)->bp = 0; 243 | (*pInfo)->db_seq = 0; 244 | _buf_to_docinfo((uint8_t*)value + sizeof(uint16_t), valuelen, (*pInfo)); 245 | 246 | free(value); 247 | 248 | return COUCHSTORE_SUCCESS; 249 | } 250 | 251 | LIBCOUCHSTORE_API 252 | couchstore_error_t couchstore_docinfos_by_id(Db *db, const sized_buf ids[], unsigned numDocs, 253 | couchstore_docinfos_options options, couchstore_changes_callback_fn callback, void *ctx) 254 | { 255 | size_t i; 256 | DocInfo *docinfo; 257 | char *err; 258 | void *value; 259 | size_t valuelen; 260 | size_t rev_meta_size, max_meta_size = 256; 261 | size_t meta_offset; 262 | 263 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + 264 | sizeof(couchstore_content_meta_flags); 265 | docinfo = (DocInfo*)malloc(sizeof(DocInfo) + max_meta_size); 266 | 267 | for (i=0;idb, db->read_options, 269 | ids[i].buf, ids[i].size, 270 | &valuelen, &err); 271 | 272 | memcpy(&rev_meta_size, (uint8_t*)value + sizeof(uint16_t) + meta_offset, 273 | sizeof(size_t)); 274 | if (rev_meta_size > max_meta_size) { 275 | max_meta_size = rev_meta_size; 276 | docinfo = (DocInfo*)realloc(docinfo, sizeof(DocInfo) + max_meta_size); 277 | } 278 | 279 | memset(docinfo, 0, sizeof(DocInfo)); 280 | docinfo->id.buf = ids[i].buf; 281 | docinfo->id.size = ids[i].size; 282 | docinfo->size = ids[i].size + valuelen; 283 | docinfo->bp = 0; 284 | docinfo->db_seq = 0; 285 | _buf_to_docinfo((uint8_t*)value + sizeof(uint16_t), valuelen, docinfo); 286 | free(value); 287 | 288 | callback(db, docinfo, ctx); 289 | } 290 | 291 | free(docinfo); 292 | 293 | return COUCHSTORE_SUCCESS; 294 | } 295 | 296 | LIBCOUCHSTORE_API 297 | couchstore_error_t couchstore_docinfos_by_sequence(Db *db, 298 | const uint64_t sequence[], 299 | unsigned numDocs, 300 | couchstore_docinfos_options options, 301 | couchstore_changes_callback_fn callback, 302 | void *ctx) 303 | { 304 | // do nothing 305 | 306 | return COUCHSTORE_SUCCESS; 307 | } 308 | 309 | LIBCOUCHSTORE_API 310 | couchstore_error_t couchstore_open_document(Db *db, 311 | const void *id, 312 | size_t idlen, 313 | Doc **pDoc, 314 | couchstore_open_options options) 315 | { 316 | char *err = NULL; 317 | void *value; 318 | size_t valuelen; 319 | 320 | value = leveldb_get(db->db, db->read_options, (char*)id, idlen, &valuelen, &err); 321 | if (err) { 322 | printf("ERR %s\n", err); 323 | } 324 | assert(err == NULL); 325 | 326 | *pDoc = (Doc *)malloc(sizeof(Doc)); 327 | (*pDoc)->id.buf = (char*)id; 328 | (*pDoc)->id.size = idlen; 329 | (*pDoc)->data.buf = (char*)value; 330 | (*pDoc)->data.size = valuelen; 331 | 332 | return COUCHSTORE_SUCCESS; 333 | } 334 | 335 | LIBCOUCHSTORE_API 336 | couchstore_error_t couchstore_walk_id_tree(Db *db, 337 | const sized_buf* startDocID, 338 | couchstore_docinfos_options options, 339 | couchstore_walk_tree_callback_fn callback, 340 | void *ctx) 341 | { 342 | int c_ret = 0; 343 | leveldb_iterator_t *lit; 344 | leveldb_readoptions_t *read_options; 345 | DocInfo doc_info; 346 | Doc doc; 347 | char *keyptr, *valueptr; 348 | size_t valuelen; 349 | 350 | read_options = leveldb_readoptions_create(); 351 | lit = leveldb_create_iterator(db->db, read_options); 352 | leveldb_iter_seek(lit, startDocID->buf, startDocID->size); 353 | 354 | while (leveldb_iter_valid(lit)) { 355 | keyptr = (char*)leveldb_iter_key(lit, &doc_info.id.size); 356 | valueptr = (char*)leveldb_iter_value(lit, &valuelen); 357 | 358 | doc_info.id.buf = (char *)malloc(doc_info.id.size); 359 | memcpy(doc_info.id.buf, keyptr, doc_info.id.size); 360 | doc.data.buf = (char *)malloc(valuelen); 361 | memcpy(doc.data.buf, valueptr, valuelen); 362 | 363 | c_ret = callback(db, 0, &doc_info, 0, NULL, ctx); 364 | 365 | free(doc_info.id.buf); 366 | free(doc.data.buf); 367 | 368 | if (c_ret != 0) { 369 | break; 370 | } 371 | 372 | leveldb_iter_next(lit); 373 | } 374 | 375 | leveldb_iter_destroy(lit); 376 | 377 | return COUCHSTORE_SUCCESS; 378 | } 379 | 380 | LIBCOUCHSTORE_API 381 | void couchstore_free_document(Doc *doc) 382 | { 383 | if (doc->id.buf) free(doc->id.buf); 384 | if (doc->data.buf) free(doc->data.buf); 385 | free(doc); 386 | } 387 | 388 | 389 | LIBCOUCHSTORE_API 390 | void couchstore_free_docinfo(DocInfo *docinfo) 391 | { 392 | //free(docinfo->rev_meta.buf); 393 | free(docinfo); 394 | } 395 | 396 | LIBCOUCHSTORE_API 397 | couchstore_error_t couchstore_commit(Db *db) 398 | { 399 | // do nothing (automatically performed at the end of each write batch) 400 | 401 | return COUCHSTORE_SUCCESS; 402 | } 403 | 404 | LIBCOUCHSTORE_API 405 | couchstore_error_t couchstore_compact_db_ex(Db* source, const char* target_filename, 406 | uint64_t flags, FileOpsInterface *ops) 407 | { 408 | return COUCHSTORE_SUCCESS; 409 | } 410 | 411 | LIBCOUCHSTORE_API 412 | couchstore_error_t couchstore_compact_db(Db* source, const char* target_filename) 413 | { 414 | return couchstore_compact_db_ex(source, target_filename, 0x0, NULL); 415 | } 416 | 417 | -------------------------------------------------------------------------------- /wrappers/couch_rocksdb.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "rocksdb/cache.h" 9 | #include "rocksdb/db.h" 10 | #include "rocksdb/table.h" 11 | #include "rocksdb/filter_policy.h" 12 | #include "rocksdb/table.h" 13 | #include "libcouchstore/couch_db.h" 14 | 15 | #define METABUF_MAXLEN (256) 16 | extern int64_t DATABUF_MAXLEN; 17 | 18 | struct _db { 19 | rocksdb::DB* db; 20 | rocksdb::Options *options; 21 | rocksdb::WriteOptions *write_options; 22 | rocksdb::ReadOptions *read_options; 23 | std::string *filename; 24 | }; 25 | 26 | static uint64_t cache_size = 0; 27 | static uint64_t wbs_size = 4*1024*1024; 28 | static int bloom_bits_per_key = 0; 29 | static int compaction_style = 0; 30 | static int compression = 0; 31 | 32 | couchstore_error_t couchstore_set_cache(uint64_t size) 33 | { 34 | cache_size = size; 35 | return COUCHSTORE_SUCCESS; 36 | } 37 | couchstore_error_t couchstore_set_wbs_size(uint64_t size) { 38 | wbs_size = size; 39 | return COUCHSTORE_SUCCESS; 40 | } 41 | couchstore_error_t couchstore_set_bloom(int bits_per_key) { 42 | bloom_bits_per_key = bits_per_key; 43 | return COUCHSTORE_SUCCESS; 44 | } 45 | couchstore_error_t couchstore_set_compaction_style(int style) { 46 | compaction_style = style; 47 | return COUCHSTORE_SUCCESS; 48 | } 49 | couchstore_error_t couchstore_set_compression(int opt) { 50 | compression = opt; 51 | return COUCHSTORE_SUCCESS; 52 | } 53 | 54 | 55 | LIBCOUCHSTORE_API 56 | couchstore_error_t couchstore_open_db(const char *filename, 57 | couchstore_open_flags flags, 58 | Db **pDb) 59 | { 60 | return couchstore_open_db_ex(filename, flags, 61 | NULL, pDb); 62 | } 63 | 64 | LIBCOUCHSTORE_API 65 | couchstore_error_t couchstore_open_db_ex(const char *filename, 66 | couchstore_open_flags flags, 67 | FileOpsInterface *ops, 68 | Db **pDb) 69 | { 70 | Db *ppdb; 71 | 72 | *pDb = (Db*)malloc(sizeof(Db)); 73 | ppdb = *pDb; 74 | 75 | ppdb->filename = new std::string(filename); 76 | 77 | ppdb->options = new rocksdb::Options(); 78 | ppdb->options->create_if_missing = true; 79 | if (!compression) { 80 | ppdb->options->compression = rocksdb::kNoCompression; 81 | } 82 | 83 | ppdb->options->max_background_compactions = 8; 84 | ppdb->options->max_background_flushes = 8; 85 | ppdb->options->max_write_buffer_number = 8; 86 | ppdb->options->write_buffer_size = wbs_size; 87 | ppdb->options->compaction_style = rocksdb::CompactionStyle(compaction_style); 88 | 89 | if (cache_size || bloom_bits_per_key) { 90 | rocksdb::BlockBasedTableOptions table_options; 91 | if (cache_size) { 92 | table_options.block_cache = rocksdb::NewLRUCache(cache_size); 93 | } 94 | if (bloom_bits_per_key) { 95 | table_options.filter_policy.reset( 96 | rocksdb::NewBloomFilterPolicy(bloom_bits_per_key)); 97 | } 98 | ppdb->options->table_factory.reset( 99 | rocksdb::NewBlockBasedTableFactory(table_options)); 100 | } 101 | ppdb->options->max_open_files = 1000; 102 | 103 | rocksdb::Status status = rocksdb::DB::Open( 104 | *ppdb->options, *ppdb->filename, &ppdb->db); 105 | 106 | ppdb->read_options = new rocksdb::ReadOptions(); 107 | ppdb->write_options = new rocksdb::WriteOptions(); 108 | ppdb->write_options->sync = true; 109 | 110 | return COUCHSTORE_SUCCESS; 111 | } 112 | 113 | couchstore_error_t couchstore_set_sync(Db *db, int sync) 114 | { 115 | db->write_options->sync = sync ? true : false; 116 | return COUCHSTORE_SUCCESS; 117 | } 118 | 119 | couchstore_error_t couchstore_disable_auto_compaction(Db *db, int cpt) 120 | { 121 | rocksdb::Status status; 122 | db->options->disable_auto_compactions = cpt; 123 | delete db->db; 124 | status = rocksdb::DB::Open(*db->options, *db->filename, &db->db); 125 | return COUCHSTORE_SUCCESS; 126 | } 127 | 128 | LIBCOUCHSTORE_API 129 | couchstore_error_t couchstore_close_db(Db *db) 130 | { 131 | delete db->db; 132 | delete db->options; 133 | delete db->write_options; 134 | delete db->read_options; 135 | delete db->filename; 136 | free(db); 137 | 138 | return COUCHSTORE_SUCCESS; 139 | } 140 | 141 | LIBCOUCHSTORE_API 142 | couchstore_error_t couchstore_db_info(Db *db, DbInfo* info) 143 | { 144 | struct stat filestat; 145 | 146 | info->filename = db->filename->c_str(); 147 | info->doc_count = 0; 148 | info->deleted_count = 0; 149 | info->header_position = 0; 150 | info->last_sequence = 0; 151 | 152 | stat(db->filename->c_str(), &filestat); 153 | info->space_used = filestat.st_size; 154 | 155 | return COUCHSTORE_SUCCESS; 156 | } 157 | 158 | size_t _docinfo_to_buf(DocInfo *docinfo, void *buf) 159 | { 160 | // [db_seq,] rev_seq, deleted, content_meta, rev_meta (size), rev_meta (buf) 161 | size_t offset = 0; 162 | 163 | memcpy((uint8_t*)buf + offset, &docinfo->rev_seq, sizeof(docinfo->rev_seq)); 164 | offset += sizeof(docinfo->rev_seq); 165 | 166 | memcpy((uint8_t*)buf + offset, &docinfo->deleted, sizeof(docinfo->deleted)); 167 | offset += sizeof(docinfo->deleted); 168 | 169 | memcpy((uint8_t*)buf + offset, &docinfo->content_meta, 170 | sizeof(docinfo->content_meta)); 171 | offset += sizeof(docinfo->content_meta); 172 | 173 | memcpy((uint8_t*)buf + offset, &docinfo->rev_meta.size, 174 | sizeof(docinfo->rev_meta.size)); 175 | offset += sizeof(docinfo->rev_meta.size); 176 | 177 | if (docinfo->rev_meta.size > 0) { 178 | memcpy((uint8_t*)buf + offset, docinfo->rev_meta.buf, docinfo->rev_meta.size); 179 | offset += docinfo->rev_meta.size; 180 | } 181 | 182 | return offset; 183 | } 184 | 185 | LIBCOUCHSTORE_API 186 | couchstore_error_t couchstore_save_documents(Db *db, Doc* const docs[], DocInfo *infos[], 187 | unsigned numdocs, couchstore_save_options options) 188 | { 189 | unsigned i; 190 | uint16_t metalen; 191 | uint8_t *buf; 192 | rocksdb::Status status; 193 | rocksdb::WriteBatch wb; 194 | 195 | buf = (uint8_t*)malloc(sizeof(metalen) + METABUF_MAXLEN + DATABUF_MAXLEN); 196 | 197 | for (i=0;idata.buf, docs[i]->data.size); 201 | 202 | wb.Put(rocksdb::Slice(docs[i]->id.buf, docs[i]->id.size), 203 | rocksdb::Slice((char*)buf, 204 | sizeof(metalen) + metalen + docs[i]->data.size)); 205 | 206 | infos[i]->db_seq = 0; 207 | } 208 | free(buf); 209 | status = db->db->Write(*db->write_options, &wb); 210 | if (!status.ok()) { 211 | printf("ERR %s\n", status.ToString().c_str()); 212 | } 213 | assert(status.ok()); 214 | 215 | return COUCHSTORE_SUCCESS; 216 | } 217 | 218 | LIBCOUCHSTORE_API 219 | couchstore_error_t couchstore_save_document(Db *db, const Doc *doc, DocInfo *info, 220 | couchstore_save_options options) 221 | { 222 | return couchstore_save_documents(db, (Doc**)&doc, (DocInfo**)&info, 1, options); 223 | } 224 | 225 | void _buf_to_docinfo(void *buf, size_t size, DocInfo *docinfo) 226 | { 227 | size_t offset = 0; 228 | 229 | memcpy(&docinfo->rev_seq, (uint8_t*)buf + offset, sizeof(docinfo->rev_seq)); 230 | offset += sizeof(docinfo->rev_seq); 231 | 232 | memcpy(&docinfo->deleted, (uint8_t*)buf + offset, sizeof(docinfo->deleted)); 233 | offset += sizeof(docinfo->deleted); 234 | 235 | memcpy(&docinfo->content_meta, (uint8_t*)buf + offset, 236 | sizeof(docinfo->content_meta)); 237 | offset += sizeof(docinfo->content_meta); 238 | 239 | memcpy(&docinfo->rev_meta.size, (uint8_t*)buf + offset, 240 | sizeof(docinfo->rev_meta.size)); 241 | offset += sizeof(docinfo->rev_meta.size); 242 | 243 | if (docinfo->rev_meta.size > 0) { 244 | //docinfo->rev_meta.buf = (char *)malloc(docinfo->rev_meta.size); 245 | docinfo->rev_meta.buf = ((char *)docinfo) + sizeof(DocInfo); 246 | memcpy(docinfo->rev_meta.buf, (uint8_t*)buf + offset, docinfo->rev_meta.size); 247 | offset += docinfo->rev_meta.size; 248 | }else{ 249 | docinfo->rev_meta.buf = NULL; 250 | } 251 | } 252 | 253 | LIBCOUCHSTORE_API 254 | couchstore_error_t couchstore_docinfo_by_id(Db *db, const void *id, size_t idlen, DocInfo **pInfo) 255 | { 256 | rocksdb::Status status; 257 | std::string *value = NULL; 258 | size_t valuelen= 0; 259 | size_t rev_meta_size; 260 | size_t meta_offset; 261 | 262 | status = db->db->Get(*db->read_options, rocksdb::Slice((char*)id, idlen), 263 | value); 264 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + 265 | sizeof(couchstore_content_meta_flags); 266 | memcpy(&rev_meta_size, (uint8_t*)value + sizeof(uint16_t) + meta_offset, 267 | sizeof(size_t)); 268 | 269 | *pInfo = (DocInfo *)malloc(sizeof(DocInfo) + rev_meta_size); 270 | (*pInfo)->id.buf = (char *)id; 271 | (*pInfo)->id.size = idlen; 272 | (*pInfo)->size = idlen + valuelen; 273 | (*pInfo)->bp = 0; 274 | (*pInfo)->db_seq = 0; 275 | _buf_to_docinfo((uint8_t*)value + sizeof(uint16_t), valuelen, (*pInfo)); 276 | 277 | free(value); 278 | 279 | return COUCHSTORE_SUCCESS; 280 | } 281 | 282 | LIBCOUCHSTORE_API 283 | couchstore_error_t couchstore_docinfos_by_id(Db *db, const sized_buf ids[], unsigned numDocs, 284 | couchstore_docinfos_options options, couchstore_changes_callback_fn callback, void *ctx) 285 | { 286 | size_t i; 287 | DocInfo *docinfo; 288 | rocksdb::Status status; 289 | std::string *value = NULL; 290 | size_t valuelen = 0; 291 | size_t rev_meta_size, max_meta_size = 256; 292 | size_t meta_offset; 293 | 294 | meta_offset = sizeof(uint64_t)*1 + sizeof(int) + sizeof(couchstore_content_meta_flags); 295 | docinfo = (DocInfo*)malloc(sizeof(DocInfo) + max_meta_size); 296 | 297 | for (i=0;idb->Get(*db->read_options, 299 | rocksdb::Slice(ids[i].buf, ids[i].size), value); 300 | 301 | memcpy(&rev_meta_size, (uint8_t*)value + sizeof(uint16_t) + meta_offset, 302 | sizeof(size_t)); 303 | if (rev_meta_size > max_meta_size) { 304 | max_meta_size = rev_meta_size; 305 | docinfo = (DocInfo*)realloc(docinfo, sizeof(DocInfo) + max_meta_size); 306 | } 307 | 308 | memset(docinfo, 0, sizeof(DocInfo)); 309 | docinfo->id.buf = ids[i].buf; 310 | docinfo->id.size = ids[i].size; 311 | docinfo->size = ids[i].size + valuelen; 312 | docinfo->bp = 0; 313 | docinfo->db_seq = 0; 314 | _buf_to_docinfo((uint8_t*)value + sizeof(uint16_t), valuelen, docinfo); 315 | free(value); 316 | 317 | callback(db, docinfo, ctx); 318 | } 319 | 320 | free(docinfo); 321 | 322 | return COUCHSTORE_SUCCESS; 323 | } 324 | 325 | LIBCOUCHSTORE_API 326 | couchstore_error_t couchstore_docinfos_by_sequence(Db *db, 327 | const uint64_t sequence[], 328 | unsigned numDocs, 329 | couchstore_docinfos_options options, 330 | couchstore_changes_callback_fn callback, 331 | void *ctx) 332 | { 333 | // do nothing 334 | 335 | return COUCHSTORE_SUCCESS; 336 | } 337 | 338 | LIBCOUCHSTORE_API 339 | couchstore_error_t couchstore_open_document(Db *db, 340 | const void *id, 341 | size_t idlen, 342 | Doc **pDoc, 343 | couchstore_open_options options) 344 | { 345 | rocksdb::Status status; 346 | std::string tmp; 347 | char* value = NULL; 348 | size_t valuelen; 349 | 350 | status = db->db->Get(*db->read_options, rocksdb::Slice((char*)id, idlen), 351 | &tmp); 352 | if (status.ok()) { 353 | valuelen = tmp.size(); 354 | value = strdup(tmp.c_str()); 355 | } else { 356 | valuelen = 0; 357 | if (!status.IsNotFound()) { 358 | printf("ERR %s\n", status.ToString().c_str()); 359 | } 360 | } 361 | assert(status.ok()); 362 | 363 | *pDoc = (Doc *)malloc(sizeof(Doc)); 364 | (*pDoc)->id.buf = (char*)id; 365 | (*pDoc)->id.size = idlen; 366 | (*pDoc)->data.buf = (char*)value; 367 | (*pDoc)->data.size = valuelen; 368 | 369 | return COUCHSTORE_SUCCESS; 370 | } 371 | 372 | LIBCOUCHSTORE_API 373 | couchstore_error_t couchstore_walk_id_tree(Db *db, 374 | const sized_buf* startDocID, 375 | couchstore_docinfos_options options, 376 | couchstore_walk_tree_callback_fn callback, 377 | void *ctx) 378 | { 379 | int c_ret = 0; 380 | rocksdb::Iterator* rit = db->db->NewIterator(rocksdb::ReadOptions()); 381 | rocksdb::Slice startkey = rocksdb::Slice(startDocID->buf, startDocID->size); 382 | rocksdb::Slice keyptr, valueptr; 383 | DocInfo doc_info; 384 | Doc doc; 385 | 386 | rit->Seek(startkey); 387 | 388 | while (rit->Valid()) { 389 | keyptr = rit->key(); 390 | valueptr = rit->value(); 391 | 392 | doc_info.id.buf = (char *)malloc(keyptr.size()); 393 | memcpy(doc_info.id.buf, keyptr.data(), keyptr.size()); 394 | doc.data.buf = (char *)malloc(valueptr.size()); 395 | memcpy(doc.data.buf, valueptr.data(), valueptr.size()); 396 | 397 | c_ret = callback(db, 0, &doc_info, 0, NULL, ctx); 398 | 399 | free(doc_info.id.buf); 400 | free(doc.data.buf); 401 | 402 | if (c_ret != 0) { 403 | break; 404 | } 405 | 406 | rit->Next(); 407 | } 408 | 409 | delete rit; 410 | 411 | return COUCHSTORE_SUCCESS; 412 | } 413 | 414 | LIBCOUCHSTORE_API 415 | void couchstore_free_document(Doc *doc) 416 | { 417 | if (doc->id.buf) free(doc->id.buf); 418 | if (doc->data.buf) free(doc->data.buf); 419 | free(doc); 420 | } 421 | 422 | 423 | LIBCOUCHSTORE_API 424 | void couchstore_free_docinfo(DocInfo *docinfo) 425 | { 426 | //free(docinfo->rev_meta.buf); 427 | free(docinfo); 428 | } 429 | 430 | LIBCOUCHSTORE_API 431 | couchstore_error_t couchstore_commit(Db *db) 432 | { 433 | // do nothing (automatically performed at the end of each write batch) 434 | 435 | return COUCHSTORE_SUCCESS; 436 | } 437 | 438 | LIBCOUCHSTORE_API 439 | couchstore_error_t couchstore_compact_db_ex(Db* source, const char* target_filename, 440 | uint64_t flags, FileOpsInterface *ops) 441 | { 442 | return COUCHSTORE_SUCCESS; 443 | } 444 | 445 | LIBCOUCHSTORE_API 446 | couchstore_error_t couchstore_compact_db(Db* source, const char* target_filename) 447 | { 448 | return couchstore_compact_db_ex(source, target_filename, 0x0, NULL); 449 | } 450 | 451 | -------------------------------------------------------------------------------- /wrappers/couch_wt.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #ifndef __STDC_FORMAT_MACROS 10 | #define __STDC_FORMAT_MACROS 11 | #endif 12 | 13 | #include 14 | 15 | #include "wiredtiger.h" 16 | #include "libcouchstore/couch_db.h" 17 | #include "memleak.h" 18 | 19 | #define METABUF_MAXLEN (256) 20 | extern int64_t DATABUF_MAXLEN; 21 | 22 | struct _db { 23 | WT_CURSOR *cursor; 24 | WT_SESSION *session; 25 | char *filename; 26 | int sync; 27 | }; 28 | 29 | static WT_CONNECTION *conn = NULL; 30 | static uint64_t cache_size = 0; 31 | static int indexing_type = 0; 32 | static size_t c_period = 15; 33 | static int compression = 0; 34 | static int split_pct = 0; 35 | static size_t leaf_page_size, int_page_size = 0; 36 | 37 | couchstore_error_t couchstore_set_cache(uint64_t size) { 38 | cache_size = size; 39 | return COUCHSTORE_SUCCESS; 40 | } 41 | couchstore_error_t couchstore_set_idx_type(int type) { 42 | indexing_type = type; 43 | return COUCHSTORE_SUCCESS; 44 | } 45 | couchstore_error_t couchstore_set_chk_period(size_t seconds) { 46 | c_period = seconds; 47 | return COUCHSTORE_SUCCESS; 48 | } 49 | couchstore_error_t couchstore_set_compression(int opt) { 50 | compression = opt; 51 | return COUCHSTORE_SUCCESS; 52 | } 53 | couchstore_error_t couchstore_set_split_pct(int pct) { 54 | split_pct = pct; 55 | return COUCHSTORE_SUCCESS; 56 | } 57 | couchstore_error_t couchstore_set_page_size(size_t leaf_pg_size, size_t int_pg_size) { 58 | leaf_page_size = leaf_pg_size; 59 | int_page_size = int_pg_size; 60 | return COUCHSTORE_SUCCESS; 61 | } 62 | 63 | #ifdef PRIu64 64 | #define _F64 PRIu64 65 | #else 66 | #define _F64 "llu" 67 | #endif 68 | couchstore_error_t couchstore_open_conn(const char *filename) 69 | { 70 | int fd; 71 | int ret; 72 | char config[256]; 73 | 74 | sprintf(config, "create," 75 | "log=(enabled)," 76 | "checkpoint=(wait=%d)," 77 | "cache_size=%" _F64, 78 | (int)c_period, 79 | cache_size); 80 | if (compression) { 81 | strcat(config, ",extensions=[libwiredtiger_snappy.so]"); 82 | } 83 | // create directory if not exist 84 | fd = open(filename, O_RDONLY, 0666); 85 | if (fd == -1) { 86 | // create 87 | char cmd[256]; 88 | 89 | sprintf(cmd, "mkdir %s\n", filename); 90 | ret = system(cmd); 91 | } 92 | 93 | ret = wiredtiger_open(filename, NULL, config, &conn); 94 | assert(ret == 0); 95 | 96 | return COUCHSTORE_SUCCESS; 97 | } 98 | 99 | couchstore_error_t couchstore_close_conn() 100 | { 101 | conn->close(conn, NULL); 102 | return COUCHSTORE_SUCCESS; 103 | } 104 | 105 | LIBCOUCHSTORE_API 106 | couchstore_error_t couchstore_open_db(const char *filename, 107 | couchstore_open_flags flags, 108 | Db **pDb) 109 | { 110 | return couchstore_open_db_ex(filename, flags, 111 | NULL, pDb); 112 | } 113 | 114 | LIBCOUCHSTORE_API 115 | couchstore_error_t couchstore_open_db_ex(const char *filename, 116 | couchstore_open_flags flags, 117 | FileOpsInterface *ops, 118 | Db **pDb) 119 | { 120 | int i, len, ret; 121 | Db *ppdb; 122 | char fileonly[256]; 123 | char table_name[256]; 124 | char table_config[256]; 125 | 126 | assert(conn); 127 | 128 | *pDb = (Db*)malloc(sizeof(Db)); 129 | ppdb = *pDb; 130 | 131 | ppdb->filename = (char*)malloc(strlen(filename)+1); 132 | strcpy(ppdb->filename, filename); 133 | 134 | // take filename only (discard directory path) 135 | len = strlen(filename); 136 | for (i=len-1; i>=0; --i) { 137 | if (filename[i] == '/') { 138 | strcpy(fileonly, filename + (i+1)); 139 | break; 140 | } 141 | if (i == 0) { // there is no directory path, filename only 142 | strcpy(fileonly, filename); 143 | } 144 | } 145 | 146 | sprintf(table_name, "table:%s", fileonly); 147 | sprintf(table_config, " "); 148 | if (indexing_type == 1) { 149 | // lsm-tree 150 | sprintf(table_config, 151 | "split_pct=%d,leaf_item_max=1KB," 152 | "type=lsm,internal_page_max=%zdKB,leaf_page_max=%zdKB," 153 | "lsm=(chunk_size=4MB," 154 | "bloom_config=(leaf_page_max=%zdMB))" 155 | , split_pct, int_page_size, leaf_page_size, leaf_page_size); 156 | } else { 157 | sprintf(table_config, 158 | "split_pct=%d,leaf_item_max=1KB," 159 | "internal_page_max=%zdKB,leaf_page_max=%zdKB" 160 | ,split_pct, int_page_size, leaf_page_size); 161 | } 162 | if (compression) { 163 | // note: compression works for C-style string format (S) only. 164 | strcat(table_config, ",block_compressor=snappy"); 165 | } 166 | strcat(table_config, ",key_format=S,value_format=S"); 167 | 168 | ret = conn->open_session(conn, NULL, NULL, &ppdb->session); 169 | if (ret != 0) { 170 | fprintf(stderr, "Error: %s\n", 171 | wiredtiger_strerror(ret)); 172 | assert(0); 173 | } 174 | ppdb->session->create(ppdb->session, table_name, table_config); 175 | ppdb->session->open_cursor(ppdb->session, table_name, NULL, NULL, &ppdb->cursor); 176 | ppdb->sync = (flags & 0x10)?(1):(0); 177 | 178 | return COUCHSTORE_SUCCESS; 179 | } 180 | 181 | LIBCOUCHSTORE_API 182 | couchstore_error_t couchstore_close_db(Db *db) 183 | { 184 | db->cursor->close(db->cursor); 185 | db->session->close(db->session, NULL); 186 | free(db->filename); 187 | free(db); 188 | 189 | return COUCHSTORE_SUCCESS; 190 | } 191 | 192 | LIBCOUCHSTORE_API 193 | couchstore_error_t couchstore_db_info(Db *db, DbInfo* info) 194 | { 195 | struct stat filestat; 196 | 197 | info->filename = db->filename; 198 | info->doc_count = 0; 199 | info->deleted_count = 0; 200 | info->header_position = 0; 201 | info->last_sequence = 0; 202 | 203 | stat(db->filename, &filestat); 204 | info->space_used = filestat.st_size; 205 | 206 | return COUCHSTORE_SUCCESS; 207 | } 208 | 209 | size_t _docinfo_to_buf(DocInfo *docinfo, void *buf) 210 | { 211 | // [db_seq,] rev_seq, deleted, content_meta, rev_meta (size), rev_meta (buf) 212 | size_t offset = 0; 213 | 214 | memcpy((uint8_t*)buf + offset, &docinfo->rev_seq, sizeof(docinfo->rev_seq)); 215 | offset += sizeof(docinfo->rev_seq); 216 | 217 | memcpy((uint8_t*)buf + offset, &docinfo->deleted, sizeof(docinfo->deleted)); 218 | offset += sizeof(docinfo->deleted); 219 | 220 | memcpy((uint8_t*)buf + offset, &docinfo->content_meta, 221 | sizeof(docinfo->content_meta)); 222 | offset += sizeof(docinfo->content_meta); 223 | 224 | memcpy((uint8_t*)buf + offset, &docinfo->rev_meta.size, 225 | sizeof(docinfo->rev_meta.size)); 226 | offset += sizeof(docinfo->rev_meta.size); 227 | 228 | if (docinfo->rev_meta.size > 0) { 229 | memcpy((uint8_t*)buf + offset, docinfo->rev_meta.buf, docinfo->rev_meta.size); 230 | offset += docinfo->rev_meta.size; 231 | } 232 | 233 | return offset; 234 | } 235 | 236 | LIBCOUCHSTORE_API 237 | couchstore_error_t couchstore_save_documents(Db *db, Doc* const docs[], DocInfo *infos[], 238 | unsigned numdocs, couchstore_save_options options) 239 | { 240 | int ret; 241 | unsigned i; 242 | uint16_t metalen; 243 | uint8_t *buf; 244 | 245 | if (db->sync) { 246 | ret = db->session->begin_transaction(db->session, "sync"); 247 | assert(ret==0); 248 | } 249 | 250 | buf = (uint8_t*)malloc(sizeof(metalen) + METABUF_MAXLEN + DATABUF_MAXLEN); 251 | 252 | for (i=0;icursor->set_key(db->cursor, docs[i]->id.buf); 254 | 255 | // Note: as WiredTiger uses C-style string, NULL character 256 | // should not be used in value. So we don't append metadata 257 | // here since it contains a number of zeros. 258 | /* 259 | metalen = _docinfo_to_buf(infos[i], buf + sizeof(metalen)); 260 | memcpy(buf, &metalen, sizeof(metalen)); 261 | */ 262 | memcpy(buf, docs[i]->data.buf, docs[i]->data.size); 263 | db->cursor->set_value(db->cursor, buf); 264 | 265 | ret = db->cursor->insert(db->cursor); 266 | if (ret != 0) { 267 | printf("ERR %d\n", ret); 268 | } 269 | 270 | infos[i]->db_seq = 0; 271 | } 272 | free(buf); 273 | 274 | if (db->sync) { 275 | ret = db->session->commit_transaction(db->session, NULL); 276 | assert(ret==0); 277 | } 278 | 279 | return COUCHSTORE_SUCCESS; 280 | } 281 | 282 | LIBCOUCHSTORE_API 283 | couchstore_error_t couchstore_save_document(Db *db, const Doc *doc, DocInfo *info, 284 | couchstore_save_options options) 285 | { 286 | return couchstore_save_documents(db, (Doc**)&doc, (DocInfo**)&info, 1, options); 287 | } 288 | 289 | void _buf_to_docinfo(void *buf, size_t size, DocInfo *docinfo) 290 | { 291 | size_t offset = 0; 292 | 293 | memcpy(&docinfo->rev_seq, (uint8_t*)buf + offset, sizeof(docinfo->rev_seq)); 294 | offset += sizeof(docinfo->rev_seq); 295 | 296 | memcpy(&docinfo->deleted, (uint8_t*)buf + offset, sizeof(docinfo->deleted)); 297 | offset += sizeof(docinfo->deleted); 298 | 299 | memcpy(&docinfo->content_meta, (uint8_t*)buf + offset, 300 | sizeof(docinfo->content_meta)); 301 | offset += sizeof(docinfo->content_meta); 302 | 303 | memcpy(&docinfo->rev_meta.size, (uint8_t*)buf + offset, 304 | sizeof(docinfo->rev_meta.size)); 305 | offset += sizeof(docinfo->rev_meta.size); 306 | 307 | if (docinfo->rev_meta.size > 0) { 308 | //docinfo->rev_meta.buf = (char *)malloc(docinfo->rev_meta.size); 309 | docinfo->rev_meta.buf = ((char *)docinfo) + sizeof(DocInfo); 310 | memcpy(docinfo->rev_meta.buf, (uint8_t*)buf + offset, docinfo->rev_meta.size); 311 | offset += docinfo->rev_meta.size; 312 | }else{ 313 | docinfo->rev_meta.buf = NULL; 314 | } 315 | } 316 | 317 | LIBCOUCHSTORE_API 318 | couchstore_error_t couchstore_open_document(Db *db, 319 | const void *id, 320 | size_t idlen, 321 | Doc **pDoc, 322 | couchstore_open_options options) 323 | { 324 | int ret; 325 | 326 | db->cursor->set_key(db->cursor, id); 327 | ret = db->cursor->search(db->cursor); 328 | assert(ret == 0); 329 | 330 | const char *value; 331 | db->cursor->get_value(db->cursor, &value); 332 | 333 | *pDoc = (Doc *)malloc(sizeof(Doc)); 334 | (*pDoc)->id.buf = (char*)id; 335 | (*pDoc)->id.size = idlen; 336 | (*pDoc)->data.size = strlen(value); 337 | (*pDoc)->data.buf = (char*)malloc((*pDoc)->data.size+1); 338 | strcpy((*pDoc)->data.buf, value); 339 | 340 | return COUCHSTORE_SUCCESS; 341 | } 342 | 343 | LIBCOUCHSTORE_API 344 | couchstore_error_t couchstore_walk_id_tree(Db *db, 345 | const sized_buf* startDocID, 346 | couchstore_docinfos_options options, 347 | couchstore_walk_tree_callback_fn callback, 348 | void *ctx) 349 | { 350 | int ret; 351 | int c_ret = 0; 352 | DocInfo doc_info; 353 | Doc doc; 354 | 355 | db->cursor->set_key(db->cursor, startDocID->buf); 356 | ret = db->cursor->search(db->cursor); 357 | assert(ret == 0); 358 | 359 | const char *key, *value; 360 | do { 361 | db->cursor->get_key(db->cursor, &key); 362 | doc_info.id.size = strlen(key); 363 | doc_info.id.buf = (char *)malloc(doc_info.id.size + 1); 364 | strcpy(doc_info.id.buf, key); 365 | 366 | db->cursor->get_value(db->cursor, &value); 367 | doc.data.size = strlen(value); 368 | doc.data.buf = (char *)malloc(doc.data.size + 1); 369 | strcpy(doc.data.buf, value); 370 | 371 | c_ret = callback(db, 0, &doc_info, 0, NULL, ctx); 372 | 373 | free(doc_info.id.buf); 374 | free(doc.data.buf); 375 | 376 | if (c_ret != 0) { 377 | break; 378 | } 379 | 380 | } while ((ret = db->cursor->next(db->cursor)) == 0); 381 | 382 | return COUCHSTORE_SUCCESS; 383 | } 384 | 385 | LIBCOUCHSTORE_API 386 | void couchstore_free_document(Doc *doc) 387 | { 388 | if (doc->id.buf) free(doc->id.buf); 389 | if (doc->data.buf) free(doc->data.buf); 390 | free(doc); 391 | } 392 | 393 | 394 | LIBCOUCHSTORE_API 395 | void couchstore_free_docinfo(DocInfo *docinfo) 396 | { 397 | free(docinfo); 398 | } 399 | 400 | LIBCOUCHSTORE_API 401 | couchstore_error_t couchstore_commit(Db *db) 402 | { 403 | // do nothing 404 | return COUCHSTORE_SUCCESS; 405 | } 406 | 407 | LIBCOUCHSTORE_API 408 | couchstore_error_t couchstore_compact_db_ex(Db* source, const char* target_filename, 409 | uint64_t flags, FileOpsInterface *ops) 410 | { 411 | return COUCHSTORE_SUCCESS; 412 | } 413 | 414 | LIBCOUCHSTORE_API 415 | couchstore_error_t couchstore_compact_db(Db* source, const char* target_filename) 416 | { 417 | return couchstore_compact_db_ex(source, target_filename, 0x0, NULL); 418 | } 419 | 420 | --------------------------------------------------------------------------------