├── .copyrightignore ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── config.cmake.h.in ├── csdata.md ├── file_format.md ├── include └── libcouchstore │ ├── couch_common.h │ ├── couch_db.h │ ├── couch_latency.h │ ├── error.h │ ├── file_ops.h │ └── json_utils.h ├── licenses ├── APL2.txt └── BSL-Couchbase.txt ├── programs ├── CMakeLists.txt └── dbdiff │ ├── CMakeLists.txt │ └── dbdiff.cc ├── python ├── couchstore.py ├── couchstore_test.py ├── example1.py └── example_read.py ├── src ├── arena.cc ├── arena.h ├── bitfield.cc ├── bitfield.h ├── btree_modify.cc ├── btree_read.cc ├── compactor.cc ├── couch_btree.h ├── couch_create.cc ├── couch_cxx_util.cc ├── couch_db.cc ├── couch_file_read.cc ├── couch_file_write.cc ├── couch_latency.cc ├── couch_latency_internal.h ├── couch_save.cc ├── couchscript.cc ├── crc32.cc ├── crc32.h ├── crypto.h ├── db_compact.cc ├── dbck.cc ├── dbdump.cc ├── dbinfo.cc ├── exception.h ├── fatbuf.h ├── file_merger.cc ├── file_merger.h ├── file_sorter.cc ├── file_sorter.h ├── internal.h ├── iobuffer.cc ├── iobuffer.h ├── log_last_internal_error.h ├── merge_sort.h ├── node_types.cc ├── node_types.h ├── os.cc ├── os.h ├── os_win.cc ├── program_getopt.cc ├── program_getopt.h ├── quicksort.c ├── quicksort.h ├── reduces.cc ├── reduces.h ├── stream.cc ├── stream.h ├── strerror.cc ├── tracking_file_ops.cc ├── tracking_file_ops.h ├── tree_writer.cc ├── tree_writer.h ├── util.cc ├── util.h └── views │ ├── bin │ ├── couch_view_file_merger.cc │ ├── couch_view_group_cleanup.cc │ ├── couch_view_group_compactor.cc │ ├── couch_view_index_builder.cc │ ├── couch_view_index_updater.cc │ ├── util.cc │ └── util.h │ ├── bitmap.cc │ ├── bitmap.h │ ├── collate_json.cc │ ├── collate_json.h │ ├── compaction.cc │ ├── compaction.h │ ├── encoding.cc │ ├── encoding.h │ ├── file_merger.cc │ ├── file_merger.h │ ├── file_sorter.cc │ ├── file_sorter.h │ ├── index_header.cc │ ├── index_header.h │ ├── keys.cc │ ├── keys.h │ ├── mapreduce │ ├── mapreduce.cc │ ├── mapreduce.h │ ├── mapreduce_c.cc │ ├── mapreduce_internal.h │ └── visibility.h │ ├── purgers.cc │ ├── purgers.h │ ├── reducers.cc │ ├── reducers.h │ ├── reductions.cc │ ├── reductions.h │ ├── sorted_list.cc │ ├── sorted_list.h │ ├── spatial.cc │ ├── spatial.h │ ├── spatial_modify.cc │ ├── util.cc │ ├── util.h │ ├── values.cc │ ├── values.h │ ├── view_group.cc │ └── view_group.h ├── tests ├── btree_purge │ ├── purge.cc │ ├── purge_tests.h │ └── tests.cc ├── bulk.lua ├── changecount.py ├── changessincefilter.lua ├── compact.lua ├── corrupt.lua ├── couchstoretest.cc ├── couchstoretest.h ├── documents.cc ├── documents.h ├── dropdel.lua ├── file_deduper_tests.cc ├── file_merger_tests.cc ├── file_sorter_tests.cc ├── gtest_internal_tests.cc ├── gtest_tests.cc ├── large.lua ├── largefile.lua ├── localdoc.lua ├── macros.h ├── mapreduce │ ├── builtin.cc │ ├── map.cc │ ├── mapreduce_tests.h │ └── reduce.cc ├── new_delete_replacement.py ├── purge.py ├── rewind.py ├── test_compact.cc ├── test_cxx_util.cc ├── test_fileops.cc ├── test_fileops.h ├── test_seek.cc ├── testapp.cc ├── testlib.lua ├── views │ ├── bitmaps.cc │ ├── cleanup.cc │ ├── collate_json_test.cc │ ├── keys.cc │ ├── reducers.cc │ ├── reductions.cc │ ├── sorted_lists.cc │ ├── spatial.cc │ ├── spatial_tests.h │ ├── tests.cc │ ├── values.cc │ └── view_tests.h ├── wrapped_fileops_test.cc └── wrapped_fileops_test.h └── view_format.md /.copyrightignore: -------------------------------------------------------------------------------- 1 | src/crc32.cc 2 | src/mergesort.cc 3 | src/llmsort.cc 4 | src/quicksort.h 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Keep the entries sorted to reduce the risk for a merge conflict 2 | *.[ao] 3 | *.dirstamp 4 | *.exe 5 | *.l[ao] 6 | *.obj 7 | *.pyc 8 | *~ 9 | .deps/ 10 | .libs/ 11 | .pc 12 | /INSTALL 13 | /Makefile 14 | /Makefile.in 15 | /a.out* 16 | /aclocal.m4 17 | /autom4te.cache 18 | /build 19 | /config.cache 20 | /config.h 21 | /config.h.in 22 | /config.log 23 | /config.status 24 | /config/compile 25 | /config/config.guess 26 | /config/config.sub 27 | /config/depcomp 28 | /config/install-sh 29 | /config/ltmain.sh 30 | /config/missing 31 | /config/mkinstalldirs 32 | /config/plugin.ac 33 | /config/ar-lib 34 | /config/test-driver 35 | /configure 36 | /couch_compact 37 | /couch_dbdump 38 | /couch_dbinfo 39 | /couch_viewgen 40 | /couch_view_file_merger 41 | /couch_view_index_builder 42 | /couchscript 43 | /docs/doxy 44 | /libcouchstore*.changes 45 | /libcouchstore*.deb 46 | /libcouchstore*.dsc 47 | /libcouchstore*.rpm 48 | /libcouchstore*.tar.gz 49 | /libcouchstore.dll 50 | /libcouchstore.exp 51 | /libcouchstore.lib 52 | /libcouchstore.pc 53 | /libtool 54 | /m4/libtool.m4 55 | /m4/ltoptions.m4 56 | /m4/ltsugar.m4 57 | /m4/ltversion.m4 58 | /m4/lt~obsolete.m4 59 | /m4/version.m4 60 | /regression 61 | /stamp-h1 62 | /test.couch 63 | /testapp 64 | /writetest 65 | xcuserdata/ 66 | 67 | /CMakeFiles/ 68 | /CTestTestfile.cmake 69 | /cmake_install.cmake 70 | /couchstore_testapp 71 | /*.dylib* 72 | /*.so* 73 | /*.dll* 74 | /Testing/ 75 | /*.exe* 76 | /*.ilk 77 | /*.pdb 78 | /*.exp 79 | /*.lib 80 | /couch_view_group_cleanup 81 | /couch_view_index_updater 82 | /purge.couch 83 | /couch_view_group_compactor 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Source code in this repository is variously licensed under the Business Source 2 | License 1.1 (BSL), the MIT license, BSD-style licenses and Apache V2.0 license. 3 | A copy of each license can be found in the licenses directory. 4 | 5 | Source code in a given file is licensed under the BSL and the copyright belongs 6 | to Couchbase Inc. unless otherwise noted at the beginning of the file. 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Couchbase (.couch) storage file C library 2 | 3 | Currently this library can only be built as part of Couchbase Server due to dependencies on the [Couchbase Platform Library][plat] and [Couchbase Server CMake project][tlm]. For instructions on building Couchbase Server, see the [Manifest Repository][manifest]. 4 | 5 | [plat]: https://github.com/couchbase/platform 6 | [tlm]: https://github.com/couchbase/tlm 7 | [manifest]: https://github.com/couchbase/manifest 8 | 9 | ## Tests: 10 | 11 | 1. `make test` 12 | 13 | This will run the native tests, and also the Lua tests if Lua was installed at the time the `configure` script ran. 14 | 15 | Tests use the CMake CTest system, and the [ctest](http://www.cmake.org/cmake/help/v2.8.8/ctest.html) command can be used to run invidual tests and print verbose output. 16 | -------------------------------------------------------------------------------- /config.cmake.h.in: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #pragma once 3 | 4 | #cmakedefine HAVE_NETINET_IN_H ${HAVE_NETINET_IN_H} 5 | #cmakedefine HAVE_ARPA_INET_H ${HAVE_ARPA_INET_H} 6 | #cmakedefine HAVE_INTTYPES_H ${HAVE_INTTYPES_H} 7 | #cmakedefine HAVE_UNISTD_H ${HAVE_UNISTD_H} 8 | #cmakedefine HAVE_FDATASYNC ${HAVE_FDATASYNC} 9 | #cmakedefine HAVE_QSORT_R ${HAVE_QSORT_R} 10 | 11 | /* Large File Support */ 12 | #define _LARGE_FILE 1 13 | #ifndef _FILE_OFFSET_BITS 14 | # define _FILE_OFFSET_BITS 64 15 | #elif (_FILE_OFFSET_BITS != 64) 16 | #error "bad things" 17 | #endif 18 | #define _LARGEFILE_SOURCE 1 19 | #ifndef O_LARGEFILE 20 | # define O_LARGEFILE 0 21 | #endif 22 | 23 | #include 24 | 25 | #ifdef HAVE_NETINET_IN_H 26 | #include 27 | #endif 28 | 29 | #ifdef HAVE_ARPA_INET_H 30 | #include 31 | #endif 32 | 33 | #ifdef HAVE_INTTYPES_H 34 | #include 35 | #endif 36 | 37 | #ifdef HAVE_UNISTD_H 38 | #include 39 | #endif 40 | 41 | #ifdef __APPLE__ 42 | /* autoconf things OS X has fdatasync but it doesn't */ 43 | #define fdatasync(FD) fsync(FD) 44 | #endif /* __APPLE__ */ 45 | 46 | #ifdef linux 47 | #undef ntohs 48 | #undef ntohl 49 | #undef htons 50 | #undef htonl 51 | #endif 52 | 53 | #if defined(WIN32) || defined(_WIN32) 54 | #define WINDOWS 55 | /* Mute: 56 | * "The POSIX name for this item is deprecated. Instead, use the ISO 57 | * C++ conformant name: _strdup." 58 | */ 59 | #pragma warning(disable: 4996) 60 | 61 | /* we need ntohl with friends (included through arpa/inet.h for *nix */ 62 | #include 63 | 64 | #endif 65 | 66 | // Sanitizer annotations 67 | #if defined(__has_attribute) 68 | # if __has_attribute(no_sanitize_undefined) 69 | // GCC only supports the top-level "no_sanitize_undefined"; so have to 70 | // turn off all UBSan checks just to disable vptr checks :( 71 | # define NO_SANITIZE_VPTR __attribute__((no_sanitize_undefined)) 72 | # elif __has_attribute(no_sanitize) 73 | // Can turn off the specific vptr sanitizer. 74 | # define NO_SANITIZE_VPTR __attribute__((no_sanitize("vptr"))) 75 | # endif 76 | #endif 77 | #if !defined(NO_SANITIZE_VPTR) 78 | # define NO_SANITIZE_VPTR 79 | #endif 80 | -------------------------------------------------------------------------------- /csdata.md: -------------------------------------------------------------------------------- 1 | ## Couchstore Data Types 2 | 3 | Struct members not listed should be considered internal to Couchstore. 4 | 5 | ### sized\_buf 6 | struct sized_buf { 7 | char* buf; 8 | size_t size; 9 | } 10 | 11 | Couchstore uses `sized_buf`s to point to data buffers. 12 | 13 | ### DocInfo 14 | struct DocInfo { 15 | sized_buf id; 16 | uint64_t db_seq; 17 | uint64_t rev_seq; 18 | sized_buf rev_meta; 19 | int deleted; 20 | uint8_t content_meta; 21 | } 22 | 23 | * `id` - Document ID 24 | * `db_seq` - Change sequence number the document was inserted at. 25 | * `rev_seq` - The version number of the document 26 | * `rev_meta` - Revision metadata. Used by ep-engine to store CAS value, expiry time, and flags 27 | * `deleted` - 1 if document should be considered "deleted" and not subject to indexing, otherwise 0. 28 | * `content_meta` - Number field used to store flags indicating metadata about the document content (is it JSON, etc.) 29 | 30 | #### The `content_meta` field 31 | 32 | * The least-significant two bits are used to encode why/whether the body is or is not JSON 33 | * 0 - Document body *is* JSON 34 | * 1 - Inserted data was not valid JSON 35 | * 2 - Inserted data was valid JSON, but contained a key reserved for internal use, such as one with a `$` prefix. 36 | * 3 - The document was inserted in non-JSON mode. 37 | * If the most-significant bit is set the document body data has been compressed with snappy. 38 | 39 | When saving documents with `save_doc` or `save_docs`, `id`, `rev_seq`, `rev_meta`, `deleted`, and `content_meta` must be set on the `DocInfo`s passed to Couchstore. The `db_seq` is determined at insert time. 40 | 41 | #### The `rev_meta` field 42 | 43 | [128 bit binary value] 44 | | 64 bits | 32 bits | 32 bits | 45 | -----------|------------|------------| 46 | CAS Value | Expiration | User flags | 47 | 48 | All values are Big Endian. 49 | 50 | * CAS Value - Random nonce value set by Memcached, used to allow ensuring value has not been modified since it was read when writing. 51 | * Expiration - 32 bit timestamp for value expiry time. 0 if item does not expire. 52 | * User flags - Memcache API allows users to set this 32 bit number along with values, it is not used by memcached internally. 53 | 54 | ### Doc 55 | struct Doc { 56 | sized_buf id; 57 | sized_buf data; 58 | } 59 | 60 | `id` contains the document ID, `data` contains the document body data. Couchstore does not compress or modify the body data in any way, unless the COMPRESS_DOC_BODIES or DECOMPRESS_DOC_BODIES flags are passed to the read and write functions, then it will check the `content_meta` of the corresponding doc info to see if it should compress or decompress the doc data. 61 | 62 | ### LocalDoc 63 | struct LocalDoc { 64 | sized_buf id; 65 | sized_buf json; 66 | int deleted; 67 | } 68 | 69 | * `id` - The local document ID, it must start with `_local/` 70 | * `json` - The local document body. 71 | * `deleted` - if set to 1 on a LocalDoc passed to save_local_doc, `json` will be ignored, and the local document with the ID in `id` will be removed, if it exists. 72 | 73 | 74 | ## C API 75 | 76 | Documented in `include/libcouchstore/couch_db.h` 77 | 78 | -------------------------------------------------------------------------------- /include/libcouchstore/couch_latency.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "couch_common.h" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | using CouchLatencyHisto = MicrosecondHistogram; 27 | using CouchLatencyMicroSec = UnsignedMicroseconds; 28 | using CouchLatencyMicroSecRep = UnsignedMicroseconds::rep; 29 | 30 | /** 31 | * Begin to collect Couchstore API latency. 32 | */ 33 | LIBCOUCHSTORE_API 34 | void couchstore_latency_collector_start(); 35 | 36 | /** 37 | * The callback function used by couchstore_get_latency_info(), 38 | * to get latency histogram data. 39 | * 40 | * @param stat_name Name of statistic item. 41 | * @param latencies Latency histogram of the statistic item. 42 | * @param elapsed_time Total elapsed time in microseconds. 43 | * @param ctx User context 44 | * @return 1 to stop getting latency info, 0 otherwise. 45 | */ 46 | using couchstore_latency_callback_fn = 47 | std::function; 51 | 52 | /** 53 | * Latency info dump options. 54 | */ 55 | struct couchstore_latency_dump_options { 56 | /** 57 | * Currently empty, but left it for future extension. 58 | */ 59 | }; 60 | 61 | /** 62 | * Dump collected latency data through the given 63 | * callback function. 64 | * 65 | * @param callback Callback function. 66 | * @param options Latency dump options. 67 | * @param ctx User context 68 | */ 69 | LIBCOUCHSTORE_API 70 | void couchstore_get_latency_info(couchstore_latency_callback_fn callback, 71 | couchstore_latency_dump_options options, 72 | void *ctx); 73 | 74 | /** 75 | * Stop collecting Couchstore API latency. 76 | */ 77 | LIBCOUCHSTORE_API 78 | void couchstore_latency_collector_stop(); 79 | 80 | -------------------------------------------------------------------------------- /include/libcouchstore/error.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #ifndef COUCHSTORE_COUCH_DB_H 19 | #error "You should include instead" 20 | #endif 21 | 22 | /** Error values returned by CouchStore API calls. */ 23 | enum couchstore_error_t { 24 | COUCHSTORE_SUCCESS = 0, 25 | COUCHSTORE_ERROR_OPEN_FILE = -1, 26 | COUCHSTORE_ERROR_CORRUPT = -2, 27 | COUCHSTORE_ERROR_ALLOC_FAIL = -3, 28 | COUCHSTORE_ERROR_READ = -4, 29 | COUCHSTORE_ERROR_DOC_NOT_FOUND = -5, 30 | COUCHSTORE_ERROR_NO_HEADER = -6, 31 | COUCHSTORE_ERROR_WRITE = -7, 32 | COUCHSTORE_ERROR_HEADER_VERSION = -8, 33 | COUCHSTORE_ERROR_CHECKSUM_FAIL = -9, 34 | COUCHSTORE_ERROR_INVALID_ARGUMENTS = -10, 35 | COUCHSTORE_ERROR_NO_SUCH_FILE = -11, 36 | COUCHSTORE_ERROR_CANCEL = -12, 37 | COUCHSTORE_ERROR_REDUCTION_TOO_LARGE = -13, 38 | COUCHSTORE_ERROR_REDUCER_FAILURE = -14, 39 | COUCHSTORE_ERROR_FILE_CLOSED = -15, 40 | COUCHSTORE_ERROR_DB_NO_LONGER_VALID = -16, 41 | COUCHSTORE_ERROR_FILE_CLOSE = -17, 42 | COUCHSTORE_ERROR_NOT_SUPPORTED = -18, 43 | 44 | /** 45 | * A non success status code for the callback of 46 | * couchstore_changes_since and couchstore_docinfos_by_id to signal that 47 | * the outer function is to stop visiting the index and return this 48 | * status code to the caller. The callback uses this status to indicate 49 | * some temporary issue (e.g. a soft out of memory condition) and the 50 | * caller should yield and resume the index scan later. 51 | */ 52 | COUCHSTORE_ERROR_SCAN_YIELD = -19, 53 | 54 | /** 55 | * A non success status code for the callback of 56 | * couchstore_changes_since and couchstore_docinfos_by_id to signal that 57 | * the outer function is to stop visiting the index and return this 58 | * status code to the caller. The callback uses this status to indicate 59 | * some non temporary issue (e.g. a change of vbucket state) and the 60 | * caller should cancel the scan. 61 | */ 62 | COUCHSTORE_ERROR_SCAN_CANCELLED = -20, 63 | 64 | COUCHSTORE_ERROR_NO_ENCRYPTION_KEY = -21, 65 | COUCHSTORE_ERROR_ENCRYPT = -22, 66 | COUCHSTORE_ERROR_DECRYPT = -23 67 | }; 68 | -------------------------------------------------------------------------------- /include/libcouchstore/json_utils.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2021 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "couch_db.h" 21 | #include 22 | 23 | namespace cb::couchstore { 24 | LIBCOUCHSTORE_API nlohmann::json to_json(const Header& header); 25 | } 26 | -------------------------------------------------------------------------------- /licenses/BSL-Couchbase.txt: -------------------------------------------------------------------------------- 1 | COUCHBASE BUSINESS SOURCE LICENSE AGREEMENT 2 | 3 | Business Source License 1.1 4 | Licensor: Couchbase, Inc. 5 | Licensed Work: Couchbase Server Version Cheshire-Cat 6 | The Licensed Work is © 2021-Present Couchbase, Inc. 7 | 8 | Additional Use Grant: You may make production use of the Licensed Work, provided 9 | you comply with the following conditions: 10 | 11 | (i) You may not prepare a derivative work based upon the Licensed Work and 12 | distribute or otherwise offer such derivative work, whether on a standalone 13 | basis or in combination with other products, applications, or services 14 | (including in any "as-a-service" offering, such as, by way of example, a 15 | software-as-a-service, database-as-a-service, or infrastructure-as-a-service 16 | offering, or any other offering based on a cloud computing or other type of 17 | hosted distribution model (collectively, "Hosted Offerings")), for a fee or 18 | otherwise on a commercial or other for-profit basis. 19 | 20 | (ii) You may not link the Licensed Work to, or otherwise include the Licensed 21 | Work in or with, any product, application, or service (including in any Hosted 22 | Offering) that is distributed or otherwise offered, whether on a standalone 23 | basis or in combination with other products, applications, or services for a fee 24 | or otherwise on a commercial or other for-profit basis. Condition (ii) shall not 25 | limit the generality of condition (i) above. 26 | 27 | 28 | Change Date: April 1, 2025 29 | 30 | Change License: Apache License, Version 2.0 31 | 32 | 33 | Notice 34 | 35 | The Business Source License (this document, or the "License") is not an Open 36 | Source license. However, the Licensed Work will eventually be made available 37 | under an Open Source License, as stated in this License. License text copyright 38 | © 2017 MariaDB Corporation Ab, All Rights Reserved. "Business Source License" is 39 | a trademark of MariaDB Corporation Ab. 40 | 41 | Terms 42 | 43 | The Licensor hereby grants You the right to copy, modify, create derivative 44 | works, redistribute, and make non-production use of the Licensed Work. The 45 | Licensor may make an Additional Use Grant, above, permitting limited production 46 | use. 47 | 48 | Effective on the Change Date, or the fourth anniversary of the first publicly 49 | available distribution of a specific version of the Licensed Work under this 50 | License, whichever comes first, the Licensor hereby grants you rights under the 51 | terms of the Change License, and the rights granted in the paragraph above 52 | terminate. 53 | 54 | If your use of the Licensed Work does not comply with the requirements currently 55 | in effect as described in this License, you must purchase a commercial license 56 | from the Licensor, its affiliated entities, or authorized resellers, or you must 57 | refrain from using the Licensed Work. 58 | 59 | All copies of the original and modified Licensed Work, and derivative works of 60 | the Licensed Work, are subject to this License. This License applies separately 61 | for each version of the Licensed Work and the Change Date may vary for each 62 | version of the Licensed Work released by Licensor. 63 | 64 | You must conspicuously display this License on each original or modified copy of 65 | the Licensed Work. If you receive the Licensed Work in original or modified form 66 | from a third party, the terms and conditions set forth in this License apply to 67 | your use of that work. 68 | 69 | Any use of the Licensed Work in violation of this License will automatically 70 | terminate your rights under this License for the current and all other versions 71 | of the Licensed Work. 72 | 73 | This License does not grant you any right in any trademark or logo of Licensor 74 | or its affiliates (provided that you may use a trademark or logo of Licensor as 75 | expressly required by this License). 76 | 77 | TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN 78 | "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS 79 | OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY, 80 | FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND TITLE. 81 | 82 | MariaDB hereby grants you permission to use this License's text to license your 83 | works, and to refer to it using the trademark "Business Source License", as long 84 | as you comply with the Covenants of Licensor below. 85 | 86 | Covenants of Licensor 87 | 88 | In consideration of the right to use this License's text and the "Business 89 | Source License" name and trademark, Licensor covenants to MariaDB, and to all 90 | other recipients of the licensed work to be provided by Licensor: 91 | 92 | 1. To specify as the Change License the GPL Version 2.0 or any later version, or 93 | a license that is compatible with GPL Version 2.0 or a later version, where 94 | "compatible" means that software provided under the Change License can be 95 | included in a program with software provided under GPL Version 2.0 or a later 96 | version. Licensor may specify additional Change Licenses without limitation. 97 | 98 | 2. To either: (a) specify an additional grant of rights to use that does not 99 | impose any additional restriction on the right granted in this License, as the 100 | Additional Use Grant; or (b) insert the text "None". 101 | 102 | 3. To specify a Change Date. 103 | 104 | 4. Not to modify this License in any other way. 105 | -------------------------------------------------------------------------------- /programs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(dbdiff) 2 | -------------------------------------------------------------------------------- /programs/dbdiff/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_EXECUTABLE(dbdiff dbdiff.cc) 2 | TARGET_LINK_LIBRARIES(dbdiff couchstore_views couchstore_program_utilities) 3 | 4 | INSTALL(TARGETS dbdiff 5 | RUNTIME DESTINATION bin) 6 | -------------------------------------------------------------------------------- /python/example1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Simple example program to create a Couchstore file with a given 4 | # number of keys. 5 | 6 | from couchstore import CouchStore, DocumentInfo, SizedBuf 7 | import os 8 | import struct 9 | import sys 10 | 11 | BATCH_SIZE = 10000 12 | REV_META_PACK = ">QII" 13 | 14 | def insert(db, key, rev, value): 15 | info = DocumentInfo(key) 16 | info.revSequence = rev 17 | # cas, exp, flags 18 | info.revMeta = str(struct.pack(REV_META_PACK, 1, 2, 3)) 19 | info.deleted = False 20 | return db.save(info, value) 21 | 22 | 23 | def insert_multi(db, keys, values): 24 | """Inserts multiple keys / values.""" 25 | 26 | ids = [] 27 | for k in keys: 28 | info = DocumentInfo(k) 29 | info.revSequence = 1 30 | # cas, exp, flags 31 | info.revMeta = str(struct.pack(REV_META_PACK, 1, 2, 3)) 32 | info.deleted = False 33 | ids.append(info) 34 | return db.saveMultiple(ids, values) 35 | 36 | 37 | def chunks(l, n): 38 | """Yield successive n-sized chunks from l.""" 39 | for i in range(0, len(l), n): 40 | yield l[i:i+n] 41 | 42 | 43 | def main(): 44 | if len(sys.argv) != 3: 45 | print("Usage: example1 ") 46 | exit(1) 47 | db = CouchStore(sys.argv[2], 'c') 48 | for batch in chunks(list(range(0, int(sys.argv[1]))), BATCH_SIZE): 49 | insert_multi(db, 50 | ["key_" + str(x) for x in batch], 51 | [str(x) for x in batch]) 52 | db.commit() 53 | db.close() 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /python/example_read.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Simple program to read the given keys from a Couchstore file. 4 | 5 | from couchstore import CouchStore, DocumentInfo, SizedBuf 6 | import argparse 7 | import os 8 | import struct 9 | import sys 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("file") 14 | parser.add_argument("keys", help="Key(s) to print", nargs="+") 15 | parser.add_argument("--unbuffered", help="Disable couchstore io buffering", action="store_true") 16 | args = parser.parse_args() 17 | 18 | db = CouchStore(args.file, 'r', unbuffered=args.unbuffered) 19 | for key in args.keys: 20 | print(db.get(key)) 21 | db.close() 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /src/arena.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** arena.h 3 | ** couchstore 4 | ** 5 | ** Created by Jens Alfke on 4/13/12. 6 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 7 | */ 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | /** Opaque arena-allocator object. */ 14 | struct arena; 15 | 16 | /** Saved position/state of an arena. */ 17 | struct arena_position; 18 | 19 | /** 20 | * Creates a new arena allocator. 21 | * @param chunk_size The size of the memory blocks the allocator sub-allocates from malloc. Pass 0 for the default (32kbytes). 22 | * @return The new arena object. 23 | */ 24 | arena* new_arena(size_t chunk_size); 25 | 26 | /** 27 | * Deletes an arena and all of its memory allocations. 28 | */ 29 | void delete_arena(arena*); 30 | 31 | /** 32 | * Allocates memory from an arena. 33 | * @param arena The arena to allocate from 34 | * @param size The number of bytes to allocate 35 | * @return A pointer to the allocated block, or NULL on failure. 36 | */ 37 | void* arena_alloc(arena*, size_t size); 38 | 39 | /** 40 | * Allocates unaligned memory from an arena. 41 | * Saves a couple of bytes if your block doesn't need to be word-aligned. 42 | */ 43 | void* arena_alloc_unaligned(arena* a, size_t size); 44 | 45 | /** 46 | * "Frees" a block allocated from an arena. This actually doesn't do anything. 47 | * In a debug build it at least checks that the pointers was (probably) allocated from the arena. 48 | * In a release build it's explicitly an inlined no-op and there's no need to call it at all. 49 | */ 50 | void arena_free(arena*, void*); 51 | 52 | /** 53 | * Captures the current state of an arena, i.e. which blocks have been allocated. 54 | * Save the return value and pass it to arena_free_from_mark later to free all blocks allocated 55 | * since this point. 56 | */ 57 | const arena_position* arena_mark(arena *a); 58 | 59 | /** 60 | * Frees all blocks from the arena that have been allocated since the corresponding call to 61 | * arena_mark. (The mark remains valid and can be used again.) 62 | */ 63 | void arena_free_from_mark(arena *a, const arena_position *mark); 64 | 65 | /** 66 | * Frees all blocks from the arena. 67 | */ 68 | void arena_free_all(arena *a); 69 | 70 | struct arena_deleter { 71 | void operator()(arena* ptr) const { 72 | delete_arena(ptr); 73 | } 74 | }; 75 | -------------------------------------------------------------------------------- /src/bitfield.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2019 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "couchstore_config.h" 19 | #include "internal.h" 20 | #include "bitfield.h" 21 | 22 | #include 23 | 24 | /* Functions for decoding raw_xx types to native integers: */ 25 | 26 | #define DECODE_RAW(DST_TYPE, FLIP_FN) \ 27 | DST_TYPE value = 0; \ 28 | memcpy((char*)&value + sizeof(value) - sizeof(raw), &(raw), sizeof(raw)); \ 29 | return FLIP_FN(value) 30 | 31 | #define DECODE_RAW_POINTER(DST_TYPE, FLIP_FN) \ 32 | DST_TYPE value = 0; \ 33 | memcpy((char*)&value + sizeof(value) - sizeof(*rawp), rawp, sizeof(*rawp)); \ 34 | return FLIP_FN(value) 35 | 36 | uint8_t couchstore_decode_raw08(raw_08 raw) 37 | { 38 | return raw.raw_bytes[0]; 39 | } 40 | 41 | uint16_t couchstore_decode_raw16(raw_16 raw) 42 | { 43 | DECODE_RAW(uint16_t, ntohs); 44 | } 45 | 46 | uint32_t couchstore_decode_raw24p(const raw_24 *rawp) 47 | { 48 | DECODE_RAW_POINTER(uint32_t, ntohl); 49 | } 50 | 51 | uint32_t couchstore_decode_raw32(raw_32 raw) 52 | { 53 | DECODE_RAW(uint32_t, ntohl); 54 | } 55 | 56 | uint64_t couchstore_decode_raw40p(const raw_40 *rawp) 57 | { 58 | DECODE_RAW_POINTER(uint64_t, ntohll); 59 | } 60 | 61 | uint64_t couchstore_decode_raw48p(const raw_48 *rawp) 62 | { 63 | DECODE_RAW_POINTER(uint64_t, ntohll); 64 | } 65 | 66 | uint64_t couchstore_decode_raw64(raw_64 raw) 67 | { 68 | DECODE_RAW(uint64_t, ntohll); 69 | } 70 | 71 | 72 | /* Functions for encoding native integers to raw_xx types: */ 73 | 74 | #define ENCODE_RAW(FLIP_FN, RAW_TYPE) \ 75 | RAW_TYPE raw; \ 76 | value = FLIP_FN(value); \ 77 | memcpy(&raw, (char*)&value + sizeof(value) - sizeof(raw), sizeof(raw)); \ 78 | return raw 79 | 80 | #define ENCODE_RAW_POINTER(FLIP_FN, RAW_TYPE) \ 81 | value = FLIP_FN(value); \ 82 | memcpy(raw, (char*)&value + sizeof(value) - sizeof(*raw), sizeof(*raw)); 83 | 84 | raw_08 couchstore_encode_raw08(uint8_t value) 85 | { 86 | ENCODE_RAW((uint8_t), raw_08); 87 | } 88 | 89 | raw_16 couchstore_encode_raw16(uint16_t value) 90 | { 91 | ENCODE_RAW(htons, raw_16); 92 | } 93 | 94 | void couchstore_encode_raw24(uint32_t value, raw_24 *raw) 95 | { 96 | ENCODE_RAW_POINTER(htonl, raw_24); 97 | } 98 | 99 | raw_32 couchstore_encode_raw32(uint32_t value) 100 | { 101 | ENCODE_RAW(htonl, raw_32); 102 | } 103 | 104 | void couchstore_encode_raw40(uint64_t value, raw_40 *raw) 105 | { 106 | ENCODE_RAW_POINTER(htonll, raw_40); 107 | } 108 | 109 | void couchstore_encode_raw48(uint64_t value, raw_48 *raw) 110 | { 111 | ENCODE_RAW_POINTER(htonll, raw_48); 112 | } 113 | 114 | raw_64 couchstore_encode_raw64(uint64_t value) 115 | { 116 | ENCODE_RAW(htonll, raw_64); 117 | } 118 | -------------------------------------------------------------------------------- /src/bitfield.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "couchstore_config.h" 4 | #include "internal.h" 5 | #include 6 | 7 | /* 8 | * Variable-width types. Since these are made out of chars they will be 9 | * byte-aligned, so structs consisting only of these will be packed. 10 | */ 11 | 12 | struct raw_08 { 13 | uint8_t raw_bytes[1]; 14 | }; 15 | 16 | struct raw_16 { 17 | uint8_t raw_bytes[2]; 18 | }; 19 | 20 | struct raw_24 { 21 | uint8_t raw_bytes[3]; 22 | }; 23 | 24 | struct raw_32 { 25 | uint8_t raw_bytes[4]; 26 | }; 27 | 28 | struct raw_40 { 29 | uint8_t raw_bytes[5]; 30 | }; 31 | 32 | struct raw_48 { 33 | uint8_t raw_bytes[6]; 34 | }; 35 | 36 | struct raw_64 { 37 | uint8_t raw_bytes[8]; 38 | }; 39 | 40 | /* Functions for decoding raw_xx types to native integers: */ 41 | #define encode_raw08(a) couchstore_encode_raw08(a) 42 | #define encode_raw16(a) couchstore_encode_raw16(a) 43 | #define encode_raw24(a, b) couchstore_encode_raw24(a, b) 44 | #define encode_raw32(a) couchstore_encode_raw32(a) 45 | #define encode_raw40(a, b) couchstore_encode_raw40(a, b) 46 | #define encode_raw48(a, b) couchstore_encode_raw48(a, b) 47 | #define encode_raw64(a) couchstore_encode_raw64(a) 48 | 49 | #define decode_raw08(a) couchstore_decode_raw08(a) 50 | #define decode_raw16(a) couchstore_decode_raw16(a) 51 | #define decode_raw24(a) couchstore_decode_raw24p(&(a)) 52 | #define decode_raw32(a) couchstore_decode_raw32(a) 53 | #define decode_raw40(a) couchstore_decode_raw40p(&(a)) 54 | #define decode_raw48(a) couchstore_decode_raw48p(&(a)) 55 | #define decode_raw64(a) couchstore_decode_raw64(a) 56 | 57 | #define dec_uint16(b) (decode_raw16(*((raw_16*)b))) 58 | #define dec_uint40(b) (decode_raw40(*((raw_40*)b))) 59 | #define dec_uint48(b) (decode_raw48(*((raw_48*)b))) 60 | #define dec_uint64(b) (decode_raw64(*((raw_64*)b))) 61 | 62 | LIBCOUCHSTORE_API 63 | uint8_t couchstore_decode_raw08(raw_08 raw); 64 | LIBCOUCHSTORE_API 65 | uint16_t couchstore_decode_raw16(raw_16 raw); 66 | LIBCOUCHSTORE_API 67 | uint32_t couchstore_decode_raw24p(const raw_24* raw); 68 | LIBCOUCHSTORE_API 69 | uint32_t couchstore_decode_raw32(raw_32 raw); 70 | LIBCOUCHSTORE_API 71 | uint64_t couchstore_decode_raw40p(const raw_40* raw); 72 | LIBCOUCHSTORE_API 73 | uint64_t couchstore_decode_raw48p(const raw_48* raw); 74 | LIBCOUCHSTORE_API 75 | uint64_t couchstore_decode_raw64(raw_64 raw); 76 | 77 | /* Functions for encoding native integers to raw_xx types: */ 78 | 79 | LIBCOUCHSTORE_API 80 | raw_08 couchstore_encode_raw08(uint8_t value); 81 | LIBCOUCHSTORE_API 82 | raw_16 couchstore_encode_raw16(uint16_t value); 83 | LIBCOUCHSTORE_API 84 | void couchstore_encode_raw24(uint32_t value, raw_24* raw); 85 | LIBCOUCHSTORE_API 86 | raw_32 couchstore_encode_raw32(uint32_t value); 87 | LIBCOUCHSTORE_API 88 | void couchstore_encode_raw40(uint64_t value, raw_40* raw); 89 | LIBCOUCHSTORE_API 90 | void couchstore_encode_raw48(uint64_t value, raw_48* raw); 91 | LIBCOUCHSTORE_API 92 | raw_64 couchstore_encode_raw64(uint64_t value); 93 | -------------------------------------------------------------------------------- /src/couch_latency.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "couch_latency_internal.h" 19 | 20 | SingletonWrapper > CouchLatency::instance( 21 | cb::RelaxedAtomic(nullptr)); 22 | 23 | CouchLatency* CouchLatency::init() { 24 | CouchLatency* tmp = instance.object.load(); 25 | if (tmp == nullptr) { 26 | // Ensure two threads don't both create an instance. 27 | std::lock_guard lock(instance.mutex); 28 | tmp = instance.object.load(); 29 | if (tmp == nullptr) { 30 | tmp = new CouchLatency(); 31 | instance.object.store(tmp); 32 | } 33 | } 34 | return tmp; 35 | } 36 | 37 | CouchLatency* CouchLatency::getInstance() { 38 | // Instance should be initialized explicitly. 39 | return instance.object.load(); 40 | } 41 | 42 | void CouchLatency::destroyInstance() { 43 | std::lock_guard lock(instance.mutex); 44 | CouchLatency* tmp = instance.object.load(); 45 | if (tmp != nullptr) { 46 | delete tmp; 47 | instance.object = nullptr; 48 | } 49 | } 50 | 51 | CouchLatency::~CouchLatency() { 52 | // Deregister all items, and reset their Histograms. 53 | // As CouchLatencyItem itself is a static unique_ptr 54 | // that is owned by each API functions, 55 | // it doesn't need to be destroyed here. 56 | for (auto& itr : itemsMap.object) { 57 | itr.second->changeRegisterStatus(false); 58 | itr.second->resetHistogram(); 59 | } 60 | } 61 | 62 | CouchLatencyItem* CouchLatency::addItem(CouchLatencyItem* item) { 63 | // This lock will be grabbed during the registration phase only. 64 | // Once an item is registered, they will run in a lock-free manner. 65 | std::lock_guard l(itemsMap.mutex); 66 | auto itr = itemsMap.object.find(item->statName); 67 | if (itr != itemsMap.object.end()) { 68 | // Already registered by other thread. 69 | // Return it. 70 | return itr->second; 71 | } 72 | itemsMap.object.insert( std::make_pair(item->statName, item) ); 73 | item->changeRegisterStatus(true); 74 | return item; 75 | } 76 | 77 | void CouchLatency::getLatencyInfo(couchstore_latency_callback_fn cb_func, 78 | couchstore_latency_dump_options options, 79 | void *ctx) { 80 | (void)options; 81 | for (auto& itr : itemsMap.object) { 82 | CouchLatencyItem *item = itr.second; 83 | if (!item->latencySum) { 84 | continue; 85 | } 86 | int ret = cb_func(item->statName.c_str(), 87 | &item->latencies, 88 | item->latencySum, 89 | ctx); 90 | if (ret) { 91 | // Abort by user. 92 | break; 93 | } 94 | } 95 | } 96 | 97 | void couchstore_latency_collector_start() { 98 | CouchLatency::init(); 99 | } 100 | 101 | void couchstore_get_latency_info(couchstore_latency_callback_fn callback, 102 | couchstore_latency_dump_options options, 103 | void *ctx) { 104 | CouchLatency* tmp = CouchLatency::getInstance(); 105 | if (!tmp) { 106 | // Latency collector is disabled. 107 | return; 108 | } 109 | tmp->getLatencyInfo(callback, options, ctx); 110 | } 111 | 112 | void couchstore_latency_collector_stop() { 113 | CouchLatency::destroyInstance(); 114 | } 115 | 116 | 117 | -------------------------------------------------------------------------------- /src/crc32.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * Couchstore evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | #pragma once 27 | 28 | #include 29 | #include 30 | 31 | typedef enum { 32 | CRC_UNKNOWN, 33 | CRC32, 34 | CRC32C 35 | } crc_mode_e; 36 | 37 | /* 38 | * Get a checksum of buf for buf_len bytes. 39 | * 40 | * mode = UNKNOWN is an invalid input (triggers assert). 41 | */ 42 | uint32_t get_checksum(const uint8_t* buf, 43 | size_t buf_len, 44 | crc_mode_e mode); 45 | 46 | /* 47 | * Perform an integrity check of buf for buf_len bytes. 48 | * 49 | * A checksum of buf is created and compared against checksum argument. 50 | * 51 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 52 | * 53 | * Returns 1 for success, 0 for failure. 54 | */ 55 | int perform_integrity_check(const uint8_t* buf, 56 | size_t buf_len, 57 | uint32_t checksum, 58 | crc_mode_e mode); 59 | 60 | /* 61 | client_hash_crc32 returns a hash which is the same as a couchbase client 62 | would e.g. when determining the vbucket from a key, this function would 63 | match the client's mapping. 64 | 65 | This hashes the key into 'hash', then does ((~hash) >> 16) & 0x7fff; 66 | */ 67 | uint32_t client_hash_crc32(const uint8_t* key, size_t key_length); 68 | -------------------------------------------------------------------------------- /src/crypto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | #pragma once 11 | 12 | #include "internal.h" 13 | 14 | /** 15 | * Convert file offset to encryption nonce 16 | * 17 | * As data chunks are always appended to the end of the file, we use the 18 | * offset in the file as a unique value for the nonce. A chunk that appears 19 | * at the beginning of a block may be read from the block boundary or one 20 | * byte ahead, as the first byte of a block is the block type which will be 21 | * skipped over. To ensure that the same nonce is used for encryption and 22 | * decryption, we need to adjust the nonce such that it is the same in 23 | * those two cases. If the offset is at the block boundary, we increment 24 | * the nonce by one. 25 | */ 26 | static inline uint64_t offset2nonce(cs_off_t offset) { 27 | const auto nonce = gsl::narrow(offset); 28 | return (nonce % COUCH_BLOCK_SIZE) ? nonce : nonce + 1; 29 | } 30 | -------------------------------------------------------------------------------- /src/exception.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | #pragma once 11 | 12 | #include 13 | 14 | #include 15 | 16 | namespace cb::couchstore { 17 | 18 | class Exception : public std::runtime_error { 19 | public: 20 | Exception(couchstore_error_t errcode, const char* what) 21 | : std::runtime_error(what), errcode(errcode) { 22 | } 23 | 24 | Exception(couchstore_error_t errcode, const std::string& what) 25 | : std::runtime_error(what), errcode(errcode) { 26 | } 27 | 28 | const couchstore_error_t errcode; 29 | }; 30 | 31 | } // namespace cb::couchstore 32 | -------------------------------------------------------------------------------- /src/fatbuf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | struct fatbuf { 19 | size_t pos; 20 | size_t size; 21 | char buf[1]; 22 | }; 23 | 24 | fatbuf* fatbuf_alloc(size_t bytes); 25 | void* fatbuf_get(fatbuf* fb, size_t bytes); 26 | void fatbuf_free(fatbuf* fb); 27 | 28 | namespace cb::couchstore { 29 | struct FatbufDeletor { 30 | void operator()(fatbuf* fb); 31 | }; 32 | 33 | using unique_fatbuf_ptr = std::unique_ptr; 34 | } // namespace cb::couchstore 35 | -------------------------------------------------------------------------------- /src/file_merger.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "couchstore_config.h" 24 | #include 25 | #include 26 | 27 | enum file_merger_error_t { 28 | FILE_MERGER_SUCCESS = 0, 29 | FILE_MERGER_ERROR_OPEN_FILE = -1, 30 | FILE_MERGER_ERROR_FILE_READ = -2, 31 | FILE_MERGER_ERROR_FILE_WRITE = -3, 32 | FILE_MERGER_ERROR_BAD_ARG = -4, 33 | FILE_MERGER_ERROR_ALLOC = -5, 34 | FILE_MERGER_SORT_ERROR = -6 35 | }; 36 | 37 | struct file_merger_record_t { 38 | void* record; 39 | unsigned filenum; 40 | }; 41 | 42 | /* 43 | * Returns the length of the record read from the file on success. 44 | * Returns 0 when the file's EOF is reached, and a negative value 45 | * on failure (a file_merger_error_t value). 46 | */ 47 | typedef int (*file_merger_read_record_t)(FILE* f, 48 | void** record_buffer, 49 | void* user_ctx); 50 | 51 | /* 52 | * Returns FILE_MERGER_SUCCESS on success, another file_merger_error_t 53 | * value otherwise. 54 | */ 55 | typedef file_merger_error_t (*file_merger_write_record_t)(FILE* f, 56 | void* record_buffer, 57 | void* user_ctx); 58 | 59 | /* 60 | * Returns 0 if both records compare equal, a negative value if the first record 61 | * is smaller than the second record, a positive value if the first record is 62 | * greater than the second record. 63 | */ 64 | typedef int (*file_merger_compare_records_t)(const void* record_buffer1, 65 | const void* record_buffer2, 66 | void* user_ctx); 67 | 68 | typedef void (*file_merger_record_free_t)(void* record, void* user_ctx); 69 | 70 | /* Callback which gets called for resulting merge records */ 71 | typedef file_merger_error_t (*file_merger_feed_record_t)(void* record_buffer, 72 | void* user_ctx); 73 | 74 | /* Among the provided records, return the winner record index */ 75 | typedef size_t (*file_merger_deduplicate_records_t)( 76 | file_merger_record_t** records, size_t len, void* user_ctx); 77 | 78 | file_merger_error_t merge_files(const char* source_files[], 79 | unsigned num_files, 80 | const char* dest_file, 81 | file_merger_read_record_t read_record, 82 | file_merger_write_record_t write_record, 83 | file_merger_feed_record_t feed_record, 84 | file_merger_compare_records_t compare_records, 85 | file_merger_deduplicate_records_t dedup_records, 86 | file_merger_record_free_t free_record, 87 | int skip_writeback, 88 | void* user_ctx); 89 | -------------------------------------------------------------------------------- /src/file_sorter.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "couchstore_config.h" 24 | #include 25 | #include "file_merger.h" 26 | 27 | typedef enum { 28 | FILE_SORTER_SUCCESS = FILE_MERGER_SUCCESS, 29 | FILE_SORTER_ERROR_OPEN_FILE = FILE_MERGER_ERROR_OPEN_FILE, 30 | FILE_SORTER_ERROR_FILE_READ = FILE_MERGER_ERROR_FILE_READ, 31 | FILE_SORTER_ERROR_FILE_WRITE = FILE_MERGER_ERROR_FILE_WRITE, 32 | FILE_SORTER_ERROR_BAD_ARG = FILE_MERGER_ERROR_BAD_ARG, 33 | FILE_SORTER_ERROR_ALLOC = FILE_MERGER_ERROR_ALLOC, 34 | FILE_SORTER_ERROR_RENAME_FILE = -10, 35 | FILE_SORTER_ERROR_DELETE_FILE = -11, 36 | FILE_SORTER_ERROR_MK_TMP_FILE = -12, 37 | FILE_SORTER_ERROR_NOT_EMPTY_TMP_FILE = -13, 38 | FILE_SORTER_ERROR_TMP_FILE_BASENAME = -14, 39 | FILE_SORTER_ERROR_MISSING_CALLBACK = -15 40 | } file_sorter_error_t; 41 | 42 | file_sorter_error_t sort_file(const char* source_file, 43 | const char* tmp_dir, 44 | unsigned num_tmp_files, 45 | unsigned max_buffer_size, 46 | file_merger_read_record_t read_record, 47 | file_merger_write_record_t write_record, 48 | file_merger_feed_record_t feed_record, 49 | file_merger_compare_records_t compare_records, 50 | file_merger_record_free_t free_record, 51 | int skip_writeback, 52 | void* user_ctx); 53 | -------------------------------------------------------------------------------- /src/iobuffer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef LIBCOUCHSTORE_IOBUFFER_H 19 | #define LIBCOUCHSTORE_IOBUFFER_H 1 20 | 21 | #include 22 | 23 | /* Buffered IO parameters */ 24 | struct buffered_file_ops_params { 25 | buffered_file_ops_params(); 26 | buffered_file_ops_params(const buffered_file_ops_params& src); 27 | buffered_file_ops_params(const bool _read_only, 28 | bool _tracing_enabled, 29 | bool _write_validation_enabled, 30 | bool _mprotect_enabled, 31 | const uint32_t _read_buffer_capacity, 32 | const uint32_t _max_read_buffers); 33 | 34 | // Flag indicating whether or not the file is being opened as read only. 35 | bool readOnly; 36 | /* Flags for tracing and validation */ 37 | bool tracing_enabled; 38 | bool write_validation_enabled; 39 | bool mprotect_enabled; 40 | // Read buffer capacity. 41 | uint32_t read_buffer_capacity; 42 | // Max read buffer count. 43 | uint32_t max_read_buffers; 44 | }; 45 | 46 | /** 47 | * Constructs a set of file ops that buffer the I/O provided by an 48 | * underlying set of raw ops. 49 | * 50 | * @param raw_ops the file ops callbacks to use for the underlying I/O 51 | * @param handle on output, a constructed (but not opened) couch_file_handle 52 | * @param params buffered IO parameters 53 | * @return the couch_file_ops to use, or NULL on failure 54 | */ 55 | FileOpsInterface* couch_get_buffered_file_ops(couchstore_error_info_t *errinfo, 56 | FileOpsInterface* raw_ops, 57 | couch_file_handle* handle, 58 | buffered_file_ops_params params); 59 | 60 | class BufferedFileOps : public FileOpsInterface { 61 | public: 62 | BufferedFileOps() {} 63 | 64 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override ; 65 | couchstore_error_t open(couchstore_error_info_t* errinfo, 66 | couch_file_handle* handle, const char* path, 67 | int oflag) override; 68 | couchstore_error_t close(couchstore_error_info_t* errinfo, 69 | couch_file_handle handle) override; 70 | couchstore_error_t set_periodic_sync(couch_file_handle handle, 71 | uint64_t period_bytes) override; 72 | couchstore_error_t set_tracing_enabled(couch_file_handle handle) override; 73 | couchstore_error_t set_write_validation_enabled( 74 | couch_file_handle handle) override; 75 | couchstore_error_t set_mprotect_enabled(couch_file_handle handle) override; 76 | ssize_t pread(couchstore_error_info_t* errinfo, 77 | couch_file_handle handle, void* buf, size_t nbytes, 78 | cs_off_t offset) override; 79 | ssize_t pwrite(couchstore_error_info_t* errinfo, 80 | couch_file_handle handle, const void* buf, 81 | size_t nbytes, cs_off_t offset) override; 82 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 83 | couch_file_handle handle) override; 84 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 85 | couch_file_handle handle) override; 86 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 87 | couch_file_handle handle, cs_off_t offset, 88 | cs_off_t len, 89 | couchstore_file_advice_t advice) override; 90 | FHStats* get_stats(couch_file_handle handle) override; 91 | 92 | void destructor(couch_file_handle handle) override; 93 | 94 | couch_file_handle constructor(couchstore_error_info_t *errinfo, 95 | FileOpsInterface* raw_ops, 96 | buffered_file_ops_params params); 97 | 98 | // Allocate memory for the read and write buffers 99 | void allocate_read_buffer(couch_file_handle handle); 100 | void allocate_write_buffer(couch_file_handle handle); 101 | 102 | void free_buffers(couch_file_handle handle) override; 103 | }; 104 | 105 | #endif // LIBCOUCHSTORE_IOBUFFER_H 106 | -------------------------------------------------------------------------------- /src/log_last_internal_error.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | #pragma once 11 | 12 | #include 13 | 14 | void log_last_internal_error(const char* format, ...) CB_FORMAT_PRINTF(1, 2); 15 | -------------------------------------------------------------------------------- /src/node_types.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | // 3 | // node_types.c 4 | // couchstore 5 | // 6 | // Created by Jens Alfke on 4/25/12. 7 | // Copyright (c) 2012 Couchbase, Inc. All rights reserved. 8 | // 9 | 10 | #include "node_types.h" 11 | 12 | #include 13 | #include 14 | 15 | size_t read_kv(const void *buf, sized_buf *key, sized_buf *value) 16 | { 17 | const raw_kv_length* kvlen = static_cast(buf); 18 | uint32_t klen, vlen; 19 | decode_kv_length(kvlen, &klen, &vlen); 20 | key->size = klen; 21 | key->buf = (char*)(kvlen + 1); 22 | value->size = vlen; 23 | value->buf = key->buf + klen; 24 | return sizeof(raw_kv_length) + klen + vlen; 25 | } 26 | 27 | void* write_kv(void *buf, sized_buf key, sized_buf value) 28 | { 29 | uint8_t *dst = static_cast(buf); 30 | *(raw_kv_length*)dst = encode_kv_length((uint32_t)key.size, (uint32_t)value.size); 31 | dst += sizeof(raw_kv_length); 32 | memcpy(dst, key.buf, key.size); 33 | dst += key.size; 34 | memcpy(dst, value.buf, value.size); 35 | dst += value.size; 36 | return dst; 37 | } 38 | 39 | node_pointer *read_root(void *buf, int size) 40 | { 41 | if (size == 0) { 42 | return nullptr; 43 | } 44 | 45 | raw_btree_root *raw = (raw_btree_root*)buf; 46 | node_pointer *ptr; 47 | uint64_t position = decode_raw48(raw->pointer); 48 | uint64_t subtreesize = decode_raw48(raw->subtreesize); 49 | int redsize = size - sizeof(*raw); 50 | 51 | ptr = (node_pointer *) cb_malloc(sizeof(node_pointer) + redsize); 52 | if (!ptr) { 53 | return nullptr; 54 | } 55 | if (redsize > 0) { 56 | buf = memcpy(reinterpret_cast(ptr + 1), raw + 1, redsize); 57 | } else { 58 | buf = nullptr; 59 | } 60 | ptr->key.buf = nullptr; 61 | ptr->key.size = 0; 62 | ptr->pointer = position; 63 | ptr->subtreesize = subtreesize; 64 | ptr->reduce_value.buf = static_cast(buf); 65 | ptr->reduce_value.size = redsize; 66 | return ptr; 67 | } 68 | 69 | size_t encode_root(void *buf, node_pointer *node) 70 | { 71 | if (!node) { 72 | return 0; 73 | } 74 | if (buf) { 75 | raw_btree_root *root = static_cast(buf); 76 | encode_raw48(node->pointer, &root->pointer); 77 | encode_raw48(node->subtreesize, &root->subtreesize); 78 | if (node->reduce_value.size > 0) { 79 | memcpy(root + 1, node->reduce_value.buf, node->reduce_value.size); 80 | } 81 | } 82 | return sizeof(raw_btree_root) + node->reduce_value.size; 83 | } 84 | 85 | void decode_kv_length(const raw_kv_length *kv, uint32_t *klen, uint32_t *vlen) 86 | { 87 | /* 12, 28 bit */ 88 | *klen = (uint16_t) ((kv->raw_kv[0] << 4) | ((kv->raw_kv[1] & 0xf0) >> 4)); 89 | memcpy(vlen, &kv->raw_kv[1], 4); 90 | *vlen = ntohl(*vlen) & 0x0FFFFFFF; 91 | } 92 | 93 | raw_kv_length encode_kv_length(size_t klen, size_t vlen) 94 | { 95 | raw_kv_length kv; 96 | uint32_t len = htonl((uint32_t)vlen); 97 | memcpy(&kv.raw_kv[1], &len, 4); 98 | kv.raw_kv[0] = (uint8_t)(klen >> 4); /* upper 8 bits of klen */ 99 | kv.raw_kv[1] |= (klen & 0xF) << 4; /* lower 4 bits of klen in upper half of byte */ 100 | return kv; 101 | } 102 | 103 | uint64_t decode_sequence_key(const sized_buf *buf) 104 | { 105 | const raw_by_seq_key *key = (const raw_by_seq_key*)buf->buf; 106 | return decode_raw48(key->sequence); 107 | } 108 | -------------------------------------------------------------------------------- /src/node_types.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** node_types.h 4 | ** couchstore 5 | ** 6 | ** Created by Jens Alfke on 4/25/12. 7 | ** Modified by Filipe Manana on 6/19/13 to fix some GCC warnings regarding 8 | ** violation of strict aliasing rules. 9 | ** 10 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 11 | ** 12 | */ 13 | 14 | #pragma once 15 | 16 | #include "bitfield.h" 17 | #include "internal.h" 18 | 19 | struct raw_file_header_v12 { 20 | raw_08 version; 21 | raw_48 update_seq; 22 | raw_48 purge_seq; 23 | raw_48 purge_ptr; 24 | raw_16 seqrootsize; 25 | raw_16 idrootsize; 26 | raw_16 localrootsize; 27 | /* Three variable-size raw_btree_root structures follow */ 28 | }; 29 | static_assert(sizeof(raw_file_header_v12) == 25, "Unexpected file header size"); 30 | 31 | struct raw_file_header_v13 { 32 | raw_08 version; 33 | raw_48 update_seq; 34 | raw_48 purge_seq; 35 | raw_48 purge_ptr; 36 | raw_16 seqrootsize; 37 | raw_16 idrootsize; 38 | raw_16 localrootsize; 39 | raw_64 timestamp; 40 | /* Three variable-size raw_btree_root structures follow */ 41 | }; 42 | static_assert(sizeof(raw_file_header_v13) == 33, "Unexpected file header size"); 43 | 44 | struct raw_file_header_v14 { 45 | raw_08 version; 46 | raw_48 update_seq; 47 | raw_48 purge_seq; 48 | raw_48 purge_ptr; 49 | raw_16 seqrootsize; 50 | raw_16 idrootsize; 51 | raw_16 localrootsize; 52 | raw_64 timestamp; 53 | raw_48 prev_header_pos; // Least significant bit is have_metadata_header 54 | /* Three variable-size raw_btree_root structures follow */ 55 | }; 56 | static_assert(sizeof(raw_file_header_v14) == 39, "Unexpected file header size"); 57 | 58 | struct raw_btree_root { 59 | raw_48 pointer; 60 | raw_48 subtreesize; 61 | /* Variable-size reduce value follows */ 62 | }; 63 | 64 | /** Packed key-and-value length type. Key length is 12 bits, value length is 28. */ 65 | struct raw_kv_length { 66 | uint8_t raw_kv[5]; 67 | }; 68 | 69 | struct raw_node_pointer { 70 | raw_48 pointer; 71 | raw_48 subtreesize; 72 | raw_16 reduce_value_size; 73 | /* Variable-size reduce value follows */ 74 | }; 75 | 76 | struct raw_by_seq_key { 77 | raw_48 sequence; 78 | }; 79 | 80 | struct raw_id_index_value { 81 | raw_48 db_seq; 82 | /* physical on-disk size of the value (including headers). */ 83 | raw_32 physical_size; 84 | raw_48 bp; /* high bit is 'deleted' flag */ 85 | raw_48 rev_seq; 86 | raw_08 content_meta; 87 | /* Variable-size rev_meta data follows */ 88 | }; 89 | 90 | struct raw_seq_index_value { 91 | /* value length - physical on-disk size of the value (including headers). */ 92 | raw_kv_length sizes; 93 | raw_48 bp; /* high bit is 'deleted' flag */ 94 | raw_48 rev_seq; 95 | raw_08 content_meta; 96 | /* Variable-size id follows */ 97 | /* Variable-size rev_meta data follows */ 98 | }; 99 | 100 | /* Mask for the 'deleted' bit in .bp fields */ 101 | #ifndef UINT64_C 102 | #define UINT64_C(x) (x ## ULL) 103 | #endif 104 | #define BP_DELETED_FLAG UINT64_C(0x800000000000) 105 | 106 | 107 | node_pointer *read_root(void *buf, int size); 108 | 109 | size_t encode_root(void *buf, node_pointer *node); 110 | 111 | 112 | /** 113 | * Reads a 12-bit key length and 28-bit value length, packed into 5 bytes big-endian. 114 | */ 115 | void decode_kv_length(const raw_kv_length *kv, uint32_t *klen, uint32_t *vlen); 116 | 117 | /** 118 | * Returns an encoded 5-byte key/value length pair. 119 | */ 120 | raw_kv_length encode_kv_length(size_t klen, size_t vlen); 121 | 122 | /** 123 | * Parses an in-memory buffer containing a 5-byte key/value length followed by key and value data, 124 | * and fills in sized_bufs to point to the key and data. 125 | * @return Number of bytes consumed from the buffer 126 | */ 127 | size_t read_kv(const void *buf, sized_buf *key, sized_buf *value); 128 | 129 | void* write_kv(void *buf, sized_buf key, sized_buf value); 130 | 131 | 132 | /** 133 | * Reads a 48-bit sequence number out of a sized_buf. 134 | */ 135 | uint64_t decode_sequence_key(const sized_buf *buf); 136 | 137 | -------------------------------------------------------------------------------- /src/os.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | class FileOps : public FileOpsInterface { 23 | public: 24 | FileOps() {} 25 | 26 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override ; 27 | couchstore_error_t open(couchstore_error_info_t* errinfo, 28 | couch_file_handle* handle, const char* path, 29 | int oflag) override; 30 | couchstore_error_t close(couchstore_error_info_t* errinfo, 31 | couch_file_handle handle) override; 32 | couchstore_error_t set_periodic_sync(couch_file_handle handle, 33 | uint64_t period_bytes) override; 34 | ssize_t pread(couchstore_error_info_t* errinfo, 35 | couch_file_handle handle, void* buf, size_t nbytes, 36 | cs_off_t offset) override; 37 | ssize_t pwrite(couchstore_error_info_t* errinfo, 38 | couch_file_handle handle, const void* buf, 39 | size_t nbytes, cs_off_t offset) override; 40 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 41 | couch_file_handle handle) override; 42 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 43 | couch_file_handle handle) override; 44 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 45 | couch_file_handle handle, cs_off_t offset, 46 | cs_off_t len, 47 | couchstore_file_advice_t advice) override; 48 | void destructor(couch_file_handle handle) override; 49 | }; 50 | -------------------------------------------------------------------------------- /src/program_getopt.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | 11 | #include "program_getopt.h" 12 | #include 13 | #include 14 | #include 15 | 16 | using cb::getopt::Argument; 17 | using cb::getopt::Option; 18 | 19 | namespace cb::couchstore { 20 | 21 | ProgramGetopt::ProgramGetopt() 22 | : dumpKeysExecutable(DESTINATION_ROOT "/bin/dump-keys"), 23 | gosecrets(DESTINATION_ROOT "/var/lib/couchbase/config/gosecrets.cfg") { 24 | addOption({ 25 | [this](auto value) { dumpKeysExecutable = std::string{value}; }, 26 | "with-dump-keys", 27 | Argument::Required, 28 | "filename", 29 | fmt::format("The \"dump-keys\" binary to use (by default {}", 30 | dumpKeysExecutable), 31 | }); 32 | addOption({[this](auto value) { gosecrets = std::string{value}; }, 33 | "with-gosecrets", 34 | Argument::Required, 35 | "filename", 36 | fmt::format("The location of gosecrets.cfg (by default {})", 37 | gosecrets)}); 38 | addOption({[this](auto value) { 39 | if (value == "-") { 40 | password = getpass(); 41 | } else { 42 | password = std::string{value}; 43 | } 44 | }, 45 | "password", 46 | Argument::Required, 47 | "password", 48 | "The password to use for authentication (use '-' to read from " 49 | "standard input)"}); 50 | addOption({[](auto) { 51 | fmt::println(stdout, "Couchbase Server {}", PRODUCT_VERSION); 52 | std::exit(EXIT_SUCCESS); 53 | }, 54 | "version", 55 | "Print program version and exit"}); 56 | } 57 | 58 | void ProgramGetopt::addOption(Option option) { 59 | parser.addOption(std::move(option)); 60 | } 61 | 62 | std::vector ProgramGetopt::parse( 63 | int argc, char* const* argv, std::function error) const { 64 | return parser.parse(argc, argv, std::move(error)); 65 | } 66 | 67 | void ProgramGetopt::usage(std::ostream& out) const { 68 | parser.usage(out); 69 | } 70 | 71 | crypto::SharedEncryptionKey ProgramGetopt::lookup(std::string_view id) { 72 | if (id.empty()) { 73 | return keyStore.getActiveKey(); 74 | } 75 | 76 | auto ret = keyStore.lookup(id); 77 | if (ret) { 78 | return ret; 79 | } 80 | 81 | auto dumpKeysRunner = cb::crypto::DumpKeysRunner::create( 82 | password, dumpKeysExecutable, gosecrets); 83 | auto key = dumpKeysRunner->lookup(id); 84 | if (key) { 85 | // add for later requests 86 | keyStore.add(key); 87 | // set it as the active key. This is useful if for command line 88 | // utilities which opens file A for reading and B for writing.. 89 | // then they'll reuse the same key (will of course fail if they 90 | // open B before A.. that'll cause the file to be unencrypted...) 91 | keyStore.setActiveKey(key); 92 | } 93 | return key; 94 | } 95 | 96 | std::ostream& operator<<(std::ostream& os, 97 | const ProgramGetopt& programOptions) { 98 | programOptions.usage(os); 99 | return os; 100 | } 101 | 102 | } // namespace cb::couchstore 103 | -------------------------------------------------------------------------------- /src/program_getopt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace cb::couchstore { 18 | 19 | class ProgramGetopt { 20 | public: 21 | ProgramGetopt(); 22 | 23 | /** 24 | * Add a command line option to the list of command line options 25 | * to accept 26 | */ 27 | void addOption(getopt::Option option); 28 | 29 | /** 30 | * Parse the command line options and call the callbacks for all 31 | * options found. 32 | * 33 | * @param argc argument count 34 | * @param argv argument vector 35 | * @param error an error callback for unknown options 36 | */ 37 | [[nodiscard]] std::vector parse( 38 | int argc, char* const* argv, std::function error) const; 39 | 40 | /// Print the common command line options to the output stream 41 | void usage(std::ostream& out) const; 42 | 43 | /// Get a lookup function which cache the result of key lookup 44 | [[nodiscard]] std::function 45 | getKeyLookupFunction() { 46 | return [this](std::string_view id) { return lookup(id); }; 47 | } 48 | 49 | protected: 50 | /// Look up the provided key id (and cache it for later use) 51 | [[nodiscard]] crypto::SharedEncryptionKey lookup(std::string_view id); 52 | 53 | getopt::CommandLineOptionsParser parser; 54 | std::string dumpKeysExecutable; 55 | std::string password; 56 | std::string gosecrets; 57 | crypto::KeyStore keyStore; 58 | }; 59 | 60 | std::ostream& operator<<(std::ostream& os, const ProgramGetopt& programOptions); 61 | 62 | } // namespace cb::couchstore 63 | -------------------------------------------------------------------------------- /src/quicksort.h: -------------------------------------------------------------------------------- 1 | #ifndef QUICKSORT_H 2 | #define QUICKSORT_H 3 | 4 | typedef int sort_cmp_t(const void *, const void *, void *); 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | void quicksort(void *a, size_t n, size_t es, sort_cmp_t *cmp, void *thunk); 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/reduces.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "couchstore_config.h" 3 | #include 4 | #include 5 | #include "reduces.h" 6 | #include "node_types.h" 7 | 8 | 9 | couchstore_error_t by_seq_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx) 10 | { 11 | raw_by_seq_reduce *raw = (raw_by_seq_reduce*)dst; 12 | 13 | (void) leaflist; 14 | (void) ctx; 15 | encode_raw40(count, &raw->count); 16 | *size_r = sizeof(*raw); 17 | 18 | return COUCHSTORE_SUCCESS; 19 | } 20 | 21 | couchstore_error_t by_seq_rereduce(char *dst, size_t *size_r, const nodelist *ptrlist, int count, void *ctx) 22 | { 23 | uint64_t total = 0; 24 | const nodelist *i = ptrlist; 25 | 26 | (void) ctx; 27 | 28 | while (i != nullptr && count > 0) { 29 | const raw_by_seq_reduce *reduce = (const raw_by_seq_reduce*) i->pointer->reduce_value.buf; 30 | total += decode_raw40(reduce->count); 31 | 32 | i = i->next; 33 | count--; 34 | } 35 | raw_by_seq_reduce *raw = (raw_by_seq_reduce*)dst; 36 | encode_raw40(total, &raw->count); 37 | *size_r = sizeof(*raw); 38 | 39 | return COUCHSTORE_SUCCESS; 40 | } 41 | 42 | 43 | static size_t encode_by_id_reduce(char *dst, uint64_t notdeleted, uint64_t deleted, uint64_t size) 44 | { 45 | raw_by_id_reduce *raw = (raw_by_id_reduce*)dst; 46 | encode_raw40(notdeleted, &raw->notdeleted); 47 | encode_raw40(deleted, &raw->deleted); 48 | encode_raw48(size, &raw->size); 49 | return sizeof(*raw); 50 | } 51 | 52 | couchstore_error_t by_id_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx) 53 | { 54 | uint64_t notdeleted = 0, deleted = 0, physical_size = 0; 55 | const nodelist *i = leaflist; 56 | 57 | (void) ctx; 58 | 59 | while (i != nullptr && count > 0) { 60 | const raw_id_index_value *raw = (const raw_id_index_value*)i->data.buf; 61 | if (decode_raw48(raw->bp) & BP_DELETED_FLAG) { 62 | deleted++; 63 | } else { 64 | notdeleted++; 65 | } 66 | physical_size += decode_raw32(raw->physical_size); 67 | 68 | i = i->next; 69 | count--; 70 | } 71 | 72 | *size_r = encode_by_id_reduce(dst, notdeleted, deleted, physical_size); 73 | 74 | return COUCHSTORE_SUCCESS; 75 | 76 | } 77 | 78 | couchstore_error_t by_id_rereduce(char *dst, size_t *size_r, const nodelist *ptrlist, int count, void *ctx) 79 | { 80 | uint64_t notdeleted = 0, deleted = 0, size = 0; 81 | const nodelist *i = ptrlist; 82 | 83 | (void) ctx; 84 | 85 | while (i != nullptr && count > 0) { 86 | const raw_by_id_reduce *reduce = (const raw_by_id_reduce*) i->pointer->reduce_value.buf; 87 | notdeleted += decode_raw40(reduce->notdeleted); 88 | deleted += decode_raw40(reduce->deleted); 89 | size += decode_raw48(reduce->size); 90 | 91 | i = i->next; 92 | count--; 93 | } 94 | 95 | *size_r = encode_by_id_reduce(dst, notdeleted, deleted, size); 96 | 97 | return COUCHSTORE_SUCCESS; 98 | } 99 | -------------------------------------------------------------------------------- /src/reduces.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "bitfield.h" 4 | #include "couch_btree.h" 5 | #include 6 | #include 7 | 8 | struct raw_by_seq_reduce { 9 | raw_40 count; 10 | }; 11 | 12 | struct raw_by_id_reduce { 13 | raw_40 notdeleted; 14 | raw_40 deleted; 15 | raw_48 size; 16 | }; 17 | 18 | couchstore_error_t by_seq_reduce(char* dst, 19 | size_t* size_r, 20 | const nodelist* leaflist, 21 | int count, 22 | void* ctx); 23 | couchstore_error_t by_seq_rereduce(char* dst, 24 | size_t* size_r, 25 | const nodelist* leaflist, 26 | int count, 27 | void* ctx); 28 | 29 | couchstore_error_t by_id_rereduce(char* dst, 30 | size_t* size_r, 31 | const nodelist* leaflist, 32 | int count, 33 | void* ctx); 34 | couchstore_error_t by_id_reduce(char* dst, 35 | size_t* size_r, 36 | const nodelist* leaflist, 37 | int count, 38 | void* ctx); 39 | -------------------------------------------------------------------------------- /src/stream.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024-Present Couchbase, Inc. 3 | * 4 | * Use of this software is governed by the Business Source License included 5 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 6 | * in that file, in accordance with the Business Source License, use of this 7 | * software will be governed by the Apache License, Version 2.0, included in 8 | * the file licenses/APL2.txt. 9 | */ 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace cb::crypto { 19 | class SymmetricCipher; 20 | } 21 | 22 | namespace cb::couchstore { 23 | 24 | /** 25 | * Provides access to a sequence of bytes (a file). 26 | */ 27 | class Stream { 28 | public: 29 | /** 30 | * Reads bytes into the buffer. 31 | * @return false on EOF 32 | */ 33 | virtual bool read(std::span buffer) = 0; 34 | 35 | /** 36 | * Writes the contents of the buffer. 37 | */ 38 | virtual void write(std::string_view buffer) = 0; 39 | 40 | /** 41 | * Writes the buffered bytes to the underlying file. 42 | */ 43 | virtual void flush() = 0; 44 | 45 | /** 46 | * Seeks to the beginning of the file. 47 | */ 48 | virtual void seek_begin() = 0; 49 | 50 | /** 51 | * Seeks to the end of the file. 52 | */ 53 | virtual void seek_end() = 0; 54 | 55 | /** 56 | * Closes the file and destroys the stream. 57 | */ 58 | virtual ~Stream() = default; 59 | }; 60 | 61 | /** 62 | * Constructs a stream of a file. 63 | * 64 | * @param path File path 65 | * @param mode File open mode to be passed to fopen() 66 | */ 67 | std::unique_ptr make_file_stream(const std::filesystem::path& path, 68 | const char* mode); 69 | 70 | /** 71 | * Constructs a stream that checksums data and uses an underlying stream. 72 | * 73 | * Overwriting existing data may produce a corrupted file. 74 | * 75 | * @param underlying The underlying stream to use 76 | * @param max_write_buffer Maximum data to buffer before checksum 77 | */ 78 | std::unique_ptr make_checksum_stream(std::unique_ptr underlying, 79 | size_t max_write_buffer = 0x10000); 80 | 81 | /** 82 | * Constructs a stream that encrypts data and uses an underlying stream. 83 | * 84 | * Overwriting existing data may produce a corrupted file. 85 | * 86 | * @param underlying The underlying stream to use 87 | * @param cipher Encryption cipher to use 88 | * @param max_write_buffer Maximum plaintext to buffer before encrypting 89 | */ 90 | std::unique_ptr make_encrypted_stream( 91 | std::unique_ptr underlying, 92 | std::shared_ptr cipher, 93 | size_t max_write_buffer = 0x10000); 94 | 95 | } // namespace cb::couchstore 96 | -------------------------------------------------------------------------------- /src/strerror.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | const char* couchstore_strerror(couchstore_error_t errcode) { 4 | switch (errcode) { 5 | case COUCHSTORE_SUCCESS: 6 | return "success"; 7 | case COUCHSTORE_ERROR_OPEN_FILE: 8 | return "error opening file"; 9 | case COUCHSTORE_ERROR_CORRUPT: 10 | return "malformed data in file"; 11 | case COUCHSTORE_ERROR_ALLOC_FAIL: 12 | return "failed to allocate buffer"; 13 | case COUCHSTORE_ERROR_READ: 14 | return "error reading file"; 15 | case COUCHSTORE_ERROR_DOC_NOT_FOUND: 16 | return "document not found"; 17 | case COUCHSTORE_ERROR_NO_HEADER: 18 | return "no header in non-empty file"; 19 | case COUCHSTORE_ERROR_WRITE: 20 | return "error writing to file"; 21 | case COUCHSTORE_ERROR_HEADER_VERSION: 22 | return "incorrect version in header"; 23 | case COUCHSTORE_ERROR_CHECKSUM_FAIL: 24 | return "checksum fail"; 25 | case COUCHSTORE_ERROR_INVALID_ARGUMENTS: 26 | return "invalid arguments"; 27 | case COUCHSTORE_ERROR_NO_SUCH_FILE: 28 | return "no such file"; 29 | case COUCHSTORE_ERROR_FILE_CLOSED: 30 | return "cannot do this operation when file is closed"; 31 | case COUCHSTORE_ERROR_DB_NO_LONGER_VALID: 32 | return "this db handle could have its file reopened and must be closed"; 33 | case COUCHSTORE_ERROR_FILE_CLOSE: 34 | return "error closing file"; 35 | case COUCHSTORE_ERROR_CANCEL: 36 | return "error cancel"; 37 | case COUCHSTORE_ERROR_NOT_SUPPORTED: 38 | return "not supported"; 39 | case COUCHSTORE_ERROR_REDUCTION_TOO_LARGE: 40 | return "reduction too large"; 41 | case COUCHSTORE_ERROR_REDUCER_FAILURE: 42 | return "reducer failure"; 43 | case COUCHSTORE_ERROR_SCAN_YIELD: 44 | return "scan yield"; 45 | case COUCHSTORE_ERROR_SCAN_CANCELLED: 46 | return "scan cancelled"; 47 | case COUCHSTORE_ERROR_NO_ENCRYPTION_KEY: 48 | return "encryption key not found"; 49 | case COUCHSTORE_ERROR_ENCRYPT: 50 | return "encryption failed"; 51 | case COUCHSTORE_ERROR_DECRYPT: 52 | return "decryption failed"; 53 | } 54 | return "couchstore_strerror unknown errcode"; 55 | } 56 | -------------------------------------------------------------------------------- /src/tracking_file_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | /** 21 | * FileOp class which builds up a map of the underlying couchstore file; by 22 | * recording read() and write() offsets and sizes. 23 | */ 24 | class TrackingFileOps : public FileOpsInterface { 25 | public: 26 | enum class Tree : uint8_t { Unknown, Sequence, Id, Local }; 27 | 28 | TrackingFileOps() { 29 | } 30 | 31 | // Implementation of FileOpsInterface /////////////////////////////////// 32 | 33 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override; 34 | 35 | couchstore_error_t open(couchstore_error_info_t* errinfo, 36 | couch_file_handle* handle, 37 | const char* path, 38 | int oflag) override; 39 | couchstore_error_t close(couchstore_error_info_t* errinfo, 40 | couch_file_handle handle) override; 41 | ssize_t pread(couchstore_error_info_t* errinfo, 42 | couch_file_handle handle, 43 | void* buf, 44 | size_t nbytes, 45 | cs_off_t offset) override; 46 | ssize_t pwrite(couchstore_error_info_t* errinfo, 47 | couch_file_handle handle, 48 | const void* buf, 49 | size_t nbytes, 50 | cs_off_t offset) override; 51 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 52 | couch_file_handle handle) override; 53 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 54 | couch_file_handle handle) override; 55 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 56 | couch_file_handle handle, 57 | cs_off_t offset, 58 | cs_off_t len, 59 | couchstore_file_advice_t advice) override; 60 | 61 | void tag(couch_file_handle handle, FileTag tag) override; 62 | 63 | void destructor(couch_file_handle handle) override; 64 | 65 | // Own methods. 66 | void setTree(couch_file_handle handle, Tree tree); 67 | 68 | /// Set true when accessing historic (old) headers and data. 69 | void setHistoricData(couch_file_handle handle, bool historic); 70 | 71 | private: 72 | /// Object representing a file, and the information we track about it. 73 | class File; 74 | }; 75 | -------------------------------------------------------------------------------- /src/tree_writer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "couch_btree.h" 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | namespace cb::couchstore { 12 | 13 | struct StreamHolder; 14 | 15 | /** 16 | * Writes key/value pairs to a tree file after external-merge-sorting them. 17 | * 18 | * Operations order: 19 | * 20 | * 
 21 |  * open()
 22 |  * add()
 23 |  * add()
 24 |  * sort()
 25 |  * write()
 26 |  * close()
 27 |  *
 28 |  * enable_encryption()
 29 |  * open()
 30 |  * add()
 31 |  * sort()
 32 |  * write()
 33 |  * close()
 34 |  *
35 | */ 36 | class TreeWriter { 37 | public: 38 | struct KeyValue { 39 | std::string key; 40 | std::string value; 41 | }; 42 | 43 | TreeWriter(); 44 | 45 | ~TreeWriter(); 46 | 47 | /** 48 | * Opens a file for adding items and sorting. 49 | * 50 | * Any previously open file must be closed before calling. 51 | * 52 | * @param file_path Path to file which may containing a series of key/value 53 | * pairs in TreeWriter format. 54 | * @param open_existing Whether to read/ add to an existing file 55 | * @param key_compare Callback function that compares two keys. 56 | * @return Error code or COUCHSTORE_SUCCESS. 57 | */ 58 | couchstore_error_t open(const char* file_path, 59 | bool open_existing, 60 | compare_callback key_compare, 61 | reduce_fn reduce, 62 | reduce_fn rereduce, 63 | void* user_reduce_ctx); 64 | 65 | /** 66 | * Closes the open key/value file, if any. 67 | */ 68 | void close(); 69 | 70 | /** 71 | * Adds a key/value pair to the file. These can be added in any order. 72 | */ 73 | couchstore_error_t add(sized_buf key, sized_buf value); 74 | 75 | /** 76 | * Sorts the key/value pairs already added to the file. 77 | */ 78 | couchstore_error_t sort(); 79 | 80 | /** 81 | * Writes the key/value pairs to a tree file, returning a pointer to the 82 | * new root. The items should first have been sorted. 83 | */ 84 | couchstore_error_t write(tree_file* to_file, node_pointer** out_root); 85 | 86 | /** 87 | * Enables encryption for the key/value file. A random key is used. 88 | */ 89 | couchstore_error_t enable_encryption(); 90 | 91 | /// Counter for key/value pairs added in add() 92 | unsigned long long num_added{0}; 93 | 94 | /// Counter for key/value pairs written to a new tree in write() 95 | unsigned long long num_written{0}; 96 | 97 | protected: 98 | /** 99 | * Creates a new stream of a file with a file name generated from the 100 | * initial file name. When the StreamHolder is destroyed the underlying 101 | * file is removed. 102 | * 103 | * @param mode fopen() mode to open the file in 104 | */ 105 | std::unique_ptr create_stream(const char* mode); 106 | 107 | /** 108 | * Creates a new stream of the given file. When the StreamHolder is 109 | * destroyed the underlying file is removed. 110 | * 111 | * @param path File path 112 | * @param mode fopen() mode to open the file in 113 | */ 114 | std::unique_ptr create_stream(std::filesystem::path path, 115 | const char* mode); 116 | 117 | /** 118 | * Reads a key/value pair from a stream. 119 | * 120 | * @return Read KeyValue record or std::nullopt on EOF 121 | */ 122 | static std::optional read_record(StreamHolder& sh); 123 | 124 | /** 125 | * Writes a key/value pair to a stream. 126 | */ 127 | static void write_record(StreamHolder& sh, 128 | std::string_view key, 129 | std::string_view value); 130 | 131 | /** 132 | * Sets the file position to the beginning of the file. 133 | */ 134 | static void rewind(StreamHolder& sh); 135 | 136 | /// Stream where key/value pairs are stored 137 | std::unique_ptr stream; 138 | /// Cipher to be used for encrypting Streams 139 | std::shared_ptr cipher; 140 | /// Callback for comparing keys 141 | compare_callback key_compare{nullptr}; 142 | reduce_fn reduce{nullptr}; 143 | reduce_fn rereduce{nullptr}; 144 | void* user_reduce_ctx{nullptr}; 145 | /// Buffer for temporary file name generation 146 | std::filesystem::path tmp_path; 147 | }; 148 | 149 | } // namespace cb::couchstore 150 | -------------------------------------------------------------------------------- /src/util.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "couchstore_config.h" 3 | #include "node_types.h" 4 | #include "util.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef DEBUG 12 | #include 13 | #endif 14 | 15 | int ebin_cmp(const sized_buf *e1, const sized_buf *e2) 16 | { 17 | size_t size; 18 | if (e2->size < e1->size) { 19 | size = e2->size; 20 | } else { 21 | size = e1->size; 22 | } 23 | 24 | int cmp = memcmp(e1->buf, e2->buf, size); 25 | if (cmp == 0) { 26 | if (size < e2->size) { 27 | return -1; 28 | } else if (size < e1->size) { 29 | return 1; 30 | } 31 | } 32 | return cmp; 33 | } 34 | 35 | int seq_cmp(const sized_buf *k1, const sized_buf *k2) 36 | { 37 | uint64_t e1val = decode_sequence_key(k1); 38 | uint64_t e2val = decode_sequence_key(k2); 39 | if (e1val == e2val) { 40 | return 0; 41 | } 42 | return (e1val < e2val ? -1 : 1); 43 | } 44 | 45 | void cb::couchstore::FatbufDeletor::operator()(fatbuf* fb) { 46 | fatbuf_free(fb); 47 | } 48 | 49 | fatbuf *fatbuf_alloc(size_t bytes) 50 | { 51 | fatbuf *fb = (fatbuf *) cb_malloc(sizeof(fatbuf) + bytes); 52 | #ifdef DEBUG 53 | memset(fb->buf, 0x44, bytes); 54 | #endif 55 | if (!fb) { 56 | return nullptr; 57 | } 58 | 59 | fb->size = bytes; 60 | fb->pos = 0; 61 | return fb; 62 | } 63 | 64 | void *fatbuf_get(fatbuf *fb, size_t bytes) 65 | { 66 | if (fb->pos + bytes > fb->size) { 67 | return nullptr; 68 | } 69 | #ifdef DEBUG 70 | if (fb->buf[fb->pos] != 0x44 && bytes > 0) { 71 | fprintf(stderr, "Fatbuf space has been written to before it was taken!\n"); 72 | } 73 | #endif 74 | void *rptr = fb->buf + fb->pos; 75 | fb->pos += bytes; 76 | return rptr; 77 | } 78 | 79 | void fatbuf_free(fatbuf *fb) 80 | { 81 | cb_free(fb); 82 | } 83 | 84 | #ifdef DEBUG 85 | void report_error(couchstore_error_t errcode, const char* file, int line) { 86 | fprintf(stderr, "Couchstore error `%s' at %s:%d\r\n", \ 87 | couchstore_strerror(errcode), file, line); 88 | } 89 | #endif 90 | 91 | sized_buf* arena_copy_buf(arena* a, const sized_buf *src) 92 | { 93 | sized_buf *nbuf = static_cast(arena_alloc(a, sizeof(sized_buf))); 94 | if (nbuf == nullptr) { 95 | return nullptr; 96 | } 97 | nbuf->buf = static_cast(arena_alloc(a, src->size)); 98 | if (nbuf->buf == nullptr) { 99 | return nullptr; 100 | } 101 | nbuf->size = src->size; 102 | memcpy(nbuf->buf, src->buf, src->size); 103 | return nbuf; 104 | } 105 | 106 | sized_buf* arena_special_copy_buf_and_revmeta(arena *a, const sized_buf *val, 107 | const DocInfo *docinfo) 108 | { 109 | sized_buf *nbuf = static_cast(arena_alloc(a, sizeof(sized_buf))); 110 | if (nbuf == nullptr) { 111 | return nullptr; 112 | } 113 | 114 | const raw_seq_index_value *raw = (const raw_seq_index_value*)val->buf; 115 | uint32_t idsize, datasize; 116 | decode_kv_length(&raw->sizes, &idsize, &datasize); 117 | 118 | nbuf->size = sizeof(*raw) + idsize + docinfo->rev_meta.size; 119 | nbuf->buf = static_cast(arena_alloc(a, nbuf->size)); 120 | if (nbuf->buf == nullptr) { 121 | return nullptr; 122 | } 123 | memcpy(nbuf->buf, val->buf, sizeof(*raw) + idsize); 124 | memcpy(nbuf->buf + sizeof(*raw) + idsize, docinfo->rev_meta.buf, 125 | docinfo->rev_meta.size); 126 | return nbuf; 127 | } 128 | 129 | cs_off_t align_to_next_block(cs_off_t offset) 130 | { 131 | if (offset % COUCH_BLOCK_SIZE != 0) { 132 | return offset + COUCH_BLOCK_SIZE - (offset % COUCH_BLOCK_SIZE); 133 | } 134 | return offset; 135 | } 136 | 137 | void log_last_internal_error(const char* format, ...) { 138 | va_list args; 139 | va_start(args, format); 140 | vsnprintf(internal_error_string, MAX_ERR_STR_LEN, format, args); 141 | va_end(args); 142 | } 143 | 144 | int strncpy_safe(char* d, const char* s, size_t n) { 145 | int b = snprintf(d, n, "%s", s); 146 | if (b < 0 || (size_t)b >= n) { 147 | return -1; 148 | } else { 149 | return 0; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "internal.h" 4 | #include "fatbuf.h" 5 | #include "arena.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | /** Plain lexicographic comparison of the contents of two sized_bufs. */ 12 | int ebin_cmp(const sized_buf *e1, const sized_buf *e2); 13 | 14 | /** Compares sequence numbers (48-bit big-endian unsigned ints) stored in sized_bufs. */ 15 | int seq_cmp(const sized_buf *k1, const sized_buf *k2); 16 | 17 | /* Copy buffer to arena */ 18 | sized_buf* arena_copy_buf(arena* a, const sized_buf *src); 19 | 20 | /* Copy rev_meta from docinfo and buffer to arena */ 21 | sized_buf* arena_special_copy_buf_and_revmeta(arena *a, const sized_buf *val, 22 | const DocInfo *docinfo); 23 | 24 | /** Offsets the pointer PTR by BYTES bytes. Result is of the same type as PTR. */ 25 | #define offsetby(PTR, BYTES) ((__typeof(PTR))((uint8_t*)(PTR) + (BYTES))) 26 | 27 | /* Aligns the offset to the start of the next block */ 28 | cs_off_t align_to_next_block(cs_off_t offset); 29 | 30 | /* Handle string truncation when copying strings. If the resulting string 'd' 31 | (including the terminating null) is not longer than n return 0 otherwise return 32 | -1 */ 33 | int strncpy_safe(char* d, const char* s, size_t n); 34 | 35 | /* Sets errcode to the result of C, and jumps to the cleanup: label if it's 36 | * nonzero. */ 37 | #ifdef DEBUG 38 | void report_error(couchstore_error_t, const char* file, int line); 39 | #define error_pass(C) \ 40 | do { \ 41 | if ((errcode = (C)) < 0) { \ 42 | report_error(errcode, __FILE__, __LINE__); \ 43 | goto cleanup; \ 44 | } \ 45 | } while (0) 46 | 47 | // Set errcode on error, without termination. 48 | #define error_tolerate(C) \ 49 | do { \ 50 | if ((C) < 0) { \ 51 | errcode = (C); \ 52 | report_error(errcode, __FILE__, __LINE__); \ 53 | } \ 54 | } while (0) 55 | #else 56 | #define error_pass(C) \ 57 | do { \ 58 | if ((errcode = (C)) < 0) { \ 59 | goto cleanup; \ 60 | } \ 61 | } while (0) 62 | 63 | #define error_tolerate(C) \ 64 | do { \ 65 | if ((C) < 0) { \ 66 | errcode = (C); \ 67 | } \ 68 | } while (0) 69 | #endif 70 | 71 | /* If the condition C evaluates to false/zero, sets errcode to E and jumps to the cleanup: label. */ 72 | #define error_unless(C, E) \ 73 | do { \ 74 | if(!(C)) { error_pass(E); } \ 75 | } while (0) 76 | 77 | /* If the parameter C is nonzero, sets errcode to E and jumps to the cleanup: label. */ 78 | #define error_nonzero(C, E) \ 79 | do { \ 80 | if((C) != 0) { error_pass(E); } \ 81 | } while (0) 82 | 83 | -------------------------------------------------------------------------------- /src/views/bin/couch_view_group_cleanup.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "couchstore_config.h" 22 | #include "../view_group.h" 23 | #include "../util.h" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "util.h" 31 | #include "../mapreduce/mapreduce.h" 32 | 33 | #define BUF_SIZE 8192 34 | 35 | int main(int argc, char *argv[]) 36 | { 37 | view_group_info_t* group_info = nullptr; 38 | uint64_t purge_count; 39 | int ret = 2; 40 | uint64_t header_pos; 41 | view_error_t error_info = {nullptr, nullptr, nullptr}; 42 | 43 | (void) argc; 44 | (void) argv; 45 | 46 | /* 47 | * Disable buffering for stdout/stderr 48 | */ 49 | setvbuf(stdout, (char*)nullptr, _IONBF, 0); 50 | setvbuf(stderr, (char*)nullptr, _IONBF, 0); 51 | 52 | if (set_binary_mode() < 0) { 53 | fprintf(stderr, "Error setting binary mode\n"); 54 | goto out; 55 | } 56 | 57 | group_info = couchstore_read_view_group_info(stdin, stderr); 58 | if (group_info == nullptr) { 59 | ret = COUCHSTORE_ERROR_ALLOC_FAIL; 60 | goto out; 61 | } 62 | 63 | ret = start_exit_listener(1 /*uses_v8*/); 64 | if (ret) { 65 | fprintf(stderr, "Error starting stdin exit listener thread\n"); 66 | goto out; 67 | } 68 | 69 | mapreduce_init(argv[0]); 70 | ret = couchstore_cleanup_view_group(group_info, 71 | &header_pos, 72 | &purge_count, 73 | &error_info); 74 | mapreduce_deinit(); 75 | 76 | if (ret != COUCHSTORE_SUCCESS) { 77 | if (error_info.error_msg != nullptr && 78 | error_info.view_name != nullptr) { 79 | fprintf(stderr, 80 | "Error cleaning up index for view `%s`, reason: %s\n", 81 | error_info.view_name, 82 | error_info.error_msg); 83 | } 84 | goto out; 85 | } 86 | 87 | fprintf(stdout, "PurgedCount %" PRIu64 "\n", purge_count); 88 | 89 | out: 90 | couchstore_free_view_group_info(group_info); 91 | cb_free((void *) error_info.error_msg); 92 | cb_free((void *) error_info.view_name); 93 | 94 | ret = (ret < 0) ? (100 + ret) : ret; 95 | _exit(ret); 96 | } 97 | -------------------------------------------------------------------------------- /src/views/bin/util.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "couchstore_config.h" 22 | #include "util.h" 23 | 24 | #include "../mapreduce/mapreduce.h" 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | static void do_exit(int ret, int uses_v8) 34 | { 35 | /* Notify and delete conditional variables, 36 | and join waiting threads when v8 is used */ 37 | if(uses_v8) { 38 | deinit_terminator_thread(true /*fatal_exit*/); 39 | } 40 | _exit(ret); 41 | } 42 | 43 | static void exit_thread_helper(int uses_v8) { 44 | char buf[4]; 45 | int len = fread(buf, 1, 4, stdin); 46 | 47 | /* If the other end closed the pipe */ 48 | if (len == 0) { 49 | do_exit(1, uses_v8); 50 | } else if (len == 4 && !strncmp(buf, "exit", 4)) { 51 | do_exit(1, uses_v8); 52 | } else { 53 | fprintf(stderr, "Error occured waiting for exit message (%d)\n", len); 54 | do_exit(2, uses_v8); 55 | } 56 | } 57 | 58 | /* Start a watcher thread to gracefully die on exit message */ 59 | int start_exit_listener(int uses_v8) { 60 | auto listener = std::thread{[uses_v8]() { exit_thread_helper(uses_v8); }}; 61 | listener.detach(); 62 | return 0; 63 | } 64 | 65 | int set_binary_mode() 66 | { 67 | try { 68 | cb::io::setBinaryMode(stdin); 69 | cb::io::setBinaryMode(stdout); 70 | cb::io::setBinaryMode(stderr); 71 | } catch (const std::system_error&) { 72 | return -1; 73 | } 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /src/views/bin/util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _BIN_UTILS_H 22 | #define _BIN_UTILS_H 23 | 24 | #include "couchstore_config.h" 25 | 26 | #include 27 | 28 | /* Start a thread to handle exit message*/ 29 | int start_exit_listener(int uses_v8); 30 | int set_binary_mode(void); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /src/views/bitmap.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "bitmap.h" 22 | #include "couchstore_config.h" 23 | #include 24 | 25 | #define CHUNK_BITS (sizeof(unsigned char) * CHAR_BIT) 26 | #define TOTAL_CHUNKS(map) sizeof(((map).chunks)) 27 | #define CHUNK_INDEX(map, bit) (TOTAL_CHUNKS(map) - 1 - ((bit) / CHUNK_BITS)) 28 | #define MAP_CHUNK(map, bit) ((map).chunks)[CHUNK_INDEX(map, bit)] 29 | #define CHUNK_OFFSET(bit) ((bit) % CHUNK_BITS) 30 | 31 | int is_bit_set(const bitmap_t* bm, uint16_t bit) { 32 | return (MAP_CHUNK(*bm, bit) & (1 << CHUNK_OFFSET(bit))) != 0; 33 | } 34 | 35 | void set_bit(bitmap_t* bm, uint16_t bit) { 36 | (MAP_CHUNK(*bm, bit)) |= (1 << CHUNK_OFFSET(bit)); 37 | } 38 | 39 | void unset_bit(bitmap_t* bm, uint16_t bit) { 40 | ((MAP_CHUNK(*bm, bit)) &= ~(1 << CHUNK_OFFSET(bit))); 41 | } 42 | 43 | void union_bitmaps(bitmap_t* dst_bm, const bitmap_t* src_bm) { 44 | unsigned int i; 45 | for (i = 0; i < 1024 / CHUNK_BITS; ++i) { 46 | dst_bm->chunks[i] |= src_bm->chunks[i]; 47 | } 48 | } 49 | 50 | void intersect_bitmaps(bitmap_t* dst_bm, const bitmap_t* src_bm) { 51 | unsigned int i; 52 | for (i = 0; i < 1024 / CHUNK_BITS; ++i) { 53 | dst_bm->chunks[i] &= src_bm->chunks[i]; 54 | } 55 | } 56 | 57 | int is_equal_bitmap(const bitmap_t* bm1, const bitmap_t* bm2) { 58 | return !memcmp(bm1, bm2, sizeof(bitmap_t)); 59 | } 60 | -------------------------------------------------------------------------------- /src/views/bitmap.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include 24 | #include 25 | 26 | struct bitmap_t { 27 | /* Big endian format. 28 | * chunk[0] msb contains msb of the 1024 bits bitmap. 29 | */ 30 | unsigned char chunks[1024 / (sizeof(unsigned char) * CHAR_BIT)]{}; 31 | }; 32 | 33 | int is_bit_set(const bitmap_t *bm, uint16_t bit); 34 | void set_bit(bitmap_t *bm, uint16_t bit); 35 | void unset_bit(bitmap_t *bm, uint16_t bit); 36 | void union_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm); 37 | void intersect_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm); 38 | int is_equal_bitmap(const bitmap_t *bm1, const bitmap_t *bm2); 39 | -------------------------------------------------------------------------------- /src/views/collate_json.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** collate_json.h 3 | ** couchstore 4 | ** 5 | ** Created by Jens Alfke on 7/9/12. 6 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 7 | */ 8 | 9 | #pragma once 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | enum CollateJSONMode { 19 | kCollateJSON_Unicode, /* Compare strings as Unicode (CouchDB's default) */ 20 | kCollateJSON_Raw, /* CouchDB's "raw" collation rules */ 21 | kCollateJSON_ASCII /* Like Unicode except strings are compared as binary 22 | UTF-8 */ 23 | }; 24 | 25 | /* Custom deleter for UCollator */ 26 | struct UCollDeleter { 27 | void operator() (UCollator* coll) { 28 | ucol_close(coll); 29 | } 30 | }; 31 | 32 | /* Custom deleter for UConverter */ 33 | struct UConvDeleter { 34 | void operator() (UConverter* cnv) { 35 | ucnv_close(cnv); 36 | } 37 | }; 38 | 39 | /** 40 | * Compares two UTF-8 JSON strings using CouchDB's collation rules. 41 | * CAREFUL: The two strings must be valid JSON, with no extraneous whitespace, 42 | * otherwise this function will return wrong results or even crash. 43 | */ 44 | int CollateJSON(const sized_buf *buf1, 45 | const sized_buf *buf2, 46 | CollateJSONMode mode); 47 | 48 | /* not part of the API -- exposed for testing only (see collate_json_test.c) */ 49 | char ConvertJSONEscape(const char **in); 50 | -------------------------------------------------------------------------------- /src/views/compaction.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | #include "couchstore_config.h" 21 | #include "bitmap.h" 22 | #include "values.h" 23 | #include "compaction.h" 24 | #include "../couch_btree.h" 25 | 26 | int view_id_btree_filter(const sized_buf *k, const sized_buf *v, 27 | const bitmap_t *bm) 28 | { 29 | return is_bit_set(bm, decode_view_btree_partition(v->buf, v->size)); 30 | } 31 | 32 | int view_btree_filter(const sized_buf *k, const sized_buf *v, 33 | const bitmap_t *bm) 34 | { 35 | return is_bit_set(bm, decode_view_btree_partition(v->buf, v->size)); 36 | } 37 | -------------------------------------------------------------------------------- /src/views/compaction.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../couch_btree.h" 24 | #include "../internal.h" 25 | #include "bitmap.h" 26 | #include 27 | #include 28 | #include 29 | 30 | /* Filter function to selectively ignore values during compaction */ 31 | typedef int (*compact_filter_fn)(const sized_buf* k, 32 | const sized_buf* v, 33 | const bitmap_t* bm); 34 | 35 | /* Function spec for updating compactor progress */ 36 | typedef void (*stats_update_fn)(uint64_t freq, uint64_t inserted); 37 | 38 | struct compactor_stats_t { 39 | uint64_t freq; 40 | uint64_t inserted; 41 | stats_update_fn update_fun; 42 | }; 43 | 44 | /* Compaction context definition */ 45 | struct view_compact_ctx_t { 46 | couchfile_modify_result* mr; 47 | arena* transient_arena; 48 | const bitmap_t* filterbm; 49 | compact_filter_fn filter_fun; 50 | compactor_stats_t* stats; 51 | }; 52 | 53 | int view_id_btree_filter(const sized_buf* k, 54 | const sized_buf* v, 55 | const bitmap_t* bm); 56 | 57 | int view_btree_filter(const sized_buf* k, 58 | const sized_buf* v, 59 | const bitmap_t* bm); 60 | -------------------------------------------------------------------------------- /src/views/encoding.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2021-Present Couchbase, Inc. 4 | * 5 | * Use of this software is governed by the Business Source License included 6 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 7 | * in that file, in accordance with the Business Source License, use of this 8 | * software will be governed by the Apache License, Version 2.0, included in 9 | * the file licenses/APL2.txt. 10 | */ 11 | 12 | #include "encoding.h" 13 | 14 | #include "../bitfield.h" 15 | 16 | #include 17 | 18 | void enc_uint16(uint16_t u, char** buf) { 19 | raw_16 k = encode_raw16(u); 20 | std::memcpy(*buf, &k, 2); 21 | *buf += 2; 22 | } 23 | -------------------------------------------------------------------------------- /src/views/encoding.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2021-Present Couchbase, Inc. 4 | * 5 | * Use of this software is governed by the Business Source License included 6 | * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified 7 | * in that file, in accordance with the Business Source License, use of this 8 | * software will be governed by the Apache License, Version 2.0, included in 9 | * the file licenses/APL2.txt. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include 15 | 16 | void enc_uint16(uint16_t u, char** buf); 17 | -------------------------------------------------------------------------------- /src/views/file_merger.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "couchstore_config.h" 22 | #include "util.h" 23 | #include "file_merger.h" 24 | #include "file_sorter.h" 25 | #include "spatial.h" 26 | 27 | static file_merger_error_t merge_view_files(const char *source_files[], 28 | unsigned num_source_files, 29 | const char *dest_path, 30 | view_file_merge_ctx_t *ctx); 31 | 32 | file_merger_error_t merge_view_kvs_ops_files(const char *source_files[], 33 | unsigned num_source_files, 34 | const char *dest_path) 35 | { 36 | view_file_merge_ctx_t ctx; 37 | 38 | ctx.key_cmp_fun = view_key_cmp; 39 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 40 | 41 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 42 | } 43 | 44 | file_merger_error_t merge_view_ids_ops_files(const char *source_files[], 45 | unsigned num_source_files, 46 | const char *dest_path) 47 | { 48 | view_file_merge_ctx_t ctx; 49 | 50 | ctx.key_cmp_fun = view_id_cmp; 51 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 52 | 53 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 54 | } 55 | 56 | file_merger_error_t merge_spatial_kvs_ops_files(const char *source_files[], 57 | unsigned num_source_files, 58 | const char *dest_path, 59 | const char *tmp_dir) 60 | { 61 | file_sorter_error_t ret; 62 | view_file_merge_ctx_t ctx; 63 | unsigned i; 64 | 65 | ctx.key_cmp_fun = spatial_merger_key_cmp; 66 | ctx.type = INCREMENTAL_UPDATE_SPATIAL_RECORD; 67 | 68 | /* The spatial kv files are not sorted, hence sort them before merge 69 | * sorting them */ 70 | for(i = 0; i < num_source_files; ++i) { 71 | ret = sort_spatial_kvs_ops_file(source_files[i], tmp_dir, &ctx); 72 | if (ret != FILE_SORTER_SUCCESS) { 73 | fprintf(stderr, "Error sorting spatial view records file (%d): %s", 74 | ret, source_files[i]); 75 | return FILE_MERGER_SORT_ERROR; 76 | } 77 | } 78 | 79 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 80 | } 81 | 82 | 83 | static file_merger_error_t merge_view_files(const char *source_files[], 84 | unsigned num_source_files, 85 | const char *dest_path, 86 | view_file_merge_ctx_t *ctx) 87 | { 88 | return merge_files(source_files, 89 | num_source_files, 90 | dest_path, 91 | read_view_record, 92 | write_view_record, 93 | nullptr, 94 | compare_view_records, 95 | dedup_view_records_merger, 96 | free_view_record, 97 | 0, 98 | ctx); 99 | } 100 | -------------------------------------------------------------------------------- /src/views/file_merger.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "couchstore_config.h" 24 | #include 25 | #include 26 | #include "../file_merger.h" 27 | 28 | /* 29 | * Merge a group files containing sorted sets of btree operations for a 30 | * view btree. 31 | */ 32 | LIBCOUCHSTORE_API 33 | file_merger_error_t merge_view_kvs_ops_files(const char* source_files[], 34 | unsigned num_source_files, 35 | const char* dest_path); 36 | 37 | /* 38 | * Merge a group files containing sorted sets of btree operations for a 39 | * view id btree (back index). 40 | */ 41 | LIBCOUCHSTORE_API 42 | file_merger_error_t merge_view_ids_ops_files(const char* source_files[], 43 | unsigned num_source_files, 44 | const char* dest_path); 45 | 46 | /* 47 | * Merge a group files containing sets of operations for a spatial view 48 | * index. 49 | */ 50 | LIBCOUCHSTORE_API 51 | file_merger_error_t merge_spatial_kvs_ops_files(const char* source_files[], 52 | unsigned num_source_files, 53 | const char* dest_path, 54 | const char* tmp_dir); 55 | -------------------------------------------------------------------------------- /src/views/file_sorter.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "file_sorter.h" 22 | #include "util.h" 23 | #include "spatial.h" 24 | 25 | #define SORT_MAX_BUFFER_SIZE (64 * 1024 * 1024) 26 | #define SORT_MAX_NUM_TMP_FILES 16 27 | 28 | 29 | static file_sorter_error_t do_sort_file(const char *file_path, 30 | const char *tmp_dir, 31 | file_merger_feed_record_t callback, 32 | int skip_writeback, 33 | view_file_merge_ctx_t *ctx); 34 | 35 | file_sorter_error_t sort_view_kvs_ops_file(const char *file_path, 36 | const char *tmp_dir) 37 | { 38 | view_file_merge_ctx_t ctx; 39 | 40 | ctx.key_cmp_fun = view_key_cmp; 41 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 42 | 43 | return do_sort_file(file_path, tmp_dir, nullptr, 0, &ctx); 44 | } 45 | 46 | file_sorter_error_t sort_view_kvs_file(const char *file_path, 47 | const char *tmp_dir, 48 | file_merger_feed_record_t callback, 49 | void *user_ctx) 50 | { 51 | view_file_merge_ctx_t ctx; 52 | 53 | ctx.key_cmp_fun = view_key_cmp; 54 | ctx.type = INITIAL_BUILD_VIEW_RECORD; 55 | ctx.user_ctx = user_ctx; 56 | 57 | return do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 58 | } 59 | 60 | file_sorter_error_t sort_view_ids_ops_file(const char *file_path, 61 | const char *tmp_dir) 62 | { 63 | view_file_merge_ctx_t ctx; 64 | 65 | ctx.key_cmp_fun = view_id_cmp; 66 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 67 | 68 | return do_sort_file(file_path, tmp_dir, nullptr, 0, &ctx); 69 | } 70 | 71 | file_sorter_error_t sort_view_ids_file(const char *file_path, 72 | const char *tmp_dir, 73 | file_merger_feed_record_t callback, 74 | void *user_ctx) 75 | { 76 | view_file_merge_ctx_t ctx; 77 | 78 | ctx.key_cmp_fun = view_id_cmp; 79 | ctx.type = INITIAL_BUILD_VIEW_RECORD; 80 | ctx.user_ctx = user_ctx; 81 | 82 | return do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 83 | } 84 | 85 | file_sorter_error_t sort_spatial_kvs_file(const char *file_path, 86 | const char *tmp_dir, 87 | file_merger_feed_record_t callback, 88 | void *user_ctx) 89 | { 90 | file_sorter_error_t ret; 91 | view_file_merge_ctx_t ctx; 92 | 93 | ctx.key_cmp_fun = spatial_key_cmp; 94 | ctx.type = INITIAL_BUILD_SPATIAL_RECORD; 95 | ctx.user_ctx = user_ctx; 96 | 97 | ret = do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 98 | 99 | return ret; 100 | } 101 | 102 | file_sorter_error_t sort_spatial_kvs_ops_file(const char *file_path, 103 | const char *tmp_dir, 104 | view_file_merge_ctx_t *ctx) 105 | { 106 | return do_sort_file(file_path, tmp_dir, nullptr, 0, ctx); 107 | } 108 | 109 | 110 | static file_sorter_error_t do_sort_file(const char *file_path, 111 | const char *tmp_dir, 112 | file_merger_feed_record_t callback, 113 | int skip_writeback, 114 | view_file_merge_ctx_t *ctx) 115 | { 116 | return sort_file(file_path, 117 | tmp_dir, 118 | SORT_MAX_NUM_TMP_FILES, 119 | SORT_MAX_BUFFER_SIZE, 120 | read_view_record, 121 | write_view_record, 122 | callback, 123 | compare_view_records, 124 | free_view_record, 125 | skip_writeback, 126 | ctx); 127 | } 128 | -------------------------------------------------------------------------------- /src/views/file_sorter.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../file_sorter.h" 24 | #include "couchstore_config.h" 25 | #include "util.h" 26 | #include 27 | 28 | /* 29 | * Sort a file containing records of btree operations for a view btree. 30 | */ 31 | LIBCOUCHSTORE_API 32 | file_sorter_error_t sort_view_kvs_ops_file(const char* file_path, 33 | const char* tmp_dir); 34 | 35 | /* 36 | * Sort a file containing view records for a view btree. 37 | */ 38 | LIBCOUCHSTORE_API 39 | file_sorter_error_t sort_view_kvs_file(const char* file_path, 40 | const char* tmp_dir, 41 | file_merger_feed_record_t callback, 42 | void* user_ctx); 43 | 44 | /* 45 | * Sort a file containing records of btree operations for a view id 46 | * btree (back index). 47 | */ 48 | LIBCOUCHSTORE_API 49 | file_sorter_error_t sort_view_ids_ops_file(const char* file_path, 50 | const char* tmp_dir); 51 | 52 | /* 53 | * Sort a file containing records for a view id btree (back index). 54 | */ 55 | LIBCOUCHSTORE_API 56 | file_sorter_error_t sort_view_ids_file(const char* file_path, 57 | const char* tmp_dir, 58 | file_merger_feed_record_t callback, 59 | void* user_ctx); 60 | 61 | /* 62 | * Sort a file containing records for a spatial index. 63 | */ 64 | LIBCOUCHSTORE_API 65 | file_sorter_error_t sort_spatial_kvs_file(const char* file_path, 66 | const char* tmp_dir, 67 | file_merger_feed_record_t callback, 68 | void* user_ctx); 69 | 70 | /* 71 | * Sort a file containing records of spatial index operations for a 72 | * spatial view. 73 | */ 74 | LIBCOUCHSTORE_API 75 | file_sorter_error_t sort_spatial_kvs_ops_file(const char* file_path, 76 | const char* tmp_dir, 77 | view_file_merge_ctx_t* ctx); 78 | 79 | /* Record file sorter */ 80 | typedef file_sorter_error_t (*sort_record_fn)( 81 | const char* file_path, 82 | const char* tmp_dir, 83 | file_merger_feed_record_t callback, 84 | void* user_ctx); 85 | -------------------------------------------------------------------------------- /src/views/index_header.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../node_types.h" 24 | #include "bitmap.h" 25 | #include "couchstore_config.h" 26 | #include "sorted_list.h" 27 | #include 28 | #include 29 | #include 30 | 31 | #define LATEST_INDEX_HEADER_VERSION 2 32 | 33 | struct part_seq_t { 34 | uint16_t part_id; 35 | uint64_t seq; 36 | }; 37 | 38 | struct index_state_transition_t { 39 | /* sorted_list instance, values of type uint16_t */ 40 | void* active; 41 | /* sorted_list instance, values of type uint16_t */ 42 | void* passive; 43 | /* sorted_list instance, values of type uint16_t */ 44 | void* unindexable; 45 | }; 46 | 47 | struct failover_log_t { 48 | unsigned char uuid[8]; 49 | uint64_t seq; 50 | }; 51 | 52 | struct part_version_t { 53 | uint16_t part_id; 54 | uint16_t num_failover_log; 55 | failover_log_t* failover_log; 56 | }; 57 | 58 | struct index_header_t { 59 | uint8_t version; 60 | /* MD5 hash */ 61 | unsigned char signature[16]; 62 | uint8_t num_views; 63 | uint16_t num_partitions; 64 | bitmap_t active_bitmask; 65 | bitmap_t passive_bitmask; 66 | bitmap_t cleanup_bitmask; 67 | /* sorted_list instance, values of type part_seq_t */ 68 | void* seqs; 69 | node_pointer* id_btree_state; 70 | /* array of num_views elements */ 71 | node_pointer** view_states; 72 | int has_replica; 73 | /* sorted_list instance, values of type uint16_t */ 74 | void* replicas_on_transfer; 75 | index_state_transition_t pending_transition; 76 | /* sorted_list instance, values of type part_seq_t */ 77 | void* unindexable_seqs; 78 | /* sorted_list instance, values of type part_ver_t */ 79 | void* part_versions; 80 | }; 81 | 82 | couchstore_error_t decode_index_header(const char* bytes, 83 | size_t len, 84 | index_header_t** header); 85 | 86 | couchstore_error_t encode_index_header(const index_header_t* header, 87 | char** buffer, 88 | size_t* buffer_size); 89 | 90 | void free_index_header(index_header_t* header); 91 | -------------------------------------------------------------------------------- /src/views/keys.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "couchstore_config.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | struct view_btree_key_t { 10 | sized_buf json_key; 11 | sized_buf doc_id; 12 | }; 13 | 14 | struct view_id_btree_key_t { 15 | uint16_t partition; 16 | sized_buf doc_id; 17 | }; 18 | 19 | /// Decode the json_key from bytes/len into the given sized_buf 20 | /// caller must free key.buf 21 | couchstore_error_t decode_view_btree_json_key(const char* bytes, 22 | size_t len, 23 | sized_buf& key); 24 | 25 | couchstore_error_t decode_view_btree_key(const char* bytes, 26 | size_t len, 27 | view_btree_key_t** key); 28 | 29 | couchstore_error_t encode_view_btree_key(const view_btree_key_t* key, 30 | char** buffer, 31 | size_t* buffer_size); 32 | 33 | void free_view_btree_key(view_btree_key_t* key); 34 | 35 | couchstore_error_t decode_view_id_btree_key(const char* bytes, 36 | size_t len, 37 | view_id_btree_key_t** key); 38 | 39 | couchstore_error_t encode_view_id_btree_key(const view_id_btree_key_t* key, 40 | char** buffer, 41 | size_t* buffer_size); 42 | 43 | void free_view_id_btree_key(view_id_btree_key_t* key); 44 | -------------------------------------------------------------------------------- /src/views/mapreduce/mapreduce_internal.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @copyright 2013 Couchbase, Inc. 3 | * 4 | * @author Filipe Manana 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | * use this file except in compliance with the License. You may obtain a copy of 8 | * the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | * License for the specific language governing permissions and limitations under 16 | * the License. 17 | **/ 18 | 19 | /** 20 | * This is a private header, do not include it in other applications/lirbaries. 21 | **/ 22 | 23 | #ifndef _MAPREDUCE_INTERNAL_H 24 | #define _MAPREDUCE_INTERNAL_H 25 | 26 | #include "mapreduce.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | 41 | #define CHECK_SUCCESS(maybe)(CheckSuccess(maybe)) 42 | 43 | template 44 | void CheckSuccess(const v8::Maybe &from) { 45 | if(!from.FromJust()) { 46 | // Fail silently 47 | } 48 | } 49 | 50 | class MapReduceError; 51 | 52 | typedef std::list json_results_list_t; 53 | typedef std::list kv_list_int_t; 54 | typedef std::vector< v8::Persistent* > function_vector_t; 55 | 56 | typedef struct { 57 | v8::Persistent jsContext; 58 | v8::Isolate *isolate; 59 | v8::ArrayBuffer::Allocator *bufAllocator; 60 | function_vector_t *functions; 61 | kv_list_int_t *kvs; 62 | std::atomic taskStartTime; 63 | std::mutex exitMutex; 64 | } mapreduce_ctx_t; 65 | 66 | 67 | void initContext(mapreduce_ctx_t *ctx, 68 | const std::list &function_sources); 69 | 70 | // Disable UBSan vtpr check as V8 doesn't include RTTI information 71 | // therefore UBSan cannot lookup valid type information for 72 | // `ctx->bufAllocator` when it is deleted in this function. 73 | void NO_SANITIZE_VPTR destroyContext(mapreduce_ctx_t* ctx); 74 | 75 | void mapDoc(mapreduce_ctx_t *ctx, 76 | const mapreduce_json_t &doc, 77 | const mapreduce_json_t &meta, 78 | mapreduce_map_result_list_t *result); 79 | 80 | json_results_list_t runReduce(mapreduce_ctx_t *ctx, 81 | const mapreduce_json_list_t &keys, 82 | const mapreduce_json_list_t &values); 83 | 84 | mapreduce_json_t runReduce(mapreduce_ctx_t *ctx, 85 | int reduceFunNum, 86 | const mapreduce_json_list_t &keys, 87 | const mapreduce_json_list_t &values); 88 | 89 | mapreduce_json_t runRereduce(mapreduce_ctx_t *ctx, 90 | int reduceFunNum, 91 | const mapreduce_json_list_t &reductions); 92 | 93 | void terminateTask(mapreduce_ctx_t *ctx); 94 | 95 | 96 | 97 | class MapReduceError { 98 | public: 99 | MapReduceError(const mapreduce_error_t error, const char *msg) 100 | : _error(error), _msg(msg) { 101 | } 102 | 103 | MapReduceError(const mapreduce_error_t error, const std::string &msg) 104 | : _error(error), _msg(msg) { 105 | } 106 | 107 | mapreduce_error_t getError() const { 108 | return _error; 109 | } 110 | 111 | const std::string& getMsg() const { 112 | return _msg; 113 | } 114 | 115 | private: 116 | const mapreduce_error_t _error; 117 | const std::string _msg; 118 | }; 119 | 120 | #endif 121 | -------------------------------------------------------------------------------- /src/views/mapreduce/visibility.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _MAPREDUCE_VISIBILITY_H 3 | #define _MAPREDUCE_VISIBILITY_H 4 | 5 | #if defined(LIBMAPREDUCE_INTERNAL) 6 | 7 | #ifdef __SUNPRO_C 8 | #define LIBMAPREDUCE_API __global 9 | #elif defined(HAVE_VISIBILITY) && HAVE_VISIBILITY 10 | #define LIBMAPREDUCE_API __attribute__ ((visibility("default"))) 11 | #elif defined(_MSC_VER) 12 | #define LIBMAPREDUCE_API extern __declspec(dllexport) 13 | #else 14 | #define LIBMAPREDUCE_API 15 | #endif 16 | 17 | #else 18 | 19 | #if defined(_MSC_VER) && !defined(LIBCOUCHSTORE_NO_VISIBILITY) 20 | #define LIBMAPREDUCE_API extern __declspec(dllimport) 21 | #else 22 | #define LIBMAPREDUCE_API 23 | #endif 24 | 25 | #endif 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/views/purgers.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | #include "couchstore_config.h" 21 | #include "purgers.h" 22 | #include "bitmap.h" 23 | #include "values.h" 24 | #include "reductions.h" 25 | #include "../couch_btree.h" 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | static int view_purgekv_action(bitmap_t *bm, uint16_t part_id, 34 | uint64_t valcount, 35 | view_purger_ctx_t *ctx) 36 | { 37 | int action = PURGE_KEEP; 38 | if (is_bit_set(bm, part_id)) { 39 | ctx->count += valcount; 40 | action = PURGE_ITEM; 41 | } 42 | 43 | return action; 44 | } 45 | 46 | static int view_purgekp_action(bitmap_t *clearbm, bitmap_t *redbm, 47 | uint64_t kvcount, 48 | view_purger_ctx_t *ctx) 49 | { 50 | int action = PURGE_PARTIAL; 51 | bitmap_t emptybm, dstbm = *clearbm; 52 | 53 | if (is_equal_bitmap(redbm, clearbm)) { 54 | action = PURGE_ITEM; 55 | ctx->count += kvcount; 56 | } else { 57 | intersect_bitmaps(&dstbm, redbm); 58 | if (is_equal_bitmap(&dstbm, &emptybm)) { 59 | action = PURGE_KEEP; 60 | } 61 | } 62 | 63 | return action; 64 | } 65 | 66 | int view_id_btree_purge_kv(const sized_buf *key, const sized_buf *val, 67 | void *ctx) 68 | { 69 | view_purger_ctx_t *purge_ctx = (view_purger_ctx_t *) ctx; 70 | (void) key; 71 | 72 | return view_purgekv_action(&purge_ctx->cbitmask, 73 | decode_view_btree_partition(val->buf, val->size), 74 | 1, 75 | purge_ctx); 76 | } 77 | 78 | int view_id_btree_purge_kp(const node_pointer *ptr, void *ctx) 79 | { 80 | int action; 81 | couchstore_error_t errcode = COUCHSTORE_SUCCESS; 82 | view_purger_ctx_t *purge_ctx = (view_purger_ctx_t *) ctx; 83 | view_id_btree_reduction_t* r = nullptr; 84 | 85 | errcode = decode_view_id_btree_reduction(ptr->reduce_value.buf, &r); 86 | if (errcode != COUCHSTORE_SUCCESS) { 87 | action = (int) errcode; 88 | goto cleanup; 89 | } 90 | 91 | action = view_purgekp_action(&purge_ctx->cbitmask, &r->partitions_bitmap, 92 | r->kv_count, 93 | purge_ctx); 94 | 95 | cleanup: 96 | free_view_id_btree_reduction(r); 97 | return action; 98 | } 99 | 100 | int view_btree_purge_kv(const sized_buf *key, const sized_buf *val, void *ctx) 101 | { 102 | view_purger_ctx_t *purge_ctx = (view_purger_ctx_t *) ctx; 103 | (void) key; 104 | 105 | auto info = decode_view_btree_partition_and_num_values(val->buf, val->size); 106 | 107 | return view_purgekv_action( 108 | &purge_ctx->cbitmask, info.partition, info.num_values, purge_ctx); 109 | } 110 | 111 | int view_btree_purge_kp(const node_pointer *ptr, void *ctx) 112 | { 113 | view_purger_ctx_t *purge_ctx = (view_purger_ctx_t *) ctx; 114 | bitmap_t partitions_bitmap; 115 | auto kv_count = decode_view_btree_reduction_partitions_bitmap( 116 | ptr->reduce_value.buf, ptr->reduce_value.size, partitions_bitmap); 117 | 118 | return view_purgekp_action( 119 | &purge_ctx->cbitmask, &partitions_bitmap, kv_count, purge_ctx); 120 | } 121 | -------------------------------------------------------------------------------- /src/views/purgers.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../couch_btree.h" 24 | #include "../internal.h" 25 | #include "bitmap.h" 26 | #include "mapreduce/mapreduce.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | struct view_purger_ctx_t { 33 | bitmap_t cbitmask; 34 | uint64_t count{0}; 35 | }; 36 | 37 | int view_id_btree_purge_kv(const sized_buf* key, 38 | const sized_buf* val, 39 | void* ctx); 40 | int view_id_btree_purge_kp(const node_pointer* ptr, void* ctx); 41 | int view_btree_purge_kv(const sized_buf* key, const sized_buf* val, void* ctx); 42 | int view_btree_purge_kp(const node_pointer* ptr, void* ctx); 43 | -------------------------------------------------------------------------------- /src/views/reducers.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * @author Fulu Li 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 10 | * use this file except in compliance with the License. You may obtain a copy of 11 | * the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 18 | * License for the specific language governing permissions and limitations under 19 | * the License. 20 | **/ 21 | 22 | #pragma once 23 | 24 | #include "../couch_btree.h" 25 | #include "../internal.h" 26 | #include "mapreduce/mapreduce.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | struct view_reducer_ctx_t { 33 | /* If not NULL, an error happened and it contains a human 34 | readable error message. */ 35 | const char* error; 36 | void* priv; 37 | }; 38 | 39 | struct stats_t { 40 | uint64_t count; 41 | double sum, min, max, sumsqr; 42 | }; 43 | 44 | view_reducer_ctx_t* make_view_reducer_ctx(const char* functions[], 45 | unsigned num_functions, 46 | char** error_msg); 47 | 48 | void free_view_reducer_ctx(view_reducer_ctx_t* ctx); 49 | 50 | couchstore_error_t view_id_btree_reduce(char* dst, 51 | size_t* size_r, 52 | const nodelist* leaflist, 53 | int count, 54 | void* ctx); 55 | 56 | couchstore_error_t view_id_btree_rereduce(char* dst, 57 | size_t* size_r, 58 | const nodelist* itmlist, 59 | int count, 60 | void* ctx); 61 | 62 | couchstore_error_t view_btree_reduce(char* dst, 63 | size_t* size_r, 64 | const nodelist* leaflist, 65 | int count, 66 | void* ctx); 67 | 68 | couchstore_error_t view_btree_rereduce(char* dst, 69 | size_t* size_r, 70 | const nodelist* nodelist, 71 | int count, 72 | void* ctx); 73 | -------------------------------------------------------------------------------- /src/views/reductions.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "bitmap.h" 4 | #include "couchstore_config.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | struct view_btree_reduction_t { 13 | uint64_t kv_count{0}; 14 | bitmap_t partitions_bitmap; 15 | /* number of elements in reduce_values */ 16 | uint8_t num_values{0}; 17 | sized_buf* reduce_values{nullptr}; 18 | std::vector buffer; 19 | }; 20 | 21 | struct view_id_btree_reduction_t { 22 | uint64_t kv_count; 23 | bitmap_t partitions_bitmap; 24 | }; 25 | 26 | // return kv_count and copy bitmap to partitions_bitmap 27 | uint64_t decode_view_btree_reduction_partitions_bitmap( 28 | const char* bytes, size_t len, bitmap_t& partitions_bitmap); 29 | 30 | couchstore_error_t decode_view_btree_reduction( 31 | const char* bytes, size_t len, view_btree_reduction_t& reduction); 32 | 33 | couchstore_error_t encode_view_btree_reduction( 34 | const view_btree_reduction_t* reduction, 35 | char* buffer, 36 | size_t* buffer_size); 37 | 38 | void free_view_btree_reduction(view_btree_reduction_t* reduction); 39 | 40 | couchstore_error_t decode_view_id_btree_reduction( 41 | const char* bytes, view_id_btree_reduction_t** reduction); 42 | 43 | couchstore_error_t encode_view_id_btree_reduction( 44 | const view_id_btree_reduction_t* reduction, 45 | char* buffer, 46 | size_t* buffer_size); 47 | 48 | void free_view_id_btree_reduction(view_id_btree_reduction_t* reduction); 49 | -------------------------------------------------------------------------------- /src/views/sorted_list.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | #pragma once 21 | 22 | #include 23 | #include 24 | 25 | /** Returns: 26 | * negative integer if a < b, 0 if a == b, positive integer if a > b 27 | **/ 28 | typedef int (*sorted_list_cmp_t)(const void *a, const void *b); 29 | 30 | 31 | void *sorted_list_create(sorted_list_cmp_t less_fun); 32 | 33 | int sorted_list_add(void *list, const void *elem, size_t elem_size); 34 | 35 | void *sorted_list_get(const void *list, const void *elem); 36 | 37 | void sorted_list_remove(void *list, const void *elem); 38 | 39 | int sorted_list_size(const void *list); 40 | 41 | void sorted_list_free(void *list); 42 | 43 | void *sorted_list_iterator(const void *list); 44 | 45 | void *sorted_list_next(void *iterator); 46 | 47 | void sorted_list_free_iterator(void *iterator); 48 | 49 | -------------------------------------------------------------------------------- /src/views/spatial.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Volker Mische 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../couch_btree.h" 24 | #include "../file_merger.h" 25 | #include "bitmap.h" 26 | #include "couchstore_config.h" 27 | #include 28 | 29 | #define ZCODE_PRECISION 32 30 | #define ZCODE_MAX_VALUE UINT32_MAX 31 | 32 | struct sized_mbb_t { 33 | double* mbb; 34 | /* the total number of values (two times the dimension) */ 35 | uint16_t num; 36 | }; 37 | 38 | /* It is used to scale up MBBs to the relative size of an enclosing one */ 39 | struct scale_factor_t { 40 | /* The offset the value needs to be shifted in order to be in the 41 | * origin of the enclosing MBB */ 42 | double* offsets; 43 | /* The scale factors for every dimension */ 44 | double* scales; 45 | /* the total number of values, one per dimension */ 46 | uint8_t dim; 47 | }; 48 | 49 | /* The context to build the initial index */ 50 | struct view_spatial_builder_ctx_t { 51 | arena* transient_arena; 52 | couchfile_modify_result* modify_result; 53 | /* Scale MBBs up for a better results when using the space filling 54 | * curve */ 55 | scale_factor_t* scale_factor; 56 | }; 57 | 58 | /* compare keys of a spatial index */ 59 | int spatial_key_cmp(const sized_buf* key1, 60 | const sized_buf* key2, 61 | const void* user_ctx); 62 | 63 | /* Compare keys of a spatial index for the file merger */ 64 | int spatial_merger_key_cmp(const sized_buf* key1, 65 | const sized_buf* key2, 66 | const void* user_ctx); 67 | 68 | /* Return the scale factor for every dimension that would be needed to 69 | * scale this MBB to the maximum value `max` (when shifted to the 70 | * origin) 71 | * Memory is dynamically allocted within the function, make sure to call 72 | * free_spatial_scale_factor() afterwards */ 73 | scale_factor_t* spatial_scale_factor(const double* mbb, 74 | uint16_t dim, 75 | uint32_t max); 76 | 77 | /* Free the memory that spatial_scale_factor() allocated */ 78 | void free_spatial_scale_factor(scale_factor_t* sf); 79 | 80 | /* Calculate the center of an multi-dimensional bounding box (MBB) */ 81 | double* spatial_center(const sized_mbb_t* mbb); 82 | 83 | /* Scales all dimensions of a (multi-dimensional) point 84 | * with the given factor and offset */ 85 | uint32_t* spatial_scale_point(const double* point, const scale_factor_t* sf); 86 | 87 | /* Set a bit on a buffer with a certain size */ 88 | void set_bit_sized(unsigned char* bitmap, uint16_t size, uint16_t bit); 89 | 90 | /* Interleave numbers bitwise. The return value is a unsigned char array 91 | * with length 4 bytes * number of numbers. 92 | * The maximum number of numbers is (2^14)-1 (16383). */ 93 | unsigned char* interleave_uint32s(uint32_t* numbers, uint16_t num); 94 | 95 | /* A reduce is used to calculate the enclosing MBB of a parent node (it's 96 | * its key) */ 97 | couchstore_error_t view_spatial_reduce(char* dst, 98 | size_t* size_r, 99 | const nodelist* leaflist, 100 | int count, 101 | void* ctx); 102 | 103 | /* Puts an item into the results set. If there are enough items they are 104 | * are flused to disk */ 105 | couchstore_error_t spatial_push_item(sized_buf* k, 106 | sized_buf* v, 107 | couchfile_modify_result* dst); 108 | 109 | /* Build an r-tree bottom-up from the already stored leaf nodes */ 110 | node_pointer* complete_new_spatial(couchfile_modify_result* mr, 111 | couchstore_error_t* errcode); 112 | 113 | /* Filter function for the compactor */ 114 | int view_spatial_filter(const sized_buf* k, 115 | const sized_buf* v, 116 | const bitmap_t* bm); 117 | -------------------------------------------------------------------------------- /src/views/util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "../file_merger.h" 24 | #include "couchstore_config.h" 25 | #include "view_group.h" 26 | #include 27 | #include 28 | 29 | #pragma pack(push, 1) 30 | struct view_file_merge_record_t { 31 | uint8_t op; 32 | uint16_t ksize; 33 | uint32_t vsize; 34 | }; 35 | #pragma pack(pop) 36 | 37 | #define VIEW_RECORD_KEY(rec) (((char*)rec) + sizeof(view_file_merge_record_t)) 38 | #define VIEW_RECORD_VAL(rec) (VIEW_RECORD_KEY(rec) + rec->ksize) 39 | 40 | enum view_record_type { 41 | INITIAL_BUILD_VIEW_RECORD, 42 | INCREMENTAL_UPDATE_VIEW_RECORD, 43 | INITIAL_BUILD_SPATIAL_RECORD, 44 | INCREMENTAL_UPDATE_SPATIAL_RECORD 45 | }; 46 | 47 | struct view_file_merge_ctx_t { 48 | FILE* src_f; 49 | FILE* dst_f; 50 | enum view_record_type type; 51 | int (*key_cmp_fun)(const sized_buf* key1, 52 | const sized_buf* key2, 53 | const void* user_ctx); 54 | const void* user_ctx; 55 | }; 56 | 57 | /* compare keys of a view btree */ 58 | int view_key_cmp(const sized_buf* key1, 59 | const sized_buf* key2, 60 | const void* user_ctx); 61 | 62 | /* compare keys of the id btree of an index */ 63 | int view_id_cmp(const sized_buf* key1, 64 | const sized_buf* key2, 65 | const void* user_ctx); 66 | 67 | /* read view index record from a file, obbeys the read record function 68 | prototype defined in src/file_merger.h */ 69 | int read_view_record(FILE* in, void** buf, void* ctx); 70 | 71 | /* write view index record from a file, obbeys the write record function 72 | prototype defined in src/file_merger.h */ 73 | file_merger_error_t write_view_record(FILE* out, void* buf, void* ctx); 74 | 75 | /* compare 2 view index records, obbeys the record compare function 76 | prototype defined in src/file_merger.h */ 77 | int compare_view_records(const void* r1, const void* r2, void* ctx); 78 | 79 | /* Pick the winner from the duplicate entries */ 80 | size_t dedup_view_records_merger(file_merger_record_t** records, 81 | size_t len, 82 | void* ctx); 83 | 84 | /* frees a view record, obbeys the record free function prototype 85 | defined in src/file_merger.h */ 86 | void free_view_record(void* record, void* ctx); 87 | 88 | LIBCOUCHSTORE_API 89 | char* couchstore_read_line(FILE* in, char* buf, int size); 90 | 91 | LIBCOUCHSTORE_API 92 | uint64_t couchstore_read_int(FILE* in, 93 | char* buf, 94 | size_t size, 95 | couchstore_error_t* ret); 96 | 97 | /* Generate appropriate view error messages */ 98 | void set_error_info(const view_btree_info_t* info, 99 | const char* red_error, 100 | couchstore_error_t ret, 101 | view_error_t* error_info); 102 | -------------------------------------------------------------------------------- /src/views/values.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _VALUES_H 3 | #define _VALUES_H 4 | 5 | #include "couchstore_config.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | typedef struct { 14 | uint16_t partition; 15 | uint16_t num_values; 16 | sized_buf *values; 17 | std::vector values_buf; 18 | } view_btree_value_t; 19 | 20 | typedef struct { 21 | uint8_t view_id; 22 | uint16_t num_keys; 23 | sized_buf *json_keys; 24 | } view_keys_mapping_t; 25 | 26 | typedef struct { 27 | uint16_t partition; 28 | uint16_t num_view_keys_map; 29 | view_keys_mapping_t *view_keys_map; 30 | } view_id_btree_value_t; 31 | 32 | couchstore_error_t decode_view_btree_value(const char* bytes, 33 | size_t len, 34 | view_btree_value_t& value); 35 | 36 | struct partition_and_value_info { 37 | uint16_t partition{0}; 38 | uint16_t num_values{0}; 39 | size_t total_sizeof_values{0}; 40 | }; 41 | partition_and_value_info decode_view_btree_partition_and_num_values( 42 | const char* bytes, size_t len); 43 | 44 | couchstore_error_t encode_view_btree_value(const view_btree_value_t *value, 45 | char **buffer, 46 | size_t *buffer_size); 47 | 48 | couchstore_error_t decode_view_id_btree_value(const char* bytes, 49 | size_t len, 50 | view_id_btree_value_t** value); 51 | 52 | void free_view_id_btree_value(view_id_btree_value_t *value); 53 | 54 | couchstore_error_t encode_view_id_btree_value(const view_id_btree_value_t *value, 55 | char **buffer, 56 | size_t *buffer_size); 57 | 58 | uint16_t decode_view_btree_partition(const char* bytes, size_t len); 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/views/view_group.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "compaction.h" 24 | #include "couchstore_config.h" 25 | #include "index_header.h" 26 | #include 27 | #include 28 | 29 | /* The maximum header size is based on the 30 | * couch_set_view_util:group_to_header_bin/1 function in couchdb */ 31 | #define MAX_VIEW_HEADER_SIZE (20 * 1024 * 1024) 32 | 33 | enum view_index_type_t { VIEW_INDEX_TYPE_MAPREDUCE, VIEW_INDEX_TYPE_SPATIAL }; 34 | 35 | struct view_error_t { 36 | const char* view_name; 37 | const char* error_msg; 38 | const char* idx_type; 39 | }; 40 | 41 | struct view_btree_info_t { 42 | int view_id; 43 | int num_reducers; 44 | const char** names; 45 | const char** reducers; 46 | }; 47 | 48 | struct view_spatial_info_t { 49 | /* Number of dimensions the multidimensional bounding box (MBB) has */ 50 | uint16_t dimension; 51 | /* The MBB that enclosed the whole spatial view*/ 52 | double* mbb; 53 | }; 54 | 55 | union view_infos_t { 56 | view_btree_info_t* btree; 57 | view_spatial_info_t* spatial; 58 | }; 59 | 60 | struct view_group_info_t { 61 | const char* filepath{nullptr}; 62 | uint64_t header_pos{0}; 63 | int num_btrees{0}; 64 | view_index_type_t type{}; 65 | view_infos_t view_infos{}; 66 | tree_file file; 67 | }; 68 | 69 | struct view_group_update_stats_t { 70 | uint64_t ids_inserted; 71 | uint64_t ids_removed; 72 | uint64_t kvs_inserted; 73 | uint64_t kvs_removed; 74 | uint64_t purged; 75 | }; 76 | 77 | struct view_btree_builder_ctx_t { 78 | arena* transient_arena; 79 | couchfile_modify_result* modify_result; 80 | }; 81 | 82 | /* Read a view group definition from an input stream, and write any 83 | errors to the optional error stream. */ 84 | LIBCOUCHSTORE_API 85 | view_group_info_t* couchstore_read_view_group_info(FILE* in_stream, 86 | FILE* error_stream); 87 | 88 | LIBCOUCHSTORE_API 89 | void couchstore_free_view_group_info(view_group_info_t* info); 90 | 91 | LIBCOUCHSTORE_API 92 | couchstore_error_t couchstore_build_view_group(view_group_info_t* info, 93 | const char* id_records_file, 94 | const char* kv_records_files[], 95 | const char* dst_file, 96 | const char* tmpdir, 97 | uint64_t* header_pos, 98 | view_error_t* error_info); 99 | 100 | couchstore_error_t read_view_group_header(view_group_info_t* info, 101 | index_header_t** header); 102 | 103 | couchstore_error_t write_view_group_header(tree_file* file, 104 | uint64_t* pos, 105 | const index_header_t* header); 106 | 107 | couchstore_error_t open_view_group_file(const char* path, 108 | couchstore_open_flags open_flags, 109 | tree_file* file); 110 | 111 | LIBCOUCHSTORE_API 112 | couchstore_error_t couchstore_cleanup_view_group(view_group_info_t* info, 113 | uint64_t* header_pos, 114 | uint64_t* purge_count, 115 | view_error_t* error_info); 116 | 117 | LIBCOUCHSTORE_API 118 | couchstore_error_t couchstore_update_view_group( 119 | view_group_info_t* info, 120 | const char* id_records_file, 121 | const char* kv_records_files[], 122 | size_t batch_size, 123 | const sized_buf* header_buf, 124 | int is_sorted, 125 | const char* tmp_dir, 126 | view_group_update_stats_t* stats, 127 | sized_buf* header_outbuf, 128 | view_error_t* error_info); 129 | 130 | LIBCOUCHSTORE_API 131 | couchstore_error_t couchstore_compact_view_group(view_group_info_t* info, 132 | const char* target_file, 133 | const sized_buf* header_buf, 134 | compactor_stats_t* stats, 135 | sized_buf* header_outbuf, 136 | view_error_t* error_info); 137 | -------------------------------------------------------------------------------- /tests/btree_purge/purge_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #pragma once 22 | 23 | #include "couchstore_config.h" 24 | #include 25 | 26 | void test_no_purge_items(); 27 | void test_all_purge_items(); 28 | void test_partial_purge_items(); 29 | void test_partial_purge_items2(); 30 | void test_partial_purge_with_stop(); 31 | void test_add_remove_purge(); 32 | 33 | void purge_tests(); 34 | void test_only_single_leafnode(); 35 | -------------------------------------------------------------------------------- /tests/btree_purge/tests.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "purge_tests.h" 22 | 23 | void purge_tests() 24 | { 25 | fprintf(stderr, "\n\nRunning purge tests\n\n"); 26 | test_no_purge_items(); 27 | test_all_purge_items(); 28 | test_partial_purge_items(); 29 | test_partial_purge_items2(); 30 | test_partial_purge_with_stop(); 31 | test_only_single_leafnode(); 32 | test_add_remove_purge(); 33 | } 34 | -------------------------------------------------------------------------------- /tests/bulk.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | testlib.check_doc(db, k, v[2]) 8 | end 9 | 10 | local found = 0 11 | db:changes(0, 0, function(di) found = found + 1 end) 12 | if found ~= #t then 13 | error("Expected " .. #t .. " results, found " .. found) 14 | end 15 | end 16 | 17 | function test_explicit(dbname) 18 | local t = { 19 | {"k1", "value 1", 1,}, 20 | {"k2", "value 2", 1,}, 21 | {"k3", "value 3", 1,}, 22 | {"k4", "value 4", 1,}, 23 | {"k5", "value 5", 1,}, 24 | {"k6", "value 6", 1,}, 25 | {"k7", "value 7", 1,} 26 | } 27 | 28 | local db = couch.open(dbname, true) 29 | 30 | for i,v in ipairs(t) do 31 | local k = v[1] 32 | testlib.assert_no_doc(db, k) 33 | end 34 | 35 | db:save_bulk(t) 36 | db:commit() 37 | 38 | check_table(db, t) 39 | end 40 | 41 | function large_txn_bulk_test(dbname) 42 | local t = {} 43 | for i = 0, 10000, 1 do 44 | table.insert(t, {"k" .. i, "value " .. i, 1}) 45 | end 46 | 47 | local db = couch.open(dbname, true) 48 | 49 | db:save_bulk(t) 50 | db:commit() 51 | 52 | check_table(db, t) 53 | end 54 | 55 | testlib.run_test("Explicit bulk test", test_explicit) 56 | testlib.run_test("Big bulk test", large_txn_bulk_test) 57 | return testlib.fail_count() 58 | -------------------------------------------------------------------------------- /tests/changecount.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from couchstore import CouchStore, DocumentInfo 4 | from tempfile import mkdtemp 5 | import os 6 | import os.path as path 7 | import struct 8 | import unittest 9 | 10 | class ChangeCountTest(unittest.TestCase): 11 | def setUp(self): 12 | self.tmpdir = mkdtemp() 13 | self.dbname = path.join(self.tmpdir, "testing.couch") 14 | self.db = CouchStore(self.dbname, 'c'); 15 | 16 | def tearDown(self): 17 | try: 18 | self.db.commit() 19 | self.db.close() 20 | except: 21 | pass 22 | try: 23 | os.remove(self.dbname) 24 | except: 25 | pass 26 | try: 27 | os.rmdir(self.tmpdir) 28 | except: 29 | pass 30 | 31 | def bulkSet(self, prefix, n): 32 | ids = [prefix + str(x) for x in range(n)] 33 | datas = ["val" + str(x) for x in range(n)] 34 | self.db.saveMultiple(ids, datas) 35 | 36 | def testRewind(self): 37 | # Save some docs 38 | self.db.save("foo1", "bar") 39 | self.db.save("foo2", "baz") 40 | self.db.save("foo3", "bell") 41 | self.db.save("foo4", "a") 42 | self.assertEqual(self.db.changesCount(0,100), 4) 43 | 44 | self.db.save("foo1", "new_bar") 45 | self.db.save("foo2", "new_baz") 46 | self.db.save("foo3", "new_bell") 47 | self.db.save("foo4", "new_a") 48 | self.assertEqual(self.db.changesCount(0,100), 4) 49 | 50 | self.bulkSet("foo", 100) 51 | self.assertEqual(self.db.changesCount(0, 108), 100) 52 | self.assertEqual(self.db.changesCount(0, 100), 92) 53 | self.assertEqual(self.db.changesCount(1, 100), 92) 54 | self.assertNotEqual(self.db.changesCount(12, 100), 92) 55 | self.assertEqual(self.db.changesCount(50, 99), 50) 56 | self.assertEqual(self.db.changesCount(50, 100), 51) 57 | self.assertEqual(self.db.changesCount(50, 108), 59) 58 | self.assertEqual(self.db.changesCount(51, 100), 50) 59 | self.assertEqual(self.db.changesCount(91, 1000), 18) 60 | self.db.save("foo88", "tval") 61 | self.assertEqual(self.db.changesCount(50, 108), 58) 62 | self.assertEqual(self.db.changesCount(50, 109), 59) 63 | 64 | 65 | if __name__ == '__main__': 66 | unittest.main() 67 | -------------------------------------------------------------------------------- /tests/changessincefilter.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function test_filter_deletes(dbname) 5 | local t = { 6 | {"k1", "value 1", 1,}, 7 | {"k2", "value 2", 1,}, 8 | {"k3", "value 3", 1,}, 9 | {"k4", "value 4", 1,}, 10 | {"k5", "value 5", 1,} 11 | } 12 | 13 | local db = couch.open(dbname, true) 14 | 15 | for i,v in ipairs(t) do 16 | local k = v[1] 17 | testlib.assert_no_doc(db, k) 18 | end 19 | 20 | db:save_bulk(t) 21 | db:commit() 22 | 23 | db:delete("k1") 24 | db:delete("k2") 25 | db:commit() 26 | 27 | local found = 0 28 | db:changes(0, 4, function(di) found = found + 1 end) 29 | if found ~= 3 then 30 | error("Expected " .. 3 .. " results, found " .. found) 31 | end 32 | 33 | db:close() 34 | end 35 | 36 | function test_filter_mutations(dbname) 37 | local t = { 38 | {"k1", "value 1", 1,}, 39 | {"k2", "value 2", 1,}, 40 | {"k3", "value 3", 1,}, 41 | {"k4", "value 4", 1,}, 42 | {"k5", "value 5", 1,} 43 | } 44 | 45 | local db = couch.open(dbname, true) 46 | 47 | for i,v in ipairs(t) do 48 | local k = v[1] 49 | testlib.assert_no_doc(db, k) 50 | end 51 | 52 | db:save_bulk(t) 53 | db:commit() 54 | 55 | db:delete("k1") 56 | db:delete("k2") 57 | db:commit() 58 | 59 | local found = 0 60 | db:changes(0, 2, function(di) found = found + 1 end) 61 | if found ~= 2 then 62 | error("Expected " .. 2 .. " results, found " .. found) 63 | end 64 | 65 | db:close() 66 | end 67 | 68 | testlib.run_test("Filter deletes", test_filter_deletes) 69 | testlib.run_test("Filter mutations", test_filter_mutations) 70 | return testlib.fail_count() 71 | -------------------------------------------------------------------------------- /tests/compact.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | testlib.check_doc(db, k, v[2]) 8 | end 9 | 10 | local found = 0 11 | db:changes(0, 0, function(di) found = found + 1 end) 12 | if found ~= #t then 13 | error("Expected " .. #t .. " results, found " .. found) 14 | end 15 | end 16 | 17 | function instuff(db, num, rev) 18 | local t = {} 19 | for i = 0, num, 1 do 20 | table.insert(t, {"k" .. i, "value " .. i, rev}) 21 | end 22 | db:save_bulk(t) 23 | end 24 | 25 | function insdata(db) 26 | instuff(db, 50000, 1) 27 | instuff(db, 25000, 2) 28 | instuff(db, 100000, 3) 29 | instuff(db, 80000, 4) 30 | instuff(db, 75000, 5) 31 | db:save_local("_local/localtest", "Local doc") 32 | db:save_local("_local/localtest2", "Another local doc") 33 | db:commit() 34 | end 35 | 36 | function into_table(db) 37 | local t = {} 38 | db:changes(0, 0, function(di) 39 | table.insert(t, {di:id(), db:get_from_docinfo(di), di:rev()}) 40 | end) 41 | return t 42 | end 43 | 44 | function compaction_test(dbname) 45 | local db = couch.open(dbname, true) 46 | outfile = os.tmpname() 47 | os.remove(outfile) 48 | insdata(db) 49 | local origdat = into_table(db) 50 | os.execute("./couch_compact " .. dbname .. " " .. outfile) 51 | local newdb = couch.open(outfile, false) 52 | check_table(newdb, origdat) 53 | testlib.check_local_doc(newdb, "_local/localtest", "Local doc") 54 | testlib.check_local_doc(newdb, "_local/localtest2", "Another local doc") 55 | db:close() 56 | newdb:close() 57 | end 58 | 59 | testlib.run_test("Compaction test", compaction_test) 60 | os.remove(outfile) 61 | return testlib.fail_count() 62 | -------------------------------------------------------------------------------- /tests/corrupt.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function simpletrunc(dbname) 5 | local db = couch.open(dbname, true) 6 | db:save("k", "original value", 1) 7 | db:commit() 8 | 9 | testlib.check_doc(db, "k", "original value") 10 | 11 | db:save("k", "second value", 1) 12 | db:commit() 13 | 14 | testlib.check_doc(db, "k", "second value") 15 | 16 | db:truncate(-10) 17 | db:close() 18 | 19 | db = couch.open(dbname, false, true) 20 | testlib.check_doc(db, "k", "original value") 21 | 22 | db:save("k", "third value", 1) 23 | db:commit() 24 | db:close() 25 | 26 | db = couch.open(dbname) 27 | testlib.check_doc(db, "k", "third value") 28 | end 29 | 30 | function do_mangle(dbname, offset) 31 | local db = couch.open(dbname, true) 32 | db:save("k", "original value", 1) 33 | db:commit() 34 | 35 | testlib.check_doc(db, "k", "original value") 36 | 37 | db:save("k", "second value", 1) 38 | db:commit() 39 | 40 | testlib.check_doc(db, "k", "second value") 41 | 42 | db:close() 43 | 44 | local f = io.open(dbname, "r+") 45 | local corruptWith = "here be some garbage" 46 | local treelen = f:seek("end") % 4096 47 | if offset > treelen then 48 | return false 49 | end 50 | f:seek("end", 0 - treelen) 51 | f:write(corruptWith) 52 | f:close() 53 | 54 | db = couch.open(dbname, false, true) 55 | testlib.check_doc(db, "k", "original value") 56 | 57 | db:save("k", "third value", 1) 58 | db:commit() 59 | db:close() 60 | 61 | db = couch.open(dbname) 62 | testlib.check_doc(db, "k", "third value") 63 | return true 64 | end 65 | 66 | -- Mangle the header with some arbitrary data from the beginning of 67 | -- the header boundary to the end. The header is located by the EOF % 68 | -- 4096. Should probably do something special when EOF *is* at a 4k 69 | -- boundary, but this test is valuable without it. 70 | function header_mangling(dbname) 71 | local i = 0 72 | while do_mangle(dbname, i) do 73 | i = i + 1 74 | os.remove(dbname) 75 | end 76 | end 77 | 78 | testlib.run_test("Simple truncation test", simpletrunc) 79 | testlib.run_test("Various mangling of headers", header_mangling) 80 | return testlib.fail_count() 81 | -------------------------------------------------------------------------------- /tests/couchstoretest.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #pragma once 18 | 19 | #include "couchstore_config.h" 20 | #include "test_fileops.h" 21 | #include "documents.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | /** 32 | * Base class for other tests 33 | * 34 | * Auto-cleans when the test is complete. 35 | * a) If db is not null, closes the db 36 | * b) removes testfile.couch. 37 | */ 38 | class CouchstoreBaseTest : public ::testing::Test { 39 | protected: 40 | CouchstoreBaseTest(); 41 | CouchstoreBaseTest( 42 | std::string file_path, bool display_latency_info = false); 43 | ~CouchstoreBaseTest() override; 44 | void clean_up(); 45 | 46 | Db* db; 47 | const std::string filePath; 48 | const bool displayLatencyInfo; 49 | }; 50 | 51 | /** 52 | * Fixture for tests that will run with and without encryption 53 | * 54 | * Extends CouchstoreBaseTest 55 | */ 56 | class CouchstoreEncryptedUnencryptedTest : public CouchstoreBaseTest { 57 | protected: 58 | CouchstoreEncryptedUnencryptedTest(); 59 | 60 | virtual bool isEncrypted() = 0; 61 | 62 | cb::couchstore::EncryptionKeyGetter getEncryptionKeyCB(); 63 | 64 | couchstore_error_t open_db(couchstore_open_flags extra_flags); 65 | 66 | private: 67 | cb::couchstore::SharedEncryptionKey sharedEncryptionKey; 68 | }; 69 | 70 | class CouchstoreTest 71 | : public CouchstoreEncryptedUnencryptedTest, 72 | public ::testing::WithParamInterface { 73 | protected: 74 | bool isEncrypted() override; 75 | }; 76 | 77 | class CouchstoreDocTest 78 | : public CouchstoreEncryptedUnencryptedTest, 79 | public ::testing::WithParamInterface> { 80 | protected: 81 | bool isEncrypted() override; 82 | }; 83 | 84 | /** 85 | * Global test class for internal only tests 86 | * 87 | * Extends CouchstoreBaseTest 88 | */ 89 | class CouchstoreInternalTest : public CouchstoreBaseTest { 90 | protected: 91 | CouchstoreInternalTest(); 92 | ~CouchstoreInternalTest() override; 93 | 94 | /** 95 | * Opens a database instance with the current filePath, ops and with 96 | * buffering disabled. 97 | * 98 | * @param extra_flags Any additional flags, other than 99 | * COUCHSTORE_OPEN_FLAG_UNBUFFERED to open the db with. 100 | */ 101 | couchstore_error_t open_db(couchstore_open_flags extra_flags); 102 | 103 | /** 104 | * Opens a database instance with the current filePath, ops and with 105 | * buffering disabled. It then populates the database with the 106 | * specified number of documents. 107 | * 108 | * @param extra_flags Any additional flags, other than 109 | * COUCHSTORE_OPEN_FLAG_UNBUFFERED to open the db with. 110 | * @param count Number of documents to populate with 111 | */ 112 | void open_db_and_populate(couchstore_open_flags extra_flags, size_t count); 113 | 114 | /** 115 | * Creates a LocalDoc object from two strings 116 | * 117 | * Note: The localDoc will just point to strings' memory 118 | * so the strings should stay alive as long as the LocalDoc 119 | * does. 120 | * 121 | * @param id ID of the document 122 | * @param json Body of the document 123 | */ 124 | LocalDoc create_local_doc(std::string& id, std::string& json); 125 | 126 | std::string compactPath; 127 | Documents documents; 128 | ::testing::NiceMock ops; 129 | DocInfo* info; 130 | Doc* doc; 131 | }; 132 | 133 | /** 134 | * Multi-threaded test. 135 | */ 136 | class CouchstoreMTTest 137 | : public ::testing::Test, 138 | public ::testing::WithParamInterface > { 139 | protected: 140 | CouchstoreMTTest(); 141 | CouchstoreMTTest(std::string file_path); 142 | 143 | void TearDown(); 144 | 145 | size_t numThreads; 146 | std::vector dbs; 147 | std::string filePath; 148 | }; 149 | 150 | /** 151 | * Test class for error injection tests 152 | */ 153 | typedef CouchstoreInternalTest FileOpsErrorInjectionTest; 154 | 155 | /** 156 | * Parameterised test class for error injection tests 157 | */ 158 | class ParameterisedFileOpsErrorInjectionTest : public FileOpsErrorInjectionTest, 159 | public ::testing::WithParamInterface { 160 | }; 161 | -------------------------------------------------------------------------------- /tests/dropdel.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | if k ~= "k99" and k ~= "k200" then 8 | testlib.check_doc(db, k, v[2]) 9 | else 10 | testlib.assert_no_doc(db, k) 11 | end 12 | end 13 | 14 | local found = 0 15 | db:changes(0, 0, function(di) found = found + 1 end) 16 | if found ~= (#t - 2) then 17 | error("Expected " .. (#t - 2) .. " results, found " .. found) 18 | end 19 | end 20 | 21 | function instuff(db, num, rev) 22 | local t = {} 23 | for i = 0, num, 1 do 24 | table.insert(t, {"k" .. i, "value " .. i, rev}) 25 | end 26 | db:save_bulk(t) 27 | end 28 | 29 | function insdata(db) 30 | instuff(db, 50000, 1) 31 | instuff(db, 25000, 2) 32 | instuff(db, 100000, 3) 33 | instuff(db, 80000, 4) 34 | instuff(db, 75000, 5) 35 | db:save_local("_local/localtest", "Local doc") 36 | db:save_local("_local/localtest2", "Another local doc") 37 | db:commit() 38 | end 39 | 40 | function into_table(db) 41 | local t = {} 42 | db:changes(0, 0, function(di) 43 | table.insert(t, {di:id(), db:get_from_docinfo(di), di:rev()}) 44 | end) 45 | return t 46 | end 47 | 48 | function compaction_test(dbname) 49 | local db = couch.open(dbname, true) 50 | outfile = os.tmpname() 51 | os.remove(outfile) 52 | insdata(db) 53 | db:delete("k99") 54 | db:delete("k200") 55 | db:commit() 56 | local origdat = into_table(db) 57 | os.execute("./couch_compact --dropdeletes " .. dbname .. " " .. outfile) 58 | local newdb = couch.open(outfile, false) 59 | check_table(newdb, origdat) 60 | testlib.check_local_doc(newdb, "_local/localtest", "Local doc") 61 | testlib.check_local_doc(newdb, "_local/localtest2", "Another local doc") 62 | db:close() 63 | newdb:close() 64 | end 65 | 66 | testlib.run_test("Deletion dropping test", compaction_test) 67 | os.remove(outfile) 68 | return testlib.fail_count() 69 | -------------------------------------------------------------------------------- /tests/large.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local alphabet_str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 5 | local alphabet = {} 6 | for i = 1, #alphabet_str do 7 | alphabet[i] = alphabet_str:sub(i, i) 8 | end 9 | 10 | 11 | local function check_table(db, t) 12 | for i,v in ipairs(t) do 13 | local k = v[1] 14 | testlib.check_doc(db, k, v[2]) 15 | end 16 | 17 | local found = 0 18 | db:changes(0, 0, function(di) found = found + 1 end) 19 | if found ~= #t then 20 | error("Expected " .. #t .. " results, found " .. found) 21 | end 22 | end 23 | 24 | function make_val(size) 25 | local chars = {} 26 | for i = 1, size do 27 | chars[i] = alphabet[math.random(1, #alphabet)] 28 | end 29 | return table.concat(chars) 30 | end 31 | 32 | local testsample = make_val(40000) 33 | 34 | function test_big_bulk(dbname) 35 | print "Making data set..." 36 | local t = {} 37 | for i = 0, 40000, 5 do 38 | local k = "k" .. i 39 | local v = testsample:sub(i) 40 | t[#t + 1] = {k, v, 1 } 41 | end 42 | 43 | print "Saving data set..." 44 | local db = couch.open(dbname, true) 45 | db:save_bulk(t) 46 | db:commit() 47 | 48 | print "Checking data set..." 49 | check_table(db, t) 50 | 51 | db:close() 52 | 53 | local db = couch.open(dbname) 54 | 55 | check_table(db, t) 56 | end 57 | 58 | function test_big_sequential(dbname) 59 | print "Making & saving data set..." 60 | local db = couch.open(dbname, true) 61 | 62 | local t = {} 63 | for i = 0, 40000, 5 do 64 | local k = "k" .. i 65 | local v = testsample:sub(i) 66 | t[#t + 1] = {k, v, 1 } 67 | db:save(k, v, 1) 68 | end 69 | 70 | db:commit() 71 | 72 | print "Checking data set..." 73 | check_table(db, t) 74 | 75 | db:close() 76 | 77 | local db = couch.open(dbname) 78 | 79 | check_table(db, t) 80 | end 81 | 82 | testlib.run_test("Big item test bulk", test_big_bulk) 83 | testlib.run_test("Big item test sequential", test_big_sequential) 84 | return testlib.fail_count() 85 | -------------------------------------------------------------------------------- /tests/largefile.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function bigdb(dbname) 5 | local db = couch.open(dbname, true) 6 | db:truncate(5 * 1024 * 1024 * 1024) 7 | db:close() 8 | 9 | db = couch.open(dbname) 10 | testlib.assert_no_doc(db, "k") 11 | 12 | db:save("k", "a value", 1) 13 | db:commit() 14 | db:close() 15 | 16 | db = couch.open(dbname) 17 | testlib.check_doc(db, "k", "a value") 18 | end 19 | 20 | testlib.run_test("A Big Database", bigdb) 21 | return testlib.fail_count() 22 | -------------------------------------------------------------------------------- /tests/localdoc.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function localtest(dbname) 5 | local key = "a_local_key" 6 | local value = "the local value" 7 | local value2 = "an updated value" 8 | 9 | local db = couch.open(dbname, true) 10 | 11 | testlib.assert_no_local_doc(key) 12 | 13 | db:save_local(key, value) 14 | db:commit() 15 | 16 | testlib.assert_no_doc(key) 17 | 18 | db:close() 19 | 20 | db = couch.open(dbname) 21 | 22 | testlib.check_local_doc(db, key, value) 23 | 24 | db:save_local(key, value2) 25 | testlib.check_local_doc(db, key, value2) 26 | db:commit() 27 | testlib.check_local_doc(db, key, value2) 28 | 29 | db:close() 30 | 31 | db = couch.open(dbname) 32 | testlib.check_local_doc(db, key, value2) 33 | 34 | db:changes(0, 0, function(di) error("Unexpectedly got a doc: " .. di:id()) end) 35 | 36 | -- Store a non-local document and verify it doesn't collide 37 | db:save(key, "non local", 1) 38 | testlib.check_local_doc(db, key, value2) 39 | testlib.check_doc(db, key, "non local") 40 | 41 | db:delete_local(key) 42 | db:commit() 43 | 44 | testlib.check_doc(db, key, "non local") 45 | testlib.assert_no_local_doc(key) 46 | 47 | end 48 | 49 | testlib.run_test("Local doc test", localtest) 50 | return testlib.fail_count() 51 | -------------------------------------------------------------------------------- /tests/macros.h: -------------------------------------------------------------------------------- 1 | #ifndef TESTS_MACROS_H 2 | #define TESTS_MACROS_H 3 | 4 | /** Custom assert() macro -- can't use the one in because it's disabled when 5 | NDEBUG is defined. */ 6 | #undef assert 7 | #define assert(expr) \ 8 | do { \ 9 | if (!(expr)) { \ 10 | fprintf(stderr, "%s:%d: assertion failed\n", \ 11 | __FILE__, __LINE__); \ 12 | fflush(stderr); \ 13 | abort(); \ 14 | } \ 15 | } while (0) 16 | 17 | #define assert_eq(A, B) \ 18 | do { \ 19 | if ((A) != (B)) { \ 20 | fprintf(stderr, "%s:%d: assertion failed: %s != %s\n", \ 21 | __FILE__, __LINE__, #A, #B); \ 22 | fflush(stderr); \ 23 | abort(); \ 24 | } \ 25 | } while (0) 26 | 27 | #define try_to(C) \ 28 | do { \ 29 | if ((errcode = (C)) < 0) { \ 30 | fprintf(stderr, \ 31 | "Couchstore error `%s' at %s:%d\r\n", \ 32 | couchstore_strerror((couchstore_error_t)errcode), \ 33 | __FILE__, \ 34 | __LINE__); \ 35 | goto cleanup; \ 36 | } \ 37 | } while (0) 38 | 39 | #define error_unless(C, E) \ 40 | do { \ 41 | if(!(C)) { try(E); } \ 42 | } while (0) 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /tests/mapreduce/mapreduce_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _MAPREDUCE_TESTS_H 22 | #define _MAPREDUCE_TESTS_H 23 | 24 | #include "couchstore_config.h" 25 | 26 | #include 27 | #include 28 | #include 29 | #include "../macros.h" 30 | #include "../../src/views/mapreduce/mapreduce.h" 31 | 32 | /* main test function */ 33 | void mapreduce_tests(void); 34 | void map_tests(void); 35 | void reduce_tests(void); 36 | void builtin_tests(void); 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /tests/new_delete_replacement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # MB-54297 regression test - If: 4 | 5 | # 1. libcouchstore.so has overriden the C++ operator new/delete 6 | # symbols (by linking global_new_replacement.cc from platform) to 7 | # use an alternative heap (jemalloc). 8 | # 2. libcouchstore.so is dlopen()ed into a binary which does not 9 | # itself override operator new / delete (such as python3). 10 | # 3. Another C++ library (e.g. snappy) has already been dlopen()ed 11 | # before libcouchstore.so and has called at least one operator new 12 | # function which is bound to the system heap. 13 | # 14 | # Then if libcouchstore.so calls a symbol in libstdc++.so.6 which in 15 | # turn calls operator new (already bound at 3); then that operator new 16 | # call will end up in the system heap, and if later deleted directly 17 | # via libcouchstore.so we will attempt to free via jemalloc and crash. 18 | 19 | 20 | # Import snappy, which depends on libstdc++.so and will call operator 21 | # new at least once. 22 | import snappy 23 | 24 | # Attempt to import couchstore - this results in the heap mismatch 25 | # described above and a segfault before the bug was fixed. 26 | import couchstore 27 | 28 | print("Package 'couchstore' loaded successfully.") 29 | -------------------------------------------------------------------------------- /tests/purge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from couchstore import CouchStore, DocumentInfo 4 | from tempfile import mkdtemp 5 | import os 6 | import os.path as path 7 | import struct 8 | import unittest 9 | 10 | REV_META_PACK = ">QII" 11 | 12 | def deleteAt(db, key, time): 13 | info = db.getInfo(key) 14 | # cas, exp, flags 15 | info.revMeta = struct.pack(REV_META_PACK, 0, time, 0) 16 | info.deleted = True 17 | return db.save(info, None) 18 | 19 | class PurgeTest(unittest.TestCase): 20 | def setUp(self): 21 | self.tmpdir = mkdtemp() 22 | self.origname = path.join(self.tmpdir, "orig.couch") 23 | self.purgedname = path.join(self.tmpdir, "purged.couch") 24 | self.origdb = CouchStore(self.origname, 'c'); 25 | 26 | def tearDown(self): 27 | try: 28 | self.origdb.close() 29 | except: 30 | pass 31 | try: 32 | self.newdb.close() 33 | except: 34 | pass 35 | try: 36 | os.remove(self.origname) 37 | except: 38 | pass 39 | try: 40 | os.remove(self.purgedname) 41 | except: 42 | pass 43 | try: 44 | os.rmdir(self.tmpdir) 45 | except: 46 | pass 47 | 48 | def testPurgeCompact(self): 49 | # Save some docs 50 | self.origdb.save("foo1", "bar") 51 | self.origdb.save("foo2", "baz") 52 | self.origdb.save("foo3", "bell") 53 | self.origdb.save("foo4", "a") 54 | 55 | # Delete some 56 | seqPurged = deleteAt(self.origdb, "foo2", 10) 57 | seqKept = deleteAt(self.origdb, "foo3", 20) 58 | seqLateDelete = deleteAt(self.origdb, "foo4", 11) 59 | self.origdb.commit() 60 | 61 | os.system(path.join(os.getcwd(), "couch_compact") + " --purge-before 15 " + 62 | self.origname + " " + self.purgedname) 63 | self.newdb = CouchStore(self.purgedname) 64 | 65 | # Check purged item is not present in key tree and kept item is 66 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 67 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 68 | self.assertRaises(KeyError, self.newdb.getInfo, "foo4") 69 | 70 | self.newdb.close() 71 | os.remove(self.purgedname) 72 | 73 | os.system(path.join(os.getcwd(), "couch_compact") + 74 | " --purge-before 15 --purge-only-upto-seq " + str(seqKept) + 75 | " " + self.origname + " " + self.purgedname) 76 | self.newdb = CouchStore(self.purgedname) 77 | 78 | # Check purged item is not present in key tree and kept item is 79 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 80 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 81 | # with purge-only-upto-seq just before deletion of foo4 we 82 | # must find it after compaction 83 | self.assertIsNotNone(self.newdb.getInfo("foo4")) 84 | 85 | self.newdb.close() 86 | os.remove(self.purgedname) 87 | 88 | os.system(path.join(os.getcwd(), "couch_compact") + 89 | " --purge-before 15 --purge-only-upto-seq " + str(seqLateDelete) + 90 | " " + self.origname + " " + self.purgedname) 91 | self.newdb = CouchStore(self.purgedname) 92 | 93 | # Check purged item is not present in key tree and kept item is 94 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 95 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 96 | # with purge-only-upto-seq just at deletion of foo4 we 97 | # must not find it after compaction 98 | self.assertRaises(KeyError, self.newdb.getInfo, "foo4") 99 | 100 | # Check purged item is not present in seq tree and kept item is 101 | self.assertRaises(KeyError, self.newdb.getInfoBySequence, seqPurged) 102 | self.assertIsNotNone(self.newdb.getInfoBySequence(seqKept)) 103 | 104 | 105 | if __name__ == '__main__': 106 | unittest.main() 107 | -------------------------------------------------------------------------------- /tests/rewind.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from couchstore import CouchStore, DocumentInfo 4 | from tempfile import mkdtemp 5 | import os 6 | import os.path as path 7 | import struct 8 | import unittest 9 | 10 | class RewindTest(unittest.TestCase): 11 | def setUp(self): 12 | self.tmpdir = mkdtemp() 13 | self.dbname = path.join(self.tmpdir, "testing.couch") 14 | self.db = CouchStore(self.dbname, 'c'); 15 | 16 | def tearDown(self): 17 | try: 18 | self.db.close() 19 | except: 20 | pass 21 | try: 22 | os.remove(self.dbname) 23 | except: 24 | pass 25 | try: 26 | os.rmdir(self.tmpdir) 27 | except: 28 | pass 29 | 30 | def testRewind(self): 31 | # Save some docs 32 | self.db.save("foo1", "bar") 33 | self.db.save("foo2", "baz") 34 | self.db.save("foo3", "bell") 35 | self.db.save("foo4", "a") 36 | self.db.commit() 37 | 38 | # Edit some docs 39 | self.db.save("foo1", "new_bar") 40 | self.db.save("foo2", "new_baz") 41 | self.db.save("foo3", "new_bell") 42 | self.db.save("foo4", "new_a") 43 | self.db.commit() 44 | 45 | # The edits happened... 46 | self.assertNotEqual(self.db["foo3"], "bell"); 47 | self.assertEqual(self.db["foo4"], "new_a"); 48 | 49 | # rewind 50 | self.db.rewindHeader() 51 | 52 | # did we go back in time? 53 | self.assertEqual(self.db["foo3"], "bell"); 54 | self.assertNotEqual(self.db["foo4"], "new_a"); 55 | 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /tests/testapp.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "views/mapreduce/mapreduce.h" 19 | #include "btree_purge/purge_tests.h" 20 | #include "views/view_tests.h" 21 | 22 | int main(int argc, char* argv[]) { 23 | mapreduce_init(argv[1]); 24 | 25 | view_tests(); 26 | purge_tests(); 27 | 28 | mapreduce_deinit(); 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /tests/testlib.lua: -------------------------------------------------------------------------------- 1 | local M = {} 2 | local failures = 0 3 | 4 | -- Verify a doc with the given key exists with the given value in the 5 | -- given database. 6 | function M.check_doc(db, key, value) 7 | local doc = db:get(key) 8 | if doc ~= value then 9 | error(string.format("Expected '%s' (%d bytes), got '%s' (%d bytes)", 10 | value, #value, doc, #doc)) 11 | end 12 | end 13 | 14 | -- Verify a local doc with the given key exists with the given value 15 | -- in the given database. 16 | function M.check_local_doc(db, key, value) 17 | local doc = db:get_local(key) 18 | if doc ~= value then 19 | error(string.format("Expected '%s' (%d bytes), got '%s' (%d bytes)", 20 | value, #value, doc, #doc)) 21 | end 22 | end 23 | 24 | -- Verify a document does not exist. 25 | function M.assert_no_doc(db, key) 26 | local succeded, result = pcall(db['get'], db, key) 27 | if succeeded then 28 | error("Expected missing doc for " .. key .. " got " .. result) 29 | end 30 | end 31 | 32 | -- Verify a local doc does not exist. 33 | function M.assert_no_local_doc(db, key) 34 | local succeded, result = pcall(db['get_local'], db, key) 35 | if succeeded then 36 | error("Expected missing local doc for " .. key .. " got " .. result) 37 | end 38 | end 39 | 40 | -- Run a named test function. 41 | -- 42 | -- The function will receive a filename it should use for the 43 | -- database. The file will be removed at the end of the test run. If 44 | -- the test function runs without error, the test is considered 45 | -- successful. 46 | function M.run_test(name, fun) 47 | local dbname = os.tmpname() 48 | 49 | local succeeded, result = pcall(fun, dbname) 50 | 51 | os.remove(dbname) 52 | 53 | if succeeded then 54 | print(name .. ": PASS") 55 | else 56 | print(name .. ": FAIL (" .. result .. ")") 57 | failures = failures + 1 58 | end 59 | end 60 | 61 | function M.fail_count() 62 | return failures 63 | end 64 | 65 | return M 66 | -------------------------------------------------------------------------------- /tests/views/bitmaps.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "view_tests.h" 22 | #include 23 | 24 | void test_bitmaps(void) 25 | { 26 | bitmap_t bm, bm1, bm2; 27 | uint16_t one_bits[] = {1023, 1013, 500, 401, 1, 7, 666, 69}; 28 | int set; 29 | uint16_t i, j; 30 | 31 | fprintf(stderr, "Running view bitmap tests\n"); 32 | for (j = 0; j < (sizeof(one_bits) / sizeof(uint16_t)); ++j) { 33 | set_bit(&bm, one_bits[j]); 34 | } 35 | 36 | for (i = 0; i < 1024; ++i) { 37 | set = 0; 38 | for (j = 0; j < (sizeof(one_bits) / sizeof(uint16_t)); ++j) { 39 | if (one_bits[j] == i) { 40 | set = 1; 41 | break; 42 | } 43 | } 44 | 45 | if (set) { 46 | cb_assert(is_bit_set(&bm, i)); 47 | } else { 48 | cb_assert(!is_bit_set(&bm, i)); 49 | } 50 | 51 | unset_bit(&bm, i); 52 | cb_assert(!is_bit_set(&bm, i)); 53 | } 54 | 55 | bm = {}; 56 | set_bit(&bm, 1023); 57 | set_bit(&bm, 514); 58 | set_bit(&bm, 0); 59 | for (i = 0; i < (sizeof((bm.chunks)) / sizeof(bm.chunks[0])); ++i) { 60 | switch (i) { 61 | case 0: 62 | cb_assert(bm.chunks[i] == 0x80); 63 | break; 64 | case 63: 65 | cb_assert(bm.chunks[i] == 0x04); 66 | break; 67 | case 127: 68 | cb_assert(bm.chunks[i] == 0x01); 69 | break; 70 | default: 71 | cb_assert(bm.chunks[i] == 0); 72 | } 73 | } 74 | 75 | set_bit(&bm1, 1023); 76 | set_bit(&bm2, 0); 77 | 78 | union_bitmaps(&bm1, &bm2); 79 | cb_assert(bm1.chunks[0] == 0x80); 80 | cb_assert(bm1.chunks[127] == 0x01); 81 | 82 | /* Tests for intersection operation */ 83 | bm1 = {}; 84 | bm2 = {}; 85 | set_bit(&bm1, 0); 86 | set_bit(&bm1, 7); 87 | set_bit(&bm2, 800); 88 | set_bit(&bm2, 801); 89 | intersect_bitmaps(&bm1, &bm2); 90 | cb_assert(bm1.chunks[0] == 0x0); 91 | cb_assert(bm1.chunks[100] == 0x0); 92 | 93 | set_bit(&bm1, 0); 94 | set_bit(&bm1, 1023); 95 | set_bit(&bm2, 7); 96 | set_bit(&bm2, 1023); 97 | 98 | intersect_bitmaps(&bm1, &bm2); 99 | cb_assert(bm1.chunks[0] == 0x80); 100 | cb_assert(bm1.chunks[127] == 0x0); 101 | 102 | /* Tests for is_equal operation */ 103 | bm1 = {}; 104 | bm2 = {}; 105 | cb_assert(is_equal_bitmap(&bm1, &bm2)); 106 | set_bit(&bm1, 7); 107 | set_bit(&bm1, 500); 108 | set_bit(&bm2, 7); 109 | set_bit(&bm2, 500); 110 | cb_assert(is_equal_bitmap(&bm1, &bm2)); 111 | set_bit(&bm2, 1000); 112 | cb_assert(!is_equal_bitmap(&bm1, &bm2)); 113 | 114 | } 115 | -------------------------------------------------------------------------------- /tests/views/keys.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | #include "view_tests.h" 4 | 5 | #include 6 | #include 7 | 8 | static view_btree_key_t *test_view_btree_key_decoding(const char *key_bin, size_t len) 9 | { 10 | view_btree_key_t* k = nullptr; 11 | 12 | cb_assert(decode_view_btree_key(key_bin, len, &k) == COUCHSTORE_SUCCESS); 13 | cb_assert(k != nullptr); 14 | cb_assert(k->json_key.size == 4); 15 | cb_assert(memcmp(k->json_key.buf, "\"23\"", k->json_key.size) == 0); 16 | cb_assert(k->doc_id.size == 12); 17 | cb_assert(memcmp(k->doc_id.buf, "doc_00000023", k->doc_id.size) == 0); 18 | 19 | sized_buf json_key{}; 20 | cb_assert(decode_view_btree_json_key(key_bin, len, json_key) == 21 | COUCHSTORE_SUCCESS); 22 | cb_assert(json_key.size == 4); 23 | cb_assert(memcmp(json_key.buf, "\"23\"", json_key.size) == 0); 24 | cb_free(json_key.buf); 25 | return k; 26 | } 27 | 28 | static view_id_btree_key_t *test_view_id_btree_key_decoding(const char *id_btree_key_bin, size_t len) 29 | { 30 | view_id_btree_key_t* k = nullptr; 31 | 32 | cb_assert(decode_view_id_btree_key(id_btree_key_bin, len, &k) == COUCHSTORE_SUCCESS); 33 | cb_assert(k != nullptr); 34 | 35 | cb_assert(k->partition == 57); 36 | cb_assert(k->doc_id.size == 12); 37 | cb_assert(memcmp(k->doc_id.buf, "doc_00000057", k->doc_id.size) == 0); 38 | 39 | return k; 40 | } 41 | 42 | static void test_view_btree_key_encoding(const view_btree_key_t *k, 43 | char **buffer, 44 | size_t *size) 45 | { 46 | couchstore_error_t res; 47 | 48 | res = encode_view_btree_key(k, buffer, size); 49 | cb_assert(res == COUCHSTORE_SUCCESS); 50 | } 51 | 52 | 53 | static void test_view_id_btree_key_encoding(const view_id_btree_key_t *k, 54 | char **buffer, 55 | size_t *size) 56 | { 57 | couchstore_error_t res; 58 | 59 | res = encode_view_id_btree_key(k, buffer, size); 60 | cb_assert(res == COUCHSTORE_SUCCESS); 61 | } 62 | 63 | void test_keys() 64 | { 65 | char key_bin[] = { 66 | 0,4,34,50,51,34,100,111,99,95,48,48,48,48,48,48,50,51 67 | }; 68 | char id_btree_key_bin[] = { 69 | 0,57,100,111,99,95,48,48,48,48,48,48,53,55 70 | }; 71 | char* k_bin2 = nullptr; 72 | size_t k_bin2_size = 0; 73 | char* id_btree_k_bin2 = nullptr; 74 | size_t id_btree_k_bin2_size = 0; 75 | char* k_bin3 = nullptr; 76 | size_t k_bin3_size = 0; 77 | char* id_btree_k_bin3 = nullptr; 78 | size_t id_btree_k_bin3_size = 0; 79 | view_btree_key_t *k; 80 | view_id_btree_key_t *id_btree_k; 81 | view_btree_key_t *k2; 82 | view_id_btree_key_t *id_btree_k2; 83 | 84 | fprintf(stderr, "Decoding a view btree key ...\n"); 85 | k = test_view_btree_key_decoding(key_bin, sizeof(key_bin)); 86 | 87 | fprintf(stderr, "Decoding a view id btree key ...\n"); 88 | id_btree_k = test_view_id_btree_key_decoding(id_btree_key_bin, sizeof(id_btree_key_bin)); 89 | 90 | fprintf(stderr, "Encoding the previously decoded view btree key ...\n"); 91 | test_view_btree_key_encoding(k, &k_bin2, &k_bin2_size); 92 | 93 | cb_assert(k_bin2_size == sizeof(key_bin)); 94 | cb_assert(memcmp(k_bin2, key_bin, k_bin2_size) == 0); 95 | 96 | fprintf(stderr, "Encoding the previously decoded view id btree key ...\n"); 97 | test_view_id_btree_key_encoding(id_btree_k, &id_btree_k_bin2, &id_btree_k_bin2_size); 98 | 99 | cb_assert(id_btree_k_bin2_size == sizeof(id_btree_key_bin)); 100 | cb_assert(memcmp(id_btree_k_bin2, id_btree_key_bin, id_btree_k_bin2_size) == 0); 101 | 102 | fprintf(stderr, "Decoding the previously encoded view btree key ...\n"); 103 | k2 = test_view_btree_key_decoding(k_bin2, k_bin2_size); 104 | 105 | fprintf(stderr, "Decoding the previously encoded view id btree key ...\n"); 106 | id_btree_k2 = test_view_id_btree_key_decoding(id_btree_k_bin2, id_btree_k_bin2_size); 107 | 108 | fprintf(stderr, "Encoding the previously decoded view btree key ...\n"); 109 | test_view_btree_key_encoding(k2, &k_bin3, &k_bin3_size); 110 | 111 | cb_assert(k_bin3_size == sizeof(key_bin)); 112 | cb_assert(memcmp(k_bin3, key_bin, k_bin3_size) == 0); 113 | 114 | fprintf(stderr, "Encoding the previously decoded view id btree key ...\n"); 115 | test_view_id_btree_key_encoding(id_btree_k2, &id_btree_k_bin3, &id_btree_k_bin3_size); 116 | 117 | cb_assert(id_btree_k_bin3_size == sizeof(id_btree_key_bin)); 118 | cb_assert(memcmp(id_btree_k_bin3, id_btree_key_bin, id_btree_k_bin3_size) == 0); 119 | 120 | free_view_btree_key(k); 121 | free_view_btree_key(k2); 122 | cb_free(k_bin2); 123 | cb_free(k_bin3); 124 | 125 | free_view_id_btree_key(id_btree_k); 126 | free_view_id_btree_key(id_btree_k2); 127 | cb_free(id_btree_k_bin2); 128 | cb_free(id_btree_k_bin3); 129 | } 130 | -------------------------------------------------------------------------------- /tests/views/spatial_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Volker Mische 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_SPATIAL_TESTS_H 22 | #define _VIEW_SPATIAL_TESTS_H 23 | 24 | #include "couchstore_config.h" 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "../macros.h" 30 | #include "../src/views/bitmap.h" 31 | #include "../src/views/spatial.h" 32 | 33 | /* Those functions are normaly static. They are declared here to prevent 34 | * compile time warning */ 35 | couchstore_error_t encode_spatial_key(const sized_mbb_t *mbb, 36 | char *key, 37 | size_t nkey); 38 | couchstore_error_t decode_spatial_key(const char *key, sized_mbb_t *mbb); 39 | couchstore_error_t expand_mbb(sized_mbb_t *original, sized_mbb_t *expander); 40 | 41 | 42 | void test_interleaving(void); 43 | void test_spatial_scale_factor(void); 44 | void test_spatial_center(void); 45 | void test_spatial_scale_point(void); 46 | void test_set_bit_sized(void); 47 | void test_encode_spatial_key(void); 48 | void test_decode_spatial_key(void); 49 | void test_expand_mbb(void); 50 | void test_view_spatial_reduce(void); 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /tests/views/tests.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "view_tests.h" 22 | #include "spatial_tests.h" 23 | 24 | 25 | void view_tests() 26 | { 27 | fprintf(stderr, "\n\nRunning view tests\n\n"); 28 | 29 | test_bitmaps(); 30 | test_sorted_lists(); 31 | test_collate_json(); 32 | test_reductions(); 33 | test_keys(); 34 | test_values(); 35 | reducer_tests(); 36 | cleanup_tests(); 37 | 38 | /* spatial tests */ 39 | test_interleaving(); 40 | test_spatial_scale_factor(); 41 | test_spatial_center(); 42 | test_spatial_scale_point(); 43 | test_set_bit_sized(); 44 | test_encode_spatial_key(); 45 | test_decode_spatial_key(); 46 | test_expand_mbb(); 47 | test_view_spatial_reduce(); 48 | } 49 | -------------------------------------------------------------------------------- /tests/views/view_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | #pragma once 21 | 22 | #include "../macros.h" 23 | #include "../src/views/bitmap.h" 24 | #include "../src/views/index_header.h" 25 | #include "../src/views/keys.h" 26 | #include "../src/views/purgers.h" 27 | #include "../src/views/reducers.h" 28 | #include "../src/views/reductions.h" 29 | #include "../src/views/sorted_list.h" 30 | #include "../src/views/values.h" 31 | #include "couchstore_config.h" 32 | #include 33 | #include 34 | #include 35 | 36 | void view_tests(); 37 | void test_bitmaps(); 38 | void test_sorted_lists(); 39 | void test_collate_json(); 40 | void test_index_headers_v1(); 41 | void test_index_headers_v2(); 42 | void test_reductions(); 43 | void test_keys(); 44 | void test_values(); 45 | void reducer_tests(); 46 | void cleanup_tests(); 47 | -------------------------------------------------------------------------------- /tests/wrapped_fileops_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "wrapped_fileops_test.h" 19 | #include "iobuffer.h" 20 | 21 | /** 22 | * This is a slightly modified version of the regular 23 | * BufferedFileOps class. 24 | * 25 | * The regular BufferedFileOps has a custom `constructor` 26 | * method outside of the FileOpsInterface. This version 27 | * uses its C++ constructor to set the File ops to be 28 | * wrapped for the `constructor` method in the interface 29 | * which allows it to share tests and bootstrapping with 30 | * other FileOps implementations. 31 | */ 32 | class TestBufferedFileOps : public BufferedFileOps { 33 | public: 34 | TestBufferedFileOps(FileOpsInterface* ops) : wrapped_ops(ops) { 35 | } 36 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override { 37 | return BufferedFileOps::constructor(errinfo, wrapped_ops.get(), 38 | buffered_file_ops_params()); 39 | } 40 | 41 | protected: 42 | std::unique_ptr wrapped_ops; 43 | }; 44 | 45 | 46 | typedef testing::Types> 47 | WrappedOpsImplementations; 48 | 49 | typedef testing::Types 50 | BufferedWrappedOpsImplementations; 51 | 52 | typedef testing::Types> 53 | UnbufferedWrappedOpsImplementations; 54 | 55 | INSTANTIATE_TYPED_TEST_SUITE_P(CouchstoreOpsTest, 56 | WrappedOpsTest, 57 | WrappedOpsImplementations 58 | ); 59 | 60 | INSTANTIATE_TYPED_TEST_SUITE_P(CouchstoreOpsTest, 61 | UnbufferedWrappedOpsTest, 62 | UnbufferedWrappedOpsImplementations 63 | ); 64 | 65 | INSTANTIATE_TYPED_TEST_SUITE_P(CouchstoreOpsTest, 66 | BufferedWrappedOpsTest, 67 | BufferedWrappedOpsImplementations 68 | ); 69 | -------------------------------------------------------------------------------- /view_format.md: -------------------------------------------------------------------------------- 1 | # The Binary (Termless) Format for Views. # 2 | 3 | **(Damien Katz -- July 19 2012)** 4 | 5 | This documents the format of the Key/Value pairs in leaf nodes, and the reduction format in inner nodes. 6 | 7 | The underlying b-tree format is the same as already used in CouchStore. 8 | 9 | * All integers are network byte order. 10 | * All strings are encoded as UTF-8. 11 | * Adjacent fields are tightly packed with no padding. 12 | 13 | 14 | ## Primary Index Key Values ## 15 | 16 | In leaf nodes, `KeyValues` have the following format: 17 | 18 | * `Key`: 19 | * `EmittedJsonKeyLength` -- 16bit integer 20 | * `EmittedJSONKey` -– JSON -– The key emitted by the map function 21 | * `UnquotedDocId` –- String -– The raw doc ID (occupies the remaining bytes) 22 | * Value: 23 | * `PartitionId` -- 16bit integer -- This is the partitionId (vbucket) from which this document id maps to. 24 | * 1 to infinity `JSONStringValue`s -- These are all the values that were emitted for this `EmittedJSONKey`. 25 | Each `JSONStringValue` is of the form: 26 | * `ValueLength` -- 24bit unsigned integer 27 | * `JSONValue` - string that is `ValueLength` bytes long 28 | 29 | (Parsing the `JSONStringValue`s is simply reading the first 24 bits, getting the length of the following string and extracting the string. If there is still buffer left, the process is repeated until the is no value buffer left.) 30 | 31 | When an emit happens, and the Key is different from all other keys emitted for that document, then there is only one `JSONStringValue`. 32 | But when multiple identical keys are emitted, the values are coalesced into a list of Values, and there will be multiple values. 33 | 34 | 35 | ### Primary Index Inner Node Reductions (KeyPointerNodes and Root) ### 36 | 37 | * `SubTreeCount` -- 40bit integer -- Count of all Values in subtree. 38 | NOTE: this is possibly greater than the `KeyId` count in the subtree, because a document can emit multiple identical keys, and they are coalesced into single `KeyId`, with all the values emitted in a list as the value. 39 | * `SubTreePartitionBitmap` -- 1024 bits -- a bitfield of all partition keys in the subtree. Currently this is hardcoded at 1024 bits in length, but in the future we may change this to a variable size. Until then, it works with any # of vbuckets ≤ 1024. 40 | * `JSONReductions` -- remaining bytes -- Zero or more `JSONReductions`, each consisting of: 41 | * `JSONLen` -- 16bit integer 42 | * `JSON` -- the actual JSON string 43 | 44 | 45 | ## Back Index ## 46 | 47 | In leaf nodes, `KeyValues` have the following format: 48 | 49 | * `Key` -- blob -- The raw docId, not quoted or JSONified in any way. 50 | * `Value`: 51 | * `PartitionId` -- 16bit integer -- This is the partitionId (vbucket) from which this document id maps to. 52 | * 1-*n* `ViewKeysMappings`, where *n* ≤ the # of mapfunctions defined in the design document. 53 | A `ViewKeysMapping` is: 54 | * `ViewId` -- 8bit integer -- the ordinal id of the map view in the group the following keys were emitted from 55 | * `NumKeys` -- 16bit integer -- the number of `JSONKeys` that follow 56 | * `JSONKeys` -- a sequence of: 57 | * `KeyLen` -- 16bit integer -- Length of the following `JSONKey` 58 | * `Key` -- JSON string -- Emitted JSON key 59 | 60 | 61 | ### Back Index Inner Node Reductions (KeyPointerNodes and Root) ### 62 | 63 | * `SubTreeCount` -- 40bit integer -- count of all Keys in subtree. 64 | * `SubTreePartitionBitmap` -- 1024 bits -- a bitfield of all partition keys in the subtree. Currently this is hardcoded at 1024 in length, but in the future we may change this to a variable size. Until then, it works with any # of vbuckets ≤ 1024. 65 | --------------------------------------------------------------------------------