├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── config.cmake.h.in ├── csdata.md ├── file_format.md ├── include └── libcouchstore │ ├── couch_common.h │ ├── couch_db.h │ ├── couch_index.h │ ├── couch_latency.h │ ├── error.h │ ├── file_ops.h │ └── visibility.h ├── programs ├── CMakeLists.txt └── dbdiff │ ├── CMakeLists.txt │ └── dbdiff.c ├── python ├── couchstore.py ├── couchstore_test.py ├── example1.py └── example_read.py ├── src ├── arena.cc ├── arena.h ├── bitfield.cc ├── bitfield.h ├── btree_modify.cc ├── btree_read.cc ├── compactor.cc ├── couch_btree.h ├── couch_create.cc ├── couch_db.cc ├── couch_file_read.cc ├── couch_file_write.cc ├── couch_latency.cc ├── couch_latency_internal.h ├── couch_save.cc ├── couchscript.cc ├── crc32.cc ├── crc32.h ├── db_compact.cc ├── dbck.cc ├── dbdump.cc ├── dbinfo.cc ├── fatbuf.h ├── file_merger.cc ├── file_merger.h ├── file_name_utils.c ├── file_name_utils.h ├── file_sorter.cc ├── file_sorter.h ├── internal.h ├── iobuffer.cc ├── iobuffer.h ├── llmsort.cc ├── mergesort.cc ├── mergesort.h ├── node_types.cc ├── node_types.h ├── os.cc ├── os.h ├── os_win.cc ├── quicksort.c ├── quicksort.h ├── reduces.cc ├── reduces.h ├── strerror.cc ├── tracking_file_ops.cc ├── tracking_file_ops.h ├── tree_writer.cc ├── tree_writer.h ├── util.cc ├── util.h ├── viewgen.c └── views │ ├── bin │ ├── couch_view_file_merger.cc │ ├── couch_view_group_cleanup.cc │ ├── couch_view_group_compactor.cc │ ├── couch_view_index_builder.cc │ ├── couch_view_index_updater.cc │ ├── util.c │ └── util.h │ ├── bitmap.c │ ├── bitmap.h │ ├── collate_json.cc │ ├── collate_json.h │ ├── compaction.cc │ ├── compaction.h │ ├── file_merger.cc │ ├── file_merger.h │ ├── file_sorter.cc │ ├── file_sorter.h │ ├── index_header.cc │ ├── index_header.h │ ├── keys.cc │ ├── keys.h │ ├── mapreduce │ ├── mapreduce.cc │ ├── mapreduce.h │ ├── mapreduce_c.cc │ ├── mapreduce_internal.h │ └── visibility.h │ ├── purgers.cc │ ├── purgers.h │ ├── reducers.cc │ ├── reducers.h │ ├── reductions.cc │ ├── reductions.h │ ├── sorted_list.c │ ├── sorted_list.h │ ├── spatial.cc │ ├── spatial.h │ ├── spatial_modify.cc │ ├── util.cc │ ├── util.h │ ├── values.cc │ ├── values.h │ ├── view_group.cc │ └── view_group.h ├── tests ├── btree_purge │ ├── purge.cc │ ├── purge_tests.h │ └── tests.cc ├── bulk.lua ├── changecount.py ├── changessincefilter.lua ├── compact.lua ├── corrupt.lua ├── couchstoredoctest.h ├── couchstoretest.cc ├── couchstoretest.h ├── documents.cc ├── documents.h ├── dropdel.lua ├── file_deduper_tests.c ├── file_merger_tests.c ├── file_sorter_tests.c ├── gtest_internal_tests.cc ├── gtest_tests.cc ├── large.lua ├── largefile.lua ├── localdoc.lua ├── macros.h ├── mapreduce │ ├── builtin.c │ ├── map.c │ ├── mapreduce_tests.h │ └── reduce.c ├── purge.py ├── rewind.py ├── test_fileops.cc ├── test_fileops.h ├── testapp.c ├── testlib.lua ├── views │ ├── bitmaps.cc │ ├── cleanup.cc │ ├── collate_json_test.cc │ ├── index_headers.cc │ ├── keys.cc │ ├── reducers.cc │ ├── reductions.cc │ ├── sorted_lists.cc │ ├── spatial.cc │ ├── spatial_tests.h │ ├── tests.cc │ ├── values.cc │ └── view_tests.h ├── wrapped_fileops_test.cc └── wrapped_fileops_test.h └── view_format.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Keep the entries sorted to reduce the risk for a merge conflict 2 | *.[ao] 3 | *.dirstamp 4 | *.exe 5 | *.l[ao] 6 | *.obj 7 | *.pyc 8 | *~ 9 | .deps/ 10 | .libs/ 11 | .pc 12 | /INSTALL 13 | /Makefile 14 | /Makefile.in 15 | /a.out* 16 | /aclocal.m4 17 | /autom4te.cache 18 | /build 19 | /config.cache 20 | /config.h 21 | /config.h.in 22 | /config.log 23 | /config.status 24 | /config/compile 25 | /config/config.guess 26 | /config/config.sub 27 | /config/depcomp 28 | /config/install-sh 29 | /config/ltmain.sh 30 | /config/missing 31 | /config/mkinstalldirs 32 | /config/plugin.ac 33 | /config/ar-lib 34 | /config/test-driver 35 | /configure 36 | /couch_compact 37 | /couch_dbdump 38 | /couch_dbinfo 39 | /couch_viewgen 40 | /couch_view_file_merger 41 | /couch_view_index_builder 42 | /couchscript 43 | /docs/doxy 44 | /libcouchstore*.changes 45 | /libcouchstore*.deb 46 | /libcouchstore*.dsc 47 | /libcouchstore*.rpm 48 | /libcouchstore*.tar.gz 49 | /libcouchstore.dll 50 | /libcouchstore.exp 51 | /libcouchstore.lib 52 | /libcouchstore.pc 53 | /libtool 54 | /m4/libtool.m4 55 | /m4/ltoptions.m4 56 | /m4/ltsugar.m4 57 | /m4/ltversion.m4 58 | /m4/lt~obsolete.m4 59 | /m4/version.m4 60 | /regression 61 | /stamp-h1 62 | /test.couch 63 | /testapp 64 | /writetest 65 | xcuserdata/ 66 | 67 | /CMakeFiles/ 68 | /CTestTestfile.cmake 69 | /cmake_install.cmake 70 | /couchstore_testapp 71 | /*.dylib* 72 | /*.so* 73 | /*.dll* 74 | /Testing/ 75 | /*.exe* 76 | /*.ilk 77 | /*.pdb 78 | /*.exp 79 | /*.lib 80 | /couch_view_group_cleanup 81 | /couch_view_index_updater 82 | /purge.couch 83 | /couch_view_group_compactor 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Couchbase (.couch) storage file C library 2 | 3 | Currently this library can only be built as part of Couchbase Server due to dependencies on the [Couchbase Platform Library][plat] and [Couchbase Server CMake project][tlm]. For instructions on building Couchbase Server, see the [Manifest Repository][manifest]. 4 | 5 | [plat]: https://github.com/couchbase/platform 6 | [tlm]: https://github.com/couchbase/tlm 7 | [manifest]: https://github.com/couchbase/manifest 8 | 9 | ## Tests: 10 | 11 | 1. `make test` 12 | 13 | This will run the native tests, and also the Lua tests if Lua was installed at the time the `configure` script ran. 14 | 15 | Tests use the CMake CTest system, and the [ctest](http://www.cmake.org/cmake/help/v2.8.8/ctest.html) command can be used to run invidual tests and print verbose output. 16 | -------------------------------------------------------------------------------- /config.cmake.h.in: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #pragma once 3 | 4 | #cmakedefine HAVE_NETINET_IN_H ${HAVE_NETINET_IN_H} 5 | #cmakedefine HAVE_ARPA_INET_H ${HAVE_ARPA_INET_H} 6 | #cmakedefine HAVE_INTTYPES_H ${HAVE_INTTYPES_H} 7 | #cmakedefine HAVE_UNISTD_H ${HAVE_UNISTD_H} 8 | #cmakedefine HAVE_FDATASYNC ${HAVE_FDATASYNC} 9 | #cmakedefine HAVE_QSORT_R ${HAVE_QSORT_R} 10 | 11 | /* Large File Support */ 12 | #define _LARGE_FILE 1 13 | #ifndef _FILE_OFFSET_BITS 14 | # define _FILE_OFFSET_BITS 64 15 | #elif (_FILE_OFFSET_BITS != 64) 16 | #error "bad things" 17 | #endif 18 | #define _LARGEFILE_SOURCE 1 19 | #ifndef O_LARGEFILE 20 | # define O_LARGEFILE 0 21 | #endif 22 | 23 | #include 24 | 25 | #ifdef HAVE_NETINET_IN_H 26 | #include 27 | #endif 28 | 29 | #ifdef HAVE_ARPA_INET_H 30 | #include 31 | #endif 32 | 33 | #ifdef HAVE_INTTYPES_H 34 | #include 35 | #endif 36 | 37 | #ifdef HAVE_UNISTD_H 38 | #include 39 | #endif 40 | 41 | #ifdef __APPLE__ 42 | /* autoconf things OS X has fdatasync but it doesn't */ 43 | #define fdatasync(FD) fsync(FD) 44 | #endif /* __APPLE__ */ 45 | 46 | #include 47 | 48 | #ifdef linux 49 | #undef ntohs 50 | #undef ntohl 51 | #undef htons 52 | #undef htonl 53 | #endif 54 | 55 | #if defined(WIN32) || defined(_WIN32) 56 | #define WINDOWS 57 | /* Mute: 58 | * "The POSIX name for this item is deprecated. Instead, use the ISO 59 | * C++ conformant name: _strdup." 60 | */ 61 | #pragma warning(disable: 4996) 62 | #define snprintf _snprintf 63 | #endif 64 | -------------------------------------------------------------------------------- /csdata.md: -------------------------------------------------------------------------------- 1 | ## Couchstore Data Types 2 | 3 | Struct members not listed should be considered internal to Couchstore. 4 | 5 | ### sized\_buf 6 | struct sized_buf { 7 | char* buf; 8 | size_t size; 9 | } 10 | 11 | Couchstore uses `sized_buf`s to point to data buffers. 12 | 13 | ### DocInfo 14 | struct DocInfo { 15 | sized_buf id; 16 | uint64_t db_seq; 17 | uint64_t rev_seq; 18 | sized_buf rev_meta; 19 | int deleted; 20 | uint8_t content_meta; 21 | } 22 | 23 | * `id` - Document ID 24 | * `db_seq` - Change sequence number the document was inserted at. 25 | * `rev_seq` - The version number of the document 26 | * `rev_meta` - Revision metadata. Used by ep-engine to store CAS value, expiry time, and flags 27 | * `deleted` - 1 if document should be considered "deleted" and not subject to indexing, otherwise 0. 28 | * `content_meta` - Number field used to store flags indicating metadata about the document content (is it JSON, etc.) 29 | 30 | #### The `content_meta` field 31 | 32 | * The least-significant two bits are used to encode why/whether the body is or is not JSON 33 | * 0 - Document body *is* JSON 34 | * 1 - Inserted data was not valid JSON 35 | * 2 - Inserted data was valid JSON, but contained a key reserved for internal use, such as one with a `$` prefix. 36 | * 3 - The document was inserted in non-JSON mode. 37 | * If the most-significant bit is set the document body data has been compressed with snappy. 38 | 39 | When saving documents with `save_doc` or `save_docs`, `id`, `rev_seq`, `rev_meta`, `deleted`, and `content_meta` must be set on the `DocInfo`s passed to Couchstore. The `db_seq` is determined at insert time. 40 | 41 | #### The `rev_meta` field 42 | 43 | [128 bit binary value] 44 | | 64 bits | 32 bits | 32 bits | 45 | -----------|------------|------------| 46 | CAS Value | Expiration | User flags | 47 | 48 | All values are Big Endian. 49 | 50 | * CAS Value - Random nonce value set by Memcached, used to allow ensuring value has not been modified since it was read when writing. 51 | * Expiration - 32 bit timestamp for value expiry time. 0 if item does not expire. 52 | * User flags - Memcache API allows users to set this 32 bit number along with values, it is not used by memcached internally. 53 | 54 | ### Doc 55 | struct Doc { 56 | sized_buf id; 57 | sized_buf data; 58 | } 59 | 60 | `id` contains the document ID, `data` contains the document body data. Couchstore does not compress or modify the body data in any way, unless the COMPRESS_DOC_BODIES or DECOMPRESS_DOC_BODIES flags are passed to the read and write functions, then it will check the `content_meta` of the corresponding doc info to see if it should compress or decompress the doc data. 61 | 62 | ### LocalDoc 63 | struct LocalDoc { 64 | sized_buf id; 65 | sized_buf json; 66 | int deleted; 67 | } 68 | 69 | * `id` - The local document ID, it must start with `_local/` 70 | * `json` - The local document body. 71 | * `deleted` - if set to 1 on a LocalDoc passed to save_local_doc, `json` will be ignored, and the local document with the ID in `id` will be removed, if it exists. 72 | 73 | 74 | ## C API 75 | 76 | Documented in `include/libcouchstore/couch_db.h` 77 | 78 | -------------------------------------------------------------------------------- /include/libcouchstore/couch_index.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef COUCHSTORE_COUCH_INDEX_H 3 | #define COUCHSTORE_COUCH_INDEX_H 4 | 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /** 12 | * Opaque reference to an open index file. 13 | */ 14 | typedef struct _CouchStoreIndex CouchStoreIndex; 15 | 16 | /* 17 | * Types of indexes. 18 | */ 19 | typedef uint64_t couchstore_index_type; 20 | enum { 21 | COUCHSTORE_VIEW_PRIMARY_INDEX = 0, /**< Primary index, maps emitted keys to values */ 22 | COUCHSTORE_VIEW_BACK_INDEX = 1, /**< Back-index, maps doc IDs to emitted keys */ 23 | }; 24 | 25 | /* 26 | * Available built-in reduce functions to use on the JSON values in primary indexes. 27 | * These are documented at 28 | */ 29 | typedef uint64_t couchstore_json_reducer; 30 | enum { 31 | COUCHSTORE_REDUCE_NONE = 0, /**< No reduction */ 32 | COUCHSTORE_REDUCE_COUNT = 1, /**< Count rows */ 33 | COUCHSTORE_REDUCE_SUM = 2, /**< Sum numeric values */ 34 | COUCHSTORE_REDUCE_STATS = 3, /**< Compute count, min, max, sum, sum of squares */ 35 | }; 36 | 37 | 38 | /** 39 | * Create a new index file. 40 | * 41 | * The file should be closed with couchstore_close_index(). 42 | * 43 | * @param filename The name of the file containing the index. Any existing file at this path 44 | * will be deleted. 45 | * @param index Pointer to where you want the handle to the index to be 46 | * stored. 47 | * @return COUCHSTORE_SUCCESS for success 48 | */ 49 | LIBCOUCHSTORE_API 50 | couchstore_error_t couchstore_create_index(const char *filename, 51 | CouchStoreIndex** index); 52 | 53 | /** 54 | * Close an open index file. 55 | * 56 | * @param index Pointer to the index handle to free. 57 | * @return COUCHSTORE_SUCCESS upon success 58 | */ 59 | LIBCOUCHSTORE_API 60 | couchstore_error_t couchstore_close_index(CouchStoreIndex* index); 61 | 62 | /** 63 | * Read an unsorted key-value file and add its contents to an index file. 64 | * Each file added will create a new independent index within the file; they are not merged. 65 | * 66 | * The key-value file is a sequence of zero or more records, each of which consists of: 67 | * key length (16 bits, big-endian) 68 | * value length (32 bits, big-endian) 69 | * key data 70 | * value data 71 | * The data formats inside the actual keys and values differ with the index types, and are 72 | * documented in "The Binary (Termless) Format for Views" (view_format.md). 73 | * 74 | * @param inputPath The path to the key-value file 75 | * @param index_type The type of index keys/values in the input file, and the type of index 76 | * to generate in the index file. Note: CouchStore can currently add only one back index. 77 | * @param reduce_function The type of JSON reduce function to apply to the data. Valid only 78 | * for primary indexes; ignored in back-indexes. 79 | * @param index The index file to write to 80 | * @return COUCHSTORE_SUCCESS on success, else an error code 81 | */ 82 | LIBCOUCHSTORE_API 83 | couchstore_error_t couchstore_index_add(const char *inputPath, 84 | couchstore_index_type index_type, 85 | couchstore_json_reducer reduce_function, 86 | CouchStoreIndex* index); 87 | 88 | #ifdef __cplusplus 89 | } 90 | #endif 91 | #endif 92 | -------------------------------------------------------------------------------- /include/libcouchstore/couch_latency.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "couch_common.h" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | using CouchLatencyHisto = MicrosecondHistogram; 27 | using CouchLatencyMicroSec = UnsignedMicroseconds; 28 | using CouchLatencyMicroSecRep = UnsignedMicroseconds::rep; 29 | 30 | /** 31 | * Begin to collect Couchstore API latency. 32 | */ 33 | LIBCOUCHSTORE_API 34 | void couchstore_latency_collector_start(); 35 | 36 | /** 37 | * The callback function used by couchstore_get_latency_info(), 38 | * to get latency histogram data. 39 | * 40 | * @param stat_name Name of statistic item. 41 | * @param latencies Latency histogram of the statistic item. 42 | * @param elapsed_time Total elapsed time in microseconds. 43 | * @param ctx User context 44 | * @return 1 to stop getting latency info, 0 otherwise. 45 | */ 46 | using couchstore_latency_callback_fn = 47 | std::function; 51 | 52 | /** 53 | * Latency info dump options. 54 | */ 55 | typedef struct _couchstore_latency_dump_options { 56 | /** 57 | * Currently empty, but left it for future extension. 58 | */ 59 | } couchstore_latency_dump_options; 60 | 61 | /** 62 | * Dump collected latency data through the given 63 | * callback function. 64 | * 65 | * @param callback Callback function. 66 | * @param options Latency dump options. 67 | * @param ctx User context 68 | */ 69 | LIBCOUCHSTORE_API 70 | void couchstore_get_latency_info(couchstore_latency_callback_fn callback, 71 | couchstore_latency_dump_options options, 72 | void *ctx); 73 | 74 | /** 75 | * Stop collecting Couchstore API latency. 76 | */ 77 | LIBCOUCHSTORE_API 78 | void couchstore_latency_collector_stop(); 79 | 80 | -------------------------------------------------------------------------------- /include/libcouchstore/error.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef LIBCOUCHSTORE_ERROR_H 3 | #define LIBCOUCHSTORE_ERROR_H 1 4 | 5 | #ifndef COUCHSTORE_COUCH_DB_H 6 | #error "You should include instead" 7 | #endif 8 | 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | 13 | /** Error values returned by CouchStore API calls. */ 14 | typedef enum { 15 | COUCHSTORE_SUCCESS = 0, 16 | COUCHSTORE_ERROR_OPEN_FILE = -1, 17 | COUCHSTORE_ERROR_CORRUPT = -2, 18 | COUCHSTORE_ERROR_ALLOC_FAIL = -3, 19 | COUCHSTORE_ERROR_READ = -4, 20 | COUCHSTORE_ERROR_DOC_NOT_FOUND = -5, 21 | COUCHSTORE_ERROR_NO_HEADER = -6, 22 | COUCHSTORE_ERROR_WRITE = -7, 23 | COUCHSTORE_ERROR_HEADER_VERSION = -8, 24 | COUCHSTORE_ERROR_CHECKSUM_FAIL = -9, 25 | COUCHSTORE_ERROR_INVALID_ARGUMENTS = -10, 26 | COUCHSTORE_ERROR_NO_SUCH_FILE = -11, 27 | COUCHSTORE_ERROR_CANCEL = -12, 28 | COUCHSTORE_ERROR_REDUCTION_TOO_LARGE = -13, 29 | COUCHSTORE_ERROR_REDUCER_FAILURE = -14, 30 | COUCHSTORE_ERROR_FILE_CLOSED = -15, 31 | COUCHSTORE_ERROR_DB_NO_LONGER_VALID = -16, 32 | COUCHSTORE_ERROR_FILE_CLOSE = -17, 33 | COUCHSTORE_ERROR_NOT_SUPPORTED = -18, 34 | } couchstore_error_t; 35 | 36 | #ifdef __cplusplus 37 | } 38 | #endif 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /include/libcouchstore/visibility.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef LIBCOUCHSTORE_VISIBILITY_H 3 | #define LIBCOUCHSTORE_VISIBILITY_H 4 | 5 | #if defined(LIBCOUCHSTORE_INTERNAL) 6 | 7 | #if defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) 8 | #define LIBCOUCHSTORE_API __global 9 | #elif defined __GNUC__ 10 | #define LIBCOUCHSTORE_API __attribute__ ((visibility("default"))) 11 | #elif defined(_MSC_VER) 12 | #define LIBCOUCHSTORE_API extern __declspec(dllexport) 13 | #else 14 | #define LIBCOUCHSTORE_API 15 | #endif 16 | 17 | #else 18 | 19 | #if defined(_MSC_VER) && !defined(LIBCOUCHSTORE_NO_VISIBILITY) 20 | #define LIBCOUCHSTORE_API extern __declspec(dllimport) 21 | #else 22 | #define LIBCOUCHSTORE_API 23 | #endif 24 | 25 | #endif 26 | 27 | 28 | #if defined(TESTAPP) 29 | #define STATIC 30 | #else 31 | #define STATIC static 32 | #endif 33 | 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /programs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_SUBDIRECTORY(dbdiff) 2 | -------------------------------------------------------------------------------- /programs/dbdiff/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_EXECUTABLE(dbdiff dbdiff.c) 2 | TARGET_LINK_LIBRARIES(dbdiff couchstore) 3 | 4 | INSTALL(TARGETS dbdiff 5 | RUNTIME DESTINATION bin) 6 | -------------------------------------------------------------------------------- /python/example1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Simple example program to create a Couchstore file with a given 4 | # number of keys. 5 | 6 | from couchstore import CouchStore, DocumentInfo, SizedBuf 7 | import os 8 | import struct 9 | import sys 10 | 11 | BATCH_SIZE = 10000 12 | REV_META_PACK = ">QII" 13 | 14 | def insert(db, key, rev, value): 15 | info = DocumentInfo(key) 16 | info.revSequence = rev 17 | # cas, exp, flags 18 | info.revMeta = str(struct.pack(REV_META_PACK, 1, 2, 3)) 19 | info.deleted = False 20 | return db.save(info, value) 21 | 22 | 23 | def insert_multi(db, keys, values): 24 | """Inserts multiple keys / values.""" 25 | 26 | ids = [] 27 | for k in keys: 28 | info = DocumentInfo(k) 29 | info.revSequence = 1 30 | # cas, exp, flags 31 | info.revMeta = str(struct.pack(REV_META_PACK, 1, 2, 3)) 32 | info.deleted = False 33 | ids.append(info) 34 | return db.saveMultiple(ids, values) 35 | 36 | 37 | def chunks(l, n): 38 | """Yield successive n-sized chunks from l.""" 39 | for i in range(0, len(l), n): 40 | yield l[i:i+n] 41 | 42 | 43 | def main(): 44 | if len(sys.argv) != 3: 45 | print "Usage: example1 " 46 | exit(1) 47 | db = CouchStore(sys.argv[2], 'c') 48 | for batch in chunks(range(0, int(sys.argv[1])), BATCH_SIZE): 49 | insert_multi(db, 50 | ["key_" + str(x) for x in batch], 51 | [str(x) for x in batch]) 52 | db.commit() 53 | db.close() 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /python/example_read.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Simple program to read the given keys from a Couchstore file. 4 | 5 | from couchstore import CouchStore, DocumentInfo, SizedBuf 6 | import argparse 7 | import os 8 | import struct 9 | import sys 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("file") 14 | parser.add_argument("keys", help="Key(s) to print", nargs="+") 15 | parser.add_argument("--unbuffered", help="Disable couchstore io buffering", action="store_true") 16 | args = parser.parse_args() 17 | 18 | db = CouchStore(args.file, 'r', unbuffered=args.unbuffered) 19 | for key in args.keys: 20 | print db.get(key) 21 | db.close() 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /src/arena.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** arena.h 3 | ** couchstore 4 | ** 5 | ** Created by Jens Alfke on 4/13/12. 6 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 7 | */ 8 | 9 | #ifndef COUCHSTORE_ARENA_H 10 | #define COUCHSTORE_ARENA_H 11 | 12 | #include 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | /** Opaque arena-allocator object. */ 19 | typedef struct arena arena; 20 | 21 | /** Saved position/state of an arena. */ 22 | typedef struct arena_position arena_position; 23 | 24 | /** 25 | * Creates a new arena allocator. 26 | * @param chunk_size The size of the memory blocks the allocator sub-allocates from malloc. Pass 0 for the default (32kbytes). 27 | * @return The new arena object. 28 | */ 29 | arena* new_arena(size_t chunk_size); 30 | 31 | /** 32 | * Deletes an arena and all of its memory allocations. 33 | */ 34 | void delete_arena(arena*); 35 | 36 | /** 37 | * Allocates memory from an arena. 38 | * @param arena The arena to allocate from 39 | * @param size The number of bytes to allocate 40 | * @return A pointer to the allocated block, or NULL on failure. 41 | */ 42 | void* arena_alloc(arena*, size_t size); 43 | 44 | /** 45 | * Allocates unaligned memory from an arena. 46 | * Saves a couple of bytes if your block doesn't need to be word-aligned. 47 | */ 48 | void* arena_alloc_unaligned(arena* a, size_t size); 49 | 50 | /** 51 | * "Frees" a block allocated from an arena. This actually doesn't do anything. 52 | * In a debug build it at least checks that the pointers was (probably) allocated from the arena. 53 | * In a release build it's explicitly an inlined no-op and there's no need to call it at all. 54 | */ 55 | void arena_free(arena*, void*); 56 | 57 | /** 58 | * Captures the current state of an arena, i.e. which blocks have been allocated. 59 | * Save the return value and pass it to arena_free_from_mark later to free all blocks allocated 60 | * since this point. 61 | */ 62 | const arena_position* arena_mark(arena *a); 63 | 64 | /** 65 | * Frees all blocks from the arena that have been allocated since the corresponding call to 66 | * arena_mark. (The mark remains valid and can be used again.) 67 | */ 68 | void arena_free_from_mark(arena *a, const arena_position *mark); 69 | 70 | /** 71 | * Frees all blocks from the arena. 72 | */ 73 | void arena_free_all(arena *a); 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif /* COUCHSTORE_ARENA_H */ 80 | -------------------------------------------------------------------------------- /src/bitfield.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | #include "internal.h" 4 | #include "bitfield.h" 5 | 6 | /* Functions for decoding raw_xx types to native integers: */ 7 | 8 | #define DECODE_RAW(DST_TYPE, FLIP_FN) \ 9 | DST_TYPE value = 0; \ 10 | memcpy((char*)&value + sizeof(value) - sizeof(raw), &(raw), sizeof(raw)); \ 11 | return FLIP_FN(value) 12 | 13 | #define DECODE_RAW_POINTER(DST_TYPE, FLIP_FN) \ 14 | DST_TYPE value = 0; \ 15 | memcpy((char*)&value + sizeof(value) - sizeof(*rawp), rawp, sizeof(*rawp)); \ 16 | return FLIP_FN(value) 17 | 18 | LIBCOUCHSTORE_API 19 | uint8_t couchstore_decode_raw08(raw_08 raw) 20 | { 21 | return raw.raw_bytes[0]; 22 | } 23 | 24 | LIBCOUCHSTORE_API 25 | uint16_t couchstore_decode_raw16(raw_16 raw) 26 | { 27 | DECODE_RAW(uint16_t, ntohs); 28 | } 29 | 30 | LIBCOUCHSTORE_API 31 | uint32_t couchstore_decode_raw24p(const raw_24 *rawp) 32 | { 33 | DECODE_RAW_POINTER(uint32_t, ntohl); 34 | } 35 | 36 | LIBCOUCHSTORE_API 37 | uint32_t couchstore_decode_raw32(raw_32 raw) 38 | { 39 | DECODE_RAW(uint32_t, ntohl); 40 | } 41 | 42 | LIBCOUCHSTORE_API 43 | uint64_t couchstore_decode_raw40p(const raw_40 *rawp) 44 | { 45 | DECODE_RAW_POINTER(uint64_t, ntohll); 46 | } 47 | 48 | LIBCOUCHSTORE_API 49 | uint64_t couchstore_decode_raw48p(const raw_48 *rawp) 50 | { 51 | DECODE_RAW_POINTER(uint64_t, ntohll); 52 | } 53 | 54 | LIBCOUCHSTORE_API 55 | uint64_t couchstore_decode_raw64(raw_64 raw) 56 | { 57 | DECODE_RAW(uint64_t, ntohll); 58 | } 59 | 60 | 61 | /* Functions for encoding native integers to raw_xx types: */ 62 | 63 | #define ENCODE_RAW(FLIP_FN, RAW_TYPE) \ 64 | RAW_TYPE raw; \ 65 | value = FLIP_FN(value); \ 66 | memcpy(&raw, (char*)&value + sizeof(value) - sizeof(raw), sizeof(raw)); \ 67 | return raw 68 | 69 | #define ENCODE_RAW_POINTER(FLIP_FN, RAW_TYPE) \ 70 | value = FLIP_FN(value); \ 71 | memcpy(raw, (char*)&value + sizeof(value) - sizeof(*raw), sizeof(*raw)); \ 72 | 73 | LIBCOUCHSTORE_API 74 | raw_08 couchstore_encode_raw08(uint8_t value) 75 | { 76 | ENCODE_RAW((uint8_t), raw_08); 77 | } 78 | 79 | LIBCOUCHSTORE_API 80 | raw_16 couchstore_encode_raw16(uint16_t value) 81 | { 82 | ENCODE_RAW(htons, raw_16); 83 | } 84 | 85 | LIBCOUCHSTORE_API 86 | void couchstore_encode_raw24(uint32_t value, raw_24 *raw) 87 | { 88 | ENCODE_RAW_POINTER(htonl, raw_24); 89 | } 90 | 91 | LIBCOUCHSTORE_API 92 | raw_32 couchstore_encode_raw32(uint32_t value) 93 | { 94 | ENCODE_RAW(htonl, raw_32); 95 | } 96 | 97 | LIBCOUCHSTORE_API 98 | void couchstore_encode_raw40(uint64_t value, raw_40 *raw) 99 | { 100 | ENCODE_RAW_POINTER(htonll, raw_40); 101 | } 102 | 103 | LIBCOUCHSTORE_API 104 | void couchstore_encode_raw48(uint64_t value, raw_48 *raw) 105 | { 106 | ENCODE_RAW_POINTER(htonll, raw_48); 107 | } 108 | 109 | LIBCOUCHSTORE_API 110 | raw_64 couchstore_encode_raw64(uint64_t value) 111 | { 112 | ENCODE_RAW(htonll, raw_64); 113 | } 114 | -------------------------------------------------------------------------------- /src/bitfield.h: -------------------------------------------------------------------------------- 1 | #ifndef COUCH_BITFIELD_H 2 | #define COUCH_BITFIELD_H 3 | 4 | #include "config.h" 5 | #include "internal.h" 6 | #include 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | /* 13 | * Variable-width types. Since these are made out of chars they will be 14 | * byte-aligned, so structs consisting only of these will be packed. 15 | */ 16 | 17 | typedef struct { 18 | uint8_t raw_bytes[1]; 19 | } raw_08; 20 | 21 | typedef struct { 22 | uint8_t raw_bytes[2]; 23 | } raw_16; 24 | 25 | typedef struct { 26 | uint8_t raw_bytes[3]; 27 | } raw_24; 28 | 29 | typedef struct { 30 | uint8_t raw_bytes[4]; 31 | } raw_32; 32 | 33 | typedef struct { 34 | uint8_t raw_bytes[5]; 35 | } raw_40; 36 | 37 | typedef struct { 38 | uint8_t raw_bytes[6]; 39 | } raw_48; 40 | 41 | typedef struct { 42 | uint8_t raw_bytes[8]; 43 | } raw_64; 44 | 45 | 46 | /* Functions for decoding raw_xx types to native integers: */ 47 | #define encode_raw08(a) couchstore_encode_raw08(a) 48 | #define encode_raw16(a) couchstore_encode_raw16(a) 49 | #define encode_raw24(a, b) couchstore_encode_raw24(a, b) 50 | #define encode_raw32(a) couchstore_encode_raw32(a) 51 | #define encode_raw40(a, b) couchstore_encode_raw40(a, b) 52 | #define encode_raw48(a, b) couchstore_encode_raw48(a, b) 53 | #define encode_raw64(a) couchstore_encode_raw64(a) 54 | 55 | #define decode_raw08(a) couchstore_decode_raw08(a) 56 | #define decode_raw16(a) couchstore_decode_raw16(a) 57 | #define decode_raw24(a) couchstore_decode_raw24p(&(a)) 58 | #define decode_raw32(a) couchstore_decode_raw32(a) 59 | #define decode_raw40(a) couchstore_decode_raw40p(&(a)) 60 | #define decode_raw48(a) couchstore_decode_raw48p(&(a)) 61 | #define decode_raw64(a) couchstore_decode_raw64(a) 62 | 63 | LIBCOUCHSTORE_API 64 | uint8_t couchstore_decode_raw08(raw_08 raw); 65 | LIBCOUCHSTORE_API 66 | uint16_t couchstore_decode_raw16(raw_16 raw); 67 | LIBCOUCHSTORE_API 68 | uint32_t couchstore_decode_raw24p(const raw_24 *raw); 69 | LIBCOUCHSTORE_API 70 | uint32_t couchstore_decode_raw32(raw_32 raw); 71 | LIBCOUCHSTORE_API 72 | uint64_t couchstore_decode_raw40p(const raw_40 *raw); 73 | LIBCOUCHSTORE_API 74 | uint64_t couchstore_decode_raw48p(const raw_48 *raw); 75 | LIBCOUCHSTORE_API 76 | uint64_t couchstore_decode_raw64(raw_64 raw); 77 | 78 | /* Functions for encoding native integers to raw_xx types: */ 79 | 80 | LIBCOUCHSTORE_API 81 | raw_08 couchstore_encode_raw08(uint8_t value); 82 | LIBCOUCHSTORE_API 83 | raw_16 couchstore_encode_raw16(uint16_t value); 84 | LIBCOUCHSTORE_API 85 | void couchstore_encode_raw24(uint32_t value, raw_24 *raw); 86 | LIBCOUCHSTORE_API 87 | raw_32 couchstore_encode_raw32(uint32_t value); 88 | LIBCOUCHSTORE_API 89 | void couchstore_encode_raw40(uint64_t value, raw_40 *raw); 90 | LIBCOUCHSTORE_API 91 | void couchstore_encode_raw48(uint64_t value, raw_48 *raw); 92 | LIBCOUCHSTORE_API 93 | raw_64 couchstore_encode_raw64(uint64_t value); 94 | 95 | #ifdef __cplusplus 96 | } 97 | #endif 98 | 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /src/compactor.cc: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "bitfield.h" 7 | 8 | static void exit_error(couchstore_error_t errcode) 9 | { 10 | fprintf(stderr, "Couchstore error: %s\n", couchstore_strerror(errcode)); 11 | exit(-1); 12 | } 13 | 14 | static void usage(const char* prog) { 15 | fprintf(stderr, "Usage: %s [--purge-before ] [--purge-only-upto-seq seq] [--dropdeletes] [--upgrade] \n", prog); 16 | exit(-1); 17 | } 18 | 19 | typedef struct { 20 | raw_64 cas; 21 | raw_32 expiry; 22 | raw_32 flags; 23 | } CouchbaseRevMeta; 24 | 25 | static int time_purge_hook(Db* target, DocInfo* info, sized_buf, void* ctx_p) { 26 | time_purge_ctx* ctx = (time_purge_ctx*) ctx_p; 27 | 28 | if (info == nullptr) { 29 | /* Compaction finished */ 30 | target->header.purge_seq = ctx->max_purged_seq; 31 | return COUCHSTORE_SUCCESS; 32 | } 33 | 34 | if(info->deleted && info->rev_meta.size >= 16) { 35 | const CouchbaseRevMeta* meta = (const CouchbaseRevMeta*)info->rev_meta.buf; 36 | uint32_t exptime = decode_raw32(meta->expiry); 37 | if(exptime < ctx->purge_before_ts 38 | && (!ctx->purge_before_seq || info->db_seq <= ctx->purge_before_seq)) { 39 | if(ctx->max_purged_seq < info->db_seq) { 40 | ctx->max_purged_seq = info->db_seq; 41 | } 42 | return COUCHSTORE_COMPACT_DROP_ITEM; 43 | } 44 | } 45 | 46 | return COUCHSTORE_COMPACT_KEEP_ITEM; 47 | } 48 | 49 | int main(int argc, char** argv) 50 | { 51 | Db* source; 52 | couchstore_error_t errcode; 53 | time_purge_ctx timepurge = {0, 0}; 54 | couchstore_compact_hook hook = NULL; 55 | couchstore_docinfo_hook dhook = NULL; 56 | void* hook_ctx = NULL; 57 | int argp = 1; 58 | couchstore_compact_flags flags = 0; 59 | FileOpsInterface* target_io_ops = couchstore_get_default_file_ops(); 60 | if(argc < 3) 61 | { 62 | usage(argv[0]); 63 | } 64 | 65 | while(argv[argp][0] == '-') { 66 | if(!strcmp(argv[argp],"--purge-before")) { 67 | argp+=2; 68 | if(argc + argp < 2) { 69 | usage(argv[0]); 70 | } 71 | hook = time_purge_hook; 72 | hook_ctx = &timepurge; 73 | timepurge.purge_before_ts = atoi(argv[argp-1]); 74 | printf("Purging items before timestamp %" PRIu64 "\n", 75 | timepurge.purge_before_ts); 76 | } 77 | 78 | if(!strcmp(argv[argp],"--purge-only-upto-seq")) { 79 | if(argc - argp < 3) { 80 | usage(argv[0]); 81 | } 82 | argp+=2; 83 | timepurge.purge_before_seq = (uint64_t)(atoll(argv[argp-1])); 84 | printf("Purging items only up-to seq %" PRIu64 "\n", 85 | timepurge.purge_before_seq); 86 | } 87 | 88 | if(!strcmp(argv[argp],"--dropdeletes")) { 89 | argp++; 90 | if(argc + argp < 2) { 91 | usage(argv[0]); 92 | } 93 | flags |= COUCHSTORE_COMPACT_FLAG_DROP_DELETES; 94 | } 95 | 96 | if(!strcmp(argv[argp], "--upgrade")) { 97 | argp++; 98 | if(argc + argp < 2) { 99 | usage(argv[0]); 100 | } 101 | flags |= COUCHSTORE_COMPACT_FLAG_UPGRADE_DB; 102 | } 103 | } 104 | 105 | errcode = couchstore_open_db(argv[argp++], COUCHSTORE_OPEN_FLAG_RDONLY, &source); 106 | if(errcode) 107 | { 108 | exit_error(errcode); 109 | } 110 | errcode = couchstore_compact_db_ex(source, argv[argp], flags, hook, dhook, hook_ctx, target_io_ops); 111 | if(errcode) 112 | { 113 | exit_error(errcode); 114 | } 115 | 116 | printf("Compacted %s -> %s\n", argv[argp - 1], argv[argp]); 117 | 118 | errcode = couchstore_close_file(source); 119 | couchstore_free_db(source); 120 | if(errcode) 121 | { 122 | exit_error(errcode); 123 | } 124 | 125 | return 0; 126 | } 127 | -------------------------------------------------------------------------------- /src/couch_file_write.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "internal.h" 13 | #include "crc32.h" 14 | #include "util.h" 15 | 16 | static ssize_t raw_write(tree_file *file, const sized_buf *buf, cs_off_t pos) 17 | { 18 | cs_off_t write_pos = pos; 19 | size_t buf_pos = 0; 20 | char blockprefix = 0; 21 | ssize_t written; 22 | size_t block_remain; 23 | while (buf_pos < buf->size) { 24 | block_remain = COUCH_BLOCK_SIZE - (write_pos % COUCH_BLOCK_SIZE); 25 | if (block_remain > (buf->size - buf_pos)) { 26 | block_remain = buf->size - buf_pos; 27 | } 28 | 29 | if (write_pos % COUCH_BLOCK_SIZE == 0) { 30 | written = file->ops->pwrite(&file->lastError, file->handle, 31 | &blockprefix, 1, write_pos); 32 | if (written < 0) { 33 | return written; 34 | } 35 | write_pos += 1; 36 | continue; 37 | } 38 | 39 | written = file->ops->pwrite(&file->lastError, file->handle, 40 | buf->buf + buf_pos, block_remain, write_pos); 41 | if (written < 0) { 42 | return written; 43 | } 44 | buf_pos += written; 45 | write_pos += written; 46 | } 47 | 48 | return (ssize_t)(write_pos - pos); 49 | } 50 | 51 | couchstore_error_t write_header(tree_file *file, sized_buf *buf, cs_off_t *pos) 52 | { 53 | cs_off_t write_pos = align_to_next_block(file->pos); 54 | ssize_t written; 55 | uint32_t size = htonl(buf->size + 4); //Len before header includes hash len. 56 | uint32_t crc32 = htonl(get_checksum(reinterpret_cast(buf->buf), 57 | buf->size, 58 | file->crc_mode)); 59 | char headerbuf[1 + 4 + 4]; 60 | 61 | *pos = write_pos; 62 | 63 | // Write the header's block header 64 | headerbuf[0] = 1; 65 | memcpy(&headerbuf[1], &size, 4); 66 | memcpy(&headerbuf[5], &crc32, 4); 67 | 68 | written = file->ops->pwrite(&file->lastError, file->handle, 69 | &headerbuf, sizeof(headerbuf), write_pos); 70 | if (written < 0) { 71 | return (couchstore_error_t)written; 72 | } 73 | write_pos += written; 74 | 75 | //Write actual header 76 | written = raw_write(file, buf, write_pos); 77 | if (written < 0) { 78 | return (couchstore_error_t)written; 79 | } 80 | write_pos += written; 81 | file->pos = write_pos; 82 | 83 | return COUCHSTORE_SUCCESS; 84 | } 85 | 86 | int db_write_buf(tree_file *file, const sized_buf *buf, cs_off_t *pos, size_t *disk_size) 87 | { 88 | cs_off_t write_pos = file->pos; 89 | cs_off_t end_pos = write_pos; 90 | ssize_t written; 91 | uint32_t size = htonl(buf->size | 0x80000000); 92 | uint32_t crc32 = htonl(get_checksum(reinterpret_cast(buf->buf), 93 | buf->size, 94 | file->crc_mode)); 95 | char headerbuf[4 + 4]; 96 | 97 | // Write the buffer's header: 98 | memcpy(&headerbuf[0], &size, 4); 99 | memcpy(&headerbuf[4], &crc32, 4); 100 | 101 | sized_buf sized_headerbuf = { headerbuf, 8 }; 102 | written = raw_write(file, &sized_headerbuf, end_pos); 103 | if (written < 0) { 104 | return (int)written; 105 | } 106 | end_pos += written; 107 | 108 | // Write actual buffer: 109 | written = raw_write(file, buf, end_pos); 110 | if (written < 0) { 111 | return (int)written; 112 | } 113 | end_pos += written; 114 | 115 | if (pos) { 116 | *pos = write_pos; 117 | } 118 | 119 | file->pos = end_pos; 120 | if (disk_size) { 121 | *disk_size = (size_t) (end_pos - write_pos); 122 | } 123 | 124 | return 0; 125 | } 126 | 127 | couchstore_error_t db_write_buf_compressed(tree_file *file, 128 | const sized_buf *buf, 129 | cs_off_t *pos, 130 | size_t *disk_size) 131 | { 132 | cb::compression::Buffer buffer; 133 | try { 134 | using cb::compression::Algorithm; 135 | if (!cb::compression::deflate(Algorithm::Snappy, 136 | {buf->buf, buf->size}, 137 | buffer)) { 138 | return COUCHSTORE_ERROR_CORRUPT; 139 | } 140 | } catch (const std::bad_alloc&) { 141 | return COUCHSTORE_ERROR_ALLOC_FAIL; 142 | } 143 | 144 | sized_buf to_write{}; 145 | to_write.buf = buffer.data(); 146 | to_write.size = buffer.size(); 147 | 148 | return static_cast(db_write_buf(file, &to_write, pos, disk_size)); 149 | } 150 | -------------------------------------------------------------------------------- /src/couch_latency.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "couch_latency_internal.h" 19 | 20 | SingletonWrapper > 21 | CouchLatency::instance(Couchbase::RelaxedAtomic(nullptr)); 22 | 23 | CouchLatency* CouchLatency::init() { 24 | CouchLatency* tmp = instance.object.load(); 25 | if (tmp == nullptr) { 26 | // Ensure two threads don't both create an instance. 27 | std::lock_guard lock(instance.mutex); 28 | tmp = instance.object.load(); 29 | if (tmp == nullptr) { 30 | tmp = new CouchLatency(); 31 | instance.object.store(tmp); 32 | } 33 | } 34 | return tmp; 35 | } 36 | 37 | CouchLatency* CouchLatency::getInstance() { 38 | // Instance should be initialized explicitly. 39 | return instance.object.load(); 40 | } 41 | 42 | void CouchLatency::destroyInstance() { 43 | std::lock_guard lock(instance.mutex); 44 | CouchLatency* tmp = instance.object.load(); 45 | if (tmp != nullptr) { 46 | delete tmp; 47 | instance.object = nullptr; 48 | } 49 | } 50 | 51 | CouchLatency::~CouchLatency() { 52 | // Deregister all items, and reset their Histograms. 53 | // As CouchLatencyItem itself is a static unique_ptr 54 | // that is owned by each API functions, 55 | // it doesn't need to be destroyed here. 56 | for (auto& itr : itemsMap.object) { 57 | itr.second->changeRegisterStatus(false); 58 | itr.second->resetHistogram(); 59 | } 60 | } 61 | 62 | CouchLatencyItem* CouchLatency::addItem(CouchLatencyItem* item) { 63 | // This lock will be grabbed during the registration phase only. 64 | // Once an item is registered, they will run in a lock-free manner. 65 | std::lock_guard l(itemsMap.mutex); 66 | auto itr = itemsMap.object.find(item->statName); 67 | if (itr != itemsMap.object.end()) { 68 | // Already registered by other thread. 69 | // Return it. 70 | return itr->second; 71 | } 72 | itemsMap.object.insert( std::make_pair(item->statName, item) ); 73 | item->changeRegisterStatus(true); 74 | return item; 75 | } 76 | 77 | void CouchLatency::getLatencyInfo(couchstore_latency_callback_fn cb_func, 78 | couchstore_latency_dump_options options, 79 | void *ctx) { 80 | (void)options; 81 | for (auto& itr : itemsMap.object) { 82 | CouchLatencyItem *item = itr.second; 83 | if (!item->latencySum) { 84 | continue; 85 | } 86 | int ret = cb_func(item->statName.c_str(), 87 | &item->latencies, 88 | item->latencySum, 89 | ctx); 90 | if (ret) { 91 | // Abort by user. 92 | break; 93 | } 94 | } 95 | } 96 | 97 | LIBCOUCHSTORE_API 98 | void couchstore_latency_collector_start() { 99 | CouchLatency::init(); 100 | } 101 | 102 | LIBCOUCHSTORE_API 103 | void couchstore_get_latency_info(couchstore_latency_callback_fn callback, 104 | couchstore_latency_dump_options options, 105 | void *ctx) { 106 | CouchLatency* tmp = CouchLatency::getInstance(); 107 | if (!tmp) { 108 | // Latency collector is disabled. 109 | return; 110 | } 111 | tmp->getLatencyInfo(callback, options, ctx); 112 | } 113 | 114 | 115 | LIBCOUCHSTORE_API 116 | void couchstore_latency_collector_stop() { 117 | CouchLatency::destroyInstance(); 118 | } 119 | 120 | 121 | -------------------------------------------------------------------------------- /src/crc32.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Checksum abstraction functions. 20 | * 21 | * Couchstore evolved to support a software CRC and platform's CRC32-C. 22 | * This module provides an API for checking and creating checksums 23 | * utilising the correct method based upon the callers crc_mode. 24 | */ 25 | 26 | #pragma once 27 | 28 | #include 29 | 30 | typedef enum { 31 | CRC_UNKNOWN, 32 | CRC32, 33 | CRC32C 34 | } crc_mode_e; 35 | 36 | /* 37 | * Get a checksum of buf for buf_len bytes. 38 | * 39 | * mode = UNKNOWN is an invalid input (triggers assert). 40 | */ 41 | uint32_t get_checksum(const uint8_t* buf, 42 | size_t buf_len, 43 | crc_mode_e mode); 44 | 45 | /* 46 | * Perform an integrity check of buf for buf_len bytes. 47 | * 48 | * A checksum of buf is created and compared against checksum argument. 49 | * 50 | * mode = UNKNOWN is an acceptable input. All modes are tried before failing. 51 | * 52 | * Returns 1 for success, 0 for failure. 53 | */ 54 | int perform_integrity_check(const uint8_t* buf, 55 | size_t buf_len, 56 | uint32_t checksum, 57 | crc_mode_e mode); 58 | 59 | /* 60 | client_hash_crc32 returns a hash which is the same as a couchbase client 61 | would e.g. when determining the vbucket from a key, this function would 62 | match the client's mapping. 63 | 64 | This hashes the key into 'hash', then does ((~hash) >> 16) & 0x7fff; 65 | */ 66 | uint32_t client_hash_crc32(const uint8_t* key, size_t key_length); 67 | -------------------------------------------------------------------------------- /src/dbinfo.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "internal.h" 12 | #include "util.h" 13 | #include "bitfield.h" 14 | 15 | 16 | static char *size_str(double size) 17 | { 18 | static char rfs[256]; 19 | int i = 0; 20 | const char *units[] = {"bytes", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}; 21 | while (size > 1024) { 22 | size /= 1024; 23 | i++; 24 | } 25 | snprintf(rfs, sizeof(rfs), "%.*f %s", i, size, units[i]); 26 | return rfs; 27 | } 28 | 29 | static void print_db_info(Db* db) 30 | { 31 | DbInfo info; 32 | couchstore_db_info(db, &info); 33 | printf(" doc count: %" PRIu64 "\n", info.doc_count); 34 | printf(" deleted doc count: %" PRIu64 "\n", info.deleted_count); 35 | printf(" data size: %s\n", size_str(info.space_used)); 36 | } 37 | 38 | static int process_file(const char *file, int iterate_headers) 39 | { 40 | Db *db; 41 | couchstore_error_t errcode; 42 | uint64_t btreesize = 0; 43 | const char* crc_strings[3] = {"warning crc is set to unknown", 44 | "CRC-32", 45 | "CRC-32C"}; 46 | 47 | errcode = couchstore_open_db(file, COUCHSTORE_OPEN_FLAG_RDONLY, &db); 48 | if (errcode != COUCHSTORE_SUCCESS) { 49 | fprintf(stderr, "Failed to open \"%s\": %s\n", 50 | file, couchstore_strerror(errcode)); 51 | return -1; 52 | } 53 | 54 | next_header: 55 | printf("DB Info (%s) - header at %" PRIu64 "\n", file, db->header.position); 56 | printf(" file format version: %" PRIu64 "\n", db->header.disk_version); 57 | printf(" update_seq: %" PRIu64 "\n", db->header.update_seq); 58 | printf(" purge_seq: %" PRIu64 "\n", db->header.purge_seq); 59 | 60 | if (db->file.crc_mode < 3) { 61 | printf(" crc: %s\n", crc_strings[db->file.crc_mode]); 62 | } else { 63 | printf(" crc: warning crc_mode is out of range %" PRIu32 "\n", db->file.crc_mode); 64 | } 65 | 66 | print_db_info(db); 67 | if (db->header.by_id_root) { 68 | btreesize += db->header.by_id_root->subtreesize; 69 | } 70 | if (db->header.by_seq_root) { 71 | btreesize += db->header.by_seq_root->subtreesize; 72 | } 73 | printf(" B-tree size: %s\n", size_str(btreesize)); 74 | printf(" total disk size: %s\n", size_str(db->file.pos)); 75 | if (iterate_headers) { 76 | if (couchstore_rewind_db_header(db) == COUCHSTORE_SUCCESS) { 77 | printf("\n"); 78 | goto next_header; 79 | } 80 | } else { 81 | couchstore_close_file(db); 82 | couchstore_free_db(db); 83 | } 84 | 85 | return 0; 86 | } 87 | 88 | static void usage(const char *name) 89 | { 90 | fprintf(stderr, "USAGE: %s [-i] \n", name); 91 | fprintf(stderr, "\t-i Iterate through all headers\n"); 92 | exit(EXIT_FAILURE); 93 | } 94 | 95 | int main(int argc, char **argv) 96 | { 97 | int error = 0; 98 | int ii; 99 | int iterate_headers = getenv("ITERATE_HEADERS") != NULL; 100 | int cmd; 101 | 102 | while ((cmd = getopt(argc, argv, "i")) != -1) { 103 | switch (cmd) { 104 | case 'i': 105 | iterate_headers = 1; 106 | break; 107 | default: 108 | usage(argv[0]); 109 | /* NOTREACHED */ 110 | } 111 | } 112 | 113 | if (optind == argc) { 114 | usage(argv[0]); 115 | } 116 | 117 | for (ii = optind; ii < argc; ++ii) { 118 | error += process_file(argv[ii], iterate_headers); 119 | } 120 | 121 | if (error) { 122 | exit(EXIT_FAILURE); 123 | } else { 124 | exit(EXIT_SUCCESS); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/fatbuf.h: -------------------------------------------------------------------------------- 1 | #ifndef COUCHSTORE_FATBUF_H 2 | #define COUCHSTORE_FATBUF_H 3 | 4 | typedef struct _fat_buffer { 5 | size_t pos; 6 | size_t size; 7 | char buf[1]; 8 | } fatbuf; 9 | #ifdef __cplusplus 10 | extern "C" { 11 | #endif 12 | fatbuf *fatbuf_alloc(size_t bytes); 13 | void *fatbuf_get(fatbuf *fb, size_t bytes); 14 | void fatbuf_free(fatbuf *fb); 15 | #ifdef __cplusplus 16 | } 17 | #endif 18 | #endif 19 | -------------------------------------------------------------------------------- /src/file_merger.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _FILE_MERGER_H 22 | #define _FILE_MERGER_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | 33 | typedef enum { 34 | FILE_MERGER_SUCCESS = 0, 35 | FILE_MERGER_ERROR_OPEN_FILE = -1, 36 | FILE_MERGER_ERROR_FILE_READ = -2, 37 | FILE_MERGER_ERROR_FILE_WRITE = -3, 38 | FILE_MERGER_ERROR_BAD_ARG = -4, 39 | FILE_MERGER_ERROR_ALLOC = -5, 40 | FILE_MERGER_SORT_ERROR = -6 41 | } file_merger_error_t; 42 | 43 | 44 | typedef struct { 45 | void *record; 46 | unsigned filenum; 47 | } file_merger_record_t; 48 | 49 | /* 50 | * Returns the length of the record read from the file on success. 51 | * Returns 0 when the file's EOF is reached, and a negative value 52 | * on failure (a file_merger_error_t value). 53 | */ 54 | typedef int (*file_merger_read_record_t)(FILE *f, 55 | void **record_buffer, 56 | void *user_ctx); 57 | 58 | /* 59 | * Returns FILE_MERGER_SUCCESS on success, another file_merger_error_t 60 | * value otherwise. 61 | */ 62 | typedef file_merger_error_t (*file_merger_write_record_t)(FILE *f, 63 | void *record_buffer, 64 | void *user_ctx); 65 | 66 | /* 67 | * Returns 0 if both records compare equal, a negative value if the first record 68 | * is smaller than the second record, a positive value if the first record is 69 | * greater than the second record. 70 | */ 71 | typedef int (*file_merger_compare_records_t)(const void *record_buffer1, 72 | const void *record_buffer2, 73 | void *user_ctx); 74 | 75 | typedef void (*file_merger_record_free_t)(void *record, 76 | void *user_ctx); 77 | 78 | /* Callback which gets called for resulting merge records */ 79 | typedef file_merger_error_t (*file_merger_feed_record_t)(void *record_buffer, 80 | void *user_ctx); 81 | 82 | /* Among the provided records, return the winner record index */ 83 | typedef size_t (*file_merger_deduplicate_records_t)( 84 | file_merger_record_t **records, 85 | size_t len, 86 | void *user_ctx); 87 | 88 | file_merger_error_t merge_files(const char *source_files[], 89 | unsigned num_files, 90 | const char *dest_file, 91 | file_merger_read_record_t read_record, 92 | file_merger_write_record_t write_record, 93 | file_merger_feed_record_t feed_record, 94 | file_merger_compare_records_t compare_records, 95 | file_merger_deduplicate_records_t dedup_records, 96 | file_merger_record_free_t free_record, 97 | int skip_writeback, 98 | void *user_ctx); 99 | 100 | 101 | #ifdef __cplusplus 102 | } 103 | #endif 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /src/file_name_utils.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #if defined(WIN32) || defined(_WIN32) 22 | # define WINDOWS 23 | # include 24 | #else 25 | # ifndef _BSD_SOURCE 26 | /* for mkstemp() */ 27 | # define _BSD_SOURCE 28 | # endif 29 | # include 30 | # include 31 | #endif 32 | 33 | #include "file_name_utils.h" 34 | 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #define TMP_FILE_SUFFIX ".XXXXXX" 41 | 42 | 43 | char *tmp_file_path(const char *tmp_dir, const char *prefix) 44 | { 45 | char *file_path; 46 | size_t tmp_dir_len, prefix_len, total_len; 47 | #ifdef WINDOWS 48 | errno_t err; 49 | #else 50 | int fd; 51 | #endif 52 | 53 | 54 | tmp_dir_len = strlen(tmp_dir); 55 | prefix_len = strlen(prefix); 56 | total_len = tmp_dir_len + 1 + prefix_len + sizeof(TMP_FILE_SUFFIX); 57 | file_path = (char *) cb_malloc(total_len); 58 | 59 | if (file_path == NULL) { 60 | return NULL; 61 | } 62 | 63 | memcpy(file_path, tmp_dir, tmp_dir_len); 64 | /* Windows specific file API functions and stdio file functions on Windows 65 | * convert forward slashes to back slashes. */ 66 | file_path[tmp_dir_len] = '/'; 67 | memcpy(file_path + tmp_dir_len + 1, prefix, prefix_len); 68 | memcpy(file_path + tmp_dir_len + 1 + prefix_len, 69 | TMP_FILE_SUFFIX, 70 | sizeof(TMP_FILE_SUFFIX)); 71 | 72 | #ifdef WINDOWS 73 | err = _mktemp_s(file_path, total_len); 74 | if (err != 0) { 75 | cb_free(file_path); 76 | return NULL; 77 | } 78 | #else 79 | fd = mkstemp(file_path); 80 | if (fd == -1) { 81 | cb_free(file_path); 82 | return NULL; 83 | } 84 | close(fd); 85 | remove(file_path); 86 | #endif 87 | 88 | return file_path; 89 | } 90 | 91 | 92 | char *file_basename(const char *path) 93 | { 94 | char *ret; 95 | #ifdef WINDOWS 96 | char drive[_MAX_DRIVE]; 97 | char dir[_MAX_DIR]; 98 | char fname[_MAX_FNAME]; 99 | char ext[_MAX_EXT]; 100 | 101 | _splitpath(path, drive, dir, fname, ext); 102 | #else 103 | char *fname; 104 | 105 | fname = basename((char *) path); 106 | #endif 107 | 108 | ret = (char *) cb_malloc(strlen(fname) + 1); 109 | if (ret != NULL) { 110 | strcpy(ret, fname); 111 | } 112 | 113 | return ret; 114 | } 115 | -------------------------------------------------------------------------------- /src/file_name_utils.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _FILE_NAME_UTILS_H 22 | #define _FILE_NAME_UTILS_H 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | char *file_basename(const char *path); 29 | 30 | char *tmp_file_path(const char *tmp_dir, const char *prefix); 31 | 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/file_sorter.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _FILE_SORTER_H 22 | #define _FILE_SORTER_H 23 | 24 | #include "config.h" 25 | #include 26 | #include "file_merger.h" 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | 33 | typedef enum { 34 | FILE_SORTER_SUCCESS = FILE_MERGER_SUCCESS, 35 | FILE_SORTER_ERROR_OPEN_FILE = FILE_MERGER_ERROR_OPEN_FILE, 36 | FILE_SORTER_ERROR_FILE_READ = FILE_MERGER_ERROR_FILE_READ, 37 | FILE_SORTER_ERROR_FILE_WRITE = FILE_MERGER_ERROR_FILE_WRITE, 38 | FILE_SORTER_ERROR_BAD_ARG = FILE_MERGER_ERROR_BAD_ARG, 39 | FILE_SORTER_ERROR_ALLOC = FILE_MERGER_ERROR_ALLOC, 40 | FILE_SORTER_ERROR_RENAME_FILE = -10, 41 | FILE_SORTER_ERROR_DELETE_FILE = -11, 42 | FILE_SORTER_ERROR_MK_TMP_FILE = -12, 43 | FILE_SORTER_ERROR_NOT_EMPTY_TMP_FILE = -13, 44 | FILE_SORTER_ERROR_TMP_FILE_BASENAME = -14, 45 | FILE_SORTER_ERROR_MISSING_CALLBACK = -15 46 | } file_sorter_error_t; 47 | 48 | 49 | file_sorter_error_t sort_file(const char *source_file, 50 | const char *tmp_dir, 51 | unsigned num_tmp_files, 52 | unsigned max_buffer_size, 53 | file_merger_read_record_t read_record, 54 | file_merger_write_record_t write_record, 55 | file_merger_feed_record_t feed_record, 56 | file_merger_compare_records_t compare_records, 57 | file_merger_record_free_t free_record, 58 | int skip_writeback, 59 | void *user_ctx); 60 | 61 | #ifdef __cplusplus 62 | } 63 | #endif 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /src/iobuffer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef LIBCOUCHSTORE_IOBUFFER_H 19 | #define LIBCOUCHSTORE_IOBUFFER_H 1 20 | 21 | #include 22 | 23 | /* Buffered IO parameters */ 24 | struct buffered_file_ops_params { 25 | buffered_file_ops_params(); 26 | buffered_file_ops_params(const buffered_file_ops_params& src); 27 | buffered_file_ops_params(const bool _read_only, 28 | const uint32_t _read_buffer_capacity, 29 | const uint32_t _max_read_buffers); 30 | 31 | // Flag indicating whether or not the file is being opened as read only. 32 | bool readOnly; 33 | // Read buffer capacity. 34 | uint32_t read_buffer_capacity; 35 | // Max read buffer count. 36 | uint32_t max_read_buffers; 37 | }; 38 | 39 | /** 40 | * Constructs a set of file ops that buffer the I/O provided by an 41 | * underlying set of raw ops. 42 | * 43 | * @param raw_ops the file ops callbacks to use for the underlying I/O 44 | * @param handle on output, a constructed (but not opened) couch_file_handle 45 | * @param params buffered IO parameters 46 | * @return the couch_file_ops to use, or NULL on failure 47 | */ 48 | FileOpsInterface* couch_get_buffered_file_ops(couchstore_error_info_t *errinfo, 49 | FileOpsInterface* raw_ops, 50 | couch_file_handle* handle, 51 | buffered_file_ops_params params); 52 | 53 | class BufferedFileOps : public FileOpsInterface { 54 | public: 55 | BufferedFileOps() {} 56 | 57 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override ; 58 | couchstore_error_t open(couchstore_error_info_t* errinfo, 59 | couch_file_handle* handle, const char* path, 60 | int oflag) override; 61 | couchstore_error_t close(couchstore_error_info_t* errinfo, 62 | couch_file_handle handle) override; 63 | couchstore_error_t set_periodic_sync(couch_file_handle handle, 64 | uint64_t period_bytes) override; 65 | ssize_t pread(couchstore_error_info_t* errinfo, 66 | couch_file_handle handle, void* buf, size_t nbytes, 67 | cs_off_t offset) override; 68 | ssize_t pwrite(couchstore_error_info_t* errinfo, 69 | couch_file_handle handle, const void* buf, 70 | size_t nbytes, cs_off_t offset) override; 71 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 72 | couch_file_handle handle) override; 73 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 74 | couch_file_handle handle) override; 75 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 76 | couch_file_handle handle, cs_off_t offset, 77 | cs_off_t len, 78 | couchstore_file_advice_t advice) override; 79 | FHStats* get_stats(couch_file_handle handle) override; 80 | 81 | void destructor(couch_file_handle handle) override; 82 | 83 | couch_file_handle constructor(couchstore_error_info_t *errinfo, 84 | FileOpsInterface* raw_ops, 85 | buffered_file_ops_params params); 86 | }; 87 | 88 | #endif // LIBCOUCHSTORE_IOBUFFER_H 89 | -------------------------------------------------------------------------------- /src/llmsort.cc: -------------------------------------------------------------------------------- 1 | /* A Linked-List Memory Sort 2 | by Philip J. Erdelsky 3 | pje@efgh.com 4 | http://www.efgh.com/software/llmsort.htm 5 | */ 6 | 7 | #include "config.h" 8 | #include "mergesort.h" 9 | #include 10 | 11 | void *sort_linked_list(void *p, unsigned idx, 12 | int (*compare)(void *, void *, void *), void *pointer, unsigned long *pcount) 13 | { 14 | unsigned base; 15 | unsigned long block_size; 16 | 17 | struct record { 18 | struct record *next[1]; 19 | /* other members not directly accessed by this function */ 20 | }; 21 | 22 | struct tape { 23 | struct record *first, *last; 24 | unsigned long count; 25 | } tape[4]; 26 | 27 | /* Distribute the records alternately to tape[0] and tape[1]. */ 28 | 29 | memset(&tape, 0, sizeof(struct tape) * 4); 30 | base = 0; 31 | while (p != NULL) { 32 | struct record *next = ((struct record *)p)->next[idx]; 33 | ((struct record *)p)->next[idx] = tape[base].first; 34 | tape[base].first = ((struct record *)p); 35 | tape[base].count++; 36 | p = next; 37 | base ^= 1; 38 | } 39 | 40 | /* If the list is empty or contains only a single record, then */ 41 | /* tape[1].count == 0L and this part is vacuous. */ 42 | 43 | for (base = 0, block_size = 1L; tape[base + 1].count != 0L; 44 | base ^= 2, block_size <<= 1) { 45 | int dest; 46 | struct tape *tape0, *tape1; 47 | tape0 = tape + base; 48 | tape1 = tape + base + 1; 49 | dest = base ^ 2; 50 | tape[dest].count = tape[dest + 1].count = 0; 51 | for (; tape0->count != 0; dest ^= 1) { 52 | unsigned long n0, n1; 53 | struct tape *output_tape = tape + dest; 54 | n0 = n1 = block_size; 55 | while (1) { 56 | struct record *chosen_record; 57 | struct tape *chosen_tape; 58 | if (n0 == 0 || tape0->count == 0) { 59 | if (n1 == 0 || tape1->count == 0) { 60 | break; 61 | } 62 | chosen_tape = tape1; 63 | n1--; 64 | } else if (n1 == 0 || tape1->count == 0) { 65 | chosen_tape = tape0; 66 | n0--; 67 | } else if ((*compare)(tape0->first, tape1->first, pointer) > 0) { 68 | chosen_tape = tape1; 69 | n1--; 70 | } else { 71 | chosen_tape = tape0; 72 | n0--; 73 | } 74 | chosen_tape->count--; 75 | chosen_record = chosen_tape->first; 76 | chosen_tape->first = chosen_record->next[idx]; 77 | if (output_tape->count == 0) { 78 | output_tape->first = chosen_record; 79 | } else { 80 | output_tape->last->next[idx] = chosen_record; 81 | } 82 | output_tape->last = chosen_record; 83 | output_tape->count++; 84 | } 85 | } 86 | } 87 | 88 | if (tape[base].count > 1L) { 89 | tape[base].last->next[idx] = NULL; 90 | } 91 | if (pcount != NULL) { 92 | *pcount = tape[base].count; 93 | } 94 | return tape[base].first; 95 | } 96 | -------------------------------------------------------------------------------- /src/mergesort.h: -------------------------------------------------------------------------------- 1 | #ifndef MERGESORT_H 2 | #define MERGESORT_H 3 | 4 | #include 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | /* 11 | * Returns the length of the record read from the file on success. 12 | * Returns 0 when the file's EOF is reached, and a negative value 13 | * on failure. 14 | */ 15 | typedef int (*mergesort_read_record_t)(FILE *f, 16 | void *record_buffer, 17 | void *pointer); 18 | 19 | /* 20 | * Returns zero if the write failed. If the write succeeded, it returns a 21 | * value different from zero. 22 | */ 23 | typedef int (*mergesort_write_record_t)(FILE *f, 24 | void *record_buffer, 25 | void *pointer); 26 | 27 | /* 28 | * Returns 0 if both records compare equal, a negative value if the first record 29 | * is smaller than the second record, a positive value if the first record is 30 | * greater than the second record. 31 | */ 32 | typedef int (*mergesort_compare_records_t)(const void *record_buffer1, 33 | const void *record_buffer2, 34 | void *pointer); 35 | 36 | typedef char *(*mergesort_record_alloc_t)(void); 37 | typedef char *(*mergesort_record_duplicate_t)(char *record); 38 | typedef void (*mergesort_record_free_t)(char *record); 39 | 40 | void *sort_linked_list(void *, unsigned, int (*)(void *, void *, void *), void *, unsigned long *); 41 | 42 | FILE *openTmpFile(char *path); 43 | 44 | int merge_sort(FILE *unsorted_file, FILE *sorted_file, 45 | char *tmp_path, 46 | mergesort_read_record_t read, 47 | mergesort_write_record_t write, 48 | mergesort_compare_records_t compare, 49 | mergesort_record_alloc_t record_alloc, 50 | mergesort_record_duplicate_t record_duplicate, 51 | mergesort_record_free_t record_free, 52 | void *pointer, 53 | unsigned long block_size, 54 | unsigned long *pcount); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/node_types.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | // 3 | // node_types.c 4 | // couchstore 5 | // 6 | // Created by Jens Alfke on 4/25/12. 7 | // Copyright (c) 2012 Couchbase, Inc. All rights reserved. 8 | // 9 | 10 | #include "node_types.h" 11 | 12 | #include 13 | #include 14 | 15 | size_t read_kv(const void *buf, sized_buf *key, sized_buf *value) 16 | { 17 | const raw_kv_length* kvlen = static_cast(buf); 18 | uint32_t klen, vlen; 19 | decode_kv_length(kvlen, &klen, &vlen); 20 | key->size = klen; 21 | key->buf = (char*)(kvlen + 1); 22 | value->size = vlen; 23 | value->buf = key->buf + klen; 24 | return sizeof(raw_kv_length) + klen + vlen; 25 | } 26 | 27 | void* write_kv(void *buf, sized_buf key, sized_buf value) 28 | { 29 | uint8_t *dst = static_cast(buf); 30 | *(raw_kv_length*)dst = encode_kv_length((uint32_t)key.size, (uint32_t)value.size); 31 | dst += sizeof(raw_kv_length); 32 | memcpy(dst, key.buf, key.size); 33 | dst += key.size; 34 | memcpy(dst, value.buf, value.size); 35 | dst += value.size; 36 | return dst; 37 | } 38 | 39 | node_pointer *read_root(void *buf, int size) 40 | { 41 | if (size == 0) { 42 | return NULL; 43 | } 44 | 45 | raw_btree_root *raw = (raw_btree_root*)buf; 46 | node_pointer *ptr; 47 | uint64_t position = decode_raw48(raw->pointer); 48 | uint64_t subtreesize = decode_raw48(raw->subtreesize); 49 | int redsize = size - sizeof(*raw); 50 | 51 | ptr = (node_pointer *) cb_malloc(sizeof(node_pointer) + redsize); 52 | if (redsize > 0) { 53 | buf = (char *) memcpy(ptr + 1, raw + 1, redsize); 54 | } else { 55 | buf = NULL; 56 | } 57 | ptr->key.buf = NULL; 58 | ptr->key.size = 0; 59 | ptr->pointer = position; 60 | ptr->subtreesize = subtreesize; 61 | ptr->reduce_value.buf = static_cast(buf); 62 | ptr->reduce_value.size = redsize; 63 | return ptr; 64 | } 65 | 66 | size_t encode_root(void *buf, node_pointer *node) 67 | { 68 | if (!node) { 69 | return 0; 70 | } 71 | if (buf) { 72 | raw_btree_root *root = static_cast(buf); 73 | encode_raw48(node->pointer, &root->pointer); 74 | encode_raw48(node->subtreesize, &root->subtreesize); 75 | memcpy(root + 1, node->reduce_value.buf, node->reduce_value.size); 76 | } 77 | return sizeof(raw_btree_root) + node->reduce_value.size; 78 | } 79 | 80 | void decode_kv_length(const raw_kv_length *kv, uint32_t *klen, uint32_t *vlen) 81 | { 82 | /* 12, 28 bit */ 83 | *klen = (uint16_t) ((kv->raw_kv[0] << 4) | ((kv->raw_kv[1] & 0xf0) >> 4)); 84 | memcpy(vlen, &kv->raw_kv[1], 4); 85 | *vlen = ntohl(*vlen) & 0x0FFFFFFF; 86 | } 87 | 88 | raw_kv_length encode_kv_length(size_t klen, size_t vlen) 89 | { 90 | raw_kv_length kv; 91 | uint32_t len = htonl((uint32_t)vlen); 92 | memcpy(&kv.raw_kv[1], &len, 4); 93 | kv.raw_kv[0] = (uint8_t)(klen >> 4); /* upper 8 bits of klen */ 94 | kv.raw_kv[1] |= (klen & 0xF) << 4; /* lower 4 bits of klen in upper half of byte */ 95 | return kv; 96 | } 97 | 98 | uint64_t decode_sequence_key(const sized_buf *buf) 99 | { 100 | const raw_by_seq_key *key = (const raw_by_seq_key*)buf->buf; 101 | return decode_raw48(key->sequence); 102 | } 103 | -------------------------------------------------------------------------------- /src/node_types.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** node_types.h 4 | ** couchstore 5 | ** 6 | ** Created by Jens Alfke on 4/25/12. 7 | ** Modified by Filipe Manana on 6/19/13 to fix some GCC warnings regarding 8 | ** violation of strict aliasing rules. 9 | ** 10 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 11 | ** 12 | */ 13 | 14 | #ifndef COUCHSTORE_NODE_TYPES_H 15 | #define COUCHSTORE_NODE_TYPES_H 16 | 17 | #include "bitfield.h" 18 | #include "internal.h" 19 | 20 | #ifdef __cplusplus 21 | extern "C" { 22 | #endif 23 | 24 | typedef struct { 25 | raw_08 version; 26 | raw_48 update_seq; 27 | raw_48 purge_seq; 28 | raw_48 purge_ptr; 29 | raw_16 seqrootsize; 30 | raw_16 idrootsize; 31 | raw_16 localrootsize; 32 | /* Three variable-size raw_btree_root structures follow */ 33 | } raw_file_header; 34 | 35 | typedef struct { 36 | raw_48 pointer; 37 | raw_48 subtreesize; 38 | /* Variable-size reduce value follows */ 39 | } raw_btree_root; 40 | 41 | /** Packed key-and-value length type. Key length is 12 bits, value length is 28. */ 42 | typedef struct { 43 | uint8_t raw_kv[5]; 44 | } raw_kv_length; 45 | 46 | typedef struct { 47 | raw_48 pointer; 48 | raw_48 subtreesize; 49 | raw_16 reduce_value_size; 50 | /* Variable-size reduce value follows */ 51 | } raw_node_pointer; 52 | 53 | typedef struct { 54 | raw_48 sequence; 55 | } raw_by_seq_key; 56 | 57 | typedef struct { 58 | raw_48 db_seq; 59 | raw_32 size; 60 | raw_48 bp; /* high bit is 'deleted' flag */ 61 | raw_48 rev_seq; 62 | raw_08 content_meta; 63 | /* Variable-size rev_meta data follows */ 64 | } raw_id_index_value; 65 | 66 | typedef struct { 67 | raw_kv_length sizes; 68 | raw_48 bp; /* high bit is 'deleted' flag */ 69 | raw_48 rev_seq; 70 | raw_08 content_meta; 71 | /* Variable-size id follows */ 72 | /* Variable-size rev_meta data follows */ 73 | } raw_seq_index_value; 74 | 75 | /* Mask for the 'deleted' bit in .bp fields */ 76 | #ifndef UINT64_C 77 | #define UINT64_C(x) (x ## ULL) 78 | #endif 79 | #define BP_DELETED_FLAG UINT64_C(0x800000000000) 80 | 81 | 82 | node_pointer *read_root(void *buf, int size); 83 | 84 | size_t encode_root(void *buf, node_pointer *node); 85 | 86 | 87 | /** 88 | * Reads a 12-bit key length and 28-bit value length, packed into 5 bytes big-endian. 89 | */ 90 | void decode_kv_length(const raw_kv_length *kv, uint32_t *klen, uint32_t *vlen); 91 | 92 | /** 93 | * Returns an encoded 5-byte key/value length pair. 94 | */ 95 | raw_kv_length encode_kv_length(size_t klen, size_t vlen); 96 | 97 | /** 98 | * Parses an in-memory buffer containing a 5-byte key/value length followed by key and value data, 99 | * and fills in sized_bufs to point to the key and data. 100 | * @return Number of bytes consumed from the buffer 101 | */ 102 | size_t read_kv(const void *buf, sized_buf *key, sized_buf *value); 103 | 104 | void* write_kv(void *buf, sized_buf key, sized_buf value); 105 | 106 | 107 | /** 108 | * Reads a 48-bit sequence number out of a sized_buf. 109 | */ 110 | uint64_t decode_sequence_key(const sized_buf *buf); 111 | 112 | #ifdef __cplusplus 113 | } 114 | #endif 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /src/os.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | class FileOps : public FileOpsInterface { 23 | public: 24 | FileOps() {} 25 | 26 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override ; 27 | couchstore_error_t open(couchstore_error_info_t* errinfo, 28 | couch_file_handle* handle, const char* path, 29 | int oflag) override; 30 | couchstore_error_t close(couchstore_error_info_t* errinfo, 31 | couch_file_handle handle) override; 32 | couchstore_error_t set_periodic_sync(couch_file_handle handle, 33 | uint64_t period_bytes) override; 34 | ssize_t pread(couchstore_error_info_t* errinfo, 35 | couch_file_handle handle, void* buf, size_t nbytes, 36 | cs_off_t offset) override; 37 | ssize_t pwrite(couchstore_error_info_t* errinfo, 38 | couch_file_handle handle, const void* buf, 39 | size_t nbytes, cs_off_t offset) override; 40 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 41 | couch_file_handle handle) override; 42 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 43 | couch_file_handle handle) override; 44 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 45 | couch_file_handle handle, cs_off_t offset, 46 | cs_off_t len, 47 | couchstore_file_advice_t advice) override; 48 | void destructor(couch_file_handle handle) override; 49 | }; 50 | -------------------------------------------------------------------------------- /src/quicksort.h: -------------------------------------------------------------------------------- 1 | #ifndef QUICKSORT_H 2 | #define QUICKSORT_H 3 | 4 | typedef int sort_cmp_t(const void *, const void *, void *); 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | void quicksort(void *a, size_t n, size_t es, sort_cmp_t *cmp, void *thunk); 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/reduces.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | #include 4 | #include 5 | #include "reduces.h" 6 | #include "node_types.h" 7 | 8 | 9 | couchstore_error_t by_seq_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx) 10 | { 11 | raw_by_seq_reduce *raw = (raw_by_seq_reduce*)dst; 12 | 13 | (void) leaflist; 14 | (void) ctx; 15 | encode_raw40(count, &raw->count); 16 | *size_r = sizeof(*raw); 17 | 18 | return COUCHSTORE_SUCCESS; 19 | } 20 | 21 | couchstore_error_t by_seq_rereduce(char *dst, size_t *size_r, const nodelist *ptrlist, int count, void *ctx) 22 | { 23 | uint64_t total = 0; 24 | const nodelist *i = ptrlist; 25 | 26 | (void) ctx; 27 | 28 | while (i != NULL && count > 0) { 29 | const raw_by_seq_reduce *reduce = (const raw_by_seq_reduce*) i->pointer->reduce_value.buf; 30 | total += decode_raw40(reduce->count); 31 | 32 | i = i->next; 33 | count--; 34 | } 35 | raw_by_seq_reduce *raw = (raw_by_seq_reduce*)dst; 36 | encode_raw40(total, &raw->count); 37 | *size_r = sizeof(*raw); 38 | 39 | return COUCHSTORE_SUCCESS; 40 | } 41 | 42 | 43 | static size_t encode_by_id_reduce(char *dst, uint64_t notdeleted, uint64_t deleted, uint64_t size) 44 | { 45 | raw_by_id_reduce *raw = (raw_by_id_reduce*)dst; 46 | encode_raw40(notdeleted, &raw->notdeleted); 47 | encode_raw40(deleted, &raw->deleted); 48 | encode_raw48(size, &raw->size); 49 | return sizeof(*raw); 50 | } 51 | 52 | couchstore_error_t by_id_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx) 53 | { 54 | uint64_t notdeleted = 0, deleted = 0, size = 0; 55 | const nodelist *i = leaflist; 56 | 57 | (void) ctx; 58 | 59 | while (i != NULL && count > 0) { 60 | const raw_id_index_value *raw = (const raw_id_index_value*)i->data.buf; 61 | if (decode_raw48(raw->bp) & BP_DELETED_FLAG) { 62 | deleted++; 63 | } else { 64 | notdeleted++; 65 | } 66 | size += decode_raw32(raw->size); 67 | 68 | i = i->next; 69 | count--; 70 | } 71 | 72 | *size_r = encode_by_id_reduce(dst, notdeleted, deleted, size); 73 | 74 | return COUCHSTORE_SUCCESS; 75 | 76 | } 77 | 78 | couchstore_error_t by_id_rereduce(char *dst, size_t *size_r, const nodelist *ptrlist, int count, void *ctx) 79 | { 80 | uint64_t notdeleted = 0, deleted = 0, size = 0; 81 | const nodelist *i = ptrlist; 82 | 83 | (void) ctx; 84 | 85 | while (i != NULL && count > 0) { 86 | const raw_by_id_reduce *reduce = (const raw_by_id_reduce*) i->pointer->reduce_value.buf; 87 | notdeleted += decode_raw40(reduce->notdeleted); 88 | deleted += decode_raw40(reduce->deleted); 89 | size += decode_raw48(reduce->size); 90 | 91 | i = i->next; 92 | count--; 93 | } 94 | 95 | *size_r = encode_by_id_reduce(dst, notdeleted, deleted, size); 96 | 97 | return COUCHSTORE_SUCCESS; 98 | } 99 | -------------------------------------------------------------------------------- /src/reduces.h: -------------------------------------------------------------------------------- 1 | #ifndef COUCHSTORE_REDUCES_H 2 | #define COUCHSTORE_REDUCES_H 1 3 | 4 | #include 5 | #include 6 | #include "couch_btree.h" 7 | #include "bitfield.h" 8 | 9 | typedef struct { 10 | raw_40 count; 11 | } raw_by_seq_reduce; 12 | 13 | typedef struct { 14 | raw_40 notdeleted; 15 | raw_40 deleted; 16 | raw_48 size; 17 | } raw_by_id_reduce; 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | couchstore_error_t by_seq_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx); 24 | couchstore_error_t by_seq_rereduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx); 25 | 26 | couchstore_error_t by_id_rereduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx); 27 | couchstore_error_t by_id_reduce(char *dst, size_t *size_r, const nodelist *leaflist, int count, void *ctx); 28 | 29 | #ifdef __cplusplus 30 | } 31 | #endif 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/strerror.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | #include 4 | #include 5 | 6 | LIBCOUCHSTORE_API 7 | const char *couchstore_strerror(couchstore_error_t errcode) 8 | { 9 | switch (errcode) { 10 | case COUCHSTORE_SUCCESS: 11 | return "success"; 12 | case COUCHSTORE_ERROR_OPEN_FILE: 13 | return "error opening file"; 14 | case COUCHSTORE_ERROR_CORRUPT: 15 | return "malformed data in file"; 16 | case COUCHSTORE_ERROR_ALLOC_FAIL: 17 | return "failed to allocate buffer"; 18 | case COUCHSTORE_ERROR_READ: 19 | return "error reading file"; 20 | case COUCHSTORE_ERROR_DOC_NOT_FOUND: 21 | return "document not found"; 22 | case COUCHSTORE_ERROR_NO_HEADER: 23 | return "no header in non-empty file"; 24 | case COUCHSTORE_ERROR_WRITE: 25 | return "error writing to file"; 26 | case COUCHSTORE_ERROR_HEADER_VERSION: 27 | return "incorrect version in header"; 28 | case COUCHSTORE_ERROR_CHECKSUM_FAIL: 29 | return "checksum fail"; 30 | case COUCHSTORE_ERROR_INVALID_ARGUMENTS: 31 | return "invalid arguments"; 32 | case COUCHSTORE_ERROR_NO_SUCH_FILE: 33 | return "no such file"; 34 | case COUCHSTORE_ERROR_FILE_CLOSED: 35 | return "cannot do this operation when file is closed"; 36 | case COUCHSTORE_ERROR_DB_NO_LONGER_VALID: 37 | return "this db handle could have its file reopened and must be closed"; 38 | case COUCHSTORE_ERROR_FILE_CLOSE: 39 | return "error closing file"; 40 | default: 41 | return NULL; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/tracking_file_ops.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | /** 21 | * FileOp class which builds up a map of the underlying couchstore file; by 22 | * recording read() and write() offsets and sizes. 23 | */ 24 | class TrackingFileOps : public FileOpsInterface { 25 | public: 26 | enum class Tree : uint8_t { Unknown, Sequence, Id, Local }; 27 | 28 | TrackingFileOps() { 29 | } 30 | 31 | // Implementation of FileOpsInterface /////////////////////////////////// 32 | 33 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override; 34 | 35 | couchstore_error_t open(couchstore_error_info_t* errinfo, 36 | couch_file_handle* handle, 37 | const char* path, 38 | int oflag) override; 39 | couchstore_error_t close(couchstore_error_info_t* errinfo, 40 | couch_file_handle handle) override; 41 | ssize_t pread(couchstore_error_info_t* errinfo, 42 | couch_file_handle handle, 43 | void* buf, 44 | size_t nbytes, 45 | cs_off_t offset) override; 46 | ssize_t pwrite(couchstore_error_info_t* errinfo, 47 | couch_file_handle handle, 48 | const void* buf, 49 | size_t nbytes, 50 | cs_off_t offset) override; 51 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 52 | couch_file_handle handle) override; 53 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 54 | couch_file_handle handle) override; 55 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 56 | couch_file_handle handle, 57 | cs_off_t offset, 58 | cs_off_t len, 59 | couchstore_file_advice_t advice) override; 60 | 61 | void tag(couch_file_handle handle, FileTag tag) override; 62 | 63 | void destructor(couch_file_handle handle) override; 64 | 65 | // Own methods. 66 | void setTree(couch_file_handle handle, Tree tree); 67 | 68 | /// Set true when accessing historic (old) headers and data. 69 | void setHistoricData(couch_file_handle handle, bool historic); 70 | 71 | private: 72 | /// Object representing a file, and the information we track about it. 73 | class File; 74 | }; 75 | -------------------------------------------------------------------------------- /src/tree_writer.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBCOUCHSTORE_TREE_WRITER_H 2 | #define LIBCOUCHSTORE_TREE_WRITER_H 3 | 4 | #include 5 | #include "couch_btree.h" 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | typedef struct TreeWriter TreeWriter; 12 | 13 | 14 | /** 15 | * Creates a new TreeWriter. 16 | * @param unsortedFilePath If non-NULL, the path to an existing file containing a series of unsorted 17 | * key/value pairs in TreeWriter format. If NULL, an empty TreeWriter will be created (using a 18 | * temporary file for the external sorting.) 19 | * @param key_compare Callback function that compares two keys. 20 | * @param out_writer The new TreeWriter pointer will be stored here. 21 | * @return Error code or COUCHSTORE_SUCCESS. 22 | */ 23 | couchstore_error_t TreeWriterOpen(char* unsortedFilePath, 24 | compare_callback key_compare, 25 | reduce_fn reduce, 26 | reduce_fn rereduce, 27 | void *user_reduce_ctx, 28 | TreeWriter** out_writer); 29 | 30 | /** 31 | * Frees a TreeWriter instance. It is safe to pass a NULL pointer. 32 | */ 33 | void TreeWriterFree(TreeWriter* writer); 34 | 35 | /** 36 | * Adds a key/value pair to a TreeWriter. These can be added in any order. 37 | */ 38 | couchstore_error_t TreeWriterAddItem(TreeWriter* writer, sized_buf key, sized_buf value); 39 | 40 | /** 41 | * Sorts the key/value pairs already added. 42 | * The keys are sorted by ebin_cmp (basic lexicographic order by byte values). 43 | * If this TreeWriter was opened on an existing data file, the contents of the file will be sorted. 44 | */ 45 | couchstore_error_t TreeWriterSort(TreeWriter* writer); 46 | 47 | /** 48 | * Writes the key/value pairs to a tree file, returning a pointer to the new root. 49 | * The items should first have been sorted. 50 | */ 51 | couchstore_error_t TreeWriterWrite(TreeWriter* writer, 52 | tree_file* to_file, 53 | node_pointer** out_root); 54 | 55 | 56 | /* 57 | * The input file format for TreeWriterOpen is as follows: 58 | * The file is binary and consists of nothing more than a series of records. 59 | * Each record looks like this: 60 | * 2 bytes: Key length (big-endian) 61 | * 4 bytes: Value length (big-endian) 62 | * 63 | * 64 | */ 65 | 66 | #ifdef __cplusplus 67 | } 68 | #endif 69 | 70 | 71 | #endif // LIBCOUCHSTORE_TREE_WRITER_H 72 | -------------------------------------------------------------------------------- /src/util.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "config.h" 3 | #include "node_types.h" 4 | #include "util.h" 5 | 6 | #include 7 | #include 8 | 9 | #ifdef DEBUG 10 | #include 11 | #endif 12 | 13 | int ebin_cmp(const sized_buf *e1, const sized_buf *e2) 14 | { 15 | size_t size; 16 | if (e2->size < e1->size) { 17 | size = e2->size; 18 | } else { 19 | size = e1->size; 20 | } 21 | 22 | int cmp = memcmp(e1->buf, e2->buf, size); 23 | if (cmp == 0) { 24 | if (size < e2->size) { 25 | return -1; 26 | } else if (size < e1->size) { 27 | return 1; 28 | } 29 | } 30 | return cmp; 31 | } 32 | 33 | int seq_cmp(const sized_buf *k1, const sized_buf *k2) 34 | { 35 | uint64_t e1val = decode_sequence_key(k1); 36 | uint64_t e2val = decode_sequence_key(k2); 37 | if (e1val == e2val) { 38 | return 0; 39 | } 40 | return (e1val < e2val ? -1 : 1); 41 | } 42 | 43 | fatbuf *fatbuf_alloc(size_t bytes) 44 | { 45 | fatbuf *fb = (fatbuf *) cb_malloc(sizeof(fatbuf) + bytes); 46 | #ifdef DEBUG 47 | memset(fb->buf, 0x44, bytes); 48 | #endif 49 | if (!fb) { 50 | return NULL; 51 | } 52 | 53 | fb->size = bytes; 54 | fb->pos = 0; 55 | return fb; 56 | } 57 | 58 | void *fatbuf_get(fatbuf *fb, size_t bytes) 59 | { 60 | if (fb->pos + bytes > fb->size) { 61 | return NULL; 62 | } 63 | #ifdef DEBUG 64 | if (fb->buf[fb->pos] != 0x44 && bytes > 0) { 65 | fprintf(stderr, "Fatbuf space has been written to before it was taken!\n"); 66 | } 67 | #endif 68 | void *rptr = fb->buf + fb->pos; 69 | fb->pos += bytes; 70 | return rptr; 71 | } 72 | 73 | void fatbuf_free(fatbuf *fb) 74 | { 75 | cb_free(fb); 76 | } 77 | 78 | #ifdef DEBUG 79 | void report_error(couchstore_error_t errcode, const char* file, int line) { 80 | fprintf(stderr, "Couchstore error `%s' at %s:%d\r\n", \ 81 | couchstore_strerror(errcode), file, line); 82 | } 83 | #endif 84 | 85 | sized_buf* arena_copy_buf(arena* a, const sized_buf *src) 86 | { 87 | sized_buf *nbuf = static_cast(arena_alloc(a, sizeof(sized_buf))); 88 | if (nbuf == NULL) { 89 | return NULL; 90 | } 91 | nbuf->buf = static_cast(arena_alloc(a, src->size)); 92 | if (nbuf->buf == NULL) { 93 | return NULL; 94 | } 95 | nbuf->size = src->size; 96 | memcpy(nbuf->buf, src->buf, src->size); 97 | return nbuf; 98 | } 99 | 100 | sized_buf* arena_special_copy_buf_and_revmeta(arena *a, const sized_buf *val, 101 | const DocInfo *docinfo) 102 | { 103 | sized_buf *nbuf = static_cast(arena_alloc(a, sizeof(sized_buf))); 104 | if (nbuf == NULL) { 105 | return NULL; 106 | } 107 | 108 | const raw_seq_index_value *raw = (const raw_seq_index_value*)val->buf; 109 | uint32_t idsize, datasize; 110 | decode_kv_length(&raw->sizes, &idsize, &datasize); 111 | 112 | nbuf->size = sizeof(*raw) + idsize + docinfo->rev_meta.size; 113 | nbuf->buf = static_cast(arena_alloc(a, nbuf->size)); 114 | if (nbuf->buf == NULL) { 115 | return NULL; 116 | } 117 | memcpy(nbuf->buf, val->buf, sizeof(*raw) + idsize); 118 | memcpy(nbuf->buf + sizeof(*raw) + idsize, docinfo->rev_meta.buf, 119 | docinfo->rev_meta.size); 120 | return nbuf; 121 | } 122 | 123 | cs_off_t align_to_next_block(cs_off_t offset) 124 | { 125 | if (offset % COUCH_BLOCK_SIZE != 0) { 126 | return offset + COUCH_BLOCK_SIZE - (offset % COUCH_BLOCK_SIZE); 127 | } 128 | return offset; 129 | } 130 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | #ifndef COUCHSTORE_UTIL_H 2 | #define COUCHSTORE_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "internal.h" 9 | #include "fatbuf.h" 10 | #include "arena.h" 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | /** Plain lexicographic comparison of the contents of two sized_bufs. */ 17 | int ebin_cmp(const sized_buf *e1, const sized_buf *e2); 18 | 19 | /** Compares sequence numbers (48-bit big-endian unsigned ints) stored in sized_bufs. */ 20 | int seq_cmp(const sized_buf *k1, const sized_buf *k2); 21 | 22 | /* Copy buffer to arena */ 23 | sized_buf* arena_copy_buf(arena* a, const sized_buf *src); 24 | 25 | /* Copy rev_meta from docinfo and buffer to arena */ 26 | sized_buf* arena_special_copy_buf_and_revmeta(arena *a, const sized_buf *val, 27 | const DocInfo *docinfo); 28 | 29 | /** Offsets the pointer PTR by BYTES bytes. Result is of the same type as PTR. */ 30 | #define offsetby(PTR, BYTES) ((__typeof(PTR))((uint8_t*)(PTR) + (BYTES))) 31 | 32 | /* Aligns the offset to the start of the next block */ 33 | cs_off_t align_to_next_block(cs_off_t offset); 34 | 35 | /* Sets errcode to the result of C, and jumps to the cleanup: label if it's nonzero. */ 36 | #ifdef DEBUG 37 | void report_error(couchstore_error_t, const char* file, int line); 38 | #define error_pass(C) \ 39 | do { \ 40 | if ((errcode = (C)) < 0) { \ 41 | report_error(errcode, __FILE__, __LINE__); \ 42 | goto cleanup; \ 43 | } \ 44 | } while (0) 45 | 46 | // Set errcode on error, without termination. 47 | #define error_tolerate(C) \ 48 | do { \ 49 | if ((C) < 0) { \ 50 | errcode = (C); \ 51 | report_error(errcode, __FILE__, __LINE__); \ 52 | } \ 53 | } while (0) 54 | #else 55 | #define error_pass(C) \ 56 | do { \ 57 | if ((errcode = (C)) < 0) { \ 58 | goto cleanup; \ 59 | } \ 60 | } while (0) 61 | 62 | #define error_tolerate(C) \ 63 | do { \ 64 | if ((C) < 0) { \ 65 | errcode = (C); \ 66 | } \ 67 | } while (0) 68 | #endif 69 | 70 | /* If the condition C evaluates to false/zero, sets errcode to E and jumps to the cleanup: label. */ 71 | #define error_unless(C, E) \ 72 | do { \ 73 | if(!(C)) { error_pass(E); } \ 74 | } while (0) 75 | 76 | /* If the parameter C is nonzero, sets errcode to E and jumps to the cleanup: label. */ 77 | #define error_nonzero(C, E) \ 78 | do { \ 79 | if((C) != 0) { error_pass(E); } \ 80 | } while (0) 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | 86 | #endif /* COUCHSTORE_UTIL_H */ 87 | -------------------------------------------------------------------------------- /src/viewgen.c: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | static int usage() { 9 | fprintf(stderr, "Usage: couch_viewgen [flag] [[flag] ...] \n" 10 | " flags: --reduce=count\n" 11 | " --reduce=sum\n" 12 | " --reduce=stats\n" 13 | " --back\n"); 14 | return EXIT_FAILURE; 15 | } 16 | 17 | 18 | int main(int argc, char** argv) 19 | { 20 | if(argc < 3) 21 | return usage(); 22 | 23 | couchstore_error_t errcode; 24 | CouchStoreIndex* index = NULL; 25 | 26 | // Process last arg first: the output filename 27 | const char* indexPath = argv[argc - 1]; 28 | errcode = couchstore_create_index(indexPath, &index); 29 | if (errcode) { 30 | fprintf(stderr, "Couldn't open database %s: %s\n", indexPath, couchstore_strerror(errcode)); 31 | goto cleanup; 32 | } 33 | 34 | couchstore_index_type indexType = COUCHSTORE_VIEW_PRIMARY_INDEX; 35 | const char* reduceName = NULL; 36 | couchstore_json_reducer reducer = COUCHSTORE_REDUCE_NONE; 37 | 38 | for (int i = 1; i < argc - 1; ++i) { 39 | const char* inputPath = argv[i]; 40 | if (strncmp(inputPath, "--reduce=", 9) == 0) { 41 | reduceName = inputPath + 9; 42 | if (strcmp(reduceName, "count") == 0) 43 | reducer = COUCHSTORE_REDUCE_COUNT; 44 | else if (strcmp(reduceName, "sum") == 0) 45 | reducer = COUCHSTORE_REDUCE_SUM; 46 | else if (strcmp(reduceName, "stats") == 0) 47 | reducer = COUCHSTORE_REDUCE_STATS; 48 | else { 49 | fprintf(stderr, "Unknown reduce function '%s'\n", reduceName); 50 | return usage(); 51 | } 52 | } else if (strcmp(inputPath, "--back") == 0) { 53 | indexType = COUCHSTORE_VIEW_BACK_INDEX; 54 | } else if (strcmp(inputPath, "--primary") == 0) { 55 | indexType = COUCHSTORE_VIEW_PRIMARY_INDEX; 56 | } else { 57 | if (indexType == COUCHSTORE_VIEW_PRIMARY_INDEX) { 58 | printf("Adding primary index %s to %s", inputPath, indexPath); 59 | if (reduceName) 60 | printf(", reducing by %s", reduceName); 61 | printf(" ...\n"); 62 | } else { 63 | if (reduceName) 64 | return usage(); 65 | printf("Adding back-index %s to %s...\n", inputPath, indexPath); 66 | } 67 | errcode = couchstore_index_add(inputPath, indexType, reducer, index); 68 | if (errcode < 0) { 69 | fprintf(stderr, "Error adding %s: %s\n", inputPath, couchstore_strerror(errcode)); 70 | goto cleanup; 71 | } 72 | indexType = COUCHSTORE_VIEW_PRIMARY_INDEX; 73 | reduceName = NULL; 74 | reducer = COUCHSTORE_REDUCE_NONE; 75 | } 76 | } 77 | printf("Done!"); 78 | 79 | cleanup: 80 | if (index) { 81 | couchstore_close_index(index); 82 | if (errcode < 0) { 83 | remove(indexPath); 84 | } 85 | } 86 | return errcode ? EXIT_FAILURE : EXIT_SUCCESS; 87 | } 88 | -------------------------------------------------------------------------------- /src/views/bin/couch_view_group_cleanup.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "config.h" 22 | #include "../view_group.h" 23 | #include "../util.h" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "util.h" 31 | #include "../mapreduce/mapreduce.h" 32 | 33 | #define BUF_SIZE 8192 34 | 35 | int main(int argc, char *argv[]) 36 | { 37 | view_group_info_t *group_info = NULL; 38 | uint64_t purge_count; 39 | int ret = 2; 40 | uint64_t header_pos; 41 | view_error_t error_info = {NULL, NULL}; 42 | cb_thread_t exit_thread; 43 | 44 | (void) argc; 45 | (void) argv; 46 | 47 | /* 48 | * Disable buffering for stdout/stderr 49 | */ 50 | setvbuf(stdout, (char *) NULL, _IONBF, 0); 51 | setvbuf(stderr, (char *) NULL, _IONBF, 0); 52 | 53 | if (set_binary_mode() < 0) { 54 | fprintf(stderr, "Error setting binary mode\n"); 55 | goto out; 56 | } 57 | 58 | group_info = couchstore_read_view_group_info(stdin, stderr); 59 | if (group_info == NULL) { 60 | ret = COUCHSTORE_ERROR_ALLOC_FAIL; 61 | goto out; 62 | } 63 | 64 | ret = start_exit_listener(&exit_thread); 65 | if (ret) { 66 | fprintf(stderr, "Error starting stdin exit listener thread\n"); 67 | goto out; 68 | } 69 | 70 | mapreduce_init(); 71 | ret = couchstore_cleanup_view_group(group_info, 72 | &header_pos, 73 | &purge_count, 74 | &error_info); 75 | mapreduce_deinit(); 76 | 77 | if (ret != COUCHSTORE_SUCCESS) { 78 | if (error_info.error_msg != NULL && error_info.view_name != NULL) { 79 | fprintf(stderr, 80 | "Error cleaning up index for view `%s`, reason: %s\n", 81 | error_info.view_name, 82 | error_info.error_msg); 83 | } 84 | goto out; 85 | } 86 | 87 | fprintf(stdout, "PurgedCount %" PRIu64 "\n", purge_count); 88 | 89 | out: 90 | couchstore_free_view_group_info(group_info); 91 | cb_free((void *) error_info.error_msg); 92 | cb_free((void *) error_info.view_name); 93 | 94 | ret = (ret < 0) ? (100 + ret) : ret; 95 | _exit(ret); 96 | } 97 | -------------------------------------------------------------------------------- /src/views/bin/util.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "config.h" 22 | #include "util.h" 23 | #include 24 | #include 25 | #include 26 | 27 | static void exit_thread_helper(void *args) 28 | { 29 | char buf[4]; 30 | int len = fread(buf, 1, 4, stdin); 31 | 32 | (void) args; 33 | 34 | /* If the other end closed the pipe */ 35 | if (len == 0) { 36 | _exit(1); 37 | } else if (len == 4 && !strncmp(buf, "exit", 4)) { 38 | _exit(1); 39 | } else { 40 | fprintf(stderr, "Error occured waiting for exit message (%d)\n", len); 41 | _exit(2); 42 | } 43 | } 44 | 45 | /* Start a watcher thread to gracefully die on exit message */ 46 | int start_exit_listener(cb_thread_t *id) 47 | { 48 | 49 | int ret = cb_create_thread(id, exit_thread_helper, NULL, 1); 50 | if (ret < 0) { 51 | /* For differentiating from couchstore_error_t */ 52 | return -ret; 53 | } 54 | 55 | return ret; 56 | } 57 | 58 | int set_binary_mode() 59 | { 60 | if (platform_set_binary_mode(stdin) < 0 || 61 | platform_set_binary_mode(stdout) < 0 || 62 | platform_set_binary_mode(stderr) < 0) { 63 | return -1; 64 | } 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /src/views/bin/util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _BIN_UTILS_H 22 | #define _BIN_UTILS_H 23 | 24 | #include "config.h" 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | /* Start a thread to handle exit message*/ 31 | int start_exit_listener(cb_thread_t *id); 32 | int set_binary_mode(void); 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/views/bitmap.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "config.h" 22 | #include "bitmap.h" 23 | #include 24 | 25 | #define CHUNK_BITS (sizeof(unsigned char) * CHAR_BIT) 26 | #define TOTAL_CHUNKS(map) sizeof(((map).chunks)) 27 | #define CHUNK_INDEX(map, bit) (TOTAL_CHUNKS(map) - 1 - ((bit) / CHUNK_BITS)) 28 | #define MAP_CHUNK(map, bit) ((map).chunks)[CHUNK_INDEX(map, bit)] 29 | #define CHUNK_OFFSET(bit) ((bit) % CHUNK_BITS) 30 | 31 | 32 | int is_bit_set(const bitmap_t *bm, uint16_t bit) 33 | { 34 | return (MAP_CHUNK(*bm, bit) & (1 << CHUNK_OFFSET(bit))) != 0; 35 | } 36 | 37 | 38 | void set_bit(bitmap_t *bm, uint16_t bit) 39 | { 40 | (MAP_CHUNK(*bm, bit)) |= (1 << CHUNK_OFFSET(bit)); 41 | } 42 | 43 | 44 | void unset_bit(bitmap_t *bm, uint16_t bit) 45 | { 46 | ((MAP_CHUNK(*bm, bit)) &= ~(1 << CHUNK_OFFSET(bit))); 47 | } 48 | 49 | void union_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm) 50 | { 51 | unsigned int i; 52 | for (i = 0; i < 1024 / CHUNK_BITS; ++i) { 53 | dst_bm->chunks[i] |= src_bm->chunks[i]; 54 | } 55 | } 56 | 57 | void intersect_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm) 58 | { 59 | unsigned int i; 60 | for (i = 0; i < 1024 / CHUNK_BITS; ++i) { 61 | dst_bm->chunks[i] &= src_bm->chunks[i]; 62 | } 63 | } 64 | 65 | int is_equal_bitmap(const bitmap_t *bm1, const bitmap_t *bm2) 66 | { 67 | return !memcmp(bm1, bm2, sizeof(bitmap_t)); 68 | } 69 | -------------------------------------------------------------------------------- /src/views/bitmap.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _BITMAP_H 22 | #define _BITMAP_H 23 | 24 | #include 25 | #include 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | 32 | typedef struct { 33 | /* Big endian format. 34 | * chunk[0] msb contains msb of the 1024 bits bitmap. 35 | */ 36 | unsigned char chunks[1024 / (sizeof(unsigned char) * CHAR_BIT)]; 37 | } bitmap_t; 38 | 39 | 40 | int is_bit_set(const bitmap_t *bm, uint16_t bit); 41 | void set_bit(bitmap_t *bm, uint16_t bit); 42 | void unset_bit(bitmap_t *bm, uint16_t bit); 43 | void union_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm); 44 | void intersect_bitmaps(bitmap_t *dst_bm, const bitmap_t *src_bm); 45 | int is_equal_bitmap(const bitmap_t *bm1, const bitmap_t *bm2); 46 | 47 | 48 | #ifdef __cplusplus 49 | } 50 | #endif 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/views/collate_json.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** collate_json.h 3 | ** couchstore 4 | ** 5 | ** Created by Jens Alfke on 7/9/12. 6 | ** Copyright (c) 2012 Couchbase, Inc. All rights reserved. 7 | */ 8 | 9 | #ifndef COUCH_COLLATE_JSON_H 10 | #define COUCH_COLLATE_JSON_H 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | typedef enum CollateJSONMode { 24 | kCollateJSON_Unicode, /* Compare strings as Unicode (CouchDB's default) */ 25 | kCollateJSON_Raw, /* CouchDB's "raw" collation rules */ 26 | kCollateJSON_ASCII /* Like Unicode except strings are compared as binary UTF-8 */ 27 | } CollateJSONMode; 28 | 29 | /* Custom deleter for UCollator */ 30 | typedef struct UCollDeleter { 31 | void operator() (UCollator* coll) { 32 | ucol_close(coll); 33 | } 34 | } UCollDeleter; 35 | 36 | /* Custom deleter for UConverter */ 37 | typedef struct UConvDeleter { 38 | void operator() (UConverter* cnv) { 39 | ucnv_close(cnv); 40 | } 41 | } UConvDeleter; 42 | 43 | 44 | /** 45 | * Compares two UTF-8 JSON strings using CouchDB's collation rules. 46 | * CAREFUL: The two strings must be valid JSON, with no extraneous whitespace, 47 | * otherwise this function will return wrong results or even crash. 48 | */ 49 | int CollateJSON(const sized_buf *buf1, 50 | const sized_buf *buf2, 51 | CollateJSONMode mode); 52 | 53 | /* not part of the API -- exposed for testing only (see collate_json_test.c) */ 54 | char ConvertJSONEscape(const char **in); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/views/compaction.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | #include "config.h" 21 | #include "bitmap.h" 22 | #include "values.h" 23 | #include "compaction.h" 24 | #include "../couch_btree.h" 25 | 26 | int view_id_btree_filter(const sized_buf *k, const sized_buf *v, 27 | const bitmap_t *bm) 28 | { 29 | int ret = 0; 30 | couchstore_error_t errcode = COUCHSTORE_SUCCESS; 31 | view_id_btree_value_t *val = NULL; 32 | (void) k; 33 | 34 | errcode = decode_view_id_btree_value(v->buf, v->size, &val); 35 | if (errcode != COUCHSTORE_SUCCESS) { 36 | ret = (int) errcode; 37 | goto cleanup; 38 | } 39 | 40 | ret = is_bit_set(bm, val->partition); 41 | 42 | cleanup: 43 | free_view_id_btree_value(val); 44 | return ret; 45 | } 46 | 47 | int view_btree_filter(const sized_buf *k, const sized_buf *v, 48 | const bitmap_t *bm) 49 | { 50 | int ret = 0; 51 | couchstore_error_t errcode = COUCHSTORE_SUCCESS; 52 | view_btree_value_t *val = NULL; 53 | (void) k; 54 | 55 | errcode = decode_view_btree_value(v->buf, v->size, &val); 56 | if (errcode != COUCHSTORE_SUCCESS) { 57 | ret = (int) errcode; 58 | goto cleanup; 59 | } 60 | 61 | ret = is_bit_set(bm, val->partition); 62 | 63 | cleanup: 64 | free_view_btree_value(val); 65 | return ret; 66 | } 67 | -------------------------------------------------------------------------------- /src/views/compaction.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2014 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_COMPACTION_H 22 | #define _VIEW_COMPACTION_H 23 | #endif 24 | 25 | #include "../couch_btree.h" 26 | #include "../internal.h" 27 | #include "bitmap.h" 28 | #include 29 | #include 30 | #include 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | /* Filter function to selectively ignore values during compaction */ 37 | typedef int (*compact_filter_fn)(const sized_buf *k, const sized_buf *v, 38 | const bitmap_t *bm); 39 | 40 | /* Function spec for updating compactor progress */ 41 | typedef void (*stats_update_fn)(uint64_t freq, uint64_t inserted); 42 | 43 | typedef struct { 44 | uint64_t freq; 45 | uint64_t inserted; 46 | stats_update_fn update_fun; 47 | } compactor_stats_t; 48 | 49 | /* Compaction context definition */ 50 | typedef struct { 51 | couchfile_modify_result *mr; 52 | arena *transient_arena; 53 | const bitmap_t *filterbm; 54 | compact_filter_fn filter_fun; 55 | compactor_stats_t *stats; 56 | } view_compact_ctx_t; 57 | 58 | int view_id_btree_filter(const sized_buf *k, const sized_buf *v, 59 | const bitmap_t *bm); 60 | 61 | int view_btree_filter(const sized_buf *k, const sized_buf *v, 62 | const bitmap_t *bm); 63 | 64 | #ifdef __cplusplus 65 | } 66 | #endif 67 | -------------------------------------------------------------------------------- /src/views/file_merger.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "config.h" 22 | #include "util.h" 23 | #include "file_merger.h" 24 | #include "file_sorter.h" 25 | #include "spatial.h" 26 | 27 | static file_merger_error_t merge_view_files(const char *source_files[], 28 | unsigned num_source_files, 29 | const char *dest_path, 30 | view_file_merge_ctx_t *ctx); 31 | 32 | 33 | LIBCOUCHSTORE_API 34 | file_merger_error_t merge_view_kvs_ops_files(const char *source_files[], 35 | unsigned num_source_files, 36 | const char *dest_path) 37 | { 38 | view_file_merge_ctx_t ctx; 39 | 40 | ctx.key_cmp_fun = view_key_cmp; 41 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 42 | 43 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 44 | } 45 | 46 | 47 | LIBCOUCHSTORE_API 48 | file_merger_error_t merge_view_ids_ops_files(const char *source_files[], 49 | unsigned num_source_files, 50 | const char *dest_path) 51 | { 52 | view_file_merge_ctx_t ctx; 53 | 54 | ctx.key_cmp_fun = view_id_cmp; 55 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 56 | 57 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 58 | } 59 | 60 | 61 | LIBCOUCHSTORE_API 62 | file_merger_error_t merge_spatial_kvs_ops_files(const char *source_files[], 63 | unsigned num_source_files, 64 | const char *dest_path, 65 | const char *tmp_dir) 66 | { 67 | file_sorter_error_t ret; 68 | view_file_merge_ctx_t ctx; 69 | unsigned i; 70 | 71 | ctx.key_cmp_fun = spatial_merger_key_cmp; 72 | ctx.type = INCREMENTAL_UPDATE_SPATIAL_RECORD; 73 | 74 | /* The spatial kv files are not sorted, hence sort them before merge 75 | * sorting them */ 76 | for(i = 0; i < num_source_files; ++i) { 77 | ret = sort_spatial_kvs_ops_file(source_files[i], tmp_dir, &ctx); 78 | if (ret != FILE_SORTER_SUCCESS) { 79 | fprintf(stderr, "Error sorting spatial view records file (%d): %s", 80 | ret, source_files[i]); 81 | return FILE_MERGER_SORT_ERROR; 82 | } 83 | } 84 | 85 | return merge_view_files(source_files, num_source_files, dest_path, &ctx); 86 | } 87 | 88 | 89 | static file_merger_error_t merge_view_files(const char *source_files[], 90 | unsigned num_source_files, 91 | const char *dest_path, 92 | view_file_merge_ctx_t *ctx) 93 | { 94 | return merge_files(source_files, num_source_files, dest_path, 95 | read_view_record, write_view_record, NULL, 96 | compare_view_records, dedup_view_records_merger, 97 | free_view_record, 0, ctx); 98 | } 99 | -------------------------------------------------------------------------------- /src/views/file_merger.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_FILE_MERGER_H 22 | #define _VIEW_FILE_MERGER_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | #include "../file_merger.h" 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | 34 | /* 35 | * Merge a group files containing sorted sets of btree operations for a 36 | * view btree. 37 | */ 38 | LIBCOUCHSTORE_API 39 | file_merger_error_t merge_view_kvs_ops_files(const char *source_files[], 40 | unsigned num_source_files, 41 | const char *dest_path); 42 | 43 | /* 44 | * Merge a group files containing sorted sets of btree operations for a 45 | * view id btree (back index). 46 | */ 47 | LIBCOUCHSTORE_API 48 | file_merger_error_t merge_view_ids_ops_files(const char *source_files[], 49 | unsigned num_source_files, 50 | const char *dest_path); 51 | 52 | 53 | /* 54 | * Merge a group files containing sets of operations for a spatial view 55 | * index. 56 | */ 57 | LIBCOUCHSTORE_API 58 | file_merger_error_t merge_spatial_kvs_ops_files(const char *source_files[], 59 | unsigned num_source_files, 60 | const char *dest_path, 61 | const char *tmp_dir); 62 | 63 | #ifdef __cplusplus 64 | } 65 | #endif 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /src/views/file_sorter.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "file_sorter.h" 22 | #include "util.h" 23 | #include "spatial.h" 24 | 25 | #define SORT_MAX_BUFFER_SIZE (64 * 1024 * 1024) 26 | #define SORT_MAX_NUM_TMP_FILES 16 27 | 28 | 29 | static file_sorter_error_t do_sort_file(const char *file_path, 30 | const char *tmp_dir, 31 | file_merger_feed_record_t callback, 32 | int skip_writeback, 33 | view_file_merge_ctx_t *ctx); 34 | 35 | LIBCOUCHSTORE_API 36 | file_sorter_error_t sort_view_kvs_ops_file(const char *file_path, 37 | const char *tmp_dir) 38 | { 39 | view_file_merge_ctx_t ctx; 40 | 41 | ctx.key_cmp_fun = view_key_cmp; 42 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 43 | 44 | return do_sort_file(file_path, tmp_dir, NULL, 0, &ctx); 45 | } 46 | 47 | 48 | LIBCOUCHSTORE_API 49 | file_sorter_error_t sort_view_kvs_file(const char *file_path, 50 | const char *tmp_dir, 51 | file_merger_feed_record_t callback, 52 | void *user_ctx) 53 | { 54 | view_file_merge_ctx_t ctx; 55 | 56 | ctx.key_cmp_fun = view_key_cmp; 57 | ctx.type = INITIAL_BUILD_VIEW_RECORD; 58 | ctx.user_ctx = user_ctx; 59 | 60 | return do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 61 | } 62 | 63 | 64 | LIBCOUCHSTORE_API 65 | file_sorter_error_t sort_view_ids_ops_file(const char *file_path, 66 | const char *tmp_dir) 67 | { 68 | view_file_merge_ctx_t ctx; 69 | 70 | ctx.key_cmp_fun = view_id_cmp; 71 | ctx.type = INCREMENTAL_UPDATE_VIEW_RECORD; 72 | 73 | return do_sort_file(file_path, tmp_dir, NULL, 0, &ctx); 74 | } 75 | 76 | 77 | LIBCOUCHSTORE_API 78 | file_sorter_error_t sort_view_ids_file(const char *file_path, 79 | const char *tmp_dir, 80 | file_merger_feed_record_t callback, 81 | void *user_ctx) 82 | { 83 | view_file_merge_ctx_t ctx; 84 | 85 | ctx.key_cmp_fun = view_id_cmp; 86 | ctx.type = INITIAL_BUILD_VIEW_RECORD; 87 | ctx.user_ctx = user_ctx; 88 | 89 | return do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 90 | } 91 | 92 | 93 | LIBCOUCHSTORE_API 94 | file_sorter_error_t sort_spatial_kvs_file(const char *file_path, 95 | const char *tmp_dir, 96 | file_merger_feed_record_t callback, 97 | void *user_ctx) 98 | { 99 | file_sorter_error_t ret; 100 | view_file_merge_ctx_t ctx; 101 | 102 | ctx.key_cmp_fun = spatial_key_cmp; 103 | ctx.type = INITIAL_BUILD_SPATIAL_RECORD; 104 | ctx.user_ctx = user_ctx; 105 | 106 | ret = do_sort_file(file_path, tmp_dir, callback, 1, &ctx); 107 | 108 | return ret; 109 | } 110 | 111 | 112 | LIBCOUCHSTORE_API 113 | file_sorter_error_t sort_spatial_kvs_ops_file(const char *file_path, 114 | const char *tmp_dir, 115 | view_file_merge_ctx_t *ctx) 116 | { 117 | return do_sort_file(file_path, tmp_dir, NULL, 0, ctx); 118 | } 119 | 120 | 121 | static file_sorter_error_t do_sort_file(const char *file_path, 122 | const char *tmp_dir, 123 | file_merger_feed_record_t callback, 124 | int skip_writeback, 125 | view_file_merge_ctx_t *ctx) 126 | { 127 | return sort_file(file_path, 128 | tmp_dir, 129 | SORT_MAX_NUM_TMP_FILES, 130 | SORT_MAX_BUFFER_SIZE, 131 | read_view_record, 132 | write_view_record, 133 | callback, 134 | compare_view_records, 135 | free_view_record, 136 | skip_writeback, 137 | ctx); 138 | } 139 | -------------------------------------------------------------------------------- /src/views/file_sorter.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_FILE_SORTER_H 22 | #define _VIEW_FILE_SORTER_H 23 | 24 | #include "config.h" 25 | #include 26 | #include "../file_sorter.h" 27 | #include "util.h" 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | 34 | /* 35 | * Sort a file containing records of btree operations for a view btree. 36 | */ 37 | LIBCOUCHSTORE_API 38 | file_sorter_error_t sort_view_kvs_ops_file(const char *file_path, 39 | const char *tmp_dir); 40 | 41 | /* 42 | * Sort a file containing view records for a view btree. 43 | */ 44 | LIBCOUCHSTORE_API 45 | file_sorter_error_t sort_view_kvs_file(const char *file_path, 46 | const char *tmp_dir, 47 | file_merger_feed_record_t callback, 48 | void *user_ctx); 49 | 50 | /* 51 | * Sort a file containing records of btree operations for a view id 52 | * btree (back index). 53 | */ 54 | LIBCOUCHSTORE_API 55 | file_sorter_error_t sort_view_ids_ops_file(const char *file_path, 56 | const char *tmp_dir); 57 | 58 | /* 59 | * Sort a file containing records for a view id btree (back index). 60 | */ 61 | LIBCOUCHSTORE_API 62 | file_sorter_error_t sort_view_ids_file(const char *file_path, 63 | const char *tmp_dir, 64 | file_merger_feed_record_t callback, 65 | void *user_ctx); 66 | 67 | /* 68 | * Sort a file containing records for a spatial index. 69 | */ 70 | LIBCOUCHSTORE_API 71 | file_sorter_error_t sort_spatial_kvs_file(const char *file_path, 72 | const char *tmp_dir, 73 | file_merger_feed_record_t callback, 74 | void *user_ctx); 75 | 76 | /* 77 | * Sort a file containing records of spatial index operations for a 78 | * spatial view. 79 | */ 80 | LIBCOUCHSTORE_API 81 | file_sorter_error_t sort_spatial_kvs_ops_file(const char *file_path, 82 | const char *tmp_dir, 83 | view_file_merge_ctx_t *ctx); 84 | 85 | /* Record file sorter */ 86 | typedef file_sorter_error_t (*sort_record_fn)(const char *file_path, 87 | const char *tmp_dir, 88 | file_merger_feed_record_t callback, 89 | void *user_ctx); 90 | 91 | #ifdef __cplusplus 92 | } 93 | #endif 94 | 95 | #endif 96 | -------------------------------------------------------------------------------- /src/views/index_header.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _INDEX_HEADER_H 22 | #define _INDEX_HEADER_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | #include 28 | #include "bitmap.h" 29 | #include "sorted_list.h" 30 | #include "../node_types.h" 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | 37 | #define LATEST_INDEX_HEADER_VERSION 2 38 | 39 | typedef struct { 40 | uint16_t part_id; 41 | uint64_t seq; 42 | } part_seq_t; 43 | 44 | 45 | typedef struct { 46 | /* sorted_list instance, values of type uint16_t */ 47 | void *active; 48 | /* sorted_list instance, values of type uint16_t */ 49 | void *passive; 50 | /* sorted_list instance, values of type uint16_t */ 51 | void *unindexable; 52 | } index_state_transition_t; 53 | 54 | 55 | typedef struct { 56 | unsigned char uuid[8]; 57 | uint64_t seq; 58 | } failover_log_t; 59 | 60 | 61 | typedef struct { 62 | uint16_t part_id; 63 | uint16_t num_failover_log; 64 | failover_log_t *failover_log; 65 | } part_version_t; 66 | 67 | 68 | typedef struct { 69 | uint8_t version; 70 | /* MD5 hash */ 71 | unsigned char signature[16]; 72 | uint8_t num_views; 73 | uint16_t num_partitions; 74 | bitmap_t active_bitmask; 75 | bitmap_t passive_bitmask; 76 | bitmap_t cleanup_bitmask; 77 | /* sorted_list instance, values of type part_seq_t */ 78 | void *seqs; 79 | node_pointer *id_btree_state; 80 | /* array of num_views elements */ 81 | node_pointer **view_states; 82 | int has_replica; 83 | /* sorted_list instance, values of type uint16_t */ 84 | void *replicas_on_transfer; 85 | index_state_transition_t pending_transition; 86 | /* sorted_list instance, values of type part_seq_t */ 87 | void *unindexable_seqs; 88 | /* sorted_list instance, values of type part_ver_t */ 89 | void *part_versions; 90 | } index_header_t; 91 | 92 | 93 | couchstore_error_t decode_index_header(const char *bytes, 94 | size_t len, 95 | index_header_t **header); 96 | 97 | couchstore_error_t encode_index_header(const index_header_t *header, 98 | char **buffer, 99 | size_t *buffer_size); 100 | 101 | void free_index_header(index_header_t *header); 102 | 103 | #ifdef __cplusplus 104 | } 105 | #endif 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /src/views/keys.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _KEYS_H 3 | #define _KEYS_H 4 | 5 | #include "config.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | 16 | typedef struct { 17 | sized_buf json_key; 18 | sized_buf doc_id; 19 | } view_btree_key_t; 20 | 21 | typedef struct { 22 | uint16_t partition; 23 | sized_buf doc_id; 24 | } view_id_btree_key_t; 25 | 26 | 27 | couchstore_error_t decode_view_btree_key(const char *bytes, 28 | size_t len, 29 | view_btree_key_t **key); 30 | 31 | couchstore_error_t encode_view_btree_key(const view_btree_key_t *key, 32 | char **buffer, 33 | size_t *buffer_size); 34 | 35 | void free_view_btree_key(view_btree_key_t *key); 36 | 37 | couchstore_error_t decode_view_id_btree_key(const char *bytes, 38 | size_t len, 39 | view_id_btree_key_t **key); 40 | 41 | couchstore_error_t encode_view_id_btree_key(const view_id_btree_key_t *key, 42 | char **buffer, 43 | size_t *buffer_size); 44 | 45 | void free_view_id_btree_key(view_id_btree_key_t *key); 46 | 47 | #ifdef __cplusplus 48 | } 49 | #endif 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/views/mapreduce/mapreduce_internal.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @copyright 2013 Couchbase, Inc. 3 | * 4 | * @author Filipe Manana 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | * use this file except in compliance with the License. You may obtain a copy of 8 | * the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | * License for the specific language governing permissions and limitations under 16 | * the License. 17 | **/ 18 | 19 | /** 20 | * This is a private header, do not include it in other applications/lirbaries. 21 | **/ 22 | 23 | #ifndef _MAPREDUCE_INTERNAL_H 24 | #define _MAPREDUCE_INTERNAL_H 25 | 26 | #include "mapreduce.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | 41 | class MapReduceError; 42 | 43 | typedef std::list json_results_list_t; 44 | typedef std::list kv_list_int_t; 45 | typedef std::vector< v8::Persistent* > function_vector_t; 46 | 47 | typedef struct { 48 | v8::Persistent jsContext; 49 | v8::Isolate *isolate; 50 | v8::ArrayBuffer::Allocator *bufAllocator; 51 | function_vector_t *functions; 52 | kv_list_int_t *kvs; 53 | std::atomic taskStartTime; 54 | std::mutex exitMutex; 55 | } mapreduce_ctx_t; 56 | 57 | 58 | void initContext(mapreduce_ctx_t *ctx, 59 | const std::list &function_sources); 60 | 61 | void destroyContext(mapreduce_ctx_t *ctx); 62 | 63 | void mapDoc(mapreduce_ctx_t *ctx, 64 | const mapreduce_json_t &doc, 65 | const mapreduce_json_t &meta, 66 | mapreduce_map_result_list_t *result); 67 | 68 | json_results_list_t runReduce(mapreduce_ctx_t *ctx, 69 | const mapreduce_json_list_t &keys, 70 | const mapreduce_json_list_t &values); 71 | 72 | mapreduce_json_t runReduce(mapreduce_ctx_t *ctx, 73 | int reduceFunNum, 74 | const mapreduce_json_list_t &keys, 75 | const mapreduce_json_list_t &values); 76 | 77 | mapreduce_json_t runRereduce(mapreduce_ctx_t *ctx, 78 | int reduceFunNum, 79 | const mapreduce_json_list_t &reductions); 80 | 81 | void terminateTask(mapreduce_ctx_t *ctx); 82 | 83 | 84 | 85 | class MapReduceError { 86 | public: 87 | MapReduceError(const mapreduce_error_t error, const char *msg) 88 | : _error(error), _msg(msg) { 89 | } 90 | 91 | MapReduceError(const mapreduce_error_t error, const std::string &msg) 92 | : _error(error), _msg(msg) { 93 | } 94 | 95 | mapreduce_error_t getError() const { 96 | return _error; 97 | } 98 | 99 | const std::string& getMsg() const { 100 | return _msg; 101 | } 102 | 103 | private: 104 | const mapreduce_error_t _error; 105 | const std::string _msg; 106 | }; 107 | 108 | #endif 109 | -------------------------------------------------------------------------------- /src/views/mapreduce/visibility.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _MAPREDUCE_VISIBILITY_H 3 | #define _MAPREDUCE_VISIBILITY_H 4 | 5 | #if defined(LIBMAPREDUCE_INTERNAL) 6 | 7 | #ifdef __SUNPRO_C 8 | #define LIBMAPREDUCE_API __global 9 | #elif defined(HAVE_VISIBILITY) && HAVE_VISIBILITY 10 | #define LIBMAPREDUCE_API __attribute__ ((visibility("default"))) 11 | #elif defined(_MSC_VER) 12 | #define LIBMAPREDUCE_API extern __declspec(dllexport) 13 | #else 14 | #define LIBMAPREDUCE_API 15 | #endif 16 | 17 | #else 18 | 19 | #if defined(_MSC_VER) && !defined(LIBCOUCHSTORE_NO_VISIBILITY) 20 | #define LIBMAPREDUCE_API extern __declspec(dllimport) 21 | #else 22 | #define LIBMAPREDUCE_API 23 | #endif 24 | 25 | #endif 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/views/purgers.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_PURGERS_H 22 | #define _VIEW_PURGERS_H 23 | #endif 24 | 25 | #include "../couch_btree.h" 26 | #include "../internal.h" 27 | #include "bitmap.h" 28 | #include "mapreduce/mapreduce.h" 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #ifdef __cplusplus 35 | extern "C" { 36 | #endif 37 | 38 | typedef struct { 39 | bitmap_t cbitmask; 40 | uint64_t count; 41 | } view_purger_ctx_t; 42 | 43 | int view_id_btree_purge_kv(const sized_buf *key, const sized_buf *val, 44 | void *ctx); 45 | int view_id_btree_purge_kp(const node_pointer *ptr, void *ctx); 46 | int view_btree_purge_kv(const sized_buf *key, const sized_buf *val, 47 | void *ctx); 48 | int view_btree_purge_kp(const node_pointer *ptr, void *ctx); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | -------------------------------------------------------------------------------- /src/views/reducers.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * @author Fulu Li 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 10 | * use this file except in compliance with the License. You may obtain a copy of 11 | * the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 18 | * License for the specific language governing permissions and limitations under 19 | * the License. 20 | **/ 21 | 22 | #ifndef _VIEW_REDUCERS_H 23 | #define _VIEW_REDUCERS_H 24 | 25 | #include "../couch_btree.h" 26 | #include "../internal.h" 27 | #include "mapreduce/mapreduce.h" 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #ifdef __cplusplus 34 | extern "C" { 35 | #endif 36 | 37 | typedef struct { 38 | /* If not NULL, an error happened and it contains a human 39 | readable error message. */ 40 | const char *error; 41 | void *priv; 42 | } view_reducer_ctx_t; 43 | 44 | typedef struct { 45 | uint64_t count; 46 | double sum, min, max, sumsqr; 47 | } stats_t; 48 | 49 | 50 | view_reducer_ctx_t *make_view_reducer_ctx(const char *functions[], 51 | unsigned num_functions, 52 | char **error_msg); 53 | 54 | void free_view_reducer_ctx(view_reducer_ctx_t *ctx); 55 | 56 | couchstore_error_t view_id_btree_reduce(char *dst, 57 | size_t *size_r, 58 | const nodelist *leaflist, 59 | int count, 60 | void *ctx); 61 | 62 | couchstore_error_t view_id_btree_rereduce(char *dst, 63 | size_t *size_r, 64 | const nodelist *itmlist, 65 | int count, 66 | void *ctx); 67 | 68 | couchstore_error_t view_btree_reduce(char *dst, 69 | size_t *size_r, 70 | const nodelist *leaflist, 71 | int count, 72 | void *ctx); 73 | 74 | couchstore_error_t view_btree_rereduce(char *dst, 75 | size_t *size_r, 76 | const nodelist *nodelist, 77 | int count, 78 | void *ctx); 79 | 80 | #ifdef __cplusplus 81 | } 82 | #endif 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /src/views/reductions.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _REDUCTIONS_H 3 | #define _REDUCTIONS_H 4 | 5 | #include "config.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "bitmap.h" 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | 17 | typedef struct { 18 | uint64_t kv_count; 19 | bitmap_t partitions_bitmap; 20 | /* number of elements in reduce_values */ 21 | uint8_t num_values; 22 | sized_buf *reduce_values; 23 | } view_btree_reduction_t; 24 | 25 | typedef struct { 26 | uint64_t kv_count; 27 | bitmap_t partitions_bitmap; 28 | } view_id_btree_reduction_t; 29 | 30 | 31 | couchstore_error_t decode_view_btree_reduction(const char *bytes, 32 | size_t len, 33 | view_btree_reduction_t **reduction); 34 | 35 | couchstore_error_t encode_view_btree_reduction(const view_btree_reduction_t *reduction, 36 | char *buffer, 37 | size_t *buffer_size); 38 | 39 | void free_view_btree_reduction(view_btree_reduction_t *reduction); 40 | 41 | couchstore_error_t decode_view_id_btree_reduction(const char *bytes, 42 | view_id_btree_reduction_t **reduction); 43 | 44 | couchstore_error_t encode_view_id_btree_reduction(const view_id_btree_reduction_t *reduction, 45 | char *buffer, 46 | size_t *buffer_size); 47 | 48 | void free_view_id_btree_reduction(view_id_btree_reduction_t *reduction); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/views/sorted_list.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _SORTED_LIST_H 22 | #define _SORTED_LIST_H 23 | 24 | #include 25 | #include 26 | 27 | 28 | #ifdef __cplusplus 29 | extern "C" { 30 | #endif 31 | 32 | 33 | /** Returns: 34 | * negative integer if a < b, 0 if a == b, positive integer if a > b 35 | **/ 36 | typedef int (*sorted_list_cmp_t)(const void *a, const void *b); 37 | 38 | 39 | void *sorted_list_create(sorted_list_cmp_t less_fun); 40 | 41 | int sorted_list_add(void *list, const void *elem, size_t elem_size); 42 | 43 | void *sorted_list_get(const void *list, const void *elem); 44 | 45 | void sorted_list_remove(void *list, const void *elem); 46 | 47 | int sorted_list_size(const void *list); 48 | 49 | void sorted_list_free(void *list); 50 | 51 | void *sorted_list_iterator(const void *list); 52 | 53 | void *sorted_list_next(void *iterator); 54 | 55 | void sorted_list_free_iterator(void *iterator); 56 | 57 | 58 | #ifdef __cplusplus 59 | } 60 | #endif 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /src/views/spatial.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Volker Mische 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_SPATIAL_UTILS_H 22 | #define _VIEW_SPATIAL_UTILS_H 23 | 24 | #include "config.h" 25 | #include 26 | #include "../file_merger.h" 27 | #include "../couch_btree.h" 28 | #include "bitmap.h" 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif 33 | #define ZCODE_PRECISION 32 34 | #define ZCODE_MAX_VALUE UINT32_MAX 35 | 36 | typedef struct { 37 | double *mbb; 38 | /* the total number of values (two times the dimension) */ 39 | uint16_t num; 40 | } sized_mbb_t; 41 | 42 | /* It is used to scale up MBBs to the relative size of an enclosing one */ 43 | typedef struct { 44 | /* The offset the value needs to be shifted in order to be in the 45 | * origin of the enclosing MBB */ 46 | double *offsets; 47 | /* The scale factors for every dimension */ 48 | double *scales; 49 | /* the total number of values, one per dimension */ 50 | uint8_t dim; 51 | } scale_factor_t; 52 | 53 | /* The context to build the initial index */ 54 | typedef struct { 55 | arena *transient_arena; 56 | couchfile_modify_result *modify_result; 57 | /* Scale MBBs up for a better results when using the space filling 58 | * curve */ 59 | scale_factor_t *scale_factor; 60 | } view_spatial_builder_ctx_t; 61 | 62 | /* compare keys of a spatial index */ 63 | int spatial_key_cmp(const sized_buf *key1, const sized_buf *key2, 64 | const void *user_ctx); 65 | 66 | /* Compare keys of a spatial index for the file merger */ 67 | int spatial_merger_key_cmp(const sized_buf *key1, const sized_buf *key2, 68 | const void *user_ctx); 69 | 70 | /* Return the scale factor for every dimension that would be needed to 71 | * scale this MBB to the maximum value `max` (when shifted to the 72 | * origin) 73 | * Memory is dynamically allocted within the function, make sure to call 74 | * free_spatial_scale_factor() afterwards */ 75 | scale_factor_t *spatial_scale_factor(const double *mbb, uint16_t dim, 76 | uint32_t max); 77 | 78 | /* Free the memory that spatial_scale_factor() allocated */ 79 | void free_spatial_scale_factor(scale_factor_t *sf); 80 | 81 | /* Calculate the center of an multi-dimensional bounding box (MBB) */ 82 | double *spatial_center(const sized_mbb_t *mbb); 83 | 84 | /* Scales all dimensions of a (multi-dimensional) point 85 | * with the given factor and offset */ 86 | uint32_t *spatial_scale_point(const double *point, 87 | const scale_factor_t *sf); 88 | 89 | /* Set a bit on a buffer with a certain size */ 90 | void set_bit_sized(unsigned char *bitmap, uint16_t size, uint16_t bit); 91 | 92 | /* Interleave numbers bitwise. The return value is a unsigned char array 93 | * with length 4 bytes * number of numbers. 94 | * The maximum number of numbers is (2^14)-1 (16383). */ 95 | unsigned char *interleave_uint32s(uint32_t *numbers, uint16_t num); 96 | 97 | /* A reduce is used to calculate the enclosing MBB of a parent node (it's 98 | * its key) */ 99 | couchstore_error_t view_spatial_reduce(char *dst, 100 | size_t *size_r, 101 | const nodelist *leaflist, 102 | int count, 103 | void *ctx); 104 | 105 | /* Puts an item into the results set. If there are enough items they are 106 | * are flused to disk */ 107 | couchstore_error_t spatial_push_item(sized_buf *k, sized_buf *v, 108 | couchfile_modify_result *dst); 109 | 110 | /* Build an r-tree bottom-up from the already stored leaf nodes */ 111 | node_pointer* complete_new_spatial(couchfile_modify_result* mr, 112 | couchstore_error_t *errcode); 113 | 114 | /* Filter function for the compactor */ 115 | int view_spatial_filter(const sized_buf *k, const sized_buf *v, 116 | const bitmap_t *bm); 117 | #ifdef __cplusplus 118 | } 119 | #endif 120 | 121 | #endif 122 | -------------------------------------------------------------------------------- /src/views/util.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_UTILS_H 22 | #define _VIEW_UTILS_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | #include "../file_merger.h" 28 | #include "view_group.h" 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif 33 | 34 | #pragma pack(push, 1) 35 | typedef struct { 36 | uint8_t op; 37 | uint16_t ksize; 38 | uint32_t vsize; 39 | } view_file_merge_record_t; 40 | #pragma pack(pop) 41 | 42 | #define VIEW_RECORD_KEY(rec) (((char *) rec) + sizeof(view_file_merge_record_t)) 43 | #define VIEW_RECORD_VAL(rec) (VIEW_RECORD_KEY(rec) + rec->ksize) 44 | 45 | enum view_record_type { 46 | INITIAL_BUILD_VIEW_RECORD, 47 | INCREMENTAL_UPDATE_VIEW_RECORD, 48 | INITIAL_BUILD_SPATIAL_RECORD, 49 | INCREMENTAL_UPDATE_SPATIAL_RECORD 50 | }; 51 | 52 | typedef struct { 53 | FILE *src_f; 54 | FILE *dst_f; 55 | enum view_record_type type; 56 | int (*key_cmp_fun)(const sized_buf *key1, const sized_buf *key2, 57 | const void *user_ctx); 58 | const void *user_ctx; 59 | } view_file_merge_ctx_t; 60 | 61 | /* compare keys of a view btree */ 62 | int view_key_cmp(const sized_buf *key1, const sized_buf *key2, 63 | const void *user_ctx); 64 | 65 | /* compare keys of the id btree of an index */ 66 | int view_id_cmp(const sized_buf *key1, const sized_buf *key2, 67 | const void *user_ctx); 68 | 69 | /* read view index record from a file, obbeys the read record function 70 | prototype defined in src/file_merger.h */ 71 | int read_view_record(FILE *in, void **buf, void *ctx); 72 | 73 | /* write view index record from a file, obbeys the write record function 74 | prototype defined in src/file_merger.h */ 75 | file_merger_error_t write_view_record(FILE *out, void *buf, void *ctx); 76 | 77 | /* compare 2 view index records, obbeys the record compare function 78 | prototype defined in src/file_merger.h */ 79 | int compare_view_records(const void *r1, const void *r2, void *ctx); 80 | 81 | /* Pick the winner from the duplicate entries */ 82 | size_t dedup_view_records_merger(file_merger_record_t **records, size_t len, void *ctx); 83 | 84 | /* frees a view record, obbeys the record free function prototype 85 | defined in src/file_merger.h */ 86 | void free_view_record(void *record, void *ctx); 87 | 88 | LIBCOUCHSTORE_API 89 | char *couchstore_read_line(FILE *in, char *buf, int size); 90 | 91 | LIBCOUCHSTORE_API 92 | uint64_t couchstore_read_int(FILE *in, char *buf, size_t size, 93 | couchstore_error_t *ret); 94 | 95 | /* Generate appropriate view error messages */ 96 | void set_error_info(const view_btree_info_t *info, 97 | const char *red_error, 98 | couchstore_error_t ret, 99 | view_error_t *error_info); 100 | 101 | #ifdef __cplusplus 102 | } 103 | #endif 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /src/views/values.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _VALUES_H 3 | #define _VALUES_H 4 | 5 | #include "config.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | typedef struct { 16 | uint16_t partition; 17 | uint16_t num_values; 18 | sized_buf *values; 19 | } view_btree_value_t; 20 | 21 | 22 | typedef struct { 23 | uint8_t view_id; 24 | uint16_t num_keys; 25 | sized_buf *json_keys; 26 | } view_keys_mapping_t; 27 | 28 | typedef struct { 29 | uint16_t partition; 30 | uint16_t num_view_keys_map; 31 | view_keys_mapping_t *view_keys_map; 32 | } view_id_btree_value_t; 33 | 34 | 35 | couchstore_error_t decode_view_btree_value(const char *bytes, 36 | size_t len, 37 | view_btree_value_t **value); 38 | 39 | couchstore_error_t encode_view_btree_value(const view_btree_value_t *value, 40 | char **buffer, 41 | size_t *buffer_size); 42 | 43 | void free_view_btree_value(view_btree_value_t *value); 44 | 45 | couchstore_error_t decode_view_id_btree_value(const char *bytes, 46 | size_t len, 47 | view_id_btree_value_t **value); 48 | 49 | couchstore_error_t encode_view_id_btree_value(const view_id_btree_value_t *value, 50 | char **buffer, 51 | size_t *buffer_size); 52 | 53 | void free_view_id_btree_value(view_id_btree_value_t *value); 54 | 55 | #ifdef __cplusplus 56 | } 57 | #endif 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /tests/btree_purge/purge_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _PURGE_TESTS_H 22 | #define _PURGE_TESTS_H 23 | 24 | #include "config.h" 25 | #include 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | void test_no_purge_items(void); 32 | void test_all_purge_items(void); 33 | void test_partial_purge_items(void); 34 | void test_partial_purge_items2(void); 35 | void test_partial_purge_with_stop(void); 36 | void test_add_remove_purge(void); 37 | 38 | void purge_tests(void); 39 | void test_only_single_leafnode(void); 40 | 41 | #ifdef __cplusplus 42 | } 43 | #endif 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /tests/btree_purge/tests.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Sarath Lakshman 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "purge_tests.h" 22 | 23 | void purge_tests() 24 | { 25 | fprintf(stderr, "\n\nRunning purge tests\n\n"); 26 | test_no_purge_items(); 27 | test_all_purge_items(); 28 | test_partial_purge_items(); 29 | test_partial_purge_items2(); 30 | test_partial_purge_with_stop(); 31 | test_only_single_leafnode(); 32 | test_add_remove_purge(); 33 | } 34 | -------------------------------------------------------------------------------- /tests/bulk.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | testlib.check_doc(db, k, v[2]) 8 | end 9 | 10 | local found = 0 11 | db:changes(0, 0, function(di) found = found + 1 end) 12 | if found ~= #t then 13 | error("Expected " .. #t .. " results, found " .. found) 14 | end 15 | end 16 | 17 | function test_explicit(dbname) 18 | local t = { 19 | {"k1", "value 1", 1,}, 20 | {"k2", "value 2", 1,}, 21 | {"k3", "value 3", 1,}, 22 | {"k4", "value 4", 1,}, 23 | {"k5", "value 5", 1,}, 24 | {"k6", "value 6", 1,}, 25 | {"k7", "value 7", 1,} 26 | } 27 | 28 | local db = couch.open(dbname, true) 29 | 30 | for i,v in ipairs(t) do 31 | local k = v[1] 32 | testlib.assert_no_doc(db, k) 33 | end 34 | 35 | db:save_bulk(t) 36 | db:commit() 37 | 38 | check_table(db, t) 39 | end 40 | 41 | function large_txn_bulk_test(dbname) 42 | local t = {} 43 | for i = 0, 10000, 1 do 44 | table.insert(t, {"k" .. i, "value " .. i, 1}) 45 | end 46 | 47 | local db = couch.open(dbname, true) 48 | 49 | db:save_bulk(t) 50 | db:commit() 51 | 52 | check_table(db, t) 53 | end 54 | 55 | testlib.run_test("Explicit bulk test", test_explicit) 56 | testlib.run_test("Big bulk test", large_txn_bulk_test) 57 | return testlib.fail_count() 58 | -------------------------------------------------------------------------------- /tests/changecount.py: -------------------------------------------------------------------------------- 1 | from couchstore import CouchStore, DocumentInfo 2 | from tempfile import mkdtemp 3 | import os 4 | import os.path as path 5 | import struct 6 | import unittest 7 | 8 | class ChangeCountTest(unittest.TestCase): 9 | def setUp(self): 10 | self.tmpdir = mkdtemp() 11 | self.dbname = path.join(self.tmpdir, "testing.couch") 12 | self.db = CouchStore(self.dbname, 'c'); 13 | 14 | def tearDown(self): 15 | try: 16 | self.db.commit() 17 | self.db.close() 18 | except: 19 | pass 20 | try: 21 | os.remove(self.dbname) 22 | except: 23 | pass 24 | try: 25 | os.rmdir(self.tmpdir) 26 | except: 27 | pass 28 | 29 | def bulkSet(self, prefix, n): 30 | ids = [prefix + str(x) for x in xrange(n)] 31 | datas = ["val" + str(x) for x in xrange(n)] 32 | self.db.saveMultiple(ids, datas) 33 | 34 | def testRewind(self): 35 | # Save some docs 36 | self.db.save("foo1", "bar") 37 | self.db.save("foo2", "baz") 38 | self.db.save("foo3", "bell") 39 | self.db.save("foo4", "a") 40 | self.assertEqual(self.db.changesCount(0,100), 4) 41 | 42 | self.db.save("foo1", "new_bar") 43 | self.db.save("foo2", "new_baz") 44 | self.db.save("foo3", "new_bell") 45 | self.db.save("foo4", "new_a") 46 | self.assertEqual(self.db.changesCount(0,100), 4) 47 | 48 | self.bulkSet("foo", 100) 49 | self.assertEqual(self.db.changesCount(0, 108), 100) 50 | self.assertEqual(self.db.changesCount(0, 100), 92) 51 | self.assertEqual(self.db.changesCount(1, 100), 92) 52 | self.assertNotEqual(self.db.changesCount(12, 100), 92) 53 | self.assertEqual(self.db.changesCount(50, 99), 50) 54 | self.assertEqual(self.db.changesCount(50, 100), 51) 55 | self.assertEqual(self.db.changesCount(50, 108), 59) 56 | self.assertEqual(self.db.changesCount(51, 100), 50) 57 | self.assertEqual(self.db.changesCount(91, 1000), 18) 58 | self.db.save("foo88", "tval") 59 | self.assertEqual(self.db.changesCount(50, 108), 58) 60 | self.assertEqual(self.db.changesCount(50, 109), 59) 61 | 62 | 63 | if __name__ == '__main__': 64 | unittest.main() 65 | -------------------------------------------------------------------------------- /tests/changessincefilter.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function test_filter_deletes(dbname) 5 | local t = { 6 | {"k1", "value 1", 1,}, 7 | {"k2", "value 2", 1,}, 8 | {"k3", "value 3", 1,}, 9 | {"k4", "value 4", 1,}, 10 | {"k5", "value 5", 1,} 11 | } 12 | 13 | local db = couch.open(dbname, true) 14 | 15 | for i,v in ipairs(t) do 16 | local k = v[1] 17 | testlib.assert_no_doc(db, k) 18 | end 19 | 20 | db:save_bulk(t) 21 | db:commit() 22 | 23 | db:delete("k1") 24 | db:delete("k2") 25 | db:commit() 26 | 27 | local found = 0 28 | db:changes(0, 4, function(di) found = found + 1 end) 29 | if found ~= 3 then 30 | error("Expected " .. 3 .. " results, found " .. found) 31 | end 32 | 33 | db:close() 34 | end 35 | 36 | function test_filter_mutations(dbname) 37 | local t = { 38 | {"k1", "value 1", 1,}, 39 | {"k2", "value 2", 1,}, 40 | {"k3", "value 3", 1,}, 41 | {"k4", "value 4", 1,}, 42 | {"k5", "value 5", 1,} 43 | } 44 | 45 | local db = couch.open(dbname, true) 46 | 47 | for i,v in ipairs(t) do 48 | local k = v[1] 49 | testlib.assert_no_doc(db, k) 50 | end 51 | 52 | db:save_bulk(t) 53 | db:commit() 54 | 55 | db:delete("k1") 56 | db:delete("k2") 57 | db:commit() 58 | 59 | local found = 0 60 | db:changes(0, 2, function(di) found = found + 1 end) 61 | if found ~= 2 then 62 | error("Expected " .. 2 .. " results, found " .. found) 63 | end 64 | 65 | db:close() 66 | end 67 | 68 | testlib.run_test("Filter deletes", test_filter_deletes) 69 | testlib.run_test("Filter mutations", test_filter_mutations) 70 | return testlib.fail_count() 71 | -------------------------------------------------------------------------------- /tests/compact.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | testlib.check_doc(db, k, v[2]) 8 | end 9 | 10 | local found = 0 11 | db:changes(0, 0, function(di) found = found + 1 end) 12 | if found ~= #t then 13 | error("Expected " .. #t .. " results, found " .. found) 14 | end 15 | end 16 | 17 | function instuff(db, num, rev) 18 | local t = {} 19 | for i = 0, num, 1 do 20 | table.insert(t, {"k" .. i, "value " .. i, rev}) 21 | end 22 | db:save_bulk(t) 23 | end 24 | 25 | function insdata(db) 26 | instuff(db, 50000, 1) 27 | instuff(db, 25000, 2) 28 | instuff(db, 100000, 3) 29 | instuff(db, 80000, 4) 30 | instuff(db, 75000, 5) 31 | db:save_local("_local/localtest", "Local doc") 32 | db:save_local("_local/localtest2", "Another local doc") 33 | db:commit() 34 | end 35 | 36 | function into_table(db) 37 | local t = {} 38 | db:changes(0, 0, function(di) 39 | table.insert(t, {di:id(), db:get_from_docinfo(di), di:rev()}) 40 | end) 41 | return t 42 | end 43 | 44 | function compaction_test(dbname) 45 | local db = couch.open(dbname, true) 46 | outfile = os.tmpname() 47 | insdata(db) 48 | local origdat = into_table(db) 49 | os.execute("./couch_compact " .. dbname .. " " .. outfile) 50 | local newdb = couch.open(outfile, false) 51 | check_table(newdb, origdat) 52 | testlib.check_local_doc(newdb, "_local/localtest", "Local doc") 53 | testlib.check_local_doc(newdb, "_local/localtest2", "Another local doc") 54 | db:close() 55 | newdb:close() 56 | end 57 | 58 | testlib.run_test("Compaction test", compaction_test) 59 | os.remove(outfile) 60 | return testlib.fail_count() 61 | -------------------------------------------------------------------------------- /tests/corrupt.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function simpletrunc(dbname) 5 | local db = couch.open(dbname, true) 6 | db:save("k", "original value", 1) 7 | db:commit() 8 | 9 | testlib.check_doc(db, "k", "original value") 10 | 11 | db:save("k", "second value", 1) 12 | db:commit() 13 | 14 | testlib.check_doc(db, "k", "second value") 15 | 16 | db:truncate(-10) 17 | db:close() 18 | 19 | db = couch.open(dbname) 20 | testlib.check_doc(db, "k", "original value") 21 | 22 | db:save("k", "third value", 1) 23 | db:commit() 24 | db:close() 25 | 26 | db = couch.open(dbname) 27 | testlib.check_doc(db, "k", "third value") 28 | end 29 | 30 | function do_mangle(dbname, offset) 31 | local db = couch.open(dbname, true) 32 | db:save("k", "original value", 1) 33 | db:commit() 34 | 35 | testlib.check_doc(db, "k", "original value") 36 | 37 | db:save("k", "second value", 1) 38 | db:commit() 39 | 40 | testlib.check_doc(db, "k", "second value") 41 | 42 | db:close() 43 | 44 | local f = io.open(dbname, "r+") 45 | local corruptWith = "here be some garbage" 46 | local treelen = f:seek("end") % 4096 47 | if offset > treelen then 48 | return false 49 | end 50 | f:seek("end", 0 - treelen) 51 | f:write(corruptWith) 52 | f:close() 53 | 54 | db = couch.open(dbname) 55 | testlib.check_doc(db, "k", "original value") 56 | 57 | db:save("k", "third value", 1) 58 | db:commit() 59 | db:close() 60 | 61 | db = couch.open(dbname) 62 | testlib.check_doc(db, "k", "third value") 63 | return true 64 | end 65 | 66 | -- Mangle the header with some arbitrary data from the beginning of 67 | -- the header boundary to the end. The header is located by the EOF % 68 | -- 4096. Should probably do something special when EOF *is* at a 4k 69 | -- boundary, but this test is valuable without it. 70 | function header_mangling(dbname) 71 | local i = 0 72 | while do_mangle(dbname, i) do 73 | i = i + 1 74 | os.remove(dbname) 75 | end 76 | end 77 | 78 | testlib.run_test("Simple truncation test", simpletrunc) 79 | testlib.run_test("Various mangling of headers", header_mangling) 80 | return testlib.fail_count() 81 | -------------------------------------------------------------------------------- /tests/couchstoredoctest.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "couchstoretest.h" 21 | 22 | #include 23 | 24 | /* 25 | CouchstoreDoctest 26 | - Subclass that enables parameterised testing 27 | */ 28 | class CouchstoreDoctest : public CouchstoreTest, 29 | public ::testing::WithParamInterface > { 30 | }; 31 | -------------------------------------------------------------------------------- /tests/couchstoretest.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #pragma once 18 | 19 | #include "config.h" 20 | #include "test_fileops.h" 21 | #include "documents.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | /* 32 | CouchstoreTest 33 | * Global test class for most of the couchstore tests. 34 | * Auto-cleans when the test is complete. 35 | a) If db is not null, closes the db 36 | b) removes testfile.couch. 37 | */ 38 | class CouchstoreTest : public ::testing::Test { 39 | protected: 40 | CouchstoreTest(); 41 | CouchstoreTest(const std::string& _filePath, 42 | const bool _display_latency_info = false); 43 | 44 | virtual ~CouchstoreTest(); 45 | void clean_up(); 46 | 47 | Db* db; 48 | std::string filePath; 49 | bool displayLatencyInfo; 50 | }; 51 | 52 | /* 53 | * Global test class for internal only tests. Extends CouchstoreTest. 54 | */ 55 | class CouchstoreInternalTest : public CouchstoreTest { 56 | protected: 57 | CouchstoreInternalTest(); 58 | virtual ~CouchstoreInternalTest(); 59 | 60 | /** 61 | * Opens a database instance with the current filePath, ops and with 62 | * buffering disabled. 63 | * 64 | * @param extra_flags Any additional flags, other than 65 | * COUCHSTORE_OPEN_FLAG_UNBUFFERED to open the db with. 66 | */ 67 | couchstore_error_t open_db(couchstore_open_flags extra_flags); 68 | 69 | /** 70 | * Opens a database instance with the current filePath, ops and with 71 | * buffering disabled. It then populates the database with the 72 | * specified number of documents. 73 | * 74 | * @param extra_flags Any additional flags, other than 75 | * COUCHSTORE_OPEN_FLAG_UNBUFFERED to open the db with. 76 | * @param count Number of documents to populate with 77 | */ 78 | void open_db_and_populate(couchstore_open_flags extra_flags, size_t count); 79 | 80 | /** 81 | * Creates a LocalDoc object from two strings 82 | * 83 | * Note: The localDoc will just point to strings' memory 84 | * so the strings should stay alive as long as the LocalDoc 85 | * does. 86 | * 87 | * @param id ID of the document 88 | * @param json Body of the document 89 | */ 90 | LocalDoc create_local_doc(std::string& id, std::string& json); 91 | 92 | std::string compactPath; 93 | Documents documents; 94 | ::testing::NiceMock ops; 95 | DocInfo* info; 96 | Doc* doc; 97 | }; 98 | 99 | /** 100 | * Multi-threaded test. 101 | */ 102 | class CouchstoreMTTest 103 | : public ::testing::Test, 104 | public ::testing::WithParamInterface > { 105 | protected: 106 | CouchstoreMTTest(); 107 | CouchstoreMTTest(const std::string& _filePath); 108 | 109 | void TearDown(); 110 | 111 | size_t numThreads; 112 | std::vector dbs; 113 | std::string filePath; 114 | }; 115 | 116 | /** 117 | * Test class for error injection tests 118 | */ 119 | typedef CouchstoreInternalTest FileOpsErrorInjectionTest; 120 | 121 | /** 122 | * Parameterised test class for error injection tests 123 | */ 124 | class ParameterisedFileOpsErrorInjectionTest : public FileOpsErrorInjectionTest, 125 | public ::testing::WithParamInterface { 126 | }; 127 | -------------------------------------------------------------------------------- /tests/documents.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | Class to assist testing of couchstore. 20 | 21 | Documents represents a set of Doc/DocInfo objects allowing convenient management 22 | of the underlying objects. 23 | 24 | **/ 25 | 26 | #pragma once 27 | 28 | #include "config.h" 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | class Documents { 36 | 37 | public: 38 | Documents(int n_docs); 39 | 40 | /** 41 | Set document at index with id and data. 42 | Note: null terminator of both id/data strings is not stored. 43 | **/ 44 | void setDoc(int index, const std::string& id, const std::string& data); 45 | 46 | /** 47 | shuffle the documents so they're no longer in the order setDoc indicated. 48 | **/ 49 | void shuffle(); 50 | 51 | /** 52 | Just generate documents. 53 | Key is doc 54 | Document is doc-data 55 | **/ 56 | void generateDocs(); 57 | 58 | void setContentMeta(int index, int flag); 59 | 60 | Doc** getDocs(); 61 | 62 | DocInfo** getDocInfos(); 63 | 64 | Doc* getDoc(int index); 65 | 66 | DocInfo* getDocInfo(int index); 67 | 68 | int getDocsCount() const; 69 | 70 | int getDocInfosCount() const; 71 | 72 | int getDeleted() const; 73 | 74 | int getCallbacks() const; 75 | 76 | int getPosition() const; 77 | 78 | void resetCounters(); 79 | 80 | /** 81 | Update the document map with the key. 82 | Expects the key to not exist (uses gtest EXPECT macro) 83 | **/ 84 | void updateDocumentMap(const std::string& key); 85 | 86 | void clearDocumentMap(); 87 | 88 | /** 89 | Couchstore callback method that checks the document against 90 | the orginal Documents data. 91 | **/ 92 | static int checkCallback(Db* db, DocInfo* info, void* ctx); 93 | 94 | /** 95 | Couchstore callback method that just counts the number of callbacks. 96 | **/ 97 | static int countCallback(Db* db, DocInfo* info, void* ctx); 98 | 99 | /** 100 | Couchstore callback method that checks the document can be opened. 101 | - Also counts callbacks and deleted documents. 102 | **/ 103 | static int docIterCheckCallback(Db *db, DocInfo *info, void *ctx); 104 | 105 | /** 106 | Couchstore callback that updates a set of document keys. 107 | - The update call expects the document to not exist in the set. 108 | **/ 109 | static int docMapUpdateCallback(Db *db, DocInfo *info, void *ctx); 110 | 111 | private: 112 | 113 | void incrementCallbacks(); 114 | 115 | void incrementDeleted(); 116 | 117 | void incrementPosition(); 118 | /** 119 | Inner class storing the data for one document. 120 | **/ 121 | class Document { 122 | public: 123 | 124 | Document(); 125 | 126 | ~Document(); 127 | 128 | /* Init a document */ 129 | void init(const std::string& id, const std::string& data, const std::vector& meta); 130 | 131 | /* Init a document with default 'zero' meta */ 132 | void init(const std::string& id, const std::string& data); 133 | 134 | void setContentMeta(int flag); 135 | 136 | private: 137 | friend Documents; 138 | 139 | Doc* getDocPointer(); 140 | 141 | DocInfo* getDocInfoPointer(); 142 | 143 | Doc doc; 144 | DocInfo docInfo; 145 | std::vector documentId; 146 | std::vector documentData; 147 | std::vector documentMeta; 148 | static std::vector zeroMeta; 149 | }; 150 | 151 | // Documents private data. 152 | std::vector docs; 153 | std::vector docInfos; 154 | std::vector documents; 155 | std::set documentMap; 156 | 157 | // Counters for the callbacks 158 | int deleted; 159 | int callbacks; 160 | int position; 161 | }; -------------------------------------------------------------------------------- /tests/dropdel.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local function check_table(db, t) 5 | for i,v in ipairs(t) do 6 | local k = v[1] 7 | if k ~= "k99" and k ~= "k200" then 8 | testlib.check_doc(db, k, v[2]) 9 | else 10 | testlib.assert_no_doc(db, k) 11 | end 12 | end 13 | 14 | local found = 0 15 | db:changes(0, 0, function(di) found = found + 1 end) 16 | if found ~= (#t - 2) then 17 | error("Expected " .. (#t - 2) .. " results, found " .. found) 18 | end 19 | end 20 | 21 | function instuff(db, num, rev) 22 | local t = {} 23 | for i = 0, num, 1 do 24 | table.insert(t, {"k" .. i, "value " .. i, rev}) 25 | end 26 | db:save_bulk(t) 27 | end 28 | 29 | function insdata(db) 30 | instuff(db, 50000, 1) 31 | instuff(db, 25000, 2) 32 | instuff(db, 100000, 3) 33 | instuff(db, 80000, 4) 34 | instuff(db, 75000, 5) 35 | db:save_local("_local/localtest", "Local doc") 36 | db:save_local("_local/localtest2", "Another local doc") 37 | db:commit() 38 | end 39 | 40 | function into_table(db) 41 | local t = {} 42 | db:changes(0, 0, function(di) 43 | table.insert(t, {di:id(), db:get_from_docinfo(di), di:rev()}) 44 | end) 45 | return t 46 | end 47 | 48 | function compaction_test(dbname) 49 | local db = couch.open(dbname, true) 50 | outfile = os.tmpname() 51 | insdata(db) 52 | db:delete("k99") 53 | db:delete("k200") 54 | db:commit() 55 | local origdat = into_table(db) 56 | os.execute("./couch_compact --dropdeletes " .. dbname .. " " .. outfile) 57 | local newdb = couch.open(outfile, false) 58 | check_table(newdb, origdat) 59 | testlib.check_local_doc(newdb, "_local/localtest", "Local doc") 60 | testlib.check_local_doc(newdb, "_local/localtest2", "Another local doc") 61 | db:close() 62 | newdb:close() 63 | end 64 | 65 | testlib.run_test("Deletion dropping test", compaction_test) 66 | os.remove(outfile) 67 | return testlib.fail_count() 68 | -------------------------------------------------------------------------------- /tests/large.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | local alphabet_str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 5 | local alphabet = {} 6 | for i = 1, #alphabet_str do 7 | alphabet[i] = alphabet_str:sub(i, i) 8 | end 9 | 10 | 11 | local function check_table(db, t) 12 | for i,v in ipairs(t) do 13 | local k = v[1] 14 | testlib.check_doc(db, k, v[2]) 15 | end 16 | 17 | local found = 0 18 | db:changes(0, 0, function(di) found = found + 1 end) 19 | if found ~= #t then 20 | error("Expected " .. #t .. " results, found " .. found) 21 | end 22 | end 23 | 24 | function make_val(size) 25 | local chars = {} 26 | for i = 1, size do 27 | chars[i] = alphabet[math.random(1, #alphabet)] 28 | end 29 | return table.concat(chars) 30 | end 31 | 32 | local testsample = make_val(40000) 33 | 34 | function test_big_bulk(dbname) 35 | print "Making data set..." 36 | local t = {} 37 | for i = 0, 40000, 5 do 38 | local k = "k" .. i 39 | local v = testsample:sub(i) 40 | t[#t + 1] = {k, v, 1 } 41 | end 42 | 43 | print "Saving data set..." 44 | local db = couch.open(dbname, true) 45 | db:save_bulk(t) 46 | db:commit() 47 | 48 | print "Checking data set..." 49 | check_table(db, t) 50 | 51 | db:close() 52 | 53 | local db = couch.open(dbname) 54 | 55 | check_table(db, t) 56 | end 57 | 58 | function test_big_sequential(dbname) 59 | print "Making & saving data set..." 60 | local db = couch.open(dbname, true) 61 | 62 | local t = {} 63 | for i = 0, 40000, 5 do 64 | local k = "k" .. i 65 | local v = testsample:sub(i) 66 | t[#t + 1] = {k, v, 1 } 67 | db:save(k, v, 1) 68 | end 69 | 70 | db:commit() 71 | 72 | print "Checking data set..." 73 | check_table(db, t) 74 | 75 | db:close() 76 | 77 | local db = couch.open(dbname) 78 | 79 | check_table(db, t) 80 | end 81 | 82 | testlib.run_test("Big item test bulk", test_big_bulk) 83 | testlib.run_test("Big item test sequential", test_big_sequential) 84 | return testlib.fail_count() 85 | -------------------------------------------------------------------------------- /tests/largefile.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function bigdb(dbname) 5 | local db = couch.open(dbname, true) 6 | db:truncate(5 * 1024 * 1024 * 1024) 7 | db:close() 8 | 9 | db = couch.open(dbname) 10 | testlib.assert_no_doc(db, "k") 11 | 12 | db:save("k", "a value", 1) 13 | db:commit() 14 | db:close() 15 | 16 | db = couch.open(dbname) 17 | testlib.check_doc(db, "k", "a value") 18 | end 19 | 20 | testlib.run_test("A Big Database", bigdb) 21 | return testlib.fail_count() 22 | -------------------------------------------------------------------------------- /tests/localdoc.lua: -------------------------------------------------------------------------------- 1 | package.path = package.path .. ";tests/?.lua" 2 | local testlib = require("testlib") 3 | 4 | function localtest(dbname) 5 | local key = "a_local_key" 6 | local value = "the local value" 7 | local value2 = "an updated value" 8 | 9 | local db = couch.open(dbname, true) 10 | 11 | testlib.assert_no_local_doc(key) 12 | 13 | db:save_local(key, value) 14 | db:commit() 15 | 16 | testlib.assert_no_doc(key) 17 | 18 | db:close() 19 | 20 | db = couch.open(dbname) 21 | 22 | testlib.check_local_doc(db, key, value) 23 | 24 | db:save_local(key, value2) 25 | testlib.check_local_doc(db, key, value2) 26 | db:commit() 27 | testlib.check_local_doc(db, key, value2) 28 | 29 | db:close() 30 | 31 | db = couch.open(dbname) 32 | testlib.check_local_doc(db, key, value2) 33 | 34 | db:changes(0, 0, function(di) error("Unexpectedly got a doc: " .. di:id()) end) 35 | 36 | -- Store a non-local document and verify it doesn't collide 37 | db:save(key, "non local", 1) 38 | testlib.check_local_doc(db, key, value2) 39 | testlib.check_doc(db, key, "non local") 40 | 41 | db:delete_local(key) 42 | db:commit() 43 | 44 | testlib.check_doc(db, key, "non local") 45 | testlib.assert_no_local_doc(key) 46 | 47 | end 48 | 49 | testlib.run_test("Local doc test", localtest) 50 | return testlib.fail_count() 51 | -------------------------------------------------------------------------------- /tests/macros.h: -------------------------------------------------------------------------------- 1 | #ifndef TESTS_MACROS_H 2 | #define TESTS_MACROS_H 3 | 4 | /** Custom assert() macro -- can't use the one in because it's disabled when 5 | NDEBUG is defined. */ 6 | #undef assert 7 | #define assert(expr) \ 8 | do { \ 9 | if (!(expr)) { \ 10 | fprintf(stderr, "%s:%d: assertion failed\n", \ 11 | __FILE__, __LINE__); \ 12 | fflush(stderr); \ 13 | abort(); \ 14 | } \ 15 | } while (0) 16 | 17 | #define assert_eq(A, B) \ 18 | do { \ 19 | if ((A) != (B)) { \ 20 | fprintf(stderr, "%s:%d: assertion failed: %s != %s\n", \ 21 | __FILE__, __LINE__, #A, #B); \ 22 | fflush(stderr); \ 23 | abort(); \ 24 | } \ 25 | } while (0) 26 | 27 | #define try_to(C) \ 28 | do { \ 29 | if ((errcode = (C)) < 0) { \ 30 | fprintf(stderr, \ 31 | "Couchstore error `%s' at %s:%d\r\n", \ 32 | couchstore_strerror((couchstore_error_t)errcode), \ 33 | __FILE__, \ 34 | __LINE__); \ 35 | goto cleanup; \ 36 | } \ 37 | } while (0) 38 | 39 | #define error_unless(C, E) \ 40 | do { \ 41 | if(!(C)) { try(E); } \ 42 | } while (0) 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /tests/mapreduce/mapreduce_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _MAPREDUCE_TESTS_H 22 | #define _MAPREDUCE_TESTS_H 23 | 24 | #include "config.h" 25 | 26 | #include 27 | #include 28 | #include 29 | #include "../macros.h" 30 | #include "../../src/views/mapreduce/mapreduce.h" 31 | 32 | /* main test function */ 33 | void mapreduce_tests(void); 34 | void map_tests(void); 35 | void reduce_tests(void); 36 | void builtin_tests(void); 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /tests/purge.py: -------------------------------------------------------------------------------- 1 | from couchstore import CouchStore, DocumentInfo 2 | from tempfile import mkdtemp 3 | import os 4 | import os.path as path 5 | import struct 6 | import unittest 7 | 8 | REV_META_PACK = ">QII" 9 | 10 | def deleteAt(db, key, time): 11 | info = db.getInfo(key) 12 | # cas, exp, flags 13 | info.revMeta = str(struct.pack(REV_META_PACK, 0, time, 0)) 14 | info.deleted = True 15 | return db.save(info, "") 16 | 17 | class PurgeTest(unittest.TestCase): 18 | def setUp(self): 19 | self.tmpdir = mkdtemp() 20 | self.origname = path.join(self.tmpdir, "orig.couch") 21 | self.purgedname = path.join(self.tmpdir, "purged.couch") 22 | self.origdb = CouchStore(self.origname, 'c'); 23 | 24 | def tearDown(self): 25 | try: 26 | self.origdb.close() 27 | except: 28 | pass 29 | try: 30 | self.newdb.close() 31 | except: 32 | pass 33 | try: 34 | os.remove(self.origname) 35 | except: 36 | pass 37 | try: 38 | os.remove(self.purgedname) 39 | except: 40 | pass 41 | try: 42 | os.rmdir(self.tmpdir) 43 | except: 44 | pass 45 | 46 | def testPurgeCompact(self): 47 | # Save some docs 48 | self.origdb.save("foo1", "bar") 49 | self.origdb.save("foo2", "baz") 50 | self.origdb.save("foo3", "bell") 51 | self.origdb.save("foo4", "a") 52 | 53 | # Delete some 54 | seqPurged = deleteAt(self.origdb, "foo2", 10) 55 | seqKept = deleteAt(self.origdb, "foo3", 20) 56 | seqLateDelete = deleteAt(self.origdb, "foo4", 11) 57 | self.origdb.commit() 58 | 59 | os.system(path.join(os.getcwd(), "couch_compact") + " --purge-before 15 " + 60 | self.origname + " " + self.purgedname) 61 | self.newdb = CouchStore(self.purgedname) 62 | 63 | # Check purged item is not present in key tree and kept item is 64 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 65 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 66 | self.assertRaises(KeyError, self.newdb.getInfo, "foo4") 67 | 68 | self.newdb.close() 69 | 70 | os.system(path.join(os.getcwd(), "couch_compact") + 71 | " --purge-before 15 --purge-only-upto-seq " + str(seqKept) + 72 | " " + self.origname + " " + self.purgedname) 73 | self.newdb = CouchStore(self.purgedname) 74 | 75 | # Check purged item is not present in key tree and kept item is 76 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 77 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 78 | # with purge-only-upto-seq just before deletion of foo4 we 79 | # must find it after compaction 80 | self.assertIsNotNone(self.newdb.getInfo("foo4")) 81 | 82 | self.newdb.close() 83 | 84 | os.system(path.join(os.getcwd(), "couch_compact") + 85 | " --purge-before 15 --purge-only-upto-seq " + str(seqLateDelete) + 86 | " " + self.origname + " " + self.purgedname) 87 | self.newdb = CouchStore(self.purgedname) 88 | 89 | # Check purged item is not present in key tree and kept item is 90 | self.assertRaises(KeyError, self.newdb.getInfo, "foo2") 91 | self.assertIsNotNone(self.newdb.getInfo("foo3")) 92 | # with purge-only-upto-seq just at deletion of foo4 we 93 | # must not find it after compaction 94 | self.assertRaises(KeyError, self.newdb.getInfo, "foo4") 95 | 96 | # Check purged item is not present in seq tree and kept item is 97 | self.assertRaises(KeyError, self.newdb.getInfoBySequence, seqPurged) 98 | self.assertIsNotNone(self.newdb.getInfoBySequence(seqKept)) 99 | 100 | 101 | if __name__ == '__main__': 102 | unittest.main() 103 | -------------------------------------------------------------------------------- /tests/rewind.py: -------------------------------------------------------------------------------- 1 | from couchstore import CouchStore, DocumentInfo 2 | from tempfile import mkdtemp 3 | import os 4 | import os.path as path 5 | import struct 6 | import unittest 7 | 8 | class RewindTest(unittest.TestCase): 9 | def setUp(self): 10 | self.tmpdir = mkdtemp() 11 | self.dbname = path.join(self.tmpdir, "testing.couch") 12 | self.db = CouchStore(self.dbname, 'c'); 13 | 14 | def tearDown(self): 15 | try: 16 | self.db.close() 17 | except: 18 | pass 19 | try: 20 | os.remove(self.dbname) 21 | except: 22 | pass 23 | try: 24 | os.rmdir(self.tmpdir) 25 | except: 26 | pass 27 | 28 | def testRewind(self): 29 | # Save some docs 30 | self.db.save("foo1", "bar") 31 | self.db.save("foo2", "baz") 32 | self.db.save("foo3", "bell") 33 | self.db.save("foo4", "a") 34 | self.db.commit() 35 | 36 | # Edit some docs 37 | self.db.save("foo1", "new_bar") 38 | self.db.save("foo2", "new_baz") 39 | self.db.save("foo3", "new_bell") 40 | self.db.save("foo4", "new_a") 41 | self.db.commit() 42 | 43 | # The edits happened... 44 | self.assertNotEqual(self.db["foo3"], "bell"); 45 | self.assertEqual(self.db["foo4"], "new_a"); 46 | 47 | # rewind 48 | self.db.rewindHeader() 49 | 50 | # did we go back in time? 51 | self.assertEqual(self.db["foo3"], "bell"); 52 | self.assertNotEqual(self.db["foo4"], "new_a"); 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /tests/testapp.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | extern void mapreduce_init(); 19 | extern void mapreduce_deinit(); 20 | extern void view_tests(); 21 | extern void purge_tests(); 22 | 23 | int main() { 24 | mapreduce_init(); 25 | 26 | view_tests(); 27 | purge_tests(); 28 | 29 | mapreduce_deinit(); 30 | return 0; 31 | } 32 | -------------------------------------------------------------------------------- /tests/testlib.lua: -------------------------------------------------------------------------------- 1 | local M = {} 2 | local failures = 0 3 | 4 | -- Verify a doc with the given key exists with the given value in the 5 | -- given database. 6 | function M.check_doc(db, key, value) 7 | local doc = db:get(key) 8 | if doc ~= value then 9 | error(string.format("Expected '%s' (%d bytes), got '%s' (%d bytes)", 10 | value, #value, doc, #doc)) 11 | end 12 | end 13 | 14 | -- Verify a local doc with the given key exists with the given value 15 | -- in the given database. 16 | function M.check_local_doc(db, key, value) 17 | local doc = db:get_local(key) 18 | if doc ~= value then 19 | error(string.format("Expected '%s' (%d bytes), got '%s' (%d bytes)", 20 | value, #value, doc, #doc)) 21 | end 22 | end 23 | 24 | -- Verify a document does not exist. 25 | function M.assert_no_doc(db, key) 26 | local succeded, result = pcall(db['get'], db, key) 27 | if succeeded then 28 | error("Expected missing doc for " .. key .. " got " .. result) 29 | end 30 | end 31 | 32 | -- Verify a local doc does not exist. 33 | function M.assert_no_local_doc(db, key) 34 | local succeded, result = pcall(db['get_local'], db, key) 35 | if succeeded then 36 | error("Expected missing local doc for " .. key .. " got " .. result) 37 | end 38 | end 39 | 40 | -- Run a named test function. 41 | -- 42 | -- The function will receive a filename it should use for the 43 | -- database. The file will be removed at the end of the test run. If 44 | -- the test function runs without error, the test is considered 45 | -- successful. 46 | function M.run_test(name, fun) 47 | local dbname = os.tmpname() 48 | 49 | local succeeded, result = pcall(fun, dbname) 50 | 51 | os.remove(dbname) 52 | 53 | if succeeded then 54 | print(name .. ": PASS") 55 | else 56 | print(name .. ": FAIL (" .. result .. ")") 57 | failures = failures + 1 58 | end 59 | end 60 | 61 | function M.fail_count() 62 | return failures 63 | end 64 | 65 | return M 66 | -------------------------------------------------------------------------------- /tests/views/bitmaps.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "view_tests.h" 22 | 23 | void test_bitmaps(void) 24 | { 25 | bitmap_t bm, bm1, bm2; 26 | uint16_t one_bits[] = {1023, 1013, 500, 401, 1, 7, 666, 69}; 27 | int set; 28 | uint16_t i, j; 29 | 30 | fprintf(stderr, "Running view bitmap tests\n"); 31 | memset(&bm, 0, sizeof(bitmap_t)); 32 | for (j = 0; j < (sizeof(one_bits) / sizeof(uint16_t)); ++j) { 33 | set_bit(&bm, one_bits[j]); 34 | } 35 | 36 | for (i = 0; i < 1024; ++i) { 37 | set = 0; 38 | for (j = 0; j < (sizeof(one_bits) / sizeof(uint16_t)); ++j) { 39 | if (one_bits[j] == i) { 40 | set = 1; 41 | break; 42 | } 43 | } 44 | 45 | if (set) { 46 | cb_assert(is_bit_set(&bm, i)); 47 | } else { 48 | cb_assert(!is_bit_set(&bm, i)); 49 | } 50 | 51 | unset_bit(&bm, i); 52 | cb_assert(!is_bit_set(&bm, i)); 53 | } 54 | 55 | memset(&bm, 0, sizeof(bitmap_t)); 56 | set_bit(&bm, 1023); 57 | set_bit(&bm, 514); 58 | set_bit(&bm, 0); 59 | for (i = 0; i < (sizeof((bm.chunks)) / sizeof(bm.chunks[0])); ++i) { 60 | switch (i) { 61 | case 0: 62 | cb_assert(bm.chunks[i] == 0x80); 63 | break; 64 | case 63: 65 | cb_assert(bm.chunks[i] == 0x04); 66 | break; 67 | case 127: 68 | cb_assert(bm.chunks[i] == 0x01); 69 | break; 70 | default: 71 | cb_assert(bm.chunks[i] == 0); 72 | } 73 | } 74 | 75 | memset(&bm1, 0, sizeof(bitmap_t)); 76 | memset(&bm2, 0, sizeof(bitmap_t)); 77 | set_bit(&bm1, 1023); 78 | set_bit(&bm2, 0); 79 | 80 | union_bitmaps(&bm1, &bm2); 81 | cb_assert(bm1.chunks[0] == 0x80); 82 | cb_assert(bm1.chunks[127] == 0x01); 83 | 84 | /* Tests for intersection operation */ 85 | memset(&bm1, 0, sizeof(bitmap_t)); 86 | memset(&bm2, 0, sizeof(bitmap_t)); 87 | set_bit(&bm1, 0); 88 | set_bit(&bm1, 7); 89 | set_bit(&bm2, 800); 90 | set_bit(&bm2, 801); 91 | intersect_bitmaps(&bm1, &bm2); 92 | cb_assert(bm1.chunks[0] == 0x0); 93 | cb_assert(bm1.chunks[100] == 0x0); 94 | 95 | set_bit(&bm1, 0); 96 | set_bit(&bm1, 1023); 97 | set_bit(&bm2, 7); 98 | set_bit(&bm2, 1023); 99 | 100 | intersect_bitmaps(&bm1, &bm2); 101 | cb_assert(bm1.chunks[0] == 0x80); 102 | cb_assert(bm1.chunks[127] == 0x0); 103 | 104 | /* Tests for is_equal operation */ 105 | memset(&bm1, 0, sizeof(bitmap_t)); 106 | memset(&bm2, 0, sizeof(bitmap_t)); 107 | cb_assert(is_equal_bitmap(&bm1, &bm2)); 108 | set_bit(&bm1, 7); 109 | set_bit(&bm1, 500); 110 | set_bit(&bm2, 7); 111 | set_bit(&bm2, 500); 112 | cb_assert(is_equal_bitmap(&bm1, &bm2)); 113 | set_bit(&bm2, 1000); 114 | cb_assert(!is_equal_bitmap(&bm1, &bm2)); 115 | 116 | } 117 | -------------------------------------------------------------------------------- /tests/views/keys.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | #include "view_tests.h" 4 | 5 | #include 6 | 7 | 8 | static view_btree_key_t *test_view_btree_key_decoding(const char *key_bin, size_t len) 9 | { 10 | view_btree_key_t *k = NULL; 11 | 12 | cb_assert(decode_view_btree_key(key_bin, len, &k) == COUCHSTORE_SUCCESS); 13 | cb_assert(k != NULL); 14 | cb_assert(k->json_key.size == 4); 15 | cb_assert(memcmp(k->json_key.buf, "\"23\"", k->json_key.size) == 0); 16 | cb_assert(k->doc_id.size == 12); 17 | cb_assert(memcmp(k->doc_id.buf, "doc_00000023", k->doc_id.size) == 0); 18 | 19 | return k; 20 | } 21 | 22 | static view_id_btree_key_t *test_view_id_btree_key_decoding(const char *id_btree_key_bin, size_t len) 23 | { 24 | view_id_btree_key_t *k = NULL; 25 | 26 | cb_assert(decode_view_id_btree_key(id_btree_key_bin, len, &k) == COUCHSTORE_SUCCESS); 27 | cb_assert(k != NULL); 28 | 29 | cb_assert(k->partition == 57); 30 | cb_assert(k->doc_id.size == 12); 31 | cb_assert(memcmp(k->doc_id.buf, "doc_00000057", k->doc_id.size) == 0); 32 | 33 | return k; 34 | } 35 | 36 | static void test_view_btree_key_encoding(const view_btree_key_t *k, 37 | char **buffer, 38 | size_t *size) 39 | { 40 | couchstore_error_t res; 41 | 42 | res = encode_view_btree_key(k, buffer, size); 43 | cb_assert(res == COUCHSTORE_SUCCESS); 44 | } 45 | 46 | 47 | static void test_view_id_btree_key_encoding(const view_id_btree_key_t *k, 48 | char **buffer, 49 | size_t *size) 50 | { 51 | couchstore_error_t res; 52 | 53 | res = encode_view_id_btree_key(k, buffer, size); 54 | cb_assert(res == COUCHSTORE_SUCCESS); 55 | } 56 | 57 | void test_keys() 58 | { 59 | char key_bin[] = { 60 | 0,4,34,50,51,34,100,111,99,95,48,48,48,48,48,48,50,51 61 | }; 62 | char id_btree_key_bin[] = { 63 | 0,57,100,111,99,95,48,48,48,48,48,48,53,55 64 | }; 65 | char *k_bin2 = NULL; 66 | size_t k_bin2_size = 0; 67 | char *id_btree_k_bin2 = NULL; 68 | size_t id_btree_k_bin2_size = 0; 69 | char *k_bin3 = NULL; 70 | size_t k_bin3_size = 0; 71 | char *id_btree_k_bin3 = NULL; 72 | size_t id_btree_k_bin3_size = 0; 73 | view_btree_key_t *k; 74 | view_id_btree_key_t *id_btree_k; 75 | view_btree_key_t *k2; 76 | view_id_btree_key_t *id_btree_k2; 77 | 78 | fprintf(stderr, "Decoding a view btree key ...\n"); 79 | k = test_view_btree_key_decoding(key_bin, sizeof(key_bin)); 80 | 81 | fprintf(stderr, "Decoding a view id btree key ...\n"); 82 | id_btree_k = test_view_id_btree_key_decoding(id_btree_key_bin, sizeof(id_btree_key_bin)); 83 | 84 | fprintf(stderr, "Encoding the previously decoded view btree key ...\n"); 85 | test_view_btree_key_encoding(k, &k_bin2, &k_bin2_size); 86 | 87 | cb_assert(k_bin2_size == sizeof(key_bin)); 88 | cb_assert(memcmp(k_bin2, key_bin, k_bin2_size) == 0); 89 | 90 | fprintf(stderr, "Encoding the previously decoded view id btree key ...\n"); 91 | test_view_id_btree_key_encoding(id_btree_k, &id_btree_k_bin2, &id_btree_k_bin2_size); 92 | 93 | cb_assert(id_btree_k_bin2_size == sizeof(id_btree_key_bin)); 94 | cb_assert(memcmp(id_btree_k_bin2, id_btree_key_bin, id_btree_k_bin2_size) == 0); 95 | 96 | fprintf(stderr, "Decoding the previously encoded view btree key ...\n"); 97 | k2 = test_view_btree_key_decoding(k_bin2, k_bin2_size); 98 | 99 | fprintf(stderr, "Decoding the previously encoded view id btree key ...\n"); 100 | id_btree_k2 = test_view_id_btree_key_decoding(id_btree_k_bin2, id_btree_k_bin2_size); 101 | 102 | fprintf(stderr, "Encoding the previously decoded view btree key ...\n"); 103 | test_view_btree_key_encoding(k2, &k_bin3, &k_bin3_size); 104 | 105 | cb_assert(k_bin3_size == sizeof(key_bin)); 106 | cb_assert(memcmp(k_bin3, key_bin, k_bin3_size) == 0); 107 | 108 | fprintf(stderr, "Encoding the previously decoded view id btree key ...\n"); 109 | test_view_id_btree_key_encoding(id_btree_k2, &id_btree_k_bin3, &id_btree_k_bin3_size); 110 | 111 | cb_assert(id_btree_k_bin3_size == sizeof(id_btree_key_bin)); 112 | cb_assert(memcmp(id_btree_k_bin3, id_btree_key_bin, id_btree_k_bin3_size) == 0); 113 | 114 | free_view_btree_key(k); 115 | free_view_btree_key(k2); 116 | cb_free(k_bin2); 117 | cb_free(k_bin3); 118 | 119 | free_view_id_btree_key(id_btree_k); 120 | free_view_id_btree_key(id_btree_k2); 121 | cb_free(id_btree_k_bin2); 122 | cb_free(id_btree_k_bin3); 123 | } 124 | -------------------------------------------------------------------------------- /tests/views/spatial_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Volker Mische 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_SPATIAL_TESTS_H 22 | #define _VIEW_SPATIAL_TESTS_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "../macros.h" 30 | #include "../src/views/bitmap.h" 31 | #include "../src/views/spatial.h" 32 | 33 | /* Those functions are normaly static. They are declared here to prevent 34 | * compile time warning */ 35 | couchstore_error_t encode_spatial_key(const sized_mbb_t *mbb, 36 | char *key, 37 | size_t nkey); 38 | couchstore_error_t decode_spatial_key(const char *key, sized_mbb_t *mbb); 39 | couchstore_error_t expand_mbb(sized_mbb_t *original, sized_mbb_t *expander); 40 | 41 | 42 | void test_interleaving(void); 43 | void test_spatial_scale_factor(void); 44 | void test_spatial_center(void); 45 | void test_spatial_scale_point(void); 46 | void test_set_bit_sized(void); 47 | void test_encode_spatial_key(void); 48 | void test_decode_spatial_key(void); 49 | void test_expand_mbb(void); 50 | void test_view_spatial_reduce(void); 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /tests/views/tests.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #include "view_tests.h" 22 | #include "spatial_tests.h" 23 | 24 | 25 | void view_tests() 26 | { 27 | fprintf(stderr, "\n\nRunning view tests\n\n"); 28 | 29 | test_bitmaps(); 30 | test_sorted_lists(); 31 | test_collate_json(); 32 | test_index_headers_v1(); 33 | test_index_headers_v2(); 34 | test_reductions(); 35 | test_keys(); 36 | test_values(); 37 | reducer_tests(); 38 | cleanup_tests(); 39 | 40 | /* spatial tests */ 41 | test_interleaving(); 42 | test_spatial_scale_factor(); 43 | test_spatial_center(); 44 | test_spatial_scale_point(); 45 | test_set_bit_sized(); 46 | test_encode_spatial_key(); 47 | test_decode_spatial_key(); 48 | test_expand_mbb(); 49 | test_view_spatial_reduce(); 50 | } 51 | -------------------------------------------------------------------------------- /tests/views/view_tests.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | /** 4 | * @copyright 2013 Couchbase, Inc. 5 | * 6 | * @author Filipe Manana 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | **/ 20 | 21 | #ifndef _VIEW_TESTS_H 22 | #define _VIEW_TESTS_H 23 | 24 | #include "config.h" 25 | #include 26 | #include 27 | #include 28 | #include "../macros.h" 29 | #include "../src/views/bitmap.h" 30 | #include "../src/views/sorted_list.h" 31 | #include "../src/views/index_header.h" 32 | #include "../src/views/reductions.h" 33 | #include "../src/views/keys.h" 34 | #include "../src/views/values.h" 35 | #include "../src/views/reducers.h" 36 | #include "../src/views/purgers.h" 37 | 38 | #ifdef __cplusplus 39 | extern "C" { 40 | #endif 41 | 42 | void view_tests(void); 43 | void test_bitmaps(void); 44 | void test_sorted_lists(void); 45 | void test_collate_json(void); 46 | void test_index_headers_v1(void); 47 | void test_index_headers_v2(void); 48 | void test_reductions(void); 49 | void test_keys(void); 50 | void test_values(void); 51 | void reducer_tests(void); 52 | void cleanup_tests(void); 53 | 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /tests/wrapped_fileops_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "wrapped_fileops_test.h" 19 | #include "iobuffer.h" 20 | 21 | /** 22 | * This is a slightly modified version of the regular 23 | * BufferedFileOps class. 24 | * 25 | * The regular BufferedFileOps has a custom `constructor` 26 | * method outside of the FileOpsInterface. This version 27 | * uses its C++ constructor to set the File ops to be 28 | * wrapped for the `constructor` method in the interface 29 | * which allows it to share tests and bootstrapping with 30 | * other FileOps implementations. 31 | */ 32 | class TestBufferedFileOps : public BufferedFileOps { 33 | public: 34 | TestBufferedFileOps(FileOpsInterface* ops) : wrapped_ops(ops) { 35 | } 36 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override { 37 | return BufferedFileOps::constructor(errinfo, wrapped_ops.get(), 38 | buffered_file_ops_params()); 39 | } 40 | 41 | protected: 42 | std::unique_ptr wrapped_ops; 43 | }; 44 | 45 | 46 | typedef testing::Types> 47 | WrappedOpsImplementations; 48 | 49 | typedef testing::Types 50 | BufferedWrappedOpsImplementations; 51 | 52 | typedef testing::Types> 53 | UnbufferedWrappedOpsImplementations; 54 | 55 | INSTANTIATE_TYPED_TEST_CASE_P(CouchstoreOpsTest, 56 | WrappedOpsTest, 57 | WrappedOpsImplementations 58 | ); 59 | 60 | INSTANTIATE_TYPED_TEST_CASE_P(CouchstoreOpsTest, 61 | UnbufferedWrappedOpsTest, 62 | UnbufferedWrappedOpsImplementations 63 | ); 64 | 65 | INSTANTIATE_TYPED_TEST_CASE_P(CouchstoreOpsTest, 66 | BufferedWrappedOpsTest, 67 | BufferedWrappedOpsImplementations 68 | ); 69 | -------------------------------------------------------------------------------- /view_format.md: -------------------------------------------------------------------------------- 1 | # The Binary (Termless) Format for Views. # 2 | 3 | **(Damien Katz -- July 19 2012)** 4 | 5 | This documents the format of the Key/Value pairs in leaf nodes, and the reduction format in inner nodes. 6 | 7 | The underlying b-tree format is the same as already used in CouchStore. 8 | 9 | * All integers are network byte order. 10 | * All strings are encoded as UTF-8. 11 | * Adjacent fields are tightly packed with no padding. 12 | 13 | 14 | ## Primary Index Key Values ## 15 | 16 | In leaf nodes, `KeyValues` have the following format: 17 | 18 | * `Key`: 19 | * `EmittedJsonKeyLength` -- 16bit integer 20 | * `EmittedJSONKey` -– JSON -– The key emitted by the map function 21 | * `UnquotedDocId` –- String -– The raw doc ID (occupies the remaining bytes) 22 | * Value: 23 | * `PartitionId` -- 16bit integer -- This is the partitionId (vbucket) from which this document id maps to. 24 | * 1 to infinity `JSONStringValue`s -- These are all the values that were emitted for this `EmittedJSONKey`. 25 | Each `JSONStringValue` is of the form: 26 | * `ValueLength` -- 24bit unsigned integer 27 | * `JSONValue` - string that is `ValueLength` bytes long 28 | 29 | (Parsing the `JSONStringValue`s is simply reading the first 24 bits, getting the length of the following string and extracting the string. If there is still buffer left, the process is repeated until the is no value buffer left.) 30 | 31 | When an emit happens, and the Key is different from all other keys emitted for that document, then there is only one `JSONStringValue`. 32 | But when multiple identical keys are emitted, the values are coalesced into a list of Values, and there will be multiple values. 33 | 34 | 35 | ### Primary Index Inner Node Reductions (KeyPointerNodes and Root) ### 36 | 37 | * `SubTreeCount` -- 40bit integer -- Count of all Values in subtree. 38 | NOTE: this is possibly greater than the `KeyId` count in the subtree, because a document can emit multiple identical keys, and they are coalesced into single `KeyId`, with all the values emitted in a list as the value. 39 | * `SubTreePartitionBitmap` -- 1024 bits -- a bitfield of all partition keys in the subtree. Currently this is hardcoded at 1024 bits in length, but in the future we may change this to a variable size. Until then, it works with any # of vbuckets ≤ 1024. 40 | * `JSONReductions` -- remaining bytes -- Zero or more `JSONReductions`, each consisting of: 41 | * `JSONLen` -- 16bit integer 42 | * `JSON` -- the actual JSON string 43 | 44 | 45 | ## Back Index ## 46 | 47 | In leaf nodes, `KeyValues` have the following format: 48 | 49 | * `Key` -- blob -- The raw docId, not quoted or JSONified in any way. 50 | * `Value`: 51 | * `PartitionId` -- 16bit integer -- This is the partitionId (vbucket) from which this document id maps to. 52 | * 1-*n* `ViewKeysMappings`, where *n* ≤ the # of mapfunctions defined in the design document. 53 | A `ViewKeysMapping` is: 54 | * `ViewId` -- 8bit integer -- the ordinal id of the map view in the group the following keys were emitted from 55 | * `NumKeys` -- 16bit integer -- the number of `JSONKeys` that follow 56 | * `JSONKeys` -- a sequence of: 57 | * `KeyLen` -- 16bit integer -- Length of the following `JSONKey` 58 | * `Key` -- JSON string -- Emitted JSON key 59 | 60 | 61 | ### Back Index Inner Node Reductions (KeyPointerNodes and Root) ### 62 | 63 | * `SubTreeCount` -- 40bit integer -- count of all Keys in subtree. 64 | * `SubTreePartitionBitmap` -- 1024 bits -- a bitfield of all partition keys in the subtree. Currently this is hardcoded at 1024 in length, but in the future we may change this to a variable size. Until then, it works with any # of vbuckets ≤ 1024. 65 | --------------------------------------------------------------------------------