├── .gitignore ├── .mailmap ├── CMakeLists.txt ├── CTestCustom.cmake.in ├── Doxyfile ├── LICENSE ├── README.md ├── benchmarks ├── access_scanner_bench.cc ├── benchmark_memory_tracker.cc ├── benchmark_memory_tracker.h └── defragmenter_bench.cc ├── configuration.json ├── docs ├── Testing.md ├── collections.md ├── engine-params.org ├── klog.org ├── protocol │ ├── addrm.md │ ├── del_with_meta.md │ ├── delrm.md │ ├── get_meta.md │ ├── set_with_meta.md │ └── setrm.md ├── stats.json └── stats.org ├── management ├── cbanalyze-core ├── cbcompact ├── cbepctl ├── cbstats ├── cbvbucketctl ├── cbvdiff ├── cli_auth_utils.py ├── clitool.py ├── mc_bin_client.py ├── mc_bin_server.py ├── memcacheConstants.py ├── tap.py └── tap_example.py ├── scripts └── unmerged-commits.py ├── src ├── access_scanner.cc ├── access_scanner.h ├── atomic.cc ├── atomic.h ├── atomic_unordered_map.h ├── atomicqueue.h ├── backfill.cc ├── backfill.h ├── bgfetcher.cc ├── bgfetcher.h ├── bloomfilter.cc ├── bloomfilter.h ├── callbacks.h ├── checkpoint.cc ├── checkpoint.h ├── checkpoint_remover.cc ├── checkpoint_remover.h ├── collections │ ├── collections_dockey.h │ ├── collections_types.h │ ├── filter.cc │ ├── filter.h │ ├── manager.cc │ ├── manager.h │ ├── manifest.cc │ ├── manifest.h │ ├── vbucket_filter.cc │ ├── vbucket_filter.h │ ├── vbucket_manifest.cc │ ├── vbucket_manifest.h │ ├── vbucket_manifest_entry.cc │ ├── vbucket_manifest_entry.h │ └── vbucket_serialised_manifest_entry.h ├── common.h ├── config.cmake.h ├── config_static.h ├── configuration.cc ├── configuration.h ├── configuration_impl.h ├── conflict_resolution.cc ├── conflict_resolution.h ├── connmap.cc ├── connmap.h ├── couch-kvstore │ ├── couch-fs-stats.cc │ ├── couch-fs-stats.h │ ├── couch-kvstore-metadata.h │ ├── couch-kvstore.cc │ └── couch-kvstore.h ├── crc32.c ├── crc32.h ├── dcp │ ├── backfill-manager.cc │ ├── backfill-manager.h │ ├── backfill.h │ ├── backfill_disk.cc │ ├── backfill_disk.h │ ├── backfill_memory.cc │ ├── backfill_memory.h │ ├── consumer.cc │ ├── consumer.h │ ├── dcp-types.h │ ├── dcpconnmap.cc │ ├── dcpconnmap.h │ ├── flow-control-manager.cc │ ├── flow-control-manager.h │ ├── flow-control.cc │ ├── flow-control.h │ ├── producer.cc │ ├── producer.h │ ├── response.cc │ ├── response.h │ ├── stream.cc │ └── stream.h ├── defragmenter.cc ├── defragmenter.h ├── defragmenter_visitor.cc ├── defragmenter_visitor.h ├── ep_bucket.cc ├── ep_bucket.h ├── ep_engine.cc ├── ep_engine.h ├── ep_time.cc ├── ep_time.h ├── ep_types.cc ├── ep_types.h ├── ep_vb.cc ├── ep_vb.h ├── ephemeral_bucket.cc ├── ephemeral_bucket.h ├── ephemeral_tombstone_purger.cc ├── ephemeral_tombstone_purger.h ├── ephemeral_vb.cc ├── ephemeral_vb.h ├── ephemeral_vb_count_visitor.cc ├── ephemeral_vb_count_visitor.h ├── executorpool.cc ├── executorpool.h ├── executorthread.cc ├── executorthread.h ├── ext_meta_parser.cc ├── ext_meta_parser.h ├── failover-table.cc ├── failover-table.h ├── fakes │ └── fake_executorpool.h ├── flusher.cc ├── flusher.h ├── forest-kvstore │ ├── forest-kvstore.cc │ └── forest-kvstore.h ├── futurequeue.h ├── globaltask.cc ├── globaltask.h ├── hash_table.cc ├── hash_table.h ├── hlc.cc ├── hlc.h ├── htresizer.cc ├── htresizer.h ├── item.cc ├── item.h ├── item_pager.cc ├── item_pager.h ├── kv_bucket.cc ├── kv_bucket.h ├── kv_bucket_iface.h ├── kvshard.cc ├── kvshard.h ├── kvstore.cc ├── kvstore.h ├── linked_list.cc ├── linked_list.h ├── locks.h ├── logger.cc ├── logger.h ├── memory_tracker.cc ├── memory_tracker.h ├── monotonic.h ├── murmurhash3.cc ├── murmurhash3.h ├── mutation_log.cc ├── mutation_log.h ├── mutation_log_entry.cc ├── mutation_log_entry.h ├── objectregistry.cc ├── objectregistry.h ├── pre_link_document_context.cc ├── pre_link_document_context.h ├── replicationthrottle.cc ├── replicationthrottle.h ├── seqlist.cc ├── seqlist.h ├── sizes.cc ├── stats.cc ├── stats.h ├── statwriter.h ├── stored-value.cc ├── stored-value.h ├── stored_value_factories.h ├── storeddockey.cc ├── storeddockey.h ├── string_utils.cc ├── string_utils.h ├── syncobject.h ├── systemevent.cc ├── systemevent.h ├── tapconnection.cc ├── tapconnection.h ├── tapconnmap.cc ├── tapconnmap.h ├── task_type.h ├── taskable.h ├── tasklogentry.h ├── taskqueue.cc ├── taskqueue.h ├── tasks.cc ├── tasks.def.h ├── tasks.h ├── testlogger.cc ├── threadlocal.h ├── threadlocal_posix.h ├── threadlocal_win32.h ├── utility.h ├── vb_count_visitor.cc ├── vb_count_visitor.h ├── vbucket.cc ├── vbucket.h ├── vbucketdeletiontask.cc ├── vbucketdeletiontask.h ├── vbucketmap.cc ├── vbucketmap.h ├── warmup.cc ├── warmup.h └── workload.h ├── tests ├── .gitignore ├── ep_perfsuite.cc ├── ep_test_apis.cc ├── ep_test_apis.h ├── ep_testsuite.cc ├── ep_testsuite_basic.cc ├── ep_testsuite_checkpoint.cc ├── ep_testsuite_common.cc ├── ep_testsuite_common.h ├── ep_testsuite_dcp.cc ├── ep_testsuite_tap.cc ├── ep_testsuite_xdcr.cc ├── mock │ ├── mock_basic_ll.h │ ├── mock_dcp.cc │ ├── mock_dcp.h │ ├── mock_dcp_backfill_mgr.h │ ├── mock_dcp_consumer.h │ ├── mock_dcp_producer.h │ ├── mock_ephemeral_vb.h │ ├── mock_global_task.h │ ├── mock_stream.h │ ├── mock_synchronous_ep_engine.cc │ └── mock_synchronous_ep_engine.h ├── module_tests │ ├── UNUSED_vbucket_test.cc │ ├── atomic_ptr_test.cc │ ├── atomic_unordered_map_test.cc │ ├── basic_ll_test.cc │ ├── bloomfilter_test.cc │ ├── breakdancer.py │ ├── checkpoint_test.cc │ ├── collections │ │ ├── collection_dockey_test.cc │ │ ├── evp_store_collections_test.cc │ │ ├── filter_test.cc │ │ ├── manifest_test.cc │ │ ├── vbucket_manifest_entry_test.cc │ │ └── vbucket_manifest_test.cc │ ├── configuration_test.cc │ ├── couch-fs-stats_test.cc │ ├── dcp_test.cc │ ├── defragmenter_test.cc │ ├── defragmenter_test.h │ ├── dirutils_test.cc │ ├── ep_unit_tests_main.cc │ ├── ephemeral_bucket_test.cc │ ├── ephemeral_bucket_test.h │ ├── ephemeral_vb_test.cc │ ├── evp_engine_test.cc │ ├── evp_engine_test.h │ ├── evp_store_rollback_test.cc │ ├── evp_store_single_threaded_test.cc │ ├── evp_store_single_threaded_test.h │ ├── evp_store_test.cc │ ├── evp_store_test.h │ ├── evp_store_with_meta.cc │ ├── executorpool_test.cc │ ├── executorpool_test.h │ ├── expiry_test.py │ ├── failover_table_test.cc │ ├── futurequeue_test.cc │ ├── gen_engine_test.py │ ├── hash_table_test.cc │ ├── hrtime_test.cc │ ├── item_pager_test.cc │ ├── kv_bucket_test.cc │ ├── kv_bucket_test.h │ ├── kvstore_test.cc │ ├── memory_tracker_test.cc │ ├── misc_test.cc │ ├── mock_hooks_api.cc │ ├── mock_hooks_api.h │ ├── mutation_log_test.cc │ ├── mutex_test.cc │ ├── stats_test.cc │ ├── stats_test.h │ ├── stored_value_test.cc │ ├── storeddockey_test.cc │ ├── string_utils_test.cc │ ├── systemevent_test.cc │ ├── test_helpers.cc │ ├── test_helpers.h │ ├── test_task.h │ ├── thread_gate.h │ ├── threadtests.h │ ├── timing_tests.cc │ ├── vbucket_test.cc │ ├── vbucket_test.h │ └── warmup_test.cc ├── suite_stubs.c └── suite_stubs.h ├── tools ├── cpplint.py ├── gencode.cc └── genconfig.cc └── wrapper └── wrapper /.gitignore: -------------------------------------------------------------------------------- 1 | # Keep the entries sorted to reduce the risk for a merge conflict 2 | *.1m 3 | *.[ao] 4 | *.dirstamp 5 | *.exe* 6 | *.ilk* 7 | *.pdb* 8 | *.dll* 9 | *.gcda 10 | *.gcno 11 | *.gcov 12 | *.l[ao] 13 | *.orig 14 | *.pyc 15 | *.tar.gz 16 | *.tcov 17 | *.log 18 | *.trs 19 | *~ 20 | .deps/ 21 | .DS_Store 22 | .generated_configuration 23 | .generated_stat-info 24 | .genstamp 25 | .libs/ 26 | /*_test 27 | /INSTALL 28 | /atomic_test 29 | /docs/doxy 30 | /gencode 31 | /genconfig 32 | /dtrace/probes.h 33 | /gen_code 34 | /gen_config 35 | /generated_suite_*.c 36 | /libtool 37 | /mc-kvstore-test 38 | /sizes 39 | /src/config.h 40 | /src/config.h.in 41 | /src/generated_configuration.* 42 | /src/stamp-h1 43 | /src/stats-info.[ch] 44 | /src/testlogger_libify.cc 45 | /tests/module_tests/*_test 46 | /wrapper/*badm-tap-registration 47 | /wrapper/*bcompact 48 | /wrapper/*bepctl 49 | /wrapper/*bstats 50 | /wrapper/*bvbucketctl 51 | Makefile 52 | Makefile.in 53 | TAGS 54 | ep-engine.vcxproj 55 | ep-engine.vcxproj.filters 56 | ep-engine.vcxproj.user 57 | tags 58 | /CMakeFiles/ 59 | /CTestTestfile.cmake 60 | /cmake_install.cmake 61 | ep-engine_sizes 62 | ep.so 63 | ep_testsuite.so 64 | generated_testsuite.so 65 | timing_tests.so 66 | Testing/Temporary/CTestCostData.txt 67 | Testing/ 68 | ep.dylib 69 | ep_testsuite.dylib 70 | generated_testsuite.dylib 71 | timing_tests.dylib 72 | ep.exp 73 | ep.lib 74 | ep_testsuite.exp 75 | ep_testsuite.lib 76 | timing_tests.exp 77 | timing_tests.lib 78 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Trond Norbye 2 | Trond Norbye 3 | Manik Taneja 4 | Manik Taneja 5 | Filipe David Manana 6 | Chiyoung Seo 7 | Chiyoung Seo 8 | -------------------------------------------------------------------------------- /CTestCustom.cmake.in: -------------------------------------------------------------------------------- 1 | # Increase the default amount of test output captured to 1MB (both 2 | # pass and fail) - we have test binaries which consist of multiple 3 | # tests. 4 | set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1048576) 5 | set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1048576) 6 | -------------------------------------------------------------------------------- /benchmarks/benchmark_memory_tracker.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | /* 25 | * A singleton which tracks memory usage for use in benchmarks. 26 | * 27 | * This class provides hooks for new and delete which are registered when the 28 | * singleton is created. 29 | * 30 | * Tracks the current allocation along with the maximum total allocation size 31 | * it has seen. 32 | */ 33 | class BenchmarkMemoryTracker { 34 | public: 35 | ~BenchmarkMemoryTracker(); 36 | 37 | static BenchmarkMemoryTracker* getInstance( 38 | const ALLOCATOR_HOOKS_API& hooks_api_); 39 | 40 | static void destroyInstance(); 41 | 42 | void reset(); 43 | 44 | size_t getMaxAlloc(); 45 | size_t getCurrentAlloc(); 46 | 47 | private: 48 | BenchmarkMemoryTracker(const ALLOCATOR_HOOKS_API& hooks_api); 49 | void connectHooks(); 50 | static void NewHook(const void* ptr, size_t); 51 | static void DeleteHook(const void* ptr); 52 | 53 | static std::atomic instance; 54 | static std::mutex instanceMutex; 55 | ALLOCATOR_HOOKS_API hooks_api; 56 | static std::atomic maxTotalAllocation; 57 | static std::atomic currentAlloc; 58 | }; 59 | -------------------------------------------------------------------------------- /docs/klog.org: -------------------------------------------------------------------------------- 1 | * Overview of the Key Mutation Log 2 | 3 | This change introduces a new on-disk binary format for logging major 4 | key events only. These events are limited to the creation and 5 | destruction of keys only -- modification of keys is out of scope, as 6 | is storing values. 7 | 8 | * Flow 9 | 10 | The logging hooks in to the persistence flow and does the following: 11 | 12 | After writing any new key or deleting any existing key from disk, a 13 | new log entry is buffered. 14 | 15 | The buffer is written when a block is full or on a flush event 16 | (configurable, usually after a commit1 or commit2 event). 17 | 18 | Before sending a commit to the underlying store, a commit1 event is 19 | logged. 20 | 21 | After the underlying store completes its commit, a commit2 is logged. 22 | 23 | * Configuration 24 | 25 | There are four new engine parameters that come with this feature: 26 | 27 | ** klog_path 28 | 29 | Where the events should be logged. An empty string (default) disables 30 | logging. 31 | 32 | ** klog_block_size 33 | 34 | The buffer/block size for log entries. The number should line up with 35 | the underlying filesystem block size. Multiples may increase 36 | throughput. 37 | 38 | ** klog_flush 39 | 40 | Configures when the buffer should be force-flushed. There are four 41 | possible values: 42 | 43 | off: never force a flush 44 | commit1: force a flush after commit1 only 45 | commit2: force a flush after commit2 only 46 | full: force a flush after both commit1 and commit2 47 | 48 | ** klog_sync 49 | 50 | Configures when the file should be fsynced. There are four 51 | possible values: 52 | 53 | off: never fsync 54 | commit1: fsync after commit1 only 55 | commit2: fsync after commit2 only 56 | full: fsync after both commit1 and commit2 57 | 58 | * Data Format 59 | 60 | Each file consists of a header and then an arbitrary number of blocks 61 | which each contain an arbitrary number of records. All fields are 62 | big-endian byte encoded. 63 | 64 | ** Header 65 | 66 | The file begins with a header of at least 4,096 bytes long. The 67 | header defines some basic info about the file. 68 | 69 | - 32-bit version number (this document describes version 1) 70 | - 32-bit block size 71 | - 32-bit block count 72 | - k/v properties to store additional tagged config 73 | - 8-bit key len 74 | - 8-bit value len 75 | - key bytes 76 | - value bytes 77 | - terminated by zero-length key 78 | 79 | If the block size listed in the header is larger 4,096 bytes, the 80 | first block itself will be this length (assume anything extra is 81 | zero-padded). 82 | 83 | ** Block 84 | 85 | - checksum (16-bits, IEEE crc32 & 0xffff) 86 | - record count (16-bits) 87 | - []record 88 | 89 | Block size is variable. I've been using 4k, for now. Blocks are 0 90 | padded at the end when we need to sync or we can't fit more entries. 91 | 92 | ** Record 93 | 94 | - rowid (64-bit) 95 | - vbucket (16-bit) 96 | - magic (0x45) 97 | - type (8-bit) 98 | - key len (8-bit) 99 | - key ([]byte) 100 | -------------------------------------------------------------------------------- /management/cbcompact: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Example Usage for local host.. 4 | # install/bin/cbcompact 127.0.0.1:12000 compact -b beer-sample 799 5 | # where 12000 is memcached port and 799 is vbucket id 6 | 7 | import cli_auth_utils 8 | from time import sleep 9 | import sys 10 | import mc_bin_client 11 | 12 | def cmd(f): 13 | f = cli_auth_utils.cmd_decorator(f) 14 | def g(*args, **kwargs): 15 | mc = args[0] 16 | 17 | try: 18 | vbucket = int(args[1]) 19 | except IndexError: 20 | print "Must specify a vbucket id after the compact argument" 21 | sys.exit(1) 22 | 23 | # These arguments are /always/ in kwargs, but if they are not specified 24 | # on the command line will have the value None. 25 | purgeBeforeTs = long(kwargs.pop('purgeBeforeTs') or 0) 26 | purgeBeforeSeq = long(kwargs.pop('purgeBeforeSeq') or 0) 27 | dropDeletes = int(kwargs.pop('dropDeletes') or 0) 28 | 29 | f(mc, vbucket, purgeBeforeTs, purgeBeforeSeq, dropDeletes, **kwargs) 30 | return g 31 | 32 | @cmd 33 | def compact(mc, vbucket, purgeBeforeTs, purgeBeforeSeq, dropDeletes): 34 | try: 35 | return mc.compact_db(vbucket, purgeBeforeTs, purgeBeforeSeq, 36 | dropDeletes) 37 | except: 38 | print "Unable to compact vbucket %d with the following parameters "\ 39 | "(purge before time: %d, purge before seqno: %d, drop deletes: "\ 40 | "%d) in requested engine."\ 41 | % (vbucket, purgeBeforeTs, purgeBeforeSeq, dropDeletes) 42 | 43 | def main(): 44 | c = cli_auth_utils.get_authed_clitool() 45 | 46 | c.addCommand('compact', compact, 'compact vbucketid') 47 | c.addOption('--purge-before', 'purgeBeforeTs', 48 | 'purge documents before this timestamp') 49 | c.addOption('--purge-only-upto-seq', 'purgeBeforeSeq', 50 | 'purge only upto this document sequence number') 51 | c.addFlag('--dropdeletes', 'dropDeletes', 'drop deleted items') 52 | 53 | c.execute() 54 | 55 | if __name__ == '__main__': 56 | main() 57 | -------------------------------------------------------------------------------- /management/cbvbucketctl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import clitool 4 | 5 | def listvb(mc, username=None, password=""): 6 | if username: 7 | mc.sasl_auth_plain(username, password) 8 | vbs = mc.stats('vbucket') 9 | for (vb, state) in sorted(list(vbs.items())): 10 | print "vbucket", vb[3:], state 11 | 12 | def setvb(mc, vbid, vbstate, username=None, password=""): 13 | if username: 14 | mc.sasl_auth_plain(username, password) 15 | mc.set_vbucket_state(int(vbid), vbstate) 16 | 17 | def rmvb(mc, vbid, username=None, password=""): 18 | if username: 19 | mc.sasl_auth_plain(username, password) 20 | mc.delete_vbucket(int(vbid)) 21 | 22 | if __name__ == '__main__': 23 | 24 | c = clitool.CliTool() 25 | 26 | c.addCommand('list', listvb, 'list [username password]') 27 | c.addCommand('set', setvb, 'set vbid vbstate [username password]') 28 | c.addCommand('rm', rmvb, 'rm vbid [username password]') 29 | 30 | c.execute() 31 | -------------------------------------------------------------------------------- /management/cli_auth_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import clitool 4 | import inspect 5 | import mc_bin_client 6 | import memcacheConstants 7 | import sys 8 | import os 9 | 10 | def cmd_decorator(f): 11 | """Decorate a function with code to authenticate based on 1-3 12 | additional arguments.""" 13 | 14 | def g(*args, **kwargs): 15 | mc = args[0] 16 | spec = inspect.getargspec(f) 17 | max = len(spec.args) 18 | defaults = len(spec.defaults) if spec.defaults else 0 19 | min = max - defaults 20 | 21 | if len(args) < min: 22 | print >> sys.stderr, ("Error: too few arguments - command " 23 | "expected a minimum of %s but was passed " 24 | "%s: %s" 25 | % (min - 1, len(args) - 1, list(args[1:]))) 26 | sys.exit(2) 27 | 28 | if spec.varargs is None: 29 | if len(args) > max: 30 | print >> sys.stderr, ("Error: too many arguments - command " 31 | "expected a maximum of %s but was passed " 32 | "%s: %s" 33 | % (max - 1, len(args) - 1, list(args[1:]))) 34 | sys.exit(2) 35 | 36 | bucket = kwargs.pop('bucketName', None) 37 | username = kwargs.pop('username', None) or bucket 38 | password = kwargs.pop('password', None) 39 | 40 | 41 | if username is not None or password is not None: 42 | bucket = bucket or 'default' 43 | username = username or bucket 44 | password = password or '' 45 | try: 46 | mc.sasl_auth_plain(username, password) 47 | except mc_bin_client.MemcachedError: 48 | print ("Authentication error for user:{0} bucket:{1}" 49 | .format(username, bucket)) 50 | sys.exit(1) 51 | 52 | mc.enable_xerror() 53 | mc.hello("{0} {1}".format(os.path.split(sys.argv[0])[1], 54 | os.getenv("EP_ENGINE_VERSION", 55 | "unknown version"))) 56 | try: 57 | if kwargs.pop('allBuckets', None): 58 | buckets = mc.list_buckets() 59 | for bucket in buckets: 60 | print '*' * 78 61 | print bucket 62 | print 63 | mc.bucket_select(bucket) 64 | f(*args, **kwargs) 65 | elif bucket is not None: 66 | mc.bucket_select(bucket) 67 | f(*args, **kwargs) 68 | else: 69 | f(*args, **kwargs) 70 | except mc_bin_client.ErrorEaccess: 71 | print ("No access to bucket:{} - permission denied " 72 | "or bucket does not exist.".format(bucket)) 73 | sys.exit(1) 74 | 75 | return g 76 | 77 | 78 | 79 | def get_authed_clitool(): 80 | c = clitool.CliTool() 81 | 82 | c.addFlag('-a', 'allBuckets', 'iterate over all buckets') 83 | c.addOption('-b', 'bucketName', 'the bucket to get stats from (Default: default)') 84 | c.addOption('-u', 'username', 'the user as which to authenticate (Default: bucketName)') 85 | c.addOption('-p', 'password', 'the password for the bucket if one exists') 86 | 87 | return c 88 | -------------------------------------------------------------------------------- /management/clitool.py: -------------------------------------------------------------------------------- 1 | import optparse 2 | import socket 3 | import sys 4 | 5 | import mc_bin_client 6 | 7 | class CliTool(object): 8 | 9 | def __init__(self, extraUsage=""): 10 | self.cmds = {} 11 | self.flags = {} 12 | self.extraUsage = extraUsage.strip() 13 | self.parser = optparse.OptionParser( 14 | usage="%prog host[:dataport] command [options]\n\n" 15 | "dataport [default:11210]") 16 | 17 | def addCommand(self, name, f, help=None): 18 | if not help: 19 | help = name 20 | self.cmds[name] = (f, help) 21 | 22 | def addFlag(self, flag, key, description): 23 | self.flags[flag] = description 24 | self.parser.add_option(flag, dest=key, action='store_true', 25 | help=description) 26 | 27 | def addHiddenFlag(self, flag, key): 28 | self.parser.add_option(flag, dest=key, action='store_true', 29 | help=optparse.SUPPRESS_HELP) 30 | 31 | def addOption(self, flag, key, description): 32 | self.flags[flag] = description 33 | self.parser.add_option(flag, dest=key, action='store', 34 | help=description) 35 | 36 | def execute(self): 37 | self.parser.usage += "\n" + self.format_command_list() 38 | 39 | opts, args = self.parser.parse_args() 40 | 41 | if len(args) < 2: 42 | print >> sys.stderr, self.parser.error("Too few arguments") 43 | sys.exit(2) 44 | 45 | 46 | hp, self.cmd = args[:2] 47 | if ':' in hp: 48 | host, port = hp.split(':', 1) 49 | try: 50 | port = int(port) 51 | except ValueError: 52 | print >> sys.stderr, self.parser.error( 53 | "invalid host[:dataport]") 54 | sys.exit(2) 55 | else: 56 | host = hp 57 | port = 11210 58 | 59 | try: 60 | mc = mc_bin_client.MemcachedClient(host, port) 61 | except socket.gaierror, e: 62 | print 'Connection error: %s' % e 63 | sys.exit(1) 64 | 65 | f = self.cmds.get(self.cmd) 66 | 67 | if not f: 68 | print self.parser.error("Unknown command") 69 | 70 | try: 71 | if callable(f[0]): 72 | f[0](mc, *args[2:], **opts.__dict__) 73 | else: 74 | getattr(mc, f[0])(*args[2:]) 75 | except socket.error, e: 76 | # "Broken pipe" is confusing, so print "Connection refused" instead. 77 | if type(e) is tuple and e[0] == 32 or \ 78 | isinstance(e, socket.error) and e.errno == 32: 79 | print >> sys.stderr, "Could not connect to %s:%d: " \ 80 | "Connection refused" % (host, port) 81 | else: 82 | raise 83 | 84 | def format_command_list(self): 85 | output = "" 86 | 87 | cmds = sorted(c[1] for c in self.cmds.values()) 88 | output += "\nCommands:\n" 89 | 90 | for c in cmds: 91 | output += " %s\n" % c 92 | 93 | output = output[:-1] 94 | 95 | output += self.extraUsage 96 | 97 | return output 98 | -------------------------------------------------------------------------------- /scripts/unmerged-commits.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Script to show which commit(s) are not yet merged between our release branches. 4 | 5 | from __future__ import print_function 6 | import subprocess 7 | import sys 8 | 9 | class bcolors: 10 | """Define ANSI color codes, if we're running under a TTY.""" 11 | if sys.stdout.isatty(): 12 | HEADER = '\033[36m' 13 | WARNING = '\033[33m' 14 | ENDC = '\033[0m' 15 | else: 16 | HEADER = '' 17 | WARNING = '' 18 | ENDC = '' 19 | 20 | # Set of branches to check for unmerged patches. Ordered by ancestory; 21 | # i.e. the oldest supported branch to the newest, which is the order 22 | # patches should be merged. 23 | branches = ['couchbase/3.0.x', 24 | 'couchbase/sherlock', 25 | 'couchbase/watson', 26 | 'couchbase/master'] 27 | 28 | total_unmerged = 0 29 | for downstream, upstream in zip(branches, branches[1:]): 30 | commits = subprocess.check_output(['git', 'cherry', '-v', 31 | upstream, downstream]) 32 | count = len(commits.splitlines()) 33 | total_unmerged += count 34 | if count > 0: 35 | print((bcolors.HEADER + 36 | "{} commits in '{}' not present in '{}':" + 37 | bcolors.ENDC).format(count, downstream, upstream)) 38 | print(commits) 39 | 40 | if total_unmerged: 41 | print((bcolors.WARNING + "Total of {} commits outstanding" + 42 | bcolors.ENDC).format(total_unmerged)) 43 | 44 | sys.exit(total_unmerged) 45 | -------------------------------------------------------------------------------- /src/access_scanner.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_ACCESS_SCANNER_H_ 19 | #define SRC_ACCESS_SCANNER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include "globaltask.h" 24 | 25 | #include 26 | 27 | // Forward declaration. 28 | class EPStats; 29 | class KVBucket; 30 | class AccessScannerValueChangeListener; 31 | 32 | class AccessScanner : public GlobalTask { 33 | friend class AccessScannerValueChangeListener; 34 | public: 35 | AccessScanner(KVBucket& _store, EPStats& st, 36 | double sleeptime = 0, 37 | bool useStartTime = false, 38 | bool completeBeforeShutdown = false); 39 | 40 | bool run(); 41 | cb::const_char_buffer getDescription(); 42 | std::atomic completedCount; 43 | 44 | private: 45 | void updateAlogTime(double sleepSecs); 46 | void deleteAlogFile(const std::string& fileName); 47 | 48 | KVBucket& store; 49 | EPStats& stats; 50 | double sleepTime; 51 | std::string alogPath; 52 | std::atomic available; 53 | uint8_t residentRatioThreshold; 54 | uint64_t maxStoredItems; 55 | }; 56 | 57 | #endif // SRC_ACCESS_SCANNER_H_ 58 | -------------------------------------------------------------------------------- /src/atomic.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2011 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "atomic.h" 21 | 22 | SpinLock::SpinLock() 23 | { 24 | lck.clear(); 25 | } 26 | 27 | SpinLock::~SpinLock() {} 28 | 29 | bool SpinLock::tryAcquire(void) { 30 | return !lck.test_and_set(std::memory_order_acquire); 31 | } 32 | 33 | 34 | void SpinLock::lock(void) { 35 | int spin = 0; 36 | while (!tryAcquire()) { 37 | ++spin; 38 | if (spin > 64) { 39 | sched_yield(); 40 | } 41 | } 42 | } 43 | 44 | void SpinLock::unlock(void) { 45 | lck.clear(std::memory_order_release); 46 | } 47 | -------------------------------------------------------------------------------- /src/backfill.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2011 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_BACKFILL_H_ 19 | #define SRC_BACKFILL_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | #include "globaltask.h" 26 | #include "kv_bucket_iface.h" 27 | 28 | #define DEFAULT_BACKFILL_SNOOZE_TIME 1.0 29 | 30 | class EventuallyPersistentEngine; 31 | class TapConnMap; 32 | 33 | enum backfill_t { 34 | ALL_MUTATIONS = 1, 35 | DELETIONS_ONLY 36 | }; 37 | 38 | /** 39 | * Dispatcher callback responsible for bulk backfilling tap queues 40 | * from a KVStore. 41 | * 42 | * Note that this is only used if the KVStore reports that it has 43 | * efficient vbucket ops. 44 | */ 45 | class BackfillDiskLoad : public GlobalTask { 46 | public: 47 | 48 | BackfillDiskLoad(const std::string &n, EventuallyPersistentEngine* e, 49 | TapConnMap &cm, KVStore *s, uint16_t vbid, 50 | uint64_t start_seqno, hrtime_t token, 51 | double sleeptime = 0, bool shutdown = false); 52 | 53 | bool run(); 54 | 55 | cb::const_char_buffer getDescription(); 56 | 57 | private: 58 | const std::string name; 59 | const std::string description; 60 | EventuallyPersistentEngine *engine; 61 | TapConnMap &connMap; 62 | KVStore *store; 63 | uint16_t vbucket; 64 | uint64_t startSeqno; 65 | hrtime_t connToken; 66 | }; 67 | 68 | /** 69 | * VBucketVisitor to backfill a Producer. This visitor basically performs 70 | * backfill from memory for only resident items if it needs to schedule a 71 | * separate disk backfill task because of low resident ratio. 72 | * 73 | * The visitor will pause if the current backfill backlog for the corresponding 74 | * producer is greater than the threshold (5000 by default). 75 | */ 76 | class BackFillVisitor : public VBucketVisitor { 77 | public: 78 | BackFillVisitor(EventuallyPersistentEngine *e, TapConnMap &cm, Producer *tc, 79 | const VBucketFilter &backfillVBfilter); 80 | 81 | virtual ~BackFillVisitor() {} 82 | 83 | void visitBucket(VBucketPtr &vb) override; 84 | 85 | void complete(void) override; 86 | 87 | private: 88 | 89 | bool pauseVisitor() override; 90 | 91 | bool checkValidity(); 92 | 93 | EventuallyPersistentEngine *engine; 94 | TapConnMap &connMap; 95 | const std::string name; 96 | hrtime_t connToken; 97 | bool valid; 98 | }; 99 | 100 | #endif // SRC_BACKFILL_H_ 101 | -------------------------------------------------------------------------------- /src/bgfetcher.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2012 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_BGFETCHER_H_ 19 | #define SRC_BGFETCHER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "item.h" 28 | #include "kvstore.h" 29 | #include "stats.h" 30 | #include "vbucket.h" 31 | 32 | // Forward declarations. 33 | class KVBucket; 34 | class KVShard; 35 | class GlobalTask; 36 | 37 | /** 38 | * Dispatcher job responsible for batching data reads and push to 39 | * underlying storage 40 | */ 41 | class BgFetcher { 42 | public: 43 | static const double sleepInterval; 44 | /** 45 | * Construct a BgFetcher 46 | * 47 | * @param s The store 48 | * @param k The shard to which this background fetcher belongs 49 | * @param st reference to statistics 50 | */ 51 | BgFetcher(KVBucket* s, KVShard* k, EPStats &st) : 52 | store(s), shard(k), taskId(0), stats(st), pendingFetch(false) {} 53 | 54 | /** 55 | * Construct a BgFetcher 56 | * 57 | * Equivalent to above constructor except stats reference is obtained 58 | * from KVBucket's reference to EPEngine's epstats. 59 | * 60 | * @param s The store 61 | * @param k The shard to which this background fetcher belongs 62 | */ 63 | BgFetcher(KVBucket& s, KVShard& k); 64 | 65 | ~BgFetcher() { 66 | LockHolder lh(queueMutex); 67 | if (!pendingVbs.empty()) { 68 | LOG(EXTENSION_LOG_DEBUG, 69 | "Terminating database reader without completing " 70 | "background fetches for %ld vbuckets.\n", pendingVbs.size()); 71 | pendingVbs.clear(); 72 | } 73 | } 74 | 75 | void start(void); 76 | void stop(void); 77 | bool run(GlobalTask *task); 78 | bool pendingJob(void) const; 79 | void notifyBGEvent(void); 80 | void setTaskId(size_t newId) { taskId = newId; } 81 | void addPendingVB(VBucket::id_type vbId) { 82 | LockHolder lh(queueMutex); 83 | pendingVbs.insert(vbId); 84 | } 85 | 86 | private: 87 | size_t doFetch(VBucket::id_type vbId, vb_bgfetch_queue_t& items); 88 | void clearItems(VBucket::id_type vbId, vb_bgfetch_queue_t& items); 89 | 90 | KVBucket* store; 91 | KVShard* shard; 92 | size_t taskId; 93 | std::mutex queueMutex; 94 | EPStats &stats; 95 | 96 | std::atomic pendingFetch; 97 | std::set pendingVbs; 98 | }; 99 | 100 | #endif // SRC_BGFETCHER_H_ 101 | -------------------------------------------------------------------------------- /src/bloomfilter.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_BLOOMFILTER_H_ 19 | #define SRC_BLOOMFILTER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | enum bfilter_status_t { 29 | BFILTER_DISABLED, 30 | BFILTER_PENDING, 31 | BFILTER_COMPACTING, 32 | BFILTER_ENABLED 33 | }; 34 | 35 | /** 36 | * A bloom filter instance for a vbucket. 37 | * We are to maintain the vbucket-number of these instances. 38 | * 39 | * Each vbucket will hold one such object. 40 | */ 41 | class BloomFilter { 42 | public: 43 | BloomFilter(size_t key_count, double false_positive_prob, 44 | bfilter_status_t newStatus = BFILTER_DISABLED); 45 | ~BloomFilter(); 46 | 47 | void setStatus(bfilter_status_t to); 48 | bfilter_status_t getStatus(); 49 | std::string getStatusString(); 50 | 51 | void addKey(const DocKey& key); 52 | bool maybeKeyExists(const DocKey& key); 53 | 54 | size_t getNumOfKeysInFilter(); 55 | size_t getFilterSize(); 56 | 57 | protected: 58 | size_t estimateFilterSize(size_t key_count, double false_positive_prob); 59 | size_t estimateNoOfHashes(size_t key_count); 60 | 61 | uint64_t hashDocKey(const DocKey& key, uint32_t iteration); 62 | 63 | size_t filterSize; 64 | size_t noOfHashes; 65 | 66 | size_t keyCounter; 67 | 68 | bfilter_status_t status; 69 | std::vector bitArray; 70 | }; 71 | 72 | #endif // SRC_BLOOMFILTER_H_ 73 | -------------------------------------------------------------------------------- /src/checkpoint_remover.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2011 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_CHECKPOINT_REMOVER_H_ 19 | #define SRC_CHECKPOINT_REMOVER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include "globaltask.h" 24 | 25 | class EPStats; 26 | class EventuallyPersistentEngine; 27 | 28 | /** 29 | * Dispatcher job responsible for removing closed unreferenced checkpoints 30 | * from memory. 31 | */ 32 | class ClosedUnrefCheckpointRemoverTask : public GlobalTask { 33 | public: 34 | 35 | /** 36 | * Construct ClosedUnrefCheckpointRemover. 37 | * @param s the store 38 | * @param st the stats 39 | */ 40 | ClosedUnrefCheckpointRemoverTask(EventuallyPersistentEngine *e, 41 | EPStats &st, size_t interval) : 42 | GlobalTask(e, TaskId::ClosedUnrefCheckpointRemoverTask, interval, false), 43 | engine(e), stats(st), sleepTime(interval), available(true) {} 44 | 45 | void cursorDroppingIfNeeded(void); 46 | 47 | bool run(void); 48 | 49 | cb::const_char_buffer getDescription() { 50 | return "Removing closed unreferenced checkpoints from memory"; 51 | } 52 | 53 | private: 54 | EventuallyPersistentEngine *engine; 55 | EPStats &stats; 56 | size_t sleepTime; 57 | std::atomic available; 58 | }; 59 | 60 | #endif // SRC_CHECKPOINT_REMOVER_H_ 61 | -------------------------------------------------------------------------------- /src/collections/collections_dockey.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #pragma once 21 | 22 | namespace Collections { 23 | 24 | /** 25 | * Collections::DocKey extends a DocKey so that the length of the collection 26 | * can be recorded (the collection is a prefix on the key). 27 | * The length of the collection only describes how many bytes are before 28 | * the collection separator, whether the collection is valid or not is not 29 | * something this class determines. 30 | */ 31 | class DocKey : public ::DocKey { 32 | public: 33 | 34 | uint32_t hash() const { 35 | return ::DocKey::hash(collectionLen); 36 | } 37 | 38 | /** 39 | * Factory method to create a Collections::DocKey from a DocKey 40 | */ 41 | static DocKey make(const ::DocKey& key, const std::string& separator) { 42 | const uint8_t* collection = findCollection(key, separator); 43 | if (collection) { 44 | return DocKey(key, collection - key.data()); 45 | } else { 46 | // No collection found, not an error - ok for DefaultNamespace. 47 | return DocKey(key, 0); 48 | } 49 | } 50 | 51 | /** 52 | * @return how many bytes of the key are a collection 53 | */ 54 | size_t getCollectionLen() const { 55 | return collectionLen; 56 | } 57 | 58 | private: 59 | DocKey(const ::DocKey& key, size_t _collectionLen) 60 | : ::DocKey(key), collectionLen(_collectionLen) { 61 | } 62 | 63 | /** 64 | * Perform a search on the DocKey looking for the separator. 65 | * 66 | * @returns pointer to the start of the separator or nullptr if none found. 67 | */ 68 | static const cb::const_byte_buffer::iterator findCollection( 69 | const ::DocKey& key, const std::string& separator) { 70 | if (key.size() == 0 || separator.size() == 0 || 71 | separator.size() > key.size()) { 72 | return nullptr; 73 | } 74 | 75 | auto rv = std::search(key.data(), 76 | key.data() + key.size(), 77 | separator.begin(), 78 | separator.end()); 79 | if (rv != (key.data() + key.size())) { 80 | return rv; 81 | } 82 | return nullptr; 83 | } 84 | 85 | uint8_t collectionLen; 86 | }; 87 | } // end namespace Collections 88 | 89 | namespace std { 90 | template <> 91 | struct hash { 92 | std::size_t operator()(const Collections::DocKey& key) const { 93 | return key.hash(); 94 | } 95 | }; 96 | } -------------------------------------------------------------------------------- /src/collections/collections_types.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | namespace Collections { 23 | 24 | // The reserved name of the system owned, default collection. 25 | const char* const _DefaultCollectionIdentifier = "$default"; 26 | static cb::const_char_buffer DefaultCollectionIdentifier( 27 | _DefaultCollectionIdentifier); 28 | 29 | // The default separator we will use for identifying collections in keys. 30 | const char* const DefaultSeparator = "::"; 31 | 32 | // SystemEvent keys 33 | const char* const SystemEventPrefix = "$collections"; 34 | const char* const CreateEventKey = "create"; 35 | const char* const DeleteEventKey = "delete"; 36 | const char* const SeparatorChangedKey = "separator"; 37 | 38 | // Couchstore private file name for manifest data 39 | const char CouchstoreManifest[] = "_local/collections_manifest"; 40 | 41 | // Length of the string excluding the zero terminator (i.e. strlen) 42 | const size_t CouchstoreManifestLen = sizeof(CouchstoreManifest) - 1; 43 | 44 | } // end namespace Collections 45 | -------------------------------------------------------------------------------- /src/collections/manager.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "collections/manager.h" 19 | #include "collections/manifest.h" 20 | #include "kv_bucket.h" 21 | #include "vbucket.h" 22 | 23 | Collections::Manager::Manager() : current(std::make_unique()) { 24 | } 25 | 26 | cb::engine_error Collections::Manager::update(KVBucket& bucket, 27 | const std::string& json) { 28 | std::unique_lock ul(lock, std::try_to_lock); 29 | if (!ul.owns_lock()) { 30 | // Make concurrent updates fail, in realiy there should only be one 31 | // admin connection making changes. 32 | return cb::engine_error(cb::engine_errc::temporary_failure, 33 | "Collections::Manager::update already locked"); 34 | } 35 | 36 | std::unique_ptr newManifest; 37 | // Construct a newManifest (will throw if JSON was illegal) 38 | try { 39 | newManifest = std::make_unique(json); 40 | } catch (std::exception& e) { 41 | LOG(EXTENSION_LOG_NOTICE, 42 | "Collections::Manager::update can't construct manifest e.what:%s", 43 | e.what()); 44 | return cb::engine_error( 45 | cb::engine_errc::invalid_arguments, 46 | "Collections::Manager::update manifest json invalid:" + json); 47 | } 48 | 49 | // Validate manifest revision is increasing 50 | if (newManifest->getRevision() <= current->getRevision()) { 51 | return cb::engine_error( 52 | cb::engine_errc::invalid_arguments, 53 | "Collections::Manager::update manifest revision:" + 54 | std::to_string(current->getRevision()) + " json:" + 55 | json); 56 | } 57 | 58 | current = std::move(newManifest); 59 | 60 | for (int i = 0; i < bucket.getVBuckets().getSize(); i++) { 61 | auto vb = bucket.getVBuckets().getBucket(i); 62 | 63 | if (vb && vb->getState() == vbucket_state_active) { 64 | vb->updateFromManifest(*current); 65 | } 66 | } 67 | 68 | return cb::engine_error(cb::engine_errc::success, 69 | "Collections::Manager::update"); 70 | } 71 | 72 | void Collections::Manager::update(VBucket& vb) const { 73 | // Lock manager updates 74 | std::lock_guard ul(lock); 75 | vb.updateFromManifest(*current); 76 | } 77 | -------------------------------------------------------------------------------- /src/collections/manager.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | class KVBucket; 26 | class VBucket; 27 | 28 | namespace Collections { 29 | 30 | class Manifest; 31 | 32 | /** 33 | * Collections::Manager provides some bucket level management functions 34 | * such as the code which enables the MCBP set_collections command. 35 | */ 36 | class Manager { 37 | public: 38 | Manager(); 39 | 40 | /** 41 | * Update the bucket with the latest JSON collections manifest. 42 | * 43 | * Locks the Manager and prevents concurrent updates, concurrent updates 44 | * are failed with TMPFAIL as in reality there should be 1 admin connection. 45 | * 46 | * @param bucket the bucket receiving a set-collections command. 47 | * @param json the json manifest form a set-collections command. 48 | * @returns engine_error indicating why the update failed. 49 | */ 50 | cb::engine_error update(KVBucket& bucket, const std::string& json); 51 | 52 | /** 53 | * Update the vbucket's manifest with the current Manifest 54 | * The Manager is locked to prevent current changing whilst this update 55 | * occurs. 56 | */ 57 | void update(VBucket& vb) const; 58 | 59 | private: 60 | mutable std::mutex lock; 61 | 62 | /// Store the most recent (current) manifest received 63 | std::unique_ptr current; 64 | }; 65 | } -------------------------------------------------------------------------------- /src/collections/manifest.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace Collections { 26 | 27 | /** 28 | * Manifest is an object that is constructed from JSON data as per 29 | * a set_collections command 30 | * 31 | * Users of this class can then obtain the revision, separator and 32 | * all collections that are included in the manifest. 33 | */ 34 | class Manifest { 35 | public: 36 | /* 37 | * Initialise the default manifest 38 | */ 39 | Manifest(); 40 | 41 | /* 42 | * Create a manifest from json. 43 | * Validates the json as per SET_COLLECTIONS rules. 44 | */ 45 | Manifest(const std::string& json); 46 | 47 | int getRevision() const { 48 | return revision; 49 | } 50 | 51 | const std::string& getSeparator() const { 52 | return separator; 53 | } 54 | 55 | bool doesDefaultCollectionExist() const { 56 | return defaultCollectionExists; 57 | } 58 | 59 | std::vector::const_iterator begin() const { 60 | return collections.begin(); 61 | } 62 | 63 | std::vector::const_iterator end() const { 64 | return collections.end(); 65 | } 66 | 67 | size_t size() const { 68 | return collections.size(); 69 | } 70 | 71 | std::vector::const_iterator find( 72 | const std::string& collection) const { 73 | return std::find(begin(), end(), collection); 74 | } 75 | 76 | private: 77 | /** 78 | * Check if the C-string input has a length > 0 and < 250. 79 | * 80 | * @param separator a C-string representing the separator. 81 | */ 82 | static bool validSeparator(const char* separator); 83 | 84 | /** 85 | * Check if the C-string represents a legal collection name. 86 | * Current validation is to ensure we block creation of _ prefixed 87 | * collections and only accept $default for $ prefixed names. 88 | * @param collection a C-string representing a collection name. 89 | */ 90 | static bool validCollection(const char* collection); 91 | 92 | int revision; 93 | bool defaultCollectionExists; 94 | std::string separator; 95 | std::vector collections; 96 | }; 97 | 98 | } // end namespace Collections 99 | -------------------------------------------------------------------------------- /src/collections/vbucket_manifest_entry.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "collections/vbucket_manifest_entry.h" 19 | 20 | void Collections::VB::ManifestEntry::throwInvalidArg( 21 | const std::string& prefix) { 22 | std::stringstream ss; 23 | ss << " " << *this; 24 | throw std::invalid_argument(prefix + ss.str()); 25 | } 26 | 27 | std::ostream& Collections::VB::operator<<( 28 | std::ostream& os, 29 | const Collections::VB::ManifestEntry& manifestEntry) { 30 | os << "ManifestEntry: collection:" << manifestEntry.getCollectionName() 31 | << ", revision:" << manifestEntry.getRevision() 32 | << ", startSeqno:" << manifestEntry.getStartSeqno() 33 | << ", endSeqno:" << manifestEntry.getEndSeqno(); 34 | return os; 35 | } -------------------------------------------------------------------------------- /src/config.cmake.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* Header files */ 4 | #cmakedefine HAVE_ALLOCA_H ${HAVE_ALLOCA_H} 5 | #cmakedefine HAVE_ARPA_INET_H ${HAVE_ARPA_INET_H} 6 | #cmakedefine HAVE_ATOMIC_H ${HAVE_ATOMIC_H} 7 | #cmakedefine HAVE_MACH_MACH_TIME_H ${HAVE_MACH_MACH_TIME_H} 8 | #cmakedefine HAVE_MEMORY ${HAVE_MEMORY} 9 | #cmakedefine HAVE_NETDB_H ${HAVE_NETDB_H} 10 | #cmakedefine HAVE_NETINET_IN_H ${HAVE_NETINET_IN_H} 11 | #cmakedefine HAVE_NETINET_IN_H ${HAVE_NETINET_IN_H} 12 | #cmakedefine HAVE_NETINET_TCP_H ${HAVE_NETINET_TCP_H} 13 | #cmakedefine HAVE_POLL_H ${HAVE_POLL_H} 14 | #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H} 15 | #cmakedefine HAVE_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H} 16 | #cmakedefine HAVE_SYS_TIME_H ${HAVE_SYS_TIME_H} 17 | #cmakedefine HAVE_SCHED_H ${HAVE_SCHED_H} 18 | #cmakedefine HAVE_TR1_MEMORY ${HAVE_TR1_MEMORY} 19 | #cmakedefine HAVE_TR1_UNORDERED_MAP ${HAVE_TR1_UNORDERED_MAP} 20 | #cmakedefine HAVE_UNISTD_H ${HAVE_UNISTD_H} 21 | #cmakedefine HAVE_UNISTD_H ${HAVE_UNISTD_H} 22 | #cmakedefine HAVE_UNORDERED_MAP ${HAVE_UNORDERED_MAP} 23 | #cmakedefine HAVE_ATOMIC ${HAVE_ATOMIC} 24 | #cmakedefine HAVE_THREAD ${HAVE_THREAD} 25 | 26 | #cmakedefine HAVE_WINSOCK2_H ${HAVE_WINSOCK2_H} 27 | #cmakedefine HAVE_WS2TCPIP_H ${HAVE_WS2TCPIP_H} 28 | 29 | /* Functions */ 30 | #cmakedefine HAVE_CLOCK_GETTIME ${HAVE_CLOCK_GETTIME} 31 | #cmakedefine HAVE_MACH_ABSOLUTE_TIME ${HAVE_MACH_ABSOLUTE_TIME} 32 | #cmakedefine HAVE_GETTIMEOFDAY ${GETTIMEOFDAY} 33 | #cmakedefine HAVE_GETOPT_LONG ${HAVE_GETOPT_LONG} 34 | 35 | /* various */ 36 | #define VERSION "${EP_ENGINE_VERSION}" 37 | 38 | #ifdef __GNUC__ 39 | #define HAVE_GCC_ATOMICS 1 40 | #endif 41 | 42 | #include "config_static.h" 43 | -------------------------------------------------------------------------------- /src/config_static.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* Consider this file as an extension to config.h, just that it contains 19 | * static text. The intention is to reduce the number of #ifdefs in the rest 20 | * of the source files without having to put all of them in AH_BOTTOM 21 | * in configure.ac. 22 | */ 23 | 24 | #ifndef SRC_CONFIG_STATIC_H 25 | #define SRC_CONFIG_STATIC_H 1 26 | 27 | #include "config.h" 28 | 29 | #define _FILE_OFFSET_BITS 64 30 | 31 | #if ((defined (__SUNPRO_C) || defined(__SUNPRO_CC)) || defined __GNUC__) 32 | #define EXPORT_FUNCTION __attribute__ ((visibility("default"))) 33 | #elif defined(_MSC_VER) 34 | #define EXPORT_FUNCTION __declspec(dllexport) 35 | #else 36 | #define EXPORT_FUNCTION 37 | #endif 38 | 39 | #if HAVE_ARPA_INET_H 40 | #include 41 | #endif 42 | 43 | #ifdef HAVE_INTTYPES_H 44 | #include 45 | #endif 46 | 47 | #ifdef HAVE_NETDB_H 48 | #include 49 | #endif 50 | 51 | #ifdef HAVE_NETINET_IN_H 52 | #include 53 | #endif 54 | 55 | #ifdef HAVE_NETINET_TCP_H 56 | #include 57 | #endif 58 | 59 | #ifdef HAVE_POLL_H 60 | #include 61 | #endif 62 | 63 | #ifdef HAVE_SYS_SOCKET_H 64 | #include 65 | #endif 66 | 67 | #ifdef HAVE_SYS_TIME_H 68 | #include 69 | #endif 70 | 71 | #include 72 | 73 | #ifdef HAVE_UNISTD_H 74 | #include 75 | #endif 76 | 77 | #ifdef WIN32 78 | #include 79 | #include 80 | #define SOCKETPAIR_AF AF_INET 81 | #define getppid() 2 82 | 83 | #include 84 | #define F_OK 0 85 | #define W_OK 2 86 | #define R_OK 4 87 | #pragma warning(disable: 4291) 88 | #pragma warning(disable: 4244) 89 | #pragma warning(disable: 4267) 90 | #pragma warning(disable: 4996) 91 | #pragma warning(disable: 4800) 92 | /* 93 | #pragma warning(disable: ) 94 | #pragma warning(disable: ) 95 | */ 96 | 97 | #define sched_yield() SwitchToThread() 98 | #if _MSC_VER < 1900 99 | #define snprintf _snprintf 100 | #endif 101 | #define sleep(a) Sleep(a * 1000) 102 | #define random() (long)rand() 103 | 104 | /* TROND FIXME */ 105 | #define IOV_MAX 1024 106 | typedef unsigned int useconds_t; 107 | #else 108 | #define INVALID_SOCKET -1 109 | #define SOCKET_ERROR -1 110 | #define SOCKETPAIR_AF AF_UNIX 111 | #endif 112 | 113 | #if defined(linux) || defined(__linux__) || defined(__linux) 114 | #undef ntohs 115 | #undef ntohl 116 | #undef htons 117 | #undef htonl 118 | #endif 119 | 120 | #ifdef HAVE_SCHED_H 121 | #include 122 | #endif 123 | 124 | #include 125 | #endif /* SRC_CONFIG_STATIC_H */ 126 | -------------------------------------------------------------------------------- /src/conflict_resolution.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2013 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_CONFLICT_RESOLUTION_H_ 19 | #define SRC_CONFLICT_RESOLUTION_H_ 1 20 | 21 | #include "config.h" 22 | #include "item.h" 23 | 24 | class ItemMetaData; 25 | class StoredValue; 26 | 27 | /** 28 | * An abstract class for doing conflict resolution for documents sent from 29 | * different datacenters. 30 | */ 31 | class ConflictResolution { 32 | public: 33 | ConflictResolution() {} 34 | 35 | virtual ~ConflictResolution() {} 36 | 37 | /** 38 | * Resolves a conflict between two documents. 39 | * 40 | * @param v the local document meta data 41 | * @param meta the remote document's meta data 42 | * @param meta_dataype datatype of remote document 43 | * @param isDelete the flag indicating if conflict resolution is 44 | * for delete operations 45 | * @return true is the remote document is the winner, false otherwise 46 | */ 47 | virtual bool resolve(const StoredValue& v, 48 | const ItemMetaData& meta, 49 | const protocol_binary_datatype_t meta_datatype, 50 | bool isDelete = false) const = 0; 51 | 52 | }; 53 | 54 | class RevisionSeqnoResolution : public ConflictResolution { 55 | public: 56 | bool resolve(const StoredValue& v, 57 | const ItemMetaData& meta, 58 | const protocol_binary_datatype_t meta_datatype, 59 | bool isDelete = false) const override; 60 | }; 61 | 62 | class LastWriteWinsResolution : public ConflictResolution { 63 | public: 64 | bool resolve(const StoredValue& v, 65 | const ItemMetaData& meta, 66 | const protocol_binary_datatype_t meta_datatype, 67 | bool isDelete = false) const override; 68 | }; 69 | 70 | #endif // SRC_CONFLICT_RESOLUTION_H_ 71 | -------------------------------------------------------------------------------- /src/couch-kvstore/couch-fs-stats.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2012 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_COUCH_KVSTORE_COUCH_FS_STATS_H_ 19 | #define SRC_COUCH_KVSTORE_COUCH_FS_STATS_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | #include 28 | #include "kvstore.h" 29 | 30 | /** 31 | * Returns an instance of StatsOps from a FileStats reference and 32 | * a reference to a base FileOps implementation to wrap 33 | */ 34 | std::unique_ptr getCouchstoreStatsOps( 35 | FileStats& stats, FileOpsInterface& base_ops); 36 | 37 | /** 38 | * FileOpsInterface implementation which records various statistics 39 | * about OS-level file operations performed by Couchstore. 40 | */ 41 | class StatsOps : public FileOpsInterface { 42 | public: 43 | StatsOps(FileStats& _stats, FileOpsInterface& ops) 44 | : stats(_stats), 45 | wrapped_ops(ops) {} 46 | 47 | couch_file_handle constructor(couchstore_error_info_t* errinfo) override ; 48 | couchstore_error_t open(couchstore_error_info_t* errinfo, 49 | couch_file_handle* handle, const char* path, 50 | int oflag) override; 51 | couchstore_error_t close(couchstore_error_info_t* errinfo, 52 | couch_file_handle handle) override; 53 | ssize_t pread(couchstore_error_info_t* errinfo, 54 | couch_file_handle handle, void* buf, size_t nbytes, 55 | cs_off_t offset) override; 56 | ssize_t pwrite(couchstore_error_info_t* errinfo, 57 | couch_file_handle handle, const void* buf, 58 | size_t nbytes, cs_off_t offset) override; 59 | cs_off_t goto_eof(couchstore_error_info_t* errinfo, 60 | couch_file_handle handle) override; 61 | couchstore_error_t sync(couchstore_error_info_t* errinfo, 62 | couch_file_handle handle) override; 63 | couchstore_error_t advise(couchstore_error_info_t* errinfo, 64 | couch_file_handle handle, cs_off_t offset, 65 | cs_off_t len, 66 | couchstore_file_advice_t advice) override; 67 | void destructor(couch_file_handle handle) override; 68 | 69 | protected: 70 | FileStats& stats; 71 | FileOpsInterface& wrapped_ops; 72 | 73 | struct StatFile { 74 | StatFile(FileOpsInterface* _orig_ops, 75 | couch_file_handle _orig_handle, 76 | cs_off_t _last_offs); 77 | 78 | FileOpsInterface* orig_ops; 79 | couch_file_handle orig_handle; 80 | cs_off_t last_offs; 81 | }; 82 | }; 83 | 84 | #endif // SRC_COUCH_KVSTORE_COUCH_FS_STATS_H_ 85 | -------------------------------------------------------------------------------- /src/crc32.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2011 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_CRC32_H_ 19 | #define SRC_CRC32_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | uint32_t crc32buf(uint8_t *buf, size_t len); 26 | 27 | #endif /* SRC_CRC32_H_ */ 28 | -------------------------------------------------------------------------------- /src/dcp/backfill.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2013 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "config.h" 21 | 22 | #include "dcp/stream.h" 23 | 24 | class ScanContext; 25 | 26 | /** 27 | * Indicates the status of the backfill that is run 28 | */ 29 | enum backfill_status_t { 30 | backfill_success, 31 | backfill_finished, 32 | backfill_snooze 33 | }; 34 | 35 | class DCPBackfill { 36 | public: 37 | DCPBackfill(const active_stream_t& s, 38 | uint64_t startSeqno, 39 | uint64_t endSeqno) 40 | : stream(s), startSeqno(startSeqno), endSeqno(endSeqno) { 41 | } 42 | 43 | virtual ~DCPBackfill() { 44 | } 45 | 46 | /** 47 | * Run the DCP backfill and return the status of the run 48 | * 49 | * @return status of the current run 50 | */ 51 | virtual backfill_status_t run() = 0; 52 | 53 | /** 54 | * Get the id of the vbucket for which this object is created 55 | * 56 | * @return vbid 57 | */ 58 | uint16_t getVBucketId() const { 59 | return stream->getVBucket(); 60 | } 61 | 62 | /** 63 | * Indicates if the DCP stream associated with the backfill is dead 64 | * 65 | * @return true if stream is in dead state; else false 66 | */ 67 | virtual bool isStreamDead() = 0; 68 | 69 | /** 70 | * Cancels the backfill 71 | */ 72 | virtual void cancel() = 0; 73 | 74 | protected: 75 | /** 76 | * Ptr to the associated Active DCP stream. Backfill can be run for only 77 | * an active DCP stream 78 | */ 79 | active_stream_t stream; 80 | 81 | /** 82 | * Start seqno of the backfill 83 | */ 84 | uint64_t startSeqno; 85 | 86 | /** 87 | * End seqno of the backfill 88 | */ 89 | uint64_t endSeqno; 90 | }; 91 | 92 | using UniqueDCPBackfillPtr = std::unique_ptr; 93 | -------------------------------------------------------------------------------- /src/dcp/flow-control.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_DCP_FLOW_CONTROL_H_ 19 | #define SRC_DCP_FLOW_CONTROL_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include "memcached/engine.h" 25 | 26 | #include 27 | 28 | class DcpConsumer; 29 | class EventuallyPersistentEngine; 30 | 31 | /** 32 | * This class handles the consumer side flow control in a DCP connection. 33 | * It is always associated with a DCP consumer. 34 | * Flow control buffer size is set when the class obj is initialized. 35 | * The class obj subsequently handles sending control messages and 36 | * sending bytes processed acks to the DCP producer 37 | */ 38 | class FlowControl { 39 | public: 40 | FlowControl(EventuallyPersistentEngine &engine, DcpConsumer* consumer); 41 | 42 | ~FlowControl(); 43 | 44 | ENGINE_ERROR_CODE handleFlowCtl(struct dcp_message_producers* producers); 45 | 46 | void incrFreedBytes(uint32_t bytes); 47 | 48 | uint32_t getFlowControlBufSize(void); 49 | 50 | void setFlowControlBufSize(uint32_t newSize); 51 | 52 | bool isBufferSufficientlyDrained(); 53 | 54 | void addStats(ADD_STAT add_stat, const void *c); 55 | 56 | private: 57 | void setBufSizeWithinBounds(size_t &bufSize); 58 | 59 | bool isBufferSufficientlyDrained_UNLOCKED(uint32_t ackable_bytes); 60 | 61 | /* Associated consumer connection handler */ 62 | DcpConsumer* consumerConn; 63 | 64 | /* Reference to ep engine instance */ 65 | EventuallyPersistentEngine &engine_; 66 | 67 | /* Indicates if flow control is enabled for this connection */ 68 | bool enabled; 69 | 70 | /* Indicates whether control msg regarding flow control has been sent to 71 | the producer */ 72 | bool pendingControl; 73 | 74 | /* Flow control buffer size */ 75 | Couchbase::RelaxedAtomic bufferSize; 76 | 77 | /* Lock while updating buffersize and pendingControl */ 78 | SpinLock bufferSizeLock; 79 | 80 | /* To keep track of when last buffer ack was sent */ 81 | rel_time_t lastBufferAck; 82 | 83 | /* Total bytes acked by this connection. This is used to for stats */ 84 | std::atomic ackedBytes; 85 | 86 | /* Bytes processed from the flow control buffer */ 87 | std::atomic freedBytes; 88 | }; 89 | 90 | #endif /* SRC_DCP_FLOW_CONTROL_H_ */ 91 | -------------------------------------------------------------------------------- /src/dcp/response.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "dcp/response.h" 21 | 22 | /* 23 | * These constants are calculated from the size of the packets that are 24 | * created by each message when it gets sent over the wire. The packet 25 | * structures are located in the protocol_binary.h file in the memcached 26 | * project. 27 | */ 28 | 29 | const uint32_t StreamRequest::baseMsgBytes = 72; 30 | const uint32_t AddStreamResponse::baseMsgBytes = 28; 31 | const uint32_t SnapshotMarkerResponse::baseMsgBytes = 24; 32 | const uint32_t SetVBucketStateResponse::baseMsgBytes = 24; 33 | const uint32_t StreamEndResponse::baseMsgBytes = 28; 34 | const uint32_t SetVBucketState::baseMsgBytes = 25; 35 | const uint32_t SnapshotMarker::baseMsgBytes = 44; 36 | 37 | const char* DcpResponse::to_string() const { 38 | switch (event_) { 39 | case Event::Mutation: 40 | return "mutation"; 41 | case Event::Deletion: 42 | return "deletion"; 43 | case Event::Expiration: 44 | return "expiration"; 45 | case Event::Flush: 46 | return "flush"; 47 | case Event::SetVbucket: 48 | return "set vbucket"; 49 | case Event::StreamReq: 50 | return "stream req"; 51 | case Event::StreamEnd: 52 | return "stream end"; 53 | case Event::SnapshotMarker: 54 | return "snapshot marker"; 55 | case Event::AddStream: 56 | return "add stream"; 57 | case Event::SystemEvent: 58 | return "system event"; 59 | } 60 | throw std::logic_error( 61 | "DcpResponse::to_string(): " + std::to_string(int(event_))); 62 | } 63 | -------------------------------------------------------------------------------- /src/defragmenter_visitor.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef DEFRAGMENTER_VISITOR_H_ 19 | #define DEFRAGMENTER_VISITOR_H_ 20 | 21 | #include "kv_bucket_iface.h" 22 | 23 | class ProgressTracker; 24 | 25 | /** Defragmentation visitor - visit all objects and defragment 26 | * 27 | */ 28 | class DefragmentVisitor : public PauseResumeEPStoreVisitor, 29 | public PauseResumeHashTableVisitor { 30 | public: 31 | DefragmentVisitor(uint8_t age_threshold_); 32 | 33 | ~DefragmentVisitor(); 34 | 35 | // Set the deadline at which point the visitor will pause visiting. 36 | void setDeadline(hrtime_t deadline_); 37 | 38 | // Implementation of PauseResumeEPStoreVisitor interface: 39 | virtual bool visit(uint16_t vbucket_id, HashTable& ht); 40 | 41 | // Implementation of PauseResumeHashTableVisitor interface: 42 | virtual bool visit(StoredValue& v); 43 | 44 | // Returns the current hashtable position. 45 | HashTable::Position getHashtablePosition() const; 46 | 47 | // Resets any held stats to zero. 48 | void clearStats(); 49 | 50 | // Returns the number of documents that have been defragmented. 51 | size_t getDefragCount() const; 52 | 53 | // Returns the number of documents that have been visited. 54 | size_t getVisitedCount() const; 55 | 56 | private: 57 | /* Configuration parameters */ 58 | 59 | // Size of the largest size class from the allocator. 60 | const size_t max_size_class; 61 | 62 | // How old a blob must be to consider it for defragmentation. 63 | const uint8_t age_threshold; 64 | 65 | /* Runtime state */ 66 | 67 | // Estimates how far we have got, and when we should pause. 68 | ProgressTracker* progressTracker; 69 | 70 | // When resuming, which vbucket should we start from? 71 | uint16_t resume_vbucket_id; 72 | 73 | // When pausing / resuming, hashtable position to use. 74 | HashTable::Position hashtable_position; 75 | 76 | /* Statistics */ 77 | // Count of how many documents have been defrag'd. 78 | size_t defrag_count; 79 | // How many documents have been visited. 80 | size_t visited_count; 81 | }; 82 | 83 | #endif /* DEFRAGMENTER_VISITOR_H_ */ 84 | -------------------------------------------------------------------------------- /src/ep_time.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2012 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "ep_time.h" 21 | 22 | #include 23 | 24 | static rel_time_t uninitialized_current_time(void) { 25 | throw std::logic_error("unitialized_current_time called"); 26 | return 0; 27 | } 28 | 29 | static time_t default_abs_time(rel_time_t notused) { 30 | throw std::logic_error("default_abs_time called"); 31 | return 0; 32 | } 33 | 34 | static rel_time_t default_reltime(time_t notused) { 35 | throw std::logic_error("default_reltime called"); 36 | return 0; 37 | } 38 | 39 | 40 | void initialize_time_functions(const SERVER_CORE_API* core_api) { 41 | if (ep_current_time == uninitialized_current_time) { 42 | ep_current_time = core_api->get_current_time; 43 | } 44 | if (ep_abs_time == default_abs_time) { 45 | ep_abs_time = core_api->abstime; 46 | } 47 | if (ep_reltime == default_reltime) { 48 | ep_reltime = core_api->realtime; 49 | } 50 | } 51 | 52 | rel_time_t (*ep_current_time)(void) = uninitialized_current_time; 53 | time_t (*ep_abs_time)(rel_time_t) = default_abs_time; 54 | rel_time_t (*ep_reltime)(time_t) = default_reltime; 55 | 56 | time_t ep_real_time(void) { 57 | return ep_abs_time(ep_current_time()); 58 | } 59 | -------------------------------------------------------------------------------- /src/ep_time.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2012 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #ifndef SRC_EP_TIME_H 18 | #define SRC_EP_TIME_H 1 19 | 20 | #include "config.h" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | /* Initializes the below time functions using the function pointers 31 | * provided by the specified SERVER_CORE_API. This function should be 32 | * called before attempting to use them, typically by the first engine 33 | * loaded. 34 | * Note: Only the first call to this function will have any effect, 35 | * i.e. once initialized the functions should not be modified to 36 | * prevent data races between the different threads which use them. 37 | */ 38 | void initialize_time_functions(const SERVER_CORE_API* core_api); 39 | 40 | extern rel_time_t (*ep_current_time)(void); 41 | extern time_t (*ep_abs_time)(rel_time_t); 42 | extern rel_time_t (*ep_reltime)(time_t); 43 | extern time_t ep_real_time(void); 44 | 45 | #ifdef __cplusplus 46 | } 47 | #endif 48 | 49 | #endif /* SRC_EP_TIME_H */ 50 | -------------------------------------------------------------------------------- /src/ep_types.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "ep_types.h" 21 | 22 | std::string to_string(HighPriorityVBNotify hpNotifyType) { 23 | using HighPriorityVBNotifyUType = 24 | std::underlying_type::type; 25 | 26 | switch (hpNotifyType) { 27 | case HighPriorityVBNotify::Seqno: 28 | return "seqno"; 29 | case HighPriorityVBNotify::ChkPersistence: 30 | return "checkpoint persistence"; 31 | default: 32 | throw std::invalid_argument( 33 | "to_string(HighPriorityVBNotify) unknown " + 34 | std::to_string( 35 | static_cast(hpNotifyType))); 36 | return ""; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/ephemeral_vb_count_visitor.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed undemor the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "ephemeral_vb_count_visitor.h" 19 | 20 | #include "ephemeral_vb.h" 21 | #include "seqlist.h" 22 | 23 | void EphemeralVBucket::CountVisitor::visitBucket(VBucketPtr& vb) { 24 | // Handle base class counts 25 | VBucketCountVisitor::visitBucket(vb); 26 | 27 | // Then append Ephemeral-specific: 28 | if (desired_state != vbucket_state_dead) { 29 | auto& ephVB = dynamic_cast(*vb); 30 | autoDeleteCount += ephVB.autoDeleteCount; 31 | seqlistCount += ephVB.seqList->getNumItems(); 32 | seqlistDeletedCount += ephVB.seqList->getNumDeletedItems(); 33 | seqlistReadRangeCount += ephVB.seqList->getRangeReadEnd() - 34 | ephVB.seqList->getRangeReadBegin(); 35 | seqlistStaleCount += ephVB.seqList->getNumStaleItems(); 36 | seqlistStaleValueBytes += ephVB.seqList->getStaleValueBytes(); 37 | seqlistStaleMetadataBytes += ephVB.seqList->getStaleMetadataBytes(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/ephemeral_vb_count_visitor.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed undemor the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "vb_count_visitor.h" 19 | 20 | #include "ephemeral_vb.h" 21 | 22 | /** 23 | * vBucket visitor that counts statistics related to Ephemeral Buckets (both 24 | * applicable to the base class and specific to Ephemeral). 25 | */ 26 | class EphemeralVBucket::CountVisitor : public VBucketCountVisitor { 27 | public: 28 | CountVisitor(vbucket_state_t state) 29 | : VBucketCountVisitor(state) { 30 | } 31 | 32 | void visitBucket(VBucketPtr &vb) override; 33 | 34 | size_t autoDeleteCount = 0; 35 | uint64_t seqlistCount = 0; 36 | uint64_t seqlistDeletedCount = 0; 37 | uint64_t seqlistReadRangeCount = 0; 38 | uint64_t seqlistStaleCount = 0; 39 | size_t seqlistStaleValueBytes = 0; 40 | size_t seqlistStaleMetadataBytes = 0; 41 | }; 42 | -------------------------------------------------------------------------------- /src/ext_meta_parser.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "ext_meta_parser.h" 19 | 20 | #include 21 | 22 | ExtendedMetaData::ExtendedMetaData(const void *meta, uint16_t nmeta) { 23 | len = nmeta; 24 | data = static_cast(meta); 25 | ret = ENGINE_SUCCESS; 26 | memoryAllocated = false; 27 | decodeMeta(); 28 | } 29 | 30 | ExtendedMetaData::~ExtendedMetaData() { 31 | if (memoryAllocated) { 32 | delete[] data; 33 | } 34 | } 35 | 36 | void ExtendedMetaData::decodeMeta() { 37 | /** 38 | * Structure of extended meta data: 39 | * | Ver (1B) | Type (1B) | Length (2B) | Field1 | ... 40 | * ... | Type (1B) | Length (2B) | Field2 | ... 41 | */ 42 | uint16_t offset = 0,bytes_left = len; 43 | 44 | if (bytes_left > 0) { 45 | uint8_t version; 46 | memcpy(&version, data, sizeof(version)); 47 | if (version == META_EXT_VERSION_ONE) { 48 | bytes_left -= sizeof(version); 49 | offset += sizeof(version); 50 | while (bytes_left != 0 && ret != ENGINE_EINVAL) { 51 | uint8_t type; 52 | uint16_t length; 53 | 54 | if (bytes_left < sizeof(type) + sizeof(length)) { 55 | ret = ENGINE_EINVAL; 56 | break; 57 | } 58 | memcpy(&type, data + offset, sizeof(type)); 59 | bytes_left -= sizeof(type); 60 | offset += sizeof(type); 61 | memcpy(&length, data + offset, sizeof(length)); 62 | length = ntohs(length); 63 | bytes_left -= sizeof(length); 64 | offset += sizeof(length); 65 | if (bytes_left < length) { 66 | ret = ENGINE_EINVAL; 67 | break; 68 | } 69 | switch (type) { 70 | case CMD_META_ADJUSTED_TIME: 71 | // Ignoring adjusted_time 72 | case CMD_META_CONFLICT_RES_MODE: 73 | // MB-21143: Now ignoring conflict_res_mode 74 | // 4.6 no longer sends, but older versions 75 | // may send it to us. 76 | break; 77 | default: 78 | ret = ENGINE_EINVAL; 79 | break; 80 | } 81 | bytes_left -= length; 82 | offset += length; 83 | } 84 | } else { 85 | ret = ENGINE_EINVAL; 86 | } 87 | } else { 88 | ret = ENGINE_EINVAL; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/ext_meta_parser.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_EXT_META_PARSER_H_ 19 | #define SRC_EXT_META_PARSER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | #include 26 | 27 | /** 28 | * Version for extras-detail in setWithMeta/delWithMeta and 29 | * DCP mutation/expiration 30 | */ 31 | enum cmd_meta_extras_version { 32 | /* Extras format: | type:1B | len:2B | field1 | type | len | field2 | ... 33 | */ 34 | META_EXT_VERSION_ONE = 0x01 35 | }; 36 | 37 | /** 38 | * Definition of extras-types for setWithMeta, delWithMeta 39 | * commands and DCP mutation/expiration messages 40 | */ 41 | enum cmd_meta_extras_type { 42 | /* adjusted time */ 43 | CMD_META_ADJUSTED_TIME = 0x01, 44 | /* conflict resolution mode is no longer sent, but could be received on upgrade.*/ 45 | CMD_META_CONFLICT_RES_MODE = 0x02 46 | }; 47 | 48 | /** 49 | * This class will be used to parse the extended meta data section 50 | * in setWithMeta/delWithMeta commands and DCP mutation/deletion 51 | * messages. 52 | */ 53 | class ExtendedMetaData { 54 | public: 55 | ExtendedMetaData() 56 | : data(nullptr), 57 | ret(ENGINE_SUCCESS), 58 | len(0), 59 | memoryAllocated(false) {} 60 | 61 | ExtendedMetaData(const void *meta, uint16_t nmeta); 62 | ~ExtendedMetaData(); 63 | 64 | ENGINE_ERROR_CODE getStatus() { 65 | return ret; 66 | } 67 | 68 | std::pair getExtMeta() { 69 | return std::make_pair(data, len); 70 | } 71 | 72 | private: 73 | /* 74 | void encodeMeta(); is currently removed as there's no extmeta to encode. 75 | Resurrect from history as required. 76 | */ 77 | void decodeMeta(); 78 | 79 | const char* data; 80 | ENGINE_ERROR_CODE ret; 81 | uint16_t len; 82 | bool memoryAllocated; 83 | }; 84 | 85 | #endif // SRC_EXT_META_PARSER_H_ 86 | -------------------------------------------------------------------------------- /src/hlc.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "hlc.h" 19 | 20 | #include "statwriter.h" 21 | 22 | void HLC::addStats(const std::string& prefix, ADD_STAT add_stat, const void *c) const { 23 | auto maxCas = getMaxHLC(); 24 | add_prefixed_stat(prefix.data(), "max_cas", maxCas, add_stat, c); 25 | 26 | // Print max_cas as a UTC human readable string 27 | auto nanoseconds = std::chrono::nanoseconds(maxCas);//duration 28 | auto seconds = std::chrono::duration_cast(nanoseconds); 29 | time_t maxCasSeconds = seconds.count(); 30 | 31 | std::tm tm; 32 | char timeString[30]; // Need to store 1970-12-31T23:23:59 33 | // Print as an ISO-8601 date format with nanosecond fractional part 34 | if (cb_gmtime_r(&maxCasSeconds, &tm) == 0 && 35 | strftime(timeString, sizeof(timeString), "%Y-%m-%dT%H:%M:%S", &tm)) { 36 | // Get the fractional nanosecond part 37 | nanoseconds -= seconds; 38 | char finalString[40];// Needs to store 1970-12-31T23:23:59.999999999 39 | const char* maxCasStr = finalString; 40 | try { 41 | checked_snprintf(finalString, sizeof(finalString), "%s.%lld", 42 | timeString, nanoseconds.count()); 43 | } catch (...) { 44 | // snprint fail, point at timeString which at least has the 45 | // majority of the time data. 46 | maxCasStr = timeString; 47 | } 48 | add_prefixed_stat(prefix.data(), "max_cas_str", maxCasStr, add_stat, c); 49 | } else { 50 | add_prefixed_stat(prefix.data(), "max_cas_str", "could not get string", add_stat, c); 51 | } 52 | 53 | add_prefixed_stat(prefix.data(), "total_abs_drift", cummulativeDrift.load(), add_stat, c); 54 | add_prefixed_stat(prefix.data(), "total_abs_drift_count", cummulativeDriftIncrements.load(), add_stat, c); 55 | add_prefixed_stat(prefix.data(), "drift_ahead_threshold_exceeded", driftAheadExceeded.load(), add_stat, c); 56 | add_prefixed_stat(prefix.data(), "drift_behind_threshold_exceeded", driftBehindExceeded.load(), add_stat, c); 57 | add_prefixed_stat(prefix.data(), "logical_clock_ticks", logicalClockTicks.load(), add_stat, c); 58 | 59 | // These are printed "as is" so we know what is being compared. Do not convert to microseconds 60 | add_prefixed_stat(prefix.data(), "drift_ahead_threshold", driftAheadThreshold.load(), add_stat, c); 61 | add_prefixed_stat(prefix.data(), "drift_behind_threshold", driftBehindThreshold.load(), add_stat, c); 62 | 63 | } 64 | 65 | void HLC::resetStats() { 66 | // Don't clear max_cas or the threshold values. 67 | cummulativeDrift = 0; 68 | cummulativeDriftIncrements = 0; 69 | driftAheadExceeded = 0; 70 | driftBehindExceeded = 0; 71 | logicalClockTicks = 0; 72 | } 73 | -------------------------------------------------------------------------------- /src/htresizer.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "ep_engine.h" 21 | #include "htresizer.h" 22 | #include "kv_bucket_iface.h" 23 | 24 | #include 25 | #include 26 | 27 | #include 28 | 29 | /** 30 | * Look at all the hash tables and make sure they're sized appropriately. 31 | */ 32 | class ResizingVisitor : public VBucketVisitor { 33 | public: 34 | ResizingVisitor() { } 35 | 36 | void visitBucket(VBucketPtr &vb) override { 37 | vb->ht.resize(); 38 | } 39 | }; 40 | 41 | HashtableResizerTask::HashtableResizerTask(KVBucketIface* s, double sleepTime) 42 | : GlobalTask(&s->getEPEngine(), 43 | TaskId::HashtableResizerTask, 44 | sleepTime, 45 | false), 46 | store(s) { 47 | } 48 | 49 | bool HashtableResizerTask::run(void) { 50 | TRACE_EVENT0("ep-engine/task", "HashtableResizerTask"); 51 | auto pv = std::make_unique(); 52 | store->visit(std::move(pv), 53 | "Hashtable resizer", 54 | TaskId::HashtableResizerVisitorTask); 55 | 56 | snooze(engine->getConfiguration().getHtResizeInterval()); 57 | return true; 58 | } 59 | -------------------------------------------------------------------------------- /src/htresizer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "config.h" 21 | 22 | #include "globaltask.h" 23 | 24 | #include 25 | 26 | class KVBucketIface; 27 | 28 | /** 29 | * Look around at hash tables and verify they're all sized 30 | * appropriately. 31 | */ 32 | class HashtableResizerTask : public GlobalTask { 33 | public: 34 | 35 | HashtableResizerTask(KVBucketIface* s, double sleepTime); 36 | 37 | bool run(void); 38 | 39 | cb::const_char_buffer getDescription() { 40 | return "Adjusting hash table sizes."; 41 | } 42 | 43 | private: 44 | KVBucketIface* store; 45 | }; 46 | -------------------------------------------------------------------------------- /src/logger.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "config.h" 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | /** 29 | * Simple logger class. Main features: 30 | * 31 | * - Common prefix for all messages emitted 32 | * - Set a minimum log level - any lower priority messages are dropped. 33 | */ 34 | class Logger { 35 | public: 36 | Logger(const std::string& prefix_ = ""); 37 | 38 | /* Log a message with the given severity. 39 | * 40 | * @param severity Severity of the log message 41 | * @param fmt printf-style format string and varargs 42 | */ 43 | virtual void log(EXTENSION_LOG_LEVEL severity, const char* fmt, ...) const CB_FORMAT_PRINTF(3, 4); 44 | 45 | /* va_list variant of the log() method. 46 | * 47 | * @param severity Severity of the log message 48 | * @param fmt printf-style format string 49 | * @param va_list Variable arguments as used by fmt. 50 | */ 51 | virtual void vlog(EXTENSION_LOG_LEVEL severity, const char* fmt, va_list va) const; 52 | 53 | // Informs the Logger class of the current logging API. 54 | static void setLoggerAPI(SERVER_LOG_API* api); 55 | 56 | // Informs the Logger class of a change in the global log level. 57 | static void setGlobalLogLevel(EXTENSION_LOG_LEVEL level); 58 | 59 | // Prefix to use for all messages. 60 | std::string prefix; 61 | 62 | // Minimum log level messages will be printed at. 63 | EXTENSION_LOG_LEVEL min_log_level; 64 | 65 | void setId(uint32_t id) { 66 | Logger::id = id; 67 | } 68 | 69 | protected: 70 | uint32_t id; 71 | 72 | private: 73 | // memcached (server) logger API which is actually used to output messages 74 | // to the underlying log file. 75 | static std::atomic logger_api; 76 | 77 | // Global log level; any message less than this will not be output. 78 | static std::atomic global_log_level; 79 | }; 80 | 81 | // The global logger instance, used by LOG() when components don't specify 82 | // their own more specific logger. Also currently used by CouchKVStore. 83 | extern Logger global_logger; 84 | -------------------------------------------------------------------------------- /src/memory_tracker.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2012 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_MEMORY_TRACKER_H_ 19 | #define SRC_MEMORY_TRACKER_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | 33 | /** 34 | * This class is used by ep-engine to hook into memcached's memory tracking 35 | * capabilities. 36 | */ 37 | class MemoryTracker { 38 | public: 39 | ~MemoryTracker(); 40 | 41 | /* Creates the singleton instance of the MemoryTracker (if it doesn't exist). 42 | * Thread-safe, so ok for multiple threads to attempt to create at the 43 | * same time. 44 | * @return The MemoryTracker singleton. 45 | */ 46 | static MemoryTracker* getInstance(const ALLOCATOR_HOOKS_API& hook_api_); 47 | 48 | static void destroyInstance(); 49 | 50 | void getAllocatorStats(std::map &alloc_stats); 51 | 52 | static bool trackingMemoryAllocations(); 53 | 54 | void updateStats(); 55 | 56 | void getDetailedStats(char* buffer, int size); 57 | 58 | size_t getFragmentation(); 59 | 60 | size_t getTotalBytesAllocated(); 61 | 62 | size_t getTotalHeapBytes(); 63 | 64 | private: 65 | MemoryTracker(const ALLOCATOR_HOOKS_API& hooks_api_); 66 | 67 | // Helper function for construction - connects the tracker 68 | // to the memory allocator via alloc_hooks. 69 | void connectHooks(); 70 | 71 | // Function for the stats updater main loop. 72 | static void statsThreadMainLoop(void* arg); 73 | 74 | static void NewHook(const void* ptr, size_t); 75 | static void DeleteHook(const void* ptr); 76 | 77 | // Wheter or not we have the ability to accurately track memory allocations 78 | static std::atomic tracking; 79 | // Singleton memory tracker and mutex guarding it's creation. 80 | static std::atomic instance; 81 | static std::mutex instance_mutex; 82 | 83 | cb_thread_t statsThreadId; 84 | allocator_stats stats; 85 | 86 | // Mutex guarding the shutdown condvar. 87 | std::mutex mutex; 88 | // Condition variable used to signal shutdown to the stats thread. 89 | std::condition_variable shutdown_cv; 90 | 91 | // Memory allocator hooks API to use (needed by New / Delete hook 92 | // functions) 93 | ALLOCATOR_HOOKS_API hooks_api; 94 | }; 95 | 96 | #endif // SRC_MEMORY_TRACKER_H_ 97 | -------------------------------------------------------------------------------- /src/monotonic.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | /** 23 | * Monotonic is a class template for simple types T. It provides guarentee 24 | * of updating the class objects by only values that are greater than what is 25 | * contained at the time of update 26 | * Note: This is not atomic/thread-safe 27 | */ 28 | template 29 | class Monotonic { 30 | public: 31 | Monotonic(const T val = std::numeric_limits::min()) : val(val) { 32 | } 33 | 34 | Monotonic(const Monotonic& other) : val(other.val) { 35 | } 36 | 37 | Monotonic& operator=(const Monotonic& other) { 38 | if (val < other.val) { 39 | val = other.val; 40 | } 41 | return *this; 42 | } 43 | 44 | Monotonic& operator=(const T& v) { 45 | if (val < v) { 46 | val = v; 47 | } 48 | return *this; 49 | } 50 | 51 | operator T() const { 52 | return val; 53 | } 54 | 55 | /* Can be used to lower the value */ 56 | void reset(const T& v) { 57 | val = v.val; 58 | } 59 | 60 | private: 61 | T val; 62 | }; 63 | -------------------------------------------------------------------------------- /src/murmurhash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | #include 12 | 13 | //----------------------------------------------------------------------------- 14 | 15 | void MurmurHash3_x86_32 (const void * key, int len, uint32_t seed, 16 | uint32_t* out); 17 | 18 | /** 19 | * The following 2 functions have been altered to just return the 20 | * 64 most significant bits. 21 | */ 22 | void MurmurHash3_x86_128 (const void * key, int len, uint32_t seed, 23 | uint64_t* out); 24 | 25 | void MurmurHash3_x64_128 (const void * key, int len, uint32_t seed, 26 | uint64_t* out); 27 | 28 | //----------------------------------------------------------------------------- 29 | 30 | #endif // _MURMURHASH3_H_ 31 | -------------------------------------------------------------------------------- /src/mutation_log_entry.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "mutation_log_entry.h" 19 | 20 | std::string to_string(MutationLogType t) { 21 | switch (t) { 22 | case MutationLogType::New: 23 | return "new"; 24 | break; 25 | case MutationLogType::Commit1: 26 | return "commit1"; 27 | break; 28 | case MutationLogType::Commit2: 29 | return "commit2"; 30 | break; 31 | case MutationLogType::NumberOfTypes: { 32 | // fall through 33 | } 34 | } 35 | throw std::invalid_argument("to_string(MutationLogType) unknown param " + 36 | std::to_string(int(t))); 37 | } 38 | 39 | // ---------------------------------------------------------------------- 40 | // Output of entries 41 | // ---------------------------------------------------------------------- 42 | std::ostream& operator<<(std::ostream& out, const MutationLogEntryV1& mle) { 43 | out << "{MutationLogEntryV1 rowid=" << mle.rowid() 44 | << ", vbucket=" << mle.vbucket() << ", magic=0x" << std::hex 45 | << static_cast(mle.magic) << std::dec 46 | << ", type=" << to_string(mle.type()) << ", key=``" << mle.key() 47 | << "''"; 48 | return out; 49 | } 50 | 51 | std::ostream& operator<<(std::ostream& out, const MutationLogEntryV2& mle) { 52 | out << "{MutationLogEntryV2" 53 | << " vbucket=" << mle.vbucket() << ", magic=0x" << std::hex 54 | << static_cast(mle.magic) << std::dec 55 | << ", type=" << to_string(mle.type()) << ", key=``" << mle.key().data() 56 | << "''"; 57 | return out; 58 | } 59 | -------------------------------------------------------------------------------- /src/objectregistry.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #ifndef SRC_OBJECTREGISTRY_H_ 18 | #define SRC_OBJECTREGISTRY_H_ 1 19 | 20 | #include "config.h" 21 | 22 | #include 23 | 24 | class EventuallyPersistentEngine; 25 | class Blob; 26 | class Item; 27 | class StoredValue; 28 | 29 | extern "C" { 30 | typedef size_t (*get_allocation_size)(const void *ptr); 31 | } 32 | 33 | class StoredValue; 34 | 35 | class ObjectRegistry { 36 | public: 37 | static void initialize(get_allocation_size func); 38 | 39 | /** 40 | * Resets the ObjectRegistry back to initial state (before initialize() 41 | * was called). 42 | */ 43 | static void reset(); 44 | 45 | static void onCreateBlob(const Blob *blob); 46 | static void onDeleteBlob(const Blob *blob); 47 | 48 | static void onCreateItem(const Item *pItem); 49 | static void onDeleteItem(const Item *pItem); 50 | 51 | static void onCreateStoredValue(const StoredValue *sv); 52 | static void onDeleteStoredValue(const StoredValue *sv); 53 | 54 | 55 | static EventuallyPersistentEngine *getCurrentEngine(); 56 | 57 | static EventuallyPersistentEngine *onSwitchThread(EventuallyPersistentEngine *engine, 58 | bool want_old_thread_local = false); 59 | 60 | static void setStats(std::atomic* init_track); 61 | static bool memoryAllocated(size_t mem); 62 | static bool memoryDeallocated(size_t mem); 63 | }; 64 | 65 | /** 66 | * To avoid mem accounting within a block 67 | */ 68 | class SystemAllocationGuard { 69 | public: 70 | SystemAllocationGuard(); 71 | ~SystemAllocationGuard(); 72 | private: 73 | EventuallyPersistentEngine* engine = nullptr; 74 | }; 75 | 76 | #endif // SRC_OBJECTREGISTRY_H_ 77 | -------------------------------------------------------------------------------- /src/pre_link_document_context.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #include "pre_link_document_context.h" 18 | #include 19 | #include "ep_engine.h" 20 | 21 | void PreLinkDocumentContext::preLink(uint64_t cas, uint64_t seqno) { 22 | // The vbucket_uuid is currently not being used by the pre_link callback 23 | item_info info = item->toItemInfo(0); 24 | info.cas = cas; 25 | info.seqno = seqno; 26 | engine.getServerApi()->document->pre_link(cookie, info); 27 | } 28 | -------------------------------------------------------------------------------- /src/pre_link_document_context.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | #pragma once 18 | 19 | #include 20 | 21 | class EventuallyPersistentEngine; 22 | class Item; 23 | 24 | /** 25 | * The PreLinkDocumetnContext is a helper class used when items are modified 26 | * by the frontend. It carries the context of the calling thread down to 27 | * where the CAS and seqno are generated, and fires the callback to the 28 | * front-end before the object is made available for other threads. 29 | * 30 | * The initial use of this is to allow the "(CAS & Seqno) macro expansion" in 31 | * extended attributes. 32 | * 33 | * See the Document API in the server interface for more description 34 | * of the pre_Link callback. 35 | */ 36 | class PreLinkDocumentContext { 37 | public: 38 | /** 39 | * Initialize the object 40 | * 41 | * @param engine_ The engine who owns the item 42 | * @param cookie_ The cookie representing the connection 43 | * @param item_ The document to operate on 44 | */ 45 | PreLinkDocumentContext(EventuallyPersistentEngine& engine_, 46 | const void* cookie_, 47 | Item* item_) 48 | : engine(engine_), cookie(cookie_), item(item_) { 49 | } 50 | 51 | /** 52 | * Set the CAS and Seqno values 53 | * 54 | * This method should be called _before_ the item is made available 55 | * for other threads 56 | * 57 | * @param cas the new CAS value for the object 58 | * @param seqno the new seqno value for the object 59 | */ 60 | void preLink(uint64_t cas, uint64_t seqno); 61 | 62 | PreLinkDocumentContext(const PreLinkDocumentContext&) = delete; 63 | 64 | protected: 65 | EventuallyPersistentEngine& engine; 66 | const void* cookie; 67 | const Item* item; 68 | }; 69 | -------------------------------------------------------------------------------- /src/replicationthrottle.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "configuration.h" 21 | #include "replicationthrottle.h" 22 | 23 | ReplicationThrottle::ReplicationThrottle(Configuration &config, EPStats &s) : 24 | queueCap(config.getReplicationThrottleQueueCap()), 25 | capPercent(config.getReplicationThrottleCapPcnt()), 26 | stats(s) 27 | {} 28 | 29 | bool ReplicationThrottle::persistenceQueueSmallEnough() const { 30 | size_t queueSize = stats.diskQueueSize.load(); 31 | if (stats.replicationThrottleWriteQueueCap == -1) { 32 | return true; 33 | } 34 | return queueSize < static_cast(stats.replicationThrottleWriteQueueCap); 35 | } 36 | 37 | bool ReplicationThrottle::hasSomeMemory() const { 38 | double memoryUsed = static_cast(stats.getTotalMemoryUsed()); 39 | double maxSize = static_cast(stats.getMaxDataSize()); 40 | 41 | return memoryUsed <= (maxSize * stats.replicationThrottleThreshold); 42 | } 43 | 44 | bool ReplicationThrottle::shouldProcess() const { 45 | return persistenceQueueSmallEnough() && hasSomeMemory(); 46 | } 47 | 48 | void ReplicationThrottle::adjustWriteQueueCap(size_t totalItems) { 49 | if (queueCap == -1) { 50 | stats.replicationThrottleWriteQueueCap.store(-1); 51 | return; 52 | } 53 | size_t qcap = static_cast(queueCap); 54 | size_t throttleCap = 0; 55 | if (capPercent > 0) { 56 | throttleCap = (static_cast(capPercent) / 100.0) * totalItems; 57 | } 58 | stats.replicationThrottleWriteQueueCap.store(throttleCap > qcap ? throttleCap : 59 | qcap); 60 | } 61 | -------------------------------------------------------------------------------- /src/replicationthrottle.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_REPLICATIONTHROTTLE_H_ 19 | #define SRC_REPLICATIONTHROTTLE_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | #include "stats.h" 26 | 27 | class Configuration; 28 | 29 | /** 30 | * Monitors various internal state to report whether we should 31 | * throttle incoming tap. 32 | */ 33 | class ReplicationThrottle { 34 | public: 35 | 36 | ReplicationThrottle(Configuration &config, EPStats &s); 37 | 38 | /** 39 | * If true, we should process incoming tap requests. 40 | */ 41 | bool shouldProcess() const; 42 | 43 | void setCapPercent(size_t perc) { capPercent = perc; } 44 | void setQueueCap(ssize_t cap) { queueCap = cap; } 45 | 46 | void adjustWriteQueueCap(size_t totalItems); 47 | 48 | private: 49 | bool persistenceQueueSmallEnough() const; 50 | bool hasSomeMemory() const; 51 | 52 | Couchbase::RelaxedAtomic queueCap; 53 | Couchbase::RelaxedAtomic capPercent; 54 | EPStats &stats; 55 | }; 56 | 57 | #endif // SRC_REPLICATIONTHROTTLE_H_ 58 | -------------------------------------------------------------------------------- /src/seqlist.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include "linked_list.h" 20 | 21 | SequenceList::RangeIterator::RangeIterator( 22 | std::unique_ptr rangeIterImpl) 23 | : rangeIterImpl(std::move(rangeIterImpl)) { 24 | } 25 | 26 | OrderedStoredValue& SequenceList::RangeIterator::operator*() const { 27 | return *(*rangeIterImpl); 28 | } 29 | 30 | SequenceList::RangeIterator& SequenceList::RangeIterator::operator++() { 31 | ++(*rangeIterImpl); 32 | return *this; 33 | } 34 | 35 | seqno_t SequenceList::RangeIterator::curr() const { 36 | return rangeIterImpl->curr(); 37 | } 38 | 39 | seqno_t SequenceList::RangeIterator::end() const { 40 | return rangeIterImpl->end(); 41 | } 42 | 43 | seqno_t SequenceList::RangeIterator::back() const { 44 | return rangeIterImpl->back(); 45 | } 46 | 47 | uint64_t SequenceList::RangeIterator::count() const { 48 | return rangeIterImpl->count(); 49 | } 50 | 51 | seqno_t SequenceList::RangeIterator::getEarlySnapShotEnd() const { 52 | return rangeIterImpl->getEarlySnapShotEnd(); 53 | } 54 | -------------------------------------------------------------------------------- /src/stats.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "stats.h" 19 | 20 | void EPStats::memAllocated(size_t sz) { 21 | if (isShutdown) { 22 | return; 23 | } 24 | 25 | if (localMemCounter.get() == nullptr) { 26 | // this HAS to be a non-bucket allocation 27 | // or else the callbacks would try to call this 28 | // function again & it would become an infinite loop 29 | SystemAllocationGuard system_alloc_guard; 30 | localMemCounter.set(new TLMemCounter()); 31 | } 32 | 33 | if (0 == sz) { 34 | return; 35 | } 36 | 37 | localMemCounter.get()->used += sz; 38 | mergeMemCounter(); 39 | } 40 | 41 | void EPStats::memDeallocated(size_t sz) { 42 | if (isShutdown) { 43 | return; 44 | } 45 | 46 | if (localMemCounter.get() == nullptr) { 47 | // this HAS to be a non-bucket allocation 48 | // or else the callbacks would try to call this 49 | // function again & it would become an infinite loop 50 | SystemAllocationGuard system_alloc_guard; 51 | localMemCounter.set(new TLMemCounter()); 52 | } 53 | 54 | if (0 == sz) { 55 | return; 56 | } 57 | 58 | localMemCounter.get()->used -= sz; 59 | mergeMemCounter(); 60 | } 61 | 62 | void EPStats::mergeMemCounter(bool force) { 63 | auto& counter = *(localMemCounter.get()); 64 | counter.count++; 65 | if (force || counter.count % mem_merge_count_threshold == 0 || 66 | std::abs(counter.used) > (long)mem_merge_bytes_threshold) { 67 | totalMemory->fetch_add(counter.used); 68 | counter.used = 0; 69 | } 70 | } -------------------------------------------------------------------------------- /src/storeddockey.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | 19 | #include "storeddockey.h" 20 | 21 | std::ostream& operator<<(std::ostream& os, const StoredDocKey& key) { 22 | return os << "ns:" << int(key.getDocNamespace()) << " " << key.c_str(); 23 | } 24 | 25 | std::ostream& operator<<(std::ostream& os, const SerialisedDocKey& key) { 26 | os << "ns:" << int(key.getDocNamespace()) << " "; 27 | for (size_t ii = 0; ii < key.size(); ++ii) { 28 | os << static_cast(key.data()[ii]); 29 | } 30 | return os; 31 | } 32 | -------------------------------------------------------------------------------- /src/string_utils.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "string_utils.h" 19 | 20 | bool cb_stob(const std::string& s) { 21 | if (s == "true") { 22 | return true; 23 | } else if (s == "false") { 24 | return false; 25 | } else { 26 | throw invalid_argument_bool("Argument was not `true` or `false`"); 27 | } 28 | } 29 | 30 | bool cb_isPrefix(const std::string& input, const std::string& prefix) { 31 | return (input.compare(0, prefix.length(), prefix) == 0); 32 | } 33 | -------------------------------------------------------------------------------- /src/string_utils.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * High level, generic string utility functions. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | #include 26 | 27 | /** 28 | * Converts a string to a boolean if viable, otherwise throws an exception 29 | * 30 | * Valid strings are 'true' and 'false' (Case-sensitive) 31 | * 32 | * @param s String to convert 33 | * @return Converted string 34 | * @throws std::invalid_argument_bool if argument is not true or false 35 | */ 36 | bool cb_stob(const std::string& s); 37 | 38 | /** 39 | * Checks input to determine whether it is prefixed with 'prefix'. 40 | */ 41 | bool cb_isPrefix(const std::string& input, 42 | const std::string& prefix); 43 | 44 | class invalid_argument_bool : public std::invalid_argument { 45 | public: 46 | invalid_argument_bool(const std::string& msg) : std::invalid_argument(msg) { 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /src/syncobject.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_SYNCOBJECT_H_ 19 | #define SRC_SYNCOBJECT_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include "utility.h" 24 | 25 | #include 26 | #include 27 | 28 | /** 29 | * Abstraction built on top of std::condition_variable & std::mutex 30 | */ 31 | class SyncObject : public std::mutex { 32 | public: 33 | SyncObject() { 34 | } 35 | 36 | ~SyncObject() { 37 | } 38 | 39 | void wait(std::unique_lock& lock) { 40 | cond.wait(lock); 41 | } 42 | 43 | template 44 | void wait(std::unique_lock& lock, Predicate pred) { 45 | cond.wait(lock, pred); 46 | } 47 | 48 | void wait_for(std::unique_lock& lock, 49 | const double secs) { 50 | cond.wait_for(lock, std::chrono::milliseconds(int64_t(secs * 1000.0))); 51 | } 52 | 53 | void wait_for(std::unique_lock& lock, 54 | const std::chrono::nanoseconds nanoSecs) { 55 | cond.wait_for(lock, nanoSecs); 56 | } 57 | 58 | void notify_all() { 59 | cond.notify_all(); 60 | } 61 | 62 | void notify_one() { 63 | cond.notify_one(); 64 | } 65 | 66 | private: 67 | std::condition_variable cond; 68 | 69 | DISALLOW_COPY_AND_ASSIGN(SyncObject); 70 | }; 71 | 72 | #endif // SRC_SYNCOBJECT_H_ 73 | -------------------------------------------------------------------------------- /src/task_type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | enum task_type_t { 23 | NO_TASK_TYPE=-1, 24 | WRITER_TASK_IDX=0, 25 | READER_TASK_IDX=1, 26 | AUXIO_TASK_IDX=2, 27 | NONIO_TASK_IDX=3, 28 | NUM_TASK_GROUPS=4 // keep this as last element of the enum 29 | }; 30 | 31 | static inline std::string to_string(const task_type_t type) { 32 | switch (type) { 33 | case WRITER_TASK_IDX: 34 | return "writer"; 35 | case READER_TASK_IDX: 36 | return "reader"; 37 | case AUXIO_TASK_IDX: 38 | return "auxIO"; 39 | case NONIO_TASK_IDX: 40 | return "nonIO"; 41 | case NO_TASK_TYPE: 42 | return "NO_TASK_TYPE"; 43 | case NUM_TASK_GROUPS: 44 | return "NUM_TASK_GROUPS"; 45 | default: 46 | throw std::invalid_argument("to_string(task_type_t) unknown type:{" + 47 | std::to_string(int(type)) + "}"); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/taskable.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | An abstract baseclass for classes which wish to create and own tasks. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include 25 | 26 | #include "globaltask.h" 27 | #include "workload.h" 28 | 29 | /* 30 | A type for identifying all tasks belonging to a task owner. 31 | */ 32 | typedef uintptr_t task_gid_t; 33 | 34 | class Taskable { 35 | public: 36 | /* 37 | Return a name for the task, used for logging 38 | */ 39 | virtual const std::string& getName() const = 0; 40 | 41 | /* 42 | Return a 'group' ID for the task. 43 | 44 | The use-case here is to allow the lookup of all tasks 45 | owned by this taskable. 46 | 47 | The address of the object is a safe GID. 48 | */ 49 | virtual task_gid_t getGID() const = 0; 50 | 51 | /* 52 | Return the workload priority for the task 53 | */ 54 | virtual bucket_priority_t getWorkloadPriority() const = 0; 55 | 56 | /* 57 | Set the taskable object's workload priority. 58 | */ 59 | virtual void setWorkloadPriority(bucket_priority_t prio) = 0; 60 | 61 | /* 62 | Return the taskable object's workload policy. 63 | */ 64 | virtual WorkLoadPolicy& getWorkLoadPolicy() = 0; 65 | 66 | /* 67 | Called with the time spent queued 68 | */ 69 | virtual void logQTime(TaskId id, const ProcessClock::duration enqTime) = 0; 70 | 71 | /* 72 | Called with the time spent running 73 | */ 74 | virtual void logRunTime(TaskId id, 75 | const ProcessClock::duration runTime) = 0; 76 | 77 | protected: 78 | virtual ~Taskable() {} 79 | }; 80 | -------------------------------------------------------------------------------- /src/tasklogentry.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef SRC_TASKLOGENTRY_H_ 18 | #define SRC_TASKLOGENTRY_H_ 1 19 | 20 | #include "config.h" 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include "task_type.h" 33 | 34 | /** 35 | * Log entry for previous job runs. 36 | */ 37 | class TaskLogEntry { 38 | public: 39 | 40 | // This is useful for the ringbuffer to initialize 41 | TaskLogEntry() : name("invalid"), 42 | duration(ProcessClock::duration::zero()) {} 43 | 44 | TaskLogEntry(const std::string &n, task_type_t type, 45 | const ProcessClock::duration d, 46 | rel_time_t t = 0) 47 | : name(n), taskType(type), ts(t), duration(d) {} 48 | 49 | /** 50 | * Get the name of the job. 51 | */ 52 | std::string getName() const { return name; } 53 | 54 | /** 55 | * Get the type of the task (Writer, Reader, AuxIO, NonIO) 56 | */ 57 | task_type_t getTaskType() const { return taskType; } 58 | 59 | /** 60 | * Get the duration this job took to run. 61 | */ 62 | ProcessClock::duration getDuration() const { return duration; } 63 | 64 | /** 65 | * Get a timestamp indicating when this thing started. 66 | */ 67 | rel_time_t getTimestamp() const { return ts; } 68 | 69 | private: 70 | std::string name; 71 | task_type_t taskType; 72 | rel_time_t ts; 73 | ProcessClock::duration duration; 74 | }; 75 | 76 | #endif // SRC_TASKLOGENTRY_H_ 77 | -------------------------------------------------------------------------------- /src/taskqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef SRC_TASKQUEUE_H_ 18 | #define SRC_TASKQUEUE_H_ 1 19 | 20 | #include "config.h" 21 | 22 | #include "futurequeue.h" 23 | #include "syncobject.h" 24 | #include "task_type.h" 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | class ExecutorPool; 32 | class ExecutorThread; 33 | 34 | class TaskQueue { 35 | friend class ExecutorPool; 36 | public: 37 | TaskQueue(ExecutorPool *m, task_type_t t, const char *nm); 38 | ~TaskQueue(); 39 | 40 | void schedule(ExTask &task); 41 | 42 | ProcessClock::time_point reschedule(ExTask &task); 43 | 44 | void checkPendingQueue(void); 45 | 46 | void doWake(size_t &numToWake); 47 | 48 | bool fetchNextTask(ExecutorThread &thread, bool toSleep); 49 | 50 | void wake(ExTask &task); 51 | 52 | static const std::string taskType2Str(task_type_t type); 53 | 54 | const std::string getName() const; 55 | 56 | const task_type_t getQueueType() const { return queueType; } 57 | 58 | size_t getReadyQueueSize(); 59 | 60 | size_t getFutureQueueSize(); 61 | 62 | size_t getPendingQueueSize(); 63 | 64 | void snooze(ExTask& task, const double secs) { 65 | futureQueue.snooze(task, secs); 66 | } 67 | 68 | private: 69 | void _schedule(ExTask &task); 70 | ProcessClock::time_point _reschedule(ExTask &task); 71 | void _checkPendingQueue(void); 72 | bool _fetchNextTask(ExecutorThread &thread, bool toSleep); 73 | void _wake(ExTask &task); 74 | bool _doSleep(ExecutorThread &thread, std::unique_lock& lock); 75 | void _doWake_UNLOCKED(size_t &numToWake); 76 | size_t _moveReadyTasks(const ProcessClock::time_point tv); 77 | ExTask _popReadyTask(void); 78 | 79 | SyncObject mutex; 80 | const std::string name; 81 | task_type_t queueType; 82 | ExecutorPool *manager; 83 | size_t sleepers; // number of threads sleeping in this taskQueue 84 | 85 | // sorted by task priority. 86 | std::priority_queue, 87 | CompareByPriority> readyQueue; 88 | 89 | // sorted by waketime. 90 | FutureQueue<> futureQueue; 91 | 92 | std::list pendingQueue; 93 | }; 94 | 95 | #endif // SRC_TASKQUEUE_H_ 96 | -------------------------------------------------------------------------------- /src/testlogger.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2011 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "common.h" 21 | 22 | void LOG(EXTENSION_LOG_LEVEL severity, const char *fmt, ...) { 23 | (void) severity; 24 | (void) fmt; 25 | } 26 | -------------------------------------------------------------------------------- /src/threadlocal.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_THREADLOCAL_H_ 19 | #define SRC_THREADLOCAL_H_ 1 20 | 21 | // thread local variable dtor 22 | using ThreadLocalDestructor = void (*)(void*); 23 | 24 | #ifdef WIN32 25 | #include "threadlocal_win32.h" 26 | template 27 | using ThreadLocal = ThreadLocalWin32; 28 | #else 29 | #include "threadlocal_posix.h" 30 | template 31 | using ThreadLocal = ThreadLocalPosix; 32 | #endif 33 | 34 | /** 35 | * Container for a thread-local pointer. 36 | */ 37 | template 38 | class ThreadLocalPtr : public ThreadLocal { 39 | public: 40 | ThreadLocalPtr(ThreadLocalDestructor dtor = nullptr) 41 | : ThreadLocal(dtor) { 42 | } 43 | 44 | ~ThreadLocalPtr() {} 45 | 46 | T *operator ->() { 47 | return ThreadLocal::get(); 48 | } 49 | 50 | T operator *() { 51 | return *ThreadLocal::get(); 52 | } 53 | 54 | void operator =(T *newValue) { 55 | this->set(newValue); 56 | } 57 | }; 58 | 59 | #endif // SRC_THREADLOCAL_H_ 60 | -------------------------------------------------------------------------------- /src/threadlocal_posix.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_THREADLOCAL_POSIX_H_ 19 | #define SRC_THREADLOCAL_POSIX_H_ 1 20 | 21 | #ifndef SRC_THREADLOCAL_H_ 22 | #error "Include threadlocal.h instead" 23 | #endif 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #define MAX_THREADS 500 33 | 34 | /** 35 | * Container of thread-local data. 36 | */ 37 | template 38 | class ThreadLocalPosix { 39 | public: 40 | ThreadLocalPosix(ThreadLocalDestructor dtor = nullptr) : dtor(dtor) { 41 | int rc = pthread_key_create(&key, dtor); 42 | if (rc != 0) { 43 | std::string msg = "Failed to create a thread-specific key: "; 44 | msg.append(strerror(rc)); 45 | throw std::system_error(rc, std::system_category(), msg); 46 | } 47 | } 48 | 49 | ~ThreadLocalPosix() { 50 | int rc = pthread_key_delete(key); 51 | if (rc != 0) { 52 | std::cerr << "~ThreadLocalPosix() - pthread_key_delete(): " 53 | << strerror(rc) << std::endl; 54 | std::cerr.flush(); 55 | } 56 | } 57 | 58 | void set(const T &newValue) { 59 | int rc = pthread_setspecific(key, newValue); 60 | if (rc != 0) { 61 | std::string msg("Failed to store thread specific value: "); 62 | msg.append(strerror(rc)); 63 | throw std::system_error(rc, std::system_category(), msg); 64 | } 65 | } 66 | 67 | T get() const { 68 | return reinterpret_cast(pthread_getspecific(key)); 69 | } 70 | 71 | void operator =(const T &newValue) { 72 | set(newValue); 73 | } 74 | 75 | operator T() const { 76 | return get(); 77 | } 78 | 79 | private: 80 | pthread_key_t key; 81 | ThreadLocalDestructor dtor; 82 | }; 83 | 84 | #endif // SRC_THREADLOCAL_POSIX_H_ 85 | -------------------------------------------------------------------------------- /src/threadlocal_win32.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2014 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_THREADLOCAL_WIN32_H_ 19 | #define SRC_THREADLOCAL_WIN32_H_ 1 20 | 21 | #ifndef SRC_THREADLOCAL_H_ 22 | #error "Include threadlocal.h instead" 23 | #endif 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | 34 | /** 35 | * Container of thread-local data. 36 | */ 37 | template 38 | class ThreadLocalWin32 { 39 | public: 40 | ThreadLocalWin32(ThreadLocalDestructor dtor = nullptr) { 41 | tlsIndex = TlsAlloc(); 42 | if (tlsIndex == TLS_OUT_OF_INDEXES) { 43 | DWORD err = GetLastError(); 44 | std::string msg("Failed to create thread local storage: "); 45 | msg.append(cb_strerror(err)); 46 | throw std::system_error(err, std::system_category(), msg); 47 | } 48 | } 49 | 50 | ~ThreadLocalWin32() { 51 | if (!TlsFree(tlsIndex)) { 52 | std::cerr << "~ThreadLocalPosix() TlsFree: " 53 | << cb_strerror() << std::endl; 54 | std::cerr.flush(); 55 | } 56 | } 57 | 58 | void set(const T &newValue) { 59 | if (!TlsSetValue(tlsIndex, newValue)) { 60 | DWORD err = GetLastError(); 61 | std::string msg("Failed to store thread specific value: "); 62 | msg.append(cb_strerror(err)); 63 | throw std::system_error(err, std::system_category(), msg); 64 | } 65 | } 66 | 67 | T get() const { 68 | return reinterpret_cast(TlsGetValue(tlsIndex)); 69 | } 70 | 71 | void operator =(const T &newValue) { 72 | set(newValue); 73 | } 74 | 75 | operator T() const { 76 | return get(); 77 | } 78 | 79 | private: 80 | // No use in Win32. Only for compatibility 81 | ThreadLocalDestructor dtor; 82 | DWORD tlsIndex; 83 | }; 84 | 85 | #endif // SRC_THREADLOCAL_WIN32_H_ 86 | -------------------------------------------------------------------------------- /src/utility.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2015 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * High level, generic utility functions and macros for wide use across 20 | * ep-engine. 21 | * 22 | * Note: Please keep this file as tight as possible - it is included by *many* 23 | * other files and hence we want to keep the preprocessor/parse overhead 24 | * as low as possible. 25 | * 26 | * As a general rule don't add anything here unless at least 50% of 27 | * the source files in ep_engine need it. 28 | */ 29 | 30 | #pragma once 31 | 32 | #include 33 | 34 | // A macro to disallow the copy constructor and operator= functions 35 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 36 | TypeName(const TypeName&) = delete; \ 37 | void operator=(const TypeName&) = delete 38 | 39 | #define DISALLOW_ASSIGN(TypeName) \ 40 | void operator=(const TypeName&) = delete 41 | 42 | 43 | // Utility functions implemented in various modules. 44 | 45 | extern void LOG(EXTENSION_LOG_LEVEL severity, const char *fmt, ...) CB_FORMAT_PRINTF(2, 3); 46 | 47 | typedef struct engine_allocator_hooks_v1 ALLOCATOR_HOOKS_API; 48 | -------------------------------------------------------------------------------- /src/vb_count_visitor.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed undemor the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "vb_count_visitor.h" 19 | 20 | void VBucketCountVisitor::visitBucket(VBucketPtr& vb) { 21 | ++numVbucket; 22 | numItems += vb->getNumItems(); 23 | numTempItems += vb->getNumTempItems(); 24 | nonResident += vb->getNumNonResidentItems(); 25 | 26 | if (vb->getHighPriorityChkSize() > 0) { 27 | chkPersistRemaining++; 28 | } 29 | 30 | if (desired_state != vbucket_state_dead) { 31 | htMemory += vb->ht.memorySize(); 32 | htItemMemory += vb->ht.getItemMemory(); 33 | htCacheSize += vb->ht.cacheSize; 34 | numEjects += vb->ht.getNumEjects(); 35 | numExpiredItems += vb->numExpiredItems; 36 | metaDataMemory += vb->ht.metaDataMemory; 37 | metaDataDisk += vb->metaDataDisk; 38 | opsCreate += vb->opsCreate; 39 | opsUpdate += vb->opsUpdate; 40 | opsDelete += vb->opsDelete; 41 | opsReject += vb->opsReject; 42 | 43 | queueSize += vb->dirtyQueueSize; 44 | queueMemory += vb->dirtyQueueMem; 45 | queueFill += vb->dirtyQueueFill; 46 | queueDrain += vb->dirtyQueueDrain; 47 | queueAge += vb->getQueueAge(); 48 | backfillQueueSize += vb->getBackfillSize(); 49 | pendingWrites += vb->dirtyQueuePendingWrites; 50 | rollbackItemCount += vb->getRollbackItemCount(); 51 | numHpVBReqs += vb->getHighPriorityChkSize(); 52 | 53 | /* 54 | * The bucket stat reports the total drift of the vbuckets. 55 | */ 56 | auto absHLCDrift = vb->getHLCDriftStats(); 57 | totalAbsHLCDrift.total += absHLCDrift.total; 58 | totalAbsHLCDrift.updates += absHLCDrift.updates; 59 | 60 | /* 61 | * Total up the exceptions 62 | */ 63 | auto driftExceptionCounters = vb->getHLCDriftExceptionCounters(); 64 | totalHLCDriftExceptionCounters.ahead += driftExceptionCounters.ahead; 65 | totalHLCDriftExceptionCounters.behind += driftExceptionCounters.behind; 66 | 67 | // Iterate over each datatype combination 68 | for (uint8_t ii = 0; ii < datatypeCounts.size(); ++ii) { 69 | datatypeCounts[ii] += vb->ht.datatypeCounts[ii]; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/vbucketdeletiontask.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "globaltask.h" 21 | #include "vbucket.h" 22 | 23 | class EPVBucket; 24 | 25 | /* 26 | * This is a NONIO task called as part of VB deletion. The task is responsible 27 | * for clearing all the VBucket's pending operations and for deleting the 28 | * VBucket (via a smart pointer). 29 | * 30 | * This task is designed to be invoked only when the VBucket has no owners. 31 | */ 32 | class VBucketMemoryDeletionTask : public GlobalTask { 33 | public: 34 | /** 35 | * @param engine required for GlobalTask construction 36 | * @param vbucket the vbucket object to delete 37 | */ 38 | VBucketMemoryDeletionTask(EventuallyPersistentEngine& eng, 39 | VBucket* vbucket, 40 | TaskId tid = TaskId::VBucketMemoryDeletionTask); 41 | 42 | cb::const_char_buffer getDescription(); 43 | 44 | bool run(); 45 | 46 | protected: 47 | /** 48 | * Call vbucket->notifyAllPendingConnsFailed and optionally perform 49 | * notifyIOComplete 50 | * 51 | * @pararm notifyIfCookieSet set to true if the function should perform 52 | * notifyIOComplete on vbucket->getDeletingCookie() 53 | */ 54 | void notifyAllPendingConnsFailed(bool notifyIfCookieSet); 55 | 56 | /** 57 | * The vbucket we are deleting is stored in a unique_ptr for RAII deletion 58 | * once this task is finished and itself deleted, the VBucket will be 59 | * deleted. 60 | */ 61 | std::unique_ptr vbucket; 62 | std::string description; 63 | }; 64 | 65 | /* 66 | * This is an AUXIO task called as part of EPVBucket deletion. The task is 67 | * responsible for clearing all the VBucket's pending operations and for 68 | * clearing the VBucket's hash table and removing the disk file. 69 | * 70 | * This task is designed to be invoked only when the EPVBucket has no owners. 71 | */ 72 | class VBucketMemoryAndDiskDeletionTask : public VBucketMemoryDeletionTask { 73 | public: 74 | /** 75 | * This task will as part of construction increase the vbucket's disk 76 | * revision so that the delete can remove the file without new 77 | * instances of the same vbucket writing to the file. 78 | * 79 | * @param engine requird for GlobalTask construction 80 | * @param shard the KVShard to use for deletion 81 | * @param vbucket the Eventually Persistent vbucket object to delete 82 | */ 83 | VBucketMemoryAndDiskDeletionTask(EventuallyPersistentEngine& engine, 84 | KVShard& shard, 85 | EPVBucket* vbucket); 86 | 87 | bool run(); 88 | 89 | protected: 90 | KVShard& shard; 91 | uint64_t vbDeleteRevision; 92 | }; -------------------------------------------------------------------------------- /src/workload.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2013 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_WORKLOAD_H_ 19 | #define SRC_WORKLOAD_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | #include 26 | 27 | enum bucket_priority_t { 28 | HIGH_BUCKET_PRIORITY=6, 29 | LOW_BUCKET_PRIORITY=2, 30 | NO_BUCKET_PRIORITY=0 31 | }; 32 | 33 | enum workload_pattern_t { 34 | READ_HEAVY, 35 | WRITE_HEAVY, 36 | MIXED 37 | }; 38 | 39 | /** 40 | * Workload optimization policy 41 | */ 42 | class WorkLoadPolicy { 43 | public: 44 | WorkLoadPolicy(int m, int s) 45 | : maxNumWorkers(m), maxNumShards(s), workloadPattern(READ_HEAVY) { } 46 | 47 | size_t getNumShards(void) { 48 | return maxNumShards; 49 | } 50 | 51 | bucket_priority_t getBucketPriority(void) { 52 | if (maxNumWorkers < HIGH_BUCKET_PRIORITY) { 53 | return LOW_BUCKET_PRIORITY; 54 | } 55 | return HIGH_BUCKET_PRIORITY; 56 | } 57 | 58 | size_t getNumWorkers(void) { 59 | return maxNumWorkers; 60 | } 61 | 62 | workload_pattern_t getWorkLoadPattern(void) { 63 | return workloadPattern.load(); 64 | } 65 | 66 | std::string stringOfWorkLoadPattern(void) { 67 | switch (workloadPattern.load()) { 68 | case READ_HEAVY: 69 | return "read_heavy"; 70 | case WRITE_HEAVY: 71 | return "write_heavy"; 72 | default: 73 | return "mixed"; 74 | } 75 | } 76 | 77 | void setWorkLoadPattern(workload_pattern_t pattern) { 78 | workloadPattern.store(pattern); 79 | } 80 | 81 | private: 82 | 83 | int maxNumWorkers; 84 | int maxNumShards; 85 | std::atomic workloadPattern; 86 | }; 87 | 88 | #endif // SRC_WORKLOAD_H_ 89 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | hash_table_test 2 | -------------------------------------------------------------------------------- /tests/mock/mock_basic_ll.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Mock of the BasicLinkedList class. Wraps the real BasicLinkedList class 20 | * and provides access to functions like getAllItemsForVerification(). 21 | */ 22 | #pragma once 23 | 24 | #include "config.h" 25 | #include "linked_list.h" 26 | 27 | #include 28 | #include 29 | 30 | class MockBasicLinkedList : public BasicLinkedList { 31 | public: 32 | MockBasicLinkedList(EPStats& st) : BasicLinkedList(0, st) { 33 | } 34 | 35 | OrderedLL& getSeqList() { 36 | return seqList; 37 | } 38 | 39 | std::vector getAllSeqnoForVerification() const { 40 | std::vector allSeqnos; 41 | std::lock_guard lckGd(writeLock); 42 | 43 | for (auto& val : seqList) { 44 | allSeqnos.push_back(val.getBySeqno()); 45 | } 46 | return allSeqnos; 47 | } 48 | 49 | /// Expose the rangeReadLock for testing. 50 | std::mutex& getRangeReadLock() { 51 | return rangeReadLock; 52 | } 53 | 54 | /* Register fake read range for testing */ 55 | void registerFakeReadRange(seqno_t start, seqno_t end) { 56 | std::lock_guard lh(rangeLock); 57 | readRange = SeqRange(start, end); 58 | } 59 | 60 | void resetReadRange() { 61 | std::lock_guard lh(rangeLock); 62 | readRange.reset(); 63 | } 64 | }; 65 | -------------------------------------------------------------------------------- /tests/mock/mock_dcp.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2013 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef TESTS_MOCK_MOCH_dcp_H_ 19 | #define TESTS_MOCK_MOCH_dcp_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | #include 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | extern std::vector > dcp_failover_log; 31 | 32 | ENGINE_ERROR_CODE mock_dcp_add_failover_log(vbucket_failover_t* entry, 33 | size_t nentries, 34 | const void *cookie); 35 | 36 | #ifdef __cplusplus 37 | } 38 | #endif 39 | 40 | void clear_dcp_data(); 41 | 42 | std::unique_ptr get_dcp_producers(ENGINE_HANDLE *_h, 43 | ENGINE_HANDLE_V1 *_h1); 44 | 45 | #endif // TESTS_MOCK_MOCH_dcp_H_ 46 | -------------------------------------------------------------------------------- /tests/mock/mock_dcp_backfill_mgr.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "dcp/backfill-manager.h" 21 | 22 | /* 23 | * Mock of the BackfillManager class. Wraps the real BackfillManager, but 24 | * exposes normally protected methods publically for test purposes. 25 | */ 26 | class MockDcpBackfillManager : public BackfillManager { 27 | public: 28 | MockDcpBackfillManager(EventuallyPersistentEngine& theEngine) 29 | : BackfillManager(theEngine) { 30 | } 31 | 32 | void setBackfillBufferSize(size_t newSize) { 33 | buffer.maxBytes = newSize; 34 | } 35 | 36 | bool getBackfillBufferFullStatus() { 37 | return buffer.full; 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /tests/mock/mock_dcp_consumer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "dcp/consumer.h" 21 | #include "mock_stream.h" 22 | 23 | /* 24 | * Mock of the DcpConsumer class. Wraps the real DcpConsumer class 25 | * and provides get/set access to lastMessageTime. 26 | */ 27 | class MockDcpConsumer: public DcpConsumer { 28 | public: 29 | MockDcpConsumer(EventuallyPersistentEngine &theEngine, const void *cookie, 30 | const std::string &name) 31 | : DcpConsumer(theEngine, cookie, name) 32 | {} 33 | 34 | void setLastMessageTime(const rel_time_t timeValue) { 35 | lastMessageTime = timeValue; 36 | } 37 | 38 | rel_time_t getLastMessageTime() { 39 | return lastMessageTime; 40 | } 41 | 42 | SingleThreadedRCPtr getVbucketStream(uint16_t vbid) { 43 | return findStream(vbid); 44 | } 45 | 46 | void public_notifyVbucketReady(uint16_t vbid) { 47 | notifyVbucketReady(vbid); 48 | } 49 | 50 | /* 51 | * Creates a PassiveStream. 52 | * @return a SingleThreadedRCPtr to the newly created MockPassiveStream. 53 | */ 54 | SingleThreadedRCPtr makePassiveStream( 55 | EventuallyPersistentEngine& e, 56 | dcp_consumer_t consumer, 57 | const std::string& name, 58 | uint32_t flags, 59 | uint32_t opaque, 60 | uint16_t vb, 61 | uint64_t start_seqno, 62 | uint64_t end_seqno, 63 | uint64_t vb_uuid, 64 | uint64_t snap_start_seqno, 65 | uint64_t snap_end_seqno, 66 | uint64_t vb_high_seqno) override { 67 | return SingleThreadedRCPtr( 68 | new MockPassiveStream(e, 69 | consumer, 70 | name, 71 | flags, 72 | opaque, 73 | vb, 74 | start_seqno, 75 | end_seqno, 76 | vb_uuid, 77 | snap_start_seqno, 78 | snap_end_seqno, 79 | vb_high_seqno)); 80 | } 81 | 82 | }; 83 | -------------------------------------------------------------------------------- /tests/mock/mock_ephemeral_vb.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Mock of the BasicLinkedList class. Wraps the real BasicLinkedList class 20 | * and provides access to functions like getAllItemsForVerification(). 21 | */ 22 | #pragma once 23 | 24 | #include "../mock/mock_basic_ll.h" 25 | #include "config.h" 26 | #include "ephemeral_vb.h" 27 | 28 | #include 29 | #include 30 | 31 | class MockEphemeralVBucket : public EphemeralVBucket { 32 | public: 33 | MockEphemeralVBucket(id_type i, 34 | vbucket_state_t newState, 35 | EPStats& st, 36 | CheckpointConfig& chkConfig, 37 | KVShard* kvshard, 38 | int64_t lastSeqno, 39 | uint64_t lastSnapStart, 40 | uint64_t lastSnapEnd, 41 | std::unique_ptr table, 42 | NewSeqnoCallback newSeqnoCb, 43 | Configuration& config, 44 | item_eviction_policy_t evictionPolicy) 45 | : EphemeralVBucket(i, 46 | newState, 47 | st, 48 | chkConfig, 49 | kvshard, 50 | lastSeqno, 51 | lastSnapStart, 52 | lastSnapEnd, 53 | std::move(table), 54 | std::move(newSeqnoCb), 55 | config, 56 | evictionPolicy) { 57 | /* we want MockBasicLinkedList instead to call certain non-public 58 | APIs of BasicLinkedList in ephemeral vbucket */ 59 | this->seqList = std::make_unique(st); 60 | mockLL = dynamic_cast((this->seqList).get()); 61 | } 62 | 63 | /* Register fake read range for testing */ 64 | void registerFakeReadRange(seqno_t start, seqno_t end) { 65 | mockLL->registerFakeReadRange(start, end); 66 | } 67 | 68 | void resetReadRange() { 69 | mockLL->resetReadRange(); 70 | } 71 | 72 | int public_getNumStaleItems() { 73 | return mockLL->getNumStaleItems(); 74 | } 75 | 76 | int public_getNumListItems() { 77 | return mockLL->getNumItems(); 78 | } 79 | 80 | MockBasicLinkedList* getLL() { 81 | return mockLL; 82 | } 83 | 84 | private: 85 | /* non owning ptr to the linkedlist in the ephemeral vbucket obj */ 86 | MockBasicLinkedList* mockLL; 87 | }; 88 | -------------------------------------------------------------------------------- /tests/mock/mock_global_task.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "globaltask.h" 21 | 22 | #include 23 | 24 | /** 25 | * Mock Task class. Doesn't actually run() or snooze() - they both do nothing. 26 | */ 27 | class MockGlobalTask : public GlobalTask { 28 | public: 29 | MockGlobalTask(Taskable& t, TaskId id) : GlobalTask(t, id) { 30 | } 31 | 32 | bool run() override { 33 | return false; 34 | } 35 | cb::const_char_buffer getDescription() override { 36 | return "MockGlobalTask"; 37 | } 38 | 39 | void snooze(const double secs) override { 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /tests/mock/mock_synchronous_ep_engine.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "mock_synchronous_ep_engine.h" 19 | 20 | #include 21 | #include 22 | #include "dcp/dcpconnmap.h" 23 | #include "dcp/flow-control-manager.h" 24 | #include "replicationthrottle.h" 25 | #include "tapconnmap.h" 26 | 27 | #include 28 | 29 | SynchronousEPEngine::SynchronousEPEngine(const std::string& extra_config) 30 | : EventuallyPersistentEngine(get_mock_server_api) { 31 | 32 | // Tests may need to create multiple failover table entries, so allow that 33 | maxFailoverEntries = 5; 34 | 35 | // Merge any extra config into the main configuration. 36 | if (extra_config.size() > 0) { 37 | if (!configuration.parseConfiguration(extra_config.c_str(), 38 | serverApi)) { 39 | throw std::invalid_argument("Unable to parse config string: " + 40 | extra_config); 41 | } 42 | } 43 | 44 | // workload is needed by EPStore's constructor (to construct the 45 | // VBucketMap). 46 | workload = new WorkLoadPolicy(/*workers*/ 1, /*shards*/ 1); 47 | 48 | // dcpConnMap_ is needed by EPStore's constructor. 49 | dcpConnMap_ = new DcpConnMap(*this); 50 | 51 | // tapConnMap is needed by queueDirty. 52 | tapConnMap = new TapConnMap(*this); 53 | 54 | // checkpointConfig is needed by CheckpointManager (via EPStore). 55 | checkpointConfig = new CheckpointConfig(*this); 56 | 57 | dcpFlowControlManager_ = new DcpFlowControlManager(*this); 58 | 59 | replicationThrottle = new ReplicationThrottle(configuration, stats); 60 | 61 | tapConfig = new TapConfig(*this); 62 | 63 | enableTraffic(true); 64 | 65 | maxItemSize = configuration.getMaxItemSize(); 66 | } 67 | 68 | void SynchronousEPEngine::setKVBucket(std::unique_ptr store) { 69 | cb_assert(kvBucket == nullptr); 70 | kvBucket = std::move(store); 71 | } 72 | 73 | void SynchronousEPEngine::initializeConnmaps() { 74 | dcpConnMap_->initialize(DCP_CONN_NOTIFIER); 75 | tapConnMap->initialize(TAP_CONN_NOTIFIER); 76 | } 77 | -------------------------------------------------------------------------------- /tests/mock/mock_synchronous_ep_engine.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | 23 | /* A class which subclasses the real EPEngine. Its main purpose is to allow 24 | * us to construct and setup an EPStore without starting all the various 25 | * background tasks which are normally started by EPEngine as part of creating 26 | * EPStore (in the initialize() method). 27 | * 28 | * The net result is a (mostly) synchronous environment - while the 29 | * ExecutorPool's threads exist, none of the normally-created background Tasks 30 | * should be running. Note however that /if/ any new tasks are created, they 31 | * will be scheduled on the ExecutorPools' threads asynchronously. 32 | */ 33 | class SynchronousEPEngine : public EventuallyPersistentEngine { 34 | public: 35 | SynchronousEPEngine(const std::string& extra_config); 36 | 37 | void setKVBucket(std::unique_ptr store); 38 | 39 | /* Allow us to call normally protected methods */ 40 | 41 | ENGINE_ERROR_CODE public_doTapVbTakeoverStats(const void* cookie, 42 | ADD_STAT add_stat, 43 | std::string& key, 44 | uint16_t vbid) { 45 | return doTapVbTakeoverStats(cookie, add_stat, key, vbid); 46 | } 47 | 48 | ENGINE_ERROR_CODE public_doDcpVbTakeoverStats(const void* cookie, 49 | ADD_STAT add_stat, 50 | std::string& key, 51 | uint16_t vbid) { 52 | return doDcpVbTakeoverStats(cookie, add_stat, key, vbid); 53 | } 54 | 55 | void public_initializeEngineCallbacks() { 56 | return initializeEngineCallbacks(); 57 | } 58 | 59 | /* 60 | * Initialize the connmap objects, which creates tasks 61 | * so must be done after executorpool is created 62 | */ 63 | void initializeConnmaps(); 64 | 65 | std::unique_ptr public_makeBucket(Configuration& config) { 66 | return makeBucket(config); 67 | } 68 | 69 | bool public_enableTraffic(bool enable) { 70 | return enableTraffic(enable); 71 | } 72 | }; 73 | -------------------------------------------------------------------------------- /tests/module_tests/collections/collection_dockey_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "collections/collections_dockey.h" 19 | #include "tests/module_tests/test_helpers.h" 20 | 21 | #include 22 | 23 | TEST(CollectionDocKeyTest, make) { 24 | auto key1 = makeStoredDocKey("beer::bud", DocNamespace::Collections); 25 | auto key2 = makeStoredDocKey("beerbud", DocNamespace::Collections); 26 | 27 | EXPECT_EQ(strlen("beer"), 28 | Collections::DocKey::make(key1, "::").getCollectionLen()); 29 | 30 | EXPECT_EQ(0, Collections::DocKey::make(key2, "::").getCollectionLen()); 31 | 32 | EXPECT_EQ(0, Collections::DocKey::make(key1, "##").getCollectionLen()); 33 | 34 | // This case is if the separator and the key are the same thing 35 | // The collection len is 0 36 | EXPECT_EQ(0, 37 | Collections::DocKey::make(key1, "beer::bud").getCollectionLen()); 38 | 39 | // If a key is beer::brewery and the separtor is brewery then the collection 40 | // is beer:: 41 | EXPECT_EQ(strlen("beer::"), 42 | Collections::DocKey::make(key1, "bud").getCollectionLen()); 43 | 44 | EXPECT_EQ(0, 45 | Collections::DocKey::make(key1, "longerthanthekey") 46 | .getCollectionLen()); 47 | } -------------------------------------------------------------------------------- /tests/module_tests/couch-fs-stats_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "tests/wrapped_fileops_test.h" 19 | #include "src/couch-kvstore/couch-fs-stats.h" 20 | 21 | class TestStatsOps : public StatsOps { 22 | public: 23 | TestStatsOps(FileOpsInterface* ops) 24 | : StatsOps(_stats, *ops), 25 | wrapped_ops(ops) {} 26 | 27 | couch_file_handle constructor(couchstore_error_info_t *errinfo) override { 28 | FileOpsInterface* orig_ops = wrapped_ops.get(); 29 | StatFile* sf = new StatFile(orig_ops, 30 | orig_ops->constructor(errinfo), 31 | 0); 32 | return reinterpret_cast(sf); 33 | } 34 | protected: 35 | FileStats _stats; 36 | std::unique_ptr wrapped_ops; 37 | }; 38 | 39 | typedef testing::Types 40 | WrappedOpsImplementation; 41 | 42 | INSTANTIATE_TYPED_TEST_CASE_P(CouchstoreOpsTest, 43 | WrappedOpsTest, 44 | WrappedOpsImplementation 45 | ); 46 | 47 | INSTANTIATE_TYPED_TEST_CASE_P(CouchstoreOpsTest, 48 | UnbufferedWrappedOpsTest, 49 | WrappedOpsImplementation 50 | ); -------------------------------------------------------------------------------- /tests/module_tests/defragmenter_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "vbucket_test.h" 21 | 22 | #include 23 | 24 | class DefragmenterTest : public VBucketTest { 25 | public: 26 | static void SetUpTestCase() { 27 | 28 | // Setup the MemoryTracker. 29 | MemoryTracker::getInstance(*get_mock_server_api()->alloc_hooks); 30 | } 31 | 32 | static void TearDownTestCase() { 33 | MemoryTracker::destroyInstance(); 34 | } 35 | 36 | protected: 37 | void SetUp() override { 38 | // Setup object registry. As we do not create a full ep-engine, we 39 | // use the "initial_tracking" for all memory tracking". 40 | ObjectRegistry::setStats(&mem_used); 41 | VBucketTest::SetUp(); 42 | } 43 | 44 | void TearDown() override { 45 | ObjectRegistry::setStats(nullptr); 46 | VBucketTest::TearDown(); 47 | } 48 | 49 | /** 50 | * Sets num_docs documents each with a doc size of docSize. This method 51 | * avoids polluting the heap to assist in measuring memory usage. 52 | * 53 | * @param docSize The size in bytes of each items blob 54 | * @param num_docs The number of docs to set 55 | */ 56 | void setDocs(size_t docSize, size_t num_docs); 57 | 58 | /** 59 | * Remove all but one document in each page. This is to create a situation 60 | * where the defragmenter runs for every document. 61 | * 62 | * @param num_docs The number of docs that have been set 63 | * @param[out] num_remaining The resulting number documents that are left after 64 | * the fragmentation 65 | */ 66 | void fragment(size_t num_docs, size_t &num_remaining); 67 | 68 | // Track of memory used (from ObjectRegistry). 69 | std::atomic mem_used{0}; 70 | }; 71 | -------------------------------------------------------------------------------- /tests/module_tests/ep_unit_tests_main.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Main function & globals for the ep_unit_test target. 20 | */ 21 | 22 | #include 23 | #include "programs/engine_testapp/mock_server.h" 24 | 25 | #include 26 | #include 27 | 28 | #include "configuration.h" 29 | #include "logger.h" 30 | #include "hash_table.h" 31 | 32 | /* static storage for environment variable set by putenv(). */ 33 | static char allow_no_stats_env[] = "ALLOW_NO_STATS_UPDATE=yeah"; 34 | 35 | 36 | int main(int argc, char **argv) { 37 | bool log_to_stderr = false; 38 | // Parse command-line options. 39 | int cmd; 40 | bool invalid_argument = false; 41 | while (!invalid_argument && 42 | (cmd = getopt(argc, argv, "v")) != EOF) { 43 | switch (cmd) { 44 | case 'v': 45 | log_to_stderr = true; 46 | break; 47 | default: 48 | std::cerr << "Usage: " << argv[0] << " [-v] [gtest_options...]" << std::endl 49 | << std::endl 50 | << " -v Verbose - Print verbose output to stderr." 51 | << std::endl << std::endl; 52 | invalid_argument = true; 53 | break; 54 | } 55 | } 56 | 57 | putenv(allow_no_stats_env); 58 | 59 | mock_init_alloc_hooks(); 60 | init_mock_server(log_to_stderr); 61 | 62 | get_mock_server_api()->log->set_level(EXTENSION_LOG_DEBUG); 63 | if (memcached_initialize_stderr_logger(get_mock_server_api) != EXTENSION_SUCCESS) { 64 | std::cerr << argv[0] << ": Failed to initialize log system" << std::endl; 65 | return 1; 66 | } 67 | Logger::setLoggerAPI(get_mock_server_api()->log); 68 | 69 | // Need to initialize ep_real_time and friends. 70 | initialize_time_functions(get_mock_server_api()->core); 71 | 72 | ::testing::InitGoogleTest(&argc, argv); 73 | return RUN_ALL_TESTS(); 74 | } 75 | -------------------------------------------------------------------------------- /tests/module_tests/ephemeral_bucket_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "ephemeral_bucket_test.h" 19 | 20 | #include "test_helpers.h" 21 | 22 | /* 23 | * Test statistics related to an individual VBucket's sequence list. 24 | */ 25 | 26 | void EphemeralBucketStatTest::addDocumentsForSeqListTesting(uint16_t vb) { 27 | // Add some documents to the vBucket to use to test the stats. 28 | store_item(vb, makeStoredDocKey("deleted"), "value"); 29 | delete_item(vb, makeStoredDocKey("deleted")); 30 | store_item(vb, makeStoredDocKey("doc"), "value"); 31 | store_item(vb, makeStoredDocKey("doc"), "value 2"); 32 | } 33 | 34 | TEST_F(EphemeralBucketStatTest, VBSeqlistStats) { 35 | // Check preconditions. 36 | auto stats = get_stat("vbucket-details 0"); 37 | ASSERT_EQ("0", stats.at("vb_0:seqlist_high_seqno")); 38 | 39 | // Add some documents to the vBucket to use to test the stats. 40 | addDocumentsForSeqListTesting(vbid); 41 | 42 | stats = get_stat("vbucket-details 0"); 43 | 44 | EXPECT_EQ("0", stats.at("vb_0:auto_delete_count")); 45 | EXPECT_EQ("2", stats.at("vb_0:seqlist_count")) 46 | << "Expected both current and deleted documents"; 47 | EXPECT_EQ("1", stats.at("vb_0:seqlist_deleted_count")); 48 | EXPECT_EQ("4", stats.at("vb_0:seqlist_high_seqno")); 49 | EXPECT_EQ("4", stats.at("vb_0:seqlist_highest_deduped_seqno")); 50 | EXPECT_EQ("0", stats.at("vb_0:seqlist_range_read_begin")); 51 | EXPECT_EQ("0", stats.at("vb_0:seqlist_range_read_end")); 52 | EXPECT_EQ("0", stats.at("vb_0:seqlist_range_read_count")); 53 | EXPECT_EQ("0", stats.at("vb_0:seqlist_stale_count")); 54 | EXPECT_EQ("0", stats.at("vb_0:seqlist_stale_value_bytes")); 55 | EXPECT_EQ("0", stats.at("vb_0:seqlist_stale_metadata_bytes")); 56 | 57 | // Trigger the "automatic" deletion of an item by paging it out. 58 | auto vb = store->getVBucket(vbid); 59 | auto key = makeStoredDocKey("doc"); 60 | auto lock = vb->ht.getLockedBucket(key); 61 | auto* value = vb->fetchValidValue( 62 | lock, key, WantsDeleted::No, TrackReference::Yes, QueueExpired::No); 63 | ASSERT_TRUE(vb->pageOut(lock, value)); 64 | 65 | stats = get_stat("vbucket-details 0"); 66 | EXPECT_EQ("1", stats.at("vb_0:auto_delete_count")); 67 | EXPECT_EQ("2", stats.at("vb_0:seqlist_deleted_count")); 68 | EXPECT_EQ("5", stats.at("vb_0:seqlist_high_seqno")); 69 | } 70 | -------------------------------------------------------------------------------- /tests/module_tests/ephemeral_bucket_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | * Unit tests for the EphemeralBucket class. 20 | */ 21 | 22 | #include "config.h" 23 | 24 | #include "stats_test.h" 25 | 26 | /** 27 | * Ephemeral Bucket test fixture class for statistics tests. 28 | */ 29 | class EphemeralBucketStatTest : public StatTest { 30 | protected: 31 | void SetUp() override { 32 | config_string += "bucket_type=ephemeral"; 33 | StatTest::SetUp(); 34 | } 35 | 36 | /// Add a number of documents to allow testing of sequence list stats. 37 | void addDocumentsForSeqListTesting(uint16_t vbid); 38 | }; 39 | -------------------------------------------------------------------------------- /tests/module_tests/evp_engine_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Unit tests for the EventuallyPersistentEngine class. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "config.h" 25 | 26 | #include 27 | 28 | #include 29 | 30 | class EventuallyPersistentEngine; 31 | 32 | class EventuallyPersistentEngineTest : public ::testing::Test { 33 | public: 34 | EventuallyPersistentEngineTest() : bucketType("persistent") { 35 | } 36 | 37 | protected: 38 | void SetUp() override; 39 | 40 | void TearDown() override; 41 | 42 | /* Helper methods for tests */ 43 | 44 | /* Stores an item into the given vbucket. */ 45 | void store_item(uint16_t vbid, const std::string& key, 46 | const std::string& value); 47 | 48 | std::string config_string; 49 | 50 | static const char test_dbname[]; 51 | 52 | const uint16_t vbid = 0; 53 | 54 | ENGINE_HANDLE* handle; 55 | ENGINE_HANDLE_V1* engine_v1; 56 | EventuallyPersistentEngine* engine; 57 | std::string bucketType; 58 | }; 59 | 60 | /* Tests parameterised over ephemeral and persistent buckets 61 | * 62 | */ 63 | class SetParamTest : public EventuallyPersistentEngineTest, 64 | public ::testing::WithParamInterface { 65 | void SetUp() override { 66 | bucketType = GetParam(); 67 | EventuallyPersistentEngineTest::SetUp(); 68 | } 69 | }; -------------------------------------------------------------------------------- /tests/module_tests/evp_store_single_threaded_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Unit tests for the EPBucket class. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "evp_store_test.h" 25 | #include "fakes/fake_executorpool.h" 26 | 27 | /* 28 | * A subclass of KVBucketTest which uses a fake ExecutorPool, 29 | * which will not spawn ExecutorThreads and hence not run any tasks 30 | * automatically in the background. All tasks must be manually run(). 31 | */ 32 | class SingleThreadedKVBucketTest : public KVBucketTest { 33 | public: 34 | /* 35 | * Run the next task from the taskQ 36 | * The task must match the expectedTaskName parameter 37 | */ 38 | ProcessClock::time_point runNextTask(TaskQueue& taskQ, 39 | const std::string& expectedTaskName); 40 | 41 | /* 42 | * Run the next task from the taskQ 43 | */ 44 | ProcessClock::time_point runNextTask(TaskQueue& taskQ); 45 | 46 | protected: 47 | void SetUp() override; 48 | 49 | void TearDown() override; 50 | 51 | /* 52 | * Change the vbucket state, and run the VBStatePeristTask (if necessary 53 | * for this bucket type). 54 | * On return the state will be changed and the task completed. 55 | */ 56 | void setVBucketStateAndRunPersistTask(uint16_t vbid, 57 | vbucket_state_t newState); 58 | 59 | /* 60 | * Set the stats isShutdown and attempt to drive all tasks to cancel 61 | */ 62 | void shutdownAndPurgeTasks(); 63 | 64 | void cancelAndPurgeTasks(); 65 | 66 | /* 67 | * Fake callback emulating dcp_add_failover_log 68 | */ 69 | static ENGINE_ERROR_CODE fakeDcpAddFailoverLog(vbucket_failover_t* entry, 70 | size_t nentries, 71 | const void *cookie) { 72 | return ENGINE_SUCCESS; 73 | } 74 | 75 | SingleThreadedExecutorPool* task_executor; 76 | }; 77 | 78 | /** 79 | * Test fixture for single-threaded tests on EPBucket. 80 | */ 81 | class SingleThreadedEPBucketTest : public SingleThreadedKVBucketTest { 82 | // Note this class is currently identical to it's parent class as the 83 | // default bucket_type in configuration.json is EPBucket, therefore 84 | // SingleThreadedKVBucketTest already defaults to creating EPBucket. 85 | // Introducing this subclass to just make the name more descriptive. 86 | }; 87 | -------------------------------------------------------------------------------- /tests/module_tests/evp_store_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Unit tests for the EPBucket class. 20 | * 21 | * These are instantiated for value and full eviction persistent buckets. 22 | */ 23 | 24 | #pragma once 25 | 26 | #include "config.h" 27 | 28 | #include "kv_bucket_test.h" 29 | 30 | /** 31 | * Test fixture for EPBucket unit tests. 32 | */ 33 | class EPBucketTest : public KVBucketTest { 34 | // Note this class is currently identical to it's parent class as the 35 | // default bucket_type in configuration.json is EPBucket, therefore 36 | // KVBucketTest already defaults to creating EPBucket. Introducing this 37 | // subclass to just make the name more descriptive. 38 | }; -------------------------------------------------------------------------------- /tests/module_tests/expiry_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import time 5 | import exceptions 6 | 7 | sys.path.append('../management') 8 | import mc_bin_client 9 | 10 | EXPIRY = 2 11 | VERBOSE = False 12 | 13 | def log(*s): 14 | if VERBOSE: 15 | print ' '.join(s) 16 | 17 | def store(mc, kprefix): 18 | k = kprefix + '.set' 19 | log(">> ", k) 20 | mc.set(k, EXPIRY, 0, k) 21 | k = kprefix + '.add' 22 | log("++ ", k) 23 | mc.add(k, EXPIRY, 0, k) 24 | 25 | def check(mc, kprefix): 26 | for suffix in ['.set', '.add']: 27 | try: 28 | k = kprefix + suffix 29 | log("<< ", k) 30 | mc.get(k) 31 | raise exceptions.Exception("Expected to fail to get " + k) 32 | except mc_bin_client.MemcachedError: 33 | pass 34 | 35 | if __name__ == '__main__': 36 | mc = mc_bin_client.MemcachedClient() 37 | 38 | if '-v' in sys.argv: 39 | VERBOSE = True 40 | 41 | store(mc, 'a1') 42 | store(mc, 'a2') 43 | log("(sleep)") 44 | time.sleep(EXPIRY + 1) 45 | check(mc, 'a2') 46 | store(mc, 'a1') 47 | store(mc, 'a2') 48 | -------------------------------------------------------------------------------- /tests/module_tests/hrtime_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "common.h" 21 | #undef NDEBUG 22 | 23 | int main(void) 24 | { 25 | hrtime_t val = 0; 26 | cb_assert(hrtime2text(val) == std::string("0 ns")); 27 | val = 9; 28 | cb_assert(hrtime2text(val) == std::string("9 ns")); 29 | val = 99; 30 | cb_assert(hrtime2text(val) == std::string("99 ns")); 31 | val = 999; 32 | cb_assert(hrtime2text(val) == std::string("999 ns")); 33 | val = 9999; 34 | cb_assert(hrtime2text(val) == std::string("9999 ns")); 35 | val *= 1000; 36 | cb_assert(hrtime2text(val) == std::string("9999 usec")); 37 | val *= 1000; 38 | cb_assert(hrtime2text(val) == std::string("9999 ms")); 39 | val = 599; 40 | val *= 1000*1000*1000; 41 | cb_assert(hrtime2text(val) == std::string("599 s")); 42 | val = 600; 43 | val *= 1000*1000*1000; 44 | cb_assert(hrtime2text(val) == std::string("0h:10m:0s")); 45 | val = 1; 46 | cb_assert(hrtime2text(val) == std::string("1 ns")); 47 | val = 10; 48 | cb_assert(hrtime2text(val) == std::string("10 ns")); 49 | val = 100; 50 | cb_assert(hrtime2text(val) == std::string("100 ns")); 51 | val = 1000; 52 | cb_assert(hrtime2text(val) == std::string("1000 ns")); 53 | val = 10000; 54 | cb_assert(hrtime2text(val) == std::string("10 usec")); 55 | val *= 1000; 56 | cb_assert(hrtime2text(val) == std::string("10 ms")); 57 | val *= 1000; 58 | cb_assert(hrtime2text(val) == std::string("10 s")); 59 | 60 | hrtime_t now = gethrtime(); 61 | usleep(200); 62 | hrtime_t later = gethrtime(); 63 | cb_assert(now + 200 < later); 64 | 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /tests/module_tests/memory_tracker_test.cc: -------------------------------------------------------------------------------- 1 | #include "config.h" 2 | 3 | #include "memory_tracker.h" 4 | #include "mock_hooks_api.h" 5 | 6 | #include 7 | 8 | #include 9 | 10 | 11 | // Test that creating the singleton MemoryTracker is thread-safe (MB-18940) 12 | TEST(MemoryTracker, SingletonIsThreadSafe_MB18940) { 13 | // Call getInstance in two different threads in parallel, should only 14 | // have one MemoryTracker created. 15 | // Note: Relies on ThreadSanitizer or similar to detect data race (or 16 | // getting lucky that you fail on first attempt). Consider running 17 | // multiple times (--gtest_repeat=XXX) to expose issue more easily. 18 | 19 | MemoryTracker* instance1; 20 | MemoryTracker* instance2; 21 | 22 | auto alloc_hooks = *getHooksApi(); 23 | 24 | // Lambda which returns (via reference param) the result of getInstance(). 25 | auto get_instance = [alloc_hooks](MemoryTracker*& t) { 26 | t = MemoryTracker::getInstance(alloc_hooks); 27 | }; 28 | 29 | // Spin up two threads to both call getInstance (and hence attempt to 30 | // create the singleton). 31 | std::thread t1{get_instance, std::ref(instance1)}; 32 | std::thread t2{get_instance, std::ref(instance2)}; 33 | 34 | t1.join(); 35 | t2.join(); 36 | 37 | MemoryTracker::destroyInstance(); 38 | 39 | EXPECT_EQ(instance1, instance2); 40 | } 41 | -------------------------------------------------------------------------------- /tests/module_tests/misc_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #ifdef _MSC_VER 21 | #define alarm(a) 22 | #endif 23 | 24 | #include 25 | 26 | #include "common.h" 27 | 28 | static void negativeValidate(const char *s) { 29 | uint16_t n; 30 | if (parseUint16(s, &n)) { 31 | std::cerr << "Expected error parsing " << s 32 | << " returned " << n << std::endl; 33 | exit(1); 34 | } 35 | } 36 | 37 | static void positiveValidate(const char *s, uint16_t expected) { 38 | uint16_t n; 39 | if (!parseUint16(s, &n)) { 40 | std::cerr << "Error parsing " << s << std::endl; 41 | exit(1); 42 | } 43 | cb_assert(n == expected); 44 | } 45 | 46 | static void testUint16() { 47 | negativeValidate("-1"); 48 | negativeValidate("65537"); 49 | negativeValidate("65536"); 50 | positiveValidate("65535", UINT16_MAX); 51 | positiveValidate("0", 0); 52 | positiveValidate("32768", 32768); 53 | } 54 | 55 | int main(int argc, char **argv) { 56 | (void)argc; (void)argv; 57 | alarm(60); 58 | testUint16(); 59 | } 60 | -------------------------------------------------------------------------------- /tests/module_tests/mock_hooks_api.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include "mock_hooks_api.h" 21 | 22 | /* 23 | * A mock implementation of the getHooksApi() function (and associated 24 | * functions). 25 | * 26 | * All hook functions will do nothing. 27 | */ 28 | 29 | extern "C" { 30 | static bool mock_add_new_hook(void (*)(const void* ptr, size_t size)) { 31 | return false; 32 | } 33 | 34 | static bool mock_remove_new_hook(void (*)(const void* ptr, size_t size)) { 35 | return false; 36 | } 37 | 38 | static bool mock_add_delete_hook(void (*)(const void* ptr)) { 39 | return false; 40 | } 41 | 42 | static bool mock_remove_delete_hook(void (*)(const void* ptr)) { 43 | return false; 44 | } 45 | 46 | static int mock_get_extra_stats_size() { 47 | return 0; 48 | } 49 | 50 | static void mock_get_allocator_stats(allocator_stats*) { 51 | // Empty 52 | } 53 | 54 | static size_t mock_get_allocation_size(const void*) { 55 | return 0; 56 | } 57 | } 58 | 59 | ALLOCATOR_HOOKS_API* getHooksApi(void) { 60 | static ALLOCATOR_HOOKS_API hooksApi; 61 | hooksApi.add_new_hook = mock_add_new_hook; 62 | hooksApi.remove_new_hook = mock_remove_new_hook; 63 | hooksApi.add_delete_hook = mock_add_delete_hook; 64 | hooksApi.remove_delete_hook = mock_remove_delete_hook; 65 | hooksApi.get_extra_stats_size = mock_get_extra_stats_size; 66 | hooksApi.get_allocator_stats = mock_get_allocator_stats; 67 | hooksApi.get_allocation_size = mock_get_allocation_size; 68 | return &hooksApi; 69 | } 70 | -------------------------------------------------------------------------------- /tests/module_tests/mock_hooks_api.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include "config.h" 21 | 22 | #include 23 | 24 | ALLOCATOR_HOOKS_API* getHooksApi(void); 25 | -------------------------------------------------------------------------------- /tests/module_tests/mutex_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "config.h" 19 | 20 | #include 21 | #include 22 | 23 | #include "common.h" 24 | #include "locks.h" 25 | 26 | #include 27 | 28 | TEST(LockTimerTest, LockHolder) { 29 | std::mutex m; 30 | { 31 | LockTimer lh(m, "LockHolder"); 32 | std::this_thread::sleep_for(std::chrono::milliseconds(2)); 33 | } 34 | } 35 | 36 | TEST(LockTimerTest, ReaderLockHolder) { 37 | cb::RWLock m; 38 | { 39 | LockTimer rlh(m, "ReaderLockHolder"); 40 | std::this_thread::sleep_for(std::chrono::milliseconds(2)); 41 | } 42 | } 43 | 44 | TEST(LockTimerTest, WriterLockHolder) { 45 | cb::RWLock m; 46 | { 47 | LockTimer wlh(m, "WriterLockHolder"); 48 | std::this_thread::sleep_for(std::chrono::milliseconds(2)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/module_tests/stats_test.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | * Test fixtures for statistics tests. 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "evp_store_single_threaded_test.h" 25 | 26 | class StatTest : public SingleThreadedEPBucketTest { 27 | protected: 28 | void SetUp() override; 29 | 30 | /** 31 | * Requests the given statistics from the bucket. 32 | * @param statkey 33 | * @return Map of stat name to value 34 | */ 35 | std::map get_stat(const char* statkey = nullptr); 36 | }; 37 | -------------------------------------------------------------------------------- /tests/module_tests/string_utils_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "string_utils.h" 19 | 20 | #include 21 | 22 | TEST(cb_stobTest, ValidWorks) { 23 | EXPECT_TRUE(cb_stob("true")); 24 | EXPECT_FALSE(cb_stob("false")); 25 | } 26 | 27 | TEST(cb_stobTest, ThrowsInvalid) { 28 | EXPECT_THROW(cb_stob("TRUE"), invalid_argument_bool); 29 | EXPECT_THROW(cb_stob("True"), invalid_argument_bool); 30 | EXPECT_THROW(cb_stob("completelyinvalid"), invalid_argument_bool); 31 | EXPECT_THROW(cb_stob("FALSE"), invalid_argument_bool); 32 | EXPECT_THROW(cb_stob("FLASE"), invalid_argument_bool); 33 | EXPECT_THROW(cb_stob("farce"), invalid_argument_bool); 34 | } -------------------------------------------------------------------------------- /tests/module_tests/systemevent_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "systemevent.h" 19 | 20 | #include 21 | 22 | TEST(SystemEventTest, make) { 23 | auto value = SystemEventFactory::make(SystemEvent::CreateCollection, 24 | "::", 25 | "SUFFIX", 26 | 0, 27 | OptionalSeqno(/*none*/)); 28 | EXPECT_EQ(queue_op::system_event, value->getOperation()); 29 | EXPECT_EQ(0, value->getNBytes()); 30 | EXPECT_NE(nullptr, strstr(value->getKey().c_str(), "SUFFIX")); 31 | EXPECT_EQ(uint32_t(SystemEvent::CreateCollection), value->getFlags()); 32 | EXPECT_EQ(-1, value->getBySeqno()); 33 | 34 | value = SystemEventFactory::make(SystemEvent::CreateCollection, 35 | "::", 36 | "SUFFIX", 37 | 100, 38 | OptionalSeqno(/*none*/)); 39 | EXPECT_EQ(queue_op::system_event, value->getOperation()); 40 | EXPECT_EQ(100, value->getNBytes()); 41 | EXPECT_NE(nullptr, strstr(value->getKey().c_str(), "SUFFIX")); 42 | EXPECT_EQ(uint32_t(SystemEvent::CreateCollection), value->getFlags()); 43 | 44 | value = SystemEventFactory::make(SystemEvent::BeginDeleteCollection, 45 | "SUFFIX", 46 | "::", 47 | 100, 48 | OptionalSeqno(/*none*/)); 49 | EXPECT_EQ(queue_op::system_event, value->getOperation()); 50 | EXPECT_EQ(100, value->getNBytes()); 51 | EXPECT_NE(nullptr, strstr(value->getKey().c_str(), "SUFFIX")); 52 | EXPECT_EQ(uint32_t(SystemEvent::BeginDeleteCollection), value->getFlags()); 53 | 54 | value = SystemEventFactory::make(SystemEvent::BeginDeleteCollection, 55 | "SUFFIX", 56 | "::", 57 | 100, 58 | OptionalSeqno(122)); 59 | EXPECT_EQ(queue_op::system_event, value->getOperation()); 60 | EXPECT_EQ(100, value->getNBytes()); 61 | EXPECT_NE(nullptr, strstr(value->getKey().c_str(), "SUFFIX")); 62 | EXPECT_EQ(uint32_t(SystemEvent::BeginDeleteCollection), value->getFlags()); 63 | EXPECT_EQ(122, value->getBySeqno()); 64 | } -------------------------------------------------------------------------------- /tests/module_tests/test_helpers.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "test_helpers.h" 19 | 20 | #include 21 | 22 | Item make_item(uint16_t vbid, 23 | const StoredDocKey& key, 24 | const std::string& value, 25 | uint32_t exptime, 26 | protocol_binary_datatype_t datatype) { 27 | uint8_t ext_meta[EXT_META_LEN] = {datatype}; 28 | Item item(key, /*flags*/0, /*exp*/exptime, value.c_str(), value.size(), 29 | ext_meta, sizeof(ext_meta)); 30 | item.setVBucketId(vbid); 31 | return item; 32 | } 33 | 34 | std::chrono::microseconds decayingSleep(std::chrono::microseconds uSeconds) { 35 | /* Max sleep time is slightly over a second */ 36 | static const std::chrono::microseconds maxSleepTime(0x1 << 20); 37 | std::this_thread::sleep_for(uSeconds); 38 | return std::min(uSeconds * 2, maxSleepTime); 39 | } 40 | -------------------------------------------------------------------------------- /tests/module_tests/test_helpers.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2016 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Helper functions / code for unit tests 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "config.h" 25 | 26 | #include "item.h" 27 | #include "programs/engine_testapp/mock_server.h" 28 | 29 | #include 30 | 31 | /// Creates an item with the given vbucket id, key and value. 32 | Item make_item( 33 | uint16_t vbid, 34 | const StoredDocKey& key, 35 | const std::string& value, 36 | uint32_t exptime = 0, 37 | protocol_binary_datatype_t datatype = PROTOCOL_BINARY_DATATYPE_JSON); 38 | 39 | /** 40 | * Create a StoredDocKey object from a std::string. 41 | * By default places the key in the default namespace, 42 | * DocNamespace::DefaultCollection. 43 | */ 44 | inline StoredDocKey makeStoredDocKey( 45 | const std::string& string, 46 | DocNamespace ns = DocNamespace::DefaultCollection) { 47 | return StoredDocKey(string, ns); 48 | } 49 | 50 | /** 51 | * Class which moves time forward when created by the given amount, and upon 52 | * destruction returns time to where it was. 53 | * 54 | * Allows tests to manipulate server time, but need to ensure any adjustments 55 | * are restored so as to not affect other later tests. 56 | */ 57 | class TimeTraveller { 58 | public: 59 | TimeTraveller(int by) 60 | : by(by) { 61 | mock_time_travel(by); 62 | } 63 | 64 | ~TimeTraveller() { 65 | // restore original timeline. 66 | mock_time_travel(-by); 67 | } 68 | 69 | private: 70 | // Amount of time travel. 71 | int by; 72 | }; 73 | 74 | /** 75 | * Function to do an exponentially increasing, but max bounded, sleep. 76 | * To do exponentially increasing sleep, must be called first with the starting 77 | * sleep time and subsequently with the sleep time returned in the previous call 78 | * 79 | * @param uSeconds Desired sleep time in micro seconds 80 | * 81 | * @return indicates the next sleep time (doubled from the current value) 82 | */ 83 | std::chrono::microseconds decayingSleep(std::chrono::microseconds uSeconds); 84 | -------------------------------------------------------------------------------- /tests/module_tests/test_task.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include "globaltask.h" 19 | 20 | class TestTask : public GlobalTask { 21 | public: 22 | TestTask(EventuallyPersistentEngine* e, TaskId id, int o = 0) 23 | : GlobalTask(e, id, 0.0, false), 24 | order(o), 25 | description(std::string("TestTask ") + 26 | GlobalTask::getTaskName(getTypeId())) { 27 | } 28 | 29 | // returning true will also drive the ExecutorPool::reschedule path. 30 | bool run() { return true; } 31 | 32 | cb::const_char_buffer getDescription() { 33 | return description; 34 | } 35 | 36 | int order; 37 | const std::string description; 38 | }; 39 | -------------------------------------------------------------------------------- /tests/module_tests/thread_gate.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | /** Object which is used to synchronise the execution of a number of threads. 21 | * Each thread calls threadUp(), and until all threads have called this 22 | * they are all blocked. 23 | */ 24 | class ThreadGate { 25 | public: 26 | ThreadGate() : n_threads(0) { 27 | } 28 | 29 | /** Create a ThreadGate. 30 | * @param n_threads Total number of threads to wait for. 31 | */ 32 | ThreadGate(size_t n_threads_) : n_threads(n_threads_) { 33 | } 34 | 35 | /* 36 | * atomically increment a threadCount 37 | * if the calling thread is the last one up, notify_all 38 | * if the calling thread is not the last one up, wait (in the function) 39 | */ 40 | void threadUp() { 41 | std::unique_lock lh(m); 42 | if (++thread_count != n_threads) { 43 | cv.wait(lh, [this]() { return isComplete(); }); 44 | } else { 45 | cv.notify_all(); // all threads accounted for, begin 46 | } 47 | } 48 | 49 | template 50 | void waitFor(std::chrono::duration timeout) { 51 | std::unique_lock lh(m); 52 | cv.wait_for(lh, timeout, [this]() { return isComplete(); }); 53 | } 54 | 55 | size_t getCount() const { 56 | return thread_count; 57 | } 58 | 59 | bool isComplete() { 60 | return thread_count == n_threads; 61 | } 62 | 63 | private: 64 | const size_t n_threads; 65 | size_t thread_count{0}; 66 | std::mutex m; 67 | std::condition_variable cv; 68 | }; -------------------------------------------------------------------------------- /tests/module_tests/warmup_test.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2017 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * Unit tests for Warmup-related functions. 20 | * 21 | */ 22 | 23 | #include "evp_engine_test.h" 24 | 25 | #include "ep_engine.h" 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | class WarmupDisabledTest : public EventuallyPersistentEngineTest { 33 | 34 | void SetUp() override { 35 | config_string = "warmup=false"; 36 | EventuallyPersistentEngineTest::SetUp(); 37 | } 38 | }; 39 | 40 | // Mock implementation of an ADD_STAT callback. 41 | class MockAddStat { 42 | public: 43 | MOCK_CONST_METHOD2(Callback, void(std::string key, std::string value)); 44 | 45 | // Static method to act as trampoline to the GoogleMock object method. 46 | static void trampoline(const char *key, const uint16_t klen, 47 | const char *val, const uint32_t vlen, 48 | const void *cookie) { 49 | auto* object = reinterpret_cast(cookie); 50 | object->Callback({key, klen}, {val, vlen}); 51 | } 52 | }; 53 | 54 | // Check we get the expected stats (i.e. none) when Warmup is disabled. 55 | TEST_F(WarmupDisabledTest, Stats) { 56 | 57 | std::string key{"warmup"}; 58 | MockAddStat add_stat; 59 | EXPECT_CALL(add_stat, Callback("ep_warmup", ::testing::_)).Times(0); 60 | EXPECT_EQ(ENGINE_KEY_ENOENT, 61 | engine->getStats(&add_stat, 62 | key.data(), 63 | key.size(), 64 | MockAddStat::trampoline)); 65 | } 66 | -------------------------------------------------------------------------------- /tests/suite_stubs.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Copyright 2010 Couchbase, Inc 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef SRC_SUITE_STUBS_H_ 19 | #define SRC_SUITE_STUBS_H_ 1 20 | 21 | #include "config.h" 22 | 23 | #include 24 | 25 | #include "ep_testsuite.h" 26 | 27 | bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 28 | void delay(int amt); 29 | 30 | void add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 31 | void append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 32 | void decr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 33 | void decrWithDefault(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 34 | void prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 35 | void flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 36 | void del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 37 | void get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 38 | void set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 39 | void setRetainCAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 40 | void incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 41 | void incrWithDefault(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 42 | void getLock(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 43 | void setUsingCAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 44 | void deleteUsingCAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 45 | void appendUsingCAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 46 | void prependUsingCAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 47 | 48 | void checkValue(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* exp); 49 | void assertNotExists(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1); 50 | 51 | #define assertHasError() cb_assert(hasError) 52 | #define assertHasNoError() cb_assert(!hasError) 53 | 54 | extern int expiry; 55 | extern int locktime; 56 | extern bool hasError; 57 | extern uint64_t cas; 58 | extern struct test_harness testHarness; 59 | 60 | engine_test_t* get_tests_0(void); 61 | engine_test_t* get_tests_1(void); 62 | engine_test_t* get_tests_2(void); 63 | engine_test_t* get_tests_3(void); 64 | engine_test_t* get_tests_4(void); 65 | engine_test_t* get_tests_5(void); 66 | engine_test_t* get_tests_6(void); 67 | engine_test_t* get_tests_7(void); 68 | engine_test_t* get_tests_8(void); 69 | engine_test_t* get_tests_9(void); 70 | 71 | #endif /* SRC_SUITE_STUBS_H_ */ 72 | -------------------------------------------------------------------------------- /wrapper/wrapper: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | export EP_ENGINE_VERSION="@EP_ENGINE_VERSION@" 4 | 5 | CWD="`pwd`" 6 | D0="`dirname "$0"`" 7 | cd "$D0/../lib/python" 8 | root="`pwd`" 9 | cd "$CWD" 10 | 11 | if test -z "$PYTHONPATH"; then 12 | PYTHONPATH="$root" 13 | else 14 | PYTHONPATH="$root:$PYTHONPATH" 15 | fi 16 | export PYTHONPATH 17 | exec "$root"/`basename "$0"` "$@" 18 | --------------------------------------------------------------------------------