├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── example ├── Makefile ├── bench_hash.cc ├── hash.cc ├── hash_test.cc ├── kv_test.cc ├── list_lock.cc ├── list_test.cc ├── main.cc ├── rush.cc ├── set_test.cc ├── simple_test.cc ├── test_bgsave.cc ├── test_server.cc ├── ttl.cc ├── zset.cc └── zset_test.cc ├── include ├── nemo.h ├── nemo_backupable.h ├── nemo_const.h ├── nemo_iterator.h ├── nemo_meta.h ├── nemo_murmur3.h ├── nemo_options.h ├── port.h ├── util.h ├── version.h └── xdebug.h ├── src ├── build_version.cc.in ├── build_version.h ├── decoder.h ├── nemo.cc ├── nemo_admin.cc ├── nemo_backupable.cc ├── nemo_bit.cc ├── nemo_hash.cc ├── nemo_hash.h ├── nemo_hyperloglog.cc ├── nemo_hyperloglog.h ├── nemo_iterator.cc ├── nemo_kv.cc ├── nemo_list.cc ├── nemo_list.h ├── nemo_meta.cc ├── nemo_mutex.h ├── nemo_set.cc ├── nemo_set.h ├── nemo_zset.cc ├── nemo_zset.h ├── port.cc └── util.cc ├── test ├── Makefile ├── gtest_1.7.0 │ ├── Makefile │ ├── include │ │ └── gtest │ │ │ ├── gtest-death-test.h │ │ │ ├── gtest-message.h │ │ │ ├── gtest-param-test.h │ │ │ ├── gtest-param-test.h.pump │ │ │ ├── gtest-printers.h │ │ │ ├── gtest-spi.h │ │ │ ├── gtest-test-part.h │ │ │ ├── gtest-typed-test.h │ │ │ ├── gtest.h │ │ │ ├── gtest_pred_impl.h │ │ │ ├── gtest_prod.h │ │ │ └── internal │ │ │ ├── gtest-death-test-internal.h │ │ │ ├── gtest-filepath.h │ │ │ ├── gtest-internal.h │ │ │ ├── gtest-linked_ptr.h │ │ │ ├── gtest-param-util-generated.h │ │ │ ├── gtest-param-util-generated.h.pump │ │ │ ├── gtest-param-util.h │ │ │ ├── gtest-port.h │ │ │ ├── gtest-string.h │ │ │ ├── gtest-tuple.h │ │ │ ├── gtest-tuple.h.pump │ │ │ ├── gtest-type-util.h │ │ │ └── gtest-type-util.h.pump │ └── src │ │ ├── gtest-all.cc │ │ ├── gtest-death-test.cc │ │ ├── gtest-filepath.cc │ │ ├── gtest-internal-inl.h │ │ ├── gtest-port.cc │ │ ├── gtest-printers.cc │ │ ├── gtest-test-part.cc │ │ ├── gtest-typed-test.cc │ │ ├── gtest.cc │ │ └── gtest_main.cc ├── include │ ├── nemo_hash_test.h │ ├── nemo_kv_test.h │ ├── nemo_list_test.h │ ├── nemo_set_test.h │ ├── nemo_test.h │ └── nemo_zset_test.h ├── main.cc ├── nemo_hash_test.cc ├── nemo_kv_test.cc ├── nemo_list_test.cc ├── nemo_set_test.cc └── nemo_zset_test.cc └── tools ├── compact ├── Makefile ├── README ├── compact.cc └── xdebug.h ├── meta_scan ├── Makefile ├── README ├── meta_scan.cc └── xdebug.h ├── nemo_performance_test ├── \ ├── case.cc ├── config.cc ├── config.h ├── main.cc ├── random_string.h └── time_keeper.h └── nemock ├── Makefile ├── README ├── nemock.cc └── xdebug.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | 30 | output/ 31 | *tags 32 | example/main 33 | example/tmp 34 | example/test* 35 | example/zset* 36 | example/hash* 37 | example/kv* 38 | example/set* 39 | example/list* 40 | example/ttl* 41 | example/bench_hash 42 | example/simple_test 43 | tmp/ 44 | test_rocksdb 45 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "3rdparty/nemo-rocksdb"] 2 | path = 3rdparty/nemo-rocksdb 3 | url = https://github.com/KernelMaker/nemo-rocksdb 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | 5 | os: 6 | - linux 7 | 8 | addons: 9 | apt: 10 | packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl'] 11 | 12 | compiler: 13 | - gcc 14 | 15 | language: cpp 16 | 17 | script: make 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CLEAN_FILES = # deliberately empty, so we can append below. 2 | CXX=g++ 3 | LDFLAGS= -lpthread -lrt 4 | CXXFLAGS=-std=c++11 -fno-builtin-memcmp -msse -msse4.2 5 | PROFILING_FLAGS=-pg 6 | ARFLAGS = rs 7 | OPT= 8 | 9 | # Set the default DEBUG_LEVEL to 0 10 | DEBUG_LEVEL?=0 11 | 12 | ifeq ($(MAKECMDGOALS),dbg) 13 | DEBUG_LEVEL=2 # compatible with rocksdb 14 | endif 15 | 16 | ifeq ($(MAKECMDGOALS),all) 17 | DEBUG_LEVEL=0 18 | endif 19 | 20 | ifeq ($(MAKECMDGOALS),clean) 21 | DEBUG_LEVEL=0 22 | endif 23 | 24 | ifeq ($(MAKECMDGOALS),static_lib) 25 | DEBUG_LEVEL=0 26 | endif 27 | 28 | # compile with -O2 if for release 29 | # if we're compiling for release, compile without debug code (-DNDEBUG) and 30 | # don't treat warnings as errors 31 | ifeq ($(DEBUG_LEVEL),0) 32 | DISABLE_WARNING_AS_ERROR=1 33 | OPT += -O2 -fno-omit-frame-pointer -DNDEBUG 34 | else 35 | $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) 36 | OPT += -O0 -D__XDEBUG__ -D_GNU_SOURCE $(PROFILING_FLAGS) 37 | endif 38 | 39 | #----------------------------------------------- 40 | 41 | SRC_DIR=src 42 | VERSION_CC=$(SRC_DIR)/build_version.cc 43 | LIB_SOURCES := $(VERSION_CC) \ 44 | $(filter-out $(VERSION_CC), $(wildcard $(SRC_DIR)/*.cc)) 45 | 46 | ifndef NEMODB_PATH 47 | $(warning Warning: missing nemo-rocksdb path, using default: $(CURDIR)/3rdparty/nemo-rocksdb) 48 | NEMODB_PATH=$(CURDIR)/3rdparty/nemo-rocksdb 49 | endif 50 | 51 | NEMODB_INCLUDE_DIR=$(NEMODB_PATH)/include 52 | 53 | AM_DEFAULT_VERBOSITY = 0 54 | 55 | AM_V_GEN = $(am__v_GEN_$(V)) 56 | am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) 57 | am__v_GEN_0 = @echo " GEN " $@; 58 | am__v_GEN_1 = 59 | AM_V_at = $(am__v_at_$(V)) 60 | am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) 61 | am__v_at_0 = @ 62 | am__v_at_1 = 63 | 64 | AM_V_CXX = $(am__v_CXX_$(V)) 65 | am__v_CXX_ = $(am__v_CXX_$(AM_DEFAULT_VERBOSITY)) 66 | am__v_CXX_0 = @echo " CXX " $@; 67 | am__v_CXX_1 = 68 | LD = $(CXX) 69 | AM_V_LD = $(am__v_LD_$(V)) 70 | am__v_LD_ = $(am__v_LD_$(AM_DEFAULT_VERBOSITY)) 71 | am__v_LD_0 = @echo " LD " $@; 72 | am__v_LD_1 = 73 | AM_V_AR = $(am__v_AR_$(V)) 74 | am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) 75 | am__v_AR_0 = @echo " AR " $@; 76 | am__v_AR_1 = 77 | 78 | AM_LINK = $(AM_V_LD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) 79 | 80 | CXXFLAGS += -g 81 | 82 | # This (the first rule) must depend on "all". 83 | default: all 84 | 85 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 86 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ 87 | -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ 88 | -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 89 | 90 | ifndef DISABLE_WARNING_AS_ERROR 91 | WARNING_FLAGS += -Werror 92 | endif 93 | 94 | CXXFLAGS += $(WARNING_FLAGS) -I./include -I$(NEMODB_INCLUDE_DIR) -I$(ROCKSDB_PATH) -I$(ROCKSDB_PATH)/include $(OPT) 95 | 96 | date := $(shell date +%F) 97 | git_sha := $(shell git rev-parse HEAD 2>/dev/null) 98 | gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ $(SRC_DIR)/build_version.cc.in 99 | # Record the version of the source that we are compiling. 100 | # We keep a record of the git revision in this file. It is then built 101 | # as a regular source file as part of the compilation process. 102 | # One can run "strings executable_filename | grep _build_" to find 103 | # the version of the source that we used to build the executable file. 104 | CLEAN_FILES += $(SRC_DIR)/build_version.cc 105 | 106 | $(SRC_DIR)/build_version.cc: FORCE 107 | $(AM_V_GEN)rm -f $@-t 108 | $(AM_V_at)$(gen_build_version) > $@-t 109 | $(AM_V_at)if test -f $@; then \ 110 | cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \ 111 | else mv -f $@-t $@; fi 112 | FORCE: 113 | 114 | LIBOBJECTS = $(LIB_SOURCES:.cc=.o) 115 | 116 | # if user didn't config LIBNAME, set the default 117 | ifeq ($(LIBNAME),) 118 | # we should only run pink in production with DEBUG_LEVEL 0 119 | ifeq ($(DEBUG_LEVEL),0) 120 | LIBNAME=libnemo 121 | else 122 | LIBNAME=libnemo_debug 123 | endif 124 | endif 125 | LIBOUTPUT = ./lib 126 | dummy := $(shell mkdir -p $(LIBOUTPUT)) 127 | LIBRARY = $(LIBOUTPUT)/${LIBNAME}.a 128 | 129 | .PHONY: clean dbg static_lib all 130 | 131 | all: $(LIBRARY) 132 | 133 | static_lib: $(LIBRARY) 134 | 135 | check: $(LIBRARY) 136 | @make -C test 137 | for t in $(notdir $(TESTS)); do echo "***** Running $$t"; ./test/$$t || exit 1; done 138 | 139 | dbg: $(LIBRARY) 140 | 141 | $(LIBRARY): $(LIBOBJECTS) 142 | $(AM_V_AR)rm -f $@ 143 | $(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS) 144 | 145 | clean: 146 | make -C ./example clean 147 | rm -f $(LIBRARY) 148 | rm -rf $(CLEAN_FILES) 149 | rm -rf $(LIBOUTPUT) 150 | find . -name "*.[oda]*" ! -path "./3rdparty/*" -exec rm -f {} \; 151 | find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; 152 | 153 | %.o: %.cc 154 | $(AM_V_CXX)$(CXX) $(CXXFLAGS) -c $< -o $@ 155 | 156 | %.d: %.cc 157 | @set -e; rm -f $@; $(CXX) -MM $(CXXFLAGS) $< > $@.$$$$; \ 158 | sed 's,\($(notdir $*)\)\.o[ :]*,$(SRC_DIR)/\1.o $@ : ,g' < $@.$$$$ > $@; \ 159 | rm -f $@.$$$$ 160 | 161 | ifneq ($(MAKECMDGOALS),clean) 162 | -include $(LIBOBJECTS:.o=.d) 163 | endif 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nemo 2 | 3 | [![Build Status](https://travis-ci.org/Qihoo360/nemo.svg?branch=master)](https://travis-ci.org/Qihoo360/nemo) 4 | 5 | A library that provide multiply data structure. Such as map, hash, list, set. We 6 | build these data structure base on rocksdb 7 | 8 | 9 | ## Performance 10 | 11 | We test the nemo library simply. We run set/get 100,000 times with a 13 bytes key and value. 12 | 13 | CPU: 24 * Intel(R) Xeon(R) CPU E5-2630 v2 @ 2.60GHz 14 | 15 | ### 1. nemo 16 | 17 | set： 4.18 micros/op; 239,140 ops/s 18 | get : 1.12 micros/op; 894,398 ops/s 19 | 20 | ### 2. leveldb 21 | 22 | set: 1.36 micros/op; 733,896 ops/s 23 | get: 0.93 micros/op; 1,072,282 ops/s 24 | 25 | ### 3. rocksdb 26 | 27 | set: 3.85 micros/op; 259,599 ops/s 28 | get: 1.00 micros/op; 998,631 ops/s 29 | 30 | ### TODO 31 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -Wno-format -DDEBUG -g -O2 -std=c++11 -D__XDEBUG__ 3 | 4 | OBJECT = main test_bgsave zset test_server hash kv_test ttl bench_hash list_lock simple_test 5 | 6 | ifndef NEMODB_PATH 7 | $(warning Warning: missing nemo-rocksdb path, using default: ../3rdparty/nemo-rocksdb) 8 | NEMODB_PATH=../3rdparty/nemo-rocksdb 9 | endif 10 | 11 | ifndef ROCKSDB_PATH 12 | $(warning Warning: missing rocksdb path, using default: ../3rdparty/nemo-rocksdb/rocksdb) 13 | ROCKSDB_PATH=../3rdparty/nemo-rocksdb/rocksdb 14 | endif 15 | 16 | LIB_PATH = -L../lib/ \ 17 | -L$(NEMODB_PATH)/lib/ \ 18 | -L$(ROCKSDB_PATH)/ 19 | 20 | LIBS = -lnemo \ 21 | -lnemodb \ 22 | -lrocksdb \ 23 | -lz \ 24 | -lbz2 \ 25 | -lsnappy \ 26 | -lrt \ 27 | -lpthread 28 | 29 | INCLUDE_PATH = -I../include/ \ 30 | -I$(NEMODB_PATH)/include \ 31 | -I$(ROCKSDB_PATH)/ \ 32 | -I$(ROCKSDB_PATH)/include 33 | 34 | OBJS = main.o test_bgsave.o zset.o test_server.o test_mset.o hash.o hash_test.o kv_test.o zset_test.o set_test.o list_test.o ttl.o bench_hash.o list_lock.o simple_test.o 35 | 36 | .PHONY: all clean 37 | 38 | all: $(OBJECT) 39 | @echo "Success, go, go, go..." 40 | 41 | main: main.o 42 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 43 | 44 | simple_test: simple_test.o 45 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 46 | 47 | zset: zset.o 48 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 49 | 50 | list_lock: list_lock.o 51 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 52 | 53 | ttl: ttl.o 54 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 55 | 56 | kv_test: kv_test.o 57 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 58 | 59 | hash_test: hash_test.o 60 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 61 | 62 | bench_hash: bench_hash.o 63 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 64 | 65 | hash: hash.o 66 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 67 | 68 | zset_test: zset_test.o 69 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 70 | 71 | set_test: set_test.o 72 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 73 | 74 | list_test: list_test.o 75 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 76 | 77 | test_mset: test_mset.o 78 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 79 | 80 | test_bgsave: test_bgsave.o 81 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 82 | 83 | test_server: test_server.o 84 | $(CXX) $(CXXFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 85 | 86 | $(OBJS): %.o : %.cc 87 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 88 | 89 | clean: 90 | rm -rf $(OBJECT) 91 | rm -f $(OBJS) 92 | 93 | -------------------------------------------------------------------------------- /example/bench_hash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "nemo.h" 10 | #include "xdebug.h" 11 | 12 | using namespace nemo; 13 | using namespace std; 14 | 15 | const int MAXN = 1024; 16 | 17 | Nemo *n; 18 | int tn; 19 | int cnt; 20 | int key_num; 21 | int length; 22 | pthread_t tid[MAXN]; 23 | int64_t total = 0; 24 | 25 | char field[MAXN]; 26 | char value[MAXN]; 27 | 28 | inline int64_t NowMicros() { 29 | struct timeval tv; 30 | gettimeofday(&tv, NULL); 31 | return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; 32 | } 33 | 34 | void gen_random(char *s, const int len) { 35 | static const char alphanum[] = 36 | "0123456789" 37 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 38 | "abcdefghijklmnopqrstuvwxyz"; 39 | 40 | for (int i = 0; i < len; ++i) { 41 | s[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; 42 | } 43 | 44 | s[len] = 0; 45 | } 46 | 47 | void* ThreadMain1(void *arg) { 48 | int64_t st, ed, t_sum = 0LL; 49 | Status s; 50 | srand(time(NULL)); 51 | 52 | for (int i = 0; i < cnt; i++) { 53 | int key_id = rand() % key_num; 54 | 55 | string key = "hash_key:" + to_string(key_id); 56 | gen_random(field, length); 57 | gen_random(value, length); 58 | st = NowMicros(); 59 | s = n->HSet(key, field, value); 60 | ed = NowMicros(); 61 | 62 | t_sum += ed - st; 63 | 64 | if (i > 0 && i % 10000 == 0) { 65 | printf (" tid:%10u run %d request, tot is %10lu\n", pthread_self(), i + 1, t_sum); 66 | } 67 | } 68 | printf (" tid:%10u end, run %d request, tot is %10lu\n", pthread_self(), cnt, t_sum); 69 | //total += t_sum; 70 | return (void *)t_sum; 71 | } 72 | 73 | int main(int argc, char* argv[]) { 74 | if (argc < 5) { 75 | printf ("Usage: ./bench thread_num query_per_thread key_num key_length\n"); 76 | exit(0); 77 | } 78 | 79 | char *pend; 80 | tn = strtol(argv[1], &pend, 10); 81 | cnt = strtol(argv[2], &pend, 10); 82 | key_num = strtol(argv[3], &pend, 10); 83 | length = strtol(argv[4], &pend, 10); 84 | 85 | 86 | printf ("thread_num %d, query %d per thread, key_num is %d, key(field)_length is %d\n", tn, cnt, key_num, length); 87 | 88 | nemo::Options options; 89 | options.target_file_size_base = 20 * 1024 * 1024; 90 | 91 | n = new Nemo("./tmp/", options); 92 | 93 | for (int i = 0; i < tn; i++) { 94 | pthread_create(&tid[i], NULL, &ThreadMain1, NULL); 95 | } 96 | 97 | void *res; 98 | for (int i = 0; i < tn; i++) { 99 | if (pthread_join(tid[i], &(res)) != 0) { 100 | printf ("pthread_join %d failed\n", tid[i]); 101 | } else { 102 | printf ("thread_join %d return %10u\n", i, (int64_t)res); 103 | } 104 | total += (int64_t)res; 105 | } 106 | printf ("Total time: %10ld us;\nTotal request: %10ld;\nQPS: %10.3lf\n", total, tn * cnt, (double)1000000.0 * tn * cnt * tn / total); 107 | 108 | delete n; 109 | 110 | return 0; 111 | } 112 | -------------------------------------------------------------------------------- /example/hash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | Nemo *n; 14 | 15 | void* ThreadMain1(void *arg) { 16 | Status s = n->HSet("tHSetKey", "field1", "value1"); 17 | log_info("Test HSet field1 OK return %s", s.ToString().c_str()); 18 | return NULL; 19 | } 20 | 21 | void* ThreadMain2(void *arg) { 22 | Status s = n->HSet("tHSetKey", "field2", "value2"); 23 | log_info("Test HSet field2 OK return %s", s.ToString().c_str()); 24 | return NULL; 25 | } 26 | 27 | void* ThreadMain3(void *arg) { 28 | Status s = n->HSet("tHSetKey", "field3", "value3"); 29 | log_info("Test HSet field3 OK return %s", s.ToString().c_str()); 30 | return NULL; 31 | } 32 | 33 | void* ThreadMain4(void *arg) { 34 | Status s = n->HSet("tHSetKey1", "field3", "value3"); 35 | log_info("Test HSet field3 OK return %s", s.ToString().c_str()); 36 | return NULL; 37 | } 38 | 39 | void* ThreadMain5(void *arg) { 40 | Status s = n->HSet("tHSetKey2", "field3", "value3"); 41 | log_info("Test HSet field3 OK return %s", s.ToString().c_str()); 42 | return NULL; 43 | } 44 | 45 | void* ThreadMain6(void *arg) { 46 | Status s = n->HSet("tHSetKey3", "field3", "value3"); 47 | log_info("Test HSet field3 OK return %s", s.ToString().c_str()); 48 | return NULL; 49 | } 50 | 51 | int main() { 52 | nemo::Options options; 53 | options.target_file_size_base = 20 * 1024 * 1024; 54 | 55 | n = new Nemo("./tmp/", options); 56 | Status s; 57 | 58 | std::string res; 59 | 60 | std::vector keys; 61 | std::vector kvs; 62 | std::vector kvss; 63 | std::vector sms; 64 | 65 | /* 66 | *************************************************HASH************************************************** 67 | */ 68 | std::vector fields; 69 | std::vector values; 70 | std::vector fvs; 71 | std::vector fvss; 72 | 73 | log_info("======Test HSet Same key======"); 74 | 75 | pthread_t tid[3]; 76 | pthread_create(&tid[0], NULL, &ThreadMain1, NULL); 77 | pthread_create(&tid[1], NULL, &ThreadMain2, NULL); 78 | pthread_create(&tid[2], NULL, &ThreadMain3, NULL); 79 | 80 | for (int i = 0; i < 3; i++) { 81 | pthread_join(tid[i], 0); 82 | } 83 | 84 | log_info("======Test HSet different key======"); 85 | 86 | pthread_create(&tid[0], NULL, &ThreadMain4, NULL); 87 | pthread_create(&tid[1], NULL, &ThreadMain5, NULL); 88 | 89 | for (int i = 0; i < 2; i++) { 90 | pthread_join(tid[i], 0); 91 | } 92 | 93 | log_info("======Test HSet different key with all the lru in use======"); 94 | // pthread_create(&tid[0], NULL, &ThreadMain4, NULL); 95 | // pthread_create(&tid[1], NULL, &ThreadMain5, NULL); 96 | // pthread_create(&tid[2], NULL, &ThreadMain6, NULL); 97 | 98 | // for (int i = 0; i < 3; i++) { 99 | // pthread_join(tid[i], 0); 100 | // } 101 | 102 | /* 103 | * Test HGetall 104 | */ 105 | log_info("======Test HGetall======"); 106 | fvs.clear(); 107 | s = n->HGetall("tHSetKey", fvs); 108 | log_info("Test HGetall OK return %s", s.ToString().c_str()); 109 | std::vector::iterator fv_iter; 110 | for (fv_iter = fvs.begin(); fv_iter != fvs.end(); fv_iter++) { 111 | log_info("Test HGetall, field: %s, val: %s", fv_iter->field.c_str(), fv_iter->val.c_str()); 112 | } 113 | log_info(""); 114 | 115 | HIterator *hit = n->HScan("tHSetKey", "", "", -1); 116 | if (hit == NULL) { 117 | log_info("HScan error!"); 118 | } 119 | for (; hit->Valid(); hit->Next()) { 120 | log_info("HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 121 | } 122 | log_info(""); 123 | delete hit; 124 | 125 | delete n; 126 | 127 | return 0; 128 | } 129 | -------------------------------------------------------------------------------- /example/hash_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | int main() 14 | { 15 | nemo::Options options; 16 | options.target_file_size_base = 20 * 1024 * 1024; 17 | 18 | Nemo *n = new Nemo("./tmp/", options); 19 | Status s; 20 | 21 | std::string res; 22 | 23 | /* 24 | *************************************************KV************************************************** 25 | */ 26 | 27 | std::vector keys; 28 | std::vector kvs; 29 | std::vector kvss; 30 | std::vector sms; 31 | 32 | /* 33 | *************************************************HASH************************************************** 34 | */ 35 | std::vector fields; 36 | std::vector values; 37 | std::vector fvs; 38 | std::vector fvss; 39 | 40 | for (int i = 0; i < 2; i++) { 41 | /* 42 | * Test HSet 43 | */ 44 | log_info("======Test HSet======"); 45 | s = n->HSet("tHSetKey", "field1", "value1"); 46 | s = n->HSet("tHSetKey", "field2", "value2"); 47 | s = n->HSet("tHSetKey", "field3", "value3"); 48 | s = n->HSet("tHSetKey", "field4", "value4"); 49 | s = n->HSet("tHSetKey", "field5", "value5"); 50 | log_info("Test HSet OK return %s", s.ToString().c_str()); 51 | log_info(""); 52 | 53 | /* 54 | * Test HGet 55 | */ 56 | log_info("======Test HGet======"); 57 | s = n->HSet("tHGetKey", "song", "tGetVal"); 58 | res = ""; 59 | s = n->HGet("tHGetKey", "song", &res); 60 | log_info("Test HGet OK return %s, result tHGetVal = %s", s.ToString().c_str(), res.c_str()); 61 | res = ""; 62 | s = n->HGet("tHGetNotFoundKey", "song", &res); 63 | log_info("Test Get NotFound return %s, result NULL = %s", s.ToString().c_str(), res.c_str()); 64 | s = n->HGet("tHGetKey", "non-field", &res); 65 | log_info("Test Get NotFound return %s, result NULL = %s", s.ToString().c_str(), res.c_str()); 66 | log_info(""); 67 | 68 | /* 69 | * Test HGetall 70 | */ 71 | log_info("======Test HGetall======"); 72 | fvs.clear(); 73 | s = n->HGetall("tHSetKey", fvs); 74 | log_info("Test HGetall OK return %s", s.ToString().c_str()); 75 | std::vector::iterator fv_iter; 76 | for (fv_iter = fvs.begin(); fv_iter != fvs.end(); fv_iter++) { 77 | log_info("Test HGetall, field: %s, val: %s", fv_iter->field.c_str(), fv_iter->val.c_str()); 78 | } 79 | log_info(""); 80 | 81 | /* 82 | * Test HScan 83 | */ 84 | log_info("======Test HScan 0 with Next======"); 85 | 86 | HIterator *hit = n->HScan("tHSetKey", "", "", -1); 87 | if (hit == NULL) { 88 | log_info("HScan error!"); 89 | } 90 | for (; hit->Valid(); hit->Next()) { 91 | log_info("HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 92 | } 93 | log_info(""); 94 | delete hit; 95 | 96 | // log_info("======Test HScan 1 with Valid======"); 97 | 98 | // hit = n->HScan("tHSetKey", "field2", "field5", -1); 99 | // if (hit == NULL) { 100 | // log_info("HScan error!"); 101 | // } 102 | // while (hit->Valid()) { 103 | // log_info("HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 104 | // bool next_ret = hit->Next(); 105 | // log_info(" After hit->Next HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 106 | // } 107 | // log_info(""); 108 | 109 | /* 110 | * Test HDelKey 111 | */ 112 | log_info("======Test HDelKey======"); 113 | 114 | int64_t del_ret; 115 | s = n->HDelKey("tHSetKey", &del_ret); 116 | log_info("Test HDelKey return %s", s.ToString().c_str()); 117 | 118 | hit = n->HScan("tHSetKey", "", "", -1); 119 | if (hit == NULL) { 120 | log_info("HScan error!"); 121 | } 122 | for (; hit->Valid(); hit->Next()) { 123 | log_info("HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 124 | } 125 | log_info(""); 126 | delete hit; 127 | } 128 | 129 | s = n->HSet("tHSetKey", "field11", "value11"); 130 | s = n->HSet("tHSetKey", "field21", "value21"); 131 | s = n->HSet("tHSetKey", "field31", "value31"); 132 | s = n->HSet("tHSetKey", "field41", "value41"); 133 | s = n->HSet("tHSetKey", "field51", "value51"); 134 | 135 | /* 136 | * Test HGetall 137 | */ 138 | log_info("======Test HGetall with new fields======"); 139 | fvs.clear(); 140 | s = n->HGetall("tHSetKey", fvs); 141 | log_info("Test HGetall OK return %s", s.ToString().c_str()); 142 | std::vector::iterator fv_iter; 143 | for (fv_iter = fvs.begin(); fv_iter != fvs.end(); fv_iter++) { 144 | log_info("Test HGetall, field: %s, val: %s", fv_iter->field.c_str(), fv_iter->val.c_str()); 145 | } 146 | log_info(""); 147 | 148 | /* 149 | * Test HGetall 150 | */ 151 | log_info("======Test Compact======"); 152 | n->Compact(); 153 | 154 | log_info(" ====== HGetall after compact======"); 155 | fvs.clear(); 156 | s = n->HGetall("tHSetKey", fvs); 157 | log_info("Test HGetall OK return %s", s.ToString().c_str()); 158 | for (fv_iter = fvs.begin(); fv_iter != fvs.end(); fv_iter++) { 159 | log_info("Test HGetall, field: %s, val: %s", fv_iter->field.c_str(), fv_iter->val.c_str()); 160 | } 161 | log_info(""); 162 | 163 | /* 164 | * Test Expire 165 | */ 166 | int64_t e_ret; 167 | int64_t ttl; 168 | 169 | log_info("======Test HExpire======"); 170 | s = n->HSet("tHSetKey", "field11", "value11"); 171 | s = n->HExpire("tHSetKey", 7, &e_ret); 172 | log_info("Test HExpire with key=tHSetKey in 7s, return %s", s.ToString().c_str()); 173 | 174 | for (int i = 0; i < 3; i++) { 175 | sleep(3); 176 | s = n->HGet("tHSetKey", "field11", &res); 177 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 178 | if (s.ok()) { 179 | n->HTTL("tHSetKey", &ttl); 180 | log_info(" new TTL is %ld, HGet field11 res:%s\n", ttl, res.c_str()); 181 | } 182 | } 183 | log_info(""); 184 | 185 | /* 186 | * Test Expireat 187 | */ 188 | log_info("======Test HExpireat======"); 189 | s = n->HSet("tHSetKey", "field12", "value11"); 190 | 191 | std::time_t t = std::time(0); 192 | s = n->HExpireat("tHSetKey", t + 8, &e_ret); 193 | log_info("Test Expireat with key=tHSetKey at timestamp=%ld in 8s, return %s", (t+8), s.ToString().c_str()); 194 | 195 | for (int i = 0; i < 3; i++) { 196 | sleep(3); 197 | s = n->HGet("tHSetKey", "field12", &res); 198 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 199 | if (s.ok()) { 200 | n->HTTL("tHSetKey", &ttl); 201 | log_info(" new TTL is %ld, HGet field12 res:%s\n", ttl, res.c_str()); 202 | } 203 | } 204 | log_info(""); 205 | 206 | s = n->HSet("tHSetKey", "field12", "value11"); 207 | s = n->HExpireat("tHSetKey", 8, &e_ret); 208 | log_info("Test HExpireat with key=tHSetKey at a passed timestamp=8, return %s", s.ToString().c_str()); 209 | s = n->HGet("tHSetKey", "field12", &res); 210 | log_info(" Get a invalid key return %s, expect ok", s.ToString().c_str()); 211 | if (s.IsNotFound()) { 212 | n->TTL("tHSetKey", &ttl); 213 | log_info(" NotFound key's TTL is %ld, HGet res:%s\n", ttl, res.c_str()); 214 | } 215 | log_info(""); 216 | 217 | /* 218 | * Test Persist 219 | */ 220 | log_info("======Test HPersist======"); 221 | s = n->HSet("tHSetKey", "field12", "value11"); 222 | s = n->HExpire("tHSetKey", 7, &e_ret); 223 | log_info("Test Persist with key=tHSetKey in 7s, return %s", s.ToString().c_str()); 224 | 225 | for (int i = 0; i < 3; i++) { 226 | sleep(3); 227 | if (i == 1) { 228 | s = n->HPersist("tHSetKey", &e_ret); 229 | log_info(" Test HPersist return %s", s.ToString().c_str()); 230 | } 231 | s = n->HGet("tHSetKey", "field12", &res); 232 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 233 | if (s.ok()) { 234 | n->HTTL("tHSetKey", &ttl); 235 | log_info(" new TTL is %ld, HGet field12 res:%s\n", ttl, res.c_str()); 236 | } 237 | } 238 | log_info(""); 239 | 240 | delete n; 241 | 242 | return 0; 243 | } 244 | -------------------------------------------------------------------------------- /example/list_lock.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | Nemo *n; 14 | 15 | void* ThreadMain1(void *arg) { 16 | int64_t llen; 17 | Status s = n->LPush("listKey", "value1", &llen); 18 | log_info("Test LPush (key, value1) OK return %s, llen=%ld", s.ToString().c_str(), llen); 19 | return NULL; 20 | } 21 | 22 | void* ThreadMain2(void *arg) { 23 | int64_t llen; 24 | Status s = n->LPush("listKey", "value2", &llen); 25 | log_info("Test LPush (key, value2) OK return %s, llen=%ld", s.ToString().c_str(), llen); 26 | return NULL; 27 | } 28 | 29 | void* ThreadMain3(void *arg) { 30 | int64_t llen; 31 | Status s = n->LPush("listKey", "value3", &llen); 32 | log_info("Test LPush (key, value3) OK return %s, llen=%ld", s.ToString().c_str(), llen); 33 | return NULL; 34 | } 35 | 36 | void* ThreadMain4(void *arg) { 37 | int64_t llen; 38 | Status s = n->LPush("listKey", "value3", &llen); 39 | log_info("Test LPush (key, value3) OK return %s, llen=%ld", s.ToString().c_str(), llen); 40 | return NULL; 41 | } 42 | 43 | void* ThreadMain5(void *arg) { 44 | int64_t llen; 45 | Status s = n->LPush("listKey1", "value3", &llen); 46 | log_info("Test LPush (key1, value3) OK return %s, llen=%ld", s.ToString().c_str(), llen); 47 | return NULL; 48 | } 49 | 50 | void* ThreadMain6(void *arg) { 51 | int64_t llen; 52 | Status s = n->LPush("listKey2", "value3", &llen); 53 | log_info("Test LPush (key2, value3) OK return %s, llen=%ld", s.ToString().c_str(), llen); 54 | return NULL; 55 | } 56 | 57 | int main() { 58 | nemo::Options options; 59 | options.target_file_size_base = 20 * 1024 * 1024; 60 | 61 | n = new Nemo("./tmp/", options); 62 | Status s; 63 | 64 | std::string res; 65 | 66 | std::vector keys; 67 | std::vector kvs; 68 | std::vector kvss; 69 | std::vector sms; 70 | 71 | /* 72 | *************************************************HASH************************************************** 73 | */ 74 | std::vector fields; 75 | std::vector values; 76 | std::vector fvs; 77 | std::vector fvss; 78 | 79 | log_info("======Test LPush Same key======"); 80 | 81 | pthread_t tid[3]; 82 | pthread_create(&tid[0], NULL, &ThreadMain1, NULL); 83 | pthread_create(&tid[1], NULL, &ThreadMain2, NULL); 84 | pthread_create(&tid[2], NULL, &ThreadMain3, NULL); 85 | 86 | for (int i = 0; i < 3; i++) { 87 | pthread_join(tid[i], 0); 88 | } 89 | 90 | log_info("======Test LPush different key======"); 91 | 92 | pthread_create(&tid[0], NULL, &ThreadMain4, NULL); 93 | pthread_create(&tid[1], NULL, &ThreadMain5, NULL); 94 | 95 | for (int i = 0; i < 2; i++) { 96 | pthread_join(tid[i], 0); 97 | } 98 | 99 | //log_info("======Test LPush different key with all the lru in use======"); 100 | // pthread_create(&tid[0], NULL, &ThreadMain4, NULL); 101 | // pthread_create(&tid[1], NULL, &ThreadMain5, NULL); 102 | // pthread_create(&tid[2], NULL, &ThreadMain6, NULL); 103 | 104 | // for (int i = 0; i < 3; i++) { 105 | // pthread_join(tid[i], 0); 106 | // } 107 | 108 | log_info(""); 109 | 110 | delete n; 111 | 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /example/rush.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include "nemo.h" 12 | #include "xdebug.h" 13 | 14 | using namespace nemo; 15 | using namespace std; 16 | 17 | 18 | char kv_key_header[] = "godblesspika_kv_key_"; 19 | char kv_value_header[] = "godblesspika_kv_value_"; 20 | 21 | char hash_key_header[] = "godblesspika_hash_key_"; 22 | char hash_field_header[] = "godblesspika_hash_field_"; 23 | char hash_value_header[] = "godblesspika_hash_value_"; 24 | 25 | char list_key_header[] = "godblesspika_list_key_"; 26 | char list_field_header[] = "godblesspika_list_field_"; 27 | 28 | char zset_key_header[] = "godblesspika_zset_key_"; 29 | char zset_member_header[] = "godblesspika_zset_member_"; 30 | 31 | char set_key_header[] = "godblesspika_set_key_"; 32 | char set_member_header[] = "godblesspika_set_member_"; 33 | 34 | Nemo *n; 35 | int64_t cnt; 36 | 37 | inline int64_t NowMicros() { 38 | struct timeval tv; 39 | gettimeofday(&tv, NULL); 40 | return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; 41 | } 42 | 43 | char* gen_str(char *key, char *prefix, int64_t id) { 44 | //snprintf (key, 256, "%s%010ld", prefix, id); 45 | snprintf (key, 256, "%s%ld", prefix, id); 46 | return key; 47 | } 48 | char* gen_str_len(char *key, int len) { 49 | for (int i = 0; i < len - 1; i++) { 50 | key[i] = 'a' + rand() % 26; 51 | } 52 | key[len - 1] = '\0'; 53 | return key; 54 | } 55 | 56 | void* call_Dump(void *arg) { 57 | int type = *((int *)arg); 58 | 59 | printf ("thread type=%d\n", type); 60 | 61 | Status s; 62 | std::string sret; 63 | int64_t iret; 64 | 65 | char key[256]; 66 | char value[256]; 67 | 68 | for (int64_t i = 0; i < cnt; i++) { 69 | if (i % 100000 == 0) printf ("thread type:%d i=%ld\n", type, i); 70 | 71 | switch (type) { 72 | case 0: { 73 | string k(gen_str(key, kv_key_header, i)); 74 | string v(gen_str(key, kv_value_header, i)); 75 | n->Set(k, v); 76 | break; 77 | } 78 | 79 | case 1: { 80 | string hash_k(gen_str(key, hash_key_header, i)); 81 | string hash_f(gen_str(key, hash_field_header, i)); 82 | string hash_v(gen_str(key, hash_value_header, i)); 83 | n->HSet(hash_k, hash_f, hash_v); 84 | break; 85 | } 86 | 87 | case 2: { 88 | string list_k(gen_str(key, list_key_header, i)); 89 | string list_f(gen_str(key, list_field_header, i)); 90 | n->LPush(list_k, list_f, &iret); 91 | break; 92 | } 93 | 94 | case 3: { 95 | string zset_k(gen_str(key, zset_key_header, i)); 96 | string zset_m(gen_str(key, zset_member_header, i)); 97 | n->ZAdd(zset_k, i, zset_m, &iret); 98 | break; 99 | } 100 | 101 | case 4: { 102 | string set_k(gen_str(key, set_key_header, i)); 103 | string set_m(gen_str(key, set_member_header, i)); 104 | n->SAdd(set_k, set_m, &iret); 105 | break; 106 | } 107 | } 108 | } 109 | } 110 | 111 | int main(int argc, char *argv[]) 112 | { 113 | nemo::Options options; 114 | options.target_file_size_base = 20 * 1024 * 1024; 115 | options.write_buffer_size = 256 * 1024 * 1024; 116 | 117 | 118 | if (argc < 3) { 119 | printf ("usage: rush count db_path\n"); 120 | exit (-1); 121 | } 122 | 123 | char *pEnd; 124 | cnt = strtoll(argv[1], &pEnd, 10); 125 | std::string path(argv[2]); 126 | 127 | 128 | int64_t st = NowMicros(); 129 | printf ("cnt=%ld db_path: %s st_timestamp:%ld\n", cnt, path.c_str(), st); 130 | 131 | n = new Nemo(path.c_str(), options); 132 | 133 | pthread_t tid[5]; 134 | 135 | for (int i = 0; i < 5; i++) { 136 | int *arg = new int; 137 | *arg = i; 138 | if (pthread_create(&tid[i], NULL, &call_Dump, arg) != 0) { 139 | printf ("pthread create %d failed\n", i); 140 | } 141 | } 142 | 143 | 144 | void *retval; 145 | for (int i = 0; i < 5; i++) { 146 | pthread_join(tid[i], &retval); 147 | } 148 | 149 | int64_t ed = NowMicros(); 150 | printf ("End ts:%ld\n", ed); 151 | 152 | std::vector nums; 153 | n->GetKeyNum(nums); 154 | 155 | for (int i = 0; i < nums.size(); i++) { 156 | printf (" key_num i=%d %ld\n", i, nums[i]); 157 | } 158 | 159 | 160 | return 0; 161 | } 162 | -------------------------------------------------------------------------------- /example/simple_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | Nemo *n; 14 | 15 | int main() { 16 | nemo::Options options; 17 | options.target_file_size_base = 20 * 1024 * 1024; 18 | 19 | n = new Nemo("./tmp/", options); 20 | 21 | std::string res; 22 | 23 | Status s = n->HSet("Key", "field1", "val1"); 24 | s = n->HSet("Key", "field2", "val2"); 25 | std::cout << "HSet return: " << s.ToString() << std::endl; 26 | 27 | int64_t l = n->HLen("Key"); 28 | std::cout << "HLen return: " << l << std::endl; 29 | delete n; 30 | 31 | return 0; 32 | } 33 | -------------------------------------------------------------------------------- /example/test_server.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "nemo.h" 8 | #include "nemo_const.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | int main() 14 | { 15 | nemo::Options options; 16 | options.max_background_flushes = 2; 17 | options.max_background_compactions = 2; 18 | Nemo *n = new Nemo("./tmp", options); 19 | Status s; 20 | 21 | 22 | std::vector nums; 23 | int64_t llen, za_res, sadd_res; 24 | 25 | // n->GetKeyNum(nums); 26 | // 27 | // for (int i = 0; i < nums.size(); i++) { 28 | // printf (" i=%d %ld\n", i, nums[i]); 29 | // } 30 | // 31 | // uint64_t num; 32 | // n->GetSpecifyKeyNum("kv", num); 33 | // printf ("kv key num: %lu\n", num); 34 | // 35 | // n->GetSpecifyKeyNum("hash", num); 36 | // printf ("hash key num: %lu\n", num); 37 | // 38 | // n->GetSpecifyKeyNum("list", num); 39 | // printf ("list key num: %lu\n", num); 40 | // 41 | // n->GetSpecifyKeyNum("zset", num); 42 | // printf ("zset key num: %lu\n", num); 43 | // 44 | // n->GetSpecifyKeyNum("set", num); 45 | // printf ("set key num: %lu\n", num); 46 | // 47 | // s = n->GetSpecifyKeyNum("invalid type", num); 48 | // printf ("test invalid type: "); 49 | // if (!s.ok()) { 50 | // printf ("SUCCESS, expect !ok\n"); 51 | // } else { 52 | // printf ("FAILED, return ok, should failed\n"); 53 | // } 54 | 55 | /* 56 | * test Compact 57 | */ 58 | std::string res; 59 | 60 | for (int i = 0; i < 1; i++) { 61 | log_info("====== i: %3d ======", i); 62 | log_info("======Test Compact======"); 63 | 64 | // Compact 65 | s = n->Compact(nemo::kALL, true); 66 | log_info(" Compact(sync=true) return %s", s.ToString().c_str()); 67 | 68 | // Compact 69 | s = n->Compact(nemo::kALL); 70 | log_info(" Compact(sync=false) return %s", s.ToString().c_str()); 71 | } 72 | 73 | char a; 74 | scanf ("%c", &a); 75 | 76 | /* 77 | * test CompactKey 78 | */ 79 | 80 | for (int i = 0; i < 1; i++) { 81 | log_info("====== i: %3d ======", i); 82 | log_info("======Test CompactKey======"); 83 | s = n->Set("key", "setval1"); 84 | s = n->HSet("key", "hashfield1", "hashVal1"); 85 | s = n->HSet("key", "hashfield2", "hashVal2"); 86 | s = n->HSet("key", "hashfield3", "hashVal3"); 87 | s = n->LPush("key", "tLPushVal1", &llen); 88 | s = n->LPush("key", "tLPushVal2", &llen); 89 | s = n->LPush("key", "tLPushVal3", &llen); 90 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 91 | s = n->ZAdd("key", 90.0, "zsetMember2", &za_res); 92 | s = n->ZAdd("key", 80.0, "zsetMember3", &za_res); 93 | s = n->SAdd("key", "member1", &sadd_res); 94 | s = n->SAdd("key", "member2", &sadd_res); 95 | s = n->SAdd("key", "member3", &sadd_res); 96 | 97 | log_info(""); 98 | log_info("Before CompactKey"); 99 | s = n->Get("key", &res); 100 | log_info(" KV: Get return %s, result res = %s", s.ToString().c_str(), res.c_str()); 101 | 102 | for (int i = 0; i < llen; i++) { 103 | s = n->LIndex("key", i, &res); 104 | log_info(" List[%d] return %s, result res = %s", i, s.ToString().c_str(), res.c_str()); 105 | } 106 | 107 | HIterator *hit = n->HScan("key", "", "", -1); 108 | if (hit == NULL) { 109 | log_info(" HScan error!"); 110 | } 111 | for (; hit->Valid(); hit->Next()) { 112 | log_info(" HScan key: %s, field: %s, value: %s", hit->key().c_str(), hit->field().c_str(), hit->value().c_str()); 113 | } 114 | log_info(""); 115 | delete hit; 116 | 117 | 118 | ZIterator *zit = n->ZScan("key", ZSET_SCORE_MIN, ZSET_SCORE_MAX, -1); 119 | if (zit == NULL) { 120 | log_info("ZScan error!"); 121 | } 122 | for (; zit->Valid(); zit->Next()) { 123 | log_info(" ZScan key: %s, score: %lf, member: %s", zit->key().c_str(), zit->score(), zit->member().c_str()); 124 | } 125 | 126 | 127 | delete zit; 128 | 129 | int64_t del_ret; 130 | log_info("======Del key======"); 131 | s = n->Del("key", &del_ret); 132 | log_info(" Del OK return %s, count is %ld", s.ToString().c_str(), del_ret); 133 | } 134 | 135 | 136 | char ch; 137 | scanf ("%c", &ch); 138 | 139 | return 0; 140 | 141 | /* 142 | * test Keys 143 | */ 144 | log_info("======Test Keys======"); 145 | s = n->Set("key", "setval1"); 146 | s = n->HSet("key", "hashfield", "tSetVal1"); 147 | s = n->LPush("key", "tLPushVal1", &llen); 148 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 149 | s = n->SAdd("key", "member1", &sadd_res); 150 | 151 | 152 | std::vector keys; 153 | std::vector::iterator smit; 154 | keys.clear(); 155 | s = n->Keys("*", keys); 156 | log_info("Test Keys(\"*\") return %s, size=%ld", s.ToString().c_str(), keys.size()); 157 | int i = 0; 158 | for (smit = keys.begin(); smit != keys.end(); smit++) { 159 | log_info(" %d : %s", i++, smit->c_str()); 160 | } 161 | 162 | keys.clear(); 163 | i = 0; 164 | s = n->Keys("*key*", keys); 165 | log_info("Test Keys(\"*key*\") return %s, size=%ld", s.ToString().c_str(), keys.size()); 166 | for (smit = keys.begin(); smit != keys.end(); smit++) { 167 | log_info(" %d : %s", i++, smit->c_str()); 168 | } 169 | 170 | keys.clear(); 171 | i = 0; 172 | s = n->Keys("t[HL]Set*", keys); 173 | log_info("Test Keys(\"t[HL]Set*\") return %s, size=%ld", s.ToString().c_str(), keys.size()); 174 | for (smit = keys.begin(); smit != keys.end(); smit++) { 175 | log_info(" %d : %s", i++, smit->c_str()); 176 | } 177 | 178 | 179 | int64_t e_ret, ttl; 180 | 181 | log_info("======Test Expire======"); 182 | s = n->Expire("key", 4, &e_ret); 183 | log_info("Test Expire with key=key in 7s, [hash, list, zset, set] return %s", s.ToString().c_str()); 184 | 185 | for (int i = 0; i < 2; i++) { 186 | sleep(3); 187 | std::string res; 188 | 189 | s = n->Get("key", &res); 190 | log_info(" after %ds, Get return %s", (i+1)*3, s.ToString().c_str()); 191 | 192 | s = n->HGet("key", "hashfield", &res); 193 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 194 | 195 | double score; 196 | s = n->ZScore("key", "zsetMember1", &score); 197 | log_info(" after %ds, ZScore return %s", (i+1)*3, s.ToString().c_str()); 198 | 199 | int ret = n->SIsMember("key", "member1"); 200 | log_info(" after %ds, SIsMember return %d, [true|false]", (i+1)*3, ret); 201 | 202 | s = n->LIndex("key", 0, &res); 203 | log_info(" after %ds, LIndex(0) return %s, val is %s", (i+1)*3, s.ToString().c_str(), res.c_str()); 204 | 205 | if (s.ok()) { 206 | s = n->TTL("key", &ttl); 207 | log_info(" new TTL is %ld, TTL return %s\n", ttl, s.ToString().c_str()); 208 | } 209 | } 210 | 211 | keys.clear(); 212 | i = 0; 213 | s = n->Keys("*", keys); 214 | log_info("Test Keys(\"*\") return %s, size=%ld", s.ToString().c_str(), keys.size()); 215 | for (smit = keys.begin(); smit != keys.end(); smit++) { 216 | log_info(" %d : %s", i++, smit->c_str()); 217 | } 218 | 219 | delete n; 220 | return 0; 221 | } 222 | -------------------------------------------------------------------------------- /example/ttl.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | int main() 14 | { 15 | nemo::Options options; 16 | options.target_file_size_base = 20 * 1024 * 1024; 17 | 18 | Nemo *n = new Nemo("./tmp/", options); 19 | Status s; 20 | 21 | std::string res; 22 | 23 | /* 24 | *************************************************KV************************************************** 25 | */ 26 | 27 | std::vector keys; 28 | std::vector kvs; 29 | std::vector kvss; 30 | std::vector sms; 31 | 32 | int64_t llen; 33 | int64_t za_res; 34 | int64_t sadd_res; 35 | 36 | /* 37 | * Test MDel 38 | */ 39 | log_info("======Test MDel======"); 40 | s = n->Set("key", "setval1"); 41 | s = n->HSet("key", "hashfield", "tSetVal1"); 42 | s = n->LPush("key", "tLPushVal1", &llen); 43 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 44 | s = n->SAdd("key", "member1", &sadd_res); 45 | 46 | /* 47 | * Test MDel 48 | */ 49 | log_info("======Test MDel======"); 50 | int64_t mcount; 51 | 52 | keys.push_back("key"); 53 | s = n->MDel(keys, &mcount); 54 | log_info("Test MDel OK return %s", s.ToString().c_str()); 55 | 56 | s = n->HGet("key", "hashfield", &res); 57 | log_info(" return %s", s.ToString().c_str()); 58 | 59 | int64_t ret; 60 | double score; 61 | s = n->ZScore("key", "zsetMember1", &score); 62 | log_info(" ZScore return %s", s.ToString().c_str()); 63 | 64 | ret = n->SIsMember("key", "member1"); 65 | log_info(" SIsMember return %ld, [true|false]", ret); 66 | 67 | s = n->LIndex("key", 0, &res); 68 | log_info(" LIndex(0) return %s, val is %s", s.ToString().c_str(), res.c_str()); 69 | log_info(""); 70 | 71 | /* 72 | * Test TTL 73 | */ 74 | log_info("======Test TTL======"); 75 | s = n->Set("key", "setval1"); 76 | s = n->HSet("key", "hashfield", "tSetVal1"); 77 | s = n->LPush("key", "tLPushVal1", &llen); 78 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 79 | s = n->SAdd("key", "member1", &sadd_res); 80 | 81 | /* 82 | * Test Expire 83 | */ 84 | int64_t e_ret; 85 | int64_t ttl; 86 | log_info("======Test Expire======"); 87 | s = n->Expire("key", 7, &e_ret); 88 | log_info("Test Expire with key=key in 7s, [hash, list, zset, set] return %s", s.ToString().c_str()); 89 | 90 | for (int i = 0; i < 3; i++) { 91 | sleep(3); 92 | std::string res; 93 | 94 | s = n->Get("key", &res); 95 | log_info(" after %ds, Get return %s", (i+1)*3, s.ToString().c_str()); 96 | 97 | s = n->HGet("key", "hashfield", &res); 98 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 99 | 100 | double score; 101 | s = n->ZScore("key", "zsetMember1", &score); 102 | log_info(" after %ds, ZScore return %s", (i+1)*3, s.ToString().c_str()); 103 | 104 | int ret = n->SIsMember("key", "member1"); 105 | log_info(" after %ds, SIsMember return %d, [true|false]", (i+1)*3, ret); 106 | 107 | s = n->LIndex("key", 0, &res); 108 | log_info(" after %ds, LIndex(0) return %s, val is %s", (i+1)*3, s.ToString().c_str(), res.c_str()); 109 | 110 | if (s.ok()) { 111 | s = n->TTL("key", &ttl); 112 | log_info(" new TTL is %ld, TTL return %s\n", ttl, s.ToString().c_str()); 113 | } 114 | } 115 | log_info(""); 116 | 117 | /* 118 | * Test Expireat 119 | */ 120 | log_info("======Test HExpireat======"); 121 | s = n->Set("key", "setval1"); 122 | s = n->HSet("key", "hashfield", "tSetVal1"); 123 | s = n->LPush("key", "tLPushVal1", &llen); 124 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 125 | s = n->SAdd("key", "member1", &sadd_res); 126 | 127 | std::time_t t = std::time(0); 128 | s = n->Expireat("key", t + 8, &e_ret); 129 | log_info("Test Expireat with key=tHSetKey at timestamp=%ld in 8s, return %s", (t+8), s.ToString().c_str()); 130 | 131 | for (int i = 0; i < 3; i++) { 132 | sleep(3); 133 | std::string res; 134 | 135 | s = n->Get("key", &res); 136 | log_info(" after %ds, Get return %s", (i+1)*3, s.ToString().c_str()); 137 | 138 | s = n->HGet("key", "hashfield", &res); 139 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 140 | 141 | double score; 142 | s = n->ZScore("key", "zsetMember1", &score); 143 | log_info(" after %ds, ZScore return %s", (i+1)*3, s.ToString().c_str()); 144 | 145 | int ret = n->SIsMember("key", "member1"); 146 | log_info(" after %ds, SIsMember return %d, [true|false]", (i+1)*3, ret); 147 | 148 | s = n->LIndex("key", 0, &res); 149 | log_info(" after %ds, LIndex(0) return %s, val is %s", (i+1)*3, s.ToString().c_str(), res.c_str()); 150 | 151 | if (s.ok()) { 152 | s = n->TTL("key", &ttl); 153 | log_info(" new TTL is %ld, TTL return %s\n", ttl, s.ToString().c_str()); 154 | } 155 | } 156 | log_info(""); 157 | 158 | s = n->Set("key", "setval1"); 159 | s = n->HSet("key", "hashfield", "tSetVal1"); 160 | s = n->LPush("key", "tLPushVal1", &llen); 161 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 162 | s = n->SAdd("key", "member1", &sadd_res); 163 | 164 | s = n->Expireat("key", 8, &e_ret); 165 | log_info("\nTest Expireat with key=key at a passed timestamp=8, return %s", s.ToString().c_str()); 166 | 167 | s = n->HGet("key", "hashfield", &res); 168 | log_info(" return %s", s.ToString().c_str()); 169 | 170 | //double score; 171 | s = n->ZScore("key", "zsetMember1", &score); 172 | log_info(" ZScore return %s", s.ToString().c_str()); 173 | 174 | ret = n->SIsMember("key", "member1"); 175 | log_info(" SIsMember return %ld, [true|false]", ret); 176 | 177 | s = n->LIndex("key", 0, &res); 178 | log_info(" LIndex(0) return %s, val is %s", s.ToString().c_str(), res.c_str()); 179 | 180 | if (s.IsNotFound()) { 181 | n->TTL("key", &ttl); 182 | log_info(" NotFound key's TTL is %ld\n", ttl); 183 | } 184 | log_info(""); 185 | 186 | /* 187 | * Test Persist 188 | */ 189 | log_info("======Test Persist======"); 190 | s = n->Set("key", "setval1"); 191 | s = n->HSet("key", "hashfield", "tSetVal1"); 192 | s = n->LPush("key", "tLPushVal1", &llen); 193 | s = n->ZAdd("key", 100.0, "zsetMember1", &za_res); 194 | s = n->SAdd("key", "member1", &sadd_res); 195 | s = n->Expire("key", 7, &e_ret); 196 | log_info("Test Persist with key=key in 7s, return %s", s.ToString().c_str()); 197 | 198 | for (int i = 0; i < 3; i++) { 199 | sleep(3); 200 | if (i == 1) { 201 | s = n->Persist("key", &e_ret); 202 | log_info(" Test key return %s", s.ToString().c_str()); 203 | } 204 | std::string res; 205 | 206 | s = n->Get("key", &res); 207 | log_info(" after %ds, Get return %s", (i+1)*3, s.ToString().c_str()); 208 | 209 | s = n->HGet("key", "hashfield", &res); 210 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 211 | 212 | double score; 213 | s = n->ZScore("key", "zsetMember1", &score); 214 | log_info(" after %ds, ZScore return %s", (i+1)*3, s.ToString().c_str()); 215 | 216 | int ret = n->SIsMember("key", "member1"); 217 | log_info(" after %ds, SIsMember return %d, [true|false]", (i+1)*3, ret); 218 | 219 | s = n->LIndex("key", 0, &res); 220 | log_info(" after %ds, LIndex(0) return %s, val is %s", (i+1)*3, s.ToString().c_str(), res.c_str()); 221 | 222 | if (s.ok()) { 223 | s = n->TTL("key", &ttl); 224 | log_info(" new TTL is %ld, TTL return %s\n", ttl, s.ToString().c_str()); 225 | } 226 | } 227 | log_info(""); 228 | 229 | delete n; 230 | 231 | return 0; 232 | } 233 | -------------------------------------------------------------------------------- /example/zset.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | #include "../src/nemo_zset.h" 11 | 12 | using namespace nemo; 13 | 14 | int main() 15 | { 16 | // double k = -0.1; 17 | // printf ("EncodeScore(1000.123456) %ld\n", EncodeScore(1000.123456)); 18 | // printf ("DecodeScore(Encode(1000.123456) %f\n", DecodeScore(EncodeScore(1000.123456))); 19 | // 20 | // printf ("EncodeScore(%lf) %ld\n", -0.1, EncodeScore(-0.1)); 21 | // printf ("DecodeScore(Encode(%lf)) %f\n", -0.1, DecodeScore(EncodeScore(-0.1))); 22 | // 23 | // k = 0; 24 | // printf ("EncodeScore(%lf) %ld\n", k, EncodeScore(k)); 25 | // printf ("DecodeScore(Encode(%lf)) %f\n", k, DecodeScore(EncodeScore(k))); 26 | // 27 | // k = -1; 28 | // printf ("EncodeScore(%lf) %ld\n", k, EncodeScore(k)); 29 | // printf ("DecodeScore(Encode(%lf)) %f\n", k, DecodeScore(EncodeScore(k))); 30 | 31 | nemo::Options options; 32 | Nemo *n = new Nemo("./tmp/", options); 33 | Status s; 34 | 35 | std::string res; 36 | 37 | /* 38 | *************************************************KV************************************************** 39 | */ 40 | 41 | std::vector keys; 42 | std::vector kvs; 43 | std::vector kvss; 44 | std::vector sms; 45 | std::vector::iterator it_sm; 46 | 47 | /* 48 | * Test ZRank 49 | */ 50 | // log_info("======Test ZRank======"); 51 | // int64_t rank; 52 | int64_t zadd_res; 53 | // s = n->ZAdd("zr", 1, "a", &zadd_res); 54 | // s = n->ZAdd("zr", 0.2, "b", &zadd_res); 55 | // 56 | // s = n->ZRange("zr", 0, -1, sms); 57 | // for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 58 | // log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 59 | // } 60 | // log_info(""); 61 | // 62 | // s = n->ZRank("zr", "a", &rank); 63 | // log_info("Test ZRank (a) return %s, rank: %ld", s.ToString().c_str(), rank); 64 | // s = n->ZRank("zr", "b", &rank); 65 | // log_info("Test ZRank (b) return %s, rank: %ld", s.ToString().c_str(), rank); 66 | 67 | /* 68 | * Test ZRemrangebyscore 69 | */ 70 | log_info("======Test ZRemrangebyscore ======"); 71 | s = n->ZAdd("zr", -1, "a", &zadd_res); 72 | s = n->ZAdd("zr", -0.1, "b", &zadd_res); 73 | s = n->ZAdd("zr", 0, "c", &zadd_res); 74 | s = n->ZAdd("zr", 1, "d", &zadd_res); 75 | s = n->ZAdd("zr", 2, "e", &zadd_res); 76 | 77 | 78 | int64_t count; 79 | s = n->ZRemrangebyscore("zr", 0, 0, &count); 80 | log_info("Test ZRremrangebyscore (0, 0) return %s, count is %ld, expect 1", s.ToString().c_str(), count); 81 | 82 | log_info(" After ZRremrangebyscore (0, 0), zrange expect 4 [a, b, d, e]:"); 83 | sms.clear(); 84 | s = n->ZRange("zr", 0, -1, sms); 85 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 86 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 87 | } 88 | log_info(""); 89 | 90 | 91 | s = n->ZRemrangebyscore("zr", -1, 0, &count); 92 | log_info("Test ZRremrangebyscore (-1, 0) return %s, count is %ld, expect 2", s.ToString().c_str(), count); 93 | 94 | log_info(" After ZRremrangebyscore (-1, 0), zrange expect 2 [d, e]:"); 95 | sms.clear(); 96 | s = n->ZRange("zr", 0, -1, sms); 97 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 98 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 99 | } 100 | log_info(""); 101 | 102 | double score; 103 | s = n->ZScore("zr", "a", &score); 104 | log_info( "zscore a return %s, score is %lf\n", s.ToString().c_str(), score); 105 | 106 | s = n->ZScore("zr", "b", &score); 107 | log_info( "zscore b return %s, score is %lf\n", s.ToString().c_str(), score); 108 | 109 | s = n->ZScore("zr", "d", &score); 110 | log_info( "zscore d return %s, score is %lf\n", s.ToString().c_str(), score); 111 | 112 | s = n->ZScore("zr", "e", &score); 113 | log_info( "zscore e return %s, score is %lf\n", s.ToString().c_str(), score); 114 | 115 | /* 116 | * Test ZUnionStore 117 | */ 118 | log_info("======Test ZUnionStore======"); 119 | s = n->ZAdd("tZUnionKey1", 0, "member1", &zadd_res); 120 | s = n->ZAdd("tZUnionKey1", 1.0, "member2", &zadd_res); 121 | s = n->ZAdd("tZUnionKey1", 2.0, "member3", &zadd_res); 122 | s = n->ZAdd("tZUnionKey1", 3.0, "member4", &zadd_res); 123 | 124 | 125 | s = n->ZAdd("tZUnionKey2", 1.0, "member1", &zadd_res); 126 | s = n->ZAdd("tZUnionKey2", 2.5, "member2", &zadd_res); 127 | s = n->ZAdd("tZUnionKey2", 2.9, "member3", &zadd_res); 128 | 129 | s = n->ZAdd("tZUnionKey3", 2.5, "member1", &zadd_res); 130 | 131 | keys.clear(); 132 | keys.push_back("tZUnionKey2"); 133 | keys.push_back("tZUnionKey1"); 134 | 135 | std::vector weights; 136 | weights.push_back(1); 137 | weights.push_back(10); 138 | 139 | int64_t union_ret; 140 | s = n->ZUnionStore("tZUnionNewKey1", 2, keys, weights, Aggregate::SUM, &union_ret); 141 | if (!s.ok()) { 142 | log_info("Test ZUnionStore [newkey1, key1, key2] return %s", s.ToString().c_str()); 143 | } else { 144 | sms.clear(); 145 | s = n->ZRange("tZUnionNewKey1", -10, -1, sms); 146 | log_info(" ZUnionStore OK, union (10 * key1, 1 * key2) should be [member1=1, member2=12.5, member3=22.9,member4=30]"); 147 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 148 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 149 | } 150 | } 151 | 152 | weights.clear(); 153 | s = n->ZUnionStore("tZUnionNewKey2", 2, keys, weights, Aggregate::MAX, &union_ret); 154 | if (!s.ok()) { 155 | log_info("Test ZUnionStore [newkey2, key1, key2] return %s", s.ToString().c_str()); 156 | } else { 157 | sms.clear(); 158 | s = n->ZRange("tZUnionNewKey2", -10, -1, sms); 159 | log_info(" ZUnionStore OK, union MAX(1 * key1, 1 * key2) should be [member1=1, member2=2.5, member3=2.9,member4=3]"); 160 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 161 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 162 | } 163 | } 164 | log_info(""); 165 | 166 | 167 | 168 | return 0; 169 | } 170 | -------------------------------------------------------------------------------- /example/zset_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | using namespace nemo; 12 | 13 | int main() 14 | { 15 | nemo::Options options; 16 | options.target_file_size_base = 20 * 1024 * 1024; 17 | 18 | Nemo *n = new Nemo("./tmp/", options); 19 | Status s; 20 | 21 | std::string res; 22 | 23 | 24 | std::vector keys; 25 | std::vector kvs; 26 | std::vector kvss; 27 | std::vector sms; 28 | 29 | /* 30 | *************************************************ZSet************************************************** 31 | */ 32 | /* 33 | * Test Expire 34 | */ 35 | int64_t e_ret; 36 | int64_t ttl; 37 | int64_t zadd_res; 38 | 39 | s = n->ZAdd("tZSetKey", 0, "field11", &zadd_res); 40 | log_info("======ZAdd return %s", s.ToString().c_str()); 41 | log_info(" ======After zadd ZCard======"); 42 | log_info(" Test ZCard, return card is %ld", n->ZCard("tZSetKey")); 43 | log_info(""); 44 | 45 | log_info("======Test ZExpire======"); 46 | s = n->ZExpire("tZSetKey", 7, &e_ret); 47 | log_info("Test ZExpire with key=tZSetKey in 7s, return %s", s.ToString().c_str()); 48 | log_info(" ======After zexpire ZCard======"); 49 | log_info(" Test ZCard, return card is %ld", n->ZCard("tZSetKey")); 50 | log_info(""); 51 | 52 | for (int i = 0; i < 3; i++) { 53 | sleep(3); 54 | double score; 55 | s = n->ZScore("tZSetKey", "field11", &score); 56 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 57 | if (s.ok()) { 58 | s = n->ZTTL("tZSetKey", &ttl); 59 | log_info(" new ZTTL return %s, ttl is %ld, ZScore tZSetKey field11 score:%lf\n", 60 | s.ToString().c_str(), ttl, score); 61 | } 62 | } 63 | log_info(""); 64 | 65 | std::vector::iterator it_sm; 66 | 67 | for (int i = 0; i < 2; i++) { 68 | s = n->ZAdd("zr", 1, "a", &zadd_res); 69 | s = n->ZAdd("zr", 0.2, "b", &zadd_res); 70 | 71 | log_info("======Test ZDelKey zrange before======"); 72 | s = n->ZRange("zr", 0, -1, sms); 73 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 74 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 75 | } 76 | log_info(""); 77 | 78 | int64_t del_ret; 79 | s = n->ZDelKey("zr", &del_ret); 80 | log_info("======ZDelKey return %s", s.ToString().c_str()); 81 | 82 | sms.clear(); 83 | log_info("======Test ZDelKey zrange after======"); 84 | s = n->ZRange("zr", 0, -1, sms); 85 | for (it_sm = sms.begin(); it_sm != sms.end(); it_sm++) { 86 | log_info(" score: %lf, member: %s", it_sm->score, it_sm->member.c_str()); 87 | } 88 | log_info(""); 89 | } 90 | 91 | /* 92 | * Test Expireat 93 | */ 94 | log_info("======Test ZExpireat======"); 95 | s = n->ZAdd("tZSetKey", 0, "field12", &zadd_res); 96 | 97 | std::time_t t = std::time(0); 98 | s = n->ZExpireat("tZSetKey", t + 8, &e_ret); 99 | log_info("Test Expireat with key=tZSetKey at timestamp=%ld in 8s, return %s", (t+8), s.ToString().c_str()); 100 | 101 | for (int i = 0; i < 3; i++) { 102 | sleep(3); 103 | double score; 104 | s = n->ZScore("tZSetKey", "field12", &score); 105 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 106 | if (s.ok()) { 107 | s = n->ZTTL("tZSetKey", &ttl); 108 | log_info(" new ZTTL return %s, ttl is %ld, ZScore tZSetKey field12 score:%lf\n", 109 | s.ToString().c_str(), ttl, score); 110 | } 111 | } 112 | log_info(""); 113 | 114 | s = n->ZAdd("tZSetKey", 0, "field12", &zadd_res); 115 | s = n->ZExpireat("tZSetKey", 8, &e_ret); 116 | log_info("Test ZExpireat with key=tZSetKey at a passed timestamp=8, return %s", s.ToString().c_str()); 117 | double score; 118 | s = n->ZScore("tZSetKey", "field12", &score); 119 | log_info(" Get a invalid key return %s, expect ok", s.ToString().c_str()); 120 | if (s.IsNotFound()) { 121 | n->ZTTL("tZSetKey", &ttl); 122 | log_info(" NotFound key's TTL is %ld, ZScore score:%lf\n", ttl, score); 123 | } 124 | log_info(""); 125 | 126 | /* 127 | * Test Persist 128 | */ 129 | log_info("======Test ZPersist======"); 130 | s = n->ZAdd("tZSetKey", 0, "field12", &zadd_res); 131 | s = n->ZExpire("tZSetKey", 7, &e_ret); 132 | log_info("Test ZPersist with key=tZSetKey in 7s, return %s", s.ToString().c_str()); 133 | 134 | for (int i = 0; i < 3; i++) { 135 | sleep(3); 136 | if (i == 1) { 137 | s = n->ZPersist("tZSetKey", &e_ret); 138 | log_info(" Test ZPersist return %s", s.ToString().c_str()); 139 | } 140 | double score; 141 | s = n->ZScore("tZSetKey", "field12", &score); 142 | log_info(" after %ds, return %s", (i+1)*3, s.ToString().c_str()); 143 | if (s.ok()) { 144 | s = n->ZTTL("tZSetKey", &ttl); 145 | log_info(" new ZTTL return %s, ttl is %ld, ZScore tZSetKey field12 score:%lf\n", 146 | s.ToString().c_str(), ttl, score); 147 | } 148 | } 149 | log_info(""); 150 | delete n; 151 | 152 | return 0; 153 | } 154 | -------------------------------------------------------------------------------- /include/nemo_backupable.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_BACKUPABLE_H_ 2 | #define NEMO_INCLUDE_NEMO_BACKUPABLE_H_ 3 | #include "db_nemo_checkpoint.h" 4 | #include "nemo.h" 5 | #include "nemo_const.h" 6 | 7 | namespace nemo { 8 | const std::string DEFAULT_BK_PATH = "dump"; //Default backup root dir 9 | const std::string DEFAULT_RS_PATH = "db"; //Default restore root dir 10 | 11 | // Arguments which will used by BackupSave Thread 12 | // p_engine for BackupEngine handler 13 | // backup_dir 14 | // key_type kv, hash, list, set or zset 15 | struct BackupSaveArgs { 16 | void *p_engine; 17 | const std::string backup_dir; 18 | const std::string key_type; 19 | Status res; 20 | 21 | BackupSaveArgs(void *_p_engine, const std::string &_backup_dir, 22 | const std::string &_key_type) 23 | : p_engine(_p_engine), backup_dir(_backup_dir), key_type(_key_type) {} 24 | }; 25 | 26 | struct BackupContent { 27 | std::vector live_files; 28 | rocksdb::VectorLogPtr live_wal_files; 29 | uint64_t manifest_file_size = 0; 30 | uint64_t sequence_number = 0; 31 | }; 32 | 33 | class BackupEngine { 34 | public: 35 | ~BackupEngine(); 36 | static Status Open(nemo::Nemo *db, BackupEngine** backup_engine_ptr); 37 | 38 | Status SetBackupContent(); 39 | 40 | Status CreateNewBackup(const std::string &dir); 41 | 42 | void StopBackup(); 43 | 44 | Status CreateNewBackupSpecify(const std::string &dir, const std::string &type); 45 | private: 46 | BackupEngine() {} 47 | 48 | std::map engines_; 49 | std::map backup_content_; 50 | std::map backup_pthread_ts_; 51 | 52 | Status NewCheckpoint(rocksdb::DBNemo *tdb, const std::string &type); 53 | std::string GetSaveDirByType(const std::string _dir, const std::string& _type) const { 54 | std::string backup_dir = _dir.empty() ? DEFAULT_BK_PATH : _dir; 55 | return backup_dir + ((backup_dir.back() != '/') ? "/" : "") + _type; 56 | } 57 | Status WaitBackupPthread(); 58 | }; 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/nemo_const.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_CONST_H_ 2 | #define NEMO_INCLUDE_NEMO_CONST_H_ 3 | 4 | namespace nemo { 5 | 6 | const int ZSET_SCORE_INTEGER_DIGITS = 13; 7 | const int ZSET_SCORE_DECIMAL_DIGITS = 5; 8 | const int64_t ZSET_SCORE_SHIFT = 1000000000000000000LL; 9 | const int64_t ZSET_SCORE_MAX = 10000000000000LL; 10 | const int64_t ZSET_SCORE_MIN = -ZSET_SCORE_MAX; 11 | const double eps = 1e-5; 12 | 13 | const std::string ALL_DB = "all"; 14 | const std::string KV_DB = "kv"; 15 | const std::string HASH_DB = "hash"; 16 | const std::string LIST_DB = "list"; 17 | const std::string ZSET_DB = "zset"; 18 | const std::string SET_DB = "set"; 19 | 20 | enum DBType { 21 | kNONE_DB = 0, 22 | kKV_DB, 23 | kHASH_DB, 24 | kLIST_DB, 25 | kZSET_DB, 26 | kSET_DB, 27 | kALL 28 | }; 29 | 30 | enum OPERATION { 31 | kNONE_OP = 0, 32 | kDEL_KEY, 33 | kCLEAN_RANGE, 34 | kCLEAN_ALL, 35 | kCLEAN_KV, 36 | kCLEAN_HASH, 37 | kCLEAN_ZSET, 38 | kCLEAN_SET, 39 | kCLEAN_LIST, 40 | }; 41 | 42 | // Property Type 43 | const std::string PROPERTY_TYPE_ROCKSDB_MEMTABLE = "rocksdb.cur-size-all-mem-tables"; 44 | const std::string PROPERTY_TYPE_ROCKSDB_TABLE_READER = "rocksdb.estimate-table-readers-mem"; 45 | const std::string PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS = "rocksdb.background-errors"; 46 | 47 | const uint32_t KEY_MAX_LENGTH = 255; 48 | 49 | enum Position { 50 | BEFORE = 0, 51 | AFTER = 1 52 | }; 53 | 54 | enum Aggregate { 55 | SUM = 0, 56 | MIN, 57 | MAX 58 | }; 59 | 60 | enum BitOpType { 61 | kBitOpNot = 1, 62 | kBitOpAnd = 2, 63 | kBitOpOr = 3, 64 | kBitOpXor = 4, 65 | kBitOpDefault = 5 66 | }; 67 | struct KV { 68 | std::string key; 69 | std::string val; 70 | }; 71 | struct KVS { 72 | std::string key; 73 | std::string val; 74 | rocksdb::Status status; 75 | }; 76 | 77 | struct FV { 78 | std::string field; 79 | std::string val; 80 | }; 81 | struct FVS { 82 | std::string field; 83 | std::string val; 84 | rocksdb::Status status; 85 | }; 86 | 87 | struct IV { 88 | int64_t index; 89 | std::string val; 90 | }; 91 | 92 | struct SM { 93 | double score; 94 | std::string member; 95 | }; 96 | 97 | 98 | namespace DataType { 99 | static const char kKv = 'k'; 100 | static const char kHash = 'h'; // hashmap(sorted by key) 101 | static const char kHSize = 'H'; 102 | static const char kList = 'l'; 103 | static const char kLMeta = 'L'; 104 | static const char kZSet = 'z'; 105 | static const char kZSize = 'Z'; 106 | static const char kZScore = 'y'; 107 | static const char kSet = 's'; 108 | static const char kSSize = 'S'; 109 | // static const char QUEUE = 'q'; 110 | // static const char QSIZE = 'Q'; 111 | } 112 | 113 | 114 | } 115 | #endif 116 | -------------------------------------------------------------------------------- /include/nemo_iterator.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_ITERATOR_H_ 2 | #define NEMO_INCLUDE_NEMO_ITERATOR_H_ 3 | #include "rocksdb/db.h" 4 | #include "nemo_const.h" 5 | 6 | namespace nemo { 7 | 8 | enum Direction { 9 | kForward = 0, 10 | kBackward = 1 11 | }; 12 | 13 | struct IteratorOptions { 14 | std::string end; 15 | uint64_t limit; 16 | rocksdb::ReadOptions read_options; 17 | Direction direction; 18 | 19 | IteratorOptions() 20 | : limit(1LL << 60), direction(kForward) {} 21 | IteratorOptions(const std::string &_end, uint64_t _limit, 22 | rocksdb::ReadOptions roptions, Direction _dir = kForward) 23 | : end(_end), limit(_limit), read_options(roptions), direction(_dir) {} 24 | }; 25 | 26 | class Iterator { 27 | public: 28 | Iterator(rocksdb::Iterator *it, const IteratorOptions& iter_options); 29 | virtual ~Iterator() { 30 | delete it_; 31 | } 32 | 33 | rocksdb::Slice key(); 34 | rocksdb::Slice value(); 35 | virtual void Skip(int64_t offset); 36 | virtual void Next(); 37 | virtual bool Valid(); 38 | virtual rocksdb::ReadOptions read_options() { return ioptions_.read_options; } 39 | int direction() { return ioptions_.direction; } 40 | 41 | protected: 42 | bool valid_; 43 | 44 | private: 45 | bool Check(); 46 | 47 | rocksdb::Iterator *it_; 48 | IteratorOptions ioptions_; 49 | 50 | //No Copying Allowed 51 | Iterator(Iterator&); 52 | void operator=(Iterator&); 53 | }; 54 | 55 | class KIterator : public Iterator { 56 | public: 57 | KIterator(rocksdb::Iterator *it, const IteratorOptions iter_options); 58 | virtual void Next(); 59 | virtual void Skip(int64_t offset); 60 | virtual bool Valid(); 61 | std::string key() { return key_; }; 62 | std::string value() { return value_; }; 63 | 64 | private: 65 | void CheckAndLoadData(); 66 | 67 | std::string key_; 68 | std::string value_; 69 | 70 | //No Copying Allowed 71 | KIterator(KIterator&); 72 | void operator=(KIterator&); 73 | }; 74 | 75 | class HIterator : public Iterator { 76 | public: 77 | HIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key); 78 | virtual void Next(); 79 | virtual void Skip(int64_t offset); 80 | virtual bool Valid(); 81 | std::string key() { return key_; }; 82 | std::string field() { return field_; }; 83 | std::string value() { return value_; }; 84 | 85 | private: 86 | void CheckAndLoadData(); 87 | 88 | std::string key_; 89 | std::string field_; 90 | std::string value_; 91 | 92 | //No Copying Allowed 93 | HIterator(HIterator&); 94 | void operator=(HIterator&); 95 | }; 96 | 97 | class ZIterator : public Iterator { 98 | public: 99 | ZIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key); 100 | virtual bool Valid(); 101 | virtual void Skip(int64_t offset); 102 | virtual void Next(); 103 | std::string key() { return key_; }; 104 | double score() { return score_; }; 105 | std::string member() { return member_; }; 106 | 107 | private: 108 | void CheckAndLoadData(); 109 | 110 | std::string key_; 111 | double score_; 112 | std::string member_; 113 | 114 | //No Copying Allowed 115 | ZIterator(ZIterator&); 116 | void operator=(ZIterator&); 117 | 118 | }; 119 | 120 | class ZLexIterator : public Iterator { 121 | public: 122 | ZLexIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key); 123 | virtual bool Valid(); 124 | virtual void Skip(int64_t offset); 125 | virtual void Next(); 126 | std::string key() { return key_; }; 127 | std::string member() { return member_; }; 128 | 129 | private: 130 | void CheckAndLoadData(); 131 | 132 | std::string key_; 133 | std::string member_; 134 | 135 | //No Copying Allowed 136 | ZLexIterator(ZLexIterator&); 137 | void operator=(ZLexIterator&); 138 | }; 139 | 140 | class SIterator : public Iterator { 141 | public: 142 | SIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key); 143 | virtual void Skip(int64_t offset); 144 | virtual void Next(); 145 | virtual bool Valid(); 146 | std::string key() { return key_; }; 147 | std::string member() { return member_; }; 148 | 149 | private: 150 | void CheckAndLoadData(); 151 | 152 | std::string key_; 153 | std::string member_; 154 | 155 | //No Copying Allowed 156 | SIterator(SIterator&); 157 | void operator=(SIterator&); 158 | }; 159 | 160 | } 161 | #endif 162 | -------------------------------------------------------------------------------- /include/nemo_meta.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_META_H_ 2 | #define NEMO_INCLUDE_META_H_ 3 | 4 | #include 5 | #include "nemo_const.h" 6 | #include "util.h" 7 | 8 | namespace nemo { 9 | 10 | class NemoMeta; 11 | typedef std::shared_ptr MetaPtr; 12 | class NemoMeta { 13 | public: 14 | virtual ~NemoMeta() {} 15 | // Construct NemoMeta from string 16 | virtual bool DecodeFrom(const std::string& raw_meta) = 0; 17 | // Encode MemoMeta to string 18 | virtual bool EncodeTo(std::string& meta) = 0; 19 | virtual std::string ToString() = 0; 20 | 21 | static bool Create(DBType type, MetaPtr &p_meta); 22 | }; 23 | 24 | struct DefaultMeta : public NemoMeta { 25 | int64_t len; 26 | 27 | DefaultMeta() : len(0) {} 28 | explicit DefaultMeta(int64_t _len):len(_len) {} 29 | virtual bool DecodeFrom(const std::string& raw_meta) { 30 | if (raw_meta.size() != sizeof(uint64_t)) { 31 | return false; 32 | } 33 | len = *(int64_t *)raw_meta.data(); 34 | return true; 35 | } 36 | virtual bool EncodeTo(std::string& raw_meta) { 37 | raw_meta.clear(); 38 | raw_meta.append((char *)&len, sizeof(int64_t)); 39 | return true; 40 | } 41 | virtual std::string ToString() { 42 | char buf[32]; 43 | std::string res("Len : "); 44 | Int64ToStr(buf, 32, len); 45 | res.append(buf); 46 | return res; 47 | } 48 | }; 49 | 50 | typedef DefaultMeta HashMeta; 51 | typedef DefaultMeta SetMeta; 52 | typedef DefaultMeta ZSetMeta; 53 | 54 | struct ListMeta : public NemoMeta { 55 | int64_t len; 56 | int64_t left; 57 | int64_t right; 58 | int64_t cur_seq; 59 | 60 | ListMeta() : len(0), left(0), right(0), cur_seq(1) {} 61 | ListMeta(int64_t _len, int64_t _left, int64_t _right, int64_t cseq) 62 | : len(_len), left(_left), right(_right), cur_seq(cseq) {} 63 | virtual bool DecodeFrom(const std::string& raw_meta); 64 | virtual bool EncodeTo(std::string& raw_meta); 65 | virtual std::string ToString(); 66 | }; 67 | 68 | } 69 | #endif 70 | -------------------------------------------------------------------------------- /include/nemo_murmur3.h: -------------------------------------------------------------------------------- 1 | #ifndef MURMUR3_H_ 2 | #define MURMUR3_H_ 3 | 4 | //----------------------------------------------------------------------------- 5 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 6 | // domain. The author hereby disclaims copyright to this source code. 7 | 8 | // Note - The x86 and x64 versions do _not_ produce the same results, as the 9 | // algorithms are optimized for their respective platforms. You can still 10 | // compile and run any of them on any platform, but your performance with the 11 | // non-native version will be less than optimal. 12 | 13 | //----------------------------------------------------------------------------- 14 | // Platform-specific functions and macros 15 | 16 | // Microsoft Visual Studio 17 | 18 | namespace nemo { 19 | 20 | #if defined(_MSC_VER) 21 | 22 | typedef unsigned char uint8_t; 23 | typedef unsigned long uint32_t; 24 | typedef unsigned __int64 uint64_t; 25 | 26 | // Other compilers 27 | 28 | #else // defined(_MSC_VER) 29 | 30 | #include 31 | 32 | #endif // !defined(_MSC_VER) 33 | 34 | #define FORCE_INLINE __attribute__((always_inline)) 35 | 36 | inline uint32_t rotl32 ( uint32_t x, uint8_t r ) 37 | { 38 | return (x << r) | (x >> (32 - r)); 39 | } 40 | 41 | #define ROTL32(x,y) rotl32(x,y) 42 | 43 | #define BIG_CONSTANT(x) (x##LLU) 44 | 45 | /* NO-OP for little-endian platforms */ 46 | #if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) 47 | # if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 48 | # define BYTESWAP(x) (x) 49 | # endif 50 | /* if __BYTE_ORDER__ is not predefined (like FreeBSD), use arch */ 51 | #elif defined(__i386) || defined(__x86_64) \ 52 | || defined(__alpha) || defined(__vax) 53 | 54 | # define BYTESWAP(x) (x) 55 | /* use __builtin_bswap32 if available */ 56 | #elif defined(__GNUC__) || defined(__clang__) 57 | #ifdef __has_builtin 58 | #if __has_builtin(__builtin_bswap32) 59 | #define BYTESWAP(x) __builtin_bswap32(x) 60 | #endif // __has_builtin(__builtin_bswap32) 61 | #endif // __has_builtin 62 | #endif // defined(__GNUC__) || defined(__clang__) 63 | /* last resort (big-endian w/o __builtin_bswap) */ 64 | #ifndef BYTESWAP 65 | # define BYTESWAP(x) ((((x)&0xFF)<<24) \ 66 | |(((x)>>24)&0xFF) \ 67 | |(((x)&0x0000FF00)<<8) \ 68 | |(((x)&0x00FF0000)>>8) ) 69 | #endif 70 | 71 | //----------------------------------------------------------------------------- 72 | // Block read - if your platform needs to do endian-swapping or can only 73 | // handle aligned reads, do the conversion here 74 | 75 | #define getblock(p, i) BYTESWAP(p[i]) 76 | 77 | //----------------------------------------------------------------------------- 78 | // Finalization mix - force all bits of a hash block to avalanche 79 | 80 | uint32_t fmix32( uint32_t h ) 81 | { 82 | h ^= h >> 16; 83 | h *= 0x85ebca6b; 84 | h ^= h >> 13; 85 | h *= 0xc2b2ae35; 86 | h ^= h >> 16; 87 | 88 | return h; 89 | } 90 | 91 | //----------------------------------------------------------------------------- 92 | 93 | #ifdef __cplusplus 94 | extern "C" 95 | #else 96 | extern 97 | #endif 98 | void MurmurHash3_x86_32( const void * key, int len, uint32_t seed, void * out ) 99 | { 100 | const uint8_t * data = (const uint8_t*)key; 101 | const int nblocks = len / 4; 102 | int i; 103 | 104 | uint32_t h1 = seed; 105 | 106 | uint32_t c1 = 0xcc9e2d51; 107 | uint32_t c2 = 0x1b873593; 108 | 109 | //---------- 110 | // body 111 | 112 | const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); 113 | 114 | for(i = -nblocks; i; i++) 115 | { 116 | uint32_t k1 = getblock(blocks,i); 117 | 118 | k1 *= c1; 119 | k1 = ROTL32(k1,15); 120 | k1 *= c2; 121 | 122 | h1 ^= k1; 123 | h1 = ROTL32(h1,13); 124 | h1 = h1*5+0xe6546b64; 125 | } 126 | 127 | //---------- 128 | // tail 129 | { 130 | const uint8_t * tail = (const uint8_t*)(data + nblocks*4); 131 | 132 | uint32_t k1 = 0; 133 | 134 | switch(len & 3) 135 | { 136 | case 3: k1 ^= tail[2] << 16; 137 | case 2: k1 ^= tail[1] << 8; 138 | case 1: k1 ^= tail[0]; 139 | k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; 140 | }; 141 | } 142 | 143 | //---------- 144 | // finalization 145 | 146 | h1 ^= len; 147 | 148 | h1 = fmix32(h1); 149 | 150 | *(uint32_t*)out = h1; 151 | } 152 | 153 | } // end namespace nemo 154 | 155 | #endif 156 | 157 | -------------------------------------------------------------------------------- /include/nemo_options.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_OPTIONS_H_ 2 | #define NEMO_INCLUDE_NEMO_OPTIONS_H_ 3 | 4 | namespace nemo { 5 | 6 | struct Options { 7 | enum CompressionType { 8 | kNoCompression, 9 | kSnappyCompression, 10 | kZlibCompression 11 | }; 12 | bool create_if_missing; 13 | int write_buffer_size; 14 | int max_open_files; 15 | bool use_bloomfilter; 16 | int write_threads; 17 | 18 | // default target_file_size_base and multiplier is the save as rocksdb 19 | int target_file_size_base; 20 | int target_file_size_multiplier; 21 | CompressionType compression; 22 | int max_background_flushes; 23 | int max_background_compactions; 24 | int max_bytes_for_level_multiplier; 25 | 26 | Options() : create_if_missing(true), 27 | write_buffer_size(64 * 1024 * 1024), 28 | max_open_files(5000), 29 | use_bloomfilter(true), 30 | write_threads(71), 31 | target_file_size_base(64 * 1024 * 1024), 32 | target_file_size_multiplier(1), 33 | compression(kSnappyCompression), 34 | max_background_flushes(1), 35 | max_background_compactions(1), 36 | max_bytes_for_level_multiplier(10) {} 37 | }; 38 | 39 | }; // end namespace nemo 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // See port_example.h for documentation for the following types/functions. 11 | 12 | #ifndef NEMO_PORT_H 13 | #define NEMO_PORT_H 14 | 15 | #undef PLATFORM_IS_LITTLE_ENDIAN 16 | #if defined(OS_MACOSX) 17 | #include 18 | #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) 19 | #define PLATFORM_IS_LITTLE_ENDIAN \ 20 | (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) 21 | #endif 22 | #elif defined(OS_SOLARIS) 23 | #include 24 | #ifdef _LITTLE_ENDIAN 25 | #define PLATFORM_IS_LITTLE_ENDIAN true 26 | #else 27 | #define PLATFORM_IS_LITTLE_ENDIAN false 28 | #endif 29 | #elif defined(OS_FREEBSD) 30 | #include 31 | #include 32 | #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) 33 | #elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ 34 | defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) 35 | #include 36 | #include 37 | #else 38 | #include 39 | #endif 40 | #include 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | 48 | #ifndef PLATFORM_IS_LITTLE_ENDIAN 49 | #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) 50 | #endif 51 | 52 | #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ 53 | defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ 54 | defined(OS_ANDROID) 55 | // Use fread/fwrite/fflush on platforms without _unlocked variants 56 | #define fread_unlocked fread 57 | #define fwrite_unlocked fwrite 58 | #define fflush_unlocked fflush 59 | #endif 60 | 61 | #if defined(OS_MACOSX) || defined(OS_FREEBSD) ||\ 62 | defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) 63 | // Use fsync() on platforms without fdatasync() 64 | #define fdatasync fsync 65 | #endif 66 | 67 | #if defined(OS_ANDROID) && __ANDROID_API__ < 9 68 | // fdatasync() was only introduced in API level 9 on Android. Use fsync() 69 | // when targetting older platforms. 70 | #define fdatasync fsync 71 | #endif 72 | 73 | namespace nemo { 74 | namespace port { 75 | 76 | static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; 77 | #undef PLATFORM_IS_LITTLE_ENDIAN 78 | 79 | class CondVar; 80 | 81 | class Mutex { 82 | public: 83 | Mutex(); 84 | ~Mutex(); 85 | 86 | void Lock(); 87 | void Unlock(); 88 | // this will assert if the mutex is not locked 89 | // it does NOT verify that mutex is held by a calling thread 90 | void AssertHeld() {} 91 | 92 | private: 93 | friend class CondVar; 94 | pthread_mutex_t mu_; 95 | 96 | // No copying 97 | Mutex(const Mutex&); 98 | void operator=(const Mutex&); 99 | }; 100 | 101 | class CondVar { 102 | public: 103 | explicit CondVar(Mutex* mu); 104 | ~CondVar(); 105 | void Wait(); 106 | void Signal(); 107 | void SignalAll(); 108 | 109 | private: 110 | pthread_cond_t cv_; 111 | Mutex* mu_; 112 | }; 113 | 114 | class RWMutex { 115 | public: 116 | RWMutex(); 117 | ~RWMutex(); 118 | 119 | void ReadLock(); 120 | void WriteLock(); 121 | void ReadUnlock(); 122 | void WriteUnlock(); 123 | void AssertHeld() { } 124 | 125 | private: 126 | pthread_rwlock_t mu_; // the underlying platform mutex 127 | 128 | // No copying allowed 129 | RWMutex(const RWMutex&); 130 | void operator=(const RWMutex&); 131 | }; 132 | 133 | class RefMutex { 134 | public: 135 | RefMutex(); 136 | ~RefMutex(); 137 | 138 | // Lock and Unlock will increase and decrease refs_, 139 | // should check refs before Unlock 140 | void Lock(); 141 | void Unlock(); 142 | 143 | void Ref(); 144 | void Unref(); 145 | bool IsLastRef() { 146 | return refs_ == 1; 147 | } 148 | 149 | private: 150 | pthread_mutex_t mu_; 151 | int refs_; 152 | 153 | // No copying 154 | RefMutex(const RefMutex&); 155 | void operator=(const RefMutex&); 156 | }; 157 | 158 | class RecordMutex { 159 | public: 160 | RecordMutex() : charge_(0) {} 161 | ~RecordMutex(); 162 | 163 | void Lock(const std::string &key); 164 | void Unlock(const std::string &key); 165 | int64_t GetUsage(); 166 | 167 | private: 168 | 169 | Mutex mutex_; 170 | 171 | std::unordered_map records_; 172 | int64_t charge_; 173 | 174 | // No copying 175 | RecordMutex(const RecordMutex&); 176 | void operator=(const RecordMutex&); 177 | }; 178 | 179 | //const int kMaxRecordMutex = 800000; 180 | //const int kMaxRecordMutex = 2; 181 | 182 | // Need to check wether the mutex is in use!!!!! 183 | //class RecordMutex { 184 | // public: 185 | // RecordMutex() : capacity_(kMaxRecordMutex) {} 186 | // ~RecordMutex(); 187 | // 188 | // void Lock(const std::string &key); 189 | // void Unlock(const std::string &key); 190 | // 191 | // //typedef std::list 192 | // typedef std::pair::iterator> data_type; 193 | // 194 | // private: 195 | // void evict(); 196 | // 197 | // int capacity_; 198 | // Mutex mutex_; 199 | // //RWMutex rw_mutex_; 200 | // std::unordered_map records_; 201 | // std::list lru_; 202 | //}; 203 | 204 | 205 | } // namespace port 206 | } // namespace nemo 207 | 208 | #endif 209 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_UTIL_H 2 | #define NEMO_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #ifndef htobe64 24 | # if __BYTE_ORDER == __LITTLE_ENDIAN 25 | # define htobe64(x) bswap_64 (x) 26 | # else 27 | # define htobe64(x) (x) 28 | # endif 29 | #endif 30 | 31 | #ifndef be64toh 32 | # if __BYTE_ORDER == __LITTLE_ENDIAN 33 | # define be64toh(x) bswap_64 (x) 34 | # else 35 | # define be64toh(x) (x) 36 | # endif 37 | #endif 38 | 39 | namespace nemo { 40 | 41 | int StrToUint64(const char *s, size_t slen, uint64_t *value); 42 | int StrToInt64(const char *s, size_t slen, int64_t *value); 43 | int StrToInt32(const char *s, size_t slen, int32_t *val); 44 | int StrToUint32(const char *s, size_t slen, uint32_t *val); 45 | int StrToDouble(const char *s, size_t slen, double *dval); 46 | int Int64ToStr(char* dst, size_t dstlen, int64_t svalue); 47 | 48 | int do_mkdir(const char *path, mode_t mode); 49 | int mkpath(const char *path, mode_t mode); 50 | 51 | int stringmatchlen(const char *pattern, int patternLen, const char *string, int stringLen, int nocase); 52 | 53 | int is_dir(const char* filename); 54 | int delete_dir(const char* dirname); 55 | } 56 | #endif 57 | -------------------------------------------------------------------------------- /include/version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 7 | // This source code is licensed under the BSD-style license found in the 8 | // LICENSE file in the root directory of this source tree. An additional grant 9 | // of patent rights can be found in the PATENTS file in the same directory. 10 | #pragma once 11 | 12 | #define PINK_MAJOR 1 13 | #define PINK_MINOR 0 14 | #define PINK_PATCH 0 15 | -------------------------------------------------------------------------------- /include/xdebug.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdebug.h 3 | * @brief debug macros 4 | * @author chenzongzhi 5 | * @version 1.0.0 6 | * @date 2014-04-25 7 | */ 8 | 9 | #ifndef __XDEBUG_H_ 10 | #define __XDEBUG_H_ 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #ifdef __XDEBUG__ 18 | #define qf_debug(fmt, arg...) \ 19 | { \ 20 | fprintf(stderr, "[----------debug--------][%s:%d]" fmt "\n", __FILE__, __LINE__, ##arg); \ 21 | } 22 | #define pint(x) qf_debug("%s = %d", #x, x) 23 | #define psint(x) qf_debug("%s = %zu", #x, x) 24 | #define psize(x) qf_debug("%s = %zu", #x, x) 25 | #define pstr(x) qf_debug("%s = %s", #x, x) 26 | // 如果A 不对, 那么就输出M 27 | #define qf_check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; exit(-1);} 28 | 29 | // 用来检测程序是否执行到这里 30 | #define sentinel(M, ...) { qf_debug(M, ##__VA_ARGS__); errno=0;} 31 | 32 | #define qf_bin_debug(buf, size) \ 33 | { \ 34 | fwrite(buf, 1, size, stderr); \ 35 | } 36 | 37 | #define _debug_time_def timeval s1, e; 38 | #define _debug_getstart gettimeofday(&s1, NULL) 39 | #define _debug_getend gettimeofday(&e, NULL) 40 | #define _debug_time ((int)(((e.tv_sec - s1.tv_sec) * 1000 + (e.tv_usec - s1.tv_usec) / 1000))) 41 | 42 | #define clean_errno() (errno == 0 ? "None" : strerror(errno)) 43 | #define log_err(M, ...) \ 44 | { \ 45 | fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__); \ 46 | exit(-1); \ 47 | } 48 | #define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 49 | #define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 50 | 51 | #else 52 | 53 | #define qf_debug(fmt, arg...) {} 54 | #define pint(x) {} 55 | #define pstr(x) {} 56 | #define qf_bin_debug(buf, size) {} 57 | 58 | #define _debug_time_def {} 59 | #define _debug_getstart {} 60 | #define _debug_getend {} 61 | #define _debug_time 0 62 | 63 | #define sentinel(M, ...) {} 64 | #define qf_check(A, M, ...) {} 65 | #define log_err(M, ...) {} 66 | #define log_warn(M, ...) {} 67 | #define log_info(M, ...) {} 68 | 69 | #endif 70 | 71 | #define qf_error(fmt, arg...) \ 72 | { \ 73 | fprintf(stderr, "[%ld][%ld][%s:%d]" fmt "\n", (long)getpid(), (long)pthread_self(), __FILE__, __LINE__, ##arg); \ 74 | fflush(stderr);\ 75 | exit(-1);\ 76 | } 77 | 78 | 79 | #endif //__XDEBUG_H_ 80 | 81 | /* vim: set ts=4 sw=4 sts=4 tw=100 */ 82 | -------------------------------------------------------------------------------- /src/build_version.cc.in: -------------------------------------------------------------------------------- 1 | #include "build_version.h" 2 | const char* nemo_build_git_sha = "pink_build_git_sha:@@GIT_SHA@@"; 3 | const char* nemo_build_git_date = "pink_build_git_date:@@GIT_DATE_TIME@@"; 4 | const char* nemo_build_compile_date = __DATE__; 5 | -------------------------------------------------------------------------------- /src/build_version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 7 | // This source code is licensed under the BSD-style license found in the 8 | // LICENSE file in the root directory of this source tree. An additional grant 9 | // of patent rights can be found in the PATENTS file in the same directory. 10 | // 11 | #pragma once 12 | 13 | // this variable tells us about the git revision 14 | extern const char* nemo_build_git_sha; 15 | 16 | // Date on which the code was compiled: 17 | extern const char* nemo_build_compile_date; 18 | -------------------------------------------------------------------------------- /src/decoder.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_DECODER_H 2 | #define NEMO_INCLUDE_DECODER_H 3 | 4 | #include 5 | 6 | #include "util.h" 7 | 8 | class Decoder { 9 | public: 10 | Decoder(const char *p, int32_t size) 11 | : ptr_(p), 12 | size_(size) {} 13 | 14 | int32_t Skip(int32_t n) { 15 | if (size_ < n) { 16 | return -1; 17 | } 18 | ptr_ += n; 19 | size_ -=n; 20 | return n; 21 | } 22 | 23 | int32_t ReadInt64(int64_t *res) { 24 | int32_t n = sizeof(int64_t); 25 | if (size_ < n) { 26 | return -1; 27 | } 28 | if (res) { 29 | *res = *(int64_t *)ptr_; 30 | } 31 | *res = be64toh(*res); 32 | ptr_ += sizeof(int64_t); 33 | size_ -= sizeof(int64_t); 34 | return sizeof(int64_t); 35 | } 36 | 37 | int32_t ReadUInt64(uint64_t *res) { 38 | int32_t n = sizeof(uint64_t); 39 | if (size_ < n) { 40 | return -1; 41 | } 42 | if (res) { 43 | *res = *(uint64_t *)ptr_; 44 | } 45 | *res = be64toh(*res); 46 | ptr_ += sizeof(uint64_t); 47 | size_ -= sizeof(uint64_t); 48 | return sizeof(uint64_t); 49 | } 50 | 51 | int32_t ReadData(std::string *res) { 52 | int32_t n = size_; 53 | if (res) { 54 | res->assign(ptr_, size_); 55 | } 56 | ptr_ += size_; 57 | size_ = 0; 58 | return n; 59 | } 60 | 61 | int32_t ReadLenData(std::string *res = NULL) { 62 | if (size_ < 1) { 63 | return -1; 64 | } 65 | int32_t len = *((uint8_t *)ptr_); 66 | ptr_ += 1; 67 | size_ -= 1; 68 | if (size_ < len) { 69 | return -1; 70 | } 71 | if (res) { 72 | res->assign(ptr_, len); 73 | } 74 | ptr_ += len; 75 | size_ -= len; 76 | return len + 1; 77 | } 78 | private: 79 | const char *ptr_; 80 | int32_t size_; 81 | //No Copying allowed 82 | Decoder(const Decoder&); 83 | void operator=(const Decoder&); 84 | }; 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /src/nemo.cc: -------------------------------------------------------------------------------- 1 | #include "nemo.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "nemo_list.h" 9 | #include "nemo_zset.h" 10 | #include "nemo_set.h" 11 | #include "nemo_hash.h" 12 | #include "port.h" 13 | #include "util.h" 14 | #include "xdebug.h" 15 | 16 | namespace nemo { 17 | 18 | Nemo::Nemo(const std::string &db_path, const Options &options) 19 | : db_path_(db_path), 20 | save_flag_(false), 21 | bgtask_flag_(true), 22 | bg_cv_(&mutex_bgtask_), 23 | scan_keynum_exit_(false), 24 | dump_to_terminate_(false) { 25 | 26 | pthread_mutex_init(&(mutex_cursors_), NULL); 27 | pthread_mutex_init(&(mutex_dump_), NULL); 28 | pthread_mutex_init(&(mutex_spop_counts_), NULL); 29 | if (db_path_[db_path_.length() - 1] != '/') { 30 | db_path_.append("/"); 31 | } 32 | 33 | mkpath(db_path_.c_str(), 0755); 34 | mkpath((db_path_ + "kv").c_str(), 0755); 35 | mkpath((db_path_ + "hash").c_str(), 0755); 36 | mkpath((db_path_ + "list").c_str(), 0755); 37 | mkpath((db_path_ + "zset").c_str(), 0755); 38 | mkpath((db_path_ + "set").c_str(), 0755); 39 | 40 | cursors_store_.cur_size_ = 0; 41 | cursors_store_.max_size_ = 5000; 42 | cursors_store_.list_.clear(); 43 | cursors_store_.map_.clear(); 44 | 45 | spop_counts_store_.cur_size_ = 0; 46 | spop_counts_store_.max_size_ = 100; 47 | spop_counts_store_.list_.clear(); 48 | spop_counts_store_.map_.clear(); 49 | 50 | // Open Options 51 | open_options_.create_if_missing = true; 52 | open_options_.write_buffer_size = options.write_buffer_size; 53 | open_options_.max_manifest_file_size = 64*1024*1024; 54 | open_options_.max_log_file_size = 512*1024*1024; 55 | open_options_.keep_log_file_num = 10; 56 | if (options.compression == Options::CompressionType::kNoCompression) { 57 | open_options_.compression = rocksdb::CompressionType::kNoCompression; 58 | } else if (options.compression == Options::CompressionType::kSnappyCompression) { 59 | open_options_.compression = rocksdb::CompressionType::kSnappyCompression; 60 | } else if (options.compression == Options::CompressionType::kZlibCompression) { 61 | open_options_.compression = rocksdb::CompressionType::kZlibCompression; 62 | } 63 | if (options.max_open_files > 0) { 64 | open_options_.max_open_files = options.max_open_files; 65 | } 66 | 67 | if (options.target_file_size_base > 0) { 68 | open_options_.target_file_size_base = (uint64_t)options.target_file_size_base; 69 | } 70 | if (options.target_file_size_multiplier > 0) { 71 | open_options_.target_file_size_multiplier = options.target_file_size_multiplier; 72 | } 73 | 74 | if (options.max_background_flushes > 0 && options.max_background_flushes <= 4) { 75 | open_options_.max_background_flushes = options.max_background_flushes; 76 | } 77 | if (options.max_background_compactions > 0 && options.max_background_compactions <= 8) { 78 | open_options_.max_background_compactions = options.max_background_compactions; 79 | } 80 | if (options.max_bytes_for_level_multiplier < 10) { 81 | open_options_.max_bytes_for_level_multiplier = 5; 82 | } 83 | if (options.max_bytes_for_level_multiplier >= 10) { 84 | open_options_.max_bytes_for_level_multiplier = 10; 85 | } 86 | 87 | //open_options_.max_bytes_for_level_base = (128 << 20); 88 | 89 | rocksdb::DBNemo *db_ttl; 90 | 91 | rocksdb::Status s = rocksdb::DBNemo::Open(open_options_, db_path_ + "kv", &db_ttl, rocksdb::kMetaPrefixKv); 92 | if (!s.ok()) { 93 | fprintf (stderr, "[FATAL] open kv db failed, %s\n", s.ToString().c_str()); 94 | exit(-1); 95 | } 96 | kv_db_ = std::unique_ptr(db_ttl); 97 | 98 | s = rocksdb::DBNemo::Open(open_options_, db_path_ + "hash", &db_ttl, rocksdb::kMetaPrefixHash); 99 | if (!s.ok()) { 100 | fprintf (stderr, "[FATAL] open hash db failed, %s\n", s.ToString().c_str()); 101 | exit(-1); 102 | } 103 | hash_db_ = std::unique_ptr(db_ttl); 104 | 105 | s = rocksdb::DBNemo::Open(open_options_, db_path_ + "list", &db_ttl, rocksdb::kMetaPrefixList); 106 | if (!s.ok()) { 107 | fprintf (stderr, "[FATAL] open list db failed, %s\n", s.ToString().c_str()); 108 | exit(-1); 109 | } 110 | list_db_ = std::unique_ptr(db_ttl); 111 | 112 | s = rocksdb::DBNemo::Open(open_options_, db_path_ + "zset", &db_ttl, rocksdb::kMetaPrefixZset); 113 | if (!s.ok()) { 114 | fprintf (stderr, "[FATAL] open zset db failed, %s\n", s.ToString().c_str()); 115 | exit(-1); 116 | } 117 | zset_db_ = std::unique_ptr(db_ttl); 118 | 119 | s = rocksdb::DBNemo::Open(open_options_, db_path_ + "set", &db_ttl, rocksdb::kMetaPrefixSet); 120 | if (!s.ok()) { 121 | fprintf (stderr, "[FATAL] open set db failed, %s\n", s.ToString().c_str()); 122 | exit(-1); 123 | } 124 | set_db_ = std::unique_ptr(db_ttl); 125 | 126 | // Add separator of Meta and data 127 | hash_db_->Put(rocksdb::WriteOptions(), "h", ""); 128 | list_db_->Put(rocksdb::WriteOptions(), "l", ""); 129 | zset_db_->Put(rocksdb::WriteOptions(), "y", ""); 130 | zset_db_->Put(rocksdb::WriteOptions(), "z", ""); 131 | set_db_->Put(rocksdb::WriteOptions(), "s", ""); 132 | 133 | // Start BGThread 134 | s = StartBGThread(); 135 | if (!s.ok()) { 136 | log_err("start bg thread error: %s", s.ToString().c_str()); 137 | fprintf (stderr, "[FATAL] start bg thread failed, %s\n", s.ToString().c_str()); 138 | exit(-1); 139 | } 140 | }; 141 | 142 | } // namespace nemo 143 | -------------------------------------------------------------------------------- /src/nemo_backupable.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "nemo_backupable.h" 6 | #include "xdebug.h" 7 | #include "util.h" 8 | 9 | namespace nemo { 10 | 11 | BackupEngine::~BackupEngine() { 12 | // Wait all children threads 13 | StopBackup(); 14 | WaitBackupPthread(); 15 | // Delete engines 16 | for (auto& engine : engines_) { 17 | delete engine.second; 18 | } 19 | engines_.clear(); 20 | } 21 | 22 | Status BackupEngine::NewCheckpoint(rocksdb::DBNemo *tdb, const std::string &type) { 23 | rocksdb::DBNemoCheckpoint* checkpoint; 24 | Status s = rocksdb::DBNemoCheckpoint::Create(tdb, &checkpoint); 25 | if (!s.ok()) { 26 | log_warn("create checkpoint failed, error %s", s.ToString().c_str()); 27 | return s; 28 | } 29 | engines_.insert(std::make_pair(type, checkpoint)); 30 | return s; 31 | } 32 | 33 | Status BackupEngine::Open(nemo::Nemo *db, 34 | BackupEngine** backup_engine_ptr) { 35 | *backup_engine_ptr = new BackupEngine(); 36 | if (!*backup_engine_ptr){ 37 | return Status::Corruption("New BackupEngine failed!"); 38 | } 39 | 40 | // Create BackupEngine for each db type 41 | rocksdb::Status s; 42 | rocksdb::DBNemo *tdb; 43 | std::string types[] = {KV_DB, HASH_DB, LIST_DB, SET_DB, ZSET_DB}; 44 | for (auto& type : types) { 45 | if ((tdb = db->GetDBByType(type)) == NULL) { 46 | s = Status::Corruption("Error db type"); 47 | } 48 | 49 | if (s.ok()) { 50 | s = (*backup_engine_ptr)->NewCheckpoint(tdb, type); 51 | } 52 | 53 | if (!s.ok()) { 54 | delete *backup_engine_ptr; 55 | break; 56 | } 57 | } 58 | return s; 59 | } 60 | 61 | Status BackupEngine::SetBackupContent() { 62 | Status s; 63 | for (auto& engine : engines_) { 64 | //Get backup content 65 | BackupContent bcontent; 66 | s = engine.second->GetCheckpointFiles(bcontent.live_files, 67 | bcontent.live_wal_files, 68 | bcontent.manifest_file_size, bcontent.sequence_number); 69 | if (!s.ok()) { 70 | log_warn("get backup files faild for type: %s", engine.first.c_str()); 71 | return s; 72 | } 73 | backup_content_[engine.first] = std::move(bcontent); 74 | } 75 | return s; 76 | } 77 | 78 | Status BackupEngine::CreateNewBackupSpecify(const std::string &backup_dir, const std::string &type) { 79 | std::map::iterator it_engine = engines_.find(type); 80 | std::map::iterator it_content = backup_content_.find(type); 81 | std::string dir = GetSaveDirByType(backup_dir, type); 82 | delete_dir(dir.c_str()); 83 | 84 | if (it_content != backup_content_.end() && 85 | it_engine != engines_.end()) { 86 | Status s = it_engine->second->CreateCheckpointWithFiles( 87 | dir, 88 | it_content->second.live_files, 89 | it_content->second.live_wal_files, 90 | it_content->second.manifest_file_size, 91 | it_content->second.sequence_number); 92 | if (!s.ok()) { 93 | log_warn("backup engine create new failed, type: %s, error %s", 94 | type.c_str(), s.ToString().c_str()); 95 | return s; 96 | } 97 | 98 | } else { 99 | log_warn("invalid db type: %s", type.c_str()); 100 | return Status::Corruption("invalid db type"); 101 | } 102 | return Status::OK(); 103 | } 104 | 105 | void* ThreadFuncSaveSpecify(void *arg) { 106 | BackupSaveArgs* arg_ptr = static_cast(arg); 107 | BackupEngine* p = static_cast(arg_ptr->p_engine); 108 | 109 | arg_ptr->res = p->CreateNewBackupSpecify(arg_ptr->backup_dir, arg_ptr->key_type); 110 | 111 | pthread_exit(&(arg_ptr->res)); 112 | } 113 | 114 | Status BackupEngine::WaitBackupPthread() { 115 | int ret; 116 | Status s = Status::OK(); 117 | for (auto& pthread : backup_pthread_ts_) { 118 | void *res; 119 | if ((ret = pthread_join(pthread.second, &res)) != 0) { 120 | log_warn("pthread_join failed with backup thread for key_type: %s, error %d", 121 | pthread.first.c_str(), ret); 122 | } 123 | Status cur_s = *(static_cast(res)); 124 | if (!cur_s.ok()) { 125 | log_warn("pthread executed failed with key_type: %s, error %s", 126 | pthread.first.c_str(), cur_s.ToString().c_str()); 127 | StopBackup(); //stop others when someone failed 128 | s = cur_s; 129 | } 130 | } 131 | backup_pthread_ts_.clear(); 132 | return s; 133 | } 134 | 135 | 136 | Status BackupEngine::CreateNewBackup(const std::string &dir) { 137 | Status s = Status::OK(); 138 | std::vector args; 139 | for (auto& engine : engines_) { 140 | pthread_t tid; 141 | BackupSaveArgs *arg = new BackupSaveArgs((void*)this, dir, engine.first); 142 | args.push_back(arg); 143 | if (pthread_create(&tid, NULL, &ThreadFuncSaveSpecify, arg) != 0) { 144 | s = Status::Corruption("pthead_create failed."); 145 | break; 146 | } 147 | if (!(backup_pthread_ts_.insert(std::make_pair(engine.first, tid)).second)) { 148 | log_warn("thread open dupilicated, type: %s", engine.first.c_str()); 149 | backup_pthread_ts_[engine.first] = tid; 150 | } 151 | } 152 | 153 | // Wait threads stop 154 | if (!s.ok()) { 155 | StopBackup(); 156 | } 157 | s = WaitBackupPthread(); 158 | 159 | for (auto& a : args) { 160 | delete a; 161 | } 162 | return s; 163 | } 164 | 165 | void BackupEngine::StopBackup() { 166 | // DEPRECATED 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /src/nemo_hash.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_HASH_H 2 | #define NEMO_INCLUDE_NEMO_HASH_H 3 | 4 | #include "nemo.h" 5 | #include "nemo_const.h" 6 | #include "decoder.h" 7 | 8 | namespace nemo { 9 | 10 | inline std::string EncodeHsizeKey(const rocksdb::Slice &name) { 11 | std::string buf; 12 | buf.append(1, DataType::kHSize); 13 | buf.append(name.data(), name.size()); 14 | return buf; 15 | } 16 | 17 | inline int DecodeHsizeKey(const rocksdb::Slice &slice, std::string *name) { 18 | Decoder decoder(slice.data(), slice.size()); 19 | if (decoder.Skip(1) == -1) { 20 | return -1; 21 | } 22 | if (decoder.ReadData(name) == -1) { 23 | return -1; 24 | } 25 | return 0; 26 | } 27 | 28 | inline std::string EncodeHashKey(const rocksdb::Slice &name, const rocksdb::Slice &key) { 29 | std::string buf; 30 | buf.append(1, DataType::kHash); 31 | buf.append(1, (uint8_t)name.size()); 32 | buf.append(name.data(), name.size()); 33 | buf.append(1, '='); 34 | buf.append(key.data(), key.size()); 35 | return buf; 36 | } 37 | 38 | inline int DecodeHashKey(const rocksdb::Slice &slice, std::string *name, std::string *key) { 39 | Decoder decoder(slice.data(), slice.size()); 40 | if (decoder.Skip(1) == -1) { 41 | return -1; 42 | } 43 | if (decoder.ReadLenData(name) == -1) { 44 | return -1; 45 | } 46 | if (decoder.Skip(1) == -1) { 47 | return -1; 48 | } 49 | if (decoder.ReadData(key) == -1) { 50 | return -1; 51 | } 52 | return 0; 53 | } 54 | 55 | } 56 | 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /src/nemo_hyperloglog.cc: -------------------------------------------------------------------------------- 1 | #include "nemo_hyperloglog.h" 2 | #include "nemo_list.h" 3 | #include "nemo_mutex.h" 4 | #include "nemo_murmur3.h" 5 | 6 | using namespace nemo; 7 | 8 | #define HLL_HASH_SEED 313 9 | 10 | HyperLogLog::HyperLogLog(uint8_t precision, std::string origin_register) { 11 | b_ = precision; 12 | m_ = 1 << precision; 13 | alpha_ = Alpha(); 14 | register_ = new char[m_]; 15 | for (uint32_t i = 0; i < m_; ++i) 16 | register_[i] = 0; 17 | if (origin_register != "") { 18 | for (uint32_t i = 0; i < m_; ++i) { 19 | register_[i] = origin_register[i]; 20 | } 21 | } 22 | } 23 | 24 | HyperLogLog::~HyperLogLog() { 25 | delete [] register_; 26 | } 27 | 28 | std::string HyperLogLog::Add(const char * str, uint32_t len) { 29 | uint32_t hash; 30 | MurmurHash3_x86_32(str, len, HLL_HASH_SEED, (void*) &hash); 31 | int index = hash & ((1 << b_) - 1); 32 | uint8_t rank = Nclz((hash << b_), 32 - b_); 33 | if (rank > register_[index]) 34 | register_[index] = rank; 35 | std::string result_str(m_, 0); 36 | for (uint32_t i = 0; i < m_; ++i) { 37 | result_str[i] = register_[i]; 38 | } 39 | return result_str; 40 | } 41 | 42 | double HyperLogLog::Estimate() const { 43 | double estimate = FirstEstimate(); 44 | 45 | if (estimate <= 2.5* m_) { 46 | uint32_t zeros = CountZero(); 47 | if (zeros != 0) { 48 | estimate = m_ * log((double)m_ / zeros); 49 | } 50 | } else if (estimate > pow(2, 32) / 30.0) { 51 | estimate = log1p(estimate * -1 / pow(2, 32)) * pow(2, 32) * -1; 52 | } 53 | return estimate; 54 | } 55 | 56 | double HyperLogLog::FirstEstimate() const { 57 | double estimate, sum = 0.0; 58 | for (uint32_t i = 0; i < m_; i++) 59 | sum += 1.0 / (1 << register_[i]); 60 | 61 | estimate = alpha_ * m_ * m_ / sum; 62 | return estimate; 63 | } 64 | 65 | double HyperLogLog::Alpha() const { 66 | switch (m_) { 67 | case 16: 68 | return 0.673; 69 | case 32: 70 | return 0.697; 71 | case 64: 72 | return 0.709; 73 | default: 74 | return 0.7213/(1 + 1.079 / m_); 75 | } 76 | } 77 | 78 | int HyperLogLog::CountZero() const { 79 | int count = 0; 80 | for(uint32_t i = 0;i < m_; i++) { 81 | if (register_[i] == 0) { 82 | count++; 83 | } 84 | } 85 | return count; 86 | } 87 | 88 | std::string HyperLogLog::Merge(const HyperLogLog & hll) { 89 | if (m_ != hll.m_) { 90 | // TODO: ERROR "number of registers doesn't match" 91 | } 92 | for (uint32_t r = 0; r < m_; r++) { 93 | if (register_[r] < hll.register_[r]) { 94 | register_[r] |= hll.register_[r]; 95 | } 96 | } 97 | 98 | std::string result_str(m_, 0); 99 | for (uint32_t i = 0; i < m_; ++i) { 100 | result_str[i] = register_[i]; 101 | } 102 | return result_str; 103 | } 104 | 105 | //::__builtin_clz(x): 返回左起第一个‘1’之前0的个数 106 | uint8_t HyperLogLog::Nclz(uint32_t x, int b) { 107 | return (uint8_t)std::min(b, ::__builtin_clz(x)) + 1; 108 | } 109 | 110 | Status Nemo::PfAdd(const std::string &key, const std::vector &values, bool & update) { 111 | if (values.size() >= KEY_MAX_LENGTH) { 112 | return Status::InvalidArgument("Invalid value length"); 113 | } 114 | 115 | Status s; 116 | std::string val, str_register = "", result = ""; 117 | s = Get(key, &val); 118 | if (s.ok()) { 119 | str_register = val; 120 | } else if (s.IsNotFound()) { 121 | str_register = ""; 122 | } 123 | HyperLogLog log(17, str_register); 124 | int previous = int(log.Estimate()); 125 | for (int i = 0; i < (int)values.size(); ++i) { 126 | result = log.Add(values[i].data(), values[i].size()); 127 | } 128 | HyperLogLog update_log(17, result); 129 | int now= int(update_log.Estimate()); 130 | if (previous != now || (s.IsNotFound() && values.size()==0)) { 131 | update = true; 132 | } 133 | s = kv_db_->Put(rocksdb::WriteOptions(), key, result); 134 | return s; 135 | } 136 | 137 | Status Nemo::PfCount(const std::vector &keys, int & result) { 138 | if (keys.size() >= KEY_MAX_LENGTH || keys.size() <= 0) { 139 | return Status::InvalidArgument("Invalid key length"); 140 | } 141 | 142 | Status s, ok; 143 | std::string value, str_register; 144 | s = Get(keys[0], &value); 145 | if (s.ok()) { 146 | str_register = std::string(value.data(), value.size()); 147 | } else if (s.IsNotFound()) { 148 | str_register = ""; 149 | } 150 | 151 | HyperLogLog first_log(17, str_register); 152 | for (int i = 1; i < (int)keys.size(); ++i) { 153 | std::string value, str_register; 154 | s = Get(keys[i], &value); 155 | if (s.ok()) { 156 | str_register = value; 157 | } else if (s.IsNotFound()) { 158 | continue; 159 | } 160 | HyperLogLog log(17, str_register); 161 | first_log.Merge(log); 162 | } 163 | result = int(first_log.Estimate()); 164 | return ok; 165 | } 166 | 167 | Status Nemo::PfMerge(const std::vector &keys) { 168 | if (keys.size() >= KEY_MAX_LENGTH || keys.size() <= 0) { 169 | return Status::InvalidArgument("Invalid key length"); 170 | } 171 | 172 | Status s; 173 | std::string value, str_register, result; 174 | s = Get(keys[0], &value); 175 | if (s.ok()) { 176 | str_register = std::string(value.data(), value.size()); 177 | } else if (s.IsNotFound()) { 178 | str_register = ""; 179 | } 180 | 181 | result = str_register; 182 | HyperLogLog first_log(17, str_register); 183 | for (int i = 1; i < (int)keys.size(); ++i) { 184 | std::string value, str_register; 185 | s = Get(keys[i], &value); 186 | if (s.ok()) { 187 | str_register = std::string(value.data(), value.size()); 188 | } else if (s.IsNotFound()) { 189 | continue; 190 | } 191 | HyperLogLog log(17, str_register); 192 | result = first_log.Merge(log); 193 | } 194 | s = kv_db_->Put(rocksdb::WriteOptions(), keys[0], result); 195 | return s; 196 | } 197 | 198 | -------------------------------------------------------------------------------- /src/nemo_hyperloglog.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_HYPERLOGLOG_H 2 | #define NEMO_INCLUDE_NEMO_HYPERLOGLOG_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "nemo.h" 11 | #include "nemo_const.h" 12 | #include "decoder.h" 13 | 14 | namespace nemo { 15 | 16 | struct HyperLogLog { 17 | public: 18 | HyperLogLog(uint8_t precision, std::string origin_register); 19 | ~HyperLogLog(); 20 | 21 | double Estimate() const; 22 | double FirstEstimate() const; 23 | int CountZero() const; 24 | double Alpha() const; 25 | uint8_t Nclz(uint32_t x, int b); 26 | 27 | std::string Add(const char * str, uint32_t len); 28 | std::string Merge(const HyperLogLog & hll); 29 | 30 | protected: 31 | uint32_t m_; // register bit width 32 | uint8_t b_; // register size 33 | double alpha_; 34 | char * register_; // registers 35 | }; 36 | 37 | } 38 | #endif 39 | -------------------------------------------------------------------------------- /src/nemo_iterator.cc: -------------------------------------------------------------------------------- 1 | #include "nemo_iterator.h" 2 | 3 | #include 4 | #include "nemo_set.h" 5 | #include "nemo_hash.h" 6 | #include "nemo_zset.h" 7 | #include "xdebug.h" 8 | 9 | nemo::Iterator::Iterator(rocksdb::Iterator *it, const IteratorOptions& iter_options) 10 | : it_(it), 11 | ioptions_(iter_options) { 12 | Check(); 13 | } 14 | 15 | bool nemo::Iterator::Check() { 16 | valid_ = false; 17 | if (ioptions_.limit == 0 || !it_->Valid()) { 18 | // make next() safe to be called after previous return false. 19 | ioptions_.limit = 0; 20 | return false; 21 | } else { 22 | if (ioptions_.direction == kForward) { 23 | if (!ioptions_.end.empty() && it_->key().compare(ioptions_.end) > 0) { 24 | ioptions_.limit = 0; 25 | return false; 26 | } 27 | } else { 28 | if(!ioptions_.end.empty() && it_->key().compare(ioptions_.end) < 0) { 29 | ioptions_.limit = 0; 30 | return false; 31 | } 32 | } 33 | ioptions_.limit --; 34 | valid_ = true; 35 | return true; 36 | } 37 | } 38 | 39 | rocksdb::Slice nemo::Iterator::key() { 40 | return it_->key(); 41 | } 42 | 43 | rocksdb::Slice nemo::Iterator::value() { 44 | return it_->value(); 45 | } 46 | 47 | bool nemo::Iterator::Valid() { 48 | return valid_; 49 | } 50 | 51 | // non-positive offset don't skip at all 52 | void nemo::Iterator::Skip(int64_t offset) { 53 | if (offset > 0) { 54 | while (offset-- > 0) { 55 | if (ioptions_.direction == kForward){ 56 | it_->Next(); 57 | } else { 58 | it_->Prev(); 59 | } 60 | 61 | if (!Check()) { 62 | return; 63 | } 64 | } 65 | } 66 | } 67 | 68 | void nemo::Iterator::Next() { 69 | if (valid_) { 70 | if (ioptions_.direction == kForward){ 71 | it_->Next(); 72 | } else { 73 | it_->Prev(); 74 | } 75 | 76 | Check(); 77 | } 78 | } 79 | 80 | // KV 81 | nemo::KIterator::KIterator(rocksdb::Iterator *it, const IteratorOptions iter_options) 82 | : Iterator(it, iter_options) { 83 | CheckAndLoadData(); 84 | } 85 | 86 | void nemo::KIterator::CheckAndLoadData() { 87 | if (valid_) { 88 | rocksdb::Slice ks = Iterator::key(); 89 | rocksdb::Slice vs = Iterator::value(); 90 | this->key_.assign(ks.data(), ks.size()); 91 | this->value_.assign(vs.data(), vs.size()); 92 | } 93 | } 94 | 95 | bool nemo::KIterator::Valid() { 96 | return valid_; 97 | } 98 | 99 | void nemo::KIterator::Next() { 100 | Iterator::Next(); 101 | CheckAndLoadData(); 102 | } 103 | 104 | void nemo::KIterator::Skip(int64_t offset) { 105 | Iterator::Skip(offset); 106 | CheckAndLoadData(); 107 | } 108 | 109 | // HASH 110 | nemo::HIterator::HIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key) 111 | : Iterator(it, iter_options) { 112 | this->key_.assign(key.data(), key.size()); 113 | CheckAndLoadData(); 114 | } 115 | 116 | // check valid and load field_, value_ 117 | void nemo::HIterator::CheckAndLoadData() { 118 | if (valid_) { 119 | rocksdb::Slice ks = Iterator::key(); 120 | 121 | if (ks[0] == DataType::kHash) { 122 | std::string k; 123 | if (DecodeHashKey(ks, &k, &this->field_) != -1) { 124 | if (k == this->key_) { 125 | rocksdb::Slice vs = Iterator::value(); 126 | this->value_.assign(vs.data(), vs.size()); 127 | return ; 128 | } 129 | } 130 | } 131 | } 132 | valid_ = false; 133 | } 134 | 135 | bool nemo::HIterator::Valid() { 136 | return valid_; 137 | } 138 | 139 | void nemo::HIterator::Next() { 140 | Iterator::Next(); 141 | CheckAndLoadData(); 142 | } 143 | 144 | void nemo::HIterator::Skip(int64_t offset) { 145 | Iterator::Skip(offset); 146 | CheckAndLoadData(); 147 | } 148 | 149 | // ZSET 150 | nemo::ZIterator::ZIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key) 151 | : Iterator(it, iter_options) { 152 | this->key_.assign(key.data(), key.size()); 153 | CheckAndLoadData(); 154 | } 155 | 156 | // check valid and assign member_ and score_ 157 | void nemo::ZIterator::CheckAndLoadData() { 158 | if (valid_) { 159 | rocksdb::Slice ks = Iterator::key(); 160 | if (ks[0] == DataType::kZScore) { 161 | std::string k; 162 | if (DecodeZScoreKey(ks, &k, &this->member_, &this->score_) != -1) { 163 | if (k == this->key_) { 164 | return ; 165 | } 166 | } 167 | } 168 | } 169 | valid_ = false; 170 | } 171 | 172 | bool nemo::ZIterator::Valid() { 173 | return valid_; 174 | } 175 | 176 | void nemo::ZIterator::Next() { 177 | Iterator::Next(); 178 | CheckAndLoadData(); 179 | } 180 | 181 | void nemo::ZIterator::Skip(int64_t offset) { 182 | Iterator::Skip(offset); 183 | CheckAndLoadData(); 184 | } 185 | 186 | // ZLexIterator 187 | nemo::ZLexIterator::ZLexIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key) 188 | : Iterator(it, iter_options) { 189 | this->key_.assign(key.data(), key.size()); 190 | CheckAndLoadData(); 191 | } 192 | 193 | void nemo::ZLexIterator::CheckAndLoadData() { 194 | if (valid_) { 195 | rocksdb::Slice ks = Iterator::key(); 196 | 197 | if (ks[0] == DataType::kZSet) { 198 | std::string k; 199 | if (DecodeZSetKey(ks, &k, &this->member_) != -1) { 200 | if (k == this->key_) { 201 | return ; 202 | } 203 | } 204 | } 205 | } 206 | valid_ = false; 207 | } 208 | 209 | bool nemo::ZLexIterator::Valid() { 210 | return valid_; 211 | } 212 | 213 | void nemo::ZLexIterator::Next() { 214 | Iterator::Next(); 215 | CheckAndLoadData(); 216 | } 217 | 218 | void nemo::ZLexIterator::Skip(int64_t offset) { 219 | Iterator::Skip(offset); 220 | CheckAndLoadData(); 221 | } 222 | 223 | // SET 224 | nemo::SIterator::SIterator(rocksdb::Iterator *it, const IteratorOptions iter_options, const rocksdb::Slice &key) 225 | : Iterator(it, iter_options) { 226 | this->key_.assign(key.data(), key.size()); 227 | CheckAndLoadData(); 228 | } 229 | 230 | // check valid and assign member_ 231 | void nemo::SIterator::CheckAndLoadData() { 232 | if (valid_) { 233 | rocksdb::Slice ks = Iterator::key(); 234 | 235 | if (ks[0] == DataType::kSet) { 236 | std::string k; 237 | if (DecodeSetKey(ks, &k, &this->member_) != -1) { 238 | if (k == this->key_) { 239 | return ; 240 | } 241 | } 242 | } 243 | } 244 | valid_ = false; 245 | } 246 | 247 | bool nemo::SIterator::Valid() { 248 | return valid_; 249 | } 250 | 251 | void nemo::SIterator::Next() { 252 | Iterator::Next(); 253 | CheckAndLoadData(); 254 | } 255 | 256 | void nemo::SIterator::Skip(int64_t offset) { 257 | Iterator::Skip(offset); 258 | CheckAndLoadData(); 259 | } 260 | -------------------------------------------------------------------------------- /src/nemo_list.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_LIST_H 2 | #define NEMO_INCLUDE_NEMO_LIST_H 3 | 4 | #include "nemo.h" 5 | #include "nemo_const.h" 6 | #include "decoder.h" 7 | 8 | namespace nemo { 9 | 10 | struct ListData { 11 | int64_t priv; 12 | int64_t next; 13 | std::string val; 14 | ListData() : priv(0), next(0) {} 15 | ListData(int64_t _priv, int64_t _next, const std::string& _value) 16 | : priv(_priv), next(_next), val(_value) {} 17 | void reset(int64_t _priv, int64_t _next, const std::string& _value) { 18 | priv = _priv; 19 | next = _next; 20 | val = _value; 21 | } 22 | }; 23 | 24 | inline std::string EncodeLMetaKey(const rocksdb::Slice &key) { 25 | std::string buf; 26 | buf.append(1, DataType::kLMeta); 27 | buf.append(key.data(), key.size()); 28 | return buf; 29 | } 30 | 31 | inline int DecodeLMetaKey(const rocksdb::Slice &slice, std::string *key) { 32 | Decoder decoder(slice.data(), slice.size()); 33 | if (decoder.Skip(1) == -1) { 34 | return -1; 35 | } 36 | if (decoder.ReadData(key) == -1) { 37 | return -1; 38 | } 39 | return 0; 40 | } 41 | 42 | inline std::string EncodeListKey(const rocksdb::Slice &key, const int64_t seq) { 43 | std::string buf; 44 | buf.append(1, DataType::kList); 45 | buf.append(1, (uint8_t)key.size()); 46 | buf.append(key.data(), key.size()); 47 | buf.append((char *)&seq, sizeof(int64_t)); 48 | return buf; 49 | } 50 | 51 | inline int DecodeListKey(const rocksdb::Slice &slice, std::string *key, int64_t *seq) { 52 | Decoder decoder(slice.data(), slice.size()); 53 | if (decoder.Skip(1) == -1) { 54 | return -1; 55 | } 56 | if (decoder.ReadLenData(key) == -1) { 57 | return -1; 58 | } 59 | if (decoder.ReadInt64(seq) == -1) { 60 | return -1; 61 | } 62 | return 0; 63 | } 64 | 65 | inline void EncodeListVal(const std::string &raw_val, const int64_t priv, const int64_t next, std::string &en_val) { 66 | en_val.clear(); 67 | en_val.append((char *)&priv, sizeof(int64_t)); 68 | en_val.append((char *)&next, sizeof(int64_t)); 69 | en_val.append(raw_val.data(), raw_val.size()); 70 | } 71 | 72 | inline void DecodeListVal(const std::string &en_val, int64_t *priv, int64_t *next, std::string &raw_val) { 73 | raw_val.clear(); 74 | *priv = *((int64_t *)en_val.data()); 75 | *next = *((int64_t *)(en_val.data() + sizeof(int64_t))); 76 | raw_val = en_val.substr(sizeof(int64_t) * 2, en_val.size() - sizeof(int64_t) * 2); 77 | } 78 | 79 | } 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /src/nemo_meta.cc: -------------------------------------------------------------------------------- 1 | #include "nemo_hash.h" 2 | #include "nemo_list.h" 3 | #include "nemo_set.h" 4 | #include "nemo_zset.h" 5 | 6 | namespace nemo { 7 | 8 | bool NemoMeta::Create(DBType type, MetaPtr &p_meta){ 9 | switch (type) { 10 | case kHASH_DB: 11 | p_meta.reset(new HashMeta()); 12 | break; 13 | case kLIST_DB: 14 | p_meta.reset(new ListMeta()); 15 | break; 16 | case kSET_DB: 17 | p_meta.reset(new SetMeta()); 18 | break; 19 | case kZSET_DB: 20 | p_meta.reset(new ZSetMeta()); 21 | default: 22 | return false; 23 | } 24 | return true; 25 | } 26 | 27 | char Nemo::GetMetaPrefix(DBType type) { 28 | switch (type) { 29 | case kHASH_DB: 30 | return DataType::kHSize; 31 | case kLIST_DB: 32 | return DataType::kLMeta; 33 | case kSET_DB: 34 | return DataType::kSSize; 35 | case kZSET_DB: 36 | return DataType::kZSize; 37 | default: 38 | return '\0'; 39 | } 40 | } 41 | 42 | Status Nemo::ScanMetasSpecify(DBType type, const std::string &pattern, 43 | std::map& metas) { 44 | switch (type) { 45 | case kHASH_DB: 46 | return ScanDBMetas(hash_db_, type, pattern, metas); 47 | case kLIST_DB: 48 | return ScanDBMetas(list_db_, type, pattern, metas); 49 | case kSET_DB: 50 | return ScanDBMetas(set_db_, type, pattern, metas); 51 | case kZSET_DB: 52 | return ScanDBMetas(zset_db_, type, pattern, metas); 53 | default: 54 | return Status::InvalidArgument("error db type"); 55 | } 56 | } 57 | 58 | Status Nemo::ScanDBMetas(std::unique_ptr &db, 59 | DBType type, const std::string &pattern, std::map& metas) { 60 | // Get Current Snapshot 61 | const rocksdb::Snapshot* psnap; 62 | psnap = db->GetSnapshot(); 63 | if (psnap == nullptr) { 64 | return Status::Corruption("GetSnapshot failed"); 65 | } 66 | Status s = ScanDBMetasOnSnap(db, psnap, type, pattern, metas); 67 | db->ReleaseSnapshot(psnap); 68 | return s; 69 | } 70 | 71 | Status Nemo::ScanDBMetasOnSnap(std::unique_ptr &db, const rocksdb::Snapshot* psnap, 72 | DBType type, const std::string &pattern, std::map& metas) { 73 | // Get meta prefix and db handler 74 | std::string prefix = std::string(1, GetMetaPrefix(type)); 75 | if (prefix.empty()) { 76 | return Status::InvalidArgument("Error db type"); 77 | } 78 | 79 | // Create Iterator 80 | rocksdb::ReadOptions iterate_options; 81 | iterate_options.snapshot = psnap; 82 | iterate_options.fill_cache = false; 83 | rocksdb::Iterator *it = db->NewIterator(iterate_options); 84 | 85 | it->Seek(prefix); 86 | MetaPtr p_meta; 87 | NemoMeta::Create(type, p_meta); 88 | for (; it->Valid(); it->Next()) { 89 | if (prefix.at(0) != it->key().ToString().at(0)) { 90 | break; 91 | } 92 | std::string key = it->key().ToString().substr(1); 93 | if (stringmatchlen(pattern.data(), pattern.size(), key.data(), key.size(), 0)) { 94 | p_meta->DecodeFrom(it->value().ToString()); 95 | metas.insert(std::make_pair(key, p_meta)); 96 | } 97 | } 98 | 99 | //db->ReleaseSnapshot(psnap); 100 | delete it; 101 | return Status::OK(); 102 | } 103 | 104 | Status Nemo::CheckMetaSpecify(DBType type, const std::string &pattern) { 105 | switch (type) { 106 | case kHASH_DB: 107 | return CheckDBMeta(hash_db_, type, pattern); 108 | case kLIST_DB: 109 | return CheckDBMeta(list_db_, type, pattern); 110 | case kSET_DB: 111 | return CheckDBMeta(set_db_, type, pattern); 112 | case kZSET_DB: 113 | return CheckDBMeta(zset_db_, type, pattern); 114 | default: 115 | return Status::InvalidArgument("error db type"); 116 | } 117 | } 118 | 119 | Status Nemo::ChecknRecover(DBType type, const std::string& key) { 120 | switch (type) { 121 | case kHASH_DB : 122 | return HChecknRecover(key); 123 | case kLIST_DB : 124 | return LChecknRecover(key); 125 | case kSET_DB : 126 | return SChecknRecover(key); 127 | case kZSET_DB : 128 | return ZChecknRecover(key); 129 | default: 130 | return Status::InvalidArgument("error db type"); 131 | } 132 | } 133 | 134 | Status Nemo::CheckDBMeta(std::unique_ptr &db, DBType type, const std::string& pattern) { 135 | // Get Current Snapshot 136 | const rocksdb::Snapshot* psnap; 137 | psnap = db->GetSnapshot(); 138 | if (psnap == nullptr) { 139 | return Status::Corruption("GetSnapshot failed"); 140 | } 141 | 142 | // ScanMetas to get all keys + metas 143 | std::vector keys; 144 | Status s = ScanKeys(db, psnap, GetMetaPrefix(type), pattern, keys); 145 | if (!s.ok()) { 146 | return s; 147 | } 148 | 149 | // Check and Recover 150 | MetaPtr pmeta; 151 | std::vector::iterator it = keys.begin(); 152 | for (; it != keys.end(); ++it) { 153 | s = ChecknRecover(type, *it); 154 | if (!s.ok()) { 155 | break; 156 | } 157 | } 158 | return s; 159 | } 160 | 161 | } 162 | -------------------------------------------------------------------------------- /src/nemo_mutex.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_MUTEX_ 2 | #define NEMO_INCLUDE_NEMO_MUTEX_ 3 | 4 | #include "port.h" 5 | 6 | #include 7 | 8 | namespace nemo { 9 | 10 | class MutexLock { 11 | public: 12 | explicit MutexLock(pthread_mutex_t *mu) 13 | : mu_(mu) { 14 | pthread_mutex_lock(this->mu_); 15 | } 16 | ~MutexLock() { pthread_mutex_unlock(this->mu_); } 17 | 18 | private: 19 | pthread_mutex_t *const mu_; 20 | // No copying allowed 21 | MutexLock(const MutexLock&); 22 | void operator=(const MutexLock&); 23 | }; 24 | 25 | class RWLock { 26 | public: 27 | explicit RWLock(pthread_rwlock_t *mu, bool is_rwlock) 28 | : mu_(mu) { 29 | if (is_rwlock) { 30 | pthread_rwlock_wrlock(this->mu_); 31 | } else { 32 | pthread_rwlock_rdlock(this->mu_); 33 | } 34 | } 35 | ~RWLock() { pthread_rwlock_unlock(this->mu_); } 36 | 37 | private: 38 | pthread_rwlock_t *const mu_; 39 | // No copying allowed 40 | RWLock(const RWLock&); 41 | void operator=(const RWLock&); 42 | }; 43 | 44 | class RecordLock { 45 | public: 46 | RecordLock(port::RecordMutex *mu, const std::string &key) 47 | : mu_(mu), key_(key) { 48 | mu_->Lock(key_); 49 | } 50 | ~RecordLock() { mu_->Unlock(key_); } 51 | 52 | private: 53 | port::RecordMutex *const mu_; 54 | std::string key_; 55 | 56 | // No copying allowed 57 | RecordLock(const RecordLock&); 58 | void operator=(const RecordLock&); 59 | }; 60 | 61 | } 62 | #endif 63 | -------------------------------------------------------------------------------- /src/nemo_set.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_SET_H_ 2 | #define NEMO_INCLUDE_NEMO_SET_H_ 3 | 4 | #include 5 | 6 | #include "nemo.h" 7 | #include "nemo_const.h" 8 | #include "decoder.h" 9 | 10 | namespace nemo { 11 | 12 | inline std::string EncodeSetKey(const rocksdb::Slice &key, const rocksdb::Slice &member) { 13 | std::string buf; 14 | buf.append(1, DataType::kSet); 15 | buf.append(1, (uint8_t)key.size()); 16 | buf.append(key.data(), key.size()); 17 | buf.append(member.data(), member.size()); 18 | return buf; 19 | } 20 | 21 | inline int DecodeSetKey(const rocksdb::Slice &slice, std::string *key, std::string *member) { 22 | Decoder decoder(slice.data(), slice.size()); 23 | if (decoder.Skip(1) == -1) { 24 | return -1; 25 | } 26 | if (decoder.ReadLenData(key) == -1) { 27 | return -1; 28 | } 29 | if (decoder.ReadData(member) == -1) { 30 | return -1; 31 | } 32 | return 0; 33 | } 34 | 35 | inline std::string EncodeSSizeKey(const rocksdb::Slice &key) { 36 | std::string buf; 37 | buf.append(1, DataType::kSSize); 38 | buf.append(key.data(), key.size()); 39 | return buf; 40 | } 41 | 42 | inline int DecodeSSizeKey(const rocksdb::Slice &slice, std::string *size) { 43 | Decoder decoder(slice.data(), slice.size()); 44 | if (decoder.Skip(1) == -1) { 45 | return -1; 46 | } 47 | if (decoder.ReadData(size) == -1) { 48 | return -1; 49 | } 50 | return 0; 51 | } 52 | 53 | } 54 | #endif 55 | 56 | -------------------------------------------------------------------------------- /src/nemo_zset.h: -------------------------------------------------------------------------------- 1 | #ifndef NEMO_INCLUDE_NEMO_ZSET_H_ 2 | #define NEMO_INCLUDE_NEMO_ZSET_H_ 3 | 4 | #include 5 | 6 | #include "util.h" 7 | #include "nemo.h" 8 | #include "nemo_const.h" 9 | #include "decoder.h" 10 | 11 | namespace nemo { 12 | 13 | inline uint64_t EncodeScore(const double score) { 14 | int64_t iscore; 15 | if (score < 0) { 16 | iscore = (int64_t)(score * 100000LL - 0.5) + ZSET_SCORE_SHIFT; 17 | } else { 18 | iscore = (int64_t)(score * 100000LL + 0.5) + ZSET_SCORE_SHIFT; 19 | } 20 | return (uint64_t)(iscore); 21 | } 22 | 23 | inline double DecodeScore(const int64_t score) { 24 | return (double)(score - ZSET_SCORE_SHIFT) / 100000.0; 25 | } 26 | 27 | inline std::string EncodeZSetKey(const rocksdb::Slice &key, const rocksdb::Slice &member) { 28 | std::string buf; 29 | buf.append(1, DataType::kZSet); 30 | buf.append(1, (uint8_t)key.size()); 31 | buf.append(key.data(), key.size()); 32 | buf.append(member.data(), member.size()); 33 | return buf; 34 | } 35 | 36 | inline std::string EncodeZScorePrefix(const rocksdb::Slice &key) { 37 | std::string buf; 38 | buf.append(1, DataType::kZScore); 39 | buf.append(1, (uint8_t)key.size()); 40 | buf.append(key.data(), key.size()); 41 | return buf; 42 | } 43 | 44 | 45 | inline int DecodeZSetKey(const rocksdb::Slice &slice, std::string *key, std::string *member) { 46 | Decoder decoder(slice.data(), slice.size()); 47 | if (decoder.Skip(1) == -1) { 48 | return -1; 49 | } 50 | if (decoder.ReadLenData(key) == -1) { 51 | return -1; 52 | } 53 | if (decoder.ReadData(member) == -1) { 54 | return -1; 55 | } 56 | return 0; 57 | } 58 | 59 | inline std::string EncodeZSizeKey(const rocksdb::Slice key) { 60 | std::string buf; 61 | buf.append(1, DataType::kZSize); 62 | buf.append(key.data(), key.size()); 63 | return buf; 64 | } 65 | 66 | inline int DecodeZSizeKey(const rocksdb::Slice &slice, std::string *size) { 67 | Decoder decoder(slice.data(), slice.size()); 68 | if (decoder.Skip(1) == -1) { 69 | return -1; 70 | } 71 | if (decoder.ReadData(size) == -1) { 72 | return -1; 73 | } 74 | return 0; 75 | } 76 | 77 | inline std::string EncodeZScoreKey(const rocksdb::Slice &key, const rocksdb::Slice &member, const double score) { 78 | std::string buf; 79 | uint64_t new_score = EncodeScore(score); 80 | buf.append(1, DataType::kZScore); 81 | buf.append(1, (uint8_t)key.size()); 82 | buf.append(key.data(), key.size()); 83 | new_score = htobe64(new_score); 84 | buf.append((char *)&new_score, sizeof(int64_t)); 85 | buf.append(member.data(), member.size()); 86 | return buf; 87 | } 88 | 89 | inline int DecodeZScoreKey(const rocksdb::Slice &slice, std::string *key, std::string *member, double *score) { 90 | Decoder decoder(slice.data(), slice.size()); 91 | if (decoder.Skip(1) == -1) { 92 | return -1; 93 | } 94 | if (decoder.ReadLenData(key) == -1) { 95 | return -1; 96 | } 97 | uint64_t iscore = 0; 98 | decoder.ReadUInt64(&iscore); 99 | //iscore = be64toh(iscore); 100 | *score = DecodeScore(iscore); 101 | if (decoder.ReadData(member) == -1) { 102 | return -1; 103 | } 104 | return 0; 105 | } 106 | 107 | } 108 | #endif 109 | 110 | -------------------------------------------------------------------------------- /src/port.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "port.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "xdebug.h" 20 | 21 | namespace nemo { 22 | namespace port { 23 | 24 | static int PthreadCall(const char* label, int result) { 25 | if (result != 0 && result != ETIMEDOUT) { 26 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 27 | abort(); 28 | } 29 | return result; 30 | } 31 | 32 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); } 33 | 34 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 35 | 36 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 37 | 38 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 39 | 40 | 41 | CondVar::CondVar(Mutex* mu) 42 | : mu_(mu) { 43 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 44 | } 45 | 46 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 47 | 48 | void CondVar::Wait() { 49 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 50 | } 51 | 52 | void CondVar::Signal() { 53 | PthreadCall("signal", pthread_cond_signal(&cv_)); 54 | } 55 | 56 | void CondVar::SignalAll() { 57 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 58 | } 59 | 60 | 61 | RWMutex::RWMutex() { PthreadCall("init mutex", pthread_rwlock_init(&mu_, nullptr)); } 62 | 63 | RWMutex::~RWMutex() { PthreadCall("destroy mutex", pthread_rwlock_destroy(&mu_)); } 64 | 65 | void RWMutex::ReadLock() { PthreadCall("read lock", pthread_rwlock_rdlock(&mu_)); } 66 | 67 | void RWMutex::WriteLock() { PthreadCall("write lock", pthread_rwlock_wrlock(&mu_)); } 68 | 69 | void RWMutex::ReadUnlock() { PthreadCall("read unlock", pthread_rwlock_unlock(&mu_)); } 70 | 71 | void RWMutex::WriteUnlock() { PthreadCall("write unlock", pthread_rwlock_unlock(&mu_)); } 72 | 73 | RefMutex::RefMutex() { 74 | refs_ = 0; 75 | PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); 76 | } 77 | 78 | RefMutex::~RefMutex() { 79 | PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); 80 | } 81 | 82 | void RefMutex::Ref() { 83 | refs_++; 84 | } 85 | void RefMutex::Unref() { 86 | --refs_; 87 | if (refs_ == 0) { 88 | delete this; 89 | } 90 | } 91 | 92 | void RefMutex::Lock() { 93 | PthreadCall("lock", pthread_mutex_lock(&mu_)); 94 | } 95 | 96 | void RefMutex::Unlock() { 97 | PthreadCall("unlock", pthread_mutex_unlock(&mu_)); 98 | } 99 | 100 | RecordMutex::~RecordMutex() { 101 | mutex_.Lock(); 102 | 103 | std::unordered_map::const_iterator it = records_.begin(); 104 | for (; it != records_.end(); it++) { 105 | delete it->second; 106 | } 107 | mutex_.Unlock(); 108 | } 109 | 110 | int64_t RecordMutex::GetUsage() { 111 | int64_t size = 0; 112 | mutex_.Lock(); 113 | size = charge_; 114 | mutex_.Unlock(); 115 | return size; 116 | } 117 | 118 | const int64_t kEstimatePairSize = sizeof(std::string) + sizeof(RefMutex) + sizeof(std::pair); 119 | 120 | void RecordMutex::Lock(const std::string &key) { 121 | mutex_.Lock(); 122 | std::unordered_map::const_iterator it = records_.find(key); 123 | 124 | if (it != records_.end()) { 125 | //log_info ("tid=(%u) >Lock key=(%s) exist, map_size=%u", pthread_self(), key.c_str(), records_.size()); 126 | RefMutex *ref_mutex = it->second; 127 | ref_mutex->Ref(); 128 | mutex_.Unlock(); 129 | 130 | ref_mutex->Lock(); 131 | //log_info ("tid=(%u) Lock key=(%s) new, map_size=%u ++", pthread_self(), key.c_str(), records_.size()); 134 | RefMutex *ref_mutex = new RefMutex(); 135 | 136 | records_.insert(std::make_pair(key, ref_mutex)); 137 | ref_mutex->Ref(); 138 | charge_ += kEstimatePairSize + key.size(); 139 | mutex_.Unlock(); 140 | 141 | ref_mutex->Lock(); 142 | //log_info ("tid=(%u) ::const_iterator it = records_.find(key); 149 | 150 | //log_info ("tid=(%u) >Unlock key=(%s) new, map_size=%u --", pthread_self(), key.c_str(), records_.size()); 151 | if (it != records_.end()) { 152 | RefMutex *ref_mutex = it->second; 153 | 154 | if (ref_mutex->IsLastRef()) { 155 | charge_ -= kEstimatePairSize + key.size(); 156 | records_.erase(it); 157 | } 158 | ref_mutex->Unlock(); 159 | ref_mutex->Unref(); 160 | } 161 | 162 | mutex_.Unlock(); 163 | //log_info ("tid=(%u) 37 | #include 38 | #include "gtest/internal/gtest-internal.h" 39 | #include "gtest/internal/gtest-string.h" 40 | 41 | namespace testing { 42 | 43 | // A copyable object representing the result of a test part (i.e. an 44 | // assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). 45 | // 46 | // Don't inherit from TestPartResult as its destructor is not virtual. 47 | class GTEST_API_ TestPartResult { 48 | public: 49 | // The possible outcomes of a test part (i.e. an assertion or an 50 | // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). 51 | enum Type { 52 | kSuccess, // Succeeded. 53 | kNonFatalFailure, // Failed but the test can continue. 54 | kFatalFailure // Failed and the test should be terminated. 55 | }; 56 | 57 | // C'tor. TestPartResult does NOT have a default constructor. 58 | // Always use this constructor (with parameters) to create a 59 | // TestPartResult object. 60 | TestPartResult(Type a_type, 61 | const char* a_file_name, 62 | int a_line_number, 63 | const char* a_message) 64 | : type_(a_type), 65 | file_name_(a_file_name == NULL ? "" : a_file_name), 66 | line_number_(a_line_number), 67 | summary_(ExtractSummary(a_message)), 68 | message_(a_message) { 69 | } 70 | 71 | // Gets the outcome of the test part. 72 | Type type() const { return type_; } 73 | 74 | // Gets the name of the source file where the test part took place, or 75 | // NULL if it's unknown. 76 | const char* file_name() const { 77 | return file_name_.empty() ? NULL : file_name_.c_str(); 78 | } 79 | 80 | // Gets the line in the source file where the test part took place, 81 | // or -1 if it's unknown. 82 | int line_number() const { return line_number_; } 83 | 84 | // Gets the summary of the failure message. 85 | const char* summary() const { return summary_.c_str(); } 86 | 87 | // Gets the message associated with the test part. 88 | const char* message() const { return message_.c_str(); } 89 | 90 | // Returns true iff the test part passed. 91 | bool passed() const { return type_ == kSuccess; } 92 | 93 | // Returns true iff the test part failed. 94 | bool failed() const { return type_ != kSuccess; } 95 | 96 | // Returns true iff the test part non-fatally failed. 97 | bool nonfatally_failed() const { return type_ == kNonFatalFailure; } 98 | 99 | // Returns true iff the test part fatally failed. 100 | bool fatally_failed() const { return type_ == kFatalFailure; } 101 | 102 | private: 103 | Type type_; 104 | 105 | // Gets the summary of the failure message by omitting the stack 106 | // trace in it. 107 | static std::string ExtractSummary(const char* message); 108 | 109 | // The name of the source file where the test part took place, or 110 | // "" if the source file is unknown. 111 | std::string file_name_; 112 | // The line in the source file where the test part took place, or -1 113 | // if the line number is unknown. 114 | int line_number_; 115 | std::string summary_; // The test failure summary. 116 | std::string message_; // The test failure message. 117 | }; 118 | 119 | // Prints a TestPartResult object. 120 | std::ostream& operator<<(std::ostream& os, const TestPartResult& result); 121 | 122 | // An array of TestPartResult objects. 123 | // 124 | // Don't inherit from TestPartResultArray as its destructor is not 125 | // virtual. 126 | class GTEST_API_ TestPartResultArray { 127 | public: 128 | TestPartResultArray() {} 129 | 130 | // Appends the given TestPartResult to the array. 131 | void Append(const TestPartResult& result); 132 | 133 | // Returns the TestPartResult at the given index (0-based). 134 | const TestPartResult& GetTestPartResult(int index) const; 135 | 136 | // Returns the number of TestPartResult objects in the array. 137 | int size() const; 138 | 139 | private: 140 | std::vector array_; 141 | 142 | GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); 143 | }; 144 | 145 | // This interface knows how to report a test part result. 146 | class TestPartResultReporterInterface { 147 | public: 148 | virtual ~TestPartResultReporterInterface() {} 149 | 150 | virtual void ReportTestPartResult(const TestPartResult& result) = 0; 151 | }; 152 | 153 | namespace internal { 154 | 155 | // This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a 156 | // statement generates new fatal failures. To do so it registers itself as the 157 | // current test part result reporter. Besides checking if fatal failures were 158 | // reported, it only delegates the reporting to the former result reporter. 159 | // The original result reporter is restored in the destructor. 160 | // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. 161 | class GTEST_API_ HasNewFatalFailureHelper 162 | : public TestPartResultReporterInterface { 163 | public: 164 | HasNewFatalFailureHelper(); 165 | virtual ~HasNewFatalFailureHelper(); 166 | virtual void ReportTestPartResult(const TestPartResult& result); 167 | bool has_new_fatal_failure() const { return has_new_fatal_failure_; } 168 | private: 169 | bool has_new_fatal_failure_; 170 | TestPartResultReporterInterface* original_reporter_; 171 | 172 | GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); 173 | }; 174 | 175 | } // namespace internal 176 | 177 | } // namespace testing 178 | 179 | #endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ 180 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/include/gtest/gtest_prod.h: -------------------------------------------------------------------------------- 1 | // Copyright 2006, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: wan@google.com (Zhanyong Wan) 31 | // 32 | // Google C++ Testing Framework definitions useful in production code. 33 | 34 | #ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ 35 | #define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ 36 | 37 | // When you need to test the private or protected members of a class, 38 | // use the FRIEND_TEST macro to declare your tests as friends of the 39 | // class. For example: 40 | // 41 | // class MyClass { 42 | // private: 43 | // void MyMethod(); 44 | // FRIEND_TEST(MyClassTest, MyMethod); 45 | // }; 46 | // 47 | // class MyClassTest : public testing::Test { 48 | // // ... 49 | // }; 50 | // 51 | // TEST_F(MyClassTest, MyMethod) { 52 | // // Can call MyClass::MyMethod() here. 53 | // } 54 | 55 | #define FRIEND_TEST(test_case_name, test_name)\ 56 | friend class test_case_name##_##test_name##_Test 57 | 58 | #endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ 59 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/include/gtest/internal/gtest-string.h: -------------------------------------------------------------------------------- 1 | // Copyright 2005, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) 31 | // 32 | // The Google C++ Testing Framework (Google Test) 33 | // 34 | // This header file declares the String class and functions used internally by 35 | // Google Test. They are subject to change without notice. They should not used 36 | // by code external to Google Test. 37 | // 38 | // This header file is #included by . 39 | // It should not be #included by other files. 40 | 41 | #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ 42 | #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ 43 | 44 | #ifdef __BORLANDC__ 45 | // string.h is not guaranteed to provide strcpy on C++ Builder. 46 | # include 47 | #endif 48 | 49 | #include 50 | #include 51 | 52 | #include "gtest/internal/gtest-port.h" 53 | 54 | namespace testing { 55 | namespace internal { 56 | 57 | // String - an abstract class holding static string utilities. 58 | class GTEST_API_ String { 59 | public: 60 | // Static utility methods 61 | 62 | // Clones a 0-terminated C string, allocating memory using new. The 63 | // caller is responsible for deleting the return value using 64 | // delete[]. Returns the cloned string, or NULL if the input is 65 | // NULL. 66 | // 67 | // This is different from strdup() in string.h, which allocates 68 | // memory using malloc(). 69 | static const char* CloneCString(const char* c_str); 70 | 71 | #if GTEST_OS_WINDOWS_MOBILE 72 | // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be 73 | // able to pass strings to Win32 APIs on CE we need to convert them 74 | // to 'Unicode', UTF-16. 75 | 76 | // Creates a UTF-16 wide string from the given ANSI string, allocating 77 | // memory using new. The caller is responsible for deleting the return 78 | // value using delete[]. Returns the wide string, or NULL if the 79 | // input is NULL. 80 | // 81 | // The wide string is created using the ANSI codepage (CP_ACP) to 82 | // match the behaviour of the ANSI versions of Win32 calls and the 83 | // C runtime. 84 | static LPCWSTR AnsiToUtf16(const char* c_str); 85 | 86 | // Creates an ANSI string from the given wide string, allocating 87 | // memory using new. The caller is responsible for deleting the return 88 | // value using delete[]. Returns the ANSI string, or NULL if the 89 | // input is NULL. 90 | // 91 | // The returned string is created using the ANSI codepage (CP_ACP) to 92 | // match the behaviour of the ANSI versions of Win32 calls and the 93 | // C runtime. 94 | static const char* Utf16ToAnsi(LPCWSTR utf16_str); 95 | #endif 96 | 97 | // Compares two C strings. Returns true iff they have the same content. 98 | // 99 | // Unlike strcmp(), this function can handle NULL argument(s). A 100 | // NULL C string is considered different to any non-NULL C string, 101 | // including the empty string. 102 | static bool CStringEquals(const char* lhs, const char* rhs); 103 | 104 | // Converts a wide C string to a String using the UTF-8 encoding. 105 | // NULL will be converted to "(null)". If an error occurred during 106 | // the conversion, "(failed to convert from wide string)" is 107 | // returned. 108 | static std::string ShowWideCString(const wchar_t* wide_c_str); 109 | 110 | // Compares two wide C strings. Returns true iff they have the same 111 | // content. 112 | // 113 | // Unlike wcscmp(), this function can handle NULL argument(s). A 114 | // NULL C string is considered different to any non-NULL C string, 115 | // including the empty string. 116 | static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); 117 | 118 | // Compares two C strings, ignoring case. Returns true iff they 119 | // have the same content. 120 | // 121 | // Unlike strcasecmp(), this function can handle NULL argument(s). 122 | // A NULL C string is considered different to any non-NULL C string, 123 | // including the empty string. 124 | static bool CaseInsensitiveCStringEquals(const char* lhs, 125 | const char* rhs); 126 | 127 | // Compares two wide C strings, ignoring case. Returns true iff they 128 | // have the same content. 129 | // 130 | // Unlike wcscasecmp(), this function can handle NULL argument(s). 131 | // A NULL C string is considered different to any non-NULL wide C string, 132 | // including the empty string. 133 | // NB: The implementations on different platforms slightly differ. 134 | // On windows, this method uses _wcsicmp which compares according to LC_CTYPE 135 | // environment variable. On GNU platform this method uses wcscasecmp 136 | // which compares according to LC_CTYPE category of the current locale. 137 | // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the 138 | // current locale. 139 | static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, 140 | const wchar_t* rhs); 141 | 142 | // Returns true iff the given string ends with the given suffix, ignoring 143 | // case. Any string is considered to end with an empty suffix. 144 | static bool EndsWithCaseInsensitive( 145 | const std::string& str, const std::string& suffix); 146 | 147 | // Formats an int value as "%02d". 148 | static std::string FormatIntWidth2(int value); // "%02d" for width == 2 149 | 150 | // Formats an int value as "%X". 151 | static std::string FormatHexInt(int value); 152 | 153 | // Formats a byte as "%02X". 154 | static std::string FormatByte(unsigned char value); 155 | 156 | private: 157 | String(); // Not meant to be instantiated. 158 | }; // class String 159 | 160 | // Gets the content of the stringstream's buffer as an std::string. Each '\0' 161 | // character in the buffer is replaced with "\\0". 162 | GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); 163 | 164 | } // namespace internal 165 | } // namespace testing 166 | 167 | #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ 168 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/src/gtest-all.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: mheule@google.com (Markus Heule) 31 | // 32 | // Google C++ Testing Framework (Google Test) 33 | // 34 | // Sometimes it's desirable to build Google Test by compiling a single file. 35 | // This file serves this purpose. 36 | 37 | // This line ensures that gtest.h can be compiled on its own, even 38 | // when it's fused. 39 | #include "gtest/gtest.h" 40 | 41 | // The following lines pull in the real gtest *.cc files. 42 | #include "src/gtest.cc" 43 | #include "src/gtest-death-test.cc" 44 | #include "src/gtest-filepath.cc" 45 | #include "src/gtest-port.cc" 46 | #include "src/gtest-printers.cc" 47 | #include "src/gtest-test-part.cc" 48 | #include "src/gtest-typed-test.cc" 49 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/src/gtest-test-part.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: mheule@google.com (Markus Heule) 31 | // 32 | // The Google C++ Testing Framework (Google Test) 33 | 34 | #include "gtest/gtest-test-part.h" 35 | 36 | // Indicates that this translation unit is part of Google Test's 37 | // implementation. It must come before gtest-internal-inl.h is 38 | // included, or there will be a compiler error. This trick is to 39 | // prevent a user from accidentally including gtest-internal-inl.h in 40 | // his code. 41 | #define GTEST_IMPLEMENTATION_ 1 42 | #include "src/gtest-internal-inl.h" 43 | #undef GTEST_IMPLEMENTATION_ 44 | 45 | namespace testing { 46 | 47 | using internal::GetUnitTestImpl; 48 | 49 | // Gets the summary of the failure message by omitting the stack trace 50 | // in it. 51 | std::string TestPartResult::ExtractSummary(const char* message) { 52 | const char* const stack_trace = strstr(message, internal::kStackTraceMarker); 53 | return stack_trace == NULL ? message : 54 | std::string(message, stack_trace); 55 | } 56 | 57 | // Prints a TestPartResult object. 58 | std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { 59 | return os 60 | << result.file_name() << ":" << result.line_number() << ": " 61 | << (result.type() == TestPartResult::kSuccess ? "Success" : 62 | result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : 63 | "Non-fatal failure") << ":\n" 64 | << result.message() << std::endl; 65 | } 66 | 67 | // Appends a TestPartResult to the array. 68 | void TestPartResultArray::Append(const TestPartResult& result) { 69 | array_.push_back(result); 70 | } 71 | 72 | // Returns the TestPartResult at the given index (0-based). 73 | const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { 74 | if (index < 0 || index >= size()) { 75 | printf("\nInvalid index (%d) into TestPartResultArray.\n", index); 76 | internal::posix::Abort(); 77 | } 78 | 79 | return array_[index]; 80 | } 81 | 82 | // Returns the number of TestPartResult objects in the array. 83 | int TestPartResultArray::size() const { 84 | return static_cast(array_.size()); 85 | } 86 | 87 | namespace internal { 88 | 89 | HasNewFatalFailureHelper::HasNewFatalFailureHelper() 90 | : has_new_fatal_failure_(false), 91 | original_reporter_(GetUnitTestImpl()-> 92 | GetTestPartResultReporterForCurrentThread()) { 93 | GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); 94 | } 95 | 96 | HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { 97 | GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( 98 | original_reporter_); 99 | } 100 | 101 | void HasNewFatalFailureHelper::ReportTestPartResult( 102 | const TestPartResult& result) { 103 | if (result.fatally_failed()) 104 | has_new_fatal_failure_ = true; 105 | original_reporter_->ReportTestPartResult(result); 106 | } 107 | 108 | } // namespace internal 109 | 110 | } // namespace testing 111 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/src/gtest-typed-test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 Google Inc. 2 | // All Rights Reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // Author: wan@google.com (Zhanyong Wan) 31 | 32 | #include "gtest/gtest-typed-test.h" 33 | #include "gtest/gtest.h" 34 | 35 | namespace testing { 36 | namespace internal { 37 | 38 | #if GTEST_HAS_TYPED_TEST_P 39 | 40 | // Skips to the first non-space char in str. Returns an empty string if str 41 | // contains only whitespace characters. 42 | static const char* SkipSpaces(const char* str) { 43 | while (IsSpace(*str)) 44 | str++; 45 | return str; 46 | } 47 | 48 | // Verifies that registered_tests match the test names in 49 | // defined_test_names_; returns registered_tests if successful, or 50 | // aborts the program otherwise. 51 | const char* TypedTestCasePState::VerifyRegisteredTestNames( 52 | const char* file, int line, const char* registered_tests) { 53 | typedef ::std::set::const_iterator DefinedTestIter; 54 | registered_ = true; 55 | 56 | // Skip initial whitespace in registered_tests since some 57 | // preprocessors prefix stringizied literals with whitespace. 58 | registered_tests = SkipSpaces(registered_tests); 59 | 60 | Message errors; 61 | ::std::set tests; 62 | for (const char* names = registered_tests; names != NULL; 63 | names = SkipComma(names)) { 64 | const std::string name = GetPrefixUntilComma(names); 65 | if (tests.count(name) != 0) { 66 | errors << "Test " << name << " is listed more than once.\n"; 67 | continue; 68 | } 69 | 70 | bool found = false; 71 | for (DefinedTestIter it = defined_test_names_.begin(); 72 | it != defined_test_names_.end(); 73 | ++it) { 74 | if (name == *it) { 75 | found = true; 76 | break; 77 | } 78 | } 79 | 80 | if (found) { 81 | tests.insert(name); 82 | } else { 83 | errors << "No test named " << name 84 | << " can be found in this test case.\n"; 85 | } 86 | } 87 | 88 | for (DefinedTestIter it = defined_test_names_.begin(); 89 | it != defined_test_names_.end(); 90 | ++it) { 91 | if (tests.count(*it) == 0) { 92 | errors << "You forgot to list test " << *it << ".\n"; 93 | } 94 | } 95 | 96 | const std::string& errors_str = errors.GetString(); 97 | if (errors_str != "") { 98 | fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), 99 | errors_str.c_str()); 100 | fflush(stderr); 101 | posix::Abort(); 102 | } 103 | 104 | return registered_tests; 105 | } 106 | 107 | #endif // GTEST_HAS_TYPED_TEST_P 108 | 109 | } // namespace internal 110 | } // namespace testing 111 | -------------------------------------------------------------------------------- /test/gtest_1.7.0/src/gtest_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2006, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | #include 31 | 32 | #include "gtest/gtest.h" 33 | 34 | GTEST_API_ int main(int argc, char **argv) { 35 | printf("Running main() from gtest_main.cc\n"); 36 | testing::InitGoogleTest(&argc, argv); 37 | return RUN_ALL_TESTS(); 38 | } 39 | -------------------------------------------------------------------------------- /test/include/nemo_hash_test.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "gtest/gtest.h" 9 | #include "xdebug.h" 10 | #include "nemo.h" 11 | #include "nemo_test.h" 12 | 13 | #ifndef NEMO_HASH_TEST_H 14 | #define NEMO_HASH_TEST_H 15 | 16 | using namespace std; 17 | 18 | class NemoHashTest : public NemoTest 19 | { 20 | protected: 21 | static const unsigned int maxHMSetNum_ = 100; 22 | static const unsigned int maxHMGetNum_ = 200; 23 | }; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /test/include/nemo_kv_test.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "gtest/gtest.h" 9 | #include "xdebug.h" 10 | #include "nemo.h" 11 | #include "nemo_test.h" 12 | 13 | #ifndef NEMO_KV_TEST_H 14 | #define NEMO_KV_TEST_H 15 | 16 | using namespace std; 17 | 18 | class NemoKVTest : public NemoTest 19 | { 20 | public: 21 | nemo::KV GetKV(const string &key, const string &val) 22 | { 23 | nemo::KV kv; 24 | kv.key = key; 25 | kv.val = val; 26 | return kv; 27 | } 28 | 29 | protected: 30 | static const unsigned int minMSetNum_ = 0; 31 | static const unsigned int maxMSetNum_ = 100; 32 | static const unsigned int maxMDelNum_ = 300; 33 | }; 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /test/include/nemo_list_test.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "gtest/gtest.h" 9 | #include "xdebug.h" 10 | #include "nemo.h" 11 | #include "nemo_test.h" 12 | 13 | #ifndef NEMO_LIST_TEST_H 14 | #define NEMO_LIST_TEST_H 15 | 16 | using namespace std; 17 | 18 | class NemoListTest : public NemoTest 19 | { 20 | public: 21 | void write_list(int64_t llen, string key) 22 | { 23 | int64_t tempLLen, index; 24 | string getVal; 25 | n_->LLen(key, &tempLLen); 26 | while(tempLLen > 0) 27 | { 28 | n_->LPop(key, &getVal); 29 | tempLLen--; 30 | } 31 | for(int64_t idx = 0; idx != llen; idx++) 32 | { 33 | n_->RPush(key, key + "_" + itoa(idx), &tempLLen); 34 | } 35 | log_message("===List from left to right is %s_0 to %s_%lld", key.c_str(), key.c_str(), llen); 36 | } 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /test/include/nemo_set_test.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "gtest/gtest.h" 9 | #include "xdebug.h" 10 | #include "nemo.h" 11 | #include "nemo_test.h" 12 | 13 | #ifndef NEMO_SET_TEST_H 14 | #define NEMO_SET_TEST_H 15 | 16 | using namespace std; 17 | 18 | class NemoSetTest : public NemoTest 19 | { 20 | public: 21 | bool isUnique(vector &members) { 22 | map tempMap; 23 | bool flag = true; 24 | for (vector::iterator iter = members.begin(); iter != members.end(); iter++) { 25 | tempMap[*iter]++; 26 | } 27 | for (map::iterator iter = tempMap.begin(); iter != tempMap.end(); iter++) { 28 | if (iter->second > 1) { 29 | flag = false; 30 | break; 31 | } 32 | } 33 | return flag; 34 | } 35 | 36 | bool isAllInKey(string key, vector &members) { //for SRandMember test 37 | bool flag = true; 38 | for (vector::iterator iter = members.begin(); iter != members.end(); iter++) { 39 | if (!(n_->SIsMember(key, *iter))) 40 | { 41 | flag = false; 42 | break; 43 | } 44 | } 45 | return flag; 46 | } 47 | 48 | 49 | void write_set(const string &key, const int64_t num) 50 | { 51 | int64_t resTemp; 52 | //numPre = n_->SCard(key); 53 | //for (int64_t index = 0; index != numPre; index++) { 54 | // n_->SRem(key, key+"_"+itoa(index), &resTemp); 55 | //} 56 | nemo::SIterator* siter = n_->SScan(key, -1); 57 | for (; siter->Valid(); siter->Next()) 58 | //while(siter->Next()) 59 | { 60 | n_->SRem(key, siter->member(), &resTemp); 61 | } 62 | delete siter; 63 | for (int64_t index = 0; index != num; index++) 64 | { 65 | n_->SAdd(key, key+"_"+itoa(index), &resTemp); 66 | } 67 | } 68 | 69 | void write_set_scope(const string &key, const int64_t numStart, const int64_t numEnd) 70 | { 71 | int64_t resTemp; 72 | //numPre = n_->SCard(key); 73 | //for (int64_t index = 0; index != numPre; index++) { 74 | // n_->SRem(key, key+"_"+itoa(index), &resTemp); 75 | //} 76 | nemo::SIterator* siter = n_->SScan(key, -1); 77 | for (; siter->Valid(); siter->Next()) 78 | { 79 | //while(siter->Next()) { 80 | n_->SRem(key, siter->member(), &resTemp); 81 | } 82 | delete siter; 83 | for (int64_t index = numStart; index != numEnd; index++) 84 | { 85 | n_->SAdd(key, itoa(index), &resTemp); 86 | } 87 | } 88 | 89 | void deleteAllMembers(string key) { 90 | nemo::SIterator* siter = n_->SScan(key, -1); 91 | int64_t resTemp; 92 | //while (siter->Next()) { 93 | for (; siter->Valid(); siter->Next()) { 94 | n_->SRem(key, siter->member(), &resTemp); 95 | } 96 | delete siter; 97 | } 98 | }; 99 | #endif 100 | -------------------------------------------------------------------------------- /test/include/nemo_test.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/include/nemo_test.h -------------------------------------------------------------------------------- /test/include/nemo_zset_test.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "gtest/gtest.h" 9 | #include "xdebug.h" 10 | #include "nemo.h" 11 | #include "nemo_test.h" 12 | 13 | #ifndef NEMO_ZSET_TEST_H 14 | #define NEMO_ZSET_TEST_H 15 | 16 | using namespace std; 17 | 18 | #define ZSET_SCORE_MIN nemo::ZSET_SCORE_MIN 19 | #define ZSET_SCORE_MAX nemo::ZSET_SCORE_MAX 20 | 21 | #define SUM nemo::SUM 22 | #define MIN nemo::MIN 23 | #define MAX nemo::MAX 24 | 25 | 26 | 27 | class NemoZSetTest : public NemoTest 28 | { 29 | public: 30 | void write_zset_random_score(string key, int64_t num, int64_t numPre = 0) { 31 | string member; 32 | double score; 33 | int64_t resTemp; 34 | n_->ZRemrangebyscore(key, ZSET_SCORE_MIN, ZSET_SCORE_MAX, &resTemp); 35 | for (int64_t index = 0; index != num; index++) { 36 | score = GetRandomFloat_(); 37 | member = itoa(index+numPre); 38 | n_->ZAdd(key, score, member, &resTemp); 39 | } 40 | } 41 | 42 | void write_zset_same_score(string key, int64_t num, double score = 1.0, int64_t numPre = 0) { 43 | string member; 44 | int64_t resTemp; 45 | n_->ZRemrangebyscore(key, ZSET_SCORE_MIN, ZSET_SCORE_MAX, &resTemp); 46 | for (int64_t index = 0; index != num; index++) { 47 | member = itoa(index+numPre); 48 | n_->ZAdd(key, score, member, &resTemp); 49 | } 50 | } 51 | 52 | void write_zset_up_score(string key, int64_t num, int64_t startScore = 0, int64_t numPre = 0) { 53 | string member; 54 | int64_t resTemp; 55 | double score; 56 | n_->ZRemrangebyscore(key, ZSET_SCORE_MIN, ZSET_SCORE_MAX, &resTemp); 57 | for (int64_t index = 0; index != num; index++) { 58 | member = itoa(index+numPre); 59 | score = index*1.0 + startScore; 60 | n_->ZAdd(key, score, member, &resTemp); 61 | } 62 | } 63 | 64 | void write_zset_up_score_scope(string key, int64_t start, int64_t end, int64_t startScore = 0, int64_t numPre = 0) { 65 | string member; 66 | int64_t resTemp; 67 | double score; 68 | n_->ZRemrangebyscore(key, ZSET_SCORE_MIN, ZSET_SCORE_MAX, &resTemp); 69 | for (int64_t index = start; index != end; index++) { 70 | member = itoa(index); 71 | score = index*1.0 + startScore; 72 | n_->ZAdd(key, score, member, &resTemp); 73 | } 74 | } 75 | 76 | bool isSorted(const vector &members) { 77 | bool flag = true; 78 | string memberPre = "deadbeaf"; 79 | for (vector::const_iterator iter = members.begin(); iter != members.end(); iter++) { 80 | if (memberPre != "deadbeaf" && iter->compare(memberPre) < 0) { 81 | flag = false; 82 | break; 83 | } 84 | memberPre = *iter; 85 | } 86 | return flag; 87 | } 88 | }; 89 | #endif 90 | -------------------------------------------------------------------------------- /test/main.cc: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | #include "nemo_test.h" 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | FILE *LOG_FILE = fopen(LOG_FILE_NAME, "w"); 7 | fclose(LOG_FILE); 8 | ::testing::InitGoogleTest(&argc, argv); 9 | return RUN_ALL_TESTS(); 10 | } 11 | -------------------------------------------------------------------------------- /test/nemo_hash_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/nemo_hash_test.cc -------------------------------------------------------------------------------- /test/nemo_kv_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/nemo_kv_test.cc -------------------------------------------------------------------------------- /test/nemo_list_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/nemo_list_test.cc -------------------------------------------------------------------------------- /test/nemo_set_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/nemo_set_test.cc -------------------------------------------------------------------------------- /test/nemo_zset_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/nemo/f47cb5c4ca8d4962ef091fee1c9798a2504ae8ca/test/nemo_zset_test.cc -------------------------------------------------------------------------------- /tools/compact/Makefile: -------------------------------------------------------------------------------- 1 | GCC = g++ 2 | CPPFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -Wno-unused-parameter -DDEBUG -D__XDEBUG__ -g -O2 -std=c++11 3 | OBJECT = compact 4 | 5 | LIB_PATH = -L ../../output/lib 6 | 7 | LIBS = -Wl,-Bstatic -lnemo -lnemodb -lrocksdb \ 8 | -Wl,-Bdynamic -lpthread\ 9 | -lsnappy \ 10 | -lrt \ 11 | -lz \ 12 | -lbz2 13 | 14 | INCLUDE_PATH = -I../../output/include/ \ 15 | -I../../3rdparty/nemo-rocksdb/rocksdb/ \ 16 | -I../../3rdparty/nemo-rocksdb/rocksdb/include 17 | 18 | .PHONY: all clean 19 | 20 | 21 | # BASE_BOJS := $(wildcard *.cpp) 22 | # BASE_BOJS += $(wildcard *.c) 23 | # OBJS := $(patsubst %.cpp,%.o,$(BASE_BOJS)) 24 | 25 | 26 | all: $(OBJECT) 27 | rm *.o 28 | 29 | $(OBJECT): $(OBJECT).o 30 | $(GCC) $(CPPFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 31 | 32 | %.o : %.cc 33 | $(GCC) $(CPPFLAGS) -c $< -o $@ $(INCLUDE_PATH) 34 | 35 | clean: 36 | rm -rf $(OBJECT) $(OBJECT).o 37 | -------------------------------------------------------------------------------- /tools/compact/README: -------------------------------------------------------------------------------- 1 | Tool for Manual Compact 2 | 3 | Usage: 4 | ./compact db_path type 5 | type is one of: kv, hash, list, zset, set, all 6 | -------------------------------------------------------------------------------- /tools/compact/compact.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "xdebug.h" 4 | #include 5 | #include "nemo.h" 6 | #include "nemo_const.h" 7 | 8 | using namespace std; 9 | 10 | void Usage() { 11 | cout << "Usage: " << endl; 12 | cout << "./compact db_path type" << endl; 13 | cout << "type is one of: kv, hash, list, zset, set, all" << endl; 14 | } 15 | 16 | int main(int argc, char **argv) 17 | { 18 | if (argc != 3) { 19 | Usage(); 20 | log_err("not enough parameter"); 21 | } 22 | std::string path(argv[1]); 23 | std::string db_type(argv[2]); 24 | 25 | // Create nemo handle 26 | nemo::Options option; 27 | option.write_buffer_size = 268435456; 28 | option.target_file_size_base = 20971520; 29 | 30 | log_info("Prepare DB..."); 31 | nemo::Nemo* db = new nemo::Nemo(path, option); 32 | assert(db); 33 | log_info("Compact Begin"); 34 | 35 | nemo::Status s; 36 | if (db_type == "kv") { 37 | s = db->Compact(nemo::kKV_DB, true); 38 | } else if (db_type == "hash") { 39 | s = db->Compact(nemo::kHASH_DB, true); 40 | } else if (db_type == "list") { 41 | s = db->Compact(nemo::kLIST_DB, true); 42 | } else if (db_type == "set") { 43 | s = db->Compact(nemo::kSET_DB, true); 44 | } else if (db_type == "zset") { 45 | s = db->Compact(nemo::kZSET_DB, true); 46 | } else if (db_type == "all") { 47 | s = db->Compact(nemo::kALL, true); 48 | } else { 49 | Usage(); 50 | log_err("Error db type : %s", db_type.c_str()); 51 | } 52 | // if (db_type == "all") { 53 | // s = db->Compact(); 54 | // } else { 55 | // s = db->CompactSpecify(db_type); 56 | // } 57 | delete db; 58 | 59 | if (!s.ok()) { 60 | Usage(); 61 | log_err("Compact Failed : %s", s.ToString().c_str()); 62 | } else { 63 | log_info("Compact Finshed"); 64 | } 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /tools/compact/xdebug.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdebug.h 3 | * @brief debug macros 4 | * @author chenzongzhi 5 | * @version 1.0.0 6 | * @date 2014-04-25 7 | */ 8 | 9 | #ifndef __XDEBUG_H_ 10 | #define __XDEBUG_H_ 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #ifdef __XDEBUG__ 18 | #define qf_debug(fmt, arg...) \ 19 | { \ 20 | fprintf(stderr, "[----------debug--------][%s:%d]" fmt "\n", __FILE__, __LINE__, ##arg); \ 21 | } 22 | #define pint(x) qf_debug("%s = %d", #x, x) 23 | #define psize(x) qf_debug("%s = %zu", #x, x) 24 | #define pstr(x) qf_debug("%s = %s", #x, x) 25 | // 如果A 不对, 那么就输出M 26 | #define qf_check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; exit(-1);} 27 | 28 | // 用来检测程序是否执行到这里 29 | #define sentinel(M, ...) { qf_debug(M, ##__VA_ARGS__); errno=0;} 30 | 31 | #define qf_bin_debug(buf, size) \ 32 | { \ 33 | fwrite(buf, 1, size, stderr); \ 34 | } 35 | 36 | #define _debug_time_def timeval s1, e; 37 | #define _debug_getstart gettimeofday(&s1, NULL) 38 | #define _debug_getend gettimeofday(&e, NULL) 39 | #define _debug_time ((int)(((e.tv_sec - s1.tv_sec) * 1000 + (e.tv_usec - s1.tv_usec) / 1000))) 40 | 41 | #define clean_errno() (errno == 0 ? "None" : strerror(errno)) 42 | #define log_err(M, ...) \ 43 | { \ 44 | fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__); \ 45 | exit(-1); \ 46 | } 47 | #define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 48 | #define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 49 | 50 | #else 51 | 52 | #define qf_debug(fmt, arg...) {} 53 | #define pint(x) {} 54 | #define pstr(x) {} 55 | #define qf_bin_debug(buf, size) {} 56 | 57 | #define _debug_time_def {} 58 | #define _debug_getstart {} 59 | #define _debug_getend {} 60 | #define _debug_time 0 61 | 62 | #define sentinel(M, ...) {} 63 | #define qf_check(A, M, ...) {} 64 | #define log_err(M, ...) {} 65 | #define log_warn(M, ...) {} 66 | #define log_info(M, ...) {} 67 | 68 | #endif 69 | 70 | #define qf_error(fmt, arg...) \ 71 | { \ 72 | fprintf(stderr, "[%ld][%ld][%s:%d]" fmt "\n", (long)getpid(), (long)pthread_self(), __FILE__, __LINE__, ##arg); \ 73 | fflush(stderr);\ 74 | exit(-1);\ 75 | } 76 | 77 | 78 | #endif //__XDEBUG_H_ 79 | 80 | /* vim: set ts=4 sw=4 sts=4 tw=100 */ 81 | -------------------------------------------------------------------------------- /tools/meta_scan/Makefile: -------------------------------------------------------------------------------- 1 | GCC = g++ 2 | CPPFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -Wno-unused-parameter -DDEBUG -D__XDEBUG__ -g -O2 -std=c++11 3 | OBJECT = meta_scan 4 | 5 | LIB_PATH = -L ../../output/lib 6 | 7 | LIBS = -Wl,-Bstatic -lnemo -lnemodb -lrocksdb \ 8 | -Wl,-Bdynamic -lpthread\ 9 | -lsnappy \ 10 | -lrt \ 11 | -lz \ 12 | -lbz2 13 | 14 | INCLUDE_PATH = -I../../output/include/ \ 15 | -I../../3rdparty/nemo-rocksdb/rocksdb/ \ 16 | -I../../3rdparty/nemo-rocksdb/rocksdb/include 17 | 18 | .PHONY: all clean 19 | 20 | 21 | # BASE_BOJS := $(wildcard *.cpp) 22 | # BASE_BOJS += $(wildcard *.c) 23 | # OBJS := $(patsubst %.cpp,%.o,$(BASE_BOJS)) 24 | 25 | 26 | all: $(OBJECT) 27 | rm *.o 28 | 29 | $(OBJECT): $(OBJECT).o 30 | $(GCC) $(CPPFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 31 | 32 | %.o : %.cc 33 | $(GCC) $(CPPFLAGS) -c $< -o $@ $(INCLUDE_PATH) 34 | 35 | clean: 36 | rm -rf $(OBJECT) $(OBJECT).o 37 | -------------------------------------------------------------------------------- /tools/meta_scan/README: -------------------------------------------------------------------------------- 1 | Tool for Meta Scan 2 | 3 | Usage: 4 | ./meta_scan db_path type pattern 5 | type is one of: kv, hash, list, zset, set, all 6 | Example: 7 | ./meta_scan ./db list \* 8 | -------------------------------------------------------------------------------- /tools/meta_scan/meta_scan.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "xdebug.h" 4 | #include 5 | #include "nemo.h" 6 | 7 | using namespace nemo; 8 | 9 | void Usage() { 10 | std::cout << "Usage: " << std::endl; 11 | std::cout << "./meta_scan db_path type pattern" << std::endl; 12 | std::cout << "type is one of: kv, hash, list, zset, set, all" << std::endl; 13 | std::cout << "Example: " << std::endl; 14 | std::cout << "./meta_scan ./db list \\*" << std::endl; 15 | } 16 | 17 | void PrintMetaSpecify(nemo::Nemo *const db, DBType type, 18 | const std::string& type_name, const std::string& pattern) { 19 | // Scan metas info 20 | std::map metas; 21 | Status s = db->ScanMetasSpecify(type, pattern, metas); 22 | if (!s.ok()) { 23 | log_err("Failed to scan metas"); 24 | return; 25 | } 26 | // Print 27 | std::cout << "---------------- Begin Scan[" << type_name << "] ----------------" << std::endl; 28 | std::map::iterator it = metas.begin(); 29 | for (; it != metas.end(); ++it) { 30 | std::cout << "Key[" << it->first 31 | << "], Metas[" << it->second->ToString() << "]" << std::endl; 32 | } 33 | std::cout << "---------------- End Scan[" << type_name << "] ----------------" << std::endl; 34 | } 35 | 36 | void PrintMeta(nemo::Nemo *const db, const std::string& type, 37 | const std::string& pattern) { 38 | bool all = false; 39 | if (type == "all") { 40 | all = true; 41 | } 42 | if (all || type == "hash") { 43 | PrintMetaSpecify(db, kHASH_DB, "hash", pattern); 44 | } 45 | if (all || type == "list") { 46 | PrintMetaSpecify(db, kLIST_DB, "list", pattern); 47 | } 48 | if (all || type == "set") { 49 | PrintMetaSpecify(db, kSET_DB, "set", pattern); 50 | } 51 | if (all || type == "zset") { 52 | PrintMetaSpecify(db, kZSET_DB, "zset", pattern); 53 | } 54 | } 55 | 56 | 57 | int main(int argc, char **argv) 58 | { 59 | if (argc != 4) { 60 | Usage(); 61 | log_err("not enough parameter"); 62 | } 63 | std::string path(argv[1]); 64 | std::string db_type(argv[2]); 65 | std::string pattern(argv[3]); 66 | 67 | if (db_type != "hash" && db_type != "list" 68 | && db_type != "set" && db_type != "zset" 69 | && db_type != "all") { 70 | Usage(); 71 | log_err("invalid type parameter"); 72 | } 73 | // Create nemo handle 74 | nemo::Options option; 75 | option.write_buffer_size = 268435456; 76 | option.target_file_size_base = 20971520; 77 | log_info("Prepare DB..."); 78 | nemo::Nemo* db = new nemo::Nemo(path, option); 79 | assert(db); 80 | log_info("SCan Begin"); 81 | 82 | PrintMeta(db, db_type, pattern); 83 | delete db; 84 | 85 | log_info("SCan Finshed"); 86 | return 0; 87 | } 88 | -------------------------------------------------------------------------------- /tools/meta_scan/xdebug.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdebug.h 3 | * @brief debug macros 4 | * @author chenzongzhi 5 | * @version 1.0.0 6 | * @date 2014-04-25 7 | */ 8 | 9 | #ifndef __XDEBUG_H_ 10 | #define __XDEBUG_H_ 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #ifdef __XDEBUG__ 18 | #define qf_debug(fmt, arg...) \ 19 | { \ 20 | fprintf(stderr, "[----------debug--------][%s:%d]" fmt "\n", __FILE__, __LINE__, ##arg); \ 21 | } 22 | #define pint(x) qf_debug("%s = %d", #x, x) 23 | #define psize(x) qf_debug("%s = %zu", #x, x) 24 | #define pstr(x) qf_debug("%s = %s", #x, x) 25 | // 如果A 不对, 那么就输出M 26 | #define qf_check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; exit(-1);} 27 | 28 | // 用来检测程序是否执行到这里 29 | #define sentinel(M, ...) { qf_debug(M, ##__VA_ARGS__); errno=0;} 30 | 31 | #define qf_bin_debug(buf, size) \ 32 | { \ 33 | fwrite(buf, 1, size, stderr); \ 34 | } 35 | 36 | #define _debug_time_def timeval s1, e; 37 | #define _debug_getstart gettimeofday(&s1, NULL) 38 | #define _debug_getend gettimeofday(&e, NULL) 39 | #define _debug_time ((int)(((e.tv_sec - s1.tv_sec) * 1000 + (e.tv_usec - s1.tv_usec) / 1000))) 40 | 41 | #define clean_errno() (errno == 0 ? "None" : strerror(errno)) 42 | #define log_err(M, ...) \ 43 | { \ 44 | fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__); \ 45 | exit(-1); \ 46 | } 47 | #define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 48 | #define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 49 | 50 | #else 51 | 52 | #define qf_debug(fmt, arg...) {} 53 | #define pint(x) {} 54 | #define pstr(x) {} 55 | #define qf_bin_debug(buf, size) {} 56 | 57 | #define _debug_time_def {} 58 | #define _debug_getstart {} 59 | #define _debug_getend {} 60 | #define _debug_time 0 61 | 62 | #define sentinel(M, ...) {} 63 | #define qf_check(A, M, ...) {} 64 | #define log_err(M, ...) {} 65 | #define log_warn(M, ...) {} 66 | #define log_info(M, ...) {} 67 | 68 | #endif 69 | 70 | #define qf_error(fmt, arg...) \ 71 | { \ 72 | fprintf(stderr, "[%ld][%ld][%s:%d]" fmt "\n", (long)getpid(), (long)pthread_self(), __FILE__, __LINE__, ##arg); \ 73 | fflush(stderr);\ 74 | exit(-1);\ 75 | } 76 | 77 | 78 | #endif //__XDEBUG_H_ 79 | 80 | /* vim: set ts=4 sw=4 sts=4 tw=100 */ 81 | -------------------------------------------------------------------------------- /tools/nemo_performance_test/\: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "nemo.h" 14 | #include "xdebug.h" 15 | 16 | #include "time_keeper.h" 17 | #include "random_string.h" 18 | #include "config.h" 19 | using namespace nemo; 20 | using std::vector; 21 | 22 | void usage(){ 23 | std::cout<<"Usage: nemo-benchmark: -t [-c ] [-r ] [-n ]\n"<< 24 | " -c number of clients(default 1)\n"<< 25 | " -r number of random keys&values (default 1000)\n"<< 26 | " -n total number of requests (default 100000) \n"<< 27 | " -d size of SET/GET key in bytes (default 3)\n"<< 28 | " -s size of SET/GET value in bytes (default 3)\n"<< 29 | " -t commend that you want to test\n"; 30 | exit(0); 31 | } 32 | 33 | void parseOption(int argc,char *argv[],config &conf){ 34 | if(argc%2==0){ 35 | usage(); 36 | } 37 | 38 | std::map cmdMap; 39 | for(int j =0; j(cmd[j],j)); 41 | } 42 | 43 | for(int i=1;i::iterator it_m=cmdMap.find(op); 75 | if(it_m!=cmdMap.end()){ 76 | conf.ops.push_back(it_m->second); 77 | op=""; 78 | } 79 | else if(op.size()>0){ 80 | std::cout<<"can't find cmd:"< &kv,const std::vector &key,const std::vector &value){ 97 | std::vector::iterator it_k=key.begin(); 98 | std::vector::iterator it_v=value.begin(); 99 | 100 | while(it_k!=key.end()&&it_v!=value.end()){ 101 | nemo::KV temp; 102 | temp.key = *it_k; 103 | temp.vlaue=*it_v; 104 | kv.push_back(temp); 105 | } 106 | } 107 | 108 | 109 | int main(int argc, char* argv[]) 110 | { 111 | int numOfKeys=1000,numOfClients=1,numOfRequests =1000; 112 | int playLoad=3,dataSize=3; 113 | int m=10; 114 | 115 | config conf; 116 | conf.numOfClients=numOfClients; 117 | conf.numOfRequests=numOfRequests; 118 | conf.playLoad=playLoad; 119 | conf.finishedRequests=0; 120 | conf.valueSize=dataSize; 121 | conf.numOfKeys=numOfKeys; 122 | conf.m=m; 123 | parseOption(argc,argv,conf); 124 | conf.latency=vector(conf.numOfRequests+3*conf.numOfClients,0); 125 | 126 | nemo::Options options; 127 | options.target_file_size_base=20*1024*1024; 128 | nemo::Nemo *np=new nemo::Nemo("./tmp",options); 129 | 130 | random_string rs; 131 | conf.key.reserve(conf.numOfKeys); 132 | conf.value.reserve(conf.numOfKeys); 133 | conf.field.reserve(conf.numOfKeys); 134 | conf.kv.reserve(conf.numOfKeys); 135 | conf.ttl.reserve(conf.numOfKeys); 136 | rs.getRandomKeyValues(conf.key,conf.value,conf.playLoad,conf.valueSize,conf.numOfKeys); 137 | // rs.getRandomKVs(conf.kv,conf.playLoad,conf.valueSize,conf.numOfKeys); 138 | getKVs(conf.key,conf.value); 139 | rs.getRandomFields(conf.field,conf.playLoad,conf.numOfKeys); 140 | rs.getRandomInts(conf.ttl,conf.numOfKeys); 141 | 142 | for(int i=0;i 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "nemo.h" 16 | #include "xdebug.h" 17 | 18 | #include "time_keeper.h" 19 | #include "random_string.h" 20 | #include "config.h" 21 | using namespace nemo; 22 | 23 | 24 | void GetKVs(std::vector &kv, const std::vector &key, const std::vector &value) { 25 | std::vector::const_iterator it_k = key.begin(); 26 | std::vector::const_iterator it_v = value.begin(); 27 | 28 | while (it_k != key.end() && it_v != value.end()) { 29 | nemo::KV temp; 30 | temp.key = *it_k++; 31 | temp.val = *it_v++; 32 | kv.push_back(temp); 33 | } 34 | } 35 | 36 | int main(int argc, char* argv[]) 37 | { 38 | 39 | std::vector num_of_keys({10000, 100000}); 40 | std::vector key_size({20}); 41 | std::vector value_size({10, 1000, 1000000}); 42 | std::vector clients({10, 20, 40}); 43 | std::vector requests({1000000}); 44 | std::vector opsv({10, 100}); 45 | std::vector op({ "set", "get", "mset", "mget", 46 | "hset", "hget", 47 | "lset", "lpush", "lpop", 48 | "zadd", "zcard", "zrem", 49 | "sadd", "srem" 50 | }); 51 | std::map cmd_map; 52 | for(int j =0; j < cmdLen; j++) { 53 | cmd_map.insert(std::pair < std::string,int>(cmd[j], j)); 54 | } 55 | 56 | //start..... 57 | for (int h = 0; h < op.size(); h++) { //operation 58 | for (int i = 0; i < clients.size(); i++) { //clients 59 | for (int j = 0; j < value_size.size(); j++) { //valueSize 60 | for (int k = 0; k < key_size.size(); k++) { //keySize 61 | int flag=1; 62 | if(op[h].compare("mset")==0||op[h].compare("mget")==0) 63 | flag=0; 64 | for (int m = 0; m < opsv.size(); m++) { //specific parameter for some operations such as mset,mget, etc. 65 | for (int n = 0; n < requests.size(); n++) { //num of requests 66 | for (int p = 0; p < num_of_keys.size(); p++) { 67 | //shell command 68 | std::system("cp -r ./tmp ./tmp2"); 69 | 70 | //parameter configure 71 | 72 | Config conf; 73 | conf.folder = "./record/"; 74 | conf.op = cmd_map.find(op[h])->second; 75 | conf.num_of_clients = clients[i]; 76 | conf.value_size = value_size[j]; 77 | conf.key_size = key_size[k]; 78 | conf.m = opsv[m]; 79 | conf.num_of_requests = requests[n]; 80 | conf.num_of_keys = num_of_keys[p]; 81 | 82 | 83 | if (conf.value_size == 1000000) conf.num_of_keys = 10000; 84 | conf.finished_requests = 0; 85 | conf.latency = std::vector(conf.num_of_requests + 3 * conf.num_of_clients, 0); 86 | RandomString rs; 87 | conf.key.reserve(conf.num_of_keys); 88 | conf.value.reserve(conf.num_of_keys); 89 | conf.field.reserve(conf.num_of_keys); 90 | conf.kv.reserve(conf.num_of_keys); 91 | conf.ttl.reserve(conf.num_of_keys); 92 | 93 | //random strings 94 | rs.GetRandomKeyValues(conf.key, conf.value, conf.key_size, conf.value_size, conf.num_of_keys); 95 | GetKVs(conf.kv,conf.key,conf.value); 96 | rs.GetRandomFields(conf.field,conf.key_size,conf.num_of_keys); 97 | rs.GetRandomInts(conf.ttl,conf.num_of_keys); 98 | //over! 99 | std::cout<<"================================\n"; 100 | std::cout<<"random string over\n"< /proc/sys/vm/drop_caches"); 129 | 130 | } 131 | } 132 | } 133 | } 134 | } 135 | } 136 | } 137 | exit(0); 138 | } 139 | 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /tools/nemo_performance_test/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nemo.h" 9 | #include "xdebug.h" 10 | 11 | #define cmdLen 89 12 | 13 | using namespace nemo; 14 | 15 | static std::string cmd[cmdLen] = {"set","setttl","get","mset","mget","kmdel","incrby","decrby","incrybyfloat","getset", 16 | "append","setnx","setxx","msetnx","getrange","setrange","strlen","kscan","scan","keys", 17 | "bitset","bitget","bitcount","bitpos","bitop","setwithexpireat","hset","hget","hdel","hexists", 18 | "hkeys","hgetall","hlen","hmset","hmget","hsetnx","hstrlen","hscan","hvals","hincrby", 19 | "hincrbyfloat","lindex","llen","lpush","lpop","lpushx","lrange","lset","ltrim","rpush", 20 | "rpop","rpushx","rpoplpush","linsert","lrem","zadd","zcard","zcount","zscan","zincrby", 21 | "zrange","zunionstore","zinterstore","zrangebyscore","zrem","zrank","zrevrank","zscore","zremrangebylex","zremrangebyrank", 22 | "zremrangebyscore","sadd","srem","scard","sscan","smembers","sunionstore","sunion","sinterstore","sinter", 23 | "sdifferstore","sdiff","sismember","spop","srandmember","smove","pfadd","pfcount","pfmerge" 24 | }; 25 | 26 | typedef struct Config { 27 | int32_t num_of_requests; 28 | int32_t num_of_clients; 29 | int64_t total_latency; 30 | int32_t key_size; 31 | int32_t value_size; 32 | int32_t num_of_keys; 33 | int32_t finished_requests; 34 | int32_t op; 35 | int32_t m; 36 | std::string folder; 37 | 38 | std::vector ops; 39 | std::vector latency; 40 | std::vector key; 41 | std::vector value; 42 | std::vector field; 43 | std::vector kv; 44 | std::vector ttl; 45 | } Config; 46 | 47 | typedef struct Client { 48 | //the index of the client 49 | int32_t no; 50 | int32_t requests; 51 | int32_t finished_requests; 52 | //the latency of every request sent by this client 53 | std::vector latency; 54 | Client(int index, int n): 55 | no(index), 56 | requests(n), 57 | finished_requests(0) { 58 | latency.reserve(n); 59 | } 60 | } Client; 61 | 62 | void ShowReports(const Config &conf, std::string file_name); 63 | void ClientOP(nemo::Nemo *np, Config *conf, Client *ct); 64 | void TestOPs(nemo::Nemo *np, Config *conf); 65 | void Reset(Config &conf); 66 | 67 | 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /tools/nemo_performance_test/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "nemo.h" 14 | #include "xdebug.h" 15 | 16 | #include "time_keeper.h" 17 | #include "random_string.h" 18 | #include "config.h" 19 | using namespace nemo; 20 | 21 | void Usage() { 22 | std::cout<<"Usage: nemo-benchmark: -t [-c ] [-r ] [-n ]\n"<< 23 | " -c number of clients(default 1)\n"<< 24 | " -r number of random keys&values (default 1000)\n"<< 25 | " -n total number of requests (default 100000) \n"<< 26 | " -d size of SET/GET key in bytes (default 3)\n"<< 27 | " -s size of SET/GET value in bytes (default 3)\n"<< 28 | " -t commend that you want to test\n"; 29 | exit(0); 30 | } 31 | 32 | void ParseOption(int argc,char *argv[],Config &conf) { 33 | if (argc%2 == 0) { 34 | Usage(); 35 | } 36 | 37 | std::map cmdMap; 38 | for (int j = 0; j < cmdLen; j++) { 39 | cmdMap.insert(std::pair(cmd[j],j)); 40 | } 41 | 42 | for (int i = 1; i < argc; i += 2) { 43 | std::string para1=argv[i]; 44 | std::string para2=argv[i+1]; 45 | if (para1.size() != 2 || para1[0] != '-') { 46 | Usage(); 47 | } 48 | 49 | switch (para1[1]) { 50 | case 'c': conf.num_of_clients = atoi(argv[i+1]); 51 | break; 52 | case 'm': conf.m = atoi(argv[i+1]); 53 | break; 54 | case 'r': conf.num_of_keys = atoi(argv[i+1]); 55 | break; 56 | case 'n': conf.num_of_requests = atoi(argv[i+1]); 57 | break; 58 | case 'd': conf.key_size = atoi(argv[i+1]); 59 | break; 60 | case 's': conf.value_size = atoi(argv[i+1]); 61 | break; 62 | case 't': { 63 | std::string op; 64 | for (unsigned int j = 0;j < para2.size(); j++) { 65 | if (para2[j] != ',' && j < para2.size()-1) { 66 | op += para2[j]; 67 | } 68 | else { 69 | if(j == para2.size()-1) { 70 | op+=para2[j]; 71 | } 72 | // std::transform(op.begin(), op.end(), op.begin(), std::tolower); 73 | std::map::iterator it_m=cmdMap.find(op); 74 | if (it_m != cmdMap.end() ) { 75 | conf.ops.push_back(it_m->second); 76 | op=""; 77 | } else if (op.size() > 0) { 78 | std::cout<<"can't find cmd:"< &kv, const std::vector &key, const std::vector &value) { 95 | std::vector::const_iterator it_k=key.begin(); 96 | std::vector::const_iterator it_v=value.begin(); 97 | 98 | while (it_k != key.end() && it_v != value.end()) { 99 | nemo::KV temp; 100 | temp.key = *it_k++; 101 | temp.val = *it_v++; 102 | kv.push_back(temp); 103 | } 104 | } 105 | 106 | 107 | int main(int argc, char* argv[]) { 108 | 109 | int num_of_keys = 1000, num_of_clients = 1, num_of_requests = 1000; 110 | int key_size = 3, value_size = 3; 111 | int m = 10; 112 | 113 | Config conf; 114 | conf.num_of_clients = num_of_clients; 115 | conf.num_of_requests = num_of_requests; 116 | conf.key_size = key_size; 117 | conf.finished_requests = 0; 118 | conf.value_size = value_size; 119 | conf.num_of_keys = num_of_keys; 120 | conf.m = m; 121 | ParseOption(argc,argv,conf); 122 | conf.latency=std::vector(conf.num_of_requests+3*conf.num_of_clients,0); 123 | 124 | 125 | RandomString rs; 126 | conf.key.reserve(conf.num_of_keys); 127 | conf.value.reserve(conf.num_of_keys); 128 | conf.field.reserve(conf.num_of_keys); 129 | conf.kv.reserve(conf.num_of_keys); 130 | conf.ttl.reserve(conf.num_of_keys); 131 | rs.GetRandomKeyValues(conf.key, conf.value, conf.key_size, conf.value_size, conf.num_of_keys); 132 | GetKVs(conf.kv, conf.key, conf.value); 133 | rs.GetRandomFields(conf.field, conf.key_size, conf.num_of_keys); 134 | rs.GetRandomInts(conf.ttl, conf.num_of_keys); 135 | 136 | nemo::Options options; 137 | options.target_file_size_base = 20*1024*1024; 138 | nemo::Nemo *np = new nemo::Nemo("./tmp",options); 139 | 140 | for (unsigned int i = 0;i < conf.ops.size(); i++) { 141 | conf.op=conf.ops[i]; 142 | 143 | TestOPs(np,&conf); 144 | 145 | ShowReports(conf,"record"); 146 | Reset(conf); 147 | } 148 | 149 | delete np; 150 | exit(0); 151 | } 152 | 153 | 154 | -------------------------------------------------------------------------------- /tools/nemo_performance_test/random_string.h: -------------------------------------------------------------------------------- 1 | #ifndef RANDOM_STRING_H 2 | #define RANDOM_STRING_H 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "nemo_const.h" 10 | 11 | class RandomString { 12 | public: 13 | RandomString() { 14 | Usrand(); 15 | } 16 | ~RandomString() {} 17 | 18 | std::string GetRandomChars(const int length) { 19 | //Usrand(); 20 | std::string randomStr; 21 | for (int index = 0; index < length; index++) { 22 | randomStr += charSet[random() % charSetLen_]; 23 | } 24 | return randomStr; 25 | } 26 | 27 | std::string GetRandomChars(const int min,const int max) { 28 | //Usrand(); 29 | int len=min + random() % (max - min + 1); 30 | return GetRandomChars(len); 31 | } 32 | 33 | int32_t GetRandomInt() { 34 | // Usrand(); 35 | return random(); 36 | } 37 | 38 | std:: string GetRandomBytes(const int length) { 39 | //Usrand(); 40 | std::string randomStr; 41 | for (int index = 0; index < length; index++) 42 | { 43 | randomStr += (char)(random() % 256); 44 | } 45 | return randomStr; 46 | } 47 | 48 | std::string GetRandomBytes(const int min,const int max) { 49 | //Usrand(); 50 | int len = min + random() % ( max - min + 1); 51 | return GetRandomBytes(len); 52 | } 53 | 54 | std::string GetRandomKey() { 55 | return GetRandomBytes(minKeyLen_, maxKeyLen_); 56 | } 57 | 58 | std::string GetRandomVal() { 59 | return GetRandomBytes(minValLen_, maxValLen_); 60 | } 61 | 62 | std::string GetRandomField() { 63 | return GetRandomBytes(minFieldLen_, maxFieldLen_); 64 | } 65 | 66 | std::string GetRandomKey(int length) { 67 | return GetRandomBytes(length); 68 | } 69 | 70 | std::string GetRandomVal(int length) { 71 | return GetRandomBytes(length); 72 | } 73 | 74 | std::string GetRandomField(int length) { 75 | return GetRandomBytes(length); 76 | } 77 | 78 | std::vector& GetRandomKeys(std::vector&v_key, int key_size, int n) { 79 | for(int i=0;i& GetRandomKeys(std::vector &v_key,int n){ 86 | for(int i=0;i& GetRandomVals(std::vector &v_value, int value_size, int n) { 94 | for(int i=0; i < n; i++) { 95 | v_value.push_back(GetRandomVal(value_size)); 96 | } 97 | return v_value; 98 | } 99 | 100 | std::vector& GetRandomVals(std::vector &v_value, int n) { 101 | for(int i = 0;i < n; i++) { 102 | v_value.push_back(GetRandomVal()); 103 | } 104 | return v_value; 105 | } 106 | 107 | std::vector& GetRandomFields(std::vector &v_field, int field_size, int n) { 108 | for(int i = 0; i < n; i++) { 109 | v_field.push_back(GetRandomField(field_size)); 110 | } 111 | return v_field; 112 | } 113 | 114 | std::vector& GetRandomFields(std::vector &v_field, int n) { 115 | for(int i = 0 ;i < n ; i++) { 116 | v_field.push_back(GetRandomField()); 117 | } 118 | return v_field; 119 | } 120 | 121 | void GetRandomInts(std::vector &v_int, int n) { 122 | // Usrand(); 123 | for(int i = 0;i < n; i++) { 124 | v_int.push_back(random()); 125 | } 126 | } 127 | 128 | void GetRandomKeyValue(std::string &key, std::string &value) { 129 | key=GetRandomKey(); 130 | value=GetRandomVal(); 131 | } 132 | 133 | void GetRandomKeyValues(std::vector &v_key,std::vector &v_value, int key_size, int value_size, int n) { 134 | for (int i = 0;i < n; i++) { 135 | v_key.push_back(GetRandomKey(key_size)); 136 | v_value.push_back(GetRandomVal(value_size)); 137 | } 138 | } 139 | 140 | void GetRandomKeyValues(std::vector &v_key, std::vector &v_value, int n) { 141 | for (int i = 0;i < n; i++) { 142 | v_key.push_back(GetRandomKey()); 143 | v_value.push_back(GetRandomVal()); 144 | } 145 | } 146 | 147 | void GetRandomKVs(std::vector &kv, int key_size, int value_size, int n) { 148 | for (int i = 0; i < n; i++) { 149 | nemo::KV temp; 150 | temp.key = GetRandomKey(key_size); 151 | temp.val = GetRandomVal(value_size); 152 | kv.push_back(temp); 153 | } 154 | 155 | } 156 | 157 | void Usrand() { 158 | struct timeval now; 159 | gettimeofday(&now,NULL); 160 | srand(now.tv_usec); 161 | } 162 | 163 | private: 164 | 165 | static const int charsSetLen_ = 62; 166 | std::string charSet="ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz"; 167 | static const int maxKeyLen_ = 254; 168 | static const int minKeyLen_ = 0; 169 | static const int maxFieldLen_ = 1024000; 170 | static const int minFieldLen_ = 0; 171 | static const int maxValLen_ = 1024000; 172 | static const int minValLen_ = 1; 173 | static const int charSetLen_ = 62; 174 | }; 175 | 176 | 177 | 178 | 179 | 180 | 181 | #endif 182 | 183 | -------------------------------------------------------------------------------- /tools/nemo_performance_test/time_keeper.h: -------------------------------------------------------------------------------- 1 | #ifndef TIME_KEEPER_H 2 | #define TIME_KEEPER_H 3 | #include 4 | #include 5 | #include 6 | 7 | class TimeKeeper{ 8 | public: 9 | TimeKeeper() { 10 | gettimeofday(&time_start,NULL); 11 | time_end = time_start; 12 | } 13 | 14 | ~TimeKeeper() { } 15 | void Restart() { 16 | gettimeofday(&time_start,NULL); 17 | } 18 | 19 | //latency in millisecond 20 | int64_t DelayMs() { 21 | gettimeofday(&time_end,NULL); 22 | int64_t total_micros = (time_end.tv_sec - time_start.tv_sec) * 1000 + (time_end.tv_usec - time_start.tv_usec) / 1000; 23 | return total_micros; 24 | } 25 | 26 | //latency in second 27 | int64_t DelaySe() { 28 | gettimeofday(&time_end,NULL); 29 | int64_t total_s = time_end.tv_sec - time_start.tv_sec; 30 | return total_s; 31 | } 32 | 33 | //latency in microsecond 34 | int64_t DelayUs() { 35 | gettimeofday(&time_end,NULL); 36 | int64_t total_us = (time_end.tv_sec-time_start.tv_sec)*1000000+( time_end.tv_usec - time_start.tv_usec); 37 | return total_us; 38 | } 39 | 40 | private: 41 | 42 | struct timeval time_start; 43 | struct timeval time_end; 44 | 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /tools/nemock/Makefile: -------------------------------------------------------------------------------- 1 | GCC = g++ 2 | CPPFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -Wno-unused-parameter -DDEBUG -D__XDEBUG__ -g -O2 -std=c++11 3 | OBJECT = nemock 4 | 5 | LIB_PATH = -L ../../output/lib 6 | 7 | LIBS = -Wl,-Bstatic -lnemo -lnemodb -lrocksdb \ 8 | -Wl,-Bdynamic -lpthread\ 9 | -lsnappy \ 10 | -lrt \ 11 | -lz \ 12 | -lbz2 13 | 14 | INCLUDE_PATH = -I../../output/include/ \ 15 | -I../../3rdparty/nemo-rocksdb/rocksdb/ \ 16 | -I../../3rdparty/nemo-rocksdb/rocksdb/include 17 | 18 | .PHONY: all clean 19 | 20 | 21 | # BASE_BOJS := $(wildcard *.cpp) 22 | # BASE_BOJS += $(wildcard *.c) 23 | # OBJS := $(patsubst %.cpp,%.o,$(BASE_BOJS)) 24 | 25 | 26 | all: $(OBJECT) 27 | rm *.o 28 | 29 | $(OBJECT): $(OBJECT).o 30 | $(GCC) $(CPPFLAGS) -o $@ $^ $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 31 | 32 | %.o : %.cc 33 | $(GCC) $(CPPFLAGS) -c $< -o $@ $(INCLUDE_PATH) 34 | 35 | clean: 36 | rm -rf $(OBJECT) $(OBJECT).o 37 | -------------------------------------------------------------------------------- /tools/nemock/README: -------------------------------------------------------------------------------- 1 | Tool for Nemo Check 2 | 3 | Usage: 4 | ./nemock db_path type pattern 5 | type is one of: kv, hash, list, zset, set, all 6 | Example: 7 | ./nemock ./db list \* 8 | -------------------------------------------------------------------------------- /tools/nemock/nemock.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "xdebug.h" 4 | #include 5 | #include "nemo.h" 6 | 7 | using namespace nemo; 8 | 9 | void Usage() { 10 | std::cout << "Usage: " << std::endl; 11 | std::cout << "./nemock db_path type pattern" << std::endl; 12 | std::cout << "type is one of: kv, hash, list, zset, set, all" << std::endl; 13 | std::cout << "Example: " << std::endl; 14 | std::cout << "./nemock ./db list \\*" << std::endl; 15 | } 16 | 17 | void ChecknRecoverSpecify(nemo::Nemo *const db, DBType type, 18 | const std::string& type_name, const std::string& pattern) { 19 | Status s = db->CheckMetaSpecify(type, pattern); 20 | if (!s.ok()) { 21 | log_err("Check and Recover %s failed : %s", type_name.c_str(), s.ToString().c_str()); 22 | } 23 | log_info("Check and Recover %s success", type_name.c_str()); 24 | } 25 | 26 | 27 | void ChecknRecover(nemo::Nemo *const db, const std::string& type, 28 | const std::string& pattern) { 29 | bool all = false; 30 | if (type == "all") { 31 | all = true; 32 | } 33 | if (all || type == "hash") { 34 | ChecknRecoverSpecify(db, kHASH_DB, type, pattern); 35 | } 36 | if (all || type == "list") { 37 | ChecknRecoverSpecify(db, kLIST_DB, type, pattern); 38 | } 39 | if (all || type == "set") { 40 | ChecknRecoverSpecify(db, kSET_DB, type, pattern); 41 | } 42 | if (all || type == "zset") { 43 | ChecknRecoverSpecify(db, kZSET_DB, type, pattern); 44 | } 45 | } 46 | 47 | 48 | int main(int argc, char **argv) 49 | { 50 | if (argc != 4) { 51 | Usage(); 52 | log_err("not enough parameter"); 53 | } 54 | std::string path(argv[1]); 55 | std::string db_type(argv[2]); 56 | std::string pattern(argv[3]); 57 | 58 | if (db_type != "hash" && db_type != "list" 59 | && db_type != "set" && db_type != "zset" 60 | && db_type != "all") { 61 | Usage(); 62 | log_err("invalid type parameter"); 63 | } 64 | // Create nemo handle 65 | nemo::Options option; 66 | option.write_buffer_size = 268435456; 67 | option.target_file_size_base = 20971520; 68 | log_info("Prepare DB..."); 69 | nemo::Nemo* db = new nemo::Nemo(path, option); 70 | assert(db); 71 | log_info("Check and Recover Begin"); 72 | ChecknRecover(db, db_type, pattern); 73 | delete db; 74 | log_info("Chekc and Recover Finshed"); 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /tools/nemock/xdebug.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdebug.h 3 | * @brief debug macros 4 | * @author chenzongzhi 5 | * @version 1.0.0 6 | * @date 2014-04-25 7 | */ 8 | 9 | #ifndef __XDEBUG_H_ 10 | #define __XDEBUG_H_ 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #ifdef __XDEBUG__ 18 | #define qf_debug(fmt, arg...) \ 19 | { \ 20 | fprintf(stderr, "[----------debug--------][%s:%d]" fmt "\n", __FILE__, __LINE__, ##arg); \ 21 | } 22 | #define pint(x) qf_debug("%s = %d", #x, x) 23 | #define psize(x) qf_debug("%s = %zu", #x, x) 24 | #define pstr(x) qf_debug("%s = %s", #x, x) 25 | // 如果A 不对, 那么就输出M 26 | #define qf_check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; exit(-1);} 27 | 28 | // 用来检测程序是否执行到这里 29 | #define sentinel(M, ...) { qf_debug(M, ##__VA_ARGS__); errno=0;} 30 | 31 | #define qf_bin_debug(buf, size) \ 32 | { \ 33 | fwrite(buf, 1, size, stderr); \ 34 | } 35 | 36 | #define _debug_time_def timeval s1, e; 37 | #define _debug_getstart gettimeofday(&s1, NULL) 38 | #define _debug_getend gettimeofday(&e, NULL) 39 | #define _debug_time ((int)(((e.tv_sec - s1.tv_sec) * 1000 + (e.tv_usec - s1.tv_usec) / 1000))) 40 | 41 | #define clean_errno() (errno == 0 ? "None" : strerror(errno)) 42 | #define log_err(M, ...) \ 43 | { \ 44 | fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__); \ 45 | exit(-1); \ 46 | } 47 | #define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 48 | #define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 49 | 50 | #else 51 | 52 | #define qf_debug(fmt, arg...) {} 53 | #define pint(x) {} 54 | #define pstr(x) {} 55 | #define qf_bin_debug(buf, size) {} 56 | 57 | #define _debug_time_def {} 58 | #define _debug_getstart {} 59 | #define _debug_getend {} 60 | #define _debug_time 0 61 | 62 | #define sentinel(M, ...) {} 63 | #define qf_check(A, M, ...) {} 64 | #define log_err(M, ...) {} 65 | #define log_warn(M, ...) {} 66 | #define log_info(M, ...) {} 67 | 68 | #endif 69 | 70 | #define qf_error(fmt, arg...) \ 71 | { \ 72 | fprintf(stderr, "[%ld][%ld][%s:%d]" fmt "\n", (long)getpid(), (long)pthread_self(), __FILE__, __LINE__, ##arg); \ 73 | fflush(stderr);\ 74 | exit(-1);\ 75 | } 76 | 77 | 78 | #endif //__XDEBUG_H_ 79 | 80 | /* vim: set ts=4 sw=4 sts=4 tw=100 */ 81 | --------------------------------------------------------------------------------