├── .gitignore ├── .gitmodules ├── .travis.yml ├── README.md ├── aof_to_pika ├── Makefile ├── README.md ├── include │ ├── aof_info.h │ ├── aof_lock.h │ └── aof_sender.h └── src │ ├── aof_info.cc │ ├── aof_lock.cc │ ├── aof_reader.cc │ └── aof_sender.cc ├── benchmark_client ├── Makefile └── benchmark_client.cc ├── binlog_sender ├── Makefile ├── binlog_consumer.cc ├── binlog_consumer.h ├── binlog_sender.cc ├── binlog_transverter.cc ├── binlog_transverter.h ├── progress_thread.cc ├── progress_thread.h ├── utils.cc └── utils.h ├── manifest_generator ├── Makefile ├── include │ ├── pika_binlog.h │ └── pika_define.h ├── manifest_generator.cc ├── pika_binlog.cc └── static_pack.sh ├── nemo_to_blackwidow ├── Makefile ├── classify_thread.cc ├── classify_thread.h ├── migrator.cc ├── migrator.h ├── nemo_to_blackwidow.cc ├── progress_thread.cc ├── progress_thread.h ├── utils.cc └── utils.h ├── pika-port ├── README.md ├── build2.sh ├── build3.sh ├── glog.sh ├── pika_port_2 │ ├── Makefile │ ├── binlog.cc │ ├── binlog.h │ ├── binlog_const.cc │ ├── binlog_const.h │ ├── binlog_log.h │ ├── binlog_receiver_thread.cc │ ├── binlog_receiver_thread.h │ ├── main.cc │ ├── master_conn.cc │ ├── master_conn.h │ ├── migrator_thread.cc │ ├── migrator_thread.h │ ├── pika2.3.6 │ │ └── include │ │ │ ├── pika_binlog.h │ │ │ ├── pika_client_conn.h │ │ │ ├── pika_command.h │ │ │ ├── pika_define.h │ │ │ └── pika_slot.h │ ├── pika_port.cc │ ├── pika_port.h │ ├── pika_sender.cc │ ├── pika_sender.h │ ├── port_conf.h │ ├── readme.md │ ├── redis_sender.cc │ ├── redis_sender.h │ ├── slaveping_thread.cc │ ├── slaveping_thread.h │ ├── sync.sh │ ├── trysync_thread.cc │ └── trysync_thread.h ├── pika_port_3 │ ├── Makefile │ ├── binlog_receiver_thread.cc │ ├── binlog_receiver_thread.h │ ├── binlog_transverter.cc │ ├── binlog_transverter.h │ ├── conf.h │ ├── const.cc │ ├── const.h │ ├── deps_source │ │ ├── libgflags.a │ │ ├── libglog.a │ │ ├── libsnappy.a │ │ └── libunwind.a │ ├── main.cc │ ├── master_conn.cc │ ├── master_conn.h │ ├── migrator_thread.cc │ ├── migrator_thread.h │ ├── pika_binlog.cc │ ├── pika_binlog.h │ ├── pika_command.h │ ├── pika_define.h │ ├── pika_port.cc │ ├── pika_port.h │ ├── pika_sender.cc │ ├── pika_sender.h │ ├── readme.md │ ├── redis_sender.cc │ ├── redis_sender.h │ ├── slaveping_thread.cc │ ├── slaveping_thread.h │ ├── static_pack.sh │ ├── trysync_thread.cc │ └── trysync_thread.h └── tool │ ├── bench.py │ ├── get_missing_keys.py │ └── sync.sh ├── pika_to_txt ├── Makefile ├── pika_to_txt.cc ├── progress_thread.cc ├── progress_thread.h ├── scan_thread.cc ├── scan_thread.h ├── write_thread.cc └── write_thread.h ├── rdb_to_pika ├── Makefile ├── README.md ├── install_rdb_tools.sh ├── protocoltopika.cc └── trans_rdb_to_pro.sh ├── ssdb_to_pika ├── Makefile ├── SSDB_client.h ├── libssdb-client.a ├── ssdb_to_pika.cc ├── thread_pool.cc └── thread_pool.h ├── ssdb_to_pika_soomth_migration ├── Makefile ├── header │ ├── include.h │ ├── slave.h │ └── version.h ├── leveldb-1.20 │ ├── .gitignore │ ├── .travis.yml │ ├── AUTHORS │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── Makefile │ ├── Makefile.bk │ ├── NEWS │ ├── README.md │ ├── TODO │ ├── build_detect_platform │ ├── build_detect_platform.bk │ ├── db │ │ ├── autocompact_test.cc │ │ ├── builder.cc │ │ ├── builder.h │ │ ├── c.cc │ │ ├── c_test.c │ │ ├── corruption_test.cc │ │ ├── db_bench.cc │ │ ├── db_impl.cc │ │ ├── db_impl.cc.bk │ │ ├── db_impl.h │ │ ├── db_iter.cc │ │ ├── db_iter.h │ │ ├── db_test.cc │ │ ├── dbformat.cc │ │ ├── dbformat.h │ │ ├── dbformat.h.bk │ │ ├── dbformat_test.cc │ │ ├── dumpfile.cc │ │ ├── fault_injection_test.cc │ │ ├── filename.cc │ │ ├── filename.h │ │ ├── filename_test.cc │ │ ├── leveldbutil.cc │ │ ├── log_format.h │ │ ├── log_reader.cc │ │ ├── log_reader.h │ │ ├── log_test.cc │ │ ├── log_writer.cc │ │ ├── log_writer.h │ │ ├── memtable.cc │ │ ├── memtable.h │ │ ├── recovery_test.cc │ │ ├── repair.cc │ │ ├── skiplist.h │ │ ├── skiplist_test.cc │ │ ├── snapshot.h │ │ ├── table_cache.cc │ │ ├── table_cache.h │ │ ├── version_edit.cc │ │ ├── version_edit.h │ │ ├── version_edit_test.cc │ │ ├── version_set.cc │ │ ├── version_set.h │ │ ├── version_set_test.cc │ │ ├── write_batch.cc │ │ ├── write_batch_internal.h │ │ └── write_batch_test.cc │ ├── doc │ │ ├── bench │ │ │ ├── db_bench_sqlite3.cc │ │ │ └── db_bench_tree_db.cc │ │ ├── benchmark.html │ │ ├── impl.md │ │ ├── index.md │ │ ├── log_format.md │ │ └── table_format.md │ ├── helpers │ │ └── memenv │ │ │ ├── memenv.cc │ │ │ ├── memenv.h │ │ │ └── memenv_test.cc │ ├── include │ │ └── leveldb │ │ │ ├── c.h │ │ │ ├── cache.h │ │ │ ├── comparator.h │ │ │ ├── db.h │ │ │ ├── dumpfile.h │ │ │ ├── env.h │ │ │ ├── filter_policy.h │ │ │ ├── iterator.h │ │ │ ├── iterator.h.bk │ │ │ ├── options.h │ │ │ ├── slice.h │ │ │ ├── status.h │ │ │ ├── table.h │ │ │ ├── table_builder.h │ │ │ └── write_batch.h │ ├── issues │ │ ├── issue178_test.cc │ │ └── issue200_test.cc │ ├── port │ │ ├── README │ │ ├── atomic_pointer.h │ │ ├── port.h │ │ ├── port_example.h │ │ ├── port_posix.cc │ │ ├── port_posix.h │ │ ├── port_posix_sse.cc │ │ ├── thread_annotations.h │ │ └── win │ │ │ └── stdint.h │ ├── table │ │ ├── block.cc │ │ ├── block.h │ │ ├── block_builder.cc │ │ ├── block_builder.h │ │ ├── filter_block.cc │ │ ├── filter_block.h │ │ ├── filter_block_test.cc │ │ ├── format.cc │ │ ├── format.h │ │ ├── iterator.cc │ │ ├── iterator_wrapper.h │ │ ├── merger.cc │ │ ├── merger.h │ │ ├── table.cc │ │ ├── table_builder.cc │ │ ├── table_test.cc │ │ ├── two_level_iterator.cc │ │ └── two_level_iterator.h │ └── util │ │ ├── arena.cc │ │ ├── arena.h │ │ ├── arena_test.cc │ │ ├── bloom.cc │ │ ├── bloom_test.cc │ │ ├── cache.cc │ │ ├── cache_test.cc │ │ ├── coding.cc │ │ ├── coding.h │ │ ├── coding_test.cc │ │ ├── comparator.cc │ │ ├── crc32c.cc │ │ ├── crc32c.h │ │ ├── crc32c_test.cc │ │ ├── env.cc │ │ ├── env_posix.cc │ │ ├── env_posix_test.cc │ │ ├── env_posix_test_helper.h │ │ ├── env_test.cc │ │ ├── filter_policy.cc │ │ ├── hash.cc │ │ ├── hash.h │ │ ├── hash_test.cc │ │ ├── histogram.cc │ │ ├── histogram.h │ │ ├── logging.cc │ │ ├── logging.h │ │ ├── mutexlock.h │ │ ├── options.cc │ │ ├── posix_logger.h │ │ ├── random.h │ │ ├── status.cc │ │ ├── testharness.cc │ │ ├── testharness.h │ │ ├── testutil.cc │ │ └── testutil.h ├── net │ ├── .gitignore │ ├── Makefile │ ├── fde.cpp │ ├── fde.h │ ├── fde_epoll.cc │ ├── fde_select.cc │ ├── link.cpp │ ├── link.h │ ├── link_redis.cpp │ ├── link_redis.h │ ├── proc.cpp │ ├── proc.h │ ├── resp.cpp │ ├── resp.h │ ├── server.cpp │ ├── server.h │ ├── test.conf │ ├── test.cpp │ ├── test2.cpp │ ├── worker.cpp │ └── worker.h ├── slave.cc ├── ssdb │ ├── .Makefile.swp │ ├── Makefile │ ├── binlog.cpp │ ├── binlog.h │ ├── const.h │ ├── iterator.cpp │ ├── iterator.h │ ├── options.cpp │ ├── options.h │ ├── ssdb.h │ ├── ssdb_impl.cpp │ ├── ssdb_impl.h │ ├── t_hash.cpp │ ├── t_hash.h │ ├── t_kv.cpp │ ├── t_kv.h │ ├── t_queue.cpp │ ├── t_queue.h │ ├── t_zset.cpp │ ├── t_zset.h │ ├── test.cpp │ ├── ttl.cpp │ └── ttl.h ├── ssdb_to_pika.cc └── util │ ├── Makefile │ ├── app.cpp │ ├── app.h │ ├── bytes.cpp │ ├── bytes.h │ ├── config.cpp │ ├── config.h │ ├── daemon.h │ ├── file.h │ ├── ip_filter.h │ ├── line.h │ ├── list.h │ ├── log.cpp │ ├── log.h │ ├── sorted_set.cpp │ ├── sorted_set.h │ ├── strings.h │ ├── test_sorted_set.cpp │ ├── test_thread.cpp │ └── thread.h └── txt_to_pika ├── Makefile ├── scan.cc ├── scan.h ├── sender.cc ├── sender.h └── txt_to_pika.cc /.gitignore: -------------------------------------------------------------------------------- 1 | #ctags 2 | tags 3 | 4 | #Compiled Object files 5 | *.o 6 | 7 | #Pika to Txt Binary file 8 | pika_to_txt/pika_to_txt 9 | 10 | #Binlog Sender Binary file 11 | binlog_sender/binlog_sender 12 | 13 | #Nemo to Blackwidow Binary file 14 | nemo_to_blackwidow/nemo_to_blackwidow 15 | 16 | #Benchmark Client Binary file 17 | benchmark_client/benchmark_client 18 | 19 | # Pika port 2 Binary file 20 | pika-port/pika_port_2/pika_port 21 | 22 | # Pika port 3 Binary file 23 | pika-port/pika_port_3/pika_port 24 | pika-port/pika_port_3/static_pika_port 25 | 26 | # manifest_generator Binary file 27 | manifest_generator/manifest_generator 28 | manifest_generator/static_manifest_generator 29 | 30 | # ssdb to pika Binary file 31 | ssdb_to_pika/ssdb_to_pika 32 | 33 | # aof to pika Binary file 34 | aof_to_pika/output/bin/aof_to_pika 35 | 36 | ## txt to pika Binary file 37 | txt_to_pika/txt_to_pika 38 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third/nemo-rocksdb"] 2 | path = third/nemo-rocksdb 3 | url = https://github.com/Qihoo360/nemo-rocksdb.git 4 | [submodule "third/blackwidow"] 5 | path = third/blackwidow 6 | url = https://github.com/Axlgrep/blackwidow.git 7 | [submodule "third/nemo"] 8 | path = third/nemo 9 | url = https://github.com/Qihoo360/nemo.git 10 | [submodule "third/rocksdb"] 11 | path = third/rocksdb 12 | url = https://github.com/facebook/rocksdb.git 13 | [submodule "third/pink"] 14 | path = third/pink 15 | url = https://github.com/PikaLabs/pink.git 16 | [submodule "third/slash"] 17 | path = third/slash 18 | url = https://github.com/Qihoo360/slash.git 19 | [submodule "third/hiredis-vip"] 20 | path = third/hiredis-vip 21 | url = https://github.com/vipshop/hiredis-vip.git 22 | [submodule "third/glog"] 23 | path = third/glog 24 | url = https://github.com/Qihoo360/glog.git 25 | [submodule "third/redis-rdb-tools"] 26 | path = third/redis-rdb-tools 27 | url = https://github.com/sripathikrishnan/redis-rdb-tools.git 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | 5 | os: 6 | - linux 7 | 8 | addons: 9 | apt: 10 | packages: ['libsnappy-dev', 'libprotobuf-dev', 'libgoogle-glog-dev'] 11 | 12 | compiler: 13 | - gcc python 14 | 15 | language: cpp python 16 | 17 | script: 18 | - git submodule init 19 | - git submodule update 20 | - cd ./binlog_sender && make && cd .. 21 | - cd ./nemo_to_blackwidow && make && cd .. 22 | - cd ./pika_to_txt && make && cd .. 23 | - cd ./txt_to_pika && make && cd .. 24 | - cd ./pika-port/pika_port_2 && make && cd .. && cd .. 25 | - cd ./pika-port/pika_port_3 && make && cd .. && cd .. 26 | - cd ./manifest_generator && make && cd .. 27 | - cd ./ssdb_to_pika && make && cd .. 28 | - cd ./rdb_to_pika && ./install_rdb_tools.sh && make && cd .. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pika-tools 2 | This project is used to store some Qihoo360 pika tools 3 | -------------------------------------------------------------------------------- /aof_to_pika/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | CXXFLAGS = -Wall -W -DDEBUG -g -O0 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 3 | OBJECT = aof_to_pika 4 | SRC_DIR = ./src 5 | OUTPUT = ./output 6 | 7 | INCLUDE_PATH = -I./include -I../.. 8 | 9 | LIBS = -lpthread 10 | 11 | .PHONY: all clean 12 | 13 | 14 | BASE_OBJS := $(wildcard $(SRC_DIR)/*.cc) 15 | BASE_OBJS += $(wildcard $(SRC_DIR)/*.c) 16 | BASE_OBJS += $(wildcard $(SRC_DIR)/*.cpp) 17 | OBJS = $(patsubst %.cc,%.o,$(BASE_OBJS)) 18 | 19 | 20 | all: $(OBJECT) 21 | rm -rf $(OUTPUT) 22 | mkdir $(OUTPUT) 23 | mkdir $(OUTPUT)/bin 24 | cp $(OBJECT) $(OUTPUT)/bin/ 25 | rm -rf $(OBJECT) 26 | @echo "Success, go, go, go..." 27 | 28 | 29 | $(OBJECT): $(OBJS) 30 | $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIBS) 31 | 32 | $(OBJS): %.o : %.cc 33 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 34 | 35 | clean: 36 | rm -rf $(SRC_DIR)/*.o 37 | rm -rf $(OUTPUT)/* 38 | rm -rf $(OUTPUT) 39 | -------------------------------------------------------------------------------- /aof_to_pika/README.md: -------------------------------------------------------------------------------- 1 | ## README 2 | 3 | #### Introduction 4 | A tool to transfer data for redis from one to another, it also support any redis-like nosql like [pika](https://github.com/baotiao/pika). The transfor progress is based on redis aof file, which simply read the aof and batch send to the destination peer. 5 | 6 | #### Feature 7 | 8 | - Continuously read new content of aof as 'tail -f' 9 | - Read response and keep statistics 10 | - More efficiency 11 | 12 | #### Usage 13 | 14 | ``` shell 15 | make 16 | cd output/bin && ./aof_to_pika -h # for more information 17 | ``` 18 | -------------------------------------------------------------------------------- /aof_to_pika/include/aof_info.h: -------------------------------------------------------------------------------- 1 | #ifndef AOF_INFO 2 | #define AOF_INFO 3 | 4 | #include 5 | 6 | #define AOF_LOG_DEBUG 1 7 | #define AOF_LOG_TRACE 2 8 | #define AOF_LOG_INFO 3 9 | #define AOF_LOG_WARN 4 10 | #define AOF_LOG_ERR 5 11 | #define AOF_LOG_FATAL 6 12 | 13 | #define LOG_FATAL(content) info_print(AOF_LOG_FATAL, content) 14 | #define LOG_ERR(content) info_print(AOF_LOG_ERR, content) 15 | #define LOG_WARN(content) info_print(AOF_LOG_WARN, content) 16 | #define LOG_INFO(content) info_print(AOF_LOG_INFO, content) 17 | #define LOG_TRACE(content) info_print(AOF_LOG_TRACE, content) 18 | #define LOG_DEBUG(content) info_print(AOF_LOG_DEBUG, content) 19 | 20 | void set_info_level(int); 21 | void info_print(int, const std::string&); 22 | #endif 23 | 24 | -------------------------------------------------------------------------------- /aof_to_pika/include/aof_lock.h: -------------------------------------------------------------------------------- 1 | #ifndef AOF_LOCK_H 2 | #define AOF_LOCK_H 3 | 4 | class CondVar; 5 | 6 | class Mutex { 7 | public: 8 | Mutex(); 9 | ~Mutex(); 10 | 11 | void Lock(); 12 | void Unlock(); 13 | void AssertHeld() { } 14 | 15 | private: 16 | friend class CondVar; 17 | pthread_mutex_t mu_; 18 | 19 | // No copying 20 | Mutex(const Mutex&); 21 | void operator=(const Mutex&); 22 | }; 23 | 24 | class CondVar { 25 | public: 26 | explicit CondVar(Mutex* mu); 27 | ~CondVar(); 28 | void Wait(); 29 | void Signal(); 30 | void SignalAll(); 31 | private: 32 | pthread_cond_t cv_; 33 | Mutex* mu_; 34 | }; 35 | 36 | #endif // AOF_LOCK_H 37 | -------------------------------------------------------------------------------- /aof_to_pika/include/aof_sender.h: -------------------------------------------------------------------------------- 1 | #ifndef AOF_SENDER_H 2 | #define AOF_SENDER_H 3 | 4 | #include 5 | #include 6 | #include "aof_lock.h" 7 | #include "aof_info.h" 8 | 9 | #define RM_NONE 0 10 | #define RM_READBLE 1 11 | #define RM_WRITABLE 2 12 | #define RM_RECONN 4 13 | 14 | #define READ_BUF_MAX 100 15 | #define MSG_BLOCK_MAX 512 * 1024 16 | 17 | typedef struct ConnInfo{ 18 | ConnInfo(){} 19 | ConnInfo(const std::string &h, const std::string &p, const std::string &a):host_(h),port_(p),auth_(a){} 20 | std::string host_; 21 | std::string port_; 22 | std::string auth_; 23 | } ConnInfo; 24 | 25 | 26 | class AOFSender 27 | { 28 | public: 29 | AOFSender():buf_wcond_(&buf_mutex_), buf_rcond_(&buf_mutex_){ sockfd_ = -1; nsucc_ = nfail_ = 0; } 30 | ~AOFSender(); 31 | bool rconnect(const std::string&, const std::string&, const std::string&); 32 | bool message_add(const std::string&); 33 | bool process(); 34 | void print_result(); 35 | 36 | private: 37 | int sockfd_; 38 | int nsucc_, nfail_; 39 | ConnInfo *conn_info_; 40 | Mutex buf_mutex_; 41 | CondVar buf_wcond_; 42 | CondVar buf_rcond_; 43 | std::deque read_buffer_; 44 | std::string to_send_; 45 | std::string current_bulk_; 46 | bool message_get_(); 47 | bool check_succ_(const std::string&, long&, long&); 48 | int mask_wait_(int fd, int mask, long long milliseconds); 49 | bool set_nonblock_(int fd); 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /aof_to_pika/src/aof_info.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "aof_info.h" 3 | 4 | short aof_info_level_ = AOF_LOG_INFO; 5 | 6 | void set_info_level(int l){ 7 | aof_info_level_ = l; 8 | } 9 | 10 | void info_print(int l, const std::string &content) { 11 | if (l > AOF_LOG_FATAL || l < AOF_LOG_DEBUG || content.empty()) return; 12 | 13 | if (l < aof_info_level_) return; 14 | if (l >= AOF_LOG_ERR) 15 | std::cerr << content << std::endl; 16 | else 17 | std::cout << content << std::endl; 18 | } 19 | 20 | -------------------------------------------------------------------------------- /aof_to_pika/src/aof_lock.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "aof_lock.h" 7 | 8 | 9 | static void PthreadCall(const std::string &label, int result) { 10 | if (result != 0) { 11 | std::cout << "pthread " << label << " : " << result << std::endl; 12 | abort(); 13 | } 14 | } 15 | 16 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 17 | 18 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 19 | 20 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 21 | 22 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 23 | 24 | CondVar::CondVar(Mutex* mu) 25 | : mu_(mu) { 26 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 27 | } 28 | 29 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 30 | 31 | void CondVar::Wait() { 32 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 33 | } 34 | 35 | void CondVar::Signal() { 36 | PthreadCall("signal", pthread_cond_signal(&cv_)); 37 | } 38 | 39 | void CondVar::SignalAll() { 40 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 41 | } 42 | -------------------------------------------------------------------------------- /benchmark_client/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | LDFLAGS=-lpthread -lsnappy 4 | 5 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 6 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ 7 | -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ 8 | -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 9 | 10 | CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g 11 | 12 | THIRD_PATH=../third 13 | 14 | SLASH_PATH=$(THIRD_PATH)/slash 15 | SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a 16 | 17 | HIREDIS_PATH=$(THIRD_PATH)/hiredis-vip 18 | HIREDIS_LIBRARY=$(HIREDIS_PATH)/libhiredis_vip.a 19 | 20 | INCLUDE_PATH = -I./ \ 21 | -I$(SLASH_PATH) \ 22 | -I$(HIREDIS_PATH) \ 23 | 24 | OBJECT : SLASH HIREDIS benchmark_client 25 | 26 | SLASH : 27 | make -C $(SLASH_PATH)/slash 28 | 29 | HIREDIS: 30 | make -C $(HIREDIS_PATH) 31 | 32 | benchmark_client : benchmark_client.cc 33 | $(CXX) $(CXXFLAGS) $(LDFLAGS) $^ -o $@ $(INCLUDE_PATH) $(SLASH_LIBRARY) $(HIREDIS_LIBRARY) 34 | 35 | clean: 36 | rm -rf benchmark_client 37 | 38 | distclean: 39 | make -C $(SLASH_PATH)/slash clean 40 | make -C $(HIREDIS_PATH) clean 41 | rm -rf benchmark_client 42 | -------------------------------------------------------------------------------- /binlog_sender/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 4 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ 5 | -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ 6 | -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 7 | 8 | CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g 9 | 10 | TARGET = binlog_sender 11 | 12 | SRC_PATH=. 13 | THIRD_PATH=../third 14 | 15 | SLASH_PATH=$(THIRD_PATH)/slash 16 | SLASH=$(SLASH_PATH)/slash/lib/libslash.a 17 | 18 | PINK_PATH=$(THIRD_PATH)/pink 19 | PINK=$(PINK_PATH)/pink/lib/libpink.a 20 | 21 | 22 | INCLUDE_PATH = -I./ \ 23 | -I$(SLASH_PATH) \ 24 | -I$(PINK_PATH) \ 25 | 26 | LIB_PATH = -L$(SLASH_PATH)/slash/lib \ 27 | -L$(PINK_PATH)/pink/lib \ 28 | 29 | LIBS = -lpink \ 30 | -lslash \ 31 | -lpthread \ 32 | 33 | SOURCE := $(wildcard $(SRC_PATH)/*.cc) 34 | OBJS := $(patsubst %.cc, %.o, $(SOURCE)) 35 | 36 | $(TARGET): $(OBJS) $(PINK) $(SLASH) 37 | $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 38 | 39 | $(OBJS): %.o : %.cc 40 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 41 | 42 | 43 | $(PINK) : 44 | make -C $(PINK_PATH)/pink SLASH_PATH=../../slash 45 | 46 | $(SLASH) : 47 | make -C $(SLASH_PATH)/slash 48 | 49 | 50 | .PHONY: clean distclean 51 | 52 | clean: 53 | rm -rf $(OBJS) 54 | rm -rf $(TARGET) 55 | 56 | distclean: 57 | rm -rf $(OBJS) 58 | rm -rf $(TARGET) 59 | make -C $(PINK_PATH)/pink clean 60 | make -C $(SLASH_PATH)/slash clean 61 | -------------------------------------------------------------------------------- /binlog_sender/binlog_consumer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The blackwidow Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_BINLOG_CONSUMER_H_ 7 | #define INCLUDE_BINLOG_CONSUMER_H_ 8 | 9 | #include "stddef.h" 10 | #include "stdint.h" 11 | 12 | #include "slash/include/slash_slice.h" 13 | #include "slash/include/slash_status.h" 14 | #include "slash/include/env.h" 15 | #include "slash/include/slash_mutex.h" 16 | 17 | enum RecordType { 18 | kZeroType = 0, 19 | kFullType = 1, 20 | kFirstType = 2, 21 | kMiddleType = 3, 22 | kLastType = 4, 23 | kEof = 5, 24 | kBadRecord = 6, 25 | kOldRecord = 7 26 | }; 27 | 28 | /* 29 | * Header is Type(1 byte), length (3 bytes), time (4 bytes) 30 | */ 31 | static const size_t kHeaderSize = 1 + 3 + 4; 32 | static const size_t kBlockSize = 64 * 1024; 33 | static const std::string kBinlogPrefix = "write2file"; 34 | 35 | class BinlogConsumer { 36 | public: 37 | BinlogConsumer(const std::string& binlog_path, 38 | uint32_t first_filenum, 39 | uint32_t last_filenum, 40 | uint64_t offset); 41 | virtual ~BinlogConsumer(); 42 | 43 | std::string NewFileName(const std::string& name, 44 | const uint32_t current); 45 | bool Init(); 46 | bool trim(); 47 | uint32_t current_filenum(); 48 | uint64_t current_offset(); 49 | slash::Status Parse(std::string* scratch); 50 | 51 | private: 52 | uint64_t get_next(bool* is_error); 53 | uint32_t ReadPhysicalRecord(slash::Slice* result); 54 | slash::Status Consume(std::string* scratch); 55 | std::string filename_; 56 | uint32_t first_filenum_; 57 | uint32_t last_filenum_; 58 | 59 | uint32_t current_filenum_; 60 | uint64_t current_offset_; 61 | uint64_t last_record_offset_; 62 | 63 | 64 | slash::Slice buffer_; 65 | char* const backing_store_; 66 | slash::SequentialFile* queue_; 67 | }; 68 | 69 | #endif // INCLUDE_BINLOG_Consumber_H_ 70 | -------------------------------------------------------------------------------- /binlog_sender/progress_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "progress_thread.h" 7 | 8 | ProgressThread::ProgressThread(BinlogConsumer* binlog_consumer) 9 | : binlog_consumer_(binlog_consumer) { 10 | } 11 | 12 | void* ProgressThread::ThreadMain() { 13 | while (!should_stop_) { 14 | printf("\rSending binlog, current file num: %d, offset: %9ld", 15 | binlog_consumer_->current_filenum(), binlog_consumer_->current_offset()); 16 | fflush(stdout); 17 | } 18 | printf("\n"); 19 | return NULL; 20 | } 21 | 22 | -------------------------------------------------------------------------------- /binlog_sender/progress_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_PROGRESS_THREAD_H_ 7 | #define INCLUDE_PROGRESS_THREAD_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include "pink/include/pink_thread.h" 13 | 14 | #include "binlog_consumer.h" 15 | 16 | extern slash::Mutex mutex; 17 | 18 | class ProgressThread : public pink::Thread { 19 | public: 20 | ProgressThread(BinlogConsumer* binlog_consumer); 21 | private: 22 | virtual void *ThreadMain(); 23 | BinlogConsumer* binlog_consumer_; 24 | }; 25 | 26 | #endif // INCLUDE_PROGRESS_THREAD_H_ 27 | -------------------------------------------------------------------------------- /binlog_sender/utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The blackwidow Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "utils.h" 7 | 8 | bool Exists(const std::string& base, const std::string& pattern) { 9 | std::string::size_type n; 10 | n = base.find(pattern); 11 | if (n == std::string::npos) { 12 | return false; 13 | } else { 14 | return true; 15 | } 16 | } 17 | 18 | bool CheckFilesStr(const std::string& files_str) { 19 | int32_t neg_count = 0; 20 | for (const auto& c : files_str) { 21 | if (isdigit(c)) { 22 | continue; 23 | } else if (c == NEG_CHAR) { 24 | neg_count++; 25 | } else { 26 | return false; 27 | } 28 | } 29 | return neg_count <= 1 ? true : false; 30 | } 31 | 32 | bool GetFileList(const std::string& files_str, std::vector* files) { 33 | std::string::size_type pos; 34 | if (Exists(files_str, NEG_STR)) { 35 | pos = files_str.find(NEG_STR); 36 | uint32_t start_file = atoi(files_str.substr(0, pos).data()); 37 | uint32_t end_file = atoi(files_str.substr(pos + 1).data()); 38 | for (uint32_t file_num = start_file; file_num <= end_file; ++file_num) { 39 | files->push_back(file_num); 40 | } 41 | } else { 42 | files->push_back(atoi(files_str.data())); 43 | } 44 | return !files->empty(); 45 | } 46 | 47 | bool CheckBinlogExists(const std::string& binlog_path, const std::vector& files) { 48 | std::string filename = binlog_path + WRITE2FILE; 49 | for (size_t idx = 0; idx < files.size(); ++idx) { 50 | std::string binlog_file = filename + std::to_string(files[idx]); 51 | if (!slash::FileExists(binlog_file)) { 52 | return false; 53 | } 54 | } 55 | return true; 56 | } 57 | -------------------------------------------------------------------------------- /binlog_sender/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The blackwidow Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_UTILS_H_ 7 | #define INCLUDE_UTILS_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include "slash/include/env.h" 13 | 14 | #define COMMA_STR "," 15 | #define COLON_STR ":" 16 | #define NEG_STR "-" 17 | #define NEG_CHAR '-' 18 | #define SPACE_STR " " 19 | #define WRITE2FILE "write2file" 20 | 21 | bool Exists(const std::string& base, const std::string& pattern); 22 | bool CheckFilesStr(const std::string& files_str); 23 | bool GetFileList(const std::string& files_str, std::vector* files); 24 | bool CheckBinlogSequential(const std::vector& files); 25 | bool CheckBinlogExists(const std::string& binlog_path, const std::vector& files); 26 | 27 | #endif // INCLUDE_UTILS_H_ 28 | -------------------------------------------------------------------------------- /manifest_generator/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | LDFLAGS=-lpthread 4 | 5 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 6 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ 7 | -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ 8 | -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 9 | 10 | CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g 11 | 12 | THIRD_PATH=$(CURDIR)/../third 13 | 14 | SLASH_PATH=$(THIRD_PATH)/slash 15 | SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a 16 | PINK_PATH=$(THIRD_PATH)/pink 17 | PINK_LIBRARY=$(PINK_PATH)/pink/lib/libpink.a 18 | 19 | 20 | INCLUDE_PATH = -I./ \ 21 | -I$(SLASH_PATH) \ 22 | -I$(PINK_PATH) \ 23 | 24 | OBJECT : SLASH PINK manifest_generator 25 | 26 | PINK : $(SLASH) 27 | make -C $(PINK_PATH)/pink NO_PB=0 SLASH_PATH=$(SLASH_PATH) 28 | 29 | SLASH : 30 | make -C $(SLASH_PATH)/slash 31 | 32 | pika_binlog.o : pika_binlog.cc 33 | $(CXX) $(CXXFLAGS) -c $^ -o $@ $(INCLUDE_PATH) 34 | 35 | manifest_generator.o : manifest_generator.cc pika_binlog.o 36 | $(CXX) $(CXXFLAGS) -c $^ -o $@ $(INCLUDE_PATH) 37 | 38 | manifest_generator : manifest_generator.o pika_binlog.o 39 | $(CXX) $^ $(SLASH_LIBRARY) -o $@ $(LDFLAGS) 40 | 41 | clean: 42 | rm -rf manifest_generator 43 | rm -rf pika_binlog.o 44 | rm -rf manifest_generator.o 45 | 46 | distclean: 47 | make -C $(SLASH_PATH)/slash clean 48 | make -C $(PINK_PATH)/pink clean 49 | rm -rf manifest_generator 50 | rm -rf pika_binlog.o 51 | rm -rf manifest_generator.o 52 | -------------------------------------------------------------------------------- /manifest_generator/static_pack.sh: -------------------------------------------------------------------------------- 1 | g++ \ 2 | manifest_generator.o \ 3 | pika_binlog.o \ 4 | ../third/slash/slash/lib/libslash.a -o static_manifest_generator -lpthread -static-libstdc++ 5 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/classify_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_CLASSIFY_THREAD_H_ 7 | #define INCLUDE_CLASSIFY_THREAD_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include 13 | 14 | #include "nemo.h" 15 | #include "pink/include/pink_thread.h" 16 | #include "slash/include/slash_mutex.h" 17 | 18 | #include "utils.h" 19 | #include "migrator.h" 20 | 21 | extern slash::Mutex mutex; 22 | 23 | class ClassifyThread : public pink::Thread { 24 | public: 25 | ClassifyThread(nemo::Nemo* nemo_db, std::vector migrators, const std::string& type); 26 | virtual ~ClassifyThread(); 27 | int64_t key_num(); 28 | bool is_finish() { return is_finish_;} 29 | std::string type() { return type_;} 30 | int64_t consume_index() { return consume_index_;} 31 | private: 32 | void PlusProcessKeyNum(); 33 | void DispatchItem(const std::string& item); 34 | virtual void *ThreadMain(); 35 | 36 | bool is_finish_; 37 | int64_t key_num_; 38 | int64_t consume_index_; 39 | pthread_rwlock_t rwlock_; 40 | nemo::Nemo* nemo_db_; 41 | std::vector migrators_; 42 | std::string type_; 43 | }; 44 | 45 | #endif // INCLUDE_CLASSIFY_THREAD_H_ 46 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/migrator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_MIGRATOR_H_ 7 | #define INCLUDE_MIGRATOR_H_ 8 | 9 | #define MAX_QUEUE_SIZE 10000 10 | 11 | #include 12 | #include "iostream" 13 | 14 | #include "blackwidow/blackwidow.h" 15 | #include "pink/include/pink_thread.h" 16 | 17 | extern slash::Mutex mutex; 18 | 19 | class Migrator: public pink::Thread { 20 | public: 21 | Migrator(int32_t migrator_id, nemo::Nemo* nemo_db, blackwidow::BlackWidow* blackwidow_db) 22 | : nemo_db_(nemo_db), 23 | blackwidow_db_(blackwidow_db), 24 | migrator_id_(migrator_id), 25 | migrate_key_num_(0), 26 | should_exit_(false), 27 | queue_cond_(&queue_mutex_) {} 28 | virtual ~Migrator() {} 29 | 30 | int32_t queue_size(); 31 | void PlusMigrateKey(); 32 | void SetShouldExit(); 33 | bool LoadItem(const std::string& item); 34 | 35 | private: 36 | virtual void *ThreadMain(); 37 | nemo::Nemo* nemo_db_; 38 | blackwidow::BlackWidow* blackwidow_db_; 39 | 40 | int32_t migrator_id_; 41 | int64_t migrate_key_num_; 42 | std::atomic should_exit_; 43 | slash::Mutex queue_mutex_; 44 | slash::CondVar queue_cond_; 45 | std::queue items_queue_; 46 | }; 47 | 48 | #endif // INCLUDE_MIGRATOR_H_ 49 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/progress_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "progress_thread.h" 7 | 8 | ProgressThread::ProgressThread(std::vector* classify_threads) 9 | : classify_threads_(classify_threads) { 10 | } 11 | 12 | bool ProgressThread::AllClassifyTreadFinish() { 13 | for (const auto& classify_thread : *classify_threads_) { 14 | if (!classify_thread->is_finish()) { 15 | return false; 16 | } 17 | } 18 | return true; 19 | } 20 | 21 | void* ProgressThread::ThreadMain() { 22 | while (true) { 23 | slash::MutexLock l(&mutex); 24 | bool is_finish = AllClassifyTreadFinish(); 25 | printf("\rstring keys: %5ld, hashes keys: %5ld, lists keys: %5ld, sets keys: %5ld, zsets keys: %5ld ", 26 | (*classify_threads_)[0]->key_num(), 27 | (*classify_threads_)[1]->key_num(), 28 | (*classify_threads_)[2]->key_num(), 29 | (*classify_threads_)[3]->key_num(), 30 | (*classify_threads_)[4]->key_num()); 31 | fflush(stdout); 32 | if (is_finish) { 33 | break; 34 | } 35 | sleep(1); 36 | } 37 | slash::MutexLock l(&mutex); 38 | printf("\nClassify keys finished\n"); 39 | return NULL; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/progress_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_PROGRESS_THREAD_H_ 7 | #define INCLUDE_PROGRESS_THREAD_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include "pink/include/pink_thread.h" 13 | 14 | #include "classify_thread.h" 15 | 16 | extern slash::Mutex mutex; 17 | 18 | class ProgressThread : public pink::Thread { 19 | public: 20 | ProgressThread(std::vector* classify_threads); 21 | private: 22 | bool AllClassifyTreadFinish(); 23 | virtual void *ThreadMain(); 24 | std::vector* classify_threads_; 25 | }; 26 | 27 | #endif // INCLUDE_PROGRESS_THREAD_H_ 28 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "iostream" 7 | #include "utils.h" 8 | 9 | void EncodeKeyValue(const std::string& key, 10 | const std::string& value, 11 | std::string* dst) { 12 | dst->clear(); 13 | int32_t key_size = key.size(); 14 | int32_t value_size = value.size(); 15 | dst->append(reinterpret_cast(&key_size), sizeof(int32_t)); 16 | dst->append(key); 17 | dst->append(reinterpret_cast(&value_size), sizeof(int32_t)); 18 | dst->append(value); 19 | return; 20 | } 21 | 22 | void DecodeKeyValue(const std::string& dst, 23 | std::string* key, 24 | std::string* value) { 25 | const char* p = dst.data(); 26 | int32_t key_size = *(reinterpret_cast(p)); 27 | p += sizeof(int32_t); 28 | key->assign(p, key_size); 29 | p += key_size; 30 | 31 | int32_t value_size = *(reinterpret_cast(p)); 32 | p += sizeof(int32_t); 33 | value->assign(p, value_size); 34 | p += value_size; 35 | return; 36 | } 37 | 38 | -------------------------------------------------------------------------------- /nemo_to_blackwidow/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_UTILS_H_ 7 | #define INCLUDE_UTILS_H_ 8 | 9 | void EncodeKeyValue(const std::string& key, const std::string& value, std::string* dst); 10 | void DecodeKeyValue(const std::string& dst, std::string* key, std::string* value); 11 | 12 | #endif // INCLUDE_UILTS_H_ 13 | -------------------------------------------------------------------------------- /pika-port/README.md: -------------------------------------------------------------------------------- 1 | # pika-port 2 | --- 3 | *Pika-port is used to move data from pika to pika/redis. This repo has been [included officially](https://github.com/Qihoo360/pika/wiki/pika%E5%88%B0redis%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7).* 4 | 5 | ## Intro 6 | --- 7 | 8 | Pika-port is like redis-port. it copy a snapshot of pika to pika/redis/codis and then transfer delta data to them. 9 | If u wanna get more details of Pika-port, pls read [Pika笔记](http://alexstocks.github.io/html/pika.html). 10 | 11 | ## Chinese Intro 12 | --- 13 | 14 | Pika-port 把自已伪装成 pika 的 slave, 通过 trysync 先把数据全量同步给 pika/redis，然后再进行增量通过。 15 | 16 | 大致流程： 17 | 18 | >- 1 发送 trysync 全量同步命令 -> 接收 dump 数据 -> 解析 dump 数据 -> 过滤 -> 发送给 pika/redis； 19 | >- 2 发送 trysync 增量同步命令 -> 接收 binlog -> 解析 binlog 数据 -> 过滤 -> 发送给 pika/redis； 20 | 21 | ## Use Case 22 | --- 23 | 24 | - [记一次pika迁移到codis](https://blog.csdn.net/wangwenjie2500/article/details/83858572) 25 | 26 | ## Compilation 27 | --- 28 | 29 | ### pika port for pika 2.3.x 30 | sh build2.sh 31 | 32 | 33 | ### pika port for pika 3.0.x 34 | sh build3.sh 35 | -------------------------------------------------------------------------------- /pika-port/build2.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # ****************************************************** 3 | # DESC : pika-port2 build script 4 | # AUTHOR : Alex Stocks 5 | # VERSION : 1.0 6 | # LICENCE : Apache License 2.0 7 | # EMAIL : alexstocks@foxmail.com 8 | # MOD : 2019-01-22 19:54 9 | # FILE : build.sh 10 | # ****************************************************** 11 | 12 | # https://github.com/facebook/rocksdb/blob/master/INSTALL.md 13 | 14 | sh glog.sh 15 | 16 | # install compression lib 17 | sudo yum install -y snappy snappy-devel 18 | sudo yum install -y zlib zlib-devel 19 | sudo yum install -y bzip2 bzip2-devel 20 | sudo yum install -y lz4-devel 21 | 22 | # download third libs 23 | # pls wait for about 30 minutes 24 | git submodule update --init --recursive --force 25 | 26 | # compile pika-port for pika 3.0.x 27 | cd pika_port_2 && make 28 | -------------------------------------------------------------------------------- /pika-port/build3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # ****************************************************** 3 | # DESC : pika-port3 build script 4 | # AUTHOR : Alex Stocks 5 | # VERSION : 1.0 6 | # LICENCE : Apache License 2.0 7 | # EMAIL : alexstocks@foxmail.com 8 | # MOD : 2019-01-22 19:54 9 | # FILE : build.sh 10 | # ****************************************************** 11 | 12 | # https://github.com/facebook/rocksdb/blob/master/INSTALL.md 13 | # install gflags 14 | git clone https://github.com/gflags/gflags.git 15 | cd gflags 16 | git checkout v2.0 17 | ./configure && make && sudo make install 18 | cd .. 19 | 20 | # install compression lib 21 | sudo yum install -y snappy snappy-devel 22 | sudo yum install -y zlib zlib-devel 23 | sudo yum install -y bzip2 bzip2-devel 24 | sudo yum install -y lz4-devel 25 | sudo yum install -y libzstd-devel 26 | sudo yum install -y libstdc++-static 27 | 28 | # download third libs 29 | # pls wait for about 30 minutes 30 | git submodule update --init --recursive --force 31 | 32 | # compile pika-port for pika 3.0.x 33 | cd pika_port_3 34 | make 35 | export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} 36 | export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH} 37 | ./pika_port 38 | -------------------------------------------------------------------------------- /pika-port/glog.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # ****************************************************** 3 | # DESC : glog build script 4 | # AUTHOR : Alex Stocks 5 | # VERSION : 1.0 6 | # LICENCE : Apache License 2.0 7 | # EMAIL : alexstocks@foxmail.com 8 | # MOD : 2019-01-22 19:54 9 | # FILE : build.sh 10 | # ****************************************************** 11 | 12 | mkdir -p deps 13 | cd deps 14 | 15 | ############## 16 | ### install cmake3 && automake for compiling glog 17 | ############## 18 | 19 | if [ ! -d "./cmake-3.13.3" ]; then 20 | wget https://github.com/Kitware/CMake/releases/download/v3.13.3/cmake-3.13.3.tar.gz 21 | tar -zxvf cmake-3.13.3.tar.gz 22 | fi 23 | cd cmake-3.13.3 && ./bootstrap && gmake && sudo gmake install && cd .. 24 | rm -f /usr/bin/cmake 25 | ln -s /usr/local/bin/cmake /usr/bin/cmake 26 | 27 | yum install -y automake 28 | autoreconf -ivf 29 | 30 | ############## 31 | ### compile libunwind 32 | ############## 33 | 34 | if [ ! -d "./libunwind-1.3.1" ]; then 35 | wget https://github.com/libunwind/libunwind/releases/download/v1.3.1/libunwind-1.3.1.tar.gz 36 | tar -xf libunwind-1.3.1.tar.gz 37 | fi 38 | 39 | mkdir libunwind 40 | cd libunwind-1.3.1 41 | ./configure --prefix=`pwd`/../libunwind 42 | make 43 | make install 44 | cd .. 45 | 46 | ############## 47 | ### compile gflags 48 | ############## 49 | 50 | # 卸载系统的gflags 51 | # sudo yum remove -y gflags-devel 52 | # 下载编译gflags 53 | if [ ! -d "./gflags-2.2.1" ]; then 54 | wget -O gflags-2.2.1.tar.gz https://github.com/gflags/gflags/archive/v2.2.1.tar.gz 55 | tar zxvf gflags-2.2.1.tar.gz 56 | fi 57 | cd gflags-2.2.1 58 | rm -rf build && mkdir -p build && cd build 59 | cmake .. -DCMAKE_INSTALL_PREFIX=../../gflags 60 | make 61 | make install 62 | cd ../.. 63 | 64 | ############## 65 | ### compile glog 66 | ############## 67 | 68 | mkdir -p glog 69 | 70 | # return to root dir 71 | cd .. 72 | 73 | cd third/glog 74 | ./configure --includedir=../../deps/gflags/include 75 | rm -rf build && mkdir -p build && cd build 76 | export CXXFLAGS="-fPIC" && cmake .. -DCMAKE_INSTALL_PREFIX=../../../deps/glog && make VERBOSE=1 && make install 77 | cd ../../.. 78 | 79 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/binlog_const.cc: -------------------------------------------------------------------------------- 1 | #include "binlog_const.h" 2 | 3 | #include "include/pika_define.h" 4 | 5 | std::string PikaState(int state) { 6 | switch (state) { 7 | case PIKA_REPL_NO_CONNECT: 8 | return "PIKA_REPL_NO_CONNECT"; 9 | 10 | case PIKA_REPL_CONNECT: 11 | return "PIKA_REPL_CONNECT"; 12 | 13 | case PIKA_REPL_CONNECTING: 14 | return "PIKA_REPL_CONNECTING"; 15 | 16 | case PIKA_REPL_CONNECTED: 17 | return "PIKA_REPL_CONNECTED"; 18 | 19 | case PIKA_REPL_WAIT_DBSYNC: 20 | return "PIKA_REPL_WAIT_DBSYNC"; 21 | 22 | case PIKA_REPL_ERROR: 23 | return "PIKA_REPL_ERROR"; 24 | 25 | default: 26 | return "PIKA_REPL_UNKNOWN"; 27 | } 28 | 29 | return "PIKA_REPL_UNKNOWN"; 30 | } 31 | 32 | std::string PikaRole(int role) { 33 | std::string roleStr = "|"; 34 | if (role == PIKA_ROLE_SINGLE) { 35 | roleStr += " PIKA_ROLE_SINGLE |"; 36 | } 37 | 38 | if (role == PIKA_ROLE_SLAVE) { 39 | roleStr += " PIKA_ROLE_SLAVE |"; 40 | } 41 | 42 | if (role == PIKA_ROLE_MASTER) { 43 | roleStr += " PIKA_ROLE_MASTER |"; 44 | } 45 | 46 | if (role == PIKA_ROLE_DOUBLE_MASTER) { 47 | roleStr += " PIKA_ROLE_DOUBLE_MASTER |"; 48 | } 49 | 50 | if (role == PIKA_ROLE_PORT) { 51 | roleStr += " PIKA_ROLE_PORT |"; 52 | } 53 | 54 | return roleStr; 55 | } 56 | 57 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/binlog_const.h: -------------------------------------------------------------------------------- 1 | #ifndef BINLOG_CONST_H_ 2 | #define BINLOG_CONST_H_ 3 | 4 | // //role 5 | // #define PIKA_ROLE_SINGLE 0 6 | // #define PIKA_ROLE_SLAVE 1 7 | // #define PIKA_ROLE_MASTER 2 8 | // #define PIKA_ROLE_DOUBLE_MASTER 3 9 | #define PIKA_ROLE_PORT 4 10 | 11 | #include 12 | #include 13 | 14 | const int64_t kTestPoint = 500000; 15 | const int64_t kTestNum = LLONG_MAX; 16 | const int64_t kDataSetNum = 5; 17 | 18 | #include 19 | std::string PikaState(int state); 20 | std::string PikaRole(int role) ; 21 | 22 | #endif 23 | 24 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/binlog_log.h: -------------------------------------------------------------------------------- 1 | #ifndef BINLOG_LOG_H__ 2 | #define BINLOG_LOG_H__ 3 | 4 | #define pline(fmt, ...) printf(fmt "\n", ##__VA_ARGS__) 5 | #define pinfo(fmt, ...) \ 6 | printf("\033[1;34;40m%s-%s-%d: " fmt "\033[0m\n", ((char*)__FILE__), \ 7 | (char*)__func__, (int)__LINE__, ##__VA_ARGS__) 8 | #define perr(fmt, ...) \ 9 | fprintf(stderr, "\033[1;31;40m%s-%s-%d: error: " fmt "\033[0m\n", \ 10 | (char*)__FILE__, (char*)__func__, (int)__LINE__, ##__VA_ARGS__) 11 | 12 | #endif 13 | 14 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/binlog_receiver_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | 8 | #include "pink/include/pink_conn.h" 9 | #include "binlog_receiver_thread.h" 10 | #include "master_conn.h" 11 | #include "pika_port.h" 12 | 13 | #include "port_conf.h" 14 | 15 | extern PikaPort* g_pika_port; 16 | 17 | BinlogReceiverThread::BinlogReceiverThread(std::string host, int port, int cron_interval) 18 | : conn_factory_(this), handles_(this) { 19 | // thread_rep_ = pink::NewHolyThread(port, &conn_factory_, 20 | // cron_interval, &handles_); 21 | thread_rep_ = pink::NewHolyThread(host, port, &conn_factory_, 22 | cron_interval, &handles_); 23 | // to prevent HolyThread::DoCronTask close the pika sender connection 24 | thread_rep_->set_keepalive_timeout(0); 25 | } 26 | 27 | BinlogReceiverThread::~BinlogReceiverThread() { 28 | thread_rep_->StopThread(); 29 | DLOG(INFO) << "BinlogReceiver thread " << thread_rep_->thread_id() << " exit!!!"; 30 | delete thread_rep_; 31 | } 32 | 33 | int BinlogReceiverThread::StartThread() { 34 | return thread_rep_->StartThread(); 35 | } 36 | 37 | bool BinlogReceiverThread::Handles::AccessHandle(std::string& ip) const { 38 | if (ip == "127.0.0.1") { 39 | ip = g_port_conf.local_ip; 40 | } 41 | if (binlog_receiver_->thread_rep_->conn_num() != 0 || 42 | !g_pika_port->ShouldAccessConnAsMaster(ip)) { 43 | DLOG(INFO) << "BinlogReceiverThread AccessHandle failed"; 44 | return false; 45 | } 46 | g_pika_port->PlusMasterConnection(); 47 | return true; 48 | } 49 | 50 | void BinlogReceiverThread::KillBinlogSender() { 51 | thread_rep_->KillAllConns(); 52 | } 53 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/binlog_receiver_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef BINLOG_RECEIVER_THREAD_H_ 7 | #define BINLOG_RECEIVER_THREAD_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "pink/include/server_thread.h" 13 | #include "slash/include/slash_mutex.h" 14 | #include "include/pika_define.h" 15 | #include "master_conn.h" 16 | 17 | class BinlogReceiverThread { 18 | public: 19 | BinlogReceiverThread(std::string host, int port, int cron_interval = 0); 20 | virtual ~BinlogReceiverThread(); 21 | int StartThread(); 22 | 23 | void KillBinlogSender(); 24 | 25 | private: 26 | class MasterConnFactory : public pink::ConnFactory { 27 | public: 28 | explicit MasterConnFactory(BinlogReceiverThread* binlog_receiver) 29 | : binlog_receiver_(binlog_receiver) { 30 | } 31 | 32 | virtual pink::PinkConn *NewPinkConn( 33 | int connfd, 34 | const std::string &ip_port, 35 | pink::ServerThread *thread, 36 | void* worker_specific_data) const override { 37 | return new MasterConn(connfd, ip_port, binlog_receiver_); 38 | } 39 | 40 | private: 41 | BinlogReceiverThread* binlog_receiver_; 42 | }; 43 | 44 | class Handles : public pink::ServerHandle { 45 | public: 46 | explicit Handles(BinlogReceiverThread* binlog_receiver) 47 | : binlog_receiver_(binlog_receiver) { 48 | } 49 | 50 | bool AccessHandle(std::string& ip) const override; 51 | 52 | private: 53 | BinlogReceiverThread* binlog_receiver_; 54 | }; 55 | 56 | MasterConnFactory conn_factory_; 57 | Handles handles_; 58 | pink::ServerThread* thread_rep_; 59 | }; 60 | #endif 61 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/master_conn.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include "master_conn.h" 8 | #include "binlog_receiver_thread.h" 9 | #include "include/pika_command.h" 10 | #include "include/pika_slot.h" 11 | 12 | #include "pika_port.h" 13 | 14 | extern PikaPort *g_pika_port; 15 | 16 | MasterConn::MasterConn(int fd, std::string ip_port, 17 | BinlogReceiverThread* binlog_receiver) 18 | : RedisConn(fd, ip_port, NULL), 19 | self_thread_(binlog_receiver) { 20 | } 21 | 22 | void MasterConn::RestoreArgs(pink::RedisCmdArgsType& argv) { 23 | raw_args_.clear(); 24 | size_t num = argv.size(); 25 | if (argv.size() > 4 && *(argv.end() - 4) == kPikaBinlogMagic) { 26 | num = argv.size()- 4; 27 | } 28 | if (argv[1].find(SlotKeyPrefix) != std::string::npos) { 29 | return; 30 | } 31 | RedisAppendLen(raw_args_, num, "*"); 32 | PikaCmdArgsType::const_iterator it = argv.begin(); 33 | for (size_t idx = 0; idx < num && it != argv.end(); ++ it, ++ idx) { 34 | RedisAppendLen(raw_args_, (*it).size(), "$"); 35 | RedisAppendContent(raw_args_, *it); 36 | } 37 | } 38 | 39 | int MasterConn::DealMessage(pink::RedisCmdArgsType& argv, std::string* response) { 40 | //no reply 41 | //eq set_is_reply(false); 42 | 43 | // if (argv.empty()) { 44 | if (argv.size() < 5) { // special chars: __PIKA_X#$SKGI\r\n1\r\n[\r\n 45 | // return -2; 46 | return 0; 47 | } 48 | 49 | // if (argv[0] == "auth") { 50 | // return 0; 51 | // } 52 | 53 | RestoreArgs(argv); 54 | if (raw_args_.size() == 0) { 55 | return 0; 56 | } 57 | 58 | // //g_pika_port->logger_->Lock(); 59 | // g_pika_port->logger()->Put(raw_args_); 60 | // //g_pika_port->logger_->Unlock(); 61 | 62 | std::string key(" "); 63 | if (1 < argv.size()) { 64 | key = argv[1]; 65 | } 66 | int ret = g_pika_port->SendRedisCommand(raw_args_, key); 67 | if (ret != 0) { 68 | DLOG(WARNING) << "send redis command:" << raw_args_ << ", ret:%d" << ret; 69 | } 70 | 71 | return 0; 72 | } 73 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/master_conn.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef MASTER_CONN_H_ 7 | #define MASTER_CONN_H_ 8 | 9 | #include "pink/include/redis_conn.h" 10 | #include "pink/include/pink_thread.h" 11 | #include "include/pika_command.h" 12 | 13 | class BinlogReceiverThread; 14 | 15 | class MasterConn: public pink::RedisConn { 16 | public: 17 | MasterConn(int fd, std::string ip_port, BinlogReceiverThread* binlog_receiver); 18 | virtual ~MasterConn() {} 19 | virtual int DealMessage(pink::RedisCmdArgsType& argv, std::string* response); 20 | 21 | private: 22 | BinlogReceiverThread* self_thread_; 23 | void RestoreArgs(pink::RedisCmdArgsType&); 24 | std::string raw_args_; 25 | }; 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/migrator_thread.cc: -------------------------------------------------------------------------------- 1 | #include "migrator_thread.h" 2 | 3 | MigratorThread::~MigratorThread() { 4 | } 5 | 6 | void MigratorThread::MigrateDB(const char type) { 7 | if (type == nemo::DataType::kKv) { 8 | nemo::KIterator *it = db_->KScan("", "", -1, false); 9 | std::string key, value; 10 | 11 | while (it->Valid()) { 12 | key = it->key(); 13 | value = it->value(); 14 | pink::RedisCmdArgsType argv; 15 | std::string cmd; 16 | 17 | int64_t ttl; 18 | db_->TTL(key, &ttl); 19 | 20 | argv.push_back("SET"); 21 | argv.push_back(key); 22 | argv.push_back(value); 23 | if (ttl > 0) { 24 | argv.push_back("EX"); 25 | argv.push_back(std::to_string(ttl)); 26 | } 27 | 28 | it->Next(); 29 | pink::SerializeRedisCommand(argv, &cmd); 30 | PlusNum(); 31 | cmd = 'k' + cmd; 32 | DispatchKey(cmd); 33 | } 34 | delete it; 35 | } else { 36 | char c_type = 'a'; 37 | switch (type) { 38 | case nemo::DataType::kHSize: 39 | c_type = 'h'; 40 | break; 41 | case nemo::DataType::kSSize: 42 | c_type = 's'; 43 | break; 44 | case nemo::DataType::kLMeta: 45 | c_type = 'l'; 46 | break; 47 | case nemo::DataType::kZSize: 48 | c_type = 'z'; 49 | break; 50 | } 51 | rocksdb::Iterator *it = db_->Scanbytype(c_type); 52 | std::string key_start = "a"; 53 | key_start[0] = type; 54 | it->Seek(key_start); 55 | for (; it->Valid(); it->Next()) { 56 | PlusNum(); 57 | DispatchKey(it->key().ToString()); 58 | } 59 | } 60 | } 61 | 62 | void MigratorThread::DispatchKey(const std::string &key) { 63 | (*senders_)[thread_index_]->LoadKey(key); 64 | thread_index_ = (thread_index_ + 1) % thread_num_; 65 | } 66 | 67 | void *MigratorThread::ThreadMain() { 68 | MigrateDB(type_); 69 | should_exit_ = true; 70 | log_info("%c keys have been dispatched completly", static_cast(type_)); 71 | return NULL; 72 | } 73 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/migrator_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef MIGRATOR_THREAD_H_ 2 | #define MIGRATOR_THREAD_H_ 3 | 4 | #include 5 | #include "nemo.h" 6 | #include "pink/include/redis_cli.h" 7 | #include "pika_sender.h" 8 | 9 | class MigratorThread : public pink::Thread { 10 | public: 11 | MigratorThread(nemo::Nemo *db, std::vector *senders, char type, int thread_num) : 12 | db_(db), 13 | senders_(senders), 14 | type_(type), 15 | thread_num_(thread_num), 16 | thread_index_(0), 17 | num_(0) 18 | { 19 | } 20 | 21 | int64_t num() { 22 | slash::MutexLock l(&num_mutex_); 23 | return num_; 24 | } 25 | 26 | virtual ~ MigratorThread(); 27 | bool should_exit_; 28 | private: 29 | nemo::Nemo *db_; 30 | std::vector *senders_; 31 | char type_; 32 | int thread_num_; 33 | int thread_index_; 34 | 35 | void MigrateDB(const char type); 36 | void DispatchKey(const std::string &key); 37 | 38 | int64_t num_; 39 | slash::Mutex num_mutex_; 40 | 41 | void PlusNum() { 42 | slash::MutexLock l(&num_mutex_); 43 | ++num_; 44 | } 45 | virtual void *ThreadMain(); 46 | }; 47 | #endif 48 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/pika2.3.6/include/pika_client_conn.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef PIKA_CLIENT_CONN_H_ 7 | #define PIKA_CLIENT_CONN_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "pink/include/redis_conn.h" 13 | #include "pink/include/pink_thread.h" 14 | #include "include/pika_command.h" 15 | 16 | class PikaWorkerSpecificData; 17 | 18 | class PikaClientConn: public pink::RedisConn { 19 | public: 20 | PikaClientConn(int fd, std::string ip_port, pink::ServerThread *server_thread, 21 | void* worker_specific_data); 22 | virtual ~PikaClientConn() {} 23 | int DealMessage(PikaCmdArgsType& argv, std::string* response) override; 24 | 25 | bool IsPubSub() { return is_pubsub_; } 26 | void SetIsPubSub(bool is_pubsub) { is_pubsub_ = is_pubsub; } 27 | 28 | private: 29 | pink::ServerThread* const server_thread_; 30 | CmdTable* const cmds_table_; 31 | bool is_pubsub_; 32 | 33 | std::string DoCmd(PikaCmdArgsType& argv, const std::string& opt); 34 | 35 | // Auth related 36 | class AuthStat { 37 | public: 38 | void Init(); 39 | bool IsAuthed(const CmdInfo* const cinfo_ptr); 40 | bool ChecknUpdate(const std::string& arg); 41 | private: 42 | enum StatType { 43 | kNoAuthed = 0, 44 | kAdminAuthed, 45 | kLimitAuthed, 46 | }; 47 | StatType stat_; 48 | }; 49 | AuthStat auth_stat_; 50 | }; 51 | 52 | struct ClientInfo { 53 | int fd; 54 | std::string ip_port; 55 | int64_t last_interaction; 56 | PikaClientConn* conn; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/pika_sender.h: -------------------------------------------------------------------------------- 1 | #ifndef PIKA_SENDER_H_ 2 | #define PIKA_SENDER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "pink/include/bg_thread.h" 9 | #include "pink/include/pink_cli.h" 10 | #include "pink/include/redis_cli.h" 11 | #include "nemo.h" 12 | 13 | class PikaSender : public pink::Thread { 14 | public: 15 | PikaSender(nemo::Nemo *db, std::string ip, int64_t port, std::string password); 16 | virtual ~PikaSender(); 17 | void LoadKey(const std::string &cmd); 18 | void Stop() { 19 | should_exit_ = true; 20 | keys_mutex_.Lock(); 21 | rsignal_.Signal(); 22 | keys_mutex_.Unlock(); 23 | } 24 | int64_t elements() { 25 | return elements_; 26 | } 27 | 28 | void SendCommand(std::string &command, const std::string &key); 29 | int QueueSize() { 30 | slash::MutexLock l(&keys_mutex_); 31 | int len = keys_queue_.size(); 32 | return len; 33 | } 34 | void ConnectRedis(); 35 | private: 36 | pink::PinkCli *cli_; 37 | slash::CondVar rsignal_; 38 | slash::CondVar wsignal_; 39 | nemo::Nemo *db_; 40 | slash::Mutex keys_mutex_; 41 | std::queue keys_queue_; 42 | std::string expire_command_; 43 | std::string ip_; 44 | int port_; 45 | std::string password_; 46 | bool should_exit_; 47 | int64_t elements_; 48 | 49 | virtual void *ThreadMain(); 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/port_conf.h: -------------------------------------------------------------------------------- 1 | #ifndef BINLOG_CONF_H_ 2 | #define BINLOG_CONF_H_ 3 | 4 | #include 5 | 6 | class PortConf { 7 | public: 8 | PortConf() { 9 | local_ip = "127.0.0.1"; 10 | local_port = 0; 11 | master_ip = "127.0.0.1"; 12 | master_port = 0; 13 | forward_ip = "127.0.0.1"; 14 | forward_port = 0; 15 | forward_thread_num = 1; 16 | filenum = size_t(UINT32_MAX); // src/pika_trysync_thread.cc:48 17 | offset = 0; 18 | log_path = "./log/"; 19 | dump_path = "./rsync_dump/"; 20 | } 21 | 22 | public: 23 | size_t filenum; 24 | size_t offset; 25 | std::string local_ip; 26 | int local_port; 27 | std::string master_ip; 28 | int master_port; 29 | std::string forward_ip; 30 | int forward_port; 31 | std::string forward_passwd; 32 | int forward_thread_num; 33 | std::string passwd; 34 | std::string log_path; 35 | std::string dump_path; 36 | }; 37 | 38 | extern PortConf g_port_conf; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/readme.md: -------------------------------------------------------------------------------- 1 | ## pika_port 2 | **pika port is like redis-port. it copy a snapshot of pika to pika/redis/codis and then transfer delta data to them.** 3 | 4 | ## Version list 5 | 6 | > V1.4 7 | * Bug Fix: filter out SlotKeyPrefix when sync snapshot data 8 | 9 | > V1.3 10 | * send redis data asynchronously by RedisSender in multiple thread mode 11 | 12 | > V1.2 13 | * send redis data asynchronously by RedisSender 14 | 15 | > V1.1 16 | * filter out SlotKeyPrefix 17 | * disable ping-pong log 18 | 19 | > V1.0 20 | * Init 21 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/redis_sender.h: -------------------------------------------------------------------------------- 1 | #ifndef REDIS_SENDER_H_ 2 | #define REDIS_SENDER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "pink/include/bg_thread.h" 11 | #include "pink/include/pink_cli.h" 12 | #include "pink/include/redis_cli.h" 13 | #include "nemo.h" 14 | 15 | class RedisSender : public pink::Thread { 16 | public: 17 | RedisSender(int id, std::string ip, int64_t port, std::string password); 18 | virtual ~RedisSender(); 19 | void Stop(void); 20 | int64_t elements() { 21 | return elements_; 22 | } 23 | 24 | void SendRedisCommand(const std::string &command); 25 | 26 | private: 27 | int SendCommand(std::string &command); 28 | void ConnectRedis(); 29 | 30 | private: 31 | int id_; 32 | pink::PinkCli *cli_; 33 | slash::CondVar rsignal_; 34 | slash::CondVar wsignal_; 35 | slash::Mutex commands_mutex_; 36 | std::queue commands_queue_; 37 | std::string ip_; 38 | int port_; 39 | std::string password_; 40 | bool should_exit_; 41 | int64_t elements_; 42 | std::atomic last_write_time_; 43 | 44 | virtual void *ThreadMain(); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/slaveping_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef SLAVEPING_THREAD_H_ 7 | #define SLAVEPING_THREAD_H_ 8 | 9 | #include "pink/include/pink_thread.h" 10 | #include "pink/include/pink_cli.h" 11 | #include "pink/include/redis_cli.h" 12 | #include "slash/include/slash_mutex.h" 13 | #include "slash/include/slash_status.h" 14 | 15 | using slash::Status; 16 | 17 | class SlavepingThread : public pink::Thread { 18 | public: 19 | SlavepingThread(int64_t sid) : sid_(sid), 20 | is_first_send_(true) { 21 | cli_ = pink::NewRedisCli(); 22 | cli_->set_connect_timeout(1500); 23 | }; 24 | virtual ~SlavepingThread() { 25 | StopThread(); 26 | delete cli_; 27 | DLOG(INFO) << " Slaveping thread " << pthread_self() << " exit!!!"; 28 | }; 29 | 30 | Status Send(); 31 | Status RecvProc(); 32 | 33 | private: 34 | int64_t sid_; 35 | bool is_first_send_; 36 | 37 | int sockfd_; 38 | pink::PinkCli *cli_; 39 | 40 | virtual void* ThreadMain(); 41 | 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [[ -f ~/pika_port ]]; then 4 | mv ~/pika_port bin 5 | fi 6 | export LD_LIBRARY_PATH=./bin 7 | 8 | # Usage: pika_port [-h] [-t local_ip -p local_port -i master_ip -o master_port -m forward_ip -n forward_port -x forward_thread_num -y forward_passwd] 9 | # -f filenum -s offset -w password -r rsync_dump_path -l log_path -h -- show this help 10 | # -t -- local host ip(OPTIONAL default: 127.0.0.1) 11 | # -p -- local port(OPTIONAL) 12 | # -i -- master ip(OPTIONAL default: 127.0.0.1) 13 | # -o -- master port(REQUIRED) 14 | # -m -- forward ip(OPTIONAL default: 127.0.0.1) 15 | # -n -- forward port(REQUIRED) 16 | # -x -- forward thread num(OPTIONAL default: 1) 17 | # -y -- forward password(OPTIONAL) 18 | # -f -- binlog filenum(OPTIONAL default: local offset) 19 | # -s -- binlog offset(OPTIONAL default: local offset) 20 | # -w -- password for master(OPTIONAL) 21 | # -r -- rsync dump data path(OPTIONAL default: ./rsync_dump) 22 | # -l -- local log path(OPTIONAL default: ./log) 23 | # -d -- daemonize(OPTIONAL) 24 | # example: ./pika_port -t 127.0.0.1 -p 12345 -i 127.0.0.1 -o 9221 -m 127.0.0.1 -n 6379 -x 7 -f 0 -s 0 -w abc -l ./log -r ./rsync_dump -d 25 | 26 | rm -rf ./rsync_dump/ 27 | rm -rf ./sync_log/ 28 | ./bin/pika_port -t 10.33.80.155 -i 10.33.80.155 -o 13333 -m 10.33.80.155 -n 16379 -x 10 -l ./sync_log/ -r ./rsync_dump 29 | -------------------------------------------------------------------------------- /pika-port/pika_port_2/trysync_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef TRYSYNC_THREAD_H_ 7 | #define TRYSYNC_THREAD_H_ 8 | 9 | #include "pink/include/pink_thread.h" 10 | #include "pink/include/redis_cli.h" 11 | 12 | 13 | class TrysyncThread : public pink::Thread { 14 | public: 15 | TrysyncThread() { 16 | cli_ = pink::NewRedisCli(); 17 | cli_->set_connect_timeout(1500); 18 | }; 19 | virtual ~TrysyncThread(); 20 | 21 | private: 22 | int sockfd_; 23 | int64_t sid_; 24 | pink::PinkCli *cli_; 25 | 26 | bool Send(); 27 | bool RecvProc(); 28 | void PrepareRsync(); 29 | bool TryUpdateMasterOffset(); 30 | int Retransmit(); 31 | 32 | virtual void* ThreadMain(); 33 | }; 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/binlog_receiver_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | 8 | #include "pink/include/pink_conn.h" 9 | #include "binlog_receiver_thread.h" 10 | #include "master_conn.h" 11 | #include "pika_port.h" 12 | 13 | #include "conf.h" 14 | 15 | extern PikaPort* g_pika_port; 16 | 17 | BinlogReceiverThread::BinlogReceiverThread(std::string host, int port, int cron_interval) 18 | : conn_factory_(this), handles_(this) { 19 | // thread_rep_ = pink::NewHolyThread(port, &conn_factory_, 20 | // cron_interval, &handles_); 21 | thread_rep_ = pink::NewHolyThread(host, port, &conn_factory_, 22 | cron_interval, &handles_); 23 | // to prevent HolyThread::DoCronTask close the pika sender connection 24 | thread_rep_->set_keepalive_timeout(0); 25 | } 26 | 27 | BinlogReceiverThread::~BinlogReceiverThread() { 28 | thread_rep_->StopThread(); 29 | LOG(INFO) << "BinlogReceiver thread " << thread_rep_->thread_id() << " exit!!!"; 30 | delete thread_rep_; 31 | } 32 | 33 | int BinlogReceiverThread::StartThread() { 34 | return thread_rep_->StartThread(); 35 | } 36 | 37 | bool BinlogReceiverThread::Handles::AccessHandle(std::string& ip) const { 38 | if (ip == "127.0.0.1") { 39 | ip = g_conf.local_ip; 40 | } 41 | LOG(INFO) << "Master Binlog Sender: " << ip << " connecting"; 42 | // if (binlog_receiver_->thread_rep_->conn_num() != 0 || 43 | // !g_pika_port->ShouldAccessConnAsMaster(ip)) { 44 | // LOG(INFO) << "BinlogReceiverThread AccessHandle failed"; 45 | // return false; 46 | // } 47 | g_pika_port->PlusMasterConnection(); 48 | return true; 49 | } 50 | 51 | void BinlogReceiverThread::KillBinlogSender() { 52 | thread_rep_->KillAllConns(); 53 | } 54 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/binlog_receiver_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef BINLOG_RECEIVER_THREAD_H_ 7 | #define BINLOG_RECEIVER_THREAD_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "pink/include/server_thread.h" 13 | #include "slash/include/slash_mutex.h" 14 | #include "pika_define.h" 15 | #include "master_conn.h" 16 | 17 | class BinlogReceiverThread { 18 | public: 19 | BinlogReceiverThread(std::string host, int port, int cron_interval = 0); 20 | virtual ~BinlogReceiverThread(); 21 | int StartThread(); 22 | 23 | void KillBinlogSender(); 24 | 25 | // uint64_t GetnPlusSerial() { 26 | // return serial_++; 27 | // } 28 | 29 | // Cmd* GetCmd(const std::string& opt) { 30 | // return GetCmdFromTable(opt, cmds_); 31 | // } 32 | 33 | 34 | private: 35 | class MasterConnFactory : public pink::ConnFactory { 36 | public: 37 | explicit MasterConnFactory(BinlogReceiverThread* binlog_receiver) 38 | : binlog_receiver_(binlog_receiver) { 39 | } 40 | 41 | virtual pink::PinkConn *NewPinkConn( 42 | int connfd, 43 | const std::string &ip_port, 44 | pink::ServerThread *thread, 45 | void* worker_specific_data) const override { 46 | return new MasterConn(connfd, ip_port, binlog_receiver_); 47 | } 48 | 49 | private: 50 | BinlogReceiverThread* binlog_receiver_; 51 | }; 52 | 53 | class Handles : public pink::ServerHandle { 54 | public: 55 | explicit Handles(BinlogReceiverThread* binlog_receiver) 56 | : binlog_receiver_(binlog_receiver) { 57 | } 58 | 59 | bool AccessHandle(std::string& ip) const override; 60 | 61 | private: 62 | BinlogReceiverThread* binlog_receiver_; 63 | }; 64 | 65 | MasterConnFactory conn_factory_; 66 | Handles handles_; 67 | pink::ServerThread* thread_rep_; 68 | 69 | // CmdTable cmds_; 70 | // uint64_t serial_; 71 | }; 72 | #endif 73 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/conf.h: -------------------------------------------------------------------------------- 1 | #ifndef BINLOG_CONF_H_ 2 | #define BINLOG_CONF_H_ 3 | 4 | #include 5 | #include 6 | 7 | class Conf { 8 | public: 9 | Conf() { 10 | local_ip = "127.0.0.1"; 11 | local_port = 0; 12 | master_ip = "127.0.0.1"; 13 | master_port = 0; 14 | forward_ip = "127.0.0.1"; 15 | forward_port = 0; 16 | forward_thread_num = 1; 17 | filenum = size_t(UINT32_MAX); // src/pika_trysync_thread.cc:48 18 | offset = 0; 19 | log_path = "./log/"; 20 | dump_path = "./rsync_dump/"; 21 | sync_batch_num = 512; 22 | wait_bgsave_timeout = 1800; //30 minutes 23 | exit_if_dbsync = false; 24 | } 25 | 26 | public: 27 | size_t filenum; 28 | size_t offset; 29 | std::string local_ip; 30 | int local_port; 31 | std::string master_ip; 32 | int master_port; 33 | std::string forward_ip; 34 | int forward_port; 35 | std::string forward_passwd; 36 | int forward_thread_num; 37 | std::string passwd; 38 | std::string log_path; 39 | std::string dump_path; 40 | size_t sync_batch_num; 41 | time_t wait_bgsave_timeout; 42 | bool exit_if_dbsync; 43 | }; 44 | 45 | extern Conf g_conf; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/const.cc: -------------------------------------------------------------------------------- 1 | #include "const.h" 2 | 3 | #include "blackwidow/blackwidow.h" 4 | 5 | #include "pika_define.h" 6 | 7 | std::string PikaState(int state) { 8 | switch (state) { 9 | case PIKA_REPL_NO_CONNECT: 10 | return "PIKA_REPL_NO_CONNECT"; 11 | 12 | case PIKA_REPL_CONNECT: 13 | return "PIKA_REPL_CONNECT"; 14 | 15 | case PIKA_REPL_CONNECTING: 16 | return "PIKA_REPL_CONNECTING"; 17 | 18 | case PIKA_REPL_CONNECTED: 19 | return "PIKA_REPL_CONNECTED"; 20 | 21 | case PIKA_REPL_WAIT_DBSYNC: 22 | return "PIKA_REPL_WAIT_DBSYNC"; 23 | 24 | case PIKA_REPL_ERROR: 25 | return "PIKA_REPL_ERROR"; 26 | 27 | default: 28 | return "PIKA_REPL_UNKNOWN"; 29 | } 30 | 31 | return "PIKA_REPL_UNKNOWN"; 32 | } 33 | 34 | std::string PikaRole(int role) { 35 | std::string roleStr = "|"; 36 | if (role == PIKA_ROLE_SINGLE) { 37 | roleStr += " PIKA_ROLE_SINGLE |"; 38 | } 39 | 40 | if (role == PIKA_ROLE_SLAVE) { 41 | roleStr += " PIKA_ROLE_SLAVE |"; 42 | } 43 | 44 | if (role == PIKA_ROLE_MASTER) { 45 | roleStr += " PIKA_ROLE_MASTER |"; 46 | } 47 | 48 | if (role == PIKA_ROLE_DOUBLE_MASTER) { 49 | roleStr += " PIKA_ROLE_DOUBLE_MASTER |"; 50 | } 51 | 52 | if (role == PIKA_ROLE_PORT) { 53 | roleStr += " PIKA_ROLE_PORT |"; 54 | } 55 | 56 | return roleStr; 57 | } 58 | 59 | const char* GetDBTypeString(int type) { 60 | switch (type) { 61 | case int(blackwidow::kStrings) : { 62 | return "blackwidow::kStrings"; 63 | } 64 | 65 | case int(blackwidow::kLists) : { 66 | return "blackwidow::kLists"; 67 | } 68 | 69 | case int(blackwidow::kHashes) : { 70 | return "blackwidow::kHashes"; 71 | } 72 | 73 | case int(blackwidow::kSets) : { 74 | return "blackwidow::kSets"; 75 | } 76 | 77 | case int(blackwidow::kZSets) : { 78 | return "blackwidow::kZSets"; 79 | } 80 | 81 | default: { 82 | return "blackwidow::Unknown"; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/const.h: -------------------------------------------------------------------------------- 1 | #ifndef BINLOG_CONST_H_ 2 | #define BINLOG_CONST_H_ 3 | 4 | // //role 5 | // #define PIKA_ROLE_SINGLE 0 6 | // #define PIKA_ROLE_SLAVE 1 7 | // #define PIKA_ROLE_MASTER 2 8 | // #define PIKA_ROLE_DOUBLE_MASTER 3 9 | #define PIKA_ROLE_PORT 4 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | const int64_t kTestPoint = 500000; 17 | const int64_t kTestNum = LLONG_MAX; 18 | const int64_t kDataSetNum = 5; 19 | 20 | std::string PikaState(int state); 21 | std::string PikaRole(int role) ; 22 | 23 | const std::string SlotKeyPrefix = "_internal:slotkey:4migrate:"; 24 | 25 | const char* GetDBTypeString(int type); 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/deps_source/libgflags.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/pika-port/pika_port_3/deps_source/libgflags.a -------------------------------------------------------------------------------- /pika-port/pika_port_3/deps_source/libglog.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/pika-port/pika_port_3/deps_source/libglog.a -------------------------------------------------------------------------------- /pika-port/pika_port_3/deps_source/libsnappy.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/pika-port/pika_port_3/deps_source/libsnappy.a -------------------------------------------------------------------------------- /pika-port/pika_port_3/deps_source/libunwind.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/pika-port/pika_port_3/deps_source/libunwind.a -------------------------------------------------------------------------------- /pika-port/pika_port_3/master_conn.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef MASTER_CONN_H_ 7 | #define MASTER_CONN_H_ 8 | 9 | /* 10 | * **************Header************** 11 | * | | | 12 | * 2 Bytes 4 Bytes 13 | */ 14 | #define HEADER_LEN 6 15 | 16 | #include "pink/include/pink_conn.h" 17 | #include "pika_command.h" 18 | 19 | #include "binlog_transverter.h" 20 | 21 | class BinlogReceiverThread; 22 | 23 | enum PortTransferOperate{ 24 | kTypePortAuth = 1, 25 | kTypePortBinlog = 2 26 | }; 27 | 28 | class MasterConn: public pink::PinkConn { 29 | public: 30 | MasterConn(int fd, std::string ip_port, void* worker_specific_data); 31 | virtual ~MasterConn(); 32 | 33 | virtual pink::ReadStatus GetRequest(); 34 | virtual pink::WriteStatus SendReply(); 35 | virtual void TryResizeBuffer(); 36 | 37 | pink::ReadStatus ReadRaw(uint32_t count); 38 | pink::ReadStatus ReadHeader(); 39 | pink::ReadStatus ReadBody(uint32_t body_lenth); 40 | void ResetStatus(); 41 | 42 | int32_t FindNextSeparators(const std::string& content, int32_t next_parse_pos); 43 | int32_t GetNextNum(const std::string& content, int32_t next_parse_pos, int32_t pos, long* value); 44 | pink::ReadStatus ParseRedisRESPArray(const std::string& content, pink::RedisCmdArgsType* argv); 45 | 46 | bool ProcessAuth(const pink::RedisCmdArgsType& argv); 47 | bool ProcessBinlogData(const pink::RedisCmdArgsType& argv, const PortBinlogItem& binlog_item); 48 | 49 | private: 50 | char* rbuf_; 51 | uint32_t rbuf_len_; 52 | uint32_t rbuf_size_; 53 | uint32_t rbuf_cur_pos_; 54 | bool is_authed_; 55 | BinlogReceiverThread* binlog_receiver_; 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/migrator_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef MIGRATOR_THREAD_H_ 2 | #define MIGRATOR_THREAD_H_ 3 | 4 | #include 5 | 6 | #include "pink/include/redis_cli.h" 7 | 8 | #include "conf.h" 9 | #include "pika_sender.h" 10 | 11 | class MigratorThread : public pink::Thread { 12 | public: 13 | MigratorThread(void *db, std::vector *senders, int type, int thread_num) : 14 | db_(db), 15 | should_exit_(false), 16 | senders_(senders), 17 | type_(type), 18 | thread_num_(thread_num), 19 | thread_index_(0), 20 | num_(0) { 21 | } 22 | 23 | virtual ~ MigratorThread(); 24 | 25 | int64_t num() { 26 | slash::MutexLock l(&num_mutex_); 27 | return num_; 28 | } 29 | 30 | void Stop() { 31 | should_exit_ = true; 32 | } 33 | 34 | private: 35 | void PlusNum() { 36 | slash::MutexLock l(&num_mutex_); 37 | ++num_; 38 | } 39 | 40 | void DispatchKey(const std::string &command, const std::string& key = ""); 41 | 42 | void MigrateDB(); 43 | void MigrateStringsDB(); 44 | void MigrateListsDB(); 45 | void MigrateHashesDB(); 46 | void MigrateSetsDB(); 47 | void MigrateZsetsDB(); 48 | 49 | virtual void *ThreadMain(); 50 | 51 | private: 52 | void* db_; 53 | bool should_exit_; 54 | 55 | std::vector *senders_; 56 | int type_; 57 | int thread_num_; 58 | int thread_index_; 59 | 60 | int64_t num_; 61 | slash::Mutex num_mutex_; 62 | }; 63 | 64 | #endif 65 | 66 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/pika_sender.h: -------------------------------------------------------------------------------- 1 | #ifndef PIKA_SENDER_H_ 2 | #define PIKA_SENDER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "pink/include/bg_thread.h" 11 | #include "pink/include/pink_cli.h" 12 | #include "pink/include/redis_cli.h" 13 | 14 | #include "conf.h" 15 | 16 | class PikaSender : public pink::Thread { 17 | public: 18 | PikaSender(std::string ip, int64_t port, std::string password); 19 | virtual ~PikaSender(); 20 | void LoadKey(const std::string &cmd); 21 | void Stop(); 22 | 23 | int64_t elements() { return elements_; } 24 | 25 | void SendCommand(std::string &command, const std::string &key); 26 | int QueueSize(); 27 | void ConnectRedis(); 28 | 29 | private: 30 | pink::PinkCli *cli_; 31 | slash::CondVar signal_; 32 | slash::Mutex keys_mutex_; 33 | std::queue keys_queue_; 34 | std::string ip_; 35 | int port_; 36 | std::string password_; 37 | std::atomic should_exit_; 38 | int32_t cnt_; 39 | int64_t elements_; 40 | 41 | virtual void *ThreadMain(); 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/readme.md: -------------------------------------------------------------------------------- 1 | ## pika_port 2 | --- 3 | **written by AlexStocks(github.com/AlexStocks) 2018/09** 4 | 5 | ## Intro 6 | 7 | Pika-port is like redis-port. it copy a snapshot of pika to pika/redis/codis and then transfer delta data to them. 8 | 9 | If u wanna get more details of Pika-port, pls read [Pika笔记](http://alexstocks.github.io/html/pika.html). 10 | 11 | This repo just supports Pika v3.0.x. 12 | 13 | ## Use Case 14 | 15 | * [记一次pika迁移到codis](https://blog.csdn.net/wangwenjie2500/article/details/83858572) 16 | 17 | ## Version list 18 | 19 | > V1.6 20 | * Improvement: add wait-bgsave-timeout to give a warning when pika-port waits bgsave data for a long time 21 | * Improvement: change rsync configure file's module name from document_${master_ip}:${master_port} to document_${slave_ip}:master_port 22 | 23 | > V1.5 24 | * Improvement: add batch parameter to speed up transfering data between pika and pika/codis 25 | 26 | > V1.4 27 | * Bug Fix: filter out SlotKeyPrefix when sync snapshot data 28 | 29 | > V1.3 30 | * send redis data asynchronously by RedisSender in multiple thread mode 31 | 32 | > V1.2 33 | * send redis data asynchronously by RedisSender 34 | 35 | > V1.1 36 | * filter out SlotKeyPrefix 37 | * disable ping-pong log 38 | 39 | > V1.0 40 | * Init 41 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/redis_sender.h: -------------------------------------------------------------------------------- 1 | #ifndef REDIS_SENDER_H_ 2 | #define REDIS_SENDER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "pink/include/bg_thread.h" 11 | #include "pink/include/pink_cli.h" 12 | #include "pink/include/redis_cli.h" 13 | 14 | class RedisSender : public pink::Thread { 15 | public: 16 | RedisSender(int id, std::string ip, int64_t port, std::string password); 17 | virtual ~RedisSender(); 18 | void Stop(void); 19 | int64_t elements() { 20 | return elements_; 21 | } 22 | 23 | void SendRedisCommand(const std::string &command); 24 | 25 | private: 26 | int SendCommand(std::string &command); 27 | void ConnectRedis(); 28 | 29 | private: 30 | int id_; 31 | pink::PinkCli *cli_; 32 | slash::CondVar rsignal_; 33 | slash::CondVar wsignal_; 34 | slash::Mutex commands_mutex_; 35 | std::queue commands_queue_; 36 | std::string ip_; 37 | int port_; 38 | std::string password_; 39 | bool should_exit_; 40 | int32_t cnt_; 41 | int64_t elements_; 42 | std::atomic last_write_time_; 43 | 44 | virtual void *ThreadMain(); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/slaveping_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef SLAVEPING_THREAD_H_ 7 | #define SLAVEPING_THREAD_H_ 8 | 9 | #include "pink/include/pink_thread.h" 10 | #include "pink/include/pink_cli.h" 11 | #include "pink/include/redis_cli.h" 12 | #include "slash/include/slash_mutex.h" 13 | #include "slash/include/slash_status.h" 14 | 15 | #include 16 | 17 | using slash::Status; 18 | 19 | class SlavepingThread : public pink::Thread { 20 | public: 21 | SlavepingThread(int64_t sid) : sid_(sid), 22 | is_first_send_(true) { 23 | cli_ = pink::NewRedisCli(); 24 | cli_->set_connect_timeout(1500); 25 | }; 26 | virtual ~SlavepingThread() { 27 | StopThread(); 28 | delete cli_; 29 | LOG(INFO) << " Slaveping thread " << pthread_self() << " exit!!!"; 30 | }; 31 | 32 | Status Send(); 33 | Status RecvProc(); 34 | 35 | private: 36 | int64_t sid_; 37 | bool is_first_send_; 38 | 39 | int sockfd_; 40 | pink::PinkCli *cli_; 41 | 42 | virtual void* ThreadMain(); 43 | 44 | }; 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/static_pack.sh: -------------------------------------------------------------------------------- 1 | g++ \ 2 | -o static_pika_port \ 3 | ./pika_port.o \ 4 | ./binlog_receiver_thread.o \ 5 | ./migrator_thread.o \ 6 | ./slaveping_thread.o \ 7 | ./binlog_transverter.o \ 8 | ./main.o ./pika_binlog.o \ 9 | ./trysync_thread.o \ 10 | ./redis_sender.o \ 11 | ./pika_sender.o \ 12 | ./const.o \ 13 | ./master_conn.o \ 14 | -L./ \ 15 | -L../../third/rocksdb/ \ 16 | -L../../third/slash/slash/lib \ 17 | -L../../third/pink/pink/lib \ 18 | -L../../third/blackwidow/lib \ 19 | -L./deps_source -lrocksdb -lpthread -lrt -lpink -lslash -lblackwidow -lrocksdb -lz -lbz2 -lsnappy -lglog -lgflags -lunwind -static-libstdc++ -llz4 -lzstd 20 | -------------------------------------------------------------------------------- /pika-port/pika_port_3/trysync_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef TRYSYNC_THREAD_H_ 7 | #define TRYSYNC_THREAD_H_ 8 | 9 | #include 10 | 11 | #include "pink/include/pink_thread.h" 12 | #include "pink/include/redis_cli.h" 13 | #include "pink/include/pink_cli.h" 14 | 15 | #include "pika_sender.h" 16 | #include "migrator_thread.h" 17 | 18 | class TrysyncThread : public pink::Thread { 19 | public: 20 | TrysyncThread() { 21 | cli_ = pink::NewRedisCli(); 22 | cli_->set_connect_timeout(1500); 23 | set_thread_name("TrysyncThread"); 24 | retransmit_flag_ = false; 25 | senders_.resize(0); 26 | migrators_.resize(0); 27 | }; 28 | virtual ~TrysyncThread(); 29 | 30 | void Stop(); 31 | 32 | private: 33 | bool Send(std::string lip); 34 | bool RecvProc(); 35 | void PrepareRsync(); 36 | bool TryUpdateMasterOffset(); 37 | int Retransmit(); 38 | 39 | virtual void* ThreadMain(); 40 | 41 | private: 42 | long sid_; 43 | int sockfd_; 44 | pink::PinkCli *cli_; 45 | 46 | slash::Mutex retransmit_mutex_; 47 | bool retransmit_flag_; 48 | 49 | std::vector senders_; 50 | std::vector migrators_; 51 | }; 52 | 53 | #endif 54 | 55 | -------------------------------------------------------------------------------- /pika-port/tool/bench.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import redis 3 | import string 4 | from random import choice 5 | 6 | def getKeys(length=8,chars=string.ascii_letters+string.digits): 7 | return ''.join([choice(chars) for i in range(length)]) 8 | 9 | def getValue(length=200,chars=string.ascii_letters+string.digits): 10 | return ''.join([choice(chars) for i in range(length)]) 11 | 12 | def setRedis(args): 13 | r = redis.StrictRedis(host=args[0], port=int(args[1])) 14 | 15 | # z, field = "set:transfer:test:pika:1", getKeys(12) 16 | # r.zadd(z, 0, field, 1, '1', 2, '2', 3, '3', 4, '4', 5, '5', 6, '6', 7, '7', 8, '8', 9, '9', 10, '10', 11, '11') 17 | 18 | num = 0 19 | max = 5000000 20 | while int(num) < int(max): 21 | num = num + 1 22 | 23 | # key, value 24 | key, value = "string:transfer:test:" + getKeys(12), getValue(30) 25 | r.set(key, value) 26 | 27 | # list 28 | l = "list:transfer:test:pika" + getKeys(15) 29 | r.lpush(l, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) 30 | 31 | # hashtable 32 | ht, field, value = "hashtable:transfer:test:pika" + getKeys(12), getKeys(12), getValue(30) 33 | r.hmset(ht, {field : value, 34 | field + str(1) : value, 35 | field + str(2) : value, 36 | field + str(3) : value, 37 | field + str(4) : value, 38 | field + str(5) : value, 39 | field + str(6) : value, 40 | field + str(7) : value, 41 | field + str(8) : value, 42 | field + str(9) : value, 43 | field + str(0) : value, 44 | field + str(10) : value, 45 | field + str(11) : value, 46 | }) 47 | 48 | # set 49 | s, field = "set:transfer:test:pika" + getKeys(12), getKeys(12) 50 | r.sadd(s, field, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) 51 | 52 | # zset 53 | z, field = "zset:transfer:test:pika" + getKeys(12), getKeys(12) 54 | r.zadd(z, 0, field, 1, '1', 2, '2', 3, '3', 4, '4', 5, '5', 6, '6', 7, '7', 8, '8', 9, '9', 10, '10', 11, '11') 55 | 56 | if __name__=="__main__": 57 | setRedis(['10.1.1.1', '10000']) 58 | -------------------------------------------------------------------------------- /pika-port/tool/get_missing_keys.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import sys 3 | 4 | """print help""" 5 | def printHelp(): 6 | """ print help prompt 7 | """ 8 | print 'Output redis missing keys that exists in pika:' 9 | print 'usage:' 10 | print ' example: ./get_missing_keys.py pika-host pika-port redis-host redis-port' 11 | 12 | def scanKey(_pikaClient, _redisClient): 13 | num = 0 14 | n, l = _pikaClient.scan(0, count=500) 15 | while True: 16 | for i in l: 17 | s = _redisClient.exists(i) 18 | if s == False: 19 | print i 20 | if n == 0: 21 | return 22 | n,l = _pikaClient.scan(n, count=500) 23 | 24 | if __name__ == "__main__": 25 | if len(sys.argv) < 5: 26 | printHelp() 27 | sys.exit(1) 28 | 29 | pikaHost = sys.argv[1] 30 | pikaPort = int(sys.argv[2]) 31 | redisHost = sys.argv[3] 32 | redisPort = int(sys.argv[4]) 33 | pikaClient = redis.Redis(host=pikaHost, port=int(pikaPort)) 34 | redisClient = redis.Redis(host=redisHost, port=int(redisPort)) 35 | scanKey(pikaClient, redisClient) 36 | 37 | -------------------------------------------------------------------------------- /pika-port/tool/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [[ -f ~/pika_port ]]; then 4 | mv ~/pika_port bin 5 | fi 6 | export LD_LIBRARY_PATH=./bin 7 | 8 | rm -rf ./rsync_dump/ 9 | rm -rf ./sync_log/ 10 | ./bin/pika_port -t 10.1.1.2 -i 10.1.1.1 -o 14443 -m 10.1.1.3 -n 6379 -x 10 -l ./sync_log/ -r ./rsync_dump -b 1024 11 | -------------------------------------------------------------------------------- /pika_to_txt/progress_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "progress_thread.h" 7 | 8 | ProgressThread::ProgressThread(ScanThread* scan_thread) 9 | : scan_thread_(scan_thread) { 10 | } 11 | 12 | void* ProgressThread::ThreadMain() { 13 | while (true) { 14 | bool is_finish = scan_thread_->is_finish(); 15 | printf("\rstrings items: %5d", scan_thread_->scan_number()); 16 | fflush(stdout); 17 | if (is_finish) { 18 | break; 19 | } 20 | } 21 | printf("\nScan strings finished\n"); 22 | return NULL; 23 | } 24 | 25 | -------------------------------------------------------------------------------- /pika_to_txt/progress_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_PROGRESS_THREAD_H_ 7 | #define INCLUDE_PROGRESS_THREAD_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include "pink/include/pink_thread.h" 13 | 14 | #include "scan_thread.h" 15 | 16 | class ProgressThread : public pink::Thread { 17 | public: 18 | ProgressThread(ScanThread* scan_thread); 19 | private: 20 | bool AllClassifyTreadFinish(); 21 | virtual void *ThreadMain(); 22 | ScanThread* scan_thread_; 23 | }; 24 | 25 | #endif // INCLUDE_PROGRESS_THREAD_H_ 26 | -------------------------------------------------------------------------------- /pika_to_txt/scan_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "scan_thread.h" 7 | 8 | extern int32_t scan_batch_limit; 9 | 10 | bool ScanThread::is_finish() { 11 | return is_finish_; 12 | } 13 | 14 | int32_t ScanThread::scan_number() { 15 | return scan_number_; 16 | } 17 | 18 | void* ScanThread::ThreadMain() { 19 | std::string key_start = ""; 20 | std::string key_end = ""; 21 | std::string pattern = "*"; 22 | std::string next_key; 23 | std::vector keys; 24 | std::vector kvs; 25 | 26 | do { 27 | blackwidow_db_->PKScanRange(blackwidow::DataType::kStrings, key_start, key_end, 28 | "*", scan_batch_limit, &keys, &kvs, &next_key); 29 | if (!kvs.empty()) { 30 | scan_number_ += kvs.size(); 31 | std::string data; 32 | for (const auto& kv : kvs) { 33 | slash::PutFixed32(&data, kv.key.size()); 34 | data.append(kv.key); 35 | slash::PutFixed32(&data, kv.value.size()); 36 | data.append(kv.value); 37 | } 38 | kvs.clear(); 39 | key_start = next_key; 40 | write_thread_->Load(data); 41 | } 42 | } while (strcmp(next_key.data(), "")); 43 | is_finish_ = true; 44 | return NULL; 45 | } 46 | 47 | -------------------------------------------------------------------------------- /pika_to_txt/scan_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_SCAN_THREAD_H_ 7 | #define INCLUDE_SCAN_THREAD_H_ 8 | 9 | #include "iostream" 10 | #include "vector" 11 | 12 | #include "slash/include/slash_coding.h" 13 | #include "pink/include/pink_thread.h" 14 | #include "blackwidow/blackwidow.h" 15 | 16 | #include "write_thread.h" 17 | 18 | class ScanThread : public pink::Thread { 19 | public: 20 | ScanThread(WriteThread* write_thread, blackwidow::BlackWidow* blackwidow_db) : 21 | is_finish_(false), 22 | scan_number_(0), 23 | write_thread_(write_thread), 24 | blackwidow_db_(blackwidow_db) {} 25 | bool is_finish(); 26 | int32_t scan_number(); 27 | private: 28 | void *ThreadMain() override; 29 | bool is_finish_; 30 | int32_t scan_number_; 31 | WriteThread* write_thread_; 32 | blackwidow::BlackWidow* blackwidow_db_; 33 | }; 34 | 35 | #endif // INCLUDE_SCAN_THREAD_H_ 36 | -------------------------------------------------------------------------------- /pika_to_txt/write_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "iostream" 7 | #include "fstream" 8 | #include "string" 9 | #include "write_thread.h" 10 | 11 | #define MAX_QUEUE_SIZE 1024 12 | 13 | WriteThread::WriteThread(const std::string& file_name) 14 | : should_stop_(false), 15 | file_name_(file_name), 16 | rsignal_(&data_queue_mutex_), 17 | wsignal_(&data_queue_mutex_) {} 18 | 19 | void WriteThread::Load(const std::string& data) { 20 | data_queue_mutex_.Lock(); 21 | if (data_queue_.size() < MAX_QUEUE_SIZE) { 22 | data_queue_.push(data); 23 | rsignal_.Signal(); 24 | data_queue_mutex_.Unlock(); 25 | } else { 26 | while (data_queue_.size() >= MAX_QUEUE_SIZE) { 27 | wsignal_.Wait(); 28 | } 29 | data_queue_.push(data); 30 | rsignal_.Signal(); 31 | data_queue_mutex_.Unlock(); 32 | } 33 | } 34 | 35 | void WriteThread::Stop() { 36 | data_queue_mutex_.Lock(); 37 | should_stop_ = true; 38 | rsignal_.Signal(); 39 | data_queue_mutex_.Unlock(); 40 | } 41 | 42 | void* WriteThread::ThreadMain() { 43 | std::fstream s(file_name_, s.binary | s.out); 44 | if (!s.is_open()) { 45 | std::cout << "failed to open " << file_name_ << ", exit..." << std::endl; 46 | exit(-1); 47 | } else { 48 | while (!should_stop_ || !data_queue_.empty()) { 49 | data_queue_mutex_.Lock(); 50 | while (data_queue_.empty() && !should_stop_) { 51 | rsignal_.Wait(); 52 | } 53 | data_queue_mutex_.Unlock(); 54 | 55 | if (data_queue_.size() > 0) { 56 | data_queue_mutex_.Lock(); 57 | std::string data = data_queue_.front(); 58 | data_queue_.pop(); 59 | wsignal_.Signal(); 60 | data_queue_mutex_.Unlock(); 61 | s.write(data.c_str(), data.size()); 62 | } 63 | } 64 | } 65 | s.close(); 66 | return NULL; 67 | } 68 | -------------------------------------------------------------------------------- /pika_to_txt/write_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018-present The pika-tools Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_WRITE_THREAD_H_ 7 | #define INCLUDE_WRITE_THREAD_H_ 8 | 9 | #include 10 | 11 | #include "pink/include/pink_thread.h" 12 | #include "slash/include/slash_mutex.h" 13 | 14 | class WriteThread : public pink::Thread { 15 | public: 16 | WriteThread(const std::string& file_name); 17 | void Load(const std::string& data); 18 | void Stop(); 19 | private: 20 | void *ThreadMain() override; 21 | bool should_stop_; 22 | std::string file_name_; 23 | slash::CondVar rsignal_; 24 | slash::CondVar wsignal_; 25 | slash::Mutex data_queue_mutex_; 26 | std::queue data_queue_; 27 | }; 28 | 29 | #endif // INCLUDE_WRITE_THREAD_H_ 30 | -------------------------------------------------------------------------------- /rdb_to_pika/Makefile: -------------------------------------------------------------------------------- 1 | CXX=g++ 2 | 3 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 4 | 5 | 6 | SRC_DIR=./ 7 | 8 | OUT_PUT=./ 9 | 10 | TARGET=redisConn 11 | 12 | THIRD=./../third 13 | 14 | HIREDIS_PATH=$(THIRD)/hiredis-vip 15 | LIB_HIREDIS=$(HIREDIS_PATH)/libhiredis_vip.a 16 | 17 | INCLUDE_PATH=-I./ \ 18 | -I$(HIREDIS_PATH) 19 | 20 | BASE_OBJ := $(wildcard $(SRC_DIR)/*.cc) 21 | BASE_OBJ += $(wildcard $(SRC_DIR)/*.cpp) 22 | BASE_OBJ += $(wildcard $(SRC_DIR)/*.c) 23 | OBJ=$(patsubst %.cc, %.o, $(BASE_OBJ)) 24 | 25 | all:$(TARGET) 26 | echo "successful" 27 | 28 | $(TARGET) : $(OBJ) $(LIB_HIREDIS) 29 | $(CXX) $(OBJ) $(LIB_HIREDIS) -o $(TARGET) 30 | 31 | $(LIB_HIREDIS): 32 | make -C $(HIREDIS_PATH) 33 | 34 | $(OBJ): %.o : %.cc 35 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 36 | .PHONEY:clean 37 | clean: 38 | rm -rf $(TARGET) *.o 39 | -------------------------------------------------------------------------------- /rdb_to_pika/README.md: -------------------------------------------------------------------------------- 1 | # This file is show you how to use this tool 2 | ## First: run install_rdb_tools.sh to install redis-rdb-tools 3 | ## Second: run trans_rdb_to_pro.sh to trans rdbfile to protocol_file 4 | ## Third: run make to generate redisConn 5 | -------------------------------------------------------------------------------- /rdb_to_pika/install_rdb_tools.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #author justforfun 3 | cd ../third/redis-rdb-tools 4 | sudo python setup.py install 5 | cd ../../rdb_to_pika 6 | 7 | 8 | -------------------------------------------------------------------------------- /rdb_to_pika/trans_rdb_to_pro.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #author justforfun 3 | rdb_filename=dump.rdb 4 | if [ $# -gt 0 ] 5 | then 6 | rdb_filename=$1 7 | fi 8 | rdb -c protocol -f protocol_file ${rdb_filename} 9 | -------------------------------------------------------------------------------- /ssdb_to_pika/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 3 | 4 | OBJECT = ssdb_to_pika 5 | 6 | SRC_DIR = . 7 | OUTPUT = . 8 | 9 | SSDB_CLIENT=./libssdb-client.a 10 | 11 | THIRD=./../third 12 | 13 | ROCKSDB_PATH=$(THIRD)/rocksdb 14 | ROCKSDB=$(THIRD)/rocksdb/librocksdb.a 15 | 16 | NEMO_PATH=$(THIRD)/nemo 17 | NEMO=$(THIRD)/nemo/lib/libnemo.a 18 | 19 | NEMO_ROCKSDB_PATH=$(THIRD)/nemo-rocksdb 20 | NEMO_ROCKSDB=$(THIRD)/nemo-rocksdb/lib/libnemodb.a 21 | 22 | BLACKWIDOW_PATH=$(THIRD)/blackwidow 23 | BLACKWIDOW=$(THIRD)/blackwidow/lib/libblackwidow.a 24 | 25 | SLASH_PATH=$(THIRD)/slash 26 | SLASH=$(THIRD)/slash/slash/lib/libslash.a 27 | 28 | INCLUDE_PATH = -I./ \ 29 | -I$(THIRD)/nemo/include \ 30 | -I$(THIRD)/nemo-rocksdb/include \ 31 | -I$(THIRD)/rocksdb \ 32 | -I$(THIRD)/rocksdb/include \ 33 | -I$(THIRD)/blackwidow/include/blackwidow \ 34 | -I$(THIRD)/blackwidow \ 35 | -I$(THIRD)/slash 36 | 37 | 38 | 39 | LIBS = -lrt -pthread -lsnappy -lbz2 -lz 40 | LIBS += $(ROCKSDB_LDFLAGS) 41 | 42 | .PHONY: all clean 43 | 44 | 45 | BASE_OBJS := $(wildcard $(SRC_DIR)/*.cc) 46 | BASE_OBJS += $(wildcard $(SRC_DIR)/*.c) 47 | BASE_OBJS += $(wildcard $(SRC_DIR)/*.cpp) 48 | OBJS = $(patsubst %.cc,%.o,$(BASE_OBJS)) 49 | 50 | 51 | all: $(OBJECT) 52 | @echo "Success, go, go, go..." 53 | 54 | 55 | $(OBJECT): $(OBJS) $(NEMO) $(NEMO_ROCKSDB) $(ROCKSDB) $(BLACKWIDOW) $(SLASH) $(SSDB_CLIENT) 56 | $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(NEMO) $(NEMO_ROCKSDB) $(ROCKSDB) $(BLACKWIDOW) $(SLASH) $(LIBS) $(SSDB_CLIENT) 57 | 58 | $(OBJS): %.o : %.cc 59 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 60 | 61 | $(NEMO): 62 | make -C $(NEMO_PATH) ROCKSDB_PATH=./../rocksdb NEMODB_PATH=./../nemo-rocksdb 63 | 64 | $(NEMO_ROCKSDB): 65 | make -C $(NEMO_ROCKSDB_PATH) ROCKSDB_PATH=./../rocksdb 66 | 67 | $(BLACKWIDOW): 68 | make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=./../rocksdb SLASH_PATH=./../slash 69 | 70 | $(SLASH): 71 | make -C $(SLASH_PATH)/slash 72 | 73 | $(ROCKSDB): 74 | make -C $(ROCKSDB_PATH) -j 24 static_lib 75 | 76 | clean: 77 | rm -rf $(SRC_DIR)/*.o 78 | rm -rf $(OBJECT) 79 | -------------------------------------------------------------------------------- /ssdb_to_pika/libssdb-client.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/ssdb_to_pika/libssdb-client.a -------------------------------------------------------------------------------- /ssdb_to_pika/thread_pool.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "thread_pool.h" 4 | 5 | ThreadPool::ThreadPool(int initial_size) : worker_size_(initial_size), is_started_(false) { 6 | Start(); 7 | } 8 | 9 | void ThreadPool::Start() { 10 | assert(workthreads_.empty()); 11 | is_started_ = true; 12 | workthreads_.reserve(worker_size_); 13 | for (int i = 0; i < worker_size_; ++i) { 14 | workthreads_.emplace_back(new std::thread(std::bind(&ThreadPool::ThreadLoop, this))); 15 | } 16 | } 17 | 18 | ThreadPool::~ThreadPool() { 19 | if (is_started_) { 20 | Stop(); 21 | } 22 | } 23 | 24 | void ThreadPool::Stop() { 25 | { 26 | std::unique_lock lock(mutex_); 27 | while (!taskque_.empty() && is_started_) { 28 | cond_.wait(lock); 29 | } 30 | is_started_ = false; 31 | cond_.notify_all(); 32 | } 33 | for (auto iter = workthreads_.begin(); iter != workthreads_.end(); ++iter) { 34 | (*iter)->join(); 35 | } 36 | workthreads_.clear(); 37 | } 38 | 39 | void ThreadPool::ThreadLoop() { 40 | while (is_started_) { 41 | Task task = Take(); 42 | if (task) { 43 | task(); 44 | } 45 | } 46 | } 47 | 48 | void ThreadPool::AddTask(const Task& tsk) { 49 | std::unique_lock lock(mutex_); 50 | taskque_.emplace_back(std::move(tsk)); 51 | cond_.notify_one(); 52 | } 53 | 54 | ThreadPool::Task ThreadPool::Take() { 55 | std::unique_lock lock(mutex_); 56 | while (taskque_.empty() && is_started_) { 57 | cond_.wait(lock); 58 | } 59 | Task task; 60 | TaskQue::size_type size = taskque_.size(); 61 | if (!taskque_.empty() && is_started_) { 62 | task = taskque_.front(); 63 | taskque_.pop_front(); 64 | assert(size - 1 == taskque_.size()); 65 | } 66 | return task; 67 | } 68 | 69 | -------------------------------------------------------------------------------- /ssdb_to_pika/thread_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef THREADPOOL_H 2 | #define THREADPOOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class ThreadPool{ 13 | public: 14 | typedef std::function Task; 15 | 16 | ThreadPool(int initial_size = 15); 17 | 18 | ~ThreadPool(); 19 | 20 | void Stop(); 21 | 22 | void AddTask(const Task& task); 23 | 24 | ThreadPool(const ThreadPool& rhs) = delete; 25 | 26 | const ThreadPool& operator = (const ThreadPool& rhs) = delete; 27 | 28 | private: 29 | bool is_started() { return is_started_;} 30 | 31 | void Start(); 32 | 33 | void ThreadLoop(); 34 | 35 | Task Take(); 36 | 37 | typedef std::vector> Worker; 38 | typedef std::deque TaskQue; 39 | int worker_size_; 40 | Worker workthreads_; 41 | TaskQue taskque_; 42 | std::mutex mutex_; 43 | std::condition_variable cond_; 44 | bool is_started_; 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | SRC_PATH=. 4 | 5 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 -DNDEBUG -D__STDC_FORMAT_MACROS -Wall -O2 -Wno-sign-compare 6 | 7 | 8 | 9 | TARGET = ssdb_to_pika 10 | 11 | THIRD = ./../third 12 | 13 | HIREDIS_PATH = $(THIRD)/hiredis-vip 14 | HIREDIS = $(HIREDIS_PATH)/libhiredis_vip.a 15 | 16 | NET_PATH = ./net 17 | NET = $(NET_PATH)/libnet.a 18 | 19 | UTIL_PATH = ./util 20 | UTIL = $(UTIL_PATH)/libutil.a 21 | 22 | SSDB_PATH = ./ssdb 23 | SSDB = $(SSDB_PATH)/libssdb.a 24 | 25 | LEVELDB_PATH = ./leveldb-1.20 26 | LEVELDB = $(LEVELDB_PATH)/out-static/libleveldb.a 27 | 28 | LIB_PATH = -L$(HIREDIS_PATH) \ 29 | -L$(NET_PATH) \ 30 | -L$(UTIL_PATH) \ 31 | -L$(SSDB_PATH) \ 32 | -L$(LEVELDB_PATH)/out-static \ 33 | 34 | LIBS = -lssdb \ 35 | -lnet \ 36 | -lutil \ 37 | -lpthread \ 38 | -lhiredis_vip \ 39 | -lleveldb \ 40 | -lsnappy \ 41 | 42 | INCLUDE_PATH = -I./ \ 43 | -I./header \ 44 | -I$(NET_PATH) \ 45 | -I$(UTIL_PATH) \ 46 | -I$(HIREDIS_PATH) \ 47 | -I$(LEVELDB_PATH)/include \ 48 | -I$(SSDB_PATH) \ 49 | 50 | SOURCE := $(wildcard $(SRC_PATH)/*.cc) 51 | 52 | OBJS := $(patsubst %.cc, %.o, $(SOURCE)) 53 | 54 | all:$(TARGET) 55 | @rm -rf ./DB/* 56 | echo "succussful" 57 | 58 | $(TARGET) : $(OBJS) $(HIREDIS) $(NET) $(UTIL) $(SSDB) $(LEVELDB) 59 | $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) 60 | 61 | $(OBJS): %.o : %.cc 62 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 63 | 64 | $(HIREDIS): 65 | make -C $(HIREDIS_PATH) 66 | rm -f $(HIREDIS_PATH)/libhiredis_vip.so 67 | 68 | $(UTIL): 69 | make -C $(UTIL_PATH) 70 | 71 | $(NET): 72 | make -C $(NET_PATH) 73 | 74 | $(LEVELDB): 75 | make -C $(LEVELDB_PATH) 76 | 77 | $(SSDB): 78 | make -C $(SSDB_PATH) 79 | 80 | .PHONY:clean distclean 81 | 82 | clean: 83 | rm -rf ssdb_to_pika $(OBJS) ./DB/data ./DB/meta 84 | 85 | distclean:clean 86 | make -C $(SSDB_PATH) clean 87 | make -C $(UTIL_PATH) clean 88 | make -C $(NET_PATH) clean 89 | make -C $(LEVELDB_PATH) clean 90 | make -C $(HIREDIS_PATH) clean 91 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/header/include.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_INCLUDE_H_ 7 | #define SSDB_INCLUDE_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "version.h" 28 | 29 | #ifndef UINT64_MAX 30 | #define UINT64_MAX 18446744073709551615ULL 31 | #endif 32 | #ifndef INT64_MAX 33 | #define INT64_MAX 0x7fffffffffffffffLL 34 | #endif 35 | 36 | 37 | static inline double microtime(){ 38 | struct timeval now; 39 | gettimeofday(&now, NULL); 40 | double ret = now.tv_sec + now.tv_usec/1000.0/1000.0; 41 | return ret; 42 | } 43 | 44 | static inline int64_t time_ms(){ 45 | struct timeval now; 46 | gettimeofday(&now, NULL); 47 | return (int64_t)now.tv_sec * 1000 + (int64_t)now.tv_usec/1000; 48 | } 49 | 50 | #endif 51 | 52 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/header/slave.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_SLAVE_H_ 7 | #define SSDB_SLAVE_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "binlog.h" 15 | 16 | class Link; 17 | class SSDB; 18 | class redisReply; 19 | class redisContext; 20 | 21 | class Slave{ 22 | private: 23 | uint64_t copy_count; 24 | uint64_t sync_count; 25 | std::string id_; 26 | SSDB *ssdb; 27 | SSDB *meta; 28 | Link *link; 29 | std::string ssdb_ip; 30 | int ssdb_port; 31 | bool is_mirror; 32 | char log_type; 33 | const char* pika_ip; 34 | const char* pika_auth; 35 | int pika_port; 36 | redisReply* reply; 37 | redisContext* conn; 38 | static const int DISCONNECTED = 0; 39 | static const int INIT = 1; 40 | static const int COPY = 2; 41 | static const int SYNC = 4; 42 | static const int OUT_OF_SYNC = 8; 43 | static std::string EXPIRE; 44 | int status; 45 | 46 | void migrate_old_status(); 47 | 48 | std::string status_key(); 49 | void load_status(); 50 | void save_status(); 51 | 52 | volatile bool thread_quit; 53 | pthread_t run_thread_tid; 54 | static void* _run_thread(void *arg); 55 | 56 | int proc(const std::vector &req); 57 | int proc_noop(const Binlog &log, const std::vector &req); 58 | int proc_copy(const Binlog &log, const std::vector &req); 59 | int proc_sync(const Binlog &log, const std::vector &req); 60 | 61 | unsigned int connect_retry; 62 | int connect(); 63 | bool connected(){ 64 | return link != NULL; 65 | } 66 | void createLocalDB(); 67 | void connectToPika(); 68 | public: 69 | int recv_timeout; 70 | uint64_t last_seq; 71 | std::string last_key; 72 | std::string ssdb_auth; 73 | Slave(const char* ssdb_ip, int ssdb_port, const char* pika_ip, 74 | int pika_port, const char* ssdb_auth, const char* pika_auth); 75 | ~Slave(); 76 | void start(); 77 | void stop(); 78 | pthread_t getTid() { 79 | return run_thread_tid; 80 | } 81 | void set_id(const std::string &id); 82 | }; 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/header/version.h: -------------------------------------------------------------------------------- 1 | #ifndef SSDB_DEPS_H 2 | #ifndef SSDB_VERSION 3 | #define SSDB_VERSION "1.9.7" 4 | #endif 5 | #endif 6 | #ifndef IOS 7 | #include 8 | //#include 9 | #endif 10 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/.gitignore: -------------------------------------------------------------------------------- 1 | build_config.mk 2 | *.a 3 | *.o 4 | *.dylib* 5 | *.so 6 | *.so.* 7 | *_test 8 | db_bench 9 | leveldbutil 10 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | compiler: 3 | - clang 4 | - gcc 5 | os: 6 | - linux 7 | - osx 8 | sudo: false 9 | before_install: 10 | - echo $LANG 11 | - echo $LC_ALL 12 | script: 13 | - make -j 4 check 14 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | Johan Bilien 13 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We'd love to accept your code patches! However, before we can take them, we 4 | have to jump a couple of legal hurdles. 5 | 6 | ## Contributor License Agreements 7 | 8 | Please fill out either the individual or corporate Contributor License 9 | Agreement as appropriate. 10 | 11 | * If you are an individual writing original source code and you're sure you 12 | own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual). 13 | * If you work for a company that wants to allow you to contribute your work, 14 | then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). 15 | 16 | Follow either of the two links above to access the appropriate CLA and 17 | instructions for how to sign and return it. 18 | 19 | ## Submitting a Patch 20 | 21 | 1. Sign the contributors license agreement above. 22 | 2. Decide which code you want to submit. A submission should be a set of changes 23 | that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues). 24 | Please don't mix more than one logical change per submission, because it makes 25 | the history hard to follow. If you want to make a change 26 | (e.g. add a sample or feature) that doesn't have a corresponding issue in the 27 | issue tracker, please create one. 28 | 3. **Submitting**: When you are ready to submit, send us a Pull Request. Be 29 | sure to include the issue number you fixed and the name you used to sign 30 | the CLA. 31 | 32 | ## Writing Code ## 33 | 34 | If your contribution contains code, please make sure that it follows 35 | [the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml). 36 | Otherwise we will have to ask you to make changes, and that's no fun for anyone. 37 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/NEWS: -------------------------------------------------------------------------------- 1 | Release 1.2 2011-05-16 2 | ---------------------- 3 | 4 | Fixes for larger databases (tested up to one billion 100-byte entries, 5 | i.e., ~100GB). 6 | 7 | (1) Place hard limit on number of level-0 files. This fixes errors 8 | of the form "too many open files". 9 | 10 | (2) Fixed memtable management. Before the fix, a heavy write burst 11 | could cause unbounded memory usage. 12 | 13 | A fix for a logging bug where the reader would incorrectly complain 14 | about corruption. 15 | 16 | Allow public access to WriteBatch contents so that users can easily 17 | wrap a DB. 18 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/TODO: -------------------------------------------------------------------------------- 1 | ss 2 | - Stats 3 | 4 | db 5 | - Maybe implement DB::BulkDeleteForRange(start_key, end_key) 6 | that would blow away files whose ranges are entirely contained 7 | within [start_key..end_key]? For Chrome, deletion of obsolete 8 | object stores, etc. can be done in the background anyway, so 9 | probably not that important. 10 | - There have been requests for MultiGet. 11 | 12 | After a range is completely deleted, what gets rid of the 13 | corresponding files if we do no future changes to that range. Make 14 | the conditions for triggering compactions fire in more situations? 15 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 6 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 7 | 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | 12 | struct Options; 13 | struct FileMetaData; 14 | 15 | class Env; 16 | class Iterator; 17 | class TableCache; 18 | class VersionEdit; 19 | 20 | // Build a Table file from the contents of *iter. The generated file 21 | // will be named according to meta->number. On success, the rest of 22 | // *meta will be filled with metadata about the generated table. 23 | // If no data is present in *iter, meta->file_size will be set to 24 | // zero, and no Table file will be produced. 25 | extern Status BuildTable(const std::string& dbname, 26 | Env* env, 27 | const Options& options, 28 | TableCache* table_cache, 29 | Iterator* iter, 30 | FileMetaData* meta); 31 | 32 | } // namespace leveldb 33 | 34 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 35 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 6 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 7 | 8 | #include 9 | #include "leveldb/db.h" 10 | #include "db/dbformat.h" 11 | 12 | namespace leveldb { 13 | 14 | class DBImpl; 15 | 16 | // Return a new iterator that converts internal keys (yielded by 17 | // "*internal_iter") that were live at the specified "sequence" number 18 | // into appropriate user keys. 19 | extern Iterator* NewDBIterator( 20 | DBImpl* db, 21 | const Comparator* user_key_comparator, 22 | Iterator* internal_iter, 23 | SequenceNumber sequence, 24 | uint32_t seed); 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 29 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/leveldbutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "leveldb/dumpfile.h" 7 | #include "leveldb/env.h" 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | namespace { 12 | 13 | class StdoutPrinter : public WritableFile { 14 | public: 15 | virtual Status Append(const Slice& data) { 16 | fwrite(data.data(), 1, data.size(), stdout); 17 | return Status::OK(); 18 | } 19 | virtual Status Close() { return Status::OK(); } 20 | virtual Status Flush() { return Status::OK(); } 21 | virtual Status Sync() { return Status::OK(); } 22 | }; 23 | 24 | bool HandleDumpCommand(Env* env, char** files, int num) { 25 | StdoutPrinter printer; 26 | bool ok = true; 27 | for (int i = 0; i < num; i++) { 28 | Status s = DumpFile(env, files[i], &printer); 29 | if (!s.ok()) { 30 | fprintf(stderr, "%s\n", s.ToString().c_str()); 31 | ok = false; 32 | } 33 | } 34 | return ok; 35 | } 36 | 37 | } // namespace 38 | } // namespace leveldb 39 | 40 | static void Usage() { 41 | fprintf( 42 | stderr, 43 | "Usage: leveldbutil command...\n" 44 | " dump files... -- dump contents of specified files\n" 45 | ); 46 | } 47 | 48 | int main(int argc, char** argv) { 49 | leveldb::Env* env = leveldb::Env::Default(); 50 | bool ok = true; 51 | if (argc < 2) { 52 | Usage(); 53 | ok = false; 54 | } else { 55 | std::string command = argv[1]; 56 | if (command == "dump") { 57 | ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); 58 | } else { 59 | Usage(); 60 | ok = false; 61 | } 62 | } 63 | return (ok ? 0 : 1); 64 | } 65 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Log format information shared by reader and writer. 6 | // See ../doc/log_format.md for more detail. 7 | 8 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 9 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 10 | 11 | namespace leveldb { 12 | namespace log { 13 | 14 | enum RecordType { 15 | // Zero is reserved for preallocated files 16 | kZeroType = 0, 17 | 18 | kFullType = 1, 19 | 20 | // For fragments 21 | kFirstType = 2, 22 | kMiddleType = 3, 23 | kLastType = 4 24 | }; 25 | static const int kMaxRecordType = kLastType; 26 | 27 | static const int kBlockSize = 32768; 28 | 29 | // Header is checksum (4 bytes), length (2 bytes), type (1 byte). 30 | static const int kHeaderSize = 4 + 2 + 1; 31 | 32 | } // namespace log 33 | } // namespace leveldb 34 | 35 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 36 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ 7 | 8 | #include 9 | #include "db/log_format.h" 10 | #include "leveldb/slice.h" 11 | #include "leveldb/status.h" 12 | 13 | namespace leveldb { 14 | 15 | class WritableFile; 16 | 17 | namespace log { 18 | 19 | class Writer { 20 | public: 21 | // Create a writer that will append data to "*dest". 22 | // "*dest" must be initially empty. 23 | // "*dest" must remain live while this Writer is in use. 24 | explicit Writer(WritableFile* dest); 25 | 26 | // Create a writer that will append data to "*dest". 27 | // "*dest" must have initial length "dest_length". 28 | // "*dest" must remain live while this Writer is in use. 29 | Writer(WritableFile* dest, uint64_t dest_length); 30 | 31 | ~Writer(); 32 | 33 | Status AddRecord(const Slice& slice); 34 | 35 | private: 36 | WritableFile* dest_; 37 | int block_offset_; // Current offset in block 38 | 39 | // crc32c values for all supported record types. These are 40 | // pre-computed to reduce the overhead of computing the crc of the 41 | // record type stored in the header. 42 | uint32_t type_crc_[kMaxRecordType + 1]; 43 | 44 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 45 | 46 | // No copying allowed 47 | Writer(const Writer&); 48 | void operator=(const Writer&); 49 | }; 50 | 51 | } // namespace log 52 | } // namespace leveldb 53 | 54 | #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ 55 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ 6 | #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/db.h" 10 | 11 | namespace leveldb { 12 | 13 | class SnapshotList; 14 | 15 | // Snapshots are kept in a doubly-linked list in the DB. 16 | // Each SnapshotImpl corresponds to a particular sequence number. 17 | class SnapshotImpl : public Snapshot { 18 | public: 19 | SequenceNumber number_; // const after creation 20 | 21 | private: 22 | friend class SnapshotList; 23 | 24 | // SnapshotImpl is kept in a doubly-linked circular list 25 | SnapshotImpl* prev_; 26 | SnapshotImpl* next_; 27 | 28 | SnapshotList* list_; // just for sanity checks 29 | }; 30 | 31 | class SnapshotList { 32 | public: 33 | SnapshotList() { 34 | list_.prev_ = &list_; 35 | list_.next_ = &list_; 36 | } 37 | 38 | bool empty() const { return list_.next_ == &list_; } 39 | SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } 40 | SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } 41 | 42 | const SnapshotImpl* New(SequenceNumber seq) { 43 | SnapshotImpl* s = new SnapshotImpl; 44 | s->number_ = seq; 45 | s->list_ = this; 46 | s->next_ = &list_; 47 | s->prev_ = list_.prev_; 48 | s->prev_->next_ = s; 49 | s->next_->prev_ = s; 50 | return s; 51 | } 52 | 53 | void Delete(const SnapshotImpl* s) { 54 | assert(s->list_ == this); 55 | s->prev_->next_ = s->next_; 56 | s->next_->prev_ = s->prev_; 57 | delete s; 58 | } 59 | 60 | private: 61 | // Dummy head of doubly-linked list of snapshots 62 | SnapshotImpl list_; 63 | }; 64 | 65 | } // namespace leveldb 66 | 67 | #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ 68 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/table_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Thread-safe (provides internal synchronization) 6 | 7 | #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 8 | #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 9 | 10 | #include 11 | #include 12 | #include "db/dbformat.h" 13 | #include "leveldb/cache.h" 14 | #include "leveldb/table.h" 15 | #include "port/port.h" 16 | 17 | namespace leveldb { 18 | 19 | class Env; 20 | 21 | class TableCache { 22 | public: 23 | TableCache(const std::string& dbname, const Options* options, int entries); 24 | ~TableCache(); 25 | 26 | // Return an iterator for the specified file number (the corresponding 27 | // file length must be exactly "file_size" bytes). If "tableptr" is 28 | // non-NULL, also sets "*tableptr" to point to the Table object 29 | // underlying the returned iterator, or NULL if no Table object underlies 30 | // the returned iterator. The returned "*tableptr" object is owned by 31 | // the cache and should not be deleted, and is valid for as long as the 32 | // returned iterator is live. 33 | Iterator* NewIterator(const ReadOptions& options, 34 | uint64_t file_number, 35 | uint64_t file_size, 36 | Table** tableptr = NULL); 37 | 38 | // If a seek to internal key "k" in specified file finds an entry, 39 | // call (*handle_result)(arg, found_key, found_value). 40 | Status Get(const ReadOptions& options, 41 | uint64_t file_number, 42 | uint64_t file_size, 43 | const Slice& k, 44 | void* arg, 45 | void (*handle_result)(void*, const Slice&, const Slice&)); 46 | 47 | // Evict any entry for the specified file number 48 | void Evict(uint64_t file_number); 49 | 50 | private: 51 | Env* const env_; 52 | const std::string dbname_; 53 | const Options* options_; 54 | Cache* cache_; 55 | 56 | Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); 57 | }; 58 | 59 | } // namespace leveldb 60 | 61 | #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 62 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/version_edit.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | static void TestEncodeDecode(const VersionEdit& edit) { 11 | std::string encoded, encoded2; 12 | edit.EncodeTo(&encoded); 13 | VersionEdit parsed; 14 | Status s = parsed.DecodeFrom(encoded); 15 | ASSERT_TRUE(s.ok()) << s.ToString(); 16 | parsed.EncodeTo(&encoded2); 17 | ASSERT_EQ(encoded, encoded2); 18 | } 19 | 20 | class VersionEditTest { }; 21 | 22 | TEST(VersionEditTest, EncodeDecode) { 23 | static const uint64_t kBig = 1ull << 50; 24 | 25 | VersionEdit edit; 26 | for (int i = 0; i < 4; i++) { 27 | TestEncodeDecode(edit); 28 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 29 | InternalKey("foo", kBig + 500 + i, kTypeValue), 30 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 31 | edit.DeleteFile(4, kBig + 700 + i); 32 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 33 | } 34 | 35 | edit.SetComparatorName("foo"); 36 | edit.SetLogNumber(kBig + 100); 37 | edit.SetNextFile(kBig + 200); 38 | edit.SetLastSequence(kBig + 1000); 39 | TestEncodeDecode(edit); 40 | } 41 | 42 | } // namespace leveldb 43 | 44 | int main(int argc, char** argv) { 45 | return leveldb::test::RunAllTests(); 46 | } 47 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/write_batch.h" 10 | 11 | namespace leveldb { 12 | 13 | class MemTable; 14 | 15 | // WriteBatchInternal provides static methods for manipulating a 16 | // WriteBatch that we don't want in the public WriteBatch interface. 17 | class WriteBatchInternal { 18 | public: 19 | // Return the number of entries in the batch. 20 | static int Count(const WriteBatch* batch); 21 | 22 | // Set the count for the number of entries in the batch. 23 | static void SetCount(WriteBatch* batch, int n); 24 | 25 | // Return the sequence number for the start of this batch. 26 | static SequenceNumber Sequence(const WriteBatch* batch); 27 | 28 | // Store the specified number as the sequence number for the start of 29 | // this batch. 30 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 31 | 32 | static Slice Contents(const WriteBatch* batch) { 33 | return Slice(batch->rep_); 34 | } 35 | 36 | static size_t ByteSize(const WriteBatch* batch) { 37 | return batch->rep_.size(); 38 | } 39 | 40 | static void SetContents(WriteBatch* batch, const Slice& contents); 41 | 42 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 43 | 44 | static void Append(WriteBatch* dst, const WriteBatch* src); 45 | }; 46 | 47 | } // namespace leveldb 48 | 49 | 50 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 51 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/helpers/memenv/memenv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 6 | #define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Env; 11 | 12 | // Returns a new environment that stores its data in memory and delegates 13 | // all non-file-storage tasks to base_env. The caller must delete the result 14 | // when it is no longer needed. 15 | // *base_env must remain live while the result is in use. 16 | Env* NewMemEnv(Env* base_env); 17 | 18 | } // namespace leveldb 19 | 20 | #endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 21 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/include/leveldb/dumpfile.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 7 | 8 | #include 9 | #include "leveldb/env.h" 10 | #include "leveldb/status.h" 11 | 12 | namespace leveldb { 13 | 14 | // Dump the contents of the file named by fname in text format to 15 | // *dst. Makes a sequence of dst->Append() calls; each call is passed 16 | // the newline-terminated text corresponding to a single item found 17 | // in the file. 18 | // 19 | // Returns a non-OK result if fname does not name a leveldb storage 20 | // file, or if the file cannot be read. 21 | Status DumpFile(Env* env, const std::string& fname, WritableFile* dst); 22 | 23 | } // namespace leveldb 24 | 25 | #endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 26 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/issues/issue200_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 200: when iterator switches direction from backward 6 | // to forward, the current key can be yielded unexpectedly if a new 7 | // mutation has been added just before the current key. 8 | 9 | #include "leveldb/db.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class Issue200 { }; 15 | 16 | TEST(Issue200, Test) { 17 | // Get rid of any state from an old run. 18 | std::string dbpath = test::TmpDir() + "/leveldb_issue200_test"; 19 | DestroyDB(dbpath, Options()); 20 | 21 | DB *db; 22 | Options options; 23 | options.create_if_missing = true; 24 | ASSERT_OK(DB::Open(options, dbpath, &db)); 25 | 26 | WriteOptions write_options; 27 | ASSERT_OK(db->Put(write_options, "1", "b")); 28 | ASSERT_OK(db->Put(write_options, "2", "c")); 29 | ASSERT_OK(db->Put(write_options, "3", "d")); 30 | ASSERT_OK(db->Put(write_options, "4", "e")); 31 | ASSERT_OK(db->Put(write_options, "5", "f")); 32 | 33 | ReadOptions read_options; 34 | Iterator *iter = db->NewIterator(read_options); 35 | 36 | // Add an element that should not be reflected in the iterator. 37 | ASSERT_OK(db->Put(write_options, "25", "cd")); 38 | 39 | iter->Seek("5"); 40 | ASSERT_EQ(iter->key().ToString(), "5"); 41 | iter->Prev(); 42 | ASSERT_EQ(iter->key().ToString(), "4"); 43 | iter->Prev(); 44 | ASSERT_EQ(iter->key().ToString(), "3"); 45 | iter->Next(); 46 | ASSERT_EQ(iter->key().ToString(), "4"); 47 | iter->Next(); 48 | ASSERT_EQ(iter->key().ToString(), "5"); 49 | 50 | delete iter; 51 | delete db; 52 | DestroyDB(dbpath, options); 53 | } 54 | 55 | } // namespace leveldb 56 | 57 | int main(int argc, char** argv) { 58 | return leveldb::test::RunAllTests(); 59 | } 60 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_posix.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/port/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_posix.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace port { 13 | 14 | static void PthreadCall(const char* label, int result) { 15 | if (result != 0) { 16 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 17 | abort(); 18 | } 19 | } 20 | 21 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 22 | 23 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 24 | 25 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 26 | 27 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 28 | 29 | CondVar::CondVar(Mutex* mu) 30 | : mu_(mu) { 31 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 32 | } 33 | 34 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 35 | 36 | void CondVar::Wait() { 37 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 38 | } 39 | 40 | void CondVar::Signal() { 41 | PthreadCall("signal", pthread_cond_signal(&cv_)); 42 | } 43 | 44 | void CondVar::SignalAll() { 45 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 46 | } 47 | 48 | void InitOnce(OnceType* once, void (*initializer)()) { 49 | PthreadCall("once", pthread_once(once, initializer)); 50 | } 51 | 52 | } // namespace port 53 | } // namespace leveldb 54 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 6 | #define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 7 | 8 | // Some environments provide custom macros to aid in static thread-safety 9 | // analysis. Provide empty definitions of such macros unless they are already 10 | // defined. 11 | 12 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 13 | #define EXCLUSIVE_LOCKS_REQUIRED(...) 14 | #endif 15 | 16 | #ifndef SHARED_LOCKS_REQUIRED 17 | #define SHARED_LOCKS_REQUIRED(...) 18 | #endif 19 | 20 | #ifndef LOCKS_EXCLUDED 21 | #define LOCKS_EXCLUDED(...) 22 | #endif 23 | 24 | #ifndef LOCK_RETURNED 25 | #define LOCK_RETURNED(x) 26 | #endif 27 | 28 | #ifndef LOCKABLE 29 | #define LOCKABLE 30 | #endif 31 | 32 | #ifndef SCOPED_LOCKABLE 33 | #define SCOPED_LOCKABLE 34 | #endif 35 | 36 | #ifndef EXCLUSIVE_LOCK_FUNCTION 37 | #define EXCLUSIVE_LOCK_FUNCTION(...) 38 | #endif 39 | 40 | #ifndef SHARED_LOCK_FUNCTION 41 | #define SHARED_LOCK_FUNCTION(...) 42 | #endif 43 | 44 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 45 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) 46 | #endif 47 | 48 | #ifndef SHARED_TRYLOCK_FUNCTION 49 | #define SHARED_TRYLOCK_FUNCTION(...) 50 | #endif 51 | 52 | #ifndef UNLOCK_FUNCTION 53 | #define UNLOCK_FUNCTION(...) 54 | #endif 55 | 56 | #ifndef NO_THREAD_SAFETY_ANALYSIS 57 | #define NO_THREAD_SAFETY_ANALYSIS 58 | #endif 59 | 60 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 61 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 7 | 8 | #include 9 | #include 10 | #include "leveldb/iterator.h" 11 | 12 | namespace leveldb { 13 | 14 | struct BlockContents; 15 | class Comparator; 16 | 17 | class Block { 18 | public: 19 | // Initialize the block with the specified contents. 20 | explicit Block(const BlockContents& contents); 21 | 22 | ~Block(); 23 | 24 | size_t size() const { return size_; } 25 | Iterator* NewIterator(const Comparator* comparator); 26 | 27 | private: 28 | uint32_t NumRestarts() const; 29 | 30 | const char* data_; 31 | size_t size_; 32 | uint32_t restart_offset_; // Offset in data_ of restart array 33 | bool owned_; // Block owns data_[] 34 | 35 | // No copying allowed 36 | Block(const Block&); 37 | void operator=(const Block&); 38 | 39 | class Iter; 40 | }; 41 | 42 | } // namespace leveldb 43 | 44 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 45 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/table/block_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 7 | 8 | #include 9 | 10 | #include 11 | #include "leveldb/slice.h" 12 | 13 | namespace leveldb { 14 | 15 | struct Options; 16 | 17 | class BlockBuilder { 18 | public: 19 | explicit BlockBuilder(const Options* options); 20 | 21 | // Reset the contents as if the BlockBuilder was just constructed. 22 | void Reset(); 23 | 24 | // REQUIRES: Finish() has not been called since the last call to Reset(). 25 | // REQUIRES: key is larger than any previously added key 26 | void Add(const Slice& key, const Slice& value); 27 | 28 | // Finish building the block and return a slice that refers to the 29 | // block contents. The returned slice will remain valid for the 30 | // lifetime of this builder or until Reset() is called. 31 | Slice Finish(); 32 | 33 | // Returns an estimate of the current (uncompressed) size of the block 34 | // we are building. 35 | size_t CurrentSizeEstimate() const; 36 | 37 | // Return true iff no entries have been added since the last Reset() 38 | bool empty() const { 39 | return buffer_.empty(); 40 | } 41 | 42 | private: 43 | const Options* options_; 44 | std::string buffer_; // Destination buffer 45 | std::vector restarts_; // Restart points 46 | int counter_; // Number of entries emitted since restart 47 | bool finished_; // Has Finish() been called? 48 | std::string last_key_; 49 | 50 | // No copying allowed 51 | BlockBuilder(const BlockBuilder&); 52 | void operator=(const BlockBuilder&); 53 | }; 54 | 55 | } // namespace leveldb 56 | 57 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 58 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/table/iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/iterator.h" 6 | 7 | namespace leveldb { 8 | 9 | Iterator::Iterator() { 10 | cleanup_.function = NULL; 11 | cleanup_.next = NULL; 12 | } 13 | 14 | Iterator::~Iterator() { 15 | if (cleanup_.function != NULL) { 16 | (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); 17 | for (Cleanup* c = cleanup_.next; c != NULL; ) { 18 | (*c->function)(c->arg1, c->arg2); 19 | Cleanup* next = c->next; 20 | delete c; 21 | c = next; 22 | } 23 | } 24 | } 25 | 26 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 27 | assert(func != NULL); 28 | Cleanup* c; 29 | if (cleanup_.function == NULL) { 30 | c = &cleanup_; 31 | } else { 32 | c = new Cleanup; 33 | c->next = cleanup_.next; 34 | cleanup_.next = c; 35 | } 36 | c->function = func; 37 | c->arg1 = arg1; 38 | c->arg2 = arg2; 39 | } 40 | 41 | namespace { 42 | class EmptyIterator : public Iterator { 43 | public: 44 | EmptyIterator(const Status& s) : status_(s) { } 45 | virtual bool Valid() const { return false; } 46 | virtual void Seek(const Slice& target) { } 47 | virtual void SeekToFirst() { } 48 | virtual void SeekToLast() { } 49 | virtual void Next() { assert(false); } 50 | virtual void Prev() { assert(false); } 51 | Slice key() const { assert(false); return Slice(); } 52 | Slice value() const { assert(false); return Slice(); } 53 | virtual Status status() const { return status_; } 54 | private: 55 | Status status_; 56 | }; 57 | } // namespace 58 | 59 | Iterator* NewEmptyIterator() { 60 | return new EmptyIterator(Status::OK()); 61 | } 62 | 63 | Iterator* NewErrorIterator(const Status& status) { 64 | return new EmptyIterator(status); 65 | } 66 | 67 | } // namespace leveldb 68 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 6 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Comparator; 11 | class Iterator; 12 | 13 | // Return an iterator that provided the union of the data in 14 | // children[0,n-1]. Takes ownership of the child iterators and 15 | // will delete them when the result iterator is deleted. 16 | // 17 | // The result does no duplicate suppression. I.e., if a particular 18 | // key is present in K child iterators, it will be yielded K times. 19 | // 20 | // REQUIRES: n >= 0 21 | extern Iterator* NewMergingIterator( 22 | const Comparator* comparator, Iterator** children, int n); 23 | 24 | } // namespace leveldb 25 | 26 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 27 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/table/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | 10 | namespace leveldb { 11 | 12 | struct ReadOptions; 13 | 14 | // Return a new two level iterator. A two-level iterator contains an 15 | // index iterator whose values point to a sequence of blocks where 16 | // each block is itself a sequence of key,value pairs. The returned 17 | // two-level iterator yields the concatenation of all key/value pairs 18 | // in the sequence of blocks. Takes ownership of "index_iter" and 19 | // will delete it when no longer needed. 20 | // 21 | // Uses a supplied function to convert an index_iter value into 22 | // an iterator over the contents of the corresponding block. 23 | extern Iterator* NewTwoLevelIterator( 24 | Iterator* index_iter, 25 | Iterator* (*block_function)( 26 | void* arg, 27 | const ReadOptions& options, 28 | const Slice& index_value), 29 | void* arg, 30 | const ReadOptions& options); 31 | 32 | } // namespace leveldb 33 | 34 | #endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 35 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/arena.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | #include 7 | 8 | namespace leveldb { 9 | 10 | static const int kBlockSize = 4096; 11 | 12 | Arena::Arena() : memory_usage_(0) { 13 | alloc_ptr_ = NULL; // First allocation will allocate a block 14 | alloc_bytes_remaining_ = 0; 15 | } 16 | 17 | Arena::~Arena() { 18 | for (size_t i = 0; i < blocks_.size(); i++) { 19 | delete[] blocks_[i]; 20 | } 21 | } 22 | 23 | char* Arena::AllocateFallback(size_t bytes) { 24 | if (bytes > kBlockSize / 4) { 25 | // Object is more than a quarter of our block size. Allocate it separately 26 | // to avoid wasting too much space in leftover bytes. 27 | char* result = AllocateNewBlock(bytes); 28 | return result; 29 | } 30 | 31 | // We waste the remaining space in the current block. 32 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 33 | alloc_bytes_remaining_ = kBlockSize; 34 | 35 | char* result = alloc_ptr_; 36 | alloc_ptr_ += bytes; 37 | alloc_bytes_remaining_ -= bytes; 38 | return result; 39 | } 40 | 41 | char* Arena::AllocateAligned(size_t bytes) { 42 | const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; 43 | assert((align & (align-1)) == 0); // Pointer size should be a power of 2 44 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); 45 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 46 | size_t needed = bytes + slop; 47 | char* result; 48 | if (needed <= alloc_bytes_remaining_) { 49 | result = alloc_ptr_ + slop; 50 | alloc_ptr_ += needed; 51 | alloc_bytes_remaining_ -= needed; 52 | } else { 53 | // AllocateFallback always returned aligned memory 54 | result = AllocateFallback(bytes); 55 | } 56 | assert((reinterpret_cast(result) & (align-1)) == 0); 57 | return result; 58 | } 59 | 60 | char* Arena::AllocateNewBlock(size_t block_bytes) { 61 | char* result = new char[block_bytes]; 62 | blocks_.push_back(result); 63 | memory_usage_.NoBarrier_Store( 64 | reinterpret_cast(MemoryUsage() + block_bytes + sizeof(char*))); 65 | return result; 66 | } 67 | 68 | } // namespace leveldb 69 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "port/port.h" 13 | 14 | namespace leveldb { 15 | 16 | class Arena { 17 | public: 18 | Arena(); 19 | ~Arena(); 20 | 21 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 22 | char* Allocate(size_t bytes); 23 | 24 | // Allocate memory with the normal alignment guarantees provided by malloc 25 | char* AllocateAligned(size_t bytes); 26 | 27 | // Returns an estimate of the total memory usage of data allocated 28 | // by the arena. 29 | size_t MemoryUsage() const { 30 | return reinterpret_cast(memory_usage_.NoBarrier_Load()); 31 | } 32 | 33 | private: 34 | char* AllocateFallback(size_t bytes); 35 | char* AllocateNewBlock(size_t block_bytes); 36 | 37 | // Allocation state 38 | char* alloc_ptr_; 39 | size_t alloc_bytes_remaining_; 40 | 41 | // Array of new[] allocated memory blocks 42 | std::vector blocks_; 43 | 44 | // Total memory usage of the arena. 45 | port::AtomicPointer memory_usage_; 46 | 47 | // No copying allowed 48 | Arena(const Arena&); 49 | void operator=(const Arena&); 50 | }; 51 | 52 | inline char* Arena::Allocate(size_t bytes) { 53 | // The semantics of what to return are a bit messy if we allow 54 | // 0-byte allocations, so we disallow them here (we don't need 55 | // them for our internal use). 56 | assert(bytes > 0); 57 | if (bytes <= alloc_bytes_remaining_) { 58 | char* result = alloc_ptr_; 59 | alloc_ptr_ += bytes; 60 | alloc_bytes_remaining_ -= bytes; 61 | return result; 62 | } 63 | return AllocateFallback(bytes); 64 | } 65 | 66 | } // namespace leveldb 67 | 68 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 69 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | #include "util/random.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | class ArenaTest { }; 13 | 14 | TEST(ArenaTest, Empty) { 15 | Arena arena; 16 | } 17 | 18 | TEST(ArenaTest, Simple) { 19 | std::vector > allocated; 20 | Arena arena; 21 | const int N = 100000; 22 | size_t bytes = 0; 23 | Random rnd(301); 24 | for (int i = 0; i < N; i++) { 25 | size_t s; 26 | if (i % (N / 10) == 0) { 27 | s = i; 28 | } else { 29 | s = rnd.OneIn(4000) ? rnd.Uniform(6000) : 30 | (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 31 | } 32 | if (s == 0) { 33 | // Our arena disallows size 0 allocations. 34 | s = 1; 35 | } 36 | char* r; 37 | if (rnd.OneIn(10)) { 38 | r = arena.AllocateAligned(s); 39 | } else { 40 | r = arena.Allocate(s); 41 | } 42 | 43 | for (size_t b = 0; b < s; b++) { 44 | // Fill the "i"th allocation with a known bit pattern 45 | r[b] = i % 256; 46 | } 47 | bytes += s; 48 | allocated.push_back(std::make_pair(s, r)); 49 | ASSERT_GE(arena.MemoryUsage(), bytes); 50 | if (i > N/10) { 51 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 52 | } 53 | } 54 | for (size_t i = 0; i < allocated.size(); i++) { 55 | size_t num_bytes = allocated[i].first; 56 | const char* p = allocated[i].second; 57 | for (size_t b = 0; b < num_bytes; b++) { 58 | // Check the "i"th allocation for the known bit pattern 59 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 60 | } 61 | } 62 | } 63 | 64 | } // namespace leveldb 65 | 66 | int main(int argc, char** argv) { 67 | return leveldb::test::RunAllTests(); 68 | } 69 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ 6 | #define STORAGE_LEVELDB_UTIL_CRC32C_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace crc32c { 13 | 14 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 15 | // crc32c of some string A. Extend() is often used to maintain the 16 | // crc32c of a stream of data. 17 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 18 | 19 | // Return the crc32c of data[0,n-1] 20 | inline uint32_t Value(const char* data, size_t n) { 21 | return Extend(0, data, n); 22 | } 23 | 24 | static const uint32_t kMaskDelta = 0xa282ead8ul; 25 | 26 | // Return a masked representation of crc. 27 | // 28 | // Motivation: it is problematic to compute the CRC of a string that 29 | // contains embedded CRCs. Therefore we recommend that CRCs stored 30 | // somewhere (e.g., in files) should be masked before being stored. 31 | inline uint32_t Mask(uint32_t crc) { 32 | // Rotate right by 15 bits and add a constant. 33 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 34 | } 35 | 36 | // Return the crc whose masked representation is masked_crc. 37 | inline uint32_t Unmask(uint32_t masked_crc) { 38 | uint32_t rot = masked_crc - kMaskDelta; 39 | return ((rot >> 17) | (rot << 15)); 40 | } 41 | 42 | } // namespace crc32c 43 | } // namespace leveldb 44 | 45 | #endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ 46 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/crc32c.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | namespace crc32c { 10 | 11 | class CRC { }; 12 | 13 | TEST(CRC, StandardResults) { 14 | // From rfc3720 section B.4. 15 | char buf[32]; 16 | 17 | memset(buf, 0, sizeof(buf)); 18 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 19 | 20 | memset(buf, 0xff, sizeof(buf)); 21 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 22 | 23 | for (int i = 0; i < 32; i++) { 24 | buf[i] = i; 25 | } 26 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 27 | 28 | for (int i = 0; i < 32; i++) { 29 | buf[i] = 31 - i; 30 | } 31 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 32 | 33 | unsigned char data[48] = { 34 | 0x01, 0xc0, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x00, 0x00, 37 | 0x00, 0x00, 0x00, 0x00, 38 | 0x14, 0x00, 0x00, 0x00, 39 | 0x00, 0x00, 0x04, 0x00, 40 | 0x00, 0x00, 0x00, 0x14, 41 | 0x00, 0x00, 0x00, 0x18, 42 | 0x28, 0x00, 0x00, 0x00, 43 | 0x00, 0x00, 0x00, 0x00, 44 | 0x02, 0x00, 0x00, 0x00, 45 | 0x00, 0x00, 0x00, 0x00, 46 | }; 47 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 48 | } 49 | 50 | TEST(CRC, Values) { 51 | ASSERT_NE(Value("a", 1), Value("foo", 3)); 52 | } 53 | 54 | TEST(CRC, Extend) { 55 | ASSERT_EQ(Value("hello world", 11), 56 | Extend(Value("hello ", 6), "world", 5)); 57 | } 58 | 59 | TEST(CRC, Mask) { 60 | uint32_t crc = Value("foo", 3); 61 | ASSERT_NE(crc, Mask(crc)); 62 | ASSERT_NE(crc, Mask(Mask(crc))); 63 | ASSERT_EQ(crc, Unmask(Mask(crc))); 64 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 65 | } 66 | 67 | } // namespace crc32c 68 | } // namespace leveldb 69 | 70 | int main(int argc, char** argv) { 71 | return leveldb::test::RunAllTests(); 72 | } 73 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/env_posix_test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 6 | #define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class EnvPosixTest; 11 | 12 | // A helper for the POSIX Env to facilitate testing. 13 | class EnvPosixTestHelper { 14 | private: 15 | friend class EnvPosixTest; 16 | 17 | // Set the maximum number of read-only files that will be opened. 18 | // Must be called before creating an Env. 19 | static void SetReadOnlyFDLimit(int limit); 20 | 21 | // Set the maximum number of read-only files that will be mapped via mmap. 22 | // Must be called before creating an Env. 23 | static void SetReadOnlyMMapLimit(int limit); 24 | }; 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 29 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/filter_policy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | namespace leveldb { 8 | 9 | FilterPolicy::~FilterPolicy() { } 10 | 11 | } // namespace leveldb 12 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "util/coding.h" 7 | #include "util/hash.h" 8 | 9 | // The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through 10 | // between switch labels. The real definition should be provided externally. 11 | // This one is a fallback version for unsupported compilers. 12 | #ifndef FALLTHROUGH_INTENDED 13 | #define FALLTHROUGH_INTENDED do { } while (0) 14 | #endif 15 | 16 | namespace leveldb { 17 | 18 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 19 | // Similar to murmur hash 20 | const uint32_t m = 0xc6a4a793; 21 | const uint32_t r = 24; 22 | const char* limit = data + n; 23 | uint32_t h = seed ^ (n * m); 24 | 25 | // Pick up four bytes at a time 26 | while (data + 4 <= limit) { 27 | uint32_t w = DecodeFixed32(data); 28 | data += 4; 29 | h += w; 30 | h *= m; 31 | h ^= (h >> 16); 32 | } 33 | 34 | // Pick up remaining bytes 35 | switch (limit - data) { 36 | case 3: 37 | h += static_cast(data[2]) << 16; 38 | FALLTHROUGH_INTENDED; 39 | case 2: 40 | h += static_cast(data[1]) << 8; 41 | FALLTHROUGH_INTENDED; 42 | case 1: 43 | h += static_cast(data[0]); 44 | h *= m; 45 | h ^= (h >> r); 46 | break; 47 | } 48 | return h; 49 | } 50 | 51 | 52 | } // namespace leveldb 53 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Simple hash function used for internal data structures 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 8 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 9 | 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 16 | 17 | } 18 | 19 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 20 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/hash_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/hash.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | class HASH { }; 11 | 12 | TEST(HASH, SignedUnsignedIssue) { 13 | const unsigned char data1[1] = {0x62}; 14 | const unsigned char data2[2] = {0xc3, 0x97}; 15 | const unsigned char data3[3] = {0xe2, 0x99, 0xa5}; 16 | const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32}; 17 | const unsigned char data5[48] = { 18 | 0x01, 0xc0, 0x00, 0x00, 19 | 0x00, 0x00, 0x00, 0x00, 20 | 0x00, 0x00, 0x00, 0x00, 21 | 0x00, 0x00, 0x00, 0x00, 22 | 0x14, 0x00, 0x00, 0x00, 23 | 0x00, 0x00, 0x04, 0x00, 24 | 0x00, 0x00, 0x00, 0x14, 25 | 0x00, 0x00, 0x00, 0x18, 26 | 0x28, 0x00, 0x00, 0x00, 27 | 0x00, 0x00, 0x00, 0x00, 28 | 0x02, 0x00, 0x00, 0x00, 29 | 0x00, 0x00, 0x00, 0x00, 30 | }; 31 | 32 | ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); 33 | ASSERT_EQ( 34 | Hash(reinterpret_cast(data1), sizeof(data1), 0xbc9f1d34), 35 | 0xef1345c4); 36 | ASSERT_EQ( 37 | Hash(reinterpret_cast(data2), sizeof(data2), 0xbc9f1d34), 38 | 0x5b663814); 39 | ASSERT_EQ( 40 | Hash(reinterpret_cast(data3), sizeof(data3), 0xbc9f1d34), 41 | 0x323c078f); 42 | ASSERT_EQ( 43 | Hash(reinterpret_cast(data4), sizeof(data4), 0xbc9f1d34), 44 | 0xed21633a); 45 | ASSERT_EQ( 46 | Hash(reinterpret_cast(data5), sizeof(data5), 0x12345678), 47 | 0xf333dabb); 48 | } 49 | 50 | } // namespace leveldb 51 | 52 | int main(int argc, char** argv) { 53 | return leveldb::test::RunAllTests(); 54 | } 55 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 6 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Histogram { 13 | public: 14 | Histogram() { } 15 | ~Histogram() { } 16 | 17 | void Clear(); 18 | void Add(double value); 19 | void Merge(const Histogram& other); 20 | 21 | std::string ToString() const; 22 | 23 | private: 24 | double min_; 25 | double max_; 26 | double num_; 27 | double sum_; 28 | double sum_squares_; 29 | 30 | enum { kNumBuckets = 154 }; 31 | static const double kBucketLimit[kNumBuckets]; 32 | double buckets_[kNumBuckets]; 33 | 34 | double Median() const; 35 | double Percentile(double p) const; 36 | double Average() const; 37 | double StandardDeviation() const; 38 | }; 39 | 40 | } // namespace leveldb 41 | 42 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 43 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "leveldb/env.h" 12 | #include "leveldb/slice.h" 13 | 14 | namespace leveldb { 15 | 16 | void AppendNumberTo(std::string* str, uint64_t num) { 17 | char buf[30]; 18 | snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); 19 | str->append(buf); 20 | } 21 | 22 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 23 | for (size_t i = 0; i < value.size(); i++) { 24 | char c = value[i]; 25 | if (c >= ' ' && c <= '~') { 26 | str->push_back(c); 27 | } else { 28 | char buf[10]; 29 | snprintf(buf, sizeof(buf), "\\x%02x", 30 | static_cast(c) & 0xff); 31 | str->append(buf); 32 | } 33 | } 34 | } 35 | 36 | std::string NumberToString(uint64_t num) { 37 | std::string r; 38 | AppendNumberTo(&r, num); 39 | return r; 40 | } 41 | 42 | std::string EscapeString(const Slice& value) { 43 | std::string r; 44 | AppendEscapedStringTo(&r, value); 45 | return r; 46 | } 47 | 48 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 49 | uint64_t v = 0; 50 | int digits = 0; 51 | while (!in->empty()) { 52 | char c = (*in)[0]; 53 | if (c >= '0' && c <= '9') { 54 | ++digits; 55 | // |delta| intentionally unit64_t to avoid Android crash (see log). 56 | const uint64_t delta = (c - '0'); 57 | static const uint64_t kMaxUint64 = ~static_cast(0); 58 | if (v > kMaxUint64/10 || 59 | (v == kMaxUint64/10 && delta > kMaxUint64%10)) { 60 | // Overflow 61 | return false; 62 | } 63 | v = (v * 10) + delta; 64 | in->remove_prefix(1); 65 | } else { 66 | break; 67 | } 68 | } 69 | *val = v; 70 | return (digits > 0); 71 | } 72 | 73 | } // namespace leveldb 74 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Slice; 19 | class WritableFile; 20 | 21 | // Append a human-readable printout of "num" to *str 22 | extern void AppendNumberTo(std::string* str, uint64_t num); 23 | 24 | // Append a human-readable printout of "value" to *str. 25 | // Escapes any non-printable characters found in "value". 26 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 27 | 28 | // Return a human-readable printout of "num" 29 | extern std::string NumberToString(uint64_t num); 30 | 31 | // Return a human-readable version of "value". 32 | // Escapes any non-printable characters found in "value". 33 | extern std::string EscapeString(const Slice& value); 34 | 35 | // Parse a human-readable number from "*in" into *value. On success, 36 | // advances "*in" past the consumed number and sets "*val" to the 37 | // numeric value. Otherwise, returns false and leaves *in in an 38 | // unspecified state. 39 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 40 | 41 | } // namespace leveldb 42 | 43 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 44 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | #include "port/thread_annotations.h" 10 | 11 | namespace leveldb { 12 | 13 | // Helper class that locks a mutex on construction and unlocks the mutex when 14 | // the destructor of the MutexLock object is invoked. 15 | // 16 | // Typical usage: 17 | // 18 | // void MyClass::MyMethod() { 19 | // MutexLock l(&mu_); // mu_ is an instance variable 20 | // ... some complex code, possibly with multiple return paths ... 21 | // } 22 | 23 | class SCOPED_LOCKABLE MutexLock { 24 | public: 25 | explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) 26 | : mu_(mu) { 27 | this->mu_->Lock(); 28 | } 29 | ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 30 | 31 | private: 32 | port::Mutex *const mu_; 33 | // No copying allowed 34 | MutexLock(const MutexLock&); 35 | void operator=(const MutexLock&); 36 | }; 37 | 38 | } // namespace leveldb 39 | 40 | 41 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 42 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/options.h" 6 | 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/env.h" 9 | 10 | namespace leveldb { 11 | 12 | Options::Options() 13 | : comparator(BytewiseComparator()), 14 | compaction_speed(4096), 15 | create_if_missing(false), 16 | error_if_exists(false), 17 | paranoid_checks(false), 18 | env(Env::Default()), 19 | info_log(NULL), 20 | write_buffer_size(4<<20), 21 | max_open_files(1000), 22 | block_cache(NULL), 23 | block_size(4096), 24 | block_restart_interval(16), 25 | max_file_size(2<<20), 26 | compression(kSnappyCompression), 27 | reuse_logs(false), 28 | filter_policy(NULL) { 29 | } 30 | 31 | } // namespace leveldb 32 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "port/port.h" 7 | #include "leveldb/status.h" 8 | 9 | namespace leveldb { 10 | 11 | const char* Status::CopyState(const char* state) { 12 | uint32_t size; 13 | memcpy(&size, state, sizeof(size)); 14 | char* result = new char[size + 5]; 15 | memcpy(result, state, size + 5); 16 | return result; 17 | } 18 | 19 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 20 | assert(code != kOk); 21 | const uint32_t len1 = msg.size(); 22 | const uint32_t len2 = msg2.size(); 23 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 24 | char* result = new char[size + 5]; 25 | memcpy(result, &size, sizeof(size)); 26 | result[4] = static_cast(code); 27 | memcpy(result + 5, msg.data(), len1); 28 | if (len2) { 29 | result[5 + len1] = ':'; 30 | result[6 + len1] = ' '; 31 | memcpy(result + 7 + len1, msg2.data(), len2); 32 | } 33 | state_ = result; 34 | } 35 | 36 | std::string Status::ToString() const { 37 | if (state_ == NULL) { 38 | return "OK"; 39 | } else { 40 | char tmp[30]; 41 | const char* type; 42 | switch (code()) { 43 | case kOk: 44 | type = "OK"; 45 | break; 46 | case kNotFound: 47 | type = "NotFound: "; 48 | break; 49 | case kCorruption: 50 | type = "Corruption: "; 51 | break; 52 | case kNotSupported: 53 | type = "Not implemented: "; 54 | break; 55 | case kInvalidArgument: 56 | type = "Invalid argument: "; 57 | break; 58 | case kIOError: 59 | type = "IO error: "; 60 | break; 61 | default: 62 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 63 | static_cast(code())); 64 | type = tmp; 65 | break; 66 | } 67 | std::string result(type); 68 | uint32_t length; 69 | memcpy(&length, state_, sizeof(length)); 70 | result.append(state_ + 5, length); 71 | return result; 72 | } 73 | } 74 | 75 | } // namespace leveldb 76 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/testharness.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testharness.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | namespace { 16 | struct Test { 17 | const char* base; 18 | const char* name; 19 | void (*func)(); 20 | }; 21 | std::vector* tests; 22 | } 23 | 24 | bool RegisterTest(const char* base, const char* name, void (*func)()) { 25 | if (tests == NULL) { 26 | tests = new std::vector; 27 | } 28 | Test t; 29 | t.base = base; 30 | t.name = name; 31 | t.func = func; 32 | tests->push_back(t); 33 | return true; 34 | } 35 | 36 | int RunAllTests() { 37 | const char* matcher = getenv("LEVELDB_TESTS"); 38 | 39 | int num = 0; 40 | if (tests != NULL) { 41 | for (size_t i = 0; i < tests->size(); i++) { 42 | const Test& t = (*tests)[i]; 43 | if (matcher != NULL) { 44 | std::string name = t.base; 45 | name.push_back('.'); 46 | name.append(t.name); 47 | if (strstr(name.c_str(), matcher) == NULL) { 48 | continue; 49 | } 50 | } 51 | fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); 52 | (*t.func)(); 53 | ++num; 54 | } 55 | } 56 | fprintf(stderr, "==== PASSED %d tests\n", num); 57 | return 0; 58 | } 59 | 60 | std::string TmpDir() { 61 | std::string dir; 62 | Status s = Env::Default()->GetTestDirectory(&dir); 63 | ASSERT_TRUE(s.ok()) << s.ToString(); 64 | return dir; 65 | } 66 | 67 | int RandomSeed() { 68 | const char* env = getenv("TEST_RANDOM_SEED"); 69 | int result = (env != NULL ? atoi(env) : 301); 70 | if (result <= 0) { 71 | result = 301; 72 | } 73 | return result; 74 | } 75 | 76 | } // namespace test 77 | } // namespace leveldb 78 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/leveldb-1.20/util/testutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testutil.h" 6 | 7 | #include "util/random.h" 8 | 9 | namespace leveldb { 10 | namespace test { 11 | 12 | Slice RandomString(Random* rnd, int len, std::string* dst) { 13 | dst->resize(len); 14 | for (int i = 0; i < len; i++) { 15 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 16 | } 17 | return Slice(*dst); 18 | } 19 | 20 | std::string RandomKey(Random* rnd, int len) { 21 | // Make sure to generate a wide variety of characters so we 22 | // test the boundary conditions for short-key optimizations. 23 | static const char kTestChars[] = { 24 | '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' 25 | }; 26 | std::string result; 27 | for (int i = 0; i < len; i++) { 28 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 29 | } 30 | return result; 31 | } 32 | 33 | 34 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 35 | size_t len, std::string* dst) { 36 | int raw = static_cast(len * compressed_fraction); 37 | if (raw < 1) raw = 1; 38 | std::string raw_data; 39 | RandomString(rnd, raw, &raw_data); 40 | 41 | // Duplicate the random data until we have filled "len" bytes 42 | dst->clear(); 43 | while (dst->size() < len) { 44 | dst->append(raw_data); 45 | } 46 | dst->resize(len); 47 | return Slice(*dst); 48 | } 49 | 50 | } // namespace test 51 | } // namespace leveldb 52 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/.gitignore: -------------------------------------------------------------------------------- 1 | test 2 | 3 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | SRC_PATH=./ 4 | 5 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 -DNDEBUG -D__STDC_FORMAT_MACROS -Wall -O2 -Wno-sign-compare 6 | 7 | TARGET=libnet.a 8 | 9 | HEADER_PATH = ./../header 10 | 11 | UTIL_PATH = ./../util 12 | 13 | LEVELDB_PATH = ./../leveldb-1.20 14 | 15 | SOURCE := $(wildcard $(SRC_PATH)*.cpp) 16 | 17 | OBJS := $(patsubst %.cpp, %.o, $(SOURCE)) 18 | 19 | INCLUDE_PATH = -I./ \ 20 | -I$(UTIL_PATH) \ 21 | -I$(LEVELDB_PATH)/include \ 22 | -I$(HEADER_PATH) \ 23 | 24 | all : $(TARGET) 25 | echo "create libnet.a successfully" 26 | 27 | $(TARGET) : $(OBJS) 28 | ar -cru $(TARGET) $(OBJS) 29 | 30 | $(OBJS): %.o : %.cpp 31 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 32 | 33 | .PHONY:clean 34 | clean: 35 | rm -rf $(OBJS) $(TARGET) 36 | 37 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/fde.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "fde.h" 7 | 8 | struct Fdevent* Fdevents::get_fde(int fd){ 9 | while((int)events.size() <= fd){ 10 | struct Fdevent *fde = new Fdevent(); 11 | fde->fd = events.size(); 12 | fde->s_flags = FDEVENT_NONE; 13 | fde->data.num = 0; 14 | fde->data.ptr = NULL; 15 | events.push_back(fde); 16 | } 17 | return events[fd]; 18 | } 19 | 20 | 21 | #ifdef HAVE_EPOLL 22 | #include "fde_epoll.cc" 23 | #else 24 | #include "fde_select.cc" 25 | #endif 26 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/fde.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef UTIL_FDE_H 7 | #define UTIL_FDE_H 8 | 9 | #include "include.h" 10 | 11 | #ifdef __linux__ 12 | #define HAVE_EPOLL 1 13 | #endif 14 | 15 | #define FDEVENT_NONE (0) 16 | #define FDEVENT_IN (1<<0) 17 | #define FDEVENT_PRI (1<<1) 18 | #define FDEVENT_OUT (1<<2) 19 | #define FDEVENT_HUP (1<<3) 20 | #define FDEVENT_ERR (1<<4) 21 | 22 | struct Fdevent{ 23 | int fd; 24 | int s_flags; // subscribed events 25 | int events; // ready events 26 | struct{ 27 | int num; 28 | void *ptr; 29 | }data; 30 | }; 31 | 32 | #include 33 | #ifdef HAVE_EPOLL 34 | #include 35 | #else 36 | #include 37 | #endif 38 | 39 | 40 | class Fdevents{ 41 | public: 42 | typedef std::vector events_t; 43 | private: 44 | #ifdef HAVE_EPOLL 45 | static const int MAX_FDS = 8 * 1024; 46 | int ep_fd; 47 | struct epoll_event ep_events[MAX_FDS]; 48 | #else 49 | int maxfd; 50 | fd_set readset; 51 | fd_set writeset; 52 | #endif 53 | events_t events; 54 | events_t ready_events; 55 | 56 | struct Fdevent *get_fde(int fd); 57 | public: 58 | Fdevents(); 59 | ~Fdevents(); 60 | 61 | bool isset(int fd, int flag); 62 | int set(int fd, int flags, int data_num, void *data_ptr); 63 | int del(int fd); 64 | int clr(int fd, int flags); 65 | const events_t* wait(int timeout_ms=-1); 66 | }; 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/link_redis.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef NET_REDIS_LINK_H_ 7 | #define NET_REDIS_LINK_H_ 8 | 9 | #include 10 | #include 11 | #include "../util/bytes.h" 12 | 13 | struct RedisRequestDesc //redis请求描述 14 | { 15 | int strategy; 16 | std::string redis_cmd; //redis命令 17 | std::string ssdb_cmd; //ssdb命令 18 | int reply_type; //回复类型 19 | }; 20 | 21 | class RedisLink 22 | { 23 | private: 24 | std::string cmd; //命令 25 | RedisRequestDesc *req_desc; //redis请求描述指针 26 | 27 | std::vector recv_bytes; //ssdb处理请求时用Bytes数据结构. 但 Bytes本身只是指针 28 | std::vector recv_string; //recv_string来存储接收到的网络请求数据 29 | int parse_req(Buffer *input); //解析请求 30 | int convert_req(); //转换请求 31 | 32 | public: 33 | RedisLink(){ 34 | req_desc = NULL; //初始化redis请求描述为空 35 | } 36 | 37 | const std::vector* recv_req(Buffer *input); //从输入缓冲区获取请求 38 | int send_resp(Buffer *output, const std::vector &resp); //讲请求发送至输入缓冲区 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/proc.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "proc.h" 7 | #include "server.h" 8 | #include "../util/log.h" 9 | 10 | ProcMap::ProcMap(){ 11 | } 12 | 13 | ProcMap::~ProcMap(){ 14 | proc_map_t::iterator it; 15 | for(it=proc_map.begin(); it!=proc_map.end(); it++){ 16 | delete it->second; 17 | } 18 | proc_map.clear(); 19 | } 20 | 21 | void ProcMap::set_proc(const std::string &c, proc_t proc){ 22 | this->set_proc(c, "t", proc); 23 | } 24 | 25 | void ProcMap::set_proc(const std::string &c, const char *sflags, proc_t proc){ 26 | Command *cmd = this->get_proc(c); 27 | if(!cmd){ 28 | cmd = new Command(); 29 | cmd->name = c; 30 | proc_map[cmd->name] = cmd; 31 | } 32 | cmd->proc = proc; 33 | cmd->flags = 0; 34 | for(const char *p=sflags; *p!='\0'; p++){ 35 | switch(*p){ 36 | case 'r': 37 | cmd->flags |= Command::FLAG_READ; 38 | break; 39 | case 'w': // w 必须和 t 同时出现, 因为某些写操作依赖单线程 40 | cmd->flags |= Command::FLAG_WRITE; 41 | cmd->flags |= Command::FLAG_THREAD; 42 | break; 43 | case 'b': 44 | cmd->flags |= Command::FLAG_BACKEND; 45 | break; 46 | case 't': 47 | cmd->flags |= Command::FLAG_THREAD; 48 | break; 49 | } 50 | } 51 | } 52 | 53 | Command* ProcMap::get_proc(const Bytes &str){ 54 | proc_map_t::iterator it = proc_map.find(str); 55 | if(it != proc_map.end()){ 56 | return it->second; 57 | } 58 | return NULL; 59 | } 60 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/resp.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef NET_RESP_H_ 7 | #define NET_RESP_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | class Response 15 | { 16 | public: 17 | std::vector resp; 18 | 19 | int size() const; 20 | void push_back(const std::string &s); 21 | void add(int s); 22 | void add(int64_t s); 23 | void add(uint64_t s); 24 | void add(double s); 25 | void add(const std::string &s); 26 | 27 | void reply_status(int status, const char *errmsg=NULL); 28 | void reply_bool(int status, const char *errmsg=NULL); 29 | void reply_int(int status, int64_t val); 30 | // the same as Redis.REPLY_BULK 31 | void reply_get(int status, const std::string *val=NULL, const char *errmsg=NULL); 32 | void reply_list(int status, const std::vector &list); 33 | }; 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/server.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef NET_SERVER_H_ 7 | #define NET_SERVER_H_ 8 | 9 | #include "include.h" 10 | #include 11 | #include 12 | 13 | #include "fde.h" 14 | #include "proc.h" 15 | #include "worker.h" 16 | 17 | class Link; 18 | class Config; 19 | class IpFilter; 20 | class Fdevents; 21 | 22 | typedef std::vector ready_list_t; 23 | 24 | class NetworkServer 25 | { 26 | private: 27 | int tick_interval; 28 | int status_report_ticks; 29 | 30 | //Config *conf; 31 | Link *serv_link; 32 | Fdevents *fdes; 33 | 34 | Link* accept_link(); 35 | int proc_result(ProcJob *job, ready_list_t *ready_list); 36 | int proc_client_event(const Fdevent *fde, ready_list_t *ready_list); 37 | 38 | int proc(ProcJob *job); 39 | 40 | int num_readers; 41 | int num_writers; 42 | ProcWorkerPool *writer; 43 | ProcWorkerPool *reader; 44 | 45 | bool readonly; 46 | 47 | NetworkServer(); 48 | 49 | protected: 50 | void usage(int argc, char **argv); 51 | 52 | public: 53 | IpFilter *ip_filter; 54 | void *data; 55 | ProcMap proc_map; 56 | int link_count; 57 | bool need_auth; 58 | std::string password; 59 | double slowlog_timeout; // in ms, but in config file, it's in seconds 60 | 61 | ~NetworkServer(); 62 | 63 | // could be called only once 64 | static NetworkServer* init(const char *conf_file, int num_readers=-1, int num_writers=-1); 65 | static NetworkServer* init(const Config &conf, int num_readers=-1, int num_writers=-1); 66 | void serve(); 67 | }; 68 | 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/test.conf: -------------------------------------------------------------------------------- 1 | pidfile = ./test.pid 2 | 3 | server: 4 | ip: 127.0.0.1 5 | port: 9001 6 | # bind to public ip 7 | #ip: 0.0.0.0 8 | # format: allow|deny: all|ip_prefix 9 | # multiple allows or denys is supported 10 | #deny: all 11 | allow: 127.0.0.1 12 | #allow: 192.168 13 | # auth password must be longer than 31 characters 14 | #auth: very-strong-passwordwwwwwwwwwwwwww 15 | readonly: no 16 | 17 | logger: 18 | level: debug 19 | output: stdout 20 | rotate: 21 | size: 1000000000 22 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/test.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "server.h" 7 | 8 | void usage(int argc, char **argv){ 9 | printf("Usage:\n"); 10 | printf(" %s [-d] /path/to/server.conf\n", argv[0]); 11 | printf("Options:\n"); 12 | printf(" -d run as daemon\n"); 13 | } 14 | 15 | DEF_PROC(hello); 16 | 17 | int main(int argc, char **argv){ 18 | bool is_daemon = false; 19 | const char *conf_file = NULL; 20 | for(int i=1; iproc_map.set_proc("hello", "w", proc_hello); 38 | serv->serve(); 39 | delete serv; 40 | return 0; 41 | } 42 | 43 | int proc_hello(NetworkServer *net, Link *link, const Request &req, Response *resp){ 44 | resp->push_back("ok"); 45 | resp->push_back("world!"); 46 | if(req.size() > 1){ 47 | // The first argument start at index 1, just like argv. 48 | resp->push_back(req[1].String()); 49 | } 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/test2.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "server.h" 7 | #include "../util/config.h" 8 | 9 | DEF_PROC(hello); 10 | 11 | int main(int argc, char **argv){ 12 | fprintf(stderr, "Usage: %s [ip] [port]\n\n", argv[0]); 13 | Config conf; 14 | conf.set("server.port", "9000"); 15 | if(argc > 1){ 16 | conf.set("server.ip", argv[1]); 17 | } 18 | if(argc > 2){ 19 | conf.set("server.port", argv[2]); 20 | } 21 | NetworkServer *serv = NetworkServer::init(conf); 22 | if(!serv){ 23 | exit(1); 24 | } 25 | // register command procedure 26 | serv->proc_map.set_proc("hello", proc_hello); 27 | serv->serve(); 28 | delete serv; 29 | return 0; 30 | } 31 | 32 | int proc_hello(NetworkServer *net, Link *link, const Request &req, Response *resp){ 33 | resp->push_back("ok"); 34 | resp->push_back("world!"); 35 | if(req.size() > 1){ 36 | // The first argument start at index 1, just like argv. 37 | resp->push_back(req[1].String()); 38 | } 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/worker.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "worker.h" 7 | #include "link.h" 8 | #include "proc.h" 9 | #include "../util/log.h" 10 | #include "include.h" 11 | 12 | ProcWorker::ProcWorker(const std::string &name){ 13 | this->name = name; 14 | } 15 | 16 | void ProcWorker::init(){ 17 | log_debug("%s %d init", this->name.c_str(), this->id); 18 | } 19 | 20 | int ProcWorker::proc(ProcJob *job){ 21 | const Request *req = job->req; 22 | 23 | proc_t p = job->cmd->proc; 24 | job->time_wait = 1000 * (microtime() - job->stime); 25 | job->result = (*p)(job->serv, job->link, *req, &job->resp); 26 | job->time_proc = 1000 * (microtime() - job->stime) - job->time_wait; 27 | 28 | if(job->link->send(job->resp.resp) == -1){ 29 | job->result = PROC_ERROR; 30 | }else{ 31 | // try to write socket before it would be added to fdevents 32 | // socket is NONBLOCK, so it won't block. 33 | if(job->link->write() < 0){ 34 | job->result = PROC_ERROR; 35 | } 36 | } 37 | 38 | return 0; 39 | } 40 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/net/worker.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef NET_WORKER_H_ 7 | #define NET_WORKER_H_ 8 | 9 | #include 10 | #include "../util/thread.h" 11 | #include "proc.h" 12 | 13 | // WARN: pipe latency is about 20 us, it is really slow! 14 | class ProcWorker : public WorkerPool::Worker{ 15 | public: 16 | ProcWorker(const std::string &name); 17 | ~ProcWorker(){} 18 | void init(); 19 | int proc(ProcJob *job); 20 | }; 21 | 22 | typedef WorkerPool ProcWorkerPool; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/.Makefile.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Axlgrep/pika-tools/fd0cc310f394eb5bded2ed515e9e5d4d32dd1c25/ssdb_to_pika_soomth_migration/ssdb/.Makefile.swp -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | SRC_PATH = ./ 4 | 5 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 -DNDEBUG -D__STDC_FORMAT_MACROS -Wall -O2 -Wno-sign-compare 6 | 7 | TARGET = libssdb.a 8 | 9 | HEADER_PATH = ./../header 10 | 11 | UTIL_PATH = ./../util 12 | 13 | LEVELDB_PATH = ./../leveldb-1.20 14 | 15 | SOURCE := $(wildcard $(SRC_PATH)*.cpp) 16 | 17 | OBJS := $(patsubst %.cpp, %.o, $(SOURCE)) 18 | 19 | INCLUDE_PATH = -I./ \ 20 | -I$(UTIL_PATH) \ 21 | -I$(LEVELDB_PATH)/include \ 22 | -I$(HEADER_PATH) \ 23 | 24 | all:$(TARGET) 25 | echo "creat libssdb.a successful" 26 | 27 | $(TARGET):ssdb.h $(OBJS) 28 | ar -cru $(TARGET) $(OBJS) 29 | 30 | $(OBJS): %.o : %.cpp 31 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 32 | 33 | .PHONY:clean 34 | clean: 35 | rm -rf $(OBJS) $(TARGET) 36 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/const.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_CONST_H_ 7 | #define SSDB_CONST_H_ 8 | 9 | static const int SSDB_SCORE_WIDTH = 9; 10 | static const int SSDB_KEY_LEN_MAX = 255; 11 | 12 | class DataType{ 13 | public: 14 | static const char SYNCLOG = 1; 15 | static const char KV = 'k'; 16 | static const char HASH = 'h'; // hashmap(sorted by key) 17 | static const char HSIZE = 'H'; 18 | static const char ZSET = 's'; // key => score 19 | static const char ZSCORE = 'z'; // key|score => "" 20 | static const char ZSIZE = 'Z'; 21 | static const char QUEUE = 'q'; 22 | static const char QSIZE = 'Q'; 23 | static const char MIN_PREFIX = HASH; 24 | static const char MAX_PREFIX = ZSET; 25 | }; 26 | 27 | class BinlogType{ 28 | public: 29 | static const char NOOP = 0; 30 | static const char SYNC = 1; 31 | static const char MIRROR = 2; 32 | static const char COPY = 3; 33 | static const char CTRL = 4; 34 | }; 35 | 36 | class BinlogCommand{ 37 | public: 38 | static const char NONE = 0; 39 | static const char KSET = 1; 40 | static const char KDEL = 2; 41 | static const char HSET = 3; 42 | static const char HDEL = 4; 43 | static const char ZSET = 5; 44 | static const char ZDEL = 6; 45 | 46 | static const char QPUSH_BACK = 10; 47 | static const char QPUSH_FRONT = 11; 48 | static const char QPOP_BACK = 12; 49 | static const char QPOP_FRONT = 13; 50 | static const char QSET = 14; 51 | 52 | static const char BEGIN = 7; 53 | static const char END = 8; 54 | }; 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_ITERATOR_H_ 7 | #define SSDB_ITERATOR_H_ 8 | 9 | #include 10 | #include 11 | #include "../util/bytes.h" 12 | 13 | namespace leveldb{ 14 | class Iterator; 15 | } 16 | 17 | class Iterator{ 18 | public: 19 | enum Direction{ 20 | FORWARD, BACKWARD 21 | }; 22 | Iterator(leveldb::Iterator *it, 23 | const std::string &end, 24 | uint64_t limit, 25 | Direction direction=Iterator::FORWARD); 26 | ~Iterator(); 27 | bool skip(uint64_t offset); 28 | bool next(); 29 | Bytes key(); 30 | Bytes val(); 31 | private: 32 | leveldb::Iterator *it; 33 | std::string end; 34 | uint64_t limit; 35 | bool is_first; 36 | int direction; 37 | }; 38 | 39 | 40 | class KIterator{ 41 | public: 42 | std::string key; 43 | std::string val; 44 | 45 | KIterator(Iterator *it); 46 | ~KIterator(); 47 | void return_val(bool onoff); 48 | bool next(); 49 | private: 50 | Iterator *it; 51 | bool return_val_; 52 | }; 53 | 54 | 55 | class HIterator{ 56 | public: 57 | std::string name; 58 | std::string key; 59 | std::string val; 60 | 61 | HIterator(Iterator *it, const Bytes &name); 62 | ~HIterator(); 63 | void return_val(bool onoff); 64 | bool next(); 65 | private: 66 | Iterator *it; 67 | bool return_val_; 68 | }; 69 | 70 | 71 | class ZIterator{ 72 | public: 73 | std::string name; 74 | std::string key; 75 | std::string score; 76 | 77 | ZIterator(Iterator *it, const Bytes &name); 78 | ~ZIterator(); 79 | bool skip(uint64_t offset); 80 | bool next(); 81 | private: 82 | Iterator *it; 83 | }; 84 | 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/options.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include "options.h" 7 | #include "../util/strings.h" 8 | 9 | #ifdef NDEBUG 10 | static const int LOG_QUEUE_SIZE = 20 * 1000 * 1000; 11 | #else 12 | static const int LOG_QUEUE_SIZE = 10000; 13 | #endif 14 | Options::Options() { 15 | compression = "yes"; 16 | this->binlog = true; 17 | binlog_capacity = LOG_QUEUE_SIZE; 18 | cache_size = 16; 19 | write_buffer_size = 16; 20 | block_size = 32; 21 | max_open_files = 500; 22 | 23 | } 24 | //Options::Options(){ 25 | // Config c; 26 | // this->load(c); 27 | //} 28 | 29 | void Options::load(const Config &conf){ 30 | cache_size = (size_t)conf.get_num("leveldb.cache_size"); 31 | max_open_files = (size_t)conf.get_num("leveldb.max_open_files"); 32 | write_buffer_size = (size_t)conf.get_num("leveldb.write_buffer_size"); 33 | block_size = (size_t)conf.get_num("leveldb.block_size"); 34 | compaction_speed = conf.get_num("leveldb.compaction_speed"); 35 | compression = conf.get_str("leveldb.compression"); 36 | std::string binlog = conf.get_str("replication.binlog"); 37 | binlog_capacity = (size_t)conf.get_num("replication.binlog.capacity"); 38 | 39 | strtolower(&compression); 40 | if(compression != "no"){ 41 | compression = "yes"; 42 | } 43 | strtolower(&binlog); 44 | if(binlog != "yes"){ 45 | this->binlog = false; 46 | }else{ 47 | this->binlog = true; 48 | } 49 | if(binlog_capacity <= 0){ 50 | binlog_capacity = LOG_QUEUE_SIZE; 51 | } 52 | 53 | if(cache_size <= 0){ 54 | cache_size = 16; 55 | } 56 | if(write_buffer_size <= 0){ 57 | write_buffer_size = 16; 58 | } 59 | if(block_size <= 0){ 60 | block_size = 32; 61 | } 62 | if(max_open_files <= 0){ 63 | max_open_files = cache_size / 1024 * 300; 64 | if(max_open_files < 500){ 65 | max_open_files = 500; 66 | } 67 | if(max_open_files > 1000){ 68 | max_open_files = 1000; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/options.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_OPTION_H_ 7 | #define SSDB_OPTION_H_ 8 | 9 | #include "../util/config.h" 10 | 11 | class Options 12 | { 13 | public: 14 | Options() ; 15 | ~Options(){} 16 | 17 | void load(const Config &conf); 18 | 19 | size_t cache_size; 20 | size_t max_open_files; 21 | size_t write_buffer_size; 22 | size_t block_size; 23 | int compaction_speed; 24 | std::string compression; 25 | bool binlog; 26 | size_t binlog_capacity; 27 | }; 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/t_hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_HASH_H_ 7 | #define SSDB_HASH_H_ 8 | 9 | #include "ssdb_impl.h" 10 | 11 | inline static 12 | std::string encode_hsize_key(const Bytes &name){ 13 | std::string buf; 14 | buf.reserve(name.size() + 1); 15 | buf.append(1, DataType::HSIZE); 16 | buf.append(name.data(), name.size()); 17 | return buf; 18 | } 19 | 20 | inline static 21 | int decode_hsize_key(const Bytes &slice, std::string *name){ 22 | Decoder decoder(slice.data(), slice.size()); 23 | if(decoder.skip(1) == -1){ 24 | return -1; 25 | } 26 | if(decoder.read_data(name) == -1){ 27 | return -1; 28 | } 29 | return 0; 30 | } 31 | 32 | inline static 33 | std::string encode_hash_key(const Bytes &name, const Bytes &key){ 34 | std::string buf; 35 | buf.reserve(128); 36 | buf.append(1, DataType::HASH); 37 | buf.append(1, (uint8_t)name.size()); 38 | buf.append(name.data(), name.size()); 39 | buf.append(1, '='); 40 | buf.append(key.data(), key.size()); 41 | return buf; 42 | } 43 | 44 | inline static 45 | int decode_hash_key(const Bytes &slice, std::string *name, std::string *key){ 46 | Decoder decoder(slice.data(), slice.size()); 47 | if(decoder.skip(1) == -1){ 48 | return -1; 49 | } 50 | if(decoder.read_8_data(name) == -1){ 51 | return -1; 52 | } 53 | if(decoder.skip(1) == -1){ 54 | return -1; 55 | } 56 | if(decoder.read_data(key) == -1){ 57 | return -1; 58 | } 59 | return 0; 60 | } 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/t_kv.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_KV_H_ 7 | #define SSDB_KV_H_ 8 | 9 | #include "ssdb_impl.h" 10 | 11 | static inline 12 | std::string encode_kv_key(const Bytes &key){ 13 | std::string buf; 14 | buf.reserve(key.size() + 1); 15 | buf.append(1, DataType::KV); 16 | buf.append(key.data(), key.size()); 17 | return buf; 18 | } 19 | 20 | static inline 21 | int decode_kv_key(const Bytes &slice, std::string *key){ 22 | Decoder decoder(slice.data(), slice.size()); 23 | if(decoder.skip(1) == -1){ 24 | return -1; 25 | } 26 | if(decoder.read_data(key) == -1){ 27 | return -1; 28 | } 29 | return 0; 30 | } 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/t_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_QUEUE_H_ 7 | #define SSDB_QUEUE_H_ 8 | 9 | #include "ssdb_impl.h" 10 | 11 | const uint64_t QFRONT_SEQ = 2; 12 | const uint64_t QBACK_SEQ = 3; 13 | const uint64_t QITEM_MIN_SEQ = 10000; 14 | const uint64_t QITEM_MAX_SEQ = 9223372036854775807ULL; 15 | const uint64_t QITEM_SEQ_INIT = QITEM_MAX_SEQ/2; 16 | 17 | inline static 18 | std::string encode_qsize_key(const Bytes &name){ 19 | std::string buf; 20 | buf.reserve(name.size() + 1); 21 | buf.append(1, DataType::QSIZE); 22 | buf.append(name.data(), name.size()); 23 | return buf; 24 | } 25 | 26 | inline static 27 | int decode_qsize_key(const Bytes &slice, std::string *name){ 28 | Decoder decoder(slice.data(), slice.size()); 29 | if(decoder.skip(1) == -1){ 30 | return -1; 31 | } 32 | if(decoder.read_data(name) == -1){ 33 | return -1; 34 | } 35 | return 0; 36 | } 37 | 38 | inline static 39 | std::string encode_qitem_key(const Bytes &name, uint64_t seq){ 40 | std::string buf; 41 | buf.reserve(128); 42 | buf.append(1, DataType::QUEUE); 43 | buf.append(1, (uint8_t)name.size()); 44 | buf.append(name.data(), name.size()); 45 | seq = big_endian(seq); 46 | buf.append((char *)&seq, sizeof(uint64_t)); 47 | return buf; 48 | } 49 | 50 | inline static 51 | int decode_qitem_key(const Bytes &slice, std::string *name, uint64_t *seq){ 52 | Decoder decoder(slice.data(), slice.size()); 53 | if(decoder.skip(1) == -1){ 54 | return -1; 55 | } 56 | if(decoder.read_8_data(name) == -1){ 57 | return -1; 58 | } 59 | if(decoder.read_uint64(seq) == -1){ 60 | return -1; 61 | } 62 | *seq = big_endian(*seq); 63 | return 0; 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/test.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include 7 | #include "ssdb.h" 8 | #include "../util/log.h" 9 | #include "../util/config.h" 10 | 11 | int main(int argc, char **argv){ 12 | set_log_level(Logger::LEVEL_TRACE); 13 | std::string work_dir = "./tmp/a"; 14 | Options opt; 15 | opt.compression = "no"; 16 | 17 | SSDB *ssdb = NULL; 18 | ssdb = SSDB::open(opt, work_dir); 19 | if(!ssdb){ 20 | log_fatal("could not open work_dir: %s", work_dir.c_str()); 21 | fprintf(stderr, "could not open work_dir: %s\n", work_dir.c_str()); 22 | exit(1); 23 | } 24 | std::string key, val; 25 | key = "a"; 26 | 27 | val.append(1024 * 1024, 'a'); 28 | ssdb->raw_set("tmp", val); 29 | ssdb->compact(); 30 | 31 | uint64_t size; 32 | size = ssdb->size(); 33 | log_debug("dbsize: %d", size); 34 | 35 | 36 | ssdb->get(key, &val); 37 | int num = str_to_int(val) + 1; 38 | ssdb->set(key, str(num)); 39 | ssdb->get(key, &val); 40 | 41 | log_debug("%s", val.c_str()); 42 | delete ssdb; 43 | } 44 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb/ttl.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef SSDB_TTL_H_ 7 | #define SSDB_TTL_H_ 8 | 9 | #include "ssdb.h" 10 | #include "../util/thread.h" 11 | #include "../util/sorted_set.h" 12 | #include 13 | 14 | class ExpirationHandler 15 | { 16 | public: 17 | Mutex mutex; 18 | 19 | ExpirationHandler(SSDB *ssdb); 20 | ~ExpirationHandler(); 21 | 22 | // "In Redis 2.6 or older the command returns -1 if the key does not exist 23 | // or if the key exist but has no associated expire. Starting with Redis 2.8.." 24 | // I stick to Redis 2.6 25 | int64_t get_ttl(const Bytes &key); 26 | // The caller must hold mutex before calling set/del functions 27 | int del_ttl(const Bytes &key); 28 | int set_ttl(const Bytes &key, int64_t ttl); 29 | 30 | private: 31 | SSDB *ssdb; 32 | volatile bool thread_quit; 33 | std::string list_name; 34 | int64_t first_timeout; 35 | SortedSet fast_keys; 36 | 37 | void start(); 38 | void stop(); 39 | void expire_loop(); 40 | static void* thread_func(void *arg); 41 | void load_expiration_keys_from_db(int num); 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/ssdb_to_pika.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "slave.h" 6 | #include "hiredis.h" 7 | 8 | void Usage() { 9 | std::cout << "Usage:" << std::endl; 10 | std::cout << "\tssdb_to_pika read data from ssdb sever and send it to pika sever" << std::endl; 11 | std::cout << "\tssdb_ip ssdb_server's ip" << std::endl; 12 | std::cout << "\tssdb_port ssdb_server's port" << std::endl; 13 | std::cout << "\tpika_ip pika_server's ip" << std::endl; 14 | std::cout << "\tpika_port pika_sever's port" << std::endl; 15 | std::cout << "\t[ssdb_auth] ssdb_server's auth" << std::endl; 16 | std::cout << "\t[pika_auth] pika_server's auth" << std::endl; 17 | std::cout << "\texample: ./ssdb_to_pika localhost 8888 localhost 9925" << std::endl; 18 | } 19 | 20 | int main(int argc, char** argv) { 21 | if (argc < 5) { 22 | Usage(); 23 | return 1; 24 | } 25 | const char* ssdb_auth = nullptr; 26 | const char* pika_auth = nullptr; 27 | if (argc >= 6) { 28 | ssdb_auth = argv[5]; 29 | } 30 | if (argc >= 7) { 31 | pika_auth = argv[6]; 32 | } 33 | std::unique_ptr slave_ptr(new Slave(argv[1], atoi(argv[2]), argv[3], 34 | atoi(argv[4]), ssdb_auth, pika_auth)); 35 | slave_ptr->start(); 36 | pthread_t tid = slave_ptr->getTid(); 37 | pthread_join(tid, NULL); 38 | if (tid == 0) { 39 | std::cout << "some error with connection" << std::endl; 40 | return 0; 41 | } 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | 3 | SRC_PATH = ./ 4 | 5 | CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 -DNDEBUG -D__STDC_FORMAT_MACROS -Wall -O2 -Wno-sign-compare 6 | 7 | TARGET = libutil.a 8 | 9 | HEADER_PATH = ./../header 10 | 11 | 12 | SOURCE := $(wildcard $(SRC_PATH)*.cpp) 13 | 14 | OBJS := $(patsubst %.cpp, %.o, $(SOURCE)) 15 | 16 | INCLUDE_PATH = -I./ \ 17 | -I$(HEADER_PATH) \ 18 | 19 | all:$(TARGET) 20 | echo "creat libutil.a successful" 21 | 22 | $(TARGET):$(OBJS) 23 | ar -cru $(TARGET) $(OBJS) 24 | 25 | $(OBJS): %.o : %.cpp 26 | $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) 27 | 28 | .PHONY:clean 29 | clean: 30 | rm -rf $(OBJS) $(TARGET) 31 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/app.h: -------------------------------------------------------------------------------- 1 | #ifndef UTIL_APP_H 2 | #define UTIL_APP_H 3 | 4 | #include 5 | 6 | class Config; 7 | 8 | class Application{ 9 | public: 10 | Application(){}; 11 | virtual ~Application(){}; 12 | 13 | int main(int argc, char **argv); 14 | 15 | virtual void usage(int argc, char **argv); 16 | virtual void welcome() = 0; 17 | virtual void run() = 0; 18 | 19 | protected: 20 | struct AppArgs{ 21 | bool is_daemon; 22 | std::string pidfile; 23 | std::string conf_file; 24 | std::string work_dir; 25 | std::string start_opt; 26 | 27 | AppArgs(){ 28 | is_daemon = false; 29 | start_opt = "start"; 30 | } 31 | }; 32 | 33 | Config *conf; 34 | AppArgs app_args; 35 | 36 | private: 37 | void parse_args(int argc, char **argv); 38 | void init(); 39 | 40 | int read_pid(); 41 | void write_pid(); 42 | void check_pidfile(); 43 | void remove_pidfile(); 44 | void kill_process(); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/daemon.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef UTIL_DAEMON_H 7 | #define UTIL_DAEMON_H 8 | 9 | int daemonize(const char *dir=NULL){ 10 | switch(fork()){ 11 | case -1: 12 | return -1; 13 | case 0: 14 | break; 15 | default: 16 | exit(0); 17 | } 18 | if(setsid() == -1){ 19 | exit(0); 20 | } 21 | if(dir != NULL){ 22 | if(chdir(dir) == -1){ 23 | exit(0); 24 | } 25 | } 26 | 27 | if(close(STDIN_FILENO) == -1){ 28 | exit(0); 29 | } 30 | if(close(STDOUT_FILENO) == -1){ 31 | exit(0); 32 | } 33 | if(close(STDERR_FILENO) == -1){ 34 | exit(0); 35 | } 36 | 37 | int fd = open("/dev/null", O_RDWR, 0); 38 | if(fd == -1){ 39 | exit(0); 40 | } 41 | if(dup2(fd, STDIN_FILENO) == -1){ 42 | exit(0); 43 | } 44 | if(dup2(fd, STDOUT_FILENO) == -1){ 45 | exit(0); 46 | } 47 | if(dup2(fd, STDERR_FILENO) == -1){ 48 | exit(0); 49 | } 50 | 51 | return 0; 52 | } 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/file.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef UTIL_FILE_H_ 7 | #define UTIL_FILE_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | static inline 15 | bool file_exists(const std::string &filename){ 16 | struct stat st; 17 | return stat(filename.c_str(), &st) == 0; 18 | } 19 | 20 | static inline 21 | bool is_dir(const std::string &filename){ 22 | struct stat st; 23 | if(stat(filename.c_str(), &st) == -1){ 24 | return false; 25 | } 26 | return (bool)S_ISDIR(st.st_mode); 27 | } 28 | 29 | static inline 30 | bool is_file(const std::string &filename){ 31 | struct stat st; 32 | if(stat(filename.c_str(), &st) == -1){ 33 | return false; 34 | } 35 | return (bool)S_ISREG(st.st_mode); 36 | } 37 | 38 | // return number of bytes read 39 | static inline 40 | int file_get_contents(const std::string &filename, std::string *content){ 41 | char buf[8192]; 42 | FILE *fp = fopen(filename.c_str(), "rb"); 43 | if(!fp){ 44 | return -1; 45 | } 46 | int ret = 0; 47 | while(!feof(fp) && !ferror(fp)){ 48 | int n = fread(buf, 1, sizeof(buf), fp); 49 | if(n > 0){ 50 | ret += n; 51 | content->append(buf, n); 52 | } 53 | } 54 | fclose(fp); 55 | return ret; 56 | } 57 | 58 | // return number of bytes written 59 | static inline 60 | int file_put_contents(const std::string &filename, const std::string &content){ 61 | FILE *fp = fopen(filename.c_str(), "wb"); 62 | if(!fp){ 63 | return -1; 64 | } 65 | int ret = fwrite(content.data(), 1, content.size(), fp); 66 | fclose(fp); 67 | return ret == (int)content.size()? ret : -1; 68 | } 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/line.h: -------------------------------------------------------------------------------- 1 | #ifndef UTIL_LINE_H 2 | #define UTIL_LINE_H 3 | 4 | #include 5 | #include 6 | #include "strings.h" 7 | 8 | class LineEncoder{ 9 | public: 10 | int write(const std::string &data){ 11 | val.append(str_escape(data)); 12 | val.append("\n"); 13 | return 0; 14 | } 15 | 16 | int write(int data){ 17 | return this->write(::str(data)); 18 | } 19 | 20 | int write(int64_t data){ 21 | return this->write(::str(data)); 22 | } 23 | 24 | std::string str(){ 25 | return val; 26 | } 27 | private: 28 | std::string val; 29 | }; 30 | 31 | class LineDecoder{ 32 | public: 33 | LineDecoder(const std::string &s){ 34 | spos = 0; 35 | epos = 0; 36 | buf = s.data(); 37 | len = (int)s.size(); 38 | } 39 | 40 | int readline(std::string *ret){ 41 | return this->read(ret); 42 | } 43 | 44 | int read(std::string *ret){ 45 | while(epos < len && buf[epos] != '\n'){ 46 | epos ++; 47 | } 48 | if(epos >= len || buf[epos] != '\n'){ 49 | return -1; 50 | } 51 | std::string line(&buf[spos], epos - spos); 52 | spos = epos + 1; 53 | epos = spos; 54 | *ret = str_unescape(line); 55 | return (int)ret->size(); 56 | } 57 | 58 | int read(int *ret){ 59 | std::string line; 60 | if(this->read(&line) == -1){ 61 | return -1; 62 | } 63 | *ret = str_to_int(line); 64 | return 0; 65 | } 66 | 67 | int read(int64_t *ret){ 68 | std::string line; 69 | if(this->read(&line) == -1){ 70 | return -1; 71 | } 72 | *ret = str_to_int64(line); 73 | return 0; 74 | } 75 | 76 | public: 77 | const char *buf; 78 | int len; 79 | int spos; 80 | int epos; 81 | }; 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/list.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The icomet Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef UTIL_LIST_H 7 | #define UTIL_LIST_H 8 | 9 | template 10 | class LinkedList{ 11 | public: 12 | class Iterator{ 13 | private: 14 | T p; 15 | public: 16 | friend class LinkedList; 17 | 18 | T next(){ 19 | T ret = p; 20 | if(p){ 21 | p = p->next; 22 | } 23 | return ret; 24 | } 25 | }; 26 | friend class Iterator; 27 | public: 28 | int size; 29 | T head; 30 | T tail; 31 | 32 | LinkedList(){ 33 | size = 0; 34 | head = NULL; 35 | tail = NULL; 36 | } 37 | 38 | Iterator iterator(){ 39 | Iterator it; 40 | it.p = this->head; 41 | return it; 42 | } 43 | 44 | bool empty() const{ 45 | return size == 0; 46 | } 47 | 48 | void remove(T t){ 49 | this->size --; 50 | if(t->prev){ 51 | t->prev->next = t->next; 52 | } 53 | if(t->next){ 54 | t->next->prev = t->prev; 55 | } 56 | if(this->head == t){ 57 | this->head = t->next; 58 | } 59 | if(this->tail == t){ 60 | this->tail = t->prev; 61 | } 62 | } 63 | 64 | T pop_front(){ 65 | T t = this->head; 66 | this->remove(t); 67 | return t; 68 | } 69 | 70 | void push_back(T t){ 71 | this->size ++; 72 | t->prev = this->tail; 73 | t->next = NULL; 74 | if(this->tail){ 75 | this->tail->next = t; 76 | }else{ // both head and tail is empty 77 | this->head = t; 78 | } 79 | this->tail = t; 80 | } 81 | }; 82 | 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/sorted_set.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #ifndef UTIL_SORTED_SET_H 7 | #define UTIL_SORTED_SET_H 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | class SortedSet 15 | { 16 | public: 17 | bool empty() const{ 18 | return size() == 0; 19 | } 20 | int size() const; 21 | int add(const std::string &key, int64_t score); 22 | // 0: not found, 1: found and deleted 23 | int del(const std::string &key); 24 | // the first item is copied into key if SortedSet not empty 25 | int front(std::string *key, int64_t *score=NULL) const; 26 | int back(std::string *key, int64_t *score=NULL) const; 27 | int64_t max_score() const; 28 | int pop_front(); 29 | int pop_back(); 30 | 31 | /* 32 | class Iterator 33 | { 34 | public: 35 | bool next(); 36 | const std::string& key(); 37 | int64_t score(); 38 | }; 39 | 40 | Iterator begin(); 41 | */ 42 | 43 | private: 44 | struct Item 45 | { 46 | std::string key; 47 | int64_t score; 48 | 49 | bool operator<(const Item& b) const{ 50 | return this->score < b.score 51 | || (this->score == b.score && this->key < b.key); 52 | } 53 | }; 54 | 55 | std::map::iterator> existed; 56 | std::set sorted_set; 57 | }; 58 | 59 | 60 | /* 61 | TODO: HashedWheel 62 | Each item is linked in two list, one is slot list, the other 63 | one is total list. 64 | */ 65 | /* 66 | template 67 | class SortedList 68 | { 69 | public: 70 | void add(const T data, int64_t score); 71 | T front(); 72 | void pop_front(); 73 | 74 | class Item 75 | { 76 | public: 77 | int64_t score; 78 | Item *prev; 79 | Item *next; 80 | //Item *slot_prev; 81 | //Item *slot_next; 82 | T data; 83 | }; 84 | }; 85 | */ 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/test_sorted_set.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "log.h" 11 | #include "sorted_set.h" 12 | #include "bytes.h" 13 | 14 | int main(int argc, char **argv){ 15 | SortedSet zset; 16 | 17 | std::vector keys; 18 | for(int i='a'; i<='z'; i++){ 19 | char buf[10]; 20 | snprintf(buf, sizeof(buf), "%c", i); 21 | keys.push_back(buf); 22 | } 23 | 24 | log_debug(""); 25 | srand(time(NULL)); 26 | for(int i=0; i<1000 * 1000; i++){ 27 | std::string &key = keys[rand() % keys.size()]; 28 | zset.add(key, rand()%30 - 15); 29 | } 30 | log_debug(""); 31 | 32 | std::string key; 33 | int64_t score; 34 | int n = 0; 35 | while(zset.front(&key, &score)){ 36 | printf("%s : %4lld\n", key.c_str(), score); 37 | zset.pop_front(); 38 | n ++; 39 | } 40 | log_debug("%d", n); 41 | 42 | { 43 | Buffer bs(8192); 44 | bs.append_record("a"); 45 | bs.append_record("bs"); 46 | dump(bs.data(), bs.size()); 47 | 48 | Bytes s; 49 | bs.read_record(&s); 50 | dump(s.data(), s.size()); 51 | bs.read_record(&s); 52 | dump(s.data(), s.size()); 53 | } 54 | 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /ssdb_to_pika_soomth_migration/util/test_thread.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2012-2014 The SSDB Authors. All rights reserved. 3 | Use of this source code is governed by a BSD-style license that can be 4 | found in the LICENSE file. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "log.h" 11 | #include "thread.h" 12 | 13 | // g++ log.o a.cpp 14 | 15 | class MyWorker : public WorkerPool::Worker{ 16 | public: 17 | MyWorker(const std::string &name){ 18 | } 19 | 20 | int proc(int *job){ 21 | usleep(200 * 1000); 22 | *job = 100000 + *job; 23 | return 0; 24 | } 25 | }; 26 | 27 | #define NUM_JOBS 10 28 | 29 | int main(){ 30 | int jobs[NUM_JOBS]; 31 | WorkerPool tp("test"); 32 | tp.start(3); 33 | 34 | log_debug("add begin"); 35 | for(int i=0; i 2 | #include 3 | #include "scan.h" 4 | 5 | ScanThread::~ScanThread() { 6 | } 7 | 8 | void ScanThread::ScanFile() { 9 | log_info("Start to scan"); 10 | 11 | std::ifstream fout(filename_, std::ios::binary); 12 | fout.seekg(0, std::ios::end); 13 | uint64_t filesize = fout.tellg(); 14 | fout.seekg(0, std::ios::beg); 15 | 16 | uint64_t index = 0; 17 | while (index < filesize) { 18 | std::string str_key, str_value, cmd; 19 | uint32_t key_len; 20 | fout.read(reinterpret_cast(&key_len), sizeof(uint32_t)); 21 | char * key = new char[key_len]; 22 | fout.read(key, key_len); 23 | str_key.append(key, key_len); 24 | 25 | uint32_t value_len; 26 | fout.read(reinterpret_cast(&value_len), sizeof(uint32_t)); 27 | char * value = new char[value_len]; 28 | fout.read(value, value_len); 29 | str_value.append(value, value_len); 30 | 31 | pink::RedisCmdArgsType argv; 32 | if (ttl_ > 0) { 33 | argv.push_back("SETEX"); 34 | argv.push_back(str_key); 35 | argv.push_back(std::to_string(ttl_)); 36 | argv.push_back(str_value); 37 | } else { 38 | argv.push_back("SET"); 39 | argv.push_back(str_key); 40 | argv.push_back(str_value); 41 | } 42 | 43 | pink::SerializeRedisCommand(argv, &cmd); 44 | 45 | DispatchCmd(cmd); 46 | num_++; 47 | 48 | delete []key; 49 | delete []value; 50 | 51 | index += key_len + value_len + sizeof(uint32_t) * 2; 52 | } 53 | fout.close(); 54 | } 55 | 56 | void ScanThread::DispatchCmd(const std::string &cmd) { 57 | senders_[thread_index_]->LoadCmd(cmd); 58 | thread_index_ = (thread_index_ + 1) % senders_.size(); 59 | } 60 | 61 | void *ScanThread::ThreadMain() { 62 | ScanFile(); 63 | log_info("Scan file complete"); 64 | return NULL; 65 | } 66 | -------------------------------------------------------------------------------- /txt_to_pika/scan.h: -------------------------------------------------------------------------------- 1 | #ifndef SCAN_H_ 2 | #define SCAN_H_ 3 | 4 | #include 5 | 6 | #include "sender.h" 7 | #include "pink/include/bg_thread.h" 8 | 9 | #include 10 | 11 | class ScanThread : public pink::Thread { 12 | public: 13 | ScanThread(std::string filename, std::vector senders, int ttl) : 14 | filename_(filename), 15 | num_(0), 16 | senders_(senders), 17 | thread_index_(0), 18 | ttl_(ttl) 19 | { 20 | } 21 | 22 | virtual ~ScanThread(); 23 | int Num() { 24 | return num_; 25 | } 26 | void DispatchCmd(const std::string &cmd); 27 | private: 28 | void ScanFile(); 29 | std::string filename_; 30 | int num_; 31 | std::vector senders_; 32 | int thread_index_; 33 | int ttl_; 34 | 35 | void* ThreadMain(); 36 | }; 37 | 38 | #endif 39 | 40 | -------------------------------------------------------------------------------- /txt_to_pika/sender.h: -------------------------------------------------------------------------------- 1 | #ifndef SENDER_H_ 2 | #define SENDER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "slash/include/xdebug.h" 9 | #include "pink/include/bg_thread.h" 10 | #include "pink/include/pink_cli.h" 11 | #include "pink/include/redis_cli.h" 12 | 13 | class SenderThread : public pink::Thread { 14 | public: 15 | SenderThread(std::string ip, int64_t port, std::string password); 16 | virtual ~SenderThread(); 17 | void LoadCmd(const std::string &cmd); 18 | void Stop() { 19 | should_exit_ = true; 20 | cmd_mutex_.Lock(); 21 | rsignal_.Signal(); 22 | cmd_mutex_.Unlock(); 23 | } 24 | int64_t elements() { 25 | return elements_; 26 | } 27 | 28 | void SendCommand(std::string &command); 29 | 30 | int QueueSize() { 31 | slash::MutexLock l(&cmd_mutex_); 32 | int len = cmd_queue_.size(); 33 | return len; 34 | } 35 | void ConnectPika(); 36 | private: 37 | pink::PinkCli *cli_; 38 | slash::CondVar rsignal_; 39 | slash::CondVar wsignal_; 40 | slash::Mutex cmd_mutex_; 41 | std::queue cmd_queue_; 42 | std::string ip_; 43 | int port_; 44 | std::string password_; 45 | bool should_exit_; 46 | int64_t elements_; 47 | 48 | virtual void *ThreadMain(); 49 | }; 50 | 51 | #endif 52 | 53 | --------------------------------------------------------------------------------